[ { "checkpoint": "checkpoint-22350", "id_acc": 0.9711449652777778, "ood_acc": 0.35872109375, "total_acc": 0.6488166118421053, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.12382552083332, "ood_avg_resp_len": 308.98884765625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 524981, "answer_accuracy": 0.6483424063389187, "avg_response_len": 243.34964457200442, "resp_tokens_sum": 197047021, "pass_at_k": { "pass@1": 0.6483424063389187, "pass@2": 0.6803206668330436, "pass@4": 0.7059131493769168, "pass@8": 0.7269858853485713, "pass@16": 0.7450104273226786, "pass@32": 0.7608087740158496, "pass@64": 0.774999417439848, "pass@128": 0.7888080935820424 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9188560103244838, "pass@2": 0.9359165592409355, "pass@4": 0.9455644327644885, "pass@8": 0.9517458870956133, "pass@16": 0.956694467156006, "pass@32": 0.9617356981168745, "pass@64": 0.9673014491619945, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 0.9999756619937694, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9943115234375, "pass@2": 0.9964159387303149, "pass@4": 0.9968543389107613, "pass@8": 0.9968749534243821, "pass@16": 0.996874999999925, "pass@32": 0.996875, "pass@64": 0.996875, "pass@128": 0.996875 }, "5": { "pass@1": 0.9741071428571428, "pass@2": 0.979054961879765, "pass@4": 0.9813411186696901, "pass@8": 0.9823550160738614, "pass@16": 0.983211768811447, "pass@32": 0.9839110645580986, "pass@64": 0.9841223976613889, "pass@128": 0.9841269841269841 }, "6": { "pass@1": 0.9783869760479041, "pass@2": 0.9840478982978925, "pass@4": 0.986558948189111, "pass@8": 0.9875558074943762, "pass@16": 0.987963102168246, "pass@32": 0.9880233341018126, "pass@64": 0.9880239520939974, "pass@128": 0.9880239520958084 }, "7": { "pass@1": 0.9787120931952663, "pass@2": 0.9838508421469508, "pass@4": 0.9854995232659527, "pass@8": 0.9862073748551169, "pass@16": 0.9870352684360165, "pass@32": 0.9882217903235329, "pass@64": 0.9895597074034385, "pass@128": 0.9911242603550295 }, "8": { "pass@1": 0.9613882211538461, "pass@2": 0.9697350406319402, "pass@4": 0.9748580922576983, "pass@8": 0.9800074606967099, "pass@16": 0.9842336623573533, "pass@32": 0.9863401319498917, "pass@64": 0.987086901456941, "pass@128": 0.9871794871794872 }, "9": { "pass@1": 0.9488556338028169, "pass@2": 0.966649453809471, "pass@4": 0.9763733091462155, "pass@8": 0.9831086093800043, "pass@16": 0.9874560548129, "pass@32": 0.9897432546798159, "pass@64": 0.9915307632141225, "pass@128": 0.9943661971830986 }, "11": { "pass@1": 0.8217289305949008, "pass@2": 0.8557530642858737, "pass@4": 0.8781868379235878, "pass@8": 0.8928124041359038, "pass@16": 0.90415402974055, "pass@32": 0.9134579174534119, "pass@64": 0.9219840469693056, "pass@128": 0.9320113314447592 }, "12": { "pass@1": 0.7351694915254238, "pass@2": 0.788351767204946, "pass@4": 0.8296121509705352, "pass@8": 0.8631458462120339, "pass@16": 0.8882166657918442, "pass@32": 0.9056702384232721, "pass@64": 0.9167729138632694, "pass@128": 0.923728813559322 }, "13": { "pass@1": 0.5215762147335423, "pass@2": 0.602112670624244, "pass@4": 0.6653502091627258, "pass@8": 0.7142055909099876, "pass@16": 0.7537658644644787, "pass@32": 0.7883226602707918, "pass@64": 0.8193713978709286, "pass@128": 0.8463949843260188 }, "14": { "pass@1": 0.3340147975077882, "pass@2": 0.397272533544288, "pass@4": 0.4527668489452836, "pass@8": 0.5028074674389132, "pass@16": 0.5481987968856181, "pass@32": 0.5876974515021655, "pass@64": 0.6210981694258276, "pass@128": 0.6510903426791277 }, "15": { "pass@1": 0.27140534682080925, "pass@2": 0.32894390617177177, "pass@4": 0.3785269061381776, "pass@8": 0.42405549456100855, "pass@16": 0.46767789842350194, "pass@32": 0.5084233147816724, "pass@64": 0.5450481432462191, "pass@128": 0.5809248554913294 }, "16": { "pass@1": 0.2229586693548387, "pass@2": 0.27569334201168394, "pass@4": 0.31796193711673143, "pass@8": 0.35152289464494024, "pass@16": 0.380881906006343, "pass@32": 0.40832885380195727, "pass@64": 0.4346231078734797, "pass@128": 0.4645161290322581 }, "17": { "pass@1": 0.1877032153179191, "pass@2": 0.2378149037367438, "pass@4": 0.28208731558482936, "pass@8": 0.3191592222713673, "pass@16": 0.3518104209106715, "pass@32": 0.3820936581550958, "pass@64": 0.41107801511864955, "pass@128": 0.4421965317919075 }, "18": { "pass@1": 0.16616071428571427, "pass@2": 0.21133858267716538, "pass@4": 0.2523077339439713, "pass@8": 0.28922715703966473, "pass@16": 0.3217381726043357, "pass@32": 0.35019897098246955, "pass@64": 0.37583858090720185, "pass@128": 0.4 }, "19": { "pass@1": 0.17372841591591592, "pass@2": 0.2298645840722612, "pass@4": 0.27871888086061297, "pass@8": 0.31920558010112116, "pass@16": 0.35332419941478427, "pass@32": 0.38412272386745866, "pass@64": 0.413135659750011, "pass@128": 0.43843843843843844 }, "20": { "pass@1": 0.16403903903903905, "pass@2": 0.21445424262845508, "pass@4": 0.25837019950208934, "pass@8": 0.29514379006139285, "pass@16": 0.32666536880092817, "pass@32": 0.3556550693689438, "pass@64": 0.3840131899528545, "pass@128": 0.4144144144144144 } } }, "teachers_in_school": { "count": 821120, "correct": 534931, "answer_accuracy": 0.6514650720966485, "avg_response_len": 252.5911377143414, "resp_tokens_sum": 207407635, "pass_at_k": { "pass@1": 0.6514650720966485, "pass@2": 0.6810825697645152, "pass@4": 0.7053459193672471, "pass@8": 0.7261223030044127, "pass@16": 0.7449127573547755, "pass@32": 0.7622321699696901, "pass@64": 0.778532305744286, "pass@128": 0.7939204988308651 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8771568251533742, "pass@2": 0.8951967900101444, "pass@4": 0.9078227647732684, "pass@8": 0.9182064081148184, "pass@16": 0.9283328420291027, "pass@32": 0.9387767921163173, "pass@64": 0.9482679411296987, "pass@128": 0.9539877300613497 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9987926136363636, "pass@2": 0.999673034478645, "pass@4": 0.9999682039745033, "pass@8": 0.9999997498310389, "pass@16": 0.9999999999945943, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999768860946746, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9761548913043478, "pass@2": 0.9806904741526873, "pass@4": 0.9844550450215461, "pass@8": 0.9874040842618942, "pass@16": 0.9900450599720992, "pass@32": 0.9923200953194031, "pass@64": 0.9941919743959912, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9873522564469914, "pass@2": 0.9888020187036076, "pass@4": 0.9894069616656083, "pass@8": 0.990037327806008, "pass@16": 0.990786092204451, "pass@32": 0.9913001711780407, "pass@64": 0.9914031358915707, "pass@128": 0.9914040114613181 }, "7": { "pass@1": 0.9812590317919075, "pass@2": 0.9833455890719586, "pass@4": 0.9854745302790909, "pass@8": 0.9875048076156542, "pass@16": 0.9886360584882343, "pass@32": 0.9891581060193411, "pass@64": 0.9898843928705129, "pass@128": 0.9913294797687862 }, "8": { "pass@1": 0.9748459507042253, "pass@2": 0.9835442774758789, "pass@4": 0.9879733210285333, "pass@8": 0.9911431418738322, "pass@16": 0.99338839349148, "pass@32": 0.9942874400881745, "pass@64": 0.9943660901808677, "pass@128": 0.9943661971830986 }, "9": { "pass@1": 0.9630660553892215, "pass@2": 0.9748923658588334, "pass@4": 0.9807665619267653, "pass@8": 0.9838964621748524, "pass@16": 0.9860050132910293, "pass@32": 0.9884616006328849, "pass@64": 0.9905750388985808, "pass@128": 0.9910179640718563 }, "11": { "pass@1": 0.8489975527108434, "pass@2": 0.882086540057869, "pass@4": 0.9039521694765564, "pass@8": 0.9186704477726292, "pass@16": 0.9294459191329357, "pass@32": 0.9399804052926921, "pass@64": 0.9501763525977892, "pass@128": 0.9578313253012049 }, "12": { "pass@1": 0.740130289968652, "pass@2": 0.7924894478315606, "pass@4": 0.8298025234110624, "pass@8": 0.8583415348389902, "pass@16": 0.8811457315129261, "pass@32": 0.898488218747374, "pass@64": 0.9119921067515657, "pass@128": 0.9216300940438872 }, "13": { "pass@1": 0.4885329131652661, "pass@2": 0.5524010785416529, "pass@4": 0.6016827056281829, "pass@8": 0.6424268441509672, "pass@16": 0.6814935198674684, "pass@32": 0.719642728564925, "pass@64": 0.7559772746835947, "pass@128": 0.7927170868347339 }, "14": { "pass@1": 0.4045138888888889, "pass@2": 0.4613771037666344, "pass@4": 0.505874457852856, "pass@8": 0.5456155359914777, "pass@16": 0.5829306088795296, "pass@32": 0.6183434242052145, "pass@64": 0.6531282479677968, "pass@128": 0.6871345029239766 }, "15": { "pass@1": 0.26129261363636364, "pass@2": 0.3235124373657837, "pass@4": 0.37589588829805376, "pass@8": 0.42043231436330086, "pass@16": 0.45993852789377043, "pass@32": 0.49402710124836785, "pass@64": 0.5220525337703099, "pass@128": 0.5454545454545454 }, "16": { "pass@1": 0.21287393162393162, "pass@2": 0.26000866478228674, "pass@4": 0.300428627244244, "pass@8": 0.33815033021506824, "pass@16": 0.3761938301457166, "pass@32": 0.41416824184837775, "pass@64": 0.4506074301431214, "pass@128": 0.4843304843304843 }, "17": { "pass@1": 0.1943662655279503, "pass@2": 0.24511312784271538, "pass@4": 0.2901540472231345, "pass@8": 0.3315383250511585, "pass@16": 0.3698795046319779, "pass@32": 0.40524016010241276, "pass@64": 0.43942643349381627, "pass@128": 0.4720496894409938 }, "18": { "pass@1": 0.17589613970588236, "pass@2": 0.22981378821213524, "pass@4": 0.2799638864994816, "pass@8": 0.32366264093031627, "pass@16": 0.3623485931753083, "pass@32": 0.39840148766647915, "pass@64": 0.43414839380764614, "pass@128": 0.4676470588235294 }, "19": { "pass@1": 0.14046223958333334, "pass@2": 0.19077498125234343, "pass@4": 0.2388992863057297, "pass@8": 0.2804475024195655, "pass@16": 0.315365405487468, "pass@32": 0.34737330408706285, "pass@64": 0.38110748946513323, "pass@128": 0.41964285714285715 }, "20": { "pass@1": 0.13994666916167664, "pass@2": 0.18604130322033105, "pass@4": 0.2294044430823391, "pass@8": 0.2700214660362194, "pass@16": 0.3089193843131219, "pass@32": 0.34388208870332215, "pass@64": 0.37594382323777076, "pass@128": 0.40718562874251496 } } }, "movie_festival_awards": { "count": 801152, "correct": 518010, "answer_accuracy": 0.6465814227512382, "avg_response_len": 270.8421323294456, "resp_tokens_sum": 216985716, "pass_at_k": { "pass@1": 0.6465814227512382, "pass@2": 0.6762224797865887, "pass@4": 0.7010356201730161, "pass@8": 0.7224846593869281, "pass@16": 0.7420975423592175, "pass@32": 0.7601357708258796, "pass@64": 0.776524499196065, "pass@128": 0.7918197795174948 }, "per_op_pass_at_k": { "10": { "pass@1": 0.884071828358209, "pass@2": 0.9018414326007759, "pass@4": 0.9138320730431082, "pass@8": 0.921710395734955, "pass@16": 0.9283555561753828, "pass@32": 0.9354921104145758, "pass@64": 0.9430789258299536, "pass@128": 0.9522388059701492 }, "2": { "pass@1": 0.9994419642857143, "pass@2": 0.9999469206974128, "pass@4": 0.9999993585623226, "pass@8": 0.9999999999514184, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9998633381924198, "pass@2": 0.9999985652303667, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9995202850877193, "pass@2": 0.9999381245107519, "pass@4": 0.9999989376327959, "pass@8": 0.9999999998454224, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9627068014705882, "pass@2": 0.9684022550949515, "pass@4": 0.972176392013498, "pass@8": 0.9752606239283936, "pass@16": 0.9784406334094804, "pass@32": 0.9813456000064947, "pass@64": 0.9835626145733192, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.9906348580441641, "pass@2": 0.9928117781117265, "pass@4": 0.9941831030979172, "pass@8": 0.9953566355803547, "pass@16": 0.9962564144193743, "pass@32": 0.9967623707930423, "pass@64": 0.996844989203979, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.9671182753164557, "pass@2": 0.9765255126831456, "pass@4": 0.9817781986112496, "pass@8": 0.9850473862203449, "pass@16": 0.9867960296777932, "pass@32": 0.987306393671264, "pass@64": 0.9873417337459038, "pass@128": 0.9873417721518988 }, "8": { "pass@1": 0.9617351726726727, "pass@2": 0.9702019933319148, "pass@4": 0.9758465687284583, "pass@8": 0.9804503225139821, "pass@16": 0.9841829457607862, "pass@32": 0.9867784982472595, "pass@64": 0.9878811072240533, "pass@128": 0.987987987987988 }, "9": { "pass@1": 0.9398110932475884, "pass@2": 0.9515331069448313, "pass@4": 0.9577947668117046, "pass@8": 0.9619601474829078, "pass@16": 0.9656259394422011, "pass@32": 0.9689428782955865, "pass@64": 0.9708018571624879, "pass@128": 0.9710610932475884 }, "11": { "pass@1": 0.8549851190476191, "pass@2": 0.8820487282839649, "pass@4": 0.8995010332041826, "pass@8": 0.9122908672389048, "pass@16": 0.9222517782526704, "pass@32": 0.9300689926212681, "pass@64": 0.9363073336659694, "pass@128": 0.9396825396825397 }, "12": { "pass@1": 0.7341121941896025, "pass@2": 0.7889543301066725, "pass@4": 0.8277259935397981, "pass@8": 0.855340272551626, "pass@16": 0.8773403999121976, "pass@32": 0.8953960226775695, "pass@64": 0.9096934117037676, "pass@128": 0.9235474006116208 }, "13": { "pass@1": 0.5405333719135802, "pass@2": 0.6106858170506467, "pass@4": 0.6639800241983641, "pass@8": 0.7044176745325131, "pass@16": 0.7376936245756492, "pass@32": 0.7672839358537915, "pass@64": 0.7959651982296765, "pass@128": 0.8271604938271605 }, "14": { "pass@1": 0.37553319732937684, "pass@2": 0.44198791151662414, "pass@4": 0.49824965397500376, "pass@8": 0.5453334095299108, "pass@16": 0.5868411116517013, "pass@32": 0.6245037035678842, "pass@64": 0.6590991243915612, "pass@128": 0.6943620178041543 }, "15": { "pass@1": 0.21554301697530864, "pass@2": 0.271911757558083, "pass@4": 0.32636879417850534, "pass@8": 0.3794379036085877, "pass@16": 0.4310885133567016, "pass@32": 0.4821266393820033, "pass@64": 0.5357953674441768, "pass@128": 0.5895061728395061 }, "16": { "pass@1": 0.20010601032448377, "pass@2": 0.24604666051146248, "pass@4": 0.285889059774608, "pass@8": 0.3214126609636647, "pass@16": 0.35628660962155617, "pass@32": 0.3924603139361884, "pass@64": 0.4309583363312713, "pass@128": 0.4749262536873156 }, "17": { "pass@1": 0.16006212349397592, "pass@2": 0.2083086282136419, "pass@4": 0.255251346217265, "pass@8": 0.30182845547006043, "pass@16": 0.34753522996493463, "pass@32": 0.3893601104123036, "pass@64": 0.4261863056840549, "pass@128": 0.45481927710843373 }, "18": { "pass@1": 0.16139112903225805, "pass@2": 0.2065325755651512, "pass@4": 0.24982900292705337, "pass@8": 0.2905043941065011, "pass@16": 0.33038934761738825, "pass@32": 0.36695064077901834, "pass@64": 0.39713096560074196, "pass@128": 0.42258064516129035 }, "19": { "pass@1": 0.16229229607250756, "pass@2": 0.21233117551204875, "pass@4": 0.25672324720739215, "pass@8": 0.29440231830730196, "pass@16": 0.3283485107029244, "pass@32": 0.35922085161504386, "pass@64": 0.3836921534435109, "pass@128": 0.40483383685800606 }, "20": { "pass@1": 0.1533643018018018, "pass@2": 0.19884586850630145, "pass@4": 0.2432588066131373, "pass@8": 0.2868569676499088, "pass@16": 0.32828460673465015, "pass@32": 0.3655857400088371, "pass@64": 0.3973743545827233, "pass@128": 0.42042042042042044 } } } } } ]