| [ |
| { |
| "checkpoint": "checkpoint-22350", |
| "id_acc": 0.9711449652777778, |
| "ood_acc": 0.35872109375, |
| "total_acc": 0.6488166118421053, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.12382552083332, |
| "ood_avg_resp_len": 308.98884765625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 524981, |
| "answer_accuracy": 0.6483424063389187, |
| "avg_response_len": 243.34964457200442, |
| "resp_tokens_sum": 197047021, |
| "pass_at_k": { |
| "pass@1": 0.6483424063389187, |
| "pass@2": 0.6803206668330436, |
| "pass@4": 0.7059131493769168, |
| "pass@8": 0.7269858853485713, |
| "pass@16": 0.7450104273226786, |
| "pass@32": 0.7608087740158496, |
| "pass@64": 0.774999417439848, |
| "pass@128": 0.7888080935820424 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9188560103244838, |
| "pass@2": 0.9359165592409355, |
| "pass@4": 0.9455644327644885, |
| "pass@8": 0.9517458870956133, |
| "pass@16": 0.956694467156006, |
| "pass@32": 0.9617356981168745, |
| "pass@64": 0.9673014491619945, |
| "pass@128": 0.9734513274336283 |
| }, |
| "2": { |
| "pass@1": 0.9999756619937694, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9943115234375, |
| "pass@2": 0.9964159387303149, |
| "pass@4": 0.9968543389107613, |
| "pass@8": 0.9968749534243821, |
| "pass@16": 0.996874999999925, |
| "pass@32": 0.996875, |
| "pass@64": 0.996875, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9741071428571428, |
| "pass@2": 0.979054961879765, |
| "pass@4": 0.9813411186696901, |
| "pass@8": 0.9823550160738614, |
| "pass@16": 0.983211768811447, |
| "pass@32": 0.9839110645580986, |
| "pass@64": 0.9841223976613889, |
| "pass@128": 0.9841269841269841 |
| }, |
| "6": { |
| "pass@1": 0.9783869760479041, |
| "pass@2": 0.9840478982978925, |
| "pass@4": 0.986558948189111, |
| "pass@8": 0.9875558074943762, |
| "pass@16": 0.987963102168246, |
| "pass@32": 0.9880233341018126, |
| "pass@64": 0.9880239520939974, |
| "pass@128": 0.9880239520958084 |
| }, |
| "7": { |
| "pass@1": 0.9787120931952663, |
| "pass@2": 0.9838508421469508, |
| "pass@4": 0.9854995232659527, |
| "pass@8": 0.9862073748551169, |
| "pass@16": 0.9870352684360165, |
| "pass@32": 0.9882217903235329, |
| "pass@64": 0.9895597074034385, |
| "pass@128": 0.9911242603550295 |
| }, |
| "8": { |
| "pass@1": 0.9613882211538461, |
| "pass@2": 0.9697350406319402, |
| "pass@4": 0.9748580922576983, |
| "pass@8": 0.9800074606967099, |
| "pass@16": 0.9842336623573533, |
| "pass@32": 0.9863401319498917, |
| "pass@64": 0.987086901456941, |
| "pass@128": 0.9871794871794872 |
| }, |
| "9": { |
| "pass@1": 0.9488556338028169, |
| "pass@2": 0.966649453809471, |
| "pass@4": 0.9763733091462155, |
| "pass@8": 0.9831086093800043, |
| "pass@16": 0.9874560548129, |
| "pass@32": 0.9897432546798159, |
| "pass@64": 0.9915307632141225, |
| "pass@128": 0.9943661971830986 |
| }, |
| "11": { |
| "pass@1": 0.8217289305949008, |
| "pass@2": 0.8557530642858737, |
| "pass@4": 0.8781868379235878, |
| "pass@8": 0.8928124041359038, |
| "pass@16": 0.90415402974055, |
| "pass@32": 0.9134579174534119, |
| "pass@64": 0.9219840469693056, |
| "pass@128": 0.9320113314447592 |
| }, |
| "12": { |
| "pass@1": 0.7351694915254238, |
| "pass@2": 0.788351767204946, |
| "pass@4": 0.8296121509705352, |
| "pass@8": 0.8631458462120339, |
| "pass@16": 0.8882166657918442, |
| "pass@32": 0.9056702384232721, |
| "pass@64": 0.9167729138632694, |
| "pass@128": 0.923728813559322 |
| }, |
| "13": { |
| "pass@1": 0.5215762147335423, |
| "pass@2": 0.602112670624244, |
| "pass@4": 0.6653502091627258, |
| "pass@8": 0.7142055909099876, |
| "pass@16": 0.7537658644644787, |
| "pass@32": 0.7883226602707918, |
| "pass@64": 0.8193713978709286, |
| "pass@128": 0.8463949843260188 |
| }, |
| "14": { |
| "pass@1": 0.3340147975077882, |
| "pass@2": 0.397272533544288, |
| "pass@4": 0.4527668489452836, |
| "pass@8": 0.5028074674389132, |
| "pass@16": 0.5481987968856181, |
| "pass@32": 0.5876974515021655, |
| "pass@64": 0.6210981694258276, |
| "pass@128": 0.6510903426791277 |
| }, |
| "15": { |
| "pass@1": 0.27140534682080925, |
| "pass@2": 0.32894390617177177, |
| "pass@4": 0.3785269061381776, |
| "pass@8": 0.42405549456100855, |
| "pass@16": 0.46767789842350194, |
| "pass@32": 0.5084233147816724, |
| "pass@64": 0.5450481432462191, |
| "pass@128": 0.5809248554913294 |
| }, |
| "16": { |
| "pass@1": 0.2229586693548387, |
| "pass@2": 0.27569334201168394, |
| "pass@4": 0.31796193711673143, |
| "pass@8": 0.35152289464494024, |
| "pass@16": 0.380881906006343, |
| "pass@32": 0.40832885380195727, |
| "pass@64": 0.4346231078734797, |
| "pass@128": 0.4645161290322581 |
| }, |
| "17": { |
| "pass@1": 0.1877032153179191, |
| "pass@2": 0.2378149037367438, |
| "pass@4": 0.28208731558482936, |
| "pass@8": 0.3191592222713673, |
| "pass@16": 0.3518104209106715, |
| "pass@32": 0.3820936581550958, |
| "pass@64": 0.41107801511864955, |
| "pass@128": 0.4421965317919075 |
| }, |
| "18": { |
| "pass@1": 0.16616071428571427, |
| "pass@2": 0.21133858267716538, |
| "pass@4": 0.2523077339439713, |
| "pass@8": 0.28922715703966473, |
| "pass@16": 0.3217381726043357, |
| "pass@32": 0.35019897098246955, |
| "pass@64": 0.37583858090720185, |
| "pass@128": 0.4 |
| }, |
| "19": { |
| "pass@1": 0.17372841591591592, |
| "pass@2": 0.2298645840722612, |
| "pass@4": 0.27871888086061297, |
| "pass@8": 0.31920558010112116, |
| "pass@16": 0.35332419941478427, |
| "pass@32": 0.38412272386745866, |
| "pass@64": 0.413135659750011, |
| "pass@128": 0.43843843843843844 |
| }, |
| "20": { |
| "pass@1": 0.16403903903903905, |
| "pass@2": 0.21445424262845508, |
| "pass@4": 0.25837019950208934, |
| "pass@8": 0.29514379006139285, |
| "pass@16": 0.32666536880092817, |
| "pass@32": 0.3556550693689438, |
| "pass@64": 0.3840131899528545, |
| "pass@128": 0.4144144144144144 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 534931, |
| "answer_accuracy": 0.6514650720966485, |
| "avg_response_len": 252.5911377143414, |
| "resp_tokens_sum": 207407635, |
| "pass_at_k": { |
| "pass@1": 0.6514650720966485, |
| "pass@2": 0.6810825697645152, |
| "pass@4": 0.7053459193672471, |
| "pass@8": 0.7261223030044127, |
| "pass@16": 0.7449127573547755, |
| "pass@32": 0.7622321699696901, |
| "pass@64": 0.778532305744286, |
| "pass@128": 0.7939204988308651 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8771568251533742, |
| "pass@2": 0.8951967900101444, |
| "pass@4": 0.9078227647732684, |
| "pass@8": 0.9182064081148184, |
| "pass@16": 0.9283328420291027, |
| "pass@32": 0.9387767921163173, |
| "pass@64": 0.9482679411296987, |
| "pass@128": 0.9539877300613497 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9987926136363636, |
| "pass@2": 0.999673034478645, |
| "pass@4": 0.9999682039745033, |
| "pass@8": 0.9999997498310389, |
| "pass@16": 0.9999999999945943, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999768860946746, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9761548913043478, |
| "pass@2": 0.9806904741526873, |
| "pass@4": 0.9844550450215461, |
| "pass@8": 0.9874040842618942, |
| "pass@16": 0.9900450599720992, |
| "pass@32": 0.9923200953194031, |
| "pass@64": 0.9941919743959912, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9873522564469914, |
| "pass@2": 0.9888020187036076, |
| "pass@4": 0.9894069616656083, |
| "pass@8": 0.990037327806008, |
| "pass@16": 0.990786092204451, |
| "pass@32": 0.9913001711780407, |
| "pass@64": 0.9914031358915707, |
| "pass@128": 0.9914040114613181 |
| }, |
| "7": { |
| "pass@1": 0.9812590317919075, |
| "pass@2": 0.9833455890719586, |
| "pass@4": 0.9854745302790909, |
| "pass@8": 0.9875048076156542, |
| "pass@16": 0.9886360584882343, |
| "pass@32": 0.9891581060193411, |
| "pass@64": 0.9898843928705129, |
| "pass@128": 0.9913294797687862 |
| }, |
| "8": { |
| "pass@1": 0.9748459507042253, |
| "pass@2": 0.9835442774758789, |
| "pass@4": 0.9879733210285333, |
| "pass@8": 0.9911431418738322, |
| "pass@16": 0.99338839349148, |
| "pass@32": 0.9942874400881745, |
| "pass@64": 0.9943660901808677, |
| "pass@128": 0.9943661971830986 |
| }, |
| "9": { |
| "pass@1": 0.9630660553892215, |
| "pass@2": 0.9748923658588334, |
| "pass@4": 0.9807665619267653, |
| "pass@8": 0.9838964621748524, |
| "pass@16": 0.9860050132910293, |
| "pass@32": 0.9884616006328849, |
| "pass@64": 0.9905750388985808, |
| "pass@128": 0.9910179640718563 |
| }, |
| "11": { |
| "pass@1": 0.8489975527108434, |
| "pass@2": 0.882086540057869, |
| "pass@4": 0.9039521694765564, |
| "pass@8": 0.9186704477726292, |
| "pass@16": 0.9294459191329357, |
| "pass@32": 0.9399804052926921, |
| "pass@64": 0.9501763525977892, |
| "pass@128": 0.9578313253012049 |
| }, |
| "12": { |
| "pass@1": 0.740130289968652, |
| "pass@2": 0.7924894478315606, |
| "pass@4": 0.8298025234110624, |
| "pass@8": 0.8583415348389902, |
| "pass@16": 0.8811457315129261, |
| "pass@32": 0.898488218747374, |
| "pass@64": 0.9119921067515657, |
| "pass@128": 0.9216300940438872 |
| }, |
| "13": { |
| "pass@1": 0.4885329131652661, |
| "pass@2": 0.5524010785416529, |
| "pass@4": 0.6016827056281829, |
| "pass@8": 0.6424268441509672, |
| "pass@16": 0.6814935198674684, |
| "pass@32": 0.719642728564925, |
| "pass@64": 0.7559772746835947, |
| "pass@128": 0.7927170868347339 |
| }, |
| "14": { |
| "pass@1": 0.4045138888888889, |
| "pass@2": 0.4613771037666344, |
| "pass@4": 0.505874457852856, |
| "pass@8": 0.5456155359914777, |
| "pass@16": 0.5829306088795296, |
| "pass@32": 0.6183434242052145, |
| "pass@64": 0.6531282479677968, |
| "pass@128": 0.6871345029239766 |
| }, |
| "15": { |
| "pass@1": 0.26129261363636364, |
| "pass@2": 0.3235124373657837, |
| "pass@4": 0.37589588829805376, |
| "pass@8": 0.42043231436330086, |
| "pass@16": 0.45993852789377043, |
| "pass@32": 0.49402710124836785, |
| "pass@64": 0.5220525337703099, |
| "pass@128": 0.5454545454545454 |
| }, |
| "16": { |
| "pass@1": 0.21287393162393162, |
| "pass@2": 0.26000866478228674, |
| "pass@4": 0.300428627244244, |
| "pass@8": 0.33815033021506824, |
| "pass@16": 0.3761938301457166, |
| "pass@32": 0.41416824184837775, |
| "pass@64": 0.4506074301431214, |
| "pass@128": 0.4843304843304843 |
| }, |
| "17": { |
| "pass@1": 0.1943662655279503, |
| "pass@2": 0.24511312784271538, |
| "pass@4": 0.2901540472231345, |
| "pass@8": 0.3315383250511585, |
| "pass@16": 0.3698795046319779, |
| "pass@32": 0.40524016010241276, |
| "pass@64": 0.43942643349381627, |
| "pass@128": 0.4720496894409938 |
| }, |
| "18": { |
| "pass@1": 0.17589613970588236, |
| "pass@2": 0.22981378821213524, |
| "pass@4": 0.2799638864994816, |
| "pass@8": 0.32366264093031627, |
| "pass@16": 0.3623485931753083, |
| "pass@32": 0.39840148766647915, |
| "pass@64": 0.43414839380764614, |
| "pass@128": 0.4676470588235294 |
| }, |
| "19": { |
| "pass@1": 0.14046223958333334, |
| "pass@2": 0.19077498125234343, |
| "pass@4": 0.2388992863057297, |
| "pass@8": 0.2804475024195655, |
| "pass@16": 0.315365405487468, |
| "pass@32": 0.34737330408706285, |
| "pass@64": 0.38110748946513323, |
| "pass@128": 0.41964285714285715 |
| }, |
| "20": { |
| "pass@1": 0.13994666916167664, |
| "pass@2": 0.18604130322033105, |
| "pass@4": 0.2294044430823391, |
| "pass@8": 0.2700214660362194, |
| "pass@16": 0.3089193843131219, |
| "pass@32": 0.34388208870332215, |
| "pass@64": 0.37594382323777076, |
| "pass@128": 0.40718562874251496 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 518010, |
| "answer_accuracy": 0.6465814227512382, |
| "avg_response_len": 270.8421323294456, |
| "resp_tokens_sum": 216985716, |
| "pass_at_k": { |
| "pass@1": 0.6465814227512382, |
| "pass@2": 0.6762224797865887, |
| "pass@4": 0.7010356201730161, |
| "pass@8": 0.7224846593869281, |
| "pass@16": 0.7420975423592175, |
| "pass@32": 0.7601357708258796, |
| "pass@64": 0.776524499196065, |
| "pass@128": 0.7918197795174948 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.884071828358209, |
| "pass@2": 0.9018414326007759, |
| "pass@4": 0.9138320730431082, |
| "pass@8": 0.921710395734955, |
| "pass@16": 0.9283555561753828, |
| "pass@32": 0.9354921104145758, |
| "pass@64": 0.9430789258299536, |
| "pass@128": 0.9522388059701492 |
| }, |
| "2": { |
| "pass@1": 0.9994419642857143, |
| "pass@2": 0.9999469206974128, |
| "pass@4": 0.9999993585623226, |
| "pass@8": 0.9999999999514184, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9998633381924198, |
| "pass@2": 0.9999985652303667, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9995202850877193, |
| "pass@2": 0.9999381245107519, |
| "pass@4": 0.9999989376327959, |
| "pass@8": 0.9999999998454224, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9627068014705882, |
| "pass@2": 0.9684022550949515, |
| "pass@4": 0.972176392013498, |
| "pass@8": 0.9752606239283936, |
| "pass@16": 0.9784406334094804, |
| "pass@32": 0.9813456000064947, |
| "pass@64": 0.9835626145733192, |
| "pass@128": 0.9852941176470589 |
| }, |
| "6": { |
| "pass@1": 0.9906348580441641, |
| "pass@2": 0.9928117781117265, |
| "pass@4": 0.9941831030979172, |
| "pass@8": 0.9953566355803547, |
| "pass@16": 0.9962564144193743, |
| "pass@32": 0.9967623707930423, |
| "pass@64": 0.996844989203979, |
| "pass@128": 0.9968454258675079 |
| }, |
| "7": { |
| "pass@1": 0.9671182753164557, |
| "pass@2": 0.9765255126831456, |
| "pass@4": 0.9817781986112496, |
| "pass@8": 0.9850473862203449, |
| "pass@16": 0.9867960296777932, |
| "pass@32": 0.987306393671264, |
| "pass@64": 0.9873417337459038, |
| "pass@128": 0.9873417721518988 |
| }, |
| "8": { |
| "pass@1": 0.9617351726726727, |
| "pass@2": 0.9702019933319148, |
| "pass@4": 0.9758465687284583, |
| "pass@8": 0.9804503225139821, |
| "pass@16": 0.9841829457607862, |
| "pass@32": 0.9867784982472595, |
| "pass@64": 0.9878811072240533, |
| "pass@128": 0.987987987987988 |
| }, |
| "9": { |
| "pass@1": 0.9398110932475884, |
| "pass@2": 0.9515331069448313, |
| "pass@4": 0.9577947668117046, |
| "pass@8": 0.9619601474829078, |
| "pass@16": 0.9656259394422011, |
| "pass@32": 0.9689428782955865, |
| "pass@64": 0.9708018571624879, |
| "pass@128": 0.9710610932475884 |
| }, |
| "11": { |
| "pass@1": 0.8549851190476191, |
| "pass@2": 0.8820487282839649, |
| "pass@4": 0.8995010332041826, |
| "pass@8": 0.9122908672389048, |
| "pass@16": 0.9222517782526704, |
| "pass@32": 0.9300689926212681, |
| "pass@64": 0.9363073336659694, |
| "pass@128": 0.9396825396825397 |
| }, |
| "12": { |
| "pass@1": 0.7341121941896025, |
| "pass@2": 0.7889543301066725, |
| "pass@4": 0.8277259935397981, |
| "pass@8": 0.855340272551626, |
| "pass@16": 0.8773403999121976, |
| "pass@32": 0.8953960226775695, |
| "pass@64": 0.9096934117037676, |
| "pass@128": 0.9235474006116208 |
| }, |
| "13": { |
| "pass@1": 0.5405333719135802, |
| "pass@2": 0.6106858170506467, |
| "pass@4": 0.6639800241983641, |
| "pass@8": 0.7044176745325131, |
| "pass@16": 0.7376936245756492, |
| "pass@32": 0.7672839358537915, |
| "pass@64": 0.7959651982296765, |
| "pass@128": 0.8271604938271605 |
| }, |
| "14": { |
| "pass@1": 0.37553319732937684, |
| "pass@2": 0.44198791151662414, |
| "pass@4": 0.49824965397500376, |
| "pass@8": 0.5453334095299108, |
| "pass@16": 0.5868411116517013, |
| "pass@32": 0.6245037035678842, |
| "pass@64": 0.6590991243915612, |
| "pass@128": 0.6943620178041543 |
| }, |
| "15": { |
| "pass@1": 0.21554301697530864, |
| "pass@2": 0.271911757558083, |
| "pass@4": 0.32636879417850534, |
| "pass@8": 0.3794379036085877, |
| "pass@16": 0.4310885133567016, |
| "pass@32": 0.4821266393820033, |
| "pass@64": 0.5357953674441768, |
| "pass@128": 0.5895061728395061 |
| }, |
| "16": { |
| "pass@1": 0.20010601032448377, |
| "pass@2": 0.24604666051146248, |
| "pass@4": 0.285889059774608, |
| "pass@8": 0.3214126609636647, |
| "pass@16": 0.35628660962155617, |
| "pass@32": 0.3924603139361884, |
| "pass@64": 0.4309583363312713, |
| "pass@128": 0.4749262536873156 |
| }, |
| "17": { |
| "pass@1": 0.16006212349397592, |
| "pass@2": 0.2083086282136419, |
| "pass@4": 0.255251346217265, |
| "pass@8": 0.30182845547006043, |
| "pass@16": 0.34753522996493463, |
| "pass@32": 0.3893601104123036, |
| "pass@64": 0.4261863056840549, |
| "pass@128": 0.45481927710843373 |
| }, |
| "18": { |
| "pass@1": 0.16139112903225805, |
| "pass@2": 0.2065325755651512, |
| "pass@4": 0.24982900292705337, |
| "pass@8": 0.2905043941065011, |
| "pass@16": 0.33038934761738825, |
| "pass@32": 0.36695064077901834, |
| "pass@64": 0.39713096560074196, |
| "pass@128": 0.42258064516129035 |
| }, |
| "19": { |
| "pass@1": 0.16229229607250756, |
| "pass@2": 0.21233117551204875, |
| "pass@4": 0.25672324720739215, |
| "pass@8": 0.29440231830730196, |
| "pass@16": 0.3283485107029244, |
| "pass@32": 0.35922085161504386, |
| "pass@64": 0.3836921534435109, |
| "pass@128": 0.40483383685800606 |
| }, |
| "20": { |
| "pass@1": 0.1533643018018018, |
| "pass@2": 0.19884586850630145, |
| "pass@4": 0.2432588066131373, |
| "pass@8": 0.2868569676499088, |
| "pass@16": 0.32828460673465015, |
| "pass@32": 0.3655857400088371, |
| "pass@64": 0.3973743545827233, |
| "pass@128": 0.42042042042042044 |
| } |
| } |
| } |
| } |
| } |
| ] |
