| [ |
| { |
| "checkpoint": "checkpoint-43615", |
| "id_acc": 0.9805737847222222, |
| "ood_acc": 0.4121578125, |
| "total_acc": 0.6814074835526316, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.14779166666668, |
| "ood_avg_resp_len": 331.915390625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 550735, |
| "answer_accuracy": 0.6801481485140689, |
| "avg_response_len": 256.70740421474864, |
| "resp_tokens_sum": 207863173, |
| "pass_at_k": { |
| "pass@1": 0.6801481485140689, |
| "pass@2": 0.730230301113263, |
| "pass@4": 0.770282546758596, |
| "pass@8": 0.80239476506394, |
| "pass@16": 0.8280953761715771, |
| "pass@32": 0.8492214535571997, |
| "pass@64": 0.867751709603317, |
| "pass@128": 0.8849193803351248 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.918809918879056, |
| "pass@2": 0.948028810419715, |
| "pass@4": 0.962817301874876, |
| "pass@8": 0.9724661916664145, |
| "pass@16": 0.9787429928484136, |
| "pass@32": 0.983707212874408, |
| "pass@64": 0.9895069940851522, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 0.9962762850467289, |
| "pass@2": 0.9984557822503495, |
| "pass@4": 0.9994738122687934, |
| "pass@8": 0.9999222873805567, |
| "pass@16": 0.9999986070165172, |
| "pass@32": 0.9999999998747847, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9998088685015291, |
| "pass@2": 0.9999984950275711, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9986083984375, |
| "pass@2": 0.9995463213582678, |
| "pass@4": 0.999937934711286, |
| "pass@8": 0.9999990142660407, |
| "pass@16": 0.9999999998879098, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9961309523809524, |
| "pass@2": 0.9971964441944757, |
| "pass@4": 0.9976364139006433, |
| "pass@8": 0.9982586134549394, |
| "pass@16": 0.9990849640116398, |
| "pass@32": 0.9997840804322056, |
| "pass@64": 0.9999954135344047, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9970761601796407, |
| "pass@2": 0.9997358874770144, |
| "pass@4": 0.9999925972015473, |
| "pass@8": 0.9999999909949934, |
| "pass@16": 0.9999999999999978, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9891595784023669, |
| "pass@2": 0.993395247053068, |
| "pass@4": 0.9951834355579812, |
| "pass@8": 0.9965973668448541, |
| "pass@16": 0.9973484845597674, |
| "pass@32": 0.9977806286548048, |
| "pass@64": 0.9985207100585047, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9747095352564102, |
| "pass@2": 0.9895912199676963, |
| "pass@4": 0.994234933674156, |
| "pass@8": 0.9961986765174754, |
| "pass@16": 0.997689406934065, |
| "pass@32": 0.9989578361290308, |
| "pass@64": 0.9998090173924294, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.976012323943662, |
| "pass@2": 0.9908849256959081, |
| "pass@4": 0.9950709141991052, |
| "pass@8": 0.9975627189798211, |
| "pass@16": 0.9993104189838761, |
| "pass@32": 0.9999512902094978, |
| "pass@64": 0.999999951254185, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8832551345609065, |
| "pass@2": 0.9303415187035764, |
| "pass@4": 0.950412839595475, |
| "pass@8": 0.9616306162855329, |
| "pass@16": 0.9699049289553977, |
| "pass@32": 0.9770236314441675, |
| "pass@64": 0.9837252279289742, |
| "pass@128": 0.9886685552407932 |
| }, |
| "12": { |
| "pass@1": 0.772753354519774, |
| "pass@2": 0.8552530973352902, |
| "pass@4": 0.8972007813112336, |
| "pass@8": 0.9213115105135973, |
| "pass@16": 0.937571762368148, |
| "pass@32": 0.9488781345620627, |
| "pass@64": 0.9575123939060747, |
| "pass@128": 0.9661016949152542 |
| }, |
| "13": { |
| "pass@1": 0.604648315047022, |
| "pass@2": 0.722908773109866, |
| "pass@4": 0.8087846775814462, |
| "pass@8": 0.8668469492435775, |
| "pass@16": 0.9029254689906228, |
| "pass@32": 0.9272910665414706, |
| "pass@64": 0.9472116958813479, |
| "pass@128": 0.9623824451410659 |
| }, |
| "14": { |
| "pass@1": 0.48761195482866043, |
| "pass@2": 0.6136661546103467, |
| "pass@4": 0.7130287466987187, |
| "pass@8": 0.7846870698190367, |
| "pass@16": 0.8326684455995763, |
| "pass@32": 0.8645961134923782, |
| "pass@64": 0.8881567987893952, |
| "pass@128": 0.9065420560747663 |
| }, |
| "15": { |
| "pass@1": 0.3537978684971098, |
| "pass@2": 0.4644254073551498, |
| "pass@4": 0.564236310417845, |
| "pass@8": 0.6463581735993315, |
| "pass@16": 0.7141209227548058, |
| "pass@32": 0.770180292774696, |
| "pass@64": 0.8143289750480056, |
| "pass@128": 0.8439306358381503 |
| }, |
| "16": { |
| "pass@1": 0.2631552419354839, |
| "pass@2": 0.3625079375158748, |
| "pass@4": 0.460933843753402, |
| "pass@8": 0.5463932807791085, |
| "pass@16": 0.6144613335763428, |
| "pass@32": 0.6705593576451186, |
| "pass@64": 0.7250965392578077, |
| "pass@128": 0.7870967741935484 |
| }, |
| "17": { |
| "pass@1": 0.2083634393063584, |
| "pass@2": 0.28992692071366777, |
| "pass@4": 0.36788058013702063, |
| "pass@8": 0.4378366215590766, |
| "pass@16": 0.5020683791358072, |
| "pass@32": 0.5617083762026661, |
| "pass@64": 0.6147709620674227, |
| "pass@128": 0.6647398843930635 |
| }, |
| "18": { |
| "pass@1": 0.19977678571428573, |
| "pass@2": 0.27297455005624305, |
| "pass@4": 0.34381426509186336, |
| "pass@8": 0.4091727005940431, |
| "pass@16": 0.469938740258817, |
| "pass@32": 0.5251539812984533, |
| "pass@64": 0.5738191964496204, |
| "pass@128": 0.6142857142857143 |
| }, |
| "19": { |
| "pass@1": 0.18184590840840842, |
| "pass@2": 0.25496965962025014, |
| "pass@4": 0.32444017921183277, |
| "pass@8": 0.3837350705173519, |
| "pass@16": 0.43252962897209885, |
| "pass@32": 0.4739766807393244, |
| "pass@64": 0.5132362741820619, |
| "pass@128": 0.5555555555555556 |
| }, |
| "20": { |
| "pass@1": 0.15554617117117117, |
| "pass@2": 0.2215896999361566, |
| "pass@4": 0.2905698581033224, |
| "pass@8": 0.35598268805423944, |
| "pass@16": 0.4129009017284111, |
| "pass@32": 0.46060170177523313, |
| "pass@64": 0.5046526510505349, |
| "pass@128": 0.5495495495495496 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 560304, |
| "answer_accuracy": 0.6823655494933749, |
| "avg_response_len": 266.6313985775526, |
| "resp_tokens_sum": 218936374, |
| "pass_at_k": { |
| "pass@1": 0.6823655494933749, |
| "pass@2": 0.7343294505373057, |
| "pass@4": 0.7772818094250238, |
| "pass@8": 0.8128419064155068, |
| "pass@16": 0.8416366011429722, |
| "pass@32": 0.8647913213523676, |
| "pass@64": 0.8843245833845497, |
| "pass@128": 0.9008573655494934 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8942916027607362, |
| "pass@2": 0.9268607947683682, |
| "pass@4": 0.943626529716761, |
| "pass@8": 0.9557011401130989, |
| "pass@16": 0.9659275382630949, |
| "pass@32": 0.9750207875973297, |
| "pass@64": 0.9827243975797911, |
| "pass@128": 0.9877300613496932 |
| }, |
| "2": { |
| "pass@1": 0.9971979483282675, |
| "pass@2": 0.999441309862863, |
| "pass@4": 0.9999647811880656, |
| "pass@8": 0.9999997930990593, |
| "pass@16": 0.9999999999966434, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.999810606060606, |
| "pass@2": 0.9999940348365545, |
| "pass@4": 0.9999999957391689, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9992141272189349, |
| "pass@2": 0.9999042683921168, |
| "pass@4": 0.9999975439460601, |
| "pass@8": 0.9999999989853603, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9879755434782609, |
| "pass@2": 0.9921498773251171, |
| "pass@4": 0.9946120753927499, |
| "pass@8": 0.9961558537811471, |
| "pass@16": 0.9974390063916145, |
| "pass@32": 0.9986817921873565, |
| "pass@64": 0.9996453541509093, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9961720988538681, |
| "pass@2": 0.9977572310087314, |
| "pass@4": 0.9984983806644514, |
| "pass@8": 0.9992385963600211, |
| "pass@16": 0.9998168739502221, |
| "pass@32": 0.9999926718551041, |
| "pass@64": 0.9999999988619943, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9913069002890174, |
| "pass@2": 0.9978817958900369, |
| "pass@4": 0.9995272355767666, |
| "pass@8": 0.9999517691551582, |
| "pass@16": 0.9999994674401029, |
| "pass@32": 0.999999999984704, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9763644366197183, |
| "pass@2": 0.9885497532438727, |
| "pass@4": 0.9928375491795917, |
| "pass@8": 0.9954941968108456, |
| "pass@16": 0.9976689556338965, |
| "pass@32": 0.9992271198351556, |
| "pass@64": 0.9999179444925249, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9684927020958084, |
| "pass@2": 0.985506595077562, |
| "pass@4": 0.9912800602394759, |
| "pass@8": 0.993879931679085, |
| "pass@16": 0.9949508972328158, |
| "pass@32": 0.9957524445505505, |
| "pass@64": 0.9966405771128148, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.8972844503012049, |
| "pass@2": 0.9420525359311259, |
| "pass@4": 0.9642173082092143, |
| "pass@8": 0.977322409108135, |
| "pass@16": 0.9847742172229766, |
| "pass@32": 0.988501950486362, |
| "pass@64": 0.9904149892908543, |
| "pass@128": 0.9909638554216867 |
| }, |
| "12": { |
| "pass@1": 0.7574451410658307, |
| "pass@2": 0.8462087015279047, |
| "pass@4": 0.8947726320651916, |
| "pass@8": 0.9222422946894413, |
| "pass@16": 0.9419412675052483, |
| "pass@32": 0.9581077775676203, |
| "pass@64": 0.9714972121458975, |
| "pass@128": 0.9811912225705329 |
| }, |
| "13": { |
| "pass@1": 0.5656512605042017, |
| "pass@2": 0.6903883108361456, |
| "pass@4": 0.781639236009364, |
| "pass@8": 0.847334486689319, |
| "pass@16": 0.892639611342463, |
| "pass@32": 0.9214866482675612, |
| "pass@64": 0.9403233190215707, |
| "pass@128": 0.9551820728291317 |
| }, |
| "14": { |
| "pass@1": 0.517703764619883, |
| "pass@2": 0.635214132361744, |
| "pass@4": 0.7256948314574706, |
| "pass@8": 0.7937242292420585, |
| "pass@16": 0.8444086090861443, |
| "pass@32": 0.8828997836691251, |
| "pass@64": 0.9113636383483578, |
| "pass@128": 0.9298245614035088 |
| }, |
| "15": { |
| "pass@1": 0.39090909090909093, |
| "pass@2": 0.5073695866141734, |
| "pass@4": 0.6082879120223603, |
| "pass@8": 0.6897176659338904, |
| "pass@16": 0.7506972840275588, |
| "pass@32": 0.7948035882498893, |
| "pass@64": 0.8300640324963526, |
| "pass@128": 0.8606060606060606 |
| }, |
| "16": { |
| "pass@1": 0.26595886752136755, |
| "pass@2": 0.36421178522556474, |
| "pass@4": 0.4586782669794478, |
| "pass@8": 0.5406111579251587, |
| "pass@16": 0.610947967085004, |
| "pass@32": 0.6732607780747382, |
| "pass@64": 0.7289286165313347, |
| "pass@128": 0.7777777777777778 |
| }, |
| "17": { |
| "pass@1": 0.24046486801242237, |
| "pass@2": 0.33193591969482056, |
| "pass@4": 0.4200885497814323, |
| "pass@8": 0.500489039451954, |
| "pass@16": 0.5712970981213321, |
| "pass@32": 0.6312626766964684, |
| "pass@64": 0.6841205818138815, |
| "pass@128": 0.7267080745341615 |
| }, |
| "18": { |
| "pass@1": 0.19278492647058823, |
| "pass@2": 0.27872061718388125, |
| "pass@4": 0.37347244838880433, |
| "pass@8": 0.4647786105965824, |
| "pass@16": 0.541464741581053, |
| "pass@32": 0.6029892933824823, |
| "pass@64": 0.6562398527040539, |
| "pass@128": 0.7058823529411765 |
| }, |
| "19": { |
| "pass@1": 0.16778273809523808, |
| "pass@2": 0.24026769603018372, |
| "pass@4": 0.3167574245741603, |
| "pass@8": 0.38910715285194514, |
| "pass@16": 0.45475982302540024, |
| "pass@32": 0.5150753252814135, |
| "pass@64": 0.5726822194758491, |
| "pass@128": 0.6279761904761905 |
| }, |
| "20": { |
| "pass@1": 0.14453125, |
| "pass@2": 0.21512572787495876, |
| "pass@4": 0.2917916301817063, |
| "pass@8": 0.3657864062821782, |
| "pass@16": 0.430406495603653, |
| "pass@32": 0.4827559541949008, |
| "pass@64": 0.5273524685833563, |
| "pass@128": 0.5658682634730539 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 546144, |
| "answer_accuracy": 0.6816983543697076, |
| "avg_response_len": 285.3670826509826, |
| "resp_tokens_sum": 228622409, |
| "pass_at_k": { |
| "pass@1": 0.6816983543697076, |
| "pass@2": 0.7355420792483992, |
| "pass@4": 0.7803627771734192, |
| "pass@8": 0.8164870287934484, |
| "pass@16": 0.8443690944030742, |
| "pass@32": 0.8662511121881104, |
| "pass@64": 0.8843690678152555, |
| "pass@128": 0.8998242530755711 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9124300373134329, |
| "pass@2": 0.9467497649547544, |
| "pass@4": 0.9660806666890516, |
| "pass@8": 0.9801777355004134, |
| "pass@16": 0.9895862348226766, |
| "pass@32": 0.9944452222553659, |
| "pass@64": 0.9965440094016991, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 0.9908705357142857, |
| "pass@2": 0.993380905511811, |
| "pass@4": 0.9950889597728856, |
| "pass@8": 0.9968958537566585, |
| "pass@16": 0.9985743490699386, |
| "pass@32": 0.9996804969826077, |
| "pass@64": 0.9999909773837989, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995444606413995, |
| "pass@2": 0.9999741741465991, |
| "pass@4": 0.9999998647217203, |
| "pass@8": 0.9999999999989906, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999543128654971, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9826056985294118, |
| "pass@2": 0.9880069621352477, |
| "pass@4": 0.991431096848188, |
| "pass@8": 0.9947064171422795, |
| "pass@16": 0.9973505752472169, |
| "pass@32": 0.9987322523655447, |
| "pass@64": 0.9996410017312131, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9951202681388013, |
| "pass@2": 0.9976775379418267, |
| "pass@4": 0.9986354657166276, |
| "pass@8": 0.9994316700259301, |
| "pass@16": 0.9999103125283976, |
| "pass@32": 0.9999986355702574, |
| "pass@64": 0.9999999999868977, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9909513449367089, |
| "pass@2": 0.9975436528206918, |
| "pass@4": 0.9995421819305815, |
| "pass@8": 0.999972065880016, |
| "pass@16": 0.9999998887474985, |
| "pass@32": 0.9999999999996607, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.977172484984985, |
| "pass@2": 0.9890679311200965, |
| "pass@4": 0.9944283839520057, |
| "pass@8": 0.9973925662821129, |
| "pass@16": 0.9990184573404439, |
| "pass@32": 0.9997945619248307, |
| "pass@64": 0.9999956614464188, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.969126808681672, |
| "pass@2": 0.9890418892574123, |
| "pass@4": 0.9944441787019384, |
| "pass@8": 0.9968056115499024, |
| "pass@16": 0.9980596798731062, |
| "pass@32": 0.9989982561402365, |
| "pass@64": 0.9998084917834613, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8862847222222222, |
| "pass@2": 0.9379390076240468, |
| "pass@4": 0.9613193696026083, |
| "pass@8": 0.9751088663981844, |
| "pass@16": 0.9831448911098957, |
| "pass@32": 0.9876844692546207, |
| "pass@64": 0.9904172544400112, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.7571435397553516, |
| "pass@2": 0.8538348202460928, |
| "pass@4": 0.9113358023481001, |
| "pass@8": 0.9436768177989697, |
| "pass@16": 0.963920288868991, |
| "pass@32": 0.9789899789119665, |
| "pass@64": 0.9888266763000944, |
| "pass@128": 0.9938837920489296 |
| }, |
| "13": { |
| "pass@1": 0.580054012345679, |
| "pass@2": 0.6993470976718188, |
| "pass@4": 0.7865744510524141, |
| "pass@8": 0.8457913987269828, |
| "pass@16": 0.8864548599706131, |
| "pass@32": 0.9167394796977104, |
| "pass@64": 0.9407926514659953, |
| "pass@128": 0.9598765432098766 |
| }, |
| "14": { |
| "pass@1": 0.49100519287833827, |
| "pass@2": 0.6159493358489684, |
| "pass@4": 0.7181933542617257, |
| "pass@8": 0.7942841966244538, |
| "pass@16": 0.8460538434175928, |
| "pass@32": 0.8798747990302571, |
| "pass@64": 0.9014204637544374, |
| "pass@128": 0.9109792284866469 |
| }, |
| "15": { |
| "pass@1": 0.35927854938271603, |
| "pass@2": 0.4777715502576067, |
| "pass@4": 0.5859670073069567, |
| "pass@8": 0.6731483255421183, |
| "pass@16": 0.7375512270638794, |
| "pass@32": 0.7884761621152888, |
| "pass@64": 0.8331642522166065, |
| "pass@128": 0.8734567901234568 |
| }, |
| "16": { |
| "pass@1": 0.2764103982300885, |
| "pass@2": 0.3803106781176689, |
| "pass@4": 0.48000900744707753, |
| "pass@8": 0.5686743712608171, |
| "pass@16": 0.644046965879172, |
| "pass@32": 0.7059575693755926, |
| "pass@64": 0.7570210187661814, |
| "pass@128": 0.799410029498525 |
| }, |
| "17": { |
| "pass@1": 0.2159026731927711, |
| "pass@2": 0.30723966238971623, |
| "pass@4": 0.4014054101671026, |
| "pass@8": 0.4887071460623403, |
| "pass@16": 0.5606714325866372, |
| "pass@32": 0.6149666253716287, |
| "pass@64": 0.6586132197438247, |
| "pass@128": 0.6957831325301205 |
| }, |
| "18": { |
| "pass@1": 0.2056703629032258, |
| "pass@2": 0.288344948564897, |
| "pass@4": 0.3727806871923268, |
| "pass@8": 0.44973651536636544, |
| "pass@16": 0.5137517898127135, |
| "pass@32": 0.5668806208375831, |
| "pass@64": 0.6144127353420628, |
| "pass@128": 0.6580645161290323 |
| }, |
| "19": { |
| "pass@1": 0.18412481117824772, |
| "pass@2": 0.2632301603349429, |
| "pass@4": 0.34316810672457493, |
| "pass@8": 0.41400733209478036, |
| "pass@16": 0.47262138865768133, |
| "pass@32": 0.5240776342631858, |
| "pass@64": 0.570154694268971, |
| "pass@128": 0.6102719033232629 |
| }, |
| "20": { |
| "pass@1": 0.1600975975975976, |
| "pass@2": 0.23573425787992722, |
| "pass@4": 0.3145617574830174, |
| "pass@8": 0.38417949220434544, |
| "pass@16": 0.4426037557475695, |
| "pass@32": 0.4945963259545575, |
| "pass@64": 0.5443321801143485, |
| "pass@128": 0.5975975975975976 |
| } |
| } |
| } |
| } |
| } |
| ] |
