[ { "checkpoint": "checkpoint-43615", "id_acc": 0.9805737847222222, "ood_acc": 0.4121578125, "total_acc": 0.6814074835526316, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.14779166666668, "ood_avg_resp_len": 331.915390625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 550735, "answer_accuracy": 0.6801481485140689, "avg_response_len": 256.70740421474864, "resp_tokens_sum": 207863173, "pass_at_k": { "pass@1": 0.6801481485140689, "pass@2": 0.730230301113263, "pass@4": 0.770282546758596, "pass@8": 0.80239476506394, "pass@16": 0.8280953761715771, "pass@32": 0.8492214535571997, "pass@64": 0.867751709603317, "pass@128": 0.8849193803351248 }, "per_op_pass_at_k": { "10": { "pass@1": 0.918809918879056, "pass@2": 0.948028810419715, "pass@4": 0.962817301874876, "pass@8": 0.9724661916664145, "pass@16": 0.9787429928484136, "pass@32": 0.983707212874408, "pass@64": 0.9895069940851522, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 0.9962762850467289, "pass@2": 0.9984557822503495, "pass@4": 0.9994738122687934, "pass@8": 0.9999222873805567, "pass@16": 0.9999986070165172, "pass@32": 0.9999999998747847, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9998088685015291, "pass@2": 0.9999984950275711, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9986083984375, "pass@2": 0.9995463213582678, "pass@4": 0.999937934711286, "pass@8": 0.9999990142660407, "pass@16": 0.9999999998879098, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9961309523809524, "pass@2": 0.9971964441944757, "pass@4": 0.9976364139006433, "pass@8": 0.9982586134549394, "pass@16": 0.9990849640116398, "pass@32": 0.9997840804322056, "pass@64": 0.9999954135344047, "pass@128": 1.0 }, "6": { "pass@1": 0.9970761601796407, "pass@2": 0.9997358874770144, "pass@4": 0.9999925972015473, "pass@8": 0.9999999909949934, "pass@16": 0.9999999999999978, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9891595784023669, "pass@2": 0.993395247053068, "pass@4": 0.9951834355579812, "pass@8": 0.9965973668448541, "pass@16": 0.9973484845597674, "pass@32": 0.9977806286548048, "pass@64": 0.9985207100585047, "pass@128": 1.0 }, "8": { "pass@1": 0.9747095352564102, "pass@2": 0.9895912199676963, "pass@4": 0.994234933674156, "pass@8": 0.9961986765174754, "pass@16": 0.997689406934065, "pass@32": 0.9989578361290308, "pass@64": 0.9998090173924294, "pass@128": 1.0 }, "9": { "pass@1": 0.976012323943662, "pass@2": 0.9908849256959081, "pass@4": 0.9950709141991052, "pass@8": 0.9975627189798211, "pass@16": 0.9993104189838761, "pass@32": 0.9999512902094978, "pass@64": 0.999999951254185, "pass@128": 1.0 }, "11": { "pass@1": 0.8832551345609065, "pass@2": 0.9303415187035764, "pass@4": 0.950412839595475, "pass@8": 0.9616306162855329, "pass@16": 0.9699049289553977, "pass@32": 0.9770236314441675, "pass@64": 0.9837252279289742, "pass@128": 0.9886685552407932 }, "12": { "pass@1": 0.772753354519774, "pass@2": 0.8552530973352902, "pass@4": 0.8972007813112336, "pass@8": 0.9213115105135973, "pass@16": 0.937571762368148, "pass@32": 0.9488781345620627, "pass@64": 0.9575123939060747, "pass@128": 0.9661016949152542 }, "13": { "pass@1": 0.604648315047022, "pass@2": 0.722908773109866, "pass@4": 0.8087846775814462, "pass@8": 0.8668469492435775, "pass@16": 0.9029254689906228, "pass@32": 0.9272910665414706, "pass@64": 0.9472116958813479, "pass@128": 0.9623824451410659 }, "14": { "pass@1": 0.48761195482866043, "pass@2": 0.6136661546103467, "pass@4": 0.7130287466987187, "pass@8": 0.7846870698190367, "pass@16": 0.8326684455995763, "pass@32": 0.8645961134923782, "pass@64": 0.8881567987893952, "pass@128": 0.9065420560747663 }, "15": { "pass@1": 0.3537978684971098, "pass@2": 0.4644254073551498, "pass@4": 0.564236310417845, "pass@8": 0.6463581735993315, "pass@16": 0.7141209227548058, "pass@32": 0.770180292774696, "pass@64": 0.8143289750480056, "pass@128": 0.8439306358381503 }, "16": { "pass@1": 0.2631552419354839, "pass@2": 0.3625079375158748, "pass@4": 0.460933843753402, "pass@8": 0.5463932807791085, "pass@16": 0.6144613335763428, "pass@32": 0.6705593576451186, "pass@64": 0.7250965392578077, "pass@128": 0.7870967741935484 }, "17": { "pass@1": 0.2083634393063584, "pass@2": 0.28992692071366777, "pass@4": 0.36788058013702063, "pass@8": 0.4378366215590766, "pass@16": 0.5020683791358072, "pass@32": 0.5617083762026661, "pass@64": 0.6147709620674227, "pass@128": 0.6647398843930635 }, "18": { "pass@1": 0.19977678571428573, "pass@2": 0.27297455005624305, "pass@4": 0.34381426509186336, "pass@8": 0.4091727005940431, "pass@16": 0.469938740258817, "pass@32": 0.5251539812984533, "pass@64": 0.5738191964496204, "pass@128": 0.6142857142857143 }, "19": { "pass@1": 0.18184590840840842, "pass@2": 0.25496965962025014, "pass@4": 0.32444017921183277, "pass@8": 0.3837350705173519, "pass@16": 0.43252962897209885, "pass@32": 0.4739766807393244, "pass@64": 0.5132362741820619, "pass@128": 0.5555555555555556 }, "20": { "pass@1": 0.15554617117117117, "pass@2": 0.2215896999361566, "pass@4": 0.2905698581033224, "pass@8": 0.35598268805423944, "pass@16": 0.4129009017284111, "pass@32": 0.46060170177523313, "pass@64": 0.5046526510505349, "pass@128": 0.5495495495495496 } } }, "teachers_in_school": { "count": 821120, "correct": 560304, "answer_accuracy": 0.6823655494933749, "avg_response_len": 266.6313985775526, "resp_tokens_sum": 218936374, "pass_at_k": { "pass@1": 0.6823655494933749, "pass@2": 0.7343294505373057, "pass@4": 0.7772818094250238, "pass@8": 0.8128419064155068, "pass@16": 0.8416366011429722, "pass@32": 0.8647913213523676, "pass@64": 0.8843245833845497, "pass@128": 0.9008573655494934 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8942916027607362, "pass@2": 0.9268607947683682, "pass@4": 0.943626529716761, "pass@8": 0.9557011401130989, "pass@16": 0.9659275382630949, "pass@32": 0.9750207875973297, "pass@64": 0.9827243975797911, "pass@128": 0.9877300613496932 }, "2": { "pass@1": 0.9971979483282675, "pass@2": 0.999441309862863, "pass@4": 0.9999647811880656, "pass@8": 0.9999997930990593, "pass@16": 0.9999999999966434, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.999810606060606, "pass@2": 0.9999940348365545, "pass@4": 0.9999999957391689, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9992141272189349, "pass@2": 0.9999042683921168, "pass@4": 0.9999975439460601, "pass@8": 0.9999999989853603, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9879755434782609, "pass@2": 0.9921498773251171, "pass@4": 0.9946120753927499, "pass@8": 0.9961558537811471, "pass@16": 0.9974390063916145, "pass@32": 0.9986817921873565, "pass@64": 0.9996453541509093, "pass@128": 1.0 }, "6": { "pass@1": 0.9961720988538681, "pass@2": 0.9977572310087314, "pass@4": 0.9984983806644514, "pass@8": 0.9992385963600211, "pass@16": 0.9998168739502221, "pass@32": 0.9999926718551041, "pass@64": 0.9999999988619943, "pass@128": 1.0 }, "7": { "pass@1": 0.9913069002890174, "pass@2": 0.9978817958900369, "pass@4": 0.9995272355767666, "pass@8": 0.9999517691551582, "pass@16": 0.9999994674401029, "pass@32": 0.999999999984704, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9763644366197183, "pass@2": 0.9885497532438727, "pass@4": 0.9928375491795917, "pass@8": 0.9954941968108456, "pass@16": 0.9976689556338965, "pass@32": 0.9992271198351556, "pass@64": 0.9999179444925249, "pass@128": 1.0 }, "9": { "pass@1": 0.9684927020958084, "pass@2": 0.985506595077562, "pass@4": 0.9912800602394759, "pass@8": 0.993879931679085, "pass@16": 0.9949508972328158, "pass@32": 0.9957524445505505, "pass@64": 0.9966405771128148, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.8972844503012049, "pass@2": 0.9420525359311259, "pass@4": 0.9642173082092143, "pass@8": 0.977322409108135, "pass@16": 0.9847742172229766, "pass@32": 0.988501950486362, "pass@64": 0.9904149892908543, "pass@128": 0.9909638554216867 }, "12": { "pass@1": 0.7574451410658307, "pass@2": 0.8462087015279047, "pass@4": 0.8947726320651916, "pass@8": 0.9222422946894413, "pass@16": 0.9419412675052483, "pass@32": 0.9581077775676203, "pass@64": 0.9714972121458975, "pass@128": 0.9811912225705329 }, "13": { "pass@1": 0.5656512605042017, "pass@2": 0.6903883108361456, "pass@4": 0.781639236009364, "pass@8": 0.847334486689319, "pass@16": 0.892639611342463, "pass@32": 0.9214866482675612, "pass@64": 0.9403233190215707, "pass@128": 0.9551820728291317 }, "14": { "pass@1": 0.517703764619883, "pass@2": 0.635214132361744, "pass@4": 0.7256948314574706, "pass@8": 0.7937242292420585, "pass@16": 0.8444086090861443, "pass@32": 0.8828997836691251, "pass@64": 0.9113636383483578, "pass@128": 0.9298245614035088 }, "15": { "pass@1": 0.39090909090909093, "pass@2": 0.5073695866141734, "pass@4": 0.6082879120223603, "pass@8": 0.6897176659338904, "pass@16": 0.7506972840275588, "pass@32": 0.7948035882498893, "pass@64": 0.8300640324963526, "pass@128": 0.8606060606060606 }, "16": { "pass@1": 0.26595886752136755, "pass@2": 0.36421178522556474, "pass@4": 0.4586782669794478, "pass@8": 0.5406111579251587, "pass@16": 0.610947967085004, "pass@32": 0.6732607780747382, "pass@64": 0.7289286165313347, "pass@128": 0.7777777777777778 }, "17": { "pass@1": 0.24046486801242237, "pass@2": 0.33193591969482056, "pass@4": 0.4200885497814323, "pass@8": 0.500489039451954, "pass@16": 0.5712970981213321, "pass@32": 0.6312626766964684, "pass@64": 0.6841205818138815, "pass@128": 0.7267080745341615 }, "18": { "pass@1": 0.19278492647058823, "pass@2": 0.27872061718388125, "pass@4": 0.37347244838880433, "pass@8": 0.4647786105965824, "pass@16": 0.541464741581053, "pass@32": 0.6029892933824823, "pass@64": 0.6562398527040539, "pass@128": 0.7058823529411765 }, "19": { "pass@1": 0.16778273809523808, "pass@2": 0.24026769603018372, "pass@4": 0.3167574245741603, "pass@8": 0.38910715285194514, "pass@16": 0.45475982302540024, "pass@32": 0.5150753252814135, "pass@64": 0.5726822194758491, "pass@128": 0.6279761904761905 }, "20": { "pass@1": 0.14453125, "pass@2": 0.21512572787495876, "pass@4": 0.2917916301817063, "pass@8": 0.3657864062821782, "pass@16": 0.430406495603653, "pass@32": 0.4827559541949008, "pass@64": 0.5273524685833563, "pass@128": 0.5658682634730539 } } }, "movie_festival_awards": { "count": 801152, "correct": 546144, "answer_accuracy": 0.6816983543697076, "avg_response_len": 285.3670826509826, "resp_tokens_sum": 228622409, "pass_at_k": { "pass@1": 0.6816983543697076, "pass@2": 0.7355420792483992, "pass@4": 0.7803627771734192, "pass@8": 0.8164870287934484, "pass@16": 0.8443690944030742, "pass@32": 0.8662511121881104, "pass@64": 0.8843690678152555, "pass@128": 0.8998242530755711 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9124300373134329, "pass@2": 0.9467497649547544, "pass@4": 0.9660806666890516, "pass@8": 0.9801777355004134, "pass@16": 0.9895862348226766, "pass@32": 0.9944452222553659, "pass@64": 0.9965440094016991, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 0.9908705357142857, "pass@2": 0.993380905511811, "pass@4": 0.9950889597728856, "pass@8": 0.9968958537566585, "pass@16": 0.9985743490699386, "pass@32": 0.9996804969826077, "pass@64": 0.9999909773837989, "pass@128": 1.0 }, "3": { "pass@1": 0.9995444606413995, "pass@2": 0.9999741741465991, "pass@4": 0.9999998647217203, "pass@8": 0.9999999999989906, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999543128654971, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9826056985294118, "pass@2": 0.9880069621352477, "pass@4": 0.991431096848188, "pass@8": 0.9947064171422795, "pass@16": 0.9973505752472169, "pass@32": 0.9987322523655447, "pass@64": 0.9996410017312131, "pass@128": 1.0 }, "6": { "pass@1": 0.9951202681388013, "pass@2": 0.9976775379418267, "pass@4": 0.9986354657166276, "pass@8": 0.9994316700259301, "pass@16": 0.9999103125283976, "pass@32": 0.9999986355702574, "pass@64": 0.9999999999868977, "pass@128": 1.0 }, "7": { "pass@1": 0.9909513449367089, "pass@2": 0.9975436528206918, "pass@4": 0.9995421819305815, "pass@8": 0.999972065880016, "pass@16": 0.9999998887474985, "pass@32": 0.9999999999996607, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.977172484984985, "pass@2": 0.9890679311200965, "pass@4": 0.9944283839520057, "pass@8": 0.9973925662821129, "pass@16": 0.9990184573404439, "pass@32": 0.9997945619248307, "pass@64": 0.9999956614464188, "pass@128": 1.0 }, "9": { "pass@1": 0.969126808681672, "pass@2": 0.9890418892574123, "pass@4": 0.9944441787019384, "pass@8": 0.9968056115499024, "pass@16": 0.9980596798731062, "pass@32": 0.9989982561402365, "pass@64": 0.9998084917834613, "pass@128": 1.0 }, "11": { "pass@1": 0.8862847222222222, "pass@2": 0.9379390076240468, "pass@4": 0.9613193696026083, "pass@8": 0.9751088663981844, "pass@16": 0.9831448911098957, "pass@32": 0.9876844692546207, "pass@64": 0.9904172544400112, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.7571435397553516, "pass@2": 0.8538348202460928, "pass@4": 0.9113358023481001, "pass@8": 0.9436768177989697, "pass@16": 0.963920288868991, "pass@32": 0.9789899789119665, "pass@64": 0.9888266763000944, "pass@128": 0.9938837920489296 }, "13": { "pass@1": 0.580054012345679, "pass@2": 0.6993470976718188, "pass@4": 0.7865744510524141, "pass@8": 0.8457913987269828, "pass@16": 0.8864548599706131, "pass@32": 0.9167394796977104, "pass@64": 0.9407926514659953, "pass@128": 0.9598765432098766 }, "14": { "pass@1": 0.49100519287833827, "pass@2": 0.6159493358489684, "pass@4": 0.7181933542617257, "pass@8": 0.7942841966244538, "pass@16": 0.8460538434175928, "pass@32": 0.8798747990302571, "pass@64": 0.9014204637544374, "pass@128": 0.9109792284866469 }, "15": { "pass@1": 0.35927854938271603, "pass@2": 0.4777715502576067, "pass@4": 0.5859670073069567, "pass@8": 0.6731483255421183, "pass@16": 0.7375512270638794, "pass@32": 0.7884761621152888, "pass@64": 0.8331642522166065, "pass@128": 0.8734567901234568 }, "16": { "pass@1": 0.2764103982300885, "pass@2": 0.3803106781176689, "pass@4": 0.48000900744707753, "pass@8": 0.5686743712608171, "pass@16": 0.644046965879172, "pass@32": 0.7059575693755926, "pass@64": 0.7570210187661814, "pass@128": 0.799410029498525 }, "17": { "pass@1": 0.2159026731927711, "pass@2": 0.30723966238971623, "pass@4": 0.4014054101671026, "pass@8": 0.4887071460623403, "pass@16": 0.5606714325866372, "pass@32": 0.6149666253716287, "pass@64": 0.6586132197438247, "pass@128": 0.6957831325301205 }, "18": { "pass@1": 0.2056703629032258, "pass@2": 0.288344948564897, "pass@4": 0.3727806871923268, "pass@8": 0.44973651536636544, "pass@16": 0.5137517898127135, "pass@32": 0.5668806208375831, "pass@64": 0.6144127353420628, "pass@128": 0.6580645161290323 }, "19": { "pass@1": 0.18412481117824772, "pass@2": 0.2632301603349429, "pass@4": 0.34316810672457493, "pass@8": 0.41400733209478036, "pass@16": 0.47262138865768133, "pass@32": 0.5240776342631858, "pass@64": 0.570154694268971, "pass@128": 0.6102719033232629 }, "20": { "pass@1": 0.1600975975975976, "pass@2": 0.23573425787992722, "pass@4": 0.3145617574830174, "pass@8": 0.38417949220434544, "pass@16": 0.4426037557475695, "pass@32": 0.4945963259545575, "pass@64": 0.5443321801143485, "pass@128": 0.5975975975975976 } } } } } ]