[ { "checkpoint": "checkpoint-21546", "id_acc": 0.6596380208333333, "ood_acc": 0.2007515625, "total_acc": 0.4181188322368421, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 207.44738020833333, "ood_avg_resp_len": 315.76594609375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 326230, "answer_accuracy": 0.4028883773316472, "avg_response_len": 252.42235170328803, "resp_tokens_sum": 204393446, "pass_at_k": { "pass@1": 0.4028883773316472, "pass@2": 0.5040771540897414, "pass@4": 0.599082216208909, "pass@8": 0.678747216971971, "pass@16": 0.7400238654947899, "pass@32": 0.7848226074309053, "pass@64": 0.8170806105461791, "pass@128": 0.8414479924122669 }, "per_op_pass_at_k": { "10": { "pass@1": 0.371427912979351, "pass@2": 0.523322997235965, "pass@4": 0.6718812374669977, "pass@8": 0.7933920591197899, "pass@16": 0.8779724027500238, "pass@32": 0.9314886148824195, "pass@64": 0.9611391857168347, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 0.9139164719626168, "pass@2": 0.9309390867613512, "pass@4": 0.9439649050877985, "pass@8": 0.9533337561204487, "pass@16": 0.9605609138316018, "pass@32": 0.966119078288411, "pass@64": 0.9697424698773338, "pass@128": 0.9719626168224299 }, "3": { "pass@1": 0.9014000382262997, "pass@2": 0.9560262105998218, "pass@4": 0.9798727122985766, "pass@8": 0.9879021076001016, "pass@16": 0.9912037087331826, "pass@32": 0.9936467679984826, "pass@64": 0.9958006824234752, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.7223876953125, "pass@2": 0.8201191098671256, "pass@4": 0.8897095618907006, "pass@8": 0.9320338826425963, "pass@16": 0.956386741149801, "pass@32": 0.9713648996011031, "pass@64": 0.9804295567954489, "pass@128": 0.984375 }, "5": { "pass@1": 0.629811507936508, "pass@2": 0.7491188601424822, "pass@4": 0.8407026499663729, "pass@8": 0.9022890763307583, "pass@16": 0.9401223531283432, "pass@32": 0.96133074752117, "pass@64": 0.9711185300201212, "pass@128": 0.9777777777777777 }, "6": { "pass@1": 0.6804126122754491, "pass@2": 0.7908088842001034, "pass@4": 0.8711181910644397, "pass@8": 0.9226813205784111, "pass@16": 0.9523163473121824, "pass@32": 0.9702453653333176, "pass@64": 0.9832286283027216, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.5659208579881657, "pass@2": 0.703538502306295, "pass@4": 0.8161317657962868, "pass@8": 0.8944902079834598, "pass@16": 0.9397179109827034, "pass@32": 0.9645216715274736, "pass@64": 0.9810000758817367, "pass@128": 0.9911242603550295 }, "8": { "pass@1": 0.5125951522435898, "pass@2": 0.6551375902230964, "pass@4": 0.7706036381870534, "pass@8": 0.8512998182377252, "pass@16": 0.9029006956766283, "pass@32": 0.9340055360413846, "pass@64": 0.9518592357512387, "pass@128": 0.9647435897435898 }, "9": { "pass@1": 0.4185519366197183, "pass@2": 0.5713655456360205, "pass@4": 0.7097021433060301, "pass@8": 0.8166626354564073, "pass@16": 0.8912920455560849, "pass@32": 0.9391179746651805, "pass@64": 0.9667627928829464, "pass@128": 0.9802816901408451 }, "11": { "pass@1": 0.3921963526912181, "pass@2": 0.5415787206397362, "pass@4": 0.6760388732392862, "pass@8": 0.7790355829347181, "pass@16": 0.8543600624135301, "pass@32": 0.9077318451268638, "pass@64": 0.9404013355393905, "pass@128": 0.9631728045325779 }, "12": { "pass@1": 0.2990819209039548, "pass@2": 0.42569940277592416, "pass@4": 0.5545588146820628, "pass@8": 0.6733555030767344, "pass@16": 0.7733669214501476, "pass@32": 0.8514951163900696, "pass@64": 0.905878813478266, "pass@128": 0.9350282485875706 }, "13": { "pass@1": 0.26119220219435735, "pass@2": 0.37001201774738973, "pass@4": 0.48377948671384713, "pass@8": 0.5913443740517459, "pass@16": 0.6861284752670558, "pass@32": 0.764477112035867, "pass@64": 0.8285231373938065, "pass@128": 0.8840125391849529 }, "14": { "pass@1": 0.21434482087227413, "pass@2": 0.3150246369612677, "pass@4": 0.4199875996808808, "pass@8": 0.5154268383070109, "pass@16": 0.597963503716804, "pass@32": 0.670862546098087, "pass@64": 0.736826979824222, "pass@128": 0.7912772585669782 }, "15": { "pass@1": 0.17119761560693642, "pass@2": 0.2615891402303036, "pass@4": 0.3643629110125685, "pass@8": 0.4638810821555635, "pass@16": 0.5488619660921231, "pass@32": 0.6166011084092652, "pass@64": 0.6693182181911753, "pass@128": 0.7138728323699421 }, "16": { "pass@1": 0.16743951612903227, "pass@2": 0.2455216535433071, "pass@4": 0.332099522539521, "pass@8": 0.4180017910081934, "pass@16": 0.49562935124361973, "pass@32": 0.5626821505928203, "pass@64": 0.6208124764624271, "pass@128": 0.667741935483871 }, "17": { "pass@1": 0.13091582369942195, "pass@2": 0.2026965232579309, "pass@4": 0.28583829468932, "pass@8": 0.36784673635324805, "pass@16": 0.4371569990362648, "pass@32": 0.49075281833375817, "pass@64": 0.5351253719194886, "pass@128": 0.5809248554913294 }, "18": { "pass@1": 0.140625, "pass@2": 0.21281390607424075, "pass@4": 0.2967270649204563, "pass@8": 0.38152916752215504, "pass@16": 0.45486813332834614, "pass@32": 0.5073866663449251, "pass@64": 0.5420570454433494, "pass@128": 0.5742857142857143 }, "19": { "pass@1": 0.11249530780780781, "pass@2": 0.17809714241800853, "pass@4": 0.25848685102425245, "pass@8": 0.3399542597368296, "pass@16": 0.40889877405287706, "pass@32": 0.46292300786984686, "pass@64": 0.5045700654986855, "pass@128": 0.5345345345345346 }, "20": { "pass@1": 0.11549831081081081, "pass@2": 0.18252775413208475, "pass@4": 0.26408397570799147, "pass@8": 0.3473522314072748, "pass@16": 0.4172471386714921, "pass@32": 0.465631560238557, "pass@64": 0.4982591181602525, "pass@128": 0.5255255255255256 } } }, "teachers_in_school": { "count": 821120, "correct": 343922, "answer_accuracy": 0.4188449922057677, "avg_response_len": 261.5548372954014, "resp_tokens_sum": 214767908, "pass_at_k": { "pass@1": 0.4188449922057677, "pass@2": 0.5203707170080039, "pass@4": 0.6137221266476559, "pass@8": 0.6911413890808137, "pass@16": 0.7503143984737648, "pass@32": 0.7929309365523514, "pass@64": 0.8235313189554284, "pass@128": 0.8483242400623539 }, "per_op_pass_at_k": { "10": { "pass@1": 0.42939992331288346, "pass@2": 0.5845611323124487, "pass@4": 0.7219675550525508, "pass@8": 0.8269844795437781, "pass@16": 0.8993004726366725, "pass@32": 0.9453174771206605, "pass@64": 0.9721708965924318, "pass@128": 0.9877300613496932 }, "2": { "pass@1": 0.9301624240121581, "pass@2": 0.9487538293085703, "pass@4": 0.9608639527012007, "pass@8": 0.9687376166611839, "pass@16": 0.9746019959801766, "pass@32": 0.9795867935302768, "pass@64": 0.9830901743667175, "pass@128": 0.9848024316109423 }, "3": { "pass@1": 0.860653409090909, "pass@2": 0.9269099707706995, "pass@4": 0.9605988600856707, "pass@8": 0.9739233460421274, "pass@16": 0.9793009784562008, "pass@32": 0.9826759601394346, "pass@64": 0.9857259818108475, "pass@128": 0.9878787878787879 }, "4": { "pass@1": 0.7838618713017751, "pass@2": 0.8724043266318782, "pass@4": 0.9301489839657617, "pass@8": 0.960023013690949, "pass@16": 0.97335069655027, "pass@32": 0.9804791565050158, "pass@64": 0.9861651795826174, "pass@128": 0.9911242603550295 }, "5": { "pass@1": 0.6375226449275362, "pass@2": 0.7545318384115028, "pass@4": 0.8361816756601076, "pass@8": 0.8851649784226489, "pass@16": 0.9156569142518054, "pass@32": 0.9365426588707715, "pass@64": 0.9514844580155023, "pass@128": 0.9623188405797102 }, "6": { "pass@1": 0.6906787249283668, "pass@2": 0.8065539477246575, "pass@4": 0.8880656017567995, "pass@8": 0.9351576339738463, "pass@16": 0.9588356055013254, "pass@32": 0.9712646483259476, "pass@64": 0.9791367802266179, "pass@128": 0.9856733524355301 }, "7": { "pass@1": 0.6023753612716763, "pass@2": 0.7327194085385282, "pass@4": 0.8357873419724274, "pass@8": 0.9079161103488637, "pass@16": 0.9501706105430924, "pass@32": 0.9716194979336098, "pass@64": 0.982082756700385, "pass@128": 0.9884393063583815 }, "8": { "pass@1": 0.5243617957746479, "pass@2": 0.673133733503383, "pass@4": 0.7976204885480864, "pass@8": 0.885373322021841, "pass@16": 0.9396246435828248, "pass@32": 0.9694345592489272, "pass@64": 0.9842985934546734, "pass@128": 0.9915492957746479 }, "9": { "pass@1": 0.48940400449101795, "pass@2": 0.6394930365646656, "pass@4": 0.7650968950737445, "pass@8": 0.8614388295275515, "pass@16": 0.9280775013732009, "pass@32": 0.9675055589240936, "pass@64": 0.9853653966250359, "pass@128": 0.9910179640718563 }, "11": { "pass@1": 0.3600809487951807, "pass@2": 0.5060478133004463, "pass@4": 0.647385364574157, "pass@8": 0.765477598410561, "pass@16": 0.8514365450245749, "pass@32": 0.9050546071868998, "pass@64": 0.9358720866999578, "pass@128": 0.9578313253012049 }, "12": { "pass@1": 0.3449990203761755, "pass@2": 0.47940900143163934, "pass@4": 0.6083281063221331, "pass@8": 0.716982685718736, "pass@16": 0.7997644746548964, "pass@32": 0.8590672882733515, "pass@64": 0.9032935756972884, "pass@128": 0.9435736677115988 }, "13": { "pass@1": 0.23816088935574228, "pass@2": 0.34847689075630256, "pass@4": 0.4657425271946047, "pass@8": 0.5719295256799126, "pass@16": 0.6597921500111992, "pass@32": 0.7322101517746895, "pass@64": 0.7924964201460486, "pass@128": 0.8487394957983193 }, "14": { "pass@1": 0.22480354532163743, "pass@2": 0.32782389660174055, "pass@4": 0.4342883085338015, "pass@8": 0.5318416120559322, "pass@16": 0.614830521987863, "pass@32": 0.6842228584791915, "pass@64": 0.7460521925082643, "pass@128": 0.804093567251462 }, "15": { "pass@1": 0.1651278409090909, "pass@2": 0.24669828203292754, "pass@4": 0.3395767744940974, "pass@8": 0.4333264190796036, "pass@16": 0.5224560313801565, "pass@32": 0.6018586127709377, "pass@64": 0.6662757719754229, "pass@128": 0.7181818181818181 }, "16": { "pass@1": 0.15731837606837606, "pass@2": 0.2312620577876484, "pass@4": 0.31380667774434196, "pass@8": 0.3952425891605544, "pass@16": 0.4682784764472947, "pass@32": 0.5316408674982099, "pass@64": 0.587040470177276, "pass@128": 0.6381766381766382 }, "17": { "pass@1": 0.1607628105590062, "pass@2": 0.2452281355455568, "pass@4": 0.3430799177082678, "pass@8": 0.4389751464459959, "pass@16": 0.5188112906635529, "pass@32": 0.5781936339374035, "pass@64": 0.6212796797280741, "pass@128": 0.65527950310559 }, "18": { "pass@1": 0.1396829044117647, "pass@2": 0.2133181594488189, "pass@4": 0.3000968705198614, "pass@8": 0.39007173591004785, "pass@16": 0.4709636864254259, "pass@32": 0.5343133483461368, "pass@64": 0.5793604777810965, "pass@128": 0.611764705882353 }, "19": { "pass@1": 0.10928199404761904, "pass@2": 0.17229316062054745, "pass@4": 0.2538720503129072, "pass@8": 0.34257004009238523, "pass@16": 0.420559560046734, "pass@32": 0.4759493512788768, "pass@64": 0.5138949420123206, "pass@128": 0.5505952380952381 }, "20": { "pass@1": 0.10689558383233533, "pass@2": 0.1719055736951294, "pass@4": 0.25167281522444424, "pass@8": 0.3348155934449863, "pass@16": 0.4058721468049287, "pass@32": 0.45523155365388795, "pass@64": 0.48832122929571226, "pass@128": 0.5149700598802395 } } }, "movie_festival_awards": { "count": 801152, "correct": 346713, "answer_accuracy": 0.4327680639878575, "avg_response_len": 279.5954313288864, "resp_tokens_sum": 223998439, "pass_at_k": { "pass@1": 0.4327680639878575, "pass@2": 0.5294413249644969, "pass@4": 0.619586927078482, "pass@8": 0.6954290251605538, "pass@16": 0.7532936024836905, "pass@32": 0.7945588840019715, "pass@64": 0.8236257194679109, "pass@128": 0.8448633967087394 }, "per_op_pass_at_k": { "10": { "pass@1": 0.47495335820895523, "pass@2": 0.623364231989658, "pass@4": 0.7471682490248422, "pass@8": 0.8398494017450072, "pass@16": 0.9037033213806163, "pass@32": 0.9430667350935134, "pass@64": 0.963281894325653, "pass@128": 0.9761194029850746 }, "2": { "pass@1": 0.9677455357142857, "pass@2": 0.9768110236220473, "pass@4": 0.9826745586266004, "pass@8": 0.9871195351191154, "pass@16": 0.9899088225558785, "pass@32": 0.9911571674097157, "pass@64": 0.9914240481035442, "pass@128": 0.9914285714285714 }, "3": { "pass@1": 0.8923104956268222, "pass@2": 0.9460619877872416, "pass@4": 0.9716916010498681, "pass@8": 0.9819850579920034, "pass@16": 0.9853747770803383, "pass@32": 0.9866242439336862, "pass@64": 0.9876146649487308, "pass@128": 0.9883381924198251 }, "4": { "pass@1": 0.8124086257309941, "pass@2": 0.8793381193304781, "pass@4": 0.9236651819290124, "pass@8": 0.952532626265685, "pass@16": 0.9713757273390042, "pass@32": 0.9841841950589343, "pass@64": 0.9915764391070508, "pass@128": 0.9941520467836257 }, "5": { "pass@1": 0.67109375, "pass@2": 0.7686660490968045, "pass@4": 0.846831118077152, "pass@8": 0.9036536299326656, "pass@16": 0.9391996061262696, "pass@32": 0.9592355455929494, "pass@64": 0.9716352127256529, "pass@128": 0.9794117647058823 }, "6": { "pass@1": 0.7108142744479495, "pass@2": 0.8149187138279639, "pass@4": 0.890687050159739, "pass@8": 0.9365963614462343, "pass@16": 0.9617556334998504, "pass@32": 0.9761391150509602, "pass@64": 0.9848625244179728, "pass@128": 0.9905362776025236 }, "7": { "pass@1": 0.628065664556962, "pass@2": 0.7528550315209811, "pass@4": 0.8482620916451892, "pass@8": 0.9137656555014311, "pass@16": 0.950796041870444, "pass@32": 0.9702759776932505, "pass@64": 0.9808180164586792, "pass@128": 0.9841772151898734 }, "8": { "pass@1": 0.5572916666666666, "pass@2": 0.7052213384644487, "pass@4": 0.8218555895603137, "pass@8": 0.902596510105195, "pass@16": 0.9517949896970779, "pass@32": 0.978889599701662, "pass@64": 0.9926690796876108, "pass@128": 1.0 }, "9": { "pass@1": 0.42710008038585207, "pass@2": 0.5741832449299945, "pass@4": 0.7114348669639761, "pass@8": 0.8201747611420893, "pass@16": 0.8954404581504106, "pass@32": 0.9422447763282753, "pass@64": 0.9695241077569068, "pass@128": 0.9839228295819936 }, "11": { "pass@1": 0.3827380952380952, "pass@2": 0.5340582427196598, "pass@4": 0.6711852259538985, "pass@8": 0.7780117481638958, "pass@16": 0.8522580284381895, "pass@32": 0.8996968127590125, "pass@64": 0.9290359324611956, "pass@128": 0.946031746031746 }, "12": { "pass@1": 0.3052131116207951, "pass@2": 0.43907156745888415, "pass@4": 0.5709099071905003, "pass@8": 0.6826387187898862, "pass@16": 0.7701434675459031, "pass@32": 0.8358149758684412, "pass@64": 0.8831344125460127, "pass@128": 0.9174311926605505 }, "13": { "pass@1": 0.23647280092592593, "pass@2": 0.3409821710654225, "pass@4": 0.45619952598517804, "pass@8": 0.5672321433778583, "pass@16": 0.6618549595277416, "pass@32": 0.7346381674583468, "pass@64": 0.7900206420715697, "pass@128": 0.8333333333333334 }, "14": { "pass@1": 0.22860255934718102, "pass@2": 0.3341221465454799, "pass@4": 0.44441233912897393, "pass@8": 0.5441630780134137, "pass@16": 0.6282250534305504, "pass@32": 0.695872480912015, "pass@64": 0.7486191756316015, "pass@128": 0.7893175074183977 }, "15": { "pass@1": 0.18356963734567902, "pass@2": 0.28285624453193353, "pass@4": 0.3948922252773957, "pass@8": 0.5030045404650186, "pass@16": 0.5949260503221181, "pass@32": 0.6677900621586327, "pass@64": 0.723897627411919, "pass@128": 0.7654320987654321 }, "16": { "pass@1": 0.1549594395280236, "pass@2": 0.22994078519499225, "pass@4": 0.31570141425905845, "pass@8": 0.40062593291544546, "pass@16": 0.47466509306307514, "pass@32": 0.537772987756483, "pass@64": 0.5930868736296269, "pass@128": 0.6371681415929203 }, "17": { "pass@1": 0.1441312123493976, "pass@2": 0.22216375343895262, "pass@4": 0.311510106229191, "pass@8": 0.3988157662825498, "pass@16": 0.47142989450471834, "pass@32": 0.5239278118241953, "pass@64": 0.564364886384367, "pass@128": 0.5993975903614458 }, "18": { "pass@1": 0.13981854838709679, "pass@2": 0.20924958724917453, "pass@4": 0.2913785064165365, "pass@8": 0.37509974076412117, "pass@16": 0.4452421080127842, "pass@32": 0.49770270518776283, "pass@64": 0.539415525844793, "pass@128": 0.5774193548387097 }, "19": { "pass@1": 0.11935895015105741, "pass@2": 0.1856285088136642, "pass@4": 0.263486335450516, "pass@8": 0.3413125930922642, "pass@16": 0.4100031762366848, "pass@32": 0.46414237314575196, "pass@64": 0.5039823180493795, "pass@128": 0.5347432024169184 }, "20": { "pass@1": 0.11969782282282282, "pass@2": 0.1870740080631812, "pass@4": 0.26937247765200556, "pass@8": 0.35681385546279937, "pass@16": 0.43545605263414, "pass@32": 0.49350455732966975, "pass@64": 0.5292488342076239, "pass@128": 0.5555555555555556 } } } } } ]