[ { "checkpoint": "checkpoint-21414", "id_acc": 0.9839626736111111, "ood_acc": 0.42973203125, "total_acc": 0.6922623355263158, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 198.20390277777778, "ood_avg_resp_len": 325.81117734375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 555786, "answer_accuracy": 0.6863860456844768, "avg_response_len": 252.82360866858994, "resp_tokens_sum": 204718355, "pass_at_k": { "pass@1": 0.6863860456844768, "pass@2": 0.7254130326411895, "pass@4": 0.7570444952926926, "pass@8": 0.7828503778901322, "pass@16": 0.8043483587566332, "pass@32": 0.8232022811118913, "pass@64": 0.839912092115579, "pass@128": 0.8542522921277268 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9289500368731564, "pass@2": 0.950207447796902, "pass@4": 0.9618721913079447, "pass@8": 0.9683629571887911, "pass@16": 0.9731712278219837, "pass@32": 0.9785933359158274, "pass@64": 0.984001133032754, "pass@128": 0.9882005899705014 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9998566513761468, "pass@2": 0.9999962375689279, "pass@4": 0.999999998566693, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9959716796875, "pass@2": 0.9967758058562992, "pass@4": 0.9970574674259467, "pass@8": 0.9972548367535184, "pass@16": 0.9976101131889564, "pass@32": 0.9982468011811024, "pass@64": 0.9992249015748031, "pass@128": 1.0 }, "5": { "pass@1": 0.9854166666666667, "pass@2": 0.9907702943382078, "pass@4": 0.9937260092488439, "pass@8": 0.9957143551495987, "pass@16": 0.9969462948049795, "pass@32": 0.9976073886711985, "pass@64": 0.9984126953975562, "pass@128": 1.0 }, "6": { "pass@1": 0.9952984655688623, "pass@2": 0.9967252994011976, "pass@4": 0.997152760845014, "pass@8": 0.9973695543515787, "pass@16": 0.9977102880718806, "pass@32": 0.9983202885567448, "pass@64": 0.9992573907303504, "pass@128": 1.0 }, "7": { "pass@1": 0.9927884615384616, "pass@2": 0.9964302666216279, "pass@4": 0.9971738165680474, "pass@8": 0.9974006550893985, "pass@16": 0.9977373852544249, "pass@32": 0.9983401667986769, "pass@64": 0.9992661790057308, "pass@128": 1.0 }, "8": { "pass@1": 0.9754607371794872, "pass@2": 0.9849183891076114, "pass@4": 0.9904976445252037, "pass@8": 0.9947075462835548, "pass@16": 0.9971160102781994, "pass@32": 0.9981897697001509, "pass@64": 0.9992050258920796, "pass@128": 1.0 }, "9": { "pass@1": 0.976056338028169, "pass@2": 0.987308694687812, "pass@4": 0.9923681648948806, "pass@8": 0.9953406620343871, "pass@16": 0.9967589374859959, "pass@32": 0.9971595298573793, "pass@64": 0.9971830836606332, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.8893634915014165, "pass@2": 0.9216428782538864, "pass@4": 0.9426526067209013, "pass@8": 0.9567089349370961, "pass@16": 0.9657084738157397, "pass@32": 0.9726498095651721, "pass@64": 0.9781467948000009, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.7853769420903954, "pass@2": 0.8501215718003475, "pass@4": 0.8864890011100305, "pass@8": 0.9091319264414983, "pass@16": 0.9259494205388434, "pass@32": 0.9402241349478002, "pass@64": 0.9528960973309343, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.663033894984326, "pass@2": 0.7490122769234571, "pass@4": 0.8104615335112882, "pass@8": 0.853904443385092, "pass@16": 0.8845206958060587, "pass@32": 0.9088142132766218, "pass@64": 0.9311986503414087, "pass@128": 0.9498432601880877 }, "14": { "pass@1": 0.5513531931464174, "pass@2": 0.6556072466701006, "pass@4": 0.7363636414607048, "pass@8": 0.7964190181968567, "pass@16": 0.8412298993660461, "pass@32": 0.8748779050942143, "pass@64": 0.8976597401578122, "pass@128": 0.9096573208722741 }, "15": { "pass@1": 0.3803964956647399, "pass@2": 0.4697129170269898, "pass@4": 0.5485355625705742, "pass@8": 0.619549863507935, "pass@16": 0.6834557901536038, "pass@32": 0.7415316551800388, "pass@64": 0.7928237964298406, "pass@128": 0.8352601156069365 }, "16": { "pass@1": 0.26701108870967744, "pass@2": 0.33717416497332997, "pass@4": 0.4000356226641024, "pass@8": 0.4562238071753039, "pass@16": 0.5083683239776283, "pass@32": 0.5558180327950206, "pass@64": 0.5974078431290065, "pass@128": 0.6387096774193548 }, "17": { "pass@1": 0.1988800578034682, "pass@2": 0.26464679293159155, "pass@4": 0.32555858534301707, "pass@8": 0.3763516256016888, "pass@16": 0.4174320117618731, "pass@32": 0.4546616675595065, "pass@64": 0.49127100273622726, "pass@128": 0.5289017341040463 }, "18": { "pass@1": 0.1835044642857143, "pass@2": 0.24339742688413935, "pass@4": 0.30136942444694426, "pass@8": 0.3541282112243157, "pass@16": 0.40178581563790733, "pass@32": 0.44530843543552245, "pass@64": 0.48633901332077, "pass@128": 0.52 }, "19": { "pass@1": 0.16138795045045046, "pass@2": 0.22111604714951164, "pass@4": 0.28016242507974787, "pass@8": 0.3330277761775915, "pass@16": 0.3799140447797912, "pass@32": 0.4232791942691067, "pass@64": 0.46275069633006266, "pass@128": 0.4984984984984985 }, "20": { "pass@1": 0.14902402402402404, "pass@2": 0.20178089014211067, "pass@4": 0.2546022451022453, "pass@8": 0.30293074849959445, "pass@16": 0.3459261371329822, "pass@32": 0.3841312900310973, "pass@64": 0.41615493031347967, "pass@128": 0.44144144144144143 } } }, "teachers_in_school": { "count": 821120, "correct": 567811, "answer_accuracy": 0.6915079403741231, "avg_response_len": 262.04109265393606, "resp_tokens_sum": 215167182, "pass_at_k": { "pass@1": 0.6915079403741231, "pass@2": 0.7278715532002332, "pass@4": 0.7581850121618644, "pass@8": 0.7843257110454198, "pass@16": 0.8070999300078672, "pass@32": 0.8268623889973439, "pass@64": 0.8442680551190365, "pass@128": 0.8601714731098987 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9213477760736196, "pass@2": 0.9384129238925657, "pass@4": 0.949690071045567, "pass@8": 0.9598488578822636, "pass@16": 0.9693063234009107, "pass@32": 0.9768461857811139, "pass@64": 0.9819663431852611, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9994554924242425, "pass@2": 0.9999593623240276, "pass@4": 0.9999997113997114, "pass@8": 0.9999999999936351, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9986362795857988, "pass@2": 0.9994190583795369, "pass@4": 0.9998904510087127, "pass@8": 0.9999965805537689, "pass@16": 0.9999999981647446, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9928442028985507, "pass@2": 0.9961813876526304, "pass@4": 0.9970337349135704, "pass@8": 0.9971010376985435, "pass@16": 0.9971014492682352, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9960154011461319, "pass@2": 0.9972760333235567, "pass@4": 0.9980329827145533, "pass@8": 0.9987262447877927, "pass@16": 0.9994665824914142, "pass@32": 0.9999245604779609, "pass@64": 0.9999996033743879, "pass@128": 1.0 }, "7": { "pass@1": 0.9939486994219653, "pass@2": 0.9967414432661234, "pass@4": 0.99792565876881, "pass@8": 0.9987121946118677, "pass@16": 0.9994619557188004, "pass@32": 0.9999239063780588, "pass@64": 0.9999995999354374, "pass@128": 1.0 }, "8": { "pass@1": 0.9861355633802817, "pass@2": 0.9926739076189421, "pass@4": 0.9949461377187008, "pass@8": 0.9959368593159335, "pass@16": 0.9971432656494152, "pass@32": 0.9986983950949794, "pass@64": 0.9998376400133082, "pass@128": 1.0 }, "9": { "pass@1": 0.9760479041916168, "pass@2": 0.9854612870715262, "pass@4": 0.9895666641969154, "pass@8": 0.9923095976462394, "pass@16": 0.9943344279338221, "pass@32": 0.9956728561711364, "pass@64": 0.9966401626703201, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.9174275225903614, "pass@2": 0.9454114440043637, "pass@4": 0.959548336766639, "pass@8": 0.9683312183957048, "pass@16": 0.9755750310615269, "pass@32": 0.9810113701132245, "pass@64": 0.9839475990610851, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.8253085815047022, "pass@2": 0.8726485171426454, "pass@4": 0.9002097548347201, "pass@8": 0.9190767954592569, "pass@16": 0.9334631448650944, "pass@32": 0.9454421699101376, "pass@64": 0.9554340968504579, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.6405593487394958, "pass@2": 0.7243505177661617, "pass@4": 0.7812124448204476, "pass@8": 0.8222758753383121, "pass@16": 0.8522745057690294, "pass@32": 0.8760415065322167, "pass@64": 0.8987697866831307, "pass@128": 0.9215686274509803 }, "14": { "pass@1": 0.51078216374269, "pass@2": 0.6018474150435142, "pass@4": 0.6741423128030043, "pass@8": 0.733084149559791, "pass@16": 0.7828366704013225, "pass@32": 0.8214649697802924, "pass@64": 0.8487004224979805, "pass@128": 0.868421052631579 }, "15": { "pass@1": 0.3926136363636364, "pass@2": 0.48546103256979206, "pass@4": 0.5685190595493744, "pass@8": 0.642916594855587, "pass@16": 0.7080450123337044, "pass@32": 0.7614434931038847, "pass@64": 0.8066676964542822, "pass@128": 0.8515151515151516 }, "16": { "pass@1": 0.2396946225071225, "pass@2": 0.3086120645175764, "pass@4": 0.373495726228666, "pass@8": 0.4313286508099069, "pass@16": 0.483636912979026, "pass@32": 0.5345251907111577, "pass@64": 0.5869590894537149, "pass@128": 0.6410256410256411 }, "17": { "pass@1": 0.22309297360248448, "pass@2": 0.2881321220716974, "pass@4": 0.3534921134664065, "pass@8": 0.41567126050668185, "pass@16": 0.47300960147900684, "pass@32": 0.5269329873814029, "pass@64": 0.5792707962209893, "pass@128": 0.6304347826086957 }, "18": { "pass@1": 0.20043658088235294, "pass@2": 0.2591159101435849, "pass@4": 0.3163269792562694, "pass@8": 0.3723714352226891, "pass@16": 0.4283322608126674, "pass@32": 0.48312989446565124, "pass@64": 0.5330274862042342, "pass@128": 0.5764705882352941 }, "19": { "pass@1": 0.17071242559523808, "pass@2": 0.22749109486314215, "pass@4": 0.28122667841743004, "pass@8": 0.3279004246347616, "pass@16": 0.3672240446708758, "pass@32": 0.40208315220229535, "pass@64": 0.4338803669895178, "pass@128": 0.46130952380952384 }, "20": { "pass@1": 0.14046126497005987, "pass@2": 0.19736241100476215, "pass@4": 0.2579008386489113, "pass@8": 0.31531915820709905, "pass@16": 0.36357010401778933, "pass@32": 0.40095651791923803, "pass@64": 0.43026012774025857, "pass@128": 0.4550898203592814 } } }, "movie_festival_awards": { "count": 801152, "correct": 559985, "answer_accuracy": 0.6989747263939926, "avg_response_len": 281.44929551445915, "resp_tokens_sum": 225483666, "pass_at_k": { "pass@1": 0.6989747263939926, "pass@2": 0.7385376868333166, "pass@4": 0.7700180920876227, "pass@8": 0.7959329900546428, "pass@16": 0.8177631127442557, "pass@32": 0.8365219907713177, "pass@64": 0.8525210488825508, "pass@128": 0.8656334877776003 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9374766791044776, "pass@2": 0.957748413444588, "pass@4": 0.9689921052778847, "pass@8": 0.9773699904515812, "pass@16": 0.9842154775267651, "pass@32": 0.9889058966234614, "pass@64": 0.9920864277402791, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9995672376093294, "pass@2": 0.9999766849934575, "pass@4": 0.9999999464352669, "pass@8": 0.9999999999999796, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999771564327485, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.979641544117647, "pass@2": 0.9847705100741082, "pass@4": 0.9893066915348816, "pass@8": 0.993388876192616, "pass@16": 0.995921967713099, "pass@32": 0.9968539794225361, "pass@64": 0.9970545741934476, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9936662066246057, "pass@2": 0.996546967758762, "pass@4": 0.9987267336851033, "pass@8": 0.9998153946135735, "pass@16": 0.9999956432882641, "pass@32": 0.9999999986498049, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9805181962025317, "pass@2": 0.9892865263380846, "pass@4": 0.9940210415747079, "pass@8": 0.9968410245494735, "pass@16": 0.9987416398964263, "pass@32": 0.999762248074338, "pass@64": 0.999995411486273, "pass@128": 1.0 }, "8": { "pass@1": 0.9846096096096096, "pass@2": 0.9932495241304298, "pass@4": 0.9962608468986419, "pass@8": 0.997408085528068, "pass@16": 0.9979931024247288, "pass@32": 0.9987431186222496, "pass@64": 0.9996334917594761, "pass@128": 1.0 }, "9": { "pass@1": 0.9752813504823151, "pass@2": 0.989830319391346, "pass@4": 0.9941949379518873, "pass@8": 0.9961845713221275, "pass@16": 0.9979604752387462, "pass@32": 0.9994346614830938, "pass@64": 0.9999767213574137, "pass@128": 1.0 }, "11": { "pass@1": 0.9069196428571429, "pass@2": 0.940245750531184, "pass@4": 0.9565850587128986, "pass@8": 0.9655123538662552, "pass@16": 0.9712187897929012, "pass@32": 0.9765592382451729, "pass@64": 0.9812519482835874, "pass@128": 0.9841269841269841 }, "12": { "pass@1": 0.8380877293577982, "pass@2": 0.8946729995906475, "pass@4": 0.928146622440543, "pass@8": 0.9504490361522508, "pass@16": 0.9655215207898006, "pass@32": 0.97460490912977, "pass@64": 0.9798295310603202, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.6760223765432098, "pass@2": 0.7619503833722171, "pass@4": 0.815976951955079, "pass@8": 0.8532820673491517, "pass@16": 0.886261996087512, "pass@32": 0.9165778983876648, "pass@64": 0.939340032189484, "pass@128": 0.9537037037037037 }, "14": { "pass@1": 0.5342637240356083, "pass@2": 0.6260302518750437, "pass@4": 0.6962082917491397, "pass@8": 0.752977965459688, "pass@16": 0.7998110559412197, "pass@32": 0.8386935155348126, "pass@64": 0.8699713083795275, "pass@128": 0.8931750741839762 }, "15": { "pass@1": 0.4252266589506173, "pass@2": 0.5177313447798191, "pass@4": 0.5954879436366747, "pass@8": 0.660013176680045, "pass@16": 0.7092053437997673, "pass@32": 0.7453400357010892, "pass@64": 0.7730020753329683, "pass@128": 0.7962962962962963 }, "16": { "pass@1": 0.26905881268436577, "pass@2": 0.3364250894246624, "pass@4": 0.39996811045743186, "pass@8": 0.4594269086371192, "pass@16": 0.5144227742772705, "pass@32": 0.5641704618116882, "pass@64": 0.6096322933350304, "pass@128": 0.6489675516224189 }, "17": { "pass@1": 0.22162085843373494, "pass@2": 0.29668230006640733, "pass@4": 0.36697059723709247, "pass@8": 0.4279641583043071, "pass@16": 0.4788697847959879, "pass@32": 0.5211855824128357, "pass@64": 0.5571144527366223, "pass@128": 0.5843373493975904 }, "18": { "pass@1": 0.22683971774193548, "pass@2": 0.29594670751841484, "pass@4": 0.35604546548616905, "pass@8": 0.40448602772267733, "pass@16": 0.4465208932223363, "pass@32": 0.48747072957279114, "pass@64": 0.5272409128446242, "pass@128": 0.5612903225806452 }, "19": { "pass@1": 0.1691842900302115, "pass@2": 0.23326102600090398, "pass@4": 0.2953160617007466, "pass@8": 0.35129132750468284, "pass@16": 0.4001914976016223, "pass@32": 0.44363235195784895, "pass@64": 0.4828626949918139, "pass@128": 0.5166163141993958 }, "20": { "pass@1": 0.14771021021021022, "pass@2": 0.20695006029651697, "pass@4": 0.269062813657302, "pass@8": 0.3278903956004891, "pass@16": 0.38248987669662604, "pass@32": 0.4341984335907677, "pass@64": 0.48148678200868184, "pass@128": 0.5255255255255256 } } } } } ]