| [ |
| { |
| "checkpoint": "checkpoint-21414", |
| "id_acc": 0.9839626736111111, |
| "ood_acc": 0.42973203125, |
| "total_acc": 0.6922623355263158, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 198.20390277777778, |
| "ood_avg_resp_len": 325.81117734375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 555786, |
| "answer_accuracy": 0.6863860456844768, |
| "avg_response_len": 252.82360866858994, |
| "resp_tokens_sum": 204718355, |
| "pass_at_k": { |
| "pass@1": 0.6863860456844768, |
| "pass@2": 0.7254130326411895, |
| "pass@4": 0.7570444952926926, |
| "pass@8": 0.7828503778901322, |
| "pass@16": 0.8043483587566332, |
| "pass@32": 0.8232022811118913, |
| "pass@64": 0.839912092115579, |
| "pass@128": 0.8542522921277268 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9289500368731564, |
| "pass@2": 0.950207447796902, |
| "pass@4": 0.9618721913079447, |
| "pass@8": 0.9683629571887911, |
| "pass@16": 0.9731712278219837, |
| "pass@32": 0.9785933359158274, |
| "pass@64": 0.984001133032754, |
| "pass@128": 0.9882005899705014 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9998566513761468, |
| "pass@2": 0.9999962375689279, |
| "pass@4": 0.999999998566693, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9959716796875, |
| "pass@2": 0.9967758058562992, |
| "pass@4": 0.9970574674259467, |
| "pass@8": 0.9972548367535184, |
| "pass@16": 0.9976101131889564, |
| "pass@32": 0.9982468011811024, |
| "pass@64": 0.9992249015748031, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9854166666666667, |
| "pass@2": 0.9907702943382078, |
| "pass@4": 0.9937260092488439, |
| "pass@8": 0.9957143551495987, |
| "pass@16": 0.9969462948049795, |
| "pass@32": 0.9976073886711985, |
| "pass@64": 0.9984126953975562, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9952984655688623, |
| "pass@2": 0.9967252994011976, |
| "pass@4": 0.997152760845014, |
| "pass@8": 0.9973695543515787, |
| "pass@16": 0.9977102880718806, |
| "pass@32": 0.9983202885567448, |
| "pass@64": 0.9992573907303504, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9927884615384616, |
| "pass@2": 0.9964302666216279, |
| "pass@4": 0.9971738165680474, |
| "pass@8": 0.9974006550893985, |
| "pass@16": 0.9977373852544249, |
| "pass@32": 0.9983401667986769, |
| "pass@64": 0.9992661790057308, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9754607371794872, |
| "pass@2": 0.9849183891076114, |
| "pass@4": 0.9904976445252037, |
| "pass@8": 0.9947075462835548, |
| "pass@16": 0.9971160102781994, |
| "pass@32": 0.9981897697001509, |
| "pass@64": 0.9992050258920796, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.976056338028169, |
| "pass@2": 0.987308694687812, |
| "pass@4": 0.9923681648948806, |
| "pass@8": 0.9953406620343871, |
| "pass@16": 0.9967589374859959, |
| "pass@32": 0.9971595298573793, |
| "pass@64": 0.9971830836606332, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.8893634915014165, |
| "pass@2": 0.9216428782538864, |
| "pass@4": 0.9426526067209013, |
| "pass@8": 0.9567089349370961, |
| "pass@16": 0.9657084738157397, |
| "pass@32": 0.9726498095651721, |
| "pass@64": 0.9781467948000009, |
| "pass@128": 0.9830028328611898 |
| }, |
| "12": { |
| "pass@1": 0.7853769420903954, |
| "pass@2": 0.8501215718003475, |
| "pass@4": 0.8864890011100305, |
| "pass@8": 0.9091319264414983, |
| "pass@16": 0.9259494205388434, |
| "pass@32": 0.9402241349478002, |
| "pass@64": 0.9528960973309343, |
| "pass@128": 0.963276836158192 |
| }, |
| "13": { |
| "pass@1": 0.663033894984326, |
| "pass@2": 0.7490122769234571, |
| "pass@4": 0.8104615335112882, |
| "pass@8": 0.853904443385092, |
| "pass@16": 0.8845206958060587, |
| "pass@32": 0.9088142132766218, |
| "pass@64": 0.9311986503414087, |
| "pass@128": 0.9498432601880877 |
| }, |
| "14": { |
| "pass@1": 0.5513531931464174, |
| "pass@2": 0.6556072466701006, |
| "pass@4": 0.7363636414607048, |
| "pass@8": 0.7964190181968567, |
| "pass@16": 0.8412298993660461, |
| "pass@32": 0.8748779050942143, |
| "pass@64": 0.8976597401578122, |
| "pass@128": 0.9096573208722741 |
| }, |
| "15": { |
| "pass@1": 0.3803964956647399, |
| "pass@2": 0.4697129170269898, |
| "pass@4": 0.5485355625705742, |
| "pass@8": 0.619549863507935, |
| "pass@16": 0.6834557901536038, |
| "pass@32": 0.7415316551800388, |
| "pass@64": 0.7928237964298406, |
| "pass@128": 0.8352601156069365 |
| }, |
| "16": { |
| "pass@1": 0.26701108870967744, |
| "pass@2": 0.33717416497332997, |
| "pass@4": 0.4000356226641024, |
| "pass@8": 0.4562238071753039, |
| "pass@16": 0.5083683239776283, |
| "pass@32": 0.5558180327950206, |
| "pass@64": 0.5974078431290065, |
| "pass@128": 0.6387096774193548 |
| }, |
| "17": { |
| "pass@1": 0.1988800578034682, |
| "pass@2": 0.26464679293159155, |
| "pass@4": 0.32555858534301707, |
| "pass@8": 0.3763516256016888, |
| "pass@16": 0.4174320117618731, |
| "pass@32": 0.4546616675595065, |
| "pass@64": 0.49127100273622726, |
| "pass@128": 0.5289017341040463 |
| }, |
| "18": { |
| "pass@1": 0.1835044642857143, |
| "pass@2": 0.24339742688413935, |
| "pass@4": 0.30136942444694426, |
| "pass@8": 0.3541282112243157, |
| "pass@16": 0.40178581563790733, |
| "pass@32": 0.44530843543552245, |
| "pass@64": 0.48633901332077, |
| "pass@128": 0.52 |
| }, |
| "19": { |
| "pass@1": 0.16138795045045046, |
| "pass@2": 0.22111604714951164, |
| "pass@4": 0.28016242507974787, |
| "pass@8": 0.3330277761775915, |
| "pass@16": 0.3799140447797912, |
| "pass@32": 0.4232791942691067, |
| "pass@64": 0.46275069633006266, |
| "pass@128": 0.4984984984984985 |
| }, |
| "20": { |
| "pass@1": 0.14902402402402404, |
| "pass@2": 0.20178089014211067, |
| "pass@4": 0.2546022451022453, |
| "pass@8": 0.30293074849959445, |
| "pass@16": 0.3459261371329822, |
| "pass@32": 0.3841312900310973, |
| "pass@64": 0.41615493031347967, |
| "pass@128": 0.44144144144144143 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 567811, |
| "answer_accuracy": 0.6915079403741231, |
| "avg_response_len": 262.04109265393606, |
| "resp_tokens_sum": 215167182, |
| "pass_at_k": { |
| "pass@1": 0.6915079403741231, |
| "pass@2": 0.7278715532002332, |
| "pass@4": 0.7581850121618644, |
| "pass@8": 0.7843257110454198, |
| "pass@16": 0.8070999300078672, |
| "pass@32": 0.8268623889973439, |
| "pass@64": 0.8442680551190365, |
| "pass@128": 0.8601714731098987 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9213477760736196, |
| "pass@2": 0.9384129238925657, |
| "pass@4": 0.949690071045567, |
| "pass@8": 0.9598488578822636, |
| "pass@16": 0.9693063234009107, |
| "pass@32": 0.9768461857811139, |
| "pass@64": 0.9819663431852611, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9994554924242425, |
| "pass@2": 0.9999593623240276, |
| "pass@4": 0.9999997113997114, |
| "pass@8": 0.9999999999936351, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9986362795857988, |
| "pass@2": 0.9994190583795369, |
| "pass@4": 0.9998904510087127, |
| "pass@8": 0.9999965805537689, |
| "pass@16": 0.9999999981647446, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9928442028985507, |
| "pass@2": 0.9961813876526304, |
| "pass@4": 0.9970337349135704, |
| "pass@8": 0.9971010376985435, |
| "pass@16": 0.9971014492682352, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9960154011461319, |
| "pass@2": 0.9972760333235567, |
| "pass@4": 0.9980329827145533, |
| "pass@8": 0.9987262447877927, |
| "pass@16": 0.9994665824914142, |
| "pass@32": 0.9999245604779609, |
| "pass@64": 0.9999996033743879, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9939486994219653, |
| "pass@2": 0.9967414432661234, |
| "pass@4": 0.99792565876881, |
| "pass@8": 0.9987121946118677, |
| "pass@16": 0.9994619557188004, |
| "pass@32": 0.9999239063780588, |
| "pass@64": 0.9999995999354374, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9861355633802817, |
| "pass@2": 0.9926739076189421, |
| "pass@4": 0.9949461377187008, |
| "pass@8": 0.9959368593159335, |
| "pass@16": 0.9971432656494152, |
| "pass@32": 0.9986983950949794, |
| "pass@64": 0.9998376400133082, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9760479041916168, |
| "pass@2": 0.9854612870715262, |
| "pass@4": 0.9895666641969154, |
| "pass@8": 0.9923095976462394, |
| "pass@16": 0.9943344279338221, |
| "pass@32": 0.9956728561711364, |
| "pass@64": 0.9966401626703201, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.9174275225903614, |
| "pass@2": 0.9454114440043637, |
| "pass@4": 0.959548336766639, |
| "pass@8": 0.9683312183957048, |
| "pass@16": 0.9755750310615269, |
| "pass@32": 0.9810113701132245, |
| "pass@64": 0.9839475990610851, |
| "pass@128": 0.9849397590361446 |
| }, |
| "12": { |
| "pass@1": 0.8253085815047022, |
| "pass@2": 0.8726485171426454, |
| "pass@4": 0.9002097548347201, |
| "pass@8": 0.9190767954592569, |
| "pass@16": 0.9334631448650944, |
| "pass@32": 0.9454421699101376, |
| "pass@64": 0.9554340968504579, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.6405593487394958, |
| "pass@2": 0.7243505177661617, |
| "pass@4": 0.7812124448204476, |
| "pass@8": 0.8222758753383121, |
| "pass@16": 0.8522745057690294, |
| "pass@32": 0.8760415065322167, |
| "pass@64": 0.8987697866831307, |
| "pass@128": 0.9215686274509803 |
| }, |
| "14": { |
| "pass@1": 0.51078216374269, |
| "pass@2": 0.6018474150435142, |
| "pass@4": 0.6741423128030043, |
| "pass@8": 0.733084149559791, |
| "pass@16": 0.7828366704013225, |
| "pass@32": 0.8214649697802924, |
| "pass@64": 0.8487004224979805, |
| "pass@128": 0.868421052631579 |
| }, |
| "15": { |
| "pass@1": 0.3926136363636364, |
| "pass@2": 0.48546103256979206, |
| "pass@4": 0.5685190595493744, |
| "pass@8": 0.642916594855587, |
| "pass@16": 0.7080450123337044, |
| "pass@32": 0.7614434931038847, |
| "pass@64": 0.8066676964542822, |
| "pass@128": 0.8515151515151516 |
| }, |
| "16": { |
| "pass@1": 0.2396946225071225, |
| "pass@2": 0.3086120645175764, |
| "pass@4": 0.373495726228666, |
| "pass@8": 0.4313286508099069, |
| "pass@16": 0.483636912979026, |
| "pass@32": 0.5345251907111577, |
| "pass@64": 0.5869590894537149, |
| "pass@128": 0.6410256410256411 |
| }, |
| "17": { |
| "pass@1": 0.22309297360248448, |
| "pass@2": 0.2881321220716974, |
| "pass@4": 0.3534921134664065, |
| "pass@8": 0.41567126050668185, |
| "pass@16": 0.47300960147900684, |
| "pass@32": 0.5269329873814029, |
| "pass@64": 0.5792707962209893, |
| "pass@128": 0.6304347826086957 |
| }, |
| "18": { |
| "pass@1": 0.20043658088235294, |
| "pass@2": 0.2591159101435849, |
| "pass@4": 0.3163269792562694, |
| "pass@8": 0.3723714352226891, |
| "pass@16": 0.4283322608126674, |
| "pass@32": 0.48312989446565124, |
| "pass@64": 0.5330274862042342, |
| "pass@128": 0.5764705882352941 |
| }, |
| "19": { |
| "pass@1": 0.17071242559523808, |
| "pass@2": 0.22749109486314215, |
| "pass@4": 0.28122667841743004, |
| "pass@8": 0.3279004246347616, |
| "pass@16": 0.3672240446708758, |
| "pass@32": 0.40208315220229535, |
| "pass@64": 0.4338803669895178, |
| "pass@128": 0.46130952380952384 |
| }, |
| "20": { |
| "pass@1": 0.14046126497005987, |
| "pass@2": 0.19736241100476215, |
| "pass@4": 0.2579008386489113, |
| "pass@8": 0.31531915820709905, |
| "pass@16": 0.36357010401778933, |
| "pass@32": 0.40095651791923803, |
| "pass@64": 0.43026012774025857, |
| "pass@128": 0.4550898203592814 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 559985, |
| "answer_accuracy": 0.6989747263939926, |
| "avg_response_len": 281.44929551445915, |
| "resp_tokens_sum": 225483666, |
| "pass_at_k": { |
| "pass@1": 0.6989747263939926, |
| "pass@2": 0.7385376868333166, |
| "pass@4": 0.7700180920876227, |
| "pass@8": 0.7959329900546428, |
| "pass@16": 0.8177631127442557, |
| "pass@32": 0.8365219907713177, |
| "pass@64": 0.8525210488825508, |
| "pass@128": 0.8656334877776003 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9374766791044776, |
| "pass@2": 0.957748413444588, |
| "pass@4": 0.9689921052778847, |
| "pass@8": 0.9773699904515812, |
| "pass@16": 0.9842154775267651, |
| "pass@32": 0.9889058966234614, |
| "pass@64": 0.9920864277402791, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9995672376093294, |
| "pass@2": 0.9999766849934575, |
| "pass@4": 0.9999999464352669, |
| "pass@8": 0.9999999999999796, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999771564327485, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.979641544117647, |
| "pass@2": 0.9847705100741082, |
| "pass@4": 0.9893066915348816, |
| "pass@8": 0.993388876192616, |
| "pass@16": 0.995921967713099, |
| "pass@32": 0.9968539794225361, |
| "pass@64": 0.9970545741934476, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9936662066246057, |
| "pass@2": 0.996546967758762, |
| "pass@4": 0.9987267336851033, |
| "pass@8": 0.9998153946135735, |
| "pass@16": 0.9999956432882641, |
| "pass@32": 0.9999999986498049, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9805181962025317, |
| "pass@2": 0.9892865263380846, |
| "pass@4": 0.9940210415747079, |
| "pass@8": 0.9968410245494735, |
| "pass@16": 0.9987416398964263, |
| "pass@32": 0.999762248074338, |
| "pass@64": 0.999995411486273, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9846096096096096, |
| "pass@2": 0.9932495241304298, |
| "pass@4": 0.9962608468986419, |
| "pass@8": 0.997408085528068, |
| "pass@16": 0.9979931024247288, |
| "pass@32": 0.9987431186222496, |
| "pass@64": 0.9996334917594761, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9752813504823151, |
| "pass@2": 0.989830319391346, |
| "pass@4": 0.9941949379518873, |
| "pass@8": 0.9961845713221275, |
| "pass@16": 0.9979604752387462, |
| "pass@32": 0.9994346614830938, |
| "pass@64": 0.9999767213574137, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9069196428571429, |
| "pass@2": 0.940245750531184, |
| "pass@4": 0.9565850587128986, |
| "pass@8": 0.9655123538662552, |
| "pass@16": 0.9712187897929012, |
| "pass@32": 0.9765592382451729, |
| "pass@64": 0.9812519482835874, |
| "pass@128": 0.9841269841269841 |
| }, |
| "12": { |
| "pass@1": 0.8380877293577982, |
| "pass@2": 0.8946729995906475, |
| "pass@4": 0.928146622440543, |
| "pass@8": 0.9504490361522508, |
| "pass@16": 0.9655215207898006, |
| "pass@32": 0.97460490912977, |
| "pass@64": 0.9798295310603202, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.6760223765432098, |
| "pass@2": 0.7619503833722171, |
| "pass@4": 0.815976951955079, |
| "pass@8": 0.8532820673491517, |
| "pass@16": 0.886261996087512, |
| "pass@32": 0.9165778983876648, |
| "pass@64": 0.939340032189484, |
| "pass@128": 0.9537037037037037 |
| }, |
| "14": { |
| "pass@1": 0.5342637240356083, |
| "pass@2": 0.6260302518750437, |
| "pass@4": 0.6962082917491397, |
| "pass@8": 0.752977965459688, |
| "pass@16": 0.7998110559412197, |
| "pass@32": 0.8386935155348126, |
| "pass@64": 0.8699713083795275, |
| "pass@128": 0.8931750741839762 |
| }, |
| "15": { |
| "pass@1": 0.4252266589506173, |
| "pass@2": 0.5177313447798191, |
| "pass@4": 0.5954879436366747, |
| "pass@8": 0.660013176680045, |
| "pass@16": 0.7092053437997673, |
| "pass@32": 0.7453400357010892, |
| "pass@64": 0.7730020753329683, |
| "pass@128": 0.7962962962962963 |
| }, |
| "16": { |
| "pass@1": 0.26905881268436577, |
| "pass@2": 0.3364250894246624, |
| "pass@4": 0.39996811045743186, |
| "pass@8": 0.4594269086371192, |
| "pass@16": 0.5144227742772705, |
| "pass@32": 0.5641704618116882, |
| "pass@64": 0.6096322933350304, |
| "pass@128": 0.6489675516224189 |
| }, |
| "17": { |
| "pass@1": 0.22162085843373494, |
| "pass@2": 0.29668230006640733, |
| "pass@4": 0.36697059723709247, |
| "pass@8": 0.4279641583043071, |
| "pass@16": 0.4788697847959879, |
| "pass@32": 0.5211855824128357, |
| "pass@64": 0.5571144527366223, |
| "pass@128": 0.5843373493975904 |
| }, |
| "18": { |
| "pass@1": 0.22683971774193548, |
| "pass@2": 0.29594670751841484, |
| "pass@4": 0.35604546548616905, |
| "pass@8": 0.40448602772267733, |
| "pass@16": 0.4465208932223363, |
| "pass@32": 0.48747072957279114, |
| "pass@64": 0.5272409128446242, |
| "pass@128": 0.5612903225806452 |
| }, |
| "19": { |
| "pass@1": 0.1691842900302115, |
| "pass@2": 0.23326102600090398, |
| "pass@4": 0.2953160617007466, |
| "pass@8": 0.35129132750468284, |
| "pass@16": 0.4001914976016223, |
| "pass@32": 0.44363235195784895, |
| "pass@64": 0.4828626949918139, |
| "pass@128": 0.5166163141993958 |
| }, |
| "20": { |
| "pass@1": 0.14771021021021022, |
| "pass@2": 0.20695006029651697, |
| "pass@4": 0.269062813657302, |
| "pass@8": 0.3278903956004891, |
| "pass@16": 0.38248987669662604, |
| "pass@32": 0.4341984335907677, |
| "pass@64": 0.48148678200868184, |
| "pass@128": 0.5255255255255256 |
| } |
| } |
| } |
| } |
| } |
| ] |
