[ { "checkpoint": "checkpoint-85656", "id_acc": 0.5962326388888889, "ood_acc": 0.16850078125, "total_acc": 0.37111060855263156, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.43200520833332, "ood_avg_resp_len": 317.42343671875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 296122, "answer_accuracy": 0.3657055208662662, "avg_response_len": 247.53881056354726, "resp_tokens_sum": 200439106, "pass_at_k": { "pass@1": 0.3657055208662662, "pass@2": 0.4645592080614758, "pass@4": 0.5663767038851872, "pass@8": 0.6576015416220393, "pass@16": 0.7305242905130579, "pass@32": 0.7864036837810774, "pass@64": 0.8300300857031987, "pass@128": 0.8645273474549479 }, "per_op_pass_at_k": { "10": { "pass@1": 0.2789684734513274, "pass@2": 0.4280777471953171, "pass@4": 0.591814418385755, "pass@8": 0.7408074226575495, "pass@16": 0.852499483578908, "pass@32": 0.9212267885243984, "pass@64": 0.9583472694281799, "pass@128": 0.976401179941003 }, "2": { "pass@1": 0.9758080218068536, "pass@2": 0.9829028380798195, "pass@4": 0.98677001268533, "pass@8": 0.9896416905457607, "pass@16": 0.992610833071678, "pass@32": 0.9956735708953123, "pass@64": 0.9981573994804587, "pass@128": 1.0 }, "3": { "pass@1": 0.9917335626911316, "pass@2": 0.996875300994486, "pass@4": 0.9981438487046914, "pass@8": 0.9989815709816434, "pass@16": 0.9996882265536386, "pass@32": 0.9999783032978147, "pass@64": 0.9999999839948809, "pass@128": 1.0 }, "4": { "pass@1": 0.8259765625, "pass@2": 0.8985351562499997, "pass@4": 0.9441926254335391, "pass@8": 0.9721901401524639, "pass@16": 0.9876806739981561, "pass@32": 0.9947236639399566, "pass@64": 0.9979643470112908, "pass@128": 1.0 }, "5": { "pass@1": 0.5100446428571429, "pass@2": 0.5957895888013999, "pass@4": 0.6638044452776733, "pass@8": 0.7162705451375567, "pass@16": 0.7588625747565263, "pass@32": 0.7968042579192405, "pass@64": 0.8300021626967635, "pass@128": 0.8539682539682539 }, "6": { "pass@1": 0.6049775449101796, "pass@2": 0.7494286770946295, "pass@4": 0.8553464221725272, "pass@8": 0.9199166321160519, "pass@16": 0.9540052822565772, "pass@32": 0.9713208827373582, "pass@64": 0.981635383428142, "pass@128": 0.9880239520958084 }, "7": { "pass@1": 0.45532082100591714, "pass@2": 0.6107571023389085, "pass@4": 0.7487294283295948, "pass@8": 0.8499193199413594, "pass@16": 0.9137974968766152, "pass@32": 0.9478084366667775, "pass@64": 0.9629303699693006, "pass@128": 0.9733727810650887 }, "8": { "pass@1": 0.384364983974359, "pass@2": 0.537295893902685, "pass@4": 0.6853231852628991, "pass@8": 0.8045472369839184, "pass@16": 0.8847355896474814, "pass@32": 0.9309828246393891, "pass@64": 0.9555114813719695, "pass@128": 0.967948717948718 }, "9": { "pass@1": 0.2938820422535211, "pass@2": 0.4350788787845184, "pass@4": 0.5845965955840028, "pass@8": 0.7199691354405304, "pass@16": 0.8255981073771215, "pass@32": 0.8950638108118301, "pass@64": 0.9340900185268076, "pass@128": 0.9577464788732394 }, "11": { "pass@1": 0.2533197592067989, "pass@2": 0.38688282103901334, "pass@4": 0.5323678361911565, "pass@8": 0.6645512690820914, "pass@16": 0.7685657965631995, "pass@32": 0.8465825707903929, "pass@64": 0.9054490438521088, "pass@128": 0.9518413597733711 }, "12": { "pass@1": 0.2280411370056497, "pass@2": 0.34637050135682185, "pass@4": 0.4763980580287633, "pass@8": 0.6009860957867592, "pass@16": 0.7120890452970247, "pass@32": 0.8080114985944731, "pass@64": 0.8822205445332022, "pass@128": 0.9265536723163842 }, "13": { "pass@1": 0.20368828369905956, "pass@2": 0.31595992335793444, "pass@4": 0.4411990043172506, "pass@8": 0.5599757590099455, "pass@16": 0.664349775924061, "pass@32": 0.7572927869826414, "pass@64": 0.8338433992759554, "pass@128": 0.8871473354231975 }, "14": { "pass@1": 0.17399240654205608, "pass@2": 0.2759075201756323, "pass@4": 0.396560774529352, "pass@8": 0.5163368704240593, "pass@16": 0.6228329513919489, "pass@32": 0.7182782309133995, "pass@64": 0.8011897358792905, "pass@128": 0.8660436137071651 }, "15": { "pass@1": 0.15435332369942195, "pass@2": 0.24282648149833852, "pass@4": 0.34869666183345543, "pass@8": 0.45676819695321474, "pass@16": 0.5541090817165775, "pass@32": 0.6384982135537257, "pass@64": 0.7144541649154501, "pass@128": 0.7774566473988439 }, "16": { "pass@1": 0.16794354838709677, "pass@2": 0.2621713868427737, "pass@4": 0.37036601170821387, "pass@8": 0.47215387892998295, "pass@16": 0.5571265858208673, "pass@32": 0.6332602700575862, "pass@64": 0.7098035889377893, "pass@128": 0.7870967741935484 }, "17": { "pass@1": 0.1376896676300578, "pass@2": 0.21637470984479543, "pass@4": 0.3087660631113307, "pass@8": 0.40222533552236417, "pass@16": 0.4867482784741931, "pass@32": 0.5601146970644826, "pass@64": 0.6261372264623557, "pass@128": 0.6878612716763006 }, "18": { "pass@1": 0.13252232142857143, "pass@2": 0.21067913385826756, "pass@4": 0.30457685414323193, "pass@8": 0.4001484284475485, "pass@16": 0.48298183754403196, "pass@32": 0.5496689805711834, "pass@64": 0.6102941385353717, "pass@128": 0.6742857142857143 }, "19": { "pass@1": 0.12868337087087087, "pass@2": 0.2052394421981036, "pass@4": 0.29570594019216084, "pass@8": 0.38331477986605983, "pass@16": 0.45245381726726425, "pass@32": 0.5028735361578468, "pass@64": 0.5445303267550561, "pass@128": 0.5795795795795796 }, "20": { "pass@1": 0.12865990990990991, "pass@2": 0.2023191991203802, "pass@4": 0.28769374689650284, "pass@8": 0.3713682642874231, "pass@16": 0.4421942211987564, "pass@32": 0.4975821498514105, "pass@64": 0.5426479505551459, "pass@128": 0.5855855855855856 } } }, "teachers_in_school": { "count": 821120, "correct": 307279, "answer_accuracy": 0.37421935892439595, "avg_response_len": 258.0393657443492, "resp_tokens_sum": 211881284, "pass_at_k": { "pass@1": 0.37421935892439595, "pass@2": 0.4751398704132225, "pass@4": 0.5782645643719765, "pass@8": 0.6704801841136393, "pass@16": 0.7445938061826964, "pass@32": 0.8012149991574135, "pass@64": 0.8437957865734628, "pass@128": 0.8762275915822292 }, "per_op_pass_at_k": { "10": { "pass@1": 0.28875095858895705, "pass@2": 0.43106990604318646, "pass@4": 0.5864057556455747, "pass@8": 0.732640888827075, "pass@16": 0.8480012279076753, "pass@32": 0.9214811867500342, "pass@64": 0.9574177953270581, "pass@128": 0.9723926380368099 }, "2": { "pass@1": 0.9847074468085106, "pass@2": 0.9920482014216307, "pass@4": 0.9937663534839298, "pass@8": 0.9939191283233895, "pass@16": 0.9939209724505289, "pass@32": 0.993920972644377, "pass@64": 0.993920972644377, "pass@128": 0.993920972644377 }, "3": { "pass@1": 0.9926846590909091, "pass@2": 0.9993184800763542, "pass@4": 0.9999869709468137, "pass@8": 0.9999999918335785, "pass@16": 0.999999999999999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.7997873520710059, "pass@2": 0.8798932319573219, "pass@4": 0.9325595224228621, "pass@8": 0.9613584139789305, "pass@16": 0.975826780702137, "pass@32": 0.984061812308243, "pass@64": 0.9891169464353992, "pass@128": 0.9911242603550295 }, "5": { "pass@1": 0.543659420289855, "pass@2": 0.636248644870478, "pass@4": 0.7052871915467087, "pass@8": 0.7567229407325373, "pass@16": 0.7959486963018207, "pass@32": 0.8241830240851653, "pass@64": 0.8421962984097515, "pass@128": 0.855072463768116 }, "6": { "pass@1": 0.605860494269341, "pass@2": 0.7515733225413447, "pass@4": 0.8630037455024421, "pass@8": 0.9309214882963924, "pass@16": 0.9633535490981642, "pass@32": 0.9774744246618872, "pass@64": 0.9856872646797787, "pass@128": 0.9914040114613181 }, "7": { "pass@1": 0.46477601156069365, "pass@2": 0.6259927859451095, "pass@4": 0.7673764361463481, "pass@8": 0.8647695309449625, "pass@16": 0.9189930954067715, "pass@32": 0.9480705098855492, "pass@64": 0.9656972328682472, "pass@128": 0.9739884393063584 }, "8": { "pass@1": 0.4154049295774648, "pass@2": 0.5812035599423316, "pass@4": 0.7328646960503166, "pass@8": 0.8477899594817437, "pass@16": 0.9197686125029038, "pass@32": 0.9579332312942341, "pass@64": 0.9763086603935157, "pass@128": 0.9830985915492958 }, "9": { "pass@1": 0.32508420658682635, "pass@2": 0.4706348867226176, "pass@4": 0.6193361934735703, "pass@8": 0.7496807347629907, "pass@16": 0.8464234084190709, "pass@32": 0.9094847682238388, "pass@64": 0.9470349431575563, "pass@128": 0.9670658682634731 }, "11": { "pass@1": 0.22632718373493976, "pass@2": 0.34282800493311844, "pass@4": 0.47730566331693486, "pass@8": 0.6155398332967894, "pass@16": 0.741496303293349, "pass@32": 0.8402857635506765, "pass@64": 0.9063125577386248, "pass@128": 0.9427710843373494 }, "12": { "pass@1": 0.24872648902821318, "pass@2": 0.3795660497864882, "pass@4": 0.5243167942565169, "pass@8": 0.6566606510122126, "pass@16": 0.7632659524563736, "pass@32": 0.8451385844187532, "pass@64": 0.9036814870973651, "pass@128": 0.9467084639498433 }, "13": { "pass@1": 0.19793855042016806, "pass@2": 0.3077669059749886, "pass@4": 0.4309012725825237, "pass@8": 0.5507201025037028, "pass@16": 0.659313944560305, "pass@32": 0.7545849536297392, "pass@64": 0.8333705544433158, "pass@128": 0.896358543417367 }, "14": { "pass@1": 0.2007721125730994, "pass@2": 0.30511487256527137, "pass@4": 0.41963403944024513, "pass@8": 0.5284681700916576, "pass@16": 0.6283094308749538, "pass@32": 0.7210936702586249, "pass@64": 0.799903130756725, "pass@128": 0.8654970760233918 }, "15": { "pass@1": 0.14105113636363636, "pass@2": 0.23116723335719394, "pass@4": 0.343928793559896, "pass@8": 0.46371082103721484, "pass@16": 0.5760989853454654, "pass@32": 0.6761599186144563, "pass@64": 0.7661271643204762, "pass@128": 0.8484848484848485 }, "16": { "pass@1": 0.15892094017094016, "pass@2": 0.250165093545999, "pass@4": 0.3544416253523868, "pass@8": 0.45494536835752714, "pass@16": 0.5440344713666627, "pass@32": 0.6268711139804762, "pass@64": 0.7067472076557446, "pass@128": 0.7749287749287749 }, "17": { "pass@1": 0.14596273291925466, "pass@2": 0.23891837739032604, "pass@4": 0.3522555392920606, "pass@8": 0.4648475282331197, "pass@16": 0.560597983365616, "pass@32": 0.6383266290994818, "pass@64": 0.7019445712769129, "pass@128": 0.7546583850931677 }, "18": { "pass@1": 0.14418658088235295, "pass@2": 0.2277142919175543, "pass@4": 0.326042368049582, "pass@8": 0.42390522986713636, "pass@16": 0.5069815929983243, "pass@32": 0.5700109209299284, "pass@64": 0.6170846108556122, "pass@128": 0.6529411764705882 }, "19": { "pass@1": 0.12446521577380952, "pass@2": 0.1989730531730408, "pass@4": 0.28920478829655233, "pass@8": 0.3814292207708836, "pass@16": 0.46396975192531037, "pass@32": 0.5318783006080356, "pass@64": 0.5880874256563429, "pass@128": 0.6398809523809523 }, "20": { "pass@1": 0.10404191616766467, "pass@2": 0.17241832771464963, "pass@4": 0.25979317517944994, "pass@8": 0.3531867827175506, "pass@16": 0.43568691680178195, "pass@32": 0.4989061678982635, "pass@64": 0.5489351028795153, "pass@128": 0.5958083832335329 } } }, "movie_festival_awards": { "count": 801152, "correct": 299140, "answer_accuracy": 0.37338732225595145, "avg_response_len": 274.94318056997923, "resp_tokens_sum": 220271279, "pass_at_k": { "pass@1": 0.37338732225595145, "pass@2": 0.47183642169449863, "pass@4": 0.5732354423653239, "pass@8": 0.6642041540865311, "pass@16": 0.7370704162700414, "pass@32": 0.7926283959547556, "pass@64": 0.8351288151632671, "pass@128": 0.8680300367470842 }, "per_op_pass_at_k": { "10": { "pass@1": 0.3172574626865672, "pass@2": 0.46422060465389586, "pass@4": 0.6129414969584025, "pass@8": 0.7433017291309109, "pass@16": 0.8425144907670432, "pass@32": 0.9081062505290839, "pass@64": 0.9492021871971893, "pass@128": 0.9731343283582089 }, "2": { "pass@1": 0.9898883928571428, "pass@2": 0.9931893278965128, "pass@4": 0.9941566200653488, "pass@8": 0.9942830392971678, "pass@16": 0.9942857133700204, "pass@32": 0.9942857142857143, "pass@64": 0.9942857142857143, "pass@128": 0.9942857142857143 }, "3": { "pass@1": 0.9862882653061225, "pass@2": 0.9972843397764055, "pass@4": 0.9996692599102955, "pass@8": 0.9999825134422476, "pass@16": 0.9999999402345775, "pass@32": 0.9999999999999069, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.8163834064327485, "pass@2": 0.890046176382557, "pass@4": 0.9372147971635119, "pass@8": 0.9636458409465761, "pass@16": 0.9772710270520469, "pass@32": 0.9842145598658137, "pass@64": 0.9873700673843149, "pass@128": 0.9883040935672515 }, "5": { "pass@1": 0.5637867647058824, "pass@2": 0.6648444737146825, "pass@4": 0.7408300522177375, "pass@8": 0.7912320906946062, "pass@16": 0.823131342775329, "pass@32": 0.8458919379220898, "pass@64": 0.8639400997706301, "pass@128": 0.8764705882352941 }, "6": { "pass@1": 0.5662460567823344, "pass@2": 0.7139843730594401, "pass@4": 0.8325656318788228, "pass@8": 0.9097002960639579, "pass@16": 0.9523965826489672, "pass@32": 0.9761397948002681, "pass@64": 0.9890095178629523, "pass@128": 0.9936908517350158 }, "7": { "pass@1": 0.4513449367088608, "pass@2": 0.6131500423602112, "pass@4": 0.7506766076708764, "pass@8": 0.8459427764388251, "pass@16": 0.9042667172900669, "pass@32": 0.9384254825081815, "pass@64": 0.9588122968919452, "pass@128": 0.9715189873417721 }, "8": { "pass@1": 0.38325825825825827, "pass@2": 0.5393948328249514, "pass@4": 0.6901599282071716, "pass@8": 0.8106203042926252, "pass@16": 0.8915132477415891, "pass@32": 0.9391020578007191, "pass@64": 0.9651795386130988, "pass@128": 0.978978978978979 }, "9": { "pass@1": 0.2740906350482315, "pass@2": 0.4123497512469298, "pass@4": 0.5664755532969952, "pass@8": 0.7106967783354227, "pass@16": 0.824073336435741, "pass@32": 0.9018407429475074, "pass@64": 0.9476444488564583, "pass@128": 0.9678456591639871 }, "11": { "pass@1": 0.26674107142857145, "pass@2": 0.40546103612048484, "pass@4": 0.5536739068330748, "pass@8": 0.6895362541177696, "pass@16": 0.8004270014665438, "pass@32": 0.8813450711545094, "pass@64": 0.9314687191900564, "pass@128": 0.9555555555555556 }, "12": { "pass@1": 0.21593081039755352, "pass@2": 0.3314182559175517, "pass@4": 0.46379726100751173, "pass@8": 0.5951990833211992, "pass@16": 0.7137224500373927, "pass@32": 0.8150697689006822, "pass@64": 0.8921259328053642, "pass@128": 0.9480122324159022 }, "13": { "pass@1": 0.20401716820987653, "pass@2": 0.31154156945659583, "pass@4": 0.43245891572581213, "pass@8": 0.5496484062356665, "pass@16": 0.652648250879313, "pass@32": 0.7420970468567396, "pass@64": 0.8164524378516309, "pass@128": 0.8734567901234568 }, "14": { "pass@1": 0.17152726261127596, "pass@2": 0.2703319586906236, "pass@4": 0.39107636971936394, "pass@8": 0.5176994047412352, "pass@16": 0.6309838722897884, "pass@32": 0.724179890916876, "pass@64": 0.8006647814674875, "pass@128": 0.8635014836795252 }, "15": { "pass@1": 0.13384934413580246, "pass@2": 0.22282522844366667, "pass@4": 0.33642364351446796, "pass@8": 0.45632627911948054, "pass@16": 0.5659365003613015, "pass@32": 0.6647865356976304, "pass@64": 0.7578901030057378, "pass@128": 0.8395061728395061 }, "16": { "pass@1": 0.1661827064896755, "pass@2": 0.2619750656167981, "pass@4": 0.3700903606628819, "pass@8": 0.46943612898581016, "pass@16": 0.5510554413402865, "pass@32": 0.61981058809925, "pass@64": 0.6839821192024573, "pass@128": 0.7492625368731564 }, "17": { "pass@1": 0.1402249623493976, "pass@2": 0.22415819775163626, "pass@4": 0.3234516700096222, "pass@8": 0.4214179191899702, "pass@16": 0.5040814253879703, "pass@32": 0.5692985561263602, "pass@64": 0.6261336188175767, "pass@128": 0.6807228915662651 }, "18": { "pass@1": 0.1180695564516129, "pass@2": 0.18816754508509018, "pass@4": 0.27230403044377516, "pass@8": 0.3606826515176, "pass@16": 0.4422537616760494, "pass@32": 0.5111308555501609, "pass@64": 0.5687212244235005, "pass@128": 0.6161290322580645 }, "19": { "pass@1": 0.13014539274924472, "pass@2": 0.20784814568118562, "pass@4": 0.2988084281194458, "pass@8": 0.38650719281063034, "pass@16": 0.45915586378086676, "pass@32": 0.5162083882367925, "pass@64": 0.565780077034896, "pass@128": 0.6163141993957704 }, "20": { "pass@1": 0.10700544294294294, "pass@2": 0.17701017947080935, "pass@4": 0.26730876151742283, "pass@8": 0.36331254904296867, "pass@16": 0.44728071981044754, "pass@32": 0.5105099300302764, "pass@64": 0.5575780342462267, "pass@128": 0.5975975975975976 } } } } } ]