[ { "checkpoint": "checkpoint-39470", "id_acc": 0.9875668402777777, "ood_acc": 0.84428203125, "total_acc": 0.9121537828947368, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.47587413194444, "ood_avg_resp_len": 382.38321875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 732223, "answer_accuracy": 0.9042826726999684, "avg_response_len": 283.21991829355045, "resp_tokens_sum": 229331098, "pass_at_k": { "pass@1": 0.9042826726999684, "pass@2": 0.9397006612193992, "pass@4": 0.9598143926138624, "pass@8": 0.9722494066653309, "pass@16": 0.9804004763829958, "pass@32": 0.9860413514495616, "pass@64": 0.9898269521281325, "pass@128": 0.992254189061018 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9300562315634219, "pass@2": 0.9487169157782271, "pass@4": 0.9599396154020569, "pass@8": 0.9683942194420995, "pass@16": 0.9757538297615362, "pass@32": 0.9826819806419275, "pass@64": 0.9892382643670711, "pass@128": 0.9941002949852508 }, "2": { "pass@1": 0.9997079439252337, "pass@2": 0.9999984668972454, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9987098623853211, "pass@2": 0.9999311475113775, "pass@4": 0.9999994467434691, "pass@8": 0.9999999999724691, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9964599609375, "pass@2": 0.9990641916830709, "pass@4": 0.9998087276199851, "pass@8": 0.9999903205715637, "pass@16": 0.9999999836447685, "pass@32": 0.9999999999999961, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9976934523809524, "pass@2": 0.9997941663542057, "pass@4": 0.9999945093172878, "pass@8": 0.9999999950689359, "pass@16": 0.9999999999999997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9965615643712575, "pass@2": 0.9996379043094914, "pass@4": 0.9999889290597658, "pass@8": 0.9999999886950237, "pass@16": 0.9999999999999977, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9929271449704142, "pass@2": 0.9980678959138986, "pass@4": 0.9994872563081982, "pass@8": 0.9999599346644329, "pass@16": 0.9999998229513717, "pass@32": 0.9999999999993783, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.985526842948718, "pass@2": 0.9970164862204728, "pass@4": 0.9995499270523874, "pass@8": 0.9999784033248174, "pass@16": 0.9999999467420494, "pass@32": 0.9999999999999449, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9843970070422535, "pass@2": 0.9951934540312746, "pass@4": 0.9971520704278161, "pass@8": 0.9975209774399802, "pass@16": 0.9978457333734444, "pass@32": 0.9984196517688809, "pass@64": 0.9993013197294001, "pass@128": 1.0 }, "11": { "pass@1": 0.9420148725212465, "pass@2": 0.9688956860208338, "pass@4": 0.9842888575719818, "pass@8": 0.9932980749767641, "pass@16": 0.9975718340081712, "pass@32": 0.999283473869785, "pass@64": 0.999918274622942, "pass@128": 1.0 }, "12": { "pass@1": 0.9253840042372882, "pass@2": 0.9572093676320124, "pass@4": 0.975868617588055, "pass@8": 0.9874239409259805, "pass@16": 0.9940122193279375, "pass@32": 0.9967254534090262, "pass@64": 0.9971693101066742, "pass@128": 0.9971751412429378 }, "13": { "pass@1": 0.9065683777429467, "pass@2": 0.9460485677436875, "pass@4": 0.969080333414436, "pass@8": 0.9827606655847345, "pass@16": 0.9899047973230172, "pass@32": 0.9936117638775492, "pass@64": 0.9964728083110211, "pass@128": 1.0 }, "14": { "pass@1": 0.8823014018691588, "pass@2": 0.9289659835160793, "pass@4": 0.956458548697768, "pass@8": 0.9735331021115504, "pass@16": 0.9854037725268124, "pass@32": 0.9928036055443255, "pass@64": 0.9970896386492748, "pass@128": 1.0 }, "15": { "pass@1": 0.9151463150289018, "pass@2": 0.9623722748168043, "pass@4": 0.9810567303545147, "pass@8": 0.9888010189100436, "pass@16": 0.993003628745617, "pass@32": 0.9959876521944565, "pass@64": 0.9970619927265407, "pass@128": 0.9971098265895953 }, "16": { "pass@1": 0.8313508064516129, "pass@2": 0.8921712280924569, "pass@4": 0.9304420479698096, "pass@8": 0.9540151944897635, "pass@16": 0.9673493337288324, "pass@32": 0.9748748941915307, "pass@64": 0.9808457757289828, "pass@128": 0.9870967741935484 }, "17": { "pass@1": 0.852194725433526, "pass@2": 0.9099348999817946, "pass@4": 0.9404819071568351, "pass@8": 0.9589799397673455, "pass@16": 0.970521670250947, "pass@32": 0.9781434694857413, "pass@64": 0.9817295210764027, "pass@128": 0.9826589595375722 }, "18": { "pass@1": 0.75921875, "pass@2": 0.8474205568053995, "pass@4": 0.9002597238738005, "pass@8": 0.9323657016061904, "pass@16": 0.9531506347489849, "pass@32": 0.9694149203288003, "pass@64": 0.980450229236788, "pass@128": 0.9857142857142858 }, "19": { "pass@1": 0.6614583333333334, "pass@2": 0.7607140851481399, "pass@4": 0.8249680473499366, "pass@8": 0.8700913512642859, "pass@16": 0.9018850425263412, "pass@32": 0.9242366811410543, "pass@64": 0.9397228170742807, "pass@128": 0.948948948948949 }, "20": { "pass@1": 0.6349239864864865, "pass@2": 0.7494402621125058, "pass@4": 0.8215973997056667, "pass@8": 0.8682736666753487, "pass@16": 0.9029783755206716, "pass@32": 0.9296180551931462, "pass@64": 0.9484160357071253, "pass@128": 0.960960960960961 } } }, "teachers_in_school": { "count": 821120, "correct": 754045, "answer_accuracy": 0.9183127922837101, "avg_response_len": 292.58281493569757, "resp_tokens_sum": 240245601, "pass_at_k": { "pass@1": 0.9183127922837101, "pass@2": 0.9482236860274367, "pass@4": 0.9651917685951739, "pass@8": 0.9757795838858602, "pass@16": 0.9827851497433899, "pass@32": 0.987475184556737, "pass@64": 0.9905849294081015, "pass@128": 0.9928293063133281 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9227137653374233, "pass@2": 0.9429375392493117, "pass@4": 0.9563598661555336, "pass@8": 0.9656292113866888, "pass@16": 0.972287380308588, "pass@32": 0.9775241038638645, "pass@64": 0.981763116867637, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 0.9997862841945289, "pass@2": 0.9999962604408492, "pass@4": 0.9999999997150812, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9969223484848485, "pass@2": 0.9985836465044142, "pass@4": 0.9996747594050742, "pass@8": 0.999983763588421, "pass@16": 0.999999969633645, "pass@32": 0.9999999999999856, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9955159023668639, "pass@2": 0.9979375844476541, "pass@4": 0.9990079075729438, "pass@8": 0.9997006903613214, "pass@16": 0.9999748631075642, "pass@32": 0.9999999112627457, "pass@64": 0.9999999999999892, "pass@128": 1.0 }, "5": { "pass@1": 0.9943161231884058, "pass@2": 0.9977643929019744, "pass@4": 0.99949508675546, "pass@8": 0.9999704874534221, "pass@16": 0.9999999192465391, "pass@32": 0.9999999999999238, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9982091690544412, "pass@2": 0.9999407756695169, "pass@4": 0.9999997013267322, "pass@8": 0.9999999999939795, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9967711343930635, "pass@2": 0.999786650584862, "pass@4": 0.9999965644648466, "pass@8": 0.9999999985045215, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9876980633802817, "pass@2": 0.9934928468448486, "pass@4": 0.995072504317242, "pass@8": 0.9960171016781834, "pass@16": 0.9971824274666318, "pass@32": 0.9985597501374172, "pass@64": 0.9996474842017711, "pass@128": 1.0 }, "9": { "pass@1": 0.9855211452095808, "pass@2": 0.9946945430006132, "pass@4": 0.9964640314421774, "pass@8": 0.996930973138194, "pass@16": 0.9970046492451531, "pass@32": 0.9970059879036105, "pass@64": 0.9970059880239521, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.9655967620481928, "pass@2": 0.9813555402713219, "pass@4": 0.9873279309589311, "pass@8": 0.9915507045621954, "pass@16": 0.9949525726882045, "pass@32": 0.9966670644801715, "pass@64": 0.9969831607359049, "pass@128": 0.9969879518072289 }, "12": { "pass@1": 0.9296385188087775, "pass@2": 0.9592457205094661, "pass@4": 0.9766451186156586, "pass@8": 0.9861626009880742, "pass@16": 0.9922937608317366, "pass@32": 0.9965541033737507, "pass@64": 0.9990781258535262, "pass@128": 1.0 }, "13": { "pass@1": 0.9335828081232493, "pass@2": 0.9606981572156414, "pass@4": 0.9748942135909476, "pass@8": 0.9834503879841167, "pass@16": 0.9898332381537198, "pass@32": 0.9943021535783841, "pass@64": 0.9967150589030925, "pass@128": 0.9971988795518207 }, "14": { "pass@1": 0.9181514985380117, "pass@2": 0.9576616964820192, "pass@4": 0.9784569757727651, "pass@8": 0.9903096357097452, "pass@16": 0.9965760665784712, "pass@32": 0.9992521427633804, "pass@64": 0.9999580995951128, "pass@128": 1.0 }, "15": { "pass@1": 0.9094933712121213, "pass@2": 0.9588310516583153, "pass@4": 0.9822578816852432, "pass@8": 0.9930328503480245, "pass@16": 0.9969244873195249, "pass@32": 0.9982614899741054, "pass@64": 0.9992483524249482, "pass@128": 1.0 }, "16": { "pass@1": 0.8821225071225072, "pass@2": 0.9243023307983942, "pass@4": 0.9472836953073169, "pass@8": 0.9623024874009348, "pass@16": 0.972876623671631, "pass@32": 0.9807417394196154, "pass@64": 0.9869335849984805, "pass@128": 0.9914529914529915 }, "17": { "pass@1": 0.8615100931677019, "pass@2": 0.9146566427837828, "pass@4": 0.943564751611017, "pass@8": 0.9607675115201083, "pass@16": 0.9711962890644932, "pass@32": 0.9778534123969457, "pass@64": 0.9834730810425003, "pass@128": 0.9906832298136646 }, "18": { "pass@1": 0.8249770220588235, "pass@2": 0.8977278977535897, "pass@4": 0.9374260079622395, "pass@8": 0.9601756057206904, "pass@16": 0.9739739781686679, "pass@32": 0.983005959288786, "pass@64": 0.98793702300017, "pass@128": 0.9911764705882353 }, "19": { "pass@1": 0.6945917038690477, "pass@2": 0.783559037542182, "pass@4": 0.8407643528375911, "pass@8": 0.8794797954531516, "pass@16": 0.9074972703542947, "pass@32": 0.9281647944725584, "pass@64": 0.9420842485989787, "pass@128": 0.9523809523809523 }, "20": { "pass@1": 0.6413501122754491, "pass@2": 0.746720879107926, "pass@4": 0.8196913748431142, "pass@8": 0.8713387468305476, "pass@16": 0.9080377494829686, "pass@32": 0.9322353462567267, "pass@64": 0.9487201780875976, "pass@128": 0.9610778443113772 } } }, "movie_festival_awards": { "count": 801152, "correct": 732090, "answer_accuracy": 0.9137966328486978, "avg_response_len": 313.07670454545456, "resp_tokens_sum": 250822028, "pass_at_k": { "pass@1": 0.9137966328486978, "pass@2": 0.9449051829931684, "pass@4": 0.9632319997636081, "pass@8": 0.9748005856481224, "pass@16": 0.9824296417295298, "pass@32": 0.9876260041923829, "pass@64": 0.9911149445638051, "pass@128": 0.9936092027480428 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9493470149253731, "pass@2": 0.9677135973674934, "pass@4": 0.9777967773617846, "pass@8": 0.9838416838151819, "pass@16": 0.9877038473718193, "pass@32": 0.9910220252708253, "pass@64": 0.9934738597354361, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 0.99984375, "pass@2": 0.9999992969628797, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9992028061224489, "pass@2": 0.9999081747434632, "pass@4": 0.9999975800218851, "pass@8": 0.9999999990001509, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9984237938596491, "pass@2": 0.9998186904268548, "pass@4": 0.9999913708703079, "pass@8": 0.9999999838662624, "pass@16": 0.9999999999999906, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9917049632352941, "pass@2": 0.9958824253126447, "pass@4": 0.9976440308012968, "pass@8": 0.9987293865869793, "pass@16": 0.9995273054917665, "pass@32": 0.9999439253027383, "pass@64": 0.9999998174959024, "pass@128": 1.0 }, "6": { "pass@1": 0.9992606466876972, "pass@2": 0.9999922377605008, "pass@4": 0.9999999982257738, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.994832871835443, "pass@2": 0.9967256491079439, "pass@4": 0.9968332677758637, "pass@8": 0.9968354423301838, "pass@16": 0.9968354430379747, "pass@32": 0.9968354430379747, "pass@64": 0.9968354430379747, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9857826576576577, "pass@2": 0.9921396544181976, "pass@4": 0.9939751430282925, "pass@8": 0.9959554355169689, "pass@16": 0.9981280304181758, "pass@32": 0.9995587097302534, "pass@64": 0.9999801661425524, "pass@128": 1.0 }, "9": { "pass@1": 0.9863092845659164, "pass@2": 0.993402188773831, "pass@4": 0.995040290582648, "pass@8": 0.995992442271155, "pass@16": 0.9969055126132675, "pass@32": 0.9975721294864157, "pass@64": 0.9983922757459707, "pass@128": 1.0 }, "11": { "pass@1": 0.9582093253968254, "pass@2": 0.979580286839145, "pass@4": 0.9909319165461457, "pass@8": 0.9965024586601782, "pass@16": 0.9987556918927141, "pass@32": 0.9997024090192648, "pass@64": 0.999990171773953, "pass@128": 1.0 }, "12": { "pass@1": 0.9369982798165137, "pass@2": 0.9629724108454334, "pass@4": 0.9807916977694299, "pass@8": 0.9921265716594561, "pass@16": 0.9975797601684947, "pass@32": 0.9994103260695886, "pass@64": 0.9999575357773413, "pass@128": 1.0 }, "13": { "pass@1": 0.8860677083333334, "pass@2": 0.9257945404393898, "pass@4": 0.952882770440732, "pass@8": 0.9705083260290297, "pass@16": 0.9827039584291745, "pass@32": 0.9909022848607238, "pass@64": 0.9954709952559196, "pass@128": 0.9969135802469136 }, "14": { "pass@1": 0.9277865356083086, "pass@2": 0.9635647883128114, "pass@4": 0.9799638896213635, "pass@8": 0.9889540682347803, "pass@16": 0.994416741293335, "pass@32": 0.9976490069232948, "pass@64": 0.9994606231626841, "pass@128": 1.0 }, "15": { "pass@1": 0.8958333333333334, "pass@2": 0.9419447026586953, "pass@4": 0.9628916451994424, "pass@8": 0.9741455706645901, "pass@16": 0.9817959566898227, "pass@32": 0.9868451198367149, "pass@64": 0.9890965712851263, "pass@128": 0.9907407407407407 }, "16": { "pass@1": 0.8752995943952803, "pass@2": 0.9210983409983046, "pass@4": 0.9490407169789616, "pass@8": 0.9661475706409914, "pass@16": 0.9755157512762427, "pass@32": 0.9813566539237482, "pass@64": 0.9858831578717764, "pass@128": 0.9911504424778761 }, "17": { "pass@1": 0.8405026355421686, "pass@2": 0.8887784158286688, "pass@4": 0.916870214265385, "pass@8": 0.9383888936653869, "pass@16": 0.9554419843774683, "pass@32": 0.9678565567484874, "pass@64": 0.977482393080175, "pass@128": 0.9879518072289156 }, "18": { "pass@1": 0.8315776209677419, "pass@2": 0.903906448437897, "pass@4": 0.9437463160854891, "pass@8": 0.9663211807044224, "pass@16": 0.9785213547715544, "pass@32": 0.98575825369811, "pass@64": 0.9908713414697528, "pass@128": 0.9935483870967742 }, "19": { "pass@1": 0.6603568731117825, "pass@2": 0.7687665776577776, "pass@4": 0.8426575621589597, "pass@8": 0.8910139633294577, "pass@16": 0.9241243059124612, "pass@32": 0.948114566321418, "pass@64": 0.964677567112477, "pass@128": 0.9758308157099698 }, "20": { "pass@1": 0.6414226726726727, "pass@2": 0.7502992658012341, "pass@4": 0.8205210395997796, "pass@8": 0.8663415393566618, "pass@16": 0.8988781493348023, "pass@32": 0.9229708469087713, "pass@64": 0.9401299955504153, "pass@128": 0.9519519519519519 } } } } } ]