| [ |
| { |
| "checkpoint": "checkpoint-39470", |
| "id_acc": 0.9875668402777777, |
| "ood_acc": 0.84428203125, |
| "total_acc": 0.9121537828947368, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.47587413194444, |
| "ood_avg_resp_len": 382.38321875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 732223, |
| "answer_accuracy": 0.9042826726999684, |
| "avg_response_len": 283.21991829355045, |
| "resp_tokens_sum": 229331098, |
| "pass_at_k": { |
| "pass@1": 0.9042826726999684, |
| "pass@2": 0.9397006612193992, |
| "pass@4": 0.9598143926138624, |
| "pass@8": 0.9722494066653309, |
| "pass@16": 0.9804004763829958, |
| "pass@32": 0.9860413514495616, |
| "pass@64": 0.9898269521281325, |
| "pass@128": 0.992254189061018 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9300562315634219, |
| "pass@2": 0.9487169157782271, |
| "pass@4": 0.9599396154020569, |
| "pass@8": 0.9683942194420995, |
| "pass@16": 0.9757538297615362, |
| "pass@32": 0.9826819806419275, |
| "pass@64": 0.9892382643670711, |
| "pass@128": 0.9941002949852508 |
| }, |
| "2": { |
| "pass@1": 0.9997079439252337, |
| "pass@2": 0.9999984668972454, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9987098623853211, |
| "pass@2": 0.9999311475113775, |
| "pass@4": 0.9999994467434691, |
| "pass@8": 0.9999999999724691, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9964599609375, |
| "pass@2": 0.9990641916830709, |
| "pass@4": 0.9998087276199851, |
| "pass@8": 0.9999903205715637, |
| "pass@16": 0.9999999836447685, |
| "pass@32": 0.9999999999999961, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9976934523809524, |
| "pass@2": 0.9997941663542057, |
| "pass@4": 0.9999945093172878, |
| "pass@8": 0.9999999950689359, |
| "pass@16": 0.9999999999999997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9965615643712575, |
| "pass@2": 0.9996379043094914, |
| "pass@4": 0.9999889290597658, |
| "pass@8": 0.9999999886950237, |
| "pass@16": 0.9999999999999977, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9929271449704142, |
| "pass@2": 0.9980678959138986, |
| "pass@4": 0.9994872563081982, |
| "pass@8": 0.9999599346644329, |
| "pass@16": 0.9999998229513717, |
| "pass@32": 0.9999999999993783, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.985526842948718, |
| "pass@2": 0.9970164862204728, |
| "pass@4": 0.9995499270523874, |
| "pass@8": 0.9999784033248174, |
| "pass@16": 0.9999999467420494, |
| "pass@32": 0.9999999999999449, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9843970070422535, |
| "pass@2": 0.9951934540312746, |
| "pass@4": 0.9971520704278161, |
| "pass@8": 0.9975209774399802, |
| "pass@16": 0.9978457333734444, |
| "pass@32": 0.9984196517688809, |
| "pass@64": 0.9993013197294001, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9420148725212465, |
| "pass@2": 0.9688956860208338, |
| "pass@4": 0.9842888575719818, |
| "pass@8": 0.9932980749767641, |
| "pass@16": 0.9975718340081712, |
| "pass@32": 0.999283473869785, |
| "pass@64": 0.999918274622942, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9253840042372882, |
| "pass@2": 0.9572093676320124, |
| "pass@4": 0.975868617588055, |
| "pass@8": 0.9874239409259805, |
| "pass@16": 0.9940122193279375, |
| "pass@32": 0.9967254534090262, |
| "pass@64": 0.9971693101066742, |
| "pass@128": 0.9971751412429378 |
| }, |
| "13": { |
| "pass@1": 0.9065683777429467, |
| "pass@2": 0.9460485677436875, |
| "pass@4": 0.969080333414436, |
| "pass@8": 0.9827606655847345, |
| "pass@16": 0.9899047973230172, |
| "pass@32": 0.9936117638775492, |
| "pass@64": 0.9964728083110211, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.8823014018691588, |
| "pass@2": 0.9289659835160793, |
| "pass@4": 0.956458548697768, |
| "pass@8": 0.9735331021115504, |
| "pass@16": 0.9854037725268124, |
| "pass@32": 0.9928036055443255, |
| "pass@64": 0.9970896386492748, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9151463150289018, |
| "pass@2": 0.9623722748168043, |
| "pass@4": 0.9810567303545147, |
| "pass@8": 0.9888010189100436, |
| "pass@16": 0.993003628745617, |
| "pass@32": 0.9959876521944565, |
| "pass@64": 0.9970619927265407, |
| "pass@128": 0.9971098265895953 |
| }, |
| "16": { |
| "pass@1": 0.8313508064516129, |
| "pass@2": 0.8921712280924569, |
| "pass@4": 0.9304420479698096, |
| "pass@8": 0.9540151944897635, |
| "pass@16": 0.9673493337288324, |
| "pass@32": 0.9748748941915307, |
| "pass@64": 0.9808457757289828, |
| "pass@128": 0.9870967741935484 |
| }, |
| "17": { |
| "pass@1": 0.852194725433526, |
| "pass@2": 0.9099348999817946, |
| "pass@4": 0.9404819071568351, |
| "pass@8": 0.9589799397673455, |
| "pass@16": 0.970521670250947, |
| "pass@32": 0.9781434694857413, |
| "pass@64": 0.9817295210764027, |
| "pass@128": 0.9826589595375722 |
| }, |
| "18": { |
| "pass@1": 0.75921875, |
| "pass@2": 0.8474205568053995, |
| "pass@4": 0.9002597238738005, |
| "pass@8": 0.9323657016061904, |
| "pass@16": 0.9531506347489849, |
| "pass@32": 0.9694149203288003, |
| "pass@64": 0.980450229236788, |
| "pass@128": 0.9857142857142858 |
| }, |
| "19": { |
| "pass@1": 0.6614583333333334, |
| "pass@2": 0.7607140851481399, |
| "pass@4": 0.8249680473499366, |
| "pass@8": 0.8700913512642859, |
| "pass@16": 0.9018850425263412, |
| "pass@32": 0.9242366811410543, |
| "pass@64": 0.9397228170742807, |
| "pass@128": 0.948948948948949 |
| }, |
| "20": { |
| "pass@1": 0.6349239864864865, |
| "pass@2": 0.7494402621125058, |
| "pass@4": 0.8215973997056667, |
| "pass@8": 0.8682736666753487, |
| "pass@16": 0.9029783755206716, |
| "pass@32": 0.9296180551931462, |
| "pass@64": 0.9484160357071253, |
| "pass@128": 0.960960960960961 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 754045, |
| "answer_accuracy": 0.9183127922837101, |
| "avg_response_len": 292.58281493569757, |
| "resp_tokens_sum": 240245601, |
| "pass_at_k": { |
| "pass@1": 0.9183127922837101, |
| "pass@2": 0.9482236860274367, |
| "pass@4": 0.9651917685951739, |
| "pass@8": 0.9757795838858602, |
| "pass@16": 0.9827851497433899, |
| "pass@32": 0.987475184556737, |
| "pass@64": 0.9905849294081015, |
| "pass@128": 0.9928293063133281 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9227137653374233, |
| "pass@2": 0.9429375392493117, |
| "pass@4": 0.9563598661555336, |
| "pass@8": 0.9656292113866888, |
| "pass@16": 0.972287380308588, |
| "pass@32": 0.9775241038638645, |
| "pass@64": 0.981763116867637, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 0.9997862841945289, |
| "pass@2": 0.9999962604408492, |
| "pass@4": 0.9999999997150812, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9969223484848485, |
| "pass@2": 0.9985836465044142, |
| "pass@4": 0.9996747594050742, |
| "pass@8": 0.999983763588421, |
| "pass@16": 0.999999969633645, |
| "pass@32": 0.9999999999999856, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9955159023668639, |
| "pass@2": 0.9979375844476541, |
| "pass@4": 0.9990079075729438, |
| "pass@8": 0.9997006903613214, |
| "pass@16": 0.9999748631075642, |
| "pass@32": 0.9999999112627457, |
| "pass@64": 0.9999999999999892, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9943161231884058, |
| "pass@2": 0.9977643929019744, |
| "pass@4": 0.99949508675546, |
| "pass@8": 0.9999704874534221, |
| "pass@16": 0.9999999192465391, |
| "pass@32": 0.9999999999999238, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9982091690544412, |
| "pass@2": 0.9999407756695169, |
| "pass@4": 0.9999997013267322, |
| "pass@8": 0.9999999999939795, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9967711343930635, |
| "pass@2": 0.999786650584862, |
| "pass@4": 0.9999965644648466, |
| "pass@8": 0.9999999985045215, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9876980633802817, |
| "pass@2": 0.9934928468448486, |
| "pass@4": 0.995072504317242, |
| "pass@8": 0.9960171016781834, |
| "pass@16": 0.9971824274666318, |
| "pass@32": 0.9985597501374172, |
| "pass@64": 0.9996474842017711, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9855211452095808, |
| "pass@2": 0.9946945430006132, |
| "pass@4": 0.9964640314421774, |
| "pass@8": 0.996930973138194, |
| "pass@16": 0.9970046492451531, |
| "pass@32": 0.9970059879036105, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.9655967620481928, |
| "pass@2": 0.9813555402713219, |
| "pass@4": 0.9873279309589311, |
| "pass@8": 0.9915507045621954, |
| "pass@16": 0.9949525726882045, |
| "pass@32": 0.9966670644801715, |
| "pass@64": 0.9969831607359049, |
| "pass@128": 0.9969879518072289 |
| }, |
| "12": { |
| "pass@1": 0.9296385188087775, |
| "pass@2": 0.9592457205094661, |
| "pass@4": 0.9766451186156586, |
| "pass@8": 0.9861626009880742, |
| "pass@16": 0.9922937608317366, |
| "pass@32": 0.9965541033737507, |
| "pass@64": 0.9990781258535262, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9335828081232493, |
| "pass@2": 0.9606981572156414, |
| "pass@4": 0.9748942135909476, |
| "pass@8": 0.9834503879841167, |
| "pass@16": 0.9898332381537198, |
| "pass@32": 0.9943021535783841, |
| "pass@64": 0.9967150589030925, |
| "pass@128": 0.9971988795518207 |
| }, |
| "14": { |
| "pass@1": 0.9181514985380117, |
| "pass@2": 0.9576616964820192, |
| "pass@4": 0.9784569757727651, |
| "pass@8": 0.9903096357097452, |
| "pass@16": 0.9965760665784712, |
| "pass@32": 0.9992521427633804, |
| "pass@64": 0.9999580995951128, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9094933712121213, |
| "pass@2": 0.9588310516583153, |
| "pass@4": 0.9822578816852432, |
| "pass@8": 0.9930328503480245, |
| "pass@16": 0.9969244873195249, |
| "pass@32": 0.9982614899741054, |
| "pass@64": 0.9992483524249482, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8821225071225072, |
| "pass@2": 0.9243023307983942, |
| "pass@4": 0.9472836953073169, |
| "pass@8": 0.9623024874009348, |
| "pass@16": 0.972876623671631, |
| "pass@32": 0.9807417394196154, |
| "pass@64": 0.9869335849984805, |
| "pass@128": 0.9914529914529915 |
| }, |
| "17": { |
| "pass@1": 0.8615100931677019, |
| "pass@2": 0.9146566427837828, |
| "pass@4": 0.943564751611017, |
| "pass@8": 0.9607675115201083, |
| "pass@16": 0.9711962890644932, |
| "pass@32": 0.9778534123969457, |
| "pass@64": 0.9834730810425003, |
| "pass@128": 0.9906832298136646 |
| }, |
| "18": { |
| "pass@1": 0.8249770220588235, |
| "pass@2": 0.8977278977535897, |
| "pass@4": 0.9374260079622395, |
| "pass@8": 0.9601756057206904, |
| "pass@16": 0.9739739781686679, |
| "pass@32": 0.983005959288786, |
| "pass@64": 0.98793702300017, |
| "pass@128": 0.9911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.6945917038690477, |
| "pass@2": 0.783559037542182, |
| "pass@4": 0.8407643528375911, |
| "pass@8": 0.8794797954531516, |
| "pass@16": 0.9074972703542947, |
| "pass@32": 0.9281647944725584, |
| "pass@64": 0.9420842485989787, |
| "pass@128": 0.9523809523809523 |
| }, |
| "20": { |
| "pass@1": 0.6413501122754491, |
| "pass@2": 0.746720879107926, |
| "pass@4": 0.8196913748431142, |
| "pass@8": 0.8713387468305476, |
| "pass@16": 0.9080377494829686, |
| "pass@32": 0.9322353462567267, |
| "pass@64": 0.9487201780875976, |
| "pass@128": 0.9610778443113772 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 732090, |
| "answer_accuracy": 0.9137966328486978, |
| "avg_response_len": 313.07670454545456, |
| "resp_tokens_sum": 250822028, |
| "pass_at_k": { |
| "pass@1": 0.9137966328486978, |
| "pass@2": 0.9449051829931684, |
| "pass@4": 0.9632319997636081, |
| "pass@8": 0.9748005856481224, |
| "pass@16": 0.9824296417295298, |
| "pass@32": 0.9876260041923829, |
| "pass@64": 0.9911149445638051, |
| "pass@128": 0.9936092027480428 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9493470149253731, |
| "pass@2": 0.9677135973674934, |
| "pass@4": 0.9777967773617846, |
| "pass@8": 0.9838416838151819, |
| "pass@16": 0.9877038473718193, |
| "pass@32": 0.9910220252708253, |
| "pass@64": 0.9934738597354361, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 0.99984375, |
| "pass@2": 0.9999992969628797, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9992028061224489, |
| "pass@2": 0.9999081747434632, |
| "pass@4": 0.9999975800218851, |
| "pass@8": 0.9999999990001509, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9984237938596491, |
| "pass@2": 0.9998186904268548, |
| "pass@4": 0.9999913708703079, |
| "pass@8": 0.9999999838662624, |
| "pass@16": 0.9999999999999906, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9917049632352941, |
| "pass@2": 0.9958824253126447, |
| "pass@4": 0.9976440308012968, |
| "pass@8": 0.9987293865869793, |
| "pass@16": 0.9995273054917665, |
| "pass@32": 0.9999439253027383, |
| "pass@64": 0.9999998174959024, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9992606466876972, |
| "pass@2": 0.9999922377605008, |
| "pass@4": 0.9999999982257738, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.994832871835443, |
| "pass@2": 0.9967256491079439, |
| "pass@4": 0.9968332677758637, |
| "pass@8": 0.9968354423301838, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 0.9968354430379747, |
| "pass@64": 0.9968354430379747, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.9857826576576577, |
| "pass@2": 0.9921396544181976, |
| "pass@4": 0.9939751430282925, |
| "pass@8": 0.9959554355169689, |
| "pass@16": 0.9981280304181758, |
| "pass@32": 0.9995587097302534, |
| "pass@64": 0.9999801661425524, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9863092845659164, |
| "pass@2": 0.993402188773831, |
| "pass@4": 0.995040290582648, |
| "pass@8": 0.995992442271155, |
| "pass@16": 0.9969055126132675, |
| "pass@32": 0.9975721294864157, |
| "pass@64": 0.9983922757459707, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9582093253968254, |
| "pass@2": 0.979580286839145, |
| "pass@4": 0.9909319165461457, |
| "pass@8": 0.9965024586601782, |
| "pass@16": 0.9987556918927141, |
| "pass@32": 0.9997024090192648, |
| "pass@64": 0.999990171773953, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9369982798165137, |
| "pass@2": 0.9629724108454334, |
| "pass@4": 0.9807916977694299, |
| "pass@8": 0.9921265716594561, |
| "pass@16": 0.9975797601684947, |
| "pass@32": 0.9994103260695886, |
| "pass@64": 0.9999575357773413, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.8860677083333334, |
| "pass@2": 0.9257945404393898, |
| "pass@4": 0.952882770440732, |
| "pass@8": 0.9705083260290297, |
| "pass@16": 0.9827039584291745, |
| "pass@32": 0.9909022848607238, |
| "pass@64": 0.9954709952559196, |
| "pass@128": 0.9969135802469136 |
| }, |
| "14": { |
| "pass@1": 0.9277865356083086, |
| "pass@2": 0.9635647883128114, |
| "pass@4": 0.9799638896213635, |
| "pass@8": 0.9889540682347803, |
| "pass@16": 0.994416741293335, |
| "pass@32": 0.9976490069232948, |
| "pass@64": 0.9994606231626841, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.8958333333333334, |
| "pass@2": 0.9419447026586953, |
| "pass@4": 0.9628916451994424, |
| "pass@8": 0.9741455706645901, |
| "pass@16": 0.9817959566898227, |
| "pass@32": 0.9868451198367149, |
| "pass@64": 0.9890965712851263, |
| "pass@128": 0.9907407407407407 |
| }, |
| "16": { |
| "pass@1": 0.8752995943952803, |
| "pass@2": 0.9210983409983046, |
| "pass@4": 0.9490407169789616, |
| "pass@8": 0.9661475706409914, |
| "pass@16": 0.9755157512762427, |
| "pass@32": 0.9813566539237482, |
| "pass@64": 0.9858831578717764, |
| "pass@128": 0.9911504424778761 |
| }, |
| "17": { |
| "pass@1": 0.8405026355421686, |
| "pass@2": 0.8887784158286688, |
| "pass@4": 0.916870214265385, |
| "pass@8": 0.9383888936653869, |
| "pass@16": 0.9554419843774683, |
| "pass@32": 0.9678565567484874, |
| "pass@64": 0.977482393080175, |
| "pass@128": 0.9879518072289156 |
| }, |
| "18": { |
| "pass@1": 0.8315776209677419, |
| "pass@2": 0.903906448437897, |
| "pass@4": 0.9437463160854891, |
| "pass@8": 0.9663211807044224, |
| "pass@16": 0.9785213547715544, |
| "pass@32": 0.98575825369811, |
| "pass@64": 0.9908713414697528, |
| "pass@128": 0.9935483870967742 |
| }, |
| "19": { |
| "pass@1": 0.6603568731117825, |
| "pass@2": 0.7687665776577776, |
| "pass@4": 0.8426575621589597, |
| "pass@8": 0.8910139633294577, |
| "pass@16": 0.9241243059124612, |
| "pass@32": 0.948114566321418, |
| "pass@64": 0.964677567112477, |
| "pass@128": 0.9758308157099698 |
| }, |
| "20": { |
| "pass@1": 0.6414226726726727, |
| "pass@2": 0.7502992658012341, |
| "pass@4": 0.8205210395997796, |
| "pass@8": 0.8663415393566618, |
| "pass@16": 0.8988781493348023, |
| "pass@32": 0.9229708469087713, |
| "pass@64": 0.9401299955504153, |
| "pass@128": 0.9519519519519519 |
| } |
| } |
| } |
| } |
| } |
| ] |
