[ { "checkpoint": "checkpoint-21318", "id_acc": 0.8035069444444445, "ood_acc": 0.2582796875, "total_acc": 0.5165452302631579, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 204.2505277777778, "ood_avg_resp_len": 323.7365421875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 412002, "answer_accuracy": 0.5088153058804932, "avg_response_len": 255.9554986365792, "resp_tokens_sum": 207254334, "pass_at_k": { "pass@1": 0.5088153058804932, "pass@2": 0.6296618356999444, "pass@4": 0.719210862962984, "pass@8": 0.7764565317805123, "pass@16": 0.8141212161317084, "pass@32": 0.8417463724587748, "pass@64": 0.8643370923890424, "pass@128": 0.88444514701233 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6165883112094396, "pass@2": 0.812616135925487, "pass@4": 0.9329178775219467, "pass@8": 0.977880396978538, "pass@16": 0.9908326861094578, "pass@32": 0.9948797565434347, "pass@64": 0.9963124723368856, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 0.9954731308411215, "pass@2": 0.9986677337061839, "pass@4": 0.9998121735951231, "pass@8": 0.9999953031319798, "pass@16": 0.9999999973162129, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9922352828746177, "pass@2": 0.9942724511786946, "pass@4": 0.9946655274742032, "pass@8": 0.9952644136002354, "pass@16": 0.9960604393384298, "pass@32": 0.9967339001105344, "pass@64": 0.9969374778695336, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.8708740234375, "pass@2": 0.9614707646407487, "pass@4": 0.9918824697694028, "pass@8": 0.9981198577441812, "pass@16": 0.999321660090812, "pass@32": 0.9998867486090719, "pass@64": 0.9999990450817442, "pass@128": 1.0 }, "5": { "pass@1": 0.8150793650793651, "pass@2": 0.9419916260467439, "pass@4": 0.982795824926646, "pass@8": 0.9889652502551732, "pass@16": 0.9919730164564239, "pass@32": 0.9950963840236445, "pass@64": 0.9979906608382471, "pass@128": 1.0 }, "6": { "pass@1": 0.8174120508982036, "pass@2": 0.9460875247536419, "pass@4": 0.9918721735381876, "pass@8": 0.9995111198131046, "pass@16": 0.9999947143124999, "pass@32": 0.9999999996428128, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.7410318047337278, "pass@2": 0.9029223256301548, "pass@4": 0.9745284013130016, "pass@8": 0.9904476913691407, "pass@16": 0.9942210562613619, "pass@32": 0.9961335951016926, "pass@64": 0.9969518672230022, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.6792117387820513, "pass@2": 0.8634866968251566, "pass@4": 0.9648952387562136, "pass@8": 0.993117944184478, "pass@16": 0.9965457958060102, "pass@32": 0.9967904117243753, "pass@64": 0.9967948715804965, "pass@128": 0.9967948717948718 }, "9": { "pass@1": 0.6066021126760563, "pass@2": 0.8094976849284685, "pass@4": 0.9378918331511383, "pass@8": 0.9845614960965662, "pass@16": 0.9947414276845604, "pass@32": 0.9974605127528204, "pass@64": 0.9991334647987112, "pass@128": 1.0 }, "11": { "pass@1": 0.5432453966005666, "pass@2": 0.7387361702839548, "pass@4": 0.8783379028754543, "pass@8": 0.9483312450627431, "pass@16": 0.9758725095070463, "pass@32": 0.9865385261621175, "pass@64": 0.9926300371235457, "pass@128": 0.9971671388101983 }, "12": { "pass@1": 0.4385151836158192, "pass@2": 0.6272646136393972, "pass@4": 0.7860603523606157, "pass@8": 0.882109409036735, "pass@16": 0.928812777941062, "pass@32": 0.9530534035343814, "pass@64": 0.9699312836176254, "pass@128": 0.9830508474576272 }, "13": { "pass@1": 0.3491134404388715, "pass@2": 0.5090198670797024, "pass@4": 0.661433884244093, "pass@8": 0.776401535792301, "pass@16": 0.8491813137976658, "pass@32": 0.895806219262245, "pass@64": 0.9278853776262804, "pass@128": 0.9498432601880877 }, "14": { "pass@1": 0.2580558800623053, "pass@2": 0.38824715762749284, "pass@4": 0.5257789931632378, "pass@8": 0.6454717743203792, "pass@16": 0.7330442660593102, "pass@32": 0.7935724413098727, "pass@64": 0.8370651897946167, "pass@128": 0.8691588785046729 }, "15": { "pass@1": 0.21414378612716764, "pass@2": 0.3176043847571796, "pass@4": 0.43071095881800886, "pass@8": 0.5377233687380446, "pass@16": 0.6287938223510475, "pass@32": 0.7027849997101823, "pass@64": 0.7643402223718212, "pass@128": 0.815028901734104 }, "16": { "pass@1": 0.1795866935483871, "pass@2": 0.2694846170942341, "pass@4": 0.36764433669581636, "pass@8": 0.4615634120719388, "pass@16": 0.5461927456759639, "pass@32": 0.6222869255543437, "pass@64": 0.6901658237191357, "pass@128": 0.7516129032258064 }, "17": { "pass@1": 0.1604949421965318, "pass@2": 0.24124876257339187, "pass@4": 0.3307531632606617, "pass@8": 0.4170569873779501, "pass@16": 0.4934542148846228, "pass@32": 0.5591805518756274, "pass@64": 0.6148043872154428, "pass@128": 0.6647398843930635 }, "18": { "pass@1": 0.1572544642857143, "pass@2": 0.23766591676040497, "pass@4": 0.32500637527451925, "pass@8": 0.4069388221484305, "pass@16": 0.4773921054222746, "pass@32": 0.536398562769051, "pass@64": 0.5866353426613642, "pass@128": 0.6342857142857142 }, "19": { "pass@1": 0.14538757507507508, "pass@2": 0.2267137954884019, "pass@4": 0.3154293936793938, "pass@8": 0.3935377471028817, "pass@16": 0.4540979116157583, "pass@32": 0.5022593830872862, "pass@64": 0.5477458679126459, "pass@128": 0.5975975975975976 }, "20": { "pass@1": 0.1489771021021021, "pass@2": 0.22310376321203096, "pass@4": 0.30419779875488545, "pass@8": 0.3789745643544472, "pass@16": 0.43775889028306053, "pass@32": 0.4829222878013473, "pass@64": 0.5246680970289664, "pass@128": 0.5705705705705706 } } }, "teachers_in_school": { "count": 821120, "correct": 425073, "answer_accuracy": 0.5176746395167576, "avg_response_len": 264.2628373441154, "resp_tokens_sum": 216991501, "pass_at_k": { "pass@1": 0.5176746395167576, "pass@2": 0.6349976371815649, "pass@4": 0.7215786593657428, "pass@8": 0.778551692727141, "pass@16": 0.8182425326536708, "pass@32": 0.8480044140913875, "pass@64": 0.871279136844935, "pass@128": 0.8910366328916601 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6133291794478528, "pass@2": 0.8027378659243504, "pass@4": 0.9205583736465452, "pass@8": 0.9674064234788113, "pass@16": 0.9829619030877876, "pass@32": 0.9895795249587055, "pass@64": 0.99280285210983, "pass@128": 0.9938650306748467 }, "2": { "pass@1": 0.9952032674772037, "pass@2": 0.9980707614340762, "pass@4": 0.9992326393281387, "pass@8": 0.9998348270328344, "pass@16": 0.9999930098124662, "pass@32": 0.9999999950594095, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9973011363636364, "pass@2": 0.9999511602242903, "pass@4": 0.9999998931951688, "pass@8": 0.9999999999996503, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.8909254807692307, "pass@2": 0.9705062928062255, "pass@4": 0.9959271720103027, "pass@8": 0.99982548865015, "pass@16": 0.9999990246171927, "pass@32": 0.9999999999818837, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.833197463768116, "pass@2": 0.9586075687549926, "pass@4": 0.9946117156551079, "pass@8": 0.9974337743926552, "pass@16": 0.9977832912102159, "pass@32": 0.9983738445737761, "pass@64": 0.9992810681273536, "pass@128": 1.0 }, "6": { "pass@1": 0.8061873209169055, "pass@2": 0.939939125848881, "pass@4": 0.9886691014446968, "pass@8": 0.9967219867607618, "pass@16": 0.9971318736800938, "pass@32": 0.9971346703833113, "pass@64": 0.997134670487106, "pass@128": 0.997134670487106 }, "7": { "pass@1": 0.7547868497109826, "pass@2": 0.9130444677074329, "pass@4": 0.9816481297857992, "pass@8": 0.9949189365426445, "pass@16": 0.9966697958873755, "pass@32": 0.9970877881943166, "pass@64": 0.9971098189590546, "pass@128": 0.9971098265895953 }, "8": { "pass@1": 0.7190801056338029, "pass@2": 0.8901581734501498, "pass@4": 0.9724684888860728, "pass@8": 0.9922722433607305, "pass@16": 0.9953948168356105, "pass@32": 0.9969816044548047, "pass@64": 0.9984237646728893, "pass@128": 1.0 }, "9": { "pass@1": 0.6311517589820359, "pass@2": 0.8219579541703993, "pass@4": 0.938733234318764, "pass@8": 0.9796654820578499, "pass@16": 0.9887555941486048, "pass@32": 0.9913159709491492, "pass@64": 0.9925046000502376, "pass@128": 0.9940119760479041 }, "11": { "pass@1": 0.5394625376506024, "pass@2": 0.7293073623233092, "pass@4": 0.8675615849223656, "pass@8": 0.9365776763371002, "pass@16": 0.9638614577987218, "pass@32": 0.976376093759271, "pass@64": 0.982875676441502, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.47754212382445144, "pass@2": 0.6714831504702196, "pass@4": 0.8286672455519861, "pass@8": 0.9186892115567936, "pass@16": 0.958047126163231, "pass@32": 0.9754520342093819, "pass@64": 0.9834511293701561, "pass@128": 0.987460815047022 }, "13": { "pass@1": 0.32169117647058826, "pass@2": 0.4791001538410642, "pass@4": 0.635133928899643, "pass@8": 0.7556170680690744, "pass@16": 0.8327301912549687, "pass@32": 0.8797769281493306, "pass@64": 0.9101362187255622, "pass@128": 0.9299719887955182 }, "14": { "pass@1": 0.26025676169590645, "pass@2": 0.3793471128608925, "pass@4": 0.5060546422925207, "pass@8": 0.622533605638087, "pass@16": 0.7172345025972773, "pass@32": 0.786939069547155, "pass@64": 0.836873056687139, "pass@128": 0.8801169590643275 }, "15": { "pass@1": 0.1910748106060606, "pass@2": 0.29230344786447154, "pass@4": 0.40653435820522416, "pass@8": 0.5187997545912606, "pass@16": 0.6193221750308796, "pass@32": 0.7046677489962385, "pass@64": 0.7760240482135111, "pass@128": 0.8303030303030303 }, "16": { "pass@1": 0.18732193732193733, "pass@2": 0.27557197377571385, "pass@4": 0.3711783807472785, "pass@8": 0.4638118388269949, "pass@16": 0.5511895035013402, "pass@32": 0.6297923984883627, "pass@64": 0.6939239833694701, "pass@128": 0.7464387464387464 }, "17": { "pass@1": 0.18356948757763975, "pass@2": 0.2795955702548052, "pass@4": 0.3870548121508104, "pass@8": 0.48968299221273387, "pass@16": 0.5737510610237383, "pass@32": 0.6364109575939241, "pass@64": 0.6850414841538945, "pass@128": 0.7267080745341615 }, "18": { "pass@1": 0.16201746323529412, "pass@2": 0.24451460456229737, "pass@4": 0.33688678152363305, "pass@8": 0.4267673684121738, "pass@16": 0.5051313488725533, "pass@32": 0.5699013612301604, "pass@64": 0.6266758961770738, "pass@128": 0.6852941176470588 }, "19": { "pass@1": 0.13688151041666666, "pass@2": 0.20690052903543316, "pass@4": 0.28606640269296696, "pass@8": 0.3618872292001535, "pass@16": 0.4282848274461741, "pass@32": 0.4847637459313529, "pass@64": 0.5332019966802806, "pass@128": 0.5803571428571429 }, "20": { "pass@1": 0.12857877994011976, "pass@2": 0.2013392015182233, "pass@4": 0.2842333639807001, "pass@8": 0.3635574876563508, "pass@16": 0.4324952325076155, "pass@32": 0.4915495567243361, "pass@64": 0.542766037298866, "pass@128": 0.5898203592814372 } } }, "movie_festival_awards": { "count": 801152, "correct": 419163, "answer_accuracy": 0.5232003415082281, "avg_response_len": 281.3867368489375, "resp_tokens_sum": 225433547, "pass_at_k": { "pass@1": 0.5232003415082281, "pass@2": 0.6375872090646272, "pass@4": 0.7202131617984124, "pass@8": 0.7740711384989063, "pass@16": 0.8116172457435462, "pass@32": 0.8402094555036361, "pass@64": 0.8630031234578873, "pass@128": 0.8817702508387921 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6579757462686567, "pass@2": 0.834109472323422, "pass@4": 0.9359926791800278, "pass@8": 0.9764007783944053, "pass@16": 0.9891302901887874, "pass@32": 0.9936090877642131, "pass@64": 0.9960931634027773, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 0.99796875, "pass@2": 0.9998998172103488, "pass@4": 0.9999994881889763, "pass@8": 0.999999999992678, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9961962463556852, "pass@2": 0.9995627539542252, "pass@4": 0.9999798044559299, "pass@8": 0.9999999488003204, "pass@16": 0.9999999999998731, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.89453125, "pass@2": 0.9731856092001663, "pass@4": 0.9966935522862264, "pass@8": 0.9998640605910729, "pass@16": 0.9999986369646572, "pass@32": 0.9999999998824618, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.8262637867647059, "pass@2": 0.9490396306160266, "pass@4": 0.9877755136306483, "pass@8": 0.9940807517906356, "pass@16": 0.9959861118116005, "pass@32": 0.9967856269501686, "pass@64": 0.9970497180460696, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.823195977917981, "pass@2": 0.951510842296132, "pass@4": 0.9916722347798, "pass@8": 0.9967021708253823, "pass@16": 0.9968451891621816, "pass@32": 0.9968454258672304, "pass@64": 0.9968454258675079, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.7539804193037974, "pass@2": 0.9107024942689124, "pass@4": 0.9759190810207264, "pass@8": 0.9894950475520311, "pass@16": 0.9926975951745108, "pass@32": 0.9947609446934582, "pass@64": 0.996358011355757, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.7272663288288288, "pass@2": 0.8983523385590324, "pass@4": 0.9783853152927954, "pass@8": 0.9969389444483746, "pass@16": 0.9995268449841545, "pass@32": 0.999969422786309, "pass@64": 0.9999999637539232, "pass@128": 1.0 }, "9": { "pass@1": 0.6400221061093248, "pass@2": 0.8392124718333046, "pass@4": 0.954813273642242, "pass@8": 0.9911535307929749, "pass@16": 0.9982191365031068, "pass@32": 0.9998064765804047, "pass@64": 0.9999988148219604, "pass@128": 1.0 }, "11": { "pass@1": 0.561954365079365, "pass@2": 0.7611267341582302, "pass@4": 0.8971184090083979, "pass@8": 0.9587528668423654, "pass@16": 0.9797623351451179, "pass@32": 0.9863107640770923, "pass@64": 0.9886859689306161, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.46318329510703365, "pass@2": 0.6548530093428686, "pass@4": 0.8143184094534056, "pass@8": 0.9063695394401549, "pass@16": 0.9468248570430656, "pass@32": 0.9654655558635091, "pass@64": 0.9777256510909348, "pass@128": 0.9877675840978594 }, "13": { "pass@1": 0.3255931712962963, "pass@2": 0.47134581267619297, "pass@4": 0.6136517368199343, "pass@8": 0.7246697236089434, "pass@16": 0.8009965803126631, "pass@32": 0.8535512416219904, "pass@64": 0.8907295465025561, "pass@128": 0.9166666666666666 }, "14": { "pass@1": 0.260084384272997, "pass@2": 0.3810208766559966, "pass@4": 0.5072055616564245, "pass@8": 0.6211221484709386, "pass@16": 0.7160103259119381, "pass@32": 0.7899355065941975, "pass@64": 0.8437498091507096, "pass@128": 0.8813056379821959 }, "15": { "pass@1": 0.18246045524691357, "pass@2": 0.28586709864391924, "pass@4": 0.4065597717530684, "pass@8": 0.5254117981658483, "pass@16": 0.6286043269038194, "pass@32": 0.7120894308563969, "pass@64": 0.7742145101941069, "pass@128": 0.8179012345679012 }, "16": { "pass@1": 0.18743086283185842, "pass@2": 0.2746723515202192, "pass@4": 0.36543294947644794, "pass@8": 0.45079641882395305, "pass@16": 0.5289121278951456, "pass@32": 0.5992388962228877, "pass@64": 0.6598870502996406, "pass@128": 0.7079646017699115 }, "17": { "pass@1": 0.1480609939759036, "pass@2": 0.23009483801347114, "pass@4": 0.32449446407677934, "pass@8": 0.41465224741986667, "pass@16": 0.4907969542374535, "pass@32": 0.5558029831195344, "pass@64": 0.6162033085573296, "pass@128": 0.6716867469879518 }, "18": { "pass@1": 0.14957157258064516, "pass@2": 0.22162377762255522, "pass@4": 0.30280550455386623, "pass@8": 0.3820271166617574, "pass@16": 0.45029393610093404, "pass@32": 0.5069387708816698, "pass@64": 0.5562547798477242, "pass@128": 0.6096774193548387 }, "19": { "pass@1": 0.14494429758308158, "pass@2": 0.2195268453981017, "pass@4": 0.2999982741960881, "pass@8": 0.3747052677520511, "pass@16": 0.43860462915881887, "pass@32": 0.49174110119745773, "pass@64": 0.535026547131782, "pass@128": 0.56797583081571 }, "20": { "pass@1": 0.14484797297297297, "pass@2": 0.2247789127710388, "pass@4": 0.3120625040113227, "pass@8": 0.39140137273801684, "pass@16": 0.45705656446114257, "pass@32": 0.5116332228353286, "pass@64": 0.5593026408401216, "pass@128": 0.6066066066066066 } } } } } ]