[ { "checkpoint": "checkpoint-21414", "id_acc": 0.4404314236111111, "ood_acc": 0.11799453125, "total_acc": 0.2707277960526316, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 205.83686111111112, "ood_avg_resp_len": 302.30122265625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 216453, "answer_accuracy": 0.267315691195068, "avg_response_len": 240.56482176730952, "resp_tokens_sum": 194792072, "pass_at_k": { "pass@1": 0.267315691195068, "pass@2": 0.3575757567506737, "pass@4": 0.46355980296468213, "pass@8": 0.5718013169071188, "pass@16": 0.666415300283059, "pass@32": 0.7380510280672923, "pass@64": 0.7880629218987633, "pass@128": 0.8234271261460638 }, "per_op_pass_at_k": { "10": { "pass@1": 0.18249907817109146, "pass@2": 0.291669932989571, "pass@4": 0.4283879288319047, "pass@8": 0.5773130882437786, "pass@16": 0.7165110679120683, "pass@32": 0.8250908597517446, "pass@64": 0.897636864211352, "pass@128": 0.9410029498525073 }, "2": { "pass@1": 0.8387850467289719, "pass@2": 0.8770620232050428, "pass@4": 0.9158330404377023, "pass@8": 0.9489748592508037, "pass@16": 0.9716519179891818, "pass@32": 0.9853529444400086, "pass@64": 0.9937954667041277, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.8111381880733946, "pass@2": 0.8610955597293457, "pass@4": 0.9098772533020528, "pass@8": 0.9511167971847219, "pass@16": 0.976515556461128, "pass@32": 0.9883934257827562, "pass@64": 0.9953847266103103, "pass@128": 1.0 }, "4": { "pass@1": 0.669482421875, "pass@2": 0.7651678610974408, "pass@4": 0.8457666476260781, "pass@8": 0.9082670737372126, "pass@16": 0.9486523603161372, "pass@32": 0.9710074419973613, "pass@64": 0.9834710705171064, "pass@128": 0.990625 }, "5": { "pass@1": 0.3938988095238095, "pass@2": 0.5420541182352206, "pass@4": 0.6801027576910021, "pass@8": 0.7897984276173151, "pass@16": 0.8690475384495403, "pass@32": 0.9209395251145625, "pass@64": 0.94971396942988, "pass@128": 0.9650793650793651 }, "6": { "pass@1": 0.37394741766467066, "pass@2": 0.5273726660851524, "pass@4": 0.6815430298009147, "pass@8": 0.8069410782443649, "pass@16": 0.8908290146104205, "pass@32": 0.9401709226432646, "pass@64": 0.9657979861571938, "pass@128": 0.9760479041916168 }, "7": { "pass@1": 0.2760724852071006, "pass@2": 0.4113903869449749, "pass@4": 0.5637461165172402, "pass@8": 0.7078811006530169, "pass@16": 0.8210646809443971, "pass@32": 0.8946519991336546, "pass@64": 0.935935567293088, "pass@128": 0.9615384615384616 }, "8": { "pass@1": 0.2428886217948718, "pass@2": 0.3690830525691498, "pass@4": 0.5166896193264306, "pass@8": 0.6660882532949959, "pass@16": 0.7933717016068363, "pass@32": 0.8817371805976045, "pass@64": 0.9338947606912962, "pass@128": 0.9615384615384616 }, "9": { "pass@1": 0.18243838028169015, "pass@2": 0.29084299101696737, "pass@4": 0.42719150599132877, "pass@8": 0.573221389556622, "pass@16": 0.704905709039916, "pass@32": 0.8052016690957006, "pass@64": 0.8691051191103008, "pass@128": 0.9014084507042254 }, "11": { "pass@1": 0.17610216005665721, "pass@2": 0.28148142468381226, "pass@4": 0.4131508514516423, "pass@8": 0.5527875096043693, "pass@16": 0.6789762038953047, "pass@32": 0.7813219444877645, "pass@64": 0.8612559757607764, "pass@128": 0.9263456090651558 }, "12": { "pass@1": 0.1612597104519774, "pass@2": 0.25932328506606156, "pass@4": 0.38093756818533275, "pass@8": 0.5091701504450877, "pass@16": 0.6263580981659973, "pass@32": 0.7227684909326607, "pass@64": 0.7995695799454732, "pass@128": 0.8615819209039548 }, "13": { "pass@1": 0.14929467084639497, "pass@2": 0.2396229296768938, "pass@4": 0.3508439281100834, "pass@8": 0.471524299365065, "pass@16": 0.5856506718640497, "pass@32": 0.681310375631426, "pass@64": 0.7585447150206255, "pass@128": 0.8181818181818182 }, "14": { "pass@1": 0.12139797507788162, "pass@2": 0.20410196973041916, "pass@4": 0.31280950249209494, "pass@8": 0.43270018777666386, "pass@16": 0.5417473391370579, "pass@32": 0.6270024330655262, "pass@64": 0.6935990061074563, "pass@128": 0.7507788161993769 }, "15": { "pass@1": 0.0994400289017341, "pass@2": 0.1643412765691138, "pass@4": 0.25314546393406, "pass@8": 0.35941326573145005, "pass@16": 0.46831664641616344, "pass@32": 0.5659098593860116, "pass@64": 0.6472013520122053, "pass@128": 0.7109826589595376 }, "16": { "pass@1": 0.11169354838709677, "pass@2": 0.1847190913131826, "pass@4": 0.2792126208014322, "pass@8": 0.38286478587916734, "pass@16": 0.4783240254920387, "pass@32": 0.555416027111267, "pass@64": 0.6126382179445582, "pass@128": 0.6612903225806451 }, "17": { "pass@1": 0.10255599710982659, "pass@2": 0.16548056244595147, "pass@4": 0.24563900628750884, "pass@8": 0.33297676668484133, "pass@16": 0.41509483243131545, "pass@32": 0.48291606901432327, "pass@64": 0.5347978778878022, "pass@128": 0.5780346820809249 }, "18": { "pass@1": 0.08895089285714286, "pass@2": 0.14654879077615304, "pass@4": 0.22479175826236006, "pass@8": 0.3169439929353406, "pass@16": 0.4069978747646183, "pass@32": 0.47939295882154837, "pass@64": 0.5320287955283105, "pass@128": 0.5771428571428572 }, "19": { "pass@1": 0.08621903153153153, "pass@2": 0.14694209465370892, "pass@4": 0.2293803981146501, "pass@8": 0.3248957284145393, "pass@16": 0.4165254606722457, "pass@32": 0.48743485676917575, "pass@64": 0.5339771828430245, "pass@128": 0.5645645645645646 }, "20": { "pass@1": 0.08230105105105105, "pass@2": 0.13847361140668232, "pass@4": 0.21802641617995944, "pass@8": 0.3131342243867788, "pass@16": 0.4031538237236701, "pass@32": 0.4687000076530978, "pass@64": 0.5080511250655994, "pass@128": 0.5285285285285285 } } }, "teachers_in_school": { "count": 821120, "correct": 225144, "answer_accuracy": 0.2741913484021824, "avg_response_len": 251.82658807482463, "resp_tokens_sum": 206779848, "pass_at_k": { "pass@1": 0.2741913484021824, "pass@2": 0.36577319397819286, "pass@4": 0.4737994723331099, "pass@8": 0.5839717493885198, "pass@16": 0.6790056800475346, "pass@32": 0.7499407345326479, "pass@64": 0.8000514435926025, "pass@128": 0.8374123148869836 }, "per_op_pass_at_k": { "10": { "pass@1": 0.19200536809815952, "pass@2": 0.3103661960774842, "pass@4": 0.4589069955212655, "pass@8": 0.6160699655217626, "pass@16": 0.7569050484893751, "pass@32": 0.8624709959517797, "pass@64": 0.9267811088053876, "pass@128": 0.9631901840490797 }, "2": { "pass@1": 0.8651453267477204, "pass@2": 0.9047007006438025, "pass@4": 0.9401708911576017, "pass@8": 0.9673511274916078, "pass@16": 0.98366795053333, "pass@32": 0.991711774893075, "pass@64": 0.9951545679051006, "pass@128": 0.9969604863221885 }, "3": { "pass@1": 0.8408854166666667, "pass@2": 0.8882236488904792, "pass@4": 0.932338428435082, "pass@8": 0.9654705362519228, "pass@16": 0.9838244373702028, "pass@32": 0.9930547331033703, "pass@64": 0.9983618153437501, "pass@128": 1.0 }, "4": { "pass@1": 0.6366494082840237, "pass@2": 0.741660066160369, "pass@4": 0.836050945092662, "pass@8": 0.9059677477589775, "pass@16": 0.946855597259587, "pass@32": 0.9696531053645359, "pass@64": 0.9819148269345508, "pass@128": 0.985207100591716 }, "5": { "pass@1": 0.3551177536231884, "pass@2": 0.49381918292822075, "pass@4": 0.6387497155246896, "pass@8": 0.7666466857124686, "pass@16": 0.8599482291029481, "pass@32": 0.9189162017525144, "pass@64": 0.9534625742799305, "pass@128": 0.9739130434782609 }, "6": { "pass@1": 0.35104763610315187, "pass@2": 0.49330306782031896, "pass@4": 0.6415392234280171, "pass@8": 0.775567796619772, "pass@16": 0.8779930569384375, "pass@32": 0.9418969165130714, "pass@64": 0.9735513661788436, "pass@128": 0.9856733524355301 }, "7": { "pass@1": 0.33408598265895956, "pass@2": 0.47440055925993363, "pass@4": 0.6220848735129203, "pass@8": 0.7551767421132735, "pass@16": 0.8544841348917069, "pass@32": 0.9146878947571145, "pass@64": 0.9438021352909666, "pass@128": 0.9566473988439307 }, "8": { "pass@1": 0.25413732394366195, "pass@2": 0.38431192469779313, "pass@4": 0.5397205181218544, "pass@8": 0.6943626581541483, "pass@16": 0.817710695410848, "pass@32": 0.8966859300249802, "pass@64": 0.9428206550894571, "pass@128": 0.971830985915493 }, "9": { "pass@1": 0.20415419161676646, "pass@2": 0.320904450940638, "pass@4": 0.4636983232634841, "pass@8": 0.6121143311826535, "pass@16": 0.7430264595297534, "pass@32": 0.8429938708009328, "pass@64": 0.912625986148803, "pass@128": 0.9580838323353293 }, "11": { "pass@1": 0.15573230421686746, "pass@2": 0.2495582724599183, "pass@4": 0.3680684676275405, "pass@8": 0.49662259086905447, "pass@16": 0.6170162594364489, "pass@32": 0.7200076987482085, "pass@64": 0.8045671207619222, "pass@128": 0.8704819277108434 }, "12": { "pass@1": 0.18047119905956113, "pass@2": 0.27988315478982045, "pass@4": 0.402113326645298, "pass@8": 0.5357618074019428, "pass@16": 0.6619916157296409, "pass@32": 0.7672372529808019, "pass@64": 0.8506168212972275, "pass@128": 0.9153605015673981 }, "13": { "pass@1": 0.1331188725490196, "pass@2": 0.21367296918767498, "pass@4": 0.3160395567675891, "pass@8": 0.4292680673053684, "pass@16": 0.5393455857147501, "pass@32": 0.6380850741657642, "pass@64": 0.7222230960815036, "pass@128": 0.7927170868347339 }, "14": { "pass@1": 0.11794133771929824, "pass@2": 0.1945354589722337, "pass@4": 0.2962274923967836, "pass@8": 0.4098757266916545, "pass@16": 0.5162275945782999, "pass@32": 0.603999749398211, "pass@64": 0.6759269353819736, "pass@128": 0.7426900584795322 }, "15": { "pass@1": 0.1011126893939394, "pass@2": 0.16904602123598192, "pass@4": 0.26009995455113555, "pass@8": 0.36595839148405934, "pass@16": 0.4725821407031842, "pass@32": 0.5677398839532518, "pass@64": 0.6513693664909158, "pass@128": 0.7272727272727273 }, "16": { "pass@1": 0.11291399572649573, "pass@2": 0.18268074062857534, "pass@4": 0.2709819667306543, "pass@8": 0.36832898523240143, "pass@16": 0.46015042128877076, "pass@32": 0.5351319160537661, "pass@64": 0.5952268034436059, "pass@128": 0.6467236467236467 }, "17": { "pass@1": 0.11769701086956522, "pass@2": 0.19582984362009093, "pass@4": 0.2977075225845214, "pass@8": 0.4106816757981006, "pass@16": 0.5135671720839062, "pass@32": 0.5929697650046102, "pass@64": 0.6530400028524697, "pass@128": 0.6987577639751553 }, "18": { "pass@1": 0.09430147058823529, "pass@2": 0.16086729967577587, "pass@4": 0.25293459052912487, "pass@8": 0.36106318527066916, "pass@16": 0.46322122493015433, "pass@32": 0.5405034298643432, "pass@64": 0.5967379412271786, "pass@128": 0.6470588235294118 }, "19": { "pass@1": 0.09419177827380952, "pass@2": 0.15729650004686918, "pass@4": 0.2404237966347958, "pass@8": 0.33286242500490154, "pass@16": 0.4162781216933555, "pass@32": 0.4755985896299961, "pass@64": 0.514052982556868, "pass@128": 0.5476190476190477 }, "20": { "pass@1": 0.08460422904191617, "pass@2": 0.14392253878070635, "pass@4": 0.22773993660597816, "pass@8": 0.32560269641149836, "pass@16": 0.413814026198524, "pass@32": 0.47310524704300333, "pass@64": 0.5072669185320547, "pass@128": 0.5299401197604791 } } }, "movie_festival_awards": { "count": 801152, "correct": 216813, "answer_accuracy": 0.2706265477712095, "avg_response_len": 277.72221625858765, "resp_tokens_sum": 222497709, "pass_at_k": { "pass@1": 0.2706265477712095, "pass@2": 0.3627233869841646, "pass@4": 0.4714831141394156, "pass@8": 0.5817573231853167, "pass@16": 0.6751043837320008, "pass@32": 0.7432362479672774, "pass@64": 0.7910686801393282, "pass@128": 0.8269691644032593 }, "per_op_pass_at_k": { "10": { "pass@1": 0.21438899253731344, "pass@2": 0.33370989540486545, "pass@4": 0.4789243028390111, "pass@8": 0.6295483040575033, "pass@16": 0.7594710983716995, "pass@32": 0.8557769210941149, "pass@64": 0.9216117810953981, "pass@128": 0.9611940298507463 }, "2": { "pass@1": 0.831875, "pass@2": 0.8827921119235095, "pass@4": 0.9307492380416736, "pass@8": 0.9679590861746582, "pass@16": 0.9877240261931111, "pass@32": 0.9945705212997372, "pass@64": 0.9968058462187176, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.7918185131195336, "pass@2": 0.8481716013176923, "pass@4": 0.9032584244207081, "pass@8": 0.9486360554236684, "pass@16": 0.9769348854369816, "pass@32": 0.9907001703503954, "pass@64": 0.9959204930452993, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.6494883040935673, "pass@2": 0.7507130070912186, "pass@4": 0.8389371421774029, "pass@8": 0.9055928823671676, "pass@16": 0.9468274932773871, "pass@32": 0.968719693628846, "pass@64": 0.9805015058372825, "pass@128": 0.9883040935672515 }, "5": { "pass@1": 0.36792279411764706, "pass@2": 0.5179806912922651, "pass@4": 0.6698513004808214, "pass@8": 0.7977126442729795, "pass@16": 0.8852094688271894, "pass@32": 0.9371548591179986, "pass@64": 0.9642397376671343, "pass@128": 0.9764705882352941 }, "6": { "pass@1": 0.31415122239747634, "pass@2": 0.4550387801485384, "pass@4": 0.607989620185489, "pass@8": 0.7480016157202245, "pass@16": 0.852043421418948, "pass@32": 0.9173663172687372, "pass@64": 0.9547247098960704, "pass@128": 0.9684542586750788 }, "7": { "pass@1": 0.3049841772151899, "pass@2": 0.4434794864447324, "pass@4": 0.597717386790259, "pass@8": 0.743221162557493, "pass@16": 0.8524529336091656, "pass@32": 0.9168331659507437, "pass@64": 0.9491947918924216, "pass@128": 0.9651898734177216 }, "8": { "pass@1": 0.22787631381381382, "pass@2": 0.35784468031023126, "pass@4": 0.513812833643542, "pass@8": 0.6671665510913155, "pass@16": 0.7915539816168905, "pass@32": 0.8796837081297789, "pass@64": 0.9353007567211731, "pass@128": 0.963963963963964 }, "9": { "pass@1": 0.18611836816720256, "pass@2": 0.2989542718434308, "pass@4": 0.4405975697973446, "pass@8": 0.594700003859233, "pass@16": 0.7340836850211218, "pass@32": 0.8371155983990625, "pass@64": 0.9047208145978293, "pass@128": 0.954983922829582 }, "11": { "pass@1": 0.15535714285714286, "pass@2": 0.25387334395700534, "pass@4": 0.3802234271311323, "pass@8": 0.5185778550023629, "pass@16": 0.6463171263078665, "pass@32": 0.7484581374990609, "pass@64": 0.824998693875125, "pass@128": 0.8825396825396825 }, "12": { "pass@1": 0.15498375382262997, "pass@2": 0.24644939379710562, "pass@4": 0.36310674726439013, "pass@8": 0.49138307752199395, "pass@16": 0.6133379352066443, "pass@32": 0.7179926806217004, "pass@64": 0.8042931141328287, "pass@128": 0.8746177370030581 }, "13": { "pass@1": 0.1466772762345679, "pass@2": 0.23146363432001538, "pass@4": 0.33295316991626056, "pass@8": 0.4391792497041196, "pass@16": 0.5405334534430788, "pass@32": 0.6326803374503219, "pass@64": 0.7157104289974534, "pass@128": 0.7901234567901234 }, "14": { "pass@1": 0.12173126854599406, "pass@2": 0.19921208731512416, "pass@4": 0.29809855481714614, "pass@8": 0.4071295133196, "pass@16": 0.5094998867710719, "pass@32": 0.5944799771400907, "pass@64": 0.6675768737310531, "pass@128": 0.7388724035608308 }, "15": { "pass@1": 0.10139371141975309, "pass@2": 0.17266027510450077, "pass@4": 0.2726622945279986, "pass@8": 0.3916678557393639, "pass@16": 0.5068999403589238, "pass@32": 0.5979407415519157, "pass@64": 0.6628955996216994, "pass@128": 0.7098765432098766 }, "16": { "pass@1": 0.11481379056047197, "pass@2": 0.18550572840452464, "pass@4": 0.27434850981016756, "pass@8": 0.3706460035215884, "pass@16": 0.4582708669852493, "pass@32": 0.5277747582628878, "pass@64": 0.588016295160178, "pass@128": 0.6519174041297935 }, "17": { "pass@1": 0.1001976656626506, "pass@2": 0.16855352018309455, "pass@4": 0.2603574520240692, "pass@8": 0.36273649965024324, "pass@16": 0.45346864578811946, "pass@32": 0.518965437902586, "pass@64": 0.5645039379656953, "pass@128": 0.5993975903614458 }, "18": { "pass@1": 0.09596774193548387, "pass@2": 0.16143557912115825, "pass@4": 0.247989123637771, "pass@8": 0.3426376649389472, "pass@16": 0.4246539381310786, "pass@32": 0.4820093556043488, "pass@64": 0.5217290432579591, "pass@128": 0.5580645161290323 }, "19": { "pass@1": 0.09238104229607251, "pass@2": 0.15827202821324066, "pass@4": 0.24653544892991106, "pass@8": 0.3432939744879859, "pass@16": 0.4281201380647039, "pass@32": 0.48765535174226415, "pass@64": 0.5268287561695072, "pass@128": 0.5619335347432024 }, "20": { "pass@1": 0.08650056306306306, "pass@2": 0.14805602846941426, "pass@4": 0.23268120257490316, "pass@8": 0.3296337420615053, "pass@16": 0.41933828064911133, "pass@32": 0.4862638392287856, "pass@64": 0.5288202968457819, "pass@128": 0.5555555555555556 } } } } } ]