[ { "checkpoint": "checkpoint-87232", "id_acc": 0.9914244791666667, "ood_acc": 0.52209296875, "total_acc": 0.7444078947368421, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.40281510416668, "ood_avg_resp_len": 332.54644921875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 598322, "answer_accuracy": 0.738917266044894, "avg_response_len": 254.97740722810622, "resp_tokens_sum": 206462346, "pass_at_k": { "pass@1": 0.738917266044894, "pass@2": 0.7694304579152076, "pass@4": 0.794609388738074, "pass@8": 0.8157616660508631, "pass@16": 0.8335155078931495, "pass@32": 0.8485443383411363, "pass@64": 0.861710944272157, "pass@128": 0.8730635472652545 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9450589970501475, "pass@2": 0.9548245766845516, "pass@4": 0.9600733434316282, "pass@8": 0.9635012658519264, "pass@16": 0.9667523755850199, "pass@32": 0.9694535779841323, "pass@64": 0.9719733470168963, "pass@128": 0.976401179941003 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9998566513761468, "pass@2": 0.9999943563533916, "pass@4": 0.9999999957000787, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9946676587301587, "pass@2": 0.9956427321584803, "pass@4": 0.9970027764386595, "pass@8": 0.9985708586291078, "pass@16": 0.9996681965873846, "pass@32": 0.999983462274152, "pass@64": 0.9999999928786398, "pass@128": 1.0 }, "6": { "pass@1": 0.9991111526946108, "pass@2": 0.9998478682398981, "pass@4": 0.9999933339170928, "pass@8": 0.9999999910116041, "pass@16": 0.9999999999999978, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9969951923076923, "pass@2": 0.9970876479289941, "pass@4": 0.997133875739645, "pass@8": 0.9972263313609467, "pass@16": 0.9974112426035503, "pass@32": 0.9977810650887574, "pass@64": 0.9985207100591716, "pass@128": 1.0 }, "8": { "pass@1": 0.998848157051282, "pass@2": 0.999936906925096, "pass@4": 0.9999996550911907, "pass@8": 0.9999999999932476, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9937940140845071, "pass@2": 0.9959642203615392, "pass@4": 0.9971389198709315, "pass@8": 0.9984679686276483, "pass@16": 0.9995073140146584, "pass@32": 0.9999460220798828, "pass@64": 0.9999998252071107, "pass@128": 1.0 }, "11": { "pass@1": 0.9419484773371105, "pass@2": 0.9577259597153754, "pass@4": 0.9652531661233563, "pass@8": 0.9696622029637655, "pass@16": 0.9735155764884466, "pass@32": 0.9768388243314153, "pass@64": 0.9792149003246603, "pass@128": 0.9801699716713881 }, "12": { "pass@1": 0.8730799788135594, "pass@2": 0.8947448173851152, "pass@4": 0.9082170043892815, "pass@8": 0.9178664664910646, "pass@16": 0.9251073995443978, "pass@32": 0.9317042928103167, "pass@64": 0.937890439820976, "pass@128": 0.943502824858757 }, "13": { "pass@1": 0.8012588166144201, "pass@2": 0.8505772838348185, "pass@4": 0.8815550211395988, "pass@8": 0.902041422244805, "pass@16": 0.9156434397761177, "pass@32": 0.9251964170363804, "pass@64": 0.9315964192268797, "pass@128": 0.9373040752351097 }, "14": { "pass@1": 0.7124951323987538, "pass@2": 0.780948362033017, "pass@4": 0.8326437676598836, "pass@8": 0.8699622045929697, "pass@16": 0.8963467542274597, "pass@32": 0.9151630022320785, "pass@64": 0.928789217051466, "pass@128": 0.9376947040498442 }, "15": { "pass@1": 0.6236000722543352, "pass@2": 0.6962096342906561, "pass@4": 0.7469215516232433, "pass@8": 0.7861022338881439, "pass@16": 0.8184478974993373, "pass@32": 0.8432674392592889, "pass@64": 0.8632962541021941, "pass@128": 0.8815028901734104 }, "16": { "pass@1": 0.35897177419354837, "pass@2": 0.44126793878587767, "pass@4": 0.5133026022956806, "pass@8": 0.576692120625883, "pass@16": 0.6305196758703814, "pass@32": 0.6740991125135573, "pass@64": 0.7084279837224753, "pass@128": 0.7322580645161291 }, "17": { "pass@1": 0.2678829479768786, "pass@2": 0.34042317145327905, "pass@4": 0.40929082681500045, "pass@8": 0.4708638843017089, "pass@16": 0.523695917831466, "pass@32": 0.5694749577617649, "pass@64": 0.6097211919243859, "pass@128": 0.6445086705202312 }, "18": { "pass@1": 0.2124330357142857, "pass@2": 0.2710053430821146, "pass@4": 0.32689066795222027, "pass@8": 0.3798762015063781, "pass@16": 0.43045459996921887, "pass@32": 0.47929737923180654, "pass@64": 0.5266289675496978, "pass@128": 0.5742857142857143 }, "19": { "pass@1": 0.1770364114114114, "pass@2": 0.2389515499751718, "pass@4": 0.29822699752621007, "pass@8": 0.35113589354995, "pass@16": 0.3973504939392282, "pass@32": 0.4402537615119897, "pass@64": 0.4817467299813639, "pass@128": 0.5165165165165165 }, "20": { "pass@1": 0.17623873873873874, "pass@2": 0.2368385856328771, "pass@4": 0.2958934598265306, "pass@8": 0.34840419959660246, "pass@16": 0.3917964017371619, "pass@32": 0.4274189429328129, "pass@64": 0.4599680186130542, "pass@128": 0.4864864864864865 } } }, "teachers_in_school": { "count": 821120, "correct": 613598, "answer_accuracy": 0.7472695830085736, "avg_response_len": 264.4864185502728, "resp_tokens_sum": 217175088, "pass_at_k": { "pass@1": 0.7472695830085736, "pass@2": 0.7764727148170102, "pass@4": 0.8001171632384544, "pass@8": 0.8200480322935797, "pass@16": 0.8371993607504471, "pass@32": 0.8521570331482223, "pass@64": 0.8651447069991107, "pass@128": 0.8760717069368668 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9302626533742331, "pass@2": 0.9415724934785761, "pass@4": 0.9490184284863168, "pass@8": 0.9545399152772625, "pass@16": 0.9589845513425366, "pass@32": 0.9634418335407142, "pass@64": 0.9685974560641929, "pass@128": 0.9723926380368099 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9970561594202898, "pass@2": 0.9971010926623303, "pass@4": 0.9971014492753624, "pass@8": 0.9971014492753624, "pass@16": 0.9971014492753624, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9999104584527221, "pass@2": 0.9999996474742233, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.997154985549133, "pass@2": 0.9972883289335943, "pass@4": 0.9974584324900139, "pass@8": 0.9977745095808112, "pass@16": 0.9983173653148847, "pass@32": 0.9991000149547781, "pass@64": 0.9998278640025487, "pass@128": 1.0 }, "8": { "pass@1": 0.9962367957746479, "pass@2": 0.9972250332704892, "pass@4": 0.9977350557412716, "pass@8": 0.9982271286415376, "pass@16": 0.9989203963246213, "pass@32": 0.9996448039393536, "pass@64": 0.9999814875425009, "pass@128": 1.0 }, "9": { "pass@1": 0.9925383607784432, "pass@2": 0.9939662996840963, "pass@4": 0.994011899710141, "pass@8": 0.9940119760478099, "pass@16": 0.9940119760479041, "pass@32": 0.9940119760479041, "pass@64": 0.9940119760479041, "pass@128": 0.9940119760479041 }, "11": { "pass@1": 0.9601609563253012, "pass@2": 0.9713681250592926, "pass@4": 0.9778057019980937, "pass@8": 0.9825910991666997, "pass@16": 0.9860410600010839, "pass@32": 0.9876684351156156, "pass@64": 0.9879473502047925, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.880240987460815, "pass@2": 0.9021062683583047, "pass@4": 0.9134986309509112, "pass@8": 0.9205803424546687, "pass@16": 0.9264893920886731, "pass@32": 0.9330255530163245, "pass@64": 0.9399906987887063, "pass@128": 0.9467084639498433 }, "13": { "pass@1": 0.7943802521008403, "pass@2": 0.8459166639096585, "pass@4": 0.8788433969388278, "pass@8": 0.8994088755249545, "pass@16": 0.9124785964224158, "pass@32": 0.9226384251736293, "pass@64": 0.931349262027494, "pass@128": 0.938375350140056 }, "14": { "pass@1": 0.7241639254385965, "pass@2": 0.7835163696643181, "pass@4": 0.822974337911708, "pass@8": 0.8516282800611411, "pass@16": 0.8749411010307703, "pass@32": 0.8942279156943614, "pass@64": 0.9076990154148188, "pass@128": 0.9152046783625731 }, "15": { "pass@1": 0.6011126893939394, "pass@2": 0.6735012526843237, "pass@4": 0.7223410215768484, "pass@8": 0.7586458316427025, "pass@16": 0.7892176036668512, "pass@32": 0.8164603540597916, "pass@64": 0.8405001463727434, "pass@128": 0.8606060606060606 }, "16": { "pass@1": 0.38695245726495725, "pass@2": 0.46460688808129746, "pass@4": 0.5310301725104875, "pass@8": 0.586507777431234, "pass@16": 0.6342449420868452, "pass@32": 0.6763208380883311, "pass@64": 0.7114651639625552, "pass@128": 0.7378917378917379 }, "17": { "pass@1": 0.2949825310559006, "pass@2": 0.36418660133515907, "pass@4": 0.4260861967176461, "pass@8": 0.4826808908534714, "pass@16": 0.5355444047838134, "pass@32": 0.5855881445188759, "pass@64": 0.6355138158131769, "pass@128": 0.6863354037267081 }, "18": { "pass@1": 0.27095588235294116, "pass@2": 0.34281387505789707, "pass@4": 0.4097198835439688, "pass@8": 0.4676197190391348, "pass@16": 0.5144302283585803, "pass@32": 0.5544524552367253, "pass@64": 0.5900773367948332, "pass@128": 0.6205882352941177 }, "19": { "pass@1": 0.18643043154761904, "pass@2": 0.23665071651199854, "pass@4": 0.28584336751432854, "pass@8": 0.3333921116126603, "pass@16": 0.3781143547100361, "pass@32": 0.4184069867713272, "pass@64": 0.4528938053761574, "pass@128": 0.48214285714285715 }, "20": { "pass@1": 0.16635479041916168, "pass@2": 0.22205343250506857, "pass@4": 0.2799436228779785, "pass@8": 0.33815938365313236, "pass@16": 0.3910483229426106, "pass@32": 0.4332796007012806, "pass@64": 0.4667753594020036, "pass@128": 0.4940119760479042 } } }, "movie_festival_awards": { "count": 801152, "correct": 598480, "answer_accuracy": 0.7470242850295574, "avg_response_len": 284.9372703307238, "resp_tokens_sum": 228278064, "pass_at_k": { "pass@1": 0.7470242850295574, "pass@2": 0.7764492752483582, "pass@4": 0.8006267307884991, "pass@8": 0.8210757158392421, "pass@16": 0.8384563691978931, "pass@32": 0.8533037383582226, "pass@64": 0.8661504703510957, "pass@128": 0.8771369228311232 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9533582089552238, "pass@2": 0.9623622781760491, "pass@4": 0.9699540755166799, "pass@8": 0.9764016729742381, "pass@16": 0.9810715661203923, "pass@32": 0.984104975073764, "pass@64": 0.9863029330743918, "pass@128": 0.9880597014925373 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999772230320699, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9911305147058823, "pass@2": 0.9915231299212598, "pass@4": 0.9918838596462206, "pass@8": 0.9925205541194468, "pass@16": 0.9936037616977195, "pass@32": 0.9951483049912878, "pass@64": 0.9966150995831404, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9998274842271293, "pass@2": 0.9999988356640751, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9994808148734177, "pass@2": 0.9999630126831456, "pass@4": 0.9999997030632247, "pass@8": 0.9999999999933531, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9897006381381381, "pass@2": 0.9923240169303161, "pass@4": 0.9943014580947651, "pass@8": 0.996399289945866, "pass@16": 0.9983737805421303, "pass@32": 0.9995889385403273, "pass@64": 0.9999802281719206, "pass@128": 1.0 }, "9": { "pass@1": 0.9933681672025724, "pass@2": 0.9962718687495252, "pass@4": 0.9975119662493954, "pass@8": 0.9983480988867165, "pass@16": 0.9991978005026352, "pass@32": 0.999840112519781, "pass@64": 0.9999978529407324, "pass@128": 1.0 }, "11": { "pass@1": 0.9660962301587301, "pass@2": 0.9800563992000997, "pass@4": 0.9856098553157043, "pass@8": 0.9871547947890077, "pass@16": 0.9873004694850344, "pass@32": 0.9873015872839117, "pass@64": 0.9873015873015873, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.8945431957186545, "pass@2": 0.9204558109995422, "pass@4": 0.9357386582411145, "pass@8": 0.9461271755458232, "pass@16": 0.9546498872896517, "pass@32": 0.9623014839946717, "pass@64": 0.9687700308988774, "pass@128": 0.9755351681957186 }, "13": { "pass@1": 0.7693383487654321, "pass@2": 0.8177102319675312, "pass@4": 0.8476258693936403, "pass@8": 0.8677171963515228, "pass@16": 0.8825881522978666, "pass@32": 0.8956357713705437, "pass@64": 0.9076377086038764, "pass@128": 0.9166666666666666 }, "14": { "pass@1": 0.7078310459940653, "pass@2": 0.7641446061823876, "pass@4": 0.8041013619588346, "pass@8": 0.8347883330046196, "pass@16": 0.8571583652568017, "pass@32": 0.8747660098007389, "pass@64": 0.8917936809048442, "pass@128": 0.9109792284866469 }, "15": { "pass@1": 0.5994888117283951, "pass@2": 0.6626423216195194, "pass@4": 0.7061492657746482, "pass@8": 0.7363345882150655, "pass@16": 0.762555667288486, "pass@32": 0.7893522163562189, "pass@64": 0.8159024934497591, "pass@128": 0.8364197530864198 }, "16": { "pass@1": 0.38262813421828906, "pass@2": 0.4605439080900284, "pass@4": 0.5269444925578988, "pass@8": 0.5858249902885159, "pass@16": 0.6374014414272734, "pass@32": 0.6810340436525093, "pass@64": 0.719601323264067, "pass@128": 0.7492625368731564 }, "17": { "pass@1": 0.29045086596385544, "pass@2": 0.36035369331657324, "pass@4": 0.4245875250156982, "pass@8": 0.48251196755235387, "pass@16": 0.5349999493590595, "pass@32": 0.5813152369542808, "pass@64": 0.6206299947132856, "pass@128": 0.6596385542168675 }, "18": { "pass@1": 0.2664818548387097, "pass@2": 0.33335899796799573, "pass@4": 0.39368648868488204, "pass@8": 0.44560317588205367, "pass@16": 0.48800080885312824, "pass@32": 0.5221118670936868, "pass@64": 0.551841095020944, "pass@128": 0.5806451612903226 }, "19": { "pass@1": 0.19399074773413896, "pass@2": 0.2584612960011418, "pass@4": 0.3207119609482348, "pass@8": 0.3767541677035014, "pass@16": 0.4255612780773268, "pass@32": 0.4675661315565914, "pass@64": 0.5024452893007847, "pass@128": 0.5287009063444109 }, "20": { "pass@1": 0.18362894144144143, "pass@2": 0.24175023941264087, "pass@4": 0.30319641717082657, "pass@8": 0.36417141782152096, "pass@16": 0.4183361652163054, "pass@32": 0.4627567871215331, "pass@64": 0.49807506355210696, "pass@128": 0.5255255255255256 } } } } } ]