[ { "checkpoint": "checkpoint-21318", "id_acc": 0.9815355902777778, "ood_acc": 0.394646875, "total_acc": 0.6726467927631579, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.10997916666668, "ood_avg_resp_len": 319.486109375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 546485, "answer_accuracy": 0.6748994724154284, "avg_response_len": 251.14894754781852, "resp_tokens_sum": 203362335, "pass_at_k": { "pass@1": 0.6748994724154284, "pass@2": 0.7191689411091793, "pass@4": 0.7552899944770523, "pass@8": 0.7857780117708147, "pass@16": 0.811338087265243, "pass@32": 0.8327335004279607, "pass@64": 0.8506094271687754, "pass@128": 0.8654758141005374 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9187177359882006, "pass@2": 0.9419193348895546, "pass@4": 0.9525911570235132, "pass@8": 0.9601124430446681, "pass@16": 0.9688315497242321, "pass@32": 0.9795041555963347, "pass@64": 0.9898968250497987, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 0.9999756619937694, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9991876911314985, "pass@2": 0.9998359580052493, "pass@4": 0.9999921440439211, "pass@8": 0.9999999874807163, "pass@16": 0.9999999999999951, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9972412109375, "pass@2": 0.9994767316683071, "pass@4": 0.9999673117032245, "pass@8": 0.999999790769164, "pass@16": 0.9999999999965491, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9859126984126985, "pass@2": 0.991292494688164, "pass@4": 0.9941161607775217, "pass@8": 0.9963759833612771, "pass@16": 0.998407301987386, "pass@32": 0.9995858450337997, "pass@64": 0.9999791349728794, "pass@128": 1.0 }, "6": { "pass@1": 0.9986433383233533, "pass@2": 0.9999635325805083, "pass@4": 0.9999999497630162, "pass@8": 0.9999999999999811, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9890208949704142, "pass@2": 0.9962824832269488, "pass@4": 0.9985037498086408, "pass@8": 0.9994197837330677, "pass@16": 0.9999007616758336, "pass@32": 0.9999981593967747, "pass@64": 0.99999999996853, "pass@128": 1.0 }, "8": { "pass@1": 0.9772636217948718, "pass@2": 0.9900265621845347, "pass@4": 0.9949976925961178, "pass@8": 0.9977808437973272, "pass@16": 0.9992447903847085, "pass@32": 0.9998833832693876, "pass@64": 0.9999990205959386, "pass@128": 1.0 }, "9": { "pass@1": 0.9679797535211268, "pass@2": 0.9858849950094272, "pass@4": 0.9922554352360883, "pass@8": 0.995677010909272, "pass@16": 0.997648471548975, "pass@32": 0.9991632867490207, "pass@64": 0.9999425475456929, "pass@128": 1.0 }, "11": { "pass@1": 0.900208038243626, "pass@2": 0.940261063772836, "pass@4": 0.9605637911585411, "pass@8": 0.9736966517888073, "pass@16": 0.9817176871122001, "pass@32": 0.9859188308542048, "pass@64": 0.9884939701677804, "pass@128": 0.9915014164305949 }, "12": { "pass@1": 0.7937853107344632, "pass@2": 0.856032644356955, "pass@4": 0.8879833495389342, "pass@8": 0.9112921328022885, "pass@16": 0.9299471191821015, "pass@32": 0.9432499614194477, "pass@64": 0.9519354007423595, "pass@128": 0.9576271186440678 }, "13": { "pass@1": 0.627228644200627, "pass@2": 0.7308244421543701, "pass@4": 0.8007843176146867, "pass@8": 0.8472013367086182, "pass@16": 0.8778727409625665, "pass@32": 0.8993320146357454, "pass@64": 0.9167319223361919, "pass@128": 0.9341692789968652 }, "14": { "pass@1": 0.46872566199376947, "pass@2": 0.5899655358500745, "pass@4": 0.6894011131202055, "pass@8": 0.7626993304280723, "pass@16": 0.8137387089660814, "pass@32": 0.8517713011755046, "pass@64": 0.8809287962069493, "pass@128": 0.9034267912772586 }, "15": { "pass@1": 0.33945989884393063, "pass@2": 0.44484704269264014, "pass@4": 0.537431375449456, "pass@8": 0.6161823443475056, "pass@16": 0.6808836029257295, "pass@32": 0.7332441335839044, "pass@64": 0.7752577955426394, "pass@128": 0.8034682080924855 }, "16": { "pass@1": 0.22444556451612904, "pass@2": 0.3037044386588771, "pass@4": 0.3856819596139193, "pass@8": 0.464336732781557, "pass@16": 0.5348499625789646, "pass@32": 0.596015201186016, "pass@64": 0.648878495267106, "pass@128": 0.6967741935483871 }, "17": { "pass@1": 0.1936416184971098, "pass@2": 0.26216233899230784, "pass@4": 0.3295796038500966, "pass@8": 0.39133321575481383, "pass@16": 0.44817939219526937, "pass@32": 0.500433802013655, "pass@64": 0.5478938944123481, "pass@128": 0.5895953757225434 }, "18": { "pass@1": 0.166875, "pass@2": 0.22814363048368944, "pass@4": 0.29212108575713747, "pass@8": 0.3567607601004869, "pass@16": 0.4191239061077216, "pass@32": 0.47647856131975425, "pass@64": 0.5270904349223088, "pass@128": 0.5685714285714286 }, "19": { "pass@1": 0.15127627627627627, "pass@2": 0.21408958170769188, "pass@4": 0.27785267325818513, "pass@8": 0.336830033058738, "pass@16": 0.3908078829480527, "pass@32": 0.4407470585272865, "pass@64": 0.48373386071585606, "pass@128": 0.5225225225225225 }, "20": { "pass@1": 0.15632038288288289, "pass@2": 0.21961528457591445, "pass@4": 0.2859781558836679, "pass@8": 0.3481888556973961, "pass@16": 0.40061764130662714, "pass@32": 0.44109860212106855, "pass@64": 0.47369522531282565, "pass@128": 0.5015015015015015 } } }, "teachers_in_school": { "count": 821120, "correct": 553346, "answer_accuracy": 0.6738917575993765, "avg_response_len": 260.4377813230709, "resp_tokens_sum": 213850671, "pass_at_k": { "pass@1": 0.6738917575993765, "pass@2": 0.7172152611988333, "pass@4": 0.7531338743120783, "pass@8": 0.7837519089282062, "pass@16": 0.8100695941203857, "pass@32": 0.8328153046805763, "pass@64": 0.8529624439757696, "pass@128": 0.8701480904130943 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9125527223926381, "pass@2": 0.9331044544949524, "pass@4": 0.9456968690838492, "pass@8": 0.9566705977659613, "pass@16": 0.9664497828821677, "pass@32": 0.9739261389620586, "pass@64": 0.9797493602093914, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9996212121212121, "pass@2": 0.9999839686232402, "pass@4": 0.9999999758552907, "pass@8": 0.9999999999999979, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9997226331360947, "pass@2": 0.9999941760238552, "pass@4": 0.9999999986133391, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9931612318840579, "pass@2": 0.9957563049184068, "pass@4": 0.9969017810273715, "pass@8": 0.9972675874824468, "pass@16": 0.9974637327583552, "pass@32": 0.9978260869564953, "pass@64": 0.9985507246376811, "pass@128": 1.0 }, "6": { "pass@1": 0.9940231017191977, "pass@2": 0.9956406662455162, "pass@4": 0.9965047204880193, "pass@8": 0.9973770261530999, "pass@16": 0.998066631116763, "pass@32": 0.9988006832926887, "pass@64": 0.9996502944295251, "pass@128": 1.0 }, "7": { "pass@1": 0.991126264450867, "pass@2": 0.9952906672431843, "pass@4": 0.9967147617205364, "pass@8": 0.9976171738733315, "pass@16": 0.9983107185422295, "pass@32": 0.9991000102569333, "pass@64": 0.9998278640025487, "pass@128": 1.0 }, "8": { "pass@1": 0.9749779929577465, "pass@2": 0.9864294527004549, "pass@4": 0.9915881530653737, "pass@8": 0.9945899766336617, "pass@16": 0.9961582248128492, "pass@32": 0.9969211944658168, "pass@64": 0.9971743778454023, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.9632765718562875, "pass@2": 0.9803109086944224, "pass@4": 0.9856916114340498, "pass@8": 0.9883274821465693, "pass@16": 0.9901297799031338, "pass@32": 0.9914736057743874, "pass@64": 0.992505694289581, "pass@128": 0.9940119760479041 }, "11": { "pass@1": 0.8991434487951807, "pass@2": 0.9375277932596532, "pass@4": 0.9547015353314265, "pass@8": 0.9633609478714571, "pass@16": 0.9693936038021645, "pass@32": 0.9742566301782772, "pass@64": 0.9782794167130561, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8104182993730408, "pass@2": 0.8787746371535066, "pass@4": 0.9125708799527014, "pass@8": 0.9304602175389681, "pass@16": 0.9416070103033272, "pass@32": 0.9500769794260939, "pass@64": 0.9590956310205097, "pass@128": 0.9686520376175548 }, "13": { "pass@1": 0.5954350490196079, "pass@2": 0.7000781611857338, "pass@4": 0.7674895590782237, "pass@8": 0.8114815716167344, "pass@16": 0.844454160581966, "pass@32": 0.8720772893119872, "pass@64": 0.8940016890795939, "pass@128": 0.907563025210084 }, "14": { "pass@1": 0.4603207236842105, "pass@2": 0.5714535991389238, "pass@4": 0.6594985125214609, "pass@8": 0.7265699264704736, "pass@16": 0.7778373120159718, "pass@32": 0.8180699356216169, "pass@64": 0.8516240506945955, "pass@128": 0.8771929824561403 }, "15": { "pass@1": 0.31363636363636366, "pass@2": 0.41937335958005234, "pass@4": 0.5162882960652643, "pass@8": 0.5983970205376589, "pass@16": 0.6648011303037484, "pass@32": 0.7190441181140116, "pass@64": 0.7636872110138734, "pass@128": 0.8 }, "16": { "pass@1": 0.23312856125356127, "pass@2": 0.3083505787738071, "pass@4": 0.3828097024730884, "pass@8": 0.45351063845009104, "pass@16": 0.5170390838126733, "pass@32": 0.569717191943559, "pass@64": 0.6154515292082063, "pass@128": 0.6552706552706553 }, "17": { "pass@1": 0.20569681677018634, "pass@2": 0.28773628160610343, "pass@4": 0.3712533041289092, "pass@8": 0.45004283295561615, "pass@16": 0.5220548916570253, "pass@32": 0.5866811888450802, "pass@64": 0.6401217163146227, "pass@128": 0.6832298136645962 }, "18": { "pass@1": 0.18230698529411765, "pass@2": 0.24840022869383982, "pass@4": 0.31653323325393146, "pass@8": 0.3846826446184044, "pass@16": 0.4524686153958313, "pass@32": 0.5189177849220697, "pass@64": 0.5829468976280522, "pass@128": 0.6411764705882353 }, "19": { "pass@1": 0.13516090029761904, "pass@2": 0.19425574732845896, "pass@4": 0.2599179865351651, "pass@8": 0.3226904065867681, "pass@16": 0.3778738096041623, "pass@32": 0.4263036119966427, "pass@64": 0.47324745553640984, "pass@128": 0.5148809523809523 }, "20": { "pass@1": 0.12696482035928144, "pass@2": 0.18205088346928192, "pass@4": 0.2436986123366316, "pass@8": 0.30770664465547454, "pass@16": 0.36782947757335144, "pass@32": 0.4220210875952675, "pass@64": 0.47310400200799735, "pass@128": 0.5209580838323353 } } }, "movie_festival_awards": { "count": 801152, "correct": 536046, "answer_accuracy": 0.669094004633328, "avg_response_len": 277.4204021209458, "resp_tokens_sum": 222255910, "pass_at_k": { "pass@1": 0.669094004633328, "pass@2": 0.7131195448632707, "pass@4": 0.7498518528669226, "pass@8": 0.7809761802925407, "pass@16": 0.8071611561775404, "pass@32": 0.8288585002889368, "pass@64": 0.8467381742920284, "pass@128": 0.8613196996325292 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9236240671641791, "pass@2": 0.9507807909272536, "pass@4": 0.9667486526870706, "pass@8": 0.9777367314708735, "pass@16": 0.9858440541829969, "pass@32": 0.9909022008929449, "pass@64": 0.9943305330613756, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9994305758017493, "pass@2": 0.9999802719175408, "pass@4": 0.9999999822361854, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9998857821637427, "pass@2": 0.9999978415527007, "pass@4": 0.9999999997259114, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9794577205882353, "pass@2": 0.986104678091709, "pass@4": 0.9914057078343147, "pass@8": 0.9952525272351784, "pass@16": 0.9976998298743082, "pass@32": 0.9990539138081224, "pass@64": 0.9998248097159032, "pass@128": 1.0 }, "6": { "pass@1": 0.996623619873817, "pass@2": 0.9981816953973025, "pass@4": 0.9989754296288672, "pass@8": 0.9996811227064588, "pass@16": 0.999973197893188, "pass@32": 0.9999999053842525, "pass@64": 0.9999999999999886, "pass@128": 1.0 }, "7": { "pass@1": 0.9818285205696202, "pass@2": 0.9903634412688129, "pass@4": 0.9938059475160541, "pass@8": 0.9956453079406357, "pass@16": 0.9968396038290583, "pass@32": 0.997595232279207, "pass@64": 0.998417683324389, "pass@128": 1.0 }, "8": { "pass@1": 0.9867445570570571, "pass@2": 0.9948784906954196, "pass@4": 0.9967002438321332, "pass@8": 0.9982105143733483, "pass@16": 0.9994664808580491, "pass@32": 0.9999560018404838, "pass@64": 0.9999999172814134, "pass@128": 1.0 }, "9": { "pass@1": 0.9711113344051447, "pass@2": 0.9879939457427148, "pass@4": 0.9937115386702061, "pass@8": 0.9961771559176027, "pass@16": 0.9975684602475261, "pass@32": 0.9986379117884371, "pass@64": 0.9996075579193004, "pass@128": 1.0 }, "11": { "pass@1": 0.8969246031746032, "pass@2": 0.9403961223597047, "pass@4": 0.9624989144214113, "pass@8": 0.9744247857115855, "pass@16": 0.9800360707668536, "pass@32": 0.9838969818396167, "pass@64": 0.9876262661924977, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.7949397935779816, "pass@2": 0.871270302078066, "pass@4": 0.9135008060918985, "pass@8": 0.9385238006991564, "pass@16": 0.9559675949973659, "pass@32": 0.9689443246955691, "pass@64": 0.978408685654852, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.5622829861111112, "pass@2": 0.6771364950908916, "pass@4": 0.7605942616547924, "pass@8": 0.8190895667375049, "pass@16": 0.8586141115589907, "pass@32": 0.8850650852501746, "pass@64": 0.9046173061888647, "pass@128": 0.9228395061728395 }, "14": { "pass@1": 0.4215968100890208, "pass@2": 0.5351282302156589, "pass@4": 0.6310083644032625, "pass@8": 0.7068240714416226, "pass@16": 0.7639057717499893, "pass@32": 0.8058978052015708, "pass@64": 0.8372551030262958, "pass@128": 0.8605341246290801 }, "15": { "pass@1": 0.29094328703703703, "pass@2": 0.38806602082725755, "pass@4": 0.4813472141213828, "pass@8": 0.5644691113048594, "pass@16": 0.6337115033449147, "pass@32": 0.6900797263618773, "pass@64": 0.7355486564030599, "pass@128": 0.7685185185185185 }, "16": { "pass@1": 0.222483407079646, "pass@2": 0.2984148897870065, "pass@4": 0.3756056095864124, "pass@8": 0.44816039374274946, "pass@16": 0.5127854787924226, "pass@32": 0.5715936942435857, "pass@64": 0.626397140540926, "pass@128": 0.672566371681416 }, "17": { "pass@1": 0.18298192771084337, "pass@2": 0.2520570717911013, "pass@4": 0.3194832942568926, "pass@8": 0.38233558376000504, "pass@16": 0.44278971561258734, "pass@32": 0.4990358885656421, "pass@64": 0.5508491199781063, "pass@128": 0.5993975903614458 }, "18": { "pass@1": 0.17641129032258066, "pass@2": 0.24202200279400554, "pass@4": 0.31029111028460155, "pass@8": 0.37480109802166267, "pass@16": 0.43077098344591513, "pass@32": 0.47823612116184727, "pass@64": 0.5179535591299852, "pass@128": 0.5483870967741935 }, "19": { "pass@1": 0.1512698262839879, "pass@2": 0.20805926921521523, "pass@4": 0.2633237125570783, "pass@8": 0.3144712609160323, "pass@16": 0.36182671356535456, "pass@32": 0.4048053375175812, "pass@64": 0.44179356967491973, "pass@128": 0.4743202416918429 }, "20": { "pass@1": 0.15599192942942944, "pass@2": 0.21430165401622092, "pass@4": 0.2763685485260287, "pass@8": 0.34237324035456, "pass@16": 0.4086498747300178, "pass@32": 0.4654420970793097, "pass@64": 0.5065305978632904, "pass@128": 0.5375375375375375 } } } } } ]