| [ |
| { |
| "checkpoint": "checkpoint-21318", |
| "id_acc": 0.9815355902777778, |
| "ood_acc": 0.394646875, |
| "total_acc": 0.6726467927631579, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.10997916666668, |
| "ood_avg_resp_len": 319.486109375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 546485, |
| "answer_accuracy": 0.6748994724154284, |
| "avg_response_len": 251.14894754781852, |
| "resp_tokens_sum": 203362335, |
| "pass_at_k": { |
| "pass@1": 0.6748994724154284, |
| "pass@2": 0.7191689411091793, |
| "pass@4": 0.7552899944770523, |
| "pass@8": 0.7857780117708147, |
| "pass@16": 0.811338087265243, |
| "pass@32": 0.8327335004279607, |
| "pass@64": 0.8506094271687754, |
| "pass@128": 0.8654758141005374 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9187177359882006, |
| "pass@2": 0.9419193348895546, |
| "pass@4": 0.9525911570235132, |
| "pass@8": 0.9601124430446681, |
| "pass@16": 0.9688315497242321, |
| "pass@32": 0.9795041555963347, |
| "pass@64": 0.9898968250497987, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 0.9999756619937694, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9991876911314985, |
| "pass@2": 0.9998359580052493, |
| "pass@4": 0.9999921440439211, |
| "pass@8": 0.9999999874807163, |
| "pass@16": 0.9999999999999951, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9972412109375, |
| "pass@2": 0.9994767316683071, |
| "pass@4": 0.9999673117032245, |
| "pass@8": 0.999999790769164, |
| "pass@16": 0.9999999999965491, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9859126984126985, |
| "pass@2": 0.991292494688164, |
| "pass@4": 0.9941161607775217, |
| "pass@8": 0.9963759833612771, |
| "pass@16": 0.998407301987386, |
| "pass@32": 0.9995858450337997, |
| "pass@64": 0.9999791349728794, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9986433383233533, |
| "pass@2": 0.9999635325805083, |
| "pass@4": 0.9999999497630162, |
| "pass@8": 0.9999999999999811, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9890208949704142, |
| "pass@2": 0.9962824832269488, |
| "pass@4": 0.9985037498086408, |
| "pass@8": 0.9994197837330677, |
| "pass@16": 0.9999007616758336, |
| "pass@32": 0.9999981593967747, |
| "pass@64": 0.99999999996853, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9772636217948718, |
| "pass@2": 0.9900265621845347, |
| "pass@4": 0.9949976925961178, |
| "pass@8": 0.9977808437973272, |
| "pass@16": 0.9992447903847085, |
| "pass@32": 0.9998833832693876, |
| "pass@64": 0.9999990205959386, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9679797535211268, |
| "pass@2": 0.9858849950094272, |
| "pass@4": 0.9922554352360883, |
| "pass@8": 0.995677010909272, |
| "pass@16": 0.997648471548975, |
| "pass@32": 0.9991632867490207, |
| "pass@64": 0.9999425475456929, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.900208038243626, |
| "pass@2": 0.940261063772836, |
| "pass@4": 0.9605637911585411, |
| "pass@8": 0.9736966517888073, |
| "pass@16": 0.9817176871122001, |
| "pass@32": 0.9859188308542048, |
| "pass@64": 0.9884939701677804, |
| "pass@128": 0.9915014164305949 |
| }, |
| "12": { |
| "pass@1": 0.7937853107344632, |
| "pass@2": 0.856032644356955, |
| "pass@4": 0.8879833495389342, |
| "pass@8": 0.9112921328022885, |
| "pass@16": 0.9299471191821015, |
| "pass@32": 0.9432499614194477, |
| "pass@64": 0.9519354007423595, |
| "pass@128": 0.9576271186440678 |
| }, |
| "13": { |
| "pass@1": 0.627228644200627, |
| "pass@2": 0.7308244421543701, |
| "pass@4": 0.8007843176146867, |
| "pass@8": 0.8472013367086182, |
| "pass@16": 0.8778727409625665, |
| "pass@32": 0.8993320146357454, |
| "pass@64": 0.9167319223361919, |
| "pass@128": 0.9341692789968652 |
| }, |
| "14": { |
| "pass@1": 0.46872566199376947, |
| "pass@2": 0.5899655358500745, |
| "pass@4": 0.6894011131202055, |
| "pass@8": 0.7626993304280723, |
| "pass@16": 0.8137387089660814, |
| "pass@32": 0.8517713011755046, |
| "pass@64": 0.8809287962069493, |
| "pass@128": 0.9034267912772586 |
| }, |
| "15": { |
| "pass@1": 0.33945989884393063, |
| "pass@2": 0.44484704269264014, |
| "pass@4": 0.537431375449456, |
| "pass@8": 0.6161823443475056, |
| "pass@16": 0.6808836029257295, |
| "pass@32": 0.7332441335839044, |
| "pass@64": 0.7752577955426394, |
| "pass@128": 0.8034682080924855 |
| }, |
| "16": { |
| "pass@1": 0.22444556451612904, |
| "pass@2": 0.3037044386588771, |
| "pass@4": 0.3856819596139193, |
| "pass@8": 0.464336732781557, |
| "pass@16": 0.5348499625789646, |
| "pass@32": 0.596015201186016, |
| "pass@64": 0.648878495267106, |
| "pass@128": 0.6967741935483871 |
| }, |
| "17": { |
| "pass@1": 0.1936416184971098, |
| "pass@2": 0.26216233899230784, |
| "pass@4": 0.3295796038500966, |
| "pass@8": 0.39133321575481383, |
| "pass@16": 0.44817939219526937, |
| "pass@32": 0.500433802013655, |
| "pass@64": 0.5478938944123481, |
| "pass@128": 0.5895953757225434 |
| }, |
| "18": { |
| "pass@1": 0.166875, |
| "pass@2": 0.22814363048368944, |
| "pass@4": 0.29212108575713747, |
| "pass@8": 0.3567607601004869, |
| "pass@16": 0.4191239061077216, |
| "pass@32": 0.47647856131975425, |
| "pass@64": 0.5270904349223088, |
| "pass@128": 0.5685714285714286 |
| }, |
| "19": { |
| "pass@1": 0.15127627627627627, |
| "pass@2": 0.21408958170769188, |
| "pass@4": 0.27785267325818513, |
| "pass@8": 0.336830033058738, |
| "pass@16": 0.3908078829480527, |
| "pass@32": 0.4407470585272865, |
| "pass@64": 0.48373386071585606, |
| "pass@128": 0.5225225225225225 |
| }, |
| "20": { |
| "pass@1": 0.15632038288288289, |
| "pass@2": 0.21961528457591445, |
| "pass@4": 0.2859781558836679, |
| "pass@8": 0.3481888556973961, |
| "pass@16": 0.40061764130662714, |
| "pass@32": 0.44109860212106855, |
| "pass@64": 0.47369522531282565, |
| "pass@128": 0.5015015015015015 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 553346, |
| "answer_accuracy": 0.6738917575993765, |
| "avg_response_len": 260.4377813230709, |
| "resp_tokens_sum": 213850671, |
| "pass_at_k": { |
| "pass@1": 0.6738917575993765, |
| "pass@2": 0.7172152611988333, |
| "pass@4": 0.7531338743120783, |
| "pass@8": 0.7837519089282062, |
| "pass@16": 0.8100695941203857, |
| "pass@32": 0.8328153046805763, |
| "pass@64": 0.8529624439757696, |
| "pass@128": 0.8701480904130943 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9125527223926381, |
| "pass@2": 0.9331044544949524, |
| "pass@4": 0.9456968690838492, |
| "pass@8": 0.9566705977659613, |
| "pass@16": 0.9664497828821677, |
| "pass@32": 0.9739261389620586, |
| "pass@64": 0.9797493602093914, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9996212121212121, |
| "pass@2": 0.9999839686232402, |
| "pass@4": 0.9999999758552907, |
| "pass@8": 0.9999999999999979, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9997226331360947, |
| "pass@2": 0.9999941760238552, |
| "pass@4": 0.9999999986133391, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9931612318840579, |
| "pass@2": 0.9957563049184068, |
| "pass@4": 0.9969017810273715, |
| "pass@8": 0.9972675874824468, |
| "pass@16": 0.9974637327583552, |
| "pass@32": 0.9978260869564953, |
| "pass@64": 0.9985507246376811, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9940231017191977, |
| "pass@2": 0.9956406662455162, |
| "pass@4": 0.9965047204880193, |
| "pass@8": 0.9973770261530999, |
| "pass@16": 0.998066631116763, |
| "pass@32": 0.9988006832926887, |
| "pass@64": 0.9996502944295251, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.991126264450867, |
| "pass@2": 0.9952906672431843, |
| "pass@4": 0.9967147617205364, |
| "pass@8": 0.9976171738733315, |
| "pass@16": 0.9983107185422295, |
| "pass@32": 0.9991000102569333, |
| "pass@64": 0.9998278640025487, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9749779929577465, |
| "pass@2": 0.9864294527004549, |
| "pass@4": 0.9915881530653737, |
| "pass@8": 0.9945899766336617, |
| "pass@16": 0.9961582248128492, |
| "pass@32": 0.9969211944658168, |
| "pass@64": 0.9971743778454023, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.9632765718562875, |
| "pass@2": 0.9803109086944224, |
| "pass@4": 0.9856916114340498, |
| "pass@8": 0.9883274821465693, |
| "pass@16": 0.9901297799031338, |
| "pass@32": 0.9914736057743874, |
| "pass@64": 0.992505694289581, |
| "pass@128": 0.9940119760479041 |
| }, |
| "11": { |
| "pass@1": 0.8991434487951807, |
| "pass@2": 0.9375277932596532, |
| "pass@4": 0.9547015353314265, |
| "pass@8": 0.9633609478714571, |
| "pass@16": 0.9693936038021645, |
| "pass@32": 0.9742566301782772, |
| "pass@64": 0.9782794167130561, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.8104182993730408, |
| "pass@2": 0.8787746371535066, |
| "pass@4": 0.9125708799527014, |
| "pass@8": 0.9304602175389681, |
| "pass@16": 0.9416070103033272, |
| "pass@32": 0.9500769794260939, |
| "pass@64": 0.9590956310205097, |
| "pass@128": 0.9686520376175548 |
| }, |
| "13": { |
| "pass@1": 0.5954350490196079, |
| "pass@2": 0.7000781611857338, |
| "pass@4": 0.7674895590782237, |
| "pass@8": 0.8114815716167344, |
| "pass@16": 0.844454160581966, |
| "pass@32": 0.8720772893119872, |
| "pass@64": 0.8940016890795939, |
| "pass@128": 0.907563025210084 |
| }, |
| "14": { |
| "pass@1": 0.4603207236842105, |
| "pass@2": 0.5714535991389238, |
| "pass@4": 0.6594985125214609, |
| "pass@8": 0.7265699264704736, |
| "pass@16": 0.7778373120159718, |
| "pass@32": 0.8180699356216169, |
| "pass@64": 0.8516240506945955, |
| "pass@128": 0.8771929824561403 |
| }, |
| "15": { |
| "pass@1": 0.31363636363636366, |
| "pass@2": 0.41937335958005234, |
| "pass@4": 0.5162882960652643, |
| "pass@8": 0.5983970205376589, |
| "pass@16": 0.6648011303037484, |
| "pass@32": 0.7190441181140116, |
| "pass@64": 0.7636872110138734, |
| "pass@128": 0.8 |
| }, |
| "16": { |
| "pass@1": 0.23312856125356127, |
| "pass@2": 0.3083505787738071, |
| "pass@4": 0.3828097024730884, |
| "pass@8": 0.45351063845009104, |
| "pass@16": 0.5170390838126733, |
| "pass@32": 0.569717191943559, |
| "pass@64": 0.6154515292082063, |
| "pass@128": 0.6552706552706553 |
| }, |
| "17": { |
| "pass@1": 0.20569681677018634, |
| "pass@2": 0.28773628160610343, |
| "pass@4": 0.3712533041289092, |
| "pass@8": 0.45004283295561615, |
| "pass@16": 0.5220548916570253, |
| "pass@32": 0.5866811888450802, |
| "pass@64": 0.6401217163146227, |
| "pass@128": 0.6832298136645962 |
| }, |
| "18": { |
| "pass@1": 0.18230698529411765, |
| "pass@2": 0.24840022869383982, |
| "pass@4": 0.31653323325393146, |
| "pass@8": 0.3846826446184044, |
| "pass@16": 0.4524686153958313, |
| "pass@32": 0.5189177849220697, |
| "pass@64": 0.5829468976280522, |
| "pass@128": 0.6411764705882353 |
| }, |
| "19": { |
| "pass@1": 0.13516090029761904, |
| "pass@2": 0.19425574732845896, |
| "pass@4": 0.2599179865351651, |
| "pass@8": 0.3226904065867681, |
| "pass@16": 0.3778738096041623, |
| "pass@32": 0.4263036119966427, |
| "pass@64": 0.47324745553640984, |
| "pass@128": 0.5148809523809523 |
| }, |
| "20": { |
| "pass@1": 0.12696482035928144, |
| "pass@2": 0.18205088346928192, |
| "pass@4": 0.2436986123366316, |
| "pass@8": 0.30770664465547454, |
| "pass@16": 0.36782947757335144, |
| "pass@32": 0.4220210875952675, |
| "pass@64": 0.47310400200799735, |
| "pass@128": 0.5209580838323353 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 536046, |
| "answer_accuracy": 0.669094004633328, |
| "avg_response_len": 277.4204021209458, |
| "resp_tokens_sum": 222255910, |
| "pass_at_k": { |
| "pass@1": 0.669094004633328, |
| "pass@2": 0.7131195448632707, |
| "pass@4": 0.7498518528669226, |
| "pass@8": 0.7809761802925407, |
| "pass@16": 0.8071611561775404, |
| "pass@32": 0.8288585002889368, |
| "pass@64": 0.8467381742920284, |
| "pass@128": 0.8613196996325292 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9236240671641791, |
| "pass@2": 0.9507807909272536, |
| "pass@4": 0.9667486526870706, |
| "pass@8": 0.9777367314708735, |
| "pass@16": 0.9858440541829969, |
| "pass@32": 0.9909022008929449, |
| "pass@64": 0.9943305330613756, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9994305758017493, |
| "pass@2": 0.9999802719175408, |
| "pass@4": 0.9999999822361854, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9998857821637427, |
| "pass@2": 0.9999978415527007, |
| "pass@4": 0.9999999997259114, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9794577205882353, |
| "pass@2": 0.986104678091709, |
| "pass@4": 0.9914057078343147, |
| "pass@8": 0.9952525272351784, |
| "pass@16": 0.9976998298743082, |
| "pass@32": 0.9990539138081224, |
| "pass@64": 0.9998248097159032, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.996623619873817, |
| "pass@2": 0.9981816953973025, |
| "pass@4": 0.9989754296288672, |
| "pass@8": 0.9996811227064588, |
| "pass@16": 0.999973197893188, |
| "pass@32": 0.9999999053842525, |
| "pass@64": 0.9999999999999886, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9818285205696202, |
| "pass@2": 0.9903634412688129, |
| "pass@4": 0.9938059475160541, |
| "pass@8": 0.9956453079406357, |
| "pass@16": 0.9968396038290583, |
| "pass@32": 0.997595232279207, |
| "pass@64": 0.998417683324389, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9867445570570571, |
| "pass@2": 0.9948784906954196, |
| "pass@4": 0.9967002438321332, |
| "pass@8": 0.9982105143733483, |
| "pass@16": 0.9994664808580491, |
| "pass@32": 0.9999560018404838, |
| "pass@64": 0.9999999172814134, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9711113344051447, |
| "pass@2": 0.9879939457427148, |
| "pass@4": 0.9937115386702061, |
| "pass@8": 0.9961771559176027, |
| "pass@16": 0.9975684602475261, |
| "pass@32": 0.9986379117884371, |
| "pass@64": 0.9996075579193004, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8969246031746032, |
| "pass@2": 0.9403961223597047, |
| "pass@4": 0.9624989144214113, |
| "pass@8": 0.9744247857115855, |
| "pass@16": 0.9800360707668536, |
| "pass@32": 0.9838969818396167, |
| "pass@64": 0.9876262661924977, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.7949397935779816, |
| "pass@2": 0.871270302078066, |
| "pass@4": 0.9135008060918985, |
| "pass@8": 0.9385238006991564, |
| "pass@16": 0.9559675949973659, |
| "pass@32": 0.9689443246955691, |
| "pass@64": 0.978408685654852, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.5622829861111112, |
| "pass@2": 0.6771364950908916, |
| "pass@4": 0.7605942616547924, |
| "pass@8": 0.8190895667375049, |
| "pass@16": 0.8586141115589907, |
| "pass@32": 0.8850650852501746, |
| "pass@64": 0.9046173061888647, |
| "pass@128": 0.9228395061728395 |
| }, |
| "14": { |
| "pass@1": 0.4215968100890208, |
| "pass@2": 0.5351282302156589, |
| "pass@4": 0.6310083644032625, |
| "pass@8": 0.7068240714416226, |
| "pass@16": 0.7639057717499893, |
| "pass@32": 0.8058978052015708, |
| "pass@64": 0.8372551030262958, |
| "pass@128": 0.8605341246290801 |
| }, |
| "15": { |
| "pass@1": 0.29094328703703703, |
| "pass@2": 0.38806602082725755, |
| "pass@4": 0.4813472141213828, |
| "pass@8": 0.5644691113048594, |
| "pass@16": 0.6337115033449147, |
| "pass@32": 0.6900797263618773, |
| "pass@64": 0.7355486564030599, |
| "pass@128": 0.7685185185185185 |
| }, |
| "16": { |
| "pass@1": 0.222483407079646, |
| "pass@2": 0.2984148897870065, |
| "pass@4": 0.3756056095864124, |
| "pass@8": 0.44816039374274946, |
| "pass@16": 0.5127854787924226, |
| "pass@32": 0.5715936942435857, |
| "pass@64": 0.626397140540926, |
| "pass@128": 0.672566371681416 |
| }, |
| "17": { |
| "pass@1": 0.18298192771084337, |
| "pass@2": 0.2520570717911013, |
| "pass@4": 0.3194832942568926, |
| "pass@8": 0.38233558376000504, |
| "pass@16": 0.44278971561258734, |
| "pass@32": 0.4990358885656421, |
| "pass@64": 0.5508491199781063, |
| "pass@128": 0.5993975903614458 |
| }, |
| "18": { |
| "pass@1": 0.17641129032258066, |
| "pass@2": 0.24202200279400554, |
| "pass@4": 0.31029111028460155, |
| "pass@8": 0.37480109802166267, |
| "pass@16": 0.43077098344591513, |
| "pass@32": 0.47823612116184727, |
| "pass@64": 0.5179535591299852, |
| "pass@128": 0.5483870967741935 |
| }, |
| "19": { |
| "pass@1": 0.1512698262839879, |
| "pass@2": 0.20805926921521523, |
| "pass@4": 0.2633237125570783, |
| "pass@8": 0.3144712609160323, |
| "pass@16": 0.36182671356535456, |
| "pass@32": 0.4048053375175812, |
| "pass@64": 0.44179356967491973, |
| "pass@128": 0.4743202416918429 |
| }, |
| "20": { |
| "pass@1": 0.15599192942942944, |
| "pass@2": 0.21430165401622092, |
| "pass@4": 0.2763685485260287, |
| "pass@8": 0.34237324035456, |
| "pass@16": 0.4086498747300178, |
| "pass@32": 0.4654420970793097, |
| "pass@64": 0.5065305978632904, |
| "pass@128": 0.5375375375375375 |
| } |
| } |
| } |
| } |
| } |
| ] |
