| [ |
| { |
| "checkpoint": "checkpoint-21414", |
| "id_acc": 0.9825416666666666, |
| "ood_acc": 0.42226640625, |
| "total_acc": 0.6876599506578948, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.4168125, |
| "ood_avg_resp_len": 330.094865625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 552209, |
| "answer_accuracy": 0.6819685128833386, |
| "avg_response_len": 256.0168155232374, |
| "resp_tokens_sum": 207303984, |
| "pass_at_k": { |
| "pass@1": 0.6819685128833386, |
| "pass@2": 0.7247032190920574, |
| "pass@4": 0.7587077436386261, |
| "pass@8": 0.7861151008390607, |
| "pass@16": 0.8092805233725615, |
| "pass@32": 0.8298467833091568, |
| "pass@64": 0.8483032154860494, |
| "pass@128": 0.8646854252292128 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9248478982300885, |
| "pass@2": 0.9491085116019793, |
| "pass@4": 0.9624991663099631, |
| "pass@8": 0.9701593210062911, |
| "pass@16": 0.9759912268469513, |
| "pass@32": 0.9812295381187107, |
| "pass@64": 0.9845646868569625, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 0.9998052959501558, |
| "pass@2": 0.9999892682807172, |
| "pass@4": 0.9999999795586298, |
| "pass@8": 0.9999999999999979, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997133027522935, |
| "pass@2": 0.9999853265188182, |
| "pass@4": 0.9999999638806617, |
| "pass@8": 0.9999999999999809, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9959716796875, |
| "pass@2": 0.997107606422244, |
| "pass@4": 0.9979930513568617, |
| "pass@8": 0.9987963980910021, |
| "pass@16": 0.9995682601800062, |
| "pass@32": 0.99995699926037, |
| "pass@64": 0.9999999140048625, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9848462301587302, |
| "pass@2": 0.9904222909636293, |
| "pass@4": 0.9933608721528855, |
| "pass@8": 0.9952308618802737, |
| "pass@16": 0.9964253623333685, |
| "pass@32": 0.9968019229794741, |
| "pass@64": 0.9968253802054875, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9938014595808383, |
| "pass@2": 0.9964022496345893, |
| "pass@4": 0.9969595811189771, |
| "pass@8": 0.997005691561411, |
| "pass@16": 0.997005988018481, |
| "pass@32": 0.9970059880239521, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 0.9970059880239521 |
| }, |
| "7": { |
| "pass@1": 0.9900147928994083, |
| "pass@2": 0.9960910200111823, |
| "pass@4": 0.9977629522456143, |
| "pass@8": 0.9984849844124919, |
| "pass@16": 0.9992618877994203, |
| "pass@32": 0.9998528845966035, |
| "pass@64": 0.999998024451384, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9765374599358975, |
| "pass@2": 0.9866522656723197, |
| "pass@4": 0.9920946723606662, |
| "pass@8": 0.9960669652526355, |
| "pass@16": 0.9982105108639956, |
| "pass@32": 0.9992585296089641, |
| "pass@64": 0.9999076317376822, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.976518485915493, |
| "pass@2": 0.9889278584895197, |
| "pass@4": 0.9931023189002782, |
| "pass@8": 0.9950372563181252, |
| "pass@16": 0.9964125011830314, |
| "pass@32": 0.9970946588086712, |
| "pass@64": 0.99718270235115, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.8844281161473088, |
| "pass@2": 0.9212723896410969, |
| "pass@4": 0.9436067501123264, |
| "pass@8": 0.9579636162051197, |
| "pass@16": 0.9670985270435172, |
| "pass@32": 0.9742570084646257, |
| "pass@64": 0.9800904955330577, |
| "pass@128": 0.9830028328611898 |
| }, |
| "12": { |
| "pass@1": 0.7716498940677966, |
| "pass@2": 0.8444291060545401, |
| "pass@4": 0.8857499737850562, |
| "pass@8": 0.9108198282777642, |
| "pass@16": 0.9293307101932994, |
| "pass@32": 0.9440249935615924, |
| "pass@64": 0.9552253462127829, |
| "pass@128": 0.963276836158192 |
| }, |
| "13": { |
| "pass@1": 0.6352860501567398, |
| "pass@2": 0.7346978130476637, |
| "pass@4": 0.8042030582775896, |
| "pass@8": 0.8533064652939295, |
| "pass@16": 0.8886264953035132, |
| "pass@32": 0.9142097203675839, |
| "pass@64": 0.9314999356483209, |
| "pass@128": 0.9435736677115988 |
| }, |
| "14": { |
| "pass@1": 0.5297410436137072, |
| "pass@2": 0.6415624923344859, |
| "pass@4": 0.7306435372564399, |
| "pass@8": 0.7979967801680269, |
| "pass@16": 0.8451427001772523, |
| "pass@32": 0.8789533711043278, |
| "pass@64": 0.9044450315775643, |
| "pass@128": 0.9252336448598131 |
| }, |
| "15": { |
| "pass@1": 0.37784501445086704, |
| "pass@2": 0.4767310460379589, |
| "pass@4": 0.5593058195218371, |
| "pass@8": 0.6291377965868806, |
| "pass@16": 0.6938041316181677, |
| "pass@32": 0.7535244975616412, |
| "pass@64": 0.8016768226609917, |
| "pass@128": 0.8294797687861272 |
| }, |
| "16": { |
| "pass@1": 0.26020665322580644, |
| "pass@2": 0.335496570993142, |
| "pass@4": 0.4038886567606469, |
| "pass@8": 0.4666672967885126, |
| "pass@16": 0.5261093963464502, |
| "pass@32": 0.5828497336957412, |
| "pass@64": 0.6393818486779441, |
| "pass@128": 0.7 |
| }, |
| "17": { |
| "pass@1": 0.19226427023121387, |
| "pass@2": 0.26076525590551186, |
| "pass@4": 0.32518011919391565, |
| "pass@8": 0.38033634075001777, |
| "pass@16": 0.4306098238185393, |
| "pass@32": 0.4796618584023391, |
| "pass@64": 0.5271427858474783, |
| "pass@128": 0.5722543352601156 |
| }, |
| "18": { |
| "pass@1": 0.18520089285714286, |
| "pass@2": 0.24933984814398197, |
| "pass@4": 0.3104361671755317, |
| "pass@8": 0.36399803537337694, |
| "pass@16": 0.41211960546818893, |
| "pass@32": 0.4593897802724865, |
| "pass@64": 0.509251678063119, |
| "pass@128": 0.5628571428571428 |
| }, |
| "19": { |
| "pass@1": 0.16439095345345345, |
| "pass@2": 0.2274970590669409, |
| "pass@4": 0.2895668528483488, |
| "pass@8": 0.3453876950121264, |
| "pass@16": 0.39525863585024174, |
| "pass@32": 0.44133377130594975, |
| "pass@64": 0.48640076574721813, |
| "pass@128": 0.5285285285285285 |
| }, |
| "20": { |
| "pass@1": 0.15127627627627627, |
| "pass@2": 0.20613576174599796, |
| "pass@4": 0.2602889900586748, |
| "pass@8": 0.30978129550909284, |
| "pass@16": 0.3539262998892409, |
| "pass@32": 0.39434409009272353, |
| "pass@64": 0.43195741841391067, |
| "pass@128": 0.46846846846846846 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 564147, |
| "answer_accuracy": 0.6870457424006235, |
| "avg_response_len": 265.64869933749026, |
| "resp_tokens_sum": 218129460, |
| "pass_at_k": { |
| "pass@1": 0.6870457424006235, |
| "pass@2": 0.7290492801075169, |
| "pass@4": 0.7632779281632992, |
| "pass@8": 0.7918571319736984, |
| "pass@16": 0.8162823884035032, |
| "pass@32": 0.8379809992378222, |
| "pass@64": 0.8575872076249258, |
| "pass@128": 0.8748246297739672 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.916267254601227, |
| "pass@2": 0.9377102102555434, |
| "pass@4": 0.9512473807913582, |
| "pass@8": 0.9620847327286483, |
| "pass@16": 0.9708337321923729, |
| "pass@32": 0.9780429832076506, |
| "pass@64": 0.98470006596974, |
| "pass@128": 0.99079754601227 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9993844696969697, |
| "pass@2": 0.9999582438558816, |
| "pass@4": 0.9999997767324539, |
| "pass@8": 0.9999999999972702, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9984744822485208, |
| "pass@2": 0.9992192231980618, |
| "pass@4": 0.9998001211387038, |
| "pass@8": 0.9999881144398141, |
| "pass@16": 0.9999999728867757, |
| "pass@32": 0.9999999999999859, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9910552536231884, |
| "pass@2": 0.9960433784092207, |
| "pass@4": 0.997278909429799, |
| "pass@8": 0.9976148285379174, |
| "pass@16": 0.9980647799249694, |
| "pass@32": 0.9987868364280551, |
| "pass@64": 0.9996462398721899, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9962168696275072, |
| "pass@2": 0.9973359627055933, |
| "pass@4": 0.997640037205235, |
| "pass@8": 0.9980702304314764, |
| "pass@16": 0.9987360751708015, |
| "pass@32": 0.9995102342761353, |
| "pass@64": 0.9999603905476594, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9923907153179191, |
| "pass@2": 0.9966742382003548, |
| "pass@4": 0.9982093324858958, |
| "pass@8": 0.999103459953119, |
| "pass@16": 0.9997474392862464, |
| "pass@32": 0.9999853533822985, |
| "pass@64": 0.999999993517332, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9815580985915493, |
| "pass@2": 0.9918754158811134, |
| "pass@4": 0.9958631642875629, |
| "pass@8": 0.9973009390170953, |
| "pass@16": 0.9978374440876846, |
| "pass@32": 0.9984196446979543, |
| "pass@64": 0.9993013197294001, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9709721182634731, |
| "pass@2": 0.9856845118817487, |
| "pass@4": 0.99032858243019, |
| "pass@8": 0.9922505487355335, |
| "pass@16": 0.9933870095289762, |
| "pass@32": 0.9944827921712119, |
| "pass@64": 0.9954997129812071, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.9080384036144579, |
| "pass@2": 0.9448522436201504, |
| "pass@4": 0.9616796697476069, |
| "pass@8": 0.9709939631976615, |
| "pass@16": 0.9778103951098883, |
| "pass@32": 0.9830680089298746, |
| "pass@64": 0.9875953271928183, |
| "pass@128": 0.9909638554216867 |
| }, |
| "12": { |
| "pass@1": 0.8134061520376176, |
| "pass@2": 0.8736304550391237, |
| "pass@4": 0.9055854940154422, |
| "pass@8": 0.9250752566405654, |
| "pass@16": 0.9386651867979705, |
| "pass@32": 0.9483623927576866, |
| "pass@64": 0.954930234395115, |
| "pass@128": 0.9592476489028213 |
| }, |
| "13": { |
| "pass@1": 0.6093312324929971, |
| "pass@2": 0.7128765384106396, |
| "pass@4": 0.784427807343409, |
| "pass@8": 0.8322995477507602, |
| "pass@16": 0.8668791417650001, |
| "pass@32": 0.8965360626211992, |
| "pass@64": 0.9225901570121836, |
| "pass@128": 0.9439775910364145 |
| }, |
| "14": { |
| "pass@1": 0.5037006578947368, |
| "pass@2": 0.6031471600589399, |
| "pass@4": 0.6821398813635136, |
| "pass@8": 0.7443594567503877, |
| "pass@16": 0.7937451409646782, |
| "pass@32": 0.8295549561038653, |
| "pass@64": 0.8540719151101315, |
| "pass@128": 0.8742690058479532 |
| }, |
| "15": { |
| "pass@1": 0.38638731060606063, |
| "pass@2": 0.49012168933428735, |
| "pass@4": 0.5826722673870308, |
| "pass@8": 0.6630281452660243, |
| "pass@16": 0.732491504570234, |
| "pass@32": 0.7933899105585001, |
| "pass@64": 0.8453957002650768, |
| "pass@128": 0.8909090909090909 |
| }, |
| "16": { |
| "pass@1": 0.23889334045584046, |
| "pass@2": 0.3124106181438857, |
| "pass@4": 0.38074211690419046, |
| "pass@8": 0.4413155535605134, |
| "pass@16": 0.4967472171694814, |
| "pass@32": 0.5513515663101722, |
| "pass@64": 0.6037091851981052, |
| "pass@128": 0.6495726495726496 |
| }, |
| "17": { |
| "pass@1": 0.2206424689440994, |
| "pass@2": 0.293285689832249, |
| "pass@4": 0.36450812903045493, |
| "pass@8": 0.43123284145879665, |
| "pass@16": 0.49317601615396117, |
| "pass@32": 0.5526968588225387, |
| "pass@64": 0.6107534973567073, |
| "pass@128": 0.6614906832298136 |
| }, |
| "18": { |
| "pass@1": 0.20032169117647058, |
| "pass@2": 0.26548170449282055, |
| "pass@4": 0.3312392085180528, |
| "pass@8": 0.397655892368413, |
| "pass@16": 0.46419447080456205, |
| "pass@32": 0.5302465468281584, |
| "pass@64": 0.594084068567451, |
| "pass@128": 0.6441176470588236 |
| }, |
| "19": { |
| "pass@1": 0.17061941964285715, |
| "pass@2": 0.23325929473659532, |
| "pass@4": 0.2938178893932901, |
| "pass@8": 0.3465596701519821, |
| "pass@16": 0.39067116880143643, |
| "pass@32": 0.43242180889559717, |
| "pass@64": 0.47503778455742324, |
| "pass@128": 0.5208333333333334 |
| }, |
| "20": { |
| "pass@1": 0.14404004491017963, |
| "pass@2": 0.20634997229949545, |
| "pass@4": 0.2734719161227602, |
| "pass@8": 0.33564373010208903, |
| "pass@16": 0.38668745398281046, |
| "pass@32": 0.42586240845329837, |
| "pass@64": 0.4586485969098191, |
| "pass@128": 0.49101796407185627 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 556033, |
| "answer_accuracy": 0.6940418297651382, |
| "avg_response_len": 284.55043737018696, |
| "resp_tokens_sum": 227968152, |
| "pass_at_k": { |
| "pass@1": 0.6940418297651382, |
| "pass@2": 0.7384633647232998, |
| "pass@4": 0.7733069706057428, |
| "pass@8": 0.8009115461768651, |
| "pass@16": 0.8233743538408184, |
| "pass@32": 0.8422516713922038, |
| "pass@64": 0.8583536041650246, |
| "pass@128": 0.8721840549608564 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9324160447761194, |
| "pass@2": 0.9569852509108, |
| "pass@4": 0.9697899050864907, |
| "pass@8": 0.9781530528094392, |
| "pass@16": 0.9842340783638477, |
| "pass@32": 0.9880054965349933, |
| "pass@64": 0.9901200990477843, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.998451166180758, |
| "pass@2": 0.9996251664332775, |
| "pass@4": 0.9999706842402718, |
| "pass@8": 0.9999998047429116, |
| "pass@16": 0.9999999999967805, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9997715643274854, |
| "pass@2": 0.999996042846618, |
| "pass@4": 0.9999999986295574, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9795266544117647, |
| "pass@2": 0.986127475104215, |
| "pass@4": 0.991238361840799, |
| "pass@8": 0.9955025618920745, |
| "pass@16": 0.9981389893347573, |
| "pass@32": 0.9995220934329105, |
| "pass@64": 0.999979772069567, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9935183359621451, |
| "pass@2": 0.9966583558955762, |
| "pass@4": 0.9988593408276646, |
| "pass@8": 0.999873815087129, |
| "pass@16": 0.9999988408563109, |
| "pass@32": 0.9999999999766422, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9809384889240507, |
| "pass@2": 0.9917790821538923, |
| "pass@4": 0.9972335925374833, |
| "pass@8": 0.9994872854003257, |
| "pass@16": 0.9999770459833613, |
| "pass@32": 0.9999999734419114, |
| "pass@64": 0.9999999999999998, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9815831456456456, |
| "pass@2": 0.9920701951951952, |
| "pass@4": 0.9958783766894002, |
| "pass@8": 0.9976620303803896, |
| "pass@16": 0.9986592364905941, |
| "pass@32": 0.9994866751847175, |
| "pass@64": 0.9999584873907899, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9723171221864951, |
| "pass@2": 0.9891170531939139, |
| "pass@4": 0.9942839329569333, |
| "pass@8": 0.9967603590122527, |
| "pass@16": 0.9987209237032028, |
| "pass@32": 0.9997957241852887, |
| "pass@64": 0.9999977644569561, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8964285714285715, |
| "pass@2": 0.9387275809273841, |
| "pass@4": 0.9594371688657963, |
| "pass@8": 0.9705618431628518, |
| "pass@16": 0.9780352242686186, |
| "pass@32": 0.9840632613312671, |
| "pass@64": 0.9890894614087125, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.821220374617737, |
| "pass@2": 0.8914681608032946, |
| "pass@4": 0.9302861282247968, |
| "pass@8": 0.9540288815663918, |
| "pass@16": 0.9683268595534348, |
| "pass@32": 0.9762805589127207, |
| "pass@64": 0.9803463816599656, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.6612895447530864, |
| "pass@2": 0.7590936983571496, |
| "pass@4": 0.8184143576381646, |
| "pass@8": 0.8560704200343588, |
| "pass@16": 0.8868263614084726, |
| "pass@32": 0.914352669868147, |
| "pass@64": 0.9374690089007075, |
| "pass@128": 0.9537037037037037 |
| }, |
| "14": { |
| "pass@1": 0.5250834569732937, |
| "pass@2": 0.6336070936236827, |
| "pass@4": 0.71655679594205, |
| "pass@8": 0.7786146810094134, |
| "pass@16": 0.8247886395298212, |
| "pass@32": 0.8566766150442523, |
| "pass@64": 0.8771299126364648, |
| "pass@128": 0.8931750741839762 |
| }, |
| "15": { |
| "pass@1": 0.4112413194444444, |
| "pass@2": 0.5148017522115288, |
| "pass@4": 0.603255610120494, |
| "pass@8": 0.6721030606815954, |
| "pass@16": 0.7209514472529268, |
| "pass@32": 0.757446229670618, |
| "pass@64": 0.7903285555038186, |
| "pass@128": 0.8209876543209876 |
| }, |
| "16": { |
| "pass@1": 0.25546183628318586, |
| "pass@2": 0.3240613313822496, |
| "pass@4": 0.3902822785979188, |
| "pass@8": 0.45340933112639226, |
| "pass@16": 0.5141112958504661, |
| "pass@32": 0.5722712975026955, |
| "pass@64": 0.6280215282043006, |
| "pass@128": 0.6784660766961652 |
| }, |
| "17": { |
| "pass@1": 0.2198324548192771, |
| "pass@2": 0.3012529942605065, |
| "pass@4": 0.37815900412674325, |
| "pass@8": 0.4417762212780109, |
| "pass@16": 0.494061647154501, |
| "pass@32": 0.5399609793146036, |
| "pass@64": 0.5796214398480122, |
| "pass@128": 0.6174698795180723 |
| }, |
| "18": { |
| "pass@1": 0.2219758064516129, |
| "pass@2": 0.2937031686563373, |
| "pass@4": 0.35714680926980896, |
| "pass@8": 0.4081763934731438, |
| "pass@16": 0.45100289005627864, |
| "pass@32": 0.48925765850501063, |
| "pass@64": 0.525177956537222, |
| "pass@128": 0.5612903225806452 |
| }, |
| "19": { |
| "pass@1": 0.16894826283987915, |
| "pass@2": 0.23550161167542888, |
| "pass@4": 0.2999038737982527, |
| "pass@8": 0.3579293787044701, |
| "pass@16": 0.40912665316978275, |
| "pass@32": 0.45602267650934875, |
| "pass@64": 0.49816792116530534, |
| "pass@128": 0.5287009063444109 |
| }, |
| "20": { |
| "pass@1": 0.15120589339339338, |
| "pass@2": 0.21511188846326643, |
| "pass@4": 0.28305867228308174, |
| "pass@8": 0.34877125001945325, |
| "pass@16": 0.4087823466909489, |
| "pass@32": 0.4612654823655196, |
| "pass@64": 0.5051002061315168, |
| "pass@128": 0.5435435435435435 |
| } |
| } |
| } |
| } |
| } |
| ] |
