[ { "checkpoint": "checkpoint-21414", "id_acc": 0.9825416666666666, "ood_acc": 0.42226640625, "total_acc": 0.6876599506578948, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.4168125, "ood_avg_resp_len": 330.094865625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 552209, "answer_accuracy": 0.6819685128833386, "avg_response_len": 256.0168155232374, "resp_tokens_sum": 207303984, "pass_at_k": { "pass@1": 0.6819685128833386, "pass@2": 0.7247032190920574, "pass@4": 0.7587077436386261, "pass@8": 0.7861151008390607, "pass@16": 0.8092805233725615, "pass@32": 0.8298467833091568, "pass@64": 0.8483032154860494, "pass@128": 0.8646854252292128 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9248478982300885, "pass@2": 0.9491085116019793, "pass@4": 0.9624991663099631, "pass@8": 0.9701593210062911, "pass@16": 0.9759912268469513, "pass@32": 0.9812295381187107, "pass@64": 0.9845646868569625, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 0.9998052959501558, "pass@2": 0.9999892682807172, "pass@4": 0.9999999795586298, "pass@8": 0.9999999999999979, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9997133027522935, "pass@2": 0.9999853265188182, "pass@4": 0.9999999638806617, "pass@8": 0.9999999999999809, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9959716796875, "pass@2": 0.997107606422244, "pass@4": 0.9979930513568617, "pass@8": 0.9987963980910021, "pass@16": 0.9995682601800062, "pass@32": 0.99995699926037, "pass@64": 0.9999999140048625, "pass@128": 1.0 }, "5": { "pass@1": 0.9848462301587302, "pass@2": 0.9904222909636293, "pass@4": 0.9933608721528855, "pass@8": 0.9952308618802737, "pass@16": 0.9964253623333685, "pass@32": 0.9968019229794741, "pass@64": 0.9968253802054875, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9938014595808383, "pass@2": 0.9964022496345893, "pass@4": 0.9969595811189771, "pass@8": 0.997005691561411, "pass@16": 0.997005988018481, "pass@32": 0.9970059880239521, "pass@64": 0.9970059880239521, "pass@128": 0.9970059880239521 }, "7": { "pass@1": 0.9900147928994083, "pass@2": 0.9960910200111823, "pass@4": 0.9977629522456143, "pass@8": 0.9984849844124919, "pass@16": 0.9992618877994203, "pass@32": 0.9998528845966035, "pass@64": 0.999998024451384, "pass@128": 1.0 }, "8": { "pass@1": 0.9765374599358975, "pass@2": 0.9866522656723197, "pass@4": 0.9920946723606662, "pass@8": 0.9960669652526355, "pass@16": 0.9982105108639956, "pass@32": 0.9992585296089641, "pass@64": 0.9999076317376822, "pass@128": 1.0 }, "9": { "pass@1": 0.976518485915493, "pass@2": 0.9889278584895197, "pass@4": 0.9931023189002782, "pass@8": 0.9950372563181252, "pass@16": 0.9964125011830314, "pass@32": 0.9970946588086712, "pass@64": 0.99718270235115, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.8844281161473088, "pass@2": 0.9212723896410969, "pass@4": 0.9436067501123264, "pass@8": 0.9579636162051197, "pass@16": 0.9670985270435172, "pass@32": 0.9742570084646257, "pass@64": 0.9800904955330577, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.7716498940677966, "pass@2": 0.8444291060545401, "pass@4": 0.8857499737850562, "pass@8": 0.9108198282777642, "pass@16": 0.9293307101932994, "pass@32": 0.9440249935615924, "pass@64": 0.9552253462127829, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.6352860501567398, "pass@2": 0.7346978130476637, "pass@4": 0.8042030582775896, "pass@8": 0.8533064652939295, "pass@16": 0.8886264953035132, "pass@32": 0.9142097203675839, "pass@64": 0.9314999356483209, "pass@128": 0.9435736677115988 }, "14": { "pass@1": 0.5297410436137072, "pass@2": 0.6415624923344859, "pass@4": 0.7306435372564399, "pass@8": 0.7979967801680269, "pass@16": 0.8451427001772523, "pass@32": 0.8789533711043278, "pass@64": 0.9044450315775643, "pass@128": 0.9252336448598131 }, "15": { "pass@1": 0.37784501445086704, "pass@2": 0.4767310460379589, "pass@4": 0.5593058195218371, "pass@8": 0.6291377965868806, "pass@16": 0.6938041316181677, "pass@32": 0.7535244975616412, "pass@64": 0.8016768226609917, "pass@128": 0.8294797687861272 }, "16": { "pass@1": 0.26020665322580644, "pass@2": 0.335496570993142, "pass@4": 0.4038886567606469, "pass@8": 0.4666672967885126, "pass@16": 0.5261093963464502, "pass@32": 0.5828497336957412, "pass@64": 0.6393818486779441, "pass@128": 0.7 }, "17": { "pass@1": 0.19226427023121387, "pass@2": 0.26076525590551186, "pass@4": 0.32518011919391565, "pass@8": 0.38033634075001777, "pass@16": 0.4306098238185393, "pass@32": 0.4796618584023391, "pass@64": 0.5271427858474783, "pass@128": 0.5722543352601156 }, "18": { "pass@1": 0.18520089285714286, "pass@2": 0.24933984814398197, "pass@4": 0.3104361671755317, "pass@8": 0.36399803537337694, "pass@16": 0.41211960546818893, "pass@32": 0.4593897802724865, "pass@64": 0.509251678063119, "pass@128": 0.5628571428571428 }, "19": { "pass@1": 0.16439095345345345, "pass@2": 0.2274970590669409, "pass@4": 0.2895668528483488, "pass@8": 0.3453876950121264, "pass@16": 0.39525863585024174, "pass@32": 0.44133377130594975, "pass@64": 0.48640076574721813, "pass@128": 0.5285285285285285 }, "20": { "pass@1": 0.15127627627627627, "pass@2": 0.20613576174599796, "pass@4": 0.2602889900586748, "pass@8": 0.30978129550909284, "pass@16": 0.3539262998892409, "pass@32": 0.39434409009272353, "pass@64": 0.43195741841391067, "pass@128": 0.46846846846846846 } } }, "teachers_in_school": { "count": 821120, "correct": 564147, "answer_accuracy": 0.6870457424006235, "avg_response_len": 265.64869933749026, "resp_tokens_sum": 218129460, "pass_at_k": { "pass@1": 0.6870457424006235, "pass@2": 0.7290492801075169, "pass@4": 0.7632779281632992, "pass@8": 0.7918571319736984, "pass@16": 0.8162823884035032, "pass@32": 0.8379809992378222, "pass@64": 0.8575872076249258, "pass@128": 0.8748246297739672 }, "per_op_pass_at_k": { "10": { "pass@1": 0.916267254601227, "pass@2": 0.9377102102555434, "pass@4": 0.9512473807913582, "pass@8": 0.9620847327286483, "pass@16": 0.9708337321923729, "pass@32": 0.9780429832076506, "pass@64": 0.98470006596974, "pass@128": 0.99079754601227 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9993844696969697, "pass@2": 0.9999582438558816, "pass@4": 0.9999997767324539, "pass@8": 0.9999999999972702, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9984744822485208, "pass@2": 0.9992192231980618, "pass@4": 0.9998001211387038, "pass@8": 0.9999881144398141, "pass@16": 0.9999999728867757, "pass@32": 0.9999999999999859, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9910552536231884, "pass@2": 0.9960433784092207, "pass@4": 0.997278909429799, "pass@8": 0.9976148285379174, "pass@16": 0.9980647799249694, "pass@32": 0.9987868364280551, "pass@64": 0.9996462398721899, "pass@128": 1.0 }, "6": { "pass@1": 0.9962168696275072, "pass@2": 0.9973359627055933, "pass@4": 0.997640037205235, "pass@8": 0.9980702304314764, "pass@16": 0.9987360751708015, "pass@32": 0.9995102342761353, "pass@64": 0.9999603905476594, "pass@128": 1.0 }, "7": { "pass@1": 0.9923907153179191, "pass@2": 0.9966742382003548, "pass@4": 0.9982093324858958, "pass@8": 0.999103459953119, "pass@16": 0.9997474392862464, "pass@32": 0.9999853533822985, "pass@64": 0.999999993517332, "pass@128": 1.0 }, "8": { "pass@1": 0.9815580985915493, "pass@2": 0.9918754158811134, "pass@4": 0.9958631642875629, "pass@8": 0.9973009390170953, "pass@16": 0.9978374440876846, "pass@32": 0.9984196446979543, "pass@64": 0.9993013197294001, "pass@128": 1.0 }, "9": { "pass@1": 0.9709721182634731, "pass@2": 0.9856845118817487, "pass@4": 0.99032858243019, "pass@8": 0.9922505487355335, "pass@16": 0.9933870095289762, "pass@32": 0.9944827921712119, "pass@64": 0.9954997129812071, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.9080384036144579, "pass@2": 0.9448522436201504, "pass@4": 0.9616796697476069, "pass@8": 0.9709939631976615, "pass@16": 0.9778103951098883, "pass@32": 0.9830680089298746, "pass@64": 0.9875953271928183, "pass@128": 0.9909638554216867 }, "12": { "pass@1": 0.8134061520376176, "pass@2": 0.8736304550391237, "pass@4": 0.9055854940154422, "pass@8": 0.9250752566405654, "pass@16": 0.9386651867979705, "pass@32": 0.9483623927576866, "pass@64": 0.954930234395115, "pass@128": 0.9592476489028213 }, "13": { "pass@1": 0.6093312324929971, "pass@2": 0.7128765384106396, "pass@4": 0.784427807343409, "pass@8": 0.8322995477507602, "pass@16": 0.8668791417650001, "pass@32": 0.8965360626211992, "pass@64": 0.9225901570121836, "pass@128": 0.9439775910364145 }, "14": { "pass@1": 0.5037006578947368, "pass@2": 0.6031471600589399, "pass@4": 0.6821398813635136, "pass@8": 0.7443594567503877, "pass@16": 0.7937451409646782, "pass@32": 0.8295549561038653, "pass@64": 0.8540719151101315, "pass@128": 0.8742690058479532 }, "15": { "pass@1": 0.38638731060606063, "pass@2": 0.49012168933428735, "pass@4": 0.5826722673870308, "pass@8": 0.6630281452660243, "pass@16": 0.732491504570234, "pass@32": 0.7933899105585001, "pass@64": 0.8453957002650768, "pass@128": 0.8909090909090909 }, "16": { "pass@1": 0.23889334045584046, "pass@2": 0.3124106181438857, "pass@4": 0.38074211690419046, "pass@8": 0.4413155535605134, "pass@16": 0.4967472171694814, "pass@32": 0.5513515663101722, "pass@64": 0.6037091851981052, "pass@128": 0.6495726495726496 }, "17": { "pass@1": 0.2206424689440994, "pass@2": 0.293285689832249, "pass@4": 0.36450812903045493, "pass@8": 0.43123284145879665, "pass@16": 0.49317601615396117, "pass@32": 0.5526968588225387, "pass@64": 0.6107534973567073, "pass@128": 0.6614906832298136 }, "18": { "pass@1": 0.20032169117647058, "pass@2": 0.26548170449282055, "pass@4": 0.3312392085180528, "pass@8": 0.397655892368413, "pass@16": 0.46419447080456205, "pass@32": 0.5302465468281584, "pass@64": 0.594084068567451, "pass@128": 0.6441176470588236 }, "19": { "pass@1": 0.17061941964285715, "pass@2": 0.23325929473659532, "pass@4": 0.2938178893932901, "pass@8": 0.3465596701519821, "pass@16": 0.39067116880143643, "pass@32": 0.43242180889559717, "pass@64": 0.47503778455742324, "pass@128": 0.5208333333333334 }, "20": { "pass@1": 0.14404004491017963, "pass@2": 0.20634997229949545, "pass@4": 0.2734719161227602, "pass@8": 0.33564373010208903, "pass@16": 0.38668745398281046, "pass@32": 0.42586240845329837, "pass@64": 0.4586485969098191, "pass@128": 0.49101796407185627 } } }, "movie_festival_awards": { "count": 801152, "correct": 556033, "answer_accuracy": 0.6940418297651382, "avg_response_len": 284.55043737018696, "resp_tokens_sum": 227968152, "pass_at_k": { "pass@1": 0.6940418297651382, "pass@2": 0.7384633647232998, "pass@4": 0.7733069706057428, "pass@8": 0.8009115461768651, "pass@16": 0.8233743538408184, "pass@32": 0.8422516713922038, "pass@64": 0.8583536041650246, "pass@128": 0.8721840549608564 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9324160447761194, "pass@2": 0.9569852509108, "pass@4": 0.9697899050864907, "pass@8": 0.9781530528094392, "pass@16": 0.9842340783638477, "pass@32": 0.9880054965349933, "pass@64": 0.9901200990477843, "pass@128": 0.991044776119403 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.998451166180758, "pass@2": 0.9996251664332775, "pass@4": 0.9999706842402718, "pass@8": 0.9999998047429116, "pass@16": 0.9999999999967805, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9997715643274854, "pass@2": 0.999996042846618, "pass@4": 0.9999999986295574, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9795266544117647, "pass@2": 0.986127475104215, "pass@4": 0.991238361840799, "pass@8": 0.9955025618920745, "pass@16": 0.9981389893347573, "pass@32": 0.9995220934329105, "pass@64": 0.999979772069567, "pass@128": 1.0 }, "6": { "pass@1": 0.9935183359621451, "pass@2": 0.9966583558955762, "pass@4": 0.9988593408276646, "pass@8": 0.999873815087129, "pass@16": 0.9999988408563109, "pass@32": 0.9999999999766422, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9809384889240507, "pass@2": 0.9917790821538923, "pass@4": 0.9972335925374833, "pass@8": 0.9994872854003257, "pass@16": 0.9999770459833613, "pass@32": 0.9999999734419114, "pass@64": 0.9999999999999998, "pass@128": 1.0 }, "8": { "pass@1": 0.9815831456456456, "pass@2": 0.9920701951951952, "pass@4": 0.9958783766894002, "pass@8": 0.9976620303803896, "pass@16": 0.9986592364905941, "pass@32": 0.9994866751847175, "pass@64": 0.9999584873907899, "pass@128": 1.0 }, "9": { "pass@1": 0.9723171221864951, "pass@2": 0.9891170531939139, "pass@4": 0.9942839329569333, "pass@8": 0.9967603590122527, "pass@16": 0.9987209237032028, "pass@32": 0.9997957241852887, "pass@64": 0.9999977644569561, "pass@128": 1.0 }, "11": { "pass@1": 0.8964285714285715, "pass@2": 0.9387275809273841, "pass@4": 0.9594371688657963, "pass@8": 0.9705618431628518, "pass@16": 0.9780352242686186, "pass@32": 0.9840632613312671, "pass@64": 0.9890894614087125, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.821220374617737, "pass@2": 0.8914681608032946, "pass@4": 0.9302861282247968, "pass@8": 0.9540288815663918, "pass@16": 0.9683268595534348, "pass@32": 0.9762805589127207, "pass@64": 0.9803463816599656, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.6612895447530864, "pass@2": 0.7590936983571496, "pass@4": 0.8184143576381646, "pass@8": 0.8560704200343588, "pass@16": 0.8868263614084726, "pass@32": 0.914352669868147, "pass@64": 0.9374690089007075, "pass@128": 0.9537037037037037 }, "14": { "pass@1": 0.5250834569732937, "pass@2": 0.6336070936236827, "pass@4": 0.71655679594205, "pass@8": 0.7786146810094134, "pass@16": 0.8247886395298212, "pass@32": 0.8566766150442523, "pass@64": 0.8771299126364648, "pass@128": 0.8931750741839762 }, "15": { "pass@1": 0.4112413194444444, "pass@2": 0.5148017522115288, "pass@4": 0.603255610120494, "pass@8": 0.6721030606815954, "pass@16": 0.7209514472529268, "pass@32": 0.757446229670618, "pass@64": 0.7903285555038186, "pass@128": 0.8209876543209876 }, "16": { "pass@1": 0.25546183628318586, "pass@2": 0.3240613313822496, "pass@4": 0.3902822785979188, "pass@8": 0.45340933112639226, "pass@16": 0.5141112958504661, "pass@32": 0.5722712975026955, "pass@64": 0.6280215282043006, "pass@128": 0.6784660766961652 }, "17": { "pass@1": 0.2198324548192771, "pass@2": 0.3012529942605065, "pass@4": 0.37815900412674325, "pass@8": 0.4417762212780109, "pass@16": 0.494061647154501, "pass@32": 0.5399609793146036, "pass@64": 0.5796214398480122, "pass@128": 0.6174698795180723 }, "18": { "pass@1": 0.2219758064516129, "pass@2": 0.2937031686563373, "pass@4": 0.35714680926980896, "pass@8": 0.4081763934731438, "pass@16": 0.45100289005627864, "pass@32": 0.48925765850501063, "pass@64": 0.525177956537222, "pass@128": 0.5612903225806452 }, "19": { "pass@1": 0.16894826283987915, "pass@2": 0.23550161167542888, "pass@4": 0.2999038737982527, "pass@8": 0.3579293787044701, "pass@16": 0.40912665316978275, "pass@32": 0.45602267650934875, "pass@64": 0.49816792116530534, "pass@128": 0.5287009063444109 }, "20": { "pass@1": 0.15120589339339338, "pass@2": 0.21511188846326643, "pass@4": 0.28305867228308174, "pass@8": 0.34877125001945325, "pass@16": 0.4087823466909489, "pass@32": 0.4612654823655196, "pass@64": 0.5051002061315168, "pass@128": 0.5435435435435435 } } } } } ]