[ { "checkpoint": "checkpoint-44699", "id_acc": 0.9833550347222222, "ood_acc": 0.36736640625, "total_acc": 0.6591504934210526, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.52830208333333, "ood_avg_resp_len": 312.80330546875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 531202, "answer_accuracy": 0.6560252331647171, "avg_response_len": 244.93973161357889, "resp_tokens_sum": 198334559, "pass_at_k": { "pass@1": 0.6560252331647171, "pass@2": 0.6894634278667485, "pass@4": 0.7172932349380673, "pass@8": 0.7407126418953518, "pass@16": 0.7603428177395862, "pass@32": 0.7770562234748629, "pass@64": 0.7919610478846388, "pass@128": 0.8055643376541258 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9344579646017699, "pass@2": 0.9526916678280263, "pass@4": 0.9628839813164948, "pass@8": 0.9702581096493935, "pass@16": 0.9760459335416097, "pass@32": 0.9812674749596855, "pass@64": 0.9866724375578277, "pass@128": 0.9911504424778761 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.999641628440367, "pass@2": 0.9999909701654267, "pass@4": 0.9999999971333858, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.996533203125, "pass@2": 0.9968619279035433, "pass@4": 0.9968749792018186, "pass@8": 0.9968749999999978, "pass@16": 0.996875, "pass@32": 0.996875, "pass@64": 0.996875, "pass@128": 0.996875 }, "5": { "pass@1": 0.9764632936507937, "pass@2": 0.9804899387576552, "pass@4": 0.9837944602162825, "pass@8": 0.9863203732706364, "pass@16": 0.9880740031976679, "pass@32": 0.9894659832300179, "pass@64": 0.9912760738735891, "pass@128": 0.9936507936507937 }, "6": { "pass@1": 0.9913454341317365, "pass@2": 0.9930778207364798, "pass@4": 0.9938670788344569, "pass@8": 0.9940085499193233, "pass@16": 0.9940119746912577, "pass@32": 0.9940119760479041, "pass@64": 0.9940119760479041, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.9867788461538461, "pass@2": 0.9901181684759818, "pass@4": 0.992015774655387, "pass@8": 0.994022348072151, "pass@16": 0.9957065225421011, "pass@32": 0.9966644429698368, "pass@64": 0.9970219764691983, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.9838741987179487, "pass@2": 0.9881515148647284, "pass@4": 0.9893485868352994, "pass@8": 0.9903211494125901, "pass@16": 0.9910836657779668, "pass@32": 0.9917911369360106, "pass@64": 0.9927947708452297, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.9845070422535211, "pass@2": 0.9931691527115452, "pass@4": 0.9958114764703706, "pass@8": 0.9972125362894183, "pass@16": 0.9986897583523467, "pass@32": 0.9997578443233471, "pass@64": 0.9999972582837349, "pass@128": 1.0 }, "11": { "pass@1": 0.8852027266288952, "pass@2": 0.9147545085989603, "pass@4": 0.9345978617049635, "pass@8": 0.9490441153165399, "pass@16": 0.9606432019579066, "pass@32": 0.9705314683575175, "pass@64": 0.9782839078839444, "pass@128": 0.9858356940509915 }, "12": { "pass@1": 0.7466454802259888, "pass@2": 0.8139510070510254, "pass@4": 0.8601370909144825, "pass@8": 0.8920004996421725, "pass@16": 0.9141258280948045, "pass@32": 0.9296279933820031, "pass@64": 0.9410879649553646, "pass@128": 0.9519774011299436 }, "13": { "pass@1": 0.5093798981191222, "pass@2": 0.5921151081134451, "pass@4": 0.6576909166722495, "pass@8": 0.7099843278394556, "pass@16": 0.75098134301228, "pass@32": 0.7835781471809989, "pass@64": 0.80972291492387, "pass@128": 0.8307210031347962 }, "14": { "pass@1": 0.3689154984423676, "pass@2": 0.44391335209851096, "pass@4": 0.5070102309057162, "pass@8": 0.5610363317684318, "pass@16": 0.6082930404376408, "pass@32": 0.6493371167503605, "pass@64": 0.6867744387040765, "pass@128": 0.7133956386292835 }, "15": { "pass@1": 0.2523708453757225, "pass@2": 0.31927668858950414, "pass@4": 0.3786499197535278, "pass@8": 0.4312203464692883, "pass@16": 0.47739438847028326, "pass@32": 0.5181956268293656, "pass@64": 0.5577668100174789, "pass@128": 0.5982658959537572 }, "16": { "pass@1": 0.19014616935483872, "pass@2": 0.2436293497586995, "pass@4": 0.29316089148130675, "pass@8": 0.33883962767516357, "pass@16": 0.3797893057690267, "pass@32": 0.41671065349623226, "pass@64": 0.45184103790572516, "pass@128": 0.4870967741935484 }, "17": { "pass@1": 0.18727420520231214, "pass@2": 0.24610352851486034, "pass@4": 0.3004370235873695, "pass@8": 0.3462447129619863, "pass@16": 0.3822512517998407, "pass@32": 0.4124269666327473, "pass@64": 0.44181302593089994, "pass@128": 0.476878612716763 }, "18": { "pass@1": 0.18810267857142857, "pass@2": 0.24303008998875142, "pass@4": 0.2925131434463549, "pass@8": 0.3346438368665325, "pass@16": 0.37033006112880934, "pass@32": 0.39964188355370805, "pass@64": 0.42305886798827047, "pass@128": 0.44285714285714284 }, "19": { "pass@1": 0.15784534534534533, "pass@2": 0.20692124210824994, "pass@4": 0.25512323628465366, "pass@8": 0.29988871819925383, "pass@16": 0.3406692381875606, "pass@32": 0.37891669047499194, "pass@64": 0.4135048107997905, "pass@128": 0.44144144144144143 }, "20": { "pass@1": 0.15195664414414414, "pass@2": 0.20411368553592957, "pass@4": 0.254373751422964, "pass@8": 0.2996420531321556, "pass@16": 0.3385611068086796, "pass@32": 0.37173141526078607, "pass@64": 0.40101691150690105, "pass@128": 0.4264264264264264 } } }, "teachers_in_school": { "count": 821120, "correct": 542547, "answer_accuracy": 0.6607402084957131, "avg_response_len": 254.24384986360093, "resp_tokens_sum": 208764710, "pass_at_k": { "pass@1": 0.6607402084957131, "pass@2": 0.692345312106834, "pass@4": 0.7187453154238392, "pass@8": 0.7414170852156288, "pass@16": 0.7608738632835582, "pass@32": 0.7776036865720104, "pass@64": 0.7921936522666159, "pass@128": 0.8053000779423227 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9242954371165644, "pass@2": 0.9467247204241344, "pass@4": 0.960173860648554, "pass@8": 0.9688716992564892, "pass@16": 0.9746263491641634, "pass@32": 0.9789321415016857, "pass@64": 0.9833402213400504, "pass@128": 0.9877300613496932 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999526515151516, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9970645340236687, "pass@2": 0.9970876479289941, "pass@4": 0.997133875739645, "pass@8": 0.9972263313609467, "pass@16": 0.9974112426035503, "pass@32": 0.9977810650887574, "pass@64": 0.9985207100591716, "pass@128": 1.0 }, "5": { "pass@1": 0.9887001811594203, "pass@2": 0.9915953440602534, "pass@4": 0.9933488841068782, "pass@8": 0.9947347459851477, "pass@16": 0.9960028065978203, "pass@32": 0.9968289314680655, "pass@64": 0.9970924755620924, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9932843839541547, "pass@2": 0.9942125843241656, "pass@4": 0.9942690063097414, "pass@8": 0.9942693409681034, "pass@16": 0.994269340974212, "pass@32": 0.994269340974212, "pass@64": 0.994269340974212, "pass@128": 0.994269340974212 }, "7": { "pass@1": 0.9916907514450867, "pass@2": 0.9933922130308135, "pass@4": 0.9947579433712405, "pass@8": 0.9961805922227138, "pass@16": 0.9969781403865762, "pass@32": 0.9971081671151855, "pass@64": 0.99710982657694, "pass@128": 0.9971098265895953 }, "8": { "pass@1": 0.9907570422535211, "pass@2": 0.9952329627370523, "pass@4": 0.9966467019967574, "pass@8": 0.9972438992106627, "pass@16": 0.9975316366671934, "pass@32": 0.9978873227410595, "pass@64": 0.9985915492957746, "pass@128": 1.0 }, "9": { "pass@1": 0.9678845434131736, "pass@2": 0.9773401032580507, "pass@4": 0.980435996118, "pass@8": 0.9821315536020941, "pass@16": 0.9838955100446117, "pass@32": 0.9854663026265753, "pass@64": 0.9865217056571726, "pass@128": 0.9880239520958084 }, "11": { "pass@1": 0.9021084337349398, "pass@2": 0.929302100132815, "pass@4": 0.9446336069813555, "pass@8": 0.955031982663235, "pass@16": 0.962953963538521, "pass@32": 0.9693316516149345, "pass@64": 0.974545578181479, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.7708170062695925, "pass@2": 0.8253758824327991, "pass@4": 0.8607174199228226, "pass@8": 0.8830145741372571, "pass@16": 0.8988607561671764, "pass@32": 0.9145445540094311, "pass@64": 0.930184391477638, "pass@128": 0.9435736677115988 }, "13": { "pass@1": 0.4949229691876751, "pass@2": 0.5704298451664127, "pass@4": 0.6317669708303267, "pass@8": 0.6809381380037726, "pass@16": 0.7209420738281904, "pass@32": 0.7554410660153358, "pass@64": 0.7887207068048872, "pass@128": 0.8263305322128851 }, "14": { "pass@1": 0.3959703947368421, "pass@2": 0.46360354215591465, "pass@4": 0.524082480369778, "pass@8": 0.5806233011224652, "pass@16": 0.6316132775599853, "pass@32": 0.6754358216679552, "pass@64": 0.712545405106953, "pass@128": 0.7456140350877193 }, "15": { "pass@1": 0.2523910984848485, "pass@2": 0.3201749284180385, "pass@4": 0.38129389508129663, "pass@8": 0.4367986120606595, "pass@16": 0.48397204169078695, "pass@32": 0.5211283773311105, "pass@64": 0.5485796081544515, "pass@128": 0.5696969696969697 }, "16": { "pass@1": 0.20877849002849003, "pass@2": 0.2670165528187184, "pass@4": 0.31650643375774584, "pass@8": 0.3575628181721004, "pass@16": 0.39263447515402217, "pass@32": 0.42433719998352165, "pass@64": 0.45382145756108494, "pass@128": 0.47863247863247865 }, "17": { "pass@1": 0.2135335791925466, "pass@2": 0.27400757140411797, "pass@4": 0.3274636615687015, "pass@8": 0.3724455659358969, "pass@16": 0.4118936793086089, "pass@32": 0.44952103938521476, "pass@64": 0.48420234899227, "pass@128": 0.5124223602484472 }, "18": { "pass@1": 0.158984375, "pass@2": 0.21292156380268631, "pass@4": 0.2651559686473013, "pass@8": 0.3149142166840468, "pass@16": 0.36073523939032814, "pass@32": 0.3976295956029731, "pass@64": 0.42512193141106314, "pass@128": 0.45 }, "19": { "pass@1": 0.14674014136904762, "pass@2": 0.19081233009936244, "pass@4": 0.23490378267448708, "pass@8": 0.2787363831910592, "pass@16": 0.32005228112975653, "pass@32": 0.35780752009771627, "pass@64": 0.3924129635883499, "pass@128": 0.4226190476190476 }, "20": { "pass@1": 0.1475252619760479, "pass@2": 0.19783648745815446, "pass@4": 0.24589100903928915, "pass@8": 0.28933610432988377, "pass@16": 0.3253420999811492, "pass@32": 0.35424371588248044, "pass@64": 0.37930679206589685, "pass@128": 0.40119760479041916 } } }, "movie_festival_awards": { "count": 801152, "correct": 529305, "answer_accuracy": 0.6606798709857805, "avg_response_len": 274.21708489774727, "resp_tokens_sum": 219689566, "pass_at_k": { "pass@1": 0.6606798709857805, "pass@2": 0.6923162142577697, "pass@4": 0.7192061071352318, "pass@8": 0.7419212559542995, "pass@16": 0.7612971729400352, "pass@32": 0.7782682183303264, "pass@64": 0.7932781986164202, "pass@128": 0.8061990733343984 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9298973880597015, "pass@2": 0.9486341667646025, "pass@4": 0.9623331648842398, "pass@8": 0.9719875662983237, "pass@16": 0.9785429498814807, "pass@32": 0.9841315033131119, "pass@64": 0.9887908906350661, "pass@128": 0.991044776119403 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999772230320699, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999543128654971, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9717601102941177, "pass@2": 0.9771971977767483, "pass@4": 0.9794758403361344, "pass@8": 0.9802533717589259, "pass@16": 0.9810081996249801, "pass@32": 0.9821723681597684, "pass@64": 0.9836483557202409, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.9996056782334385, "pass@2": 0.9999910734245759, "pass@4": 0.9999999896503473, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9830152294303798, "pass@2": 0.9880324616266324, "pass@4": 0.9902660212370608, "pass@8": 0.991350756420527, "pass@16": 0.9922229217269881, "pass@32": 0.9934765985754536, "pass@64": 0.9950646865344362, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9827796546546547, "pass@2": 0.9879839238845144, "pass@4": 0.9901089140321423, "pass@8": 0.9912163301188336, "pass@16": 0.9919635210522875, "pass@32": 0.9927370227462073, "pass@64": 0.9936274857534592, "pass@128": 0.993993993993994 }, "9": { "pass@1": 0.9745779742765274, "pass@2": 0.9838013804845936, "pass@4": 0.9882725915289527, "pass@8": 0.9912003897231443, "pass@16": 0.9930943945884175, "pass@32": 0.9942506476636976, "pass@64": 0.9951758657948274, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.8968005952380952, "pass@2": 0.9253007436570432, "pass@4": 0.9426268133150022, "pass@8": 0.9550882186965979, "pass@16": 0.965702093414588, "pass@32": 0.9758799151725375, "pass@64": 0.9852208180666315, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.7458428899082569, "pass@2": 0.8013131636928408, "pass@4": 0.8399700091960991, "pass@8": 0.8693948834655879, "pass@16": 0.8932526941911637, "pass@32": 0.9134654564118084, "pass@64": 0.929910476447298, "pass@128": 0.9388379204892966 }, "13": { "pass@1": 0.5423659336419753, "pass@2": 0.6123213005492364, "pass@4": 0.6643283845885002, "pass@8": 0.7066937664543848, "pass@16": 0.7454170853733546, "pass@32": 0.7799614307727357, "pass@64": 0.8075608604900163, "pass@128": 0.8302469135802469 }, "14": { "pass@1": 0.38826038575667654, "pass@2": 0.45652023119699037, "pass@4": 0.5185281256571415, "pass@8": 0.5713009485900146, "pass@16": 0.615971139816462, "pass@32": 0.6542321591059677, "pass@64": 0.6889103089980461, "pass@128": 0.7270029673590505 }, "15": { "pass@1": 0.2394386574074074, "pass@2": 0.30811567390881694, "pass@4": 0.3726164692376415, "pass@8": 0.42934240469128404, "pass@16": 0.47968165853761174, "pass@32": 0.5248203944523677, "pass@64": 0.5650033650072848, "pass@128": 0.6049382716049383 }, "16": { "pass@1": 0.2054526179941003, "pass@2": 0.25541919262304597, "pass@4": 0.2998856860259724, "pass@8": 0.33891998228181563, "pass@16": 0.37319465701364124, "pass@32": 0.4050990191660384, "pass@64": 0.43727254875461835, "pass@128": 0.46607669616519176 }, "17": { "pass@1": 0.1750988328313253, "pass@2": 0.23081190411251296, "pass@4": 0.2838856912464256, "pass@8": 0.33224730248156936, "pass@16": 0.37542718922860013, "pass@32": 0.415505560472016, "pass@64": 0.4525529833380625, "pass@128": 0.4819277108433735 }, "18": { "pass@1": 0.1844758064516129, "pass@2": 0.23592360934721868, "pass@4": 0.2844569100233439, "pass@8": 0.3272416885972945, "pass@16": 0.36229334243959777, "pass@32": 0.39137086403441174, "pass@64": 0.4171094098187473, "pass@128": 0.43870967741935485 }, "19": { "pass@1": 0.1746365181268882, "pass@2": 0.23602458845778723, "pass@4": 0.2923745359813408, "pass@8": 0.33760097581230863, "pass@16": 0.37204435321267093, "pass@32": 0.3998882313604565, "pass@64": 0.4237688040283907, "pass@128": 0.44410876132930516 }, "20": { "pass@1": 0.14153997747747749, "pass@2": 0.19203073644510646, "pass@4": 0.24307466690537555, "pass@8": 0.2914056515389387, "pass@16": 0.3347423022916823, "pass@32": 0.3709634296544144, "pass@64": 0.40021479133203836, "pass@128": 0.42342342342342343 } } } } } ]