| [ |
| { |
| "checkpoint": "checkpoint-44699", |
| "id_acc": 0.9833550347222222, |
| "ood_acc": 0.36736640625, |
| "total_acc": 0.6591504934210526, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.52830208333333, |
| "ood_avg_resp_len": 312.80330546875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 531202, |
| "answer_accuracy": 0.6560252331647171, |
| "avg_response_len": 244.93973161357889, |
| "resp_tokens_sum": 198334559, |
| "pass_at_k": { |
| "pass@1": 0.6560252331647171, |
| "pass@2": 0.6894634278667485, |
| "pass@4": 0.7172932349380673, |
| "pass@8": 0.7407126418953518, |
| "pass@16": 0.7603428177395862, |
| "pass@32": 0.7770562234748629, |
| "pass@64": 0.7919610478846388, |
| "pass@128": 0.8055643376541258 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9344579646017699, |
| "pass@2": 0.9526916678280263, |
| "pass@4": 0.9628839813164948, |
| "pass@8": 0.9702581096493935, |
| "pass@16": 0.9760459335416097, |
| "pass@32": 0.9812674749596855, |
| "pass@64": 0.9866724375578277, |
| "pass@128": 0.9911504424778761 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.999641628440367, |
| "pass@2": 0.9999909701654267, |
| "pass@4": 0.9999999971333858, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.996533203125, |
| "pass@2": 0.9968619279035433, |
| "pass@4": 0.9968749792018186, |
| "pass@8": 0.9968749999999978, |
| "pass@16": 0.996875, |
| "pass@32": 0.996875, |
| "pass@64": 0.996875, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9764632936507937, |
| "pass@2": 0.9804899387576552, |
| "pass@4": 0.9837944602162825, |
| "pass@8": 0.9863203732706364, |
| "pass@16": 0.9880740031976679, |
| "pass@32": 0.9894659832300179, |
| "pass@64": 0.9912760738735891, |
| "pass@128": 0.9936507936507937 |
| }, |
| "6": { |
| "pass@1": 0.9913454341317365, |
| "pass@2": 0.9930778207364798, |
| "pass@4": 0.9938670788344569, |
| "pass@8": 0.9940085499193233, |
| "pass@16": 0.9940119746912577, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 0.9940119760479041, |
| "pass@128": 0.9940119760479041 |
| }, |
| "7": { |
| "pass@1": 0.9867788461538461, |
| "pass@2": 0.9901181684759818, |
| "pass@4": 0.992015774655387, |
| "pass@8": 0.994022348072151, |
| "pass@16": 0.9957065225421011, |
| "pass@32": 0.9966644429698368, |
| "pass@64": 0.9970219764691983, |
| "pass@128": 0.9970414201183432 |
| }, |
| "8": { |
| "pass@1": 0.9838741987179487, |
| "pass@2": 0.9881515148647284, |
| "pass@4": 0.9893485868352994, |
| "pass@8": 0.9903211494125901, |
| "pass@16": 0.9910836657779668, |
| "pass@32": 0.9917911369360106, |
| "pass@64": 0.9927947708452297, |
| "pass@128": 0.9935897435897436 |
| }, |
| "9": { |
| "pass@1": 0.9845070422535211, |
| "pass@2": 0.9931691527115452, |
| "pass@4": 0.9958114764703706, |
| "pass@8": 0.9972125362894183, |
| "pass@16": 0.9986897583523467, |
| "pass@32": 0.9997578443233471, |
| "pass@64": 0.9999972582837349, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8852027266288952, |
| "pass@2": 0.9147545085989603, |
| "pass@4": 0.9345978617049635, |
| "pass@8": 0.9490441153165399, |
| "pass@16": 0.9606432019579066, |
| "pass@32": 0.9705314683575175, |
| "pass@64": 0.9782839078839444, |
| "pass@128": 0.9858356940509915 |
| }, |
| "12": { |
| "pass@1": 0.7466454802259888, |
| "pass@2": 0.8139510070510254, |
| "pass@4": 0.8601370909144825, |
| "pass@8": 0.8920004996421725, |
| "pass@16": 0.9141258280948045, |
| "pass@32": 0.9296279933820031, |
| "pass@64": 0.9410879649553646, |
| "pass@128": 0.9519774011299436 |
| }, |
| "13": { |
| "pass@1": 0.5093798981191222, |
| "pass@2": 0.5921151081134451, |
| "pass@4": 0.6576909166722495, |
| "pass@8": 0.7099843278394556, |
| "pass@16": 0.75098134301228, |
| "pass@32": 0.7835781471809989, |
| "pass@64": 0.80972291492387, |
| "pass@128": 0.8307210031347962 |
| }, |
| "14": { |
| "pass@1": 0.3689154984423676, |
| "pass@2": 0.44391335209851096, |
| "pass@4": 0.5070102309057162, |
| "pass@8": 0.5610363317684318, |
| "pass@16": 0.6082930404376408, |
| "pass@32": 0.6493371167503605, |
| "pass@64": 0.6867744387040765, |
| "pass@128": 0.7133956386292835 |
| }, |
| "15": { |
| "pass@1": 0.2523708453757225, |
| "pass@2": 0.31927668858950414, |
| "pass@4": 0.3786499197535278, |
| "pass@8": 0.4312203464692883, |
| "pass@16": 0.47739438847028326, |
| "pass@32": 0.5181956268293656, |
| "pass@64": 0.5577668100174789, |
| "pass@128": 0.5982658959537572 |
| }, |
| "16": { |
| "pass@1": 0.19014616935483872, |
| "pass@2": 0.2436293497586995, |
| "pass@4": 0.29316089148130675, |
| "pass@8": 0.33883962767516357, |
| "pass@16": 0.3797893057690267, |
| "pass@32": 0.41671065349623226, |
| "pass@64": 0.45184103790572516, |
| "pass@128": 0.4870967741935484 |
| }, |
| "17": { |
| "pass@1": 0.18727420520231214, |
| "pass@2": 0.24610352851486034, |
| "pass@4": 0.3004370235873695, |
| "pass@8": 0.3462447129619863, |
| "pass@16": 0.3822512517998407, |
| "pass@32": 0.4124269666327473, |
| "pass@64": 0.44181302593089994, |
| "pass@128": 0.476878612716763 |
| }, |
| "18": { |
| "pass@1": 0.18810267857142857, |
| "pass@2": 0.24303008998875142, |
| "pass@4": 0.2925131434463549, |
| "pass@8": 0.3346438368665325, |
| "pass@16": 0.37033006112880934, |
| "pass@32": 0.39964188355370805, |
| "pass@64": 0.42305886798827047, |
| "pass@128": 0.44285714285714284 |
| }, |
| "19": { |
| "pass@1": 0.15784534534534533, |
| "pass@2": 0.20692124210824994, |
| "pass@4": 0.25512323628465366, |
| "pass@8": 0.29988871819925383, |
| "pass@16": 0.3406692381875606, |
| "pass@32": 0.37891669047499194, |
| "pass@64": 0.4135048107997905, |
| "pass@128": 0.44144144144144143 |
| }, |
| "20": { |
| "pass@1": 0.15195664414414414, |
| "pass@2": 0.20411368553592957, |
| "pass@4": 0.254373751422964, |
| "pass@8": 0.2996420531321556, |
| "pass@16": 0.3385611068086796, |
| "pass@32": 0.37173141526078607, |
| "pass@64": 0.40101691150690105, |
| "pass@128": 0.4264264264264264 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 542547, |
| "answer_accuracy": 0.6607402084957131, |
| "avg_response_len": 254.24384986360093, |
| "resp_tokens_sum": 208764710, |
| "pass_at_k": { |
| "pass@1": 0.6607402084957131, |
| "pass@2": 0.692345312106834, |
| "pass@4": 0.7187453154238392, |
| "pass@8": 0.7414170852156288, |
| "pass@16": 0.7608738632835582, |
| "pass@32": 0.7776036865720104, |
| "pass@64": 0.7921936522666159, |
| "pass@128": 0.8053000779423227 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9242954371165644, |
| "pass@2": 0.9467247204241344, |
| "pass@4": 0.960173860648554, |
| "pass@8": 0.9688716992564892, |
| "pass@16": 0.9746263491641634, |
| "pass@32": 0.9789321415016857, |
| "pass@64": 0.9833402213400504, |
| "pass@128": 0.9877300613496932 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999526515151516, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9970645340236687, |
| "pass@2": 0.9970876479289941, |
| "pass@4": 0.997133875739645, |
| "pass@8": 0.9972263313609467, |
| "pass@16": 0.9974112426035503, |
| "pass@32": 0.9977810650887574, |
| "pass@64": 0.9985207100591716, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9887001811594203, |
| "pass@2": 0.9915953440602534, |
| "pass@4": 0.9933488841068782, |
| "pass@8": 0.9947347459851477, |
| "pass@16": 0.9960028065978203, |
| "pass@32": 0.9968289314680655, |
| "pass@64": 0.9970924755620924, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9932843839541547, |
| "pass@2": 0.9942125843241656, |
| "pass@4": 0.9942690063097414, |
| "pass@8": 0.9942693409681034, |
| "pass@16": 0.994269340974212, |
| "pass@32": 0.994269340974212, |
| "pass@64": 0.994269340974212, |
| "pass@128": 0.994269340974212 |
| }, |
| "7": { |
| "pass@1": 0.9916907514450867, |
| "pass@2": 0.9933922130308135, |
| "pass@4": 0.9947579433712405, |
| "pass@8": 0.9961805922227138, |
| "pass@16": 0.9969781403865762, |
| "pass@32": 0.9971081671151855, |
| "pass@64": 0.99710982657694, |
| "pass@128": 0.9971098265895953 |
| }, |
| "8": { |
| "pass@1": 0.9907570422535211, |
| "pass@2": 0.9952329627370523, |
| "pass@4": 0.9966467019967574, |
| "pass@8": 0.9972438992106627, |
| "pass@16": 0.9975316366671934, |
| "pass@32": 0.9978873227410595, |
| "pass@64": 0.9985915492957746, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9678845434131736, |
| "pass@2": 0.9773401032580507, |
| "pass@4": 0.980435996118, |
| "pass@8": 0.9821315536020941, |
| "pass@16": 0.9838955100446117, |
| "pass@32": 0.9854663026265753, |
| "pass@64": 0.9865217056571726, |
| "pass@128": 0.9880239520958084 |
| }, |
| "11": { |
| "pass@1": 0.9021084337349398, |
| "pass@2": 0.929302100132815, |
| "pass@4": 0.9446336069813555, |
| "pass@8": 0.955031982663235, |
| "pass@16": 0.962953963538521, |
| "pass@32": 0.9693316516149345, |
| "pass@64": 0.974545578181479, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.7708170062695925, |
| "pass@2": 0.8253758824327991, |
| "pass@4": 0.8607174199228226, |
| "pass@8": 0.8830145741372571, |
| "pass@16": 0.8988607561671764, |
| "pass@32": 0.9145445540094311, |
| "pass@64": 0.930184391477638, |
| "pass@128": 0.9435736677115988 |
| }, |
| "13": { |
| "pass@1": 0.4949229691876751, |
| "pass@2": 0.5704298451664127, |
| "pass@4": 0.6317669708303267, |
| "pass@8": 0.6809381380037726, |
| "pass@16": 0.7209420738281904, |
| "pass@32": 0.7554410660153358, |
| "pass@64": 0.7887207068048872, |
| "pass@128": 0.8263305322128851 |
| }, |
| "14": { |
| "pass@1": 0.3959703947368421, |
| "pass@2": 0.46360354215591465, |
| "pass@4": 0.524082480369778, |
| "pass@8": 0.5806233011224652, |
| "pass@16": 0.6316132775599853, |
| "pass@32": 0.6754358216679552, |
| "pass@64": 0.712545405106953, |
| "pass@128": 0.7456140350877193 |
| }, |
| "15": { |
| "pass@1": 0.2523910984848485, |
| "pass@2": 0.3201749284180385, |
| "pass@4": 0.38129389508129663, |
| "pass@8": 0.4367986120606595, |
| "pass@16": 0.48397204169078695, |
| "pass@32": 0.5211283773311105, |
| "pass@64": 0.5485796081544515, |
| "pass@128": 0.5696969696969697 |
| }, |
| "16": { |
| "pass@1": 0.20877849002849003, |
| "pass@2": 0.2670165528187184, |
| "pass@4": 0.31650643375774584, |
| "pass@8": 0.3575628181721004, |
| "pass@16": 0.39263447515402217, |
| "pass@32": 0.42433719998352165, |
| "pass@64": 0.45382145756108494, |
| "pass@128": 0.47863247863247865 |
| }, |
| "17": { |
| "pass@1": 0.2135335791925466, |
| "pass@2": 0.27400757140411797, |
| "pass@4": 0.3274636615687015, |
| "pass@8": 0.3724455659358969, |
| "pass@16": 0.4118936793086089, |
| "pass@32": 0.44952103938521476, |
| "pass@64": 0.48420234899227, |
| "pass@128": 0.5124223602484472 |
| }, |
| "18": { |
| "pass@1": 0.158984375, |
| "pass@2": 0.21292156380268631, |
| "pass@4": 0.2651559686473013, |
| "pass@8": 0.3149142166840468, |
| "pass@16": 0.36073523939032814, |
| "pass@32": 0.3976295956029731, |
| "pass@64": 0.42512193141106314, |
| "pass@128": 0.45 |
| }, |
| "19": { |
| "pass@1": 0.14674014136904762, |
| "pass@2": 0.19081233009936244, |
| "pass@4": 0.23490378267448708, |
| "pass@8": 0.2787363831910592, |
| "pass@16": 0.32005228112975653, |
| "pass@32": 0.35780752009771627, |
| "pass@64": 0.3924129635883499, |
| "pass@128": 0.4226190476190476 |
| }, |
| "20": { |
| "pass@1": 0.1475252619760479, |
| "pass@2": 0.19783648745815446, |
| "pass@4": 0.24589100903928915, |
| "pass@8": 0.28933610432988377, |
| "pass@16": 0.3253420999811492, |
| "pass@32": 0.35424371588248044, |
| "pass@64": 0.37930679206589685, |
| "pass@128": 0.40119760479041916 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 529305, |
| "answer_accuracy": 0.6606798709857805, |
| "avg_response_len": 274.21708489774727, |
| "resp_tokens_sum": 219689566, |
| "pass_at_k": { |
| "pass@1": 0.6606798709857805, |
| "pass@2": 0.6923162142577697, |
| "pass@4": 0.7192061071352318, |
| "pass@8": 0.7419212559542995, |
| "pass@16": 0.7612971729400352, |
| "pass@32": 0.7782682183303264, |
| "pass@64": 0.7932781986164202, |
| "pass@128": 0.8061990733343984 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9298973880597015, |
| "pass@2": 0.9486341667646025, |
| "pass@4": 0.9623331648842398, |
| "pass@8": 0.9719875662983237, |
| "pass@16": 0.9785429498814807, |
| "pass@32": 0.9841315033131119, |
| "pass@64": 0.9887908906350661, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999772230320699, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999543128654971, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9717601102941177, |
| "pass@2": 0.9771971977767483, |
| "pass@4": 0.9794758403361344, |
| "pass@8": 0.9802533717589259, |
| "pass@16": 0.9810081996249801, |
| "pass@32": 0.9821723681597684, |
| "pass@64": 0.9836483557202409, |
| "pass@128": 0.9852941176470589 |
| }, |
| "6": { |
| "pass@1": 0.9996056782334385, |
| "pass@2": 0.9999910734245759, |
| "pass@4": 0.9999999896503473, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9830152294303798, |
| "pass@2": 0.9880324616266324, |
| "pass@4": 0.9902660212370608, |
| "pass@8": 0.991350756420527, |
| "pass@16": 0.9922229217269881, |
| "pass@32": 0.9934765985754536, |
| "pass@64": 0.9950646865344362, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.9827796546546547, |
| "pass@2": 0.9879839238845144, |
| "pass@4": 0.9901089140321423, |
| "pass@8": 0.9912163301188336, |
| "pass@16": 0.9919635210522875, |
| "pass@32": 0.9927370227462073, |
| "pass@64": 0.9936274857534592, |
| "pass@128": 0.993993993993994 |
| }, |
| "9": { |
| "pass@1": 0.9745779742765274, |
| "pass@2": 0.9838013804845936, |
| "pass@4": 0.9882725915289527, |
| "pass@8": 0.9912003897231443, |
| "pass@16": 0.9930943945884175, |
| "pass@32": 0.9942506476636976, |
| "pass@64": 0.9951758657948274, |
| "pass@128": 0.9967845659163987 |
| }, |
| "11": { |
| "pass@1": 0.8968005952380952, |
| "pass@2": 0.9253007436570432, |
| "pass@4": 0.9426268133150022, |
| "pass@8": 0.9550882186965979, |
| "pass@16": 0.965702093414588, |
| "pass@32": 0.9758799151725375, |
| "pass@64": 0.9852208180666315, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.7458428899082569, |
| "pass@2": 0.8013131636928408, |
| "pass@4": 0.8399700091960991, |
| "pass@8": 0.8693948834655879, |
| "pass@16": 0.8932526941911637, |
| "pass@32": 0.9134654564118084, |
| "pass@64": 0.929910476447298, |
| "pass@128": 0.9388379204892966 |
| }, |
| "13": { |
| "pass@1": 0.5423659336419753, |
| "pass@2": 0.6123213005492364, |
| "pass@4": 0.6643283845885002, |
| "pass@8": 0.7066937664543848, |
| "pass@16": 0.7454170853733546, |
| "pass@32": 0.7799614307727357, |
| "pass@64": 0.8075608604900163, |
| "pass@128": 0.8302469135802469 |
| }, |
| "14": { |
| "pass@1": 0.38826038575667654, |
| "pass@2": 0.45652023119699037, |
| "pass@4": 0.5185281256571415, |
| "pass@8": 0.5713009485900146, |
| "pass@16": 0.615971139816462, |
| "pass@32": 0.6542321591059677, |
| "pass@64": 0.6889103089980461, |
| "pass@128": 0.7270029673590505 |
| }, |
| "15": { |
| "pass@1": 0.2394386574074074, |
| "pass@2": 0.30811567390881694, |
| "pass@4": 0.3726164692376415, |
| "pass@8": 0.42934240469128404, |
| "pass@16": 0.47968165853761174, |
| "pass@32": 0.5248203944523677, |
| "pass@64": 0.5650033650072848, |
| "pass@128": 0.6049382716049383 |
| }, |
| "16": { |
| "pass@1": 0.2054526179941003, |
| "pass@2": 0.25541919262304597, |
| "pass@4": 0.2998856860259724, |
| "pass@8": 0.33891998228181563, |
| "pass@16": 0.37319465701364124, |
| "pass@32": 0.4050990191660384, |
| "pass@64": 0.43727254875461835, |
| "pass@128": 0.46607669616519176 |
| }, |
| "17": { |
| "pass@1": 0.1750988328313253, |
| "pass@2": 0.23081190411251296, |
| "pass@4": 0.2838856912464256, |
| "pass@8": 0.33224730248156936, |
| "pass@16": 0.37542718922860013, |
| "pass@32": 0.415505560472016, |
| "pass@64": 0.4525529833380625, |
| "pass@128": 0.4819277108433735 |
| }, |
| "18": { |
| "pass@1": 0.1844758064516129, |
| "pass@2": 0.23592360934721868, |
| "pass@4": 0.2844569100233439, |
| "pass@8": 0.3272416885972945, |
| "pass@16": 0.36229334243959777, |
| "pass@32": 0.39137086403441174, |
| "pass@64": 0.4171094098187473, |
| "pass@128": 0.43870967741935485 |
| }, |
| "19": { |
| "pass@1": 0.1746365181268882, |
| "pass@2": 0.23602458845778723, |
| "pass@4": 0.2923745359813408, |
| "pass@8": 0.33760097581230863, |
| "pass@16": 0.37204435321267093, |
| "pass@32": 0.3998882313604565, |
| "pass@64": 0.4237688040283907, |
| "pass@128": 0.44410876132930516 |
| }, |
| "20": { |
| "pass@1": 0.14153997747747749, |
| "pass@2": 0.19203073644510646, |
| "pass@4": 0.24307466690537555, |
| "pass@8": 0.2914056515389387, |
| "pass@16": 0.3347423022916823, |
| "pass@32": 0.3709634296544144, |
| "pass@64": 0.40021479133203836, |
| "pass@128": 0.42342342342342343 |
| } |
| } |
| } |
| } |
| } |
| ] |
