[ { "checkpoint": "checkpoint-21808", "id_acc": 0.9757300347222222, "ood_acc": 0.35559765625, "total_acc": 0.649344572368421, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.41278559027776, "ood_avg_resp_len": 316.2850265625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 523405, "answer_accuracy": 0.6463960737432817, "avg_response_len": 246.91074533670565, "resp_tokens_sum": 199930544, "pass_at_k": { "pass@1": 0.6463960737432817, "pass@2": 0.6836626853679749, "pass@4": 0.7149836901759768, "pass@8": 0.7411302959656122, "pass@16": 0.762904361502207, "pass@32": 0.7807357410333592, "pass@64": 0.7951122370172407, "pass@128": 0.807777426493835 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9079784292035398, "pass@2": 0.9283034864004833, "pass@4": 0.9397207392770592, "pass@8": 0.9474065863832337, "pass@16": 0.953277630712324, "pass@32": 0.9587798739641823, "pass@64": 0.9641137000779781, "pass@128": 0.9705014749262537 }, "2": { "pass@1": 0.9999026479750779, "pass@2": 0.9999977003458681, "pass@4": 0.9999999997079805, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9992115825688074, "pass@2": 0.9999002955765851, "pass@4": 0.9999974601798628, "pass@8": 0.9999999989512287, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999365234375, "pass@2": 0.9999569389763779, "pass@4": 0.9999997020880201, "pass@8": 0.9999999999934361, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9670138888888888, "pass@2": 0.9722800274965628, "pass@4": 0.9764746609054821, "pass@8": 0.9805577074470837, "pass@16": 0.9849640666818144, "pass@32": 0.9887401621413379, "pass@64": 0.9903685676767175, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.9776618637724551, "pass@2": 0.9838419863029846, "pass@4": 0.9863429900603738, "pass@8": 0.9880363231052547, "pass@16": 0.9891851085789134, "pass@32": 0.9900843472432467, "pass@64": 0.9908396435351152, "pass@128": 0.9910179640718563 }, "7": { "pass@1": 0.9761002218934911, "pass@2": 0.9821877329590457, "pass@4": 0.9854717781209303, "pass@8": 0.9882141392067795, "pass@16": 0.9909368540370078, "pass@32": 0.9934787972166277, "pass@64": 0.995374785040164, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.9728565705128205, "pass@2": 0.9847504037956794, "pass@4": 0.9898392754992162, "pass@8": 0.9928595606012538, "pass@16": 0.9949857414228591, "pass@32": 0.9963289537684682, "pass@64": 0.9967736090563776, "pass@128": 0.9967948717948718 }, "9": { "pass@1": 0.9606514084507042, "pass@2": 0.9767279860263948, "pass@4": 0.9843846306183557, "pass@8": 0.9893238732854907, "pass@16": 0.9925283410495083, "pass@32": 0.9947305999396088, "pass@64": 0.9964951038189134, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.8500354107648725, "pass@2": 0.8847818752648838, "pass@4": 0.9057016992918374, "pass@8": 0.9209417546235552, "pass@16": 0.9350945491946546, "pass@32": 0.9489809420090249, "pass@64": 0.9604307908065416, "pass@128": 0.9688385269121813 }, "12": { "pass@1": 0.7215969279661016, "pass@2": 0.7881981516081679, "pass@4": 0.8320213376399981, "pass@8": 0.8641739560917514, "pass@16": 0.8889098921541904, "pass@32": 0.9070642083248085, "pass@64": 0.9203748363482039, "pass@128": 0.9322033898305084 }, "13": { "pass@1": 0.47176234326018807, "pass@2": 0.5514414354651594, "pass@4": 0.61641166121868, "pass@8": 0.6679790136854478, "pass@16": 0.7104903504508953, "pass@32": 0.7483723201148265, "pass@64": 0.782034213606473, "pass@128": 0.8119122257053292 }, "14": { "pass@1": 0.3773364485981308, "pass@2": 0.44722178784310845, "pass@4": 0.5053335929387331, "pass@8": 0.5553041100778835, "pass@16": 0.5983519667569652, "pass@32": 0.6341758147700675, "pass@64": 0.6635061144623973, "pass@128": 0.6853582554517134 }, "15": { "pass@1": 0.2905979046242775, "pass@2": 0.3691705828137089, "pass@4": 0.44011951603954086, "pass@8": 0.49954595991520595, "pass@16": 0.5489780070558901, "pass@32": 0.589607486805191, "pass@64": 0.622846434960636, "pass@128": 0.6502890173410405 }, "16": { "pass@1": 0.18639112903225807, "pass@2": 0.24502595567691135, "pass@4": 0.3011878971781754, "pass@8": 0.3503457986058463, "pass@16": 0.3939021510117161, "pass@32": 0.43316290463944823, "pass@64": 0.46712538073865945, "pass@128": 0.5 }, "17": { "pass@1": 0.18806448699421965, "pass@2": 0.24722041270310846, "pass@4": 0.30208944176414365, "pass@8": 0.3499756198131455, "pass@16": 0.39130847250177325, "pass@32": 0.4240642793091486, "pass@64": 0.44695993145996443, "pass@128": 0.4653179190751445 }, "18": { "pass@1": 0.1703794642857143, "pass@2": 0.23251019403824516, "pass@4": 0.2904351298944774, "pass@8": 0.3390466063051701, "pass@16": 0.3793374554178496, "pass@32": 0.4127509001659112, "pass@64": 0.4405737754821389, "pass@128": 0.4685714285714286 }, "19": { "pass@1": 0.14318224474474475, "pass@2": 0.20463980220377853, "pass@4": 0.2685763572909241, "pass@8": 0.3284336355160867, "pass@16": 0.3783702554316468, "pass@32": 0.4147896381558507, "pass@64": 0.4437101594568495, "pass@128": 0.4744744744744745 }, "20": { "pass@1": 0.15111204954954954, "pass@2": 0.21561509836608267, "pass@4": 0.281267438698541, "pass@8": 0.3375878425646831, "pass@16": 0.381100671812623, "pass@32": 0.4144019892616114, "pass@64": 0.4406480739566761, "pass@128": 0.4624624624624625 } } }, "teachers_in_school": { "count": 821120, "correct": 531860, "answer_accuracy": 0.647725058456742, "avg_response_len": 256.26843944855807, "resp_tokens_sum": 210427141, "pass_at_k": { "pass@1": 0.647725058456742, "pass@2": 0.6831084180777097, "pass@4": 0.7134677922128366, "pass@8": 0.7397314767573606, "pass@16": 0.7620730851567378, "pass@32": 0.7808543069415151, "pass@64": 0.7969666514692069, "pass@128": 0.8116913484021824 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9004026073619632, "pass@2": 0.9242703402009566, "pass@4": 0.9396745401073328, "pass@8": 0.9505904177376397, "pass@16": 0.9586655869893964, "pass@32": 0.96514230835694, "pass@64": 0.9707695920724272, "pass@128": 0.9754601226993865 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999763257575758, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999075443786982, "pass@2": 0.9999992720029819, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9834692028985508, "pass@2": 0.9852240956293506, "pass@4": 0.9858998532792097, "pass@8": 0.986317596894847, "pass@16": 0.9869360286784715, "pass@32": 0.9877361308097358, "pass@64": 0.988322264064818, "pass@128": 0.9884057971014493 }, "6": { "pass@1": 0.9928142908309455, "pass@2": 0.9974357275003949, "pass@4": 0.999494777783866, "pass@8": 0.9999737827585785, "pass@16": 0.9999999348964089, "pass@32": 0.9999999999999439, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9855491329479769, "pass@2": 0.9887166605980608, "pass@4": 0.9913939513883019, "pass@8": 0.9943285343504065, "pass@16": 0.9968946491921471, "pass@32": 0.9987241613729214, "pass@64": 0.999808867257577, "pass@128": 1.0 }, "8": { "pass@1": 0.9807658450704225, "pass@2": 0.9904752827991573, "pass@4": 0.9946537202215919, "pass@8": 0.9965723900602302, "pass@16": 0.9974438326135088, "pass@32": 0.9978865571381552, "pass@64": 0.9985915492939871, "pass@128": 1.0 }, "9": { "pass@1": 0.9550196482035929, "pass@2": 0.9680099692347592, "pass@4": 0.973621248223463, "pass@8": 0.9772931092950905, "pass@16": 0.9797760833989643, "pass@32": 0.9813793644561303, "pass@64": 0.9820069826621478, "pass@128": 0.9820359281437125 }, "11": { "pass@1": 0.8364551957831325, "pass@2": 0.8805564284697844, "pass@4": 0.9119869102393826, "pass@8": 0.9332457070554272, "pass@16": 0.9464105210220005, "pass@32": 0.9554480120240839, "pass@64": 0.9620750241307124, "pass@128": 0.9668674698795181 }, "12": { "pass@1": 0.7179907915360502, "pass@2": 0.7807096641818678, "pass@4": 0.8237406890557176, "pass@8": 0.8544050832000054, "pass@16": 0.8745406088754614, "pass@32": 0.8870839775741599, "pass@64": 0.8953727964648683, "pass@128": 0.8996865203761756 }, "13": { "pass@1": 0.4339110644257703, "pass@2": 0.5064138352191269, "pass@4": 0.5668354349613859, "pass@8": 0.6208126022401135, "pass@16": 0.6694676564972795, "pass@32": 0.7126055948949221, "pass@64": 0.7509963818526764, "pass@128": 0.7843137254901961 }, "14": { "pass@1": 0.39722679093567254, "pass@2": 0.46368376444720705, "pass@4": 0.5167455819667276, "pass@8": 0.56171751122582, "pass@16": 0.6024369898411556, "pass@32": 0.6426240615167135, "pass@64": 0.6832214593635492, "pass@128": 0.7222222222222222 }, "15": { "pass@1": 0.24644886363636365, "pass@2": 0.32176874552612744, "pass@4": 0.3923603217211486, "pass@8": 0.4581419440373477, "pass@16": 0.5190329686132177, "pass@32": 0.5727951896253346, "pass@64": 0.6198279692563144, "pass@128": 0.6636363636363637 }, "16": { "pass@1": 0.2040820868945869, "pass@2": 0.26635372501514243, "pass@4": 0.3238963121062858, "pass@8": 0.37332193265107516, "pass@16": 0.4134811645828353, "pass@32": 0.4443577173332517, "pass@64": 0.47302414732746867, "pass@128": 0.50997150997151 }, "17": { "pass@1": 0.19650135869565216, "pass@2": 0.2574690052330416, "pass@4": 0.3152138539941824, "pass@8": 0.3709270117297045, "pass@16": 0.42297098948583206, "pass@32": 0.4692510903823389, "pass@64": 0.5108174522501921, "pass@128": 0.5527950310559007 }, "18": { "pass@1": 0.18683363970588235, "pass@2": 0.24651024779990735, "pass@4": 0.3035248188645537, "pass@8": 0.3556424980931623, "pass@16": 0.4017419832141523, "pass@32": 0.43944869520791696, "pass@64": 0.4673817173155303, "pass@128": 0.49117647058823527 }, "19": { "pass@1": 0.14808872767857142, "pass@2": 0.2049320250984251, "pass@4": 0.2627956976918065, "pass@8": 0.3139157657154089, "pass@16": 0.3560561707464758, "pass@32": 0.39169649693737485, "pass@64": 0.4226012391904779, "pass@128": 0.44642857142857145 }, "20": { "pass@1": 0.13087107035928144, "pass@2": 0.1879670776557122, "pass@4": 0.24696301351178396, "pass@8": 0.30177241862067616, "pass@16": 0.34838594581032317, "pass@32": 0.38534870948871147, "pass@64": 0.4129125683339026, "pass@128": 0.4341317365269461 } } }, "movie_festival_awards": { "count": 801152, "correct": 523941, "answer_accuracy": 0.6539845123022847, "avg_response_len": 275.54656045294774, "resp_tokens_sum": 220754678, "pass_at_k": { "pass@1": 0.6539845123022847, "pass@2": 0.689856626143393, "pass@4": 0.7200727996005184, "pass@8": 0.7453208731590747, "pass@16": 0.7665777414461511, "pass@32": 0.7847634494980638, "pass@64": 0.8007918914069807, "pass@128": 0.8149864195558396 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9029617537313432, "pass@2": 0.9262189299565168, "pass@4": 0.9432951989210303, "pass@8": 0.9559143658871644, "pass@16": 0.9651937507145258, "pass@32": 0.9716208086596739, "pass@64": 0.9761076892137535, "pass@128": 0.9791044776119403 }, "2": { "pass@1": 0.9999107142857143, "pass@2": 0.9999989454443194, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9991116982507289, "pass@2": 0.9997976974816924, "pass@4": 0.9999873259282823, "pass@8": 0.9999999629758403, "pass@16": 0.9999999999999313, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999314692982456, "pass@2": 0.9999996402587834, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9659926470588235, "pass@2": 0.9708096919870312, "pass@4": 0.9745823887822845, "pass@8": 0.9771404920721685, "pass@16": 0.9789554066534717, "pass@32": 0.9809376766950135, "pass@64": 0.9837387679474603, "pass@128": 0.9882352941176471 }, "6": { "pass@1": 0.9929268533123028, "pass@2": 0.9952720199210117, "pass@4": 0.9963477722816194, "pass@8": 0.996781255357137, "pass@16": 0.996844514797445, "pass@32": 0.9968454258207375, "pass@64": 0.9968454258675079, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.9869956487341772, "pass@2": 0.9934614210605002, "pass@4": 0.9959656588614716, "pass@8": 0.9971545915353225, "pass@16": 0.9978742203032149, "pass@32": 0.9986754832496939, "pass@64": 0.9996137745440047, "pass@128": 1.0 }, "8": { "pass@1": 0.9711195570570571, "pass@2": 0.9796935222624201, "pass@4": 0.9847409628413564, "pass@8": 0.9876525069319597, "pass@16": 0.9891184230382688, "pass@32": 0.9900539597715055, "pass@64": 0.9908121349376716, "pass@128": 0.990990990990991 }, "9": { "pass@1": 0.9665393890675241, "pass@2": 0.9782261943945113, "pass@4": 0.9827812531271213, "pass@8": 0.9852322346734745, "pass@16": 0.9870234208375871, "pass@32": 0.9888026081069605, "pass@64": 0.9901177403407886, "pass@128": 0.9903536977491961 }, "11": { "pass@1": 0.8699652777777778, "pass@2": 0.9050528840144983, "pass@4": 0.9261437665529902, "pass@8": 0.9379237916730481, "pass@16": 0.9436162837580702, "pass@32": 0.9481522394334617, "pass@64": 0.9538037350418224, "pass@128": 0.9587301587301588 }, "12": { "pass@1": 0.6851586391437309, "pass@2": 0.7609573280117508, "pass@4": 0.8146087567609088, "pass@8": 0.8498854256731503, "pass@16": 0.8726716568222734, "pass@32": 0.8890453884635856, "pass@64": 0.9038027299053867, "pass@128": 0.9204892966360856 }, "13": { "pass@1": 0.48818479938271603, "pass@2": 0.5643306630941965, "pass@4": 0.6234116620839059, "pass@8": 0.670892987256112, "pass@16": 0.710554304400132, "pass@32": 0.7441477272144019, "pass@64": 0.7721765126609678, "pass@128": 0.7932098765432098 }, "14": { "pass@1": 0.39139002225519287, "pass@2": 0.4689131901446295, "pass@4": 0.5358667138974095, "pass@8": 0.5899477624146936, "pass@16": 0.6321371774697618, "pass@32": 0.6666508816034391, "pass@64": 0.697774235354575, "pass@128": 0.7270029673590505 }, "15": { "pass@1": 0.29598283179012347, "pass@2": 0.3657031477836101, "pass@4": 0.4284062113763555, "pass@8": 0.488394802453985, "pass@16": 0.5458289659825917, "pass@32": 0.5968433317356244, "pass@64": 0.6421028185366104, "pass@128": 0.6820987654320988 }, "16": { "pass@1": 0.20973912241887904, "pass@2": 0.2661744680974612, "pass@4": 0.32020332027080706, "pass@8": 0.37020648522530164, "pass@16": 0.4155200084120026, "pass@32": 0.4563823340406111, "pass@64": 0.4954197121915054, "pass@128": 0.5339233038348082 }, "17": { "pass@1": 0.1867940512048193, "pass@2": 0.2508834550801631, "pass@4": 0.3094012797534344, "pass@8": 0.3578539351904199, "pass@16": 0.3979111457071782, "pass@32": 0.43246884560924453, "pass@64": 0.46455603309265586, "pass@128": 0.49698795180722893 }, "18": { "pass@1": 0.17842741935483872, "pass@2": 0.2330915036830072, "pass@4": 0.286078386068677, "pass@8": 0.3361083917487688, "pass@16": 0.3821960914627827, "pass@32": 0.422830010224237, "pass@64": 0.45738747568942023, "pass@128": 0.4870967741935484 }, "19": { "pass@1": 0.1512698262839879, "pass@2": 0.20847594083783336, "pass@4": 0.2632785100314122, "pass@8": 0.31349632397118765, "pass@16": 0.3602084544258579, "pass@32": 0.40216106105046995, "pass@64": 0.4354416751759285, "pass@128": 0.459214501510574 }, "20": { "pass@1": 0.16378096846846846, "pass@2": 0.22388222375919223, "pass@4": 0.2818586584334616, "pass@8": 0.3335726763870422, "pass@16": 0.37771145894953373, "pass@32": 0.41447893995579, "pass@64": 0.44587055010516763, "pass@128": 0.47147147147147145 } } } } } ]