| [ |
| { |
| "checkpoint": "checkpoint-21808", |
| "id_acc": 0.9757300347222222, |
| "ood_acc": 0.35559765625, |
| "total_acc": 0.649344572368421, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.41278559027776, |
| "ood_avg_resp_len": 316.2850265625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 523405, |
| "answer_accuracy": 0.6463960737432817, |
| "avg_response_len": 246.91074533670565, |
| "resp_tokens_sum": 199930544, |
| "pass_at_k": { |
| "pass@1": 0.6463960737432817, |
| "pass@2": 0.6836626853679749, |
| "pass@4": 0.7149836901759768, |
| "pass@8": 0.7411302959656122, |
| "pass@16": 0.762904361502207, |
| "pass@32": 0.7807357410333592, |
| "pass@64": 0.7951122370172407, |
| "pass@128": 0.807777426493835 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9079784292035398, |
| "pass@2": 0.9283034864004833, |
| "pass@4": 0.9397207392770592, |
| "pass@8": 0.9474065863832337, |
| "pass@16": 0.953277630712324, |
| "pass@32": 0.9587798739641823, |
| "pass@64": 0.9641137000779781, |
| "pass@128": 0.9705014749262537 |
| }, |
| "2": { |
| "pass@1": 0.9999026479750779, |
| "pass@2": 0.9999977003458681, |
| "pass@4": 0.9999999997079805, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9992115825688074, |
| "pass@2": 0.9999002955765851, |
| "pass@4": 0.9999974601798628, |
| "pass@8": 0.9999999989512287, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999365234375, |
| "pass@2": 0.9999569389763779, |
| "pass@4": 0.9999997020880201, |
| "pass@8": 0.9999999999934361, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9670138888888888, |
| "pass@2": 0.9722800274965628, |
| "pass@4": 0.9764746609054821, |
| "pass@8": 0.9805577074470837, |
| "pass@16": 0.9849640666818144, |
| "pass@32": 0.9887401621413379, |
| "pass@64": 0.9903685676767175, |
| "pass@128": 0.9904761904761905 |
| }, |
| "6": { |
| "pass@1": 0.9776618637724551, |
| "pass@2": 0.9838419863029846, |
| "pass@4": 0.9863429900603738, |
| "pass@8": 0.9880363231052547, |
| "pass@16": 0.9891851085789134, |
| "pass@32": 0.9900843472432467, |
| "pass@64": 0.9908396435351152, |
| "pass@128": 0.9910179640718563 |
| }, |
| "7": { |
| "pass@1": 0.9761002218934911, |
| "pass@2": 0.9821877329590457, |
| "pass@4": 0.9854717781209303, |
| "pass@8": 0.9882141392067795, |
| "pass@16": 0.9909368540370078, |
| "pass@32": 0.9934787972166277, |
| "pass@64": 0.995374785040164, |
| "pass@128": 0.9970414201183432 |
| }, |
| "8": { |
| "pass@1": 0.9728565705128205, |
| "pass@2": 0.9847504037956794, |
| "pass@4": 0.9898392754992162, |
| "pass@8": 0.9928595606012538, |
| "pass@16": 0.9949857414228591, |
| "pass@32": 0.9963289537684682, |
| "pass@64": 0.9967736090563776, |
| "pass@128": 0.9967948717948718 |
| }, |
| "9": { |
| "pass@1": 0.9606514084507042, |
| "pass@2": 0.9767279860263948, |
| "pass@4": 0.9843846306183557, |
| "pass@8": 0.9893238732854907, |
| "pass@16": 0.9925283410495083, |
| "pass@32": 0.9947305999396088, |
| "pass@64": 0.9964951038189134, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.8500354107648725, |
| "pass@2": 0.8847818752648838, |
| "pass@4": 0.9057016992918374, |
| "pass@8": 0.9209417546235552, |
| "pass@16": 0.9350945491946546, |
| "pass@32": 0.9489809420090249, |
| "pass@64": 0.9604307908065416, |
| "pass@128": 0.9688385269121813 |
| }, |
| "12": { |
| "pass@1": 0.7215969279661016, |
| "pass@2": 0.7881981516081679, |
| "pass@4": 0.8320213376399981, |
| "pass@8": 0.8641739560917514, |
| "pass@16": 0.8889098921541904, |
| "pass@32": 0.9070642083248085, |
| "pass@64": 0.9203748363482039, |
| "pass@128": 0.9322033898305084 |
| }, |
| "13": { |
| "pass@1": 0.47176234326018807, |
| "pass@2": 0.5514414354651594, |
| "pass@4": 0.61641166121868, |
| "pass@8": 0.6679790136854478, |
| "pass@16": 0.7104903504508953, |
| "pass@32": 0.7483723201148265, |
| "pass@64": 0.782034213606473, |
| "pass@128": 0.8119122257053292 |
| }, |
| "14": { |
| "pass@1": 0.3773364485981308, |
| "pass@2": 0.44722178784310845, |
| "pass@4": 0.5053335929387331, |
| "pass@8": 0.5553041100778835, |
| "pass@16": 0.5983519667569652, |
| "pass@32": 0.6341758147700675, |
| "pass@64": 0.6635061144623973, |
| "pass@128": 0.6853582554517134 |
| }, |
| "15": { |
| "pass@1": 0.2905979046242775, |
| "pass@2": 0.3691705828137089, |
| "pass@4": 0.44011951603954086, |
| "pass@8": 0.49954595991520595, |
| "pass@16": 0.5489780070558901, |
| "pass@32": 0.589607486805191, |
| "pass@64": 0.622846434960636, |
| "pass@128": 0.6502890173410405 |
| }, |
| "16": { |
| "pass@1": 0.18639112903225807, |
| "pass@2": 0.24502595567691135, |
| "pass@4": 0.3011878971781754, |
| "pass@8": 0.3503457986058463, |
| "pass@16": 0.3939021510117161, |
| "pass@32": 0.43316290463944823, |
| "pass@64": 0.46712538073865945, |
| "pass@128": 0.5 |
| }, |
| "17": { |
| "pass@1": 0.18806448699421965, |
| "pass@2": 0.24722041270310846, |
| "pass@4": 0.30208944176414365, |
| "pass@8": 0.3499756198131455, |
| "pass@16": 0.39130847250177325, |
| "pass@32": 0.4240642793091486, |
| "pass@64": 0.44695993145996443, |
| "pass@128": 0.4653179190751445 |
| }, |
| "18": { |
| "pass@1": 0.1703794642857143, |
| "pass@2": 0.23251019403824516, |
| "pass@4": 0.2904351298944774, |
| "pass@8": 0.3390466063051701, |
| "pass@16": 0.3793374554178496, |
| "pass@32": 0.4127509001659112, |
| "pass@64": 0.4405737754821389, |
| "pass@128": 0.4685714285714286 |
| }, |
| "19": { |
| "pass@1": 0.14318224474474475, |
| "pass@2": 0.20463980220377853, |
| "pass@4": 0.2685763572909241, |
| "pass@8": 0.3284336355160867, |
| "pass@16": 0.3783702554316468, |
| "pass@32": 0.4147896381558507, |
| "pass@64": 0.4437101594568495, |
| "pass@128": 0.4744744744744745 |
| }, |
| "20": { |
| "pass@1": 0.15111204954954954, |
| "pass@2": 0.21561509836608267, |
| "pass@4": 0.281267438698541, |
| "pass@8": 0.3375878425646831, |
| "pass@16": 0.381100671812623, |
| "pass@32": 0.4144019892616114, |
| "pass@64": 0.4406480739566761, |
| "pass@128": 0.4624624624624625 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 531860, |
| "answer_accuracy": 0.647725058456742, |
| "avg_response_len": 256.26843944855807, |
| "resp_tokens_sum": 210427141, |
| "pass_at_k": { |
| "pass@1": 0.647725058456742, |
| "pass@2": 0.6831084180777097, |
| "pass@4": 0.7134677922128366, |
| "pass@8": 0.7397314767573606, |
| "pass@16": 0.7620730851567378, |
| "pass@32": 0.7808543069415151, |
| "pass@64": 0.7969666514692069, |
| "pass@128": 0.8116913484021824 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9004026073619632, |
| "pass@2": 0.9242703402009566, |
| "pass@4": 0.9396745401073328, |
| "pass@8": 0.9505904177376397, |
| "pass@16": 0.9586655869893964, |
| "pass@32": 0.96514230835694, |
| "pass@64": 0.9707695920724272, |
| "pass@128": 0.9754601226993865 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999763257575758, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999075443786982, |
| "pass@2": 0.9999992720029819, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9834692028985508, |
| "pass@2": 0.9852240956293506, |
| "pass@4": 0.9858998532792097, |
| "pass@8": 0.986317596894847, |
| "pass@16": 0.9869360286784715, |
| "pass@32": 0.9877361308097358, |
| "pass@64": 0.988322264064818, |
| "pass@128": 0.9884057971014493 |
| }, |
| "6": { |
| "pass@1": 0.9928142908309455, |
| "pass@2": 0.9974357275003949, |
| "pass@4": 0.999494777783866, |
| "pass@8": 0.9999737827585785, |
| "pass@16": 0.9999999348964089, |
| "pass@32": 0.9999999999999439, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9855491329479769, |
| "pass@2": 0.9887166605980608, |
| "pass@4": 0.9913939513883019, |
| "pass@8": 0.9943285343504065, |
| "pass@16": 0.9968946491921471, |
| "pass@32": 0.9987241613729214, |
| "pass@64": 0.999808867257577, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9807658450704225, |
| "pass@2": 0.9904752827991573, |
| "pass@4": 0.9946537202215919, |
| "pass@8": 0.9965723900602302, |
| "pass@16": 0.9974438326135088, |
| "pass@32": 0.9978865571381552, |
| "pass@64": 0.9985915492939871, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9550196482035929, |
| "pass@2": 0.9680099692347592, |
| "pass@4": 0.973621248223463, |
| "pass@8": 0.9772931092950905, |
| "pass@16": 0.9797760833989643, |
| "pass@32": 0.9813793644561303, |
| "pass@64": 0.9820069826621478, |
| "pass@128": 0.9820359281437125 |
| }, |
| "11": { |
| "pass@1": 0.8364551957831325, |
| "pass@2": 0.8805564284697844, |
| "pass@4": 0.9119869102393826, |
| "pass@8": 0.9332457070554272, |
| "pass@16": 0.9464105210220005, |
| "pass@32": 0.9554480120240839, |
| "pass@64": 0.9620750241307124, |
| "pass@128": 0.9668674698795181 |
| }, |
| "12": { |
| "pass@1": 0.7179907915360502, |
| "pass@2": 0.7807096641818678, |
| "pass@4": 0.8237406890557176, |
| "pass@8": 0.8544050832000054, |
| "pass@16": 0.8745406088754614, |
| "pass@32": 0.8870839775741599, |
| "pass@64": 0.8953727964648683, |
| "pass@128": 0.8996865203761756 |
| }, |
| "13": { |
| "pass@1": 0.4339110644257703, |
| "pass@2": 0.5064138352191269, |
| "pass@4": 0.5668354349613859, |
| "pass@8": 0.6208126022401135, |
| "pass@16": 0.6694676564972795, |
| "pass@32": 0.7126055948949221, |
| "pass@64": 0.7509963818526764, |
| "pass@128": 0.7843137254901961 |
| }, |
| "14": { |
| "pass@1": 0.39722679093567254, |
| "pass@2": 0.46368376444720705, |
| "pass@4": 0.5167455819667276, |
| "pass@8": 0.56171751122582, |
| "pass@16": 0.6024369898411556, |
| "pass@32": 0.6426240615167135, |
| "pass@64": 0.6832214593635492, |
| "pass@128": 0.7222222222222222 |
| }, |
| "15": { |
| "pass@1": 0.24644886363636365, |
| "pass@2": 0.32176874552612744, |
| "pass@4": 0.3923603217211486, |
| "pass@8": 0.4581419440373477, |
| "pass@16": 0.5190329686132177, |
| "pass@32": 0.5727951896253346, |
| "pass@64": 0.6198279692563144, |
| "pass@128": 0.6636363636363637 |
| }, |
| "16": { |
| "pass@1": 0.2040820868945869, |
| "pass@2": 0.26635372501514243, |
| "pass@4": 0.3238963121062858, |
| "pass@8": 0.37332193265107516, |
| "pass@16": 0.4134811645828353, |
| "pass@32": 0.4443577173332517, |
| "pass@64": 0.47302414732746867, |
| "pass@128": 0.50997150997151 |
| }, |
| "17": { |
| "pass@1": 0.19650135869565216, |
| "pass@2": 0.2574690052330416, |
| "pass@4": 0.3152138539941824, |
| "pass@8": 0.3709270117297045, |
| "pass@16": 0.42297098948583206, |
| "pass@32": 0.4692510903823389, |
| "pass@64": 0.5108174522501921, |
| "pass@128": 0.5527950310559007 |
| }, |
| "18": { |
| "pass@1": 0.18683363970588235, |
| "pass@2": 0.24651024779990735, |
| "pass@4": 0.3035248188645537, |
| "pass@8": 0.3556424980931623, |
| "pass@16": 0.4017419832141523, |
| "pass@32": 0.43944869520791696, |
| "pass@64": 0.4673817173155303, |
| "pass@128": 0.49117647058823527 |
| }, |
| "19": { |
| "pass@1": 0.14808872767857142, |
| "pass@2": 0.2049320250984251, |
| "pass@4": 0.2627956976918065, |
| "pass@8": 0.3139157657154089, |
| "pass@16": 0.3560561707464758, |
| "pass@32": 0.39169649693737485, |
| "pass@64": 0.4226012391904779, |
| "pass@128": 0.44642857142857145 |
| }, |
| "20": { |
| "pass@1": 0.13087107035928144, |
| "pass@2": 0.1879670776557122, |
| "pass@4": 0.24696301351178396, |
| "pass@8": 0.30177241862067616, |
| "pass@16": 0.34838594581032317, |
| "pass@32": 0.38534870948871147, |
| "pass@64": 0.4129125683339026, |
| "pass@128": 0.4341317365269461 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 523941, |
| "answer_accuracy": 0.6539845123022847, |
| "avg_response_len": 275.54656045294774, |
| "resp_tokens_sum": 220754678, |
| "pass_at_k": { |
| "pass@1": 0.6539845123022847, |
| "pass@2": 0.689856626143393, |
| "pass@4": 0.7200727996005184, |
| "pass@8": 0.7453208731590747, |
| "pass@16": 0.7665777414461511, |
| "pass@32": 0.7847634494980638, |
| "pass@64": 0.8007918914069807, |
| "pass@128": 0.8149864195558396 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9029617537313432, |
| "pass@2": 0.9262189299565168, |
| "pass@4": 0.9432951989210303, |
| "pass@8": 0.9559143658871644, |
| "pass@16": 0.9651937507145258, |
| "pass@32": 0.9716208086596739, |
| "pass@64": 0.9761076892137535, |
| "pass@128": 0.9791044776119403 |
| }, |
| "2": { |
| "pass@1": 0.9999107142857143, |
| "pass@2": 0.9999989454443194, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9991116982507289, |
| "pass@2": 0.9997976974816924, |
| "pass@4": 0.9999873259282823, |
| "pass@8": 0.9999999629758403, |
| "pass@16": 0.9999999999999313, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999314692982456, |
| "pass@2": 0.9999996402587834, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9659926470588235, |
| "pass@2": 0.9708096919870312, |
| "pass@4": 0.9745823887822845, |
| "pass@8": 0.9771404920721685, |
| "pass@16": 0.9789554066534717, |
| "pass@32": 0.9809376766950135, |
| "pass@64": 0.9837387679474603, |
| "pass@128": 0.9882352941176471 |
| }, |
| "6": { |
| "pass@1": 0.9929268533123028, |
| "pass@2": 0.9952720199210117, |
| "pass@4": 0.9963477722816194, |
| "pass@8": 0.996781255357137, |
| "pass@16": 0.996844514797445, |
| "pass@32": 0.9968454258207375, |
| "pass@64": 0.9968454258675079, |
| "pass@128": 0.9968454258675079 |
| }, |
| "7": { |
| "pass@1": 0.9869956487341772, |
| "pass@2": 0.9934614210605002, |
| "pass@4": 0.9959656588614716, |
| "pass@8": 0.9971545915353225, |
| "pass@16": 0.9978742203032149, |
| "pass@32": 0.9986754832496939, |
| "pass@64": 0.9996137745440047, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9711195570570571, |
| "pass@2": 0.9796935222624201, |
| "pass@4": 0.9847409628413564, |
| "pass@8": 0.9876525069319597, |
| "pass@16": 0.9891184230382688, |
| "pass@32": 0.9900539597715055, |
| "pass@64": 0.9908121349376716, |
| "pass@128": 0.990990990990991 |
| }, |
| "9": { |
| "pass@1": 0.9665393890675241, |
| "pass@2": 0.9782261943945113, |
| "pass@4": 0.9827812531271213, |
| "pass@8": 0.9852322346734745, |
| "pass@16": 0.9870234208375871, |
| "pass@32": 0.9888026081069605, |
| "pass@64": 0.9901177403407886, |
| "pass@128": 0.9903536977491961 |
| }, |
| "11": { |
| "pass@1": 0.8699652777777778, |
| "pass@2": 0.9050528840144983, |
| "pass@4": 0.9261437665529902, |
| "pass@8": 0.9379237916730481, |
| "pass@16": 0.9436162837580702, |
| "pass@32": 0.9481522394334617, |
| "pass@64": 0.9538037350418224, |
| "pass@128": 0.9587301587301588 |
| }, |
| "12": { |
| "pass@1": 0.6851586391437309, |
| "pass@2": 0.7609573280117508, |
| "pass@4": 0.8146087567609088, |
| "pass@8": 0.8498854256731503, |
| "pass@16": 0.8726716568222734, |
| "pass@32": 0.8890453884635856, |
| "pass@64": 0.9038027299053867, |
| "pass@128": 0.9204892966360856 |
| }, |
| "13": { |
| "pass@1": 0.48818479938271603, |
| "pass@2": 0.5643306630941965, |
| "pass@4": 0.6234116620839059, |
| "pass@8": 0.670892987256112, |
| "pass@16": 0.710554304400132, |
| "pass@32": 0.7441477272144019, |
| "pass@64": 0.7721765126609678, |
| "pass@128": 0.7932098765432098 |
| }, |
| "14": { |
| "pass@1": 0.39139002225519287, |
| "pass@2": 0.4689131901446295, |
| "pass@4": 0.5358667138974095, |
| "pass@8": 0.5899477624146936, |
| "pass@16": 0.6321371774697618, |
| "pass@32": 0.6666508816034391, |
| "pass@64": 0.697774235354575, |
| "pass@128": 0.7270029673590505 |
| }, |
| "15": { |
| "pass@1": 0.29598283179012347, |
| "pass@2": 0.3657031477836101, |
| "pass@4": 0.4284062113763555, |
| "pass@8": 0.488394802453985, |
| "pass@16": 0.5458289659825917, |
| "pass@32": 0.5968433317356244, |
| "pass@64": 0.6421028185366104, |
| "pass@128": 0.6820987654320988 |
| }, |
| "16": { |
| "pass@1": 0.20973912241887904, |
| "pass@2": 0.2661744680974612, |
| "pass@4": 0.32020332027080706, |
| "pass@8": 0.37020648522530164, |
| "pass@16": 0.4155200084120026, |
| "pass@32": 0.4563823340406111, |
| "pass@64": 0.4954197121915054, |
| "pass@128": 0.5339233038348082 |
| }, |
| "17": { |
| "pass@1": 0.1867940512048193, |
| "pass@2": 0.2508834550801631, |
| "pass@4": 0.3094012797534344, |
| "pass@8": 0.3578539351904199, |
| "pass@16": 0.3979111457071782, |
| "pass@32": 0.43246884560924453, |
| "pass@64": 0.46455603309265586, |
| "pass@128": 0.49698795180722893 |
| }, |
| "18": { |
| "pass@1": 0.17842741935483872, |
| "pass@2": 0.2330915036830072, |
| "pass@4": 0.286078386068677, |
| "pass@8": 0.3361083917487688, |
| "pass@16": 0.3821960914627827, |
| "pass@32": 0.422830010224237, |
| "pass@64": 0.45738747568942023, |
| "pass@128": 0.4870967741935484 |
| }, |
| "19": { |
| "pass@1": 0.1512698262839879, |
| "pass@2": 0.20847594083783336, |
| "pass@4": 0.2632785100314122, |
| "pass@8": 0.31349632397118765, |
| "pass@16": 0.3602084544258579, |
| "pass@32": 0.40216106105046995, |
| "pass@64": 0.4354416751759285, |
| "pass@128": 0.459214501510574 |
| }, |
| "20": { |
| "pass@1": 0.16378096846846846, |
| "pass@2": 0.22388222375919223, |
| "pass@4": 0.2818586584334616, |
| "pass@8": 0.3335726763870422, |
| "pass@16": 0.37771145894953373, |
| "pass@32": 0.41447893995579, |
| "pass@64": 0.44587055010516763, |
| "pass@128": 0.47147147147147145 |
| } |
| } |
| } |
| } |
| } |
| ] |
