[ { "checkpoint": "checkpoint-21808", "id_acc": 0.890703125, "ood_acc": 0.30851953125, "total_acc": 0.5842907072368421, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.43872916666666, "ood_avg_resp_len": 328.6832890625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 467039, "answer_accuracy": 0.5767850438665824, "avg_response_len": 254.83193245139108, "resp_tokens_sum": 206344551, "pass_at_k": { "pass@1": 0.5767850438665824, "pass@2": 0.6287842201040128, "pass@4": 0.6708340107650578, "pass@8": 0.7045834515269525, "pass@16": 0.7321452790267492, "pass@32": 0.7549469874285673, "pass@64": 0.7737805410052538, "pass@128": 0.7892823269048371 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8082595870206489, "pass@2": 0.8647357617355352, "pass@4": 0.8976401423273417, "pass@8": 0.9173720749762039, "pass@16": 0.9291881918718146, "pass@32": 0.9361090250317172, "pass@64": 0.9411341859417028, "pass@128": 0.943952802359882 }, "2": { "pass@1": 0.9132593457943925, "pass@2": 0.9192192827532073, "pass@4": 0.9254616957926987, "pass@8": 0.9326086884253552, "pass@16": 0.9412402480297669, "pass@32": 0.9502751094866533, "pass@64": 0.9567691395661726, "pass@128": 0.9595015576323987 }, "3": { "pass@1": 0.9742689220183486, "pass@2": 0.9795918213778326, "pass@4": 0.9831441336461383, "pass@8": 0.985580114929864, "pass@16": 0.9875498957766747, "pass@32": 0.9897440229097163, "pass@64": 0.9922786486226282, "pass@128": 0.9938837920489296 }, "4": { "pass@1": 0.9580322265625, "pass@2": 0.9734970933809055, "pass@4": 0.983606354869704, "pass@8": 0.9907801440498372, "pass@16": 0.9961476900698585, "pass@32": 0.9991019833663184, "pass@64": 0.9999517511298771, "pass@128": 1.0 }, "5": { "pass@1": 0.9450892857142857, "pass@2": 0.9643052430946132, "pass@4": 0.9760748763547413, "pass@8": 0.9826861265629934, "pass@16": 0.9857626408128157, "pass@32": 0.9875250624374391, "pass@64": 0.9888449939948887, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.8729883982035929, "pass@2": 0.9114836272337217, "pass@4": 0.9363393078859157, "pass@8": 0.9538367547277492, "pass@16": 0.966520089086936, "pass@32": 0.9746123891484606, "pass@64": 0.979363177663249, "pass@128": 0.9820359281437125 }, "7": { "pass@1": 0.8500600961538461, "pass@2": 0.8949580382518755, "pass@4": 0.9245661620330001, "pass@8": 0.9468657824828772, "pass@16": 0.9647895774794542, "pass@32": 0.977670264977721, "pass@64": 0.9850630781424923, "pass@128": 0.9881656804733728 }, "8": { "pass@1": 0.8358122996794872, "pass@2": 0.8931357100494657, "pass@4": 0.9322278360036731, "pass@8": 0.9569910688945663, "pass@16": 0.9718355835350385, "pass@32": 0.9799294253747101, "pass@64": 0.9839658835248849, "pass@128": 0.9871794871794872 }, "9": { "pass@1": 0.7866857394366197, "pass@2": 0.8491883386935789, "pass@4": 0.8889232061661304, "pass@8": 0.9148338855375537, "pass@16": 0.9351348435701646, "pass@32": 0.9531606564413371, "pass@64": 0.9678114627207628, "pass@128": 0.9746478873239437 }, "11": { "pass@1": 0.7324938031161473, "pass@2": 0.8083786191474651, "pass@4": 0.8603668985427808, "pass@8": 0.8951230630417605, "pass@16": 0.9192438398166457, "pass@32": 0.9382015773061856, "pass@64": 0.9543511427116407, "pass@128": 0.9688385269121813 }, "12": { "pass@1": 0.5774629237288136, "pass@2": 0.6690998404066018, "pass@4": 0.7407094531615751, "pass@8": 0.7936093234416746, "pass@16": 0.8316808174882109, "pass@32": 0.8604117124933197, "pass@64": 0.8853741576695201, "pass@128": 0.9124293785310734 }, "13": { "pass@1": 0.37955525078369906, "pass@2": 0.4627638813467281, "pass@4": 0.5334049649553994, "pass@8": 0.5908909515982965, "pass@16": 0.6383243449452396, "pass@32": 0.6785025472136269, "pass@64": 0.7141232799137383, "pass@128": 0.7492163009404389 }, "14": { "pass@1": 0.3085572429906542, "pass@2": 0.3798913643388032, "pass@4": 0.4410535917239317, "pass@8": 0.4929612888049262, "pass@16": 0.5385958716440408, "pass@32": 0.5820172888780842, "pass@64": 0.6215417900208923, "pass@128": 0.6510903426791277 }, "15": { "pass@1": 0.2654895231213873, "pass@2": 0.33615796106230944, "pass@4": 0.3985344978553981, "pass@8": 0.4518694766045028, "pass@16": 0.4984802935633324, "pass@32": 0.5388247206741961, "pass@64": 0.5739738417672182, "pass@128": 0.6098265895953757 }, "16": { "pass@1": 0.17903225806451614, "pass@2": 0.23331375412750824, "pass@4": 0.2866873023936522, "pass@8": 0.3375684152141981, "pass@16": 0.3862721919687068, "pass@32": 0.43088065063116227, "pass@64": 0.4678904281213043, "pass@128": 0.5 }, "17": { "pass@1": 0.16988800578034682, "pass@2": 0.22384549519821573, "pass@4": 0.2748313011632217, "pass@8": 0.3193916584849494, "pass@16": 0.358615204863562, "pass@32": 0.39272698262401673, "pass@64": 0.4211641810459257, "pass@128": 0.44508670520231214 }, "18": { "pass@1": 0.15439732142857143, "pass@2": 0.21321674634420695, "pass@4": 0.2686094804220902, "pass@8": 0.3154271003708264, "pass@16": 0.3563084554457416, "pass@32": 0.3923554436588413, "pass@64": 0.4227785903208911, "pass@128": 0.44571428571428573 }, "19": { "pass@1": 0.14902402402402404, "pass@2": 0.20243558026530464, "pass@4": 0.25758560106788453, "pass@8": 0.3117411757269937, "pass@16": 0.3584763615622298, "pass@32": 0.3972802657507109, "pass@64": 0.43147037921139075, "pass@128": 0.4594594594594595 }, "20": { "pass@1": 0.14071884384384384, "pass@2": 0.20093001466032961, "pass@4": 0.26321763270582166, "pass@8": 0.32019753114376637, "pass@16": 0.367272720530513, "pass@32": 0.4034641339305123, "pass@64": 0.43107760881473256, "pass@128": 0.45045045045045046 } } }, "teachers_in_school": { "count": 821120, "correct": 479150, "answer_accuracy": 0.5835322486360094, "avg_response_len": 265.8948083106002, "resp_tokens_sum": 218331545, "pass_at_k": { "pass@1": 0.5835322486360094, "pass@2": 0.633331217281108, "pass@4": 0.6740935293990435, "pass@8": 0.7072492136270658, "pass@16": 0.7340684890976236, "pass@32": 0.7561529413080658, "pass@64": 0.7747532859097668, "pass@128": 0.7906469212782541 }, "per_op_pass_at_k": { "10": { "pass@1": 0.793472009202454, "pass@2": 0.8493992590937638, "pass@4": 0.8877754004867485, "pass@8": 0.9141527086586699, "pass@16": 0.9311651079164657, "pass@32": 0.9426145462430192, "pass@64": 0.9516821714118398, "pass@128": 0.9601226993865031 }, "2": { "pass@1": 0.9402783054711246, "pass@2": 0.9459940346552426, "pass@4": 0.9491347127391752, "pass@8": 0.9519259878204085, "pass@16": 0.9553404745234914, "pass@32": 0.9597092653562085, "pass@64": 0.9640150627253267, "pass@128": 0.9665653495440729 }, "3": { "pass@1": 0.9669981060606061, "pass@2": 0.976074847888332, "pass@4": 0.9835954573860082, "pass@8": 0.9887292837477384, "pass@16": 0.990873589188666, "pass@32": 0.9916485785890059, "pass@64": 0.9924242355459135, "pass@128": 0.9939393939393939 }, "4": { "pass@1": 0.965051775147929, "pass@2": 0.9720598384429018, "pass@4": 0.9769207664685405, "pass@8": 0.9815084547831039, "pass@16": 0.9856876197176295, "pass@32": 0.9890133314572516, "pass@64": 0.990732185832648, "pass@128": 0.9911242603550295 }, "5": { "pass@1": 0.9434329710144927, "pass@2": 0.9612539940659597, "pass@4": 0.9702148456986353, "pass@8": 0.9758259545604623, "pass@16": 0.9803524893764493, "pass@32": 0.9842124262842136, "pass@64": 0.988043997616301, "pass@128": 0.991304347826087 }, "6": { "pass@1": 0.8868642550143266, "pass@2": 0.9194193054396137, "pass@4": 0.939174852516644, "pass@8": 0.952673887441346, "pass@16": 0.9630499625273448, "pass@32": 0.9720429085728259, "pass@64": 0.9808653443318179, "pass@128": 0.9885386819484241 }, "7": { "pass@1": 0.8915281791907514, "pass@2": 0.92678630353648, "pass@4": 0.950118883170673, "pass@8": 0.9654538312470585, "pass@16": 0.9751158531352345, "pass@32": 0.9812282655305531, "pass@64": 0.9852157051970267, "pass@128": 0.9884393063583815 }, "8": { "pass@1": 0.8579005281690141, "pass@2": 0.8997251718975272, "pass@4": 0.9258677762110714, "pass@8": 0.9437238727011946, "pass@16": 0.9556631355474996, "pass@32": 0.9640877913679654, "pass@64": 0.9721959531135995, "pass@128": 0.9802816901408451 }, "9": { "pass@1": 0.7888285928143712, "pass@2": 0.851565815219954, "pass@4": 0.8928656674277982, "pass@8": 0.9188092984095537, "pass@16": 0.9350753145380785, "pass@32": 0.9466458701980125, "pass@64": 0.9550391747311027, "pass@128": 0.9610778443113772 }, "11": { "pass@1": 0.7130788780120482, "pass@2": 0.7939841304193148, "pass@4": 0.8469520404452451, "pass@8": 0.8823839807042658, "pass@16": 0.9067006633768472, "pass@32": 0.9217781709792446, "pass@64": 0.9309741583437446, "pass@128": 0.9397590361445783 }, "12": { "pass@1": 0.5889988244514106, "pass@2": 0.6738543029397968, "pass@4": 0.7364737050893712, "pass@8": 0.7833416397619604, "pass@16": 0.8210170887885584, "pass@32": 0.8530612652715264, "pass@64": 0.8769674396793894, "pass@128": 0.8934169278996865 }, "13": { "pass@1": 0.37489058123249297, "pass@2": 0.45928484582809503, "pass@4": 0.5333312377444418, "pass@8": 0.5955567982367712, "pass@16": 0.6468370691536759, "pass@32": 0.6914674094335568, "pass@64": 0.7330451891460972, "pass@128": 0.7675070028011205 }, "14": { "pass@1": 0.3155381944444444, "pass@2": 0.3891212817147854, "pass@4": 0.453302549078295, "pass@8": 0.507271840880927, "pass@16": 0.5527430887420266, "pass@32": 0.5921199769806936, "pass@64": 0.6270367990392346, "pass@128": 0.6578947368421053 }, "15": { "pass@1": 0.2109848484848485, "pass@2": 0.27266687544738727, "pass@4": 0.3313393129836044, "pass@8": 0.3861669387313994, "pass@16": 0.43703563287263564, "pass@32": 0.4823231852380329, "pass@64": 0.5239700082246377, "pass@128": 0.5666666666666667 }, "16": { "pass@1": 0.1877003205128205, "pass@2": 0.2486519112995491, "pass@4": 0.30775032955282305, "pass@8": 0.3612737419871499, "pass@16": 0.40587271912690925, "pass@32": 0.43920306525713915, "pass@64": 0.4626650339424816, "pass@128": 0.48148148148148145 }, "17": { "pass@1": 0.19596758540372672, "pass@2": 0.25669413605907954, "pass@4": 0.31698872724372196, "pass@8": 0.37355683740742207, "pass@16": 0.4234816279489652, "pass@32": 0.46557943249256295, "pass@64": 0.4995003793338017, "pass@128": 0.5279503105590062 }, "18": { "pass@1": 0.17325367647058823, "pass@2": 0.23055631947660954, "pass@4": 0.28691845293235396, "pass@8": 0.3373942841337351, "pass@16": 0.38137141561201654, "pass@32": 0.4221056033676291, "pass@64": 0.45916635273358053, "pass@128": 0.48823529411764705 }, "19": { "pass@1": 0.15171595982142858, "pass@2": 0.20929048907949008, "pass@4": 0.263858470258182, "pass@8": 0.31035082337712583, "pass@16": 0.3495046566647495, "pass@32": 0.3829455856819846, "pass@64": 0.41029909549921423, "pass@128": 0.43154761904761907 }, "20": { "pass@1": 0.13075411676646706, "pass@2": 0.18799286269979726, "pass@4": 0.24704597469852202, "pass@8": 0.30007091187136975, "pass@16": 0.34343731208913214, "pass@32": 0.37867318149442, "pass@64": 0.41008857357648265, "pass@128": 0.44011976047904194 } } }, "movie_festival_awards": { "count": 801152, "correct": 474806, "answer_accuracy": 0.5926540781274964, "avg_response_len": 283.2720008188209, "resp_tokens_sum": 226943930, "pass_at_k": { "pass@1": 0.5926540781274964, "pass@2": 0.6393053844982961, "pass@4": 0.6776084430021845, "pass@8": 0.7091740146867663, "pass@16": 0.7353773789381717, "pass@32": 0.7576234233134128, "pass@64": 0.7773517089235912, "pass@128": 0.7961335676625659 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8008861940298507, "pass@2": 0.8529277823481023, "pass@4": 0.8856773796932103, "pass@8": 0.9071853613797309, "pass@16": 0.922739041071155, "pass@32": 0.93425539456196, "pass@64": 0.9436669840253802, "pass@128": 0.9522388059701492 }, "2": { "pass@1": 0.9342410714285714, "pass@2": 0.9433903965129361, "pass@4": 0.9506338338957628, "pass@8": 0.9568800274460122, "pass@16": 0.9613865780552503, "pass@32": 0.964584323870877, "pass@64": 0.968582583102937, "pass@128": 0.9742857142857143 }, "3": { "pass@1": 0.978156887755102, "pass@2": 0.9854187949082893, "pass@4": 0.9897041988738288, "pass@8": 0.9925314831336094, "pass@16": 0.9949651689420985, "pass@32": 0.9969385354552436, "pass@64": 0.9984540418035831, "pass@128": 1.0 }, "4": { "pass@1": 0.9693667763157895, "pass@2": 0.9765919987797581, "pass@4": 0.9815825623222536, "pass@8": 0.9844233166382033, "pass@16": 0.9863282582142694, "pass@32": 0.9881606478134689, "pass@64": 0.9896775917125719, "pass@128": 0.9912280701754386 }, "5": { "pass@1": 0.9251608455882353, "pass@2": 0.9427270292959704, "pass@4": 0.9546608984538691, "pass@8": 0.9638809428859377, "pass@16": 0.9718929334226678, "pass@32": 0.9793735683916001, "pass@64": 0.9842883511825552, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.9004337539432177, "pass@2": 0.9270256340197225, "pass@4": 0.9445922334432167, "pass@8": 0.9583119300991347, "pass@16": 0.9698035441893214, "pass@32": 0.9800977802382987, "pass@64": 0.98842612091269, "pass@128": 0.9936908517350158 }, "7": { "pass@1": 0.8817246835443038, "pass@2": 0.9180894454549986, "pass@4": 0.9400556796500119, "pass@8": 0.954207112641725, "pass@16": 0.9646399152221633, "pass@32": 0.9730050298001863, "pass@64": 0.980426538061284, "pass@128": 0.9873417721518988 }, "8": { "pass@1": 0.8465183933933934, "pass@2": 0.8873936682745736, "pass@4": 0.9118258303297669, "pass@8": 0.927138893708802, "pass@16": 0.9391865911461822, "pass@32": 0.9500528993936376, "pass@64": 0.9598739626007716, "pass@128": 0.9669669669669669 }, "9": { "pass@1": 0.8351587620578779, "pass@2": 0.8831090019495152, "pass@4": 0.9114603200725307, "pass@8": 0.9301541604057866, "pass@16": 0.9436524287358187, "pass@32": 0.9539569362661837, "pass@64": 0.9625513320792103, "pass@128": 0.9710610932475884 }, "11": { "pass@1": 0.7725198412698413, "pass@2": 0.8347190976127983, "pass@4": 0.873448953107052, "pass@8": 0.8999402716418193, "pass@16": 0.921179742389368, "pass@32": 0.9396846498700138, "pass@64": 0.9556165664314662, "pass@128": 0.9682539682539683 }, "12": { "pass@1": 0.5756402905198776, "pass@2": 0.660415387440102, "pass@4": 0.7272169780382953, "pass@8": 0.7786842246092575, "pass@16": 0.8154482412117705, "pass@32": 0.8406595602173778, "pass@64": 0.8608165987491957, "pass@128": 0.8807339449541285 }, "13": { "pass@1": 0.39448302469135804, "pass@2": 0.47080356274910107, "pass@4": 0.5377397492558798, "pass@8": 0.59349137895756, "pass@16": 0.6390058492431547, "pass@32": 0.6799437836098674, "pass@64": 0.7212962307025573, "pass@128": 0.7654320987654321 }, "14": { "pass@1": 0.3428922477744807, "pass@2": 0.4210904460384586, "pass@4": 0.4858289081631857, "pass@8": 0.5362440374450701, "pass@16": 0.5772242927210018, "pass@32": 0.6131834924204059, "pass@64": 0.6455434319815009, "pass@128": 0.6765578635014837 }, "15": { "pass@1": 0.27056809413580246, "pass@2": 0.34628619556236, "pass@4": 0.4140252838765522, "pass@8": 0.4736736812919801, "pass@16": 0.5259024276983834, "pass@32": 0.5715661180455387, "pass@64": 0.6127073029932402, "pass@128": 0.6512345679012346 }, "16": { "pass@1": 0.1816233407079646, "pass@2": 0.23493717046431134, "pass@4": 0.28664469540864906, "pass@8": 0.3319588231575276, "pass@16": 0.3707286043155161, "pass@32": 0.40553125034803245, "pass@64": 0.439920634081472, "pass@128": 0.4808259587020649 }, "17": { "pass@1": 0.1729339231927711, "pass@2": 0.2311502407266863, "pass@4": 0.290240824942063, "pass@8": 0.34476655743241874, "pass@16": 0.39019497501102995, "pass@32": 0.4279541230581569, "pass@64": 0.4627532617837625, "pass@128": 0.49698795180722893 }, "18": { "pass@1": 0.15579637096774193, "pass@2": 0.20977306642113286, "pass@4": 0.26256573744814155, "pass@8": 0.310313267504495, "pass@16": 0.3522023621260945, "pass@32": 0.3894700390706386, "pass@64": 0.4218560462182471, "pass@128": 0.45161290322580644 }, "19": { "pass@1": 0.14449584592145015, "pass@2": 0.19614825927159402, "pass@4": 0.24853690541325835, "pass@8": 0.2985191260368612, "pass@16": 0.34577644795107754, "pass@32": 0.3880271665126702, "pass@64": 0.4212440414798256, "pass@128": 0.4471299093655589 }, "20": { "pass@1": 0.1475929054054054, "pass@2": 0.2010962734387931, "pass@4": 0.2587126730217282, "pass@8": 0.31578019467143653, "pass@16": 0.3662754203197754, "pass@32": 0.40700090737768396, "pass@64": 0.4424362537448303, "pass@128": 0.4774774774774775 } } } } } ]