| [ |
| { |
| "checkpoint": "checkpoint-21808", |
| "id_acc": 0.890703125, |
| "ood_acc": 0.30851953125, |
| "total_acc": 0.5842907072368421, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.43872916666666, |
| "ood_avg_resp_len": 328.6832890625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 467039, |
| "answer_accuracy": 0.5767850438665824, |
| "avg_response_len": 254.83193245139108, |
| "resp_tokens_sum": 206344551, |
| "pass_at_k": { |
| "pass@1": 0.5767850438665824, |
| "pass@2": 0.6287842201040128, |
| "pass@4": 0.6708340107650578, |
| "pass@8": 0.7045834515269525, |
| "pass@16": 0.7321452790267492, |
| "pass@32": 0.7549469874285673, |
| "pass@64": 0.7737805410052538, |
| "pass@128": 0.7892823269048371 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8082595870206489, |
| "pass@2": 0.8647357617355352, |
| "pass@4": 0.8976401423273417, |
| "pass@8": 0.9173720749762039, |
| "pass@16": 0.9291881918718146, |
| "pass@32": 0.9361090250317172, |
| "pass@64": 0.9411341859417028, |
| "pass@128": 0.943952802359882 |
| }, |
| "2": { |
| "pass@1": 0.9132593457943925, |
| "pass@2": 0.9192192827532073, |
| "pass@4": 0.9254616957926987, |
| "pass@8": 0.9326086884253552, |
| "pass@16": 0.9412402480297669, |
| "pass@32": 0.9502751094866533, |
| "pass@64": 0.9567691395661726, |
| "pass@128": 0.9595015576323987 |
| }, |
| "3": { |
| "pass@1": 0.9742689220183486, |
| "pass@2": 0.9795918213778326, |
| "pass@4": 0.9831441336461383, |
| "pass@8": 0.985580114929864, |
| "pass@16": 0.9875498957766747, |
| "pass@32": 0.9897440229097163, |
| "pass@64": 0.9922786486226282, |
| "pass@128": 0.9938837920489296 |
| }, |
| "4": { |
| "pass@1": 0.9580322265625, |
| "pass@2": 0.9734970933809055, |
| "pass@4": 0.983606354869704, |
| "pass@8": 0.9907801440498372, |
| "pass@16": 0.9961476900698585, |
| "pass@32": 0.9991019833663184, |
| "pass@64": 0.9999517511298771, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9450892857142857, |
| "pass@2": 0.9643052430946132, |
| "pass@4": 0.9760748763547413, |
| "pass@8": 0.9826861265629934, |
| "pass@16": 0.9857626408128157, |
| "pass@32": 0.9875250624374391, |
| "pass@64": 0.9888449939948887, |
| "pass@128": 0.9904761904761905 |
| }, |
| "6": { |
| "pass@1": 0.8729883982035929, |
| "pass@2": 0.9114836272337217, |
| "pass@4": 0.9363393078859157, |
| "pass@8": 0.9538367547277492, |
| "pass@16": 0.966520089086936, |
| "pass@32": 0.9746123891484606, |
| "pass@64": 0.979363177663249, |
| "pass@128": 0.9820359281437125 |
| }, |
| "7": { |
| "pass@1": 0.8500600961538461, |
| "pass@2": 0.8949580382518755, |
| "pass@4": 0.9245661620330001, |
| "pass@8": 0.9468657824828772, |
| "pass@16": 0.9647895774794542, |
| "pass@32": 0.977670264977721, |
| "pass@64": 0.9850630781424923, |
| "pass@128": 0.9881656804733728 |
| }, |
| "8": { |
| "pass@1": 0.8358122996794872, |
| "pass@2": 0.8931357100494657, |
| "pass@4": 0.9322278360036731, |
| "pass@8": 0.9569910688945663, |
| "pass@16": 0.9718355835350385, |
| "pass@32": 0.9799294253747101, |
| "pass@64": 0.9839658835248849, |
| "pass@128": 0.9871794871794872 |
| }, |
| "9": { |
| "pass@1": 0.7866857394366197, |
| "pass@2": 0.8491883386935789, |
| "pass@4": 0.8889232061661304, |
| "pass@8": 0.9148338855375537, |
| "pass@16": 0.9351348435701646, |
| "pass@32": 0.9531606564413371, |
| "pass@64": 0.9678114627207628, |
| "pass@128": 0.9746478873239437 |
| }, |
| "11": { |
| "pass@1": 0.7324938031161473, |
| "pass@2": 0.8083786191474651, |
| "pass@4": 0.8603668985427808, |
| "pass@8": 0.8951230630417605, |
| "pass@16": 0.9192438398166457, |
| "pass@32": 0.9382015773061856, |
| "pass@64": 0.9543511427116407, |
| "pass@128": 0.9688385269121813 |
| }, |
| "12": { |
| "pass@1": 0.5774629237288136, |
| "pass@2": 0.6690998404066018, |
| "pass@4": 0.7407094531615751, |
| "pass@8": 0.7936093234416746, |
| "pass@16": 0.8316808174882109, |
| "pass@32": 0.8604117124933197, |
| "pass@64": 0.8853741576695201, |
| "pass@128": 0.9124293785310734 |
| }, |
| "13": { |
| "pass@1": 0.37955525078369906, |
| "pass@2": 0.4627638813467281, |
| "pass@4": 0.5334049649553994, |
| "pass@8": 0.5908909515982965, |
| "pass@16": 0.6383243449452396, |
| "pass@32": 0.6785025472136269, |
| "pass@64": 0.7141232799137383, |
| "pass@128": 0.7492163009404389 |
| }, |
| "14": { |
| "pass@1": 0.3085572429906542, |
| "pass@2": 0.3798913643388032, |
| "pass@4": 0.4410535917239317, |
| "pass@8": 0.4929612888049262, |
| "pass@16": 0.5385958716440408, |
| "pass@32": 0.5820172888780842, |
| "pass@64": 0.6215417900208923, |
| "pass@128": 0.6510903426791277 |
| }, |
| "15": { |
| "pass@1": 0.2654895231213873, |
| "pass@2": 0.33615796106230944, |
| "pass@4": 0.3985344978553981, |
| "pass@8": 0.4518694766045028, |
| "pass@16": 0.4984802935633324, |
| "pass@32": 0.5388247206741961, |
| "pass@64": 0.5739738417672182, |
| "pass@128": 0.6098265895953757 |
| }, |
| "16": { |
| "pass@1": 0.17903225806451614, |
| "pass@2": 0.23331375412750824, |
| "pass@4": 0.2866873023936522, |
| "pass@8": 0.3375684152141981, |
| "pass@16": 0.3862721919687068, |
| "pass@32": 0.43088065063116227, |
| "pass@64": 0.4678904281213043, |
| "pass@128": 0.5 |
| }, |
| "17": { |
| "pass@1": 0.16988800578034682, |
| "pass@2": 0.22384549519821573, |
| "pass@4": 0.2748313011632217, |
| "pass@8": 0.3193916584849494, |
| "pass@16": 0.358615204863562, |
| "pass@32": 0.39272698262401673, |
| "pass@64": 0.4211641810459257, |
| "pass@128": 0.44508670520231214 |
| }, |
| "18": { |
| "pass@1": 0.15439732142857143, |
| "pass@2": 0.21321674634420695, |
| "pass@4": 0.2686094804220902, |
| "pass@8": 0.3154271003708264, |
| "pass@16": 0.3563084554457416, |
| "pass@32": 0.3923554436588413, |
| "pass@64": 0.4227785903208911, |
| "pass@128": 0.44571428571428573 |
| }, |
| "19": { |
| "pass@1": 0.14902402402402404, |
| "pass@2": 0.20243558026530464, |
| "pass@4": 0.25758560106788453, |
| "pass@8": 0.3117411757269937, |
| "pass@16": 0.3584763615622298, |
| "pass@32": 0.3972802657507109, |
| "pass@64": 0.43147037921139075, |
| "pass@128": 0.4594594594594595 |
| }, |
| "20": { |
| "pass@1": 0.14071884384384384, |
| "pass@2": 0.20093001466032961, |
| "pass@4": 0.26321763270582166, |
| "pass@8": 0.32019753114376637, |
| "pass@16": 0.367272720530513, |
| "pass@32": 0.4034641339305123, |
| "pass@64": 0.43107760881473256, |
| "pass@128": 0.45045045045045046 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 479150, |
| "answer_accuracy": 0.5835322486360094, |
| "avg_response_len": 265.8948083106002, |
| "resp_tokens_sum": 218331545, |
| "pass_at_k": { |
| "pass@1": 0.5835322486360094, |
| "pass@2": 0.633331217281108, |
| "pass@4": 0.6740935293990435, |
| "pass@8": 0.7072492136270658, |
| "pass@16": 0.7340684890976236, |
| "pass@32": 0.7561529413080658, |
| "pass@64": 0.7747532859097668, |
| "pass@128": 0.7906469212782541 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.793472009202454, |
| "pass@2": 0.8493992590937638, |
| "pass@4": 0.8877754004867485, |
| "pass@8": 0.9141527086586699, |
| "pass@16": 0.9311651079164657, |
| "pass@32": 0.9426145462430192, |
| "pass@64": 0.9516821714118398, |
| "pass@128": 0.9601226993865031 |
| }, |
| "2": { |
| "pass@1": 0.9402783054711246, |
| "pass@2": 0.9459940346552426, |
| "pass@4": 0.9491347127391752, |
| "pass@8": 0.9519259878204085, |
| "pass@16": 0.9553404745234914, |
| "pass@32": 0.9597092653562085, |
| "pass@64": 0.9640150627253267, |
| "pass@128": 0.9665653495440729 |
| }, |
| "3": { |
| "pass@1": 0.9669981060606061, |
| "pass@2": 0.976074847888332, |
| "pass@4": 0.9835954573860082, |
| "pass@8": 0.9887292837477384, |
| "pass@16": 0.990873589188666, |
| "pass@32": 0.9916485785890059, |
| "pass@64": 0.9924242355459135, |
| "pass@128": 0.9939393939393939 |
| }, |
| "4": { |
| "pass@1": 0.965051775147929, |
| "pass@2": 0.9720598384429018, |
| "pass@4": 0.9769207664685405, |
| "pass@8": 0.9815084547831039, |
| "pass@16": 0.9856876197176295, |
| "pass@32": 0.9890133314572516, |
| "pass@64": 0.990732185832648, |
| "pass@128": 0.9911242603550295 |
| }, |
| "5": { |
| "pass@1": 0.9434329710144927, |
| "pass@2": 0.9612539940659597, |
| "pass@4": 0.9702148456986353, |
| "pass@8": 0.9758259545604623, |
| "pass@16": 0.9803524893764493, |
| "pass@32": 0.9842124262842136, |
| "pass@64": 0.988043997616301, |
| "pass@128": 0.991304347826087 |
| }, |
| "6": { |
| "pass@1": 0.8868642550143266, |
| "pass@2": 0.9194193054396137, |
| "pass@4": 0.939174852516644, |
| "pass@8": 0.952673887441346, |
| "pass@16": 0.9630499625273448, |
| "pass@32": 0.9720429085728259, |
| "pass@64": 0.9808653443318179, |
| "pass@128": 0.9885386819484241 |
| }, |
| "7": { |
| "pass@1": 0.8915281791907514, |
| "pass@2": 0.92678630353648, |
| "pass@4": 0.950118883170673, |
| "pass@8": 0.9654538312470585, |
| "pass@16": 0.9751158531352345, |
| "pass@32": 0.9812282655305531, |
| "pass@64": 0.9852157051970267, |
| "pass@128": 0.9884393063583815 |
| }, |
| "8": { |
| "pass@1": 0.8579005281690141, |
| "pass@2": 0.8997251718975272, |
| "pass@4": 0.9258677762110714, |
| "pass@8": 0.9437238727011946, |
| "pass@16": 0.9556631355474996, |
| "pass@32": 0.9640877913679654, |
| "pass@64": 0.9721959531135995, |
| "pass@128": 0.9802816901408451 |
| }, |
| "9": { |
| "pass@1": 0.7888285928143712, |
| "pass@2": 0.851565815219954, |
| "pass@4": 0.8928656674277982, |
| "pass@8": 0.9188092984095537, |
| "pass@16": 0.9350753145380785, |
| "pass@32": 0.9466458701980125, |
| "pass@64": 0.9550391747311027, |
| "pass@128": 0.9610778443113772 |
| }, |
| "11": { |
| "pass@1": 0.7130788780120482, |
| "pass@2": 0.7939841304193148, |
| "pass@4": 0.8469520404452451, |
| "pass@8": 0.8823839807042658, |
| "pass@16": 0.9067006633768472, |
| "pass@32": 0.9217781709792446, |
| "pass@64": 0.9309741583437446, |
| "pass@128": 0.9397590361445783 |
| }, |
| "12": { |
| "pass@1": 0.5889988244514106, |
| "pass@2": 0.6738543029397968, |
| "pass@4": 0.7364737050893712, |
| "pass@8": 0.7833416397619604, |
| "pass@16": 0.8210170887885584, |
| "pass@32": 0.8530612652715264, |
| "pass@64": 0.8769674396793894, |
| "pass@128": 0.8934169278996865 |
| }, |
| "13": { |
| "pass@1": 0.37489058123249297, |
| "pass@2": 0.45928484582809503, |
| "pass@4": 0.5333312377444418, |
| "pass@8": 0.5955567982367712, |
| "pass@16": 0.6468370691536759, |
| "pass@32": 0.6914674094335568, |
| "pass@64": 0.7330451891460972, |
| "pass@128": 0.7675070028011205 |
| }, |
| "14": { |
| "pass@1": 0.3155381944444444, |
| "pass@2": 0.3891212817147854, |
| "pass@4": 0.453302549078295, |
| "pass@8": 0.507271840880927, |
| "pass@16": 0.5527430887420266, |
| "pass@32": 0.5921199769806936, |
| "pass@64": 0.6270367990392346, |
| "pass@128": 0.6578947368421053 |
| }, |
| "15": { |
| "pass@1": 0.2109848484848485, |
| "pass@2": 0.27266687544738727, |
| "pass@4": 0.3313393129836044, |
| "pass@8": 0.3861669387313994, |
| "pass@16": 0.43703563287263564, |
| "pass@32": 0.4823231852380329, |
| "pass@64": 0.5239700082246377, |
| "pass@128": 0.5666666666666667 |
| }, |
| "16": { |
| "pass@1": 0.1877003205128205, |
| "pass@2": 0.2486519112995491, |
| "pass@4": 0.30775032955282305, |
| "pass@8": 0.3612737419871499, |
| "pass@16": 0.40587271912690925, |
| "pass@32": 0.43920306525713915, |
| "pass@64": 0.4626650339424816, |
| "pass@128": 0.48148148148148145 |
| }, |
| "17": { |
| "pass@1": 0.19596758540372672, |
| "pass@2": 0.25669413605907954, |
| "pass@4": 0.31698872724372196, |
| "pass@8": 0.37355683740742207, |
| "pass@16": 0.4234816279489652, |
| "pass@32": 0.46557943249256295, |
| "pass@64": 0.4995003793338017, |
| "pass@128": 0.5279503105590062 |
| }, |
| "18": { |
| "pass@1": 0.17325367647058823, |
| "pass@2": 0.23055631947660954, |
| "pass@4": 0.28691845293235396, |
| "pass@8": 0.3373942841337351, |
| "pass@16": 0.38137141561201654, |
| "pass@32": 0.4221056033676291, |
| "pass@64": 0.45916635273358053, |
| "pass@128": 0.48823529411764705 |
| }, |
| "19": { |
| "pass@1": 0.15171595982142858, |
| "pass@2": 0.20929048907949008, |
| "pass@4": 0.263858470258182, |
| "pass@8": 0.31035082337712583, |
| "pass@16": 0.3495046566647495, |
| "pass@32": 0.3829455856819846, |
| "pass@64": 0.41029909549921423, |
| "pass@128": 0.43154761904761907 |
| }, |
| "20": { |
| "pass@1": 0.13075411676646706, |
| "pass@2": 0.18799286269979726, |
| "pass@4": 0.24704597469852202, |
| "pass@8": 0.30007091187136975, |
| "pass@16": 0.34343731208913214, |
| "pass@32": 0.37867318149442, |
| "pass@64": 0.41008857357648265, |
| "pass@128": 0.44011976047904194 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 474806, |
| "answer_accuracy": 0.5926540781274964, |
| "avg_response_len": 283.2720008188209, |
| "resp_tokens_sum": 226943930, |
| "pass_at_k": { |
| "pass@1": 0.5926540781274964, |
| "pass@2": 0.6393053844982961, |
| "pass@4": 0.6776084430021845, |
| "pass@8": 0.7091740146867663, |
| "pass@16": 0.7353773789381717, |
| "pass@32": 0.7576234233134128, |
| "pass@64": 0.7773517089235912, |
| "pass@128": 0.7961335676625659 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8008861940298507, |
| "pass@2": 0.8529277823481023, |
| "pass@4": 0.8856773796932103, |
| "pass@8": 0.9071853613797309, |
| "pass@16": 0.922739041071155, |
| "pass@32": 0.93425539456196, |
| "pass@64": 0.9436669840253802, |
| "pass@128": 0.9522388059701492 |
| }, |
| "2": { |
| "pass@1": 0.9342410714285714, |
| "pass@2": 0.9433903965129361, |
| "pass@4": 0.9506338338957628, |
| "pass@8": 0.9568800274460122, |
| "pass@16": 0.9613865780552503, |
| "pass@32": 0.964584323870877, |
| "pass@64": 0.968582583102937, |
| "pass@128": 0.9742857142857143 |
| }, |
| "3": { |
| "pass@1": 0.978156887755102, |
| "pass@2": 0.9854187949082893, |
| "pass@4": 0.9897041988738288, |
| "pass@8": 0.9925314831336094, |
| "pass@16": 0.9949651689420985, |
| "pass@32": 0.9969385354552436, |
| "pass@64": 0.9984540418035831, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9693667763157895, |
| "pass@2": 0.9765919987797581, |
| "pass@4": 0.9815825623222536, |
| "pass@8": 0.9844233166382033, |
| "pass@16": 0.9863282582142694, |
| "pass@32": 0.9881606478134689, |
| "pass@64": 0.9896775917125719, |
| "pass@128": 0.9912280701754386 |
| }, |
| "5": { |
| "pass@1": 0.9251608455882353, |
| "pass@2": 0.9427270292959704, |
| "pass@4": 0.9546608984538691, |
| "pass@8": 0.9638809428859377, |
| "pass@16": 0.9718929334226678, |
| "pass@32": 0.9793735683916001, |
| "pass@64": 0.9842883511825552, |
| "pass@128": 0.9852941176470589 |
| }, |
| "6": { |
| "pass@1": 0.9004337539432177, |
| "pass@2": 0.9270256340197225, |
| "pass@4": 0.9445922334432167, |
| "pass@8": 0.9583119300991347, |
| "pass@16": 0.9698035441893214, |
| "pass@32": 0.9800977802382987, |
| "pass@64": 0.98842612091269, |
| "pass@128": 0.9936908517350158 |
| }, |
| "7": { |
| "pass@1": 0.8817246835443038, |
| "pass@2": 0.9180894454549986, |
| "pass@4": 0.9400556796500119, |
| "pass@8": 0.954207112641725, |
| "pass@16": 0.9646399152221633, |
| "pass@32": 0.9730050298001863, |
| "pass@64": 0.980426538061284, |
| "pass@128": 0.9873417721518988 |
| }, |
| "8": { |
| "pass@1": 0.8465183933933934, |
| "pass@2": 0.8873936682745736, |
| "pass@4": 0.9118258303297669, |
| "pass@8": 0.927138893708802, |
| "pass@16": 0.9391865911461822, |
| "pass@32": 0.9500528993936376, |
| "pass@64": 0.9598739626007716, |
| "pass@128": 0.9669669669669669 |
| }, |
| "9": { |
| "pass@1": 0.8351587620578779, |
| "pass@2": 0.8831090019495152, |
| "pass@4": 0.9114603200725307, |
| "pass@8": 0.9301541604057866, |
| "pass@16": 0.9436524287358187, |
| "pass@32": 0.9539569362661837, |
| "pass@64": 0.9625513320792103, |
| "pass@128": 0.9710610932475884 |
| }, |
| "11": { |
| "pass@1": 0.7725198412698413, |
| "pass@2": 0.8347190976127983, |
| "pass@4": 0.873448953107052, |
| "pass@8": 0.8999402716418193, |
| "pass@16": 0.921179742389368, |
| "pass@32": 0.9396846498700138, |
| "pass@64": 0.9556165664314662, |
| "pass@128": 0.9682539682539683 |
| }, |
| "12": { |
| "pass@1": 0.5756402905198776, |
| "pass@2": 0.660415387440102, |
| "pass@4": 0.7272169780382953, |
| "pass@8": 0.7786842246092575, |
| "pass@16": 0.8154482412117705, |
| "pass@32": 0.8406595602173778, |
| "pass@64": 0.8608165987491957, |
| "pass@128": 0.8807339449541285 |
| }, |
| "13": { |
| "pass@1": 0.39448302469135804, |
| "pass@2": 0.47080356274910107, |
| "pass@4": 0.5377397492558798, |
| "pass@8": 0.59349137895756, |
| "pass@16": 0.6390058492431547, |
| "pass@32": 0.6799437836098674, |
| "pass@64": 0.7212962307025573, |
| "pass@128": 0.7654320987654321 |
| }, |
| "14": { |
| "pass@1": 0.3428922477744807, |
| "pass@2": 0.4210904460384586, |
| "pass@4": 0.4858289081631857, |
| "pass@8": 0.5362440374450701, |
| "pass@16": 0.5772242927210018, |
| "pass@32": 0.6131834924204059, |
| "pass@64": 0.6455434319815009, |
| "pass@128": 0.6765578635014837 |
| }, |
| "15": { |
| "pass@1": 0.27056809413580246, |
| "pass@2": 0.34628619556236, |
| "pass@4": 0.4140252838765522, |
| "pass@8": 0.4736736812919801, |
| "pass@16": 0.5259024276983834, |
| "pass@32": 0.5715661180455387, |
| "pass@64": 0.6127073029932402, |
| "pass@128": 0.6512345679012346 |
| }, |
| "16": { |
| "pass@1": 0.1816233407079646, |
| "pass@2": 0.23493717046431134, |
| "pass@4": 0.28664469540864906, |
| "pass@8": 0.3319588231575276, |
| "pass@16": 0.3707286043155161, |
| "pass@32": 0.40553125034803245, |
| "pass@64": 0.439920634081472, |
| "pass@128": 0.4808259587020649 |
| }, |
| "17": { |
| "pass@1": 0.1729339231927711, |
| "pass@2": 0.2311502407266863, |
| "pass@4": 0.290240824942063, |
| "pass@8": 0.34476655743241874, |
| "pass@16": 0.39019497501102995, |
| "pass@32": 0.4279541230581569, |
| "pass@64": 0.4627532617837625, |
| "pass@128": 0.49698795180722893 |
| }, |
| "18": { |
| "pass@1": 0.15579637096774193, |
| "pass@2": 0.20977306642113286, |
| "pass@4": 0.26256573744814155, |
| "pass@8": 0.310313267504495, |
| "pass@16": 0.3522023621260945, |
| "pass@32": 0.3894700390706386, |
| "pass@64": 0.4218560462182471, |
| "pass@128": 0.45161290322580644 |
| }, |
| "19": { |
| "pass@1": 0.14449584592145015, |
| "pass@2": 0.19614825927159402, |
| "pass@4": 0.24853690541325835, |
| "pass@8": 0.2985191260368612, |
| "pass@16": 0.34577644795107754, |
| "pass@32": 0.3880271665126702, |
| "pass@64": 0.4212440414798256, |
| "pass@128": 0.4471299093655589 |
| }, |
| "20": { |
| "pass@1": 0.1475929054054054, |
| "pass@2": 0.2010962734387931, |
| "pass@4": 0.2587126730217282, |
| "pass@8": 0.31578019467143653, |
| "pass@16": 0.3662754203197754, |
| "pass@32": 0.40700090737768396, |
| "pass@64": 0.4424362537448303, |
| "pass@128": 0.4774774774774775 |
| } |
| } |
| } |
| } |
| } |
| ] |
