| [ |
| { |
| "checkpoint": "checkpoint-21414", |
| "id_acc": 0.4404314236111111, |
| "ood_acc": 0.11799453125, |
| "total_acc": 0.2707277960526316, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 205.83686111111112, |
| "ood_avg_resp_len": 302.30122265625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 216453, |
| "answer_accuracy": 0.267315691195068, |
| "avg_response_len": 240.56482176730952, |
| "resp_tokens_sum": 194792072, |
| "pass_at_k": { |
| "pass@1": 0.267315691195068, |
| "pass@2": 0.3575757567506737, |
| "pass@4": 0.46355980296468213, |
| "pass@8": 0.5718013169071188, |
| "pass@16": 0.666415300283059, |
| "pass@32": 0.7380510280672923, |
| "pass@64": 0.7880629218987633, |
| "pass@128": 0.8234271261460638 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.18249907817109146, |
| "pass@2": 0.291669932989571, |
| "pass@4": 0.4283879288319047, |
| "pass@8": 0.5773130882437786, |
| "pass@16": 0.7165110679120683, |
| "pass@32": 0.8250908597517446, |
| "pass@64": 0.897636864211352, |
| "pass@128": 0.9410029498525073 |
| }, |
| "2": { |
| "pass@1": 0.8387850467289719, |
| "pass@2": 0.8770620232050428, |
| "pass@4": 0.9158330404377023, |
| "pass@8": 0.9489748592508037, |
| "pass@16": 0.9716519179891818, |
| "pass@32": 0.9853529444400086, |
| "pass@64": 0.9937954667041277, |
| "pass@128": 0.9968847352024922 |
| }, |
| "3": { |
| "pass@1": 0.8111381880733946, |
| "pass@2": 0.8610955597293457, |
| "pass@4": 0.9098772533020528, |
| "pass@8": 0.9511167971847219, |
| "pass@16": 0.976515556461128, |
| "pass@32": 0.9883934257827562, |
| "pass@64": 0.9953847266103103, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.669482421875, |
| "pass@2": 0.7651678610974408, |
| "pass@4": 0.8457666476260781, |
| "pass@8": 0.9082670737372126, |
| "pass@16": 0.9486523603161372, |
| "pass@32": 0.9710074419973613, |
| "pass@64": 0.9834710705171064, |
| "pass@128": 0.990625 |
| }, |
| "5": { |
| "pass@1": 0.3938988095238095, |
| "pass@2": 0.5420541182352206, |
| "pass@4": 0.6801027576910021, |
| "pass@8": 0.7897984276173151, |
| "pass@16": 0.8690475384495403, |
| "pass@32": 0.9209395251145625, |
| "pass@64": 0.94971396942988, |
| "pass@128": 0.9650793650793651 |
| }, |
| "6": { |
| "pass@1": 0.37394741766467066, |
| "pass@2": 0.5273726660851524, |
| "pass@4": 0.6815430298009147, |
| "pass@8": 0.8069410782443649, |
| "pass@16": 0.8908290146104205, |
| "pass@32": 0.9401709226432646, |
| "pass@64": 0.9657979861571938, |
| "pass@128": 0.9760479041916168 |
| }, |
| "7": { |
| "pass@1": 0.2760724852071006, |
| "pass@2": 0.4113903869449749, |
| "pass@4": 0.5637461165172402, |
| "pass@8": 0.7078811006530169, |
| "pass@16": 0.8210646809443971, |
| "pass@32": 0.8946519991336546, |
| "pass@64": 0.935935567293088, |
| "pass@128": 0.9615384615384616 |
| }, |
| "8": { |
| "pass@1": 0.2428886217948718, |
| "pass@2": 0.3690830525691498, |
| "pass@4": 0.5166896193264306, |
| "pass@8": 0.6660882532949959, |
| "pass@16": 0.7933717016068363, |
| "pass@32": 0.8817371805976045, |
| "pass@64": 0.9338947606912962, |
| "pass@128": 0.9615384615384616 |
| }, |
| "9": { |
| "pass@1": 0.18243838028169015, |
| "pass@2": 0.29084299101696737, |
| "pass@4": 0.42719150599132877, |
| "pass@8": 0.573221389556622, |
| "pass@16": 0.704905709039916, |
| "pass@32": 0.8052016690957006, |
| "pass@64": 0.8691051191103008, |
| "pass@128": 0.9014084507042254 |
| }, |
| "11": { |
| "pass@1": 0.17610216005665721, |
| "pass@2": 0.28148142468381226, |
| "pass@4": 0.4131508514516423, |
| "pass@8": 0.5527875096043693, |
| "pass@16": 0.6789762038953047, |
| "pass@32": 0.7813219444877645, |
| "pass@64": 0.8612559757607764, |
| "pass@128": 0.9263456090651558 |
| }, |
| "12": { |
| "pass@1": 0.1612597104519774, |
| "pass@2": 0.25932328506606156, |
| "pass@4": 0.38093756818533275, |
| "pass@8": 0.5091701504450877, |
| "pass@16": 0.6263580981659973, |
| "pass@32": 0.7227684909326607, |
| "pass@64": 0.7995695799454732, |
| "pass@128": 0.8615819209039548 |
| }, |
| "13": { |
| "pass@1": 0.14929467084639497, |
| "pass@2": 0.2396229296768938, |
| "pass@4": 0.3508439281100834, |
| "pass@8": 0.471524299365065, |
| "pass@16": 0.5856506718640497, |
| "pass@32": 0.681310375631426, |
| "pass@64": 0.7585447150206255, |
| "pass@128": 0.8181818181818182 |
| }, |
| "14": { |
| "pass@1": 0.12139797507788162, |
| "pass@2": 0.20410196973041916, |
| "pass@4": 0.31280950249209494, |
| "pass@8": 0.43270018777666386, |
| "pass@16": 0.5417473391370579, |
| "pass@32": 0.6270024330655262, |
| "pass@64": 0.6935990061074563, |
| "pass@128": 0.7507788161993769 |
| }, |
| "15": { |
| "pass@1": 0.0994400289017341, |
| "pass@2": 0.1643412765691138, |
| "pass@4": 0.25314546393406, |
| "pass@8": 0.35941326573145005, |
| "pass@16": 0.46831664641616344, |
| "pass@32": 0.5659098593860116, |
| "pass@64": 0.6472013520122053, |
| "pass@128": 0.7109826589595376 |
| }, |
| "16": { |
| "pass@1": 0.11169354838709677, |
| "pass@2": 0.1847190913131826, |
| "pass@4": 0.2792126208014322, |
| "pass@8": 0.38286478587916734, |
| "pass@16": 0.4783240254920387, |
| "pass@32": 0.555416027111267, |
| "pass@64": 0.6126382179445582, |
| "pass@128": 0.6612903225806451 |
| }, |
| "17": { |
| "pass@1": 0.10255599710982659, |
| "pass@2": 0.16548056244595147, |
| "pass@4": 0.24563900628750884, |
| "pass@8": 0.33297676668484133, |
| "pass@16": 0.41509483243131545, |
| "pass@32": 0.48291606901432327, |
| "pass@64": 0.5347978778878022, |
| "pass@128": 0.5780346820809249 |
| }, |
| "18": { |
| "pass@1": 0.08895089285714286, |
| "pass@2": 0.14654879077615304, |
| "pass@4": 0.22479175826236006, |
| "pass@8": 0.3169439929353406, |
| "pass@16": 0.4069978747646183, |
| "pass@32": 0.47939295882154837, |
| "pass@64": 0.5320287955283105, |
| "pass@128": 0.5771428571428572 |
| }, |
| "19": { |
| "pass@1": 0.08621903153153153, |
| "pass@2": 0.14694209465370892, |
| "pass@4": 0.2293803981146501, |
| "pass@8": 0.3248957284145393, |
| "pass@16": 0.4165254606722457, |
| "pass@32": 0.48743485676917575, |
| "pass@64": 0.5339771828430245, |
| "pass@128": 0.5645645645645646 |
| }, |
| "20": { |
| "pass@1": 0.08230105105105105, |
| "pass@2": 0.13847361140668232, |
| "pass@4": 0.21802641617995944, |
| "pass@8": 0.3131342243867788, |
| "pass@16": 0.4031538237236701, |
| "pass@32": 0.4687000076530978, |
| "pass@64": 0.5080511250655994, |
| "pass@128": 0.5285285285285285 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 225144, |
| "answer_accuracy": 0.2741913484021824, |
| "avg_response_len": 251.82658807482463, |
| "resp_tokens_sum": 206779848, |
| "pass_at_k": { |
| "pass@1": 0.2741913484021824, |
| "pass@2": 0.36577319397819286, |
| "pass@4": 0.4737994723331099, |
| "pass@8": 0.5839717493885198, |
| "pass@16": 0.6790056800475346, |
| "pass@32": 0.7499407345326479, |
| "pass@64": 0.8000514435926025, |
| "pass@128": 0.8374123148869836 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.19200536809815952, |
| "pass@2": 0.3103661960774842, |
| "pass@4": 0.4589069955212655, |
| "pass@8": 0.6160699655217626, |
| "pass@16": 0.7569050484893751, |
| "pass@32": 0.8624709959517797, |
| "pass@64": 0.9267811088053876, |
| "pass@128": 0.9631901840490797 |
| }, |
| "2": { |
| "pass@1": 0.8651453267477204, |
| "pass@2": 0.9047007006438025, |
| "pass@4": 0.9401708911576017, |
| "pass@8": 0.9673511274916078, |
| "pass@16": 0.98366795053333, |
| "pass@32": 0.991711774893075, |
| "pass@64": 0.9951545679051006, |
| "pass@128": 0.9969604863221885 |
| }, |
| "3": { |
| "pass@1": 0.8408854166666667, |
| "pass@2": 0.8882236488904792, |
| "pass@4": 0.932338428435082, |
| "pass@8": 0.9654705362519228, |
| "pass@16": 0.9838244373702028, |
| "pass@32": 0.9930547331033703, |
| "pass@64": 0.9983618153437501, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.6366494082840237, |
| "pass@2": 0.741660066160369, |
| "pass@4": 0.836050945092662, |
| "pass@8": 0.9059677477589775, |
| "pass@16": 0.946855597259587, |
| "pass@32": 0.9696531053645359, |
| "pass@64": 0.9819148269345508, |
| "pass@128": 0.985207100591716 |
| }, |
| "5": { |
| "pass@1": 0.3551177536231884, |
| "pass@2": 0.49381918292822075, |
| "pass@4": 0.6387497155246896, |
| "pass@8": 0.7666466857124686, |
| "pass@16": 0.8599482291029481, |
| "pass@32": 0.9189162017525144, |
| "pass@64": 0.9534625742799305, |
| "pass@128": 0.9739130434782609 |
| }, |
| "6": { |
| "pass@1": 0.35104763610315187, |
| "pass@2": 0.49330306782031896, |
| "pass@4": 0.6415392234280171, |
| "pass@8": 0.775567796619772, |
| "pass@16": 0.8779930569384375, |
| "pass@32": 0.9418969165130714, |
| "pass@64": 0.9735513661788436, |
| "pass@128": 0.9856733524355301 |
| }, |
| "7": { |
| "pass@1": 0.33408598265895956, |
| "pass@2": 0.47440055925993363, |
| "pass@4": 0.6220848735129203, |
| "pass@8": 0.7551767421132735, |
| "pass@16": 0.8544841348917069, |
| "pass@32": 0.9146878947571145, |
| "pass@64": 0.9438021352909666, |
| "pass@128": 0.9566473988439307 |
| }, |
| "8": { |
| "pass@1": 0.25413732394366195, |
| "pass@2": 0.38431192469779313, |
| "pass@4": 0.5397205181218544, |
| "pass@8": 0.6943626581541483, |
| "pass@16": 0.817710695410848, |
| "pass@32": 0.8966859300249802, |
| "pass@64": 0.9428206550894571, |
| "pass@128": 0.971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.20415419161676646, |
| "pass@2": 0.320904450940638, |
| "pass@4": 0.4636983232634841, |
| "pass@8": 0.6121143311826535, |
| "pass@16": 0.7430264595297534, |
| "pass@32": 0.8429938708009328, |
| "pass@64": 0.912625986148803, |
| "pass@128": 0.9580838323353293 |
| }, |
| "11": { |
| "pass@1": 0.15573230421686746, |
| "pass@2": 0.2495582724599183, |
| "pass@4": 0.3680684676275405, |
| "pass@8": 0.49662259086905447, |
| "pass@16": 0.6170162594364489, |
| "pass@32": 0.7200076987482085, |
| "pass@64": 0.8045671207619222, |
| "pass@128": 0.8704819277108434 |
| }, |
| "12": { |
| "pass@1": 0.18047119905956113, |
| "pass@2": 0.27988315478982045, |
| "pass@4": 0.402113326645298, |
| "pass@8": 0.5357618074019428, |
| "pass@16": 0.6619916157296409, |
| "pass@32": 0.7672372529808019, |
| "pass@64": 0.8506168212972275, |
| "pass@128": 0.9153605015673981 |
| }, |
| "13": { |
| "pass@1": 0.1331188725490196, |
| "pass@2": 0.21367296918767498, |
| "pass@4": 0.3160395567675891, |
| "pass@8": 0.4292680673053684, |
| "pass@16": 0.5393455857147501, |
| "pass@32": 0.6380850741657642, |
| "pass@64": 0.7222230960815036, |
| "pass@128": 0.7927170868347339 |
| }, |
| "14": { |
| "pass@1": 0.11794133771929824, |
| "pass@2": 0.1945354589722337, |
| "pass@4": 0.2962274923967836, |
| "pass@8": 0.4098757266916545, |
| "pass@16": 0.5162275945782999, |
| "pass@32": 0.603999749398211, |
| "pass@64": 0.6759269353819736, |
| "pass@128": 0.7426900584795322 |
| }, |
| "15": { |
| "pass@1": 0.1011126893939394, |
| "pass@2": 0.16904602123598192, |
| "pass@4": 0.26009995455113555, |
| "pass@8": 0.36595839148405934, |
| "pass@16": 0.4725821407031842, |
| "pass@32": 0.5677398839532518, |
| "pass@64": 0.6513693664909158, |
| "pass@128": 0.7272727272727273 |
| }, |
| "16": { |
| "pass@1": 0.11291399572649573, |
| "pass@2": 0.18268074062857534, |
| "pass@4": 0.2709819667306543, |
| "pass@8": 0.36832898523240143, |
| "pass@16": 0.46015042128877076, |
| "pass@32": 0.5351319160537661, |
| "pass@64": 0.5952268034436059, |
| "pass@128": 0.6467236467236467 |
| }, |
| "17": { |
| "pass@1": 0.11769701086956522, |
| "pass@2": 0.19582984362009093, |
| "pass@4": 0.2977075225845214, |
| "pass@8": 0.4106816757981006, |
| "pass@16": 0.5135671720839062, |
| "pass@32": 0.5929697650046102, |
| "pass@64": 0.6530400028524697, |
| "pass@128": 0.6987577639751553 |
| }, |
| "18": { |
| "pass@1": 0.09430147058823529, |
| "pass@2": 0.16086729967577587, |
| "pass@4": 0.25293459052912487, |
| "pass@8": 0.36106318527066916, |
| "pass@16": 0.46322122493015433, |
| "pass@32": 0.5405034298643432, |
| "pass@64": 0.5967379412271786, |
| "pass@128": 0.6470588235294118 |
| }, |
| "19": { |
| "pass@1": 0.09419177827380952, |
| "pass@2": 0.15729650004686918, |
| "pass@4": 0.2404237966347958, |
| "pass@8": 0.33286242500490154, |
| "pass@16": 0.4162781216933555, |
| "pass@32": 0.4755985896299961, |
| "pass@64": 0.514052982556868, |
| "pass@128": 0.5476190476190477 |
| }, |
| "20": { |
| "pass@1": 0.08460422904191617, |
| "pass@2": 0.14392253878070635, |
| "pass@4": 0.22773993660597816, |
| "pass@8": 0.32560269641149836, |
| "pass@16": 0.413814026198524, |
| "pass@32": 0.47310524704300333, |
| "pass@64": 0.5072669185320547, |
| "pass@128": 0.5299401197604791 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 216813, |
| "answer_accuracy": 0.2706265477712095, |
| "avg_response_len": 277.72221625858765, |
| "resp_tokens_sum": 222497709, |
| "pass_at_k": { |
| "pass@1": 0.2706265477712095, |
| "pass@2": 0.3627233869841646, |
| "pass@4": 0.4714831141394156, |
| "pass@8": 0.5817573231853167, |
| "pass@16": 0.6751043837320008, |
| "pass@32": 0.7432362479672774, |
| "pass@64": 0.7910686801393282, |
| "pass@128": 0.8269691644032593 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.21438899253731344, |
| "pass@2": 0.33370989540486545, |
| "pass@4": 0.4789243028390111, |
| "pass@8": 0.6295483040575033, |
| "pass@16": 0.7594710983716995, |
| "pass@32": 0.8557769210941149, |
| "pass@64": 0.9216117810953981, |
| "pass@128": 0.9611940298507463 |
| }, |
| "2": { |
| "pass@1": 0.831875, |
| "pass@2": 0.8827921119235095, |
| "pass@4": 0.9307492380416736, |
| "pass@8": 0.9679590861746582, |
| "pass@16": 0.9877240261931111, |
| "pass@32": 0.9945705212997372, |
| "pass@64": 0.9968058462187176, |
| "pass@128": 0.9971428571428571 |
| }, |
| "3": { |
| "pass@1": 0.7918185131195336, |
| "pass@2": 0.8481716013176923, |
| "pass@4": 0.9032584244207081, |
| "pass@8": 0.9486360554236684, |
| "pass@16": 0.9769348854369816, |
| "pass@32": 0.9907001703503954, |
| "pass@64": 0.9959204930452993, |
| "pass@128": 0.9970845481049563 |
| }, |
| "4": { |
| "pass@1": 0.6494883040935673, |
| "pass@2": 0.7507130070912186, |
| "pass@4": 0.8389371421774029, |
| "pass@8": 0.9055928823671676, |
| "pass@16": 0.9468274932773871, |
| "pass@32": 0.968719693628846, |
| "pass@64": 0.9805015058372825, |
| "pass@128": 0.9883040935672515 |
| }, |
| "5": { |
| "pass@1": 0.36792279411764706, |
| "pass@2": 0.5179806912922651, |
| "pass@4": 0.6698513004808214, |
| "pass@8": 0.7977126442729795, |
| "pass@16": 0.8852094688271894, |
| "pass@32": 0.9371548591179986, |
| "pass@64": 0.9642397376671343, |
| "pass@128": 0.9764705882352941 |
| }, |
| "6": { |
| "pass@1": 0.31415122239747634, |
| "pass@2": 0.4550387801485384, |
| "pass@4": 0.607989620185489, |
| "pass@8": 0.7480016157202245, |
| "pass@16": 0.852043421418948, |
| "pass@32": 0.9173663172687372, |
| "pass@64": 0.9547247098960704, |
| "pass@128": 0.9684542586750788 |
| }, |
| "7": { |
| "pass@1": 0.3049841772151899, |
| "pass@2": 0.4434794864447324, |
| "pass@4": 0.597717386790259, |
| "pass@8": 0.743221162557493, |
| "pass@16": 0.8524529336091656, |
| "pass@32": 0.9168331659507437, |
| "pass@64": 0.9491947918924216, |
| "pass@128": 0.9651898734177216 |
| }, |
| "8": { |
| "pass@1": 0.22787631381381382, |
| "pass@2": 0.35784468031023126, |
| "pass@4": 0.513812833643542, |
| "pass@8": 0.6671665510913155, |
| "pass@16": 0.7915539816168905, |
| "pass@32": 0.8796837081297789, |
| "pass@64": 0.9353007567211731, |
| "pass@128": 0.963963963963964 |
| }, |
| "9": { |
| "pass@1": 0.18611836816720256, |
| "pass@2": 0.2989542718434308, |
| "pass@4": 0.4405975697973446, |
| "pass@8": 0.594700003859233, |
| "pass@16": 0.7340836850211218, |
| "pass@32": 0.8371155983990625, |
| "pass@64": 0.9047208145978293, |
| "pass@128": 0.954983922829582 |
| }, |
| "11": { |
| "pass@1": 0.15535714285714286, |
| "pass@2": 0.25387334395700534, |
| "pass@4": 0.3802234271311323, |
| "pass@8": 0.5185778550023629, |
| "pass@16": 0.6463171263078665, |
| "pass@32": 0.7484581374990609, |
| "pass@64": 0.824998693875125, |
| "pass@128": 0.8825396825396825 |
| }, |
| "12": { |
| "pass@1": 0.15498375382262997, |
| "pass@2": 0.24644939379710562, |
| "pass@4": 0.36310674726439013, |
| "pass@8": 0.49138307752199395, |
| "pass@16": 0.6133379352066443, |
| "pass@32": 0.7179926806217004, |
| "pass@64": 0.8042931141328287, |
| "pass@128": 0.8746177370030581 |
| }, |
| "13": { |
| "pass@1": 0.1466772762345679, |
| "pass@2": 0.23146363432001538, |
| "pass@4": 0.33295316991626056, |
| "pass@8": 0.4391792497041196, |
| "pass@16": 0.5405334534430788, |
| "pass@32": 0.6326803374503219, |
| "pass@64": 0.7157104289974534, |
| "pass@128": 0.7901234567901234 |
| }, |
| "14": { |
| "pass@1": 0.12173126854599406, |
| "pass@2": 0.19921208731512416, |
| "pass@4": 0.29809855481714614, |
| "pass@8": 0.4071295133196, |
| "pass@16": 0.5094998867710719, |
| "pass@32": 0.5944799771400907, |
| "pass@64": 0.6675768737310531, |
| "pass@128": 0.7388724035608308 |
| }, |
| "15": { |
| "pass@1": 0.10139371141975309, |
| "pass@2": 0.17266027510450077, |
| "pass@4": 0.2726622945279986, |
| "pass@8": 0.3916678557393639, |
| "pass@16": 0.5068999403589238, |
| "pass@32": 0.5979407415519157, |
| "pass@64": 0.6628955996216994, |
| "pass@128": 0.7098765432098766 |
| }, |
| "16": { |
| "pass@1": 0.11481379056047197, |
| "pass@2": 0.18550572840452464, |
| "pass@4": 0.27434850981016756, |
| "pass@8": 0.3706460035215884, |
| "pass@16": 0.4582708669852493, |
| "pass@32": 0.5277747582628878, |
| "pass@64": 0.588016295160178, |
| "pass@128": 0.6519174041297935 |
| }, |
| "17": { |
| "pass@1": 0.1001976656626506, |
| "pass@2": 0.16855352018309455, |
| "pass@4": 0.2603574520240692, |
| "pass@8": 0.36273649965024324, |
| "pass@16": 0.45346864578811946, |
| "pass@32": 0.518965437902586, |
| "pass@64": 0.5645039379656953, |
| "pass@128": 0.5993975903614458 |
| }, |
| "18": { |
| "pass@1": 0.09596774193548387, |
| "pass@2": 0.16143557912115825, |
| "pass@4": 0.247989123637771, |
| "pass@8": 0.3426376649389472, |
| "pass@16": 0.4246539381310786, |
| "pass@32": 0.4820093556043488, |
| "pass@64": 0.5217290432579591, |
| "pass@128": 0.5580645161290323 |
| }, |
| "19": { |
| "pass@1": 0.09238104229607251, |
| "pass@2": 0.15827202821324066, |
| "pass@4": 0.24653544892991106, |
| "pass@8": 0.3432939744879859, |
| "pass@16": 0.4281201380647039, |
| "pass@32": 0.48765535174226415, |
| "pass@64": 0.5268287561695072, |
| "pass@128": 0.5619335347432024 |
| }, |
| "20": { |
| "pass@1": 0.08650056306306306, |
| "pass@2": 0.14805602846941426, |
| "pass@4": 0.23268120257490316, |
| "pass@8": 0.3296337420615053, |
| "pass@16": 0.41933828064911133, |
| "pass@32": 0.4862638392287856, |
| "pass@64": 0.5288202968457819, |
| "pass@128": 0.5555555555555556 |
| } |
| } |
| } |
| } |
| } |
| ] |
