| [ |
| { |
| "checkpoint": "checkpoint-21414", |
| "id_acc": 0.5163628472222223, |
| "ood_acc": 0.14568359375, |
| "total_acc": 0.3212685032894737, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.46446354166667, |
| "ood_avg_resp_len": 302.6413921875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 256705, |
| "answer_accuracy": 0.3170262112709453, |
| "avg_response_len": 240.0941365495574, |
| "resp_tokens_sum": 194410945, |
| "pass_at_k": { |
| "pass@1": 0.3170262112709453, |
| "pass@2": 0.41333096242603934, |
| "pass@4": 0.5188438225967936, |
| "pass@8": 0.6212990365018248, |
| "pass@16": 0.7097565650424552, |
| "pass@32": 0.7780028734739614, |
| "pass@64": 0.8260987824256207, |
| "pass@128": 0.8594688586784698 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.25410213864306785, |
| "pass@2": 0.38541848129050216, |
| "pass@4": 0.5397663588511608, |
| "pass@8": 0.6911220142903763, |
| "pass@16": 0.8144716731548343, |
| "pass@32": 0.9007734251817849, |
| "pass@64": 0.9547486096517519, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 0.9106795171339563, |
| "pass@2": 0.9424638034439621, |
| "pass@4": 0.9664033990494175, |
| "pass@8": 0.9818446668613697, |
| "pass@16": 0.9903789463115695, |
| "pass@32": 0.994020154983159, |
| "pass@64": 0.9959263337220218, |
| "pass@128": 0.9968847352024922 |
| }, |
| "3": { |
| "pass@1": 0.8564124617737003, |
| "pass@2": 0.8995242782152231, |
| "pass@4": 0.9379065466587319, |
| "pass@8": 0.9688289456978036, |
| "pass@16": 0.988510504308861, |
| "pass@32": 0.9967707993454415, |
| "pass@64": 0.999420725755249, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.7214599609375, |
| "pass@2": 0.8123266024852361, |
| "pass@4": 0.8851466221995687, |
| "pass@8": 0.9351398158085544, |
| "pass@16": 0.9643246165797228, |
| "pass@32": 0.9816108955922624, |
| "pass@64": 0.9923375102880352, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.4746031746031746, |
| "pass@2": 0.6117000999875012, |
| "pass@4": 0.7313267404074488, |
| "pass@8": 0.8257029432030616, |
| "pass@16": 0.8926087586123375, |
| "pass@32": 0.936348525554623, |
| "pass@64": 0.9630246909720346, |
| "pass@128": 0.9777777777777777 |
| }, |
| "6": { |
| "pass@1": 0.4722118263473054, |
| "pass@2": 0.6342759854307132, |
| "pass@4": 0.7775939507374454, |
| "pass@8": 0.8825466618292312, |
| "pass@16": 0.9464171107787874, |
| "pass@32": 0.9786132080116102, |
| "pass@64": 0.9937169652841435, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.3753004807692308, |
| "pass@2": 0.5153170572613338, |
| "pass@4": 0.6561292772944801, |
| "pass@8": 0.7761565488757287, |
| "pass@16": 0.8626775371075649, |
| "pass@32": 0.9154326493792188, |
| "pass@64": 0.9457564239431862, |
| "pass@128": 0.9615384615384616 |
| }, |
| "8": { |
| "pass@1": 0.3215144230769231, |
| "pass@2": 0.46476094033918824, |
| "pass@4": 0.6165568053993246, |
| "pass@8": 0.7534847622641432, |
| "pass@16": 0.8589420967295084, |
| "pass@32": 0.9294160107049367, |
| "pass@64": 0.968232374522714, |
| "pass@128": 0.9839743589743589 |
| }, |
| "9": { |
| "pass@1": 0.2617517605633803, |
| "pass@2": 0.39134794000221806, |
| "pass@4": 0.5420022101110598, |
| "pass@8": 0.6911958532785958, |
| "pass@16": 0.8130187441216671, |
| "pass@32": 0.8937970226619533, |
| "pass@64": 0.9389010514280172, |
| "pass@128": 0.9633802816901409 |
| }, |
| "11": { |
| "pass@1": 0.23074539660056656, |
| "pass@2": 0.35009535812272746, |
| "pass@4": 0.4882828888598553, |
| "pass@8": 0.6247398677588178, |
| "pass@16": 0.7429827393881699, |
| "pass@32": 0.8357673540792772, |
| "pass@64": 0.9005930374066281, |
| "pass@128": 0.943342776203966 |
| }, |
| "12": { |
| "pass@1": 0.19639389124293785, |
| "pass@2": 0.3063894356955379, |
| "pass@4": 0.440758561959416, |
| "pass@8": 0.5857390531049586, |
| "pass@16": 0.7220385954008949, |
| "pass@32": 0.8282172555034433, |
| "pass@64": 0.8965112418457605, |
| "pass@128": 0.9378531073446328 |
| }, |
| "13": { |
| "pass@1": 0.17932014106583072, |
| "pass@2": 0.2773797145360748, |
| "pass@4": 0.3952843552275407, |
| "pass@8": 0.5244591507479613, |
| "pass@16": 0.6515171712008817, |
| "pass@32": 0.7609219998509404, |
| "pass@64": 0.8391465004588685, |
| "pass@128": 0.8871473354231975 |
| }, |
| "14": { |
| "pass@1": 0.14987344236760125, |
| "pass@2": 0.24167831824760222, |
| "pass@4": 0.3558844169011584, |
| "pass@8": 0.48062360747413807, |
| "pass@16": 0.6031438802092132, |
| "pass@32": 0.7101192293253078, |
| "pass@64": 0.791392407450985, |
| "pass@128": 0.8442367601246106 |
| }, |
| "15": { |
| "pass@1": 0.1279127528901734, |
| "pass@2": 0.19832997189477028, |
| "pass@4": 0.2873977274697598, |
| "pass@8": 0.39199371493524304, |
| "pass@16": 0.5038338391917309, |
| "pass@32": 0.6065491304457705, |
| "pass@64": 0.6887834435073522, |
| "pass@128": 0.7572254335260116 |
| }, |
| "16": { |
| "pass@1": 0.12358870967741936, |
| "pass@2": 0.19655353060706127, |
| "pass@4": 0.28869364908015527, |
| "pass@8": 0.38926244247020075, |
| "pass@16": 0.48742943166589325, |
| "pass@32": 0.5751892983598913, |
| "pass@64": 0.6502279989309359, |
| "pass@128": 0.7161290322580646 |
| }, |
| "17": { |
| "pass@1": 0.12741600433526012, |
| "pass@2": 0.19630173012152363, |
| "pass@4": 0.2774420586335666, |
| "pass@8": 0.36357971010177026, |
| "pass@16": 0.4478440310684387, |
| "pass@32": 0.5246952747803684, |
| "pass@64": 0.5915975129658583, |
| "pass@128": 0.6502890173410405 |
| }, |
| "18": { |
| "pass@1": 0.10390625, |
| "pass@2": 0.16584434758155223, |
| "pass@4": 0.24453465691788526, |
| "pass@8": 0.33183968973437294, |
| "pass@16": 0.4167819865707351, |
| "pass@32": 0.49231203177580335, |
| "pass@64": 0.5558379435748657, |
| "pass@128": 0.6057142857142858 |
| }, |
| "19": { |
| "pass@1": 0.10846002252252253, |
| "pass@2": 0.17541778983708112, |
| "pass@4": 0.2587067981930187, |
| "pass@8": 0.34694823379659046, |
| "pass@16": 0.42791437185014847, |
| "pass@32": 0.4934379723821667, |
| "pass@64": 0.5438193608394496, |
| "pass@128": 0.5915915915915916 |
| }, |
| "20": { |
| "pass@1": 0.10027214714714715, |
| "pass@2": 0.15846235014542095, |
| "pass@4": 0.23135669668543687, |
| "pass@8": 0.31203855130831787, |
| "pass@16": 0.39199227760299216, |
| "pass@32": 0.4606754660817298, |
| "pass@64": 0.5124782566863452, |
| "pass@128": 0.5495495495495496 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 262888, |
| "answer_accuracy": 0.32015783320342944, |
| "avg_response_len": 249.93761204208886, |
| "resp_tokens_sum": 205228772, |
| "pass_at_k": { |
| "pass@1": 0.32015783320342944, |
| "pass@2": 0.4156968626680902, |
| "pass@4": 0.5208065187194584, |
| "pass@8": 0.6237177484598363, |
| "pass@16": 0.7124685902904693, |
| "pass@32": 0.7799126469709882, |
| "pass@64": 0.8275756037833933, |
| "pass@128": 0.8618862042088854 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.26540931748466257, |
| "pass@2": 0.39662335152891176, |
| "pass@4": 0.5456323869217269, |
| "pass@8": 0.6920447324686265, |
| "pass@16": 0.8146591405440967, |
| "pass@32": 0.9003614283097305, |
| "pass@64": 0.9499486824301553, |
| "pass@128": 0.9754601226993865 |
| }, |
| "2": { |
| "pass@1": 0.9039228723404256, |
| "pass@2": 0.936999273029701, |
| "pass@4": 0.9640325722012707, |
| "pass@8": 0.9822922126341371, |
| "pass@16": 0.9921029236893512, |
| "pass@32": 0.9961241979706833, |
| "pass@64": 0.9969365657746452, |
| "pass@128": 0.9969604863221885 |
| }, |
| "3": { |
| "pass@1": 0.8807528409090909, |
| "pass@2": 0.9155679581245529, |
| "pass@4": 0.9469561969526535, |
| "pass@8": 0.9723452248893673, |
| "pass@16": 0.9888141155481235, |
| "pass@32": 0.9966414290903808, |
| "pass@64": 0.9994041833835215, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.6924232618343196, |
| "pass@2": 0.7907765689791734, |
| "pass@4": 0.8758891791632553, |
| "pass@8": 0.936007704442523, |
| "pass@16": 0.9686286192064628, |
| "pass@32": 0.9837451220106904, |
| "pass@64": 0.9917408064315401, |
| "pass@128": 0.9970414201183432 |
| }, |
| "5": { |
| "pass@1": 0.44732789855072463, |
| "pass@2": 0.578949132717106, |
| "pass@4": 0.7002346964781571, |
| "pass@8": 0.7998659410224632, |
| "pass@16": 0.8748176184318499, |
| "pass@32": 0.9272979570411638, |
| "pass@64": 0.9597548745587382, |
| "pass@128": 0.9768115942028985 |
| }, |
| "6": { |
| "pass@1": 0.4549606017191977, |
| "pass@2": 0.611524702186224, |
| "pass@4": 0.748689749383046, |
| "pass@8": 0.8528006981800831, |
| "pass@16": 0.9231486355342031, |
| "pass@32": 0.9622876414728101, |
| "pass@64": 0.9801267017916749, |
| "pass@128": 0.9885386819484241 |
| }, |
| "7": { |
| "pass@1": 0.4048274927745665, |
| "pass@2": 0.5412742222702657, |
| "pass@4": 0.674239169977307, |
| "pass@8": 0.787814474108333, |
| "pass@16": 0.8702585537862398, |
| "pass@32": 0.9193176128192155, |
| "pass@64": 0.9434355530262379, |
| "pass@128": 0.9566473988439307 |
| }, |
| "8": { |
| "pass@1": 0.34401408450704224, |
| "pass@2": 0.48745494621270946, |
| "pass@4": 0.635357102985366, |
| "pass@8": 0.7675013657375953, |
| "pass@16": 0.8688231467359476, |
| "pass@32": 0.9331341673035908, |
| "pass@64": 0.9671424869278781, |
| "pass@128": 0.9802816901408451 |
| }, |
| "9": { |
| "pass@1": 0.277905127245509, |
| "pass@2": 0.40908488141826543, |
| "pass@4": 0.5569567148043622, |
| "pass@8": 0.7026706835230918, |
| "pass@16": 0.8241962201311076, |
| "pass@32": 0.9040788025953181, |
| "pass@64": 0.9458140176070623, |
| "pass@128": 0.9700598802395209 |
| }, |
| "11": { |
| "pass@1": 0.19147684487951808, |
| "pass@2": 0.2949832054596335, |
| "pass@4": 0.4249811888160067, |
| "pass@8": 0.5684708001105898, |
| "pass@16": 0.7033851584127203, |
| "pass@32": 0.8127286622022419, |
| "pass@64": 0.8939142475671594, |
| "pass@128": 0.9487951807228916 |
| }, |
| "12": { |
| "pass@1": 0.2179418103448276, |
| "pass@2": 0.33208283452225207, |
| "pass@4": 0.47159283704348853, |
| "pass@8": 0.6208105407618292, |
| "pass@16": 0.7542302648948708, |
| "pass@32": 0.8506687264615499, |
| "pass@64": 0.9116059188654317, |
| "pass@128": 0.9529780564263323 |
| }, |
| "13": { |
| "pass@1": 0.16940213585434175, |
| "pass@2": 0.2634938325723991, |
| "pass@4": 0.3793841751923866, |
| "pass@8": 0.5050673959610599, |
| "pass@16": 0.6265313995042749, |
| "pass@32": 0.7333003831218914, |
| "pass@64": 0.8201189132557245, |
| "pass@128": 0.8795518207282913 |
| }, |
| "14": { |
| "pass@1": 0.14487390350877194, |
| "pass@2": 0.2287026004973063, |
| "pass@4": 0.3352070892454231, |
| "pass@8": 0.4521805285221764, |
| "pass@16": 0.5621008322504216, |
| "pass@32": 0.6559120049884942, |
| "pass@64": 0.7329353333346605, |
| "pass@128": 0.7953216374269005 |
| }, |
| "15": { |
| "pass@1": 0.12080965909090909, |
| "pass@2": 0.19719637318062513, |
| "pass@4": 0.2982539980797855, |
| "pass@8": 0.4146947642198226, |
| "pass@16": 0.5306220264944798, |
| "pass@32": 0.632068217942676, |
| "pass@64": 0.713947397576443, |
| "pass@128": 0.7818181818181819 |
| }, |
| "16": { |
| "pass@1": 0.13054220085470086, |
| "pass@2": 0.20016067703075577, |
| "pass@4": 0.28242962231216834, |
| "pass@8": 0.37245942247155317, |
| "pass@16": 0.4633370433089357, |
| "pass@32": 0.5449004689861886, |
| "pass@64": 0.6146582022395269, |
| "pass@128": 0.6723646723646723 |
| }, |
| "17": { |
| "pass@1": 0.12473311335403726, |
| "pass@2": 0.19863702499144123, |
| "pass@4": 0.2904211032238981, |
| "pass@8": 0.39107439466974725, |
| "pass@16": 0.48594380207333926, |
| "pass@32": 0.565189798800697, |
| "pass@64": 0.6291464350799738, |
| "pass@128": 0.6832298136645962 |
| }, |
| "18": { |
| "pass@1": 0.11567095588235295, |
| "pass@2": 0.1858296665122742, |
| "pass@4": 0.2744835213613003, |
| "pass@8": 0.3731498178040868, |
| "pass@16": 0.47119057826231775, |
| "pass@32": 0.5583474575059797, |
| "pass@64": 0.6331969527222595, |
| "pass@128": 0.6970588235294117 |
| }, |
| "19": { |
| "pass@1": 0.11228143601190477, |
| "pass@2": 0.1775589818850769, |
| "pass@4": 0.2564224229225811, |
| "pass@8": 0.33700818889898293, |
| "pass@16": 0.4081891566185268, |
| "pass@32": 0.46506532057111194, |
| "pass@64": 0.5090195808415485, |
| "pass@128": 0.5505952380952381 |
| }, |
| "20": { |
| "pass@1": 0.09300149700598802, |
| "pass@2": 0.15357940614833324, |
| "pass@4": 0.23198116028909568, |
| "pass@8": 0.3187552978474917, |
| "pass@16": 0.40235214790860857, |
| "pass@32": 0.47365640144881727, |
| "pass@64": 0.5272035572429615, |
| "pass@128": 0.5688622754491018 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 261732, |
| "answer_accuracy": 0.32669455983383927, |
| "avg_response_len": 272.9523573554082, |
| "resp_tokens_sum": 218676327, |
| "pass_at_k": { |
| "pass@1": 0.32669455983383927, |
| "pass@2": 0.423339878763563, |
| "pass@4": 0.5288904041172903, |
| "pass@8": 0.629584657101439, |
| "pass@16": 0.7138453133463799, |
| "pass@32": 0.7778640458840944, |
| "pass@64": 0.824266594191614, |
| "pass@128": 0.8586036108004473 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.2988106343283582, |
| "pass@2": 0.43811001586555404, |
| "pass@4": 0.592827761361919, |
| "pass@8": 0.7389906797871061, |
| "pass@16": 0.8525613765094223, |
| "pass@32": 0.9236767364419983, |
| "pass@64": 0.9630391911668797, |
| "pass@128": 0.982089552238806 |
| }, |
| "2": { |
| "pass@1": 0.8912946428571429, |
| "pass@2": 0.9280339566929132, |
| "pass@4": 0.9595391234131446, |
| "pass@8": 0.9812508611442692, |
| "pass@16": 0.9925072517907899, |
| "pass@32": 0.9966186521833359, |
| "pass@64": 0.9983968451130126, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.8721073250728864, |
| "pass@2": 0.9140296796446361, |
| "pass@4": 0.9490690140591022, |
| "pass@8": 0.9730759865091045, |
| "pass@16": 0.9857702689101124, |
| "pass@32": 0.9920553316855298, |
| "pass@64": 0.9955026148242196, |
| "pass@128": 0.9970845481049563 |
| }, |
| "4": { |
| "pass@1": 0.7161686769005848, |
| "pass@2": 0.8083079356034446, |
| "pass@4": 0.8882740408545416, |
| "pass@8": 0.9440402847981656, |
| "pass@16": 0.9733790763978329, |
| "pass@32": 0.9852732332074049, |
| "pass@64": 0.9894172655511996, |
| "pass@128": 0.9912280701754386 |
| }, |
| "5": { |
| "pass@1": 0.4732766544117647, |
| "pass@2": 0.6120990620657709, |
| "pass@4": 0.7379644536160923, |
| "pass@8": 0.8328578936686588, |
| "pass@16": 0.8941166055439531, |
| "pass@32": 0.9329952890031863, |
| "pass@64": 0.9585346672880898, |
| "pass@128": 0.9764705882352941 |
| }, |
| "6": { |
| "pass@1": 0.4296382097791798, |
| "pass@2": 0.5824524485208274, |
| "pass@4": 0.7232217291844827, |
| "pass@8": 0.831538757783547, |
| "pass@16": 0.9022300063004003, |
| "pass@32": 0.9425257163922585, |
| "pass@64": 0.9631168749883116, |
| "pass@128": 0.9747634069400631 |
| }, |
| "7": { |
| "pass@1": 0.38983386075949367, |
| "pass@2": 0.5293648148609583, |
| "pass@4": 0.665921777776196, |
| "pass@8": 0.7814408029242328, |
| "pass@16": 0.8668908924653281, |
| "pass@32": 0.924388953311442, |
| "pass@64": 0.9611535974011868, |
| "pass@128": 0.9841772151898734 |
| }, |
| "8": { |
| "pass@1": 0.3031625375375375, |
| "pass@2": 0.44241460949138073, |
| "pass@4": 0.5955823466886454, |
| "pass@8": 0.736405779811164, |
| "pass@16": 0.8433245202487523, |
| "pass@32": 0.9120078894889249, |
| "pass@64": 0.9486454274049398, |
| "pass@128": 0.960960960960961 |
| }, |
| "9": { |
| "pass@1": 0.24593046623794212, |
| "pass@2": 0.37445249006253595, |
| "pass@4": 0.5259498379623767, |
| "pass@8": 0.6801062895678363, |
| "pass@16": 0.8112116977754945, |
| "pass@32": 0.9045034497592848, |
| "pass@64": 0.9614410982686454, |
| "pass@128": 0.9903536977491961 |
| }, |
| "11": { |
| "pass@1": 0.2273313492063492, |
| "pass@2": 0.34733900449943744, |
| "pass@4": 0.48797233292267006, |
| "pass@8": 0.6312245582865806, |
| "pass@16": 0.755738770752698, |
| "pass@32": 0.8506597987618932, |
| "pass@64": 0.9176150703846887, |
| "pass@128": 0.9619047619047619 |
| }, |
| "12": { |
| "pass@1": 0.20637423547400613, |
| "pass@2": 0.31808833887163207, |
| "pass@4": 0.45083233947820794, |
| "pass@8": 0.5880314127552496, |
| "pass@16": 0.7108223889541407, |
| "pass@32": 0.8044281622631726, |
| "pass@64": 0.8670414062590106, |
| "pass@128": 0.908256880733945 |
| }, |
| "13": { |
| "pass@1": 0.18993537808641975, |
| "pass@2": 0.2897349962331098, |
| "pass@4": 0.4045277045809091, |
| "pass@8": 0.5189595494585346, |
| "pass@16": 0.6230551493144086, |
| "pass@32": 0.7152991866459357, |
| "pass@64": 0.7936777363002678, |
| "pass@128": 0.8549382716049383 |
| }, |
| "14": { |
| "pass@1": 0.16450296735905046, |
| "pass@2": 0.2570091444893571, |
| "pass@4": 0.3680840732260101, |
| "pass@8": 0.4827612360891505, |
| "pass@16": 0.5865991822960027, |
| "pass@32": 0.6762563749288109, |
| "pass@64": 0.7509042429339469, |
| "pass@128": 0.8100890207715133 |
| }, |
| "15": { |
| "pass@1": 0.11622299382716049, |
| "pass@2": 0.19063654369592675, |
| "pass@4": 0.29270876470302304, |
| "pass@8": 0.4155669062840721, |
| "pass@16": 0.5390361333862039, |
| "pass@32": 0.6411043054391663, |
| "pass@64": 0.7135058242529809, |
| "pass@128": 0.7685185185185185 |
| }, |
| "16": { |
| "pass@1": 0.13880438790560473, |
| "pass@2": 0.20914410726314084, |
| "pass@4": 0.29200838169565074, |
| "pass@8": 0.38077294241430154, |
| "pass@16": 0.46737511475798427, |
| "pass@32": 0.5497080689560159, |
| "pass@64": 0.6309065509538583, |
| "pass@128": 0.7109144542772862 |
| }, |
| "17": { |
| "pass@1": 0.12768260542168675, |
| "pass@2": 0.19840273989659413, |
| "pass@4": 0.2831356568004299, |
| "pass@8": 0.3738445163007728, |
| "pass@16": 0.4613568303598599, |
| "pass@32": 0.5385335841890195, |
| "pass@64": 0.6024723538594325, |
| "pass@128": 0.6536144578313253 |
| }, |
| "18": { |
| "pass@1": 0.10819052419354838, |
| "pass@2": 0.17318469011938017, |
| "pass@4": 0.25503082659022464, |
| "pass@8": 0.34397619629010034, |
| "pass@16": 0.42672772692874866, |
| "pass@32": 0.4951718346534467, |
| "pass@64": 0.5523887072386519, |
| "pass@128": 0.6129032258064516 |
| }, |
| "19": { |
| "pass@1": 0.12157760574018127, |
| "pass@2": 0.1908757463662964, |
| "pass@4": 0.27040493408867705, |
| "pass@8": 0.3496266345867802, |
| "pass@16": 0.42369818096696776, |
| "pass@32": 0.48808375812634547, |
| "pass@64": 0.5379706412146679, |
| "pass@128": 0.5770392749244713 |
| }, |
| "20": { |
| "pass@1": 0.10203171921921922, |
| "pass@2": 0.16524644723463625, |
| "pass@4": 0.24453744633272184, |
| "pass@8": 0.33137040399755197, |
| "pass@16": 0.4145973832634668, |
| "pass@32": 0.48491618782761853, |
| "pass@64": 0.5411038749238117, |
| "pass@128": 0.5885885885885885 |
| } |
| } |
| } |
| } |
| } |
| ] |
