AR-Diff / SDAR-100M-midtrain-ar_diff-block_curriculum-2_16 /composition /autoregressive /summary.json
| [ | |
| { | |
| "checkpoint": "checkpoint-39470", | |
| "id_acc": 0.9900894097222223, | |
| "ood_acc": 0.87708359375, | |
| "total_acc": 0.9306126644736842, | |
| "id_avg_loss": 0.0, | |
| "ood_avg_loss": 0.0, | |
| "total_avg_loss": 0.0, | |
| "id_avg_resp_len": 200.35493229166667, | |
| "ood_avg_resp_len": 383.22090859375, | |
| "count_id": 1152000, | |
| "count_ood": 1280000, | |
| "count_total": 2432000, | |
| "template_metrics": { | |
| "crazy_zootopia": { | |
| "count": 809728, | |
| "correct": 751583, | |
| "answer_accuracy": 0.9281919360575404, | |
| "avg_response_len": 283.65910527979764, | |
| "resp_tokens_sum": 229686720, | |
| "pass_at_k": { | |
| "pass@1": 0.9281919360575404, | |
| "pass@2": 0.9499665173848083, | |
| "pass@4": 0.9640744475635182, | |
| "pass@8": 0.9737280900028716, | |
| "pass@16": 0.980602757648027, | |
| "pass@32": 0.9857206319262101, | |
| "pass@64": 0.9895329614718181, | |
| "pass@128": 0.9920961112867531 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.9392514749262537, | |
| "pass@2": 0.9528702268134623, | |
| "pass@4": 0.9647196697204883, | |
| "pass@8": 0.9752039016665607, | |
| "pass@16": 0.9841804774412467, | |
| "pass@32": 0.9916424699073122, | |
| "pass@64": 0.9975593656716509, | |
| "pass@128": 1.0 | |
| }, | |
| "2": { | |
| "pass@1": 0.9999756619937694, | |
| "pass@2": 1.0, | |
| "pass@4": 1.0, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "3": { | |
| "pass@1": 0.9964162844036697, | |
| "pass@2": 0.9992663259409089, | |
| "pass@4": 0.9999513056854131, | |
| "pass@8": 0.9999997214289341, | |
| "pass@16": 0.9999999999945326, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "4": { | |
| "pass@1": 0.99814453125, | |
| "pass@2": 0.9990688053641733, | |
| "pass@4": 0.9997314116985375, | |
| "pass@8": 0.9999793655718723, | |
| "pass@16": 0.9999999169698948, | |
| "pass@32": 0.9999999999998159, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "5": { | |
| "pass@1": 0.9982886904761905, | |
| "pass@2": 0.9999117297837768, | |
| "pass@4": 0.9999994979794193, | |
| "pass@8": 0.9999999999856092, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "6": { | |
| "pass@1": 0.9993450598802395, | |
| "pass@2": 0.9999620591494178, | |
| "pass@4": 0.9999997176625378, | |
| "pass@8": 0.9999999999937114, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "7": { | |
| "pass@1": 0.9974112426035503, | |
| "pass@2": 0.9990299439733494, | |
| "pass@4": 0.999743682649876, | |
| "pass@8": 0.9999804637850213, | |
| "pass@16": 0.9999999213916163, | |
| "pass@32": 0.9999999999998258, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "8": { | |
| "pass@1": 0.9911608573717948, | |
| "pass@2": 0.9963473052947709, | |
| "pass@4": 0.9984855673088942, | |
| "pass@8": 0.9994635469329705, | |
| "pass@16": 0.9999228935849399, | |
| "pass@32": 0.9999990402568896, | |
| "pass@64": 0.9999999999948799, | |
| "pass@128": 1.0 | |
| }, | |
| "9": { | |
| "pass@1": 0.9897447183098591, | |
| "pass@2": 0.9935465648220028, | |
| "pass@4": 0.9950367868629096, | |
| "pass@8": 0.9961759013482411, | |
| "pass@16": 0.9975146715824619, | |
| "pass@32": 0.9990370240116533, | |
| "pass@64": 0.9999221200063837, | |
| "pass@128": 1.0 | |
| }, | |
| "11": { | |
| "pass@1": 0.9560685198300283, | |
| "pass@2": 0.972025147219558, | |
| "pass@4": 0.9833676431380923, | |
| "pass@8": 0.9902832202553371, | |
| "pass@16": 0.9946957855807873, | |
| "pass@32": 0.997549645536328, | |
| "pass@64": 0.9992147611382037, | |
| "pass@128": 1.0 | |
| }, | |
| "12": { | |
| "pass@1": 0.943215925141243, | |
| "pass@2": 0.9660565138573776, | |
| "pass@4": 0.9814934266691235, | |
| "pass@8": 0.9904757233909742, | |
| "pass@16": 0.9957168814771249, | |
| "pass@32": 0.9984134645336938, | |
| "pass@64": 0.9996455904087194, | |
| "pass@128": 1.0 | |
| }, | |
| "13": { | |
| "pass@1": 0.9108787225705329, | |
| "pass@2": 0.9434656776837063, | |
| "pass@4": 0.9667659061230197, | |
| "pass@8": 0.9808162550485344, | |
| "pass@16": 0.9881944688814267, | |
| "pass@32": 0.9930606223263609, | |
| "pass@64": 0.9966675055170503, | |
| "pass@128": 1.0 | |
| }, | |
| "14": { | |
| "pass@1": 0.9186380451713395, | |
| "pass@2": 0.9460888249074009, | |
| "pass@4": 0.9653372817299704, | |
| "pass@8": 0.9779994019698924, | |
| "pass@16": 0.9860626693919856, | |
| "pass@32": 0.9919049064530143, | |
| "pass@64": 0.9963014598578678, | |
| "pass@128": 1.0 | |
| }, | |
| "15": { | |
| "pass@1": 0.9521540823699421, | |
| "pass@2": 0.9741484513677119, | |
| "pass@4": 0.985112245633313, | |
| "pass@8": 0.9908469302823427, | |
| "pass@16": 0.9934400862091718, | |
| "pass@32": 0.994992084796452, | |
| "pass@64": 0.9963530196364373, | |
| "pass@128": 0.9971098265895953 | |
| }, | |
| "16": { | |
| "pass@1": 0.8628276209677419, | |
| "pass@2": 0.9018546005842007, | |
| "pass@4": 0.9287450191709906, | |
| "pass@8": 0.9468683936291992, | |
| "pass@16": 0.9613419346632679, | |
| "pass@32": 0.9736111775442221, | |
| "pass@64": 0.9823099408096163, | |
| "pass@128": 0.9870967741935484 | |
| }, | |
| "17": { | |
| "pass@1": 0.884709176300578, | |
| "pass@2": 0.9211279214646582, | |
| "pass@4": 0.9415992674867953, | |
| "pass@8": 0.9565759689885837, | |
| "pass@16": 0.9692118557186207, | |
| "pass@32": 0.9795491000031725, | |
| "pass@64": 0.98639472632689, | |
| "pass@128": 0.9884393063583815 | |
| }, | |
| "18": { | |
| "pass@1": 0.82296875, | |
| "pass@2": 0.8701905230596182, | |
| "pass@4": 0.8998161420001068, | |
| "pass@8": 0.9205166862174053, | |
| "pass@16": 0.9355630508173823, | |
| "pass@32": 0.9477589149914283, | |
| "pass@64": 0.9590373492455058, | |
| "pass@128": 0.9685714285714285 | |
| }, | |
| "19": { | |
| "pass@1": 0.7173188813813813, | |
| "pass@2": 0.7844284983802695, | |
| "pass@4": 0.8315280263390493, | |
| "pass@8": 0.8687886876667537, | |
| "pass@16": 0.8990525439832475, | |
| "pass@32": 0.9216565068026786, | |
| "pass@64": 0.9364336919565242, | |
| "pass@128": 0.9459459459459459 | |
| }, | |
| "20": { | |
| "pass@1": 0.7638419669669669, | |
| "pass@2": 0.8345680818613895, | |
| "pass@4": 0.8795241003095331, | |
| "pass@8": 0.9096289064963395, | |
| "pass@16": 0.9287254882088332, | |
| "pass@32": 0.9412760341681837, | |
| "pass@64": 0.9527062784829503, | |
| "pass@128": 0.963963963963964 | |
| } | |
| } | |
| }, | |
| "teachers_in_school": { | |
| "count": 821120, | |
| "correct": 766904, | |
| "answer_accuracy": 0.933973109898675, | |
| "avg_response_len": 292.6585639127046, | |
| "resp_tokens_sum": 240307800, | |
| "pass_at_k": { | |
| "pass@1": 0.933973109898675, | |
| "pass@2": 0.9541137397892366, | |
| "pass@4": 0.9676575266507484, | |
| "pass@8": 0.9769106752250937, | |
| "pass@16": 0.9831209304949401, | |
| "pass@32": 0.9874264895458542, | |
| "pass@64": 0.9906284085148328, | |
| "pass@128": 0.9929851909586905 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.9263564033742331, | |
| "pass@2": 0.943031511158881, | |
| "pass@4": 0.9574623036959338, | |
| "pass@8": 0.9692292659689711, | |
| "pass@16": 0.9785950061939993, | |
| "pass@32": 0.985816650956142, | |
| "pass@64": 0.9927118784293236, | |
| "pass@128": 1.0 | |
| }, | |
| "2": { | |
| "pass@1": 1.0, | |
| "pass@2": 1.0, | |
| "pass@4": 1.0, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "3": { | |
| "pass@1": 0.9957859848484848, | |
| "pass@2": 0.9979222590073968, | |
| "pass@4": 0.9992018259081249, | |
| "pass@8": 0.9998472966011496, | |
| "pass@16": 0.999994268588126, | |
| "pass@32": 0.9999999968257123, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "4": { | |
| "pass@1": 0.9996764053254438, | |
| "pass@2": 0.9999919920328006, | |
| "pass@4": 0.999999990293373, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "5": { | |
| "pass@1": 0.996286231884058, | |
| "pass@2": 0.9990335786830994, | |
| "pass@4": 0.9998567722512948, | |
| "pass@8": 0.9999954258079576, | |
| "pass@16": 0.9999999965927238, | |
| "pass@32": 0.9999999999999999, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "6": { | |
| "pass@1": 0.9997089899713467, | |
| "pass@2": 0.9999936545360197, | |
| "pass@4": 0.9999999959711339, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "7": { | |
| "pass@1": 0.9963872832369942, | |
| "pass@2": 0.9973061080515223, | |
| "pass@4": 0.9977670850211643, | |
| "pass@8": 0.9983013813782712, | |
| "pass@16": 0.999038785005553, | |
| "pass@32": 0.9997319443095561, | |
| "pass@64": 0.9999910524153849, | |
| "pass@128": 1.0 | |
| }, | |
| "8": { | |
| "pass@1": 0.9921654929577465, | |
| "pass@2": 0.9949418459576357, | |
| "pass@4": 0.9969772593566649, | |
| "pass@8": 0.998742979098145, | |
| "pass@16": 0.9996984788969664, | |
| "pass@32": 0.9999800029461254, | |
| "pass@64": 0.9999999852572564, | |
| "pass@128": 1.0 | |
| }, | |
| "9": { | |
| "pass@1": 0.9932634730538922, | |
| "pass@2": 0.9957594653213259, | |
| "pass@4": 0.9968023138200539, | |
| "pass@8": 0.9971748134078343, | |
| "pass@16": 0.9973801780816433, | |
| "pass@32": 0.9977544910178684, | |
| "pass@64": 0.9985029940119761, | |
| "pass@128": 1.0 | |
| }, | |
| "11": { | |
| "pass@1": 0.9698795180722891, | |
| "pass@2": 0.9801018789725835, | |
| "pass@4": 0.9859459858556837, | |
| "pass@8": 0.9903607486554723, | |
| "pass@16": 0.9944354686155655, | |
| "pass@32": 0.997734424893915, | |
| "pass@64": 0.9996912428943647, | |
| "pass@128": 1.0 | |
| }, | |
| "12": { | |
| "pass@1": 0.9312793887147336, | |
| "pass@2": 0.9578514921136424, | |
| "pass@4": 0.9785946927088653, | |
| "pass@8": 0.9911220322102386, | |
| "pass@16": 0.9969256254585147, | |
| "pass@32": 0.999356859530079, | |
| "pass@64": 0.9999770727927124, | |
| "pass@128": 1.0 | |
| }, | |
| "13": { | |
| "pass@1": 0.9410014005602241, | |
| "pass@2": 0.9629413281060455, | |
| "pass@4": 0.9783460898795218, | |
| "pass@8": 0.9885164313887403, | |
| "pass@16": 0.9938448297567265, | |
| "pass@32": 0.99557976291731, | |
| "pass@64": 0.9965040977444787, | |
| "pass@128": 0.9971988795518207 | |
| }, | |
| "14": { | |
| "pass@1": 0.938093932748538, | |
| "pass@2": 0.9617224553345302, | |
| "pass@4": 0.9780836096145873, | |
| "pass@8": 0.9891690673671412, | |
| "pass@16": 0.9953444109429628, | |
| "pass@32": 0.9985866789814201, | |
| "pass@64": 0.9998999204723729, | |
| "pass@128": 1.0 | |
| }, | |
| "15": { | |
| "pass@1": 0.946875, | |
| "pass@2": 0.9708445180147938, | |
| "pass@4": 0.9852852896228875, | |
| "pass@8": 0.9932527012437333, | |
| "pass@16": 0.9965859651004563, | |
| "pass@32": 0.9976875706827749, | |
| "pass@64": 0.9984848257468957, | |
| "pass@128": 1.0 | |
| }, | |
| "16": { | |
| "pass@1": 0.8979255698005698, | |
| "pass@2": 0.9284205560042177, | |
| "pass@4": 0.9476878840999577, | |
| "pass@8": 0.9601126751922175, | |
| "pass@16": 0.9681501715762721, | |
| "pass@32": 0.9745545081734679, | |
| "pass@64": 0.9806737994273104, | |
| "pass@128": 0.9857549857549858 | |
| }, | |
| "17": { | |
| "pass@1": 0.883103649068323, | |
| "pass@2": 0.9193444484520957, | |
| "pass@4": 0.9431839631265031, | |
| "pass@8": 0.9588158770909269, | |
| "pass@16": 0.969734921800441, | |
| "pass@32": 0.9777583996276256, | |
| "pass@64": 0.9834804448754739, | |
| "pass@128": 0.9875776397515528 | |
| }, | |
| "18": { | |
| "pass@1": 0.8681985294117647, | |
| "pass@2": 0.9134885942566008, | |
| "pass@4": 0.9388149804803808, | |
| "pass@8": 0.956182315324883, | |
| "pass@16": 0.969199828469146, | |
| "pass@32": 0.9786850953286746, | |
| "pass@64": 0.98495366662752, | |
| "pass@128": 0.9882352941176471 | |
| }, | |
| "19": { | |
| "pass@1": 0.7562779017857143, | |
| "pass@2": 0.8173970929415075, | |
| "pass@4": 0.8586776217704924, | |
| "pass@8": 0.8873359588818277, | |
| "pass@16": 0.9093487259281797, | |
| "pass@32": 0.9286139494295144, | |
| "pass@64": 0.9465647885804772, | |
| "pass@128": 0.9613095238095238 | |
| }, | |
| "20": { | |
| "pass@1": 0.7049962574850299, | |
| "pass@2": 0.7821469511763871, | |
| "pass@4": 0.8387314392587147, | |
| "pass@8": 0.8803005442248173, | |
| "pass@16": 0.9090276962957424, | |
| "pass@32": 0.927911101416634, | |
| "pass@64": 0.9395962270441276, | |
| "pass@128": 0.9461077844311377 | |
| } | |
| } | |
| }, | |
| "movie_festival_awards": { | |
| "count": 801152, | |
| "correct": 744763, | |
| "answer_accuracy": 0.9296151042498801, | |
| "avg_response_len": 313.71964995406614, | |
| "resp_tokens_sum": 251337125, | |
| "pass_at_k": { | |
| "pass@1": 0.9296151042498801, | |
| "pass@2": 0.951017935747319, | |
| "pass@4": 0.9651440403059918, | |
| "pass@8": 0.974651918770272, | |
| "pass@16": 0.9810899167866008, | |
| "pass@32": 0.9857287259116158, | |
| "pass@64": 0.9891995583526448, | |
| "pass@128": 0.9915321936411567 | |
| }, | |
| "per_op_pass_at_k": { | |
| "10": { | |
| "pass@1": 0.9518889925373134, | |
| "pass@2": 0.9682519978845928, | |
| "pass@4": 0.9806292692891003, | |
| "pass@8": 0.9888008490677124, | |
| "pass@16": 0.9928884788352844, | |
| "pass@32": 0.9939672147924427, | |
| "pass@64": 0.9940297978227394, | |
| "pass@128": 0.9940298507462687 | |
| }, | |
| "2": { | |
| "pass@1": 1.0, | |
| "pass@2": 1.0, | |
| "pass@4": 1.0, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "3": { | |
| "pass@1": 0.9998177842565598, | |
| "pass@2": 0.9999960543835081, | |
| "pass@4": 0.9999999986335526, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "4": { | |
| "pass@1": 0.9992233187134503, | |
| "pass@2": 0.9998215683565871, | |
| "pass@4": 0.9999901437758876, | |
| "pass@8": 0.9999999784883499, | |
| "pass@16": 0.9999999999999812, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "5": { | |
| "pass@1": 0.9940946691176471, | |
| "pass@2": 0.9972234686197314, | |
| "pass@4": 0.9979508072520346, | |
| "pass@8": 0.9985971409541529, | |
| "pass@16": 0.9993657240569218, | |
| "pass@32": 0.999893411003342, | |
| "pass@64": 0.9999991012534062, | |
| "pass@128": 1.0 | |
| }, | |
| "6": { | |
| "pass@1": 0.9998767744479495, | |
| "pass@2": 0.9999976713281503, | |
| "pass@4": 0.9999999997042957, | |
| "pass@8": 1.0, | |
| "pass@16": 1.0, | |
| "pass@32": 1.0, | |
| "pass@64": 1.0, | |
| "pass@128": 1.0 | |
| }, | |
| "7": { | |
| "pass@1": 0.9922122231012658, | |
| "pass@2": 0.9943055105651352, | |
| "pass@4": 0.9964102716986326, | |
| "pass@8": 0.9984351822242498, | |
| "pass@16": 0.9996795269701011, | |
| "pass@32": 0.9999902592414398, | |
| "pass@64": 0.9999999992697213, | |
| "pass@128": 1.0 | |
| }, | |
| "8": { | |
| "pass@1": 0.9930555555555556, | |
| "pass@2": 0.9965503150788586, | |
| "pass@4": 0.9984686342698152, | |
| "pass@8": 0.9996429053090278, | |
| "pass@16": 0.9999814246833897, | |
| "pass@32": 0.999999972034264, | |
| "pass@64": 0.9999999999999997, | |
| "pass@128": 1.0 | |
| }, | |
| "9": { | |
| "pass@1": 0.9924135852090032, | |
| "pass@2": 0.9953062099653139, | |
| "pass@4": 0.9966570869758643, | |
| "pass@8": 0.9975856061587218, | |
| "pass@16": 0.9983673492925828, | |
| "pass@32": 0.9992571222626693, | |
| "pass@64": 0.9999073347342836, | |
| "pass@128": 1.0 | |
| }, | |
| "11": { | |
| "pass@1": 0.9599702380952381, | |
| "pass@2": 0.9765876921634795, | |
| "pass@4": 0.989391186815934, | |
| "pass@8": 0.9963180825233957, | |
| "pass@16": 0.9992307852616401, | |
| "pass@32": 0.9999490539138667, | |
| "pass@64": 0.9999999121061375, | |
| "pass@128": 1.0 | |
| }, | |
| "12": { | |
| "pass@1": 0.939865252293578, | |
| "pass@2": 0.9603624124106047, | |
| "pass@4": 0.9760637159460566, | |
| "pass@8": 0.9866032202849236, | |
| "pass@16": 0.9926096934140741, | |
| "pass@32": 0.9959497775979793, | |
| "pass@64": 0.9984405444345842, | |
| "pass@128": 1.0 | |
| }, | |
| "13": { | |
| "pass@1": 0.9073832947530864, | |
| "pass@2": 0.9443962191358023, | |
| "pass@4": 0.9685251580242282, | |
| "pass@8": 0.9828957795842923, | |
| "pass@16": 0.9915318725469653, | |
| "pass@32": 0.996952650279957, | |
| "pass@64": 0.9995923226260632, | |
| "pass@128": 1.0 | |
| }, | |
| "14": { | |
| "pass@1": 0.9464252596439169, | |
| "pass@2": 0.9695290777821917, | |
| "pass@4": 0.9824109808974172, | |
| "pass@8": 0.9893288765335709, | |
| "pass@16": 0.9930767880716179, | |
| "pass@32": 0.9956401539621311, | |
| "pass@64": 0.996931099741985, | |
| "pass@128": 0.9970326409495549 | |
| }, | |
| "15": { | |
| "pass@1": 0.9288194444444444, | |
| "pass@2": 0.9530141197628071, | |
| "pass@4": 0.9679944280691763, | |
| "pass@8": 0.9773730797152483, | |
| "pass@16": 0.9826136664666306, | |
| "pass@32": 0.9854947655239579, | |
| "pass@64": 0.9876295766722463, | |
| "pass@128": 0.9907407407407407 | |
| }, | |
| "16": { | |
| "pass@1": 0.8941049041297935, | |
| "pass@2": 0.932356267275219, | |
| "pass@4": 0.9575642494356341, | |
| "pass@8": 0.9720565681989967, | |
| "pass@16": 0.9788168570235897, | |
| "pass@32": 0.9821212614436396, | |
| "pass@64": 0.9848713086920479, | |
| "pass@128": 0.9882005899705014 | |
| }, | |
| "17": { | |
| "pass@1": 0.8530685240963856, | |
| "pass@2": 0.8834865791907793, | |
| "pass@4": 0.9036101018246212, | |
| "pass@8": 0.9202718219997545, | |
| "pass@16": 0.9349628711134209, | |
| "pass@32": 0.9486017624674481, | |
| "pass@64": 0.9615355997566628, | |
| "pass@128": 0.9728915662650602 | |
| }, | |
| "18": { | |
| "pass@1": 0.8661290322580645, | |
| "pass@2": 0.9098425196850394, | |
| "pass@4": 0.9353601161749935, | |
| "pass@8": 0.9524239545873169, | |
| "pass@16": 0.9650574815675628, | |
| "pass@32": 0.9742872492835252, | |
| "pass@64": 0.9800047314029565, | |
| "pass@128": 0.9838709677419355 | |
| }, | |
| "19": { | |
| "pass@1": 0.7272233761329305, | |
| "pass@2": 0.7992720698432335, | |
| "pass@4": 0.8498617207403449, | |
| "pass@8": 0.886109822147316, | |
| "pass@16": 0.9119255337966973, | |
| "pass@32": 0.9312105063485466, | |
| "pass@64": 0.946458090289178, | |
| "pass@128": 0.9546827794561934 | |
| }, | |
| "20": { | |
| "pass@1": 0.714573948948949, | |
| "pass@2": 0.7877462846704968, | |
| "pass@4": 0.8363218603868204, | |
| "pass@8": 0.8718850949032955, | |
| "pass@16": 0.9008473092143473, | |
| "pass@32": 0.9259147136391661, | |
| "pass@64": 0.945745298634043, | |
| "pass@128": 0.9579579579579579 | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ] |