| [ |
| { |
| "checkpoint": "checkpoint-44699", |
| "id_acc": 0.911046875, |
| "ood_acc": 0.29633828125, |
| "total_acc": 0.5875160361842106, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 199.46338975694445, |
| "ood_avg_resp_len": 318.24450546875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 469664, |
| "answer_accuracy": 0.580026873221625, |
| "avg_response_len": 248.33461236365793, |
| "resp_tokens_sum": 201083489, |
| "pass_at_k": { |
| "pass@1": 0.580026873221625, |
| "pass@2": 0.6442908562587665, |
| "pass@4": 0.6958034834989211, |
| "pass@8": 0.7365417777176548, |
| "pass@16": 0.7686865636898084, |
| "pass@32": 0.7945016220873287, |
| "pass@64": 0.8162913610635678, |
| "pass@128": 0.8362314258615239 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.7771248156342183, |
| "pass@2": 0.8634121751097489, |
| "pass@4": 0.9133753164703967, |
| "pass@8": 0.9424125973654682, |
| "pass@16": 0.9595853388576279, |
| "pass@32": 0.9704237748229196, |
| "pass@64": 0.9789232030426774, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 0.9363317757009346, |
| "pass@2": 0.944525060097628, |
| "pass@4": 0.9532450491001708, |
| "pass@8": 0.9627298066354898, |
| "pass@16": 0.9719688257345186, |
| "pass@32": 0.980109835516427, |
| "pass@64": 0.987954208653673, |
| "pass@128": 0.9968847352024922 |
| }, |
| "3": { |
| "pass@1": 0.9741016819571865, |
| "pass@2": 0.9827925214910066, |
| "pass@4": 0.9867644511752545, |
| "pass@8": 0.9889207398015514, |
| "pass@16": 0.9904735766349954, |
| "pass@32": 0.9920811096141816, |
| "pass@64": 0.9934682845780995, |
| "pass@128": 0.9938837920489296 |
| }, |
| "4": { |
| "pass@1": 0.9593505859375, |
| "pass@2": 0.9731499138779528, |
| "pass@4": 0.9823881259959691, |
| "pass@8": 0.9881178094134416, |
| "pass@16": 0.9913778625190709, |
| "pass@32": 0.9934385296963498, |
| "pass@64": 0.9951254225202119, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9149305555555556, |
| "pass@2": 0.9386279840019998, |
| "pass@4": 0.9542117681718351, |
| "pass@8": 0.9660540883231847, |
| "pass@16": 0.9757455198041897, |
| "pass@32": 0.9831868336772416, |
| "pass@64": 0.9889152632850954, |
| "pass@128": 0.9936507936507937 |
| }, |
| "6": { |
| "pass@1": 0.9309739895209581, |
| "pass@2": 0.9547627186571742, |
| "pass@4": 0.9694351802581562, |
| "pass@8": 0.9786972797798579, |
| "pass@16": 0.9846500715770806, |
| "pass@32": 0.9890982222982732, |
| "pass@64": 0.9920427308990417, |
| "pass@128": 0.9940119760479041 |
| }, |
| "7": { |
| "pass@1": 0.9030833949704142, |
| "pass@2": 0.9393964322322136, |
| "pass@4": 0.9593146475218703, |
| "pass@8": 0.9720034544718734, |
| "pass@16": 0.980721876863249, |
| "pass@32": 0.9867746427167278, |
| "pass@64": 0.9916786492683689, |
| "pass@128": 0.9970414201183432 |
| }, |
| "8": { |
| "pass@1": 0.8989383012820513, |
| "pass@2": 0.9453968869876843, |
| "pass@4": 0.9636373202748688, |
| "pass@8": 0.9731567959470013, |
| "pass@16": 0.9791146723545264, |
| "pass@32": 0.983025910711685, |
| "pass@64": 0.9854191370827627, |
| "pass@128": 0.9871794871794872 |
| }, |
| "9": { |
| "pass@1": 0.8684198943661972, |
| "pass@2": 0.9273798796717314, |
| "pass@4": 0.9541264833093043, |
| "pass@8": 0.9697246361156203, |
| "pass@16": 0.9795548818702162, |
| "pass@32": 0.9850109862577768, |
| "pass@64": 0.9880098458356275, |
| "pass@128": 0.9887323943661972 |
| }, |
| "11": { |
| "pass@1": 0.7133719900849859, |
| "pass@2": 0.8189317938480065, |
| "pass@4": 0.883105831317827, |
| "pass@8": 0.9197420263846189, |
| "pass@16": 0.9425990411933546, |
| "pass@32": 0.9584202607176746, |
| "pass@64": 0.9703830576083085, |
| "pass@128": 0.9773371104815864 |
| }, |
| "12": { |
| "pass@1": 0.4903778248587571, |
| "pass@2": 0.6256558204324039, |
| "pass@4": 0.7312299754903507, |
| "pass@8": 0.8046130198855828, |
| "pass@16": 0.8548554975791646, |
| "pass@32": 0.8929254084348788, |
| "pass@64": 0.9249538202932568, |
| "pass@128": 0.9519774011299436 |
| }, |
| "13": { |
| "pass@1": 0.35450137147335425, |
| "pass@2": 0.46927568002863274, |
| "pass@4": 0.5712045930583124, |
| "pass@8": 0.6563511124051389, |
| "pass@16": 0.7233043809932352, |
| "pass@32": 0.7757701552474985, |
| "pass@64": 0.8189983379952339, |
| "pass@128": 0.8557993730407524 |
| }, |
| "14": { |
| "pass@1": 0.2768204828660436, |
| "pass@2": 0.3809104943949764, |
| "pass@4": 0.4742459108499289, |
| "pass@8": 0.5537739274477774, |
| "pass@16": 0.6249179084010938, |
| "pass@32": 0.6868947826587918, |
| "pass@64": 0.7398453986973906, |
| "pass@128": 0.7881619937694704 |
| }, |
| "15": { |
| "pass@1": 0.2188854768786127, |
| "pass@2": 0.3133277957307358, |
| "pass@4": 0.4072432692661974, |
| "pass@8": 0.4865732250040521, |
| "pass@16": 0.551303533167993, |
| "pass@32": 0.6076449232032669, |
| "pass@64": 0.6589186848327896, |
| "pass@128": 0.7109826589595376 |
| }, |
| "16": { |
| "pass@1": 0.17726814516129033, |
| "pass@2": 0.24868594424688859, |
| "pass@4": 0.3202650595086902, |
| "pass@8": 0.3873123584530507, |
| "pass@16": 0.4491814670825519, |
| "pass@32": 0.5042842654352826, |
| "pass@64": 0.5534790693723503, |
| "pass@128": 0.603225806451613 |
| }, |
| "17": { |
| "pass@1": 0.17332008670520233, |
| "pass@2": 0.24559291224796342, |
| "pass@4": 0.31781556992875903, |
| "pass@8": 0.3825153496828191, |
| "pass@16": 0.4364200708299305, |
| "pass@32": 0.48270561555806835, |
| "pass@64": 0.5258694560653755, |
| "pass@128": 0.5722543352601156 |
| }, |
| "18": { |
| "pass@1": 0.1690625, |
| "pass@2": 0.23962985095613049, |
| "pass@4": 0.30801571669612715, |
| "pass@8": 0.3677881716863713, |
| "pass@16": 0.415209138215289, |
| "pass@32": 0.45032772234979906, |
| "pass@64": 0.47887185765730594, |
| "pass@128": 0.5057142857142857 |
| }, |
| "19": { |
| "pass@1": 0.16786317567567569, |
| "pass@2": 0.23250772918587886, |
| "pass@4": 0.29525881421353867, |
| "pass@8": 0.3543126353276815, |
| "pass@16": 0.4073455414188086, |
| "pass@32": 0.45058219908918623, |
| "pass@64": 0.4831636850314373, |
| "pass@128": 0.5105105105105106 |
| }, |
| "20": { |
| "pass@1": 0.16061373873873874, |
| "pass@2": 0.22986273675250057, |
| "pass@4": 0.2993586457661261, |
| "pass@8": 0.35899482853932685, |
| "pass@16": 0.4044574963197297, |
| "pass@32": 0.4399765302164153, |
| "pass@64": 0.470242670848185, |
| "pass@128": 0.4954954954954955 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 483167, |
| "answer_accuracy": 0.5884243472330476, |
| "avg_response_len": 257.7551100935308, |
| "resp_tokens_sum": 211647876, |
| "pass_at_k": { |
| "pass@1": 0.5884243472330476, |
| "pass@2": 0.6496861594073996, |
| "pass@4": 0.6991931554687777, |
| "pass@8": 0.7400875770824119, |
| "pass@16": 0.7735274891578671, |
| "pass@32": 0.8004030340738901, |
| "pass@64": 0.8224152654722662, |
| "pass@128": 0.8420888542478566 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.790260736196319, |
| "pass@2": 0.8626221257427176, |
| "pass@4": 0.9026658627027448, |
| "pass@8": 0.9305973202449752, |
| "pass@16": 0.9520073304178969, |
| "pass@32": 0.9671571441192927, |
| "pass@64": 0.9765058182369718, |
| "pass@128": 0.9815950920245399 |
| }, |
| "2": { |
| "pass@1": 0.9332256838905775, |
| "pass@2": 0.9407051761481944, |
| "pass@4": 0.9479080504374642, |
| "pass@8": 0.9536852149913407, |
| "pass@16": 0.9571820803552304, |
| "pass@32": 0.9588901496981889, |
| "pass@64": 0.9604862016565554, |
| "pass@128": 0.9635258358662614 |
| }, |
| "3": { |
| "pass@1": 0.9784090909090909, |
| "pass@2": 0.9849987324027679, |
| "pass@4": 0.9886542293008828, |
| "pass@8": 0.9912084467155227, |
| "pass@16": 0.9930326420416548, |
| "pass@32": 0.9947539086640631, |
| "pass@64": 0.9964193746484429, |
| "pass@128": 0.996969696969697 |
| }, |
| "4": { |
| "pass@1": 0.9647512943786982, |
| "pass@2": 0.9728151353492055, |
| "pass@4": 0.9783021487143988, |
| "pass@8": 0.9830129515701709, |
| "pass@16": 0.9867103391258935, |
| "pass@32": 0.9897313735773864, |
| "pass@64": 0.9922419638528047, |
| "pass@128": 0.9940828402366864 |
| }, |
| "5": { |
| "pass@1": 0.9416666666666667, |
| "pass@2": 0.9582031695766293, |
| "pass@4": 0.9686339471153061, |
| "pass@8": 0.9767519230405057, |
| "pass@16": 0.9834265774804692, |
| "pass@32": 0.988204034915414, |
| "pass@64": 0.9920156285108184, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9282772206303725, |
| "pass@2": 0.9563044299799198, |
| "pass@4": 0.9710311721421637, |
| "pass@8": 0.9806274820666016, |
| "pass@16": 0.9861484876564786, |
| "pass@32": 0.9888576883172377, |
| "pass@64": 0.9899679684332302, |
| "pass@128": 0.9914040114613181 |
| }, |
| "7": { |
| "pass@1": 0.9178332731213873, |
| "pass@2": 0.9538034511401392, |
| "pass@4": 0.9726097203889971, |
| "pass@8": 0.9839673817925544, |
| "pass@16": 0.9899827277601754, |
| "pass@32": 0.9929938813526252, |
| "pass@64": 0.9941355497813412, |
| "pass@128": 0.9942196531791907 |
| }, |
| "8": { |
| "pass@1": 0.9106954225352113, |
| "pass@2": 0.9566988050349345, |
| "pass@4": 0.9760180632350526, |
| "pass@8": 0.9860794596604422, |
| "pass@16": 0.9915585599451117, |
| "pass@32": 0.993776910042879, |
| "pass@64": 0.9943270956076777, |
| "pass@128": 0.9943661971830986 |
| }, |
| "9": { |
| "pass@1": 0.8456212574850299, |
| "pass@2": 0.91207705160545, |
| "pass@4": 0.9416968105408975, |
| "pass@8": 0.957179736673861, |
| "pass@16": 0.9682130101641933, |
| "pass@32": 0.9766842369771581, |
| "pass@64": 0.982990924027013, |
| "pass@128": 0.9880239520958084 |
| }, |
| "11": { |
| "pass@1": 0.7269390060240963, |
| "pass@2": 0.8169354336637885, |
| "pass@4": 0.8682271524229653, |
| "pass@8": 0.8982871087556397, |
| "pass@16": 0.9182913182805088, |
| "pass@32": 0.9348106777196764, |
| "pass@64": 0.9505494615239103, |
| "pass@128": 0.9668674698795181 |
| }, |
| "12": { |
| "pass@1": 0.5305152821316614, |
| "pass@2": 0.6572589354034508, |
| "pass@4": 0.7495488091418038, |
| "pass@8": 0.8106145929971147, |
| "pass@16": 0.8533804682435675, |
| "pass@32": 0.8861680726994876, |
| "pass@64": 0.9133851877237164, |
| "pass@128": 0.9373040752351097 |
| }, |
| "13": { |
| "pass@1": 0.33661589635854344, |
| "pass@2": 0.45304883764529447, |
| "pass@4": 0.5610518070745358, |
| "pass@8": 0.6485457230363358, |
| "pass@16": 0.713696536353939, |
| "pass@32": 0.7628602973367136, |
| "pass@64": 0.8040627756332077, |
| "pass@128": 0.84593837535014 |
| }, |
| "14": { |
| "pass@1": 0.3016949926900585, |
| "pass@2": 0.40240868328958873, |
| "pass@4": 0.49299692554878016, |
| "pass@8": 0.5730546778713073, |
| "pass@16": 0.6428674497544992, |
| "pass@32": 0.6986742397132528, |
| "pass@64": 0.74331622198312, |
| "pass@128": 0.7894736842105263 |
| }, |
| "15": { |
| "pass@1": 0.2177556818181818, |
| "pass@2": 0.30779870555953226, |
| "pass@4": 0.3992948816057082, |
| "pass@8": 0.48379835112074754, |
| "pass@16": 0.5545522992571181, |
| "pass@32": 0.6100068408376694, |
| "pass@64": 0.657364349482816, |
| "pass@128": 0.706060606060606 |
| }, |
| "16": { |
| "pass@1": 0.1894141737891738, |
| "pass@2": 0.26630815779437816, |
| "pass@4": 0.34345242501738554, |
| "pass@8": 0.4152680205722794, |
| "pass@16": 0.47843937487739835, |
| "pass@32": 0.5340027097177347, |
| "pass@64": 0.582983948272668, |
| "pass@128": 0.6267806267806267 |
| }, |
| "17": { |
| "pass@1": 0.19863645186335405, |
| "pass@2": 0.279192852252164, |
| "pass@4": 0.3574614560990435, |
| "pass@8": 0.4286738857288767, |
| "pass@16": 0.4910720735207935, |
| "pass@32": 0.5474434388067502, |
| "pass@64": 0.5990456601352024, |
| "pass@128": 0.6428571428571429 |
| }, |
| "18": { |
| "pass@1": 0.16017922794117648, |
| "pass@2": 0.23308497857804533, |
| "pass@4": 0.3089515182844792, |
| "pass@8": 0.38097280125521266, |
| "pass@16": 0.44501758863725593, |
| "pass@32": 0.4989253440660845, |
| "pass@64": 0.541355544376601, |
| "pass@128": 0.5735294117647058 |
| }, |
| "19": { |
| "pass@1": 0.15239025297619047, |
| "pass@2": 0.21226228557367818, |
| "pass@4": 0.27195060968941387, |
| "pass@8": 0.33047229886625246, |
| "pass@16": 0.38482815880394616, |
| "pass@32": 0.43015119890887243, |
| "pass@64": 0.4641203141576692, |
| "pass@128": 0.4880952380952381 |
| }, |
| "20": { |
| "pass@1": 0.14095247005988024, |
| "pass@2": 0.20596835364703656, |
| "pass@4": 0.27415568525491196, |
| "pass@8": 0.3394977707056415, |
| "pass@16": 0.39794876064215573, |
| "pass@32": 0.44576809211309787, |
| "pass@64": 0.4837252202930043, |
| "pass@128": 0.5149700598802395 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 476008, |
| "answer_accuracy": 0.5941544176386004, |
| "avg_response_len": 280.10093839870586, |
| "resp_tokens_sum": 224403427, |
| "pass_at_k": { |
| "pass@1": 0.5941544176386004, |
| "pass@2": 0.6584226194594812, |
| "pass@4": 0.7107949916349928, |
| "pass@8": 0.7528320912137726, |
| "pass@16": 0.7859981786824436, |
| "pass@32": 0.8124843985122424, |
| "pass@64": 0.8346996523138441, |
| "pass@128": 0.8539702827927784 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8151119402985074, |
| "pass@2": 0.894133050299682, |
| "pass@4": 0.9361917815310395, |
| "pass@8": 0.9607177294918307, |
| "pass@16": 0.974920028102238, |
| "pass@32": 0.9823346279403201, |
| "pass@64": 0.9868942200091164, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 0.915625, |
| "pass@2": 0.9211642294713162, |
| "pass@4": 0.9274200423161388, |
| "pass@8": 0.9339374424325937, |
| "pass@16": 0.9398469239256345, |
| "pass@32": 0.9449957416667745, |
| "pass@64": 0.9503416731237622, |
| "pass@128": 0.9571428571428572 |
| }, |
| "3": { |
| "pass@1": 0.9802523688046647, |
| "pass@2": 0.9851332757512453, |
| "pass@4": 0.988258999695009, |
| "pass@8": 0.9907579902047114, |
| "pass@16": 0.9924813282950841, |
| "pass@32": 0.9939612434255555, |
| "pass@64": 0.9954531454092972, |
| "pass@128": 0.9970845481049563 |
| }, |
| "4": { |
| "pass@1": 0.9657574926900585, |
| "pass@2": 0.9755307621909105, |
| "pass@4": 0.9809962987082755, |
| "pass@8": 0.9845367405017438, |
| "pass@16": 0.9873080289127392, |
| "pass@32": 0.9893515838691211, |
| "pass@64": 0.9912087730850718, |
| "pass@128": 0.9941520467836257 |
| }, |
| "5": { |
| "pass@1": 0.9165670955882353, |
| "pass@2": 0.9386376794812418, |
| "pass@4": 0.9540271523963915, |
| "pass@8": 0.9649185404033847, |
| "pass@16": 0.9742903210799776, |
| "pass@32": 0.9831582523177278, |
| "pass@64": 0.9901467246024266, |
| "pass@128": 0.9941176470588236 |
| }, |
| "6": { |
| "pass@1": 0.9536179022082019, |
| "pass@2": 0.9753758476365533, |
| "pass@4": 0.9836247020778551, |
| "pass@8": 0.9886263415095881, |
| "pass@16": 0.992890469889173, |
| "pass@32": 0.99654159266549, |
| "pass@64": 0.9990623094181035, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9324564873417721, |
| "pass@2": 0.9607023385328421, |
| "pass@4": 0.9730168865363347, |
| "pass@8": 0.9809211880879508, |
| "pass@16": 0.9860712788116768, |
| "pass@32": 0.9893128993348704, |
| "pass@64": 0.9920828692662229, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.8937922297297297, |
| "pass@2": 0.9417517911612395, |
| "pass@4": 0.9637633128066195, |
| "pass@8": 0.9750247388470051, |
| "pass@16": 0.9810504117686079, |
| "pass@32": 0.9852425954657849, |
| "pass@64": 0.9892275719681659, |
| "pass@128": 0.993993993993994 |
| }, |
| "9": { |
| "pass@1": 0.8511605707395499, |
| "pass@2": 0.9153317815277108, |
| "pass@4": 0.9480443674444222, |
| "pass@8": 0.9669171484266056, |
| "pass@16": 0.9779593801637884, |
| "pass@32": 0.9841731235226076, |
| "pass@64": 0.9879175577920956, |
| "pass@128": 0.9903536977491961 |
| }, |
| "11": { |
| "pass@1": 0.732093253968254, |
| "pass@2": 0.8374496156730411, |
| "pass@4": 0.8996038209509525, |
| "pass@8": 0.9368779024733572, |
| "pass@16": 0.9593199778817806, |
| "pass@32": 0.9713870939894751, |
| "pass@64": 0.9776932080779764, |
| "pass@128": 0.9809523809523809 |
| }, |
| "12": { |
| "pass@1": 0.5414038608562691, |
| "pass@2": 0.6731839497700401, |
| "pass@4": 0.7681141700750705, |
| "pass@8": 0.8300111166340215, |
| "pass@16": 0.8687917152417938, |
| "pass@32": 0.8943989454268281, |
| "pass@64": 0.9125531792181364, |
| "pass@128": 0.926605504587156 |
| }, |
| "13": { |
| "pass@1": 0.37449363425925924, |
| "pass@2": 0.49851071133469455, |
| "pass@4": 0.6065300379119271, |
| "pass@8": 0.6926563227419713, |
| "pass@16": 0.7617440863768723, |
| "pass@32": 0.8180242034665416, |
| "pass@64": 0.8632191058162716, |
| "pass@128": 0.8981481481481481 |
| }, |
| "14": { |
| "pass@1": 0.3313473664688427, |
| "pass@2": 0.43997705845930957, |
| "pass@4": 0.5421407949006373, |
| "pass@8": 0.629466788219989, |
| "pass@16": 0.6999395215454379, |
| "pass@32": 0.7571167495219389, |
| "pass@64": 0.8039831562649183, |
| "pass@128": 0.8367952522255193 |
| }, |
| "15": { |
| "pass@1": 0.20691068672839505, |
| "pass@2": 0.3100724063137941, |
| "pass@4": 0.41896026306896816, |
| "pass@8": 0.5175979655164591, |
| "pass@16": 0.598636322043697, |
| "pass@32": 0.662957709462422, |
| "pass@64": 0.7151067850864165, |
| "pass@128": 0.7561728395061729 |
| }, |
| "16": { |
| "pass@1": 0.20342459439528024, |
| "pass@2": 0.28151348338094906, |
| "pass@4": 0.35446680531084046, |
| "pass@8": 0.4200273095429436, |
| "pass@16": 0.4821220436808709, |
| "pass@32": 0.5409573975320442, |
| "pass@64": 0.5938332249407281, |
| "pass@128": 0.640117994100295 |
| }, |
| "17": { |
| "pass@1": 0.1840879141566265, |
| "pass@2": 0.2620826564130538, |
| "pass@4": 0.3423274769494175, |
| "pass@8": 0.4160604187774409, |
| "pass@16": 0.47882764147063955, |
| "pass@32": 0.5320816045911806, |
| "pass@64": 0.5799399739308471, |
| "pass@128": 0.6295180722891566 |
| }, |
| "18": { |
| "pass@1": 0.1681451612903226, |
| "pass@2": 0.24457550165100325, |
| "pass@4": 0.3209164985425209, |
| "pass@8": 0.38626919031550705, |
| "pass@16": 0.4366118977394541, |
| "pass@32": 0.47500204091998266, |
| "pass@64": 0.5101726723947231, |
| "pass@128": 0.5451612903225806 |
| }, |
| "19": { |
| "pass@1": 0.16460536253776434, |
| "pass@2": 0.23573317850940825, |
| "pass@4": 0.30449151739340746, |
| "pass@8": 0.3636900639671202, |
| "pass@16": 0.41398196429348005, |
| "pass@32": 0.4596670842841002, |
| "pass@64": 0.5020669682882645, |
| "pass@128": 0.540785498489426 |
| }, |
| "20": { |
| "pass@1": 0.13051332582582584, |
| "pass@2": 0.20101166619375274, |
| "pass@4": 0.27972318691019477, |
| "pass@8": 0.35571918790213874, |
| "pass@16": 0.41978209084984897, |
| "pass@32": 0.46961062281713356, |
| "pass@64": 0.5115038624020116, |
| "pass@128": 0.5495495495495496 |
| } |
| } |
| } |
| } |
| } |
| ] |
