| [ |
| { |
| "checkpoint": "checkpoint-21414", |
| "id_acc": 0.6735208333333333, |
| "ood_acc": 0.22256484375, |
| "total_acc": 0.4361755756578947, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.16801041666668, |
| "ood_avg_resp_len": 311.04166015625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 349315, |
| "answer_accuracy": 0.43139795091685107, |
| "avg_response_len": 243.84849109824535, |
| "resp_tokens_sum": 197450951, |
| "pass_at_k": { |
| "pass@1": 0.43139795091685107, |
| "pass@2": 0.5321828105668215, |
| "pass@4": 0.6249946485161172, |
| "pass@8": 0.7011739986140003, |
| "pass@16": 0.7595677606039941, |
| "pass@32": 0.8034274001625996, |
| "pass@64": 0.8371937190423179, |
| "pass@128": 0.8632627252608284 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.4817247418879056, |
| "pass@2": 0.6352787552551507, |
| "pass@4": 0.7659861662756753, |
| "pass@8": 0.8593352023758045, |
| "pass@16": 0.9180698213960385, |
| "pass@32": 0.9501475446176046, |
| "pass@64": 0.9676992562490706, |
| "pass@128": 0.9793510324483776 |
| }, |
| "2": { |
| "pass@1": 0.9578222352024922, |
| "pass@2": 0.9749559999509407, |
| "pass@4": 0.9878366366587354, |
| "pass@8": 0.9955344824449235, |
| "pass@16": 0.9990460809752442, |
| "pass@32": 0.999941873566355, |
| "pass@64": 0.9999999112660973, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9316704892966361, |
| "pass@2": 0.949972459004551, |
| "pass@4": 0.9662891631091987, |
| "pass@8": 0.9793003255083925, |
| "pass@16": 0.9883260233328266, |
| "pass@32": 0.9940213091054283, |
| "pass@64": 0.9975285790814391, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.80537109375, |
| "pass@2": 0.8725716658464571, |
| "pass@4": 0.9234813232330332, |
| "pass@8": 0.956524622745434, |
| "pass@16": 0.9763327780316118, |
| "pass@32": 0.9879384001567445, |
| "pass@64": 0.9945018158032546, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.6077628968253969, |
| "pass@2": 0.730289963754531, |
| "pass@4": 0.8259401024276726, |
| "pass@8": 0.8936040569679891, |
| "pass@16": 0.9394573037217293, |
| "pass@32": 0.9679654619746364, |
| "pass@64": 0.9831936120425192, |
| "pass@128": 0.9904761904761905 |
| }, |
| "6": { |
| "pass@1": 0.6897688997005988, |
| "pass@2": 0.8093785362346175, |
| "pass@4": 0.8870302679230957, |
| "pass@8": 0.9347081087434049, |
| "pass@16": 0.9643132303145109, |
| "pass@32": 0.9813625491033848, |
| "pass@64": 0.9897868533872056, |
| "pass@128": 0.9940119760479041 |
| }, |
| "7": { |
| "pass@1": 0.5860068417159763, |
| "pass@2": 0.7297689628663284, |
| "pass@4": 0.8420394427397755, |
| "pass@8": 0.9139281489117121, |
| "pass@16": 0.9525323666683377, |
| "pass@32": 0.9726538898839411, |
| "pass@64": 0.9848167796739252, |
| "pass@128": 0.9911242603550295 |
| }, |
| "8": { |
| "pass@1": 0.505859375, |
| "pass@2": 0.6564822613819913, |
| "pass@4": 0.7820809541235229, |
| "pass@8": 0.8685459407960715, |
| "pass@16": 0.9199025646370973, |
| "pass@32": 0.9504180479169292, |
| "pass@64": 0.9696105221054736, |
| "pass@128": 0.9807692307692307 |
| }, |
| "9": { |
| "pass@1": 0.4445862676056338, |
| "pass@2": 0.6046994565820115, |
| "pass@4": 0.7470850805621114, |
| "pass@8": 0.8488156642340807, |
| "pass@16": 0.9112835745002482, |
| "pass@32": 0.9467934160269519, |
| "pass@64": 0.9656020765152085, |
| "pass@128": 0.9746478873239437 |
| }, |
| "11": { |
| "pass@1": 0.4284038597733711, |
| "pass@2": 0.5775366794182597, |
| "pass@4": 0.7094636056470274, |
| "pass@8": 0.8094007817227927, |
| "pass@16": 0.8754949614771798, |
| "pass@32": 0.9179393032615564, |
| "pass@64": 0.946461553261782, |
| "pass@128": 0.9660056657223796 |
| }, |
| "12": { |
| "pass@1": 0.3552921963276836, |
| "pass@2": 0.49981093465011817, |
| "pass@4": 0.6435842768065005, |
| "pass@8": 0.7688598850974901, |
| "pass@16": 0.8616190363221634, |
| "pass@32": 0.9181819016489402, |
| "pass@64": 0.9516632170824267, |
| "pass@128": 0.9745762711864406 |
| }, |
| "13": { |
| "pass@1": 0.312157131661442, |
| "pass@2": 0.438362763187125, |
| "pass@4": 0.5662403240935008, |
| "pass@8": 0.6810592242120193, |
| "pass@16": 0.7738279317724085, |
| "pass@32": 0.8394025480984975, |
| "pass@64": 0.8794518854511277, |
| "pass@128": 0.9028213166144201 |
| }, |
| "14": { |
| "pass@1": 0.2306756230529595, |
| "pass@2": 0.3417531336620305, |
| "pass@4": 0.4677710703802214, |
| "pass@8": 0.5904887371828694, |
| "pass@16": 0.693758241859579, |
| "pass@32": 0.7724491730569445, |
| "pass@64": 0.8299225014352747, |
| "pass@128": 0.8660436137071651 |
| }, |
| "15": { |
| "pass@1": 0.19102239884393063, |
| "pass@2": 0.2817613274316142, |
| "pass@4": 0.37920384635807797, |
| "pass@8": 0.47406381308256057, |
| "pass@16": 0.563398032120974, |
| "pass@32": 0.644849763727135, |
| "pass@64": 0.7142993770422786, |
| "pass@128": 0.7687861271676301 |
| }, |
| "16": { |
| "pass@1": 0.17033770161290324, |
| "pass@2": 0.2407516033782067, |
| "pass@4": 0.3186744777870508, |
| "pass@8": 0.398733871999996, |
| "pass@16": 0.47617025411495906, |
| "pass@32": 0.5478840483451405, |
| "pass@64": 0.6112842658780524, |
| "pass@128": 0.6645161290322581 |
| }, |
| "17": { |
| "pass@1": 0.158891799132948, |
| "pass@2": 0.2312060500204816, |
| "pass@4": 0.30643091163460046, |
| "pass@8": 0.3764142529721565, |
| "pass@16": 0.44026184076793384, |
| "pass@32": 0.5005950804847814, |
| "pass@64": 0.5607158592132552, |
| "pass@128": 0.6213872832369942 |
| }, |
| "18": { |
| "pass@1": 0.13390625, |
| "pass@2": 0.19612204724409452, |
| "pass@4": 0.26780344635491976, |
| "pass@8": 0.3450560776528902, |
| "pass@16": 0.42315467172835713, |
| "pass@32": 0.4987441792586379, |
| "pass@64": 0.571169957667318, |
| "pass@128": 0.6342857142857142 |
| }, |
| "19": { |
| "pass@1": 0.1423141891891892, |
| "pass@2": 0.21137439112340686, |
| "pass@4": 0.28596790322380866, |
| "pass@8": 0.3565394692761307, |
| "pass@16": 0.42062869719073964, |
| "pass@32": 0.4799759936316975, |
| "pass@64": 0.534573698675929, |
| "pass@128": 0.5855855855855856 |
| }, |
| "20": { |
| "pass@1": 0.12483577327327328, |
| "pass@2": 0.18133216878295616, |
| "pass@4": 0.2437110065633688, |
| "pass@8": 0.3046702644410265, |
| "pass@16": 0.36142120928657884, |
| "pass@32": 0.4167471278795742, |
| "pass@64": 0.4733304200251955, |
| "pass@128": 0.5255255255255256 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 360298, |
| "answer_accuracy": 0.43878848402182385, |
| "avg_response_len": 254.20170011691349, |
| "resp_tokens_sum": 208730100, |
| "pass_at_k": { |
| "pass@1": 0.43878848402182385, |
| "pass@2": 0.5369486884823405, |
| "pass@4": 0.6291625673593271, |
| "pass@8": 0.7060254278303971, |
| "pass@16": 0.7656998211258677, |
| "pass@32": 0.8113387439377043, |
| "pass@64": 0.8458824812617933, |
| "pass@128": 0.8717069368667186 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.4443539110429448, |
| "pass@2": 0.6050519147625724, |
| "pass@4": 0.7508354329558494, |
| "pass@8": 0.857326700302133, |
| "pass@16": 0.9225485327660016, |
| "pass@32": 0.9569656699143689, |
| "pass@64": 0.9739071748568061, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 0.9620298252279635, |
| "pass@2": 0.9795812591245241, |
| "pass@4": 0.9908501828608812, |
| "pass@8": 0.9970251835629206, |
| "pass@16": 0.9994825746890394, |
| "pass@32": 0.9999799709303407, |
| "pass@64": 0.9999999930684337, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.95625, |
| "pass@2": 0.9672706394655214, |
| "pass@4": 0.9773673733965071, |
| "pass@8": 0.9849303085747515, |
| "pass@16": 0.9890761521939984, |
| "pass@32": 0.9919127406602036, |
| "pass@64": 0.9945956899378898, |
| "pass@128": 0.996969696969697 |
| }, |
| "4": { |
| "pass@1": 0.8583811020710059, |
| "pass@2": 0.9200091436425475, |
| "pass@4": 0.9599025718123985, |
| "pass@8": 0.9819837740587968, |
| "pass@16": 0.992273429855794, |
| "pass@32": 0.9959718861628852, |
| "pass@64": 0.9969509246945819, |
| "pass@128": 0.9970414201183432 |
| }, |
| "5": { |
| "pass@1": 0.6773097826086957, |
| "pass@2": 0.7912629807143674, |
| "pass@4": 0.8727454549159614, |
| "pass@8": 0.9256251584652402, |
| "pass@16": 0.9592054453081453, |
| "pass@32": 0.9789833888272682, |
| "pass@64": 0.9877626724630637, |
| "pass@128": 0.991304347826087 |
| }, |
| "6": { |
| "pass@1": 0.6456617120343839, |
| "pass@2": 0.7710739909302169, |
| "pass@4": 0.8655726544210627, |
| "pass@8": 0.9255087613679311, |
| "pass@16": 0.9588882660952172, |
| "pass@32": 0.9775911162407339, |
| "pass@64": 0.9877839315981168, |
| "pass@128": 0.9914040114613181 |
| }, |
| "7": { |
| "pass@1": 0.5831376445086706, |
| "pass@2": 0.714123873515088, |
| "pass@4": 0.8156219727411236, |
| "pass@8": 0.8806761278760264, |
| "pass@16": 0.9226814672747715, |
| "pass@32": 0.9539407684583533, |
| "pass@64": 0.9767353629407124, |
| "pass@128": 0.9884393063583815 |
| }, |
| "8": { |
| "pass@1": 0.550088028169014, |
| "pass@2": 0.6991391260951532, |
| "pass@4": 0.8196430778899111, |
| "pass@8": 0.89965991789473, |
| "pass@16": 0.9473028675491105, |
| "pass@32": 0.9751506124431925, |
| "pass@64": 0.9897350817974143, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.4807728293413174, |
| "pass@2": 0.6240647396152582, |
| "pass@4": 0.7528413109102381, |
| "pass@8": 0.8474207898913209, |
| "pass@16": 0.9070636547105337, |
| "pass@32": 0.9438653815276806, |
| "pass@64": 0.9669455167868922, |
| "pass@128": 0.9820359281437125 |
| }, |
| "11": { |
| "pass@1": 0.3729527484939759, |
| "pass@2": 0.5160652452329001, |
| "pass@4": 0.6554858650010614, |
| "pass@8": 0.7738427672123679, |
| "pass@16": 0.8616539041891845, |
| "pass@32": 0.9187370376853801, |
| "pass@64": 0.9524868366449002, |
| "pass@128": 0.9728915662650602 |
| }, |
| "12": { |
| "pass@1": 0.35881171630094044, |
| "pass@2": 0.49483036309332806, |
| "pass@4": 0.6288250344098836, |
| "pass@8": 0.7445689588528022, |
| "pass@16": 0.833712635378144, |
| "pass@32": 0.8972884331775789, |
| "pass@64": 0.9391571594435375, |
| "pass@128": 0.9623824451410659 |
| }, |
| "13": { |
| "pass@1": 0.28223476890756305, |
| "pass@2": 0.40754165839564177, |
| "pass@4": 0.5387995284728062, |
| "pass@8": 0.6576386299571035, |
| "pass@16": 0.7543768701454598, |
| "pass@32": 0.8278516793921898, |
| "pass@64": 0.8779019857732024, |
| "pass@128": 0.9103641456582633 |
| }, |
| "14": { |
| "pass@1": 0.24312408625730994, |
| "pass@2": 0.34659735172906, |
| "pass@4": 0.45804868278526556, |
| "pass@8": 0.5631887917582749, |
| "pass@16": 0.6514521850564117, |
| "pass@32": 0.7211406756976264, |
| "pass@64": 0.7736040979948913, |
| "pass@128": 0.8128654970760234 |
| }, |
| "15": { |
| "pass@1": 0.18742897727272728, |
| "pass@2": 0.2791256561679787, |
| "pass@4": 0.3839303893831451, |
| "pass@8": 0.49218108231025, |
| "pass@16": 0.5963661486084444, |
| "pass@32": 0.6899128048407284, |
| "pass@64": 0.7652887002306195, |
| "pass@128": 0.8151515151515152 |
| }, |
| "16": { |
| "pass@1": 0.16312767094017094, |
| "pass@2": 0.2328744363685309, |
| "pass@4": 0.3091983616898317, |
| "pass@8": 0.3862526954911408, |
| "pass@16": 0.46173026023197594, |
| "pass@32": 0.5376746561844317, |
| "pass@64": 0.6142307830288806, |
| "pass@128": 0.6894586894586895 |
| }, |
| "17": { |
| "pass@1": 0.16386840062111802, |
| "pass@2": 0.23810797427495464, |
| "pass@4": 0.32328601500511184, |
| "pass@8": 0.41261976880930534, |
| "pass@16": 0.49869282658803993, |
| "pass@32": 0.5769880576200863, |
| "pass@64": 0.644537560302228, |
| "pass@128": 0.6987577639751553 |
| }, |
| "18": { |
| "pass@1": 0.14855238970588236, |
| "pass@2": 0.21801760074108376, |
| "pass@4": 0.2986294655263678, |
| "pass@8": 0.3826632892944409, |
| "pass@16": 0.46140967549262235, |
| "pass@32": 0.5326616971221702, |
| "pass@64": 0.6004062585650338, |
| "pass@128": 0.6676470588235294 |
| }, |
| "19": { |
| "pass@1": 0.14308965773809523, |
| "pass@2": 0.2120781044947507, |
| "pass@4": 0.2880920523885407, |
| "pass@8": 0.35969615277468225, |
| "pass@16": 0.42255468128199647, |
| "pass@32": 0.47800113481593476, |
| "pass@64": 0.5267301888452244, |
| "pass@128": 0.5684523809523809 |
| }, |
| "20": { |
| "pass@1": 0.11522267964071856, |
| "pass@2": 0.1793600299401198, |
| "pass@4": 0.25643922728221846, |
| "pass@8": 0.33409736909282767, |
| "pass@16": 0.4014528278273527, |
| "pass@32": 0.4552548863035034, |
| "pass@64": 0.49782715039176945, |
| "pass@128": 0.5299401197604791 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 351166, |
| "answer_accuracy": 0.43832631011343665, |
| "avg_response_len": 273.4684828846461, |
| "resp_tokens_sum": 219089822, |
| "pass_at_k": { |
| "pass@1": 0.43832631011343665, |
| "pass@2": 0.5365698394312263, |
| "pass@4": 0.6283680451181478, |
| "pass@8": 0.7046852593995614, |
| "pass@16": 0.7642488013994941, |
| "pass@32": 0.8099337635927337, |
| "pass@64": 0.8454458711849641, |
| "pass@128": 0.8739415242051446 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.5032882462686568, |
| "pass@2": 0.6513496738747213, |
| "pass@4": 0.7781951312056141, |
| "pass@8": 0.8691466965727291, |
| "pass@16": 0.9260160693597145, |
| "pass@32": 0.9589344283151598, |
| "pass@64": 0.9799965043319006, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 0.94046875, |
| "pass@2": 0.9615220753655792, |
| "pass@4": 0.9791949306336708, |
| "pass@8": 0.9910785397841038, |
| "pass@16": 0.9969192482415177, |
| "pass@32": 0.9991217509879896, |
| "pass@64": 0.9999135257784807, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.940620444606414, |
| "pass@2": 0.9625026543238218, |
| "pass@4": 0.9799381504972225, |
| "pass@8": 0.9913868834403958, |
| "pass@16": 0.9972989677312782, |
| "pass@32": 0.9995030708111858, |
| "pass@64": 0.9999804748054121, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.8034082602339181, |
| "pass@2": 0.8737722032278858, |
| "pass@4": 0.9267439889640982, |
| "pass@8": 0.9596337325518417, |
| "pass@16": 0.9772015339150868, |
| "pass@32": 0.9862561581036814, |
| "pass@64": 0.9904384863967542, |
| "pass@128": 0.9912280701754386 |
| }, |
| "5": { |
| "pass@1": 0.6249080882352941, |
| "pass@2": 0.745866503589625, |
| "pass@4": 0.8407373869626591, |
| "pass@8": 0.9027665657756448, |
| "pass@16": 0.9395420959632879, |
| "pass@32": 0.9609190845435237, |
| "pass@64": 0.9743793589403449, |
| "pass@128": 0.9823529411764705 |
| }, |
| "6": { |
| "pass@1": 0.6496944006309149, |
| "pass@2": 0.7748069531036532, |
| "pass@4": 0.8661763146128815, |
| "pass@8": 0.9248608618126145, |
| "pass@16": 0.9607558923277141, |
| "pass@32": 0.9806711476271339, |
| "pass@64": 0.9897304603908488, |
| "pass@128": 0.9936908517350158 |
| }, |
| "7": { |
| "pass@1": 0.5810176028481012, |
| "pass@2": 0.7143181407604903, |
| "pass@4": 0.8142075656286634, |
| "pass@8": 0.8796060112889925, |
| "pass@16": 0.9231276994518164, |
| "pass@32": 0.951785145268383, |
| "pass@64": 0.9706941203196205, |
| "pass@128": 0.9810126582278481 |
| }, |
| "8": { |
| "pass@1": 0.5080705705705706, |
| "pass@2": 0.6495878260149914, |
| "pass@4": 0.7686508476980916, |
| "pass@8": 0.854349197922715, |
| "pass@16": 0.911571770067261, |
| "pass@32": 0.9476719843601782, |
| "pass@64": 0.9698413001028867, |
| "pass@128": 0.984984984984985 |
| }, |
| "9": { |
| "pass@1": 0.45955586816720256, |
| "pass@2": 0.6173716516444286, |
| "pass@4": 0.7601637984560605, |
| "pass@8": 0.8648801163870649, |
| "pass@16": 0.9301138762865712, |
| "pass@32": 0.9680547564025581, |
| "pass@64": 0.9880174104714309, |
| "pass@128": 0.9967845659163987 |
| }, |
| "11": { |
| "pass@1": 0.4020337301587302, |
| "pass@2": 0.5445221691038621, |
| "pass@4": 0.6799382666452413, |
| "pass@8": 0.7912959175692053, |
| "pass@16": 0.8715446459982027, |
| "pass@32": 0.9231247651466039, |
| "pass@64": 0.9547388131097485, |
| "pass@128": 0.9746031746031746 |
| }, |
| "12": { |
| "pass@1": 0.35464449541284404, |
| "pass@2": 0.49375549314936534, |
| "pass@4": 0.6321710660594023, |
| "pass@8": 0.7495656664698744, |
| "pass@16": 0.8368415645553295, |
| "pass@32": 0.8947223371149916, |
| "pass@64": 0.9320921468349798, |
| "pass@128": 0.9602446483180428 |
| }, |
| "13": { |
| "pass@1": 0.31233121141975306, |
| "pass@2": 0.4359784345533197, |
| "pass@4": 0.5601907490151696, |
| "pass@8": 0.6717671604124168, |
| "pass@16": 0.7678836429673943, |
| "pass@32": 0.8445813822543843, |
| "pass@64": 0.8977074759369114, |
| "pass@128": 0.9320987654320988 |
| }, |
| "14": { |
| "pass@1": 0.24656899109792285, |
| "pass@2": 0.35336872648893636, |
| "pass@4": 0.4694582441846106, |
| "pass@8": 0.5798708865407574, |
| "pass@16": 0.672736222941444, |
| "pass@32": 0.7462808456444997, |
| "pass@64": 0.8034686946130885, |
| "pass@128": 0.8427299703264095 |
| }, |
| "15": { |
| "pass@1": 0.1877170138888889, |
| "pass@2": 0.28082508566637493, |
| "pass@4": 0.38921946533303714, |
| "pass@8": 0.5000600226201365, |
| "pass@16": 0.602305374771351, |
| "pass@32": 0.6889864561370689, |
| "pass@64": 0.7561926767193276, |
| "pass@128": 0.808641975308642 |
| }, |
| "16": { |
| "pass@1": 0.175954092920354, |
| "pass@2": 0.24397544886535194, |
| "pass@4": 0.31938721570201956, |
| "pass@8": 0.4002087109263321, |
| "pass@16": 0.4841250558928012, |
| "pass@32": 0.5685579542130321, |
| "pass@64": 0.6513467529523246, |
| "pass@128": 0.7286135693215339 |
| }, |
| "17": { |
| "pass@1": 0.15173192771084337, |
| "pass@2": 0.2243319982686652, |
| "pass@4": 0.3052337293493432, |
| "pass@8": 0.38670322107789923, |
| "pass@16": 0.4610427598188654, |
| "pass@32": 0.5284630277297131, |
| "pass@64": 0.5901698510578611, |
| "pass@128": 0.641566265060241 |
| }, |
| "18": { |
| "pass@1": 0.15791330645161292, |
| "pass@2": 0.22585883921767838, |
| "pass@4": 0.3001777695126821, |
| "pass@8": 0.3720267021326365, |
| "pass@16": 0.43655216978213546, |
| "pass@32": 0.4955005756619598, |
| "pass@64": 0.5516453645056283, |
| "pass@128": 0.6064516129032258 |
| }, |
| "19": { |
| "pass@1": 0.14999527945619334, |
| "pass@2": 0.2162124289316555, |
| "pass@4": 0.2851994351348076, |
| "pass@8": 0.35096117976459, |
| "pass@16": 0.4119157773312138, |
| "pass@32": 0.4694671341734886, |
| "pass@64": 0.5252148649384238, |
| "pass@128": 0.5861027190332326 |
| }, |
| "20": { |
| "pass@1": 0.11364489489489489, |
| "pass@2": 0.17525042266676127, |
| "pass@4": 0.2480603184737043, |
| "pass@8": 0.3236715042660682, |
| "pass@16": 0.39577047962332407, |
| "pass@32": 0.46366567785399415, |
| "pass@64": 0.5281737272922968, |
| "pass@128": 0.5915915915915916 |
| } |
| } |
| } |
| } |
| } |
| ] |
