| [ |
| { |
| "checkpoint": "checkpoint-38796", |
| "id_acc": 0.7004678819444444, |
| "ood_acc": 0.331778125, |
| "total_acc": 0.5064206414473684, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 198.95842534722223, |
| "ood_avg_resp_len": 371.70684140625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 401802, |
| "answer_accuracy": 0.49621848324375595, |
| "avg_response_len": 276.35492288768575, |
| "resp_tokens_sum": 223772319, |
| "pass_at_k": { |
| "pass@1": 0.49621848324375595, |
| "pass@2": 0.6231529911240028, |
| "pass@4": 0.740067849446409, |
| "pass@8": 0.8317378367298127, |
| "pass@16": 0.8949596717478773, |
| "pass@32": 0.9350559220713367, |
| "pass@64": 0.9600439120925355, |
| "pass@128": 0.9753398672146696 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.5095409292035398, |
| "pass@2": 0.6676066418135783, |
| "pass@4": 0.792585876710101, |
| "pass@8": 0.8760817636863799, |
| "pass@16": 0.9283330831148673, |
| "pass@32": 0.9602328298448352, |
| "pass@64": 0.9791876634794598, |
| "pass@128": 0.9911504424778761 |
| }, |
| "2": { |
| "pass@1": 0.9343360591900312, |
| "pass@2": 0.9592853901439887, |
| "pass@4": 0.9779000282674944, |
| "pass@8": 0.9878251114374832, |
| "pass@16": 0.9922969056587208, |
| "pass@32": 0.9936688924295337, |
| "pass@64": 0.9937693490770984, |
| "pass@128": 0.9937694704049844 |
| }, |
| "3": { |
| "pass@1": 0.9195575305810397, |
| "pass@2": 0.9484343771822101, |
| "pass@4": 0.9708153361563746, |
| "pass@8": 0.984238780440124, |
| "pass@16": 0.9897953555580769, |
| "pass@32": 0.990792748570468, |
| "pass@64": 0.9908256717392108, |
| "pass@128": 0.9908256880733946 |
| }, |
| "4": { |
| "pass@1": 0.7866455078125, |
| "pass@2": 0.8459907111220479, |
| "pass@4": 0.8906744697928376, |
| "pass@8": 0.9215507509441558, |
| "pass@16": 0.9437763487973065, |
| "pass@32": 0.9611323349556897, |
| "pass@64": 0.9735555459387797, |
| "pass@128": 0.98125 |
| }, |
| "5": { |
| "pass@1": 0.6709573412698413, |
| "pass@2": 0.7741130014998127, |
| "pass@4": 0.8445359746698321, |
| "pass@8": 0.8894758925880082, |
| "pass@16": 0.9211739208974751, |
| "pass@32": 0.9452852732182677, |
| "pass@64": 0.964295583575878, |
| "pass@128": 0.9777777777777777 |
| }, |
| "6": { |
| "pass@1": 0.7365503368263473, |
| "pass@2": 0.8420378730727522, |
| "pass@4": 0.9124382115970529, |
| "pass@8": 0.9505378525143745, |
| "pass@16": 0.9685954065179547, |
| "pass@32": 0.97972634565819, |
| "pass@64": 0.9880378752295537, |
| "pass@128": 0.9940119760479041 |
| }, |
| "7": { |
| "pass@1": 0.6175110946745562, |
| "pass@2": 0.7572351983646276, |
| "pass@4": 0.8602960354807709, |
| "pass@8": 0.9199085087926454, |
| "pass@16": 0.9515635930824008, |
| "pass@32": 0.9699748728805815, |
| "pass@64": 0.9825938231042854, |
| "pass@128": 0.9911242603550295 |
| }, |
| "8": { |
| "pass@1": 0.5553635817307693, |
| "pass@2": 0.705602191853422, |
| "pass@4": 0.821673709956928, |
| "pass@8": 0.8943235864636814, |
| "pass@16": 0.9343372725706356, |
| "pass@32": 0.9560323516975169, |
| "pass@64": 0.9697434965886538, |
| "pass@128": 0.9807692307692307 |
| }, |
| "9": { |
| "pass@1": 0.479137323943662, |
| "pass@2": 0.6481472496395699, |
| "pass@4": 0.7957091023833289, |
| "pass@8": 0.8944526189705332, |
| "pass@16": 0.9475698851465509, |
| "pass@32": 0.9725689666299432, |
| "pass@64": 0.9838555505224045, |
| "pass@128": 0.9887323943661972 |
| }, |
| "11": { |
| "pass@1": 0.4631506728045326, |
| "pass@2": 0.6252648836742432, |
| "pass@4": 0.767624272798053, |
| "pass@8": 0.871077584229325, |
| "pass@16": 0.9329364153366072, |
| "pass@32": 0.9639094963103593, |
| "pass@64": 0.9780315644820233, |
| "pass@128": 0.9858356940509915 |
| }, |
| "12": { |
| "pass@1": 0.4075962217514124, |
| "pass@2": 0.5693932392455183, |
| "pass@4": 0.7213146841814255, |
| "pass@8": 0.8371093473559977, |
| "pass@16": 0.9086267449422752, |
| "pass@32": 0.9469433243538209, |
| "pass@64": 0.9686512377816593, |
| "pass@128": 0.9830508474576272 |
| }, |
| "13": { |
| "pass@1": 0.39882934952978055, |
| "pass@2": 0.5654986516673662, |
| "pass@4": 0.7186619650600091, |
| "pass@8": 0.8329572629488781, |
| "pass@16": 0.9032880963070734, |
| "pass@32": 0.9407221789825072, |
| "pass@64": 0.9613466943723601, |
| "pass@128": 0.9749216300940439 |
| }, |
| "14": { |
| "pass@1": 0.35604069314641745, |
| "pass@2": 0.5193354919420117, |
| "pass@4": 0.684745257018106, |
| "pass@8": 0.8188914601096546, |
| "pass@16": 0.9064022768205324, |
| "pass@32": 0.9520687996104888, |
| "pass@64": 0.9726630069903832, |
| "pass@128": 0.9813084112149533 |
| }, |
| "15": { |
| "pass@1": 0.33921152456647397, |
| "pass@2": 0.4931290820854757, |
| "pass@4": 0.650143435556827, |
| "pass@8": 0.7837146243177703, |
| "pass@16": 0.8784636507395034, |
| "pass@32": 0.9341449136359978, |
| "pass@64": 0.9638817286290657, |
| "pass@128": 0.9797687861271677 |
| }, |
| "16": { |
| "pass@1": 0.3211945564516129, |
| "pass@2": 0.47427451104902224, |
| "pass@4": 0.6338670295245349, |
| "pass@8": 0.7704789743204787, |
| "pass@16": 0.8674678402262296, |
| "pass@32": 0.9269334345131763, |
| "pass@64": 0.95867323469044, |
| "pass@128": 0.9774193548387097 |
| }, |
| "17": { |
| "pass@1": 0.2943686777456647, |
| "pass@2": 0.43410561080515186, |
| "pass@4": 0.5843995006404546, |
| "pass@8": 0.7225977142959461, |
| "pass@16": 0.8319339455322081, |
| "pass@32": 0.9085076669393349, |
| "pass@64": 0.9564128585594077, |
| "pass@128": 0.976878612716763 |
| }, |
| "18": { |
| "pass@1": 0.2702678571428571, |
| "pass@2": 0.40575260123734536, |
| "pass@4": 0.5552940701162348, |
| "pass@8": 0.6981102462106233, |
| "pass@16": 0.8141778003682741, |
| "pass@32": 0.894818854884761, |
| "pass@64": 0.9449149724164841, |
| "pass@128": 0.9685714285714285 |
| }, |
| "19": { |
| "pass@1": 0.23252158408408408, |
| "pass@2": 0.34998618204819004, |
| "pass@4": 0.48075028543729326, |
| "pass@8": 0.6082667201968596, |
| "pass@16": 0.7195758859617793, |
| "pass@32": 0.8085267342570868, |
| "pass@64": 0.8769663601857598, |
| "pass@128": 0.9279279279279279 |
| }, |
| "20": { |
| "pass@1": 0.19352946696696696, |
| "pass@2": 0.29999401468397513, |
| "pass@4": 0.42863513513513496, |
| "pass@8": 0.5593568000768633, |
| "pass@16": 0.6727429737905991, |
| "pass@32": 0.763426098355866, |
| "pass@64": 0.8337823046158129, |
| "pass@128": 0.8858858858858859 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 420450, |
| "answer_accuracy": 0.5120445245518317, |
| "avg_response_len": 286.99597988113794, |
| "resp_tokens_sum": 235658139, |
| "pass_at_k": { |
| "pass@1": 0.5120445245518317, |
| "pass@2": 0.6380669421753976, |
| "pass@4": 0.7534741674827613, |
| "pass@8": 0.8433371855207411, |
| "pass@16": 0.904921209981468, |
| "pass@32": 0.9434274968633595, |
| "pass@64": 0.9665472920084971, |
| "pass@128": 0.9800467653936087 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.4727521088957055, |
| "pass@2": 0.6345907957345057, |
| "pass@4": 0.77542852455943, |
| "pass@8": 0.8752167459860357, |
| "pass@16": 0.9336091179643451, |
| "pass@32": 0.9627348671897347, |
| "pass@64": 0.9780142800175672, |
| "pass@128": 0.9877300613496932 |
| }, |
| "2": { |
| "pass@1": 0.9381648936170213, |
| "pass@2": 0.9569980614125363, |
| "pass@4": 0.971669399322805, |
| "pass@8": 0.9823670230919718, |
| "pass@16": 0.9902618175090221, |
| "pass@32": 0.9948116541729152, |
| "pass@64": 0.9965635996372417, |
| "pass@128": 0.9969604863221885 |
| }, |
| "3": { |
| "pass@1": 0.950686553030303, |
| "pass@2": 0.9672959914101645, |
| "pass@4": 0.9803795730079196, |
| "pass@8": 0.9893317365210512, |
| "pass@16": 0.9939045470160155, |
| "pass@32": 0.9959745105782435, |
| "pass@64": 0.9968729838687728, |
| "pass@128": 0.996969696969697 |
| }, |
| "4": { |
| "pass@1": 0.8468703772189349, |
| "pass@2": 0.9065033429623076, |
| "pass@4": 0.9489256611044922, |
| "pass@8": 0.9743967832425964, |
| "pass@16": 0.9872230174014897, |
| "pass@32": 0.9930567852310904, |
| "pass@64": 0.9965815370797454, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.7758605072463768, |
| "pass@2": 0.8736209774049982, |
| "pass@4": 0.9324389600756424, |
| "pass@8": 0.9618868319100985, |
| "pass@16": 0.9760629639219559, |
| "pass@32": 0.9825935008857577, |
| "pass@64": 0.9869077034509552, |
| "pass@128": 0.991304347826087 |
| }, |
| "6": { |
| "pass@1": 0.6769340974212035, |
| "pass@2": 0.7943660740473344, |
| "pass@4": 0.8786454614018506, |
| "pass@8": 0.9272978475082277, |
| "pass@16": 0.9543528148099992, |
| "pass@32": 0.9711407927719327, |
| "pass@64": 0.9821259539329217, |
| "pass@128": 0.9885386819484241 |
| }, |
| "7": { |
| "pass@1": 0.6066428829479769, |
| "pass@2": 0.7398836818988667, |
| "pass@4": 0.8397892167380809, |
| "pass@8": 0.9042434643738783, |
| "pass@16": 0.9441212894901903, |
| "pass@32": 0.96651931919803, |
| "pass@64": 0.9778479793557311, |
| "pass@128": 0.9826589595375722 |
| }, |
| "8": { |
| "pass@1": 0.5825044014084507, |
| "pass@2": 0.7344803565487412, |
| "pass@4": 0.8506169114424071, |
| "pass@8": 0.9214648743029239, |
| "pass@16": 0.9596192806093881, |
| "pass@32": 0.9801233610769453, |
| "pass@64": 0.9908749402461413, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.5183383233532934, |
| "pass@2": 0.6750163550851055, |
| "pass@4": 0.8066507348071007, |
| "pass@8": 0.8964758234235036, |
| "pass@16": 0.9496126362729699, |
| "pass@32": 0.9775149761319307, |
| "pass@64": 0.9920008501197732, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.4303934487951807, |
| "pass@2": 0.5927442545773641, |
| "pass@4": 0.7420111311387282, |
| "pass@8": 0.851233626656646, |
| "pass@16": 0.9172232851276387, |
| "pass@32": 0.9531123039455416, |
| "pass@64": 0.9715619962195704, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.41724627742946707, |
| "pass@2": 0.577324331078913, |
| "pass@4": 0.7282632558861173, |
| "pass@8": 0.8475615268998387, |
| "pass@16": 0.9247642940425248, |
| "pass@32": 0.9665189848115742, |
| "pass@64": 0.9858146148658699, |
| "pass@128": 0.9937304075235109 |
| }, |
| "13": { |
| "pass@1": 0.3948266806722689, |
| "pass@2": 0.5571427882176495, |
| "pass@4": 0.7088043327567247, |
| "pass@8": 0.8258372669093875, |
| "pass@16": 0.9026431000892041, |
| "pass@32": 0.9463133172020857, |
| "pass@64": 0.9691138082317794, |
| "pass@128": 0.9831932773109243 |
| }, |
| "14": { |
| "pass@1": 0.41015625, |
| "pass@2": 0.5688616636736198, |
| "pass@4": 0.7169494355968666, |
| "pass@8": 0.831644961695096, |
| "pass@16": 0.9084661438307753, |
| "pass@32": 0.9549772171257439, |
| "pass@64": 0.9812690835166658, |
| "pass@128": 0.9941520467836257 |
| }, |
| "15": { |
| "pass@1": 0.33780776515151517, |
| "pass@2": 0.4990828561202573, |
| "pass@4": 0.6665392868505067, |
| "pass@8": 0.807065398160388, |
| "pass@16": 0.8994904127312967, |
| "pass@32": 0.9489088718389327, |
| "pass@64": 0.9717879055264091, |
| "pass@128": 0.9787878787878788 |
| }, |
| "16": { |
| "pass@1": 0.32298344017094016, |
| "pass@2": 0.4738822711263657, |
| "pass@4": 0.6295656763524213, |
| "pass@8": 0.7637132042842444, |
| "pass@16": 0.8619682864031663, |
| "pass@32": 0.9226160339647616, |
| "pass@64": 0.9550762847040853, |
| "pass@128": 0.9743589743589743 |
| }, |
| "17": { |
| "pass@1": 0.32038528726708076, |
| "pass@2": 0.4652963301462315, |
| "pass@4": 0.6150197741198496, |
| "pass@8": 0.7468171297086604, |
| "pass@16": 0.8433732629472633, |
| "pass@32": 0.9032106328795099, |
| "pass@64": 0.9403173018208404, |
| "pass@128": 0.968944099378882 |
| }, |
| "18": { |
| "pass@1": 0.2850873161764706, |
| "pass@2": 0.4292677454840204, |
| "pass@4": 0.5829545498908223, |
| "pass@8": 0.7183212578697612, |
| "pass@16": 0.8203104066913215, |
| "pass@32": 0.8889514779071954, |
| "pass@64": 0.932731144763619, |
| "pass@128": 0.9558823529411765 |
| }, |
| "19": { |
| "pass@1": 0.24262927827380953, |
| "pass@2": 0.3626660925196849, |
| "pass@4": 0.4944864068330747, |
| "pass@8": 0.621680543117198, |
| "pass@16": 0.7348198506994026, |
| "pass@32": 0.8285702067226429, |
| "pass@64": 0.8994453819696483, |
| "pass@128": 0.9434523809523809 |
| }, |
| "20": { |
| "pass@1": 0.19344124251497005, |
| "pass@2": 0.3052857130227736, |
| "pass@4": 0.43797066861777034, |
| "pass@8": 0.5705444770411404, |
| "pass@16": 0.6877959075104455, |
| "pass@32": 0.7850349116495576, |
| "pass@64": 0.8579923825438611, |
| "pass@128": 0.907185628742515 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 409363, |
| "answer_accuracy": 0.5109679561431538, |
| "avg_response_len": 306.50164388280876, |
| "resp_tokens_sum": 245554405, |
| "pass_at_k": { |
| "pass@1": 0.5109679561431538, |
| "pass@2": 0.6347774558965905, |
| "pass@4": 0.7470422575742857, |
| "pass@8": 0.8347069618219127, |
| "pass@16": 0.8956642572378208, |
| "pass@32": 0.9350350049432259, |
| "pass@64": 0.9601454722895479, |
| "pass@128": 0.975235660648666 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.5484141791044777, |
| "pass@2": 0.7068614848983433, |
| "pass@4": 0.8289243484489802, |
| "pass@8": 0.9026957799440277, |
| "pass@16": 0.9424595065592135, |
| "pass@32": 0.9669448671204803, |
| "pass@64": 0.9840754473966935, |
| "pass@128": 0.9970149253731343 |
| }, |
| "2": { |
| "pass@1": 0.9351785714285714, |
| "pass@2": 0.9614377109111365, |
| "pass@4": 0.9807448331458566, |
| "pass@8": 0.9900186920873438, |
| "pass@16": 0.9932031478280128, |
| "pass@32": 0.994165464267091, |
| "pass@64": 0.994284838023708, |
| "pass@128": 0.9942857142857143 |
| }, |
| "3": { |
| "pass@1": 0.9484784985422741, |
| "pass@2": 0.9667359421271322, |
| "pass@4": 0.9812006491171111, |
| "pass@8": 0.9911249057844892, |
| "pass@16": 0.9963589159034837, |
| "pass@32": 0.9986878666696775, |
| "pass@64": 0.9998071628884445, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.7850420321637427, |
| "pass@2": 0.8495130542892672, |
| "pass@4": 0.8935691234867573, |
| "pass@8": 0.9234471775450245, |
| "pass@16": 0.9452483337671119, |
| "pass@32": 0.961587906311577, |
| "pass@64": 0.9729863836696224, |
| "pass@128": 0.97953216374269 |
| }, |
| "5": { |
| "pass@1": 0.7065027573529412, |
| "pass@2": 0.8072114549559978, |
| "pass@4": 0.8746589903284145, |
| "pass@8": 0.9123061051506803, |
| "pass@16": 0.9350173413401953, |
| "pass@32": 0.9502013517528385, |
| "pass@64": 0.9608592051459295, |
| "pass@128": 0.9705882352941176 |
| }, |
| "6": { |
| "pass@1": 0.6834828470031545, |
| "pass@2": 0.7923822157778388, |
| "pass@4": 0.8667003048120562, |
| "pass@8": 0.9103088471153025, |
| "pass@16": 0.9358442713049175, |
| "pass@32": 0.9527734551605721, |
| "pass@64": 0.9654646330828386, |
| "pass@128": 0.9747634069400631 |
| }, |
| "7": { |
| "pass@1": 0.6329855617088608, |
| "pass@2": 0.7641914494667598, |
| "pass@4": 0.8574156207135493, |
| "pass@8": 0.9165753219580909, |
| "pass@16": 0.9548959002581976, |
| "pass@32": 0.9780368342011752, |
| "pass@64": 0.990102927075729, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.5770457957957958, |
| "pass@2": 0.7276407805443237, |
| "pass@4": 0.8416383075989373, |
| "pass@8": 0.911326672634802, |
| "pass@16": 0.949508092764589, |
| "pass@32": 0.9704760825523712, |
| "pass@64": 0.98276801411379, |
| "pass@128": 0.987987987987988 |
| }, |
| "9": { |
| "pass@1": 0.5122588424437299, |
| "pass@2": 0.6717535509026002, |
| "pass@4": 0.8050184239429874, |
| "pass@8": 0.8946679048869003, |
| "pass@16": 0.9452181368847813, |
| "pass@32": 0.9728178175087213, |
| "pass@64": 0.9876158326332224, |
| "pass@128": 0.9935691318327974 |
| }, |
| "11": { |
| "pass@1": 0.4708829365079365, |
| "pass@2": 0.6231510123734529, |
| "pass@4": 0.7544366772605811, |
| "pass@8": 0.851528774724346, |
| "pass@16": 0.9152272927693456, |
| "pass@32": 0.9535655892565871, |
| "pass@64": 0.9750103454894454, |
| "pass@128": 0.9841269841269841 |
| }, |
| "12": { |
| "pass@1": 0.43018922018348627, |
| "pass@2": 0.593299260757543, |
| "pass@4": 0.7401128262063567, |
| "pass@8": 0.849012069087901, |
| "pass@16": 0.916627492993055, |
| "pass@32": 0.9528701633670399, |
| "pass@64": 0.9721467280169921, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.39754533179012347, |
| "pass@2": 0.5523874945319335, |
| "pass@4": 0.6953536343258018, |
| "pass@8": 0.8065616697672022, |
| "pass@16": 0.8807671612053861, |
| "pass@32": 0.9246210517968243, |
| "pass@64": 0.9498011940741536, |
| "pass@128": 0.9660493827160493 |
| }, |
| "14": { |
| "pass@1": 0.38318341988130566, |
| "pass@2": 0.5392996185658544, |
| "pass@4": 0.6927449403579738, |
| "pass@8": 0.8184434802350886, |
| "pass@16": 0.9013070276922327, |
| "pass@32": 0.9460098896212308, |
| "pass@64": 0.9683616043843487, |
| "pass@128": 0.9792284866468842 |
| }, |
| "15": { |
| "pass@1": 0.3361304012345679, |
| "pass@2": 0.4982395863711481, |
| "pass@4": 0.6646675548658267, |
| "pass@8": 0.8041232007332922, |
| "pass@16": 0.8965907609063177, |
| "pass@32": 0.9461555988848293, |
| "pass@64": 0.9736658319254623, |
| "pass@128": 0.9907407407407407 |
| }, |
| "16": { |
| "pass@1": 0.3105872050147493, |
| "pass@2": 0.4608186421387594, |
| "pass@4": 0.6136408792374404, |
| "pass@8": 0.7452051705964161, |
| "pass@16": 0.844733132645075, |
| "pass@32": 0.9102423990647945, |
| "pass@64": 0.9475723541577755, |
| "pass@128": 0.9646017699115044 |
| }, |
| "17": { |
| "pass@1": 0.28572100903614456, |
| "pass@2": 0.4279750646285927, |
| "pass@4": 0.5799197509949807, |
| "pass@8": 0.7145075723252433, |
| "pass@16": 0.8179494250317864, |
| "pass@32": 0.8902762163574444, |
| "pass@64": 0.9388825184567935, |
| "pass@128": 0.9698795180722891 |
| }, |
| "18": { |
| "pass@1": 0.26066028225806454, |
| "pass@2": 0.39178943357886703, |
| "pass@4": 0.5395992352770422, |
| "pass@8": 0.6798815082751558, |
| "pass@16": 0.7896970588371004, |
| "pass@32": 0.8658843755573179, |
| "pass@64": 0.9173571118838663, |
| "pass@128": 0.9451612903225807 |
| }, |
| "19": { |
| "pass@1": 0.24596393504531722, |
| "pass@2": 0.37129121369269935, |
| "pass@4": 0.5090978352970231, |
| "pass@8": 0.6405254066302252, |
| "pass@16": 0.7512558089644261, |
| "pass@32": 0.8355988987470783, |
| "pass@64": 0.8976115860640348, |
| "pass@128": 0.9395770392749244 |
| }, |
| "20": { |
| "pass@1": 0.20038006756756757, |
| "pass@2": 0.3129104744508288, |
| "pass@4": 0.44692457558796145, |
| "pass@8": 0.5810482888254881, |
| "pass@16": 0.6969763264168647, |
| "pass@32": 0.7902839304149739, |
| "pass@64": 0.8623890297183104, |
| "pass@128": 0.9129129129129129 |
| } |
| } |
| } |
| } |
| } |
| ] |
