| [ |
| { |
| "checkpoint": "checkpoint-21546", |
| "id_acc": 0.8766857638888889, |
| "ood_acc": 0.329509375, |
| "total_acc": 0.5886981907894737, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 199.3285625, |
| "ood_avg_resp_len": 317.06312109375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 476007, |
| "answer_accuracy": 0.5878603679260196, |
| "avg_response_len": 249.26987334018338, |
| "resp_tokens_sum": 201840796, |
| "pass_at_k": { |
| "pass@1": 0.5878603679260196, |
| "pass@2": 0.6611123113024427, |
| "pass@4": 0.715592607199787, |
| "pass@8": 0.7548355615364056, |
| "pass@16": 0.7838130901019593, |
| "pass@32": 0.8065087365881829, |
| "pass@64": 0.8256173393051519, |
| "pass@128": 0.8423964590578564 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.7548626474926253, |
| "pass@2": 0.8580390739321302, |
| "pass@4": 0.9158941481872283, |
| "pass@8": 0.9422260710100495, |
| "pass@16": 0.9541179241145453, |
| "pass@32": 0.9627354979168701, |
| "pass@64": 0.9700056559069217, |
| "pass@128": 0.9734513274336283 |
| }, |
| "2": { |
| "pass@1": 0.9521271417445483, |
| "pass@2": 0.9600596070351018, |
| "pass@4": 0.967638862899147, |
| "pass@8": 0.9736349720045984, |
| "pass@16": 0.9777725738486086, |
| "pass@32": 0.9807478572199139, |
| "pass@64": 0.983644411229528, |
| "pass@128": 0.9875389408099688 |
| }, |
| "3": { |
| "pass@1": 0.9738627675840978, |
| "pass@2": 0.9860556779599798, |
| "pass@4": 0.9911855877533655, |
| "pass@8": 0.9935549584465483, |
| "pass@16": 0.9950485984443537, |
| "pass@32": 0.9964154468880052, |
| "pass@64": 0.9980977119454602, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9468017578125, |
| "pass@2": 0.9668841504675199, |
| "pass@4": 0.9788314503655793, |
| "pass@8": 0.98423139689816, |
| "pass@16": 0.9869775056679988, |
| "pass@32": 0.988862756689439, |
| "pass@64": 0.9906228273186573, |
| "pass@128": 0.99375 |
| }, |
| "5": { |
| "pass@1": 0.8940972222222222, |
| "pass@2": 0.925370656792901, |
| "pass@4": 0.9444052689842337, |
| "pass@8": 0.956068245645845, |
| "pass@16": 0.9641325616691484, |
| "pass@32": 0.9702121366978933, |
| "pass@64": 0.9736402664360028, |
| "pass@128": 0.9746031746031746 |
| }, |
| "6": { |
| "pass@1": 0.874625748502994, |
| "pass@2": 0.9345000942995885, |
| "pass@4": 0.9660633426622565, |
| "pass@8": 0.9817471947893537, |
| "pass@16": 0.9901813682097385, |
| "pass@32": 0.9945595172809533, |
| "pass@64": 0.996547366723047, |
| "pass@128": 0.9970059880239521 |
| }, |
| "7": { |
| "pass@1": 0.8708394970414202, |
| "pass@2": 0.9296885919955274, |
| "pass@4": 0.9574697000707745, |
| "pass@8": 0.9709659057101204, |
| "pass@16": 0.9797032271920411, |
| "pass@32": 0.9856532059055091, |
| "pass@64": 0.9903706220860208, |
| "pass@128": 0.9970414201183432 |
| }, |
| "8": { |
| "pass@1": 0.8371394230769231, |
| "pass@2": 0.921493681228548, |
| "pass@4": 0.9620283975921279, |
| "pass@8": 0.9787379257212695, |
| "pass@16": 0.9860862836307394, |
| "pass@32": 0.9901621390004783, |
| "pass@64": 0.993815134792471, |
| "pass@128": 0.9967948717948718 |
| }, |
| "9": { |
| "pass@1": 0.7703125, |
| "pass@2": 0.8768198264389484, |
| "pass@4": 0.9369461128978587, |
| "pass@8": 0.9678318575372462, |
| "pass@16": 0.9839113625433821, |
| "pass@32": 0.993005940915507, |
| "pass@64": 0.9981911215016727, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.7148326841359773, |
| "pass@2": 0.8247275183466801, |
| "pass@4": 0.8932718025154789, |
| "pass@8": 0.9331762977429914, |
| "pass@16": 0.9563250136105693, |
| "pass@32": 0.9703408496523926, |
| "pass@64": 0.9791441852491506, |
| "pass@128": 0.9886685552407932 |
| }, |
| "12": { |
| "pass@1": 0.610787429378531, |
| "pass@2": 0.7386105504025978, |
| "pass@4": 0.8302062342842734, |
| "pass@8": 0.8869758223344203, |
| "pass@16": 0.9190229298739069, |
| "pass@32": 0.9382509994759017, |
| "pass@64": 0.9504507398120785, |
| "pass@128": 0.96045197740113 |
| }, |
| "13": { |
| "pass@1": 0.4938528605015674, |
| "pass@2": 0.6135426437192993, |
| "pass@4": 0.7050806219755448, |
| "pass@8": 0.7709119554640657, |
| "pass@16": 0.8185553713018926, |
| "pass@32": 0.8529551509357786, |
| "pass@64": 0.8789136115726013, |
| "pass@128": 0.896551724137931 |
| }, |
| "14": { |
| "pass@1": 0.3722984813084112, |
| "pass@2": 0.4832002600142271, |
| "pass@4": 0.5817811873983043, |
| "pass@8": 0.6600182956969374, |
| "pass@16": 0.7170059764733474, |
| "pass@32": 0.759723872506481, |
| "pass@64": 0.7973315503500162, |
| "pass@128": 0.8348909657320872 |
| }, |
| "15": { |
| "pass@1": 0.2915236632947977, |
| "pass@2": 0.38716340573938335, |
| "pass@4": 0.4724536182435286, |
| "pass@8": 0.5428825309662252, |
| "pass@16": 0.6009330341758621, |
| "pass@32": 0.6513234112092107, |
| "pass@64": 0.6978355864283469, |
| "pass@128": 0.7398843930635838 |
| }, |
| "16": { |
| "pass@1": 0.2159022177419355, |
| "pass@2": 0.294323485521971, |
| "pass@4": 0.3682993205486411, |
| "pass@8": 0.4345881141350131, |
| "pass@16": 0.49488662642953934, |
| "pass@32": 0.5483336279194447, |
| "pass@64": 0.5945119316894635, |
| "pass@128": 0.635483870967742 |
| }, |
| "17": { |
| "pass@1": 0.17767792630057805, |
| "pass@2": 0.24014361260297645, |
| "pass@4": 0.3024947796445963, |
| "pass@8": 0.36105936344277334, |
| "pass@16": 0.41425311433468764, |
| "pass@32": 0.4629499781277512, |
| "pass@64": 0.5069431688928129, |
| "pass@128": 0.5491329479768786 |
| }, |
| "18": { |
| "pass@1": 0.17955357142857142, |
| "pass@2": 0.24916057367829025, |
| "pass@4": 0.3168455503776314, |
| "pass@8": 0.37584058796751063, |
| "pass@16": 0.4238660306777394, |
| "pass@32": 0.4629081102911771, |
| "pass@64": 0.4982544518716383, |
| "pass@128": 0.5285714285714286 |
| }, |
| "19": { |
| "pass@1": 0.15315315315315314, |
| "pass@2": 0.21435005379395142, |
| "pass@4": 0.27603142118496454, |
| "pass@8": 0.3325890592148852, |
| "pass@16": 0.38104382491172395, |
| "pass@32": 0.4215814504369368, |
| "pass@64": 0.456708682959238, |
| "pass@128": 0.4864864864864865 |
| }, |
| "20": { |
| "pass@1": 0.13323479729729729, |
| "pass@2": 0.19331757939041397, |
| "pass@4": 0.2577742100931077, |
| "pass@8": 0.3186896897271893, |
| "pass@16": 0.37015786252682287, |
| "pass@32": 0.4131022452662304, |
| "pass@64": 0.4508702308920766, |
| "pass@128": 0.48348348348348347 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 482715, |
| "answer_accuracy": 0.5878738795791114, |
| "avg_response_len": 257.7196682579891, |
| "resp_tokens_sum": 211618774, |
| "pass_at_k": { |
| "pass@1": 0.5878738795791114, |
| "pass@2": 0.6570554679301156, |
| "pass@4": 0.710301292594661, |
| "pass@8": 0.750695245054591, |
| "pass@16": 0.7818350494228476, |
| "pass@32": 0.8066651512685107, |
| "pass@64": 0.8272927747768041, |
| "pass@128": 0.8455183164458301 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.7257716641104295, |
| "pass@2": 0.829145482102314, |
| "pass@4": 0.8909685864612009, |
| "pass@8": 0.923828926172029, |
| "pass@16": 0.9424341083861851, |
| "pass@32": 0.9547993211002885, |
| "pass@64": 0.9635684888575998, |
| "pass@128": 0.9693251533742331 |
| }, |
| "2": { |
| "pass@1": 0.969961056231003, |
| "pass@2": 0.9800300062226267, |
| "pass@4": 0.9871420935604933, |
| "pass@8": 0.9922565877731916, |
| "pass@16": 0.9959260151333992, |
| "pass@32": 0.9983344837584649, |
| "pass@64": 0.9996191683758167, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9798295454545455, |
| "pass@2": 0.9895311381531855, |
| "pass@4": 0.9931646833918487, |
| "pass@8": 0.9950567296773937, |
| "pass@16": 0.9963382395784798, |
| "pass@32": 0.9977819591272412, |
| "pass@64": 0.9992064994093033, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9674093934911243, |
| "pass@2": 0.9813876642361278, |
| "pass@4": 0.9896786493589189, |
| "pass@8": 0.9942140522761386, |
| "pass@16": 0.9964544182239704, |
| "pass@32": 0.9981567480120942, |
| "pass@64": 0.9996475795537232, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.8929347826086956, |
| "pass@2": 0.9245599395184297, |
| "pass@4": 0.9455650755612065, |
| "pass@8": 0.9585984031211746, |
| "pass@16": 0.9661113498054109, |
| "pass@32": 0.9713447438413154, |
| "pass@64": 0.9761022999756817, |
| "pass@128": 0.9797101449275363 |
| }, |
| "6": { |
| "pass@1": 0.8609643624641834, |
| "pass@2": 0.9185087313584368, |
| "pass@4": 0.9530109303135099, |
| "pass@8": 0.9730129786135173, |
| "pass@16": 0.9845427515918137, |
| "pass@32": 0.9913868693492569, |
| "pass@64": 0.9955681335039379, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8571396315028902, |
| "pass@2": 0.9151441815347509, |
| "pass@4": 0.9492684707764129, |
| "pass@8": 0.9707522712002761, |
| "pass@16": 0.9846804474521327, |
| "pass@32": 0.9932585974949043, |
| "pass@64": 0.9976034786090924, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8400528169014084, |
| "pass@2": 0.9146213402462021, |
| "pass@4": 0.9519286311042101, |
| "pass@8": 0.9705870784036298, |
| "pass@16": 0.9806850356855231, |
| "pass@32": 0.9870184734714211, |
| "pass@64": 0.9913704266572482, |
| "pass@128": 0.9943661971830986 |
| }, |
| "9": { |
| "pass@1": 0.7795424775449101, |
| "pass@2": 0.8696370791880805, |
| "pass@4": 0.9201581606191439, |
| "pass@8": 0.9472707355109283, |
| "pass@16": 0.9634659109310358, |
| "pass@32": 0.9732413865436497, |
| "pass@64": 0.979391838196961, |
| "pass@128": 0.9850299401197605 |
| }, |
| "11": { |
| "pass@1": 0.7141613328313253, |
| "pass@2": 0.8234709260269424, |
| "pass@4": 0.8924713595665003, |
| "pass@8": 0.9303646055635199, |
| "pass@16": 0.9515900662979302, |
| "pass@32": 0.9648603754748566, |
| "pass@64": 0.9738234865676904, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.625342868338558, |
| "pass@2": 0.7412462512033179, |
| "pass@4": 0.8177234021295925, |
| "pass@8": 0.864841454508076, |
| "pass@16": 0.8973079096792422, |
| "pass@32": 0.9206851353500622, |
| "pass@64": 0.9360387543338561, |
| "pass@128": 0.9467084639498433 |
| }, |
| "13": { |
| "pass@1": 0.45526960784313725, |
| "pass@2": 0.573507700324224, |
| "pass@4": 0.6657258832666921, |
| "pass@8": 0.7338637568424262, |
| "pass@16": 0.7852004994961729, |
| "pass@32": 0.8273113155682195, |
| "pass@64": 0.8624061133822667, |
| "pass@128": 0.8907563025210085 |
| }, |
| "14": { |
| "pass@1": 0.35798154239766083, |
| "pass@2": 0.45727353570935186, |
| "pass@4": 0.545599068756756, |
| "pass@8": 0.6210483819213921, |
| "pass@16": 0.68125171142204, |
| "pass@32": 0.7270073779808284, |
| "pass@64": 0.7631432072915221, |
| "pass@128": 0.7953216374269005 |
| }, |
| "15": { |
| "pass@1": 0.2765151515151515, |
| "pass@2": 0.36803939990455725, |
| "pass@4": 0.45397499403483615, |
| "pass@8": 0.5263473308035848, |
| "pass@16": 0.5859555392776711, |
| "pass@32": 0.6386314053079184, |
| "pass@64": 0.688474647802565, |
| "pass@128": 0.7333333333333333 |
| }, |
| "16": { |
| "pass@1": 0.19366542022792022, |
| "pass@2": 0.26023509881777596, |
| "pass@4": 0.3251300206063985, |
| "pass@8": 0.38605801417611135, |
| "pass@16": 0.44536576095291924, |
| "pass@32": 0.502109158651507, |
| "pass@64": 0.5541207044234497, |
| "pass@128": 0.603988603988604 |
| }, |
| "17": { |
| "pass@1": 0.20055318322981366, |
| "pass@2": 0.2769511572602337, |
| "pass@4": 0.3524708197383709, |
| "pass@8": 0.4216463758523163, |
| "pass@16": 0.4810756708240572, |
| "pass@32": 0.5287041717254924, |
| "pass@64": 0.5680075699156929, |
| "pass@128": 0.6055900621118012 |
| }, |
| "18": { |
| "pass@1": 0.19087775735294119, |
| "pass@2": 0.26312348019916604, |
| "pass@4": 0.3357639033172321, |
| "pass@8": 0.400534103180857, |
| "pass@16": 0.45257040732607284, |
| "pass@32": 0.49584444773542125, |
| "pass@64": 0.5358095049537671, |
| "pass@128": 0.5735294117647058 |
| }, |
| "19": { |
| "pass@1": 0.14274088541666666, |
| "pass@2": 0.20004298779058863, |
| "pass@4": 0.26311496163425996, |
| "pass@8": 0.3261681997932631, |
| "pass@16": 0.3816211319972022, |
| "pass@32": 0.4273529590398094, |
| "pass@64": 0.466436005280597, |
| "pass@128": 0.5 |
| }, |
| "20": { |
| "pass@1": 0.12951440868263472, |
| "pass@2": 0.18884229572351371, |
| "pass@4": 0.25496642738145764, |
| "pass@8": 0.3193190721203961, |
| "pass@16": 0.3750415414954998, |
| "pass@32": 0.4213703302393265, |
| "pass@64": 0.46037260804896324, |
| "pass@128": 0.49700598802395207 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 472992, |
| "answer_accuracy": 0.5903898386323694, |
| "avg_response_len": 277.1106219543857, |
| "resp_tokens_sum": 222007729, |
| "pass_at_k": { |
| "pass@1": 0.5903898386323694, |
| "pass@2": 0.6585257779977994, |
| "pass@4": 0.7106201799966714, |
| "pass@8": 0.750469997444581, |
| "pass@16": 0.7816516072340053, |
| "pass@32": 0.8062069138724718, |
| "pass@64": 0.8254911858450941, |
| "pass@128": 0.8407093784949673 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.7465951492537314, |
| "pass@2": 0.8382885033493949, |
| "pass@4": 0.8949848927466156, |
| "pass@8": 0.9280509232871795, |
| "pass@16": 0.9478438805819164, |
| "pass@32": 0.9594792695269607, |
| "pass@64": 0.9666690334826524, |
| "pass@128": 0.9731343283582089 |
| }, |
| "2": { |
| "pass@1": 0.9866741071428572, |
| "pass@2": 0.9914500140607426, |
| "pass@4": 0.9947826811827094, |
| "pass@8": 0.9964610707643552, |
| "pass@16": 0.9970452137532337, |
| "pass@32": 0.9971410795160666, |
| "pass@64": 0.9971428571124661, |
| "pass@128": 0.9971428571428571 |
| }, |
| "3": { |
| "pass@1": 0.9861060495626822, |
| "pass@2": 0.9948298076260877, |
| "pass@4": 0.9969340503355447, |
| "pass@8": 0.9972608802887731, |
| "pass@16": 0.9974489749023661, |
| "pass@32": 0.997813411078717, |
| "pass@64": 0.9985422740524781, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9719937865497076, |
| "pass@2": 0.981466132522908, |
| "pass@4": 0.9844356083778999, |
| "pass@8": 0.9855420893824135, |
| "pass@16": 0.9860609498652024, |
| "pass@32": 0.9866636688638668, |
| "pass@64": 0.9875788552746696, |
| "pass@128": 0.9883040935672515 |
| }, |
| "5": { |
| "pass@1": 0.9102711397058824, |
| "pass@2": 0.9282122076192687, |
| "pass@4": 0.940072399314056, |
| "pass@8": 0.9506308126253655, |
| "pass@16": 0.9599675251090849, |
| "pass@32": 0.9676808245522525, |
| "pass@64": 0.9735033828674721, |
| "pass@128": 0.9764705882352941 |
| }, |
| "6": { |
| "pass@1": 0.871870070977918, |
| "pass@2": 0.9295623338880746, |
| "pass@4": 0.9614348610603484, |
| "pass@8": 0.9783258917269404, |
| "pass@16": 0.9877960079219414, |
| "pass@32": 0.9940389600521968, |
| "pass@64": 0.99779786673376, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8647151898734177, |
| "pass@2": 0.9250473437655732, |
| "pass@4": 0.9549791028890373, |
| "pass@8": 0.9688374702660314, |
| "pass@16": 0.9767573806272281, |
| "pass@32": 0.9830239517261676, |
| "pass@64": 0.9885077193436845, |
| "pass@128": 0.9936708860759493 |
| }, |
| "8": { |
| "pass@1": 0.8060482357357357, |
| "pass@2": 0.883660235038188, |
| "pass@4": 0.9271438722186747, |
| "pass@8": 0.9515870560611387, |
| "pass@16": 0.9657731015433726, |
| "pass@32": 0.975180094457099, |
| "pass@64": 0.9815049830533622, |
| "pass@128": 0.984984984984985 |
| }, |
| "9": { |
| "pass@1": 0.7751456993569131, |
| "pass@2": 0.8718941470238243, |
| "pass@4": 0.9229569807592373, |
| "pass@8": 0.9491012856677312, |
| "pass@16": 0.9649184196419777, |
| "pass@32": 0.9739737051321179, |
| "pass@64": 0.9781618463030722, |
| "pass@128": 0.9807073954983923 |
| }, |
| "11": { |
| "pass@1": 0.6679811507936508, |
| "pass@2": 0.7739470066241724, |
| "pass@4": 0.8426609691645686, |
| "pass@8": 0.8882811236312015, |
| "pass@16": 0.9217558302441041, |
| "pass@32": 0.9453042212257897, |
| "pass@64": 0.958881139521916, |
| "pass@128": 0.9682539682539683 |
| }, |
| "12": { |
| "pass@1": 0.6055045871559633, |
| "pass@2": 0.7249553775674832, |
| "pass@4": 0.8070858390407616, |
| "pass@8": 0.8604887130755426, |
| "pass@16": 0.8984640220204004, |
| "pass@32": 0.926717664021228, |
| "pass@64": 0.9479442969224088, |
| "pass@128": 0.9663608562691132 |
| }, |
| "13": { |
| "pass@1": 0.44849537037037035, |
| "pass@2": 0.5667077531107221, |
| "pass@4": 0.6617636250329818, |
| "pass@8": 0.7328063948739421, |
| "pass@16": 0.7862427039253898, |
| "pass@32": 0.828873576525592, |
| "pass@64": 0.8648600832732204, |
| "pass@128": 0.8919753086419753 |
| }, |
| "14": { |
| "pass@1": 0.3864753338278932, |
| "pass@2": 0.4937969491109606, |
| "pass@4": 0.582945069644484, |
| "pass@8": 0.6549811145473605, |
| "pass@16": 0.7130843331326536, |
| "pass@32": 0.7584331108720384, |
| "pass@64": 0.7946619410176868, |
| "pass@128": 0.8249258160237388 |
| }, |
| "15": { |
| "pass@1": 0.2794897762345679, |
| "pass@2": 0.37721836419753096, |
| "pass@4": 0.4652975750716343, |
| "pass@8": 0.5389522735789639, |
| "pass@16": 0.6011663209854563, |
| "pass@32": 0.65646740261954, |
| "pass@64": 0.705641491781776, |
| "pass@128": 0.7469135802469136 |
| }, |
| "16": { |
| "pass@1": 0.21928005162241887, |
| "pass@2": 0.2919903955589622, |
| "pass@4": 0.36209677274853913, |
| "pass@8": 0.42627676402045145, |
| "pass@16": 0.4839805580211237, |
| "pass@32": 0.5352047911685343, |
| "pass@64": 0.5795251816926208, |
| "pass@128": 0.616519174041298 |
| }, |
| "17": { |
| "pass@1": 0.16629800451807228, |
| "pass@2": 0.23740372414856273, |
| "pass@4": 0.312128824070184, |
| "pass@8": 0.3813466482123387, |
| "pass@16": 0.4392210735431096, |
| "pass@32": 0.48650114540932765, |
| "pass@64": 0.5239750597065395, |
| "pass@128": 0.5542168674698795 |
| }, |
| "18": { |
| "pass@1": 0.17905745967741934, |
| "pass@2": 0.24398614109728226, |
| "pass@4": 0.31053360880293185, |
| "pass@8": 0.3724229822072563, |
| "pass@16": 0.42312145386859323, |
| "pass@32": 0.46238044186642907, |
| "pass@64": 0.495589661049704, |
| "pass@128": 0.5258064516129032 |
| }, |
| "19": { |
| "pass@1": 0.15674565709969787, |
| "pass@2": 0.2175691206556129, |
| "pass@4": 0.277465813846532, |
| "pass@8": 0.33085507142386955, |
| "pass@16": 0.37441386794026765, |
| "pass@32": 0.40776816099529734, |
| "pass@64": 0.43326206050798804, |
| "pass@128": 0.4501510574018127 |
| }, |
| "20": { |
| "pass@1": 0.14738175675675674, |
| "pass@2": 0.21154693078905676, |
| "pass@4": 0.2815054278125145, |
| "pass@8": 0.35082918530995677, |
| "pass@16": 0.41341781482613965, |
| "pass@32": 0.464339425513764, |
| "pass@64": 0.5009669935181226, |
| "pass@128": 0.5255255255255256 |
| } |
| } |
| } |
| } |
| } |
| ] |
