| [ |
| { |
| "checkpoint": "checkpoint-21546", |
| "id_acc": 0.9797482638888889, |
| "ood_acc": 0.41910390625, |
| "total_acc": 0.6846722861842105, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.59117881944445, |
| "ood_avg_resp_len": 317.97294609375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 551351, |
| "answer_accuracy": 0.6809088978027189, |
| "avg_response_len": 248.23700551296238, |
| "resp_tokens_sum": 201004454, |
| "pass_at_k": { |
| "pass@1": 0.6809088978027189, |
| "pass@2": 0.7179658930087301, |
| "pass@4": 0.7469747311907298, |
| "pass@8": 0.7700900354015988, |
| "pass@16": 0.7887883821084508, |
| "pass@32": 0.8044023259925125, |
| "pass@64": 0.8181816148916696, |
| "pass@128": 0.8313310148593108 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9221515486725663, |
| "pass@2": 0.9428836259958658, |
| "pass@4": 0.953557931641288, |
| "pass@8": 0.9603148730351716, |
| "pass@16": 0.9648846877139904, |
| "pass@32": 0.9680322261071029, |
| "pass@64": 0.9708755602536454, |
| "pass@128": 0.9734513274336283 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9978736620795107, |
| "pass@2": 0.9985266319921019, |
| "pass@4": 0.9993000800358671, |
| "pass@8": 0.9998489340659815, |
| "pass@16": 0.9999942173708687, |
| "pass@32": 0.9999999967965904, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9907470703125, |
| "pass@2": 0.9910037063238188, |
| "pass@4": 0.991343688484252, |
| "pass@8": 0.9919133814083988, |
| "pass@16": 0.9927106862872591, |
| "pass@32": 0.9934601647847077, |
| "pass@64": 0.9937403254241349, |
| "pass@128": 0.99375 |
| }, |
| "5": { |
| "pass@1": 0.9669642857142857, |
| "pass@2": 0.9718843738282714, |
| "pass@4": 0.9763600398164514, |
| "pass@8": 0.9802523092172719, |
| "pass@16": 0.9830724326754721, |
| "pass@32": 0.9851478752003268, |
| "pass@64": 0.987257685923108, |
| "pass@128": 0.9904761904761905 |
| }, |
| "6": { |
| "pass@1": 0.9956025449101796, |
| "pass@2": 0.9989277105238344, |
| "pass@4": 0.9996942885881777, |
| "pass@8": 0.9999718501697081, |
| "pass@16": 0.9999998309063756, |
| "pass@32": 0.999999999998971, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9926960059171598, |
| "pass@2": 0.9966541257047011, |
| "pass@4": 0.9984301580460726, |
| "pass@8": 0.9994525764888945, |
| "pass@16": 0.9999158567707788, |
| "pass@32": 0.9999987203425054, |
| "pass@64": 0.9999999999877117, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9768880208333334, |
| "pass@2": 0.9874886432465175, |
| "pass@4": 0.9895625994467037, |
| "pass@8": 0.9907045168965894, |
| "pass@16": 0.9916775319238652, |
| "pass@32": 0.9925913730797592, |
| "pass@64": 0.9933988491820491, |
| "pass@128": 0.9935897435897436 |
| }, |
| "9": { |
| "pass@1": 0.9691681338028169, |
| "pass@2": 0.9880122269047353, |
| "pass@4": 0.9939504940403571, |
| "pass@8": 0.9970743852484422, |
| "pass@16": 0.9988665501562034, |
| "pass@32": 0.9997334583192324, |
| "pass@64": 0.9999912787935963, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8784525495750708, |
| "pass@2": 0.9102096623987864, |
| "pass@4": 0.9281930339444113, |
| "pass@8": 0.9398208465107002, |
| "pass@16": 0.9490450774095365, |
| "pass@32": 0.9577149931620413, |
| "pass@64": 0.9651973316007761, |
| "pass@128": 0.9716713881019831 |
| }, |
| "12": { |
| "pass@1": 0.798927436440678, |
| "pass@2": 0.8551790699096934, |
| "pass@4": 0.8885044120544257, |
| "pass@8": 0.9085335882562506, |
| "pass@16": 0.9217108196453194, |
| "pass@32": 0.9324307195802108, |
| "pass@64": 0.9433761539311727, |
| "pass@128": 0.9548022598870056 |
| }, |
| "13": { |
| "pass@1": 0.6550989420062696, |
| "pass@2": 0.7359161719694916, |
| "pass@4": 0.7893547024294372, |
| "pass@8": 0.8251472079307166, |
| "pass@16": 0.8499261844339199, |
| "pass@32": 0.8690833324313215, |
| "pass@64": 0.887678042849582, |
| "pass@128": 0.9090909090909091 |
| }, |
| "14": { |
| "pass@1": 0.4941588785046729, |
| "pass@2": 0.5814315193661542, |
| "pass@4": 0.6522121046201, |
| "pass@8": 0.7054398474703508, |
| "pass@16": 0.7459692840506352, |
| "pass@32": 0.7797983464083315, |
| "pass@64": 0.8106658828186043, |
| "pass@128": 0.8411214953271028 |
| }, |
| "15": { |
| "pass@1": 0.36416184971098264, |
| "pass@2": 0.44411809885758474, |
| "pass@4": 0.5126349126879371, |
| "pass@8": 0.5702821524993034, |
| "pass@16": 0.6204606367913936, |
| "pass@32": 0.6645106758255857, |
| "pass@64": 0.7048887406631363, |
| "pass@128": 0.7456647398843931 |
| }, |
| "16": { |
| "pass@1": 0.26522177419354837, |
| "pass@2": 0.3349060198120396, |
| "pass@4": 0.39511467790316557, |
| "pass@8": 0.44689684761816845, |
| "pass@16": 0.4919076005163578, |
| "pass@32": 0.5289581705941678, |
| "pass@64": 0.557172877415548, |
| "pass@128": 0.5806451612903226 |
| }, |
| "17": { |
| "pass@1": 0.20822796242774566, |
| "pass@2": 0.2647282212917027, |
| "pass@4": 0.3119168785260225, |
| "pass@8": 0.353016882625172, |
| "pass@16": 0.39074588899191076, |
| "pass@32": 0.42409412095318333, |
| "pass@64": 0.450657781803306, |
| "pass@128": 0.47398843930635837 |
| }, |
| "18": { |
| "pass@1": 0.17683035714285714, |
| "pass@2": 0.23971140326209206, |
| "pass@4": 0.3019964647276231, |
| "pass@8": 0.35749287383033507, |
| "pass@16": 0.40207686154338784, |
| "pass@32": 0.437154461831571, |
| "pass@64": 0.467470346580905, |
| "pass@128": 0.4942857142857143 |
| }, |
| "19": { |
| "pass@1": 0.15927646396396397, |
| "pass@2": 0.21241775732425341, |
| "pass@4": 0.2633034674156721, |
| "pass@8": 0.3066705995032504, |
| "pass@16": 0.34242761359959684, |
| "pass@32": 0.3739619165348702, |
| "pass@64": 0.40473258265774553, |
| "pass@128": 0.43543543543543545 |
| }, |
| "20": { |
| "pass@1": 0.15995683183183182, |
| "pass@2": 0.21900899423045087, |
| "pass@4": 0.2765225937410976, |
| "pass@8": 0.32656027758034484, |
| "pass@16": 0.3677726314607193, |
| "pass@32": 0.4017703052549921, |
| "pass@64": 0.4317523725638508, |
| "pass@128": 0.4594594594594595 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 561524, |
| "answer_accuracy": 0.6838513250194855, |
| "avg_response_len": 257.001773187841, |
| "resp_tokens_sum": 211029296, |
| "pass_at_k": { |
| "pass@1": 0.6838513250194855, |
| "pass@2": 0.7182096587108185, |
| "pass@4": 0.7457970707480713, |
| "pass@8": 0.7688692845996465, |
| "pass@16": 0.7884647649459229, |
| "pass@32": 0.8053975893041211, |
| "pass@64": 0.8202428182245672, |
| "pass@128": 0.8325798908807482 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.902032208588957, |
| "pass@2": 0.928237539853147, |
| "pass@4": 0.9445890421672748, |
| "pass@8": 0.9554783664156684, |
| "pass@16": 0.9637377662720875, |
| "pass@32": 0.9702591907243098, |
| "pass@64": 0.975278890857173, |
| "pass@128": 0.9785276073619632 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9966619318181819, |
| "pass@2": 0.9969518014793606, |
| "pass@4": 0.9969696373180625, |
| "pass@8": 0.9969696969696016, |
| "pass@16": 0.996969696969697, |
| "pass@32": 0.996969696969697, |
| "pass@64": 0.996969696969697, |
| "pass@128": 0.996969696969697 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9875226449275363, |
| "pass@2": 0.9883676395070182, |
| "pass@4": 0.9884954541008462, |
| "pass@8": 0.9885869564329156, |
| "pass@16": 0.988768115942029, |
| "pass@32": 0.9891304347826086, |
| "pass@64": 0.9898550724637681, |
| "pass@128": 0.991304347826087 |
| }, |
| "6": { |
| "pass@1": 0.9941350286532952, |
| "pass@2": 0.9977561734314014, |
| "pass@4": 0.9985594472610693, |
| "pass@8": 0.9992944900423253, |
| "pass@16": 0.9998437548382668, |
| "pass@32": 0.9999948232370918, |
| "pass@64": 0.9999999995301811, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9928648843930635, |
| "pass@2": 0.994768672340813, |
| "pass@4": 0.9953643609758319, |
| "pass@8": 0.9960753760142124, |
| "pass@16": 0.9967665713625197, |
| "pass@32": 0.9970812204794673, |
| "pass@64": 0.9971097917066677, |
| "pass@128": 0.9971098265895953 |
| }, |
| "8": { |
| "pass@1": 0.9756382042253521, |
| "pass@2": 0.9838094016857049, |
| "pass@4": 0.9872804353681142, |
| "pass@8": 0.9894908076168281, |
| "pass@16": 0.9907958224370905, |
| "pass@32": 0.991898265474723, |
| "pass@64": 0.9929392340213707, |
| "pass@128": 0.9943661971830986 |
| }, |
| "9": { |
| "pass@1": 0.9612649700598802, |
| "pass@2": 0.9778384913244379, |
| "pass@4": 0.9821942192667527, |
| "pass@8": 0.983871105890422, |
| "pass@16": 0.9847190114530727, |
| "pass@32": 0.985008695002592, |
| "pass@64": 0.985029924450078, |
| "pass@128": 0.9850299401197605 |
| }, |
| "11": { |
| "pass@1": 0.8986257530120482, |
| "pass@2": 0.9296826825016609, |
| "pass@4": 0.9489027942478575, |
| "pass@8": 0.9620662130010917, |
| "pass@16": 0.9704384000316348, |
| "pass@32": 0.9762796497655822, |
| "pass@64": 0.9813886069516573, |
| "pass@128": 0.9849397590361446 |
| }, |
| "12": { |
| "pass@1": 0.8237166927899686, |
| "pass@2": 0.8677596543084939, |
| "pass@4": 0.8935126335109363, |
| "pass@8": 0.9109319966161393, |
| "pass@16": 0.9238355493757084, |
| "pass@32": 0.9343567941155181, |
| "pass@64": 0.9447863035001147, |
| "pass@128": 0.9561128526645768 |
| }, |
| "13": { |
| "pass@1": 0.5984768907563025, |
| "pass@2": 0.6788860721453939, |
| "pass@4": 0.7344253578071649, |
| "pass@8": 0.7775856673290666, |
| "pass@16": 0.8146780213187624, |
| "pass@32": 0.8475796206853868, |
| "pass@64": 0.8756093944707887, |
| "pass@128": 0.8935574229691877 |
| }, |
| "14": { |
| "pass@1": 0.48645376461988304, |
| "pass@2": 0.5669478404015286, |
| "pass@4": 0.6318697369517405, |
| "pass@8": 0.68492196122115, |
| "pass@16": 0.7292486257783408, |
| "pass@32": 0.7674189239076338, |
| "pass@64": 0.7990489164697606, |
| "pass@128": 0.8216374269005848 |
| }, |
| "15": { |
| "pass@1": 0.38359375, |
| "pass@2": 0.46019185456931494, |
| "pass@4": 0.5203419740146114, |
| "pass@8": 0.5682835301967295, |
| "pass@16": 0.6074261748472496, |
| "pass@32": 0.6400943942606238, |
| "pass@64": 0.6706869139430693, |
| "pass@128": 0.7 |
| }, |
| "16": { |
| "pass@1": 0.24243233618233617, |
| "pass@2": 0.3018344662045447, |
| "pass@4": 0.3575915131335078, |
| "pass@8": 0.4085050939332824, |
| "pass@16": 0.4546885384517169, |
| "pass@32": 0.49601365095000066, |
| "pass@64": 0.5296128337074691, |
| "pass@128": 0.5527065527065527 |
| }, |
| "17": { |
| "pass@1": 0.2296195652173913, |
| "pass@2": 0.29389167726316806, |
| "pass@4": 0.3527888585937627, |
| "pass@8": 0.4015310017732799, |
| "pass@16": 0.4411821281483737, |
| "pass@32": 0.47576190293681564, |
| "pass@64": 0.5078436970671439, |
| "pass@128": 0.5372670807453416 |
| }, |
| "18": { |
| "pass@1": 0.21167279411764706, |
| "pass@2": 0.26834254863362644, |
| "pass@4": 0.3236940603012858, |
| "pass@8": 0.3749530206752505, |
| "pass@16": 0.4188367625939018, |
| "pass@32": 0.45438906059280315, |
| "pass@64": 0.4836671899265589, |
| "pass@128": 0.5088235294117647 |
| }, |
| "19": { |
| "pass@1": 0.14625186011904762, |
| "pass@2": 0.1951546828130859, |
| "pass@4": 0.24635013452559498, |
| "pass@8": 0.29440603480328803, |
| "pass@16": 0.3372258333217718, |
| "pass@32": 0.37594895700766007, |
| "pass@64": 0.411830564552925, |
| "pass@128": 0.44642857142857145 |
| }, |
| "20": { |
| "pass@1": 0.1511742140718563, |
| "pass@2": 0.20507213918619455, |
| "pass@4": 0.25712532359353274, |
| "pass@8": 0.3054460965504392, |
| "pass@16": 0.351170875678362, |
| "pass@32": 0.39341225491743276, |
| "pass@64": 0.4319755908517848, |
| "pass@128": 0.46407185628742514 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 552248, |
| "answer_accuracy": 0.6893173829685253, |
| "avg_response_len": 276.4077965230069, |
| "resp_tokens_sum": 221444659, |
| "pass_at_k": { |
| "pass@1": 0.6893173829685253, |
| "pass@2": 0.7236746211439762, |
| "pass@4": 0.7508164814845082, |
| "pass@8": 0.7726614849119308, |
| "pass@16": 0.790704121070079, |
| "pass@32": 0.8062121975664928, |
| "pass@64": 0.8199103369645999, |
| "pass@128": 0.8324013420674229 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9242537313432836, |
| "pass@2": 0.9467549065695148, |
| "pass@4": 0.960208038827236, |
| "pass@8": 0.9686813621778994, |
| "pass@16": 0.9736968715149406, |
| "pass@32": 0.9769121125587591, |
| "pass@64": 0.9802306191654232, |
| "pass@128": 0.9850746268656716 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997494533527697, |
| "pass@2": 0.9999802719175408, |
| "pass@4": 0.9999999098144802, |
| "pass@8": 0.9999999999996636, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999771564327485, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9793658088235294, |
| "pass@2": 0.982123161764706, |
| "pass@4": 0.9840028584662212, |
| "pass@8": 0.9856092514818663, |
| "pass@16": 0.9870776742919471, |
| "pass@32": 0.9886067946246756, |
| "pass@64": 0.9902717923000518, |
| "pass@128": 0.9911764705882353 |
| }, |
| "6": { |
| "pass@1": 0.9942576892744479, |
| "pass@2": 0.9967119153481209, |
| "pass@4": 0.9968445709861974, |
| "pass@8": 0.9968454258391087, |
| "pass@16": 0.9968454258675079, |
| "pass@32": 0.9968454258675079, |
| "pass@64": 0.9968454258675079, |
| "pass@128": 0.9968454258675079 |
| }, |
| "7": { |
| "pass@1": 0.9869709256329114, |
| "pass@2": 0.9900737721768167, |
| "pass@4": 0.9904884912526759, |
| "pass@8": 0.9905062765613193, |
| "pass@16": 0.9905063291137864, |
| "pass@32": 0.990506329113924, |
| "pass@64": 0.990506329113924, |
| "pass@128": 0.990506329113924 |
| }, |
| "8": { |
| "pass@1": 0.9748263888888888, |
| "pass@2": 0.9825904891111588, |
| "pass@4": 0.9846056751352027, |
| "pass@8": 0.9865519125295662, |
| "pass@16": 0.9890839997485931, |
| "pass@32": 0.9917298739262829, |
| "pass@64": 0.9936165397127583, |
| "pass@128": 0.993993993993994 |
| }, |
| "9": { |
| "pass@1": 0.9695538585209004, |
| "pass@2": 0.9819578860419779, |
| "pass@4": 0.9849974913706525, |
| "pass@8": 0.9868120170628933, |
| "pass@16": 0.9885209776830189, |
| "pass@32": 0.9898635914465111, |
| "pass@64": 0.9903321210779717, |
| "pass@128": 0.9903536977491961 |
| }, |
| "11": { |
| "pass@1": 0.8884672619047619, |
| "pass@2": 0.9179161198600174, |
| "pass@4": 0.9360529898048454, |
| "pass@8": 0.9490142660043712, |
| "pass@16": 0.958196320440275, |
| "pass@32": 0.9647608513937915, |
| "pass@64": 0.9700160392867752, |
| "pass@128": 0.9746031746031746 |
| }, |
| "12": { |
| "pass@1": 0.8331660932721713, |
| "pass@2": 0.8894891220592839, |
| "pass@4": 0.9224128669696099, |
| "pass@8": 0.9411316213230453, |
| "pass@16": 0.9521785605790735, |
| "pass@32": 0.9594392629935055, |
| "pass@64": 0.9642912699886997, |
| "pass@128": 0.9663608562691132 |
| }, |
| "13": { |
| "pass@1": 0.5963059413580247, |
| "pass@2": 0.6718958849761836, |
| "pass@4": 0.7258452271012411, |
| "pass@8": 0.7661379023691128, |
| "pass@16": 0.7994569530613119, |
| "pass@32": 0.8298630341960512, |
| "pass@64": 0.8566462178607112, |
| "pass@128": 0.8765432098765432 |
| }, |
| "14": { |
| "pass@1": 0.5224406528189911, |
| "pass@2": 0.6048735805743123, |
| "pass@4": 0.667954188960801, |
| "pass@8": 0.714907466749094, |
| "pass@16": 0.7518824252298536, |
| "pass@32": 0.7844183856251077, |
| "pass@64": 0.814024580720177, |
| "pass@128": 0.8427299703264095 |
| }, |
| "15": { |
| "pass@1": 0.39817226080246915, |
| "pass@2": 0.46172277510450077, |
| "pass@4": 0.516345238673869, |
| "pass@8": 0.5643374864301807, |
| "pass@16": 0.6066126531536483, |
| "pass@32": 0.6453878305545641, |
| "pass@64": 0.6818109357935861, |
| "pass@128": 0.7160493827160493 |
| }, |
| "16": { |
| "pass@1": 0.2555309734513274, |
| "pass@2": 0.31953203028824917, |
| "pass@4": 0.378190404296808, |
| "pass@8": 0.4293594132098313, |
| "pass@16": 0.47360209270500697, |
| "pass@32": 0.5121977074655605, |
| "pass@64": 0.5497008456356067, |
| "pass@128": 0.5870206489675516 |
| }, |
| "17": { |
| "pass@1": 0.21279649849397592, |
| "pass@2": 0.2769127691869842, |
| "pass@4": 0.3335075267323512, |
| "pass@8": 0.3784226935397351, |
| "pass@16": 0.41479042093392043, |
| "pass@32": 0.44466458523149915, |
| "pass@64": 0.4693555810200645, |
| "pass@128": 0.4939759036144578 |
| }, |
| "18": { |
| "pass@1": 0.21242439516129033, |
| "pass@2": 0.2739847917195834, |
| "pass@4": 0.33244598044196094, |
| "pass@8": 0.38322910594677906, |
| "pass@16": 0.4248379459806415, |
| "pass@32": 0.458828979622347, |
| "pass@64": 0.48675039253010505, |
| "pass@128": 0.5096774193548387 |
| }, |
| "19": { |
| "pass@1": 0.17817692598187312, |
| "pass@2": 0.23166459012298704, |
| "pass@4": 0.27631746239424, |
| "pass@8": 0.31228890701309736, |
| "pass@16": 0.34476007943757514, |
| "pass@32": 0.3760017219184328, |
| "pass@64": 0.40610132605649063, |
| "pass@128": 0.4350453172205438 |
| }, |
| "20": { |
| "pass@1": 0.15322353603603603, |
| "pass@2": 0.2071318365609704, |
| "pass@4": 0.2625266365915972, |
| "pass@8": 0.3153373034222012, |
| "pass@16": 0.36097657025856844, |
| "pass@32": 0.39841696589607256, |
| "pass@64": 0.42861872538402224, |
| "pass@128": 0.45645645645645644 |
| } |
| } |
| } |
| } |
| } |
| ] |
