[ { "checkpoint": "checkpoint-21546", "id_acc": 0.9797482638888889, "ood_acc": 0.41910390625, "total_acc": 0.6846722861842105, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.59117881944445, "ood_avg_resp_len": 317.97294609375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 551351, "answer_accuracy": 0.6809088978027189, "avg_response_len": 248.23700551296238, "resp_tokens_sum": 201004454, "pass_at_k": { "pass@1": 0.6809088978027189, "pass@2": 0.7179658930087301, "pass@4": 0.7469747311907298, "pass@8": 0.7700900354015988, "pass@16": 0.7887883821084508, "pass@32": 0.8044023259925125, "pass@64": 0.8181816148916696, "pass@128": 0.8313310148593108 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9221515486725663, "pass@2": 0.9428836259958658, "pass@4": 0.953557931641288, "pass@8": 0.9603148730351716, "pass@16": 0.9648846877139904, "pass@32": 0.9680322261071029, "pass@64": 0.9708755602536454, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9978736620795107, "pass@2": 0.9985266319921019, "pass@4": 0.9993000800358671, "pass@8": 0.9998489340659815, "pass@16": 0.9999942173708687, "pass@32": 0.9999999967965904, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9907470703125, "pass@2": 0.9910037063238188, "pass@4": 0.991343688484252, "pass@8": 0.9919133814083988, "pass@16": 0.9927106862872591, "pass@32": 0.9934601647847077, "pass@64": 0.9937403254241349, "pass@128": 0.99375 }, "5": { "pass@1": 0.9669642857142857, "pass@2": 0.9718843738282714, "pass@4": 0.9763600398164514, "pass@8": 0.9802523092172719, "pass@16": 0.9830724326754721, "pass@32": 0.9851478752003268, "pass@64": 0.987257685923108, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.9956025449101796, "pass@2": 0.9989277105238344, "pass@4": 0.9996942885881777, "pass@8": 0.9999718501697081, "pass@16": 0.9999998309063756, "pass@32": 0.999999999998971, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9926960059171598, "pass@2": 0.9966541257047011, "pass@4": 0.9984301580460726, "pass@8": 0.9994525764888945, "pass@16": 0.9999158567707788, "pass@32": 0.9999987203425054, "pass@64": 0.9999999999877117, "pass@128": 1.0 }, "8": { "pass@1": 0.9768880208333334, "pass@2": 0.9874886432465175, "pass@4": 0.9895625994467037, "pass@8": 0.9907045168965894, "pass@16": 0.9916775319238652, "pass@32": 0.9925913730797592, "pass@64": 0.9933988491820491, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.9691681338028169, "pass@2": 0.9880122269047353, "pass@4": 0.9939504940403571, "pass@8": 0.9970743852484422, "pass@16": 0.9988665501562034, "pass@32": 0.9997334583192324, "pass@64": 0.9999912787935963, "pass@128": 1.0 }, "11": { "pass@1": 0.8784525495750708, "pass@2": 0.9102096623987864, "pass@4": 0.9281930339444113, "pass@8": 0.9398208465107002, "pass@16": 0.9490450774095365, "pass@32": 0.9577149931620413, "pass@64": 0.9651973316007761, "pass@128": 0.9716713881019831 }, "12": { "pass@1": 0.798927436440678, "pass@2": 0.8551790699096934, "pass@4": 0.8885044120544257, "pass@8": 0.9085335882562506, "pass@16": 0.9217108196453194, "pass@32": 0.9324307195802108, "pass@64": 0.9433761539311727, "pass@128": 0.9548022598870056 }, "13": { "pass@1": 0.6550989420062696, "pass@2": 0.7359161719694916, "pass@4": 0.7893547024294372, "pass@8": 0.8251472079307166, "pass@16": 0.8499261844339199, "pass@32": 0.8690833324313215, "pass@64": 0.887678042849582, "pass@128": 0.9090909090909091 }, "14": { "pass@1": 0.4941588785046729, "pass@2": 0.5814315193661542, "pass@4": 0.6522121046201, "pass@8": 0.7054398474703508, "pass@16": 0.7459692840506352, "pass@32": 0.7797983464083315, "pass@64": 0.8106658828186043, "pass@128": 0.8411214953271028 }, "15": { "pass@1": 0.36416184971098264, "pass@2": 0.44411809885758474, "pass@4": 0.5126349126879371, "pass@8": 0.5702821524993034, "pass@16": 0.6204606367913936, "pass@32": 0.6645106758255857, "pass@64": 0.7048887406631363, "pass@128": 0.7456647398843931 }, "16": { "pass@1": 0.26522177419354837, "pass@2": 0.3349060198120396, "pass@4": 0.39511467790316557, "pass@8": 0.44689684761816845, "pass@16": 0.4919076005163578, "pass@32": 0.5289581705941678, "pass@64": 0.557172877415548, "pass@128": 0.5806451612903226 }, "17": { "pass@1": 0.20822796242774566, "pass@2": 0.2647282212917027, "pass@4": 0.3119168785260225, "pass@8": 0.353016882625172, "pass@16": 0.39074588899191076, "pass@32": 0.42409412095318333, "pass@64": 0.450657781803306, "pass@128": 0.47398843930635837 }, "18": { "pass@1": 0.17683035714285714, "pass@2": 0.23971140326209206, "pass@4": 0.3019964647276231, "pass@8": 0.35749287383033507, "pass@16": 0.40207686154338784, "pass@32": 0.437154461831571, "pass@64": 0.467470346580905, "pass@128": 0.4942857142857143 }, "19": { "pass@1": 0.15927646396396397, "pass@2": 0.21241775732425341, "pass@4": 0.2633034674156721, "pass@8": 0.3066705995032504, "pass@16": 0.34242761359959684, "pass@32": 0.3739619165348702, "pass@64": 0.40473258265774553, "pass@128": 0.43543543543543545 }, "20": { "pass@1": 0.15995683183183182, "pass@2": 0.21900899423045087, "pass@4": 0.2765225937410976, "pass@8": 0.32656027758034484, "pass@16": 0.3677726314607193, "pass@32": 0.4017703052549921, "pass@64": 0.4317523725638508, "pass@128": 0.4594594594594595 } } }, "teachers_in_school": { "count": 821120, "correct": 561524, "answer_accuracy": 0.6838513250194855, "avg_response_len": 257.001773187841, "resp_tokens_sum": 211029296, "pass_at_k": { "pass@1": 0.6838513250194855, "pass@2": 0.7182096587108185, "pass@4": 0.7457970707480713, "pass@8": 0.7688692845996465, "pass@16": 0.7884647649459229, "pass@32": 0.8053975893041211, "pass@64": 0.8202428182245672, "pass@128": 0.8325798908807482 }, "per_op_pass_at_k": { "10": { "pass@1": 0.902032208588957, "pass@2": 0.928237539853147, "pass@4": 0.9445890421672748, "pass@8": 0.9554783664156684, "pass@16": 0.9637377662720875, "pass@32": 0.9702591907243098, "pass@64": 0.975278890857173, "pass@128": 0.9785276073619632 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9966619318181819, "pass@2": 0.9969518014793606, "pass@4": 0.9969696373180625, "pass@8": 0.9969696969696016, "pass@16": 0.996969696969697, "pass@32": 0.996969696969697, "pass@64": 0.996969696969697, "pass@128": 0.996969696969697 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9875226449275363, "pass@2": 0.9883676395070182, "pass@4": 0.9884954541008462, "pass@8": 0.9885869564329156, "pass@16": 0.988768115942029, "pass@32": 0.9891304347826086, "pass@64": 0.9898550724637681, "pass@128": 0.991304347826087 }, "6": { "pass@1": 0.9941350286532952, "pass@2": 0.9977561734314014, "pass@4": 0.9985594472610693, "pass@8": 0.9992944900423253, "pass@16": 0.9998437548382668, "pass@32": 0.9999948232370918, "pass@64": 0.9999999995301811, "pass@128": 1.0 }, "7": { "pass@1": 0.9928648843930635, "pass@2": 0.994768672340813, "pass@4": 0.9953643609758319, "pass@8": 0.9960753760142124, "pass@16": 0.9967665713625197, "pass@32": 0.9970812204794673, "pass@64": 0.9971097917066677, "pass@128": 0.9971098265895953 }, "8": { "pass@1": 0.9756382042253521, "pass@2": 0.9838094016857049, "pass@4": 0.9872804353681142, "pass@8": 0.9894908076168281, "pass@16": 0.9907958224370905, "pass@32": 0.991898265474723, "pass@64": 0.9929392340213707, "pass@128": 0.9943661971830986 }, "9": { "pass@1": 0.9612649700598802, "pass@2": 0.9778384913244379, "pass@4": 0.9821942192667527, "pass@8": 0.983871105890422, "pass@16": 0.9847190114530727, "pass@32": 0.985008695002592, "pass@64": 0.985029924450078, "pass@128": 0.9850299401197605 }, "11": { "pass@1": 0.8986257530120482, "pass@2": 0.9296826825016609, "pass@4": 0.9489027942478575, "pass@8": 0.9620662130010917, "pass@16": 0.9704384000316348, "pass@32": 0.9762796497655822, "pass@64": 0.9813886069516573, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.8237166927899686, "pass@2": 0.8677596543084939, "pass@4": 0.8935126335109363, "pass@8": 0.9109319966161393, "pass@16": 0.9238355493757084, "pass@32": 0.9343567941155181, "pass@64": 0.9447863035001147, "pass@128": 0.9561128526645768 }, "13": { "pass@1": 0.5984768907563025, "pass@2": 0.6788860721453939, "pass@4": 0.7344253578071649, "pass@8": 0.7775856673290666, "pass@16": 0.8146780213187624, "pass@32": 0.8475796206853868, "pass@64": 0.8756093944707887, "pass@128": 0.8935574229691877 }, "14": { "pass@1": 0.48645376461988304, "pass@2": 0.5669478404015286, "pass@4": 0.6318697369517405, "pass@8": 0.68492196122115, "pass@16": 0.7292486257783408, "pass@32": 0.7674189239076338, "pass@64": 0.7990489164697606, "pass@128": 0.8216374269005848 }, "15": { "pass@1": 0.38359375, "pass@2": 0.46019185456931494, "pass@4": 0.5203419740146114, "pass@8": 0.5682835301967295, "pass@16": 0.6074261748472496, "pass@32": 0.6400943942606238, "pass@64": 0.6706869139430693, "pass@128": 0.7 }, "16": { "pass@1": 0.24243233618233617, "pass@2": 0.3018344662045447, "pass@4": 0.3575915131335078, "pass@8": 0.4085050939332824, "pass@16": 0.4546885384517169, "pass@32": 0.49601365095000066, "pass@64": 0.5296128337074691, "pass@128": 0.5527065527065527 }, "17": { "pass@1": 0.2296195652173913, "pass@2": 0.29389167726316806, "pass@4": 0.3527888585937627, "pass@8": 0.4015310017732799, "pass@16": 0.4411821281483737, "pass@32": 0.47576190293681564, "pass@64": 0.5078436970671439, "pass@128": 0.5372670807453416 }, "18": { "pass@1": 0.21167279411764706, "pass@2": 0.26834254863362644, "pass@4": 0.3236940603012858, "pass@8": 0.3749530206752505, "pass@16": 0.4188367625939018, "pass@32": 0.45438906059280315, "pass@64": 0.4836671899265589, "pass@128": 0.5088235294117647 }, "19": { "pass@1": 0.14625186011904762, "pass@2": 0.1951546828130859, "pass@4": 0.24635013452559498, "pass@8": 0.29440603480328803, "pass@16": 0.3372258333217718, "pass@32": 0.37594895700766007, "pass@64": 0.411830564552925, "pass@128": 0.44642857142857145 }, "20": { "pass@1": 0.1511742140718563, "pass@2": 0.20507213918619455, "pass@4": 0.25712532359353274, "pass@8": 0.3054460965504392, "pass@16": 0.351170875678362, "pass@32": 0.39341225491743276, "pass@64": 0.4319755908517848, "pass@128": 0.46407185628742514 } } }, "movie_festival_awards": { "count": 801152, "correct": 552248, "answer_accuracy": 0.6893173829685253, "avg_response_len": 276.4077965230069, "resp_tokens_sum": 221444659, "pass_at_k": { "pass@1": 0.6893173829685253, "pass@2": 0.7236746211439762, "pass@4": 0.7508164814845082, "pass@8": 0.7726614849119308, "pass@16": 0.790704121070079, "pass@32": 0.8062121975664928, "pass@64": 0.8199103369645999, "pass@128": 0.8324013420674229 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9242537313432836, "pass@2": 0.9467549065695148, "pass@4": 0.960208038827236, "pass@8": 0.9686813621778994, "pass@16": 0.9736968715149406, "pass@32": 0.9769121125587591, "pass@64": 0.9802306191654232, "pass@128": 0.9850746268656716 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9997494533527697, "pass@2": 0.9999802719175408, "pass@4": 0.9999999098144802, "pass@8": 0.9999999999996636, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999771564327485, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9793658088235294, "pass@2": 0.982123161764706, "pass@4": 0.9840028584662212, "pass@8": 0.9856092514818663, "pass@16": 0.9870776742919471, "pass@32": 0.9886067946246756, "pass@64": 0.9902717923000518, "pass@128": 0.9911764705882353 }, "6": { "pass@1": 0.9942576892744479, "pass@2": 0.9967119153481209, "pass@4": 0.9968445709861974, "pass@8": 0.9968454258391087, "pass@16": 0.9968454258675079, "pass@32": 0.9968454258675079, "pass@64": 0.9968454258675079, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.9869709256329114, "pass@2": 0.9900737721768167, "pass@4": 0.9904884912526759, "pass@8": 0.9905062765613193, "pass@16": 0.9905063291137864, "pass@32": 0.990506329113924, "pass@64": 0.990506329113924, "pass@128": 0.990506329113924 }, "8": { "pass@1": 0.9748263888888888, "pass@2": 0.9825904891111588, "pass@4": 0.9846056751352027, "pass@8": 0.9865519125295662, "pass@16": 0.9890839997485931, "pass@32": 0.9917298739262829, "pass@64": 0.9936165397127583, "pass@128": 0.993993993993994 }, "9": { "pass@1": 0.9695538585209004, "pass@2": 0.9819578860419779, "pass@4": 0.9849974913706525, "pass@8": 0.9868120170628933, "pass@16": 0.9885209776830189, "pass@32": 0.9898635914465111, "pass@64": 0.9903321210779717, "pass@128": 0.9903536977491961 }, "11": { "pass@1": 0.8884672619047619, "pass@2": 0.9179161198600174, "pass@4": 0.9360529898048454, "pass@8": 0.9490142660043712, "pass@16": 0.958196320440275, "pass@32": 0.9647608513937915, "pass@64": 0.9700160392867752, "pass@128": 0.9746031746031746 }, "12": { "pass@1": 0.8331660932721713, "pass@2": 0.8894891220592839, "pass@4": 0.9224128669696099, "pass@8": 0.9411316213230453, "pass@16": 0.9521785605790735, "pass@32": 0.9594392629935055, "pass@64": 0.9642912699886997, "pass@128": 0.9663608562691132 }, "13": { "pass@1": 0.5963059413580247, "pass@2": 0.6718958849761836, "pass@4": 0.7258452271012411, "pass@8": 0.7661379023691128, "pass@16": 0.7994569530613119, "pass@32": 0.8298630341960512, "pass@64": 0.8566462178607112, "pass@128": 0.8765432098765432 }, "14": { "pass@1": 0.5224406528189911, "pass@2": 0.6048735805743123, "pass@4": 0.667954188960801, "pass@8": 0.714907466749094, "pass@16": 0.7518824252298536, "pass@32": 0.7844183856251077, "pass@64": 0.814024580720177, "pass@128": 0.8427299703264095 }, "15": { "pass@1": 0.39817226080246915, "pass@2": 0.46172277510450077, "pass@4": 0.516345238673869, "pass@8": 0.5643374864301807, "pass@16": 0.6066126531536483, "pass@32": 0.6453878305545641, "pass@64": 0.6818109357935861, "pass@128": 0.7160493827160493 }, "16": { "pass@1": 0.2555309734513274, "pass@2": 0.31953203028824917, "pass@4": 0.378190404296808, "pass@8": 0.4293594132098313, "pass@16": 0.47360209270500697, "pass@32": 0.5121977074655605, "pass@64": 0.5497008456356067, "pass@128": 0.5870206489675516 }, "17": { "pass@1": 0.21279649849397592, "pass@2": 0.2769127691869842, "pass@4": 0.3335075267323512, "pass@8": 0.3784226935397351, "pass@16": 0.41479042093392043, "pass@32": 0.44466458523149915, "pass@64": 0.4693555810200645, "pass@128": 0.4939759036144578 }, "18": { "pass@1": 0.21242439516129033, "pass@2": 0.2739847917195834, "pass@4": 0.33244598044196094, "pass@8": 0.38322910594677906, "pass@16": 0.4248379459806415, "pass@32": 0.458828979622347, "pass@64": 0.48675039253010505, "pass@128": 0.5096774193548387 }, "19": { "pass@1": 0.17817692598187312, "pass@2": 0.23166459012298704, "pass@4": 0.27631746239424, "pass@8": 0.31228890701309736, "pass@16": 0.34476007943757514, "pass@32": 0.3760017219184328, "pass@64": 0.40610132605649063, "pass@128": 0.4350453172205438 }, "20": { "pass@1": 0.15322353603603603, "pass@2": 0.2071318365609704, "pass@4": 0.2625266365915972, "pass@8": 0.3153373034222012, "pass@16": 0.36097657025856844, "pass@32": 0.39841696589607256, "pass@64": 0.42861872538402224, "pass@128": 0.45645645645645644 } } } } } ]