[ { "checkpoint": "checkpoint-21414", "id_acc": 0.6735208333333333, "ood_acc": 0.22256484375, "total_acc": 0.4361755756578947, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.16801041666668, "ood_avg_resp_len": 311.04166015625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 349315, "answer_accuracy": 0.43139795091685107, "avg_response_len": 243.84849109824535, "resp_tokens_sum": 197450951, "pass_at_k": { "pass@1": 0.43139795091685107, "pass@2": 0.5321828105668215, "pass@4": 0.6249946485161172, "pass@8": 0.7011739986140003, "pass@16": 0.7595677606039941, "pass@32": 0.8034274001625996, "pass@64": 0.8371937190423179, "pass@128": 0.8632627252608284 }, "per_op_pass_at_k": { "10": { "pass@1": 0.4817247418879056, "pass@2": 0.6352787552551507, "pass@4": 0.7659861662756753, "pass@8": 0.8593352023758045, "pass@16": 0.9180698213960385, "pass@32": 0.9501475446176046, "pass@64": 0.9676992562490706, "pass@128": 0.9793510324483776 }, "2": { "pass@1": 0.9578222352024922, "pass@2": 0.9749559999509407, "pass@4": 0.9878366366587354, "pass@8": 0.9955344824449235, "pass@16": 0.9990460809752442, "pass@32": 0.999941873566355, "pass@64": 0.9999999112660973, "pass@128": 1.0 }, "3": { "pass@1": 0.9316704892966361, "pass@2": 0.949972459004551, "pass@4": 0.9662891631091987, "pass@8": 0.9793003255083925, "pass@16": 0.9883260233328266, "pass@32": 0.9940213091054283, "pass@64": 0.9975285790814391, "pass@128": 1.0 }, "4": { "pass@1": 0.80537109375, "pass@2": 0.8725716658464571, "pass@4": 0.9234813232330332, "pass@8": 0.956524622745434, "pass@16": 0.9763327780316118, "pass@32": 0.9879384001567445, "pass@64": 0.9945018158032546, "pass@128": 0.996875 }, "5": { "pass@1": 0.6077628968253969, "pass@2": 0.730289963754531, "pass@4": 0.8259401024276726, "pass@8": 0.8936040569679891, "pass@16": 0.9394573037217293, "pass@32": 0.9679654619746364, "pass@64": 0.9831936120425192, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.6897688997005988, "pass@2": 0.8093785362346175, "pass@4": 0.8870302679230957, "pass@8": 0.9347081087434049, "pass@16": 0.9643132303145109, "pass@32": 0.9813625491033848, "pass@64": 0.9897868533872056, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.5860068417159763, "pass@2": 0.7297689628663284, "pass@4": 0.8420394427397755, "pass@8": 0.9139281489117121, "pass@16": 0.9525323666683377, "pass@32": 0.9726538898839411, "pass@64": 0.9848167796739252, "pass@128": 0.9911242603550295 }, "8": { "pass@1": 0.505859375, "pass@2": 0.6564822613819913, "pass@4": 0.7820809541235229, "pass@8": 0.8685459407960715, "pass@16": 0.9199025646370973, "pass@32": 0.9504180479169292, "pass@64": 0.9696105221054736, "pass@128": 0.9807692307692307 }, "9": { "pass@1": 0.4445862676056338, "pass@2": 0.6046994565820115, "pass@4": 0.7470850805621114, "pass@8": 0.8488156642340807, "pass@16": 0.9112835745002482, "pass@32": 0.9467934160269519, "pass@64": 0.9656020765152085, "pass@128": 0.9746478873239437 }, "11": { "pass@1": 0.4284038597733711, "pass@2": 0.5775366794182597, "pass@4": 0.7094636056470274, "pass@8": 0.8094007817227927, "pass@16": 0.8754949614771798, "pass@32": 0.9179393032615564, "pass@64": 0.946461553261782, "pass@128": 0.9660056657223796 }, "12": { "pass@1": 0.3552921963276836, "pass@2": 0.49981093465011817, "pass@4": 0.6435842768065005, "pass@8": 0.7688598850974901, "pass@16": 0.8616190363221634, "pass@32": 0.9181819016489402, "pass@64": 0.9516632170824267, "pass@128": 0.9745762711864406 }, "13": { "pass@1": 0.312157131661442, "pass@2": 0.438362763187125, "pass@4": 0.5662403240935008, "pass@8": 0.6810592242120193, "pass@16": 0.7738279317724085, "pass@32": 0.8394025480984975, "pass@64": 0.8794518854511277, "pass@128": 0.9028213166144201 }, "14": { "pass@1": 0.2306756230529595, "pass@2": 0.3417531336620305, "pass@4": 0.4677710703802214, "pass@8": 0.5904887371828694, "pass@16": 0.693758241859579, "pass@32": 0.7724491730569445, "pass@64": 0.8299225014352747, "pass@128": 0.8660436137071651 }, "15": { "pass@1": 0.19102239884393063, "pass@2": 0.2817613274316142, "pass@4": 0.37920384635807797, "pass@8": 0.47406381308256057, "pass@16": 0.563398032120974, "pass@32": 0.644849763727135, "pass@64": 0.7142993770422786, "pass@128": 0.7687861271676301 }, "16": { "pass@1": 0.17033770161290324, "pass@2": 0.2407516033782067, "pass@4": 0.3186744777870508, "pass@8": 0.398733871999996, "pass@16": 0.47617025411495906, "pass@32": 0.5478840483451405, "pass@64": 0.6112842658780524, "pass@128": 0.6645161290322581 }, "17": { "pass@1": 0.158891799132948, "pass@2": 0.2312060500204816, "pass@4": 0.30643091163460046, "pass@8": 0.3764142529721565, "pass@16": 0.44026184076793384, "pass@32": 0.5005950804847814, "pass@64": 0.5607158592132552, "pass@128": 0.6213872832369942 }, "18": { "pass@1": 0.13390625, "pass@2": 0.19612204724409452, "pass@4": 0.26780344635491976, "pass@8": 0.3450560776528902, "pass@16": 0.42315467172835713, "pass@32": 0.4987441792586379, "pass@64": 0.571169957667318, "pass@128": 0.6342857142857142 }, "19": { "pass@1": 0.1423141891891892, "pass@2": 0.21137439112340686, "pass@4": 0.28596790322380866, "pass@8": 0.3565394692761307, "pass@16": 0.42062869719073964, "pass@32": 0.4799759936316975, "pass@64": 0.534573698675929, "pass@128": 0.5855855855855856 }, "20": { "pass@1": 0.12483577327327328, "pass@2": 0.18133216878295616, "pass@4": 0.2437110065633688, "pass@8": 0.3046702644410265, "pass@16": 0.36142120928657884, "pass@32": 0.4167471278795742, "pass@64": 0.4733304200251955, "pass@128": 0.5255255255255256 } } }, "teachers_in_school": { "count": 821120, "correct": 360298, "answer_accuracy": 0.43878848402182385, "avg_response_len": 254.20170011691349, "resp_tokens_sum": 208730100, "pass_at_k": { "pass@1": 0.43878848402182385, "pass@2": 0.5369486884823405, "pass@4": 0.6291625673593271, "pass@8": 0.7060254278303971, "pass@16": 0.7656998211258677, "pass@32": 0.8113387439377043, "pass@64": 0.8458824812617933, "pass@128": 0.8717069368667186 }, "per_op_pass_at_k": { "10": { "pass@1": 0.4443539110429448, "pass@2": 0.6050519147625724, "pass@4": 0.7508354329558494, "pass@8": 0.857326700302133, "pass@16": 0.9225485327660016, "pass@32": 0.9569656699143689, "pass@64": 0.9739071748568061, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 0.9620298252279635, "pass@2": 0.9795812591245241, "pass@4": 0.9908501828608812, "pass@8": 0.9970251835629206, "pass@16": 0.9994825746890394, "pass@32": 0.9999799709303407, "pass@64": 0.9999999930684337, "pass@128": 1.0 }, "3": { "pass@1": 0.95625, "pass@2": 0.9672706394655214, "pass@4": 0.9773673733965071, "pass@8": 0.9849303085747515, "pass@16": 0.9890761521939984, "pass@32": 0.9919127406602036, "pass@64": 0.9945956899378898, "pass@128": 0.996969696969697 }, "4": { "pass@1": 0.8583811020710059, "pass@2": 0.9200091436425475, "pass@4": 0.9599025718123985, "pass@8": 0.9819837740587968, "pass@16": 0.992273429855794, "pass@32": 0.9959718861628852, "pass@64": 0.9969509246945819, "pass@128": 0.9970414201183432 }, "5": { "pass@1": 0.6773097826086957, "pass@2": 0.7912629807143674, "pass@4": 0.8727454549159614, "pass@8": 0.9256251584652402, "pass@16": 0.9592054453081453, "pass@32": 0.9789833888272682, "pass@64": 0.9877626724630637, "pass@128": 0.991304347826087 }, "6": { "pass@1": 0.6456617120343839, "pass@2": 0.7710739909302169, "pass@4": 0.8655726544210627, "pass@8": 0.9255087613679311, "pass@16": 0.9588882660952172, "pass@32": 0.9775911162407339, "pass@64": 0.9877839315981168, "pass@128": 0.9914040114613181 }, "7": { "pass@1": 0.5831376445086706, "pass@2": 0.714123873515088, "pass@4": 0.8156219727411236, "pass@8": 0.8806761278760264, "pass@16": 0.9226814672747715, "pass@32": 0.9539407684583533, "pass@64": 0.9767353629407124, "pass@128": 0.9884393063583815 }, "8": { "pass@1": 0.550088028169014, "pass@2": 0.6991391260951532, "pass@4": 0.8196430778899111, "pass@8": 0.89965991789473, "pass@16": 0.9473028675491105, "pass@32": 0.9751506124431925, "pass@64": 0.9897350817974143, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.4807728293413174, "pass@2": 0.6240647396152582, "pass@4": 0.7528413109102381, "pass@8": 0.8474207898913209, "pass@16": 0.9070636547105337, "pass@32": 0.9438653815276806, "pass@64": 0.9669455167868922, "pass@128": 0.9820359281437125 }, "11": { "pass@1": 0.3729527484939759, "pass@2": 0.5160652452329001, "pass@4": 0.6554858650010614, "pass@8": 0.7738427672123679, "pass@16": 0.8616539041891845, "pass@32": 0.9187370376853801, "pass@64": 0.9524868366449002, "pass@128": 0.9728915662650602 }, "12": { "pass@1": 0.35881171630094044, "pass@2": 0.49483036309332806, "pass@4": 0.6288250344098836, "pass@8": 0.7445689588528022, "pass@16": 0.833712635378144, "pass@32": 0.8972884331775789, "pass@64": 0.9391571594435375, "pass@128": 0.9623824451410659 }, "13": { "pass@1": 0.28223476890756305, "pass@2": 0.40754165839564177, "pass@4": 0.5387995284728062, "pass@8": 0.6576386299571035, "pass@16": 0.7543768701454598, "pass@32": 0.8278516793921898, "pass@64": 0.8779019857732024, "pass@128": 0.9103641456582633 }, "14": { "pass@1": 0.24312408625730994, "pass@2": 0.34659735172906, "pass@4": 0.45804868278526556, "pass@8": 0.5631887917582749, "pass@16": 0.6514521850564117, "pass@32": 0.7211406756976264, "pass@64": 0.7736040979948913, "pass@128": 0.8128654970760234 }, "15": { "pass@1": 0.18742897727272728, "pass@2": 0.2791256561679787, "pass@4": 0.3839303893831451, "pass@8": 0.49218108231025, "pass@16": 0.5963661486084444, "pass@32": 0.6899128048407284, "pass@64": 0.7652887002306195, "pass@128": 0.8151515151515152 }, "16": { "pass@1": 0.16312767094017094, "pass@2": 0.2328744363685309, "pass@4": 0.3091983616898317, "pass@8": 0.3862526954911408, "pass@16": 0.46173026023197594, "pass@32": 0.5376746561844317, "pass@64": 0.6142307830288806, "pass@128": 0.6894586894586895 }, "17": { "pass@1": 0.16386840062111802, "pass@2": 0.23810797427495464, "pass@4": 0.32328601500511184, "pass@8": 0.41261976880930534, "pass@16": 0.49869282658803993, "pass@32": 0.5769880576200863, "pass@64": 0.644537560302228, "pass@128": 0.6987577639751553 }, "18": { "pass@1": 0.14855238970588236, "pass@2": 0.21801760074108376, "pass@4": 0.2986294655263678, "pass@8": 0.3826632892944409, "pass@16": 0.46140967549262235, "pass@32": 0.5326616971221702, "pass@64": 0.6004062585650338, "pass@128": 0.6676470588235294 }, "19": { "pass@1": 0.14308965773809523, "pass@2": 0.2120781044947507, "pass@4": 0.2880920523885407, "pass@8": 0.35969615277468225, "pass@16": 0.42255468128199647, "pass@32": 0.47800113481593476, "pass@64": 0.5267301888452244, "pass@128": 0.5684523809523809 }, "20": { "pass@1": 0.11522267964071856, "pass@2": 0.1793600299401198, "pass@4": 0.25643922728221846, "pass@8": 0.33409736909282767, "pass@16": 0.4014528278273527, "pass@32": 0.4552548863035034, "pass@64": 0.49782715039176945, "pass@128": 0.5299401197604791 } } }, "movie_festival_awards": { "count": 801152, "correct": 351166, "answer_accuracy": 0.43832631011343665, "avg_response_len": 273.4684828846461, "resp_tokens_sum": 219089822, "pass_at_k": { "pass@1": 0.43832631011343665, "pass@2": 0.5365698394312263, "pass@4": 0.6283680451181478, "pass@8": 0.7046852593995614, "pass@16": 0.7642488013994941, "pass@32": 0.8099337635927337, "pass@64": 0.8454458711849641, "pass@128": 0.8739415242051446 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5032882462686568, "pass@2": 0.6513496738747213, "pass@4": 0.7781951312056141, "pass@8": 0.8691466965727291, "pass@16": 0.9260160693597145, "pass@32": 0.9589344283151598, "pass@64": 0.9799965043319006, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 0.94046875, "pass@2": 0.9615220753655792, "pass@4": 0.9791949306336708, "pass@8": 0.9910785397841038, "pass@16": 0.9969192482415177, "pass@32": 0.9991217509879896, "pass@64": 0.9999135257784807, "pass@128": 1.0 }, "3": { "pass@1": 0.940620444606414, "pass@2": 0.9625026543238218, "pass@4": 0.9799381504972225, "pass@8": 0.9913868834403958, "pass@16": 0.9972989677312782, "pass@32": 0.9995030708111858, "pass@64": 0.9999804748054121, "pass@128": 1.0 }, "4": { "pass@1": 0.8034082602339181, "pass@2": 0.8737722032278858, "pass@4": 0.9267439889640982, "pass@8": 0.9596337325518417, "pass@16": 0.9772015339150868, "pass@32": 0.9862561581036814, "pass@64": 0.9904384863967542, "pass@128": 0.9912280701754386 }, "5": { "pass@1": 0.6249080882352941, "pass@2": 0.745866503589625, "pass@4": 0.8407373869626591, "pass@8": 0.9027665657756448, "pass@16": 0.9395420959632879, "pass@32": 0.9609190845435237, "pass@64": 0.9743793589403449, "pass@128": 0.9823529411764705 }, "6": { "pass@1": 0.6496944006309149, "pass@2": 0.7748069531036532, "pass@4": 0.8661763146128815, "pass@8": 0.9248608618126145, "pass@16": 0.9607558923277141, "pass@32": 0.9806711476271339, "pass@64": 0.9897304603908488, "pass@128": 0.9936908517350158 }, "7": { "pass@1": 0.5810176028481012, "pass@2": 0.7143181407604903, "pass@4": 0.8142075656286634, "pass@8": 0.8796060112889925, "pass@16": 0.9231276994518164, "pass@32": 0.951785145268383, "pass@64": 0.9706941203196205, "pass@128": 0.9810126582278481 }, "8": { "pass@1": 0.5080705705705706, "pass@2": 0.6495878260149914, "pass@4": 0.7686508476980916, "pass@8": 0.854349197922715, "pass@16": 0.911571770067261, "pass@32": 0.9476719843601782, "pass@64": 0.9698413001028867, "pass@128": 0.984984984984985 }, "9": { "pass@1": 0.45955586816720256, "pass@2": 0.6173716516444286, "pass@4": 0.7601637984560605, "pass@8": 0.8648801163870649, "pass@16": 0.9301138762865712, "pass@32": 0.9680547564025581, "pass@64": 0.9880174104714309, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.4020337301587302, "pass@2": 0.5445221691038621, "pass@4": 0.6799382666452413, "pass@8": 0.7912959175692053, "pass@16": 0.8715446459982027, "pass@32": 0.9231247651466039, "pass@64": 0.9547388131097485, "pass@128": 0.9746031746031746 }, "12": { "pass@1": 0.35464449541284404, "pass@2": 0.49375549314936534, "pass@4": 0.6321710660594023, "pass@8": 0.7495656664698744, "pass@16": 0.8368415645553295, "pass@32": 0.8947223371149916, "pass@64": 0.9320921468349798, "pass@128": 0.9602446483180428 }, "13": { "pass@1": 0.31233121141975306, "pass@2": 0.4359784345533197, "pass@4": 0.5601907490151696, "pass@8": 0.6717671604124168, "pass@16": 0.7678836429673943, "pass@32": 0.8445813822543843, "pass@64": 0.8977074759369114, "pass@128": 0.9320987654320988 }, "14": { "pass@1": 0.24656899109792285, "pass@2": 0.35336872648893636, "pass@4": 0.4694582441846106, "pass@8": 0.5798708865407574, "pass@16": 0.672736222941444, "pass@32": 0.7462808456444997, "pass@64": 0.8034686946130885, "pass@128": 0.8427299703264095 }, "15": { "pass@1": 0.1877170138888889, "pass@2": 0.28082508566637493, "pass@4": 0.38921946533303714, "pass@8": 0.5000600226201365, "pass@16": 0.602305374771351, "pass@32": 0.6889864561370689, "pass@64": 0.7561926767193276, "pass@128": 0.808641975308642 }, "16": { "pass@1": 0.175954092920354, "pass@2": 0.24397544886535194, "pass@4": 0.31938721570201956, "pass@8": 0.4002087109263321, "pass@16": 0.4841250558928012, "pass@32": 0.5685579542130321, "pass@64": 0.6513467529523246, "pass@128": 0.7286135693215339 }, "17": { "pass@1": 0.15173192771084337, "pass@2": 0.2243319982686652, "pass@4": 0.3052337293493432, "pass@8": 0.38670322107789923, "pass@16": 0.4610427598188654, "pass@32": 0.5284630277297131, "pass@64": 0.5901698510578611, "pass@128": 0.641566265060241 }, "18": { "pass@1": 0.15791330645161292, "pass@2": 0.22585883921767838, "pass@4": 0.3001777695126821, "pass@8": 0.3720267021326365, "pass@16": 0.43655216978213546, "pass@32": 0.4955005756619598, "pass@64": 0.5516453645056283, "pass@128": 0.6064516129032258 }, "19": { "pass@1": 0.14999527945619334, "pass@2": 0.2162124289316555, "pass@4": 0.2851994351348076, "pass@8": 0.35096117976459, "pass@16": 0.4119157773312138, "pass@32": 0.4694671341734886, "pass@64": 0.5252148649384238, "pass@128": 0.5861027190332326 }, "20": { "pass@1": 0.11364489489489489, "pass@2": 0.17525042266676127, "pass@4": 0.2480603184737043, "pass@8": 0.3236715042660682, "pass@16": 0.39577047962332407, "pass@32": 0.46366567785399415, "pass@64": 0.5281737272922968, "pass@128": 0.5915915915915916 } } } } } ]