[ { "checkpoint": "checkpoint-21350", "id_acc": 0.509234375, "ood_acc": 0.1478125, "total_acc": 0.3190123355263158, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 202.11198611111112, "ood_avg_resp_len": 301.16630703125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 260336, "answer_accuracy": 0.32151043313310146, "avg_response_len": 242.01525203525134, "resp_tokens_sum": 195966526, "pass_at_k": { "pass@1": 0.32151043313310146, "pass@2": 0.4099008614305211, "pass@4": 0.5029745765210857, "pass@8": 0.5885455898222649, "pass@16": 0.6584088671873528, "pass@32": 0.71223867333972, "pass@64": 0.7538778515608776, "pass@128": 0.786911160290863 }, "per_op_pass_at_k": { "10": { "pass@1": 0.2340523598820059, "pass@2": 0.3490363621582701, "pass@4": 0.47703086422825464, "pass@8": 0.601095237968881, "pass@16": 0.7081131385563241, "pass@32": 0.7955993715227827, "pass@64": 0.8638342379925815, "pass@128": 0.911504424778761 }, "2": { "pass@1": 0.9397390965732088, "pass@2": 0.9527060796722839, "pass@4": 0.9634816159078247, "pass@8": 0.97184488771153, "pass@16": 0.9779010470744166, "pass@32": 0.9834545773716056, "pass@64": 0.9890862112444283, "pass@128": 0.9937694704049844 }, "3": { "pass@1": 0.9171683868501529, "pass@2": 0.9446715698668398, "pass@4": 0.9611175976856101, "pass@8": 0.9714190867433976, "pass@16": 0.9792196020103332, "pass@32": 0.9848030044380172, "pass@64": 0.9877250665899934, "pass@128": 0.9908256880733946 }, "4": { "pass@1": 0.777978515625, "pass@2": 0.8460526113435035, "pass@4": 0.8948536686820393, "pass@8": 0.9276525455102214, "pass@16": 0.9486645850423827, "pass@32": 0.9630959185836433, "pass@64": 0.9733635031273581, "pass@128": 0.978125 }, "5": { "pass@1": 0.42544642857142856, "pass@2": 0.5282945100612425, "pass@4": 0.6184357633867192, "pass@8": 0.6987714941711862, "pass@16": 0.7683401121051054, "pass@32": 0.8247387249798265, "pass@64": 0.8683272151365441, "pass@128": 0.8984126984126984 }, "6": { "pass@1": 0.4719311377245509, "pass@2": 0.6100170475977178, "pass@4": 0.7327381440718109, "pass@8": 0.8212352253202607, "pass@16": 0.8766424175779357, "pass@32": 0.9137167660624057, "pass@64": 0.9411640584150373, "pass@128": 0.9610778443113772 }, "7": { "pass@1": 0.36415957840236685, "pass@2": 0.48880959383590367, "pass@4": 0.6072677575805979, "pass@8": 0.7089181989531216, "pass@16": 0.7888269810111848, "pass@32": 0.8487011389145268, "pass@64": 0.8983636353290456, "pass@128": 0.9437869822485208 }, "8": { "pass@1": 0.3150040064102564, "pass@2": 0.43504725671310324, "pass@4": 0.5564820518228488, "pass@8": 0.6692766640774171, "pass@16": 0.7639736770960015, "pass@32": 0.8368242107183997, "pass@64": 0.8889416956818764, "pass@128": 0.9230769230769231 }, "9": { "pass@1": 0.23921654929577466, "pass@2": 0.3568800598868798, "pass@4": 0.4833165853388044, "pass@8": 0.5962302356670168, "pass@16": 0.6890966235919163, "pass@32": 0.7682846086626541, "pass@64": 0.8373558560961706, "pass@128": 0.8873239436619719 }, "11": { "pass@1": 0.21746635977337112, "pass@2": 0.3272846913965782, "pass@4": 0.44665189983334197, "pass@8": 0.5553283625674275, "pass@16": 0.6448423394148377, "pass@32": 0.7179749189998534, "pass@64": 0.777423089022195, "pass@128": 0.8243626062322946 }, "12": { "pass@1": 0.18498411016949154, "pass@2": 0.2839403582232306, "pass@4": 0.3926915541807275, "pass@8": 0.494440493747792, "pass@16": 0.5843288803240919, "pass@32": 0.6652268589464182, "pass@64": 0.735752190146036, "pass@128": 0.7937853107344632 }, "13": { "pass@1": 0.18647139498432602, "pass@2": 0.2839080974008344, "pass@4": 0.38877480009356197, "pass@8": 0.4846227863805249, "pass@16": 0.5638918640501261, "pass@32": 0.6319251904268449, "pass@64": 0.6916457926217192, "pass@128": 0.7429467084639498 }, "14": { "pass@1": 0.16109326323987538, "pass@2": 0.25444944746976716, "pass@4": 0.3592573200546192, "pass@8": 0.4555773271112732, "pass@16": 0.5327569175982596, "pass@32": 0.5947606734348753, "pass@64": 0.6511536361169645, "pass@128": 0.7102803738317757 }, "15": { "pass@1": 0.14234104046242774, "pass@2": 0.2236904612898822, "pass@4": 0.31877335600390994, "pass@8": 0.4121823992074167, "pass@16": 0.49071930914631096, "pass@32": 0.5529727050149277, "pass@64": 0.6061524184861755, "pass@128": 0.6560693641618497 }, "16": { "pass@1": 0.14392641129032258, "pass@2": 0.22453287719075446, "pass@4": 0.3174367695973486, "pass@8": 0.4091923824210985, "pass@16": 0.48592865391022005, "pass@32": 0.5402647597178885, "pass@64": 0.5777915977164921, "pass@128": 0.6096774193548387 }, "17": { "pass@1": 0.11477149566473989, "pass@2": 0.18487011287606392, "pass@4": 0.27174867872368563, "pass@8": 0.3607717520233434, "pass@16": 0.43365161352207626, "pass@32": 0.482409295928184, "pass@64": 0.5150428298119281, "pass@128": 0.546242774566474 }, "18": { "pass@1": 0.12763392857142858, "pass@2": 0.2035601799775029, "pass@4": 0.2933536606138519, "pass@8": 0.3812815069370258, "pass@16": 0.45235960903548417, "pass@32": 0.501900940088412, "pass@64": 0.5345649813685637, "pass@128": 0.56 }, "19": { "pass@1": 0.11732826576576577, "pass@2": 0.18959707739235296, "pass@4": 0.2780769194391241, "pass@8": 0.3682895575262585, "pass@16": 0.44274252772123246, "pass@32": 0.4905652476370396, "pass@64": 0.5157753123580949, "pass@128": 0.5315315315315315 }, "20": { "pass@1": 0.10604354354354355, "pass@2": 0.17358192345889195, "pass@4": 0.25876359036201546, "pass@8": 0.3487342283712976, "pass@16": 0.4243679405449727, "pass@32": 0.47493279620096895, "pass@64": 0.5034854391857208, "pass@128": 0.5165165165165165 } } }, "teachers_in_school": { "count": 821120, "correct": 256406, "answer_accuracy": 0.31226373733437257, "avg_response_len": 252.55651061964147, "resp_tokens_sum": 207379202, "pass_at_k": { "pass@1": 0.31226373733437257, "pass@2": 0.4041371953651974, "pass@4": 0.5020694377398108, "pass@8": 0.5920825799919917, "pass@16": 0.6640724218550299, "pass@32": 0.7175980090991305, "pass@64": 0.7592845415933916, "pass@128": 0.7950116913484022 }, "per_op_pass_at_k": { "10": { "pass@1": 0.22354294478527606, "pass@2": 0.3360058088981208, "pass@4": 0.46179167040469643, "pass@8": 0.5850830032020309, "pass@16": 0.6922560235978716, "pass@32": 0.779310015924251, "pass@64": 0.8474751226546069, "pass@128": 0.8987730061349694 }, "2": { "pass@1": 0.8912186550151976, "pass@2": 0.9116783440633751, "pass@4": 0.9292722276546739, "pass@8": 0.9451709089562008, "pass@16": 0.9578299275067208, "pass@32": 0.967487067518195, "pass@64": 0.9747033816302779, "pass@128": 0.9787234042553191 }, "3": { "pass@1": 0.848626893939394, "pass@2": 0.8895933995466476, "pass@4": 0.9138172435831877, "pass@8": 0.9303741145081666, "pass@16": 0.9424221740566364, "pass@32": 0.9518335581963728, "pass@64": 0.9600967272076638, "pass@128": 0.9666666666666667 }, "4": { "pass@1": 0.725938424556213, "pass@2": 0.7984489295531847, "pass@4": 0.8548653993139906, "pass@8": 0.8967305917882408, "pass@16": 0.9268802741732332, "pass@32": 0.945689901633452, "pass@64": 0.95690937985168, "pass@128": 0.9674556213017751 }, "5": { "pass@1": 0.4459239130434783, "pass@2": 0.5601492068926165, "pass@4": 0.6647551846780018, "pass@8": 0.7498131323981773, "pass@16": 0.8119176290092686, "pass@32": 0.8557433256995092, "pass@64": 0.8852069468068376, "pass@128": 0.9072463768115943 }, "6": { "pass@1": 0.42238986389684813, "pass@2": 0.560813164158563, "pass@4": 0.6923920269278656, "pass@8": 0.796637969019255, "pass@16": 0.8679993396877942, "pass@32": 0.9132326395237702, "pass@64": 0.9407776590161546, "pass@128": 0.9598853868194842 }, "7": { "pass@1": 0.3537978684971098, "pass@2": 0.48195561763233363, "pass@4": 0.6084160362360775, "pass@8": 0.717139403598144, "pass@16": 0.8003843219004848, "pass@32": 0.8634777033100696, "pass@64": 0.915658946495517, "pass@128": 0.9566473988439307 }, "8": { "pass@1": 0.28818221830985913, "pass@2": 0.41921301430631, "pass@4": 0.5555384356004792, "pass@8": 0.6756259834552851, "pass@16": 0.7670445900118551, "pass@32": 0.8316106463558992, "pass@64": 0.8783679240426463, "pass@128": 0.9154929577464789 }, "9": { "pass@1": 0.25715755988023953, "pass@2": 0.37767722429157424, "pass@4": 0.5053444070801026, "pass@8": 0.6198841515478999, "pass@16": 0.7131677032398571, "pass@32": 0.7895863734944806, "pass@64": 0.8534291934377334, "pass@128": 0.9011976047904192 }, "11": { "pass@1": 0.1899472891566265, "pass@2": 0.29249144708756303, "pass@4": 0.4056045608192049, "pass@8": 0.5118326321094175, "pass@16": 0.6064758426416722, "pass@32": 0.691489508808827, "pass@64": 0.767702363812573, "pass@128": 0.8373493975903614 }, "12": { "pass@1": 0.21497844827586207, "pass@2": 0.3253735683607731, "pass@4": 0.44188718318517406, "pass@8": 0.547290961654598, "pass@16": 0.6366487702338203, "pass@32": 0.7110514969364836, "pass@64": 0.7759151125242277, "pass@128": 0.8307210031347962 }, "13": { "pass@1": 0.16281512605042017, "pass@2": 0.25351139471536654, "pass@4": 0.3567216876251812, "pass@8": 0.45448099582903717, "pass@16": 0.5343941235795181, "pass@32": 0.5985284837482586, "pass@64": 0.6597611542390495, "pass@128": 0.7282913165266106 }, "14": { "pass@1": 0.16132127192982457, "pass@2": 0.24900207786526687, "pass@4": 0.34654788052809166, "pass@8": 0.43636616447770876, "pass@16": 0.5097873294578236, "pass@32": 0.5690943424021969, "pass@64": 0.6227856094077184, "pass@128": 0.6754385964912281 }, "15": { "pass@1": 0.13856534090909092, "pass@2": 0.22398182116439982, "pass@4": 0.32483539131472194, "pass@8": 0.4210097549573201, "pass@16": 0.49911894400022283, "pass@32": 0.5627604370589941, "pass@64": 0.6207404034236885, "pass@128": 0.6727272727272727 }, "16": { "pass@1": 0.13837695868945868, "pass@2": 0.21823368833703485, "pass@4": 0.3100300085352579, "pass@8": 0.39765295225678154, "pass@16": 0.46724374386741413, "pass@32": 0.5191649589933541, "pass@64": 0.5639517418293533, "pass@128": 0.6068376068376068 }, "17": { "pass@1": 0.1343652950310559, "pass@2": 0.2217845221792929, "pass@4": 0.3313873420131489, "pass@8": 0.44396581707866756, "pass@16": 0.5348156239501506, "pass@32": 0.5924449604975142, "pass@64": 0.6296611056432868, "pass@128": 0.6645962732919255 }, "18": { "pass@1": 0.13033088235294119, "pass@2": 0.2081334674617878, "pass@4": 0.302660001599065, "pass@8": 0.39741988375269416, "pass@16": 0.4747293286693812, "pass@32": 0.5260981930569909, "pass@64": 0.5574985566060943, "pass@128": 0.5852941176470589 }, "19": { "pass@1": 0.11130487351190477, "pass@2": 0.1813290184430071, "pass@4": 0.2708650369596659, "pass@8": 0.3643281841752635, "pass@16": 0.4387280487490098, "pass@32": 0.4830895220764344, "pass@64": 0.506235461720585, "pass@128": 0.5208333333333334 }, "20": { "pass@1": 0.10656811377245509, "pass@2": 0.17586099945777736, "pass@4": 0.2645760077146045, "pass@8": 0.3582559738491696, "pass@16": 0.4345901738446862, "pass@32": 0.48211568706570346, "pass@64": 0.5091873400019051, "pass@128": 0.5299401197604791 } } }, "movie_festival_awards": { "count": 801152, "correct": 259096, "answer_accuracy": 0.32340429781115193, "avg_response_len": 268.33878340190125, "resp_tokens_sum": 214980153, "pass_at_k": { "pass@1": 0.32340429781115193, "pass@2": 0.4089581278549485, "pass@4": 0.4999869338808328, "pass@8": 0.5851361993699952, "pass@16": 0.655583087089333, "pass@32": 0.7106789122766686, "pass@64": 0.75461773383224, "pass@128": 0.7916600095861959 }, "per_op_pass_at_k": { "10": { "pass@1": 0.25944496268656714, "pass@2": 0.37380861440827357, "pass@4": 0.4947208124171043, "pass@8": 0.6084540057533921, "pass@16": 0.7069548544384052, "pass@32": 0.7889803052552564, "pass@64": 0.8535451186415071, "pass@128": 0.9014925373134328 }, "2": { "pass@1": 0.9660044642857143, "pass@2": 0.9723727502812146, "pass@4": 0.9782737465852483, "pass@8": 0.9840867443239417, "pass@16": 0.9897300202044366, "pass@32": 0.9943372627596654, "pass@64": 0.9967877861964524, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9045189504373178, "pass@2": 0.9409872075939486, "pass@4": 0.963359374538824, "pass@8": 0.9778173387389365, "pass@16": 0.9862698501499768, "pass@32": 0.9905573089869245, "pass@64": 0.9926020223300518, "pass@128": 0.9941690962099126 }, "4": { "pass@1": 0.7404513888888888, "pass@2": 0.8004220483952662, "pass@4": 0.8438767870573194, "pass@8": 0.8764562526018601, "pass@16": 0.9012650591890069, "pass@32": 0.9202351939936253, "pass@64": 0.9359478097107635, "pass@128": 0.9502923976608187 }, "5": { "pass@1": 0.44046415441176473, "pass@2": 0.539720356646595, "pass@4": 0.632906808156333, "pass@8": 0.7201508139502907, "pass@16": 0.7958080919156965, "pass@32": 0.8564772070244233, "pass@64": 0.9020604729390345, "pass@128": 0.9323529411764706 }, "6": { "pass@1": 0.4135695977917981, "pass@2": 0.5535905015027699, "pass@4": 0.6846301371240265, "pass@8": 0.7918933708969056, "pass@16": 0.870299357531166, "pass@32": 0.921637925223863, "pass@64": 0.953726180491391, "pass@128": 0.9747634069400631 }, "7": { "pass@1": 0.3732693829113924, "pass@2": 0.5000556756453705, "pass@4": 0.6184612249813708, "pass@8": 0.71713744327495, "pass@16": 0.7982010440452766, "pass@32": 0.8671220101820218, "pass@64": 0.9208143511726397, "pass@128": 0.9525316455696202 }, "8": { "pass@1": 0.29450544294294295, "pass@2": 0.4251602734624386, "pass@4": 0.5522831794111326, "pass@8": 0.6573037179247492, "pass@16": 0.739586440476168, "pass@32": 0.8057187779468007, "pass@64": 0.859777333695693, "pass@128": 0.9069069069069069 }, "9": { "pass@1": 0.21402733118971062, "pass@2": 0.3169770014178291, "pass@4": 0.43098392071971686, "pass@8": 0.540710406429895, "pass@16": 0.6397980025166145, "pass@32": 0.732216820941264, "pass@64": 0.8158523025431184, "pass@128": 0.8842443729903537 }, "11": { "pass@1": 0.21999007936507936, "pass@2": 0.32811679790026244, "pass@4": 0.4464493072294533, "pass@8": 0.5552910257350235, "pass@16": 0.6437437236941529, "pass@32": 0.7177650886723408, "pass@64": 0.7823839715220456, "pass@128": 0.8380952380952381 }, "12": { "pass@1": 0.18274560397553516, "pass@2": 0.2797676021575284, "pass@4": 0.38757090627433033, "pass@8": 0.4914502937919992, "pass@16": 0.5856451067997539, "pass@32": 0.6711780315675037, "pass@64": 0.7491794923736346, "pass@128": 0.8195718654434251 }, "13": { "pass@1": 0.17450327932098766, "pass@2": 0.26490693654612624, "pass@4": 0.3661309399403777, "pass@8": 0.4624642420940969, "pass@16": 0.5448557983915544, "pass@32": 0.6185134734378167, "pass@64": 0.6895867643018863, "pass@128": 0.7561728395061729 }, "14": { "pass@1": 0.1650361646884273, "pass@2": 0.2522824715530736, "pass@4": 0.3488312738726651, "pass@8": 0.43660343721379263, "pass@16": 0.5027340358101761, "pass@32": 0.5535550253369403, "pass@64": 0.6047111303759262, "pass@128": 0.6646884272997032 }, "15": { "pass@1": 0.11781442901234568, "pass@2": 0.19408977957616405, "pass@4": 0.2919006570937892, "pass@8": 0.3949211679701401, "pass@16": 0.48446269102984507, "pass@32": 0.5532427056623543, "pass@64": 0.6071269026647454, "pass@128": 0.6604938271604939 }, "16": { "pass@1": 0.1358545353982301, "pass@2": 0.21153033760713535, "pass@4": 0.2991320606494985, "pass@8": 0.3867938722175179, "pass@16": 0.46065202878578043, "pass@32": 0.5153170845256568, "pass@64": 0.5579031332192951, "pass@128": 0.5988200589970502 }, "17": { "pass@1": 0.11241057981927711, "pass@2": 0.18683555580590078, "pass@4": 0.2822885405514067, "pass@8": 0.3811760461419187, "pass@16": 0.4616358779338103, "pass@32": 0.5172659436791867, "pass@64": 0.5580279888684868, "pass@128": 0.5933734939759037 }, "18": { "pass@1": 0.11902721774193549, "pass@2": 0.18856561150622297, "pass@4": 0.2713082698332063, "pass@8": 0.3538581464718504, "pass@16": 0.41980580105441206, "pass@32": 0.4647936660551533, "pass@64": 0.49533100083021947, "pass@128": 0.5225806451612903 }, "19": { "pass@1": 0.11131042296072508, "pass@2": 0.18102727953945336, "pass@4": 0.26652312078814944, "pass@8": 0.3546306422885341, "pass@16": 0.42932109990881356, "pass@32": 0.4792171969760829, "pass@64": 0.5081471983782093, "pass@128": 0.5287009063444109 }, "20": { "pass@1": 0.1073338963963964, "pass@2": 0.17943607978056803, "pass@4": 0.2735159763250315, "pass@8": 0.3727452360201581, "pass@16": 0.45258346490831136, "pass@32": 0.5016080590418188, "pass@64": 0.5292435042316577, "pass@128": 0.5465465465465466 } } } } } ]