[ { "checkpoint": "checkpoint-21318", "id_acc": 0.6852916666666666, "ood_acc": 0.19446875, "total_acc": 0.4269638157894737, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 208.45466059027777, "ood_avg_resp_len": 326.81123984375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 342449, "answer_accuracy": 0.4229185603066709, "avg_response_len": 260.1732470656813, "resp_tokens_sum": 210669563, "pass_at_k": { "pass@1": 0.4229185603066709, "pass@2": 0.5425882489401411, "pass@4": 0.6514530540720866, "pass@8": 0.7349339274393267, "pass@16": 0.7921521525323136, "pass@32": 0.8304860340709597, "pass@64": 0.8569206377719508, "pass@128": 0.8765412582990831 }, "per_op_pass_at_k": { "10": { "pass@1": 0.4465339233038348, "pass@2": 0.6388332404245938, "pass@4": 0.8110225173733812, "pass@8": 0.921477109538525, "pass@16": 0.9717416308535904, "pass@32": 0.9891458432827139, "pass@64": 0.9936897894228782, "pass@128": 0.9941002949852508 }, "2": { "pass@1": 0.9911653037383178, "pass@2": 0.9966566861677336, "pass@4": 0.999050767310628, "pass@8": 0.9998905695126493, "pass@16": 0.999998216195341, "pass@32": 0.9999999997961639, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.985282874617737, "pass@2": 0.9922294511064558, "pass@4": 0.9940402065567491, "pass@8": 0.9947209661426557, "pass@16": 0.9953911602416506, "pass@32": 0.9962353673679719, "pass@64": 0.9968537648390282, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.785400390625, "pass@2": 0.903405665600394, "pass@4": 0.962644184418353, "pass@8": 0.9868863476913816, "pass@16": 0.9941653982094479, "pass@32": 0.9961020872725843, "pass@64": 0.9967848997121347, "pass@128": 0.996875 }, "5": { "pass@1": 0.7234375, "pass@2": 0.8706743688288962, "pass@4": 0.9531064816302729, "pass@8": 0.9811308550360431, "pass@16": 0.9899558341220963, "pass@32": 0.9954884071162514, "pass@64": 0.9991944736715411, "pass@128": 1.0 }, "6": { "pass@1": 0.6733953967065869, "pass@2": 0.8406031195483057, "pass@4": 0.9395720931590136, "pass@8": 0.9803569606728914, "pass@16": 0.9937328027575888, "pass@32": 0.9981249379712359, "pass@64": 0.9996228304870413, "pass@128": 1.0 }, "7": { "pass@1": 0.6056998890532544, "pass@2": 0.7893496948236497, "pass@4": 0.9154080571659309, "pass@8": 0.9736429870586445, "pass@16": 0.9920458218503939, "pass@32": 0.9961656958235556, "pass@64": 0.9969556692193592, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.5085887419871795, "pass@2": 0.694472179108621, "pass@4": 0.8478189676410628, "pass@8": 0.9408661377431812, "pass@16": 0.9802409507990216, "pass@32": 0.992799258045408, "pass@64": 0.9964515393636147, "pass@128": 0.9967948717948718 }, "9": { "pass@1": 0.4210607394366197, "pass@2": 0.6056206332483088, "pass@4": 0.776658242103539, "pass@8": 0.8980321860923599, "pass@16": 0.9616868959115926, "pass@32": 0.9882404462840045, "pass@64": 0.9979170192930646, "pass@128": 1.0 }, "11": { "pass@1": 0.37329585694050993, "pass@2": 0.551695882871228, "pass@4": 0.7211304773163977, "pass@8": 0.8466730517924583, "pass@16": 0.9240023608454788, "pass@32": 0.9649779620357039, "pass@64": 0.9830596785000053, "pass@128": 0.9915014164305949 }, "12": { "pass@1": 0.292041843220339, "pass@2": 0.445378881400418, "pass@4": 0.6104767092143142, "pass@8": 0.7569505135104562, "pass@16": 0.8642031150634764, "pass@32": 0.9290966759540392, "pass@64": 0.9615571251109591, "pass@128": 0.9774011299435028 }, "13": { "pass@1": 0.24546923981191224, "pass@2": 0.37470611285266464, "pass@4": 0.515894614368345, "pass@8": 0.6490237078227582, "pass@16": 0.7596894299609254, "pass@32": 0.840893650138219, "pass@64": 0.8948081728732732, "pass@128": 0.9278996865203761 }, "14": { "pass@1": 0.18959306853582555, "pass@2": 0.3006176870998604, "pass@4": 0.427712137618312, "pass@8": 0.5498629890140165, "pass@16": 0.6543743903066959, "pass@32": 0.737543883904964, "pass@64": 0.8032422314864625, "pass@128": 0.8629283489096573 }, "15": { "pass@1": 0.1723491690751445, "pass@2": 0.2642844545082154, "pass@4": 0.37061671906257376, "pass@8": 0.47872579891713424, "pass@16": 0.5760957682298359, "pass@32": 0.6578081839366763, "pass@64": 0.726064350257364, "pass@128": 0.7832369942196532 }, "16": { "pass@1": 0.15972782258064516, "pass@2": 0.2509338487426974, "pass@4": 0.3550295015542411, "pass@8": 0.45324696431136946, "pass@16": 0.5356608353740641, "pass@32": 0.6073546211496147, "pass@64": 0.6720452169438437, "pass@128": 0.7258064516129032 }, "17": { "pass@1": 0.13990245664739884, "pass@2": 0.22101185938282278, "pass@4": 0.3164436180484664, "pass@8": 0.41076839730264847, "pass@16": 0.49225585447228964, "pass@32": 0.5578174018891956, "pass@64": 0.6074973845792984, "pass@128": 0.6445086705202312 }, "18": { "pass@1": 0.14140625, "pass@2": 0.22287331271091107, "pass@4": 0.3158783496705768, "pass@8": 0.4047143605454004, "pass@16": 0.4806212432861746, "pass@32": 0.5441695355192144, "pass@64": 0.5929268117187024, "pass@128": 0.6285714285714286 }, "19": { "pass@1": 0.12178584834834835, "pass@2": 0.2003935530018208, "pass@4": 0.2966635212265133, "pass@8": 0.3916437060986376, "pass@16": 0.4675764001606751, "pass@32": 0.5210871242284472, "pass@64": 0.5603378293253486, "pass@128": 0.5975975975975976 }, "20": { "pass@1": 0.1343140015015015, "pass@2": 0.2102024514672154, "pass@4": 0.29733897001613524, "pass@8": 0.3787117342156637, "pass@16": 0.44053976907172987, "pass@32": 0.4840102941432425, "pass@64": 0.5188376365719, "pass@128": 0.5495495495495496 } } }, "teachers_in_school": { "count": 821120, "correct": 353535, "answer_accuracy": 0.4305521726422447, "avg_response_len": 267.12930509547937, "resp_tokens_sum": 219345215, "pass_at_k": { "pass@1": 0.4305521726422447, "pass@2": 0.5488958043095424, "pass@4": 0.6565102379301194, "pass@8": 0.7393861279939664, "pass@16": 0.7967114541993309, "pass@32": 0.8354859692386322, "pass@64": 0.8628640490879613, "pass@128": 0.883398285268901 }, "per_op_pass_at_k": { "10": { "pass@1": 0.4417657208588957, "pass@2": 0.6254098533887251, "pass@4": 0.7895586344460009, "pass@8": 0.902180920679623, "pass@16": 0.9602459166775801, "pass@32": 0.9838844395250028, "pass@64": 0.9940783135740863, "pass@128": 1.0 }, "2": { "pass@1": 0.9897178951367781, "pass@2": 0.9944665743244863, "pass@4": 0.9962265064511312, "pass@8": 0.9976958285113193, "pass@16": 0.9990759530607468, "pass@32": 0.9998433687033346, "pass@64": 0.9999979699106412, "pass@128": 1.0 }, "3": { "pass@1": 0.9911458333333333, "pass@2": 0.9987286745406827, "pass@4": 0.9999071513788048, "pass@8": 0.9999993026951626, "pass@16": 0.9999999999823642, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.8233635355029586, "pass@2": 0.9237856281740668, "pass@4": 0.9751466158261278, "pass@8": 0.9947431792867425, "pass@16": 0.9994093992634441, "pass@32": 0.999977674603606, "pass@64": 0.9999999845146609, "pass@128": 1.0 }, "5": { "pass@1": 0.7294157608695652, "pass@2": 0.8852718817756482, "pass@4": 0.9686271036772574, "pass@8": 0.9933307796493893, "pass@16": 0.9985040585747371, "pass@32": 0.9996314109396737, "pass@64": 0.9999809509319557, "pass@128": 1.0 }, "6": { "pass@1": 0.656294770773639, "pass@2": 0.8270861065361097, "pass@4": 0.9347942541387194, "pass@8": 0.9814209598618784, "pass@16": 0.9945548668728172, "pass@32": 0.9968666716435007, "pass@64": 0.9971318787705731, "pass@128": 0.997134670487106 }, "7": { "pass@1": 0.5994851878612717, "pass@2": 0.7854668511902057, "pass@4": 0.9165456852214282, "pass@8": 0.9749830657252201, "pass@16": 0.9907475606943794, "pass@32": 0.9941723292399648, "pass@64": 0.9955813786159017, "pass@128": 0.9971098265895953 }, "8": { "pass@1": 0.537962147887324, "pass@2": 0.7267418487301769, "pass@4": 0.8734284560380663, "pass@8": 0.9558601430499954, "pass@16": 0.9882046049377172, "pass@32": 0.9968186949583511, "pass@64": 0.9991768774132447, "pass@128": 1.0 }, "9": { "pass@1": 0.4456633607784431, "pass@2": 0.6276234440567688, "pass@4": 0.7922213463960706, "pass@8": 0.9065005181236976, "pass@16": 0.9658771118804983, "pass@32": 0.9887563191805588, "pass@64": 0.9949775085345346, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.36782285391566266, "pass@2": 0.5311914488663316, "pass@4": 0.6897431396000198, "pass@8": 0.8171699659496217, "pass@16": 0.9025748190987217, "pass@32": 0.9493687309679908, "pass@64": 0.9691301516674868, "pass@128": 0.9759036144578314 }, "12": { "pass@1": 0.315071512539185, "pass@2": 0.47797929059807936, "pass@4": 0.6508310780313902, "pass@8": 0.7995801723582354, "pass@16": 0.9021519816505987, "pass@32": 0.9578641087525421, "pass@64": 0.9810057964321914, "pass@128": 0.987460815047022 }, "13": { "pass@1": 0.22306109943977592, "pass@2": 0.34787379518736644, "pass@4": 0.4913096419670232, "pass@8": 0.6284722553035195, "pass@16": 0.7405570071656749, "pass@32": 0.8204934325140724, "pass@64": 0.8717410777978305, "pass@128": 0.9019607843137255 }, "14": { "pass@1": 0.20394736842105263, "pass@2": 0.3087752394437537, "pass@4": 0.4268146111670252, "pass@8": 0.5400761487356827, "pass@16": 0.6359658047968749, "pass@32": 0.7131556435958423, "pass@64": 0.777729951776535, "pass@128": 0.8333333333333334 }, "15": { "pass@1": 0.15125473484848484, "pass@2": 0.2402495675256501, "pass@4": 0.34591075291724904, "pass@8": 0.4546747425102622, "pass@16": 0.5561022181855334, "pass@32": 0.6466769364046635, "pass@64": 0.7251665061652584, "pass@128": 0.793939393939394 }, "16": { "pass@1": 0.15936609686609687, "pass@2": 0.24812613567534839, "pass@4": 0.3483905905992523, "pass@8": 0.4440684963898884, "pass@16": 0.5276683868893259, "pass@32": 0.602690676439445, "pass@64": 0.6705441942266881, "pass@128": 0.7236467236467237 }, "17": { "pass@1": 0.15518245341614906, "pass@2": 0.24886558847263648, "pass@4": 0.3614898456986356, "pass@8": 0.47296045985238105, "pass@16": 0.5633214268791019, "pass@32": 0.6301423383268817, "pass@64": 0.6824064953907748, "pass@128": 0.7236024844720497 }, "18": { "pass@1": 0.14912683823529413, "pass@2": 0.23567189671144048, "pass@4": 0.33769619532852513, "pass@8": 0.43589205343587717, "pass@16": 0.5173551198042894, "pass@32": 0.5809180935714433, "pass@64": 0.6302462830224216, "pass@128": 0.6735294117647059 }, "19": { "pass@1": 0.12544177827380953, "pass@2": 0.19975115415260586, "pass@4": 0.2883934862494865, "pass@8": 0.37491759062354046, "pass@16": 0.4466349227835348, "pass@32": 0.5024021294291547, "pass@64": 0.5451126921232157, "pass@128": 0.5833333333333334 }, "20": { "pass@1": 0.11484842814371257, "pass@2": 0.1869559355698052, "pass@4": 0.2755867404673218, "pass@8": 0.36559495414186255, "pass@16": 0.44360557051556626, "pass@32": 0.5070050690768639, "pass@64": 0.5568275703889851, "pass@128": 0.592814371257485 } } }, "movie_festival_awards": { "count": 801152, "correct": 342392, "answer_accuracy": 0.42737458060393035, "avg_response_len": 285.1436156934015, "resp_tokens_sum": 228443378, "pass_at_k": { "pass@1": 0.42737458060393035, "pass@2": 0.5446091879661872, "pass@4": 0.6511326482965425, "pass@8": 0.7337778342152393, "pass@16": 0.7916555036993614, "pass@32": 0.8305669216442872, "pass@64": 0.8574869133315559, "pass@128": 0.8780955424189167 }, "per_op_pass_at_k": { "10": { "pass@1": 0.47397388059701495, "pass@2": 0.6609736749324245, "pass@4": 0.8215659850354525, "pass@8": 0.9247115909157001, "pass@16": 0.973320962987217, "pass@32": 0.9910160984123851, "pass@64": 0.9968016858355323, "pass@128": 1.0 }, "2": { "pass@1": 0.9850223214285714, "pass@2": 0.990562078177728, "pass@4": 0.9936565247201243, "pass@8": 0.99628322582052, "pass@16": 0.9982468124419266, "pass@32": 0.9994705409631336, "pass@64": 0.9999604250630065, "pass@128": 1.0 }, "3": { "pass@1": 0.9744670189504373, "pass@2": 0.9893956176396318, "pass@4": 0.9947748540361026, "pass@8": 0.9971638249363372, "pass@16": 0.9985952804093334, "pass@32": 0.9994997945753495, "pass@64": 0.9999596976399271, "pass@128": 1.0 }, "4": { "pass@1": 0.8097359283625731, "pass@2": 0.9175131377492288, "pass@4": 0.9695436792988581, "pass@8": 0.990537740763135, "pass@16": 0.9977759727154517, "pass@32": 0.999775258702344, "pass@64": 0.9999986832160754, "pass@128": 1.0 }, "5": { "pass@1": 0.72890625, "pass@2": 0.878439815886985, "pass@4": 0.959648049140916, "pass@8": 0.9871259824810613, "pass@16": 0.9948913084304853, "pass@32": 0.9968584084214227, "pass@64": 0.9970579866112305, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.648166403785489, "pass@2": 0.8241907089098095, "pass@4": 0.9350052029182461, "pass@8": 0.9819223507609767, "pass@16": 0.9944646316781948, "pass@32": 0.9966362978181063, "pass@64": 0.9968442880734899, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.5740704113924051, "pass@2": 0.7606453391308681, "pass@4": 0.893493643595183, "pass@8": 0.9581101204574126, "pass@16": 0.9818771997853405, "pass@32": 0.9913735408888709, "pass@64": 0.9956743455011157, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.5260416666666666, "pass@2": 0.7125529811307374, "pass@4": 0.859053190423269, "pass@8": 0.9460967479622385, "pass@16": 0.9838518118032488, "pass@32": 0.9956321875252359, "pass@64": 0.9990078136203314, "pass@128": 1.0 }, "9": { "pass@1": 0.42235229099678456, "pass@2": 0.6138682210041272, "pass@4": 0.7884813126253103, "pass@8": 0.9053748270240659, "pass@16": 0.9647517139116322, "pass@32": 0.989277240255102, "pass@64": 0.997988016037548, "pass@128": 1.0 }, "11": { "pass@1": 0.36907242063492063, "pass@2": 0.547036307961505, "pass@4": 0.7195467402884156, "pass@8": 0.8510037478551823, "pass@16": 0.9309048284121005, "pass@32": 0.9686246787005046, "pass@64": 0.9828193966686908, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.2889908256880734, "pass@2": 0.4415784752823327, "pass@4": 0.6112021381501623, "pass@8": 0.7652150574070996, "pass@16": 0.8759575044156851, "pass@32": 0.9368876121500012, "pass@64": 0.9662670959057992, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.22470582561728394, "pass@2": 0.3401763907115776, "pass@4": 0.4711223767746623, "pass@8": 0.5984699407411349, "pass@16": 0.707213501007334, "pass@32": 0.7918201843485843, "pass@64": 0.8525426276869016, "pass@128": 0.8888888888888888 }, "14": { "pass@1": 0.19002689169139467, "pass@2": 0.2904014258510713, "pass@4": 0.4036141504196249, "pass@8": 0.5152772210378236, "pass@16": 0.6185174402869785, "pass@32": 0.7102964169679666, "pass@64": 0.7856532696274455, "pass@128": 0.8427299703264095 }, "15": { "pass@1": 0.13638117283950618, "pass@2": 0.22569710253232228, "pass@4": 0.34052704494114155, "pass@8": 0.4628335534068289, "pass@16": 0.5718785905319632, "pass@32": 0.6565004014279554, "pass@64": 0.7200795400563395, "pass@128": 0.7746913580246914 }, "16": { "pass@1": 0.16745022123893805, "pass@2": 0.25647965879265094, "pass@4": 0.3525406710776198, "pass@8": 0.44303220730310716, "pass@16": 0.5225488709949, "pass@32": 0.5886508031370119, "pass@64": 0.6441438599474358, "pass@128": 0.696165191740413 }, "17": { "pass@1": 0.13114175451807228, "pass@2": 0.21291008028175676, "pass@4": 0.31454568470733324, "pass@8": 0.41770793556733127, "pass@16": 0.5038636015595315, "pass@32": 0.5709741441319872, "pass@64": 0.6271183508670329, "pass@128": 0.677710843373494 }, "18": { "pass@1": 0.13253528225806452, "pass@2": 0.20849234823469648, "pass@4": 0.2975716369123214, "pass@8": 0.3861958592550186, "pass@16": 0.46449155962219885, "pass@32": 0.5306578674461371, "pass@64": 0.586280118296387, "pass@128": 0.635483870967742 }, "19": { "pass@1": 0.12665219033232628, "pass@2": 0.20248716904155864, "pass@4": 0.2918663957601977, "pass@8": 0.3797575055446779, "pass@16": 0.4547335616087513, "pass@32": 0.5116890151826592, "pass@64": 0.5518702077421631, "pass@128": 0.5830815709969789 }, "20": { "pass@1": 0.13182713963963963, "pass@2": 0.2154030260575536, "pass@4": 0.3154102786701214, "pass@8": 0.41165280468807414, "pass@16": 0.4890331740618884, "pass@32": 0.5450977224339447, "pass@64": 0.5843774991785532, "pass@128": 0.6156156156156156 } } } } } ]