[ { "checkpoint": "checkpoint-42828", "id_acc": 0.4487638888888889, "ood_acc": 0.13439375, "total_acc": 0.2833059210526316, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 204.87644010416668, "ood_avg_resp_len": 314.86191484375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 224253, "answer_accuracy": 0.2769485555643377, "avg_response_len": 249.4732712713405, "resp_tokens_sum": 202005493, "pass_at_k": { "pass@1": 0.2769485555643377, "pass@2": 0.3713698318214332, "pass@4": 0.47568788097662684, "pass@8": 0.5759088361249274, "pass@16": 0.6595118676241185, "pass@32": 0.7230610799994464, "pass@64": 0.7715846034812562, "pass@128": 0.8101485931078091 }, "per_op_pass_at_k": { "10": { "pass@1": 0.19429848820058998, "pass@2": 0.304934107379277, "pass@4": 0.434951352319898, "pass@8": 0.5684199490328546, "pass@16": 0.6891009886552397, "pass@32": 0.7857931447451506, "pass@64": 0.860108943525792, "pass@128": 0.9233038348082596 }, "2": { "pass@1": 0.831386292834891, "pass@2": 0.8734914268893955, "pass@4": 0.9105459002204167, "pass@8": 0.9401008638276085, "pass@16": 0.9611783814748307, "pass@32": 0.9741949948603676, "pass@64": 0.9817242745943495, "pass@128": 0.9875389408099688 }, "3": { "pass@1": 0.8260225535168195, "pass@2": 0.8906170988947479, "pass@4": 0.9285593713056506, "pass@8": 0.9526553375068405, "pass@16": 0.9699066685395095, "pass@32": 0.9816788551557367, "pass@64": 0.988712453516909, "pass@128": 0.9938837920489296 }, "4": { "pass@1": 0.5572509765625, "pass@2": 0.6628467950295271, "pass@4": 0.7544799485025304, "pass@8": 0.8284263532654119, "pass@16": 0.884775967340774, "pass@32": 0.9236644996257792, "pass@64": 0.948151095592842, "pass@128": 0.965625 }, "5": { "pass@1": 0.3347718253968254, "pass@2": 0.4497898700162479, "pass@4": 0.5641970530469402, "pass@8": 0.6677945563217111, "pass@16": 0.7584282589253484, "pass@32": 0.8321889687382951, "pass@64": 0.8851659220568333, "pass@128": 0.9174603174603174 }, "6": { "pass@1": 0.4252432634730539, "pass@2": 0.5709287625536326, "pass@4": 0.7067664802565847, "pass@8": 0.8127587788249542, "pass@16": 0.8845402446547445, "pass@32": 0.9276708113313906, "pass@64": 0.9525643996495025, "pass@128": 0.9670658682634731 }, "7": { "pass@1": 0.3097956730769231, "pass@2": 0.4431132210082467, "pass@4": 0.5825773815070447, "pass@8": 0.7077649850344365, "pass@16": 0.804085063866593, "pass@32": 0.8732138745287983, "pass@64": 0.9242260451555102, "pass@128": 0.9526627218934911 }, "8": { "pass@1": 0.27226061698717946, "pass@2": 0.3969216888754288, "pass@4": 0.5352359978440191, "pass@8": 0.668389018888514, "pass@16": 0.777516858096813, "pass@32": 0.8581637138287497, "pass@64": 0.916168827905763, "pass@128": 0.9551282051282052 }, "9": { "pass@1": 0.20517165492957745, "pass@2": 0.3156790645447484, "pass@4": 0.44685785738050343, "pass@8": 0.5801721703370608, "pass@16": 0.6942797587019922, "pass@32": 0.7812453529979867, "pass@64": 0.8455093152480154, "pass@128": 0.8929577464788733 }, "11": { "pass@1": 0.1927452195467422, "pass@2": 0.3026219301376279, "pass@4": 0.431418140455531, "pass@8": 0.5596011558602536, "pass@16": 0.6691124471752867, "pass@32": 0.7558813134298097, "pass@64": 0.821310706535779, "pass@128": 0.8640226628895185 }, "12": { "pass@1": 0.16684322033898305, "pass@2": 0.264343248142711, "pass@4": 0.3767520347032891, "pass@8": 0.4853319445797364, "pass@16": 0.5806646367336986, "pass@32": 0.6630849144980867, "pass@64": 0.7337648428979751, "pass@128": 0.7909604519774012 }, "13": { "pass@1": 0.16465027429467086, "pass@2": 0.25840046713400666, "pass@4": 0.3634469285579116, "pass@8": 0.46413359882342115, "pass@16": 0.5558490901969614, "pass@32": 0.6407655550139999, "pass@64": 0.7175643568137752, "pass@128": 0.780564263322884 }, "14": { "pass@1": 0.14057632398753894, "pass@2": 0.23038280042681575, "pass@4": 0.3386714943342362, "pass@8": 0.4454643097630123, "pass@16": 0.5379967864190127, "pass@32": 0.6166861756042253, "pass@64": 0.689484801493184, "pass@128": 0.7632398753894081 }, "15": { "pass@1": 0.11599078757225434, "pass@2": 0.193432891652633, "pass@4": 0.2944971575084906, "pass@8": 0.40651592812419884, "pass@16": 0.5089160038620619, "pass@32": 0.5890422875655702, "pass@64": 0.6509615060463998, "pass@128": 0.708092485549133 }, "16": { "pass@1": 0.1469758064516129, "pass@2": 0.23466432245364482, "pass@4": 0.3392853496135562, "pass@8": 0.441151099642495, "pass@16": 0.5216397014566494, "pass@32": 0.5830837413676024, "pass@64": 0.6388692766638284, "pass@128": 0.6935483870967742 }, "17": { "pass@1": 0.11547145953757225, "pass@2": 0.1877869549633608, "pass@4": 0.2775737121552002, "pass@8": 0.36958965536460775, "pass@16": 0.4467152912085064, "pass@32": 0.5056999218405851, "pass@64": 0.5550444893472826, "pass@128": 0.6011560693641619 }, "18": { "pass@1": 0.11703125, "pass@2": 0.1914370078740158, "pass@4": 0.2851955653757565, "pass@8": 0.3820999147912633, "pass@16": 0.46089162638401937, "pass@32": 0.5131813025594367, "pass@64": 0.5494175506884521, "pass@128": 0.5828571428571429 }, "19": { "pass@1": 0.11031343843843844, "pass@2": 0.1848742196921331, "pass@4": 0.2798845093687614, "pass@8": 0.3767028297947531, "pass@16": 0.45188701937385706, "pass@32": 0.4971710103722426, "pass@64": 0.5240927048521262, "pass@128": 0.5435435435435435 }, "20": { "pass@1": 0.09830142642642643, "pass@2": 0.1625456657444847, "pass@4": 0.24512006128738417, "pass@8": 0.3354723396365215, "pass@16": 0.41607247504734884, "pass@32": 0.47314849238736145, "pass@64": 0.5110935339188466, "pass@128": 0.5405405405405406 } } }, "teachers_in_school": { "count": 821120, "correct": 234790, "answer_accuracy": 0.2859387178487919, "avg_response_len": 258.47156566640683, "resp_tokens_sum": 212236172, "pass_at_k": { "pass@1": 0.2859387178487919, "pass@2": 0.3777207700947009, "pass@4": 0.4797066342447682, "pass@8": 0.5781080472364224, "pass@16": 0.659984755265672, "pass@32": 0.7221385448329016, "pass@64": 0.7697332178023997, "pass@128": 0.8079501169134841 }, "per_op_pass_at_k": { "10": { "pass@1": 0.1877396472392638, "pass@2": 0.2979068040191296, "pass@4": 0.42743402032567995, "pass@8": 0.5592994362559849, "pass@16": 0.6798039506691061, "pass@32": 0.7821547762425294, "pass@64": 0.8609090332078575, "pass@128": 0.911042944785276 }, "2": { "pass@1": 0.8750237462006079, "pass@2": 0.9054504822535481, "pass@4": 0.9290282496982711, "pass@8": 0.9469597285175716, "pass@16": 0.9616501693834948, "pass@32": 0.9731913245513274, "pass@64": 0.982534585697019, "pass@128": 0.9908814589665653 }, "3": { "pass@1": 0.8613162878787879, "pass@2": 0.9110038624433314, "pass@4": 0.9384561071911464, "pass@8": 0.9562286327789103, "pass@16": 0.9687754714930751, "pass@32": 0.9774223917946637, "pass@64": 0.9834020912247246, "pass@128": 0.9878787878787879 }, "4": { "pass@1": 0.5816383136094675, "pass@2": 0.6702755905511808, "pass@4": 0.7480943742054436, "pass@8": 0.8105270906947715, "pass@16": 0.8574687882367842, "pass@32": 0.8923947023004354, "pass@64": 0.9173499844593105, "pass@128": 0.9349112426035503 }, "5": { "pass@1": 0.35183423913043477, "pass@2": 0.44896653543307047, "pass@4": 0.5367537307836514, "pass@8": 0.6184585802569412, "pass@16": 0.6973476991211023, "pass@32": 0.7694325969464841, "pass@64": 0.8289443827711489, "pass@128": 0.8753623188405797 }, "6": { "pass@1": 0.44206661891117477, "pass@2": 0.5813061926088036, "pass@4": 0.7101583094555867, "pass@8": 0.8137002926251504, "pass@16": 0.8888854739106302, "pass@32": 0.9397605232654371, "pass@64": 0.9715154997083257, "pass@128": 0.9885386819484241 }, "7": { "pass@1": 0.31550307080924855, "pass@2": 0.4467138500750988, "pass@4": 0.5860489259651785, "pass@8": 0.7147282946538364, "pass@16": 0.8169303698336912, "pass@32": 0.8862546718118719, "pass@64": 0.9261973005402598, "pass@128": 0.9450867052023122 }, "8": { "pass@1": 0.25985915492957745, "pass@2": 0.3925993955861151, "pass@4": 0.5404817329171882, "pass@8": 0.6789521589715024, "pass@16": 0.78910166835959, "pass@32": 0.8686986141098911, "pass@64": 0.923139404679525, "pass@128": 0.9605633802816902 }, "9": { "pass@1": 0.21641092814371257, "pass@2": 0.33585609293224555, "pass@4": 0.47451746422789937, "pass@8": 0.6121577136393171, "pass@16": 0.730587365488769, "pass@32": 0.8226909510349335, "pass@64": 0.889786424123189, "pass@128": 0.9341317365269461 }, "11": { "pass@1": 0.16799228162650603, "pass@2": 0.2671806813869653, "pass@4": 0.3836231012350866, "pass@8": 0.49547203715780913, "pass@16": 0.5913990690849538, "pass@32": 0.6777404005188927, "pass@64": 0.7607330836250256, "pass@128": 0.8403614457831325 }, "12": { "pass@1": 0.19803095611285265, "pass@2": 0.3043818496531975, "pass@4": 0.4234678795048738, "pass@8": 0.5373117655964851, "pass@16": 0.633112223326805, "pass@32": 0.7141837879299113, "pass@64": 0.787705464533175, "pass@128": 0.8526645768025078 }, "13": { "pass@1": 0.14377626050420167, "pass@2": 0.2336692058713247, "pass@4": 0.3419758614731982, "pass@8": 0.44945168699137733, "pass@16": 0.5414633284811603, "pass@32": 0.6177322308195665, "pass@64": 0.686910562781833, "pass@128": 0.7563025210084033 }, "14": { "pass@1": 0.14761513157894737, "pass@2": 0.2372986168669705, "pass@4": 0.3444328192528565, "pass@8": 0.44664556769660785, "pass@16": 0.5269598579960679, "pass@32": 0.5898055394127177, "pass@64": 0.6485638274319744, "pass@128": 0.7076023391812866 }, "15": { "pass@1": 0.12208806818181818, "pass@2": 0.19942510737294203, "pass@4": 0.2963789963754532, "pass@8": 0.3982598483278521, "pass@16": 0.4878949107801932, "pass@32": 0.5587040931915987, "pass@64": 0.6197747324074336, "pass@128": 0.6727272727272727 }, "16": { "pass@1": 0.1268028846153846, "pass@2": 0.2082698897413465, "pass@4": 0.31007912766246093, "pass@8": 0.4124197783852443, "pass@16": 0.4938064368685502, "pass@32": 0.5518641332595097, "pass@64": 0.5992631357352116, "pass@128": 0.6438746438746439 }, "17": { "pass@1": 0.13327348602484473, "pass@2": 0.22033259769159275, "pass@4": 0.32994341555519846, "pass@8": 0.4416166613059523, "pass@16": 0.5330099260532796, "pass@32": 0.597999239811453, "pass@64": 0.6442029397091859, "pass@128": 0.6832298136645962 }, "18": { "pass@1": 0.11691176470588235, "pass@2": 0.19327307202408514, "pass@4": 0.29130406658726493, "pass@8": 0.39555143896715, "pass@16": 0.48208167098196425, "pass@32": 0.5367870440715797, "pass@64": 0.569770393006611, "pass@128": 0.5970588235294118 }, "19": { "pass@1": 0.10398065476190477, "pass@2": 0.16996581482002251, "pass@4": 0.25625598474297845, "pass@8": 0.35065165885240784, "pass@16": 0.4305015663253454, "pass@32": 0.4829005293848349, "pass@64": 0.5146124285334518, "pass@128": 0.5357142857142857 }, "20": { "pass@1": 0.09541074101796407, "pass@2": 0.1615064654156255, "pass@4": 0.24927919302003418, "pass@8": 0.3450166044834941, "pass@16": 0.4261222996576307, "pass@32": 0.4774343147460568, "pass@64": 0.5063468164477533, "pass@128": 0.5299401197604791 } } }, "movie_festival_awards": { "count": 801152, "correct": 229957, "answer_accuracy": 0.2870329225914683, "avg_response_len": 280.59499945079085, "resp_tokens_sum": 224799245, "pass_at_k": { "pass@1": 0.2870329225914683, "pass@2": 0.3787641858715626, "pass@4": 0.4802384863509366, "pass@8": 0.5777949264058008, "pass@16": 0.6588539403003395, "pass@32": 0.7211879136421478, "pass@64": 0.7699639648290318, "pass@128": 0.8101933216168717 }, "per_op_pass_at_k": { "10": { "pass@1": 0.2155783582089552, "pass@2": 0.33451969972969814, "pass@4": 0.4693261846000589, "pass@8": 0.5978972579074031, "pass@16": 0.7044582550974517, "pass@32": 0.7876023011362222, "pass@64": 0.8552567047585439, "pass@128": 0.9194029850746268 }, "2": { "pass@1": 0.9069642857142857, "pass@2": 0.9312689820022496, "pass@4": 0.9486790698484115, "pass@8": 0.9622491070080178, "pass@16": 0.9721970036911877, "pass@32": 0.9795142189097774, "pass@64": 0.984230110753602, "pass@128": 0.9857142857142858 }, "3": { "pass@1": 0.8650464650145773, "pass@2": 0.9146810650581944, "pass@4": 0.9394977516476619, "pass@8": 0.9532998856091857, "pass@16": 0.9628501996761378, "pass@32": 0.9709821548050687, "pass@64": 0.97883034020522, "pass@128": 0.9883381924198251 }, "4": { "pass@1": 0.5795869883040936, "pass@2": 0.685957069922181, "pass@4": 0.7741738061251112, "pass@8": 0.8446660971516667, "pass@16": 0.8991103940142082, "pass@32": 0.9343024245485025, "pass@64": 0.9508020706569525, "pass@128": 0.9590643274853801 }, "5": { "pass@1": 0.3434283088235294, "pass@2": 0.4411019279759142, "pass@4": 0.5363170749795979, "pass@8": 0.6291732271577856, "pass@16": 0.7177730817023343, "pass@32": 0.7982552067000316, "pass@64": 0.8593381500839656, "pass@128": 0.8970588235294118 }, "6": { "pass@1": 0.4251281545741325, "pass@2": 0.5811580950843291, "pass@4": 0.7208497665118351, "pass@8": 0.8232985290012133, "pass@16": 0.8898679892074591, "pass@32": 0.9340736625308671, "pass@64": 0.9627191767527687, "pass@128": 0.9779179810725552 }, "7": { "pass@1": 0.3076542721518987, "pass@2": 0.44246525528256747, "pass@4": 0.5810167029160911, "pass@8": 0.7043732363576187, "pass@16": 0.8016622464263247, "pass@32": 0.8742022822931801, "pass@64": 0.9264441039386613, "pass@128": 0.9651898734177216 }, "8": { "pass@1": 0.2358061186186186, "pass@2": 0.35777263483956373, "pass@4": 0.4963302315251132, "pass@8": 0.6330193138275914, "pass@16": 0.7494470352400429, "pass@32": 0.8392101017513182, "pass@64": 0.9060906232833575, "pass@128": 0.9519519519519519 }, "9": { "pass@1": 0.18197347266881028, "pass@2": 0.2925186564802387, "pass@4": 0.42774186194973196, "pass@8": 0.5688716036414828, "pass@16": 0.6958608696659061, "pass@32": 0.7992127490405743, "pass@64": 0.8781366109068879, "pass@128": 0.9292604501607717 }, "11": { "pass@1": 0.16783234126984126, "pass@2": 0.2776492782152234, "pass@4": 0.41207455228810697, "pass@8": 0.5425249589903164, "pass@16": 0.6496929013278826, "pass@32": 0.7376433064662824, "pass@64": 0.8115952261395818, "pass@128": 0.8698412698412699 }, "12": { "pass@1": 0.16243788226299694, "pass@2": 0.25898054672638415, "pass@4": 0.37313383304151193, "pass@8": 0.4853343503793426, "pass@16": 0.5823023549839643, "pass@32": 0.6685330995309777, "pass@64": 0.7505451517849687, "pass@128": 0.8256880733944955 }, "13": { "pass@1": 0.15593653549382716, "pass@2": 0.2489063867016624, "pass@4": 0.35850821048989273, "pass@8": 0.46484399413550886, "pass@16": 0.554682742479938, "pass@32": 0.6296884159566101, "pass@64": 0.6960370266813714, "pass@128": 0.7561728395061729 }, "14": { "pass@1": 0.14308234421364985, "pass@2": 0.2288688842029019, "pass@4": 0.3325652593685433, "pass@8": 0.43882321384252904, "pass@16": 0.530629508814399, "pass@32": 0.6042841232586478, "pass@64": 0.6681575340775399, "pass@128": 0.7299703264094956 }, "15": { "pass@1": 0.10551697530864197, "pass@2": 0.18038467847769024, "pass@4": 0.28229776833451387, "pass@8": 0.3971891617144702, "pass@16": 0.49889005466932873, "pass@32": 0.5715708980528333, "pass@64": 0.6243133684428923, "pass@128": 0.6697530864197531 }, "16": { "pass@1": 0.13889657079646017, "pass@2": 0.22407265463498482, "pass@4": 0.32410092350181907, "pass@8": 0.41857070650905354, "pass@16": 0.49424763506606695, "pass@32": 0.5532521898450039, "pass@64": 0.6042886991067128, "pass@128": 0.6519174041297935 }, "17": { "pass@1": 0.11502259036144578, "pass@2": 0.19192616924390482, "pass@4": 0.28962638376904687, "pass@8": 0.3891213228204151, "pass@16": 0.4674030310535856, "pass@32": 0.5186417774810506, "pass@64": 0.560044060038099, "pass@128": 0.6054216867469879 }, "18": { "pass@1": 0.11020665322580646, "pass@2": 0.18079954597409195, "pass@4": 0.2687805620063621, "pass@8": 0.35802912525157776, "pass@16": 0.429463810012179, "pass@32": 0.48079673280382973, "pass@64": 0.5232930609521405, "pass@128": 0.5612903225806452 }, "19": { "pass@1": 0.1171875, "pass@2": 0.19060626650331858, "pass@4": 0.2793686684179584, "pass@8": 0.36774681277531496, "pass@16": 0.43928069476041537, "pass@32": 0.48800335846972764, "pass@64": 0.5236880280647143, "pass@128": 0.5589123867069486 }, "20": { "pass@1": 0.09292886636636637, "pass@2": 0.15924191908443877, "pass@4": 0.25056567760111054, "pass@8": 0.35344805682641933, "pass@16": 0.44386425978250205, "pass@32": 0.506770362429784, "pass@64": 0.5464396237727965, "pass@128": 0.5765765765765766 } } } } } ]