[ { "checkpoint": "checkpoint-22350", "id_acc": 0.8321892361111111, "ood_acc": 0.2584078125, "total_acc": 0.5301990131578947, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.74917100694444, "ood_avg_resp_len": 304.17006484375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 433879, "answer_accuracy": 0.5358330204710717, "avg_response_len": 243.30185197992412, "resp_tokens_sum": 197008322, "pass_at_k": { "pass@1": 0.5358330204710717, "pass@2": 0.6076064854518745, "pass@4": 0.6674381751571705, "pass@8": 0.7160587355579924, "pass@16": 0.7550128453944618, "pass@32": 0.78618341453662, "pass@64": 0.8111926583790442, "pass@128": 0.8322794815049004 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6943906710914455, "pass@2": 0.7924839732422831, "pass@4": 0.860622414178315, "pass@8": 0.9069794084790083, "pass@16": 0.938909065525345, "pass@32": 0.961220354064231, "pass@64": 0.9761939203554444, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 0.9308557242990654, "pass@2": 0.9428547446464054, "pass@4": 0.9537001116682842, "pass@8": 0.9648737739918477, "pass@16": 0.9749592956005654, "pass@32": 0.982755395867794, "pass@64": 0.9887116454415467, "pass@128": 0.9937694704049844 }, "3": { "pass@1": 0.9621081804281345, "pass@2": 0.9769709118928944, "pass@4": 0.9848477245963518, "pass@8": 0.9902888229327741, "pass@16": 0.9939032445179141, "pass@32": 0.9963332215254794, "pass@64": 0.9984785531300948, "pass@128": 1.0 }, "4": { "pass@1": 0.9477783203125, "pass@2": 0.9676934670275597, "pass@4": 0.9787657553547993, "pass@8": 0.985568835042546, "pass@16": 0.9906626007427637, "pass@32": 0.9951571846591477, "pass@64": 0.9986095925944035, "pass@128": 1.0 }, "5": { "pass@1": 0.9187748015873016, "pass@2": 0.9491169072615924, "pass@4": 0.966745741603728, "pass@8": 0.9769276192432061, "pass@16": 0.9838399495702144, "pass@32": 0.9891182703066478, "pass@64": 0.9919019584676965, "pass@128": 0.9936507936507937 }, "6": { "pass@1": 0.8201019835329342, "pass@2": 0.8826146918760904, "pass@4": 0.9236047901385079, "pass@8": 0.9489790206766302, "pass@16": 0.9630538085476542, "pass@32": 0.9706400767227515, "pass@64": 0.9751205236185113, "pass@128": 0.9790419161676647 }, "7": { "pass@1": 0.8473095414201184, "pass@2": 0.8948160788333407, "pass@4": 0.9233494061208317, "pass@8": 0.9446898395536166, "pass@16": 0.9609689351179754, "pass@32": 0.9718427623312778, "pass@64": 0.9791843428978619, "pass@128": 0.985207100591716 }, "8": { "pass@1": 0.7699068509615384, "pass@2": 0.8474366072329903, "pass@4": 0.8956249834756224, "pass@8": 0.9289230761890581, "pass@16": 0.952910327741177, "pass@32": 0.9687048044336607, "pass@64": 0.9776952048260006, "pass@128": 0.9839743589743589 }, "9": { "pass@1": 0.7081205985915493, "pass@2": 0.8081637462570703, "pass@4": 0.873297600035911, "pass@8": 0.9156458120013241, "pass@16": 0.9428910229292787, "pass@32": 0.9607310443761353, "pass@64": 0.9728863441329083, "pass@128": 0.9802816901408451 }, "11": { "pass@1": 0.5862473441926346, "pass@2": 0.6982166358100419, "pass@4": 0.7848791272726872, "pass@8": 0.8518025967130893, "pass@16": 0.8993367128402658, "pass@32": 0.9296627844085906, "pass@64": 0.9481568861295607, "pass@128": 0.9603399433427762 }, "12": { "pass@1": 0.4626588983050847, "pass@2": 0.5915618466123937, "pass@4": 0.7009537414814668, "pass@8": 0.7852130266152862, "pass@16": 0.8448068180092854, "pass@32": 0.8845265457694631, "pass@64": 0.9105669496967504, "pass@128": 0.9293785310734464 }, "13": { "pass@1": 0.359203565830721, "pass@2": 0.47705713289561363, "pass@4": 0.5781522086384967, "pass@8": 0.6589915053523162, "pass@16": 0.7248504805274102, "pass@32": 0.7792684090834485, "pass@64": 0.8205216396970003, "pass@128": 0.8495297805642633 }, "14": { "pass@1": 0.23035922897196262, "pass@2": 0.3267222109794686, "pass@4": 0.42098008076093296, "pass@8": 0.5040391320187068, "pass@16": 0.5757914308258796, "pass@32": 0.6382070925379484, "pass@64": 0.6902304607748279, "pass@128": 0.7320872274143302 }, "15": { "pass@1": 0.1915643063583815, "pass@2": 0.2748797420463337, "pass@4": 0.35926651934042914, "pass@8": 0.4369129733469511, "pass@16": 0.5078082628147159, "pass@32": 0.5739481885721509, "pass@64": 0.6338527371652254, "pass@128": 0.6878612716763006 }, "16": { "pass@1": 0.1787046370967742, "pass@2": 0.25333494729489453, "pass@4": 0.3314841881659954, "pass@8": 0.4064332975106817, "pass@16": 0.47415692528401443, "pass@32": 0.5324868128405753, "pass@64": 0.5803103772512649, "pass@128": 0.6161290322580645 }, "17": { "pass@1": 0.16708815028901733, "pass@2": 0.24038220836557264, "pass@4": 0.3126962039788378, "pass@8": 0.37412953912282093, "pass@16": 0.4247942184063712, "pass@32": 0.4686977316823931, "pass@64": 0.5104069609249928, "pass@128": 0.5549132947976878 }, "18": { "pass@1": 0.15174107142857143, "pass@2": 0.21740122328458925, "pass@4": 0.2878790993090149, "pass@8": 0.3559641056655266, "pass@16": 0.41501722330270596, "pass@32": 0.46579089458779094, "pass@64": 0.5139033629581451, "pass@128": 0.5628571428571428 }, "19": { "pass@1": 0.1649540165165165, "pass@2": 0.23666493757537074, "pass@4": 0.30362395044087953, "pass@8": 0.36108731681792244, "pass@16": 0.41238462464001546, "pass@32": 0.4586458618410122, "pass@64": 0.49736060514722336, "pass@128": 0.5315315315315315 }, "20": { "pass@1": 0.14857826576576577, "pass@2": 0.2135457307701402, "pass@4": 0.2781666238792225, "pass@8": 0.3375520764569661, "pass@16": 0.38947846469059344, "pass@32": 0.4322996299171272, "pass@64": 0.4686906739688465, "pass@128": 0.5045045045045045 } } }, "teachers_in_school": { "count": 821120, "correct": 431233, "answer_accuracy": 0.5251765880748246, "avg_response_len": 250.86816908612627, "resp_tokens_sum": 205992871, "pass_at_k": { "pass@1": 0.5251765880748246, "pass@2": 0.5990236113071689, "pass@4": 0.6605269380444797, "pass@8": 0.7110892406005488, "pass@16": 0.7520712251847446, "pass@32": 0.7855516781058292, "pass@64": 0.8138543322827327, "pass@128": 0.8380358534684333 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6537576687116564, "pass@2": 0.7460161948698127, "pass@4": 0.8060986609802601, "pass@8": 0.847945424633065, "pass@16": 0.8813404119372923, "pass@32": 0.9103958247721612, "pass@64": 0.9355294903589877, "pass@128": 0.9570552147239264 }, "2": { "pass@1": 0.9330357142857143, "pass@2": 0.9584501322308118, "pass@4": 0.9728141189798079, "pass@8": 0.9825223285201696, "pass@16": 0.988427837610654, "pass@32": 0.9919575137068457, "pass@64": 0.9936774918251292, "pass@128": 0.993920972644377 }, "3": { "pass@1": 0.9535274621212121, "pass@2": 0.9845360594130279, "pass@4": 0.9929094837577115, "pass@8": 0.9964002893104172, "pass@16": 0.9984903628634235, "pass@32": 0.9996016193444951, "pass@64": 0.9999800782817796, "pass@128": 1.0 }, "4": { "pass@1": 0.9252727440828402, "pass@2": 0.9541336398686112, "pass@4": 0.9712754627010378, "pass@8": 0.9821602965278723, "pass@16": 0.9889760898065934, "pass@32": 0.99406958704192, "pass@64": 0.9982805345757635, "pass@128": 1.0 }, "5": { "pass@1": 0.9472373188405797, "pass@2": 0.9669444682186469, "pass@4": 0.9776280263880057, "pass@8": 0.9841109345296649, "pass@16": 0.9871814520804815, "pass@32": 0.9891453339567674, "pass@64": 0.991220735025819, "pass@128": 0.9942028985507246 }, "6": { "pass@1": 0.7716690544412608, "pass@2": 0.8399441317149118, "pass@4": 0.88877262772311, "pass@8": 0.9236277234039011, "pass@16": 0.947779342166902, "pass@32": 0.9623149622458524, "pass@64": 0.9719698435364026, "pass@128": 0.9799426934097422 }, "7": { "pass@1": 0.8232252528901735, "pass@2": 0.8818197140548907, "pass@4": 0.9189946032757454, "pass@8": 0.9443311493694878, "pass@16": 0.9613068615394603, "pass@32": 0.973776344644011, "pass@64": 0.9830204417073324, "pass@128": 0.9884393063583815 }, "8": { "pass@1": 0.7471390845070423, "pass@2": 0.8324141205500724, "pass@4": 0.8866519479079198, "pass@8": 0.9229622296115595, "pass@16": 0.9482430767490266, "pass@32": 0.9668919515308674, "pass@64": 0.9814110244645726, "pass@128": 0.9915492957746479 }, "9": { "pass@1": 0.6907980913173652, "pass@2": 0.7911919762836525, "pass@4": 0.8568220283280452, "pass@8": 0.9022026761726983, "pass@16": 0.9342429287522777, "pass@32": 0.9557694238038187, "pass@64": 0.968969940690975, "pass@128": 0.9760479041916168 }, "11": { "pass@1": 0.5349444653614458, "pass@2": 0.6472616598282896, "pass@4": 0.7313714308866506, "pass@8": 0.7953237183465092, "pass@16": 0.8460691697512974, "pass@32": 0.8874826145843286, "pass@64": 0.9217594495766532, "pass@128": 0.9487951807228916 }, "12": { "pass@1": 0.4509943181818182, "pass@2": 0.5793468300298671, "pass@4": 0.6863166373403949, "pass@8": 0.7673792999388869, "pass@16": 0.8274085956239333, "pass@32": 0.8702289262058087, "pass@64": 0.901284376599011, "pass@128": 0.9310344827586207 }, "13": { "pass@1": 0.29072566526610644, "pass@2": 0.40266313218200644, "pass@4": 0.5093617699048123, "pass@8": 0.5996578531299764, "pass@16": 0.6733180939052346, "pass@32": 0.7334280975334988, "pass@64": 0.7814138765854218, "pass@128": 0.8235294117647058 }, "14": { "pass@1": 0.2385325292397661, "pass@2": 0.32814910266150954, "pass@4": 0.4164776088844155, "pass@8": 0.5006383588810326, "pass@16": 0.5789538739078068, "pass@32": 0.6484920703430255, "pass@64": 0.7104199418336355, "pass@128": 0.7631578947368421 }, "15": { "pass@1": 0.1773910984848485, "pass@2": 0.26216893342877584, "pass@4": 0.3520031467657451, "pass@8": 0.43594597119352674, "pass@16": 0.510650496174987, "pass@32": 0.5791911558699068, "pass@64": 0.6454691653281225, "pass@128": 0.7090909090909091 }, "16": { "pass@1": 0.18540776353276353, "pass@2": 0.26307989546178523, "pass@4": 0.33983510180885507, "pass@8": 0.40605426855918836, "pass@16": 0.45938682436814693, "pass@32": 0.5041509463348236, "pass@64": 0.5477951643139377, "pass@128": 0.5925925925925926 }, "17": { "pass@1": 0.19257084627329193, "pass@2": 0.2739552257054825, "pass@4": 0.356384882402122, "pass@8": 0.4314625004676136, "pass@16": 0.49515481662757777, "pass@32": 0.5487970243528865, "pass@64": 0.5973562474326223, "pass@128": 0.639751552795031 }, "18": { "pass@1": 0.16482077205882353, "pass@2": 0.23924559981472898, "pass@4": 0.31721665040031777, "pass@8": 0.3896711923876134, "pass@16": 0.4539420417330226, "pass@32": 0.5117977902329494, "pass@64": 0.5613726317248905, "pass@128": 0.5941176470588235 }, "19": { "pass@1": 0.14569382440476192, "pass@2": 0.21112570889576301, "pass@4": 0.27935337240211033, "pass@8": 0.34295164313173365, "pass@16": 0.3982470420257465, "pass@32": 0.4451936526320672, "pass@64": 0.4832365573017363, "pass@128": 0.5119047619047619 }, "20": { "pass@1": 0.14116298652694612, "pass@2": 0.210082541609694, "pass@4": 0.28193506883870056, "pass@8": 0.34853051627177145, "pass@16": 0.4043639567430765, "pass@32": 0.4477221435505894, "pass@64": 0.48468313273594765, "pass@128": 0.5239520958083832 } } }, "movie_festival_awards": { "count": 801152, "correct": 424332, "answer_accuracy": 0.5296523006870107, "avg_response_len": 267.2945146489056, "resp_tokens_sum": 214143535, "pass_at_k": { "pass@1": 0.5296523006870107, "pass@2": 0.6046481452535212, "pass@4": 0.6667228500895, "pass@8": 0.7173735309314182, "pass@16": 0.758131133309332, "pass@32": 0.7910589349365911, "pass@64": 0.8179951826110925, "pass@128": 0.8403898386323694 }, "per_op_pass_at_k": { "10": { "pass@1": 0.6852378731343284, "pass@2": 0.7834590580561754, "pass@4": 0.8437291223858209, "pass@8": 0.8810008174002717, "pass@16": 0.9081553902326077, "pass@32": 0.932425299977489, "pass@64": 0.9556776289164318, "pass@128": 0.9791044776119403 }, "2": { "pass@1": 0.9315401785714286, "pass@2": 0.954753585489314, "pass@4": 0.9696069607370507, "pass@8": 0.9788367325735242, "pass@16": 0.9835176370135428, "pass@32": 0.9863311311106975, "pass@64": 0.9889418407176944, "pass@128": 0.9914285714285714 }, "3": { "pass@1": 0.9608236151603499, "pass@2": 0.9816564702371388, "pass@4": 0.9905631080007128, "pass@8": 0.996430338052377, "pass@16": 0.9992393738766351, "pass@32": 0.9999502865055028, "pass@64": 0.9999999185821354, "pass@128": 1.0 }, "4": { "pass@1": 0.9050621345029239, "pass@2": 0.9440120355021416, "pass@4": 0.9680004067474015, "pass@8": 0.984340659490216, "pass@16": 0.9928412933334729, "pass@32": 0.9964188428980713, "pass@64": 0.9983444427656412, "pass@128": 1.0 }, "5": { "pass@1": 0.9003216911764705, "pass@2": 0.9268205042843912, "pass@4": 0.9463646645272281, "pass@8": 0.9632171704788541, "pass@16": 0.9757164148383949, "pass@32": 0.9834181485021496, "pass@64": 0.9877429121098109, "pass@128": 0.9911764705882353 }, "6": { "pass@1": 0.8128450315457413, "pass@2": 0.8733140415807641, "pass@4": 0.9138919448948998, "pass@8": 0.9426343651866598, "pass@16": 0.9625392372821305, "pass@32": 0.9741035535727569, "pass@64": 0.9804410838777078, "pass@128": 0.9842271293375394 }, "7": { "pass@1": 0.7880241297468354, "pass@2": 0.8616713283663908, "pass@4": 0.9114520427826266, "pass@8": 0.9435628508320384, "pass@16": 0.9629477011367917, "pass@32": 0.9742916300599795, "pass@64": 0.9818103307680968, "pass@128": 0.9873417721518988 }, "8": { "pass@1": 0.7663288288288288, "pass@2": 0.8553814641413071, "pass@4": 0.9128960433999799, "pass@8": 0.9495830700873962, "pass@16": 0.9718125161329592, "pass@32": 0.9853249653977751, "pass@64": 0.9933116829048084, "pass@128": 0.996996996996997 }, "9": { "pass@1": 0.6727039790996785, "pass@2": 0.7811534736815446, "pass@4": 0.8535718273358913, "pass@8": 0.9004946604118668, "pass@16": 0.9332582598657084, "pass@32": 0.9588299872546514, "pass@64": 0.9769867806714441, "pass@128": 0.9839228295819936 }, "11": { "pass@1": 0.5628224206349206, "pass@2": 0.6886682133483313, "pass@4": 0.7795924515387953, "pass@8": 0.843846333850522, "pass@16": 0.8911162090697101, "pass@32": 0.92447878151729, "pass@64": 0.9459007713570995, "pass@128": 0.9587301587301588 }, "12": { "pass@1": 0.47842603211009177, "pass@2": 0.6046798622649233, "pass@4": 0.7040755857352686, "pass@8": 0.7796449097929059, "pass@16": 0.8354664617911026, "pass@32": 0.8750244266732382, "pass@64": 0.9055190650001771, "pass@128": 0.9327217125382263 }, "13": { "pass@1": 0.32978877314814814, "pass@2": 0.4449008761057645, "pass@4": 0.5485310094340058, "pass@8": 0.6357017178809099, "pass@16": 0.7077049545874511, "pass@32": 0.7677406341601419, "pass@64": 0.8184612944758107, "pass@128": 0.8611111111111112 }, "14": { "pass@1": 0.24547941394658754, "pass@2": 0.33984986506694087, "pass@4": 0.4344585242868378, "pass@8": 0.5230985598476801, "pass@16": 0.6032954304388765, "pass@32": 0.6722870419315873, "pass@64": 0.7308149824893481, "pass@128": 0.7804154302670623 }, "15": { "pass@1": 0.1517891589506173, "pass@2": 0.23095328144745775, "pass@4": 0.32680433059293523, "pass@8": 0.4268563984986593, "pass@16": 0.5184743795580643, "pass@32": 0.5953989749015376, "pass@64": 0.6532845826632232, "pass@128": 0.6975308641975309 }, "16": { "pass@1": 0.19750184365781712, "pass@2": 0.2707752653705895, "pass@4": 0.3384619665904593, "pass@8": 0.39868858340697155, "pass@16": 0.4537976264326197, "pass@32": 0.5075626238811257, "pass@64": 0.5621747076841135, "pass@128": 0.6135693215339233 }, "17": { "pass@1": 0.16789815512048192, "pass@2": 0.24503390036524042, "pass@4": 0.3252749837934414, "pass@8": 0.39844549811174756, "pass@16": 0.4604747765970229, "pass@32": 0.5131333897557869, "pass@64": 0.5606722490567707, "pass@128": 0.6054216867469879 }, "18": { "pass@1": 0.16890120967741937, "pass@2": 0.24259866332232669, "pass@4": 0.31384557797210855, "pass@8": 0.37329870800292475, "pass@16": 0.4203020018185082, "pass@32": 0.46182948474688756, "pass@64": 0.5007418217465677, "pass@128": 0.535483870967742 }, "19": { "pass@1": 0.14680891238670696, "pass@2": 0.20984675702357458, "pass@4": 0.27491625404830444, "pass@8": 0.3400342250777418, "pass@16": 0.4028458907394394, "pass@32": 0.45635539239742556, "pass@64": 0.4952112213686042, "pass@128": 0.5226586102719033 }, "20": { "pass@1": 0.14135228978978978, "pass@2": 0.2111279586673286, "pass@4": 0.2834856467828909, "pass@8": 0.34874205971355804, "pass@16": 0.40399217073375576, "pass@32": 0.45152029441970093, "pass@64": 0.49416572663356273, "pass@128": 0.5345345345345346 } } } } } ]