[ { "checkpoint": "checkpoint-21546", "id_acc": 0.8766857638888889, "ood_acc": 0.329509375, "total_acc": 0.5886981907894737, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.3285625, "ood_avg_resp_len": 317.06312109375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 476007, "answer_accuracy": 0.5878603679260196, "avg_response_len": 249.26987334018338, "resp_tokens_sum": 201840796, "pass_at_k": { "pass@1": 0.5878603679260196, "pass@2": 0.6611123113024427, "pass@4": 0.715592607199787, "pass@8": 0.7548355615364056, "pass@16": 0.7838130901019593, "pass@32": 0.8065087365881829, "pass@64": 0.8256173393051519, "pass@128": 0.8423964590578564 }, "per_op_pass_at_k": { "10": { "pass@1": 0.7548626474926253, "pass@2": 0.8580390739321302, "pass@4": 0.9158941481872283, "pass@8": 0.9422260710100495, "pass@16": 0.9541179241145453, "pass@32": 0.9627354979168701, "pass@64": 0.9700056559069217, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 0.9521271417445483, "pass@2": 0.9600596070351018, "pass@4": 0.967638862899147, "pass@8": 0.9736349720045984, "pass@16": 0.9777725738486086, "pass@32": 0.9807478572199139, "pass@64": 0.983644411229528, "pass@128": 0.9875389408099688 }, "3": { "pass@1": 0.9738627675840978, "pass@2": 0.9860556779599798, "pass@4": 0.9911855877533655, "pass@8": 0.9935549584465483, "pass@16": 0.9950485984443537, "pass@32": 0.9964154468880052, "pass@64": 0.9980977119454602, "pass@128": 1.0 }, "4": { "pass@1": 0.9468017578125, "pass@2": 0.9668841504675199, "pass@4": 0.9788314503655793, "pass@8": 0.98423139689816, "pass@16": 0.9869775056679988, "pass@32": 0.988862756689439, "pass@64": 0.9906228273186573, "pass@128": 0.99375 }, "5": { "pass@1": 0.8940972222222222, "pass@2": 0.925370656792901, "pass@4": 0.9444052689842337, "pass@8": 0.956068245645845, "pass@16": 0.9641325616691484, "pass@32": 0.9702121366978933, "pass@64": 0.9736402664360028, "pass@128": 0.9746031746031746 }, "6": { "pass@1": 0.874625748502994, "pass@2": 0.9345000942995885, "pass@4": 0.9660633426622565, "pass@8": 0.9817471947893537, "pass@16": 0.9901813682097385, "pass@32": 0.9945595172809533, "pass@64": 0.996547366723047, "pass@128": 0.9970059880239521 }, "7": { "pass@1": 0.8708394970414202, "pass@2": 0.9296885919955274, "pass@4": 0.9574697000707745, "pass@8": 0.9709659057101204, "pass@16": 0.9797032271920411, "pass@32": 0.9856532059055091, "pass@64": 0.9903706220860208, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.8371394230769231, "pass@2": 0.921493681228548, "pass@4": 0.9620283975921279, "pass@8": 0.9787379257212695, "pass@16": 0.9860862836307394, "pass@32": 0.9901621390004783, "pass@64": 0.993815134792471, "pass@128": 0.9967948717948718 }, "9": { "pass@1": 0.7703125, "pass@2": 0.8768198264389484, "pass@4": 0.9369461128978587, "pass@8": 0.9678318575372462, "pass@16": 0.9839113625433821, "pass@32": 0.993005940915507, "pass@64": 0.9981911215016727, "pass@128": 1.0 }, "11": { "pass@1": 0.7148326841359773, "pass@2": 0.8247275183466801, "pass@4": 0.8932718025154789, "pass@8": 0.9331762977429914, "pass@16": 0.9563250136105693, "pass@32": 0.9703408496523926, "pass@64": 0.9791441852491506, "pass@128": 0.9886685552407932 }, "12": { "pass@1": 0.610787429378531, "pass@2": 0.7386105504025978, "pass@4": 0.8302062342842734, "pass@8": 0.8869758223344203, "pass@16": 0.9190229298739069, "pass@32": 0.9382509994759017, "pass@64": 0.9504507398120785, "pass@128": 0.96045197740113 }, "13": { "pass@1": 0.4938528605015674, "pass@2": 0.6135426437192993, "pass@4": 0.7050806219755448, "pass@8": 0.7709119554640657, "pass@16": 0.8185553713018926, "pass@32": 0.8529551509357786, "pass@64": 0.8789136115726013, "pass@128": 0.896551724137931 }, "14": { "pass@1": 0.3722984813084112, "pass@2": 0.4832002600142271, "pass@4": 0.5817811873983043, "pass@8": 0.6600182956969374, "pass@16": 0.7170059764733474, "pass@32": 0.759723872506481, "pass@64": 0.7973315503500162, "pass@128": 0.8348909657320872 }, "15": { "pass@1": 0.2915236632947977, "pass@2": 0.38716340573938335, "pass@4": 0.4724536182435286, "pass@8": 0.5428825309662252, "pass@16": 0.6009330341758621, "pass@32": 0.6513234112092107, "pass@64": 0.6978355864283469, "pass@128": 0.7398843930635838 }, "16": { "pass@1": 0.2159022177419355, "pass@2": 0.294323485521971, "pass@4": 0.3682993205486411, "pass@8": 0.4345881141350131, "pass@16": 0.49488662642953934, "pass@32": 0.5483336279194447, "pass@64": 0.5945119316894635, "pass@128": 0.635483870967742 }, "17": { "pass@1": 0.17767792630057805, "pass@2": 0.24014361260297645, "pass@4": 0.3024947796445963, "pass@8": 0.36105936344277334, "pass@16": 0.41425311433468764, "pass@32": 0.4629499781277512, "pass@64": 0.5069431688928129, "pass@128": 0.5491329479768786 }, "18": { "pass@1": 0.17955357142857142, "pass@2": 0.24916057367829025, "pass@4": 0.3168455503776314, "pass@8": 0.37584058796751063, "pass@16": 0.4238660306777394, "pass@32": 0.4629081102911771, "pass@64": 0.4982544518716383, "pass@128": 0.5285714285714286 }, "19": { "pass@1": 0.15315315315315314, "pass@2": 0.21435005379395142, "pass@4": 0.27603142118496454, "pass@8": 0.3325890592148852, "pass@16": 0.38104382491172395, "pass@32": 0.4215814504369368, "pass@64": 0.456708682959238, "pass@128": 0.4864864864864865 }, "20": { "pass@1": 0.13323479729729729, "pass@2": 0.19331757939041397, "pass@4": 0.2577742100931077, "pass@8": 0.3186896897271893, "pass@16": 0.37015786252682287, "pass@32": 0.4131022452662304, "pass@64": 0.4508702308920766, "pass@128": 0.48348348348348347 } } }, "teachers_in_school": { "count": 821120, "correct": 482715, "answer_accuracy": 0.5878738795791114, "avg_response_len": 257.7196682579891, "resp_tokens_sum": 211618774, "pass_at_k": { "pass@1": 0.5878738795791114, "pass@2": 0.6570554679301156, "pass@4": 0.710301292594661, "pass@8": 0.750695245054591, "pass@16": 0.7818350494228476, "pass@32": 0.8066651512685107, "pass@64": 0.8272927747768041, "pass@128": 0.8455183164458301 }, "per_op_pass_at_k": { "10": { "pass@1": 0.7257716641104295, "pass@2": 0.829145482102314, "pass@4": 0.8909685864612009, "pass@8": 0.923828926172029, "pass@16": 0.9424341083861851, "pass@32": 0.9547993211002885, "pass@64": 0.9635684888575998, "pass@128": 0.9693251533742331 }, "2": { "pass@1": 0.969961056231003, "pass@2": 0.9800300062226267, "pass@4": 0.9871420935604933, "pass@8": 0.9922565877731916, "pass@16": 0.9959260151333992, "pass@32": 0.9983344837584649, "pass@64": 0.9996191683758167, "pass@128": 1.0 }, "3": { "pass@1": 0.9798295454545455, "pass@2": 0.9895311381531855, "pass@4": 0.9931646833918487, "pass@8": 0.9950567296773937, "pass@16": 0.9963382395784798, "pass@32": 0.9977819591272412, "pass@64": 0.9992064994093033, "pass@128": 1.0 }, "4": { "pass@1": 0.9674093934911243, "pass@2": 0.9813876642361278, "pass@4": 0.9896786493589189, "pass@8": 0.9942140522761386, "pass@16": 0.9964544182239704, "pass@32": 0.9981567480120942, "pass@64": 0.9996475795537232, "pass@128": 1.0 }, "5": { "pass@1": 0.8929347826086956, "pass@2": 0.9245599395184297, "pass@4": 0.9455650755612065, "pass@8": 0.9585984031211746, "pass@16": 0.9661113498054109, "pass@32": 0.9713447438413154, "pass@64": 0.9761022999756817, "pass@128": 0.9797101449275363 }, "6": { "pass@1": 0.8609643624641834, "pass@2": 0.9185087313584368, "pass@4": 0.9530109303135099, "pass@8": 0.9730129786135173, "pass@16": 0.9845427515918137, "pass@32": 0.9913868693492569, "pass@64": 0.9955681335039379, "pass@128": 1.0 }, "7": { "pass@1": 0.8571396315028902, "pass@2": 0.9151441815347509, "pass@4": 0.9492684707764129, "pass@8": 0.9707522712002761, "pass@16": 0.9846804474521327, "pass@32": 0.9932585974949043, "pass@64": 0.9976034786090924, "pass@128": 1.0 }, "8": { "pass@1": 0.8400528169014084, "pass@2": 0.9146213402462021, "pass@4": 0.9519286311042101, "pass@8": 0.9705870784036298, "pass@16": 0.9806850356855231, "pass@32": 0.9870184734714211, "pass@64": 0.9913704266572482, "pass@128": 0.9943661971830986 }, "9": { "pass@1": 0.7795424775449101, "pass@2": 0.8696370791880805, "pass@4": 0.9201581606191439, "pass@8": 0.9472707355109283, "pass@16": 0.9634659109310358, "pass@32": 0.9732413865436497, "pass@64": 0.979391838196961, "pass@128": 0.9850299401197605 }, "11": { "pass@1": 0.7141613328313253, "pass@2": 0.8234709260269424, "pass@4": 0.8924713595665003, "pass@8": 0.9303646055635199, "pass@16": 0.9515900662979302, "pass@32": 0.9648603754748566, "pass@64": 0.9738234865676904, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.625342868338558, "pass@2": 0.7412462512033179, "pass@4": 0.8177234021295925, "pass@8": 0.864841454508076, "pass@16": 0.8973079096792422, "pass@32": 0.9206851353500622, "pass@64": 0.9360387543338561, "pass@128": 0.9467084639498433 }, "13": { "pass@1": 0.45526960784313725, "pass@2": 0.573507700324224, "pass@4": 0.6657258832666921, "pass@8": 0.7338637568424262, "pass@16": 0.7852004994961729, "pass@32": 0.8273113155682195, "pass@64": 0.8624061133822667, "pass@128": 0.8907563025210085 }, "14": { "pass@1": 0.35798154239766083, "pass@2": 0.45727353570935186, "pass@4": 0.545599068756756, "pass@8": 0.6210483819213921, "pass@16": 0.68125171142204, "pass@32": 0.7270073779808284, "pass@64": 0.7631432072915221, "pass@128": 0.7953216374269005 }, "15": { "pass@1": 0.2765151515151515, "pass@2": 0.36803939990455725, "pass@4": 0.45397499403483615, "pass@8": 0.5263473308035848, "pass@16": 0.5859555392776711, "pass@32": 0.6386314053079184, "pass@64": 0.688474647802565, "pass@128": 0.7333333333333333 }, "16": { "pass@1": 0.19366542022792022, "pass@2": 0.26023509881777596, "pass@4": 0.3251300206063985, "pass@8": 0.38605801417611135, "pass@16": 0.44536576095291924, "pass@32": 0.502109158651507, "pass@64": 0.5541207044234497, "pass@128": 0.603988603988604 }, "17": { "pass@1": 0.20055318322981366, "pass@2": 0.2769511572602337, "pass@4": 0.3524708197383709, "pass@8": 0.4216463758523163, "pass@16": 0.4810756708240572, "pass@32": 0.5287041717254924, "pass@64": 0.5680075699156929, "pass@128": 0.6055900621118012 }, "18": { "pass@1": 0.19087775735294119, "pass@2": 0.26312348019916604, "pass@4": 0.3357639033172321, "pass@8": 0.400534103180857, "pass@16": 0.45257040732607284, "pass@32": 0.49584444773542125, "pass@64": 0.5358095049537671, "pass@128": 0.5735294117647058 }, "19": { "pass@1": 0.14274088541666666, "pass@2": 0.20004298779058863, "pass@4": 0.26311496163425996, "pass@8": 0.3261681997932631, "pass@16": 0.3816211319972022, "pass@32": 0.4273529590398094, "pass@64": 0.466436005280597, "pass@128": 0.5 }, "20": { "pass@1": 0.12951440868263472, "pass@2": 0.18884229572351371, "pass@4": 0.25496642738145764, "pass@8": 0.3193190721203961, "pass@16": 0.3750415414954998, "pass@32": 0.4213703302393265, "pass@64": 0.46037260804896324, "pass@128": 0.49700598802395207 } } }, "movie_festival_awards": { "count": 801152, "correct": 472992, "answer_accuracy": 0.5903898386323694, "avg_response_len": 277.1106219543857, "resp_tokens_sum": 222007729, "pass_at_k": { "pass@1": 0.5903898386323694, "pass@2": 0.6585257779977994, "pass@4": 0.7106201799966714, "pass@8": 0.750469997444581, "pass@16": 0.7816516072340053, "pass@32": 0.8062069138724718, "pass@64": 0.8254911858450941, "pass@128": 0.8407093784949673 }, "per_op_pass_at_k": { "10": { "pass@1": 0.7465951492537314, "pass@2": 0.8382885033493949, "pass@4": 0.8949848927466156, "pass@8": 0.9280509232871795, "pass@16": 0.9478438805819164, "pass@32": 0.9594792695269607, "pass@64": 0.9666690334826524, "pass@128": 0.9731343283582089 }, "2": { "pass@1": 0.9866741071428572, "pass@2": 0.9914500140607426, "pass@4": 0.9947826811827094, "pass@8": 0.9964610707643552, "pass@16": 0.9970452137532337, "pass@32": 0.9971410795160666, "pass@64": 0.9971428571124661, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9861060495626822, "pass@2": 0.9948298076260877, "pass@4": 0.9969340503355447, "pass@8": 0.9972608802887731, "pass@16": 0.9974489749023661, "pass@32": 0.997813411078717, "pass@64": 0.9985422740524781, "pass@128": 1.0 }, "4": { "pass@1": 0.9719937865497076, "pass@2": 0.981466132522908, "pass@4": 0.9844356083778999, "pass@8": 0.9855420893824135, "pass@16": 0.9860609498652024, "pass@32": 0.9866636688638668, "pass@64": 0.9875788552746696, "pass@128": 0.9883040935672515 }, "5": { "pass@1": 0.9102711397058824, "pass@2": 0.9282122076192687, "pass@4": 0.940072399314056, "pass@8": 0.9506308126253655, "pass@16": 0.9599675251090849, "pass@32": 0.9676808245522525, "pass@64": 0.9735033828674721, "pass@128": 0.9764705882352941 }, "6": { "pass@1": 0.871870070977918, "pass@2": 0.9295623338880746, "pass@4": 0.9614348610603484, "pass@8": 0.9783258917269404, "pass@16": 0.9877960079219414, "pass@32": 0.9940389600521968, "pass@64": 0.99779786673376, "pass@128": 1.0 }, "7": { "pass@1": 0.8647151898734177, "pass@2": 0.9250473437655732, "pass@4": 0.9549791028890373, "pass@8": 0.9688374702660314, "pass@16": 0.9767573806272281, "pass@32": 0.9830239517261676, "pass@64": 0.9885077193436845, "pass@128": 0.9936708860759493 }, "8": { "pass@1": 0.8060482357357357, "pass@2": 0.883660235038188, "pass@4": 0.9271438722186747, "pass@8": 0.9515870560611387, "pass@16": 0.9657731015433726, "pass@32": 0.975180094457099, "pass@64": 0.9815049830533622, "pass@128": 0.984984984984985 }, "9": { "pass@1": 0.7751456993569131, "pass@2": 0.8718941470238243, "pass@4": 0.9229569807592373, "pass@8": 0.9491012856677312, "pass@16": 0.9649184196419777, "pass@32": 0.9739737051321179, "pass@64": 0.9781618463030722, "pass@128": 0.9807073954983923 }, "11": { "pass@1": 0.6679811507936508, "pass@2": 0.7739470066241724, "pass@4": 0.8426609691645686, "pass@8": 0.8882811236312015, "pass@16": 0.9217558302441041, "pass@32": 0.9453042212257897, "pass@64": 0.958881139521916, "pass@128": 0.9682539682539683 }, "12": { "pass@1": 0.6055045871559633, "pass@2": 0.7249553775674832, "pass@4": 0.8070858390407616, "pass@8": 0.8604887130755426, "pass@16": 0.8984640220204004, "pass@32": 0.926717664021228, "pass@64": 0.9479442969224088, "pass@128": 0.9663608562691132 }, "13": { "pass@1": 0.44849537037037035, "pass@2": 0.5667077531107221, "pass@4": 0.6617636250329818, "pass@8": 0.7328063948739421, "pass@16": 0.7862427039253898, "pass@32": 0.828873576525592, "pass@64": 0.8648600832732204, "pass@128": 0.8919753086419753 }, "14": { "pass@1": 0.3864753338278932, "pass@2": 0.4937969491109606, "pass@4": 0.582945069644484, "pass@8": 0.6549811145473605, "pass@16": 0.7130843331326536, "pass@32": 0.7584331108720384, "pass@64": 0.7946619410176868, "pass@128": 0.8249258160237388 }, "15": { "pass@1": 0.2794897762345679, "pass@2": 0.37721836419753096, "pass@4": 0.4652975750716343, "pass@8": 0.5389522735789639, "pass@16": 0.6011663209854563, "pass@32": 0.65646740261954, "pass@64": 0.705641491781776, "pass@128": 0.7469135802469136 }, "16": { "pass@1": 0.21928005162241887, "pass@2": 0.2919903955589622, "pass@4": 0.36209677274853913, "pass@8": 0.42627676402045145, "pass@16": 0.4839805580211237, "pass@32": 0.5352047911685343, "pass@64": 0.5795251816926208, "pass@128": 0.616519174041298 }, "17": { "pass@1": 0.16629800451807228, "pass@2": 0.23740372414856273, "pass@4": 0.312128824070184, "pass@8": 0.3813466482123387, "pass@16": 0.4392210735431096, "pass@32": 0.48650114540932765, "pass@64": 0.5239750597065395, "pass@128": 0.5542168674698795 }, "18": { "pass@1": 0.17905745967741934, "pass@2": 0.24398614109728226, "pass@4": 0.31053360880293185, "pass@8": 0.3724229822072563, "pass@16": 0.42312145386859323, "pass@32": 0.46238044186642907, "pass@64": 0.495589661049704, "pass@128": 0.5258064516129032 }, "19": { "pass@1": 0.15674565709969787, "pass@2": 0.2175691206556129, "pass@4": 0.277465813846532, "pass@8": 0.33085507142386955, "pass@16": 0.37441386794026765, "pass@32": 0.40776816099529734, "pass@64": 0.43326206050798804, "pass@128": 0.4501510574018127 }, "20": { "pass@1": 0.14738175675675674, "pass@2": 0.21154693078905676, "pass@4": 0.2815054278125145, "pass@8": 0.35082918530995677, "pass@16": 0.41341781482613965, "pass@32": 0.464339425513764, "pass@64": 0.5009669935181226, "pass@128": 0.5255255255255256 } } } } } ]