[ { "checkpoint": "checkpoint-42828", "id_acc": 0.7638064236111111, "ood_acc": 0.26778984375, "total_acc": 0.5027450657894736, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 194.81892274305557, "ood_avg_resp_len": 329.1969828125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 406947, "answer_accuracy": 0.5025724687796396, "avg_response_len": 252.37878398672146, "resp_tokens_sum": 204358168, "pass_at_k": { "pass@1": 0.5025724687796396, "pass@2": 0.6136163776042443, "pass@4": 0.7087776433691901, "pass@8": 0.778848798243971, "pass@16": 0.8267637766621762, "pass@32": 0.8599245759312861, "pass@64": 0.8848061815918338, "pass@128": 0.904995257666772 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5961928466076696, "pass@2": 0.7596196838780109, "pass@4": 0.8756493018571788, "pass@8": 0.9365228324321908, "pass@16": 0.9606329559490022, "pass@32": 0.9688884108408146, "pass@64": 0.9737911492704813, "pass@128": 0.9793510324483776 }, "2": { "pass@1": 0.9976878894080997, "pass@2": 0.9985906952927612, "pass@4": 0.9993798701564174, "pass@8": 0.9998843836043162, "pass@16": 0.9999967738107939, "pass@32": 0.9999999991592076, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9983514908256881, "pass@2": 0.9998521364588602, "pass@4": 0.9999973724614697, "pass@8": 0.99999999928849, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9142578125, "pass@2": 0.9533737543061023, "pass@4": 0.9760740093035247, "pass@8": 0.9864437669797045, "pass@16": 0.9911445712028863, "pass@32": 0.9944248711665763, "pass@64": 0.9964253512379372, "pass@128": 0.996875 }, "5": { "pass@1": 0.794890873015873, "pass@2": 0.894600674915636, "pass@4": 0.94974954797317, "pass@8": 0.9755730371991035, "pass@16": 0.9874647018460322, "pass@32": 0.9937251962720715, "pass@64": 0.9976983546461972, "pass@128": 1.0 }, "6": { "pass@1": 0.770279752994012, "pass@2": 0.8859394891319728, "pass@4": 0.9532805356665737, "pass@8": 0.9830399819257302, "pass@16": 0.9944878419475502, "pass@32": 0.9986150659952652, "pass@64": 0.999872031232074, "pass@128": 1.0 }, "7": { "pass@1": 0.6746486686390533, "pass@2": 0.8198156420351302, "pass@4": 0.9145076452166845, "pass@8": 0.9619153600090276, "pass@16": 0.9826155678869178, "pass@32": 0.992917007771169, "pass@64": 0.9985178232847783, "pass@128": 1.0 }, "8": { "pass@1": 0.6353415464743589, "pass@2": 0.7960370450989301, "pass@4": 0.9076204095281363, "pass@8": 0.9645666794146627, "pass@16": 0.9884791332293995, "pass@32": 0.9974799702985278, "pass@64": 0.9998132509778702, "pass@128": 1.0 }, "9": { "pass@1": 0.5373899647887324, "pass@2": 0.7189194715537314, "pass@4": 0.8618524614880887, "pass@8": 0.9448509571689906, "pass@16": 0.9817401961596698, "pass@32": 0.9947024763761073, "pass@64": 0.998955947935806, "pass@128": 1.0 }, "11": { "pass@1": 0.5171963526912181, "pass@2": 0.6901484882112825, "pass@4": 0.8268923876548013, "pass@8": 0.9124617472551908, "pass@16": 0.9559764167265765, "pass@32": 0.9757553214879822, "pass@64": 0.9844442937777405, "pass@128": 0.9886685552407932 }, "12": { "pass@1": 0.435204802259887, "pass@2": 0.6030044708394502, "pass@4": 0.748447401966703, "pass@8": 0.8497352014720185, "pass@16": 0.9089616085075132, "pass@32": 0.9431542535728017, "pass@64": 0.9651132899120077, "pass@128": 0.9774011299435028 }, "13": { "pass@1": 0.3713019200626959, "pass@2": 0.523809101399551, "pass@4": 0.6686971683398507, "pass@8": 0.7846809994799563, "pass@16": 0.8617996533225794, "pass@32": 0.9064250879752345, "pass@64": 0.9331490503349423, "pass@128": 0.9529780564263323 }, "14": { "pass@1": 0.30415206386292837, "pass@2": 0.44828767753329857, "pass@4": 0.5943040942896155, "pass@8": 0.716445667438578, "pass@16": 0.8051037312119722, "pass@32": 0.8657429264678448, "pass@64": 0.9090951189790747, "pass@128": 0.9470404984423676 }, "15": { "pass@1": 0.24954841040462428, "pass@2": 0.37298704826817164, "pass@4": 0.5029955495989294, "pass@8": 0.6201905378692797, "pass@16": 0.7157051200779855, "pass@32": 0.7883710262863396, "pass@64": 0.8416705019935963, "pass@128": 0.8815028901734104 }, "16": { "pass@1": 0.21121471774193548, "pass@2": 0.3187492062484124, "pass@4": 0.4356203729574124, "pass@8": 0.545153874059209, "pass@16": 0.6407560148339182, "pass@32": 0.7208517883776376, "pass@64": 0.7881858215754862, "pass@128": 0.8419354838709677 }, "17": { "pass@1": 0.17341040462427745, "pass@2": 0.26089931045468995, "pass@4": 0.35955679374976973, "pass@8": 0.45545195127969673, "pass@16": 0.5399904831688518, "pass@32": 0.6110066282578624, "pass@64": 0.6704129618632997, "pass@128": 0.7225433526011561 }, "18": { "pass@1": 0.15582589285714285, "pass@2": 0.23676708380202446, "pass@4": 0.3290234993840056, "pass@8": 0.42050899990842666, "pass@16": 0.5023571496853358, "pass@32": 0.5734286017598653, "pass@64": 0.6377678650847342, "pass@128": 0.6971428571428572 }, "19": { "pass@1": 0.14161036036036037, "pass@2": 0.22125718237922962, "pass@4": 0.3123289304490093, "pass@8": 0.3986272218604655, "pass@16": 0.47052017433388255, "pass@32": 0.527380938890095, "pass@64": 0.5741233473287567, "pass@128": 0.6156156156156156 }, "20": { "pass@1": 0.14102383633633633, "pass@2": 0.21171503688728094, "pass@4": 0.2923553308651733, "pass@8": 0.37377671407570623, "pass@16": 0.4469443695731371, "pass@32": 0.5086362984246378, "pass@64": 0.5628735775041814, "pass@128": 0.6126126126126126 } } }, "teachers_in_school": { "count": 821120, "correct": 415101, "answer_accuracy": 0.5055302513639907, "avg_response_len": 263.1199654130943, "resp_tokens_sum": 216053066, "pass_at_k": { "pass@1": 0.5055302513639907, "pass@2": 0.6167045893912595, "pass@4": 0.7131499286624569, "pass@8": 0.7848576100860732, "pass@16": 0.8334102197462872, "pass@32": 0.8664360830608916, "pass@64": 0.8913671235680202, "pass@128": 0.911925175370226 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5438075153374233, "pass@2": 0.7155738249359938, "pass@4": 0.8482569538853645, "pass@8": 0.9247060702289117, "pass@16": 0.9606764223030863, "pass@32": 0.9775510243733617, "pass@64": 0.9867887531474895, "pass@128": 0.99079754601227 }, "2": { "pass@1": 0.992686170212766, "pass@2": 0.9948932580235983, "pass@4": 0.996548313679635, "pass@8": 0.9981614876492373, "pass@16": 0.9994732641458764, "pass@32": 0.9999644243028596, "pass@64": 0.9999999628162317, "pass@128": 1.0 }, "3": { "pass@1": 0.9937973484848485, "pass@2": 0.9945269625387737, "pass@4": 0.995079583233914, "pass@8": 0.9960043744311183, "pass@16": 0.997362339583687, "pass@32": 0.998850274274236, "pass@64": 0.9998151400249071, "pass@128": 1.0 }, "4": { "pass@1": 0.933385724852071, "pass@2": 0.9726342280902018, "pass@4": 0.9899115271463842, "pass@8": 0.9961884271819588, "pass@16": 0.9981460451848981, "pass@32": 0.9990776110322851, "pass@64": 0.999823789772943, "pass@128": 1.0 }, "5": { "pass@1": 0.8190896739130434, "pass@2": 0.9131440431359124, "pass@4": 0.9604721708699447, "pass@8": 0.9786022704945484, "pass@16": 0.9854326450754342, "pass@32": 0.9896840841129063, "pass@64": 0.9937767545499087, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.7318230659025788, "pass@2": 0.853087350247051, "pass@4": 0.9329505317028773, "pass@8": 0.9718729806164039, "pass@16": 0.9874429073861202, "pass@32": 0.9933592781253554, "pass@64": 0.996150603505014, "pass@128": 0.997134670487106 }, "7": { "pass@1": 0.6908417630057804, "pass@2": 0.8355122234991585, "pass@4": 0.9259327110845249, "pass@8": 0.9657232728447545, "pass@16": 0.9799154383926086, "pass@32": 0.987047671764001, "pass@64": 0.9922131555149458, "pass@128": 0.9942196531791907 }, "8": { "pass@1": 0.6376320422535211, "pass@2": 0.8039948846623045, "pass@4": 0.9164696530891375, "pass@8": 0.969848565415213, "pass@16": 0.9884038113446275, "pass@32": 0.9941602181010866, "pass@64": 0.9971833475810209, "pass@128": 1.0 }, "9": { "pass@1": 0.5755988023952096, "pass@2": 0.7414146853929929, "pass@4": 0.8681606595582738, "pass@8": 0.941494152221592, "pass@16": 0.9739024093533677, "pass@32": 0.9868868137718635, "pass@64": 0.9927593268163736, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.4614787274096386, "pass@2": 0.6289481251778768, "pass@4": 0.7768584167942858, "pass@8": 0.8823223386877882, "pass@16": 0.9427012275107226, "pass@32": 0.9688437100059571, "pass@64": 0.9783544805563124, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.4521698667711599, "pass@2": 0.6230932046503593, "pass@4": 0.7703201394496527, "pass@8": 0.8716570043484047, "pass@16": 0.9280077925551187, "pass@32": 0.9550018756047801, "pass@64": 0.9660625310485854, "pass@128": 0.9717868338557993 }, "13": { "pass@1": 0.3567270658263305, "pass@2": 0.5099872626215837, "pass@4": 0.6578685560838504, "pass@8": 0.7772300365473884, "pass@16": 0.8555928488258188, "pass@32": 0.9011742109700471, "pass@64": 0.9303704417034507, "pass@128": 0.9523809523809523 }, "14": { "pass@1": 0.3323967470760234, "pass@2": 0.4757476861444953, "pass@4": 0.6150902066518003, "pass@8": 0.7283701824607279, "pass@16": 0.8069417810438068, "pass@32": 0.8571822437247654, "pass@64": 0.8945694038536045, "pass@128": 0.9298245614035088 }, "15": { "pass@1": 0.2434185606060606, "pass@2": 0.36151984908136464, "pass@4": 0.4901963163695448, "pass@8": 0.6128158772102645, "pass@16": 0.7157703886521702, "pass@32": 0.7941160880333268, "pass@64": 0.8492354664830497, "pass@128": 0.8848484848484849 }, "16": { "pass@1": 0.1988292378917379, "pass@2": 0.30013200473338286, "pass@4": 0.4110917996895686, "pass@8": 0.5151463559434297, "pass@16": 0.6051339051747506, "pass@32": 0.6838217166573823, "pass@64": 0.7560222940461221, "pass@128": 0.8233618233618234 }, "17": { "pass@1": 0.1983695652173913, "pass@2": 0.30658913899349527, "pass@4": 0.42618183159946577, "pass@8": 0.5340680663265758, "pass@16": 0.618370732659288, "pass@32": 0.6845467437656324, "pass@64": 0.7440009642404236, "pass@128": 0.8012422360248447 }, "18": { "pass@1": 0.16353400735294119, "pass@2": 0.25522702929597024, "pass@4": 0.3619508745230373, "pass@8": 0.46639634535611824, "pass@16": 0.5540833993978276, "pass@32": 0.6223032123495336, "pass@64": 0.6775643958034884, "pass@128": 0.7264705882352941 }, "19": { "pass@1": 0.14448474702380953, "pass@2": 0.21798508096644167, "pass@4": 0.30426176778125946, "pass@8": 0.39128929215247166, "pass@16": 0.4700711724164494, "pass@32": 0.5399918946084133, "pass@64": 0.6015360772969983, "pass@128": 0.6547619047619048 }, "20": { "pass@1": 0.12995883233532934, "pass@2": 0.20353350876986184, "pass@4": 0.29159784620859525, "pass@8": 0.3811237163449579, "pass@16": 0.45964420951180207, "pass@32": 0.5219112961870748, "pass@64": 0.5730840359592941, "pass@128": 0.6137724550898204 } } }, "movie_festival_awards": { "count": 801152, "correct": 400628, "answer_accuracy": 0.5000649065345902, "avg_response_len": 281.3352559813868, "resp_tokens_sum": 225392303, "pass_at_k": { "pass@1": 0.5000649065345902, "pass@2": 0.6103195650232244, "pass@4": 0.7064286998825201, "pass@8": 0.7783712513258505, "pass@16": 0.8281078442464707, "pass@32": 0.8625593473715938, "pass@64": 0.8876228519333529, "pass@128": 0.907013899984023 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5870335820895523, "pass@2": 0.7577733870019981, "pass@4": 0.880753396963439, "pass@8": 0.9473623446101491, "pass@16": 0.9759983171238966, "pass@32": 0.9859594955268647, "pass@64": 0.9894615713657725, "pass@128": 0.991044776119403 }, "2": { "pass@1": 0.9900669642857143, "pass@2": 0.991916127671541, "pass@4": 0.9936696981627297, "pass@8": 0.9953110922921324, "pass@16": 0.9968387123422647, "pass@32": 0.9984497961412884, "pass@64": 0.9997474915629107, "pass@128": 1.0 }, "3": { "pass@1": 0.9979956268221575, "pass@2": 0.999621579509194, "pass@4": 0.999968567340161, "pass@8": 0.99999975922343, "pass@16": 0.9999999999947993, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9050392909356725, "pass@2": 0.9572253303863328, "pass@4": 0.9842237557476368, "pass@8": 0.9937768813991182, "pass@16": 0.9968527053025525, "pass@32": 0.9982664901804696, "pass@64": 0.9992743503931047, "pass@128": 1.0 }, "5": { "pass@1": 0.7528492647058823, "pass@2": 0.863650706345531, "pass@4": 0.9333525565738104, "pass@8": 0.9660227461686778, "pass@16": 0.9788446430220072, "pass@32": 0.9857440751253994, "pass@64": 0.9908227122501815, "pass@128": 0.9941176470588236 }, "6": { "pass@1": 0.7334631309148265, "pass@2": 0.85599066357833, "pass@4": 0.9375967866989801, "pass@8": 0.9775949027886589, "pass@16": 0.992955917554611, "pass@32": 0.9984939993931751, "pass@64": 0.9998956310797713, "pass@128": 1.0 }, "7": { "pass@1": 0.6655211629746836, "pass@2": 0.8138696987441444, "pass@4": 0.9095275688442417, "pass@8": 0.9588090317998087, "pass@16": 0.9824500620137617, "pass@32": 0.993098936423399, "pass@64": 0.9973843970952397, "pass@128": 1.0 }, "8": { "pass@1": 0.6062077702702703, "pass@2": 0.7712748427561416, "pass@4": 0.8953629433708176, "pass@8": 0.9638294728471273, "pass@16": 0.9908266464333062, "pass@32": 0.9983304781303394, "pass@64": 0.9998135038671645, "pass@128": 1.0 }, "9": { "pass@1": 0.5452170418006431, "pass@2": 0.7200408417094971, "pass@4": 0.8566020424094902, "pass@8": 0.9366537967909059, "pass@16": 0.9726401308835425, "pass@32": 0.9868106206266033, "pass@64": 0.993172338236693, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.5193204365079365, "pass@2": 0.6937250031246089, "pass@4": 0.8327515706370029, "pass@8": 0.9199520734107461, "pass@16": 0.9648824495942682, "pass@32": 0.9852639879739948, "pass@64": 0.9940219878639607, "pass@128": 0.9968253968253968 }, "12": { "pass@1": 0.4114822247706422, "pass@2": 0.5805370945604271, "pass@4": 0.7374564925370567, "pass@8": 0.8546288115348385, "pass@16": 0.9252189474343845, "pass@32": 0.9613500309324735, "pass@64": 0.9792605949864354, "pass@128": 0.9908256880733946 }, "13": { "pass@1": 0.3473186728395062, "pass@2": 0.48654589834256845, "pass@4": 0.6228602443213108, "pass@8": 0.7370374613510134, "pass@16": 0.8212246754468296, "pass@32": 0.8785089408856174, "pass@64": 0.9155469572108085, "pass@128": 0.9382716049382716 }, "14": { "pass@1": 0.302763353115727, "pass@2": 0.4393772342811747, "pass@4": 0.5812337206921829, "pass@8": 0.7044582723264429, "pass@16": 0.7965432964422573, "pass@32": 0.8588847914988257, "pass@64": 0.8974336095198548, "pass@128": 0.9228486646884273 }, "15": { "pass@1": 0.23396508487654322, "pass@2": 0.3543128614999515, "pass@4": 0.4862309537696679, "pass@8": 0.6049426375187965, "pass@16": 0.6974339017879962, "pass@32": 0.7649831585883168, "pass@64": 0.8146309282794365, "pass@128": 0.8549382716049383 }, "16": { "pass@1": 0.21102968289085547, "pass@2": 0.31148272187768566, "pass@4": 0.42092105300996674, "pass@8": 0.5274955208393359, "pass@16": 0.6251729588230321, "pass@32": 0.7151963712477419, "pass@64": 0.7952666121181906, "pass@128": 0.855457227138643 }, "17": { "pass@1": 0.16773343373493976, "pass@2": 0.2616457463713118, "pass@4": 0.3689029873713071, "pass@8": 0.4728816422571547, "pass@16": 0.5617307806554981, "pass@32": 0.6346774139142679, "pass@64": 0.6943223299165555, "pass@128": 0.7409638554216867 }, "18": { "pass@1": 0.17552923387096775, "pass@2": 0.26093908750317496, "pass@4": 0.35391009198446943, "pass@8": 0.4437184648651771, "pass@16": 0.5250520514402012, "pass@32": 0.5967975861389322, "pass@64": 0.6566854833295221, "pass@128": 0.7032258064516129 }, "19": { "pass@1": 0.15903512084592145, "pass@2": 0.2373675274163237, "pass@4": 0.32128599436777355, "pass@8": 0.40200679847511533, "pass@16": 0.47251039414116286, "pass@32": 0.5292500787241331, "pass@64": 0.575524305858063, "pass@128": 0.6163141993957704 }, "20": { "pass@1": 0.12579767267267267, "pass@2": 0.1951881753801045, "pass@4": 0.2782783728610501, "pass@8": 0.36563991218626535, "pass@16": 0.4454211481401893, "pass@32": 0.5097664447120562, "pass@64": 0.5649682015817358, "pass@128": 0.6246246246246246 } } } } } ]