[ { "checkpoint": "checkpoint-39008", "id_acc": 0.9902135416666666, "ood_acc": 0.87520703125, "total_acc": 0.9296837993421052, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.16405208333333, "ood_avg_resp_len": 381.9587625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 750630, "answer_accuracy": 0.9270149976288334, "avg_response_len": 282.90025539440404, "resp_tokens_sum": 229072258, "pass_at_k": { "pass@1": 0.9270149976288334, "pass@2": 0.9493477619236584, "pass@4": 0.96401552496224, "pass@8": 0.9738049633109391, "pass@16": 0.980471830554453, "pass@32": 0.9851893630710853, "pass@64": 0.9886710958308421, "pass@128": 0.9914638001896934 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9435149336283186, "pass@2": 0.954635492880868, "pass@4": 0.9636101311451112, "pass@8": 0.9714900775767249, "pass@16": 0.9780677121400383, "pass@32": 0.9842094464543852, "pass@64": 0.9901773717362713, "pass@128": 0.9941002949852508 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9997371941896025, "pass@2": 0.9999939801102844, "pass@4": 0.9999999982800315, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9909474206349206, "pass@2": 0.994433508311461, "pass@4": 0.9964954672332624, "pass@8": 0.9978625121315736, "pass@16": 0.9989254195211111, "pass@32": 0.9997055216891618, "pass@64": 0.9999901718594374, "pass@128": 1.0 }, "6": { "pass@1": 0.9992281062874252, "pass@2": 0.9999447463341034, "pass@4": 0.9999994695648073, "pass@8": 0.9999999999730462, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.997133875739645, "pass@2": 0.9989840801612077, "pass@4": 0.9997141855973625, "pass@8": 0.9999752317035955, "pass@16": 0.9999998695305009, "pass@32": 0.999999999999429, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9951923076923077, "pass@2": 0.9986281199525542, "pass@4": 0.9998137005710825, "pass@8": 0.9999957663923499, "pass@16": 0.9999999980003716, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.991637323943662, "pass@2": 0.9929615587224134, "pass@4": 0.9940380704172542, "pass@8": 0.9954732852979431, "pass@16": 0.9970755009921725, "pass@32": 0.9986975516024429, "pass@64": 0.9998376400088083, "pass@128": 1.0 }, "11": { "pass@1": 0.9542315864022662, "pass@2": 0.969464140327006, "pass@4": 0.9810940441934841, "pass@8": 0.9893951399204369, "pass@16": 0.9945360284144443, "pass@32": 0.9977323646041456, "pass@64": 0.9996420458747602, "pass@128": 1.0 }, "12": { "pass@1": 0.9427304025423728, "pass@2": 0.9646079396325459, "pass@4": 0.979820588898845, "pass@8": 0.989370272818801, "pass@16": 0.9949116559332181, "pass@32": 0.9979107888230833, "pass@64": 0.9992908315443338, "pass@128": 1.0 }, "13": { "pass@1": 0.914576802507837, "pass@2": 0.9446635956359689, "pass@4": 0.9665589881202152, "pass@8": 0.979981660826714, "pass@16": 0.9874232215331469, "pass@32": 0.9921527270385655, "pass@64": 0.996205173271306, "pass@128": 1.0 }, "14": { "pass@1": 0.9100467289719626, "pass@2": 0.9379875266759877, "pass@4": 0.958589766816531, "pass@8": 0.9731059061694559, "pass@16": 0.9830554969430491, "pass@32": 0.9893467916867982, "pass@64": 0.992726298871782, "pass@128": 0.9937694704049844 }, "15": { "pass@1": 0.9557216401734104, "pass@2": 0.9786739480451507, "pass@4": 0.988442469889963, "pass@8": 0.9914905405416587, "pass@16": 0.992362600034752, "pass@32": 0.9933207386637914, "pass@64": 0.9949478858495289, "pass@128": 0.9971098265895953 }, "16": { "pass@1": 0.8498739919354839, "pass@2": 0.8913004826009657, "pass@4": 0.9222687491684507, "pass@8": 0.9436176461985871, "pass@16": 0.9585810148724633, "pass@32": 0.9695494764061836, "pass@64": 0.9774985888021867, "pass@128": 0.9838709677419355 }, "17": { "pass@1": 0.8969698338150289, "pass@2": 0.9344089225570076, "pass@4": 0.956744095300948, "pass@8": 0.97040257318339, "pass@16": 0.9791461578401852, "pass@32": 0.9845951312505338, "pass@64": 0.9879973643102052, "pass@128": 0.9913294797687862 }, "18": { "pass@1": 0.8265625, "pass@2": 0.8718394966254224, "pass@4": 0.9008116189940538, "pass@8": 0.9218754716646158, "pass@16": 0.9370514134261215, "pass@32": 0.9485340476411356, "pass@64": 0.9583897263711767, "pass@128": 0.9685714285714285 }, "19": { "pass@1": 0.7230902777777778, "pass@2": 0.7983750236456933, "pass@4": 0.8509800430351607, "pass@8": 0.8887121617215229, "pass@16": 0.9156199204084648, "pass@32": 0.9346616186743513, "pass@64": 0.9488236564767019, "pass@128": 0.960960960960961 }, "20": { "pass@1": 0.7268909534534534, "pass@2": 0.8097002021706746, "pass@4": 0.8596189023669333, "pass@8": 0.8914002622308476, "pass@16": 0.9137724953284576, "pass@32": 0.9295212910338141, "pass@64": 0.9403257272551829, "pass@128": 0.948948948948949 } } }, "teachers_in_school": { "count": 821120, "correct": 766541, "answer_accuracy": 0.9335310307872174, "avg_response_len": 292.15439521628997, "resp_tokens_sum": 239893817, "pass_at_k": { "pass@1": 0.9335310307872174, "pass@2": 0.953042742465052, "pass@4": 0.9665239741758695, "pass@8": 0.9762235468442136, "pass@16": 0.9828267648985132, "pass@32": 0.9872906061783097, "pass@64": 0.9907222191039753, "pass@128": 0.9937646141855028 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9211320935582822, "pass@2": 0.9365198994009952, "pass@4": 0.9526557933134124, "pass@8": 0.9657088477738169, "pass@16": 0.9746686601446526, "pass@32": 0.9816210687882091, "pass@64": 0.9878415021526776, "pass@128": 0.9938650306748467 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9968513257575757, "pass@2": 0.9974964954664758, "pass@4": 0.9979660738430424, "pass@8": 0.9986534604223928, "pass@16": 0.9994358706064412, "pass@32": 0.9999202169903283, "pass@64": 0.9999995805383678, "pass@128": 1.0 }, "4": { "pass@1": 0.9999537721893491, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9962635869565217, "pass@2": 0.9972030839895015, "pass@4": 0.9974499779918814, "pass@8": 0.9977680587306922, "pass@16": 0.9983124881128989, "pass@32": 0.999097406302473, "pass@64": 0.9998273650576287, "pass@128": 1.0 }, "6": { "pass@1": 0.9998209169054442, "pass@2": 0.9999901292782528, "pass@4": 0.9999999811986252, "pass@8": 0.999999999999998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9987129696531792, "pass@2": 0.9996746421419143, "pass@4": 0.9999665655051787, "pass@8": 0.9999997068622991, "pass@16": 0.9999999999917892, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9898767605633803, "pass@2": 0.9936876178329822, "pass@4": 0.9957979374937288, "pass@8": 0.9972295980502696, "pass@16": 0.9982648879987673, "pass@32": 0.9991210789903305, "pass@64": 0.9998322279837888, "pass@128": 1.0 }, "9": { "pass@1": 0.9966785179640718, "pass@2": 0.9970903419538876, "pass@4": 0.9971908938029453, "pass@8": 0.997369925503324, "pass@16": 0.9977102880852468, "pass@32": 0.9983202885567448, "pass@64": 0.9992573907303504, "pass@128": 1.0 }, "11": { "pass@1": 0.973621046686747, "pass@2": 0.9814229852480792, "pass@4": 0.9863388201286587, "pass@8": 0.9898787999691451, "pass@16": 0.993064840318199, "pass@32": 0.9960242380201543, "pass@64": 0.9987056869995018, "pass@128": 1.0 }, "12": { "pass@1": 0.9421532131661442, "pass@2": 0.9639698985510823, "pass@4": 0.9794393557388399, "pass@8": 0.9891352473786128, "pass@16": 0.99374911404456, "pass@32": 0.9954805067246403, "pass@64": 0.9964824145461987, "pass@128": 0.9968652037617555 }, "13": { "pass@1": 0.9447435224089635, "pass@2": 0.9655367068087075, "pass@4": 0.9791388313855723, "pass@8": 0.9881338330670343, "pass@16": 0.9938460585401995, "pass@32": 0.9969037090102838, "pass@64": 0.9985099383695012, "pass@128": 1.0 }, "14": { "pass@1": 0.9357638888888888, "pass@2": 0.9592812514389654, "pass@4": 0.976130010064531, "pass@8": 0.9864184088906494, "pass@16": 0.9913667647399901, "pass@32": 0.9940353616501455, "pass@64": 0.9955747091064813, "pass@128": 0.9970760233918129 }, "15": { "pass@1": 0.9528645833333333, "pass@2": 0.9747278394178001, "pass@4": 0.9865421262683073, "pass@8": 0.993495451097799, "pass@16": 0.9974149003571077, "pass@32": 0.99942427013389, "pass@64": 0.999978739438185, "pass@128": 1.0 }, "16": { "pass@1": 0.8898459757834758, "pass@2": 0.918867913946654, "pass@4": 0.940958512130428, "pass@8": 0.9576799933576514, "pass@16": 0.9685457081346035, "pass@32": 0.9757692082156921, "pass@64": 0.9819115581804977, "pass@128": 0.9886039886039886 }, "17": { "pass@1": 0.8886354813664596, "pass@2": 0.9250734368122461, "pass@4": 0.9469407821499012, "pass@8": 0.9609358541243732, "pass@16": 0.9696919174554715, "pass@32": 0.9768628206172025, "pass@64": 0.9848878654881961, "pass@128": 0.9937888198757764 }, "18": { "pass@1": 0.865234375, "pass@2": 0.9126758626679023, "pass@4": 0.9434961421182645, "pass@8": 0.9639815917003957, "pass@16": 0.9758078148025395, "pass@32": 0.9821697623332426, "pass@64": 0.9862873674165625, "pass@128": 0.9882352941176471 }, "19": { "pass@1": 0.7456752232142857, "pass@2": 0.8095109937429702, "pass@4": 0.8518101247388715, "pass@8": 0.8841118867969516, "pass@16": 0.9092287569498064, "pass@32": 0.9274672406486854, "pass@64": 0.9428158675268603, "pass@128": 0.9583333333333334 }, "20": { "pass@1": 0.6919910179640718, "pass@2": 0.7697962834174169, "pass@4": 0.8282520268798735, "pass@8": 0.8748525418555514, "pass@16": 0.9103714017754003, "pass@32": 0.9346016347947679, "pass@64": 0.9506273862852723, "pass@128": 0.9640718562874252 } } }, "movie_festival_awards": { "count": 801152, "correct": 743820, "answer_accuracy": 0.9284380492091389, "avg_response_len": 312.7123554581403, "resp_tokens_sum": 250530129, "pass_at_k": { "pass@1": 0.9284380492091389, "pass@2": 0.9505996797682092, "pass@4": 0.9653567733704562, "pass@8": 0.9753979459646634, "pass@16": 0.98229150543528, "pass@32": 0.9872608599596581, "pass@64": 0.9909840482108517, "pass@128": 0.9936092027480428 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9478311567164179, "pass@2": 0.9607738864731464, "pass@4": 0.9701930362249498, "pass@8": 0.9777165144485844, "pass@16": 0.9850490284905359, "pass@32": 0.9921181247105286, "pass@64": 0.9976074742078478, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999772230320699, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9994745979532164, "pass@2": 0.9999316491688539, "pass@4": 0.9999986720409949, "pass@8": 0.9999999997423706, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9941865808823529, "pass@2": 0.9964205071792498, "pass@4": 0.9969975253093362, "pass@8": 0.9970582244883547, "pass@16": 0.9970588234979403, "pass@32": 0.9970588235294118, "pass@64": 0.9970588235294118, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9979544558359621, "pass@2": 0.9992621991355971, "pass@4": 0.9998455858435678, "pass@8": 0.9999935023705825, "pass@16": 0.9999999931461893, "pass@32": 0.9999999999999997, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9960195806962026, "pass@2": 0.9982615961078442, "pass@4": 0.9994898540174567, "pass@8": 0.9999461399811752, "pass@16": 0.9999994164199608, "pass@32": 0.9999999999832518, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9913898273273273, "pass@2": 0.9939999054172283, "pass@4": 0.9949980785059523, "pass@8": 0.9957539554064397, "pass@16": 0.9965150597531441, "pass@32": 0.9969397435790592, "pass@64": 0.9969968106564769, "pass@128": 0.996996996996997 }, "9": { "pass@1": 0.9915846061093248, "pass@2": 0.9950103014152974, "pass@4": 0.9964494795867558, "pass@8": 0.9972464958966785, "pass@16": 0.9978505020760328, "pass@32": 0.9986542070682007, "pass@64": 0.999607565131529, "pass@128": 1.0 }, "11": { "pass@1": 0.9640625, "pass@2": 0.9797177696537934, "pass@4": 0.9911180554811603, "pass@8": 0.9973277243916092, "pass@16": 0.9995487596571645, "pass@32": 0.9999809674873567, "pass@64": 0.99999999279412, "pass@128": 1.0 }, "12": { "pass@1": 0.9453363914373089, "pass@2": 0.9667115148450481, "pass@4": 0.9831256795308844, "pass@8": 0.9930450861150011, "pass@16": 0.9978774666448946, "pass@32": 0.9996537102787295, "pass@64": 0.999990478361063, "pass@128": 1.0 }, "13": { "pass@1": 0.9026572145061729, "pass@2": 0.9364265122241666, "pass@4": 0.9617531060932193, "pass@8": 0.9778678586981546, "pass@16": 0.9868936266865624, "pass@32": 0.9917170187367842, "pass@64": 0.9945546397415536, "pass@128": 0.9969135802469136 }, "14": { "pass@1": 0.9428319732937686, "pass@2": 0.9654668479403722, "pass@4": 0.9797224478987603, "pass@8": 0.9890457880872057, "pass@16": 0.994350926646944, "pass@32": 0.9976499071092195, "pass@64": 0.9996521834093685, "pass@128": 1.0 }, "15": { "pass@1": 0.9220196759259259, "pass@2": 0.9517492496597652, "pass@4": 0.9698473547288068, "pass@8": 0.9799816784659696, "pass@16": 0.9860164971157882, "pass@32": 0.9894908053800399, "pass@64": 0.9919161725258471, "pass@128": 0.9938271604938271 }, "16": { "pass@1": 0.8983222713864307, "pass@2": 0.9338663972313197, "pass@4": 0.9579245663982268, "pass@8": 0.9727914706482771, "pass@16": 0.9807054356298547, "pass@32": 0.9851133092513668, "pass@64": 0.9889024393806544, "pass@128": 0.9941002949852508 }, "17": { "pass@1": 0.8501035391566265, "pass@2": 0.8853027760648893, "pass@4": 0.9105946367020382, "pass@8": 0.9327234486879625, "pass@16": 0.9505718186178311, "pass@32": 0.9637574221371371, "pass@64": 0.9733407462659791, "pass@128": 0.9819277108433735 }, "18": { "pass@1": 0.8662550403225806, "pass@2": 0.9099980949961902, "pass@4": 0.9349534958331817, "pass@8": 0.9524300861471281, "pass@16": 0.9665201701946451, "pass@32": 0.9776554085806844, "pass@64": 0.9853613753348571, "pass@128": 0.9903225806451613 }, "19": { "pass@1": 0.7413378021148036, "pass@2": 0.8133043509289437, "pass@4": 0.8603601790146319, "pass@8": 0.8925875144159812, "pass@16": 0.9169766051361591, "pass@32": 0.9370711730923081, "pass@64": 0.9531285297469417, "pass@128": 0.9637462235649547 }, "20": { "pass@1": 0.6867961711711712, "pass@2": 0.7743428714147219, "pass@4": 0.8342244167677225, "pass@8": 0.8771601288105502, "pass@16": 0.9078914256137317, "pass@32": 0.9314685436519544, "pass@64": 0.9509367136317464, "pass@128": 0.963963963963964 } } } } } ]