[ { "checkpoint": "checkpoint-44699", "id_acc": 0.911046875, "ood_acc": 0.29633828125, "total_acc": 0.5875160361842106, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.46338975694445, "ood_avg_resp_len": 318.24450546875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 469664, "answer_accuracy": 0.580026873221625, "avg_response_len": 248.33461236365793, "resp_tokens_sum": 201083489, "pass_at_k": { "pass@1": 0.580026873221625, "pass@2": 0.6442908562587665, "pass@4": 0.6958034834989211, "pass@8": 0.7365417777176548, "pass@16": 0.7686865636898084, "pass@32": 0.7945016220873287, "pass@64": 0.8162913610635678, "pass@128": 0.8362314258615239 }, "per_op_pass_at_k": { "10": { "pass@1": 0.7771248156342183, "pass@2": 0.8634121751097489, "pass@4": 0.9133753164703967, "pass@8": 0.9424125973654682, "pass@16": 0.9595853388576279, "pass@32": 0.9704237748229196, "pass@64": 0.9789232030426774, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 0.9363317757009346, "pass@2": 0.944525060097628, "pass@4": 0.9532450491001708, "pass@8": 0.9627298066354898, "pass@16": 0.9719688257345186, "pass@32": 0.980109835516427, "pass@64": 0.987954208653673, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.9741016819571865, "pass@2": 0.9827925214910066, "pass@4": 0.9867644511752545, "pass@8": 0.9889207398015514, "pass@16": 0.9904735766349954, "pass@32": 0.9920811096141816, "pass@64": 0.9934682845780995, "pass@128": 0.9938837920489296 }, "4": { "pass@1": 0.9593505859375, "pass@2": 0.9731499138779528, "pass@4": 0.9823881259959691, "pass@8": 0.9881178094134416, "pass@16": 0.9913778625190709, "pass@32": 0.9934385296963498, "pass@64": 0.9951254225202119, "pass@128": 0.996875 }, "5": { "pass@1": 0.9149305555555556, "pass@2": 0.9386279840019998, "pass@4": 0.9542117681718351, "pass@8": 0.9660540883231847, "pass@16": 0.9757455198041897, "pass@32": 0.9831868336772416, "pass@64": 0.9889152632850954, "pass@128": 0.9936507936507937 }, "6": { "pass@1": 0.9309739895209581, "pass@2": 0.9547627186571742, "pass@4": 0.9694351802581562, "pass@8": 0.9786972797798579, "pass@16": 0.9846500715770806, "pass@32": 0.9890982222982732, "pass@64": 0.9920427308990417, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.9030833949704142, "pass@2": 0.9393964322322136, "pass@4": 0.9593146475218703, "pass@8": 0.9720034544718734, "pass@16": 0.980721876863249, "pass@32": 0.9867746427167278, "pass@64": 0.9916786492683689, "pass@128": 0.9970414201183432 }, "8": { "pass@1": 0.8989383012820513, "pass@2": 0.9453968869876843, "pass@4": 0.9636373202748688, "pass@8": 0.9731567959470013, "pass@16": 0.9791146723545264, "pass@32": 0.983025910711685, "pass@64": 0.9854191370827627, "pass@128": 0.9871794871794872 }, "9": { "pass@1": 0.8684198943661972, "pass@2": 0.9273798796717314, "pass@4": 0.9541264833093043, "pass@8": 0.9697246361156203, "pass@16": 0.9795548818702162, "pass@32": 0.9850109862577768, "pass@64": 0.9880098458356275, "pass@128": 0.9887323943661972 }, "11": { "pass@1": 0.7133719900849859, "pass@2": 0.8189317938480065, "pass@4": 0.883105831317827, "pass@8": 0.9197420263846189, "pass@16": 0.9425990411933546, "pass@32": 0.9584202607176746, "pass@64": 0.9703830576083085, "pass@128": 0.9773371104815864 }, "12": { "pass@1": 0.4903778248587571, "pass@2": 0.6256558204324039, "pass@4": 0.7312299754903507, "pass@8": 0.8046130198855828, "pass@16": 0.8548554975791646, "pass@32": 0.8929254084348788, "pass@64": 0.9249538202932568, "pass@128": 0.9519774011299436 }, "13": { "pass@1": 0.35450137147335425, "pass@2": 0.46927568002863274, "pass@4": 0.5712045930583124, "pass@8": 0.6563511124051389, "pass@16": 0.7233043809932352, "pass@32": 0.7757701552474985, "pass@64": 0.8189983379952339, "pass@128": 0.8557993730407524 }, "14": { "pass@1": 0.2768204828660436, "pass@2": 0.3809104943949764, "pass@4": 0.4742459108499289, "pass@8": 0.5537739274477774, "pass@16": 0.6249179084010938, "pass@32": 0.6868947826587918, "pass@64": 0.7398453986973906, "pass@128": 0.7881619937694704 }, "15": { "pass@1": 0.2188854768786127, "pass@2": 0.3133277957307358, "pass@4": 0.4072432692661974, "pass@8": 0.4865732250040521, "pass@16": 0.551303533167993, "pass@32": 0.6076449232032669, "pass@64": 0.6589186848327896, "pass@128": 0.7109826589595376 }, "16": { "pass@1": 0.17726814516129033, "pass@2": 0.24868594424688859, "pass@4": 0.3202650595086902, "pass@8": 0.3873123584530507, "pass@16": 0.4491814670825519, "pass@32": 0.5042842654352826, "pass@64": 0.5534790693723503, "pass@128": 0.603225806451613 }, "17": { "pass@1": 0.17332008670520233, "pass@2": 0.24559291224796342, "pass@4": 0.31781556992875903, "pass@8": 0.3825153496828191, "pass@16": 0.4364200708299305, "pass@32": 0.48270561555806835, "pass@64": 0.5258694560653755, "pass@128": 0.5722543352601156 }, "18": { "pass@1": 0.1690625, "pass@2": 0.23962985095613049, "pass@4": 0.30801571669612715, "pass@8": 0.3677881716863713, "pass@16": 0.415209138215289, "pass@32": 0.45032772234979906, "pass@64": 0.47887185765730594, "pass@128": 0.5057142857142857 }, "19": { "pass@1": 0.16786317567567569, "pass@2": 0.23250772918587886, "pass@4": 0.29525881421353867, "pass@8": 0.3543126353276815, "pass@16": 0.4073455414188086, "pass@32": 0.45058219908918623, "pass@64": 0.4831636850314373, "pass@128": 0.5105105105105106 }, "20": { "pass@1": 0.16061373873873874, "pass@2": 0.22986273675250057, "pass@4": 0.2993586457661261, "pass@8": 0.35899482853932685, "pass@16": 0.4044574963197297, "pass@32": 0.4399765302164153, "pass@64": 0.470242670848185, "pass@128": 0.4954954954954955 } } }, "teachers_in_school": { "count": 821120, "correct": 483167, "answer_accuracy": 0.5884243472330476, "avg_response_len": 257.7551100935308, "resp_tokens_sum": 211647876, "pass_at_k": { "pass@1": 0.5884243472330476, "pass@2": 0.6496861594073996, "pass@4": 0.6991931554687777, "pass@8": 0.7400875770824119, "pass@16": 0.7735274891578671, "pass@32": 0.8004030340738901, "pass@64": 0.8224152654722662, "pass@128": 0.8420888542478566 }, "per_op_pass_at_k": { "10": { "pass@1": 0.790260736196319, "pass@2": 0.8626221257427176, "pass@4": 0.9026658627027448, "pass@8": 0.9305973202449752, "pass@16": 0.9520073304178969, "pass@32": 0.9671571441192927, "pass@64": 0.9765058182369718, "pass@128": 0.9815950920245399 }, "2": { "pass@1": 0.9332256838905775, "pass@2": 0.9407051761481944, "pass@4": 0.9479080504374642, "pass@8": 0.9536852149913407, "pass@16": 0.9571820803552304, "pass@32": 0.9588901496981889, "pass@64": 0.9604862016565554, "pass@128": 0.9635258358662614 }, "3": { "pass@1": 0.9784090909090909, "pass@2": 0.9849987324027679, "pass@4": 0.9886542293008828, "pass@8": 0.9912084467155227, "pass@16": 0.9930326420416548, "pass@32": 0.9947539086640631, "pass@64": 0.9964193746484429, "pass@128": 0.996969696969697 }, "4": { "pass@1": 0.9647512943786982, "pass@2": 0.9728151353492055, "pass@4": 0.9783021487143988, "pass@8": 0.9830129515701709, "pass@16": 0.9867103391258935, "pass@32": 0.9897313735773864, "pass@64": 0.9922419638528047, "pass@128": 0.9940828402366864 }, "5": { "pass@1": 0.9416666666666667, "pass@2": 0.9582031695766293, "pass@4": 0.9686339471153061, "pass@8": 0.9767519230405057, "pass@16": 0.9834265774804692, "pass@32": 0.988204034915414, "pass@64": 0.9920156285108184, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9282772206303725, "pass@2": 0.9563044299799198, "pass@4": 0.9710311721421637, "pass@8": 0.9806274820666016, "pass@16": 0.9861484876564786, "pass@32": 0.9888576883172377, "pass@64": 0.9899679684332302, "pass@128": 0.9914040114613181 }, "7": { "pass@1": 0.9178332731213873, "pass@2": 0.9538034511401392, "pass@4": 0.9726097203889971, "pass@8": 0.9839673817925544, "pass@16": 0.9899827277601754, "pass@32": 0.9929938813526252, "pass@64": 0.9941355497813412, "pass@128": 0.9942196531791907 }, "8": { "pass@1": 0.9106954225352113, "pass@2": 0.9566988050349345, "pass@4": 0.9760180632350526, "pass@8": 0.9860794596604422, "pass@16": 0.9915585599451117, "pass@32": 0.993776910042879, "pass@64": 0.9943270956076777, "pass@128": 0.9943661971830986 }, "9": { "pass@1": 0.8456212574850299, "pass@2": 0.91207705160545, "pass@4": 0.9416968105408975, "pass@8": 0.957179736673861, "pass@16": 0.9682130101641933, "pass@32": 0.9766842369771581, "pass@64": 0.982990924027013, "pass@128": 0.9880239520958084 }, "11": { "pass@1": 0.7269390060240963, "pass@2": 0.8169354336637885, "pass@4": 0.8682271524229653, "pass@8": 0.8982871087556397, "pass@16": 0.9182913182805088, "pass@32": 0.9348106777196764, "pass@64": 0.9505494615239103, "pass@128": 0.9668674698795181 }, "12": { "pass@1": 0.5305152821316614, "pass@2": 0.6572589354034508, "pass@4": 0.7495488091418038, "pass@8": 0.8106145929971147, "pass@16": 0.8533804682435675, "pass@32": 0.8861680726994876, "pass@64": 0.9133851877237164, "pass@128": 0.9373040752351097 }, "13": { "pass@1": 0.33661589635854344, "pass@2": 0.45304883764529447, "pass@4": 0.5610518070745358, "pass@8": 0.6485457230363358, "pass@16": 0.713696536353939, "pass@32": 0.7628602973367136, "pass@64": 0.8040627756332077, "pass@128": 0.84593837535014 }, "14": { "pass@1": 0.3016949926900585, "pass@2": 0.40240868328958873, "pass@4": 0.49299692554878016, "pass@8": 0.5730546778713073, "pass@16": 0.6428674497544992, "pass@32": 0.6986742397132528, "pass@64": 0.74331622198312, "pass@128": 0.7894736842105263 }, "15": { "pass@1": 0.2177556818181818, "pass@2": 0.30779870555953226, "pass@4": 0.3992948816057082, "pass@8": 0.48379835112074754, "pass@16": 0.5545522992571181, "pass@32": 0.6100068408376694, "pass@64": 0.657364349482816, "pass@128": 0.706060606060606 }, "16": { "pass@1": 0.1894141737891738, "pass@2": 0.26630815779437816, "pass@4": 0.34345242501738554, "pass@8": 0.4152680205722794, "pass@16": 0.47843937487739835, "pass@32": 0.5340027097177347, "pass@64": 0.582983948272668, "pass@128": 0.6267806267806267 }, "17": { "pass@1": 0.19863645186335405, "pass@2": 0.279192852252164, "pass@4": 0.3574614560990435, "pass@8": 0.4286738857288767, "pass@16": 0.4910720735207935, "pass@32": 0.5474434388067502, "pass@64": 0.5990456601352024, "pass@128": 0.6428571428571429 }, "18": { "pass@1": 0.16017922794117648, "pass@2": 0.23308497857804533, "pass@4": 0.3089515182844792, "pass@8": 0.38097280125521266, "pass@16": 0.44501758863725593, "pass@32": 0.4989253440660845, "pass@64": 0.541355544376601, "pass@128": 0.5735294117647058 }, "19": { "pass@1": 0.15239025297619047, "pass@2": 0.21226228557367818, "pass@4": 0.27195060968941387, "pass@8": 0.33047229886625246, "pass@16": 0.38482815880394616, "pass@32": 0.43015119890887243, "pass@64": 0.4641203141576692, "pass@128": 0.4880952380952381 }, "20": { "pass@1": 0.14095247005988024, "pass@2": 0.20596835364703656, "pass@4": 0.27415568525491196, "pass@8": 0.3394977707056415, "pass@16": 0.39794876064215573, "pass@32": 0.44576809211309787, "pass@64": 0.4837252202930043, "pass@128": 0.5149700598802395 } } }, "movie_festival_awards": { "count": 801152, "correct": 476008, "answer_accuracy": 0.5941544176386004, "avg_response_len": 280.10093839870586, "resp_tokens_sum": 224403427, "pass_at_k": { "pass@1": 0.5941544176386004, "pass@2": 0.6584226194594812, "pass@4": 0.7107949916349928, "pass@8": 0.7528320912137726, "pass@16": 0.7859981786824436, "pass@32": 0.8124843985122424, "pass@64": 0.8346996523138441, "pass@128": 0.8539702827927784 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8151119402985074, "pass@2": 0.894133050299682, "pass@4": 0.9361917815310395, "pass@8": 0.9607177294918307, "pass@16": 0.974920028102238, "pass@32": 0.9823346279403201, "pass@64": 0.9868942200091164, "pass@128": 0.991044776119403 }, "2": { "pass@1": 0.915625, "pass@2": 0.9211642294713162, "pass@4": 0.9274200423161388, "pass@8": 0.9339374424325937, "pass@16": 0.9398469239256345, "pass@32": 0.9449957416667745, "pass@64": 0.9503416731237622, "pass@128": 0.9571428571428572 }, "3": { "pass@1": 0.9802523688046647, "pass@2": 0.9851332757512453, "pass@4": 0.988258999695009, "pass@8": 0.9907579902047114, "pass@16": 0.9924813282950841, "pass@32": 0.9939612434255555, "pass@64": 0.9954531454092972, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.9657574926900585, "pass@2": 0.9755307621909105, "pass@4": 0.9809962987082755, "pass@8": 0.9845367405017438, "pass@16": 0.9873080289127392, "pass@32": 0.9893515838691211, "pass@64": 0.9912087730850718, "pass@128": 0.9941520467836257 }, "5": { "pass@1": 0.9165670955882353, "pass@2": 0.9386376794812418, "pass@4": 0.9540271523963915, "pass@8": 0.9649185404033847, "pass@16": 0.9742903210799776, "pass@32": 0.9831582523177278, "pass@64": 0.9901467246024266, "pass@128": 0.9941176470588236 }, "6": { "pass@1": 0.9536179022082019, "pass@2": 0.9753758476365533, "pass@4": 0.9836247020778551, "pass@8": 0.9886263415095881, "pass@16": 0.992890469889173, "pass@32": 0.99654159266549, "pass@64": 0.9990623094181035, "pass@128": 1.0 }, "7": { "pass@1": 0.9324564873417721, "pass@2": 0.9607023385328421, "pass@4": 0.9730168865363347, "pass@8": 0.9809211880879508, "pass@16": 0.9860712788116768, "pass@32": 0.9893128993348704, "pass@64": 0.9920828692662229, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.8937922297297297, "pass@2": 0.9417517911612395, "pass@4": 0.9637633128066195, "pass@8": 0.9750247388470051, "pass@16": 0.9810504117686079, "pass@32": 0.9852425954657849, "pass@64": 0.9892275719681659, "pass@128": 0.993993993993994 }, "9": { "pass@1": 0.8511605707395499, "pass@2": 0.9153317815277108, "pass@4": 0.9480443674444222, "pass@8": 0.9669171484266056, "pass@16": 0.9779593801637884, "pass@32": 0.9841731235226076, "pass@64": 0.9879175577920956, "pass@128": 0.9903536977491961 }, "11": { "pass@1": 0.732093253968254, "pass@2": 0.8374496156730411, "pass@4": 0.8996038209509525, "pass@8": 0.9368779024733572, "pass@16": 0.9593199778817806, "pass@32": 0.9713870939894751, "pass@64": 0.9776932080779764, "pass@128": 0.9809523809523809 }, "12": { "pass@1": 0.5414038608562691, "pass@2": 0.6731839497700401, "pass@4": 0.7681141700750705, "pass@8": 0.8300111166340215, "pass@16": 0.8687917152417938, "pass@32": 0.8943989454268281, "pass@64": 0.9125531792181364, "pass@128": 0.926605504587156 }, "13": { "pass@1": 0.37449363425925924, "pass@2": 0.49851071133469455, "pass@4": 0.6065300379119271, "pass@8": 0.6926563227419713, "pass@16": 0.7617440863768723, "pass@32": 0.8180242034665416, "pass@64": 0.8632191058162716, "pass@128": 0.8981481481481481 }, "14": { "pass@1": 0.3313473664688427, "pass@2": 0.43997705845930957, "pass@4": 0.5421407949006373, "pass@8": 0.629466788219989, "pass@16": 0.6999395215454379, "pass@32": 0.7571167495219389, "pass@64": 0.8039831562649183, "pass@128": 0.8367952522255193 }, "15": { "pass@1": 0.20691068672839505, "pass@2": 0.3100724063137941, "pass@4": 0.41896026306896816, "pass@8": 0.5175979655164591, "pass@16": 0.598636322043697, "pass@32": 0.662957709462422, "pass@64": 0.7151067850864165, "pass@128": 0.7561728395061729 }, "16": { "pass@1": 0.20342459439528024, "pass@2": 0.28151348338094906, "pass@4": 0.35446680531084046, "pass@8": 0.4200273095429436, "pass@16": 0.4821220436808709, "pass@32": 0.5409573975320442, "pass@64": 0.5938332249407281, "pass@128": 0.640117994100295 }, "17": { "pass@1": 0.1840879141566265, "pass@2": 0.2620826564130538, "pass@4": 0.3423274769494175, "pass@8": 0.4160604187774409, "pass@16": 0.47882764147063955, "pass@32": 0.5320816045911806, "pass@64": 0.5799399739308471, "pass@128": 0.6295180722891566 }, "18": { "pass@1": 0.1681451612903226, "pass@2": 0.24457550165100325, "pass@4": 0.3209164985425209, "pass@8": 0.38626919031550705, "pass@16": 0.4366118977394541, "pass@32": 0.47500204091998266, "pass@64": 0.5101726723947231, "pass@128": 0.5451612903225806 }, "19": { "pass@1": 0.16460536253776434, "pass@2": 0.23573317850940825, "pass@4": 0.30449151739340746, "pass@8": 0.3636900639671202, "pass@16": 0.41398196429348005, "pass@32": 0.4596670842841002, "pass@64": 0.5020669682882645, "pass@128": 0.540785498489426 }, "20": { "pass@1": 0.13051332582582584, "pass@2": 0.20101166619375274, "pass@4": 0.27972318691019477, "pass@8": 0.35571918790213874, "pass@16": 0.41978209084984897, "pass@32": 0.46961062281713356, "pass@64": 0.5115038624020116, "pass@128": 0.5495495495495496 } } } } } ]