[ { "checkpoint": "checkpoint-85276", "id_acc": 0.9949305555555555, "ood_acc": 0.44683984375, "total_acc": 0.7064617598684211, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.0108559027778, "ood_avg_resp_len": 327.42891796875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 576855, "answer_accuracy": 0.7124058943250079, "avg_response_len": 256.26435790784063, "resp_tokens_sum": 207504426, "pass_at_k": { "pass@1": 0.7124058943250079, "pass@2": 0.7482848133624225, "pass@4": 0.7797965772724061, "pass@8": 0.8060704977403156, "pass@16": 0.8268832725059063, "pass@32": 0.8434735001446694, "pass@64": 0.8574080645300883, "pass@128": 0.8695858362314258 }, "per_op_pass_at_k": { "10": { "pass@1": 0.964855272861357, "pass@2": 0.9744083600445962, "pass@4": 0.980863755415529, "pass@8": 0.9845045622809016, "pass@16": 0.9864817601267433, "pass@32": 0.988192742885706, "pass@64": 0.9911504413060571, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9985367063492063, "pass@2": 0.99933172415948, "pass@4": 0.999864562762988, "pass@8": 0.999995076184736, "pass@16": 0.999999996268659, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9997688609467456, "pass@2": 0.9999836200670922, "pass@4": 0.9999999417602385, "pass@8": 0.9999999999999069, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.999824719551282, "pass@2": 0.9999976340096912, "pass@4": 0.9999999996995569, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.997337147887324, "pass@2": 0.9977261700121991, "pass@4": 0.9981745219347582, "pass@8": 0.9988346121771267, "pass@16": 0.9995479293349335, "pass@32": 0.9999462946812021, "pass@64": 0.9999998252073432, "pass@128": 1.0 }, "11": { "pass@1": 0.9568431303116147, "pass@2": 0.966704122147621, "pass@4": 0.9737702782725814, "pass@8": 0.9797293556840062, "pass@16": 0.9839171023985908, "pass@32": 0.9864784015256126, "pass@64": 0.9885864447521222, "pass@128": 0.9915014164305949 }, "12": { "pass@1": 0.8533501059322034, "pass@2": 0.8813041477601309, "pass@4": 0.8969065471688916, "pass@8": 0.9103012428913164, "pass@16": 0.9232330989994582, "pass@32": 0.9356599220900689, "pass@64": 0.9473396438691338, "pass@128": 0.9548022598870056 }, "13": { "pass@1": 0.760384012539185, "pass@2": 0.8294336077308518, "pass@4": 0.874541486977137, "pass@8": 0.9027689973847379, "pass@16": 0.9198466852847369, "pass@32": 0.9308948171804944, "pass@64": 0.9399058726440269, "pass@128": 0.9498432601880877 }, "14": { "pass@1": 0.6034852024922118, "pass@2": 0.6929789259695341, "pass@4": 0.7630072742075453, "pass@8": 0.8164932601872855, "pass@16": 0.8526563967709537, "pass@32": 0.8763087593659933, "pass@64": 0.8936422032813724, "pass@128": 0.9065420560747663 }, "15": { "pass@1": 0.4252619219653179, "pass@2": 0.5253370209594465, "pass@4": 0.6099591160209015, "pass@8": 0.6784789856366449, "pass@16": 0.7314473595873143, "pass@32": 0.7713284427298922, "pass@64": 0.8031345386516523, "pass@128": 0.8294797687861272 }, "16": { "pass@1": 0.26829637096774195, "pass@2": 0.3590876619253239, "pass@4": 0.44438377874136703, "pass@8": 0.5160205150620084, "pass@16": 0.572072708861716, "pass@32": 0.6173416875891375, "pass@64": 0.6591309169128403, "pass@128": 0.6967741935483871 }, "17": { "pass@1": 0.210305274566474, "pass@2": 0.2880021534067633, "pass@4": 0.36638274885075767, "pass@8": 0.4354115772958514, "pass@16": 0.4924820483803051, "pass@32": 0.5414044485040482, "pass@64": 0.5843553236837202, "pass@128": 0.6242774566473989 }, "18": { "pass@1": 0.19066964285714286, "pass@2": 0.2540649606299211, "pass@4": 0.3153792696448659, "pass@8": 0.3744374956178368, "pass@16": 0.4304686242139373, "pass@32": 0.4799023237679476, "pass@64": 0.5191885486106175, "pass@128": 0.5485714285714286 }, "19": { "pass@1": 0.17203922672672672, "pass@2": 0.2452760339079237, "pass@4": 0.31713847114831356, "pass@8": 0.3807029920118585, "pass@16": 0.43294372077217796, "pass@32": 0.47306262676523364, "pass@64": 0.5035309613007347, "pass@128": 0.5315315315315315 }, "20": { "pass@1": 0.16786317567567569, "pass@2": 0.23649646201319421, "pass@4": 0.3076818460755471, "pass@8": 0.3678730399252492, "pass@16": 0.41342134802722214, "pass@32": 0.450621229184501, "pass@64": 0.48393762875113866, "pass@128": 0.5135135135135135 } } }, "teachers_in_school": { "count": 821120, "correct": 578426, "answer_accuracy": 0.704435405300078, "avg_response_len": 263.05899381332813, "resp_tokens_sum": 216003001, "pass_at_k": { "pass@1": 0.704435405300078, "pass@2": 0.7394285546608842, "pass@4": 0.7702634267559836, "pass@8": 0.7961202574484137, "pass@16": 0.8169219087902136, "pass@32": 0.8337421048859038, "pass@64": 0.8480070379164071, "pass@128": 0.8607950116913484 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9569833205521472, "pass@2": 0.9659142372349164, "pass@4": 0.9719421844700393, "pass@8": 0.9748675318830776, "pass@16": 0.9757861748653367, "pass@32": 0.9762266888126843, "pass@64": 0.9769938650304216, "pass@128": 0.9785276073619632 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9998414855072464, "pass@2": 0.9999967904827115, "pass@4": 0.9999999997282949, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9999776146131805, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9948283450704225, "pass@2": 0.9955154153266054, "pass@4": 0.9962789690901313, "pass@8": 0.9971273245981712, "pass@16": 0.997797475275284, "pass@32": 0.9984192527437615, "pass@64": 0.9993013197287653, "pass@128": 1.0 }, "9": { "pass@1": 0.9957662799401198, "pass@2": 0.9974428603423076, "pass@4": 0.9988750589372435, "pass@8": 0.9997585789040461, "pass@16": 0.9999876161726882, "pass@32": 0.9999999824517899, "pass@64": 0.9999999999999998, "pass@128": 1.0 }, "11": { "pass@1": 0.9643731174698795, "pass@2": 0.9750883455080163, "pass@4": 0.9803072865138847, "pass@8": 0.9823661486047269, "pass@16": 0.9832962407555205, "pass@32": 0.9844320989052803, "pass@64": 0.9860781709515227, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.8863636363636364, "pass@2": 0.9133642287660749, "pass@4": 0.9258904760729354, "pass@8": 0.9354300779654707, "pass@16": 0.945221377305145, "pass@32": 0.9540520042583904, "pass@64": 0.9598380142135106, "pass@128": 0.9623824451410659 }, "13": { "pass@1": 0.6867997198879552, "pass@2": 0.7681042397273864, "pass@4": 0.824483724986057, "pass@8": 0.8627893286369971, "pass@16": 0.8897924787212615, "pass@32": 0.9118141519741789, "pass@64": 0.9305813041850224, "pass@128": 0.9439775910364145 }, "14": { "pass@1": 0.5445906432748538, "pass@2": 0.6282847970483952, "pass@4": 0.6919824232497251, "pass@8": 0.7401227434606403, "pass@16": 0.7772650766591268, "pass@32": 0.8067142511689581, "pass@64": 0.8289431092869494, "pass@128": 0.8450292397660819 }, "15": { "pass@1": 0.3677556818181818, "pass@2": 0.4612525351944643, "pass@4": 0.5395015228210107, "pass@8": 0.6038984673310392, "pass@16": 0.6548475782041733, "pass@32": 0.695211253240757, "pass@64": 0.7307349783554408, "pass@128": 0.7636363636363637 }, "16": { "pass@1": 0.23595530626780628, "pass@2": 0.30808173217129914, "pass@4": 0.37879288913672104, "pass@8": 0.44199327843430775, "pass@16": 0.49717687261406224, "pass@32": 0.5475518095537767, "pass@64": 0.5956511172414417, "pass@128": 0.6410256410256411 }, "17": { "pass@1": 0.23090547360248448, "pass@2": 0.31294512948109743, "pass@4": 0.3923752244478753, "pass@8": 0.4620048777355182, "pass@16": 0.5184937076070995, "pass@32": 0.5641996443460395, "pass@64": 0.6058905683329744, "pass@128": 0.6490683229813664 }, "18": { "pass@1": 0.19181985294117648, "pass@2": 0.2639264416396479, "pass@4": 0.3402037715873749, "pass@8": 0.41292313703410055, "pass@16": 0.4755867515992984, "pass@32": 0.5242663505584598, "pass@64": 0.5607374408176437, "pass@128": 0.5911764705882353 }, "19": { "pass@1": 0.17740885416666666, "pass@2": 0.24643025578833894, "pass@4": 0.3146818307867766, "pass@8": 0.37225657621985075, "pass@16": 0.4180985090398595, "pass@32": 0.4553241566350863, "pass@64": 0.48681976016382017, "pass@128": 0.5178571428571429 }, "20": { "pass@1": 0.1394320733532934, "pass@2": 0.2010787725729644, "pass@4": 0.26803790703183067, "pass@8": 0.32950827537338695, "pass@16": 0.37733200724855503, "pass@32": 0.4121459313058152, "pass@64": 0.43980782684006975, "pass@128": 0.46407185628742514 } } }, "movie_festival_awards": { "count": 801152, "correct": 562834, "answer_accuracy": 0.7025308555679821, "avg_response_len": 282.1113771169516, "resp_tokens_sum": 226014094, "pass_at_k": { "pass@1": 0.7025308555679821, "pass@2": 0.736340083665345, "pass@4": 0.7661264757077474, "pass@8": 0.791289508043559, "pass@16": 0.8116318796222768, "pass@32": 0.8281989841030529, "pass@64": 0.8425176638526883, "pass@128": 0.8555679821057677 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9654617537313432, "pass@2": 0.974182483253026, "pass@4": 0.9814832566638125, "pass@8": 0.9865134381095033, "pass@16": 0.990010567366735, "pass@32": 0.9930920761226464, "pass@64": 0.9962352974533647, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9999080882352941, "pass@2": 0.9999992762853173, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9999258306962026, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9954016516516516, "pass@2": 0.9980015694828687, "pass@4": 0.9994780266768457, "pass@8": 0.9999541707109151, "pass@16": 0.9999996450540588, "pass@32": 0.9999999999945218, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.994599075562701, "pass@2": 0.9959007962630074, "pass@4": 0.9973834390074231, "pass@8": 0.9989100635493507, "pass@16": 0.9998018508583297, "pass@32": 0.9999941260810753, "pass@64": 0.9999999994727716, "pass@128": 1.0 }, "11": { "pass@1": 0.9669642857142857, "pass@2": 0.977667635295588, "pass@4": 0.9834914713041819, "pass@8": 0.987697919919454, "pass@16": 0.9903756780268099, "pass@32": 0.9920980050763724, "pass@64": 0.9932587543558059, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.8925602064220184, "pass@2": 0.927709778106865, "pass@4": 0.9466711294115757, "pass@8": 0.9594735205095442, "pass@16": 0.968631391961948, "pass@32": 0.9747149407936225, "pass@64": 0.9783565974863963, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.6791087962962963, "pass@2": 0.7509379252940609, "pass@4": 0.8003636541960031, "pass@8": 0.836590700158435, "pass@16": 0.8634558161248805, "pass@32": 0.8838359924574606, "pass@64": 0.8998857884792347, "pass@128": 0.9135802469135802 }, "14": { "pass@1": 0.5279349035608308, "pass@2": 0.6192675354564355, "pass@4": 0.6896984853896232, "pass@8": 0.7423055698396044, "pass@16": 0.7843204109962838, "pass@32": 0.8204913013870107, "pass@64": 0.8477207957073809, "pass@128": 0.8664688427299704 }, "15": { "pass@1": 0.36140046296296297, "pass@2": 0.4426244137017595, "pass@4": 0.5132164547139938, "pass@8": 0.5721977881442194, "pass@16": 0.6197088824105395, "pass@32": 0.658967059013711, "pass@64": 0.6955853807708048, "pass@128": 0.7345679012345679 }, "16": { "pass@1": 0.21766685103244837, "pass@2": 0.2878512385896452, "pass@4": 0.35677422180634516, "pass@8": 0.4192935319550409, "pass@16": 0.4744116086421201, "pass@32": 0.5235822279834264, "pass@64": 0.5713460103429768, "pass@128": 0.6194690265486725 }, "17": { "pass@1": 0.19837161144578314, "pass@2": 0.26883901254624787, "pass@4": 0.33928608895650064, "pass@8": 0.40176271614166426, "pass@16": 0.4530815948293738, "pass@32": 0.49619312239293656, "pass@64": 0.536260897714176, "pass@128": 0.5692771084337349 }, "18": { "pass@1": 0.19788306451612903, "pass@2": 0.2630076041402083, "pass@4": 0.3281824866649735, "pass@8": 0.38767325712032175, "pass@16": 0.4352260441352987, "pass@32": 0.47043974442309966, "pass@64": 0.4992493451488665, "pass@128": 0.5258064516129032 }, "19": { "pass@1": 0.17987632175226587, "pass@2": 0.247976856697671, "pass@4": 0.3147488179347673, "pass@8": 0.37247260884174876, "pass@16": 0.41889090519709676, "pass@32": 0.45572123675521775, "pass@64": 0.4860676102567291, "pass@128": 0.513595166163142 }, "20": { "pass@1": 0.15899493243243243, "pass@2": 0.2257040505071999, "pass@4": 0.29573212638960666, "pass@8": 0.3606103751183562, "pass@16": 0.41446968279744983, "pass@32": 0.457917316065741, "pass@64": 0.49484972985591286, "pass@128": 0.5285285285285285 } } } } } ]