| [ |
| { |
| "checkpoint": "checkpoint-85276", |
| "id_acc": 0.9949305555555555, |
| "ood_acc": 0.44683984375, |
| "total_acc": 0.7064617598684211, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.0108559027778, |
| "ood_avg_resp_len": 327.42891796875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 576855, |
| "answer_accuracy": 0.7124058943250079, |
| "avg_response_len": 256.26435790784063, |
| "resp_tokens_sum": 207504426, |
| "pass_at_k": { |
| "pass@1": 0.7124058943250079, |
| "pass@2": 0.7482848133624225, |
| "pass@4": 0.7797965772724061, |
| "pass@8": 0.8060704977403156, |
| "pass@16": 0.8268832725059063, |
| "pass@32": 0.8434735001446694, |
| "pass@64": 0.8574080645300883, |
| "pass@128": 0.8695858362314258 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.964855272861357, |
| "pass@2": 0.9744083600445962, |
| "pass@4": 0.980863755415529, |
| "pass@8": 0.9845045622809016, |
| "pass@16": 0.9864817601267433, |
| "pass@32": 0.988192742885706, |
| "pass@64": 0.9911504413060571, |
| "pass@128": 0.9970501474926253 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9985367063492063, |
| "pass@2": 0.99933172415948, |
| "pass@4": 0.999864562762988, |
| "pass@8": 0.999995076184736, |
| "pass@16": 0.999999996268659, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9997688609467456, |
| "pass@2": 0.9999836200670922, |
| "pass@4": 0.9999999417602385, |
| "pass@8": 0.9999999999999069, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.999824719551282, |
| "pass@2": 0.9999976340096912, |
| "pass@4": 0.9999999996995569, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.997337147887324, |
| "pass@2": 0.9977261700121991, |
| "pass@4": 0.9981745219347582, |
| "pass@8": 0.9988346121771267, |
| "pass@16": 0.9995479293349335, |
| "pass@32": 0.9999462946812021, |
| "pass@64": 0.9999998252073432, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9568431303116147, |
| "pass@2": 0.966704122147621, |
| "pass@4": 0.9737702782725814, |
| "pass@8": 0.9797293556840062, |
| "pass@16": 0.9839171023985908, |
| "pass@32": 0.9864784015256126, |
| "pass@64": 0.9885864447521222, |
| "pass@128": 0.9915014164305949 |
| }, |
| "12": { |
| "pass@1": 0.8533501059322034, |
| "pass@2": 0.8813041477601309, |
| "pass@4": 0.8969065471688916, |
| "pass@8": 0.9103012428913164, |
| "pass@16": 0.9232330989994582, |
| "pass@32": 0.9356599220900689, |
| "pass@64": 0.9473396438691338, |
| "pass@128": 0.9548022598870056 |
| }, |
| "13": { |
| "pass@1": 0.760384012539185, |
| "pass@2": 0.8294336077308518, |
| "pass@4": 0.874541486977137, |
| "pass@8": 0.9027689973847379, |
| "pass@16": 0.9198466852847369, |
| "pass@32": 0.9308948171804944, |
| "pass@64": 0.9399058726440269, |
| "pass@128": 0.9498432601880877 |
| }, |
| "14": { |
| "pass@1": 0.6034852024922118, |
| "pass@2": 0.6929789259695341, |
| "pass@4": 0.7630072742075453, |
| "pass@8": 0.8164932601872855, |
| "pass@16": 0.8526563967709537, |
| "pass@32": 0.8763087593659933, |
| "pass@64": 0.8936422032813724, |
| "pass@128": 0.9065420560747663 |
| }, |
| "15": { |
| "pass@1": 0.4252619219653179, |
| "pass@2": 0.5253370209594465, |
| "pass@4": 0.6099591160209015, |
| "pass@8": 0.6784789856366449, |
| "pass@16": 0.7314473595873143, |
| "pass@32": 0.7713284427298922, |
| "pass@64": 0.8031345386516523, |
| "pass@128": 0.8294797687861272 |
| }, |
| "16": { |
| "pass@1": 0.26829637096774195, |
| "pass@2": 0.3590876619253239, |
| "pass@4": 0.44438377874136703, |
| "pass@8": 0.5160205150620084, |
| "pass@16": 0.572072708861716, |
| "pass@32": 0.6173416875891375, |
| "pass@64": 0.6591309169128403, |
| "pass@128": 0.6967741935483871 |
| }, |
| "17": { |
| "pass@1": 0.210305274566474, |
| "pass@2": 0.2880021534067633, |
| "pass@4": 0.36638274885075767, |
| "pass@8": 0.4354115772958514, |
| "pass@16": 0.4924820483803051, |
| "pass@32": 0.5414044485040482, |
| "pass@64": 0.5843553236837202, |
| "pass@128": 0.6242774566473989 |
| }, |
| "18": { |
| "pass@1": 0.19066964285714286, |
| "pass@2": 0.2540649606299211, |
| "pass@4": 0.3153792696448659, |
| "pass@8": 0.3744374956178368, |
| "pass@16": 0.4304686242139373, |
| "pass@32": 0.4799023237679476, |
| "pass@64": 0.5191885486106175, |
| "pass@128": 0.5485714285714286 |
| }, |
| "19": { |
| "pass@1": 0.17203922672672672, |
| "pass@2": 0.2452760339079237, |
| "pass@4": 0.31713847114831356, |
| "pass@8": 0.3807029920118585, |
| "pass@16": 0.43294372077217796, |
| "pass@32": 0.47306262676523364, |
| "pass@64": 0.5035309613007347, |
| "pass@128": 0.5315315315315315 |
| }, |
| "20": { |
| "pass@1": 0.16786317567567569, |
| "pass@2": 0.23649646201319421, |
| "pass@4": 0.3076818460755471, |
| "pass@8": 0.3678730399252492, |
| "pass@16": 0.41342134802722214, |
| "pass@32": 0.450621229184501, |
| "pass@64": 0.48393762875113866, |
| "pass@128": 0.5135135135135135 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 578426, |
| "answer_accuracy": 0.704435405300078, |
| "avg_response_len": 263.05899381332813, |
| "resp_tokens_sum": 216003001, |
| "pass_at_k": { |
| "pass@1": 0.704435405300078, |
| "pass@2": 0.7394285546608842, |
| "pass@4": 0.7702634267559836, |
| "pass@8": 0.7961202574484137, |
| "pass@16": 0.8169219087902136, |
| "pass@32": 0.8337421048859038, |
| "pass@64": 0.8480070379164071, |
| "pass@128": 0.8607950116913484 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9569833205521472, |
| "pass@2": 0.9659142372349164, |
| "pass@4": 0.9719421844700393, |
| "pass@8": 0.9748675318830776, |
| "pass@16": 0.9757861748653367, |
| "pass@32": 0.9762266888126843, |
| "pass@64": 0.9769938650304216, |
| "pass@128": 0.9785276073619632 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9998414855072464, |
| "pass@2": 0.9999967904827115, |
| "pass@4": 0.9999999997282949, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999776146131805, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9948283450704225, |
| "pass@2": 0.9955154153266054, |
| "pass@4": 0.9962789690901313, |
| "pass@8": 0.9971273245981712, |
| "pass@16": 0.997797475275284, |
| "pass@32": 0.9984192527437615, |
| "pass@64": 0.9993013197287653, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9957662799401198, |
| "pass@2": 0.9974428603423076, |
| "pass@4": 0.9988750589372435, |
| "pass@8": 0.9997585789040461, |
| "pass@16": 0.9999876161726882, |
| "pass@32": 0.9999999824517899, |
| "pass@64": 0.9999999999999998, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9643731174698795, |
| "pass@2": 0.9750883455080163, |
| "pass@4": 0.9803072865138847, |
| "pass@8": 0.9823661486047269, |
| "pass@16": 0.9832962407555205, |
| "pass@32": 0.9844320989052803, |
| "pass@64": 0.9860781709515227, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.8863636363636364, |
| "pass@2": 0.9133642287660749, |
| "pass@4": 0.9258904760729354, |
| "pass@8": 0.9354300779654707, |
| "pass@16": 0.945221377305145, |
| "pass@32": 0.9540520042583904, |
| "pass@64": 0.9598380142135106, |
| "pass@128": 0.9623824451410659 |
| }, |
| "13": { |
| "pass@1": 0.6867997198879552, |
| "pass@2": 0.7681042397273864, |
| "pass@4": 0.824483724986057, |
| "pass@8": 0.8627893286369971, |
| "pass@16": 0.8897924787212615, |
| "pass@32": 0.9118141519741789, |
| "pass@64": 0.9305813041850224, |
| "pass@128": 0.9439775910364145 |
| }, |
| "14": { |
| "pass@1": 0.5445906432748538, |
| "pass@2": 0.6282847970483952, |
| "pass@4": 0.6919824232497251, |
| "pass@8": 0.7401227434606403, |
| "pass@16": 0.7772650766591268, |
| "pass@32": 0.8067142511689581, |
| "pass@64": 0.8289431092869494, |
| "pass@128": 0.8450292397660819 |
| }, |
| "15": { |
| "pass@1": 0.3677556818181818, |
| "pass@2": 0.4612525351944643, |
| "pass@4": 0.5395015228210107, |
| "pass@8": 0.6038984673310392, |
| "pass@16": 0.6548475782041733, |
| "pass@32": 0.695211253240757, |
| "pass@64": 0.7307349783554408, |
| "pass@128": 0.7636363636363637 |
| }, |
| "16": { |
| "pass@1": 0.23595530626780628, |
| "pass@2": 0.30808173217129914, |
| "pass@4": 0.37879288913672104, |
| "pass@8": 0.44199327843430775, |
| "pass@16": 0.49717687261406224, |
| "pass@32": 0.5475518095537767, |
| "pass@64": 0.5956511172414417, |
| "pass@128": 0.6410256410256411 |
| }, |
| "17": { |
| "pass@1": 0.23090547360248448, |
| "pass@2": 0.31294512948109743, |
| "pass@4": 0.3923752244478753, |
| "pass@8": 0.4620048777355182, |
| "pass@16": 0.5184937076070995, |
| "pass@32": 0.5641996443460395, |
| "pass@64": 0.6058905683329744, |
| "pass@128": 0.6490683229813664 |
| }, |
| "18": { |
| "pass@1": 0.19181985294117648, |
| "pass@2": 0.2639264416396479, |
| "pass@4": 0.3402037715873749, |
| "pass@8": 0.41292313703410055, |
| "pass@16": 0.4755867515992984, |
| "pass@32": 0.5242663505584598, |
| "pass@64": 0.5607374408176437, |
| "pass@128": 0.5911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.17740885416666666, |
| "pass@2": 0.24643025578833894, |
| "pass@4": 0.3146818307867766, |
| "pass@8": 0.37225657621985075, |
| "pass@16": 0.4180985090398595, |
| "pass@32": 0.4553241566350863, |
| "pass@64": 0.48681976016382017, |
| "pass@128": 0.5178571428571429 |
| }, |
| "20": { |
| "pass@1": 0.1394320733532934, |
| "pass@2": 0.2010787725729644, |
| "pass@4": 0.26803790703183067, |
| "pass@8": 0.32950827537338695, |
| "pass@16": 0.37733200724855503, |
| "pass@32": 0.4121459313058152, |
| "pass@64": 0.43980782684006975, |
| "pass@128": 0.46407185628742514 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 562834, |
| "answer_accuracy": 0.7025308555679821, |
| "avg_response_len": 282.1113771169516, |
| "resp_tokens_sum": 226014094, |
| "pass_at_k": { |
| "pass@1": 0.7025308555679821, |
| "pass@2": 0.736340083665345, |
| "pass@4": 0.7661264757077474, |
| "pass@8": 0.791289508043559, |
| "pass@16": 0.8116318796222768, |
| "pass@32": 0.8281989841030529, |
| "pass@64": 0.8425176638526883, |
| "pass@128": 0.8555679821057677 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9654617537313432, |
| "pass@2": 0.974182483253026, |
| "pass@4": 0.9814832566638125, |
| "pass@8": 0.9865134381095033, |
| "pass@16": 0.990010567366735, |
| "pass@32": 0.9930920761226464, |
| "pass@64": 0.9962352974533647, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9999080882352941, |
| "pass@2": 0.9999992762853173, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9999258306962026, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9954016516516516, |
| "pass@2": 0.9980015694828687, |
| "pass@4": 0.9994780266768457, |
| "pass@8": 0.9999541707109151, |
| "pass@16": 0.9999996450540588, |
| "pass@32": 0.9999999999945218, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.994599075562701, |
| "pass@2": 0.9959007962630074, |
| "pass@4": 0.9973834390074231, |
| "pass@8": 0.9989100635493507, |
| "pass@16": 0.9998018508583297, |
| "pass@32": 0.9999941260810753, |
| "pass@64": 0.9999999994727716, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9669642857142857, |
| "pass@2": 0.977667635295588, |
| "pass@4": 0.9834914713041819, |
| "pass@8": 0.987697919919454, |
| "pass@16": 0.9903756780268099, |
| "pass@32": 0.9920980050763724, |
| "pass@64": 0.9932587543558059, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.8925602064220184, |
| "pass@2": 0.927709778106865, |
| "pass@4": 0.9466711294115757, |
| "pass@8": 0.9594735205095442, |
| "pass@16": 0.968631391961948, |
| "pass@32": 0.9747149407936225, |
| "pass@64": 0.9783565974863963, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.6791087962962963, |
| "pass@2": 0.7509379252940609, |
| "pass@4": 0.8003636541960031, |
| "pass@8": 0.836590700158435, |
| "pass@16": 0.8634558161248805, |
| "pass@32": 0.8838359924574606, |
| "pass@64": 0.8998857884792347, |
| "pass@128": 0.9135802469135802 |
| }, |
| "14": { |
| "pass@1": 0.5279349035608308, |
| "pass@2": 0.6192675354564355, |
| "pass@4": 0.6896984853896232, |
| "pass@8": 0.7423055698396044, |
| "pass@16": 0.7843204109962838, |
| "pass@32": 0.8204913013870107, |
| "pass@64": 0.8477207957073809, |
| "pass@128": 0.8664688427299704 |
| }, |
| "15": { |
| "pass@1": 0.36140046296296297, |
| "pass@2": 0.4426244137017595, |
| "pass@4": 0.5132164547139938, |
| "pass@8": 0.5721977881442194, |
| "pass@16": 0.6197088824105395, |
| "pass@32": 0.658967059013711, |
| "pass@64": 0.6955853807708048, |
| "pass@128": 0.7345679012345679 |
| }, |
| "16": { |
| "pass@1": 0.21766685103244837, |
| "pass@2": 0.2878512385896452, |
| "pass@4": 0.35677422180634516, |
| "pass@8": 0.4192935319550409, |
| "pass@16": 0.4744116086421201, |
| "pass@32": 0.5235822279834264, |
| "pass@64": 0.5713460103429768, |
| "pass@128": 0.6194690265486725 |
| }, |
| "17": { |
| "pass@1": 0.19837161144578314, |
| "pass@2": 0.26883901254624787, |
| "pass@4": 0.33928608895650064, |
| "pass@8": 0.40176271614166426, |
| "pass@16": 0.4530815948293738, |
| "pass@32": 0.49619312239293656, |
| "pass@64": 0.536260897714176, |
| "pass@128": 0.5692771084337349 |
| }, |
| "18": { |
| "pass@1": 0.19788306451612903, |
| "pass@2": 0.2630076041402083, |
| "pass@4": 0.3281824866649735, |
| "pass@8": 0.38767325712032175, |
| "pass@16": 0.4352260441352987, |
| "pass@32": 0.47043974442309966, |
| "pass@64": 0.4992493451488665, |
| "pass@128": 0.5258064516129032 |
| }, |
| "19": { |
| "pass@1": 0.17987632175226587, |
| "pass@2": 0.247976856697671, |
| "pass@4": 0.3147488179347673, |
| "pass@8": 0.37247260884174876, |
| "pass@16": 0.41889090519709676, |
| "pass@32": 0.45572123675521775, |
| "pass@64": 0.4860676102567291, |
| "pass@128": 0.513595166163142 |
| }, |
| "20": { |
| "pass@1": 0.15899493243243243, |
| "pass@2": 0.2257040505071999, |
| "pass@4": 0.29573212638960666, |
| "pass@8": 0.3606103751183562, |
| "pass@16": 0.41446968279744983, |
| "pass@32": 0.457917316065741, |
| "pass@64": 0.49484972985591286, |
| "pass@128": 0.5285285285285285 |
| } |
| } |
| } |
| } |
| } |
| ] |
