[ { "checkpoint": "checkpoint-43615", "id_acc": 0.9825190972222222, "ood_acc": 0.45401953125, "total_acc": 0.7043614309210526, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.3422282986111, "ood_avg_resp_len": 328.5712453125, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 567145, "answer_accuracy": 0.7004142131678786, "avg_response_len": 252.55946194277584, "resp_tokens_sum": 204504468, "pass_at_k": { "pass@1": 0.7004142131678786, "pass@2": 0.7395955729510206, "pass@4": 0.7698265951867037, "pass@8": 0.7947565208670186, "pass@16": 0.8161362718077232, "pass@32": 0.8348131982541847, "pass@64": 0.8511208520691765, "pass@128": 0.8654758141005374 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9277055678466076, "pass@2": 0.9488087357443151, "pass@4": 0.9601653258497553, "pass@8": 0.9663311672233376, "pass@16": 0.9698972937132337, "pass@32": 0.9728622723733015, "pass@64": 0.9752809834995639, "pass@128": 0.976401179941003 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999609375, "pass@2": 0.9999696265994095, "pass@4": 0.9999997905535245, "pass@8": 0.9999999999971869, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9999503968253968, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9957662799401198, "pass@2": 0.9997635143099631, "pass@4": 0.999998917519292, "pass@8": 0.9999999999868238, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9913785133136095, "pass@2": 0.9970337761496528, "pass@4": 0.997718585750006, "pass@8": 0.9982611889854031, "pass@16": 0.9990160343548016, "pass@32": 0.9997255997961729, "pass@64": 0.9999908406382343, "pass@128": 1.0 }, "8": { "pass@1": 0.9798928285256411, "pass@2": 0.9949072058600847, "pass@4": 0.9975075756756363, "pass@8": 0.9982414406851464, "pass@16": 0.9990761655849738, "pass@32": 0.9997820042825152, "pass@64": 0.999995369433774, "pass@128": 1.0 }, "9": { "pass@1": 0.9655809859154929, "pass@2": 0.9891687229677276, "pass@4": 0.9945222701035602, "pass@8": 0.9967932938229055, "pass@16": 0.9987868476144616, "pass@32": 0.9998391201766423, "pass@64": 0.9999989639740793, "pass@128": 1.0 }, "11": { "pass@1": 0.9152133498583569, "pass@2": 0.9418670953135112, "pass@4": 0.9525116567404991, "pass@8": 0.9593008519186704, "pass@16": 0.965339162220901, "pass@32": 0.9706969576879204, "pass@64": 0.9747305623568889, "pass@128": 0.9773371104815864 }, "12": { "pass@1": 0.8249691031073446, "pass@2": 0.8652749510654386, "pass@4": 0.886044204220235, "pass@8": 0.9030516492001283, "pass@16": 0.9185576391527194, "pass@32": 0.9310006414932023, "pass@64": 0.9402646449798419, "pass@128": 0.9491525423728814 }, "13": { "pass@1": 0.6906837774294671, "pass@2": 0.7665915107496364, "pass@4": 0.8156440448862381, "pass@8": 0.850177610982185, "pass@16": 0.8783236992195879, "pass@32": 0.9048766301553047, "pass@64": 0.9293481960106035, "pass@128": 0.9435736677115988 }, "14": { "pass@1": 0.5863755841121495, "pass@2": 0.6817359169180951, "pass@4": 0.751071299498777, "pass@8": 0.8000867531511684, "pass@16": 0.8356650823330366, "pass@32": 0.8617799218043426, "pass@64": 0.8799863924826393, "pass@128": 0.8940809968847352 }, "15": { "pass@1": 0.4189396676300578, "pass@2": 0.5180344260616264, "pass@4": 0.6040508860705992, "pass@8": 0.6743815532963773, "pass@16": 0.7309694062186467, "pass@32": 0.7763246525532387, "pass@64": 0.8115786910425721, "pass@128": 0.838150289017341 }, "16": { "pass@1": 0.28606350806451614, "pass@2": 0.36844837439674855, "pass@4": 0.4472324213505569, "pass@8": 0.5179145938648841, "pass@16": 0.578643100153927, "pass@32": 0.6322087046156032, "pass@64": 0.683953269991507, "pass@128": 0.7322580645161291 }, "17": { "pass@1": 0.23051390895953758, "pass@2": 0.30222793682581583, "pass@4": 0.3681365696339981, "pass@8": 0.42571137624084293, "pass@16": 0.47504055200905826, "pass@32": 0.5195842752225408, "pass@64": 0.5641670330751434, "pass@128": 0.615606936416185 }, "18": { "pass@1": 0.19892857142857143, "pass@2": 0.2626008858267716, "pass@4": 0.3208692021533023, "pass@8": 0.37249567470288364, "pass@16": 0.4221343361554889, "pass@32": 0.47083877776919847, "pass@64": 0.5156649496135026, "pass@128": 0.5542857142857143 }, "19": { "pass@1": 0.17759947447447447, "pass@2": 0.2406274384620841, "pass@4": 0.30124126629441594, "pass@8": 0.3539290191523181, "pass@16": 0.39861072878847414, "pass@32": 0.4374562488689359, "pass@64": 0.4728657691794338, "pass@128": 0.5075075075075075 }, "20": { "pass@1": 0.1545608108108108, "pass@2": 0.2081844393606203, "pass@4": 0.26157267109629323, "pass@8": 0.31397218376186337, "pass@16": 0.3652350803897118, "pass@32": 0.41169042788730165, "pass@64": 0.4496195738157476, "pass@128": 0.4804804804804805 } } }, "teachers_in_school": { "count": 821120, "correct": 579870, "answer_accuracy": 0.7061939789555729, "avg_response_len": 262.5659087587685, "resp_tokens_sum": 215598119, "pass_at_k": { "pass@1": 0.7061939789555729, "pass@2": 0.7452381345088053, "pass@4": 0.7749703109305348, "pass@8": 0.7992528682477295, "pass@16": 0.819876792222174, "pass@32": 0.8378528939015578, "pass@64": 0.8540891913392115, "pass@128": 0.8693686671862821 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9090059432515337, "pass@2": 0.9270842139268634, "pass@4": 0.9345829163992535, "pass@8": 0.9405544205539004, "pass@16": 0.946824201238343, "pass@32": 0.9524117803349368, "pass@64": 0.9569699348737799, "pass@128": 0.9631901840490797 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.996195652173913, "pass@2": 0.9969256390505534, "pass@4": 0.9970928810529117, "pass@8": 0.9971014332819161, "pass@16": 0.9971014492753529, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9916726361031518, "pass@2": 0.999209989734449, "pass@4": 0.9999890933225033, "pass@8": 0.9999999968452457, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9938132225433526, "pass@2": 0.9995878800464246, "pass@4": 0.9999964821051992, "pass@8": 0.9999999997261103, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9757702464788732, "pass@2": 0.9908201175557283, "pass@4": 0.9951669697977892, "pass@8": 0.9969802958142218, "pass@16": 0.9980060212746866, "pass@32": 0.9988184994691299, "pass@64": 0.9996562048705918, "pass@128": 1.0 }, "9": { "pass@1": 0.9669956961077845, "pass@2": 0.9858933707388369, "pass@4": 0.9899473833547747, "pass@8": 0.9909252150955133, "pass@16": 0.9913709733917561, "pass@32": 0.9917664068906267, "pass@64": 0.9925149700598765, "pass@128": 0.9940119760479041 }, "11": { "pass@1": 0.9235692771084337, "pass@2": 0.9509915893890526, "pass@4": 0.9639130351552438, "pass@8": 0.9727737593702174, "pass@16": 0.9796306262631516, "pass@32": 0.9840085952101105, "pass@64": 0.9867755529100372, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.82641065830721, "pass@2": 0.8706954403524789, "pass@4": 0.8927595551339781, "pass@8": 0.9066140729642067, "pass@16": 0.9191512022482687, "pass@32": 0.9317707221772884, "pass@64": 0.9434131089152301, "pass@128": 0.9529780564263323 }, "13": { "pass@1": 0.657234768907563, "pass@2": 0.7425564221751693, "pass@4": 0.8012479739927459, "pass@8": 0.8442492970489506, "pass@16": 0.8782426089455424, "pass@32": 0.9053033979633102, "pass@64": 0.9249559823901916, "pass@128": 0.938375350140056 }, "14": { "pass@1": 0.5948464912280702, "pass@2": 0.6767214336694752, "pass@4": 0.7351692696855875, "pass@8": 0.7802167089053011, "pass@16": 0.8166149968068924, "pass@32": 0.8456027296875692, "pass@64": 0.8682169255438028, "pass@128": 0.8888888888888888 }, "15": { "pass@1": 0.4621212121212121, "pass@2": 0.5555893581484137, "pass@4": 0.6282997971844423, "pass@8": 0.6857743323657854, "pass@16": 0.7351256550346867, "pass@32": 0.7776288146237075, "pass@64": 0.8132651239824699, "pass@128": 0.8454545454545455 }, "16": { "pass@1": 0.3003917378917379, "pass@2": 0.3826286537900711, "pass@4": 0.45496413802975494, "pass@8": 0.5125153672145101, "pass@16": 0.5576001745102152, "pass@32": 0.5973186639162141, "pass@64": 0.6387977667357648, "pass@128": 0.6866096866096866 }, "17": { "pass@1": 0.26210694875776397, "pass@2": 0.3371254034821733, "pass@4": 0.4045208762724534, "pass@8": 0.4633616799532274, "pass@16": 0.5141234559202151, "pass@32": 0.5609182165634445, "pass@64": 0.6065126277737566, "pass@128": 0.6459627329192547 }, "18": { "pass@1": 0.21994485294117647, "pass@2": 0.2876186892079666, "pass@4": 0.35397602643419546, "pass@8": 0.41577271551011696, "pass@16": 0.4692777647810956, "pass@32": 0.5149172959863509, "pass@64": 0.5566269269161073, "pass@128": 0.5970588235294118 }, "19": { "pass@1": 0.17896670386904762, "pass@2": 0.24228453376921633, "pass@4": 0.3052222350554393, "pass@8": 0.36137853712505047, "pass@16": 0.41033944710580444, "pass@32": 0.4536026470267038, "pass@64": 0.4931087740130446, "pass@128": 0.5327380952380952 }, "20": { "pass@1": 0.14464820359281438, "pass@2": 0.19913384353340563, "pass@4": 0.2525909067691389, "pass@8": 0.3026967108449422, "pass@16": 0.34996399288652164, "pass@32": 0.39467057046628035, "pass@64": 0.4374371107253553, "pass@128": 0.47604790419161674 } } }, "movie_festival_awards": { "count": 801152, "correct": 565992, "answer_accuracy": 0.7064726793417478, "avg_response_len": 282.91117540741334, "resp_tokens_sum": 226654854, "pass_at_k": { "pass@1": 0.7064726793417478, "pass@2": 0.7465973014606943, "pass@4": 0.7777583073624791, "pass@8": 0.8028225638276483, "pass@16": 0.8233834153374825, "pass@32": 0.8408147920202843, "pass@64": 0.8557843704790238, "pass@128": 0.868349576609682 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9315531716417911, "pass@2": 0.9500055088729578, "pass@4": 0.9595039831774755, "pass@8": 0.9673470409588951, "pass@16": 0.9746367138289137, "pass@32": 0.9802078501456624, "pass@64": 0.9835196793541271, "pass@128": 0.9850746268656716 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999314692982456, "pass@2": 0.9999996402587834, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9902113970588236, "pass@2": 0.9909560994673461, "pass@4": 0.9913272637795277, "pass@8": 0.9915337428304501, "pass@16": 0.9918683418196722, "pass@32": 0.992467577582214, "pass@64": 0.9933881426586383, "pass@128": 0.9941176470588236 }, "6": { "pass@1": 0.9931240141955836, "pass@2": 0.999324685163566, "pass@4": 0.9999875186145893, "pass@8": 0.9999999956424814, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9916683148734177, "pass@2": 0.9989651338084321, "pass@4": 0.9999520768962424, "pass@8": 0.9999998305577984, "pass@16": 0.9999999999987065, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9695007507507507, "pass@2": 0.985661104017403, "pass@4": 0.9899353118585399, "pass@8": 0.9923632575228235, "pass@16": 0.9945284865421475, "pass@32": 0.9961461919806662, "pass@64": 0.9969084473566544, "pass@128": 0.996996996996997 }, "9": { "pass@1": 0.958324959807074, "pass@2": 0.9830643783072135, "pass@4": 0.9897614975579816, "pass@8": 0.9921204975281055, "pass@16": 0.9937353286137367, "pass@32": 0.9950919800668229, "pass@64": 0.9967841208283975, "pass@128": 1.0 }, "11": { "pass@1": 0.9302827380952381, "pass@2": 0.9605592269716289, "pass@4": 0.974174619839187, "pass@8": 0.9801028124837545, "pass@16": 0.9834570559922984, "pass@32": 0.9862572223400154, "pass@64": 0.9886073538787452, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.8466408639143731, "pass@2": 0.8960929410773188, "pass@4": 0.9223581401522054, "pass@8": 0.9378106996557768, "pass@16": 0.9472559198752548, "pass@32": 0.9537005538684983, "pass@64": 0.9584471767968687, "pass@128": 0.9602446483180428 }, "13": { "pass@1": 0.679735725308642, "pass@2": 0.7567773646349767, "pass@4": 0.805612143678799, "pass@8": 0.839658816993664, "pass@16": 0.8658793001302546, "pass@32": 0.8862026550356525, "pass@64": 0.9017406716529552, "pass@128": 0.9135802469135802 }, "14": { "pass@1": 0.5714716246290801, "pass@2": 0.6545268288978713, "pass@4": 0.7162127416750945, "pass@8": 0.7610036057794388, "pass@16": 0.7964032510766361, "pass@32": 0.8272994531619295, "pass@64": 0.8525827941386422, "pass@128": 0.8724035608308606 }, "15": { "pass@1": 0.43429301697530864, "pass@2": 0.5223389502527462, "pass@4": 0.5966668286834513, "pass@8": 0.659581656347112, "pass@16": 0.7088315725579833, "pass@32": 0.7452782492230502, "pass@64": 0.7739088213987012, "pass@128": 0.7993827160493827 }, "16": { "pass@1": 0.3088126843657817, "pass@2": 0.3951749878057278, "pass@4": 0.474545925951734, "pass@8": 0.5420576203264432, "pass@16": 0.5976382087845169, "pass@32": 0.6438050130672477, "pass@64": 0.6833998326354856, "pass@128": 0.7227138643067846 }, "17": { "pass@1": 0.23270425451807228, "pass@2": 0.306759024286121, "pass@4": 0.37694031525426774, "pass@8": 0.4393868852582266, "pass@16": 0.4924237323769399, "pass@32": 0.5367887415364578, "pass@64": 0.5768039245457116, "pass@128": 0.6144578313253012 }, "18": { "pass@1": 0.22270665322580646, "pass@2": 0.29104608521717046, "pass@4": 0.35537309620571617, "pass@8": 0.41243105277516273, "pass@16": 0.4632090305336774, "pass@32": 0.5069404871616056, "pass@64": 0.5423413744862255, "pass@128": 0.5709677419354838 }, "19": { "pass@1": 0.17421166918429004, "pass@2": 0.2406351101410661, "pass@4": 0.3060777475511935, "pass@8": 0.3655529463948891, "pass@16": 0.41785418696358484, "pass@32": 0.46570097986105186, "pass@64": 0.5092918882760151, "pass@128": 0.5438066465256798 }, "20": { "pass@1": 0.1710304054054054, "pass@2": 0.24044455380577423, "pass@4": 0.30747180645212135, "pass@8": 0.3621113126399097, "pass@16": 0.40679923108349025, "pass@32": 0.4503535060851414, "pass@64": 0.4931701838218701, "pass@128": 0.5255255255255256 } } } } } ]