| [ |
| { |
| "checkpoint": "checkpoint-43615", |
| "id_acc": 0.9825190972222222, |
| "ood_acc": 0.45401953125, |
| "total_acc": 0.7043614309210526, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.3422282986111, |
| "ood_avg_resp_len": 328.5712453125, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 567145, |
| "answer_accuracy": 0.7004142131678786, |
| "avg_response_len": 252.55946194277584, |
| "resp_tokens_sum": 204504468, |
| "pass_at_k": { |
| "pass@1": 0.7004142131678786, |
| "pass@2": 0.7395955729510206, |
| "pass@4": 0.7698265951867037, |
| "pass@8": 0.7947565208670186, |
| "pass@16": 0.8161362718077232, |
| "pass@32": 0.8348131982541847, |
| "pass@64": 0.8511208520691765, |
| "pass@128": 0.8654758141005374 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9277055678466076, |
| "pass@2": 0.9488087357443151, |
| "pass@4": 0.9601653258497553, |
| "pass@8": 0.9663311672233376, |
| "pass@16": 0.9698972937132337, |
| "pass@32": 0.9728622723733015, |
| "pass@64": 0.9752809834995639, |
| "pass@128": 0.976401179941003 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999609375, |
| "pass@2": 0.9999696265994095, |
| "pass@4": 0.9999997905535245, |
| "pass@8": 0.9999999999971869, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9999503968253968, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9957662799401198, |
| "pass@2": 0.9997635143099631, |
| "pass@4": 0.999998917519292, |
| "pass@8": 0.9999999999868238, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9913785133136095, |
| "pass@2": 0.9970337761496528, |
| "pass@4": 0.997718585750006, |
| "pass@8": 0.9982611889854031, |
| "pass@16": 0.9990160343548016, |
| "pass@32": 0.9997255997961729, |
| "pass@64": 0.9999908406382343, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9798928285256411, |
| "pass@2": 0.9949072058600847, |
| "pass@4": 0.9975075756756363, |
| "pass@8": 0.9982414406851464, |
| "pass@16": 0.9990761655849738, |
| "pass@32": 0.9997820042825152, |
| "pass@64": 0.999995369433774, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9655809859154929, |
| "pass@2": 0.9891687229677276, |
| "pass@4": 0.9945222701035602, |
| "pass@8": 0.9967932938229055, |
| "pass@16": 0.9987868476144616, |
| "pass@32": 0.9998391201766423, |
| "pass@64": 0.9999989639740793, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9152133498583569, |
| "pass@2": 0.9418670953135112, |
| "pass@4": 0.9525116567404991, |
| "pass@8": 0.9593008519186704, |
| "pass@16": 0.965339162220901, |
| "pass@32": 0.9706969576879204, |
| "pass@64": 0.9747305623568889, |
| "pass@128": 0.9773371104815864 |
| }, |
| "12": { |
| "pass@1": 0.8249691031073446, |
| "pass@2": 0.8652749510654386, |
| "pass@4": 0.886044204220235, |
| "pass@8": 0.9030516492001283, |
| "pass@16": 0.9185576391527194, |
| "pass@32": 0.9310006414932023, |
| "pass@64": 0.9402646449798419, |
| "pass@128": 0.9491525423728814 |
| }, |
| "13": { |
| "pass@1": 0.6906837774294671, |
| "pass@2": 0.7665915107496364, |
| "pass@4": 0.8156440448862381, |
| "pass@8": 0.850177610982185, |
| "pass@16": 0.8783236992195879, |
| "pass@32": 0.9048766301553047, |
| "pass@64": 0.9293481960106035, |
| "pass@128": 0.9435736677115988 |
| }, |
| "14": { |
| "pass@1": 0.5863755841121495, |
| "pass@2": 0.6817359169180951, |
| "pass@4": 0.751071299498777, |
| "pass@8": 0.8000867531511684, |
| "pass@16": 0.8356650823330366, |
| "pass@32": 0.8617799218043426, |
| "pass@64": 0.8799863924826393, |
| "pass@128": 0.8940809968847352 |
| }, |
| "15": { |
| "pass@1": 0.4189396676300578, |
| "pass@2": 0.5180344260616264, |
| "pass@4": 0.6040508860705992, |
| "pass@8": 0.6743815532963773, |
| "pass@16": 0.7309694062186467, |
| "pass@32": 0.7763246525532387, |
| "pass@64": 0.8115786910425721, |
| "pass@128": 0.838150289017341 |
| }, |
| "16": { |
| "pass@1": 0.28606350806451614, |
| "pass@2": 0.36844837439674855, |
| "pass@4": 0.4472324213505569, |
| "pass@8": 0.5179145938648841, |
| "pass@16": 0.578643100153927, |
| "pass@32": 0.6322087046156032, |
| "pass@64": 0.683953269991507, |
| "pass@128": 0.7322580645161291 |
| }, |
| "17": { |
| "pass@1": 0.23051390895953758, |
| "pass@2": 0.30222793682581583, |
| "pass@4": 0.3681365696339981, |
| "pass@8": 0.42571137624084293, |
| "pass@16": 0.47504055200905826, |
| "pass@32": 0.5195842752225408, |
| "pass@64": 0.5641670330751434, |
| "pass@128": 0.615606936416185 |
| }, |
| "18": { |
| "pass@1": 0.19892857142857143, |
| "pass@2": 0.2626008858267716, |
| "pass@4": 0.3208692021533023, |
| "pass@8": 0.37249567470288364, |
| "pass@16": 0.4221343361554889, |
| "pass@32": 0.47083877776919847, |
| "pass@64": 0.5156649496135026, |
| "pass@128": 0.5542857142857143 |
| }, |
| "19": { |
| "pass@1": 0.17759947447447447, |
| "pass@2": 0.2406274384620841, |
| "pass@4": 0.30124126629441594, |
| "pass@8": 0.3539290191523181, |
| "pass@16": 0.39861072878847414, |
| "pass@32": 0.4374562488689359, |
| "pass@64": 0.4728657691794338, |
| "pass@128": 0.5075075075075075 |
| }, |
| "20": { |
| "pass@1": 0.1545608108108108, |
| "pass@2": 0.2081844393606203, |
| "pass@4": 0.26157267109629323, |
| "pass@8": 0.31397218376186337, |
| "pass@16": 0.3652350803897118, |
| "pass@32": 0.41169042788730165, |
| "pass@64": 0.4496195738157476, |
| "pass@128": 0.4804804804804805 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 579870, |
| "answer_accuracy": 0.7061939789555729, |
| "avg_response_len": 262.5659087587685, |
| "resp_tokens_sum": 215598119, |
| "pass_at_k": { |
| "pass@1": 0.7061939789555729, |
| "pass@2": 0.7452381345088053, |
| "pass@4": 0.7749703109305348, |
| "pass@8": 0.7992528682477295, |
| "pass@16": 0.819876792222174, |
| "pass@32": 0.8378528939015578, |
| "pass@64": 0.8540891913392115, |
| "pass@128": 0.8693686671862821 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9090059432515337, |
| "pass@2": 0.9270842139268634, |
| "pass@4": 0.9345829163992535, |
| "pass@8": 0.9405544205539004, |
| "pass@16": 0.946824201238343, |
| "pass@32": 0.9524117803349368, |
| "pass@64": 0.9569699348737799, |
| "pass@128": 0.9631901840490797 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.996195652173913, |
| "pass@2": 0.9969256390505534, |
| "pass@4": 0.9970928810529117, |
| "pass@8": 0.9971014332819161, |
| "pass@16": 0.9971014492753529, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9916726361031518, |
| "pass@2": 0.999209989734449, |
| "pass@4": 0.9999890933225033, |
| "pass@8": 0.9999999968452457, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9938132225433526, |
| "pass@2": 0.9995878800464246, |
| "pass@4": 0.9999964821051992, |
| "pass@8": 0.9999999997261103, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9757702464788732, |
| "pass@2": 0.9908201175557283, |
| "pass@4": 0.9951669697977892, |
| "pass@8": 0.9969802958142218, |
| "pass@16": 0.9980060212746866, |
| "pass@32": 0.9988184994691299, |
| "pass@64": 0.9996562048705918, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9669956961077845, |
| "pass@2": 0.9858933707388369, |
| "pass@4": 0.9899473833547747, |
| "pass@8": 0.9909252150955133, |
| "pass@16": 0.9913709733917561, |
| "pass@32": 0.9917664068906267, |
| "pass@64": 0.9925149700598765, |
| "pass@128": 0.9940119760479041 |
| }, |
| "11": { |
| "pass@1": 0.9235692771084337, |
| "pass@2": 0.9509915893890526, |
| "pass@4": 0.9639130351552438, |
| "pass@8": 0.9727737593702174, |
| "pass@16": 0.9796306262631516, |
| "pass@32": 0.9840085952101105, |
| "pass@64": 0.9867755529100372, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.82641065830721, |
| "pass@2": 0.8706954403524789, |
| "pass@4": 0.8927595551339781, |
| "pass@8": 0.9066140729642067, |
| "pass@16": 0.9191512022482687, |
| "pass@32": 0.9317707221772884, |
| "pass@64": 0.9434131089152301, |
| "pass@128": 0.9529780564263323 |
| }, |
| "13": { |
| "pass@1": 0.657234768907563, |
| "pass@2": 0.7425564221751693, |
| "pass@4": 0.8012479739927459, |
| "pass@8": 0.8442492970489506, |
| "pass@16": 0.8782426089455424, |
| "pass@32": 0.9053033979633102, |
| "pass@64": 0.9249559823901916, |
| "pass@128": 0.938375350140056 |
| }, |
| "14": { |
| "pass@1": 0.5948464912280702, |
| "pass@2": 0.6767214336694752, |
| "pass@4": 0.7351692696855875, |
| "pass@8": 0.7802167089053011, |
| "pass@16": 0.8166149968068924, |
| "pass@32": 0.8456027296875692, |
| "pass@64": 0.8682169255438028, |
| "pass@128": 0.8888888888888888 |
| }, |
| "15": { |
| "pass@1": 0.4621212121212121, |
| "pass@2": 0.5555893581484137, |
| "pass@4": 0.6282997971844423, |
| "pass@8": 0.6857743323657854, |
| "pass@16": 0.7351256550346867, |
| "pass@32": 0.7776288146237075, |
| "pass@64": 0.8132651239824699, |
| "pass@128": 0.8454545454545455 |
| }, |
| "16": { |
| "pass@1": 0.3003917378917379, |
| "pass@2": 0.3826286537900711, |
| "pass@4": 0.45496413802975494, |
| "pass@8": 0.5125153672145101, |
| "pass@16": 0.5576001745102152, |
| "pass@32": 0.5973186639162141, |
| "pass@64": 0.6387977667357648, |
| "pass@128": 0.6866096866096866 |
| }, |
| "17": { |
| "pass@1": 0.26210694875776397, |
| "pass@2": 0.3371254034821733, |
| "pass@4": 0.4045208762724534, |
| "pass@8": 0.4633616799532274, |
| "pass@16": 0.5141234559202151, |
| "pass@32": 0.5609182165634445, |
| "pass@64": 0.6065126277737566, |
| "pass@128": 0.6459627329192547 |
| }, |
| "18": { |
| "pass@1": 0.21994485294117647, |
| "pass@2": 0.2876186892079666, |
| "pass@4": 0.35397602643419546, |
| "pass@8": 0.41577271551011696, |
| "pass@16": 0.4692777647810956, |
| "pass@32": 0.5149172959863509, |
| "pass@64": 0.5566269269161073, |
| "pass@128": 0.5970588235294118 |
| }, |
| "19": { |
| "pass@1": 0.17896670386904762, |
| "pass@2": 0.24228453376921633, |
| "pass@4": 0.3052222350554393, |
| "pass@8": 0.36137853712505047, |
| "pass@16": 0.41033944710580444, |
| "pass@32": 0.4536026470267038, |
| "pass@64": 0.4931087740130446, |
| "pass@128": 0.5327380952380952 |
| }, |
| "20": { |
| "pass@1": 0.14464820359281438, |
| "pass@2": 0.19913384353340563, |
| "pass@4": 0.2525909067691389, |
| "pass@8": 0.3026967108449422, |
| "pass@16": 0.34996399288652164, |
| "pass@32": 0.39467057046628035, |
| "pass@64": 0.4374371107253553, |
| "pass@128": 0.47604790419161674 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 565992, |
| "answer_accuracy": 0.7064726793417478, |
| "avg_response_len": 282.91117540741334, |
| "resp_tokens_sum": 226654854, |
| "pass_at_k": { |
| "pass@1": 0.7064726793417478, |
| "pass@2": 0.7465973014606943, |
| "pass@4": 0.7777583073624791, |
| "pass@8": 0.8028225638276483, |
| "pass@16": 0.8233834153374825, |
| "pass@32": 0.8408147920202843, |
| "pass@64": 0.8557843704790238, |
| "pass@128": 0.868349576609682 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9315531716417911, |
| "pass@2": 0.9500055088729578, |
| "pass@4": 0.9595039831774755, |
| "pass@8": 0.9673470409588951, |
| "pass@16": 0.9746367138289137, |
| "pass@32": 0.9802078501456624, |
| "pass@64": 0.9835196793541271, |
| "pass@128": 0.9850746268656716 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999314692982456, |
| "pass@2": 0.9999996402587834, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9902113970588236, |
| "pass@2": 0.9909560994673461, |
| "pass@4": 0.9913272637795277, |
| "pass@8": 0.9915337428304501, |
| "pass@16": 0.9918683418196722, |
| "pass@32": 0.992467577582214, |
| "pass@64": 0.9933881426586383, |
| "pass@128": 0.9941176470588236 |
| }, |
| "6": { |
| "pass@1": 0.9931240141955836, |
| "pass@2": 0.999324685163566, |
| "pass@4": 0.9999875186145893, |
| "pass@8": 0.9999999956424814, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9916683148734177, |
| "pass@2": 0.9989651338084321, |
| "pass@4": 0.9999520768962424, |
| "pass@8": 0.9999998305577984, |
| "pass@16": 0.9999999999987065, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9695007507507507, |
| "pass@2": 0.985661104017403, |
| "pass@4": 0.9899353118585399, |
| "pass@8": 0.9923632575228235, |
| "pass@16": 0.9945284865421475, |
| "pass@32": 0.9961461919806662, |
| "pass@64": 0.9969084473566544, |
| "pass@128": 0.996996996996997 |
| }, |
| "9": { |
| "pass@1": 0.958324959807074, |
| "pass@2": 0.9830643783072135, |
| "pass@4": 0.9897614975579816, |
| "pass@8": 0.9921204975281055, |
| "pass@16": 0.9937353286137367, |
| "pass@32": 0.9950919800668229, |
| "pass@64": 0.9967841208283975, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9302827380952381, |
| "pass@2": 0.9605592269716289, |
| "pass@4": 0.974174619839187, |
| "pass@8": 0.9801028124837545, |
| "pass@16": 0.9834570559922984, |
| "pass@32": 0.9862572223400154, |
| "pass@64": 0.9886073538787452, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.8466408639143731, |
| "pass@2": 0.8960929410773188, |
| "pass@4": 0.9223581401522054, |
| "pass@8": 0.9378106996557768, |
| "pass@16": 0.9472559198752548, |
| "pass@32": 0.9537005538684983, |
| "pass@64": 0.9584471767968687, |
| "pass@128": 0.9602446483180428 |
| }, |
| "13": { |
| "pass@1": 0.679735725308642, |
| "pass@2": 0.7567773646349767, |
| "pass@4": 0.805612143678799, |
| "pass@8": 0.839658816993664, |
| "pass@16": 0.8658793001302546, |
| "pass@32": 0.8862026550356525, |
| "pass@64": 0.9017406716529552, |
| "pass@128": 0.9135802469135802 |
| }, |
| "14": { |
| "pass@1": 0.5714716246290801, |
| "pass@2": 0.6545268288978713, |
| "pass@4": 0.7162127416750945, |
| "pass@8": 0.7610036057794388, |
| "pass@16": 0.7964032510766361, |
| "pass@32": 0.8272994531619295, |
| "pass@64": 0.8525827941386422, |
| "pass@128": 0.8724035608308606 |
| }, |
| "15": { |
| "pass@1": 0.43429301697530864, |
| "pass@2": 0.5223389502527462, |
| "pass@4": 0.5966668286834513, |
| "pass@8": 0.659581656347112, |
| "pass@16": 0.7088315725579833, |
| "pass@32": 0.7452782492230502, |
| "pass@64": 0.7739088213987012, |
| "pass@128": 0.7993827160493827 |
| }, |
| "16": { |
| "pass@1": 0.3088126843657817, |
| "pass@2": 0.3951749878057278, |
| "pass@4": 0.474545925951734, |
| "pass@8": 0.5420576203264432, |
| "pass@16": 0.5976382087845169, |
| "pass@32": 0.6438050130672477, |
| "pass@64": 0.6833998326354856, |
| "pass@128": 0.7227138643067846 |
| }, |
| "17": { |
| "pass@1": 0.23270425451807228, |
| "pass@2": 0.306759024286121, |
| "pass@4": 0.37694031525426774, |
| "pass@8": 0.4393868852582266, |
| "pass@16": 0.4924237323769399, |
| "pass@32": 0.5367887415364578, |
| "pass@64": 0.5768039245457116, |
| "pass@128": 0.6144578313253012 |
| }, |
| "18": { |
| "pass@1": 0.22270665322580646, |
| "pass@2": 0.29104608521717046, |
| "pass@4": 0.35537309620571617, |
| "pass@8": 0.41243105277516273, |
| "pass@16": 0.4632090305336774, |
| "pass@32": 0.5069404871616056, |
| "pass@64": 0.5423413744862255, |
| "pass@128": 0.5709677419354838 |
| }, |
| "19": { |
| "pass@1": 0.17421166918429004, |
| "pass@2": 0.2406351101410661, |
| "pass@4": 0.3060777475511935, |
| "pass@8": 0.3655529463948891, |
| "pass@16": 0.41785418696358484, |
| "pass@32": 0.46570097986105186, |
| "pass@64": 0.5092918882760151, |
| "pass@128": 0.5438066465256798 |
| }, |
| "20": { |
| "pass@1": 0.1710304054054054, |
| "pass@2": 0.24044455380577423, |
| "pass@4": 0.30747180645212135, |
| "pass@8": 0.3621113126399097, |
| "pass@16": 0.40679923108349025, |
| "pass@32": 0.4503535060851414, |
| "pass@64": 0.4931701838218701, |
| "pass@128": 0.5255255255255256 |
| } |
| } |
| } |
| } |
| } |
| ] |
