| [ |
| { |
| "checkpoint": "checkpoint-43092", |
| "id_acc": 0.9890286458333334, |
| "ood_acc": 0.45655625, |
| "total_acc": 0.7087800164473684, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.0608220486111, |
| "ood_avg_resp_len": 337.71591484375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 572786, |
| "answer_accuracy": 0.7073807500790389, |
| "avg_response_len": 259.53983806907996, |
| "resp_tokens_sum": 210156674, |
| "pass_at_k": { |
| "pass@1": 0.7073807500790389, |
| "pass@2": 0.7484131737598306, |
| "pass@4": 0.7826223199089866, |
| "pass@8": 0.8112873192411284, |
| "pass@16": 0.8348865510699676, |
| "pass@32": 0.8543960303613137, |
| "pass@64": 0.8710845727829702, |
| "pass@128": 0.8857097692064496 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.933766592920354, |
| "pass@2": 0.9475370473602304, |
| "pass@4": 0.9553202182691763, |
| "pass@8": 0.9613865974516098, |
| "pass@16": 0.9667406524526152, |
| "pass@32": 0.9709195009812132, |
| "pass@64": 0.9745526757724312, |
| "pass@128": 0.9793510324483776 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999044342507645, |
| "pass@2": 0.9999988712706783, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.996142578125, |
| "pass@2": 0.9967488927165353, |
| "pass@4": 0.9968706206646043, |
| "pass@8": 0.9968749965852275, |
| "pass@16": 0.9968749999999998, |
| "pass@32": 0.996875, |
| "pass@64": 0.996875, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9925347222222223, |
| "pass@2": 0.99652543432071, |
| "pass@4": 0.9985260767999237, |
| "pass@8": 0.9996086870710147, |
| "pass@16": 0.9999673965062219, |
| "pass@32": 0.9999998586121597, |
| "pass@64": 0.9999999999999666, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9990175898203593, |
| "pass@2": 0.9998707064218019, |
| "pass@4": 0.9999974362299026, |
| "pass@8": 0.9999999993034019, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9967871671597633, |
| "pass@2": 0.9986597574896333, |
| "pass@4": 0.9994644609882345, |
| "pass@8": 0.9999097856251264, |
| "pass@16": 0.9999979755459689, |
| "pass@32": 0.999999999687371, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9962940705128205, |
| "pass@2": 0.9995362658994551, |
| "pass@4": 0.9999801467124302, |
| "pass@8": 0.9999999460636346, |
| "pass@16": 0.9999999999998607, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9914832746478873, |
| "pass@2": 0.9945464123322614, |
| "pass@4": 0.9954065231282707, |
| "pass@8": 0.9962856938141759, |
| "pass@16": 0.9975480903170094, |
| "pass@32": 0.9989312408746419, |
| "pass@64": 0.9998281583330123, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9224504249291785, |
| "pass@2": 0.9485899126720354, |
| "pass@4": 0.9635718603517338, |
| "pass@8": 0.9746340552270142, |
| "pass@16": 0.9827315024722435, |
| "pass@32": 0.9890347097729495, |
| "pass@64": 0.9939729033360523, |
| "pass@128": 0.9971671388101983 |
| }, |
| "12": { |
| "pass@1": 0.8250132415254238, |
| "pass@2": 0.8754302626896221, |
| "pass@4": 0.9020065444679585, |
| "pass@8": 0.9187468401482781, |
| "pass@16": 0.9300900627208345, |
| "pass@32": 0.9374870101619674, |
| "pass@64": 0.943320106586296, |
| "pass@128": 0.9463276836158192 |
| }, |
| "13": { |
| "pass@1": 0.6892878134796239, |
| "pass@2": 0.7743891621208006, |
| "pass@4": 0.8298121179209959, |
| "pass@8": 0.8653668213711395, |
| "pass@16": 0.8887256261906442, |
| "pass@32": 0.9066502334100086, |
| "pass@64": 0.9226060356153544, |
| "pass@128": 0.9341692789968652 |
| }, |
| "14": { |
| "pass@1": 0.6008810358255452, |
| "pass@2": 0.6981550641450188, |
| "pass@4": 0.7680769103628403, |
| "pass@8": 0.8199068539406262, |
| "pass@16": 0.8589850392938971, |
| "pass@32": 0.8882216939153829, |
| "pass@64": 0.9114461262603831, |
| "pass@128": 0.9283489096573209 |
| }, |
| "15": { |
| "pass@1": 0.499683887283237, |
| "pass@2": 0.6034264627235901, |
| "pass@4": 0.685617021951013, |
| "pass@8": 0.753911944629328, |
| "pass@16": 0.8109761641534945, |
| "pass@32": 0.8549149812310414, |
| "pass@64": 0.888116284661785, |
| "pass@128": 0.9132947976878613 |
| }, |
| "16": { |
| "pass@1": 0.28996975806451614, |
| "pass@2": 0.392073993522987, |
| "pass@4": 0.48502713118521434, |
| "pass@8": 0.5628008190252322, |
| "pass@16": 0.6288144586114016, |
| "pass@32": 0.6871668599368553, |
| "pass@64": 0.7371638065675952, |
| "pass@128": 0.7806451612903226 |
| }, |
| "17": { |
| "pass@1": 0.23787481936416185, |
| "pass@2": 0.3213355104455874, |
| "pass@4": 0.4036086006770826, |
| "pass@8": 0.478981711950451, |
| "pass@16": 0.5446522133794389, |
| "pass@32": 0.6005623952808441, |
| "pass@64": 0.6508987513647807, |
| "pass@128": 0.7023121387283237 |
| }, |
| "18": { |
| "pass@1": 0.18537946428571428, |
| "pass@2": 0.25326877109111356, |
| "pass@4": 0.3241999343832021, |
| "pass@8": 0.3925693688574649, |
| "pass@16": 0.4538384179215893, |
| "pass@32": 0.510303574324437, |
| "pass@64": 0.563145543835638, |
| "pass@128": 0.6114285714285714 |
| }, |
| "19": { |
| "pass@1": 0.16910660660660662, |
| "pass@2": 0.24374275850653804, |
| "pass@4": 0.31945002989491195, |
| "pass@8": 0.38608697149641263, |
| "pass@16": 0.44213983771966503, |
| "pass@32": 0.4886300555889962, |
| "pass@64": 0.5260766337906572, |
| "pass@128": 0.5585585585585585 |
| }, |
| "20": { |
| "pass@1": 0.146631006006006, |
| "pass@2": 0.20644241682627507, |
| "pass@4": 0.27179707378919976, |
| "pass@8": 0.3340115890129607, |
| "pass@16": 0.3849817874397895, |
| "pass@32": 0.4260355274333975, |
| "pass@64": 0.4629757589609945, |
| "pass@128": 0.4984984984984985 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 581884, |
| "answer_accuracy": 0.7086467264224474, |
| "avg_response_len": 269.1478894680437, |
| "resp_tokens_sum": 221002715, |
| "pass_at_k": { |
| "pass@1": 0.7086467264224474, |
| "pass@2": 0.7529284181083906, |
| "pass@4": 0.789696794033826, |
| "pass@8": 0.8200934667162189, |
| "pass@16": 0.8447522358027476, |
| "pass@32": 0.8648850453631034, |
| "pass@64": 0.8819242262338707, |
| "pass@128": 0.8964925954793452 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.92578125, |
| "pass@2": 0.9429499933578087, |
| "pass@4": 0.9526008393889411, |
| "pass@8": 0.9601664585552419, |
| "pass@16": 0.966387221960968, |
| "pass@32": 0.9720659871471612, |
| "pass@64": 0.9785593196538308, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999768860946746, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.994633152173913, |
| "pass@2": 0.9967352076914299, |
| "pass@4": 0.9977504507588726, |
| "pass@8": 0.9984084344178916, |
| "pass@16": 0.999164532036249, |
| "pass@32": 0.9998028560467964, |
| "pass@64": 0.9999958123575, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9963287965616046, |
| "pass@2": 0.9987809658642242, |
| "pass@4": 0.999815062694527, |
| "pass@8": 0.9999952074688148, |
| "pass@16": 0.9999999974425797, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9959808526011561, |
| "pass@2": 0.9983575650857949, |
| "pass@4": 0.999492153618081, |
| "pass@8": 0.9999396093759136, |
| "pass@16": 0.9999991648348607, |
| "pass@32": 0.9999999999571497, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9943882042253521, |
| "pass@2": 0.9975542724853055, |
| "pass@4": 0.9981729394741148, |
| "pass@8": 0.9988346119276097, |
| "pass@16": 0.9995479293349335, |
| "pass@32": 0.9999462946812021, |
| "pass@64": 0.9999998252073432, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9845387350299402, |
| "pass@2": 0.9911594134565516, |
| "pass@4": 0.9930956371845733, |
| "pass@8": 0.9940558490829383, |
| "pass@16": 0.9947300775819929, |
| "pass@32": 0.99550884869657, |
| "pass@64": 0.9970059880239206, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9327230798192772, |
| "pass@2": 0.9553884830661227, |
| "pass@4": 0.9655985720158469, |
| "pass@8": 0.9732440311714325, |
| "pass@16": 0.9797502401646335, |
| "pass@32": 0.9842853142934843, |
| "pass@64": 0.9868220768962014, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.8315047021943573, |
| "pass@2": 0.8758245809986922, |
| "pass@4": 0.8984926463933383, |
| "pass@8": 0.9122377949607192, |
| "pass@16": 0.9210956330412908, |
| "pass@32": 0.9279964265749576, |
| "pass@64": 0.9346488207894688, |
| "pass@128": 0.9404388714733543 |
| }, |
| "13": { |
| "pass@1": 0.6733849789915967, |
| "pass@2": 0.7632339845386972, |
| "pass@4": 0.8257816785506843, |
| "pass@8": 0.873118872788884, |
| "pass@16": 0.9058035124961593, |
| "pass@32": 0.9262089981982806, |
| "pass@64": 0.9400121851516173, |
| "pass@128": 0.9495798319327731 |
| }, |
| "14": { |
| "pass@1": 0.5953947368421053, |
| "pass@2": 0.7060957429663403, |
| "pass@4": 0.7819589659296974, |
| "pass@8": 0.8308866697243269, |
| "pass@16": 0.8652589646722112, |
| "pass@32": 0.891447718658418, |
| "pass@64": 0.9121577137047078, |
| "pass@128": 0.9298245614035088 |
| }, |
| "15": { |
| "pass@1": 0.45750473484848486, |
| "pass@2": 0.5845785612025762, |
| "pass@4": 0.6833397441797047, |
| "pass@8": 0.7585455044018924, |
| "pass@16": 0.8174234806184377, |
| "pass@32": 0.8595616298534416, |
| "pass@64": 0.8883757848949035, |
| "pass@128": 0.9121212121212121 |
| }, |
| "16": { |
| "pass@1": 0.3191773504273504, |
| "pass@2": 0.42424659858222846, |
| "pass@4": 0.5176986851002598, |
| "pass@8": 0.5958284212821107, |
| "pass@16": 0.6588898912568301, |
| "pass@32": 0.7107162679661149, |
| "pass@64": 0.7570844039780159, |
| "pass@128": 0.7977207977207977 |
| }, |
| "17": { |
| "pass@1": 0.2440799689440994, |
| "pass@2": 0.33687093461143414, |
| "pass@4": 0.42972677648601365, |
| "pass@8": 0.5149193235169248, |
| "pass@16": 0.5886558969071508, |
| "pass@32": 0.6524703951284391, |
| "pass@64": 0.7085742614052603, |
| "pass@128": 0.7546583850931677 |
| }, |
| "18": { |
| "pass@1": 0.2026424632352941, |
| "pass@2": 0.28368240504863346, |
| "pass@4": 0.36881430501334383, |
| "pass@8": 0.4510080695137622, |
| "pass@16": 0.5247043964142323, |
| "pass@32": 0.5886140567119187, |
| "pass@64": 0.6429471720895035, |
| "pass@128": 0.6882352941176471 |
| }, |
| "19": { |
| "pass@1": 0.15924944196428573, |
| "pass@2": 0.22613738224128227, |
| "pass@4": 0.29628468651240036, |
| "pass@8": 0.359848732678431, |
| "pass@16": 0.4142965991221225, |
| "pass@32": 0.4622006165063772, |
| "pass@64": 0.5040379662872626, |
| "pass@128": 0.5386904761904762 |
| }, |
| "20": { |
| "pass@1": 0.140437874251497, |
| "pass@2": 0.20751729808100344, |
| "pass@4": 0.2795247786204868, |
| "pass@8": 0.34537950497075726, |
| "pass@16": 0.4002546883745909, |
| "pass@32": 0.4489385049952736, |
| "pass@64": 0.49459444957064275, |
| "pass@128": 0.5389221556886228 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 569083, |
| "answer_accuracy": 0.7103308735420993, |
| "avg_response_len": 289.06755397227994, |
| "resp_tokens_sum": 231587049, |
| "pass_at_k": { |
| "pass@1": 0.7103308735420993, |
| "pass@2": 0.7545694829366925, |
| "pass@4": 0.7923817946027174, |
| "pass@8": 0.8244789529898521, |
| "pass@16": 0.8508752930876641, |
| "pass@32": 0.8721800395204266, |
| "pass@64": 0.8893696261444074, |
| "pass@128": 0.9036587314267455 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9393889925373134, |
| "pass@2": 0.9549491714655072, |
| "pass@4": 0.965057642608107, |
| "pass@8": 0.9737208157945008, |
| "pass@16": 0.98106388712431, |
| "pass@32": 0.9870307725573805, |
| "pass@64": 0.9912841778496856, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999772230320699, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999314692982456, |
| "pass@2": 0.9999996402587834, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9917279411764706, |
| "pass@2": 0.9963271479851784, |
| "pass@4": 0.9986156570502216, |
| "pass@8": 0.9997295335042793, |
| "pass@16": 0.9999901896087987, |
| "pass@32": 0.999999994425445, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9997535488958991, |
| "pass@2": 0.999999611888025, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9959454113924051, |
| "pass@2": 0.9986816941592744, |
| "pass@4": 0.9996448105181474, |
| "pass@8": 0.9999666000503677, |
| "pass@16": 0.9999997719707885, |
| "pass@32": 0.9999999999980841, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9924455705705706, |
| "pass@2": 0.9977377722210402, |
| "pass@4": 0.9995844652301346, |
| "pass@8": 0.9999838536555166, |
| "pass@16": 0.9999999819342317, |
| "pass@32": 0.9999999999999983, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9865856109324759, |
| "pass@2": 0.9923918272273846, |
| "pass@4": 0.9953768842600464, |
| "pass@8": 0.9978683483863066, |
| "pass@16": 0.9994365168263357, |
| "pass@32": 0.9999623618899749, |
| "pass@64": 0.9999999606641166, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9373511904761904, |
| "pass@2": 0.962304321334833, |
| "pass@4": 0.9718105858791461, |
| "pass@8": 0.976898166730419, |
| "pass@16": 0.9816211121446493, |
| "pass@32": 0.9857731100700239, |
| "pass@64": 0.9888041381121763, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.8498423165137615, |
| "pass@2": 0.9028085042982016, |
| "pass@4": 0.9311867590519071, |
| "pass@8": 0.948102987517338, |
| "pass@16": 0.9594348788726959, |
| "pass@32": 0.9668654821854931, |
| "pass@64": 0.971696629405173, |
| "pass@128": 0.9755351681957186 |
| }, |
| "13": { |
| "pass@1": 0.6694396219135802, |
| "pass@2": 0.7591893895207539, |
| "pass@4": 0.8213380865007612, |
| "pass@8": 0.8666384821132164, |
| "pass@16": 0.8992577884036135, |
| "pass@32": 0.9251175647819574, |
| "pass@64": 0.9452858855374175, |
| "pass@128": 0.9598765432098766 |
| }, |
| "14": { |
| "pass@1": 0.5999165430267063, |
| "pass@2": 0.7062372952639077, |
| "pass@4": 0.7839603653957199, |
| "pass@8": 0.8398434324267073, |
| "pass@16": 0.8811202144761617, |
| "pass@32": 0.9119805574487586, |
| "pass@64": 0.9328961358161704, |
| "pass@128": 0.9436201780415431 |
| }, |
| "15": { |
| "pass@1": 0.4603105709876543, |
| "pass@2": 0.5742631780402441, |
| "pass@4": 0.6655581154207579, |
| "pass@8": 0.7361133291928023, |
| "pass@16": 0.7895663565246654, |
| "pass@32": 0.8312652600547796, |
| "pass@64": 0.8664726963298978, |
| "pass@128": 0.8981481481481481 |
| }, |
| "16": { |
| "pass@1": 0.3001474926253687, |
| "pass@2": 0.4084765434464496, |
| "pass@4": 0.5108166163964012, |
| "pass@8": 0.6000877946591848, |
| "pass@16": 0.6762192856729624, |
| "pass@32": 0.7409432910171935, |
| "pass@64": 0.7943751083189771, |
| "pass@128": 0.8377581120943953 |
| }, |
| "17": { |
| "pass@1": 0.22797439759036145, |
| "pass@2": 0.31540902784365804, |
| "pass@4": 0.4051189386341765, |
| "pass@8": 0.49084683699379594, |
| "pass@16": 0.5660672610116918, |
| "pass@32": 0.6294375403203648, |
| "pass@64": 0.6849477865640023, |
| "pass@128": 0.7349397590361446 |
| }, |
| "18": { |
| "pass@1": 0.2153225806451613, |
| "pass@2": 0.29553673482346937, |
| "pass@4": 0.37796009561304855, |
| "pass@8": 0.4543094148417844, |
| "pass@16": 0.5207927067666392, |
| "pass@32": 0.5782161804753672, |
| "pass@64": 0.6254156054050343, |
| "pass@128": 0.6612903225806451 |
| }, |
| "19": { |
| "pass@1": 0.16467617069486404, |
| "pass@2": 0.2393482973333015, |
| "pass@4": 0.3177286392826274, |
| "pass@8": 0.3914122829378995, |
| "pass@16": 0.45648896396624944, |
| "pass@32": 0.5101591560041855, |
| "pass@64": 0.5542824876096353, |
| "pass@128": 0.595166163141994 |
| }, |
| "20": { |
| "pass@1": 0.15040822072072071, |
| "pass@2": 0.22015802712161, |
| "pass@4": 0.29973835900016993, |
| "pass@8": 0.3787759978843816, |
| "pass@16": 0.4455617672687102, |
| "pass@32": 0.49543394840778265, |
| "pass@64": 0.534018784210231, |
| "pass@128": 0.5705705705705706 |
| } |
| } |
| } |
| } |
| } |
| ] |
