[ { "checkpoint": "checkpoint-43092", "id_acc": 0.9890286458333334, "ood_acc": 0.45655625, "total_acc": 0.7087800164473684, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.0608220486111, "ood_avg_resp_len": 337.71591484375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 572786, "answer_accuracy": 0.7073807500790389, "avg_response_len": 259.53983806907996, "resp_tokens_sum": 210156674, "pass_at_k": { "pass@1": 0.7073807500790389, "pass@2": 0.7484131737598306, "pass@4": 0.7826223199089866, "pass@8": 0.8112873192411284, "pass@16": 0.8348865510699676, "pass@32": 0.8543960303613137, "pass@64": 0.8710845727829702, "pass@128": 0.8857097692064496 }, "per_op_pass_at_k": { "10": { "pass@1": 0.933766592920354, "pass@2": 0.9475370473602304, "pass@4": 0.9553202182691763, "pass@8": 0.9613865974516098, "pass@16": 0.9667406524526152, "pass@32": 0.9709195009812132, "pass@64": 0.9745526757724312, "pass@128": 0.9793510324483776 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999044342507645, "pass@2": 0.9999988712706783, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.996142578125, "pass@2": 0.9967488927165353, "pass@4": 0.9968706206646043, "pass@8": 0.9968749965852275, "pass@16": 0.9968749999999998, "pass@32": 0.996875, "pass@64": 0.996875, "pass@128": 0.996875 }, "5": { "pass@1": 0.9925347222222223, "pass@2": 0.99652543432071, "pass@4": 0.9985260767999237, "pass@8": 0.9996086870710147, "pass@16": 0.9999673965062219, "pass@32": 0.9999998586121597, "pass@64": 0.9999999999999666, "pass@128": 1.0 }, "6": { "pass@1": 0.9990175898203593, "pass@2": 0.9998707064218019, "pass@4": 0.9999974362299026, "pass@8": 0.9999999993034019, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9967871671597633, "pass@2": 0.9986597574896333, "pass@4": 0.9994644609882345, "pass@8": 0.9999097856251264, "pass@16": 0.9999979755459689, "pass@32": 0.999999999687371, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9962940705128205, "pass@2": 0.9995362658994551, "pass@4": 0.9999801467124302, "pass@8": 0.9999999460636346, "pass@16": 0.9999999999998607, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9914832746478873, "pass@2": 0.9945464123322614, "pass@4": 0.9954065231282707, "pass@8": 0.9962856938141759, "pass@16": 0.9975480903170094, "pass@32": 0.9989312408746419, "pass@64": 0.9998281583330123, "pass@128": 1.0 }, "11": { "pass@1": 0.9224504249291785, "pass@2": 0.9485899126720354, "pass@4": 0.9635718603517338, "pass@8": 0.9746340552270142, "pass@16": 0.9827315024722435, "pass@32": 0.9890347097729495, "pass@64": 0.9939729033360523, "pass@128": 0.9971671388101983 }, "12": { "pass@1": 0.8250132415254238, "pass@2": 0.8754302626896221, "pass@4": 0.9020065444679585, "pass@8": 0.9187468401482781, "pass@16": 0.9300900627208345, "pass@32": 0.9374870101619674, "pass@64": 0.943320106586296, "pass@128": 0.9463276836158192 }, "13": { "pass@1": 0.6892878134796239, "pass@2": 0.7743891621208006, "pass@4": 0.8298121179209959, "pass@8": 0.8653668213711395, "pass@16": 0.8887256261906442, "pass@32": 0.9066502334100086, "pass@64": 0.9226060356153544, "pass@128": 0.9341692789968652 }, "14": { "pass@1": 0.6008810358255452, "pass@2": 0.6981550641450188, "pass@4": 0.7680769103628403, "pass@8": 0.8199068539406262, "pass@16": 0.8589850392938971, "pass@32": 0.8882216939153829, "pass@64": 0.9114461262603831, "pass@128": 0.9283489096573209 }, "15": { "pass@1": 0.499683887283237, "pass@2": 0.6034264627235901, "pass@4": 0.685617021951013, "pass@8": 0.753911944629328, "pass@16": 0.8109761641534945, "pass@32": 0.8549149812310414, "pass@64": 0.888116284661785, "pass@128": 0.9132947976878613 }, "16": { "pass@1": 0.28996975806451614, "pass@2": 0.392073993522987, "pass@4": 0.48502713118521434, "pass@8": 0.5628008190252322, "pass@16": 0.6288144586114016, "pass@32": 0.6871668599368553, "pass@64": 0.7371638065675952, "pass@128": 0.7806451612903226 }, "17": { "pass@1": 0.23787481936416185, "pass@2": 0.3213355104455874, "pass@4": 0.4036086006770826, "pass@8": 0.478981711950451, "pass@16": 0.5446522133794389, "pass@32": 0.6005623952808441, "pass@64": 0.6508987513647807, "pass@128": 0.7023121387283237 }, "18": { "pass@1": 0.18537946428571428, "pass@2": 0.25326877109111356, "pass@4": 0.3241999343832021, "pass@8": 0.3925693688574649, "pass@16": 0.4538384179215893, "pass@32": 0.510303574324437, "pass@64": 0.563145543835638, "pass@128": 0.6114285714285714 }, "19": { "pass@1": 0.16910660660660662, "pass@2": 0.24374275850653804, "pass@4": 0.31945002989491195, "pass@8": 0.38608697149641263, "pass@16": 0.44213983771966503, "pass@32": 0.4886300555889962, "pass@64": 0.5260766337906572, "pass@128": 0.5585585585585585 }, "20": { "pass@1": 0.146631006006006, "pass@2": 0.20644241682627507, "pass@4": 0.27179707378919976, "pass@8": 0.3340115890129607, "pass@16": 0.3849817874397895, "pass@32": 0.4260355274333975, "pass@64": 0.4629757589609945, "pass@128": 0.4984984984984985 } } }, "teachers_in_school": { "count": 821120, "correct": 581884, "answer_accuracy": 0.7086467264224474, "avg_response_len": 269.1478894680437, "resp_tokens_sum": 221002715, "pass_at_k": { "pass@1": 0.7086467264224474, "pass@2": 0.7529284181083906, "pass@4": 0.789696794033826, "pass@8": 0.8200934667162189, "pass@16": 0.8447522358027476, "pass@32": 0.8648850453631034, "pass@64": 0.8819242262338707, "pass@128": 0.8964925954793452 }, "per_op_pass_at_k": { "10": { "pass@1": 0.92578125, "pass@2": 0.9429499933578087, "pass@4": 0.9526008393889411, "pass@8": 0.9601664585552419, "pass@16": 0.966387221960968, "pass@32": 0.9720659871471612, "pass@64": 0.9785593196538308, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999768860946746, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.994633152173913, "pass@2": 0.9967352076914299, "pass@4": 0.9977504507588726, "pass@8": 0.9984084344178916, "pass@16": 0.999164532036249, "pass@32": 0.9998028560467964, "pass@64": 0.9999958123575, "pass@128": 1.0 }, "6": { "pass@1": 0.9963287965616046, "pass@2": 0.9987809658642242, "pass@4": 0.999815062694527, "pass@8": 0.9999952074688148, "pass@16": 0.9999999974425797, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9959808526011561, "pass@2": 0.9983575650857949, "pass@4": 0.999492153618081, "pass@8": 0.9999396093759136, "pass@16": 0.9999991648348607, "pass@32": 0.9999999999571497, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9943882042253521, "pass@2": 0.9975542724853055, "pass@4": 0.9981729394741148, "pass@8": 0.9988346119276097, "pass@16": 0.9995479293349335, "pass@32": 0.9999462946812021, "pass@64": 0.9999998252073432, "pass@128": 1.0 }, "9": { "pass@1": 0.9845387350299402, "pass@2": 0.9911594134565516, "pass@4": 0.9930956371845733, "pass@8": 0.9940558490829383, "pass@16": 0.9947300775819929, "pass@32": 0.99550884869657, "pass@64": 0.9970059880239206, "pass@128": 1.0 }, "11": { "pass@1": 0.9327230798192772, "pass@2": 0.9553884830661227, "pass@4": 0.9655985720158469, "pass@8": 0.9732440311714325, "pass@16": 0.9797502401646335, "pass@32": 0.9842853142934843, "pass@64": 0.9868220768962014, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.8315047021943573, "pass@2": 0.8758245809986922, "pass@4": 0.8984926463933383, "pass@8": 0.9122377949607192, "pass@16": 0.9210956330412908, "pass@32": 0.9279964265749576, "pass@64": 0.9346488207894688, "pass@128": 0.9404388714733543 }, "13": { "pass@1": 0.6733849789915967, "pass@2": 0.7632339845386972, "pass@4": 0.8257816785506843, "pass@8": 0.873118872788884, "pass@16": 0.9058035124961593, "pass@32": 0.9262089981982806, "pass@64": 0.9400121851516173, "pass@128": 0.9495798319327731 }, "14": { "pass@1": 0.5953947368421053, "pass@2": 0.7060957429663403, "pass@4": 0.7819589659296974, "pass@8": 0.8308866697243269, "pass@16": 0.8652589646722112, "pass@32": 0.891447718658418, "pass@64": 0.9121577137047078, "pass@128": 0.9298245614035088 }, "15": { "pass@1": 0.45750473484848486, "pass@2": 0.5845785612025762, "pass@4": 0.6833397441797047, "pass@8": 0.7585455044018924, "pass@16": 0.8174234806184377, "pass@32": 0.8595616298534416, "pass@64": 0.8883757848949035, "pass@128": 0.9121212121212121 }, "16": { "pass@1": 0.3191773504273504, "pass@2": 0.42424659858222846, "pass@4": 0.5176986851002598, "pass@8": 0.5958284212821107, "pass@16": 0.6588898912568301, "pass@32": 0.7107162679661149, "pass@64": 0.7570844039780159, "pass@128": 0.7977207977207977 }, "17": { "pass@1": 0.2440799689440994, "pass@2": 0.33687093461143414, "pass@4": 0.42972677648601365, "pass@8": 0.5149193235169248, "pass@16": 0.5886558969071508, "pass@32": 0.6524703951284391, "pass@64": 0.7085742614052603, "pass@128": 0.7546583850931677 }, "18": { "pass@1": 0.2026424632352941, "pass@2": 0.28368240504863346, "pass@4": 0.36881430501334383, "pass@8": 0.4510080695137622, "pass@16": 0.5247043964142323, "pass@32": 0.5886140567119187, "pass@64": 0.6429471720895035, "pass@128": 0.6882352941176471 }, "19": { "pass@1": 0.15924944196428573, "pass@2": 0.22613738224128227, "pass@4": 0.29628468651240036, "pass@8": 0.359848732678431, "pass@16": 0.4142965991221225, "pass@32": 0.4622006165063772, "pass@64": 0.5040379662872626, "pass@128": 0.5386904761904762 }, "20": { "pass@1": 0.140437874251497, "pass@2": 0.20751729808100344, "pass@4": 0.2795247786204868, "pass@8": 0.34537950497075726, "pass@16": 0.4002546883745909, "pass@32": 0.4489385049952736, "pass@64": 0.49459444957064275, "pass@128": 0.5389221556886228 } } }, "movie_festival_awards": { "count": 801152, "correct": 569083, "answer_accuracy": 0.7103308735420993, "avg_response_len": 289.06755397227994, "resp_tokens_sum": 231587049, "pass_at_k": { "pass@1": 0.7103308735420993, "pass@2": 0.7545694829366925, "pass@4": 0.7923817946027174, "pass@8": 0.8244789529898521, "pass@16": 0.8508752930876641, "pass@32": 0.8721800395204266, "pass@64": 0.8893696261444074, "pass@128": 0.9036587314267455 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9393889925373134, "pass@2": 0.9549491714655072, "pass@4": 0.965057642608107, "pass@8": 0.9737208157945008, "pass@16": 0.98106388712431, "pass@32": 0.9870307725573805, "pass@64": 0.9912841778496856, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999772230320699, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999314692982456, "pass@2": 0.9999996402587834, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9917279411764706, "pass@2": 0.9963271479851784, "pass@4": 0.9986156570502216, "pass@8": 0.9997295335042793, "pass@16": 0.9999901896087987, "pass@32": 0.999999994425445, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9997535488958991, "pass@2": 0.999999611888025, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9959454113924051, "pass@2": 0.9986816941592744, "pass@4": 0.9996448105181474, "pass@8": 0.9999666000503677, "pass@16": 0.9999997719707885, "pass@32": 0.9999999999980841, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9924455705705706, "pass@2": 0.9977377722210402, "pass@4": 0.9995844652301346, "pass@8": 0.9999838536555166, "pass@16": 0.9999999819342317, "pass@32": 0.9999999999999983, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9865856109324759, "pass@2": 0.9923918272273846, "pass@4": 0.9953768842600464, "pass@8": 0.9978683483863066, "pass@16": 0.9994365168263357, "pass@32": 0.9999623618899749, "pass@64": 0.9999999606641166, "pass@128": 1.0 }, "11": { "pass@1": 0.9373511904761904, "pass@2": 0.962304321334833, "pass@4": 0.9718105858791461, "pass@8": 0.976898166730419, "pass@16": 0.9816211121446493, "pass@32": 0.9857731100700239, "pass@64": 0.9888041381121763, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.8498423165137615, "pass@2": 0.9028085042982016, "pass@4": 0.9311867590519071, "pass@8": 0.948102987517338, "pass@16": 0.9594348788726959, "pass@32": 0.9668654821854931, "pass@64": 0.971696629405173, "pass@128": 0.9755351681957186 }, "13": { "pass@1": 0.6694396219135802, "pass@2": 0.7591893895207539, "pass@4": 0.8213380865007612, "pass@8": 0.8666384821132164, "pass@16": 0.8992577884036135, "pass@32": 0.9251175647819574, "pass@64": 0.9452858855374175, "pass@128": 0.9598765432098766 }, "14": { "pass@1": 0.5999165430267063, "pass@2": 0.7062372952639077, "pass@4": 0.7839603653957199, "pass@8": 0.8398434324267073, "pass@16": 0.8811202144761617, "pass@32": 0.9119805574487586, "pass@64": 0.9328961358161704, "pass@128": 0.9436201780415431 }, "15": { "pass@1": 0.4603105709876543, "pass@2": 0.5742631780402441, "pass@4": 0.6655581154207579, "pass@8": 0.7361133291928023, "pass@16": 0.7895663565246654, "pass@32": 0.8312652600547796, "pass@64": 0.8664726963298978, "pass@128": 0.8981481481481481 }, "16": { "pass@1": 0.3001474926253687, "pass@2": 0.4084765434464496, "pass@4": 0.5108166163964012, "pass@8": 0.6000877946591848, "pass@16": 0.6762192856729624, "pass@32": 0.7409432910171935, "pass@64": 0.7943751083189771, "pass@128": 0.8377581120943953 }, "17": { "pass@1": 0.22797439759036145, "pass@2": 0.31540902784365804, "pass@4": 0.4051189386341765, "pass@8": 0.49084683699379594, "pass@16": 0.5660672610116918, "pass@32": 0.6294375403203648, "pass@64": 0.6849477865640023, "pass@128": 0.7349397590361446 }, "18": { "pass@1": 0.2153225806451613, "pass@2": 0.29553673482346937, "pass@4": 0.37796009561304855, "pass@8": 0.4543094148417844, "pass@16": 0.5207927067666392, "pass@32": 0.5782161804753672, "pass@64": 0.6254156054050343, "pass@128": 0.6612903225806451 }, "19": { "pass@1": 0.16467617069486404, "pass@2": 0.2393482973333015, "pass@4": 0.3177286392826274, "pass@8": 0.3914122829378995, "pass@16": 0.45648896396624944, "pass@32": 0.5101591560041855, "pass@64": 0.5542824876096353, "pass@128": 0.595166163141994 }, "20": { "pass@1": 0.15040822072072071, "pass@2": 0.22015802712161, "pass@4": 0.29973835900016993, "pass@8": 0.3787759978843816, "pass@16": 0.4455617672687102, "pass@32": 0.49543394840778265, "pass@64": 0.534018784210231, "pass@128": 0.5705705705705706 } } } } } ]