[ { "checkpoint": "checkpoint-42828", "id_acc": 0.9894826388888889, "ood_acc": 0.470528125, "total_acc": 0.7163486842105263, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.09417708333334, "ood_avg_resp_len": 344.22949609375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 574771, "answer_accuracy": 0.7098321905627569, "avg_response_len": 262.7198664242807, "resp_tokens_sum": 212731632, "pass_at_k": { "pass@1": 0.7098321905627569, "pass@2": 0.7544343266197929, "pass@4": 0.7921442557488494, "pass@8": 0.8234180639046024, "pass@16": 0.8490259583479542, "pass@32": 0.8706260639313723, "pass@64": 0.8894478974698937, "pass@128": 0.9054694909895669 }, "per_op_pass_at_k": { "10": { "pass@1": 0.938191371681416, "pass@2": 0.9507075581260308, "pass@4": 0.9574170305592333, "pass@8": 0.9626028420434352, "pass@16": 0.9676380108556495, "pass@32": 0.9733761722067171, "pass@64": 0.9794348064762847, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999044342507645, "pass@2": 0.9999988712706783, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9985107421875, "pass@2": 0.999341781496063, "pass@4": 0.9998666789698163, "pass@8": 0.9999951531193496, "pass@16": 0.9999999963269612, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.994469246031746, "pass@2": 0.9972573740782403, "pass@4": 0.9988838163086756, "pass@8": 0.999703452073703, "pass@16": 0.999977476461648, "pass@32": 0.9999999361951407, "pass@64": 0.999999999999996, "pass@128": 1.0 }, "6": { "pass@1": 0.9982924775449101, "pass@2": 0.9991354643076052, "pass@4": 0.9997573747330986, "pass@8": 0.9999824898608188, "pass@16": 0.9999999386330103, "pass@32": 0.9999999999999043, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.993805473372781, "pass@2": 0.9953204351674976, "pass@4": 0.9963636679290828, "pass@8": 0.9978022115058525, "pass@16": 0.9992398726779238, "pass@32": 0.9999300060627638, "pass@64": 0.9999998828759263, "pass@128": 1.0 }, "8": { "pass@1": 0.9918619791666666, "pass@2": 0.9953259861447606, "pass@4": 0.9971175333852498, "pass@8": 0.9986976735909107, "pass@16": 0.9997214713252663, "pass@32": 0.9999882289469381, "pass@64": 0.999999996955866, "pass@128": 1.0 }, "9": { "pass@1": 0.9906029929577465, "pass@2": 0.9971175973161807, "pass@4": 0.9994117220382662, "pass@8": 0.9999626653484882, "pass@16": 0.9999997920087084, "pass@32": 0.9999999999982944, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.91983888101983, "pass@2": 0.946985972318262, "pass@4": 0.9615740933410231, "pass@8": 0.9704805355870788, "pass@16": 0.9768131739959114, "pass@32": 0.9819557012317468, "pass@64": 0.98653133809928, "pass@128": 0.9915014164305949 }, "12": { "pass@1": 0.822607697740113, "pass@2": 0.8753218281507184, "pass@4": 0.9018579549907951, "pass@8": 0.9178689747222858, "pass@16": 0.9318606981543764, "pass@32": 0.9459486582670955, "pass@64": 0.9575915981731089, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.6809855015673981, "pass@2": 0.7670173000024677, "pass@4": 0.8201065466346483, "pass@8": 0.8561603281573825, "pass@16": 0.882208233859429, "pass@32": 0.899328313039419, "pass@64": 0.9099842471218675, "pass@128": 0.9216300940438872 }, "14": { "pass@1": 0.5971816588785047, "pass@2": 0.6971129375475261, "pass@4": 0.7666536472660535, "pass@8": 0.8149757096073096, "pass@16": 0.8522813198771078, "pass@32": 0.8827592151983151, "pass@64": 0.907156078560458, "pass@128": 0.9252336448598131 }, "15": { "pass@1": 0.49001986994219654, "pass@2": 0.5955823158936784, "pass@4": 0.682237448552312, "pass@8": 0.7510216556505724, "pass@16": 0.8021241041232691, "pass@32": 0.840262532442047, "pass@64": 0.871512978537945, "pass@128": 0.8988439306358381 }, "16": { "pass@1": 0.3142641129032258, "pass@2": 0.42235323533147057, "pass@4": 0.5219397541033175, "pass@8": 0.6054150479280793, "pass@16": 0.6731704410957603, "pass@32": 0.7300474628311022, "pass@64": 0.7812425186016846, "pass@128": 0.8258064516129032 }, "17": { "pass@1": 0.2594608020231214, "pass@2": 0.3552075747576351, "pass@4": 0.44889092467126573, "pass@8": 0.5298944076343943, "pass@16": 0.5938477057646299, "pass@32": 0.6458432672898637, "pass@64": 0.6926852959730377, "pass@128": 0.7312138728323699 }, "18": { "pass@1": 0.20560267857142858, "pass@2": 0.2975752249718784, "pass@4": 0.3935471034870643, "pass@8": 0.479720452534764, "pass@16": 0.5518197621634817, "pass@32": 0.6137227694722202, "pass@64": 0.6675575336255497, "pass@128": 0.7114285714285714 }, "19": { "pass@1": 0.16469594594594594, "pass@2": 0.2434220637960795, "pass@4": 0.3257808131528604, "pass@8": 0.4000305781451452, "pass@16": 0.46376379967114273, "pass@32": 0.5206510590669114, "pass@64": 0.571185394124095, "pass@128": 0.6126126126126126 }, "20": { "pass@1": 0.15843186936936937, "pass@2": 0.22814768508666133, "pass@4": 0.30495890800812053, "pass@8": 0.383387374477363, "pass@16": 0.4573577428718796, "pass@32": 0.5261218888926433, "pass@64": 0.5905595183935609, "pass@128": 0.6516516516516516 } } }, "teachers_in_school": { "count": 821120, "correct": 592767, "answer_accuracy": 0.7219005748246298, "avg_response_len": 273.3977908222915, "resp_tokens_sum": 224492394, "pass_at_k": { "pass@1": 0.7219005748246298, "pass@2": 0.7684856788653494, "pass@4": 0.8070809353721593, "pass@8": 0.8377910566802691, "pass@16": 0.8615161766040658, "pass@32": 0.8800848254091402, "pass@64": 0.89531877294161, "pass@128": 0.9078721745908028 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9266919095092024, "pass@2": 0.9459019944688664, "pass@4": 0.9580926648931151, "pass@8": 0.9655181079663312, "pass@16": 0.9705065116715573, "pass@32": 0.9745265481735442, "pass@64": 0.9789238933060727, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9993134469696969, "pass@2": 0.9998590730136006, "pass@4": 0.9999941839656407, "pass@8": 0.9999999934122665, "pass@16": 0.999999999999999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9967391304347826, "pass@2": 0.9973970814789457, "pass@4": 0.9976917420648507, "pass@8": 0.9981758657979531, "pass@16": 0.9988891034656363, "pass@32": 0.9996345084013638, "pass@64": 0.99998095094953, "pass@128": 1.0 }, "6": { "pass@1": 0.9988135744985673, "pass@2": 0.9997299652550595, "pass@4": 0.9999778987153827, "pass@8": 0.9999998766975703, "pass@16": 0.9999999999988423, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9965227601156069, "pass@2": 0.9971318726958264, "pass@4": 0.9972870163267165, "pass@8": 0.9974611417052052, "pass@16": 0.9977897000591688, "pass@32": 0.9983785444449502, "pass@64": 0.9992831459651359, "pass@128": 1.0 }, "8": { "pass@1": 0.9943221830985915, "pass@2": 0.9966639403349228, "pass@4": 0.9979784254080916, "pass@8": 0.9990666990790485, "pass@16": 0.9997528296434962, "pass@32": 0.9999857246359625, "pass@64": 0.9999999936816814, "pass@128": 1.0 }, "9": { "pass@1": 0.9866672904191617, "pass@2": 0.9917241059220143, "pass@4": 0.9938411520603335, "pass@8": 0.9956474719604435, "pass@16": 0.9972573036700675, "pass@32": 0.998297458460058, "pass@64": 0.9992573828256583, "pass@128": 1.0 }, "11": { "pass@1": 0.9343232304216867, "pass@2": 0.9600106874347788, "pass@4": 0.9705750945288463, "pass@8": 0.9756324867411611, "pass@16": 0.9785410759514516, "pass@32": 0.9804200441280092, "pass@64": 0.9815556641572076, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8473011363636364, "pass@2": 0.8946973810875521, "pass@4": 0.9175820971046328, "pass@8": 0.9309735895345088, "pass@16": 0.9410367488667444, "pass@32": 0.948965953999329, "pass@64": 0.9554065505058231, "pass@128": 0.9623824451410659 }, "13": { "pass@1": 0.6850709033613446, "pass@2": 0.7874598166038067, "pass@4": 0.8496326567372344, "pass@8": 0.8870374883664758, "pass@16": 0.9118338376968488, "pass@32": 0.9300755296021455, "pass@64": 0.9456425249067281, "pass@128": 0.957983193277311 }, "14": { "pass@1": 0.6223958333333334, "pass@2": 0.7197378493806696, "pass@4": 0.7886473072444895, "pass@8": 0.8338802937283072, "pass@16": 0.8612657118281954, "pass@32": 0.8797224978366543, "pass@64": 0.8969039766342238, "pass@128": 0.9152046783625731 }, "15": { "pass@1": 0.5188920454545455, "pass@2": 0.6297918903602958, "pass@4": 0.7113841423231179, "pass@8": 0.7704131884610596, "pass@16": 0.8132045442176113, "pass@32": 0.8445315913779556, "pass@64": 0.8667790210259716, "pass@128": 0.8818181818181818 }, "16": { "pass@1": 0.31828703703703703, "pass@2": 0.4308475362855286, "pass@4": 0.5321735987595564, "pass@8": 0.6133575183418821, "pass@16": 0.6771684148710823, "pass@32": 0.7313473883636334, "pass@64": 0.7795209648610576, "pass@128": 0.8176638176638177 }, "17": { "pass@1": 0.29660811335403725, "pass@2": 0.404082047488629, "pass@4": 0.5058153114206997, "pass@8": 0.59019338398362, "pass@16": 0.6588107355969026, "pass@32": 0.7140254963144298, "pass@64": 0.755545829423683, "pass@128": 0.7888198757763976 }, "18": { "pass@1": 0.25089613970588237, "pass@2": 0.3484751331635016, "pass@4": 0.4468376891307704, "pass@8": 0.5344262099305134, "pass@16": 0.602894653464828, "pass@32": 0.652252172748721, "pass@64": 0.6898146169752933, "pass@128": 0.7176470588235294 }, "19": { "pass@1": 0.1790829613095238, "pass@2": 0.2575305528215224, "pass@4": 0.3426924809622011, "pass@8": 0.4214982239741903, "pass@16": 0.48576795980818427, "pass@32": 0.5364442340319796, "pass@64": 0.5787094815498106, "pass@128": 0.6160714285714286 }, "20": { "pass@1": 0.15112743263473055, "pass@2": 0.22712756082323543, "pass@4": 0.311789831192508, "pass@8": 0.3931681982311315, "pass@16": 0.4636711942826472, "pass@32": 0.5235392610475729, "pass@64": 0.5746665246604618, "pass@128": 0.6167664670658682 } } }, "movie_festival_awards": { "count": 801152, "correct": 574622, "answer_accuracy": 0.7172446676785429, "avg_response_len": 291.95236484462373, "resp_tokens_sum": 233898221, "pass_at_k": { "pass@1": 0.7172446676785429, "pass@2": 0.762473200009304, "pass@4": 0.8005009265997003, "pass@8": 0.8318729010553301, "pass@16": 0.8572073607122934, "pass@32": 0.8776230304314351, "pass@64": 0.8943118328037649, "pass@128": 0.9079725195718166 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9461054104477612, "pass@2": 0.9629205106357972, "pass@4": 0.9724614089283613, "pass@8": 0.9784954370979474, "pass@16": 0.9833977623356791, "pass@32": 0.9880483413656285, "pass@64": 0.9915697437769286, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999543128654971, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9866957720588235, "pass@2": 0.991383091130153, "pass@4": 0.9954316220582722, "pass@8": 0.998009907801721, "pass@16": 0.9992383013303037, "pass@32": 0.9998536614272977, "pass@64": 0.9999980360722474, "pass@128": 1.0 }, "6": { "pass@1": 0.999728903785489, "pass@2": 0.9999992237760501, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9992335838607594, "pass@2": 0.9999707994866939, "pass@4": 0.999999947791336, "pass@8": 0.99999999999998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9920701951951952, "pass@2": 0.99714071847438, "pass@4": 0.9993788327134785, "pass@8": 0.9999618621335632, "pass@16": 0.9999998522509821, "pass@32": 0.9999999999994185, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9870880225080386, "pass@2": 0.9951408493050107, "pass@4": 0.9981870624290932, "pass@8": 0.9994770626159226, "pass@16": 0.9999345493968065, "pass@32": 0.9999993363019254, "pass@64": 0.9999999999980551, "pass@128": 1.0 }, "11": { "pass@1": 0.9361111111111111, "pass@2": 0.964721597300337, "pass@4": 0.9788161914284527, "pass@8": 0.9862816248572933, "pass@16": 0.9897694199344508, "pass@32": 0.9919058827484885, "pass@64": 0.9933702121063085, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.8474053899082569, "pass@2": 0.9069223464326136, "pass@4": 0.9367416807990738, "pass@8": 0.9538386375839678, "pass@16": 0.9652243644935586, "pass@32": 0.9741221163718324, "pass@64": 0.9806778665797814, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.6504870756172839, "pass@2": 0.737235102556625, "pass@4": 0.7968340239298788, "pass@8": 0.8394478736620564, "pass@16": 0.8707955001207655, "pass@32": 0.8962127601427606, "pass@64": 0.9199947039243642, "pass@128": 0.9444444444444444 }, "14": { "pass@1": 0.5979460311572701, "pass@2": 0.6981321847473071, "pass@4": 0.7671013725287299, "pass@8": 0.8156496766290819, "pass@16": 0.8534954255336188, "pass@32": 0.8846802898454802, "pass@64": 0.9111804207009937, "pass@128": 0.9317507418397626 }, "15": { "pass@1": 0.5047019675925926, "pass@2": 0.6032917760279963, "pass@4": 0.6773077448073619, "pass@8": 0.7323858349826511, "pass@16": 0.7744864686864456, "pass@32": 0.8079447450839151, "pass@64": 0.8351820095707687, "pass@128": 0.8549382716049383 }, "16": { "pass@1": 0.32107300884955753, "pass@2": 0.4297624439644157, "pass@4": 0.5362244711114648, "pass@8": 0.6299412145504063, "pass@16": 0.7046635110365608, "pass@32": 0.7583830063556092, "pass@64": 0.7949665227100015, "pass@128": 0.8200589970501475 }, "17": { "pass@1": 0.252941453313253, "pass@2": 0.34630216238971623, "pass@4": 0.4382089559023495, "pass@8": 0.5172900182449583, "pass@16": 0.5816590133785813, "pass@32": 0.6347724043312906, "pass@64": 0.6815166946995262, "pass@128": 0.7228915662650602 }, "18": { "pass@1": 0.24987399193548387, "pass@2": 0.3411508604267206, "pass@4": 0.42806011889642825, "pass@8": 0.5009790895393157, "pass@16": 0.5617827302678425, "pass@32": 0.6148278041185289, "pass@64": 0.6618773335461938, "pass@128": 0.7 }, "19": { "pass@1": 0.18797205438066467, "pass@2": 0.26968615445916705, "pass@4": 0.3539405954165095, "pass@8": 0.43323175405120734, "pass@16": 0.5015421311382682, "pass@32": 0.5558973363316422, "pass@64": 0.5980831369200011, "pass@128": 0.6374622356495468 }, "20": { "pass@1": 0.1547954204204204, "pass@2": 0.2324537874488661, "pass@4": 0.32128977881143245, "pass@8": 0.4114801380719811, "pass@16": 0.49201842931368517, "pass@32": 0.5597262016579201, "pass@64": 0.6158544846756309, "pass@128": 0.6606606606606606 } } } } } ]