[ { "checkpoint": "checkpoint-85656", "id_acc": 0.9929774305555555, "ood_acc": 0.4781125, "total_acc": 0.7219958881578947, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.76994010416666, "ood_avg_resp_len": 339.323959375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 582274, "answer_accuracy": 0.7190982650964275, "avg_response_len": 260.7669427264464, "resp_tokens_sum": 211150295, "pass_at_k": { "pass@1": 0.7190982650964275, "pass@2": 0.7589688715922986, "pass@4": 0.7932620020861227, "pass@8": 0.8222878929551545, "pass@16": 0.8463803610796929, "pass@32": 0.8663935257186914, "pass@64": 0.8830698445425451, "pass@128": 0.8964590578564654 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9570658185840708, "pass@2": 0.9683732840917013, "pass@4": 0.9756881081236524, "pass@8": 0.9813912893648465, "pass@16": 0.9867117841326309, "pass@32": 0.9916970042118012, "pass@64": 0.9962000480088795, "pass@128": 1.0 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999755859375, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9967757936507936, "pass@2": 0.9981674165729285, "pass@4": 0.9991240716934194, "pass@8": 0.999773855222213, "pass@16": 0.9999870485045957, "pass@32": 0.9999999813944173, "pass@64": 0.9999999999999998, "pass@128": 1.0 }, "6": { "pass@1": 0.9999532185628742, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9982664571005917, "pass@2": 0.9996865972836976, "pass@4": 0.9999851585674571, "pass@8": 0.9999999736396237, "pass@16": 0.9999999999999807, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9986979166666666, "pass@2": 0.9998899814506359, "pass@4": 0.9999984557218807, "pass@8": 0.9999999997174955, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9965669014084507, "pass@2": 0.9971723549961184, "pass@4": 0.9971830946307767, "pass@8": 0.9971830985915493, "pass@16": 0.9971830985915493, "pass@32": 0.9971830985915493, "pass@64": 0.9971830985915493, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.9157445113314447, "pass@2": 0.9303819483170127, "pass@4": 0.9399434625381455, "pass@8": 0.9476715847990214, "pass@16": 0.954937662674825, "pass@32": 0.9618599495743576, "pass@64": 0.9670685637911212, "pass@128": 0.9688385269121813 }, "12": { "pass@1": 0.8374382062146892, "pass@2": 0.8719144812936519, "pass@4": 0.8911755683611579, "pass@8": 0.9053322956848698, "pass@16": 0.9172509395306903, "pass@32": 0.9285946036232826, "pass@64": 0.9412245539437281, "pass@128": 0.9548022598870056 }, "13": { "pass@1": 0.7017535266457681, "pass@2": 0.7656527688643157, "pass@4": 0.8069338072552844, "pass@8": 0.836520332654227, "pass@16": 0.8591068301655219, "pass@32": 0.8770989104903102, "pass@64": 0.8922208279506292, "pass@128": 0.9059561128526645 }, "14": { "pass@1": 0.6434968847352025, "pass@2": 0.7339530134667749, "pass@4": 0.7953253649952631, "pass@8": 0.8356009050741141, "pass@16": 0.8650482090662411, "pass@32": 0.8884090804371552, "pass@64": 0.9065243627350636, "pass@128": 0.9190031152647975 }, "15": { "pass@1": 0.5236407153179191, "pass@2": 0.6345384967684675, "pass@4": 0.720673474883558, "pass@8": 0.7874274695235931, "pass@16": 0.836765843937725, "pass@32": 0.870818671155785, "pass@64": 0.8910456399594974, "pass@128": 0.8988439306358381 }, "16": { "pass@1": 0.31060987903225806, "pass@2": 0.41360133032766044, "pass@4": 0.5086247351137556, "pass@8": 0.5870659401872468, "pass@16": 0.6479544799527384, "pass@32": 0.6986068415947772, "pass@64": 0.7455048157861739, "pass@128": 0.7903225806451613 }, "17": { "pass@1": 0.2576770231213873, "pass@2": 0.3527515674070362, "pass@4": 0.4467490030147967, "pass@8": 0.5316486877319969, "pass@16": 0.6051723920177607, "pass@32": 0.6689063240851189, "pass@64": 0.7234904728932069, "pass@128": 0.7658959537572254 }, "18": { "pass@1": 0.2110267857142857, "pass@2": 0.29613364735658004, "pass@4": 0.384165431463924, "pass@8": 0.4662889678969746, "pass@16": 0.5357500897786317, "pass@32": 0.5902205339958437, "pass@64": 0.6345563661728516, "pass@128": 0.6742857142857143 }, "19": { "pass@1": 0.173681493993994, "pass@2": 0.24716990612659911, "pass@4": 0.32139107752296747, "pass@8": 0.3897048009291002, "pass@16": 0.4521456987554777, "pass@32": 0.5096928722985181, "pass@64": 0.5592708746194527, "pass@128": 0.5975975975975976 }, "20": { "pass@1": 0.17227383633633633, "pass@2": 0.24018629850322742, "pass@4": 0.31119543052613924, "pass@8": 0.3805273280744088, "pass@16": 0.44240840872374637, "pass@32": 0.4948028929687008, "pass@64": 0.538554484333375, "pass@128": 0.5735735735735735 } } }, "teachers_in_school": { "count": 821120, "correct": 596124, "answer_accuracy": 0.7259888932190179, "avg_response_len": 270.2181629968823, "resp_tokens_sum": 221881538, "pass_at_k": { "pass@1": 0.7259888932190179, "pass@2": 0.766912620979365, "pass@4": 0.8019209021173521, "pass@8": 0.8308038764426154, "pass@16": 0.8538239206472131, "pass@32": 0.8723197323980967, "pass@64": 0.8882274904806203, "pass@128": 0.9025720966484801 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9357026457055214, "pass@2": 0.9485471716342208, "pass@4": 0.9583134079099005, "pass@8": 0.965796860977208, "pass@16": 0.9720041059256965, "pass@32": 0.9780843207095543, "pass@64": 0.9840777547518659, "pass@128": 0.9877300613496932 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999763257575758, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999768860946746, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9993206521739131, "pass@2": 0.9999322435239073, "pass@4": 0.9999991343473369, "pass@8": 0.9999999999112675, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9995299068767909, "pass@2": 0.9999576969067977, "pass@4": 0.9999996293443263, "pass@8": 0.9999999999871032, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9993000361271677, "pass@2": 0.9998751905921442, "pass@4": 0.9999952453558912, "pass@8": 0.9999999955120726, "pass@16": 0.9999999999999997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9959727112676057, "pass@2": 0.9982917683264945, "pass@4": 0.9995983774035287, "pass@8": 0.9999709649128528, "pass@16": 0.9999998400094289, "pass@32": 0.9999999999990319, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9944330089820359, "pass@2": 0.9963245261445614, "pass@4": 0.9975550485081578, "pass@8": 0.9985324513055662, "pass@16": 0.9993539825001094, "pass@32": 0.9998914962254856, "pass@64": 0.999999085108258, "pass@128": 1.0 }, "11": { "pass@1": 0.9468655873493976, "pass@2": 0.9604431505549762, "pass@4": 0.9680644524103162, "pass@8": 0.9726580226305667, "pass@16": 0.9756575108892106, "pass@32": 0.9783564878872142, "pass@64": 0.9817601740575576, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.8422070924764891, "pass@2": 0.8776021739935335, "pass@4": 0.8994707533619423, "pass@8": 0.9159269312012803, "pass@16": 0.9295581121779324, "pass@32": 0.9422592174940747, "pass@64": 0.9548234063200668, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.7061887254901961, "pass@2": 0.7791781082511747, "pass@4": 0.8260199885728562, "pass@8": 0.8570431299381749, "pass@16": 0.8790524526209159, "pass@32": 0.8947079171348, "pass@64": 0.9077045369983032, "pass@128": 0.9215686274509803 }, "14": { "pass@1": 0.6597222222222222, "pass@2": 0.7535168301330757, "pass@4": 0.8180587859631577, "pass@8": 0.8601767977854613, "pass@16": 0.8856046539907487, "pass@32": 0.9014211318157292, "pass@64": 0.9136951455705985, "pass@128": 0.9269005847953217 }, "15": { "pass@1": 0.5109611742424243, "pass@2": 0.6282312544738723, "pass@4": 0.7217386031291542, "pass@8": 0.7923839668825581, "pass@16": 0.8425049221743166, "pass@32": 0.8757183743773951, "pass@64": 0.8958328669936695, "pass@128": 0.9090909090909091 }, "16": { "pass@1": 0.3457532051282051, "pass@2": 0.4452947988873186, "pass@4": 0.532903049779034, "pass@8": 0.6048692395044325, "pass@16": 0.6625327790912872, "pass@32": 0.7115800723039244, "pass@64": 0.757156872133029, "pass@128": 0.8005698005698005 }, "17": { "pass@1": 0.2784355590062112, "pass@2": 0.3800656881205065, "pass@4": 0.477290026246719, "pass@8": 0.5595626443779531, "pass@16": 0.625599140698911, "pass@32": 0.6790160770546658, "pass@64": 0.7283353115816313, "pass@128": 0.7763975155279503 }, "18": { "pass@1": 0.23963694852941175, "pass@2": 0.3307752431681334, "pass@4": 0.42159714759919714, "pass@8": 0.5055019564772302, "pass@16": 0.5780629837271026, "pass@32": 0.6386943668162116, "pass@64": 0.6911164787819537, "pass@128": 0.7352941176470589 }, "19": { "pass@1": 0.17643229166666666, "pass@2": 0.24522813554555678, "pass@4": 0.3185077649334011, "pass@8": 0.3868667149725815, "pass@16": 0.4454668435699914, "pass@32": 0.4963013158912973, "pass@64": 0.5420733465747838, "pass@128": 0.5833333333333334 }, "20": { "pass@1": 0.1466364146706587, "pass@2": 0.2122326459286152, "pass@4": 0.2832578821547007, "pass@8": 0.35382239266907534, "pass@16": 0.41688701623004404, "pass@32": 0.46919719938448934, "pass@64": 0.5122032313480497, "pass@128": 0.5508982035928144 } } }, "movie_festival_awards": { "count": 801152, "correct": 577496, "answer_accuracy": 0.7208320019172392, "avg_response_len": 288.8812684733983, "resp_tokens_sum": 231437806, "pass_at_k": { "pass@1": 0.7208320019172392, "pass@2": 0.7610648540117928, "pass@4": 0.7956922751747378, "pass@8": 0.824719333058601, "pass@16": 0.8483699736521619, "pass@32": 0.8678111148940243, "pass@64": 0.8841115817330713, "pass@128": 0.8980667838312829 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9563199626865672, "pass@2": 0.9663844899518158, "pass@4": 0.9732305133500102, "pass@8": 0.9785252509761181, "pass@16": 0.9830552767950438, "pass@32": 0.9872244258703623, "pass@64": 0.9910618926107597, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.996920955882353, "pass@2": 0.999296911185734, "pass@4": 0.999915258386819, "pass@8": 0.9999981750560808, "pass@16": 0.9999999995322573, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9986691640378549, "pass@2": 0.9994651816985022, "pass@4": 0.9999134103702337, "pass@8": 0.9999980443690937, "pass@16": 0.9999999994983201, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9996785996835443, "pass@2": 0.9999937705571613, "pass@4": 0.9999999955503979, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9931024774774775, "pass@2": 0.9966164491262917, "pass@4": 0.998535216318681, "pass@8": 0.9994952286242749, "pass@16": 0.9999277542615488, "pass@32": 0.9999991007812299, "pass@64": 0.9999999999952028, "pass@128": 1.0 }, "9": { "pass@1": 0.9935942524115756, "pass@2": 0.9967580607387904, "pass@4": 0.9984507620831962, "pass@8": 0.9995185509250936, "pass@16": 0.9999447697863701, "pass@32": 0.9999995445205452, "pass@64": 0.9999999999992755, "pass@128": 1.0 }, "11": { "pass@1": 0.933531746031746, "pass@2": 0.9515041088613923, "pass@4": 0.9643743076758263, "pass@8": 0.9733412595679051, "pass@16": 0.9793917714194672, "pass@32": 0.9834661512791815, "pass@64": 0.9856061741231261, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.8517297400611621, "pass@2": 0.892629999518409, "pass@4": 0.9156304034243424, "pass@8": 0.9315569919301774, "pass@16": 0.9442163881132603, "pass@32": 0.9550475627343186, "pass@64": 0.9651791869569815, "pass@128": 0.9724770642201835 }, "13": { "pass@1": 0.6634355709876543, "pass@2": 0.7349605387868178, "pass@4": 0.7827549337004169, "pass@8": 0.8185453779861791, "pass@16": 0.8474015318736398, "pass@32": 0.8702366775562574, "pass@64": 0.8869076964727599, "pass@128": 0.8981481481481481 }, "14": { "pass@1": 0.6352698442136498, "pass@2": 0.7258190173835839, "pass@4": 0.7881565101098262, "pass@8": 0.828713185282778, "pass@16": 0.8561651647024184, "pass@32": 0.8781020595086385, "pass@64": 0.8971976264132405, "pass@128": 0.913946587537092 }, "15": { "pass@1": 0.5290075231481481, "pass@2": 0.6366762205939533, "pass@4": 0.7172502033889275, "pass@8": 0.7746811978937012, "pass@16": 0.8138282124679664, "pass@32": 0.8408839816842546, "pass@64": 0.8593529920260841, "pass@128": 0.8734567901234568 }, "16": { "pass@1": 0.3200129056047198, "pass@2": 0.419621600120781, "pass@4": 0.510766843856907, "pass@8": 0.587493043139971, "pass@16": 0.6510022787317294, "pass@32": 0.7076012028954857, "pass@64": 0.7594490626705939, "pass@128": 0.8053097345132744 }, "17": { "pass@1": 0.2574124623493976, "pass@2": 0.35109927900578686, "pass@4": 0.4391238452121193, "pass@8": 0.5116352714481722, "pass@16": 0.5700288921857198, "pass@32": 0.6232034647147692, "pass@64": 0.6743981124495337, "pass@128": 0.7198795180722891 }, "18": { "pass@1": 0.22880544354838708, "pass@2": 0.3054177514605029, "pass@4": 0.3843950726925263, "pass@8": 0.46068492782560816, "pass@16": 0.5260139043388721, "pass@32": 0.5775948095148585, "pass@64": 0.6185270051584428, "pass@128": 0.6548387096774193 }, "19": { "pass@1": 0.16887745468277945, "pass@2": 0.2465554898065991, "pass@4": 0.33081693168263315, "pass@8": 0.4119898536696145, "pass@16": 0.4832835378374356, "pass@32": 0.542532246435064, "pass@64": 0.5936834175175014, "pass@128": 0.6404833836858006 }, "20": { "pass@1": 0.1543262012012012, "pass@2": 0.22396535314842403, "pass@4": 0.30279329132281885, "pass@8": 0.38295495014077957, "pass@16": 0.4554402145987438, "pass@32": 0.513757434375952, "pass@64": 0.5580152117413981, "pass@128": 0.5945945945945946 } } } } } ]