| [ |
| { |
| "checkpoint": "checkpoint-85656", |
| "id_acc": 0.9929774305555555, |
| "ood_acc": 0.4781125, |
| "total_acc": 0.7219958881578947, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 199.76994010416666, |
| "ood_avg_resp_len": 339.323959375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 582274, |
| "answer_accuracy": 0.7190982650964275, |
| "avg_response_len": 260.7669427264464, |
| "resp_tokens_sum": 211150295, |
| "pass_at_k": { |
| "pass@1": 0.7190982650964275, |
| "pass@2": 0.7589688715922986, |
| "pass@4": 0.7932620020861227, |
| "pass@8": 0.8222878929551545, |
| "pass@16": 0.8463803610796929, |
| "pass@32": 0.8663935257186914, |
| "pass@64": 0.8830698445425451, |
| "pass@128": 0.8964590578564654 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9570658185840708, |
| "pass@2": 0.9683732840917013, |
| "pass@4": 0.9756881081236524, |
| "pass@8": 0.9813912893648465, |
| "pass@16": 0.9867117841326309, |
| "pass@32": 0.9916970042118012, |
| "pass@64": 0.9962000480088795, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999755859375, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9967757936507936, |
| "pass@2": 0.9981674165729285, |
| "pass@4": 0.9991240716934194, |
| "pass@8": 0.999773855222213, |
| "pass@16": 0.9999870485045957, |
| "pass@32": 0.9999999813944173, |
| "pass@64": 0.9999999999999998, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999532185628742, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9982664571005917, |
| "pass@2": 0.9996865972836976, |
| "pass@4": 0.9999851585674571, |
| "pass@8": 0.9999999736396237, |
| "pass@16": 0.9999999999999807, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9986979166666666, |
| "pass@2": 0.9998899814506359, |
| "pass@4": 0.9999984557218807, |
| "pass@8": 0.9999999997174955, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9965669014084507, |
| "pass@2": 0.9971723549961184, |
| "pass@4": 0.9971830946307767, |
| "pass@8": 0.9971830985915493, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.9157445113314447, |
| "pass@2": 0.9303819483170127, |
| "pass@4": 0.9399434625381455, |
| "pass@8": 0.9476715847990214, |
| "pass@16": 0.954937662674825, |
| "pass@32": 0.9618599495743576, |
| "pass@64": 0.9670685637911212, |
| "pass@128": 0.9688385269121813 |
| }, |
| "12": { |
| "pass@1": 0.8374382062146892, |
| "pass@2": 0.8719144812936519, |
| "pass@4": 0.8911755683611579, |
| "pass@8": 0.9053322956848698, |
| "pass@16": 0.9172509395306903, |
| "pass@32": 0.9285946036232826, |
| "pass@64": 0.9412245539437281, |
| "pass@128": 0.9548022598870056 |
| }, |
| "13": { |
| "pass@1": 0.7017535266457681, |
| "pass@2": 0.7656527688643157, |
| "pass@4": 0.8069338072552844, |
| "pass@8": 0.836520332654227, |
| "pass@16": 0.8591068301655219, |
| "pass@32": 0.8770989104903102, |
| "pass@64": 0.8922208279506292, |
| "pass@128": 0.9059561128526645 |
| }, |
| "14": { |
| "pass@1": 0.6434968847352025, |
| "pass@2": 0.7339530134667749, |
| "pass@4": 0.7953253649952631, |
| "pass@8": 0.8356009050741141, |
| "pass@16": 0.8650482090662411, |
| "pass@32": 0.8884090804371552, |
| "pass@64": 0.9065243627350636, |
| "pass@128": 0.9190031152647975 |
| }, |
| "15": { |
| "pass@1": 0.5236407153179191, |
| "pass@2": 0.6345384967684675, |
| "pass@4": 0.720673474883558, |
| "pass@8": 0.7874274695235931, |
| "pass@16": 0.836765843937725, |
| "pass@32": 0.870818671155785, |
| "pass@64": 0.8910456399594974, |
| "pass@128": 0.8988439306358381 |
| }, |
| "16": { |
| "pass@1": 0.31060987903225806, |
| "pass@2": 0.41360133032766044, |
| "pass@4": 0.5086247351137556, |
| "pass@8": 0.5870659401872468, |
| "pass@16": 0.6479544799527384, |
| "pass@32": 0.6986068415947772, |
| "pass@64": 0.7455048157861739, |
| "pass@128": 0.7903225806451613 |
| }, |
| "17": { |
| "pass@1": 0.2576770231213873, |
| "pass@2": 0.3527515674070362, |
| "pass@4": 0.4467490030147967, |
| "pass@8": 0.5316486877319969, |
| "pass@16": 0.6051723920177607, |
| "pass@32": 0.6689063240851189, |
| "pass@64": 0.7234904728932069, |
| "pass@128": 0.7658959537572254 |
| }, |
| "18": { |
| "pass@1": 0.2110267857142857, |
| "pass@2": 0.29613364735658004, |
| "pass@4": 0.384165431463924, |
| "pass@8": 0.4662889678969746, |
| "pass@16": 0.5357500897786317, |
| "pass@32": 0.5902205339958437, |
| "pass@64": 0.6345563661728516, |
| "pass@128": 0.6742857142857143 |
| }, |
| "19": { |
| "pass@1": 0.173681493993994, |
| "pass@2": 0.24716990612659911, |
| "pass@4": 0.32139107752296747, |
| "pass@8": 0.3897048009291002, |
| "pass@16": 0.4521456987554777, |
| "pass@32": 0.5096928722985181, |
| "pass@64": 0.5592708746194527, |
| "pass@128": 0.5975975975975976 |
| }, |
| "20": { |
| "pass@1": 0.17227383633633633, |
| "pass@2": 0.24018629850322742, |
| "pass@4": 0.31119543052613924, |
| "pass@8": 0.3805273280744088, |
| "pass@16": 0.44240840872374637, |
| "pass@32": 0.4948028929687008, |
| "pass@64": 0.538554484333375, |
| "pass@128": 0.5735735735735735 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 596124, |
| "answer_accuracy": 0.7259888932190179, |
| "avg_response_len": 270.2181629968823, |
| "resp_tokens_sum": 221881538, |
| "pass_at_k": { |
| "pass@1": 0.7259888932190179, |
| "pass@2": 0.766912620979365, |
| "pass@4": 0.8019209021173521, |
| "pass@8": 0.8308038764426154, |
| "pass@16": 0.8538239206472131, |
| "pass@32": 0.8723197323980967, |
| "pass@64": 0.8882274904806203, |
| "pass@128": 0.9025720966484801 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9357026457055214, |
| "pass@2": 0.9485471716342208, |
| "pass@4": 0.9583134079099005, |
| "pass@8": 0.965796860977208, |
| "pass@16": 0.9720041059256965, |
| "pass@32": 0.9780843207095543, |
| "pass@64": 0.9840777547518659, |
| "pass@128": 0.9877300613496932 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999763257575758, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999768860946746, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9993206521739131, |
| "pass@2": 0.9999322435239073, |
| "pass@4": 0.9999991343473369, |
| "pass@8": 0.9999999999112675, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9995299068767909, |
| "pass@2": 0.9999576969067977, |
| "pass@4": 0.9999996293443263, |
| "pass@8": 0.9999999999871032, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9993000361271677, |
| "pass@2": 0.9998751905921442, |
| "pass@4": 0.9999952453558912, |
| "pass@8": 0.9999999955120726, |
| "pass@16": 0.9999999999999997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9959727112676057, |
| "pass@2": 0.9982917683264945, |
| "pass@4": 0.9995983774035287, |
| "pass@8": 0.9999709649128528, |
| "pass@16": 0.9999998400094289, |
| "pass@32": 0.9999999999990319, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9944330089820359, |
| "pass@2": 0.9963245261445614, |
| "pass@4": 0.9975550485081578, |
| "pass@8": 0.9985324513055662, |
| "pass@16": 0.9993539825001094, |
| "pass@32": 0.9998914962254856, |
| "pass@64": 0.999999085108258, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9468655873493976, |
| "pass@2": 0.9604431505549762, |
| "pass@4": 0.9680644524103162, |
| "pass@8": 0.9726580226305667, |
| "pass@16": 0.9756575108892106, |
| "pass@32": 0.9783564878872142, |
| "pass@64": 0.9817601740575576, |
| "pass@128": 0.9849397590361446 |
| }, |
| "12": { |
| "pass@1": 0.8422070924764891, |
| "pass@2": 0.8776021739935335, |
| "pass@4": 0.8994707533619423, |
| "pass@8": 0.9159269312012803, |
| "pass@16": 0.9295581121779324, |
| "pass@32": 0.9422592174940747, |
| "pass@64": 0.9548234063200668, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.7061887254901961, |
| "pass@2": 0.7791781082511747, |
| "pass@4": 0.8260199885728562, |
| "pass@8": 0.8570431299381749, |
| "pass@16": 0.8790524526209159, |
| "pass@32": 0.8947079171348, |
| "pass@64": 0.9077045369983032, |
| "pass@128": 0.9215686274509803 |
| }, |
| "14": { |
| "pass@1": 0.6597222222222222, |
| "pass@2": 0.7535168301330757, |
| "pass@4": 0.8180587859631577, |
| "pass@8": 0.8601767977854613, |
| "pass@16": 0.8856046539907487, |
| "pass@32": 0.9014211318157292, |
| "pass@64": 0.9136951455705985, |
| "pass@128": 0.9269005847953217 |
| }, |
| "15": { |
| "pass@1": 0.5109611742424243, |
| "pass@2": 0.6282312544738723, |
| "pass@4": 0.7217386031291542, |
| "pass@8": 0.7923839668825581, |
| "pass@16": 0.8425049221743166, |
| "pass@32": 0.8757183743773951, |
| "pass@64": 0.8958328669936695, |
| "pass@128": 0.9090909090909091 |
| }, |
| "16": { |
| "pass@1": 0.3457532051282051, |
| "pass@2": 0.4452947988873186, |
| "pass@4": 0.532903049779034, |
| "pass@8": 0.6048692395044325, |
| "pass@16": 0.6625327790912872, |
| "pass@32": 0.7115800723039244, |
| "pass@64": 0.757156872133029, |
| "pass@128": 0.8005698005698005 |
| }, |
| "17": { |
| "pass@1": 0.2784355590062112, |
| "pass@2": 0.3800656881205065, |
| "pass@4": 0.477290026246719, |
| "pass@8": 0.5595626443779531, |
| "pass@16": 0.625599140698911, |
| "pass@32": 0.6790160770546658, |
| "pass@64": 0.7283353115816313, |
| "pass@128": 0.7763975155279503 |
| }, |
| "18": { |
| "pass@1": 0.23963694852941175, |
| "pass@2": 0.3307752431681334, |
| "pass@4": 0.42159714759919714, |
| "pass@8": 0.5055019564772302, |
| "pass@16": 0.5780629837271026, |
| "pass@32": 0.6386943668162116, |
| "pass@64": 0.6911164787819537, |
| "pass@128": 0.7352941176470589 |
| }, |
| "19": { |
| "pass@1": 0.17643229166666666, |
| "pass@2": 0.24522813554555678, |
| "pass@4": 0.3185077649334011, |
| "pass@8": 0.3868667149725815, |
| "pass@16": 0.4454668435699914, |
| "pass@32": 0.4963013158912973, |
| "pass@64": 0.5420733465747838, |
| "pass@128": 0.5833333333333334 |
| }, |
| "20": { |
| "pass@1": 0.1466364146706587, |
| "pass@2": 0.2122326459286152, |
| "pass@4": 0.2832578821547007, |
| "pass@8": 0.35382239266907534, |
| "pass@16": 0.41688701623004404, |
| "pass@32": 0.46919719938448934, |
| "pass@64": 0.5122032313480497, |
| "pass@128": 0.5508982035928144 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 577496, |
| "answer_accuracy": 0.7208320019172392, |
| "avg_response_len": 288.8812684733983, |
| "resp_tokens_sum": 231437806, |
| "pass_at_k": { |
| "pass@1": 0.7208320019172392, |
| "pass@2": 0.7610648540117928, |
| "pass@4": 0.7956922751747378, |
| "pass@8": 0.824719333058601, |
| "pass@16": 0.8483699736521619, |
| "pass@32": 0.8678111148940243, |
| "pass@64": 0.8841115817330713, |
| "pass@128": 0.8980667838312829 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9563199626865672, |
| "pass@2": 0.9663844899518158, |
| "pass@4": 0.9732305133500102, |
| "pass@8": 0.9785252509761181, |
| "pass@16": 0.9830552767950438, |
| "pass@32": 0.9872244258703623, |
| "pass@64": 0.9910618926107597, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.996920955882353, |
| "pass@2": 0.999296911185734, |
| "pass@4": 0.999915258386819, |
| "pass@8": 0.9999981750560808, |
| "pass@16": 0.9999999995322573, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9986691640378549, |
| "pass@2": 0.9994651816985022, |
| "pass@4": 0.9999134103702337, |
| "pass@8": 0.9999980443690937, |
| "pass@16": 0.9999999994983201, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9996785996835443, |
| "pass@2": 0.9999937705571613, |
| "pass@4": 0.9999999955503979, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9931024774774775, |
| "pass@2": 0.9966164491262917, |
| "pass@4": 0.998535216318681, |
| "pass@8": 0.9994952286242749, |
| "pass@16": 0.9999277542615488, |
| "pass@32": 0.9999991007812299, |
| "pass@64": 0.9999999999952028, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9935942524115756, |
| "pass@2": 0.9967580607387904, |
| "pass@4": 0.9984507620831962, |
| "pass@8": 0.9995185509250936, |
| "pass@16": 0.9999447697863701, |
| "pass@32": 0.9999995445205452, |
| "pass@64": 0.9999999999992755, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.933531746031746, |
| "pass@2": 0.9515041088613923, |
| "pass@4": 0.9643743076758263, |
| "pass@8": 0.9733412595679051, |
| "pass@16": 0.9793917714194672, |
| "pass@32": 0.9834661512791815, |
| "pass@64": 0.9856061741231261, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.8517297400611621, |
| "pass@2": 0.892629999518409, |
| "pass@4": 0.9156304034243424, |
| "pass@8": 0.9315569919301774, |
| "pass@16": 0.9442163881132603, |
| "pass@32": 0.9550475627343186, |
| "pass@64": 0.9651791869569815, |
| "pass@128": 0.9724770642201835 |
| }, |
| "13": { |
| "pass@1": 0.6634355709876543, |
| "pass@2": 0.7349605387868178, |
| "pass@4": 0.7827549337004169, |
| "pass@8": 0.8185453779861791, |
| "pass@16": 0.8474015318736398, |
| "pass@32": 0.8702366775562574, |
| "pass@64": 0.8869076964727599, |
| "pass@128": 0.8981481481481481 |
| }, |
| "14": { |
| "pass@1": 0.6352698442136498, |
| "pass@2": 0.7258190173835839, |
| "pass@4": 0.7881565101098262, |
| "pass@8": 0.828713185282778, |
| "pass@16": 0.8561651647024184, |
| "pass@32": 0.8781020595086385, |
| "pass@64": 0.8971976264132405, |
| "pass@128": 0.913946587537092 |
| }, |
| "15": { |
| "pass@1": 0.5290075231481481, |
| "pass@2": 0.6366762205939533, |
| "pass@4": 0.7172502033889275, |
| "pass@8": 0.7746811978937012, |
| "pass@16": 0.8138282124679664, |
| "pass@32": 0.8408839816842546, |
| "pass@64": 0.8593529920260841, |
| "pass@128": 0.8734567901234568 |
| }, |
| "16": { |
| "pass@1": 0.3200129056047198, |
| "pass@2": 0.419621600120781, |
| "pass@4": 0.510766843856907, |
| "pass@8": 0.587493043139971, |
| "pass@16": 0.6510022787317294, |
| "pass@32": 0.7076012028954857, |
| "pass@64": 0.7594490626705939, |
| "pass@128": 0.8053097345132744 |
| }, |
| "17": { |
| "pass@1": 0.2574124623493976, |
| "pass@2": 0.35109927900578686, |
| "pass@4": 0.4391238452121193, |
| "pass@8": 0.5116352714481722, |
| "pass@16": 0.5700288921857198, |
| "pass@32": 0.6232034647147692, |
| "pass@64": 0.6743981124495337, |
| "pass@128": 0.7198795180722891 |
| }, |
| "18": { |
| "pass@1": 0.22880544354838708, |
| "pass@2": 0.3054177514605029, |
| "pass@4": 0.3843950726925263, |
| "pass@8": 0.46068492782560816, |
| "pass@16": 0.5260139043388721, |
| "pass@32": 0.5775948095148585, |
| "pass@64": 0.6185270051584428, |
| "pass@128": 0.6548387096774193 |
| }, |
| "19": { |
| "pass@1": 0.16887745468277945, |
| "pass@2": 0.2465554898065991, |
| "pass@4": 0.33081693168263315, |
| "pass@8": 0.4119898536696145, |
| "pass@16": 0.4832835378374356, |
| "pass@32": 0.542532246435064, |
| "pass@64": 0.5936834175175014, |
| "pass@128": 0.6404833836858006 |
| }, |
| "20": { |
| "pass@1": 0.1543262012012012, |
| "pass@2": 0.22396535314842403, |
| "pass@4": 0.30279329132281885, |
| "pass@8": 0.38295495014077957, |
| "pass@16": 0.4554402145987438, |
| "pass@32": 0.513757434375952, |
| "pass@64": 0.5580152117413981, |
| "pass@128": 0.5945945945945946 |
| } |
| } |
| } |
| } |
| } |
| ] |
