[ { "checkpoint": "checkpoint-22350", "id_acc": 0.9704357638888889, "ood_acc": 0.353303125, "total_acc": 0.6456291118421053, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 196.0621032986111, "ood_avg_resp_len": 308.236484375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 519278, "answer_accuracy": 0.6412993005058488, "avg_response_len": 242.80526176691433, "resp_tokens_sum": 196606219, "pass_at_k": { "pass@1": 0.6412993005058488, "pass@2": 0.669765941427828, "pass@4": 0.6926322673248675, "pass@8": 0.7119556252626574, "pass@16": 0.7288212599295653, "pass@32": 0.7440110633943208, "pass@64": 0.7579828058690048, "pass@128": 0.7701549162187796 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9232577433628318, "pass@2": 0.9413916422781228, "pass@4": 0.9499321857444808, "pass@8": 0.9535468343450594, "pass@16": 0.9556178797377718, "pass@32": 0.9578424969991439, "pass@64": 0.9603845492642211, "pass@128": 0.9616519174041298 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9871337890625, "pass@2": 0.9921521284448819, "pass@4": 0.9948190980033745, "pass@8": 0.9961782344257655, "pass@16": 0.9967684498096553, "pass@32": 0.9968730556584248, "pass@64": 0.9968749999667598, "pass@128": 0.996875 }, "5": { "pass@1": 0.9740575396825397, "pass@2": 0.9775235126859141, "pass@4": 0.979264063123062, "pass@8": 0.980565740435064, "pass@16": 0.9821661054021831, "pass@32": 0.9838715548767617, "pass@64": 0.9855250123602817, "pass@128": 0.9873015873015873 }, "6": { "pass@1": 0.9662939745508982, "pass@2": 0.9776823076288367, "pass@4": 0.9837688846154713, "pass@8": 0.987005164857485, "pass@16": 0.9892880596338539, "pass@32": 0.991436482518866, "pass@64": 0.993178757094046, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.9721015162721893, "pass@2": 0.9787191911661931, "pass@4": 0.9822101936777133, "pass@8": 0.9845253393977582, "pass@16": 0.986677006424018, "pass@32": 0.98918608445364, "pass@64": 0.9918771028850546, "pass@128": 0.9940828402366864 }, "8": { "pass@1": 0.9414563301282052, "pass@2": 0.9573518416868564, "pass@4": 0.9671929527438877, "pass@8": 0.9740068690727082, "pass@16": 0.978930348505969, "pass@32": 0.9823055554325953, "pass@64": 0.9838388835534627, "pass@128": 0.9839743589743589 }, "9": { "pass@1": 0.9477112676056338, "pass@2": 0.966034642896751, "pass@4": 0.974834940878869, "pass@8": 0.979053728156307, "pass@16": 0.9807582382316763, "pass@32": 0.9815156079864068, "pass@64": 0.9823999112442319, "pass@128": 0.9830985915492958 }, "11": { "pass@1": 0.80875973796034, "pass@2": 0.8375635720818182, "pass@4": 0.8604180507004612, "pass@8": 0.8809243943571061, "pass@16": 0.8994562696166066, "pass@32": 0.9151986207448092, "pass@64": 0.9273850927591833, "pass@128": 0.9376770538243626 }, "12": { "pass@1": 0.7334701624293786, "pass@2": 0.7817570704880107, "pass@4": 0.8171735532528774, "pass@8": 0.8454429859144871, "pass@16": 0.8691713238808132, "pass@32": 0.8891811697633311, "pass@64": 0.906188340693502, "pass@128": 0.9180790960451978 }, "13": { "pass@1": 0.5085961990595611, "pass@2": 0.5776671994174711, "pass@4": 0.6343787514413363, "pass@8": 0.6803702374386639, "pass@16": 0.7154196073579733, "pass@32": 0.7422161611800479, "pass@64": 0.76511848727167, "pass@128": 0.786833855799373 }, "14": { "pass@1": 0.30926304517133957, "pass@2": 0.3654867141315279, "pass@4": 0.4139567650422203, "pass@8": 0.45759488846050517, "pass@16": 0.49837696805182397, "pass@32": 0.5345293388750569, "pass@64": 0.5677636542177898, "pass@128": 0.6012461059190031 }, "15": { "pass@1": 0.2457550578034682, "pass@2": 0.29157006679259007, "pass@4": 0.33352885432312285, "pass@8": 0.37290426380904296, "pass@16": 0.4105600444996007, "pass@32": 0.4447149669473933, "pass@64": 0.4736651101410065, "pass@128": 0.49710982658959535 }, "16": { "pass@1": 0.22184979838709679, "pass@2": 0.26663068326136635, "pass@4": 0.30263250511024836, "pass@8": 0.3322350800902826, "pass@16": 0.3567520519571173, "pass@32": 0.3789453955002686, "pass@64": 0.4006622686591528, "pass@128": 0.42258064516129035 }, "17": { "pass@1": 0.1810196893063584, "pass@2": 0.22069610224841837, "pass@4": 0.2561653288642387, "pass@8": 0.2871249636415423, "pass@16": 0.31473971022818265, "pass@32": 0.3398529280723456, "pass@64": 0.3637549328856826, "pass@128": 0.38439306358381503 }, "18": { "pass@1": 0.1610044642857143, "pass@2": 0.19778894825646798, "pass@4": 0.23138643687396207, "pass@8": 0.26264321026468734, "pass@16": 0.29186077506817065, "pass@32": 0.32167058957108174, "pass@64": 0.3512736526766329, "pass@128": 0.3742857142857143 }, "19": { "pass@1": 0.16532939189189189, "pass@2": 0.2147416855832209, "pass@4": 0.2589971599833805, "pass@8": 0.296842774153626, "pass@16": 0.32952997822341357, "pass@32": 0.3608921725080611, "pass@64": 0.3914425486882824, "pass@128": 0.4144144144144144 }, "20": { "pass@1": 0.17171077327327328, "pass@2": 0.2125411582842685, "pass@4": 0.24931663215521463, "pass@8": 0.28445708954372556, "pass@16": 0.31796378367059186, "pass@32": 0.3504592271011099, "pass@64": 0.3830969877606695, "pass@128": 0.4174174174174174 } } }, "teachers_in_school": { "count": 821120, "correct": 531958, "answer_accuracy": 0.6478444076383476, "avg_response_len": 252.07089584957131, "resp_tokens_sum": 206980454, "pass_at_k": { "pass@1": 0.6478444076383476, "pass@2": 0.6735689605439996, "pass@4": 0.6949296075324933, "pass@8": 0.7136456940962927, "pass@16": 0.7308214637042161, "pass@32": 0.7465964425889449, "pass@64": 0.7609117522322542, "pass@128": 0.7742790335151988 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8846098542944786, "pass@2": 0.9002780662769918, "pass@4": 0.9112371256507044, "pass@8": 0.920703275994343, "pass@16": 0.928180327350798, "pass@32": 0.9340597884127482, "pass@64": 0.9389806489566199, "pass@128": 0.941717791411043 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9972064393939394, "pass@2": 0.9974264047959914, "pass@4": 0.9978202057129222, "pass@8": 0.9984495141444165, "pass@16": 0.9992439943304724, "pass@32": 0.9998493181625818, "pass@64": 0.9999979765592963, "pass@128": 1.0 }, "4": { "pass@1": 0.9979428624260355, "pass@2": 0.9990397719330942, "pass@4": 0.9997296856938741, "pass@8": 0.999977983651016, "pass@16": 0.999999898523723, "pass@32": 0.9999999999996828, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9704483695652174, "pass@2": 0.9743309939518432, "pass@4": 0.9780603405552569, "pass@8": 0.9814549315371892, "pass@16": 0.9847236492950605, "pass@32": 0.9876855114215582, "pass@64": 0.9896688751350827, "pass@128": 0.991304347826087 }, "6": { "pass@1": 0.9924785100286533, "pass@2": 0.9948129357218599, "pass@4": 0.9960934836583821, "pass@8": 0.9969392940630192, "pass@16": 0.9974515227801705, "pass@32": 0.9978507255774547, "pass@64": 0.9985673352433165, "pass@128": 1.0 }, "7": { "pass@1": 0.9881909320809249, "pass@2": 0.9905351087797553, "pass@4": 0.9912523347876314, "pass@8": 0.9913285922353743, "pass@16": 0.9913294796978018, "pass@32": 0.9913294797687862, "pass@64": 0.9913294797687862, "pass@128": 0.9913294797687862 }, "8": { "pass@1": 0.968794014084507, "pass@2": 0.9768676527669956, "pass@4": 0.9810858080239967, "pass@8": 0.9833780732591677, "pass@16": 0.9854529291572443, "pass@32": 0.9874980186150435, "pass@64": 0.9886114136046574, "pass@128": 0.9887323943661972 }, "9": { "pass@1": 0.9583645209580839, "pass@2": 0.9689787501768117, "pass@4": 0.9740201422239886, "pass@8": 0.9778193850824559, "pass@16": 0.9817966272607928, "pass@32": 0.9852710419406909, "pass@64": 0.9874857148809907, "pass@128": 0.9880239520958084 }, "11": { "pass@1": 0.8485739834337349, "pass@2": 0.8717270657432884, "pass@4": 0.8875434908927042, "pass@8": 0.900419181851555, "pass@16": 0.911604703248638, "pass@32": 0.9220797702645877, "pass@64": 0.9313253131337487, "pass@128": 0.9397590361445783 }, "12": { "pass@1": 0.7480162617554859, "pass@2": 0.7935103392738131, "pass@4": 0.8251117530763199, "pass@8": 0.8481994805867784, "pass@16": 0.8673263961259938, "pass@32": 0.8842887166756281, "pass@64": 0.8996039865796801, "pass@128": 0.9153605015673981 }, "13": { "pass@1": 0.4614189425770308, "pass@2": 0.5181631707801232, "pass@4": 0.5654716180435426, "pass@8": 0.6084657762411573, "pass@16": 0.650034763496025, "pass@32": 0.6886026576493296, "pass@64": 0.7222463189761011, "pass@128": 0.7535014005602241 }, "14": { "pass@1": 0.38966557017543857, "pass@2": 0.43866052516461757, "pass@4": 0.4775347327746312, "pass@8": 0.5106014603664322, "pass@16": 0.5421969559925761, "pass@32": 0.5743889726779975, "pass@64": 0.6071513767259988, "pass@128": 0.6432748538011696 }, "15": { "pass@1": 0.25414299242424243, "pass@2": 0.3020221904080171, "pass@4": 0.34498625342286765, "pass@8": 0.38554057450292495, "pass@16": 0.42102806619022154, "pass@32": 0.4483743656957863, "pass@64": 0.46978329116358775, "pass@128": 0.4909090909090909 }, "16": { "pass@1": 0.19816150284900286, "pass@2": 0.2385198645041165, "pass@4": 0.2712453993464492, "pass@8": 0.3002115489394276, "pass@16": 0.33021755688763793, "pass@32": 0.3629484109481779, "pass@64": 0.39703810753951246, "pass@128": 0.4301994301994302 }, "17": { "pass@1": 0.19232822204968944, "pass@2": 0.24024803455274613, "pass@4": 0.28360422241474476, "pass@8": 0.32167776261409275, "pass@16": 0.3578691530927828, "pass@32": 0.3935301291607172, "pass@64": 0.4269889693410022, "pass@128": 0.45652173913043476 }, "18": { "pass@1": 0.1627297794117647, "pass@2": 0.21125484888837412, "pass@4": 0.2557750033084101, "pass@8": 0.294215639407568, "pass@16": 0.32724684342872096, "pass@32": 0.3559548424763023, "pass@64": 0.382033785221312, "pass@128": 0.4088235294117647 }, "19": { "pass@1": 0.14957682291666666, "pass@2": 0.19273652805118105, "pass@4": 0.23311154353473673, "pass@8": 0.27019836298649635, "pass@16": 0.3039869333375479, "pass@32": 0.3345311995737657, "pass@64": 0.36212989556933894, "pass@128": 0.38392857142857145 }, "20": { "pass@1": 0.13704622005988024, "pass@2": 0.18041611167428911, "pass@4": 0.22258806767791758, "pass@8": 0.26292508542741727, "pass@16": 0.299340227526878, "pass@32": 0.330316341657262, "pass@64": 0.35724850294988825, "pass@128": 0.38023952095808383 } } }, "movie_festival_awards": { "count": 801152, "correct": 518934, "answer_accuracy": 0.6477347619428023, "avg_response_len": 270.6347484622144, "resp_tokens_sum": 216819570, "pass_at_k": { "pass@1": 0.6477347619428023, "pass@2": 0.6729965542532144, "pass@4": 0.694529736541958, "pass@8": 0.7137905490411974, "pass@16": 0.7312837376191367, "pass@32": 0.7469832429372135, "pass@64": 0.7611960808325885, "pass@128": 0.7737657772807157 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8892957089552239, "pass@2": 0.9057556704665646, "pass@4": 0.9162532422253183, "pass@8": 0.9243341199575077, "pass@16": 0.9307240765602192, "pass@32": 0.9350739008984293, "pass@64": 0.9379850574461723, "pass@128": 0.9402985074626866 }, "2": { "pass@1": 0.9997098214285715, "pass@2": 0.999972581552306, "pass@4": 0.9999998085060796, "pass@8": 0.9999999999974281, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9971984329446064, "pass@2": 0.9973955344229929, "pass@4": 0.9976782940397756, "pass@8": 0.9981652294469208, "pass@16": 0.9988826259348237, "pass@32": 0.9996323772550161, "pass@64": 0.9999808398763494, "pass@128": 1.0 }, "4": { "pass@1": 0.9945860745614035, "pass@2": 0.9950729842980154, "pass@4": 0.9958536131448481, "pass@8": 0.9970718402735894, "pass@16": 0.9985582880872056, "pass@32": 0.9996730624062373, "pass@64": 0.9999907663284503, "pass@128": 1.0 }, "5": { "pass@1": 0.9663143382352941, "pass@2": 0.9694860178323298, "pass@4": 0.9722351474448048, "pass@8": 0.9755392992634085, "pass@16": 0.9793933800658496, "pass@32": 0.9823811114856312, "pass@64": 0.9838078008574993, "pass@128": 0.9852941176470589 }, "6": { "pass@1": 0.9973629731861199, "pass@2": 0.9984681220348246, "pass@4": 0.9992779798424248, "pass@8": 0.9998441685791019, "pass@16": 0.9999940349535459, "pass@32": 0.9999999966955364, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9751038370253164, "pass@2": 0.9806318835343366, "pass@4": 0.9847448105774753, "pass@8": 0.9874530271372075, "pass@16": 0.9892156409124943, "pass@32": 0.9902597707262977, "pass@64": 0.9905017189678624, "pass@128": 0.990506329113924 }, "8": { "pass@1": 0.9673423423423423, "pass@2": 0.9762660051784069, "pass@4": 0.9823950601332488, "pass@8": 0.9866693202134098, "pass@16": 0.9895049053205298, "pass@32": 0.9919861992734489, "pass@64": 0.9937088592336957, "pass@128": 0.993993993993994 }, "9": { "pass@1": 0.9417956189710611, "pass@2": 0.9539905720687647, "pass@4": 0.9617673268132481, "pass@8": 0.9670705477659864, "pass@16": 0.9716396337451848, "pass@32": 0.9758638941571824, "pass@64": 0.9788092784862537, "pass@128": 0.9807073954983923 }, "11": { "pass@1": 0.8631200396825397, "pass@2": 0.8873668135233092, "pass@4": 0.9023296802185442, "pass@8": 0.9139362203436046, "pass@16": 0.9238145883317608, "pass@32": 0.9313328902460989, "pass@64": 0.9375069285089583, "pass@128": 0.9428571428571428 }, "12": { "pass@1": 0.7282826834862385, "pass@2": 0.7779322882804786, "pass@4": 0.8170610029824253, "pass@8": 0.8498968778339464, "pass@16": 0.8772922076732741, "pass@32": 0.8990817030643646, "pass@64": 0.9153620294937252, "pass@128": 0.926605504587156 }, "13": { "pass@1": 0.5295138888888888, "pass@2": 0.5883043373918536, "pass@4": 0.6362882339360357, "pass@8": 0.6759673449866654, "pass@16": 0.7097663096176011, "pass@32": 0.7375562398906225, "pass@64": 0.7616677793162406, "pass@128": 0.7808641975308642 }, "14": { "pass@1": 0.3577058605341246, "pass@2": 0.40876794726512283, "pass@4": 0.4565106505603714, "pass@8": 0.4996678465065184, "pass@16": 0.5361346834509522, "pass@32": 0.5672766953822151, "pass@64": 0.5954790295606865, "pass@128": 0.6172106824925816 }, "15": { "pass@1": 0.23336226851851852, "pass@2": 0.28335064887722383, "pass@4": 0.3287279220305795, "pass@8": 0.37229036077603983, "pass@16": 0.4153575676276682, "pass@32": 0.4545569214715594, "pass@64": 0.48957127857625266, "pass@128": 0.5216049382716049 }, "16": { "pass@1": 0.22121589233038347, "pass@2": 0.26422882842078366, "pass@4": 0.29995856214875805, "pass@8": 0.33195147122739216, "pass@16": 0.3610778167272766, "pass@32": 0.3877967539917674, "pass@64": 0.4144776267893019, "pass@128": 0.4424778761061947 }, "17": { "pass@1": 0.16177993222891565, "pass@2": 0.20444165935395123, "pass@4": 0.24705405621360577, "pass@8": 0.288054495373893, "pass@16": 0.3239910809999265, "pass@32": 0.3561570186864724, "pass@64": 0.38991113117665654, "pass@128": 0.4246987951807229 }, "18": { "pass@1": 0.16353326612903227, "pass@2": 0.19809975869951738, "pass@4": 0.2304538341981446, "pass@8": 0.26223427015498135, "pass@16": 0.2945088081050385, "pass@32": 0.32849514432266463, "pass@64": 0.3624810386483276, "pass@128": 0.3935483870967742 }, "19": { "pass@1": 0.14520392749244712, "pass@2": 0.18464128327425824, "pass@4": 0.21987888617397153, "pass@8": 0.2509003220465607, "pass@16": 0.2791341129996524, "pass@32": 0.30528950755645506, "pass@64": 0.3276331403258758, "pass@128": 0.34441087613293053 }, "20": { "pass@1": 0.15343468468468469, "pass@2": 0.193956382563666, "pass@4": 0.23039945232071216, "pass@8": 0.26507179724896024, "pass@16": 0.3007939940823713, "pass@32": 0.3370014674190456, "pass@64": 0.3718108648660131, "pass@128": 0.40540540540540543 } } } } } ]