| [ |
| { |
| "checkpoint": "checkpoint-22350", |
| "id_acc": 0.9704357638888889, |
| "ood_acc": 0.353303125, |
| "total_acc": 0.6456291118421053, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 196.0621032986111, |
| "ood_avg_resp_len": 308.236484375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 519278, |
| "answer_accuracy": 0.6412993005058488, |
| "avg_response_len": 242.80526176691433, |
| "resp_tokens_sum": 196606219, |
| "pass_at_k": { |
| "pass@1": 0.6412993005058488, |
| "pass@2": 0.669765941427828, |
| "pass@4": 0.6926322673248675, |
| "pass@8": 0.7119556252626574, |
| "pass@16": 0.7288212599295653, |
| "pass@32": 0.7440110633943208, |
| "pass@64": 0.7579828058690048, |
| "pass@128": 0.7701549162187796 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9232577433628318, |
| "pass@2": 0.9413916422781228, |
| "pass@4": 0.9499321857444808, |
| "pass@8": 0.9535468343450594, |
| "pass@16": 0.9556178797377718, |
| "pass@32": 0.9578424969991439, |
| "pass@64": 0.9603845492642211, |
| "pass@128": 0.9616519174041298 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9871337890625, |
| "pass@2": 0.9921521284448819, |
| "pass@4": 0.9948190980033745, |
| "pass@8": 0.9961782344257655, |
| "pass@16": 0.9967684498096553, |
| "pass@32": 0.9968730556584248, |
| "pass@64": 0.9968749999667598, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9740575396825397, |
| "pass@2": 0.9775235126859141, |
| "pass@4": 0.979264063123062, |
| "pass@8": 0.980565740435064, |
| "pass@16": 0.9821661054021831, |
| "pass@32": 0.9838715548767617, |
| "pass@64": 0.9855250123602817, |
| "pass@128": 0.9873015873015873 |
| }, |
| "6": { |
| "pass@1": 0.9662939745508982, |
| "pass@2": 0.9776823076288367, |
| "pass@4": 0.9837688846154713, |
| "pass@8": 0.987005164857485, |
| "pass@16": 0.9892880596338539, |
| "pass@32": 0.991436482518866, |
| "pass@64": 0.993178757094046, |
| "pass@128": 0.9940119760479041 |
| }, |
| "7": { |
| "pass@1": 0.9721015162721893, |
| "pass@2": 0.9787191911661931, |
| "pass@4": 0.9822101936777133, |
| "pass@8": 0.9845253393977582, |
| "pass@16": 0.986677006424018, |
| "pass@32": 0.98918608445364, |
| "pass@64": 0.9918771028850546, |
| "pass@128": 0.9940828402366864 |
| }, |
| "8": { |
| "pass@1": 0.9414563301282052, |
| "pass@2": 0.9573518416868564, |
| "pass@4": 0.9671929527438877, |
| "pass@8": 0.9740068690727082, |
| "pass@16": 0.978930348505969, |
| "pass@32": 0.9823055554325953, |
| "pass@64": 0.9838388835534627, |
| "pass@128": 0.9839743589743589 |
| }, |
| "9": { |
| "pass@1": 0.9477112676056338, |
| "pass@2": 0.966034642896751, |
| "pass@4": 0.974834940878869, |
| "pass@8": 0.979053728156307, |
| "pass@16": 0.9807582382316763, |
| "pass@32": 0.9815156079864068, |
| "pass@64": 0.9823999112442319, |
| "pass@128": 0.9830985915492958 |
| }, |
| "11": { |
| "pass@1": 0.80875973796034, |
| "pass@2": 0.8375635720818182, |
| "pass@4": 0.8604180507004612, |
| "pass@8": 0.8809243943571061, |
| "pass@16": 0.8994562696166066, |
| "pass@32": 0.9151986207448092, |
| "pass@64": 0.9273850927591833, |
| "pass@128": 0.9376770538243626 |
| }, |
| "12": { |
| "pass@1": 0.7334701624293786, |
| "pass@2": 0.7817570704880107, |
| "pass@4": 0.8171735532528774, |
| "pass@8": 0.8454429859144871, |
| "pass@16": 0.8691713238808132, |
| "pass@32": 0.8891811697633311, |
| "pass@64": 0.906188340693502, |
| "pass@128": 0.9180790960451978 |
| }, |
| "13": { |
| "pass@1": 0.5085961990595611, |
| "pass@2": 0.5776671994174711, |
| "pass@4": 0.6343787514413363, |
| "pass@8": 0.6803702374386639, |
| "pass@16": 0.7154196073579733, |
| "pass@32": 0.7422161611800479, |
| "pass@64": 0.76511848727167, |
| "pass@128": 0.786833855799373 |
| }, |
| "14": { |
| "pass@1": 0.30926304517133957, |
| "pass@2": 0.3654867141315279, |
| "pass@4": 0.4139567650422203, |
| "pass@8": 0.45759488846050517, |
| "pass@16": 0.49837696805182397, |
| "pass@32": 0.5345293388750569, |
| "pass@64": 0.5677636542177898, |
| "pass@128": 0.6012461059190031 |
| }, |
| "15": { |
| "pass@1": 0.2457550578034682, |
| "pass@2": 0.29157006679259007, |
| "pass@4": 0.33352885432312285, |
| "pass@8": 0.37290426380904296, |
| "pass@16": 0.4105600444996007, |
| "pass@32": 0.4447149669473933, |
| "pass@64": 0.4736651101410065, |
| "pass@128": 0.49710982658959535 |
| }, |
| "16": { |
| "pass@1": 0.22184979838709679, |
| "pass@2": 0.26663068326136635, |
| "pass@4": 0.30263250511024836, |
| "pass@8": 0.3322350800902826, |
| "pass@16": 0.3567520519571173, |
| "pass@32": 0.3789453955002686, |
| "pass@64": 0.4006622686591528, |
| "pass@128": 0.42258064516129035 |
| }, |
| "17": { |
| "pass@1": 0.1810196893063584, |
| "pass@2": 0.22069610224841837, |
| "pass@4": 0.2561653288642387, |
| "pass@8": 0.2871249636415423, |
| "pass@16": 0.31473971022818265, |
| "pass@32": 0.3398529280723456, |
| "pass@64": 0.3637549328856826, |
| "pass@128": 0.38439306358381503 |
| }, |
| "18": { |
| "pass@1": 0.1610044642857143, |
| "pass@2": 0.19778894825646798, |
| "pass@4": 0.23138643687396207, |
| "pass@8": 0.26264321026468734, |
| "pass@16": 0.29186077506817065, |
| "pass@32": 0.32167058957108174, |
| "pass@64": 0.3512736526766329, |
| "pass@128": 0.3742857142857143 |
| }, |
| "19": { |
| "pass@1": 0.16532939189189189, |
| "pass@2": 0.2147416855832209, |
| "pass@4": 0.2589971599833805, |
| "pass@8": 0.296842774153626, |
| "pass@16": 0.32952997822341357, |
| "pass@32": 0.3608921725080611, |
| "pass@64": 0.3914425486882824, |
| "pass@128": 0.4144144144144144 |
| }, |
| "20": { |
| "pass@1": 0.17171077327327328, |
| "pass@2": 0.2125411582842685, |
| "pass@4": 0.24931663215521463, |
| "pass@8": 0.28445708954372556, |
| "pass@16": 0.31796378367059186, |
| "pass@32": 0.3504592271011099, |
| "pass@64": 0.3830969877606695, |
| "pass@128": 0.4174174174174174 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 531958, |
| "answer_accuracy": 0.6478444076383476, |
| "avg_response_len": 252.07089584957131, |
| "resp_tokens_sum": 206980454, |
| "pass_at_k": { |
| "pass@1": 0.6478444076383476, |
| "pass@2": 0.6735689605439996, |
| "pass@4": 0.6949296075324933, |
| "pass@8": 0.7136456940962927, |
| "pass@16": 0.7308214637042161, |
| "pass@32": 0.7465964425889449, |
| "pass@64": 0.7609117522322542, |
| "pass@128": 0.7742790335151988 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8846098542944786, |
| "pass@2": 0.9002780662769918, |
| "pass@4": 0.9112371256507044, |
| "pass@8": 0.920703275994343, |
| "pass@16": 0.928180327350798, |
| "pass@32": 0.9340597884127482, |
| "pass@64": 0.9389806489566199, |
| "pass@128": 0.941717791411043 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9972064393939394, |
| "pass@2": 0.9974264047959914, |
| "pass@4": 0.9978202057129222, |
| "pass@8": 0.9984495141444165, |
| "pass@16": 0.9992439943304724, |
| "pass@32": 0.9998493181625818, |
| "pass@64": 0.9999979765592963, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9979428624260355, |
| "pass@2": 0.9990397719330942, |
| "pass@4": 0.9997296856938741, |
| "pass@8": 0.999977983651016, |
| "pass@16": 0.999999898523723, |
| "pass@32": 0.9999999999996828, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9704483695652174, |
| "pass@2": 0.9743309939518432, |
| "pass@4": 0.9780603405552569, |
| "pass@8": 0.9814549315371892, |
| "pass@16": 0.9847236492950605, |
| "pass@32": 0.9876855114215582, |
| "pass@64": 0.9896688751350827, |
| "pass@128": 0.991304347826087 |
| }, |
| "6": { |
| "pass@1": 0.9924785100286533, |
| "pass@2": 0.9948129357218599, |
| "pass@4": 0.9960934836583821, |
| "pass@8": 0.9969392940630192, |
| "pass@16": 0.9974515227801705, |
| "pass@32": 0.9978507255774547, |
| "pass@64": 0.9985673352433165, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9881909320809249, |
| "pass@2": 0.9905351087797553, |
| "pass@4": 0.9912523347876314, |
| "pass@8": 0.9913285922353743, |
| "pass@16": 0.9913294796978018, |
| "pass@32": 0.9913294797687862, |
| "pass@64": 0.9913294797687862, |
| "pass@128": 0.9913294797687862 |
| }, |
| "8": { |
| "pass@1": 0.968794014084507, |
| "pass@2": 0.9768676527669956, |
| "pass@4": 0.9810858080239967, |
| "pass@8": 0.9833780732591677, |
| "pass@16": 0.9854529291572443, |
| "pass@32": 0.9874980186150435, |
| "pass@64": 0.9886114136046574, |
| "pass@128": 0.9887323943661972 |
| }, |
| "9": { |
| "pass@1": 0.9583645209580839, |
| "pass@2": 0.9689787501768117, |
| "pass@4": 0.9740201422239886, |
| "pass@8": 0.9778193850824559, |
| "pass@16": 0.9817966272607928, |
| "pass@32": 0.9852710419406909, |
| "pass@64": 0.9874857148809907, |
| "pass@128": 0.9880239520958084 |
| }, |
| "11": { |
| "pass@1": 0.8485739834337349, |
| "pass@2": 0.8717270657432884, |
| "pass@4": 0.8875434908927042, |
| "pass@8": 0.900419181851555, |
| "pass@16": 0.911604703248638, |
| "pass@32": 0.9220797702645877, |
| "pass@64": 0.9313253131337487, |
| "pass@128": 0.9397590361445783 |
| }, |
| "12": { |
| "pass@1": 0.7480162617554859, |
| "pass@2": 0.7935103392738131, |
| "pass@4": 0.8251117530763199, |
| "pass@8": 0.8481994805867784, |
| "pass@16": 0.8673263961259938, |
| "pass@32": 0.8842887166756281, |
| "pass@64": 0.8996039865796801, |
| "pass@128": 0.9153605015673981 |
| }, |
| "13": { |
| "pass@1": 0.4614189425770308, |
| "pass@2": 0.5181631707801232, |
| "pass@4": 0.5654716180435426, |
| "pass@8": 0.6084657762411573, |
| "pass@16": 0.650034763496025, |
| "pass@32": 0.6886026576493296, |
| "pass@64": 0.7222463189761011, |
| "pass@128": 0.7535014005602241 |
| }, |
| "14": { |
| "pass@1": 0.38966557017543857, |
| "pass@2": 0.43866052516461757, |
| "pass@4": 0.4775347327746312, |
| "pass@8": 0.5106014603664322, |
| "pass@16": 0.5421969559925761, |
| "pass@32": 0.5743889726779975, |
| "pass@64": 0.6071513767259988, |
| "pass@128": 0.6432748538011696 |
| }, |
| "15": { |
| "pass@1": 0.25414299242424243, |
| "pass@2": 0.3020221904080171, |
| "pass@4": 0.34498625342286765, |
| "pass@8": 0.38554057450292495, |
| "pass@16": 0.42102806619022154, |
| "pass@32": 0.4483743656957863, |
| "pass@64": 0.46978329116358775, |
| "pass@128": 0.4909090909090909 |
| }, |
| "16": { |
| "pass@1": 0.19816150284900286, |
| "pass@2": 0.2385198645041165, |
| "pass@4": 0.2712453993464492, |
| "pass@8": 0.3002115489394276, |
| "pass@16": 0.33021755688763793, |
| "pass@32": 0.3629484109481779, |
| "pass@64": 0.39703810753951246, |
| "pass@128": 0.4301994301994302 |
| }, |
| "17": { |
| "pass@1": 0.19232822204968944, |
| "pass@2": 0.24024803455274613, |
| "pass@4": 0.28360422241474476, |
| "pass@8": 0.32167776261409275, |
| "pass@16": 0.3578691530927828, |
| "pass@32": 0.3935301291607172, |
| "pass@64": 0.4269889693410022, |
| "pass@128": 0.45652173913043476 |
| }, |
| "18": { |
| "pass@1": 0.1627297794117647, |
| "pass@2": 0.21125484888837412, |
| "pass@4": 0.2557750033084101, |
| "pass@8": 0.294215639407568, |
| "pass@16": 0.32724684342872096, |
| "pass@32": 0.3559548424763023, |
| "pass@64": 0.382033785221312, |
| "pass@128": 0.4088235294117647 |
| }, |
| "19": { |
| "pass@1": 0.14957682291666666, |
| "pass@2": 0.19273652805118105, |
| "pass@4": 0.23311154353473673, |
| "pass@8": 0.27019836298649635, |
| "pass@16": 0.3039869333375479, |
| "pass@32": 0.3345311995737657, |
| "pass@64": 0.36212989556933894, |
| "pass@128": 0.38392857142857145 |
| }, |
| "20": { |
| "pass@1": 0.13704622005988024, |
| "pass@2": 0.18041611167428911, |
| "pass@4": 0.22258806767791758, |
| "pass@8": 0.26292508542741727, |
| "pass@16": 0.299340227526878, |
| "pass@32": 0.330316341657262, |
| "pass@64": 0.35724850294988825, |
| "pass@128": 0.38023952095808383 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 518934, |
| "answer_accuracy": 0.6477347619428023, |
| "avg_response_len": 270.6347484622144, |
| "resp_tokens_sum": 216819570, |
| "pass_at_k": { |
| "pass@1": 0.6477347619428023, |
| "pass@2": 0.6729965542532144, |
| "pass@4": 0.694529736541958, |
| "pass@8": 0.7137905490411974, |
| "pass@16": 0.7312837376191367, |
| "pass@32": 0.7469832429372135, |
| "pass@64": 0.7611960808325885, |
| "pass@128": 0.7737657772807157 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.8892957089552239, |
| "pass@2": 0.9057556704665646, |
| "pass@4": 0.9162532422253183, |
| "pass@8": 0.9243341199575077, |
| "pass@16": 0.9307240765602192, |
| "pass@32": 0.9350739008984293, |
| "pass@64": 0.9379850574461723, |
| "pass@128": 0.9402985074626866 |
| }, |
| "2": { |
| "pass@1": 0.9997098214285715, |
| "pass@2": 0.999972581552306, |
| "pass@4": 0.9999998085060796, |
| "pass@8": 0.9999999999974281, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9971984329446064, |
| "pass@2": 0.9973955344229929, |
| "pass@4": 0.9976782940397756, |
| "pass@8": 0.9981652294469208, |
| "pass@16": 0.9988826259348237, |
| "pass@32": 0.9996323772550161, |
| "pass@64": 0.9999808398763494, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9945860745614035, |
| "pass@2": 0.9950729842980154, |
| "pass@4": 0.9958536131448481, |
| "pass@8": 0.9970718402735894, |
| "pass@16": 0.9985582880872056, |
| "pass@32": 0.9996730624062373, |
| "pass@64": 0.9999907663284503, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9663143382352941, |
| "pass@2": 0.9694860178323298, |
| "pass@4": 0.9722351474448048, |
| "pass@8": 0.9755392992634085, |
| "pass@16": 0.9793933800658496, |
| "pass@32": 0.9823811114856312, |
| "pass@64": 0.9838078008574993, |
| "pass@128": 0.9852941176470589 |
| }, |
| "6": { |
| "pass@1": 0.9973629731861199, |
| "pass@2": 0.9984681220348246, |
| "pass@4": 0.9992779798424248, |
| "pass@8": 0.9998441685791019, |
| "pass@16": 0.9999940349535459, |
| "pass@32": 0.9999999966955364, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9751038370253164, |
| "pass@2": 0.9806318835343366, |
| "pass@4": 0.9847448105774753, |
| "pass@8": 0.9874530271372075, |
| "pass@16": 0.9892156409124943, |
| "pass@32": 0.9902597707262977, |
| "pass@64": 0.9905017189678624, |
| "pass@128": 0.990506329113924 |
| }, |
| "8": { |
| "pass@1": 0.9673423423423423, |
| "pass@2": 0.9762660051784069, |
| "pass@4": 0.9823950601332488, |
| "pass@8": 0.9866693202134098, |
| "pass@16": 0.9895049053205298, |
| "pass@32": 0.9919861992734489, |
| "pass@64": 0.9937088592336957, |
| "pass@128": 0.993993993993994 |
| }, |
| "9": { |
| "pass@1": 0.9417956189710611, |
| "pass@2": 0.9539905720687647, |
| "pass@4": 0.9617673268132481, |
| "pass@8": 0.9670705477659864, |
| "pass@16": 0.9716396337451848, |
| "pass@32": 0.9758638941571824, |
| "pass@64": 0.9788092784862537, |
| "pass@128": 0.9807073954983923 |
| }, |
| "11": { |
| "pass@1": 0.8631200396825397, |
| "pass@2": 0.8873668135233092, |
| "pass@4": 0.9023296802185442, |
| "pass@8": 0.9139362203436046, |
| "pass@16": 0.9238145883317608, |
| "pass@32": 0.9313328902460989, |
| "pass@64": 0.9375069285089583, |
| "pass@128": 0.9428571428571428 |
| }, |
| "12": { |
| "pass@1": 0.7282826834862385, |
| "pass@2": 0.7779322882804786, |
| "pass@4": 0.8170610029824253, |
| "pass@8": 0.8498968778339464, |
| "pass@16": 0.8772922076732741, |
| "pass@32": 0.8990817030643646, |
| "pass@64": 0.9153620294937252, |
| "pass@128": 0.926605504587156 |
| }, |
| "13": { |
| "pass@1": 0.5295138888888888, |
| "pass@2": 0.5883043373918536, |
| "pass@4": 0.6362882339360357, |
| "pass@8": 0.6759673449866654, |
| "pass@16": 0.7097663096176011, |
| "pass@32": 0.7375562398906225, |
| "pass@64": 0.7616677793162406, |
| "pass@128": 0.7808641975308642 |
| }, |
| "14": { |
| "pass@1": 0.3577058605341246, |
| "pass@2": 0.40876794726512283, |
| "pass@4": 0.4565106505603714, |
| "pass@8": 0.4996678465065184, |
| "pass@16": 0.5361346834509522, |
| "pass@32": 0.5672766953822151, |
| "pass@64": 0.5954790295606865, |
| "pass@128": 0.6172106824925816 |
| }, |
| "15": { |
| "pass@1": 0.23336226851851852, |
| "pass@2": 0.28335064887722383, |
| "pass@4": 0.3287279220305795, |
| "pass@8": 0.37229036077603983, |
| "pass@16": 0.4153575676276682, |
| "pass@32": 0.4545569214715594, |
| "pass@64": 0.48957127857625266, |
| "pass@128": 0.5216049382716049 |
| }, |
| "16": { |
| "pass@1": 0.22121589233038347, |
| "pass@2": 0.26422882842078366, |
| "pass@4": 0.29995856214875805, |
| "pass@8": 0.33195147122739216, |
| "pass@16": 0.3610778167272766, |
| "pass@32": 0.3877967539917674, |
| "pass@64": 0.4144776267893019, |
| "pass@128": 0.4424778761061947 |
| }, |
| "17": { |
| "pass@1": 0.16177993222891565, |
| "pass@2": 0.20444165935395123, |
| "pass@4": 0.24705405621360577, |
| "pass@8": 0.288054495373893, |
| "pass@16": 0.3239910809999265, |
| "pass@32": 0.3561570186864724, |
| "pass@64": 0.38991113117665654, |
| "pass@128": 0.4246987951807229 |
| }, |
| "18": { |
| "pass@1": 0.16353326612903227, |
| "pass@2": 0.19809975869951738, |
| "pass@4": 0.2304538341981446, |
| "pass@8": 0.26223427015498135, |
| "pass@16": 0.2945088081050385, |
| "pass@32": 0.32849514432266463, |
| "pass@64": 0.3624810386483276, |
| "pass@128": 0.3935483870967742 |
| }, |
| "19": { |
| "pass@1": 0.14520392749244712, |
| "pass@2": 0.18464128327425824, |
| "pass@4": 0.21987888617397153, |
| "pass@8": 0.2509003220465607, |
| "pass@16": 0.2791341129996524, |
| "pass@32": 0.30528950755645506, |
| "pass@64": 0.3276331403258758, |
| "pass@128": 0.34441087613293053 |
| }, |
| "20": { |
| "pass@1": 0.15343468468468469, |
| "pass@2": 0.193956382563666, |
| "pass@4": 0.23039945232071216, |
| "pass@8": 0.26507179724896024, |
| "pass@16": 0.3007939940823713, |
| "pass@32": 0.3370014674190456, |
| "pass@64": 0.3718108648660131, |
| "pass@128": 0.40540540540540543 |
| } |
| } |
| } |
| } |
| } |
| ] |
