| [ |
| { |
| "checkpoint": "checkpoint-42828", |
| "id_acc": 0.9894826388888889, |
| "ood_acc": 0.470528125, |
| "total_acc": 0.7163486842105263, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.09417708333334, |
| "ood_avg_resp_len": 344.22949609375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 574771, |
| "answer_accuracy": 0.7098321905627569, |
| "avg_response_len": 262.7198664242807, |
| "resp_tokens_sum": 212731632, |
| "pass_at_k": { |
| "pass@1": 0.7098321905627569, |
| "pass@2": 0.7544343266197929, |
| "pass@4": 0.7921442557488494, |
| "pass@8": 0.8234180639046024, |
| "pass@16": 0.8490259583479542, |
| "pass@32": 0.8706260639313723, |
| "pass@64": 0.8894478974698937, |
| "pass@128": 0.9054694909895669 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.938191371681416, |
| "pass@2": 0.9507075581260308, |
| "pass@4": 0.9574170305592333, |
| "pass@8": 0.9626028420434352, |
| "pass@16": 0.9676380108556495, |
| "pass@32": 0.9733761722067171, |
| "pass@64": 0.9794348064762847, |
| "pass@128": 0.9852507374631269 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999044342507645, |
| "pass@2": 0.9999988712706783, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9985107421875, |
| "pass@2": 0.999341781496063, |
| "pass@4": 0.9998666789698163, |
| "pass@8": 0.9999951531193496, |
| "pass@16": 0.9999999963269612, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.994469246031746, |
| "pass@2": 0.9972573740782403, |
| "pass@4": 0.9988838163086756, |
| "pass@8": 0.999703452073703, |
| "pass@16": 0.999977476461648, |
| "pass@32": 0.9999999361951407, |
| "pass@64": 0.999999999999996, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9982924775449101, |
| "pass@2": 0.9991354643076052, |
| "pass@4": 0.9997573747330986, |
| "pass@8": 0.9999824898608188, |
| "pass@16": 0.9999999386330103, |
| "pass@32": 0.9999999999999043, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.993805473372781, |
| "pass@2": 0.9953204351674976, |
| "pass@4": 0.9963636679290828, |
| "pass@8": 0.9978022115058525, |
| "pass@16": 0.9992398726779238, |
| "pass@32": 0.9999300060627638, |
| "pass@64": 0.9999998828759263, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9918619791666666, |
| "pass@2": 0.9953259861447606, |
| "pass@4": 0.9971175333852498, |
| "pass@8": 0.9986976735909107, |
| "pass@16": 0.9997214713252663, |
| "pass@32": 0.9999882289469381, |
| "pass@64": 0.999999996955866, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9906029929577465, |
| "pass@2": 0.9971175973161807, |
| "pass@4": 0.9994117220382662, |
| "pass@8": 0.9999626653484882, |
| "pass@16": 0.9999997920087084, |
| "pass@32": 0.9999999999982944, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.91983888101983, |
| "pass@2": 0.946985972318262, |
| "pass@4": 0.9615740933410231, |
| "pass@8": 0.9704805355870788, |
| "pass@16": 0.9768131739959114, |
| "pass@32": 0.9819557012317468, |
| "pass@64": 0.98653133809928, |
| "pass@128": 0.9915014164305949 |
| }, |
| "12": { |
| "pass@1": 0.822607697740113, |
| "pass@2": 0.8753218281507184, |
| "pass@4": 0.9018579549907951, |
| "pass@8": 0.9178689747222858, |
| "pass@16": 0.9318606981543764, |
| "pass@32": 0.9459486582670955, |
| "pass@64": 0.9575915981731089, |
| "pass@128": 0.963276836158192 |
| }, |
| "13": { |
| "pass@1": 0.6809855015673981, |
| "pass@2": 0.7670173000024677, |
| "pass@4": 0.8201065466346483, |
| "pass@8": 0.8561603281573825, |
| "pass@16": 0.882208233859429, |
| "pass@32": 0.899328313039419, |
| "pass@64": 0.9099842471218675, |
| "pass@128": 0.9216300940438872 |
| }, |
| "14": { |
| "pass@1": 0.5971816588785047, |
| "pass@2": 0.6971129375475261, |
| "pass@4": 0.7666536472660535, |
| "pass@8": 0.8149757096073096, |
| "pass@16": 0.8522813198771078, |
| "pass@32": 0.8827592151983151, |
| "pass@64": 0.907156078560458, |
| "pass@128": 0.9252336448598131 |
| }, |
| "15": { |
| "pass@1": 0.49001986994219654, |
| "pass@2": 0.5955823158936784, |
| "pass@4": 0.682237448552312, |
| "pass@8": 0.7510216556505724, |
| "pass@16": 0.8021241041232691, |
| "pass@32": 0.840262532442047, |
| "pass@64": 0.871512978537945, |
| "pass@128": 0.8988439306358381 |
| }, |
| "16": { |
| "pass@1": 0.3142641129032258, |
| "pass@2": 0.42235323533147057, |
| "pass@4": 0.5219397541033175, |
| "pass@8": 0.6054150479280793, |
| "pass@16": 0.6731704410957603, |
| "pass@32": 0.7300474628311022, |
| "pass@64": 0.7812425186016846, |
| "pass@128": 0.8258064516129032 |
| }, |
| "17": { |
| "pass@1": 0.2594608020231214, |
| "pass@2": 0.3552075747576351, |
| "pass@4": 0.44889092467126573, |
| "pass@8": 0.5298944076343943, |
| "pass@16": 0.5938477057646299, |
| "pass@32": 0.6458432672898637, |
| "pass@64": 0.6926852959730377, |
| "pass@128": 0.7312138728323699 |
| }, |
| "18": { |
| "pass@1": 0.20560267857142858, |
| "pass@2": 0.2975752249718784, |
| "pass@4": 0.3935471034870643, |
| "pass@8": 0.479720452534764, |
| "pass@16": 0.5518197621634817, |
| "pass@32": 0.6137227694722202, |
| "pass@64": 0.6675575336255497, |
| "pass@128": 0.7114285714285714 |
| }, |
| "19": { |
| "pass@1": 0.16469594594594594, |
| "pass@2": 0.2434220637960795, |
| "pass@4": 0.3257808131528604, |
| "pass@8": 0.4000305781451452, |
| "pass@16": 0.46376379967114273, |
| "pass@32": 0.5206510590669114, |
| "pass@64": 0.571185394124095, |
| "pass@128": 0.6126126126126126 |
| }, |
| "20": { |
| "pass@1": 0.15843186936936937, |
| "pass@2": 0.22814768508666133, |
| "pass@4": 0.30495890800812053, |
| "pass@8": 0.383387374477363, |
| "pass@16": 0.4573577428718796, |
| "pass@32": 0.5261218888926433, |
| "pass@64": 0.5905595183935609, |
| "pass@128": 0.6516516516516516 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 592767, |
| "answer_accuracy": 0.7219005748246298, |
| "avg_response_len": 273.3977908222915, |
| "resp_tokens_sum": 224492394, |
| "pass_at_k": { |
| "pass@1": 0.7219005748246298, |
| "pass@2": 0.7684856788653494, |
| "pass@4": 0.8070809353721593, |
| "pass@8": 0.8377910566802691, |
| "pass@16": 0.8615161766040658, |
| "pass@32": 0.8800848254091402, |
| "pass@64": 0.89531877294161, |
| "pass@128": 0.9078721745908028 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9266919095092024, |
| "pass@2": 0.9459019944688664, |
| "pass@4": 0.9580926648931151, |
| "pass@8": 0.9655181079663312, |
| "pass@16": 0.9705065116715573, |
| "pass@32": 0.9745265481735442, |
| "pass@64": 0.9789238933060727, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9993134469696969, |
| "pass@2": 0.9998590730136006, |
| "pass@4": 0.9999941839656407, |
| "pass@8": 0.9999999934122665, |
| "pass@16": 0.999999999999999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9967391304347826, |
| "pass@2": 0.9973970814789457, |
| "pass@4": 0.9976917420648507, |
| "pass@8": 0.9981758657979531, |
| "pass@16": 0.9988891034656363, |
| "pass@32": 0.9996345084013638, |
| "pass@64": 0.99998095094953, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9988135744985673, |
| "pass@2": 0.9997299652550595, |
| "pass@4": 0.9999778987153827, |
| "pass@8": 0.9999998766975703, |
| "pass@16": 0.9999999999988423, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9965227601156069, |
| "pass@2": 0.9971318726958264, |
| "pass@4": 0.9972870163267165, |
| "pass@8": 0.9974611417052052, |
| "pass@16": 0.9977897000591688, |
| "pass@32": 0.9983785444449502, |
| "pass@64": 0.9992831459651359, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9943221830985915, |
| "pass@2": 0.9966639403349228, |
| "pass@4": 0.9979784254080916, |
| "pass@8": 0.9990666990790485, |
| "pass@16": 0.9997528296434962, |
| "pass@32": 0.9999857246359625, |
| "pass@64": 0.9999999936816814, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9866672904191617, |
| "pass@2": 0.9917241059220143, |
| "pass@4": 0.9938411520603335, |
| "pass@8": 0.9956474719604435, |
| "pass@16": 0.9972573036700675, |
| "pass@32": 0.998297458460058, |
| "pass@64": 0.9992573828256583, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9343232304216867, |
| "pass@2": 0.9600106874347788, |
| "pass@4": 0.9705750945288463, |
| "pass@8": 0.9756324867411611, |
| "pass@16": 0.9785410759514516, |
| "pass@32": 0.9804200441280092, |
| "pass@64": 0.9815556641572076, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.8473011363636364, |
| "pass@2": 0.8946973810875521, |
| "pass@4": 0.9175820971046328, |
| "pass@8": 0.9309735895345088, |
| "pass@16": 0.9410367488667444, |
| "pass@32": 0.948965953999329, |
| "pass@64": 0.9554065505058231, |
| "pass@128": 0.9623824451410659 |
| }, |
| "13": { |
| "pass@1": 0.6850709033613446, |
| "pass@2": 0.7874598166038067, |
| "pass@4": 0.8496326567372344, |
| "pass@8": 0.8870374883664758, |
| "pass@16": 0.9118338376968488, |
| "pass@32": 0.9300755296021455, |
| "pass@64": 0.9456425249067281, |
| "pass@128": 0.957983193277311 |
| }, |
| "14": { |
| "pass@1": 0.6223958333333334, |
| "pass@2": 0.7197378493806696, |
| "pass@4": 0.7886473072444895, |
| "pass@8": 0.8338802937283072, |
| "pass@16": 0.8612657118281954, |
| "pass@32": 0.8797224978366543, |
| "pass@64": 0.8969039766342238, |
| "pass@128": 0.9152046783625731 |
| }, |
| "15": { |
| "pass@1": 0.5188920454545455, |
| "pass@2": 0.6297918903602958, |
| "pass@4": 0.7113841423231179, |
| "pass@8": 0.7704131884610596, |
| "pass@16": 0.8132045442176113, |
| "pass@32": 0.8445315913779556, |
| "pass@64": 0.8667790210259716, |
| "pass@128": 0.8818181818181818 |
| }, |
| "16": { |
| "pass@1": 0.31828703703703703, |
| "pass@2": 0.4308475362855286, |
| "pass@4": 0.5321735987595564, |
| "pass@8": 0.6133575183418821, |
| "pass@16": 0.6771684148710823, |
| "pass@32": 0.7313473883636334, |
| "pass@64": 0.7795209648610576, |
| "pass@128": 0.8176638176638177 |
| }, |
| "17": { |
| "pass@1": 0.29660811335403725, |
| "pass@2": 0.404082047488629, |
| "pass@4": 0.5058153114206997, |
| "pass@8": 0.59019338398362, |
| "pass@16": 0.6588107355969026, |
| "pass@32": 0.7140254963144298, |
| "pass@64": 0.755545829423683, |
| "pass@128": 0.7888198757763976 |
| }, |
| "18": { |
| "pass@1": 0.25089613970588237, |
| "pass@2": 0.3484751331635016, |
| "pass@4": 0.4468376891307704, |
| "pass@8": 0.5344262099305134, |
| "pass@16": 0.602894653464828, |
| "pass@32": 0.652252172748721, |
| "pass@64": 0.6898146169752933, |
| "pass@128": 0.7176470588235294 |
| }, |
| "19": { |
| "pass@1": 0.1790829613095238, |
| "pass@2": 0.2575305528215224, |
| "pass@4": 0.3426924809622011, |
| "pass@8": 0.4214982239741903, |
| "pass@16": 0.48576795980818427, |
| "pass@32": 0.5364442340319796, |
| "pass@64": 0.5787094815498106, |
| "pass@128": 0.6160714285714286 |
| }, |
| "20": { |
| "pass@1": 0.15112743263473055, |
| "pass@2": 0.22712756082323543, |
| "pass@4": 0.311789831192508, |
| "pass@8": 0.3931681982311315, |
| "pass@16": 0.4636711942826472, |
| "pass@32": 0.5235392610475729, |
| "pass@64": 0.5746665246604618, |
| "pass@128": 0.6167664670658682 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 574622, |
| "answer_accuracy": 0.7172446676785429, |
| "avg_response_len": 291.95236484462373, |
| "resp_tokens_sum": 233898221, |
| "pass_at_k": { |
| "pass@1": 0.7172446676785429, |
| "pass@2": 0.762473200009304, |
| "pass@4": 0.8005009265997003, |
| "pass@8": 0.8318729010553301, |
| "pass@16": 0.8572073607122934, |
| "pass@32": 0.8776230304314351, |
| "pass@64": 0.8943118328037649, |
| "pass@128": 0.9079725195718166 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9461054104477612, |
| "pass@2": 0.9629205106357972, |
| "pass@4": 0.9724614089283613, |
| "pass@8": 0.9784954370979474, |
| "pass@16": 0.9833977623356791, |
| "pass@32": 0.9880483413656285, |
| "pass@64": 0.9915697437769286, |
| "pass@128": 0.9940298507462687 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999543128654971, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9866957720588235, |
| "pass@2": 0.991383091130153, |
| "pass@4": 0.9954316220582722, |
| "pass@8": 0.998009907801721, |
| "pass@16": 0.9992383013303037, |
| "pass@32": 0.9998536614272977, |
| "pass@64": 0.9999980360722474, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.999728903785489, |
| "pass@2": 0.9999992237760501, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9992335838607594, |
| "pass@2": 0.9999707994866939, |
| "pass@4": 0.999999947791336, |
| "pass@8": 0.99999999999998, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9920701951951952, |
| "pass@2": 0.99714071847438, |
| "pass@4": 0.9993788327134785, |
| "pass@8": 0.9999618621335632, |
| "pass@16": 0.9999998522509821, |
| "pass@32": 0.9999999999994185, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9870880225080386, |
| "pass@2": 0.9951408493050107, |
| "pass@4": 0.9981870624290932, |
| "pass@8": 0.9994770626159226, |
| "pass@16": 0.9999345493968065, |
| "pass@32": 0.9999993363019254, |
| "pass@64": 0.9999999999980551, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9361111111111111, |
| "pass@2": 0.964721597300337, |
| "pass@4": 0.9788161914284527, |
| "pass@8": 0.9862816248572933, |
| "pass@16": 0.9897694199344508, |
| "pass@32": 0.9919058827484885, |
| "pass@64": 0.9933702121063085, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.8474053899082569, |
| "pass@2": 0.9069223464326136, |
| "pass@4": 0.9367416807990738, |
| "pass@8": 0.9538386375839678, |
| "pass@16": 0.9652243644935586, |
| "pass@32": 0.9741221163718324, |
| "pass@64": 0.9806778665797814, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.6504870756172839, |
| "pass@2": 0.737235102556625, |
| "pass@4": 0.7968340239298788, |
| "pass@8": 0.8394478736620564, |
| "pass@16": 0.8707955001207655, |
| "pass@32": 0.8962127601427606, |
| "pass@64": 0.9199947039243642, |
| "pass@128": 0.9444444444444444 |
| }, |
| "14": { |
| "pass@1": 0.5979460311572701, |
| "pass@2": 0.6981321847473071, |
| "pass@4": 0.7671013725287299, |
| "pass@8": 0.8156496766290819, |
| "pass@16": 0.8534954255336188, |
| "pass@32": 0.8846802898454802, |
| "pass@64": 0.9111804207009937, |
| "pass@128": 0.9317507418397626 |
| }, |
| "15": { |
| "pass@1": 0.5047019675925926, |
| "pass@2": 0.6032917760279963, |
| "pass@4": 0.6773077448073619, |
| "pass@8": 0.7323858349826511, |
| "pass@16": 0.7744864686864456, |
| "pass@32": 0.8079447450839151, |
| "pass@64": 0.8351820095707687, |
| "pass@128": 0.8549382716049383 |
| }, |
| "16": { |
| "pass@1": 0.32107300884955753, |
| "pass@2": 0.4297624439644157, |
| "pass@4": 0.5362244711114648, |
| "pass@8": 0.6299412145504063, |
| "pass@16": 0.7046635110365608, |
| "pass@32": 0.7583830063556092, |
| "pass@64": 0.7949665227100015, |
| "pass@128": 0.8200589970501475 |
| }, |
| "17": { |
| "pass@1": 0.252941453313253, |
| "pass@2": 0.34630216238971623, |
| "pass@4": 0.4382089559023495, |
| "pass@8": 0.5172900182449583, |
| "pass@16": 0.5816590133785813, |
| "pass@32": 0.6347724043312906, |
| "pass@64": 0.6815166946995262, |
| "pass@128": 0.7228915662650602 |
| }, |
| "18": { |
| "pass@1": 0.24987399193548387, |
| "pass@2": 0.3411508604267206, |
| "pass@4": 0.42806011889642825, |
| "pass@8": 0.5009790895393157, |
| "pass@16": 0.5617827302678425, |
| "pass@32": 0.6148278041185289, |
| "pass@64": 0.6618773335461938, |
| "pass@128": 0.7 |
| }, |
| "19": { |
| "pass@1": 0.18797205438066467, |
| "pass@2": 0.26968615445916705, |
| "pass@4": 0.3539405954165095, |
| "pass@8": 0.43323175405120734, |
| "pass@16": 0.5015421311382682, |
| "pass@32": 0.5558973363316422, |
| "pass@64": 0.5980831369200011, |
| "pass@128": 0.6374622356495468 |
| }, |
| "20": { |
| "pass@1": 0.1547954204204204, |
| "pass@2": 0.2324537874488661, |
| "pass@4": 0.32128977881143245, |
| "pass@8": 0.4114801380719811, |
| "pass@16": 0.49201842931368517, |
| "pass@32": 0.5597262016579201, |
| "pass@64": 0.6158544846756309, |
| "pass@128": 0.6606606606606606 |
| } |
| } |
| } |
| } |
| } |
| ] |
