[ { "checkpoint": "checkpoint-21546", "id_acc": 0.9822352430555555, "ood_acc": 0.42139921875, "total_acc": 0.6870583881578948, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.35695486111112, "ood_avg_resp_len": 319.1119796875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 552886, "answer_accuracy": 0.6828045961112867, "avg_response_len": 249.1386391479608, "resp_tokens_sum": 201734532, "pass_at_k": { "pass@1": 0.6828045961112867, "pass@2": 0.7217115855449684, "pass@4": 0.7536723871913236, "pass@8": 0.7797075439923347, "pass@16": 0.800334627046473, "pass@32": 0.817100237695868, "pass@64": 0.8315237481736786, "pass@128": 0.8444514701233007 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9263228244837758, "pass@2": 0.9494500238078647, "pass@4": 0.9622405067176335, "pass@8": 0.9693425057647705, "pass@16": 0.9736827240563464, "pass@32": 0.9773741161261673, "pass@64": 0.98185566893626, "pass@128": 0.9882005899705014 }, "2": { "pass@1": 0.9999756619937694, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9965596330275229, "pass@2": 0.9981052397120085, "pass@4": 0.999307317089951, "pass@8": 0.999884609180392, "pass@16": 0.9999968325205879, "pass@32": 0.9999999991746348, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9903076171875, "pass@2": 0.9908726008858268, "pass@4": 0.9911945356439821, "pass@8": 0.9917235482142811, "pass@16": 0.9926657644356954, "pass@32": 0.9941486923509562, "pass@64": 0.9959137795275591, "pass@128": 0.996875 }, "5": { "pass@1": 0.9702628968253968, "pass@2": 0.9771036432945881, "pass@4": 0.9829424443968311, "pass@8": 0.9876832076495944, "pass@16": 0.990954409287743, "pass@32": 0.9930190209714992, "pass@64": 0.9948502012050503, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9956727170658682, "pass@2": 0.9975463688764207, "pass@4": 0.9990146021230881, "pass@8": 0.9998480009664775, "pass@16": 0.9999970866060833, "pass@32": 0.9999999996392369, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9940366124260355, "pass@2": 0.9978666047383871, "pass@4": 0.9993812003159368, "pass@8": 0.9999391517555988, "pass@16": 0.9999995060547365, "pass@32": 0.9999999999906712, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9850260416666666, "pass@2": 0.991938282354129, "pass@4": 0.9944415109169044, "pass@8": 0.9959676378945411, "pass@16": 0.996687988729117, "pass@32": 0.9967934570195686, "pass@64": 0.9967948717815591, "pass@128": 0.9967948717948718 }, "9": { "pass@1": 0.975594190140845, "pass@2": 0.9893763169568592, "pass@4": 0.9948080432727598, "pass@8": 0.9979627772031147, "pass@16": 0.9995681970104956, "pass@32": 0.9999716229169351, "pass@64": 0.9999999660012311, "pass@128": 1.0 }, "11": { "pass@1": 0.8872388456090652, "pass@2": 0.9170830452142494, "pass@4": 0.9344475201046042, "pass@8": 0.9461166366837217, "pass@16": 0.9548923315039325, "pass@32": 0.9624344015568419, "pass@64": 0.9692996308589977, "pass@128": 0.9745042492917847 }, "12": { "pass@1": 0.7989495056497176, "pass@2": 0.8576045976244497, "pass@4": 0.894183136271101, "pass@8": 0.9153254162241761, "pass@16": 0.9267540179385538, "pass@32": 0.9355352501223162, "pass@64": 0.9451007362794269, "pass@128": 0.9548022598870056 }, "13": { "pass@1": 0.661662421630094, "pass@2": 0.7403545621158639, "pass@4": 0.7929696285025494, "pass@8": 0.8331626588813359, "pass@16": 0.8660458043380581, "pass@32": 0.8937908906213489, "pass@64": 0.9188840917684389, "pass@128": 0.9435736677115988 }, "14": { "pass@1": 0.48459404205607476, "pass@2": 0.586037726592587, "pass@4": 0.6693883112741745, "pass@8": 0.7337120781768929, "pass@16": 0.7796279839990717, "pass@32": 0.8097918673125195, "pass@64": 0.8326759446166977, "pass@128": 0.8566978193146417 }, "15": { "pass@1": 0.3742097182080925, "pass@2": 0.46095314562377676, "pass@4": 0.5352501313961474, "pass@8": 0.5976363331573419, "pass@16": 0.6497626597389893, "pass@32": 0.6938640362470392, "pass@64": 0.7308787729367424, "pass@128": 0.7630057803468208 }, "16": { "pass@1": 0.2728074596774194, "pass@2": 0.35024368173736364, "pass@4": 0.41988924942849887, "pass@8": 0.47919964105067914, "pass@16": 0.527871351126739, "pass@32": 0.5696261229797946, "pass@64": 0.6054567059936721, "pass@128": 0.632258064516129 }, "17": { "pass@1": 0.21050848988439305, "pass@2": 0.2739904305675663, "pass@4": 0.3313912844528826, "pass@8": 0.38125538448297414, "pass@16": 0.4236245875549352, "pass@32": 0.4619779463348103, "pass@64": 0.49696813014542734, "pass@128": 0.5260115606936416 }, "18": { "pass@1": 0.1708705357142857, "pass@2": 0.2367473987626546, "pass@4": 0.30414279554341417, "pass@8": 0.3648555156644124, "pass@16": 0.41361773639831223, "pass@32": 0.4516806081441688, "pass@64": 0.48544624589394647, "pass@128": 0.5228571428571429 }, "19": { "pass@1": 0.15662537537537538, "pass@2": 0.21153806365420533, "pass@4": 0.266338078235716, "pass@8": 0.3166763163826258, "pass@16": 0.360523558204304, "pass@32": 0.39946955168091575, "pass@64": 0.4333781248476876, "pass@128": 0.4594594594594595 }, "20": { "pass@1": 0.15707113363363365, "pass@2": 0.21765934241327928, "pass@4": 0.27828566361637236, "pass@8": 0.3325391712216513, "pass@16": 0.3774587634032869, "pass@32": 0.41180155357726206, "pass@64": 0.43666908597212345, "pass@128": 0.45645645645645644 } } }, "teachers_in_school": { "count": 821120, "correct": 563959, "answer_accuracy": 0.6868167868277475, "avg_response_len": 257.8540128117693, "resp_tokens_sum": 211729087, "pass_at_k": { "pass@1": 0.6868167868277475, "pass@2": 0.7234032563934145, "pass@4": 0.7540754758149335, "pass@8": 0.7798046148261166, "pass@16": 0.8016922862089578, "pass@32": 0.8207486969269729, "pass@64": 0.8371004495427515, "pass@128": 0.8508183943881528 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9111148389570553, "pass@2": 0.9357741624800733, "pass@4": 0.9522138641795541, "pass@8": 0.9627448771437234, "pass@16": 0.9721080231114817, "pass@32": 0.9821285716890078, "pass@64": 0.9909827842779778, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9971117424242424, "pass@2": 0.9987439602720115, "pass@4": 0.9997009899330764, "pass@8": 0.9999796152070378, "pass@16": 0.9999999194724303, "pass@32": 0.9999999999998216, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9998844304733728, "pass@2": 0.9999978160089457, "pass@4": 0.9999999997226677, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9863677536231884, "pass@2": 0.9885042222982998, "pass@4": 0.9891683455328956, "pass@8": 0.9898325450069416, "pass@16": 0.9908752954090346, "pass@32": 0.9922112515258646, "pass@64": 0.9934649176276082, "pass@128": 0.9942028985507246 }, "6": { "pass@1": 0.9970003581661891, "pass@2": 0.9985179816348173, "pass@4": 0.9994818757433256, "pass@8": 0.9999207959785322, "pass@16": 0.9999984128084317, "pass@32": 0.9999999998125675, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9931809971098265, "pass@2": 0.9950314477037914, "pass@4": 0.9958010177376672, "pass@8": 0.9966315484192274, "pass@16": 0.9973368465089668, "pass@32": 0.997828695449064, "pass@64": 0.9985549131017303, "pass@128": 1.0 }, "8": { "pass@1": 0.9819762323943662, "pass@2": 0.9893354219807033, "pass@4": 0.9935911558590389, "pass@8": 0.9965179072847223, "pass@16": 0.9981742152631411, "pass@32": 0.999119008113794, "pass@64": 0.9998322278254919, "pass@128": 1.0 }, "9": { "pass@1": 0.9731942365269461, "pass@2": 0.9808708125088408, "pass@4": 0.9824776866402178, "pass@8": 0.9832461410936331, "pass@16": 0.9842818929540148, "pass@32": 0.9858385141976975, "pass@64": 0.9874802206610402, "pass@128": 0.9880239520958084 }, "11": { "pass@1": 0.9079207454819277, "pass@2": 0.93692449423679, "pass@4": 0.9551024624369245, "pass@8": 0.9671201826101439, "pass@16": 0.9746815567314097, "pass@32": 0.9788712127467571, "pass@64": 0.9815593127389013, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.8269004702194357, "pass@2": 0.8749591180608697, "pass@4": 0.9040032188374567, "pass@8": 0.9231087891699578, "pass@16": 0.9369859413563386, "pass@32": 0.9479069421599462, "pass@64": 0.9560498758422292, "pass@128": 0.9623824451410659 }, "13": { "pass@1": 0.6125262605042017, "pass@2": 0.6965695234786827, "pass@4": 0.7562029827679098, "pass@8": 0.8020303886222311, "pass@16": 0.838161985699861, "pass@32": 0.8655366886816216, "pass@64": 0.8870740770303306, "pass@128": 0.9047619047619048 }, "14": { "pass@1": 0.4824561403508772, "pass@2": 0.5683321246028454, "pass@4": 0.6404780016533019, "pass@8": 0.6993768785639923, "pass@16": 0.7467342166602401, "pass@32": 0.7841871616007284, "pass@64": 0.8128076930956771, "pass@128": 0.8362573099415205 }, "15": { "pass@1": 0.39372632575757577, "pass@2": 0.4847448401336197, "pass@4": 0.557197479292361, "pass@8": 0.613319566937348, "pass@16": 0.6601720515546455, "pass@32": 0.702804164645992, "pass@64": 0.7402769189342715, "pass@128": 0.7727272727272727 }, "16": { "pass@1": 0.24263265669515668, "pass@2": 0.30694675785719094, "pass@4": 0.36897373752426255, "pass@8": 0.4271218724609568, "pass@16": 0.48113967991388973, "pass@32": 0.5317545339132025, "pass@64": 0.5758856604621956, "pass@128": 0.6096866096866097 }, "17": { "pass@1": 0.22648971273291926, "pass@2": 0.2974423968063773, "pass@4": 0.3655080216098764, "pass@8": 0.4248164502069451, "pass@16": 0.4753100911077305, "pass@32": 0.5195282979937258, "pass@64": 0.5565277964456121, "pass@128": 0.5838509316770186 }, "18": { "pass@1": 0.21300551470588236, "pass@2": 0.2771649924733672, "pass@4": 0.3376023183682923, "pass@8": 0.39235717588844027, "pass@16": 0.44123591873712553, "pass@32": 0.48463721239549934, "pass@64": 0.5242288971716302, "pass@128": 0.5617647058823529 }, "19": { "pass@1": 0.14706566220238096, "pass@2": 0.20007777348143976, "pass@4": 0.25505773050690084, "pass@8": 0.306085461392637, "pass@16": 0.3516417990127139, "pass@32": 0.39376010269129014, "pass@64": 0.43223091722228596, "pass@128": 0.46726190476190477 }, "20": { "pass@1": 0.14556044161676646, "pass@2": 0.20420244648498276, "pass@4": 0.26505273339709784, "pass@8": 0.3224901187290411, "pass@16": 0.37382821128207, "pass@32": 0.4189057357484239, "pass@64": 0.45902656447888046, "pass@128": 0.4940119760479042 } } }, "movie_festival_awards": { "count": 801152, "correct": 554081, "answer_accuracy": 0.6916053383128296, "avg_response_len": 277.5439953966289, "resp_tokens_sum": 222354927, "pass_at_k": { "pass@1": 0.6916053383128296, "pass@2": 0.7286108201040862, "pass@4": 0.7588274034696325, "pass@8": 0.7837869789700335, "pass@16": 0.8045777313882145, "pass@32": 0.8226117310463649, "pass@64": 0.8388624663967905, "pass@128": 0.8528518932736859 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9279617537313433, "pass@2": 0.9513434304853687, "pass@4": 0.9654792511010749, "pass@8": 0.9737231562263299, "pass@16": 0.97817026758166, "pass@32": 0.981386045278353, "pass@64": 0.9839306760109867, "pass@128": 0.9850746268656716 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9996127915451894, "pass@2": 0.9999770436858658, "pass@4": 0.9999999464352669, "pass@8": 0.9999999999999796, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9781479779411765, "pass@2": 0.9830361278369619, "pass@4": 0.9859722322393524, "pass@8": 0.9881811181957687, "pass@16": 0.9897299162379272, "pass@32": 0.9906714940716781, "pass@64": 0.9911358126184439, "pass@128": 0.9911764705882353 }, "6": { "pass@1": 0.996352523659306, "pass@2": 0.9968279608286346, "pass@4": 0.9968454007326369, "pass@8": 0.9968454258675057, "pass@16": 0.9968454258675079, "pass@32": 0.9968454258675079, "pass@64": 0.9968454258675079, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.9880834651898734, "pass@2": 0.9925585411890763, "pass@4": 0.9936028700526358, "pass@8": 0.9936705108502834, "pass@16": 0.9936708860669233, "pass@32": 0.9936708860759493, "pass@64": 0.9936708860759493, "pass@128": 0.9936708860759493 }, "8": { "pass@1": 0.9793308933933934, "pass@2": 0.9852753836513678, "pass@4": 0.988564877025507, "pass@8": 0.9918839873821754, "pass@16": 0.9945610320902971, "pass@32": 0.9960152183579682, "pass@64": 0.9968180581304334, "pass@128": 0.996996996996997 }, "9": { "pass@1": 0.975633038585209, "pass@2": 0.985183210117224, "pass@4": 0.9885799165578577, "pass@8": 0.9912897529136566, "pass@16": 0.9942023916344718, "pass@32": 0.9970803734644477, "pass@64": 0.9993715908946497, "pass@128": 1.0 }, "11": { "pass@1": 0.8945436507936508, "pass@2": 0.9223569710036246, "pass@4": 0.9402420390903516, "pass@8": 0.9529961646045297, "pass@16": 0.96257165667739, "pass@32": 0.9707572036881429, "pass@64": 0.9780558610114967, "pass@128": 0.9841269841269841 }, "12": { "pass@1": 0.8388761467889908, "pass@2": 0.8935348641912881, "pass@4": 0.9239817218948547, "pass@8": 0.9433164183874273, "pass@16": 0.9572791903461986, "pass@32": 0.9678375527483201, "pass@64": 0.975711998782937, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.6124614197530864, "pass@2": 0.6951154217701951, "pass@4": 0.752679211973503, "pass@8": 0.7942920488824928, "pass@16": 0.8279101424953335, "pass@32": 0.8575816947638316, "pass@64": 0.884275234929319, "pass@128": 0.9074074074074074 }, "14": { "pass@1": 0.5233911350148368, "pass@2": 0.614512021308909, "pass@4": 0.6847058470435999, "pass@8": 0.7369969091059508, "pass@16": 0.7790020662193811, "pass@32": 0.8157176244438528, "pass@64": 0.8475338299324224, "pass@128": 0.8724035608308606 }, "15": { "pass@1": 0.40183738425925924, "pass@2": 0.4773633439049284, "pass@4": 0.5415484597990062, "pass@8": 0.5997374246066435, "pass@16": 0.652959196898339, "pass@32": 0.7031944084122549, "pass@64": 0.750283610070048, "pass@128": 0.7901234567901234 }, "16": { "pass@1": 0.2575589970501475, "pass@2": 0.32797111989872935, "pass@4": 0.39335200992575037, "pass@8": 0.4518741747379751, "pass@16": 0.5021914236768155, "pass@32": 0.5466943347804367, "pass@64": 0.5881361312764039, "pass@128": 0.6224188790560472 }, "17": { "pass@1": 0.21653802710843373, "pass@2": 0.2869075959349209, "pass@4": 0.34959244514616383, "pass@8": 0.4005336949460392, "pass@16": 0.44448347713010716, "pass@32": 0.48757226058553715, "pass@64": 0.5302459429332484, "pass@128": 0.572289156626506 }, "18": { "pass@1": 0.2128024193548387, "pass@2": 0.28386779273558566, "pass@4": 0.35476632437074396, "pass@8": 0.41725617116976693, "pass@16": 0.46536508744664384, "pass@32": 0.49949986996412565, "pass@64": 0.5271047913394351, "pass@128": 0.5516129032258065 }, "19": { "pass@1": 0.16871223564954682, "pass@2": 0.22346459666484297, "pass@4": 0.27490075381438345, "pass@8": 0.32159746324003746, "pass@16": 0.3655734909622184, "pass@32": 0.40807790411245964, "pass@64": 0.4465981236371106, "pass@128": 0.4773413897280967 }, "20": { "pass@1": 0.1522850975975976, "pass@2": 0.21085788051831356, "pass@4": 0.2715947052226581, "pass@8": 0.3282373203373321, "pass@16": 0.37425196123074994, "pass@32": 0.4095715191404, "pass@64": 0.4416954781218096, "pass@128": 0.4744744744744745 } } } } } ]