| [ |
| { |
| "checkpoint": "checkpoint-21546", |
| "id_acc": 0.9822352430555555, |
| "ood_acc": 0.42139921875, |
| "total_acc": 0.6870583881578948, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.35695486111112, |
| "ood_avg_resp_len": 319.1119796875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 552886, |
| "answer_accuracy": 0.6828045961112867, |
| "avg_response_len": 249.1386391479608, |
| "resp_tokens_sum": 201734532, |
| "pass_at_k": { |
| "pass@1": 0.6828045961112867, |
| "pass@2": 0.7217115855449684, |
| "pass@4": 0.7536723871913236, |
| "pass@8": 0.7797075439923347, |
| "pass@16": 0.800334627046473, |
| "pass@32": 0.817100237695868, |
| "pass@64": 0.8315237481736786, |
| "pass@128": 0.8444514701233007 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9263228244837758, |
| "pass@2": 0.9494500238078647, |
| "pass@4": 0.9622405067176335, |
| "pass@8": 0.9693425057647705, |
| "pass@16": 0.9736827240563464, |
| "pass@32": 0.9773741161261673, |
| "pass@64": 0.98185566893626, |
| "pass@128": 0.9882005899705014 |
| }, |
| "2": { |
| "pass@1": 0.9999756619937694, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9965596330275229, |
| "pass@2": 0.9981052397120085, |
| "pass@4": 0.999307317089951, |
| "pass@8": 0.999884609180392, |
| "pass@16": 0.9999968325205879, |
| "pass@32": 0.9999999991746348, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9903076171875, |
| "pass@2": 0.9908726008858268, |
| "pass@4": 0.9911945356439821, |
| "pass@8": 0.9917235482142811, |
| "pass@16": 0.9926657644356954, |
| "pass@32": 0.9941486923509562, |
| "pass@64": 0.9959137795275591, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9702628968253968, |
| "pass@2": 0.9771036432945881, |
| "pass@4": 0.9829424443968311, |
| "pass@8": 0.9876832076495944, |
| "pass@16": 0.990954409287743, |
| "pass@32": 0.9930190209714992, |
| "pass@64": 0.9948502012050503, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9956727170658682, |
| "pass@2": 0.9975463688764207, |
| "pass@4": 0.9990146021230881, |
| "pass@8": 0.9998480009664775, |
| "pass@16": 0.9999970866060833, |
| "pass@32": 0.9999999996392369, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9940366124260355, |
| "pass@2": 0.9978666047383871, |
| "pass@4": 0.9993812003159368, |
| "pass@8": 0.9999391517555988, |
| "pass@16": 0.9999995060547365, |
| "pass@32": 0.9999999999906712, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9850260416666666, |
| "pass@2": 0.991938282354129, |
| "pass@4": 0.9944415109169044, |
| "pass@8": 0.9959676378945411, |
| "pass@16": 0.996687988729117, |
| "pass@32": 0.9967934570195686, |
| "pass@64": 0.9967948717815591, |
| "pass@128": 0.9967948717948718 |
| }, |
| "9": { |
| "pass@1": 0.975594190140845, |
| "pass@2": 0.9893763169568592, |
| "pass@4": 0.9948080432727598, |
| "pass@8": 0.9979627772031147, |
| "pass@16": 0.9995681970104956, |
| "pass@32": 0.9999716229169351, |
| "pass@64": 0.9999999660012311, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8872388456090652, |
| "pass@2": 0.9170830452142494, |
| "pass@4": 0.9344475201046042, |
| "pass@8": 0.9461166366837217, |
| "pass@16": 0.9548923315039325, |
| "pass@32": 0.9624344015568419, |
| "pass@64": 0.9692996308589977, |
| "pass@128": 0.9745042492917847 |
| }, |
| "12": { |
| "pass@1": 0.7989495056497176, |
| "pass@2": 0.8576045976244497, |
| "pass@4": 0.894183136271101, |
| "pass@8": 0.9153254162241761, |
| "pass@16": 0.9267540179385538, |
| "pass@32": 0.9355352501223162, |
| "pass@64": 0.9451007362794269, |
| "pass@128": 0.9548022598870056 |
| }, |
| "13": { |
| "pass@1": 0.661662421630094, |
| "pass@2": 0.7403545621158639, |
| "pass@4": 0.7929696285025494, |
| "pass@8": 0.8331626588813359, |
| "pass@16": 0.8660458043380581, |
| "pass@32": 0.8937908906213489, |
| "pass@64": 0.9188840917684389, |
| "pass@128": 0.9435736677115988 |
| }, |
| "14": { |
| "pass@1": 0.48459404205607476, |
| "pass@2": 0.586037726592587, |
| "pass@4": 0.6693883112741745, |
| "pass@8": 0.7337120781768929, |
| "pass@16": 0.7796279839990717, |
| "pass@32": 0.8097918673125195, |
| "pass@64": 0.8326759446166977, |
| "pass@128": 0.8566978193146417 |
| }, |
| "15": { |
| "pass@1": 0.3742097182080925, |
| "pass@2": 0.46095314562377676, |
| "pass@4": 0.5352501313961474, |
| "pass@8": 0.5976363331573419, |
| "pass@16": 0.6497626597389893, |
| "pass@32": 0.6938640362470392, |
| "pass@64": 0.7308787729367424, |
| "pass@128": 0.7630057803468208 |
| }, |
| "16": { |
| "pass@1": 0.2728074596774194, |
| "pass@2": 0.35024368173736364, |
| "pass@4": 0.41988924942849887, |
| "pass@8": 0.47919964105067914, |
| "pass@16": 0.527871351126739, |
| "pass@32": 0.5696261229797946, |
| "pass@64": 0.6054567059936721, |
| "pass@128": 0.632258064516129 |
| }, |
| "17": { |
| "pass@1": 0.21050848988439305, |
| "pass@2": 0.2739904305675663, |
| "pass@4": 0.3313912844528826, |
| "pass@8": 0.38125538448297414, |
| "pass@16": 0.4236245875549352, |
| "pass@32": 0.4619779463348103, |
| "pass@64": 0.49696813014542734, |
| "pass@128": 0.5260115606936416 |
| }, |
| "18": { |
| "pass@1": 0.1708705357142857, |
| "pass@2": 0.2367473987626546, |
| "pass@4": 0.30414279554341417, |
| "pass@8": 0.3648555156644124, |
| "pass@16": 0.41361773639831223, |
| "pass@32": 0.4516806081441688, |
| "pass@64": 0.48544624589394647, |
| "pass@128": 0.5228571428571429 |
| }, |
| "19": { |
| "pass@1": 0.15662537537537538, |
| "pass@2": 0.21153806365420533, |
| "pass@4": 0.266338078235716, |
| "pass@8": 0.3166763163826258, |
| "pass@16": 0.360523558204304, |
| "pass@32": 0.39946955168091575, |
| "pass@64": 0.4333781248476876, |
| "pass@128": 0.4594594594594595 |
| }, |
| "20": { |
| "pass@1": 0.15707113363363365, |
| "pass@2": 0.21765934241327928, |
| "pass@4": 0.27828566361637236, |
| "pass@8": 0.3325391712216513, |
| "pass@16": 0.3774587634032869, |
| "pass@32": 0.41180155357726206, |
| "pass@64": 0.43666908597212345, |
| "pass@128": 0.45645645645645644 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 563959, |
| "answer_accuracy": 0.6868167868277475, |
| "avg_response_len": 257.8540128117693, |
| "resp_tokens_sum": 211729087, |
| "pass_at_k": { |
| "pass@1": 0.6868167868277475, |
| "pass@2": 0.7234032563934145, |
| "pass@4": 0.7540754758149335, |
| "pass@8": 0.7798046148261166, |
| "pass@16": 0.8016922862089578, |
| "pass@32": 0.8207486969269729, |
| "pass@64": 0.8371004495427515, |
| "pass@128": 0.8508183943881528 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9111148389570553, |
| "pass@2": 0.9357741624800733, |
| "pass@4": 0.9522138641795541, |
| "pass@8": 0.9627448771437234, |
| "pass@16": 0.9721080231114817, |
| "pass@32": 0.9821285716890078, |
| "pass@64": 0.9909827842779778, |
| "pass@128": 0.9969325153374233 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9971117424242424, |
| "pass@2": 0.9987439602720115, |
| "pass@4": 0.9997009899330764, |
| "pass@8": 0.9999796152070378, |
| "pass@16": 0.9999999194724303, |
| "pass@32": 0.9999999999998216, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9998844304733728, |
| "pass@2": 0.9999978160089457, |
| "pass@4": 0.9999999997226677, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9863677536231884, |
| "pass@2": 0.9885042222982998, |
| "pass@4": 0.9891683455328956, |
| "pass@8": 0.9898325450069416, |
| "pass@16": 0.9908752954090346, |
| "pass@32": 0.9922112515258646, |
| "pass@64": 0.9934649176276082, |
| "pass@128": 0.9942028985507246 |
| }, |
| "6": { |
| "pass@1": 0.9970003581661891, |
| "pass@2": 0.9985179816348173, |
| "pass@4": 0.9994818757433256, |
| "pass@8": 0.9999207959785322, |
| "pass@16": 0.9999984128084317, |
| "pass@32": 0.9999999998125675, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9931809971098265, |
| "pass@2": 0.9950314477037914, |
| "pass@4": 0.9958010177376672, |
| "pass@8": 0.9966315484192274, |
| "pass@16": 0.9973368465089668, |
| "pass@32": 0.997828695449064, |
| "pass@64": 0.9985549131017303, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9819762323943662, |
| "pass@2": 0.9893354219807033, |
| "pass@4": 0.9935911558590389, |
| "pass@8": 0.9965179072847223, |
| "pass@16": 0.9981742152631411, |
| "pass@32": 0.999119008113794, |
| "pass@64": 0.9998322278254919, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9731942365269461, |
| "pass@2": 0.9808708125088408, |
| "pass@4": 0.9824776866402178, |
| "pass@8": 0.9832461410936331, |
| "pass@16": 0.9842818929540148, |
| "pass@32": 0.9858385141976975, |
| "pass@64": 0.9874802206610402, |
| "pass@128": 0.9880239520958084 |
| }, |
| "11": { |
| "pass@1": 0.9079207454819277, |
| "pass@2": 0.93692449423679, |
| "pass@4": 0.9551024624369245, |
| "pass@8": 0.9671201826101439, |
| "pass@16": 0.9746815567314097, |
| "pass@32": 0.9788712127467571, |
| "pass@64": 0.9815593127389013, |
| "pass@128": 0.9849397590361446 |
| }, |
| "12": { |
| "pass@1": 0.8269004702194357, |
| "pass@2": 0.8749591180608697, |
| "pass@4": 0.9040032188374567, |
| "pass@8": 0.9231087891699578, |
| "pass@16": 0.9369859413563386, |
| "pass@32": 0.9479069421599462, |
| "pass@64": 0.9560498758422292, |
| "pass@128": 0.9623824451410659 |
| }, |
| "13": { |
| "pass@1": 0.6125262605042017, |
| "pass@2": 0.6965695234786827, |
| "pass@4": 0.7562029827679098, |
| "pass@8": 0.8020303886222311, |
| "pass@16": 0.838161985699861, |
| "pass@32": 0.8655366886816216, |
| "pass@64": 0.8870740770303306, |
| "pass@128": 0.9047619047619048 |
| }, |
| "14": { |
| "pass@1": 0.4824561403508772, |
| "pass@2": 0.5683321246028454, |
| "pass@4": 0.6404780016533019, |
| "pass@8": 0.6993768785639923, |
| "pass@16": 0.7467342166602401, |
| "pass@32": 0.7841871616007284, |
| "pass@64": 0.8128076930956771, |
| "pass@128": 0.8362573099415205 |
| }, |
| "15": { |
| "pass@1": 0.39372632575757577, |
| "pass@2": 0.4847448401336197, |
| "pass@4": 0.557197479292361, |
| "pass@8": 0.613319566937348, |
| "pass@16": 0.6601720515546455, |
| "pass@32": 0.702804164645992, |
| "pass@64": 0.7402769189342715, |
| "pass@128": 0.7727272727272727 |
| }, |
| "16": { |
| "pass@1": 0.24263265669515668, |
| "pass@2": 0.30694675785719094, |
| "pass@4": 0.36897373752426255, |
| "pass@8": 0.4271218724609568, |
| "pass@16": 0.48113967991388973, |
| "pass@32": 0.5317545339132025, |
| "pass@64": 0.5758856604621956, |
| "pass@128": 0.6096866096866097 |
| }, |
| "17": { |
| "pass@1": 0.22648971273291926, |
| "pass@2": 0.2974423968063773, |
| "pass@4": 0.3655080216098764, |
| "pass@8": 0.4248164502069451, |
| "pass@16": 0.4753100911077305, |
| "pass@32": 0.5195282979937258, |
| "pass@64": 0.5565277964456121, |
| "pass@128": 0.5838509316770186 |
| }, |
| "18": { |
| "pass@1": 0.21300551470588236, |
| "pass@2": 0.2771649924733672, |
| "pass@4": 0.3376023183682923, |
| "pass@8": 0.39235717588844027, |
| "pass@16": 0.44123591873712553, |
| "pass@32": 0.48463721239549934, |
| "pass@64": 0.5242288971716302, |
| "pass@128": 0.5617647058823529 |
| }, |
| "19": { |
| "pass@1": 0.14706566220238096, |
| "pass@2": 0.20007777348143976, |
| "pass@4": 0.25505773050690084, |
| "pass@8": 0.306085461392637, |
| "pass@16": 0.3516417990127139, |
| "pass@32": 0.39376010269129014, |
| "pass@64": 0.43223091722228596, |
| "pass@128": 0.46726190476190477 |
| }, |
| "20": { |
| "pass@1": 0.14556044161676646, |
| "pass@2": 0.20420244648498276, |
| "pass@4": 0.26505273339709784, |
| "pass@8": 0.3224901187290411, |
| "pass@16": 0.37382821128207, |
| "pass@32": 0.4189057357484239, |
| "pass@64": 0.45902656447888046, |
| "pass@128": 0.4940119760479042 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 554081, |
| "answer_accuracy": 0.6916053383128296, |
| "avg_response_len": 277.5439953966289, |
| "resp_tokens_sum": 222354927, |
| "pass_at_k": { |
| "pass@1": 0.6916053383128296, |
| "pass@2": 0.7286108201040862, |
| "pass@4": 0.7588274034696325, |
| "pass@8": 0.7837869789700335, |
| "pass@16": 0.8045777313882145, |
| "pass@32": 0.8226117310463649, |
| "pass@64": 0.8388624663967905, |
| "pass@128": 0.8528518932736859 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9279617537313433, |
| "pass@2": 0.9513434304853687, |
| "pass@4": 0.9654792511010749, |
| "pass@8": 0.9737231562263299, |
| "pass@16": 0.97817026758166, |
| "pass@32": 0.981386045278353, |
| "pass@64": 0.9839306760109867, |
| "pass@128": 0.9850746268656716 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9996127915451894, |
| "pass@2": 0.9999770436858658, |
| "pass@4": 0.9999999464352669, |
| "pass@8": 0.9999999999999796, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9781479779411765, |
| "pass@2": 0.9830361278369619, |
| "pass@4": 0.9859722322393524, |
| "pass@8": 0.9881811181957687, |
| "pass@16": 0.9897299162379272, |
| "pass@32": 0.9906714940716781, |
| "pass@64": 0.9911358126184439, |
| "pass@128": 0.9911764705882353 |
| }, |
| "6": { |
| "pass@1": 0.996352523659306, |
| "pass@2": 0.9968279608286346, |
| "pass@4": 0.9968454007326369, |
| "pass@8": 0.9968454258675057, |
| "pass@16": 0.9968454258675079, |
| "pass@32": 0.9968454258675079, |
| "pass@64": 0.9968454258675079, |
| "pass@128": 0.9968454258675079 |
| }, |
| "7": { |
| "pass@1": 0.9880834651898734, |
| "pass@2": 0.9925585411890763, |
| "pass@4": 0.9936028700526358, |
| "pass@8": 0.9936705108502834, |
| "pass@16": 0.9936708860669233, |
| "pass@32": 0.9936708860759493, |
| "pass@64": 0.9936708860759493, |
| "pass@128": 0.9936708860759493 |
| }, |
| "8": { |
| "pass@1": 0.9793308933933934, |
| "pass@2": 0.9852753836513678, |
| "pass@4": 0.988564877025507, |
| "pass@8": 0.9918839873821754, |
| "pass@16": 0.9945610320902971, |
| "pass@32": 0.9960152183579682, |
| "pass@64": 0.9968180581304334, |
| "pass@128": 0.996996996996997 |
| }, |
| "9": { |
| "pass@1": 0.975633038585209, |
| "pass@2": 0.985183210117224, |
| "pass@4": 0.9885799165578577, |
| "pass@8": 0.9912897529136566, |
| "pass@16": 0.9942023916344718, |
| "pass@32": 0.9970803734644477, |
| "pass@64": 0.9993715908946497, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8945436507936508, |
| "pass@2": 0.9223569710036246, |
| "pass@4": 0.9402420390903516, |
| "pass@8": 0.9529961646045297, |
| "pass@16": 0.96257165667739, |
| "pass@32": 0.9707572036881429, |
| "pass@64": 0.9780558610114967, |
| "pass@128": 0.9841269841269841 |
| }, |
| "12": { |
| "pass@1": 0.8388761467889908, |
| "pass@2": 0.8935348641912881, |
| "pass@4": 0.9239817218948547, |
| "pass@8": 0.9433164183874273, |
| "pass@16": 0.9572791903461986, |
| "pass@32": 0.9678375527483201, |
| "pass@64": 0.975711998782937, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.6124614197530864, |
| "pass@2": 0.6951154217701951, |
| "pass@4": 0.752679211973503, |
| "pass@8": 0.7942920488824928, |
| "pass@16": 0.8279101424953335, |
| "pass@32": 0.8575816947638316, |
| "pass@64": 0.884275234929319, |
| "pass@128": 0.9074074074074074 |
| }, |
| "14": { |
| "pass@1": 0.5233911350148368, |
| "pass@2": 0.614512021308909, |
| "pass@4": 0.6847058470435999, |
| "pass@8": 0.7369969091059508, |
| "pass@16": 0.7790020662193811, |
| "pass@32": 0.8157176244438528, |
| "pass@64": 0.8475338299324224, |
| "pass@128": 0.8724035608308606 |
| }, |
| "15": { |
| "pass@1": 0.40183738425925924, |
| "pass@2": 0.4773633439049284, |
| "pass@4": 0.5415484597990062, |
| "pass@8": 0.5997374246066435, |
| "pass@16": 0.652959196898339, |
| "pass@32": 0.7031944084122549, |
| "pass@64": 0.750283610070048, |
| "pass@128": 0.7901234567901234 |
| }, |
| "16": { |
| "pass@1": 0.2575589970501475, |
| "pass@2": 0.32797111989872935, |
| "pass@4": 0.39335200992575037, |
| "pass@8": 0.4518741747379751, |
| "pass@16": 0.5021914236768155, |
| "pass@32": 0.5466943347804367, |
| "pass@64": 0.5881361312764039, |
| "pass@128": 0.6224188790560472 |
| }, |
| "17": { |
| "pass@1": 0.21653802710843373, |
| "pass@2": 0.2869075959349209, |
| "pass@4": 0.34959244514616383, |
| "pass@8": 0.4005336949460392, |
| "pass@16": 0.44448347713010716, |
| "pass@32": 0.48757226058553715, |
| "pass@64": 0.5302459429332484, |
| "pass@128": 0.572289156626506 |
| }, |
| "18": { |
| "pass@1": 0.2128024193548387, |
| "pass@2": 0.28386779273558566, |
| "pass@4": 0.35476632437074396, |
| "pass@8": 0.41725617116976693, |
| "pass@16": 0.46536508744664384, |
| "pass@32": 0.49949986996412565, |
| "pass@64": 0.5271047913394351, |
| "pass@128": 0.5516129032258065 |
| }, |
| "19": { |
| "pass@1": 0.16871223564954682, |
| "pass@2": 0.22346459666484297, |
| "pass@4": 0.27490075381438345, |
| "pass@8": 0.32159746324003746, |
| "pass@16": 0.3655734909622184, |
| "pass@32": 0.40807790411245964, |
| "pass@64": 0.4465981236371106, |
| "pass@128": 0.4773413897280967 |
| }, |
| "20": { |
| "pass@1": 0.1522850975975976, |
| "pass@2": 0.21085788051831356, |
| "pass@4": 0.2715947052226581, |
| "pass@8": 0.3282373203373321, |
| "pass@16": 0.37425196123074994, |
| "pass@32": 0.4095715191404, |
| "pass@64": 0.4416954781218096, |
| "pass@128": 0.4744744744744745 |
| } |
| } |
| } |
| } |
| } |
| ] |
