[ { "checkpoint": "checkpoint-21350", "id_acc": 0.6244479166666667, "ood_acc": 0.19060625, "total_acc": 0.39611019736842107, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 198.69680642361112, "ood_avg_resp_len": 306.29846640625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 323829, "answer_accuracy": 0.39992318408156813, "avg_response_len": 243.85018055445778, "resp_tokens_sum": 197452319, "pass_at_k": { "pass@1": 0.39992318408156813, "pass@2": 0.4944797055210301, "pass@4": 0.5840710188405998, "pass@8": 0.660719265536069, "pass@16": 0.7230338617727872, "pass@32": 0.7728459366315423, "pass@64": 0.8121533397812276, "pass@128": 0.8430287701549162 }, "per_op_pass_at_k": { "10": { "pass@1": 0.36114952064896755, "pass@2": 0.5025727736743082, "pass@4": 0.6456373453871359, "pass@8": 0.7715342066613812, "pass@16": 0.8645454062463671, "pass@32": 0.9234167949172833, "pass@64": 0.9545074396989176, "pass@128": 0.9646017699115044 }, "2": { "pass@1": 0.9437792056074766, "pass@2": 0.9547834339048741, "pass@4": 0.9639342809952494, "pass@8": 0.9714543860845367, "pass@16": 0.9780852797401268, "pass@32": 0.9838669377540457, "pass@64": 0.9889642323239781, "pass@128": 0.9937694704049844 }, "3": { "pass@1": 0.9333906727828746, "pass@2": 0.9471856263093257, "pass@4": 0.9564936748158777, "pass@8": 0.9643997413788679, "pass@16": 0.9714202643835527, "pass@32": 0.9771898049285574, "pass@64": 0.982185906508893, "pass@128": 0.9877675840978594 }, "4": { "pass@1": 0.846826171875, "pass@2": 0.8998592827263778, "pass@4": 0.9362568458239592, "pass@8": 0.9610523510160874, "pass@16": 0.9772996456506924, "pass@32": 0.9865836582491718, "pass@64": 0.9911079218798541, "pass@128": 0.99375 }, "5": { "pass@1": 0.590203373015873, "pass@2": 0.6948537682789656, "pass@4": 0.7772942653001701, "pass@8": 0.8367794936962248, "pass@16": 0.8804916680795036, "pass@32": 0.9143427896247542, "pass@64": 0.9375744510001492, "pass@128": 0.9523809523809523 }, "6": { "pass@1": 0.6659571482035929, "pass@2": 0.7866637541845445, "pass@4": 0.8693523220712676, "pass@8": 0.9201696057829176, "pass@16": 0.9509777220112018, "pass@32": 0.9715084608001282, "pass@64": 0.9841061290050542, "pass@128": 0.9910179640718563 }, "7": { "pass@1": 0.5229289940828402, "pass@2": 0.6506014711363743, "pass@4": 0.7598703474750563, "pass@8": 0.843843292045879, "pass@16": 0.9012108173980592, "pass@32": 0.9370625269608213, "pass@64": 0.960044360287011, "pass@128": 0.9733727810650887 }, "8": { "pass@1": 0.4805438701923077, "pass@2": 0.626625435342217, "pass@4": 0.7501883081922448, "pass@8": 0.8398731877822811, "pass@16": 0.8993453994361422, "pass@32": 0.9372353301463613, "pass@64": 0.9600822667646836, "pass@128": 0.9743589743589743 }, "9": { "pass@1": 0.37691461267605636, "pass@2": 0.5194996257069978, "pass@4": 0.6534315924417888, "pass@8": 0.7672971099907365, "pass@16": 0.856289359327201, "pass@32": 0.9170800615077654, "pass@64": 0.9537005371934129, "pass@128": 0.971830985915493 }, "11": { "pass@1": 0.3295414305949009, "pass@2": 0.4673450709330597, "pass@4": 0.5997836868302225, "pass@8": 0.7113282977329743, "pass@16": 0.7996943213346072, "pass@32": 0.8639812545790467, "pass@64": 0.9078105095951646, "pass@128": 0.9348441926345609 }, "12": { "pass@1": 0.2501765536723164, "pass@2": 0.3753962031229148, "pass@4": 0.5053718790977395, "pass@8": 0.6213835158955956, "pass@16": 0.7194175565450607, "pass@32": 0.8002603777466087, "pass@64": 0.863669554304307, "pass@128": 0.9067796610169492 }, "13": { "pass@1": 0.24708561912225704, "pass@2": 0.3578045164515093, "pass@4": 0.4703984194373826, "pass@8": 0.5756403999380573, "pass@16": 0.6725806558811607, "pass@32": 0.7608977414595537, "pass@64": 0.8395201593334661, "pass@128": 0.9059561128526645 }, "14": { "pass@1": 0.2110105140186916, "pass@2": 0.314535577182525, "pass@4": 0.42529508636186836, "pass@8": 0.5282425677297654, "pass@16": 0.6195701240372914, "pass@32": 0.7011807434818168, "pass@64": 0.7712958911735933, "pass@128": 0.8286604361370716 }, "15": { "pass@1": 0.174516799132948, "pass@2": 0.26026530710937146, "pass@4": 0.3516182513854841, "pass@8": 0.4383474727139665, "pass@16": 0.5179759923282242, "pass@32": 0.592788340884816, "pass@64": 0.6628031726152839, "pass@128": 0.7254335260115607 }, "16": { "pass@1": 0.17643649193548386, "pass@2": 0.256484950469901, "pass@4": 0.3398621137075606, "pass@8": 0.4208168272956852, "pass@16": 0.49688550526352426, "pass@32": 0.5672181973038024, "pass@64": 0.6332357350352547, "pass@128": 0.7 }, "17": { "pass@1": 0.1539694725433526, "pass@2": 0.2301268575622411, "pass@4": 0.3082558133990478, "pass@8": 0.3769771190958141, "pass@16": 0.4362476355948833, "pass@32": 0.49174618367651685, "pass@64": 0.5481902103997353, "pass@128": 0.6098265895953757 }, "18": { "pass@1": 0.14058035714285713, "pass@2": 0.21339742688413954, "pass@4": 0.2935735435749102, "pass@8": 0.37007484386718004, "pass@16": 0.43467331394651476, "pass@32": 0.48700053725037323, "pass@64": 0.5293119476612531, "pass@128": 0.5685714285714286 }, "19": { "pass@1": 0.1365897147147147, "pass@2": 0.2067106476555297, "pass@4": 0.28071525490620003, "pass@8": 0.348540194766144, "pass@16": 0.40951359580054114, "pass@32": 0.4652272616603531, "pass@64": 0.5140670759360153, "pass@128": 0.5555555555555556 }, "20": { "pass@1": 0.13518205705705705, "pass@2": 0.19940080336241758, "pass@4": 0.26722059320287667, "pass@8": 0.33167000256711, "pass@16": 0.3880052020125598, "pass@32": 0.4359332930515247, "pass@64": 0.47551369120416725, "pass@128": 0.5045045045045045 } } }, "teachers_in_school": { "count": 821120, "correct": 318978, "answer_accuracy": 0.38846697194076385, "avg_response_len": 253.20947851714732, "resp_tokens_sum": 207915367, "pass_at_k": { "pass@1": 0.38846697194076385, "pass@2": 0.4831616006714161, "pass@4": 0.5734167920114881, "pass@8": 0.6516151494643043, "pass@16": 0.7160463847331318, "pass@32": 0.7682086317953974, "pass@64": 0.8096786629277101, "pass@128": 0.8413094310210444 }, "per_op_pass_at_k": { "10": { "pass@1": 0.3624664493865031, "pass@2": 0.4975967344572724, "pass@4": 0.6321679976927728, "pass@8": 0.7529265654036871, "pass@16": 0.8500658601313674, "pass@32": 0.917990814785285, "pass@64": 0.9590980548606445, "pass@128": 0.9815950920245399 }, "2": { "pass@1": 0.9269329407294833, "pass@2": 0.9438456579230784, "pass@4": 0.9553312027106032, "pass@8": 0.9639957189471136, "pass@16": 0.9706872409005898, "pass@32": 0.9753433482006632, "pass@64": 0.9778610939418738, "pass@128": 0.9787234042553191 }, "3": { "pass@1": 0.881155303030303, "pass@2": 0.9059744840133619, "pass@4": 0.9232438743452522, "pass@8": 0.9368277008371412, "pass@16": 0.9468825073475181, "pass@32": 0.9552734657415504, "pass@64": 0.9645054655673708, "pass@128": 0.9757575757575757 }, "4": { "pass@1": 0.8148345044378699, "pass@2": 0.8734522783394674, "pass@4": 0.9115685165833552, "pass@8": 0.9380823088154047, "pass@16": 0.956343947805061, "pass@32": 0.9685328260459767, "pass@64": 0.9768866649882043, "pass@128": 0.9822485207100592 }, "5": { "pass@1": 0.6144021739130435, "pass@2": 0.7229213026360837, "pass@4": 0.8079759513756427, "pass@8": 0.8658693081389128, "pass@16": 0.902718231976262, "pass@32": 0.9294425193718734, "pass@64": 0.949326977779095, "pass@128": 0.9594202898550724 }, "6": { "pass@1": 0.5740732449856734, "pass@2": 0.7082373288134823, "pass@4": 0.8135206498184862, "pass@8": 0.8863266213359048, "pass@16": 0.9320016713493857, "pass@32": 0.9581798591821608, "pass@64": 0.9729124958024994, "pass@128": 0.9828080229226361 }, "7": { "pass@1": 0.49701950867052025, "pass@2": 0.6282756246870876, "pass@4": 0.7404991438931393, "pass@8": 0.8254763539044299, "pass@16": 0.8839952514643473, "pass@32": 0.9212432414261512, "pass@64": 0.944875483972104, "pass@128": 0.9653179190751445 }, "8": { "pass@1": 0.43058978873239434, "pass@2": 0.5753673616502167, "pass@4": 0.7030660641011416, "pass@8": 0.8001631029757508, "pass@16": 0.8668395813069671, "pass@32": 0.9117980434621504, "pass@64": 0.9437722729837049, "pass@128": 0.971830985915493 }, "9": { "pass@1": 0.3983205464071856, "pass@2": 0.5390116666273753, "pass@4": 0.6679228606903187, "pass@8": 0.774652988649831, "pass@16": 0.8587257409476783, "pass@32": 0.9206947819975753, "pass@64": 0.9582371946330216, "pass@128": 0.9760479041916168 }, "11": { "pass@1": 0.29918109939759036, "pass@2": 0.4210441668247794, "pass@4": 0.5394813751631948, "pass@8": 0.6456911887862564, "pass@16": 0.7395810077099041, "pass@32": 0.8209640589794067, "pass@64": 0.887567370372437, "pass@128": 0.9307228915662651 }, "12": { "pass@1": 0.28950333072100315, "pass@2": 0.41677362821810293, "pass@4": 0.5441571644257631, "pass@8": 0.6591465794235026, "pass@16": 0.7601174561613118, "pass@32": 0.8408860799581142, "pass@64": 0.8961900157902011, "pass@128": 0.9341692789968652 }, "13": { "pass@1": 0.21962535014005602, "pass@2": 0.32394020600366136, "pass@4": 0.4328213529506295, "pass@8": 0.5350798081641757, "pass@16": 0.6285083457669532, "pass@32": 0.714978858889809, "pass@64": 0.7926042040647505, "pass@128": 0.8543417366946778 }, "14": { "pass@1": 0.20360471491228072, "pass@2": 0.2982582049776674, "pass@4": 0.39714510000942854, "pass@8": 0.491090990747798, "pass@16": 0.5775013194565408, "pass@32": 0.6566763801674114, "pass@64": 0.727151911027562, "pass@128": 0.783625730994152 }, "15": { "pass@1": 0.15722064393939394, "pass@2": 0.2387232313290383, "pass@4": 0.32951538074786096, "pass@8": 0.41899566652474257, "pass@16": 0.5013357299356879, "pass@32": 0.5751957873786209, "pass@64": 0.639451132054347, "pass@128": 0.6909090909090909 }, "16": { "pass@1": 0.16415153133903135, "pass@2": 0.24567567074948962, "pass@4": 0.33242294766573, "pass@8": 0.41320291592274866, "pass@16": 0.48563958155008496, "pass@32": 0.5530816266561013, "pass@64": 0.6152416612226966, "pass@128": 0.6638176638176638 }, "17": { "pass@1": 0.16037461180124224, "pass@2": 0.24287831038783184, "pass@4": 0.33337522590343915, "pass@8": 0.4201866766107831, "pass@16": 0.49633960489111445, "pass@32": 0.5619214680954321, "pass@64": 0.6206053593384304, "pass@128": 0.6708074534161491 }, "18": { "pass@1": 0.1348345588235294, "pass@2": 0.20740287748957853, "pass@4": 0.28915632430137395, "pass@8": 0.37054043763698213, "pass@16": 0.4462654321967189, "pass@32": 0.5117364598301063, "pass@64": 0.5669886295013616, "pass@128": 0.611764705882353 }, "19": { "pass@1": 0.13102213541666666, "pass@2": 0.19896023739220098, "pass@4": 0.27425777079204383, "pass@8": 0.3457286602947428, "pass@16": 0.408312844903427, "pass@32": 0.4615735021120364, "pass@64": 0.5057884604835114, "pass@128": 0.5446428571428571 }, "20": { "pass@1": 0.1250935628742515, "pass@2": 0.18957017068225754, "pass@4": 0.2618971112892326, "pass@8": 0.3306961799461177, "pass@16": 0.38798512293652654, "pass@32": 0.4361992048056054, "pass@64": 0.48030832094747544, "pass@128": 0.5209580838323353 } } }, "movie_festival_awards": { "count": 801152, "correct": 320533, "answer_accuracy": 0.4000901202268733, "avg_response_len": 269.1038304841029, "resp_tokens_sum": 215593072, "pass_at_k": { "pass@1": 0.4000901202268733, "pass@2": 0.4915955276370618, "pass@4": 0.5797298176040884, "pass@8": 0.656446372260374, "pass@16": 0.7187723264801746, "pass@32": 0.7682261309010882, "pass@64": 0.8071850532733251, "pass@128": 0.8384725994567822 }, "per_op_pass_at_k": { "10": { "pass@1": 0.38957555970149255, "pass@2": 0.5278385386061816, "pass@4": 0.6573339027584231, "pass@8": 0.7652352904379972, "pass@16": 0.8462123530244711, "pass@32": 0.9007894990247578, "pass@64": 0.9329989074885636, "pass@128": 0.9492537313432836 }, "2": { "pass@1": 0.9677678571428572, "pass@2": 0.9728785854893138, "pass@4": 0.9782819976967164, "pass@8": 0.9833322679920727, "pass@16": 0.987546562066378, "pass@32": 0.9910945657332776, "pass@64": 0.9942553750832824, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9237427113702624, "pass@2": 0.9485807258786534, "pass@4": 0.9676794325636404, "pass@8": 0.9822012944091195, "pass@16": 0.9910301853418775, "pass@32": 0.9949440882394144, "pass@64": 0.9963556520655231, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.8395010964912281, "pass@2": 0.8924888192429892, "pass@4": 0.9275407303801936, "pass@8": 0.9516473139663537, "pass@16": 0.9668355189678384, "pass@32": 0.9752623599544581, "pass@64": 0.980767093553529, "pass@128": 0.9853801169590644 }, "5": { "pass@1": 0.5912683823529412, "pass@2": 0.6982844343446042, "pass@4": 0.7874484240940461, "pass@8": 0.8546648972248505, "pass@16": 0.9016930542819938, "pass@32": 0.9313893737852658, "pass@64": 0.9468139136974493, "pass@128": 0.9529411764705882 }, "6": { "pass@1": 0.6101389984227129, "pass@2": 0.7372598363098934, "pass@4": 0.8294707784949593, "pass@8": 0.8884357436921327, "pass@16": 0.9239306097873485, "pass@32": 0.9486099594928609, "pass@64": 0.9684781300081131, "pass@128": 0.9810725552050473 }, "7": { "pass@1": 0.5148091376582279, "pass@2": 0.6504655729841523, "pass@4": 0.7609237038605933, "pass@8": 0.8428334528178655, "pass@16": 0.8999233899205381, "pass@32": 0.9376737302572167, "pass@64": 0.9596856059802206, "pass@128": 0.9683544303797469 }, "8": { "pass@1": 0.45765296546546547, "pass@2": 0.5974745661015346, "pass@4": 0.7241982536529772, "pass@8": 0.8253190393429889, "pass@16": 0.8937941835338469, "pass@32": 0.9343942484480311, "pass@64": 0.9562481389043579, "pass@128": 0.96996996996997 }, "9": { "pass@1": 0.3453577170418006, "pass@2": 0.4762565036585059, "pass@4": 0.6051359099606117, "pass@8": 0.716427365466166, "pass@16": 0.8080941860304234, "pass@32": 0.8798992794463033, "pass@64": 0.9278435692980506, "pass@128": 0.9581993569131833 }, "11": { "pass@1": 0.32242063492063494, "pass@2": 0.45908206786651673, "pass@4": 0.5897093937662551, "pass@8": 0.7004402636150577, "pass@16": 0.7896761702070251, "pass@32": 0.8602341105634611, "pass@64": 0.9128581401462108, "pass@128": 0.9523809523809523 }, "12": { "pass@1": 0.2635225535168196, "pass@2": 0.3877930181319079, "pass@4": 0.5181792000770544, "pass@8": 0.6405584256764115, "pass@16": 0.7469117642026594, "pass@32": 0.8305788497806939, "pass@64": 0.8900975412573625, "pass@128": 0.9296636085626911 }, "13": { "pass@1": 0.22899787808641975, "pass@2": 0.3279974117818605, "pass@4": 0.431257729936536, "pass@8": 0.529076076642035, "pass@16": 0.6174130400830578, "pass@32": 0.6948931455581896, "pass@64": 0.7644153598458592, "pass@128": 0.8364197530864198 }, "14": { "pass@1": 0.2097783753709199, "pass@2": 0.3072589312834414, "pass@4": 0.40860501663924065, "pass@8": 0.5003493943525822, "pass@16": 0.5797323311010636, "pass@32": 0.6504938152935147, "pass@64": 0.714900085130906, "pass@128": 0.771513353115727 }, "15": { "pass@1": 0.1486062885802469, "pass@2": 0.23368465660542423, "pass@4": 0.3351522572988561, "pass@8": 0.4362885891668056, "pass@16": 0.5222229432999017, "pass@32": 0.5923627487289759, "pass@64": 0.65235085460358, "pass@128": 0.7067901234567902 }, "16": { "pass@1": 0.16894819321533924, "pass@2": 0.24454161150210196, "pass@4": 0.3213824090572752, "pass@8": 0.39293382315692404, "pass@16": 0.4604045860512616, "pass@32": 0.5278180398696859, "pass@64": 0.5974230289076049, "pass@128": 0.6666666666666666 }, "17": { "pass@1": 0.13337725903614459, "pass@2": 0.20455357354615303, "pass@4": 0.28689263634967316, "pass@8": 0.3693797529401078, "pass@16": 0.44313391916126493, "pass@32": 0.5065971685899989, "pass@64": 0.5609107151651793, "pass@128": 0.608433734939759 }, "18": { "pass@1": 0.14690020161290324, "pass@2": 0.21565833756667505, "pass@4": 0.2908736779878321, "pass@8": 0.3619471264143349, "pass@16": 0.42385086814033907, "pass@32": 0.47860570791588575, "pass@64": 0.5266059947925121, "pass@128": 0.5709677419354838 }, "19": { "pass@1": 0.14307968277945618, "pass@2": 0.20948621155172828, "pass@4": 0.2771546254036975, "pass@8": 0.3391845110184821, "pass@16": 0.39618189679402127, "pass@32": 0.45002902801284544, "pass@64": 0.4975762422706243, "pass@128": 0.5347432024169184 }, "20": { "pass@1": 0.11660097597597598, "pass@2": 0.18397235818495664, "pass@4": 0.2672153804535694, "pass@8": 0.3537960139658834, "pass@16": 0.4293593992377488, "pass@32": 0.4897216597246643, "pass@64": 0.5406000531835978, "pass@128": 0.5825825825825826 } } } } } ]