diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18036_metrics.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18036_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9ae862327ee5ae4338359cce8cebcd7d25733c2c --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18036_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18036", + "id": { + "answer_accuracy": 0.2986241319444444, + "avg_response_len": 160.385515625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2986241319444444, + "pass@2": 0.3560722058180293, + "pass@4": 0.4123714342373923, + "pass@8": 0.4685796073680717, + "pass@16": 0.5261624839810676, + "pass@32": 0.5853959103135282, + "pass@64": 0.644963890707408, + "pass@128": 0.7011111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.166390625, + "pass@2": 0.2259514025590554, + "pass@4": 0.2863963988563925, + "pass@8": 0.3482629040513863, + "pass@16": 0.41321660035034496, + "pass@32": 0.48238182073841385, + "pass@64": 0.5576115458586041, + "pass@128": 0.638 + }, + "2": { + "pass@1": 0.990671875, + "pass@2": 0.9935901820866144, + "pass@4": 0.9952708424259467, + "pass@8": 0.9959823235301756, + "pass@16": 0.9963081298372323, + "pass@32": 0.9965811591259394, + "pass@64": 0.9968779527543081, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.4917734375, + "pass@2": 0.5920674212598435, + "pass@4": 0.6712201013310835, + "pass@8": 0.7366605597467424, + "pass@16": 0.7940310747737966, + "pass@32": 0.8437691716985969, + "pass@64": 0.8848637470151715, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.142328125, + "pass@2": 0.1895428149606297, + "pass@4": 0.24196070744281933, + "pass@8": 0.3001219863904964, + "pass@16": 0.3630831070094905, + "pass@32": 0.42937258936494427, + "pass@64": 0.49665674917381397, + "pass@128": 0.563 + }, + "5": { + "pass@1": 0.1859609375, + "pass@2": 0.24305327263779522, + "pass@4": 0.30010716901012346, + "pass@8": 0.3565661151339041, + "pass@16": 0.41086951982193237, + "pass@32": 0.4622424201710075, + "pass@64": 0.5113145302510105, + "pass@128": 0.555 + }, + "6": { + "pass@1": 0.18025, + "pass@2": 0.24015661909448793, + "pass@4": 0.30253015729283816, + "pass@8": 0.36917539613042805, + "pass@16": 0.43996762480457535, + "pass@32": 0.5125727302311399, + "pass@64": 0.5855432171663093, + "pass@128": 0.659 + }, + "7": { + "pass@1": 0.1811171875, + "pass@2": 0.24470780019685043, + "pass@4": 0.3085118221784774, + "pass@8": 0.3723172427071303, + "pass@16": 0.4393734735260297, + "pass@32": 0.5125028024150173, + "pass@64": 0.5887981414335282, + "pass@128": 0.659 + }, + "8": { + "pass@1": 0.1779296875, + "pass@2": 0.2424523868110235, + "pass@4": 0.30925016619797496, + "pass@8": 0.3778363723592985, + "pass@16": 0.44976204475875387, + "pass@32": 0.5269236739085034, + "pass@64": 0.606891191761134, + "pass@128": 0.681 + }, + "9": { + "pass@1": 0.1711953125, + "pass@2": 0.23312795275590525, + "pass@4": 0.2960955434008245, + "pass@8": 0.3602935662629644, + "pass@16": 0.42885078094733525, + "pass@32": 0.502216825168209, + "pass@64": 0.5761179409527961, + "pass@128": 0.644 + } + }, + "per_op_accuracy": { + "10": 0.166390625, + "2": 0.990671875, + "3": 0.4917734375, + "4": 0.142328125, + "5": 0.1859609375, + "6": 0.18025, + "7": 0.1811171875, + "8": 0.1779296875, + "9": 0.1711953125 + }, + "per_op_avg_response_len": { + "10": 192.0604375, + "2": 107.388515625, + "3": 133.7745078125, + "4": 132.2058984375, + "5": 157.1836953125, + "6": 173.123, + "7": 166.8762890625, + "8": 184.4205078125, + "9": 196.4367890625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 192.0604375, + "2": 107.388515625, + "3": 133.7745078125, + "4": 132.2058984375, + "5": 157.1836953125, + "6": 173.123, + "7": 166.8762890625, + "8": 184.4205078125, + "9": 196.4367890625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 344015, + "answer_accuracy": 0.2986241319444444, + "avg_response_len": 160.385515625, + "resp_tokens_sum": 184764114, + "pass_at_k": { + "pass@1": 0.2986241319444444, + "pass@2": 0.3560722058180293, + "pass@4": 0.4123714342373923, + "pass@8": 0.4685796073680717, + "pass@16": 0.5261624839810676, + "pass@32": 0.5853959103135282, + "pass@64": 0.644963890707408, + "pass@128": 0.7011111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.166390625, + "pass@2": 0.2259514025590554, + "pass@4": 0.2863963988563925, + "pass@8": 0.3482629040513863, + "pass@16": 0.41321660035034496, + "pass@32": 0.48238182073841385, + "pass@64": 0.5576115458586041, + "pass@128": 0.638 + }, + "2": { + "pass@1": 0.990671875, + "pass@2": 0.9935901820866144, + "pass@4": 0.9952708424259467, + "pass@8": 0.9959823235301756, + "pass@16": 0.9963081298372323, + "pass@32": 0.9965811591259394, + "pass@64": 0.9968779527543081, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.4917734375, + "pass@2": 0.5920674212598435, + "pass@4": 0.6712201013310835, + "pass@8": 0.7366605597467424, + "pass@16": 0.7940310747737966, + "pass@32": 0.8437691716985969, + "pass@64": 0.8848637470151715, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.142328125, + "pass@2": 0.1895428149606297, + "pass@4": 0.24196070744281933, + "pass@8": 0.3001219863904964, + "pass@16": 0.3630831070094905, + "pass@32": 0.42937258936494427, + "pass@64": 0.49665674917381397, + "pass@128": 0.563 + }, + "5": { + "pass@1": 0.1859609375, + "pass@2": 0.24305327263779522, + "pass@4": 0.30010716901012346, + "pass@8": 0.3565661151339041, + "pass@16": 0.41086951982193237, + "pass@32": 0.4622424201710075, + "pass@64": 0.5113145302510105, + "pass@128": 0.555 + }, + "6": { + "pass@1": 0.18025, + "pass@2": 0.24015661909448793, + "pass@4": 0.30253015729283816, + "pass@8": 0.36917539613042805, + "pass@16": 0.43996762480457535, + "pass@32": 0.5125727302311399, + "pass@64": 0.5855432171663093, + "pass@128": 0.659 + }, + "7": { + "pass@1": 0.1811171875, + "pass@2": 0.24470780019685043, + "pass@4": 0.3085118221784774, + "pass@8": 0.3723172427071303, + "pass@16": 0.4393734735260297, + "pass@32": 0.5125028024150173, + "pass@64": 0.5887981414335282, + "pass@128": 0.659 + }, + "8": { + "pass@1": 0.1779296875, + "pass@2": 0.2424523868110235, + "pass@4": 0.30925016619797496, + "pass@8": 0.3778363723592985, + "pass@16": 0.44976204475875387, + "pass@32": 0.5269236739085034, + "pass@64": 0.606891191761134, + "pass@128": 0.681 + }, + "9": { + "pass@1": 0.1711953125, + "pass@2": 0.23312795275590525, + "pass@4": 0.2960955434008245, + "pass@8": 0.3602935662629644, + "pass@16": 0.42885078094733525, + "pass@32": 0.502216825168209, + "pass@64": 0.5761179409527961, + "pass@128": 0.644 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.13812578125, + "avg_response_len": 230.7461453125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.13812578125, + "pass@2": 0.19947488927165416, + "pass@4": 0.26303507414698396, + "pass@8": 0.3224787618250951, + "pass@16": 0.3764864423923888, + "pass@32": 0.42794615329664565, + "pass@64": 0.4815194974145096, + "pass@128": 0.5411 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1615234375, + "pass@2": 0.22522293307086616, + "pass@4": 0.2913539652230965, + "pass@8": 0.35650526969125756, + "pass@16": 0.4212366708514859, + "pass@32": 0.48704905366123813, + "pass@64": 0.5537880291775845, + "pass@128": 0.621 + }, + "12": { + "pass@1": 0.177640625, + "pass@2": 0.24494746555118121, + "pass@4": 0.3095038772028492, + "pass@8": 0.3681241151088833, + "pass@16": 0.4231968431059328, + "pass@32": 0.47780202700665486, + "pass@64": 0.5342497999364783, + "pass@128": 0.591 + }, + "13": { + "pass@1": 0.14421875, + "pass@2": 0.20472244094488207, + "pass@4": 0.26368166094863116, + "pass@8": 0.3173714718771548, + "pass@16": 0.3698519072480568, + "pass@32": 0.4256450382815248, + "pass@64": 0.4867350069868675, + "pass@128": 0.555 + }, + "14": { + "pass@1": 0.142953125, + "pass@2": 0.20548954232283478, + "pass@4": 0.269536325084364, + "pass@8": 0.3294002337381809, + "pass@16": 0.38382653185693555, + "pass@32": 0.4364604107914041, + "pass@64": 0.49171555804313116, + "pass@128": 0.551 + }, + "15": { + "pass@1": 0.123890625, + "pass@2": 0.18273597440944872, + "pass@4": 0.2469230581177352, + "pass@8": 0.31021002735044045, + "pass@16": 0.3699086146579923, + "pass@32": 0.42638194215710434, + "pass@64": 0.4835918195756341, + "pass@128": 0.545 + }, + "16": { + "pass@1": 0.1468671875, + "pass@2": 0.21468307086614163, + "pass@4": 0.2857840742407197, + "pass@8": 0.35146961344131555, + "pass@16": 0.40863693047698385, + "pass@32": 0.4600975407571595, + "pass@64": 0.512396775641509, + "pass@128": 0.572 + }, + "17": { + "pass@1": 0.133984375, + "pass@2": 0.1954734251968503, + "pass@4": 0.260024570022497, + "pass@8": 0.3201508423744681, + "pass@16": 0.37182972549526555, + "pass@32": 0.41791018039462813, + "pass@64": 0.4650061313740662, + "pass@128": 0.519 + }, + "18": { + "pass@1": 0.121015625, + "pass@2": 0.18066990649606313, + "pass@4": 0.24538407902137208, + "pass@8": 0.3063208445164039, + "pass@16": 0.35994733861579264, + "pass@32": 0.4099684151222696, + "pass@64": 0.4629988412040062, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.117140625, + "pass@2": 0.17400295275590555, + "pass@4": 0.23292370350581162, + "pass@8": 0.28577010302695205, + "pass@16": 0.33105295248792116, + "pass@32": 0.37251698273524714, + "pass@64": 0.4171026028709156, + "pass@128": 0.472 + }, + "20": { + "pass@1": 0.1120234375, + "pass@2": 0.16680118110236225, + "pass@4": 0.2252354281027368, + "pass@8": 0.279465097125832, + "pass@16": 0.3253769091274793, + "pass@32": 0.36562994205924854, + "pass@64": 0.4076104093349335, + "pass@128": 0.46 + } + }, + "per_op_accuracy": { + "11": 0.1615234375, + "12": 0.177640625, + "13": 0.14421875, + "14": 0.142953125, + "15": 0.123890625, + "16": 0.1468671875, + "17": 0.133984375, + "18": 0.121015625, + "19": 0.117140625, + "20": 0.1120234375 + }, + "per_op_avg_response_len": { + "11": 202.6476640625, + "12": 206.102765625, + "13": 214.878890625, + "14": 230.0282421875, + "15": 234.52325, + "16": 248.9111015625, + "17": 246.7481640625, + "18": 251.7229375, + "19": 235.2643515625, + "20": 236.6340859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 202.6476640625, + "12": 206.102765625, + "13": 214.878890625, + "14": 230.0282421875, + "15": 234.52325, + "16": 248.9111015625, + "17": 246.7481640625, + "18": 251.7229375, + "19": 235.2643515625, + "20": 236.6340859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 176801, + "answer_accuracy": 0.13812578125, + "avg_response_len": 230.7461453125, + "resp_tokens_sum": 295355066, + "pass_at_k": { + "pass@1": 0.13812578125, + "pass@2": 0.19947488927165416, + "pass@4": 0.26303507414698396, + "pass@8": 0.3224787618250951, + "pass@16": 0.3764864423923888, + "pass@32": 0.42794615329664565, + "pass@64": 0.4815194974145096, + "pass@128": 0.5411 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1615234375, + "pass@2": 0.22522293307086616, + "pass@4": 0.2913539652230965, + "pass@8": 0.35650526969125756, + "pass@16": 0.4212366708514859, + "pass@32": 0.48704905366123813, + "pass@64": 0.5537880291775845, + "pass@128": 0.621 + }, + "12": { + "pass@1": 0.177640625, + "pass@2": 0.24494746555118121, + "pass@4": 0.3095038772028492, + "pass@8": 0.3681241151088833, + "pass@16": 0.4231968431059328, + "pass@32": 0.47780202700665486, + "pass@64": 0.5342497999364783, + "pass@128": 0.591 + }, + "13": { + "pass@1": 0.14421875, + "pass@2": 0.20472244094488207, + "pass@4": 0.26368166094863116, + "pass@8": 0.3173714718771548, + "pass@16": 0.3698519072480568, + "pass@32": 0.4256450382815248, + "pass@64": 0.4867350069868675, + "pass@128": 0.555 + }, + "14": { + "pass@1": 0.142953125, + "pass@2": 0.20548954232283478, + "pass@4": 0.269536325084364, + "pass@8": 0.3294002337381809, + "pass@16": 0.38382653185693555, + "pass@32": 0.4364604107914041, + "pass@64": 0.49171555804313116, + "pass@128": 0.551 + }, + "15": { + "pass@1": 0.123890625, + "pass@2": 0.18273597440944872, + "pass@4": 0.2469230581177352, + "pass@8": 0.31021002735044045, + "pass@16": 0.3699086146579923, + "pass@32": 0.42638194215710434, + "pass@64": 0.4835918195756341, + "pass@128": 0.545 + }, + "16": { + "pass@1": 0.1468671875, + "pass@2": 0.21468307086614163, + "pass@4": 0.2857840742407197, + "pass@8": 0.35146961344131555, + "pass@16": 0.40863693047698385, + "pass@32": 0.4600975407571595, + "pass@64": 0.512396775641509, + "pass@128": 0.572 + }, + "17": { + "pass@1": 0.133984375, + "pass@2": 0.1954734251968503, + "pass@4": 0.260024570022497, + "pass@8": 0.3201508423744681, + "pass@16": 0.37182972549526555, + "pass@32": 0.41791018039462813, + "pass@64": 0.4650061313740662, + "pass@128": 0.519 + }, + "18": { + "pass@1": 0.121015625, + "pass@2": 0.18066990649606313, + "pass@4": 0.24538407902137208, + "pass@8": 0.3063208445164039, + "pass@16": 0.35994733861579264, + "pass@32": 0.4099684151222696, + "pass@64": 0.4629988412040062, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.117140625, + "pass@2": 0.17400295275590555, + "pass@4": 0.23292370350581162, + "pass@8": 0.28577010302695205, + "pass@16": 0.33105295248792116, + "pass@32": 0.37251698273524714, + "pass@64": 0.4171026028709156, + "pass@128": 0.472 + }, + "20": { + "pass@1": 0.1120234375, + "pass@2": 0.16680118110236225, + "pass@4": 0.2252354281027368, + "pass@8": 0.279465097125832, + "pass@16": 0.3253769091274793, + "pass@32": 0.36562994205924854, + "pass@64": 0.4076104093349335, + "pass@128": 0.46 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.2141513157894737, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.2141513157894737, + "pass@2": 0.2736525655304598, + "pass@4": 0.3337733499792709, + "pass@8": 0.3916844255033439, + "pass@16": 0.44738561998703563, + "pass@32": 0.5025276171467539, + "pass@64": 0.5589405258164144, + "pass@128": 0.6168947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.166390625, + "pass@2": 0.2259514025590554, + "pass@4": 0.2863963988563925, + "pass@8": 0.3482629040513863, + "pass@16": 0.41321660035034496, + "pass@32": 0.48238182073841385, + "pass@64": 0.5576115458586041, + "pass@128": 0.638 + }, + "2": { + "pass@1": 0.990671875, + "pass@2": 0.9935901820866144, + "pass@4": 0.9952708424259467, + "pass@8": 0.9959823235301756, + "pass@16": 0.9963081298372323, + "pass@32": 0.9965811591259394, + "pass@64": 0.9968779527543081, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.4917734375, + "pass@2": 0.5920674212598435, + "pass@4": 0.6712201013310835, + "pass@8": 0.7366605597467424, + "pass@16": 0.7940310747737966, + "pass@32": 0.8437691716985969, + "pass@64": 0.8848637470151715, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.142328125, + "pass@2": 0.1895428149606297, + "pass@4": 0.24196070744281933, + "pass@8": 0.3001219863904964, + "pass@16": 0.3630831070094905, + "pass@32": 0.42937258936494427, + "pass@64": 0.49665674917381397, + "pass@128": 0.563 + }, + "5": { + "pass@1": 0.1859609375, + "pass@2": 0.24305327263779522, + "pass@4": 0.30010716901012346, + "pass@8": 0.3565661151339041, + "pass@16": 0.41086951982193237, + "pass@32": 0.4622424201710075, + "pass@64": 0.5113145302510105, + "pass@128": 0.555 + }, + "6": { + "pass@1": 0.18025, + "pass@2": 0.24015661909448793, + "pass@4": 0.30253015729283816, + "pass@8": 0.36917539613042805, + "pass@16": 0.43996762480457535, + "pass@32": 0.5125727302311399, + "pass@64": 0.5855432171663093, + "pass@128": 0.659 + }, + "7": { + "pass@1": 0.1811171875, + "pass@2": 0.24470780019685043, + "pass@4": 0.3085118221784774, + "pass@8": 0.3723172427071303, + "pass@16": 0.4393734735260297, + "pass@32": 0.5125028024150173, + "pass@64": 0.5887981414335282, + "pass@128": 0.659 + }, + "8": { + "pass@1": 0.1779296875, + "pass@2": 0.2424523868110235, + "pass@4": 0.30925016619797496, + "pass@8": 0.3778363723592985, + "pass@16": 0.44976204475875387, + "pass@32": 0.5269236739085034, + "pass@64": 0.606891191761134, + "pass@128": 0.681 + }, + "9": { + "pass@1": 0.1711953125, + "pass@2": 0.23312795275590525, + "pass@4": 0.2960955434008245, + "pass@8": 0.3602935662629644, + "pass@16": 0.42885078094733525, + "pass@32": 0.502216825168209, + "pass@64": 0.5761179409527961, + "pass@128": 0.644 + }, + "11": { + "pass@1": 0.1615234375, + "pass@2": 0.22522293307086616, + "pass@4": 0.2913539652230965, + "pass@8": 0.35650526969125756, + "pass@16": 0.4212366708514859, + "pass@32": 0.48704905366123813, + "pass@64": 0.5537880291775845, + "pass@128": 0.621 + }, + "12": { + "pass@1": 0.177640625, + "pass@2": 0.24494746555118121, + "pass@4": 0.3095038772028492, + "pass@8": 0.3681241151088833, + "pass@16": 0.4231968431059328, + "pass@32": 0.47780202700665486, + "pass@64": 0.5342497999364783, + "pass@128": 0.591 + }, + "13": { + "pass@1": 0.14421875, + "pass@2": 0.20472244094488207, + "pass@4": 0.26368166094863116, + "pass@8": 0.3173714718771548, + "pass@16": 0.3698519072480568, + "pass@32": 0.4256450382815248, + "pass@64": 0.4867350069868675, + "pass@128": 0.555 + }, + "14": { + "pass@1": 0.142953125, + "pass@2": 0.20548954232283478, + "pass@4": 0.269536325084364, + "pass@8": 0.3294002337381809, + "pass@16": 0.38382653185693555, + "pass@32": 0.4364604107914041, + "pass@64": 0.49171555804313116, + "pass@128": 0.551 + }, + "15": { + "pass@1": 0.123890625, + "pass@2": 0.18273597440944872, + "pass@4": 0.2469230581177352, + "pass@8": 0.31021002735044045, + "pass@16": 0.3699086146579923, + "pass@32": 0.42638194215710434, + "pass@64": 0.4835918195756341, + "pass@128": 0.545 + }, + "16": { + "pass@1": 0.1468671875, + "pass@2": 0.21468307086614163, + "pass@4": 0.2857840742407197, + "pass@8": 0.35146961344131555, + "pass@16": 0.40863693047698385, + "pass@32": 0.4600975407571595, + "pass@64": 0.512396775641509, + "pass@128": 0.572 + }, + "17": { + "pass@1": 0.133984375, + "pass@2": 0.1954734251968503, + "pass@4": 0.260024570022497, + "pass@8": 0.3201508423744681, + "pass@16": 0.37182972549526555, + "pass@32": 0.41791018039462813, + "pass@64": 0.4650061313740662, + "pass@128": 0.519 + }, + "18": { + "pass@1": 0.121015625, + "pass@2": 0.18066990649606313, + "pass@4": 0.24538407902137208, + "pass@8": 0.3063208445164039, + "pass@16": 0.35994733861579264, + "pass@32": 0.4099684151222696, + "pass@64": 0.4629988412040062, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.117140625, + "pass@2": 0.17400295275590555, + "pass@4": 0.23292370350581162, + "pass@8": 0.28577010302695205, + "pass@16": 0.33105295248792116, + "pass@32": 0.37251698273524714, + "pass@64": 0.4171026028709156, + "pass@128": 0.472 + }, + "20": { + "pass@1": 0.1120234375, + "pass@2": 0.16680118110236225, + "pass@4": 0.2252354281027368, + "pass@8": 0.279465097125832, + "pass@16": 0.3253769091274793, + "pass@32": 0.36562994205924854, + "pass@64": 0.4076104093349335, + "pass@128": 0.46 + } + }, + "per_op_accuracy": { + "10": 0.166390625, + "2": 0.990671875, + "3": 0.4917734375, + "4": 0.142328125, + "5": 0.1859609375, + "6": 0.18025, + "7": 0.1811171875, + "8": 0.1779296875, + "9": 0.1711953125, + "11": 0.1615234375, + "12": 0.177640625, + "13": 0.14421875, + "14": 0.142953125, + "15": 0.123890625, + "16": 0.1468671875, + "17": 0.133984375, + "18": 0.121015625, + "19": 0.117140625, + "20": 0.1120234375 + }, + "per_op_avg_response_len": { + "10": 192.0604375, + "2": 107.388515625, + "3": 133.7745078125, + "4": 132.2058984375, + "5": 157.1836953125, + "6": 173.123, + "7": 166.8762890625, + "8": 184.4205078125, + "9": 196.4367890625, + "11": 202.6476640625, + "12": 206.102765625, + "13": 214.878890625, + "14": 230.0282421875, + "15": 234.52325, + "16": 248.9111015625, + "17": 246.7481640625, + "18": 251.7229375, + "19": 235.2643515625, + "20": 236.6340859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 192.0604375, + "2": 107.388515625, + "3": 133.7745078125, + "4": 132.2058984375, + "5": 157.1836953125, + "6": 173.123, + "7": 166.8762890625, + "8": 184.4205078125, + "9": 196.4367890625, + "11": 202.6476640625, + "12": 206.102765625, + "13": 214.878890625, + "14": 230.0282421875, + "15": 234.52325, + "16": 248.9111015625, + "17": 246.7481640625, + "18": 251.7229375, + "19": 235.2643515625, + "20": 236.6340859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 520816, + "answer_accuracy": 0.2141513157894737, + "avg_response_len": 197.41742598684212, + "resp_tokens_sum": 480119180, + "pass_at_k": { + "pass@1": 0.2141513157894737, + "pass@2": 0.2736525655304598, + "pass@4": 0.3337733499792709, + "pass@8": 0.3916844255033439, + "pass@16": 0.44738561998703563, + "pass@32": 0.5025276171467539, + "pass@64": 0.5589405258164144, + "pass@128": 0.6168947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.166390625, + "pass@2": 0.2259514025590554, + "pass@4": 0.2863963988563925, + "pass@8": 0.3482629040513863, + "pass@16": 0.41321660035034496, + "pass@32": 0.48238182073841385, + "pass@64": 0.5576115458586041, + "pass@128": 0.638 + }, + "2": { + "pass@1": 0.990671875, + "pass@2": 0.9935901820866144, + "pass@4": 0.9952708424259467, + "pass@8": 0.9959823235301756, + "pass@16": 0.9963081298372323, + "pass@32": 0.9965811591259394, + "pass@64": 0.9968779527543081, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.4917734375, + "pass@2": 0.5920674212598435, + "pass@4": 0.6712201013310835, + "pass@8": 0.7366605597467424, + "pass@16": 0.7940310747737966, + "pass@32": 0.8437691716985969, + "pass@64": 0.8848637470151715, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.142328125, + "pass@2": 0.1895428149606297, + "pass@4": 0.24196070744281933, + "pass@8": 0.3001219863904964, + "pass@16": 0.3630831070094905, + "pass@32": 0.42937258936494427, + "pass@64": 0.49665674917381397, + "pass@128": 0.563 + }, + "5": { + "pass@1": 0.1859609375, + "pass@2": 0.24305327263779522, + "pass@4": 0.30010716901012346, + "pass@8": 0.3565661151339041, + "pass@16": 0.41086951982193237, + "pass@32": 0.4622424201710075, + "pass@64": 0.5113145302510105, + "pass@128": 0.555 + }, + "6": { + "pass@1": 0.18025, + "pass@2": 0.24015661909448793, + "pass@4": 0.30253015729283816, + "pass@8": 0.36917539613042805, + "pass@16": 0.43996762480457535, + "pass@32": 0.5125727302311399, + "pass@64": 0.5855432171663093, + "pass@128": 0.659 + }, + "7": { + "pass@1": 0.1811171875, + "pass@2": 0.24470780019685043, + "pass@4": 0.3085118221784774, + "pass@8": 0.3723172427071303, + "pass@16": 0.4393734735260297, + "pass@32": 0.5125028024150173, + "pass@64": 0.5887981414335282, + "pass@128": 0.659 + }, + "8": { + "pass@1": 0.1779296875, + "pass@2": 0.2424523868110235, + "pass@4": 0.30925016619797496, + "pass@8": 0.3778363723592985, + "pass@16": 0.44976204475875387, + "pass@32": 0.5269236739085034, + "pass@64": 0.606891191761134, + "pass@128": 0.681 + }, + "9": { + "pass@1": 0.1711953125, + "pass@2": 0.23312795275590525, + "pass@4": 0.2960955434008245, + "pass@8": 0.3602935662629644, + "pass@16": 0.42885078094733525, + "pass@32": 0.502216825168209, + "pass@64": 0.5761179409527961, + "pass@128": 0.644 + }, + "11": { + "pass@1": 0.1615234375, + "pass@2": 0.22522293307086616, + "pass@4": 0.2913539652230965, + "pass@8": 0.35650526969125756, + "pass@16": 0.4212366708514859, + "pass@32": 0.48704905366123813, + "pass@64": 0.5537880291775845, + "pass@128": 0.621 + }, + "12": { + "pass@1": 0.177640625, + "pass@2": 0.24494746555118121, + "pass@4": 0.3095038772028492, + "pass@8": 0.3681241151088833, + "pass@16": 0.4231968431059328, + "pass@32": 0.47780202700665486, + "pass@64": 0.5342497999364783, + "pass@128": 0.591 + }, + "13": { + "pass@1": 0.14421875, + "pass@2": 0.20472244094488207, + "pass@4": 0.26368166094863116, + "pass@8": 0.3173714718771548, + "pass@16": 0.3698519072480568, + "pass@32": 0.4256450382815248, + "pass@64": 0.4867350069868675, + "pass@128": 0.555 + }, + "14": { + "pass@1": 0.142953125, + "pass@2": 0.20548954232283478, + "pass@4": 0.269536325084364, + "pass@8": 0.3294002337381809, + "pass@16": 0.38382653185693555, + "pass@32": 0.4364604107914041, + "pass@64": 0.49171555804313116, + "pass@128": 0.551 + }, + "15": { + "pass@1": 0.123890625, + "pass@2": 0.18273597440944872, + "pass@4": 0.2469230581177352, + "pass@8": 0.31021002735044045, + "pass@16": 0.3699086146579923, + "pass@32": 0.42638194215710434, + "pass@64": 0.4835918195756341, + "pass@128": 0.545 + }, + "16": { + "pass@1": 0.1468671875, + "pass@2": 0.21468307086614163, + "pass@4": 0.2857840742407197, + "pass@8": 0.35146961344131555, + "pass@16": 0.40863693047698385, + "pass@32": 0.4600975407571595, + "pass@64": 0.512396775641509, + "pass@128": 0.572 + }, + "17": { + "pass@1": 0.133984375, + "pass@2": 0.1954734251968503, + "pass@4": 0.260024570022497, + "pass@8": 0.3201508423744681, + "pass@16": 0.37182972549526555, + "pass@32": 0.41791018039462813, + "pass@64": 0.4650061313740662, + "pass@128": 0.519 + }, + "18": { + "pass@1": 0.121015625, + "pass@2": 0.18066990649606313, + "pass@4": 0.24538407902137208, + "pass@8": 0.3063208445164039, + "pass@16": 0.35994733861579264, + "pass@32": 0.4099684151222696, + "pass@64": 0.4629988412040062, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.117140625, + "pass@2": 0.17400295275590555, + "pass@4": 0.23292370350581162, + "pass@8": 0.28577010302695205, + "pass@16": 0.33105295248792116, + "pass@32": 0.37251698273524714, + "pass@64": 0.4171026028709156, + "pass@128": 0.472 + }, + "20": { + "pass@1": 0.1120234375, + "pass@2": 0.16680118110236225, + "pass@4": 0.2252354281027368, + "pass@8": 0.279465097125832, + "pass@16": 0.3253769091274793, + "pass@32": 0.36562994205924854, + "pass@64": 0.4076104093349335, + "pass@128": 0.46 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1e4d66c790300603a661b5590f0afa7b9ce42b61 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive", "results_dir": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18036_id_generations.jsonl", "total": 1152000, "missing_solution": 1, "missing_gold": 0, "parse_fail": 0, "perfect": 127050, "value_mismatch": 715836, "dependency_mismatch": 812741, "answer_mismatch": 807976, "extra_nodes": 708866, "missing_nodes": 1022319, "perfect_with_extra": 8, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 715836, "dependency_mismatch": 812741, "answer_mismatch": 807976, "extra_nodes": 708866, "missing_nodes": 1022319, "perfect": 127050, "perfect_with_extra": 8, "missing_solution": 1}}, "results_path": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/checkpoint-18036_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18036_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 1069621, "dependency_mismatch": 1104202, "answer_mismatch": 1103206, "extra_nodes": 1026153, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 1069621, "dependency_mismatch": 1104202, "answer_mismatch": 1103206, "extra_nodes": 1026153, "missing_nodes": 1280000}}, "results_path": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/checkpoint-18036_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 1, "missing_gold": 0, "parse_fail": 0, "perfect": 127050, "value_mismatch": 1785457, "dependency_mismatch": 1916943, "answer_mismatch": 1911182, "extra_nodes": 1735019, "missing_nodes": 2302319, "perfect_with_extra": 8, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1785457, "dependency_mismatch": 1916943, "answer_mismatch": 1911182, "extra_nodes": 1735019, "missing_nodes": 2302319, "perfect": 127050, "perfect_with_extra": 8, "missing_solution": 1}}}} diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..a0044f827a8a3fc6964176e8bddca7e2761ab166 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "checkpoint-18036_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.988, + "pass@2": 0.994, + "pass@4": 0.995, + "pass@8": 0.996, + "pass@16": 0.997, + "pass@32": 0.997, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.001, + "pass@16": 0.001, + "pass@32": 0.005, + "pass@64": 0.007, + "pass@128": 0.01 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.002, + "pass@32": 0.004, + "pass@64": 0.004, + "pass@128": 0.004 + }, + "5": { + "pass@1": 0.001, + "pass@2": 0.003, + "pass@4": 0.004, + "pass@8": 0.004, + "pass@16": 0.005, + "pass@32": 0.008, + "pass@64": 0.01, + "pass@128": 0.013 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.988, + "3": 0.0, + "4": 0.0, + "5": 0.001, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.994, + "3": 0.0, + "4": 0.001, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.995, + "3": 0.0, + "4": 0.001, + "5": 0.004, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.996, + "3": 0.001, + "4": 0.001, + "5": 0.004, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.997, + "3": 0.001, + "4": 0.002, + "5": 0.005, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.997, + "3": 0.005, + "4": 0.004, + "5": 0.008, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.997, + "3": 0.007, + "4": 0.004, + "5": 0.01, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.997, + "3": 0.01, + "4": 0.004, + "5": 0.013, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.10988888888888888, + "pass@2": 0.11088888888888888, + "pass@4": 0.1111111111111111, + "pass@8": 0.11133333333333334, + "pass@16": 0.11166666666666666, + "pass@32": 0.11266666666666666, + "pass@64": 0.1131111111111111, + "pass@128": 0.11377777777777778 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.988, + "pass@2": 0.994, + "pass@4": 0.995, + "pass@8": 0.996, + "pass@16": 0.997, + "pass@32": 0.997, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.001, + "pass@16": 0.001, + "pass@32": 0.005, + "pass@64": 0.007, + "pass@128": 0.01 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.002, + "pass@32": 0.004, + "pass@64": 0.004, + "pass@128": 0.004 + }, + "5": { + "pass@1": 0.001, + "pass@2": 0.003, + "pass@4": 0.004, + "pass@8": 0.004, + "pass@16": 0.005, + "pass@32": 0.008, + "pass@64": 0.01, + "pass@128": 0.013 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.988, + "3": 0.0, + "4": 0.0, + "5": 0.001, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 0.994, + "3": 0.0, + "4": 0.001, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 0.995, + "3": 0.0, + "4": 0.001, + "5": 0.004, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 0.996, + "3": 0.001, + "4": 0.001, + "5": 0.004, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 0.997, + "3": 0.001, + "4": 0.002, + "5": 0.005, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 0.997, + "3": 0.005, + "4": 0.004, + "5": 0.008, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 0.997, + "3": 0.007, + "4": 0.004, + "5": 0.01, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 0.997, + "3": 0.01, + "4": 0.004, + "5": 0.013, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.10988888888888888, + "pass@2": 0.11088888888888888, + "pass@4": 0.1111111111111111, + "pass@8": 0.11133333333333334, + "pass@16": 0.11166666666666666, + "pass@32": 0.11266666666666666, + "pass@64": 0.1131111111111111, + "pass@128": 0.11377777777777778 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-18036_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/summary.csv b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..00e906cace273a63ab04415b44b6166dfa156742 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18036,0.2986241319444444,0.13812578125,0.2141513157894737,0.0,0.0,0.0,160.385515625,230.7461453125,1152000,1280000,2432000 diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/summary.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2d295c7c124b95bef3d335a9bf762e5075cc39ce --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/autoregressive/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18036", + "id_acc": 0.2986241319444444, + "ood_acc": 0.13812578125, + "total_acc": 0.2141513157894737, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 160.385515625, + "ood_avg_resp_len": 230.7461453125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 520816, + "answer_accuracy": 0.2141513157894737, + "avg_response_len": 197.41742598684212, + "resp_tokens_sum": 480119180, + "pass_at_k": { + "pass@1": 0.2141513157894737, + "pass@2": 0.2736525655304598, + "pass@4": 0.3337733499792709, + "pass@8": 0.3916844255033439, + "pass@16": 0.44738561998703563, + "pass@32": 0.5025276171467539, + "pass@64": 0.5589405258164144, + "pass@128": 0.6168947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.166390625, + "pass@2": 0.2259514025590554, + "pass@4": 0.2863963988563925, + "pass@8": 0.3482629040513863, + "pass@16": 0.41321660035034496, + "pass@32": 0.48238182073841385, + "pass@64": 0.5576115458586041, + "pass@128": 0.638 + }, + "2": { + "pass@1": 0.990671875, + "pass@2": 0.9935901820866144, + "pass@4": 0.9952708424259467, + "pass@8": 0.9959823235301756, + "pass@16": 0.9963081298372323, + "pass@32": 0.9965811591259394, + "pass@64": 0.9968779527543081, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.4917734375, + "pass@2": 0.5920674212598435, + "pass@4": 0.6712201013310835, + "pass@8": 0.7366605597467424, + "pass@16": 0.7940310747737966, + "pass@32": 0.8437691716985969, + "pass@64": 0.8848637470151715, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.142328125, + "pass@2": 0.1895428149606297, + "pass@4": 0.24196070744281933, + "pass@8": 0.3001219863904964, + "pass@16": 0.3630831070094905, + "pass@32": 0.42937258936494427, + "pass@64": 0.49665674917381397, + "pass@128": 0.563 + }, + "5": { + "pass@1": 0.1859609375, + "pass@2": 0.24305327263779522, + "pass@4": 0.30010716901012346, + "pass@8": 0.3565661151339041, + "pass@16": 0.41086951982193237, + "pass@32": 0.4622424201710075, + "pass@64": 0.5113145302510105, + "pass@128": 0.555 + }, + "6": { + "pass@1": 0.18025, + "pass@2": 0.24015661909448793, + "pass@4": 0.30253015729283816, + "pass@8": 0.36917539613042805, + "pass@16": 0.43996762480457535, + "pass@32": 0.5125727302311399, + "pass@64": 0.5855432171663093, + "pass@128": 0.659 + }, + "7": { + "pass@1": 0.1811171875, + "pass@2": 0.24470780019685043, + "pass@4": 0.3085118221784774, + "pass@8": 0.3723172427071303, + "pass@16": 0.4393734735260297, + "pass@32": 0.5125028024150173, + "pass@64": 0.5887981414335282, + "pass@128": 0.659 + }, + "8": { + "pass@1": 0.1779296875, + "pass@2": 0.2424523868110235, + "pass@4": 0.30925016619797496, + "pass@8": 0.3778363723592985, + "pass@16": 0.44976204475875387, + "pass@32": 0.5269236739085034, + "pass@64": 0.606891191761134, + "pass@128": 0.681 + }, + "9": { + "pass@1": 0.1711953125, + "pass@2": 0.23312795275590525, + "pass@4": 0.2960955434008245, + "pass@8": 0.3602935662629644, + "pass@16": 0.42885078094733525, + "pass@32": 0.502216825168209, + "pass@64": 0.5761179409527961, + "pass@128": 0.644 + }, + "11": { + "pass@1": 0.1615234375, + "pass@2": 0.22522293307086616, + "pass@4": 0.2913539652230965, + "pass@8": 0.35650526969125756, + "pass@16": 0.4212366708514859, + "pass@32": 0.48704905366123813, + "pass@64": 0.5537880291775845, + "pass@128": 0.621 + }, + "12": { + "pass@1": 0.177640625, + "pass@2": 0.24494746555118121, + "pass@4": 0.3095038772028492, + "pass@8": 0.3681241151088833, + "pass@16": 0.4231968431059328, + "pass@32": 0.47780202700665486, + "pass@64": 0.5342497999364783, + "pass@128": 0.591 + }, + "13": { + "pass@1": 0.14421875, + "pass@2": 0.20472244094488207, + "pass@4": 0.26368166094863116, + "pass@8": 0.3173714718771548, + "pass@16": 0.3698519072480568, + "pass@32": 0.4256450382815248, + "pass@64": 0.4867350069868675, + "pass@128": 0.555 + }, + "14": { + "pass@1": 0.142953125, + "pass@2": 0.20548954232283478, + "pass@4": 0.269536325084364, + "pass@8": 0.3294002337381809, + "pass@16": 0.38382653185693555, + "pass@32": 0.4364604107914041, + "pass@64": 0.49171555804313116, + "pass@128": 0.551 + }, + "15": { + "pass@1": 0.123890625, + "pass@2": 0.18273597440944872, + "pass@4": 0.2469230581177352, + "pass@8": 0.31021002735044045, + "pass@16": 0.3699086146579923, + "pass@32": 0.42638194215710434, + "pass@64": 0.4835918195756341, + "pass@128": 0.545 + }, + "16": { + "pass@1": 0.1468671875, + "pass@2": 0.21468307086614163, + "pass@4": 0.2857840742407197, + "pass@8": 0.35146961344131555, + "pass@16": 0.40863693047698385, + "pass@32": 0.4600975407571595, + "pass@64": 0.512396775641509, + "pass@128": 0.572 + }, + "17": { + "pass@1": 0.133984375, + "pass@2": 0.1954734251968503, + "pass@4": 0.260024570022497, + "pass@8": 0.3201508423744681, + "pass@16": 0.37182972549526555, + "pass@32": 0.41791018039462813, + "pass@64": 0.4650061313740662, + "pass@128": 0.519 + }, + "18": { + "pass@1": 0.121015625, + "pass@2": 0.18066990649606313, + "pass@4": 0.24538407902137208, + "pass@8": 0.3063208445164039, + "pass@16": 0.35994733861579264, + "pass@32": 0.4099684151222696, + "pass@64": 0.4629988412040062, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.117140625, + "pass@2": 0.17400295275590555, + "pass@4": 0.23292370350581162, + "pass@8": 0.28577010302695205, + "pass@16": 0.33105295248792116, + "pass@32": 0.37251698273524714, + "pass@64": 0.4171026028709156, + "pass@128": 0.472 + }, + "20": { + "pass@1": 0.1120234375, + "pass@2": 0.16680118110236225, + "pass@4": 0.2252354281027368, + "pass@8": 0.279465097125832, + "pass@16": 0.3253769091274793, + "pass@32": 0.36562994205924854, + "pass@64": 0.4076104093349335, + "pass@128": 0.46 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/checkpoint-18036_metrics.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/checkpoint-18036_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..96cd1ac3702f4677cc5161857da7d2b1f9c023cc --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/checkpoint-18036_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18036", + "id": { + "answer_accuracy": 0.2539678819444444, + "avg_response_len": 140.28355381944445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2539678819444444, + "pass@2": 0.31150337653106247, + "pass@4": 0.36996179091155695, + "pass@8": 0.42720659932606386, + "pass@16": 0.4826607528512722, + "pass@32": 0.5362825487283354, + "pass@64": 0.5879061744882603, + "pass@128": 0.6363333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1390390625, + "pass@2": 0.19477608267716526, + "pass@4": 0.2535477263779526, + "pass@8": 0.31036623797543145, + "pass@16": 0.36406534600530344, + "pass@32": 0.4162172276390307, + "pass@64": 0.4682437110617285, + "pass@128": 0.52 + }, + "2": { + "pass@1": 0.816015625, + "pass@2": 0.8640276820866137, + "pass@4": 0.8999403779527553, + "pass@8": 0.9268988385403476, + "pass@16": 0.9467968309560193, + "pass@32": 0.9619097924912621, + "pass@64": 0.9731399839814393, + "pass@128": 0.981 + }, + "3": { + "pass@1": 0.4730390625, + "pass@2": 0.5563313238188982, + "pass@4": 0.6292127719347588, + "pass@8": 0.6967211168191564, + "pass@16": 0.7596796787302875, + "pass@32": 0.8151303463422954, + "pass@64": 0.8596418578325105, + "pass@128": 0.893 + }, + "4": { + "pass@1": 0.121515625, + "pass@2": 0.16837598425196837, + "pass@4": 0.22318688338957593, + "pass@8": 0.28403936570795274, + "pass@16": 0.3471201745075693, + "pass@32": 0.4091258045012325, + "pass@64": 0.4679068956534565, + "pass@128": 0.519 + }, + "5": { + "pass@1": 0.1600625, + "pass@2": 0.21501291830708652, + "pass@4": 0.27184010433070854, + "pass@8": 0.32608437172555776, + "pass@16": 0.38011897134833406, + "pass@32": 0.4367956870909163, + "pass@64": 0.49536962436913273, + "pass@128": 0.551 + }, + "6": { + "pass@1": 0.1438515625, + "pass@2": 0.19685014763779518, + "pass@4": 0.25503478805774243, + "pass@8": 0.3177334994334665, + "pass@16": 0.3854366415099119, + "pass@32": 0.45631431258216576, + "pass@64": 0.5280629662112165, + "pass@128": 0.597 + }, + "7": { + "pass@1": 0.1457265625, + "pass@2": 0.20062942913385823, + "pass@4": 0.26060725421822256, + "pass@8": 0.3233561204386788, + "pass@16": 0.38675781863133524, + "pass@32": 0.4482787138257641, + "pass@64": 0.5082246941939706, + "pass@128": 0.568 + }, + "8": { + "pass@1": 0.1484453125, + "pass@2": 0.21044365157480324, + "pass@4": 0.27630699446944096, + "pass@8": 0.34013712500411397, + "pass@16": 0.40028769322138275, + "pass@32": 0.4584471816928049, + "pass@64": 0.515661887346291, + "pass@128": 0.572 + }, + "9": { + "pass@1": 0.138015625, + "pass@2": 0.19708316929133862, + "pass@4": 0.25997921747281566, + "pass@8": 0.31952271828976914, + "pass@16": 0.3736836207512015, + "pass@32": 0.4243238723895534, + "pass@64": 0.4749039497446198, + "pass@128": 0.526 + } + }, + "per_op_accuracy": { + "10": 0.1390390625, + "2": 0.816015625, + "3": 0.4730390625, + "4": 0.121515625, + "5": 0.1600625, + "6": 0.1438515625, + "7": 0.1457265625, + "8": 0.1484453125, + "9": 0.138015625 + }, + "per_op_avg_response_len": { + "10": 161.1008828125, + "2": 112.0468671875, + "3": 115.9029375, + "4": 124.847546875, + "5": 141.48946875, + "6": 147.19890625, + "7": 142.2397578125, + "8": 155.7766484375, + "9": 161.94896875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 161.1008828125, + "2": 112.0468671875, + "3": 115.9029375, + "4": 124.847546875, + "5": 141.48946875, + "6": 147.19890625, + "7": 142.2397578125, + "8": 155.7766484375, + "9": 161.94896875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 292571, + "answer_accuracy": 0.2539678819444444, + "avg_response_len": 140.28355381944445, + "resp_tokens_sum": 161606654, + "pass_at_k": { + "pass@1": 0.2539678819444444, + "pass@2": 0.31150337653106247, + "pass@4": 0.36996179091155695, + "pass@8": 0.42720659932606386, + "pass@16": 0.4826607528512722, + "pass@32": 0.5362825487283354, + "pass@64": 0.5879061744882603, + "pass@128": 0.6363333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1390390625, + "pass@2": 0.19477608267716526, + "pass@4": 0.2535477263779526, + "pass@8": 0.31036623797543145, + "pass@16": 0.36406534600530344, + "pass@32": 0.4162172276390307, + "pass@64": 0.4682437110617285, + "pass@128": 0.52 + }, + "2": { + "pass@1": 0.816015625, + "pass@2": 0.8640276820866137, + "pass@4": 0.8999403779527553, + "pass@8": 0.9268988385403476, + "pass@16": 0.9467968309560193, + "pass@32": 0.9619097924912621, + "pass@64": 0.9731399839814393, + "pass@128": 0.981 + }, + "3": { + "pass@1": 0.4730390625, + "pass@2": 0.5563313238188982, + "pass@4": 0.6292127719347588, + "pass@8": 0.6967211168191564, + "pass@16": 0.7596796787302875, + "pass@32": 0.8151303463422954, + "pass@64": 0.8596418578325105, + "pass@128": 0.893 + }, + "4": { + "pass@1": 0.121515625, + "pass@2": 0.16837598425196837, + "pass@4": 0.22318688338957593, + "pass@8": 0.28403936570795274, + "pass@16": 0.3471201745075693, + "pass@32": 0.4091258045012325, + "pass@64": 0.4679068956534565, + "pass@128": 0.519 + }, + "5": { + "pass@1": 0.1600625, + "pass@2": 0.21501291830708652, + "pass@4": 0.27184010433070854, + "pass@8": 0.32608437172555776, + "pass@16": 0.38011897134833406, + "pass@32": 0.4367956870909163, + "pass@64": 0.49536962436913273, + "pass@128": 0.551 + }, + "6": { + "pass@1": 0.1438515625, + "pass@2": 0.19685014763779518, + "pass@4": 0.25503478805774243, + "pass@8": 0.3177334994334665, + "pass@16": 0.3854366415099119, + "pass@32": 0.45631431258216576, + "pass@64": 0.5280629662112165, + "pass@128": 0.597 + }, + "7": { + "pass@1": 0.1457265625, + "pass@2": 0.20062942913385823, + "pass@4": 0.26060725421822256, + "pass@8": 0.3233561204386788, + "pass@16": 0.38675781863133524, + "pass@32": 0.4482787138257641, + "pass@64": 0.5082246941939706, + "pass@128": 0.568 + }, + "8": { + "pass@1": 0.1484453125, + "pass@2": 0.21044365157480324, + "pass@4": 0.27630699446944096, + "pass@8": 0.34013712500411397, + "pass@16": 0.40028769322138275, + "pass@32": 0.4584471816928049, + "pass@64": 0.515661887346291, + "pass@128": 0.572 + }, + "9": { + "pass@1": 0.138015625, + "pass@2": 0.19708316929133862, + "pass@4": 0.25997921747281566, + "pass@8": 0.31952271828976914, + "pass@16": 0.3736836207512015, + "pass@32": 0.4243238723895534, + "pass@64": 0.4749039497446198, + "pass@128": 0.526 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.11763046875, + "avg_response_len": 186.45809296875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.11763046875, + "pass@2": 0.17655755413385835, + "pass@4": 0.23935241960067696, + "pass@8": 0.2962832649934929, + "pass@16": 0.34353640059955715, + "pass@32": 0.3819276350857189, + "pass@64": 0.415250334433132, + "pass@128": 0.4492 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1353515625, + "pass@2": 0.19709227362204723, + "pass@4": 0.26180325712410907, + "pass@8": 0.32148119114967133, + "pass@16": 0.3741421028525149, + "pass@32": 0.4213193028440292, + "pass@64": 0.4673698056385636, + "pass@128": 0.516 + }, + "12": { + "pass@1": 0.1424375, + "pass@2": 0.20833083169291336, + "pass@4": 0.2745519436632921, + "pass@8": 0.33219240556801255, + "pass@16": 0.37990862171801504, + "pass@32": 0.4214549194913383, + "pass@64": 0.4604914124534296, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1203828125, + "pass@2": 0.17965551181102368, + "pass@4": 0.24155033173978213, + "pass@8": 0.29627520307163613, + "pass@16": 0.34087442766048354, + "pass@32": 0.37780344084458917, + "pass@64": 0.4120212581585064, + "pass@128": 0.449 + }, + "14": { + "pass@1": 0.124828125, + "pass@2": 0.18710949803149607, + "pass@4": 0.25215789857517795, + "pass@8": 0.3100059216731181, + "pass@16": 0.3571145485284577, + "pass@32": 0.3953640847893672, + "pass@64": 0.4303271787501644, + "pass@128": 0.467 + }, + "15": { + "pass@1": 0.1088359375, + "pass@2": 0.1623578986220471, + "pass@4": 0.22226705989876264, + "pass@8": 0.28059483532399837, + "pass@16": 0.33222815336636113, + "pass@32": 0.3757246194656453, + "pass@64": 0.4139863603129349, + "pass@128": 0.455 + }, + "16": { + "pass@1": 0.1261640625, + "pass@2": 0.19020841535433086, + "pass@4": 0.25936223303337075, + "pass@8": 0.32254732224368526, + "pass@16": 0.37439127444998255, + "pass@32": 0.4150412087203277, + "pass@64": 0.4487921536917901, + "pass@128": 0.481 + }, + "17": { + "pass@1": 0.1153203125, + "pass@2": 0.17593159448818885, + "pass@4": 0.23957447065991735, + "pass@8": 0.29513532413364124, + "pass@16": 0.339505521865133, + "pass@32": 0.37348327354714267, + "pass@64": 0.40013921779086925, + "pass@128": 0.427 + }, + "18": { + "pass@1": 0.1076875, + "pass@2": 0.16468454724409443, + "pass@4": 0.22661534476940354, + "pass@8": 0.2824372639390208, + "pass@16": 0.3281661844114503, + "pass@32": 0.3651101622226279, + "pass@64": 0.3968644654256433, + "pass@128": 0.428 + }, + "19": { + "pass@1": 0.1011875, + "pass@2": 0.15374261811023618, + "pass@4": 0.21080185629921244, + "pass@8": 0.2630630246139682, + "pass@16": 0.3066213308405542, + "pass@32": 0.34058448672793357, + "pass@64": 0.3664017231546799, + "pass@128": 0.39 + }, + "20": { + "pass@1": 0.094109375, + "pass@2": 0.14646235236220462, + "pass@4": 0.20483980024371942, + "pass@8": 0.2591001582181367, + "pass@16": 0.3024118403025776, + "pass@32": 0.33339085220419806, + "pass@64": 0.3561097689547827, + "pass@128": 0.377 + } + }, + "per_op_accuracy": { + "11": 0.1353515625, + "12": 0.1424375, + "13": 0.1203828125, + "14": 0.124828125, + "15": 0.1088359375, + "16": 0.1261640625, + "17": 0.1153203125, + "18": 0.1076875, + "19": 0.1011875, + "20": 0.094109375 + }, + "per_op_avg_response_len": { + "11": 171.16228125, + "12": 176.8704375, + "13": 179.542796875, + "14": 185.5136640625, + "15": 186.4975390625, + "16": 197.9320546875, + "17": 194.2960390625, + "18": 196.7378359375, + "19": 188.2073828125, + "20": 187.8208984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 171.16228125, + "12": 176.8704375, + "13": 179.542796875, + "14": 185.5136640625, + "15": 186.4975390625, + "16": 197.9320546875, + "17": 194.2960390625, + "18": 196.7378359375, + "19": 188.2073828125, + "20": 187.8208984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 150567, + "answer_accuracy": 0.11763046875, + "avg_response_len": 186.45809296875, + "resp_tokens_sum": 238666359, + "pass_at_k": { + "pass@1": 0.11763046875, + "pass@2": 0.17655755413385835, + "pass@4": 0.23935241960067696, + "pass@8": 0.2962832649934929, + "pass@16": 0.34353640059955715, + "pass@32": 0.3819276350857189, + "pass@64": 0.415250334433132, + "pass@128": 0.4492 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1353515625, + "pass@2": 0.19709227362204723, + "pass@4": 0.26180325712410907, + "pass@8": 0.32148119114967133, + "pass@16": 0.3741421028525149, + "pass@32": 0.4213193028440292, + "pass@64": 0.4673698056385636, + "pass@128": 0.516 + }, + "12": { + "pass@1": 0.1424375, + "pass@2": 0.20833083169291336, + "pass@4": 0.2745519436632921, + "pass@8": 0.33219240556801255, + "pass@16": 0.37990862171801504, + "pass@32": 0.4214549194913383, + "pass@64": 0.4604914124534296, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1203828125, + "pass@2": 0.17965551181102368, + "pass@4": 0.24155033173978213, + "pass@8": 0.29627520307163613, + "pass@16": 0.34087442766048354, + "pass@32": 0.37780344084458917, + "pass@64": 0.4120212581585064, + "pass@128": 0.449 + }, + "14": { + "pass@1": 0.124828125, + "pass@2": 0.18710949803149607, + "pass@4": 0.25215789857517795, + "pass@8": 0.3100059216731181, + "pass@16": 0.3571145485284577, + "pass@32": 0.3953640847893672, + "pass@64": 0.4303271787501644, + "pass@128": 0.467 + }, + "15": { + "pass@1": 0.1088359375, + "pass@2": 0.1623578986220471, + "pass@4": 0.22226705989876264, + "pass@8": 0.28059483532399837, + "pass@16": 0.33222815336636113, + "pass@32": 0.3757246194656453, + "pass@64": 0.4139863603129349, + "pass@128": 0.455 + }, + "16": { + "pass@1": 0.1261640625, + "pass@2": 0.19020841535433086, + "pass@4": 0.25936223303337075, + "pass@8": 0.32254732224368526, + "pass@16": 0.37439127444998255, + "pass@32": 0.4150412087203277, + "pass@64": 0.4487921536917901, + "pass@128": 0.481 + }, + "17": { + "pass@1": 0.1153203125, + "pass@2": 0.17593159448818885, + "pass@4": 0.23957447065991735, + "pass@8": 0.29513532413364124, + "pass@16": 0.339505521865133, + "pass@32": 0.37348327354714267, + "pass@64": 0.40013921779086925, + "pass@128": 0.427 + }, + "18": { + "pass@1": 0.1076875, + "pass@2": 0.16468454724409443, + "pass@4": 0.22661534476940354, + "pass@8": 0.2824372639390208, + "pass@16": 0.3281661844114503, + "pass@32": 0.3651101622226279, + "pass@64": 0.3968644654256433, + "pass@128": 0.428 + }, + "19": { + "pass@1": 0.1011875, + "pass@2": 0.15374261811023618, + "pass@4": 0.21080185629921244, + "pass@8": 0.2630630246139682, + "pass@16": 0.3066213308405542, + "pass@32": 0.34058448672793357, + "pass@64": 0.3664017231546799, + "pass@128": 0.39 + }, + "20": { + "pass@1": 0.094109375, + "pass@2": 0.14646235236220462, + "pass@4": 0.20483980024371942, + "pass@8": 0.2591001582181367, + "pass@16": 0.3024118403025776, + "pass@32": 0.33339085220419806, + "pass@64": 0.3561097689547827, + "pass@128": 0.377 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.18221134868421052, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.18221134868421052, + "pass@2": 0.24047925947990395, + "pass@4": 0.30122001653740244, + "pass@8": 0.35829958125628747, + "pass@16": 0.40943740956090274, + "pass@32": 0.455043120495383, + "pass@64": 0.4970346797224109, + "pass@128": 0.5378421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1390390625, + "pass@2": 0.19477608267716526, + "pass@4": 0.2535477263779526, + "pass@8": 0.31036623797543145, + "pass@16": 0.36406534600530344, + "pass@32": 0.4162172276390307, + "pass@64": 0.4682437110617285, + "pass@128": 0.52 + }, + "2": { + "pass@1": 0.816015625, + "pass@2": 0.8640276820866137, + "pass@4": 0.8999403779527553, + "pass@8": 0.9268988385403476, + "pass@16": 0.9467968309560193, + "pass@32": 0.9619097924912621, + "pass@64": 0.9731399839814393, + "pass@128": 0.981 + }, + "3": { + "pass@1": 0.4730390625, + "pass@2": 0.5563313238188982, + "pass@4": 0.6292127719347588, + "pass@8": 0.6967211168191564, + "pass@16": 0.7596796787302875, + "pass@32": 0.8151303463422954, + "pass@64": 0.8596418578325105, + "pass@128": 0.893 + }, + "4": { + "pass@1": 0.121515625, + "pass@2": 0.16837598425196837, + "pass@4": 0.22318688338957593, + "pass@8": 0.28403936570795274, + "pass@16": 0.3471201745075693, + "pass@32": 0.4091258045012325, + "pass@64": 0.4679068956534565, + "pass@128": 0.519 + }, + "5": { + "pass@1": 0.1600625, + "pass@2": 0.21501291830708652, + "pass@4": 0.27184010433070854, + "pass@8": 0.32608437172555776, + "pass@16": 0.38011897134833406, + "pass@32": 0.4367956870909163, + "pass@64": 0.49536962436913273, + "pass@128": 0.551 + }, + "6": { + "pass@1": 0.1438515625, + "pass@2": 0.19685014763779518, + "pass@4": 0.25503478805774243, + "pass@8": 0.3177334994334665, + "pass@16": 0.3854366415099119, + "pass@32": 0.45631431258216576, + "pass@64": 0.5280629662112165, + "pass@128": 0.597 + }, + "7": { + "pass@1": 0.1457265625, + "pass@2": 0.20062942913385823, + "pass@4": 0.26060725421822256, + "pass@8": 0.3233561204386788, + "pass@16": 0.38675781863133524, + "pass@32": 0.4482787138257641, + "pass@64": 0.5082246941939706, + "pass@128": 0.568 + }, + "8": { + "pass@1": 0.1484453125, + "pass@2": 0.21044365157480324, + "pass@4": 0.27630699446944096, + "pass@8": 0.34013712500411397, + "pass@16": 0.40028769322138275, + "pass@32": 0.4584471816928049, + "pass@64": 0.515661887346291, + "pass@128": 0.572 + }, + "9": { + "pass@1": 0.138015625, + "pass@2": 0.19708316929133862, + "pass@4": 0.25997921747281566, + "pass@8": 0.31952271828976914, + "pass@16": 0.3736836207512015, + "pass@32": 0.4243238723895534, + "pass@64": 0.4749039497446198, + "pass@128": 0.526 + }, + "11": { + "pass@1": 0.1353515625, + "pass@2": 0.19709227362204723, + "pass@4": 0.26180325712410907, + "pass@8": 0.32148119114967133, + "pass@16": 0.3741421028525149, + "pass@32": 0.4213193028440292, + "pass@64": 0.4673698056385636, + "pass@128": 0.516 + }, + "12": { + "pass@1": 0.1424375, + "pass@2": 0.20833083169291336, + "pass@4": 0.2745519436632921, + "pass@8": 0.33219240556801255, + "pass@16": 0.37990862171801504, + "pass@32": 0.4214549194913383, + "pass@64": 0.4604914124534296, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1203828125, + "pass@2": 0.17965551181102368, + "pass@4": 0.24155033173978213, + "pass@8": 0.29627520307163613, + "pass@16": 0.34087442766048354, + "pass@32": 0.37780344084458917, + "pass@64": 0.4120212581585064, + "pass@128": 0.449 + }, + "14": { + "pass@1": 0.124828125, + "pass@2": 0.18710949803149607, + "pass@4": 0.25215789857517795, + "pass@8": 0.3100059216731181, + "pass@16": 0.3571145485284577, + "pass@32": 0.3953640847893672, + "pass@64": 0.4303271787501644, + "pass@128": 0.467 + }, + "15": { + "pass@1": 0.1088359375, + "pass@2": 0.1623578986220471, + "pass@4": 0.22226705989876264, + "pass@8": 0.28059483532399837, + "pass@16": 0.33222815336636113, + "pass@32": 0.3757246194656453, + "pass@64": 0.4139863603129349, + "pass@128": 0.455 + }, + "16": { + "pass@1": 0.1261640625, + "pass@2": 0.19020841535433086, + "pass@4": 0.25936223303337075, + "pass@8": 0.32254732224368526, + "pass@16": 0.37439127444998255, + "pass@32": 0.4150412087203277, + "pass@64": 0.4487921536917901, + "pass@128": 0.481 + }, + "17": { + "pass@1": 0.1153203125, + "pass@2": 0.17593159448818885, + "pass@4": 0.23957447065991735, + "pass@8": 0.29513532413364124, + "pass@16": 0.339505521865133, + "pass@32": 0.37348327354714267, + "pass@64": 0.40013921779086925, + "pass@128": 0.427 + }, + "18": { + "pass@1": 0.1076875, + "pass@2": 0.16468454724409443, + "pass@4": 0.22661534476940354, + "pass@8": 0.2824372639390208, + "pass@16": 0.3281661844114503, + "pass@32": 0.3651101622226279, + "pass@64": 0.3968644654256433, + "pass@128": 0.428 + }, + "19": { + "pass@1": 0.1011875, + "pass@2": 0.15374261811023618, + "pass@4": 0.21080185629921244, + "pass@8": 0.2630630246139682, + "pass@16": 0.3066213308405542, + "pass@32": 0.34058448672793357, + "pass@64": 0.3664017231546799, + "pass@128": 0.39 + }, + "20": { + "pass@1": 0.094109375, + "pass@2": 0.14646235236220462, + "pass@4": 0.20483980024371942, + "pass@8": 0.2591001582181367, + "pass@16": 0.3024118403025776, + "pass@32": 0.33339085220419806, + "pass@64": 0.3561097689547827, + "pass@128": 0.377 + } + }, + "per_op_accuracy": { + "10": 0.1390390625, + "2": 0.816015625, + "3": 0.4730390625, + "4": 0.121515625, + "5": 0.1600625, + "6": 0.1438515625, + "7": 0.1457265625, + "8": 0.1484453125, + "9": 0.138015625, + "11": 0.1353515625, + "12": 0.1424375, + "13": 0.1203828125, + "14": 0.124828125, + "15": 0.1088359375, + "16": 0.1261640625, + "17": 0.1153203125, + "18": 0.1076875, + "19": 0.1011875, + "20": 0.094109375 + }, + "per_op_avg_response_len": { + "10": 161.1008828125, + "2": 112.0468671875, + "3": 115.9029375, + "4": 124.847546875, + "5": 141.48946875, + "6": 147.19890625, + "7": 142.2397578125, + "8": 155.7766484375, + "9": 161.94896875, + "11": 171.16228125, + "12": 176.8704375, + "13": 179.542796875, + "14": 185.5136640625, + "15": 186.4975390625, + "16": 197.9320546875, + "17": 194.2960390625, + "18": 196.7378359375, + "19": 188.2073828125, + "20": 187.8208984375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 161.1008828125, + "2": 112.0468671875, + "3": 115.9029375, + "4": 124.847546875, + "5": 141.48946875, + "6": 147.19890625, + "7": 142.2397578125, + "8": 155.7766484375, + "9": 161.94896875, + "11": 171.16228125, + "12": 176.8704375, + "13": 179.542796875, + "14": 185.5136640625, + "15": 186.4975390625, + "16": 197.9320546875, + "17": 194.2960390625, + "18": 196.7378359375, + "19": 188.2073828125, + "20": 187.8208984375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 443138, + "answer_accuracy": 0.18221134868421052, + "avg_response_len": 164.58594284539473, + "resp_tokens_sum": 400273013, + "pass_at_k": { + "pass@1": 0.18221134868421052, + "pass@2": 0.24047925947990395, + "pass@4": 0.30122001653740244, + "pass@8": 0.35829958125628747, + "pass@16": 0.40943740956090274, + "pass@32": 0.455043120495383, + "pass@64": 0.4970346797224109, + "pass@128": 0.5378421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1390390625, + "pass@2": 0.19477608267716526, + "pass@4": 0.2535477263779526, + "pass@8": 0.31036623797543145, + "pass@16": 0.36406534600530344, + "pass@32": 0.4162172276390307, + "pass@64": 0.4682437110617285, + "pass@128": 0.52 + }, + "2": { + "pass@1": 0.816015625, + "pass@2": 0.8640276820866137, + "pass@4": 0.8999403779527553, + "pass@8": 0.9268988385403476, + "pass@16": 0.9467968309560193, + "pass@32": 0.9619097924912621, + "pass@64": 0.9731399839814393, + "pass@128": 0.981 + }, + "3": { + "pass@1": 0.4730390625, + "pass@2": 0.5563313238188982, + "pass@4": 0.6292127719347588, + "pass@8": 0.6967211168191564, + "pass@16": 0.7596796787302875, + "pass@32": 0.8151303463422954, + "pass@64": 0.8596418578325105, + "pass@128": 0.893 + }, + "4": { + "pass@1": 0.121515625, + "pass@2": 0.16837598425196837, + "pass@4": 0.22318688338957593, + "pass@8": 0.28403936570795274, + "pass@16": 0.3471201745075693, + "pass@32": 0.4091258045012325, + "pass@64": 0.4679068956534565, + "pass@128": 0.519 + }, + "5": { + "pass@1": 0.1600625, + "pass@2": 0.21501291830708652, + "pass@4": 0.27184010433070854, + "pass@8": 0.32608437172555776, + "pass@16": 0.38011897134833406, + "pass@32": 0.4367956870909163, + "pass@64": 0.49536962436913273, + "pass@128": 0.551 + }, + "6": { + "pass@1": 0.1438515625, + "pass@2": 0.19685014763779518, + "pass@4": 0.25503478805774243, + "pass@8": 0.3177334994334665, + "pass@16": 0.3854366415099119, + "pass@32": 0.45631431258216576, + "pass@64": 0.5280629662112165, + "pass@128": 0.597 + }, + "7": { + "pass@1": 0.1457265625, + "pass@2": 0.20062942913385823, + "pass@4": 0.26060725421822256, + "pass@8": 0.3233561204386788, + "pass@16": 0.38675781863133524, + "pass@32": 0.4482787138257641, + "pass@64": 0.5082246941939706, + "pass@128": 0.568 + }, + "8": { + "pass@1": 0.1484453125, + "pass@2": 0.21044365157480324, + "pass@4": 0.27630699446944096, + "pass@8": 0.34013712500411397, + "pass@16": 0.40028769322138275, + "pass@32": 0.4584471816928049, + "pass@64": 0.515661887346291, + "pass@128": 0.572 + }, + "9": { + "pass@1": 0.138015625, + "pass@2": 0.19708316929133862, + "pass@4": 0.25997921747281566, + "pass@8": 0.31952271828976914, + "pass@16": 0.3736836207512015, + "pass@32": 0.4243238723895534, + "pass@64": 0.4749039497446198, + "pass@128": 0.526 + }, + "11": { + "pass@1": 0.1353515625, + "pass@2": 0.19709227362204723, + "pass@4": 0.26180325712410907, + "pass@8": 0.32148119114967133, + "pass@16": 0.3741421028525149, + "pass@32": 0.4213193028440292, + "pass@64": 0.4673698056385636, + "pass@128": 0.516 + }, + "12": { + "pass@1": 0.1424375, + "pass@2": 0.20833083169291336, + "pass@4": 0.2745519436632921, + "pass@8": 0.33219240556801255, + "pass@16": 0.37990862171801504, + "pass@32": 0.4214549194913383, + "pass@64": 0.4604914124534296, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1203828125, + "pass@2": 0.17965551181102368, + "pass@4": 0.24155033173978213, + "pass@8": 0.29627520307163613, + "pass@16": 0.34087442766048354, + "pass@32": 0.37780344084458917, + "pass@64": 0.4120212581585064, + "pass@128": 0.449 + }, + "14": { + "pass@1": 0.124828125, + "pass@2": 0.18710949803149607, + "pass@4": 0.25215789857517795, + "pass@8": 0.3100059216731181, + "pass@16": 0.3571145485284577, + "pass@32": 0.3953640847893672, + "pass@64": 0.4303271787501644, + "pass@128": 0.467 + }, + "15": { + "pass@1": 0.1088359375, + "pass@2": 0.1623578986220471, + "pass@4": 0.22226705989876264, + "pass@8": 0.28059483532399837, + "pass@16": 0.33222815336636113, + "pass@32": 0.3757246194656453, + "pass@64": 0.4139863603129349, + "pass@128": 0.455 + }, + "16": { + "pass@1": 0.1261640625, + "pass@2": 0.19020841535433086, + "pass@4": 0.25936223303337075, + "pass@8": 0.32254732224368526, + "pass@16": 0.37439127444998255, + "pass@32": 0.4150412087203277, + "pass@64": 0.4487921536917901, + "pass@128": 0.481 + }, + "17": { + "pass@1": 0.1153203125, + "pass@2": 0.17593159448818885, + "pass@4": 0.23957447065991735, + "pass@8": 0.29513532413364124, + "pass@16": 0.339505521865133, + "pass@32": 0.37348327354714267, + "pass@64": 0.40013921779086925, + "pass@128": 0.427 + }, + "18": { + "pass@1": 0.1076875, + "pass@2": 0.16468454724409443, + "pass@4": 0.22661534476940354, + "pass@8": 0.2824372639390208, + "pass@16": 0.3281661844114503, + "pass@32": 0.3651101622226279, + "pass@64": 0.3968644654256433, + "pass@128": 0.428 + }, + "19": { + "pass@1": 0.1011875, + "pass@2": 0.15374261811023618, + "pass@4": 0.21080185629921244, + "pass@8": 0.2630630246139682, + "pass@16": 0.3066213308405542, + "pass@32": 0.34058448672793357, + "pass@64": 0.3664017231546799, + "pass@128": 0.39 + }, + "20": { + "pass@1": 0.094109375, + "pass@2": 0.14646235236220462, + "pass@4": 0.20483980024371942, + "pass@8": 0.2591001582181367, + "pass@16": 0.3024118403025776, + "pass@32": 0.33339085220419806, + "pass@64": 0.3561097689547827, + "pass@128": 0.377 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c10c19fcb68cef8c71969b48148840dae31531e2 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18036", + "id_acc": 0.2539678819444444, + "ood_acc": 0.11763046875, + "total_acc": 0.18221134868421052, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 140.28355381944445, + "ood_avg_resp_len": 186.45809296875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 443138, + "answer_accuracy": 0.18221134868421052, + "avg_response_len": 164.58594284539473, + "resp_tokens_sum": 400273013, + "pass_at_k": { + "pass@1": 0.18221134868421052, + "pass@2": 0.24047925947990395, + "pass@4": 0.30122001653740244, + "pass@8": 0.35829958125628747, + "pass@16": 0.40943740956090274, + "pass@32": 0.455043120495383, + "pass@64": 0.4970346797224109, + "pass@128": 0.5378421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1390390625, + "pass@2": 0.19477608267716526, + "pass@4": 0.2535477263779526, + "pass@8": 0.31036623797543145, + "pass@16": 0.36406534600530344, + "pass@32": 0.4162172276390307, + "pass@64": 0.4682437110617285, + "pass@128": 0.52 + }, + "2": { + "pass@1": 0.816015625, + "pass@2": 0.8640276820866137, + "pass@4": 0.8999403779527553, + "pass@8": 0.9268988385403476, + "pass@16": 0.9467968309560193, + "pass@32": 0.9619097924912621, + "pass@64": 0.9731399839814393, + "pass@128": 0.981 + }, + "3": { + "pass@1": 0.4730390625, + "pass@2": 0.5563313238188982, + "pass@4": 0.6292127719347588, + "pass@8": 0.6967211168191564, + "pass@16": 0.7596796787302875, + "pass@32": 0.8151303463422954, + "pass@64": 0.8596418578325105, + "pass@128": 0.893 + }, + "4": { + "pass@1": 0.121515625, + "pass@2": 0.16837598425196837, + "pass@4": 0.22318688338957593, + "pass@8": 0.28403936570795274, + "pass@16": 0.3471201745075693, + "pass@32": 0.4091258045012325, + "pass@64": 0.4679068956534565, + "pass@128": 0.519 + }, + "5": { + "pass@1": 0.1600625, + "pass@2": 0.21501291830708652, + "pass@4": 0.27184010433070854, + "pass@8": 0.32608437172555776, + "pass@16": 0.38011897134833406, + "pass@32": 0.4367956870909163, + "pass@64": 0.49536962436913273, + "pass@128": 0.551 + }, + "6": { + "pass@1": 0.1438515625, + "pass@2": 0.19685014763779518, + "pass@4": 0.25503478805774243, + "pass@8": 0.3177334994334665, + "pass@16": 0.3854366415099119, + "pass@32": 0.45631431258216576, + "pass@64": 0.5280629662112165, + "pass@128": 0.597 + }, + "7": { + "pass@1": 0.1457265625, + "pass@2": 0.20062942913385823, + "pass@4": 0.26060725421822256, + "pass@8": 0.3233561204386788, + "pass@16": 0.38675781863133524, + "pass@32": 0.4482787138257641, + "pass@64": 0.5082246941939706, + "pass@128": 0.568 + }, + "8": { + "pass@1": 0.1484453125, + "pass@2": 0.21044365157480324, + "pass@4": 0.27630699446944096, + "pass@8": 0.34013712500411397, + "pass@16": 0.40028769322138275, + "pass@32": 0.4584471816928049, + "pass@64": 0.515661887346291, + "pass@128": 0.572 + }, + "9": { + "pass@1": 0.138015625, + "pass@2": 0.19708316929133862, + "pass@4": 0.25997921747281566, + "pass@8": 0.31952271828976914, + "pass@16": 0.3736836207512015, + "pass@32": 0.4243238723895534, + "pass@64": 0.4749039497446198, + "pass@128": 0.526 + }, + "11": { + "pass@1": 0.1353515625, + "pass@2": 0.19709227362204723, + "pass@4": 0.26180325712410907, + "pass@8": 0.32148119114967133, + "pass@16": 0.3741421028525149, + "pass@32": 0.4213193028440292, + "pass@64": 0.4673698056385636, + "pass@128": 0.516 + }, + "12": { + "pass@1": 0.1424375, + "pass@2": 0.20833083169291336, + "pass@4": 0.2745519436632921, + "pass@8": 0.33219240556801255, + "pass@16": 0.37990862171801504, + "pass@32": 0.4214549194913383, + "pass@64": 0.4604914124534296, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1203828125, + "pass@2": 0.17965551181102368, + "pass@4": 0.24155033173978213, + "pass@8": 0.29627520307163613, + "pass@16": 0.34087442766048354, + "pass@32": 0.37780344084458917, + "pass@64": 0.4120212581585064, + "pass@128": 0.449 + }, + "14": { + "pass@1": 0.124828125, + "pass@2": 0.18710949803149607, + "pass@4": 0.25215789857517795, + "pass@8": 0.3100059216731181, + "pass@16": 0.3571145485284577, + "pass@32": 0.3953640847893672, + "pass@64": 0.4303271787501644, + "pass@128": 0.467 + }, + "15": { + "pass@1": 0.1088359375, + "pass@2": 0.1623578986220471, + "pass@4": 0.22226705989876264, + "pass@8": 0.28059483532399837, + "pass@16": 0.33222815336636113, + "pass@32": 0.3757246194656453, + "pass@64": 0.4139863603129349, + "pass@128": 0.455 + }, + "16": { + "pass@1": 0.1261640625, + "pass@2": 0.19020841535433086, + "pass@4": 0.25936223303337075, + "pass@8": 0.32254732224368526, + "pass@16": 0.37439127444998255, + "pass@32": 0.4150412087203277, + "pass@64": 0.4487921536917901, + "pass@128": 0.481 + }, + "17": { + "pass@1": 0.1153203125, + "pass@2": 0.17593159448818885, + "pass@4": 0.23957447065991735, + "pass@8": 0.29513532413364124, + "pass@16": 0.339505521865133, + "pass@32": 0.37348327354714267, + "pass@64": 0.40013921779086925, + "pass@128": 0.427 + }, + "18": { + "pass@1": 0.1076875, + "pass@2": 0.16468454724409443, + "pass@4": 0.22661534476940354, + "pass@8": 0.2824372639390208, + "pass@16": 0.3281661844114503, + "pass@32": 0.3651101622226279, + "pass@64": 0.3968644654256433, + "pass@128": 0.428 + }, + "19": { + "pass@1": 0.1011875, + "pass@2": 0.15374261811023618, + "pass@4": 0.21080185629921244, + "pass@8": 0.2630630246139682, + "pass@16": 0.3066213308405542, + "pass@32": 0.34058448672793357, + "pass@64": 0.3664017231546799, + "pass@128": 0.39 + }, + "20": { + "pass@1": 0.094109375, + "pass@2": 0.14646235236220462, + "pass@4": 0.20483980024371942, + "pass@8": 0.2591001582181367, + "pass@16": 0.3024118403025776, + "pass@32": 0.33339085220419806, + "pass@64": 0.3561097689547827, + "pass@128": 0.377 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/checkpoint-18036_metrics.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/checkpoint-18036_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d7013d27f85ecfbb2ad4af637cf8ff63ff860e50 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/checkpoint-18036_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18036", + "id": { + "answer_accuracy": 0.302703125, + "avg_response_len": 149.27326909722223, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.302703125, + "pass@2": 0.35166968777340873, + "pass@4": 0.39928614034704357, + "pass@8": 0.4468817410696084, + "pass@16": 0.4953417448608443, + "pass@32": 0.544688246818182, + "pass@64": 0.5940642607047667, + "pass@128": 0.6412222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1630625, + "pass@2": 0.21347588582677168, + "pass@4": 0.2642949128233969, + "pass@8": 0.31614253754271593, + "pass@16": 0.3694169699008454, + "pass@32": 0.4248465293831709, + "pass@64": 0.4837865875765387, + "pass@128": 0.545 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.52928125, + "pass@2": 0.6198010580708667, + "pass@4": 0.6962650952380958, + "pass@8": 0.7637662647889308, + "pass@16": 0.8220606906568304, + "pass@32": 0.8700264578065076, + "pass@64": 0.9065823126823773, + "pass@128": 0.931 + }, + "4": { + "pass@1": 0.1338515625, + "pass@2": 0.1732834645669292, + "pass@4": 0.21536872956505432, + "pass@8": 0.2610682422232634, + "pass@16": 0.3087937305407377, + "pass@32": 0.357163171617181, + "pass@64": 0.4060268924072924, + "pass@128": 0.452 + }, + "5": { + "pass@1": 0.181765625, + "pass@2": 0.22961564960629918, + "pass@4": 0.27551404124484413, + "pass@8": 0.3202793232631325, + "pass@16": 0.364273250405269, + "pass@32": 0.4068932680783598, + "pass@64": 0.4481304829573861, + "pass@128": 0.486 + }, + "6": { + "pass@1": 0.1842734375, + "pass@2": 0.2351831938976377, + "pass@4": 0.28776895697412797, + "pass@8": 0.3442406937577007, + "pass@16": 0.4048360947297735, + "pass@32": 0.46698028111218115, + "pass@64": 0.5288423301720054, + "pass@128": 0.59 + }, + "7": { + "pass@1": 0.183625, + "pass@2": 0.2380729576771655, + "pass@4": 0.29126410292463395, + "pass@8": 0.34462512272993623, + "pass@16": 0.40132947887771286, + "pass@32": 0.462340171523925, + "pass@64": 0.5258123951529414, + "pass@128": 0.588 + }, + "8": { + "pass@1": 0.1748515625, + "pass@2": 0.22887499999999986, + "pass@4": 0.2856317171916009, + "pass@8": 0.3444107444166686, + "pass@16": 0.40623534068656814, + "pass@32": 0.4721663410578541, + "pass@64": 0.5404229646806321, + "pass@128": 0.606 + }, + "9": { + "pass@1": 0.1736171875, + "pass@2": 0.22671998031496055, + "pass@4": 0.2774677071616045, + "pass@8": 0.32740274090403493, + "pass@16": 0.38113014794977174, + "pass@32": 0.4417780007844633, + "pass@64": 0.5069743807137554, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "10": 0.1630625, + "2": 1.0, + "3": 0.52928125, + "4": 0.1338515625, + "5": 0.181765625, + "6": 0.1842734375, + "7": 0.183625, + "8": 0.1748515625, + "9": 0.1736171875 + }, + "per_op_avg_response_len": { + "10": 175.88946875, + "2": 106.7163984375, + "3": 122.8328046875, + "4": 126.4015859375, + "5": 148.3601796875, + "6": 160.4829765625, + "7": 154.3978515625, + "8": 169.5115703125, + "9": 178.8665859375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 175.88946875, + "2": 106.7163984375, + "3": 122.8328046875, + "4": 126.4015859375, + "5": 148.3601796875, + "6": 160.4829765625, + "7": 154.3978515625, + "8": 169.5115703125, + "9": 178.8665859375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 348714, + "answer_accuracy": 0.302703125, + "avg_response_len": 149.27326909722223, + "resp_tokens_sum": 171962806, + "pass_at_k": { + "pass@1": 0.302703125, + "pass@2": 0.35166968777340873, + "pass@4": 0.39928614034704357, + "pass@8": 0.4468817410696084, + "pass@16": 0.4953417448608443, + "pass@32": 0.544688246818182, + "pass@64": 0.5940642607047667, + "pass@128": 0.6412222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1630625, + "pass@2": 0.21347588582677168, + "pass@4": 0.2642949128233969, + "pass@8": 0.31614253754271593, + "pass@16": 0.3694169699008454, + "pass@32": 0.4248465293831709, + "pass@64": 0.4837865875765387, + "pass@128": 0.545 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.52928125, + "pass@2": 0.6198010580708667, + "pass@4": 0.6962650952380958, + "pass@8": 0.7637662647889308, + "pass@16": 0.8220606906568304, + "pass@32": 0.8700264578065076, + "pass@64": 0.9065823126823773, + "pass@128": 0.931 + }, + "4": { + "pass@1": 0.1338515625, + "pass@2": 0.1732834645669292, + "pass@4": 0.21536872956505432, + "pass@8": 0.2610682422232634, + "pass@16": 0.3087937305407377, + "pass@32": 0.357163171617181, + "pass@64": 0.4060268924072924, + "pass@128": 0.452 + }, + "5": { + "pass@1": 0.181765625, + "pass@2": 0.22961564960629918, + "pass@4": 0.27551404124484413, + "pass@8": 0.3202793232631325, + "pass@16": 0.364273250405269, + "pass@32": 0.4068932680783598, + "pass@64": 0.4481304829573861, + "pass@128": 0.486 + }, + "6": { + "pass@1": 0.1842734375, + "pass@2": 0.2351831938976377, + "pass@4": 0.28776895697412797, + "pass@8": 0.3442406937577007, + "pass@16": 0.4048360947297735, + "pass@32": 0.46698028111218115, + "pass@64": 0.5288423301720054, + "pass@128": 0.59 + }, + "7": { + "pass@1": 0.183625, + "pass@2": 0.2380729576771655, + "pass@4": 0.29126410292463395, + "pass@8": 0.34462512272993623, + "pass@16": 0.40132947887771286, + "pass@32": 0.462340171523925, + "pass@64": 0.5258123951529414, + "pass@128": 0.588 + }, + "8": { + "pass@1": 0.1748515625, + "pass@2": 0.22887499999999986, + "pass@4": 0.2856317171916009, + "pass@8": 0.3444107444166686, + "pass@16": 0.40623534068656814, + "pass@32": 0.4721663410578541, + "pass@64": 0.5404229646806321, + "pass@128": 0.606 + }, + "9": { + "pass@1": 0.1736171875, + "pass@2": 0.22671998031496055, + "pass@4": 0.2774677071616045, + "pass@8": 0.32740274090403493, + "pass@16": 0.38113014794977174, + "pass@32": 0.4417780007844633, + "pass@64": 0.5069743807137554, + "pass@128": 0.573 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.13537890625, + "avg_response_len": 210.17079140625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.13537890625, + "pass@2": 0.19357584891732357, + "pass@4": 0.25351657412823647, + "pass@8": 0.309148662218035, + "pass@16": 0.35890781035424235, + "pass@32": 0.4051408578274514, + "pass@64": 0.4518268153075732, + "pass@128": 0.5018 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1568515625, + "pass@2": 0.21649015748031486, + "pass@4": 0.27808150487439043, + "pass@8": 0.3379698366489983, + "pass@16": 0.39541510217003295, + "pass@32": 0.45272628033786183, + "pass@64": 0.5118699617248653, + "pass@128": 0.571 + }, + "12": { + "pass@1": 0.1706015625, + "pass@2": 0.23184793307086585, + "pass@4": 0.2905048297712786, + "pass@8": 0.3439697993410916, + "pass@16": 0.39338020765977705, + "pass@32": 0.4416526597953732, + "pass@64": 0.49099154198546185, + "pass@128": 0.544 + }, + "13": { + "pass@1": 0.1420625, + "pass@2": 0.19764628444881913, + "pass@4": 0.2517021288901386, + "pass@8": 0.299950352104967, + "pass@16": 0.34482794101641207, + "pass@32": 0.39177753118195113, + "pass@64": 0.444609224813425, + "pass@128": 0.505 + }, + "14": { + "pass@1": 0.141546875, + "pass@2": 0.20225910433070854, + "pass@4": 0.2641557774653163, + "pass@8": 0.3203023199092991, + "pass@16": 0.36996708324609545, + "pass@32": 0.41707634769054264, + "pass@64": 0.46608213362971446, + "pass@128": 0.516 + }, + "15": { + "pass@1": 0.12409375, + "pass@2": 0.18118996062992132, + "pass@4": 0.24321870303712, + "pass@8": 0.30384184286284144, + "pass@16": 0.35935116156199787, + "pass@32": 0.4108054431690119, + "pass@64": 0.46359649250178275, + "pass@128": 0.522 + }, + "16": { + "pass@1": 0.1452890625, + "pass@2": 0.20948732775590556, + "pass@4": 0.27656139304461935, + "pass@8": 0.33881602696840146, + "pass@16": 0.3930596363283811, + "pass@32": 0.44199125490498103, + "pass@64": 0.49106896702427183, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.1324921875, + "pass@2": 0.19231680610236243, + "pass@4": 0.2537987380014995, + "pass@8": 0.31021429963217123, + "pass@16": 0.3584734267298275, + "pass@32": 0.4005223851988628, + "pass@64": 0.441732300984372, + "pass@128": 0.485 + }, + "18": { + "pass@1": 0.11909375, + "pass@2": 0.17544918799212592, + "pass@4": 0.23614921475440537, + "pass@8": 0.2937008838552852, + "pass@16": 0.3449585035623667, + "pass@32": 0.3906721303045243, + "pass@64": 0.4346289098535683, + "pass@128": 0.484 + }, + "19": { + "pass@1": 0.115890625, + "pass@2": 0.17102312992126, + "pass@4": 0.22782487017247816, + "pass@8": 0.278406883897172, + "pass@16": 0.3214901014043795, + "pass@32": 0.35996239752451015, + "pass@64": 0.39745835975035204, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1058671875, + "pass@2": 0.1580485974409449, + "pass@4": 0.21316858127109095, + "pass@8": 0.2643143769600797, + "pass@16": 0.3081549398631067, + "pass@32": 0.3442221481669153, + "pass@64": 0.37623026080795646, + "pass@128": 0.41 + } + }, + "per_op_accuracy": { + "11": 0.1568515625, + "12": 0.1706015625, + "13": 0.1420625, + "14": 0.141546875, + "15": 0.12409375, + "16": 0.1452890625, + "17": 0.1324921875, + "18": 0.11909375, + "19": 0.115890625, + "20": 0.1058671875 + }, + "per_op_avg_response_len": { + "11": 186.479703125, + "12": 189.926765625, + "13": 197.90084375, + "14": 209.4173359375, + "15": 214.27421875, + "16": 225.2639140625, + "17": 222.732359375, + "18": 225.9647734375, + "19": 213.84275, + "20": 215.90525 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 186.479703125, + "12": 189.926765625, + "13": 197.90084375, + "14": 209.4173359375, + "15": 214.27421875, + "16": 225.2639140625, + "17": 222.732359375, + "18": 225.9647734375, + "19": 213.84275, + "20": 215.90525 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 173285, + "answer_accuracy": 0.13537890625, + "avg_response_len": 210.17079140625, + "resp_tokens_sum": 269018613, + "pass_at_k": { + "pass@1": 0.13537890625, + "pass@2": 0.19357584891732357, + "pass@4": 0.25351657412823647, + "pass@8": 0.309148662218035, + "pass@16": 0.35890781035424235, + "pass@32": 0.4051408578274514, + "pass@64": 0.4518268153075732, + "pass@128": 0.5018 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1568515625, + "pass@2": 0.21649015748031486, + "pass@4": 0.27808150487439043, + "pass@8": 0.3379698366489983, + "pass@16": 0.39541510217003295, + "pass@32": 0.45272628033786183, + "pass@64": 0.5118699617248653, + "pass@128": 0.571 + }, + "12": { + "pass@1": 0.1706015625, + "pass@2": 0.23184793307086585, + "pass@4": 0.2905048297712786, + "pass@8": 0.3439697993410916, + "pass@16": 0.39338020765977705, + "pass@32": 0.4416526597953732, + "pass@64": 0.49099154198546185, + "pass@128": 0.544 + }, + "13": { + "pass@1": 0.1420625, + "pass@2": 0.19764628444881913, + "pass@4": 0.2517021288901386, + "pass@8": 0.299950352104967, + "pass@16": 0.34482794101641207, + "pass@32": 0.39177753118195113, + "pass@64": 0.444609224813425, + "pass@128": 0.505 + }, + "14": { + "pass@1": 0.141546875, + "pass@2": 0.20225910433070854, + "pass@4": 0.2641557774653163, + "pass@8": 0.3203023199092991, + "pass@16": 0.36996708324609545, + "pass@32": 0.41707634769054264, + "pass@64": 0.46608213362971446, + "pass@128": 0.516 + }, + "15": { + "pass@1": 0.12409375, + "pass@2": 0.18118996062992132, + "pass@4": 0.24321870303712, + "pass@8": 0.30384184286284144, + "pass@16": 0.35935116156199787, + "pass@32": 0.4108054431690119, + "pass@64": 0.46359649250178275, + "pass@128": 0.522 + }, + "16": { + "pass@1": 0.1452890625, + "pass@2": 0.20948732775590556, + "pass@4": 0.27656139304461935, + "pass@8": 0.33881602696840146, + "pass@16": 0.3930596363283811, + "pass@32": 0.44199125490498103, + "pass@64": 0.49106896702427183, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.1324921875, + "pass@2": 0.19231680610236243, + "pass@4": 0.2537987380014995, + "pass@8": 0.31021429963217123, + "pass@16": 0.3584734267298275, + "pass@32": 0.4005223851988628, + "pass@64": 0.441732300984372, + "pass@128": 0.485 + }, + "18": { + "pass@1": 0.11909375, + "pass@2": 0.17544918799212592, + "pass@4": 0.23614921475440537, + "pass@8": 0.2937008838552852, + "pass@16": 0.3449585035623667, + "pass@32": 0.3906721303045243, + "pass@64": 0.4346289098535683, + "pass@128": 0.484 + }, + "19": { + "pass@1": 0.115890625, + "pass@2": 0.17102312992126, + "pass@4": 0.22782487017247816, + "pass@8": 0.278406883897172, + "pass@16": 0.3214901014043795, + "pass@32": 0.35996239752451015, + "pass@64": 0.39745835975035204, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1058671875, + "pass@2": 0.1580485974409449, + "pass@4": 0.21316858127109095, + "pass@8": 0.2643143769600797, + "pass@16": 0.3081549398631067, + "pass@32": 0.3442221481669153, + "pass@64": 0.37623026080795646, + "pass@128": 0.41 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.21463774671052632, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.21463774671052632, + "pass@2": 0.26846240416493955, + "pass@4": 0.3225653160213468, + "pass@8": 0.37439064693719876, + "pass@16": 0.4235344109100096, + "pass@32": 0.47124225261253944, + "pass@64": 0.5192024473378285, + "pass@128": 0.5678421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1630625, + "pass@2": 0.21347588582677168, + "pass@4": 0.2642949128233969, + "pass@8": 0.31614253754271593, + "pass@16": 0.3694169699008454, + "pass@32": 0.4248465293831709, + "pass@64": 0.4837865875765387, + "pass@128": 0.545 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.52928125, + "pass@2": 0.6198010580708667, + "pass@4": 0.6962650952380958, + "pass@8": 0.7637662647889308, + "pass@16": 0.8220606906568304, + "pass@32": 0.8700264578065076, + "pass@64": 0.9065823126823773, + "pass@128": 0.931 + }, + "4": { + "pass@1": 0.1338515625, + "pass@2": 0.1732834645669292, + "pass@4": 0.21536872956505432, + "pass@8": 0.2610682422232634, + "pass@16": 0.3087937305407377, + "pass@32": 0.357163171617181, + "pass@64": 0.4060268924072924, + "pass@128": 0.452 + }, + "5": { + "pass@1": 0.181765625, + "pass@2": 0.22961564960629918, + "pass@4": 0.27551404124484413, + "pass@8": 0.3202793232631325, + "pass@16": 0.364273250405269, + "pass@32": 0.4068932680783598, + "pass@64": 0.4481304829573861, + "pass@128": 0.486 + }, + "6": { + "pass@1": 0.1842734375, + "pass@2": 0.2351831938976377, + "pass@4": 0.28776895697412797, + "pass@8": 0.3442406937577007, + "pass@16": 0.4048360947297735, + "pass@32": 0.46698028111218115, + "pass@64": 0.5288423301720054, + "pass@128": 0.59 + }, + "7": { + "pass@1": 0.183625, + "pass@2": 0.2380729576771655, + "pass@4": 0.29126410292463395, + "pass@8": 0.34462512272993623, + "pass@16": 0.40132947887771286, + "pass@32": 0.462340171523925, + "pass@64": 0.5258123951529414, + "pass@128": 0.588 + }, + "8": { + "pass@1": 0.1748515625, + "pass@2": 0.22887499999999986, + "pass@4": 0.2856317171916009, + "pass@8": 0.3444107444166686, + "pass@16": 0.40623534068656814, + "pass@32": 0.4721663410578541, + "pass@64": 0.5404229646806321, + "pass@128": 0.606 + }, + "9": { + "pass@1": 0.1736171875, + "pass@2": 0.22671998031496055, + "pass@4": 0.2774677071616045, + "pass@8": 0.32740274090403493, + "pass@16": 0.38113014794977174, + "pass@32": 0.4417780007844633, + "pass@64": 0.5069743807137554, + "pass@128": 0.573 + }, + "11": { + "pass@1": 0.1568515625, + "pass@2": 0.21649015748031486, + "pass@4": 0.27808150487439043, + "pass@8": 0.3379698366489983, + "pass@16": 0.39541510217003295, + "pass@32": 0.45272628033786183, + "pass@64": 0.5118699617248653, + "pass@128": 0.571 + }, + "12": { + "pass@1": 0.1706015625, + "pass@2": 0.23184793307086585, + "pass@4": 0.2905048297712786, + "pass@8": 0.3439697993410916, + "pass@16": 0.39338020765977705, + "pass@32": 0.4416526597953732, + "pass@64": 0.49099154198546185, + "pass@128": 0.544 + }, + "13": { + "pass@1": 0.1420625, + "pass@2": 0.19764628444881913, + "pass@4": 0.2517021288901386, + "pass@8": 0.299950352104967, + "pass@16": 0.34482794101641207, + "pass@32": 0.39177753118195113, + "pass@64": 0.444609224813425, + "pass@128": 0.505 + }, + "14": { + "pass@1": 0.141546875, + "pass@2": 0.20225910433070854, + "pass@4": 0.2641557774653163, + "pass@8": 0.3203023199092991, + "pass@16": 0.36996708324609545, + "pass@32": 0.41707634769054264, + "pass@64": 0.46608213362971446, + "pass@128": 0.516 + }, + "15": { + "pass@1": 0.12409375, + "pass@2": 0.18118996062992132, + "pass@4": 0.24321870303712, + "pass@8": 0.30384184286284144, + "pass@16": 0.35935116156199787, + "pass@32": 0.4108054431690119, + "pass@64": 0.46359649250178275, + "pass@128": 0.522 + }, + "16": { + "pass@1": 0.1452890625, + "pass@2": 0.20948732775590556, + "pass@4": 0.27656139304461935, + "pass@8": 0.33881602696840146, + "pass@16": 0.3930596363283811, + "pass@32": 0.44199125490498103, + "pass@64": 0.49106896702427183, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.1324921875, + "pass@2": 0.19231680610236243, + "pass@4": 0.2537987380014995, + "pass@8": 0.31021429963217123, + "pass@16": 0.3584734267298275, + "pass@32": 0.4005223851988628, + "pass@64": 0.441732300984372, + "pass@128": 0.485 + }, + "18": { + "pass@1": 0.11909375, + "pass@2": 0.17544918799212592, + "pass@4": 0.23614921475440537, + "pass@8": 0.2937008838552852, + "pass@16": 0.3449585035623667, + "pass@32": 0.3906721303045243, + "pass@64": 0.4346289098535683, + "pass@128": 0.484 + }, + "19": { + "pass@1": 0.115890625, + "pass@2": 0.17102312992126, + "pass@4": 0.22782487017247816, + "pass@8": 0.278406883897172, + "pass@16": 0.3214901014043795, + "pass@32": 0.35996239752451015, + "pass@64": 0.39745835975035204, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1058671875, + "pass@2": 0.1580485974409449, + "pass@4": 0.21316858127109095, + "pass@8": 0.2643143769600797, + "pass@16": 0.3081549398631067, + "pass@32": 0.3442221481669153, + "pass@64": 0.37623026080795646, + "pass@128": 0.41 + } + }, + "per_op_accuracy": { + "10": 0.1630625, + "2": 1.0, + "3": 0.52928125, + "4": 0.1338515625, + "5": 0.181765625, + "6": 0.1842734375, + "7": 0.183625, + "8": 0.1748515625, + "9": 0.1736171875, + "11": 0.1568515625, + "12": 0.1706015625, + "13": 0.1420625, + "14": 0.141546875, + "15": 0.12409375, + "16": 0.1452890625, + "17": 0.1324921875, + "18": 0.11909375, + "19": 0.115890625, + "20": 0.1058671875 + }, + "per_op_avg_response_len": { + "10": 175.88946875, + "2": 106.7163984375, + "3": 122.8328046875, + "4": 126.4015859375, + "5": 148.3601796875, + "6": 160.4829765625, + "7": 154.3978515625, + "8": 169.5115703125, + "9": 178.8665859375, + "11": 186.479703125, + "12": 189.926765625, + "13": 197.90084375, + "14": 209.4173359375, + "15": 214.27421875, + "16": 225.2639140625, + "17": 222.732359375, + "18": 225.9647734375, + "19": 213.84275, + "20": 215.90525 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 175.88946875, + "2": 106.7163984375, + "3": 122.8328046875, + "4": 126.4015859375, + "5": 148.3601796875, + "6": 160.4829765625, + "7": 154.3978515625, + "8": 169.5115703125, + "9": 178.8665859375, + "11": 186.479703125, + "12": 189.926765625, + "13": 197.90084375, + "14": 209.4173359375, + "15": 214.27421875, + "16": 225.2639140625, + "17": 222.732359375, + "18": 225.9647734375, + "19": 213.84275, + "20": 215.90525 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 521999, + "answer_accuracy": 0.21463774671052632, + "avg_response_len": 181.32459662828947, + "resp_tokens_sum": 440981419, + "pass_at_k": { + "pass@1": 0.21463774671052632, + "pass@2": 0.26846240416493955, + "pass@4": 0.3225653160213468, + "pass@8": 0.37439064693719876, + "pass@16": 0.4235344109100096, + "pass@32": 0.47124225261253944, + "pass@64": 0.5192024473378285, + "pass@128": 0.5678421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1630625, + "pass@2": 0.21347588582677168, + "pass@4": 0.2642949128233969, + "pass@8": 0.31614253754271593, + "pass@16": 0.3694169699008454, + "pass@32": 0.4248465293831709, + "pass@64": 0.4837865875765387, + "pass@128": 0.545 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.52928125, + "pass@2": 0.6198010580708667, + "pass@4": 0.6962650952380958, + "pass@8": 0.7637662647889308, + "pass@16": 0.8220606906568304, + "pass@32": 0.8700264578065076, + "pass@64": 0.9065823126823773, + "pass@128": 0.931 + }, + "4": { + "pass@1": 0.1338515625, + "pass@2": 0.1732834645669292, + "pass@4": 0.21536872956505432, + "pass@8": 0.2610682422232634, + "pass@16": 0.3087937305407377, + "pass@32": 0.357163171617181, + "pass@64": 0.4060268924072924, + "pass@128": 0.452 + }, + "5": { + "pass@1": 0.181765625, + "pass@2": 0.22961564960629918, + "pass@4": 0.27551404124484413, + "pass@8": 0.3202793232631325, + "pass@16": 0.364273250405269, + "pass@32": 0.4068932680783598, + "pass@64": 0.4481304829573861, + "pass@128": 0.486 + }, + "6": { + "pass@1": 0.1842734375, + "pass@2": 0.2351831938976377, + "pass@4": 0.28776895697412797, + "pass@8": 0.3442406937577007, + "pass@16": 0.4048360947297735, + "pass@32": 0.46698028111218115, + "pass@64": 0.5288423301720054, + "pass@128": 0.59 + }, + "7": { + "pass@1": 0.183625, + "pass@2": 0.2380729576771655, + "pass@4": 0.29126410292463395, + "pass@8": 0.34462512272993623, + "pass@16": 0.40132947887771286, + "pass@32": 0.462340171523925, + "pass@64": 0.5258123951529414, + "pass@128": 0.588 + }, + "8": { + "pass@1": 0.1748515625, + "pass@2": 0.22887499999999986, + "pass@4": 0.2856317171916009, + "pass@8": 0.3444107444166686, + "pass@16": 0.40623534068656814, + "pass@32": 0.4721663410578541, + "pass@64": 0.5404229646806321, + "pass@128": 0.606 + }, + "9": { + "pass@1": 0.1736171875, + "pass@2": 0.22671998031496055, + "pass@4": 0.2774677071616045, + "pass@8": 0.32740274090403493, + "pass@16": 0.38113014794977174, + "pass@32": 0.4417780007844633, + "pass@64": 0.5069743807137554, + "pass@128": 0.573 + }, + "11": { + "pass@1": 0.1568515625, + "pass@2": 0.21649015748031486, + "pass@4": 0.27808150487439043, + "pass@8": 0.3379698366489983, + "pass@16": 0.39541510217003295, + "pass@32": 0.45272628033786183, + "pass@64": 0.5118699617248653, + "pass@128": 0.571 + }, + "12": { + "pass@1": 0.1706015625, + "pass@2": 0.23184793307086585, + "pass@4": 0.2905048297712786, + "pass@8": 0.3439697993410916, + "pass@16": 0.39338020765977705, + "pass@32": 0.4416526597953732, + "pass@64": 0.49099154198546185, + "pass@128": 0.544 + }, + "13": { + "pass@1": 0.1420625, + "pass@2": 0.19764628444881913, + "pass@4": 0.2517021288901386, + "pass@8": 0.299950352104967, + "pass@16": 0.34482794101641207, + "pass@32": 0.39177753118195113, + "pass@64": 0.444609224813425, + "pass@128": 0.505 + }, + "14": { + "pass@1": 0.141546875, + "pass@2": 0.20225910433070854, + "pass@4": 0.2641557774653163, + "pass@8": 0.3203023199092991, + "pass@16": 0.36996708324609545, + "pass@32": 0.41707634769054264, + "pass@64": 0.46608213362971446, + "pass@128": 0.516 + }, + "15": { + "pass@1": 0.12409375, + "pass@2": 0.18118996062992132, + "pass@4": 0.24321870303712, + "pass@8": 0.30384184286284144, + "pass@16": 0.35935116156199787, + "pass@32": 0.4108054431690119, + "pass@64": 0.46359649250178275, + "pass@128": 0.522 + }, + "16": { + "pass@1": 0.1452890625, + "pass@2": 0.20948732775590556, + "pass@4": 0.27656139304461935, + "pass@8": 0.33881602696840146, + "pass@16": 0.3930596363283811, + "pass@32": 0.44199125490498103, + "pass@64": 0.49106896702427183, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.1324921875, + "pass@2": 0.19231680610236243, + "pass@4": 0.2537987380014995, + "pass@8": 0.31021429963217123, + "pass@16": 0.3584734267298275, + "pass@32": 0.4005223851988628, + "pass@64": 0.441732300984372, + "pass@128": 0.485 + }, + "18": { + "pass@1": 0.11909375, + "pass@2": 0.17544918799212592, + "pass@4": 0.23614921475440537, + "pass@8": 0.2937008838552852, + "pass@16": 0.3449585035623667, + "pass@32": 0.3906721303045243, + "pass@64": 0.4346289098535683, + "pass@128": 0.484 + }, + "19": { + "pass@1": 0.115890625, + "pass@2": 0.17102312992126, + "pass@4": 0.22782487017247816, + "pass@8": 0.278406883897172, + "pass@16": 0.3214901014043795, + "pass@32": 0.35996239752451015, + "pass@64": 0.39745835975035204, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1058671875, + "pass@2": 0.1580485974409449, + "pass@4": 0.21316858127109095, + "pass@8": 0.2643143769600797, + "pass@16": 0.3081549398631067, + "pass@32": 0.3442221481669153, + "pass@64": 0.37623026080795646, + "pass@128": 0.41 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.csv b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..dcd5350fcecbc17459cd307e5d80ff2b5e9b9015 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18036,0.302703125,0.13537890625,0.21463774671052632,0.0,0.0,0.0,149.27326909722223,210.17079140625,1152000,1280000,2432000 diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..521e610db7b9333f11a0a522eaad2a173b47a61e --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18036", + "id_acc": 0.302703125, + "ood_acc": 0.13537890625, + "total_acc": 0.21463774671052632, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 149.27326909722223, + "ood_avg_resp_len": 210.17079140625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 521999, + "answer_accuracy": 0.21463774671052632, + "avg_response_len": 181.32459662828947, + "resp_tokens_sum": 440981419, + "pass_at_k": { + "pass@1": 0.21463774671052632, + "pass@2": 0.26846240416493955, + "pass@4": 0.3225653160213468, + "pass@8": 0.37439064693719876, + "pass@16": 0.4235344109100096, + "pass@32": 0.47124225261253944, + "pass@64": 0.5192024473378285, + "pass@128": 0.5678421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1630625, + "pass@2": 0.21347588582677168, + "pass@4": 0.2642949128233969, + "pass@8": 0.31614253754271593, + "pass@16": 0.3694169699008454, + "pass@32": 0.4248465293831709, + "pass@64": 0.4837865875765387, + "pass@128": 0.545 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.52928125, + "pass@2": 0.6198010580708667, + "pass@4": 0.6962650952380958, + "pass@8": 0.7637662647889308, + "pass@16": 0.8220606906568304, + "pass@32": 0.8700264578065076, + "pass@64": 0.9065823126823773, + "pass@128": 0.931 + }, + "4": { + "pass@1": 0.1338515625, + "pass@2": 0.1732834645669292, + "pass@4": 0.21536872956505432, + "pass@8": 0.2610682422232634, + "pass@16": 0.3087937305407377, + "pass@32": 0.357163171617181, + "pass@64": 0.4060268924072924, + "pass@128": 0.452 + }, + "5": { + "pass@1": 0.181765625, + "pass@2": 0.22961564960629918, + "pass@4": 0.27551404124484413, + "pass@8": 0.3202793232631325, + "pass@16": 0.364273250405269, + "pass@32": 0.4068932680783598, + "pass@64": 0.4481304829573861, + "pass@128": 0.486 + }, + "6": { + "pass@1": 0.1842734375, + "pass@2": 0.2351831938976377, + "pass@4": 0.28776895697412797, + "pass@8": 0.3442406937577007, + "pass@16": 0.4048360947297735, + "pass@32": 0.46698028111218115, + "pass@64": 0.5288423301720054, + "pass@128": 0.59 + }, + "7": { + "pass@1": 0.183625, + "pass@2": 0.2380729576771655, + "pass@4": 0.29126410292463395, + "pass@8": 0.34462512272993623, + "pass@16": 0.40132947887771286, + "pass@32": 0.462340171523925, + "pass@64": 0.5258123951529414, + "pass@128": 0.588 + }, + "8": { + "pass@1": 0.1748515625, + "pass@2": 0.22887499999999986, + "pass@4": 0.2856317171916009, + "pass@8": 0.3444107444166686, + "pass@16": 0.40623534068656814, + "pass@32": 0.4721663410578541, + "pass@64": 0.5404229646806321, + "pass@128": 0.606 + }, + "9": { + "pass@1": 0.1736171875, + "pass@2": 0.22671998031496055, + "pass@4": 0.2774677071616045, + "pass@8": 0.32740274090403493, + "pass@16": 0.38113014794977174, + "pass@32": 0.4417780007844633, + "pass@64": 0.5069743807137554, + "pass@128": 0.573 + }, + "11": { + "pass@1": 0.1568515625, + "pass@2": 0.21649015748031486, + "pass@4": 0.27808150487439043, + "pass@8": 0.3379698366489983, + "pass@16": 0.39541510217003295, + "pass@32": 0.45272628033786183, + "pass@64": 0.5118699617248653, + "pass@128": 0.571 + }, + "12": { + "pass@1": 0.1706015625, + "pass@2": 0.23184793307086585, + "pass@4": 0.2905048297712786, + "pass@8": 0.3439697993410916, + "pass@16": 0.39338020765977705, + "pass@32": 0.4416526597953732, + "pass@64": 0.49099154198546185, + "pass@128": 0.544 + }, + "13": { + "pass@1": 0.1420625, + "pass@2": 0.19764628444881913, + "pass@4": 0.2517021288901386, + "pass@8": 0.299950352104967, + "pass@16": 0.34482794101641207, + "pass@32": 0.39177753118195113, + "pass@64": 0.444609224813425, + "pass@128": 0.505 + }, + "14": { + "pass@1": 0.141546875, + "pass@2": 0.20225910433070854, + "pass@4": 0.2641557774653163, + "pass@8": 0.3203023199092991, + "pass@16": 0.36996708324609545, + "pass@32": 0.41707634769054264, + "pass@64": 0.46608213362971446, + "pass@128": 0.516 + }, + "15": { + "pass@1": 0.12409375, + "pass@2": 0.18118996062992132, + "pass@4": 0.24321870303712, + "pass@8": 0.30384184286284144, + "pass@16": 0.35935116156199787, + "pass@32": 0.4108054431690119, + "pass@64": 0.46359649250178275, + "pass@128": 0.522 + }, + "16": { + "pass@1": 0.1452890625, + "pass@2": 0.20948732775590556, + "pass@4": 0.27656139304461935, + "pass@8": 0.33881602696840146, + "pass@16": 0.3930596363283811, + "pass@32": 0.44199125490498103, + "pass@64": 0.49106896702427183, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.1324921875, + "pass@2": 0.19231680610236243, + "pass@4": 0.2537987380014995, + "pass@8": 0.31021429963217123, + "pass@16": 0.3584734267298275, + "pass@32": 0.4005223851988628, + "pass@64": 0.441732300984372, + "pass@128": 0.485 + }, + "18": { + "pass@1": 0.11909375, + "pass@2": 0.17544918799212592, + "pass@4": 0.23614921475440537, + "pass@8": 0.2937008838552852, + "pass@16": 0.3449585035623667, + "pass@32": 0.3906721303045243, + "pass@64": 0.4346289098535683, + "pass@128": 0.484 + }, + "19": { + "pass@1": 0.115890625, + "pass@2": 0.17102312992126, + "pass@4": 0.22782487017247816, + "pass@8": 0.278406883897172, + "pass@16": 0.3214901014043795, + "pass@32": 0.35996239752451015, + "pass@64": 0.39745835975035204, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.1058671875, + "pass@2": 0.1580485974409449, + "pass@4": 0.21316858127109095, + "pass@8": 0.2643143769600797, + "pass@16": 0.3081549398631067, + "pass@32": 0.3442221481669153, + "pass@64": 0.37623026080795646, + "pass@128": 0.41 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/checkpoint-18036_metrics.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/checkpoint-18036_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..dfc0a729ad92aad100fe230bb3e069fb9353d3f2 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/checkpoint-18036_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18036", + "id": { + "answer_accuracy": 0.3013984375, + "avg_response_len": 143.08181510416668, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.3013984375, + "pass@2": 0.34618633803587584, + "pass@4": 0.3898556645419372, + "pass@8": 0.4331095794746263, + "pass@16": 0.4768079370188185, + "pass@32": 0.5212313592884714, + "pass@64": 0.5654912723307691, + "pass@128": 0.6075555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1592421875, + "pass@2": 0.20772305610236216, + "pass@4": 0.25526453608923866, + "pass@8": 0.3012407553460957, + "pass@16": 0.3478237348021007, + "pass@32": 0.39664880885279796, + "pass@64": 0.4479086788606886, + "pass@128": 0.499 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.544046875, + "pass@2": 0.6235316190944886, + "pass@4": 0.6941555884889387, + "pass@8": 0.7575594972302828, + "pass@16": 0.8129146470531659, + "pass@32": 0.8597756497658611, + "pass@64": 0.897532543179026, + "pass@128": 0.925 + }, + "4": { + "pass@1": 0.1323984375, + "pass@2": 0.1711313976377952, + "pass@4": 0.2122244947506561, + "pass@8": 0.2554727870148705, + "pass@16": 0.29988200409425175, + "pass@32": 0.3442308953433778, + "pass@64": 0.3875010123975943, + "pass@128": 0.429 + }, + "5": { + "pass@1": 0.1825234375, + "pass@2": 0.22873646653543295, + "pass@4": 0.27240803890138693, + "pass@8": 0.31307946336216225, + "pass@16": 0.35183148299958383, + "pass@32": 0.3894461842248169, + "pass@64": 0.4246001486132662, + "pass@128": 0.453 + }, + "6": { + "pass@1": 0.1844375, + "pass@2": 0.23024064960629906, + "pass@4": 0.2765699090738656, + "pass@8": 0.3271302954428923, + "pass@16": 0.3823534027019154, + "pass@32": 0.4401454906309613, + "pass@64": 0.49908083353119365, + "pass@128": 0.561 + }, + "7": { + "pass@1": 0.176953125, + "pass@2": 0.2268052411417323, + "pass@4": 0.2755276186726656, + "pass@8": 0.32264272309627734, + "pass@16": 0.3704820289853954, + "pass@32": 0.42211152475520264, + "pass@64": 0.477123089082523, + "pass@128": 0.532 + }, + "8": { + "pass@1": 0.1684140625, + "pass@2": 0.2162971210629921, + "pass@4": 0.2658955424634419, + "pass@8": 0.31886406291169084, + "pass@16": 0.3753421843690717, + "pass@32": 0.434354859079704, + "pass@64": 0.49411717498128066, + "pass@128": 0.551 + }, + "9": { + "pass@1": 0.1645703125, + "pass@2": 0.21121149114173213, + "pass@4": 0.25665525243719506, + "pass@8": 0.3019966308672843, + "pass@16": 0.35064194816380945, + "pass@32": 0.40436882094353244, + "pass@64": 0.4615579703313727, + "pass@128": 0.518 + } + }, + "per_op_accuracy": { + "10": 0.1592421875, + "2": 1.0, + "3": 0.544046875, + "4": 0.1323984375, + "5": 0.1825234375, + "6": 0.1844375, + "7": 0.176953125, + "8": 0.1684140625, + "9": 0.1645703125 + }, + "per_op_avg_response_len": { + "10": 166.137296875, + "2": 106.34078125, + "3": 116.6402109375, + "4": 123.73853125, + "5": 144.2860703125, + "6": 152.681921875, + "7": 147.155, + "8": 161.573203125, + "9": 169.1833203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 166.137296875, + "2": 106.34078125, + "3": 116.6402109375, + "4": 123.73853125, + "5": 144.2860703125, + "6": 152.681921875, + "7": 147.155, + "8": 161.573203125, + "9": 169.1833203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 347211, + "answer_accuracy": 0.3013984375, + "avg_response_len": 143.08181510416668, + "resp_tokens_sum": 164830251, + "pass_at_k": { + "pass@1": 0.3013984375, + "pass@2": 0.34618633803587584, + "pass@4": 0.3898556645419372, + "pass@8": 0.4331095794746263, + "pass@16": 0.4768079370188185, + "pass@32": 0.5212313592884714, + "pass@64": 0.5654912723307691, + "pass@128": 0.6075555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1592421875, + "pass@2": 0.20772305610236216, + "pass@4": 0.25526453608923866, + "pass@8": 0.3012407553460957, + "pass@16": 0.3478237348021007, + "pass@32": 0.39664880885279796, + "pass@64": 0.4479086788606886, + "pass@128": 0.499 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.544046875, + "pass@2": 0.6235316190944886, + "pass@4": 0.6941555884889387, + "pass@8": 0.7575594972302828, + "pass@16": 0.8129146470531659, + "pass@32": 0.8597756497658611, + "pass@64": 0.897532543179026, + "pass@128": 0.925 + }, + "4": { + "pass@1": 0.1323984375, + "pass@2": 0.1711313976377952, + "pass@4": 0.2122244947506561, + "pass@8": 0.2554727870148705, + "pass@16": 0.29988200409425175, + "pass@32": 0.3442308953433778, + "pass@64": 0.3875010123975943, + "pass@128": 0.429 + }, + "5": { + "pass@1": 0.1825234375, + "pass@2": 0.22873646653543295, + "pass@4": 0.27240803890138693, + "pass@8": 0.31307946336216225, + "pass@16": 0.35183148299958383, + "pass@32": 0.3894461842248169, + "pass@64": 0.4246001486132662, + "pass@128": 0.453 + }, + "6": { + "pass@1": 0.1844375, + "pass@2": 0.23024064960629906, + "pass@4": 0.2765699090738656, + "pass@8": 0.3271302954428923, + "pass@16": 0.3823534027019154, + "pass@32": 0.4401454906309613, + "pass@64": 0.49908083353119365, + "pass@128": 0.561 + }, + "7": { + "pass@1": 0.176953125, + "pass@2": 0.2268052411417323, + "pass@4": 0.2755276186726656, + "pass@8": 0.32264272309627734, + "pass@16": 0.3704820289853954, + "pass@32": 0.42211152475520264, + "pass@64": 0.477123089082523, + "pass@128": 0.532 + }, + "8": { + "pass@1": 0.1684140625, + "pass@2": 0.2162971210629921, + "pass@4": 0.2658955424634419, + "pass@8": 0.31886406291169084, + "pass@16": 0.3753421843690717, + "pass@32": 0.434354859079704, + "pass@64": 0.49411717498128066, + "pass@128": 0.551 + }, + "9": { + "pass@1": 0.1645703125, + "pass@2": 0.21121149114173213, + "pass@4": 0.25665525243719506, + "pass@8": 0.3019966308672843, + "pass@16": 0.35064194816380945, + "pass@32": 0.40436882094353244, + "pass@64": 0.4615579703313727, + "pass@128": 0.518 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.13433359375, + "avg_response_len": 196.4847953125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.13433359375, + "pass@2": 0.18849779773622113, + "pass@4": 0.24367690543682238, + "pass@8": 0.2949828593778681, + "pass@16": 0.3411553588633753, + "pass@32": 0.383746956013363, + "pass@64": 0.425921664231556, + "pass@128": 0.4702 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1600859375, + "pass@2": 0.21548031496062972, + "pass@4": 0.2706893131796023, + "pass@8": 0.3231774330405756, + "pass@16": 0.3725548916337546, + "pass@32": 0.42074269712866724, + "pass@64": 0.46975063763889346, + "pass@128": 0.521 + }, + "12": { + "pass@1": 0.162421875, + "pass@2": 0.21815649606299212, + "pass@4": 0.273061603580802, + "pass@8": 0.3238364142759003, + "pass@16": 0.37060905439189873, + "pass@32": 0.4151891740113344, + "pass@64": 0.4593731471228355, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1389765625, + "pass@2": 0.18828223425196838, + "pass@4": 0.23619160376827888, + "pass@8": 0.2806350469549499, + "pass@16": 0.3233458543828078, + "pass@32": 0.3675383511680379, + "pass@64": 0.4148052729669554, + "pass@128": 0.463 + }, + "14": { + "pass@1": 0.13959375, + "pass@2": 0.19457344980314956, + "pass@4": 0.2501803342707158, + "pass@8": 0.30221649281300034, + "pass@16": 0.35030926745908336, + "pass@32": 0.3967151621944879, + "pass@64": 0.4444123835148911, + "pass@128": 0.493 + }, + "15": { + "pass@1": 0.1236171875, + "pass@2": 0.1764902805118111, + "pass@4": 0.2318053493625793, + "pass@8": 0.28467220477037813, + "pass@16": 0.3335075031590315, + "pass@32": 0.37923753527046705, + "pass@64": 0.4248889725202785, + "pass@128": 0.473 + }, + "16": { + "pass@1": 0.1456484375, + "pass@2": 0.2079420521653543, + "pass@4": 0.27151239491938495, + "pass@8": 0.3292299127334275, + "pass@16": 0.37950110445976687, + "pass@32": 0.42543712400323264, + "pass@64": 0.47100500681297097, + "pass@128": 0.52 + }, + "17": { + "pass@1": 0.12915625, + "pass@2": 0.18509018208661418, + "pass@4": 0.2442903911698535, + "pass@8": 0.3009775405788867, + "pass@16": 0.34962310350361114, + "pass@32": 0.38791881679904056, + "pass@64": 0.4216590089189286, + "pass@128": 0.458 + }, + "18": { + "pass@1": 0.1213203125, + "pass@2": 0.17376377952755906, + "pass@4": 0.22896580718035223, + "pass@8": 0.2808072699232147, + "pass@16": 0.3265065620637678, + "pass@32": 0.36735639811045945, + "pass@64": 0.40726548312163546, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.114671875, + "pass@2": 0.1660674212598425, + "pass@4": 0.2184041025496812, + "pass@8": 0.26563625308460304, + "pass@16": 0.3076778841212495, + "pass@32": 0.34615382617388063, + "pass@64": 0.3835141854426309, + "pass@128": 0.425 + }, + "20": { + "pass@1": 0.10784375, + "pass@2": 0.15913176673228346, + "pass@4": 0.21166815438695144, + "pass@8": 0.25864002560370153, + "pass@16": 0.2979183634587359, + "pass@32": 0.33118047527404426, + "pass@64": 0.36254254425558835, + "pass@128": 0.398 + } + }, + "per_op_accuracy": { + "11": 0.1600859375, + "12": 0.162421875, + "13": 0.1389765625, + "14": 0.13959375, + "15": 0.1236171875, + "16": 0.1456484375, + "17": 0.12915625, + "18": 0.1213203125, + "19": 0.114671875, + "20": 0.10784375 + }, + "per_op_avg_response_len": { + "11": 176.529625, + "12": 181.8053984375, + "13": 187.584625, + "14": 195.7017265625, + "15": 200.0345, + "16": 210.587484375, + "17": 205.8522734375, + "18": 209.2062578125, + "19": 197.86275, + "20": 199.6833125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 176.529625, + "12": 181.8053984375, + "13": 187.584625, + "14": 195.7017265625, + "15": 200.0345, + "16": 210.587484375, + "17": 205.8522734375, + "18": 209.2062578125, + "19": 197.86275, + "20": 199.6833125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 171947, + "answer_accuracy": 0.13433359375, + "avg_response_len": 196.4847953125, + "resp_tokens_sum": 251500538, + "pass_at_k": { + "pass@1": 0.13433359375, + "pass@2": 0.18849779773622113, + "pass@4": 0.24367690543682238, + "pass@8": 0.2949828593778681, + "pass@16": 0.3411553588633753, + "pass@32": 0.383746956013363, + "pass@64": 0.425921664231556, + "pass@128": 0.4702 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1600859375, + "pass@2": 0.21548031496062972, + "pass@4": 0.2706893131796023, + "pass@8": 0.3231774330405756, + "pass@16": 0.3725548916337546, + "pass@32": 0.42074269712866724, + "pass@64": 0.46975063763889346, + "pass@128": 0.521 + }, + "12": { + "pass@1": 0.162421875, + "pass@2": 0.21815649606299212, + "pass@4": 0.273061603580802, + "pass@8": 0.3238364142759003, + "pass@16": 0.37060905439189873, + "pass@32": 0.4151891740113344, + "pass@64": 0.4593731471228355, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1389765625, + "pass@2": 0.18828223425196838, + "pass@4": 0.23619160376827888, + "pass@8": 0.2806350469549499, + "pass@16": 0.3233458543828078, + "pass@32": 0.3675383511680379, + "pass@64": 0.4148052729669554, + "pass@128": 0.463 + }, + "14": { + "pass@1": 0.13959375, + "pass@2": 0.19457344980314956, + "pass@4": 0.2501803342707158, + "pass@8": 0.30221649281300034, + "pass@16": 0.35030926745908336, + "pass@32": 0.3967151621944879, + "pass@64": 0.4444123835148911, + "pass@128": 0.493 + }, + "15": { + "pass@1": 0.1236171875, + "pass@2": 0.1764902805118111, + "pass@4": 0.2318053493625793, + "pass@8": 0.28467220477037813, + "pass@16": 0.3335075031590315, + "pass@32": 0.37923753527046705, + "pass@64": 0.4248889725202785, + "pass@128": 0.473 + }, + "16": { + "pass@1": 0.1456484375, + "pass@2": 0.2079420521653543, + "pass@4": 0.27151239491938495, + "pass@8": 0.3292299127334275, + "pass@16": 0.37950110445976687, + "pass@32": 0.42543712400323264, + "pass@64": 0.47100500681297097, + "pass@128": 0.52 + }, + "17": { + "pass@1": 0.12915625, + "pass@2": 0.18509018208661418, + "pass@4": 0.2442903911698535, + "pass@8": 0.3009775405788867, + "pass@16": 0.34962310350361114, + "pass@32": 0.38791881679904056, + "pass@64": 0.4216590089189286, + "pass@128": 0.458 + }, + "18": { + "pass@1": 0.1213203125, + "pass@2": 0.17376377952755906, + "pass@4": 0.22896580718035223, + "pass@8": 0.2808072699232147, + "pass@16": 0.3265065620637678, + "pass@32": 0.36735639811045945, + "pass@64": 0.40726548312163546, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.114671875, + "pass@2": 0.1660674212598425, + "pass@4": 0.2184041025496812, + "pass@8": 0.26563625308460304, + "pass@16": 0.3076778841212495, + "pass@32": 0.34615382617388063, + "pass@64": 0.3835141854426309, + "pass@128": 0.425 + }, + "20": { + "pass@1": 0.10784375, + "pass@2": 0.15913176673228346, + "pass@4": 0.21166815438695144, + "pass@8": 0.25864002560370153, + "pass@16": 0.2979183634587359, + "pass@32": 0.33118047527404426, + "pass@64": 0.36254254425558835, + "pass@128": 0.398 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.21346957236842104, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.21346957236842104, + "pass@2": 0.2631923694571082, + "pass@4": 0.3129194755392376, + "pass@8": 0.3604113057394877, + "pass@16": 0.4054118432528026, + "pass@32": 0.4488711470384168, + "pass@64": 0.4920335838575066, + "pass@128": 0.5352631578947369 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1592421875, + "pass@2": 0.20772305610236216, + "pass@4": 0.25526453608923866, + "pass@8": 0.3012407553460957, + "pass@16": 0.3478237348021007, + "pass@32": 0.39664880885279796, + "pass@64": 0.4479086788606886, + "pass@128": 0.499 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.544046875, + "pass@2": 0.6235316190944886, + "pass@4": 0.6941555884889387, + "pass@8": 0.7575594972302828, + "pass@16": 0.8129146470531659, + "pass@32": 0.8597756497658611, + "pass@64": 0.897532543179026, + "pass@128": 0.925 + }, + "4": { + "pass@1": 0.1323984375, + "pass@2": 0.1711313976377952, + "pass@4": 0.2122244947506561, + "pass@8": 0.2554727870148705, + "pass@16": 0.29988200409425175, + "pass@32": 0.3442308953433778, + "pass@64": 0.3875010123975943, + "pass@128": 0.429 + }, + "5": { + "pass@1": 0.1825234375, + "pass@2": 0.22873646653543295, + "pass@4": 0.27240803890138693, + "pass@8": 0.31307946336216225, + "pass@16": 0.35183148299958383, + "pass@32": 0.3894461842248169, + "pass@64": 0.4246001486132662, + "pass@128": 0.453 + }, + "6": { + "pass@1": 0.1844375, + "pass@2": 0.23024064960629906, + "pass@4": 0.2765699090738656, + "pass@8": 0.3271302954428923, + "pass@16": 0.3823534027019154, + "pass@32": 0.4401454906309613, + "pass@64": 0.49908083353119365, + "pass@128": 0.561 + }, + "7": { + "pass@1": 0.176953125, + "pass@2": 0.2268052411417323, + "pass@4": 0.2755276186726656, + "pass@8": 0.32264272309627734, + "pass@16": 0.3704820289853954, + "pass@32": 0.42211152475520264, + "pass@64": 0.477123089082523, + "pass@128": 0.532 + }, + "8": { + "pass@1": 0.1684140625, + "pass@2": 0.2162971210629921, + "pass@4": 0.2658955424634419, + "pass@8": 0.31886406291169084, + "pass@16": 0.3753421843690717, + "pass@32": 0.434354859079704, + "pass@64": 0.49411717498128066, + "pass@128": 0.551 + }, + "9": { + "pass@1": 0.1645703125, + "pass@2": 0.21121149114173213, + "pass@4": 0.25665525243719506, + "pass@8": 0.3019966308672843, + "pass@16": 0.35064194816380945, + "pass@32": 0.40436882094353244, + "pass@64": 0.4615579703313727, + "pass@128": 0.518 + }, + "11": { + "pass@1": 0.1600859375, + "pass@2": 0.21548031496062972, + "pass@4": 0.2706893131796023, + "pass@8": 0.3231774330405756, + "pass@16": 0.3725548916337546, + "pass@32": 0.42074269712866724, + "pass@64": 0.46975063763889346, + "pass@128": 0.521 + }, + "12": { + "pass@1": 0.162421875, + "pass@2": 0.21815649606299212, + "pass@4": 0.273061603580802, + "pass@8": 0.3238364142759003, + "pass@16": 0.37060905439189873, + "pass@32": 0.4151891740113344, + "pass@64": 0.4593731471228355, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1389765625, + "pass@2": 0.18828223425196838, + "pass@4": 0.23619160376827888, + "pass@8": 0.2806350469549499, + "pass@16": 0.3233458543828078, + "pass@32": 0.3675383511680379, + "pass@64": 0.4148052729669554, + "pass@128": 0.463 + }, + "14": { + "pass@1": 0.13959375, + "pass@2": 0.19457344980314956, + "pass@4": 0.2501803342707158, + "pass@8": 0.30221649281300034, + "pass@16": 0.35030926745908336, + "pass@32": 0.3967151621944879, + "pass@64": 0.4444123835148911, + "pass@128": 0.493 + }, + "15": { + "pass@1": 0.1236171875, + "pass@2": 0.1764902805118111, + "pass@4": 0.2318053493625793, + "pass@8": 0.28467220477037813, + "pass@16": 0.3335075031590315, + "pass@32": 0.37923753527046705, + "pass@64": 0.4248889725202785, + "pass@128": 0.473 + }, + "16": { + "pass@1": 0.1456484375, + "pass@2": 0.2079420521653543, + "pass@4": 0.27151239491938495, + "pass@8": 0.3292299127334275, + "pass@16": 0.37950110445976687, + "pass@32": 0.42543712400323264, + "pass@64": 0.47100500681297097, + "pass@128": 0.52 + }, + "17": { + "pass@1": 0.12915625, + "pass@2": 0.18509018208661418, + "pass@4": 0.2442903911698535, + "pass@8": 0.3009775405788867, + "pass@16": 0.34962310350361114, + "pass@32": 0.38791881679904056, + "pass@64": 0.4216590089189286, + "pass@128": 0.458 + }, + "18": { + "pass@1": 0.1213203125, + "pass@2": 0.17376377952755906, + "pass@4": 0.22896580718035223, + "pass@8": 0.2808072699232147, + "pass@16": 0.3265065620637678, + "pass@32": 0.36735639811045945, + "pass@64": 0.40726548312163546, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.114671875, + "pass@2": 0.1660674212598425, + "pass@4": 0.2184041025496812, + "pass@8": 0.26563625308460304, + "pass@16": 0.3076778841212495, + "pass@32": 0.34615382617388063, + "pass@64": 0.3835141854426309, + "pass@128": 0.425 + }, + "20": { + "pass@1": 0.10784375, + "pass@2": 0.15913176673228346, + "pass@4": 0.21166815438695144, + "pass@8": 0.25864002560370153, + "pass@16": 0.2979183634587359, + "pass@32": 0.33118047527404426, + "pass@64": 0.36254254425558835, + "pass@128": 0.398 + } + }, + "per_op_accuracy": { + "10": 0.1592421875, + "2": 1.0, + "3": 0.544046875, + "4": 0.1323984375, + "5": 0.1825234375, + "6": 0.1844375, + "7": 0.176953125, + "8": 0.1684140625, + "9": 0.1645703125, + "11": 0.1600859375, + "12": 0.162421875, + "13": 0.1389765625, + "14": 0.13959375, + "15": 0.1236171875, + "16": 0.1456484375, + "17": 0.12915625, + "18": 0.1213203125, + "19": 0.114671875, + "20": 0.10784375 + }, + "per_op_avg_response_len": { + "10": 166.137296875, + "2": 106.34078125, + "3": 116.6402109375, + "4": 123.73853125, + "5": 144.2860703125, + "6": 152.681921875, + "7": 147.155, + "8": 161.573203125, + "9": 169.1833203125, + "11": 176.529625, + "12": 181.8053984375, + "13": 187.584625, + "14": 195.7017265625, + "15": 200.0345, + "16": 210.587484375, + "17": 205.8522734375, + "18": 209.2062578125, + "19": 197.86275, + "20": 199.6833125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 166.137296875, + "2": 106.34078125, + "3": 116.6402109375, + "4": 123.73853125, + "5": 144.2860703125, + "6": 152.681921875, + "7": 147.155, + "8": 161.573203125, + "9": 169.1833203125, + "11": 176.529625, + "12": 181.8053984375, + "13": 187.584625, + "14": 195.7017265625, + "15": 200.0345, + "16": 210.587484375, + "17": 205.8522734375, + "18": 209.2062578125, + "19": 197.86275, + "20": 199.6833125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 519158, + "answer_accuracy": 0.21346957236842104, + "avg_response_len": 171.18864679276317, + "resp_tokens_sum": 416330789, + "pass_at_k": { + "pass@1": 0.21346957236842104, + "pass@2": 0.2631923694571082, + "pass@4": 0.3129194755392376, + "pass@8": 0.3604113057394877, + "pass@16": 0.4054118432528026, + "pass@32": 0.4488711470384168, + "pass@64": 0.4920335838575066, + "pass@128": 0.5352631578947369 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1592421875, + "pass@2": 0.20772305610236216, + "pass@4": 0.25526453608923866, + "pass@8": 0.3012407553460957, + "pass@16": 0.3478237348021007, + "pass@32": 0.39664880885279796, + "pass@64": 0.4479086788606886, + "pass@128": 0.499 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.544046875, + "pass@2": 0.6235316190944886, + "pass@4": 0.6941555884889387, + "pass@8": 0.7575594972302828, + "pass@16": 0.8129146470531659, + "pass@32": 0.8597756497658611, + "pass@64": 0.897532543179026, + "pass@128": 0.925 + }, + "4": { + "pass@1": 0.1323984375, + "pass@2": 0.1711313976377952, + "pass@4": 0.2122244947506561, + "pass@8": 0.2554727870148705, + "pass@16": 0.29988200409425175, + "pass@32": 0.3442308953433778, + "pass@64": 0.3875010123975943, + "pass@128": 0.429 + }, + "5": { + "pass@1": 0.1825234375, + "pass@2": 0.22873646653543295, + "pass@4": 0.27240803890138693, + "pass@8": 0.31307946336216225, + "pass@16": 0.35183148299958383, + "pass@32": 0.3894461842248169, + "pass@64": 0.4246001486132662, + "pass@128": 0.453 + }, + "6": { + "pass@1": 0.1844375, + "pass@2": 0.23024064960629906, + "pass@4": 0.2765699090738656, + "pass@8": 0.3271302954428923, + "pass@16": 0.3823534027019154, + "pass@32": 0.4401454906309613, + "pass@64": 0.49908083353119365, + "pass@128": 0.561 + }, + "7": { + "pass@1": 0.176953125, + "pass@2": 0.2268052411417323, + "pass@4": 0.2755276186726656, + "pass@8": 0.32264272309627734, + "pass@16": 0.3704820289853954, + "pass@32": 0.42211152475520264, + "pass@64": 0.477123089082523, + "pass@128": 0.532 + }, + "8": { + "pass@1": 0.1684140625, + "pass@2": 0.2162971210629921, + "pass@4": 0.2658955424634419, + "pass@8": 0.31886406291169084, + "pass@16": 0.3753421843690717, + "pass@32": 0.434354859079704, + "pass@64": 0.49411717498128066, + "pass@128": 0.551 + }, + "9": { + "pass@1": 0.1645703125, + "pass@2": 0.21121149114173213, + "pass@4": 0.25665525243719506, + "pass@8": 0.3019966308672843, + "pass@16": 0.35064194816380945, + "pass@32": 0.40436882094353244, + "pass@64": 0.4615579703313727, + "pass@128": 0.518 + }, + "11": { + "pass@1": 0.1600859375, + "pass@2": 0.21548031496062972, + "pass@4": 0.2706893131796023, + "pass@8": 0.3231774330405756, + "pass@16": 0.3725548916337546, + "pass@32": 0.42074269712866724, + "pass@64": 0.46975063763889346, + "pass@128": 0.521 + }, + "12": { + "pass@1": 0.162421875, + "pass@2": 0.21815649606299212, + "pass@4": 0.273061603580802, + "pass@8": 0.3238364142759003, + "pass@16": 0.37060905439189873, + "pass@32": 0.4151891740113344, + "pass@64": 0.4593731471228355, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1389765625, + "pass@2": 0.18828223425196838, + "pass@4": 0.23619160376827888, + "pass@8": 0.2806350469549499, + "pass@16": 0.3233458543828078, + "pass@32": 0.3675383511680379, + "pass@64": 0.4148052729669554, + "pass@128": 0.463 + }, + "14": { + "pass@1": 0.13959375, + "pass@2": 0.19457344980314956, + "pass@4": 0.2501803342707158, + "pass@8": 0.30221649281300034, + "pass@16": 0.35030926745908336, + "pass@32": 0.3967151621944879, + "pass@64": 0.4444123835148911, + "pass@128": 0.493 + }, + "15": { + "pass@1": 0.1236171875, + "pass@2": 0.1764902805118111, + "pass@4": 0.2318053493625793, + "pass@8": 0.28467220477037813, + "pass@16": 0.3335075031590315, + "pass@32": 0.37923753527046705, + "pass@64": 0.4248889725202785, + "pass@128": 0.473 + }, + "16": { + "pass@1": 0.1456484375, + "pass@2": 0.2079420521653543, + "pass@4": 0.27151239491938495, + "pass@8": 0.3292299127334275, + "pass@16": 0.37950110445976687, + "pass@32": 0.42543712400323264, + "pass@64": 0.47100500681297097, + "pass@128": 0.52 + }, + "17": { + "pass@1": 0.12915625, + "pass@2": 0.18509018208661418, + "pass@4": 0.2442903911698535, + "pass@8": 0.3009775405788867, + "pass@16": 0.34962310350361114, + "pass@32": 0.38791881679904056, + "pass@64": 0.4216590089189286, + "pass@128": 0.458 + }, + "18": { + "pass@1": 0.1213203125, + "pass@2": 0.17376377952755906, + "pass@4": 0.22896580718035223, + "pass@8": 0.2808072699232147, + "pass@16": 0.3265065620637678, + "pass@32": 0.36735639811045945, + "pass@64": 0.40726548312163546, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.114671875, + "pass@2": 0.1660674212598425, + "pass@4": 0.2184041025496812, + "pass@8": 0.26563625308460304, + "pass@16": 0.3076778841212495, + "pass@32": 0.34615382617388063, + "pass@64": 0.3835141854426309, + "pass@128": 0.425 + }, + "20": { + "pass@1": 0.10784375, + "pass@2": 0.15913176673228346, + "pass@4": 0.21166815438695144, + "pass@8": 0.25864002560370153, + "pass@16": 0.2979183634587359, + "pass@32": 0.33118047527404426, + "pass@64": 0.36254254425558835, + "pass@128": 0.398 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.csv b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f4f7e2bc4bd8540353c11009644b5e74520a9095 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18036,0.3013984375,0.13433359375,0.21346957236842104,0.0,0.0,0.0,143.08181510416668,196.4847953125,1152000,1280000,2432000 diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..993d4ab2e39ccae9555894beabc419f8962321cd --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18036", + "id_acc": 0.3013984375, + "ood_acc": 0.13433359375, + "total_acc": 0.21346957236842104, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 143.08181510416668, + "ood_avg_resp_len": 196.4847953125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 519158, + "answer_accuracy": 0.21346957236842104, + "avg_response_len": 171.18864679276317, + "resp_tokens_sum": 416330789, + "pass_at_k": { + "pass@1": 0.21346957236842104, + "pass@2": 0.2631923694571082, + "pass@4": 0.3129194755392376, + "pass@8": 0.3604113057394877, + "pass@16": 0.4054118432528026, + "pass@32": 0.4488711470384168, + "pass@64": 0.4920335838575066, + "pass@128": 0.5352631578947369 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1592421875, + "pass@2": 0.20772305610236216, + "pass@4": 0.25526453608923866, + "pass@8": 0.3012407553460957, + "pass@16": 0.3478237348021007, + "pass@32": 0.39664880885279796, + "pass@64": 0.4479086788606886, + "pass@128": 0.499 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.544046875, + "pass@2": 0.6235316190944886, + "pass@4": 0.6941555884889387, + "pass@8": 0.7575594972302828, + "pass@16": 0.8129146470531659, + "pass@32": 0.8597756497658611, + "pass@64": 0.897532543179026, + "pass@128": 0.925 + }, + "4": { + "pass@1": 0.1323984375, + "pass@2": 0.1711313976377952, + "pass@4": 0.2122244947506561, + "pass@8": 0.2554727870148705, + "pass@16": 0.29988200409425175, + "pass@32": 0.3442308953433778, + "pass@64": 0.3875010123975943, + "pass@128": 0.429 + }, + "5": { + "pass@1": 0.1825234375, + "pass@2": 0.22873646653543295, + "pass@4": 0.27240803890138693, + "pass@8": 0.31307946336216225, + "pass@16": 0.35183148299958383, + "pass@32": 0.3894461842248169, + "pass@64": 0.4246001486132662, + "pass@128": 0.453 + }, + "6": { + "pass@1": 0.1844375, + "pass@2": 0.23024064960629906, + "pass@4": 0.2765699090738656, + "pass@8": 0.3271302954428923, + "pass@16": 0.3823534027019154, + "pass@32": 0.4401454906309613, + "pass@64": 0.49908083353119365, + "pass@128": 0.561 + }, + "7": { + "pass@1": 0.176953125, + "pass@2": 0.2268052411417323, + "pass@4": 0.2755276186726656, + "pass@8": 0.32264272309627734, + "pass@16": 0.3704820289853954, + "pass@32": 0.42211152475520264, + "pass@64": 0.477123089082523, + "pass@128": 0.532 + }, + "8": { + "pass@1": 0.1684140625, + "pass@2": 0.2162971210629921, + "pass@4": 0.2658955424634419, + "pass@8": 0.31886406291169084, + "pass@16": 0.3753421843690717, + "pass@32": 0.434354859079704, + "pass@64": 0.49411717498128066, + "pass@128": 0.551 + }, + "9": { + "pass@1": 0.1645703125, + "pass@2": 0.21121149114173213, + "pass@4": 0.25665525243719506, + "pass@8": 0.3019966308672843, + "pass@16": 0.35064194816380945, + "pass@32": 0.40436882094353244, + "pass@64": 0.4615579703313727, + "pass@128": 0.518 + }, + "11": { + "pass@1": 0.1600859375, + "pass@2": 0.21548031496062972, + "pass@4": 0.2706893131796023, + "pass@8": 0.3231774330405756, + "pass@16": 0.3725548916337546, + "pass@32": 0.42074269712866724, + "pass@64": 0.46975063763889346, + "pass@128": 0.521 + }, + "12": { + "pass@1": 0.162421875, + "pass@2": 0.21815649606299212, + "pass@4": 0.273061603580802, + "pass@8": 0.3238364142759003, + "pass@16": 0.37060905439189873, + "pass@32": 0.4151891740113344, + "pass@64": 0.4593731471228355, + "pass@128": 0.502 + }, + "13": { + "pass@1": 0.1389765625, + "pass@2": 0.18828223425196838, + "pass@4": 0.23619160376827888, + "pass@8": 0.2806350469549499, + "pass@16": 0.3233458543828078, + "pass@32": 0.3675383511680379, + "pass@64": 0.4148052729669554, + "pass@128": 0.463 + }, + "14": { + "pass@1": 0.13959375, + "pass@2": 0.19457344980314956, + "pass@4": 0.2501803342707158, + "pass@8": 0.30221649281300034, + "pass@16": 0.35030926745908336, + "pass@32": 0.3967151621944879, + "pass@64": 0.4444123835148911, + "pass@128": 0.493 + }, + "15": { + "pass@1": 0.1236171875, + "pass@2": 0.1764902805118111, + "pass@4": 0.2318053493625793, + "pass@8": 0.28467220477037813, + "pass@16": 0.3335075031590315, + "pass@32": 0.37923753527046705, + "pass@64": 0.4248889725202785, + "pass@128": 0.473 + }, + "16": { + "pass@1": 0.1456484375, + "pass@2": 0.2079420521653543, + "pass@4": 0.27151239491938495, + "pass@8": 0.3292299127334275, + "pass@16": 0.37950110445976687, + "pass@32": 0.42543712400323264, + "pass@64": 0.47100500681297097, + "pass@128": 0.52 + }, + "17": { + "pass@1": 0.12915625, + "pass@2": 0.18509018208661418, + "pass@4": 0.2442903911698535, + "pass@8": 0.3009775405788867, + "pass@16": 0.34962310350361114, + "pass@32": 0.38791881679904056, + "pass@64": 0.4216590089189286, + "pass@128": 0.458 + }, + "18": { + "pass@1": 0.1213203125, + "pass@2": 0.17376377952755906, + "pass@4": 0.22896580718035223, + "pass@8": 0.2808072699232147, + "pass@16": 0.3265065620637678, + "pass@32": 0.36735639811045945, + "pass@64": 0.40726548312163546, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.114671875, + "pass@2": 0.1660674212598425, + "pass@4": 0.2184041025496812, + "pass@8": 0.26563625308460304, + "pass@16": 0.3076778841212495, + "pass@32": 0.34615382617388063, + "pass@64": 0.3835141854426309, + "pass@128": 0.425 + }, + "20": { + "pass@1": 0.10784375, + "pass@2": 0.15913176673228346, + "pass@4": 0.21166815438695144, + "pass@8": 0.25864002560370153, + "pass@16": 0.2979183634587359, + "pass@32": 0.33118047527404426, + "pass@64": 0.36254254425558835, + "pass@128": 0.398 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/checkpoint-18036_metrics.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/checkpoint-18036_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5fa8261c19ee4c49e0de40c79ab2d19fc3054810 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/checkpoint-18036_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18036", + "id": { + "answer_accuracy": 0.29051475694444445, + "avg_response_len": 142.94665972222222, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.29051475694444445, + "pass@2": 0.34386664752843976, + "pass@4": 0.39664924225097326, + "pass@8": 0.44844192680726014, + "pass@16": 0.499307176185741, + "pass@32": 0.5488681487182293, + "pass@64": 0.5961773347563447, + "pass@128": 0.6401111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1478984375, + "pass@2": 0.19923855807086627, + "pass@4": 0.25262500703037083, + "pass@8": 0.30650540226228934, + "pass@16": 0.35994912682498653, + "pass@32": 0.41290081123119726, + "pass@64": 0.4673808844065095, + "pass@128": 0.523 + }, + "2": { + "pass@1": 0.9587890625, + "pass@2": 0.9759959399606299, + "pass@4": 0.9859024143232096, + "pass@8": 0.9921983842033142, + "pass@16": 0.9959833778773702, + "pass@32": 0.9979885223455225, + "pass@64": 0.9988623805075786, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.5236328125, + "pass@2": 0.6070394931102369, + "pass@4": 0.6788192813085867, + "pass@8": 0.7424375231451571, + "pass@16": 0.7995801911522875, + "pass@32": 0.8495603915797477, + "pass@64": 0.8884488903916155, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.1328125, + "pass@2": 0.1752502460629921, + "pass@4": 0.2221315449943755, + "pass@8": 0.2728284436391267, + "pass@16": 0.3264936988844912, + "pass@32": 0.38123826139965056, + "pass@64": 0.43536047324376165, + "pass@128": 0.487 + }, + "5": { + "pass@1": 0.1868515625, + "pass@2": 0.24461724901574805, + "pass@4": 0.3004909116985375, + "pass@8": 0.35149775683040163, + "pass@16": 0.3993127693843177, + "pass@32": 0.4462225810693977, + "pass@64": 0.49302187438693307, + "pass@128": 0.538 + }, + "6": { + "pass@1": 0.1715625, + "pass@2": 0.22371198326771635, + "pass@4": 0.27955061801649767, + "pass@8": 0.34080354755334136, + "pass@16": 0.4062840574005593, + "pass@32": 0.47319170709674596, + "pass@64": 0.5371530434190255, + "pass@128": 0.596 + }, + "7": { + "pass@1": 0.1707265625, + "pass@2": 0.22783009350393704, + "pass@4": 0.2871581517622791, + "pass@8": 0.34684211803384596, + "pass@16": 0.40588472724342184, + "pass@32": 0.46268461053160126, + "pass@64": 0.5158797090415914, + "pass@128": 0.566 + }, + "8": { + "pass@1": 0.16203125, + "pass@2": 0.22241843011811016, + "pass@4": 0.28555184289463803, + "pass@8": 0.3488563330532779, + "pass@16": 0.410999142354151, + "pass@32": 0.470813795785859, + "pass@64": 0.5283060687421842, + "pass@128": 0.583 + }, + "9": { + "pass@1": 0.160328125, + "pass@2": 0.21869783464566916, + "pass@4": 0.27761340823022107, + "pass@8": 0.3340078325444927, + "pass@16": 0.3892774945499858, + "pass@32": 0.44521265742436017, + "pass@64": 0.5011826886679033, + "pass@128": 0.555 + } + }, + "per_op_accuracy": { + "10": 0.1478984375, + "2": 0.9587890625, + "3": 0.5236328125, + "4": 0.1328125, + "5": 0.1868515625, + "6": 0.1715625, + "7": 0.1707265625, + "8": 0.16203125, + "9": 0.160328125 + }, + "per_op_avg_response_len": { + "10": 165.2404453125, + "2": 107.311953125, + "3": 116.4744765625, + "4": 124.00365625, + "5": 145.1775, + "6": 152.1250234375, + "7": 146.6023984375, + "8": 161.05984375, + "9": 168.524640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 165.2404453125, + "2": 107.311953125, + "3": 116.4744765625, + "4": 124.00365625, + "5": 145.1775, + "6": 152.1250234375, + "7": 146.6023984375, + "8": 161.05984375, + "9": 168.524640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 334673, + "answer_accuracy": 0.29051475694444445, + "avg_response_len": 142.94665972222222, + "resp_tokens_sum": 164674552, + "pass_at_k": { + "pass@1": 0.29051475694444445, + "pass@2": 0.34386664752843976, + "pass@4": 0.39664924225097326, + "pass@8": 0.44844192680726014, + "pass@16": 0.499307176185741, + "pass@32": 0.5488681487182293, + "pass@64": 0.5961773347563447, + "pass@128": 0.6401111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1478984375, + "pass@2": 0.19923855807086627, + "pass@4": 0.25262500703037083, + "pass@8": 0.30650540226228934, + "pass@16": 0.35994912682498653, + "pass@32": 0.41290081123119726, + "pass@64": 0.4673808844065095, + "pass@128": 0.523 + }, + "2": { + "pass@1": 0.9587890625, + "pass@2": 0.9759959399606299, + "pass@4": 0.9859024143232096, + "pass@8": 0.9921983842033142, + "pass@16": 0.9959833778773702, + "pass@32": 0.9979885223455225, + "pass@64": 0.9988623805075786, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.5236328125, + "pass@2": 0.6070394931102369, + "pass@4": 0.6788192813085867, + "pass@8": 0.7424375231451571, + "pass@16": 0.7995801911522875, + "pass@32": 0.8495603915797477, + "pass@64": 0.8884488903916155, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.1328125, + "pass@2": 0.1752502460629921, + "pass@4": 0.2221315449943755, + "pass@8": 0.2728284436391267, + "pass@16": 0.3264936988844912, + "pass@32": 0.38123826139965056, + "pass@64": 0.43536047324376165, + "pass@128": 0.487 + }, + "5": { + "pass@1": 0.1868515625, + "pass@2": 0.24461724901574805, + "pass@4": 0.3004909116985375, + "pass@8": 0.35149775683040163, + "pass@16": 0.3993127693843177, + "pass@32": 0.4462225810693977, + "pass@64": 0.49302187438693307, + "pass@128": 0.538 + }, + "6": { + "pass@1": 0.1715625, + "pass@2": 0.22371198326771635, + "pass@4": 0.27955061801649767, + "pass@8": 0.34080354755334136, + "pass@16": 0.4062840574005593, + "pass@32": 0.47319170709674596, + "pass@64": 0.5371530434190255, + "pass@128": 0.596 + }, + "7": { + "pass@1": 0.1707265625, + "pass@2": 0.22783009350393704, + "pass@4": 0.2871581517622791, + "pass@8": 0.34684211803384596, + "pass@16": 0.40588472724342184, + "pass@32": 0.46268461053160126, + "pass@64": 0.5158797090415914, + "pass@128": 0.566 + }, + "8": { + "pass@1": 0.16203125, + "pass@2": 0.22241843011811016, + "pass@4": 0.28555184289463803, + "pass@8": 0.3488563330532779, + "pass@16": 0.410999142354151, + "pass@32": 0.470813795785859, + "pass@64": 0.5283060687421842, + "pass@128": 0.583 + }, + "9": { + "pass@1": 0.160328125, + "pass@2": 0.21869783464566916, + "pass@4": 0.27761340823022107, + "pass@8": 0.3340078325444927, + "pass@16": 0.3892774945499858, + "pass@32": 0.44521265742436017, + "pass@64": 0.5011826886679033, + "pass@128": 0.555 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.12907890625, + "avg_response_len": 193.69520546875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.12907890625, + "pass@2": 0.18724101870078808, + "pass@4": 0.24785211215785752, + "pass@8": 0.30334483840556176, + "pass@16": 0.3506148984602512, + "pass@32": 0.3910240745475175, + "pass@64": 0.4288613233997954, + "pass@128": 0.4684 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1494296875, + "pass@2": 0.2121313976377953, + "pass@4": 0.27820185901762257, + "pass@8": 0.3399698940576726, + "pass@16": 0.3939905585452765, + "pass@32": 0.4423371372495392, + "pass@64": 0.48918812515801696, + "pass@128": 0.537 + }, + "12": { + "pass@1": 0.160625, + "pass@2": 0.22663988681102348, + "pass@4": 0.2916212170978629, + "pass@8": 0.34822754981947357, + "pass@16": 0.3963911014309076, + "pass@32": 0.4398473857490613, + "pass@64": 0.4820694974925308, + "pass@128": 0.525 + }, + "13": { + "pass@1": 0.128828125, + "pass@2": 0.18485014763779528, + "pass@4": 0.24237966141732273, + "pass@8": 0.294481229789422, + "pass@16": 0.33967200111532, + "pass@32": 0.38147261968651247, + "pass@64": 0.4239452576013823, + "pass@128": 0.47 + }, + "14": { + "pass@1": 0.1373359375, + "pass@2": 0.19902374507874016, + "pass@4": 0.26239655764904346, + "pass@8": 0.3195219412709867, + "pass@16": 0.3674092323231115, + "pass@32": 0.40875126333943446, + "pass@64": 0.4485776234182972, + "pass@128": 0.486 + }, + "15": { + "pass@1": 0.1211796875, + "pass@2": 0.17508575295275589, + "pass@4": 0.2326256533558303, + "pass@8": 0.28774299435707185, + "pass@16": 0.3365395191749295, + "pass@32": 0.3781282218037708, + "pass@64": 0.41767681213174834, + "pass@128": 0.463 + }, + "16": { + "pass@1": 0.1435546875, + "pass@2": 0.20907381889763788, + "pass@4": 0.2768720108736405, + "pass@8": 0.33846787114434645, + "pass@16": 0.38942821097635366, + "pass@32": 0.4309488696250114, + "pass@64": 0.4682743075454802, + "pass@128": 0.505 + }, + "17": { + "pass@1": 0.128046875, + "pass@2": 0.18688361220472433, + "pass@4": 0.24821866788526425, + "pass@8": 0.3037966743819872, + "pass@16": 0.34922241180981833, + "pass@32": 0.3843204415947109, + "pass@64": 0.41371488728847383, + "pass@128": 0.444 + }, + "18": { + "pass@1": 0.1144609375, + "pass@2": 0.16997982283464572, + "pass@4": 0.22844402062242222, + "pass@8": 0.28092987058321883, + "pass@16": 0.3247783041463269, + "pass@32": 0.3622571724362258, + "pass@64": 0.39707136692912126, + "pass@128": 0.434 + }, + "19": { + "pass@1": 0.105515625, + "pass@2": 0.1550834153543307, + "pass@4": 0.2072736702287215, + "pass@8": 0.25674054817131314, + "pass@16": 0.3010017683454511, + "pass@32": 0.34017178013356214, + "pass@64": 0.3777464467051679, + "pass@128": 0.418 + }, + "20": { + "pass@1": 0.1018125, + "pass@2": 0.15365858759842518, + "pass@4": 0.210487803430821, + "pass@8": 0.26356981048007855, + "pass@16": 0.307715876734973, + "pass@32": 0.342005853857356, + "pass@64": 0.3703489097277736, + "pass@128": 0.402 + } + }, + "per_op_accuracy": { + "11": 0.1494296875, + "12": 0.160625, + "13": 0.128828125, + "14": 0.1373359375, + "15": 0.1211796875, + "16": 0.1435546875, + "17": 0.128046875, + "18": 0.1144609375, + "19": 0.105515625, + "20": 0.1018125 + }, + "per_op_avg_response_len": { + "11": 177.91121875, + "12": 181.398046875, + "13": 185.36875, + "14": 192.35971875, + "15": 196.3850546875, + "16": 208.8321796875, + "17": 201.32846875, + "18": 203.226375, + "19": 193.799375, + "20": 196.3428671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 177.91121875, + "12": 181.398046875, + "13": 185.36875, + "14": 192.35971875, + "15": 196.3850546875, + "16": 208.8321796875, + "17": 201.32846875, + "18": 203.226375, + "19": 193.799375, + "20": 196.3428671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 165221, + "answer_accuracy": 0.12907890625, + "avg_response_len": 193.69520546875, + "resp_tokens_sum": 247929863, + "pass_at_k": { + "pass@1": 0.12907890625, + "pass@2": 0.18724101870078808, + "pass@4": 0.24785211215785752, + "pass@8": 0.30334483840556176, + "pass@16": 0.3506148984602512, + "pass@32": 0.3910240745475175, + "pass@64": 0.4288613233997954, + "pass@128": 0.4684 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1494296875, + "pass@2": 0.2121313976377953, + "pass@4": 0.27820185901762257, + "pass@8": 0.3399698940576726, + "pass@16": 0.3939905585452765, + "pass@32": 0.4423371372495392, + "pass@64": 0.48918812515801696, + "pass@128": 0.537 + }, + "12": { + "pass@1": 0.160625, + "pass@2": 0.22663988681102348, + "pass@4": 0.2916212170978629, + "pass@8": 0.34822754981947357, + "pass@16": 0.3963911014309076, + "pass@32": 0.4398473857490613, + "pass@64": 0.4820694974925308, + "pass@128": 0.525 + }, + "13": { + "pass@1": 0.128828125, + "pass@2": 0.18485014763779528, + "pass@4": 0.24237966141732273, + "pass@8": 0.294481229789422, + "pass@16": 0.33967200111532, + "pass@32": 0.38147261968651247, + "pass@64": 0.4239452576013823, + "pass@128": 0.47 + }, + "14": { + "pass@1": 0.1373359375, + "pass@2": 0.19902374507874016, + "pass@4": 0.26239655764904346, + "pass@8": 0.3195219412709867, + "pass@16": 0.3674092323231115, + "pass@32": 0.40875126333943446, + "pass@64": 0.4485776234182972, + "pass@128": 0.486 + }, + "15": { + "pass@1": 0.1211796875, + "pass@2": 0.17508575295275589, + "pass@4": 0.2326256533558303, + "pass@8": 0.28774299435707185, + "pass@16": 0.3365395191749295, + "pass@32": 0.3781282218037708, + "pass@64": 0.41767681213174834, + "pass@128": 0.463 + }, + "16": { + "pass@1": 0.1435546875, + "pass@2": 0.20907381889763788, + "pass@4": 0.2768720108736405, + "pass@8": 0.33846787114434645, + "pass@16": 0.38942821097635366, + "pass@32": 0.4309488696250114, + "pass@64": 0.4682743075454802, + "pass@128": 0.505 + }, + "17": { + "pass@1": 0.128046875, + "pass@2": 0.18688361220472433, + "pass@4": 0.24821866788526425, + "pass@8": 0.3037966743819872, + "pass@16": 0.34922241180981833, + "pass@32": 0.3843204415947109, + "pass@64": 0.41371488728847383, + "pass@128": 0.444 + }, + "18": { + "pass@1": 0.1144609375, + "pass@2": 0.16997982283464572, + "pass@4": 0.22844402062242222, + "pass@8": 0.28092987058321883, + "pass@16": 0.3247783041463269, + "pass@32": 0.3622571724362258, + "pass@64": 0.39707136692912126, + "pass@128": 0.434 + }, + "19": { + "pass@1": 0.105515625, + "pass@2": 0.1550834153543307, + "pass@4": 0.2072736702287215, + "pass@8": 0.25674054817131314, + "pass@16": 0.3010017683454511, + "pass@32": 0.34017178013356214, + "pass@64": 0.3777464467051679, + "pass@128": 0.418 + }, + "20": { + "pass@1": 0.1018125, + "pass@2": 0.15365858759842518, + "pass@4": 0.210487803430821, + "pass@8": 0.26356981048007855, + "pass@16": 0.307715876734973, + "pass@32": 0.342005853857356, + "pass@64": 0.3703489097277736, + "pass@128": 0.402 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.2055485197368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.2055485197368421, + "pass@2": 0.26143210604019984, + "pass@4": 0.3183349632545875, + "pass@8": 0.3720750381747851, + "pass@16": 0.4210480826460184, + "pass@32": 0.46579232020733236, + "pass@64": 0.5081162761476423, + "pass@128": 0.5497368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1478984375, + "pass@2": 0.19923855807086627, + "pass@4": 0.25262500703037083, + "pass@8": 0.30650540226228934, + "pass@16": 0.35994912682498653, + "pass@32": 0.41290081123119726, + "pass@64": 0.4673808844065095, + "pass@128": 0.523 + }, + "2": { + "pass@1": 0.9587890625, + "pass@2": 0.9759959399606299, + "pass@4": 0.9859024143232096, + "pass@8": 0.9921983842033142, + "pass@16": 0.9959833778773702, + "pass@32": 0.9979885223455225, + "pass@64": 0.9988623805075786, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.5236328125, + "pass@2": 0.6070394931102369, + "pass@4": 0.6788192813085867, + "pass@8": 0.7424375231451571, + "pass@16": 0.7995801911522875, + "pass@32": 0.8495603915797477, + "pass@64": 0.8884488903916155, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.1328125, + "pass@2": 0.1752502460629921, + "pass@4": 0.2221315449943755, + "pass@8": 0.2728284436391267, + "pass@16": 0.3264936988844912, + "pass@32": 0.38123826139965056, + "pass@64": 0.43536047324376165, + "pass@128": 0.487 + }, + "5": { + "pass@1": 0.1868515625, + "pass@2": 0.24461724901574805, + "pass@4": 0.3004909116985375, + "pass@8": 0.35149775683040163, + "pass@16": 0.3993127693843177, + "pass@32": 0.4462225810693977, + "pass@64": 0.49302187438693307, + "pass@128": 0.538 + }, + "6": { + "pass@1": 0.1715625, + "pass@2": 0.22371198326771635, + "pass@4": 0.27955061801649767, + "pass@8": 0.34080354755334136, + "pass@16": 0.4062840574005593, + "pass@32": 0.47319170709674596, + "pass@64": 0.5371530434190255, + "pass@128": 0.596 + }, + "7": { + "pass@1": 0.1707265625, + "pass@2": 0.22783009350393704, + "pass@4": 0.2871581517622791, + "pass@8": 0.34684211803384596, + "pass@16": 0.40588472724342184, + "pass@32": 0.46268461053160126, + "pass@64": 0.5158797090415914, + "pass@128": 0.566 + }, + "8": { + "pass@1": 0.16203125, + "pass@2": 0.22241843011811016, + "pass@4": 0.28555184289463803, + "pass@8": 0.3488563330532779, + "pass@16": 0.410999142354151, + "pass@32": 0.470813795785859, + "pass@64": 0.5283060687421842, + "pass@128": 0.583 + }, + "9": { + "pass@1": 0.160328125, + "pass@2": 0.21869783464566916, + "pass@4": 0.27761340823022107, + "pass@8": 0.3340078325444927, + "pass@16": 0.3892774945499858, + "pass@32": 0.44521265742436017, + "pass@64": 0.5011826886679033, + "pass@128": 0.555 + }, + "11": { + "pass@1": 0.1494296875, + "pass@2": 0.2121313976377953, + "pass@4": 0.27820185901762257, + "pass@8": 0.3399698940576726, + "pass@16": 0.3939905585452765, + "pass@32": 0.4423371372495392, + "pass@64": 0.48918812515801696, + "pass@128": 0.537 + }, + "12": { + "pass@1": 0.160625, + "pass@2": 0.22663988681102348, + "pass@4": 0.2916212170978629, + "pass@8": 0.34822754981947357, + "pass@16": 0.3963911014309076, + "pass@32": 0.4398473857490613, + "pass@64": 0.4820694974925308, + "pass@128": 0.525 + }, + "13": { + "pass@1": 0.128828125, + "pass@2": 0.18485014763779528, + "pass@4": 0.24237966141732273, + "pass@8": 0.294481229789422, + "pass@16": 0.33967200111532, + "pass@32": 0.38147261968651247, + "pass@64": 0.4239452576013823, + "pass@128": 0.47 + }, + "14": { + "pass@1": 0.1373359375, + "pass@2": 0.19902374507874016, + "pass@4": 0.26239655764904346, + "pass@8": 0.3195219412709867, + "pass@16": 0.3674092323231115, + "pass@32": 0.40875126333943446, + "pass@64": 0.4485776234182972, + "pass@128": 0.486 + }, + "15": { + "pass@1": 0.1211796875, + "pass@2": 0.17508575295275589, + "pass@4": 0.2326256533558303, + "pass@8": 0.28774299435707185, + "pass@16": 0.3365395191749295, + "pass@32": 0.3781282218037708, + "pass@64": 0.41767681213174834, + "pass@128": 0.463 + }, + "16": { + "pass@1": 0.1435546875, + "pass@2": 0.20907381889763788, + "pass@4": 0.2768720108736405, + "pass@8": 0.33846787114434645, + "pass@16": 0.38942821097635366, + "pass@32": 0.4309488696250114, + "pass@64": 0.4682743075454802, + "pass@128": 0.505 + }, + "17": { + "pass@1": 0.128046875, + "pass@2": 0.18688361220472433, + "pass@4": 0.24821866788526425, + "pass@8": 0.3037966743819872, + "pass@16": 0.34922241180981833, + "pass@32": 0.3843204415947109, + "pass@64": 0.41371488728847383, + "pass@128": 0.444 + }, + "18": { + "pass@1": 0.1144609375, + "pass@2": 0.16997982283464572, + "pass@4": 0.22844402062242222, + "pass@8": 0.28092987058321883, + "pass@16": 0.3247783041463269, + "pass@32": 0.3622571724362258, + "pass@64": 0.39707136692912126, + "pass@128": 0.434 + }, + "19": { + "pass@1": 0.105515625, + "pass@2": 0.1550834153543307, + "pass@4": 0.2072736702287215, + "pass@8": 0.25674054817131314, + "pass@16": 0.3010017683454511, + "pass@32": 0.34017178013356214, + "pass@64": 0.3777464467051679, + "pass@128": 0.418 + }, + "20": { + "pass@1": 0.1018125, + "pass@2": 0.15365858759842518, + "pass@4": 0.210487803430821, + "pass@8": 0.26356981048007855, + "pass@16": 0.307715876734973, + "pass@32": 0.342005853857356, + "pass@64": 0.3703489097277736, + "pass@128": 0.402 + } + }, + "per_op_accuracy": { + "10": 0.1478984375, + "2": 0.9587890625, + "3": 0.5236328125, + "4": 0.1328125, + "5": 0.1868515625, + "6": 0.1715625, + "7": 0.1707265625, + "8": 0.16203125, + "9": 0.160328125, + "11": 0.1494296875, + "12": 0.160625, + "13": 0.128828125, + "14": 0.1373359375, + "15": 0.1211796875, + "16": 0.1435546875, + "17": 0.128046875, + "18": 0.1144609375, + "19": 0.105515625, + "20": 0.1018125 + }, + "per_op_avg_response_len": { + "10": 165.2404453125, + "2": 107.311953125, + "3": 116.4744765625, + "4": 124.00365625, + "5": 145.1775, + "6": 152.1250234375, + "7": 146.6023984375, + "8": 161.05984375, + "9": 168.524640625, + "11": 177.91121875, + "12": 181.398046875, + "13": 185.36875, + "14": 192.35971875, + "15": 196.3850546875, + "16": 208.8321796875, + "17": 201.32846875, + "18": 203.226375, + "19": 193.799375, + "20": 196.3428671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 165.2404453125, + "2": 107.311953125, + "3": 116.4744765625, + "4": 124.00365625, + "5": 145.1775, + "6": 152.1250234375, + "7": 146.6023984375, + "8": 161.05984375, + "9": 168.524640625, + "11": 177.91121875, + "12": 181.398046875, + "13": 185.36875, + "14": 192.35971875, + "15": 196.3850546875, + "16": 208.8321796875, + "17": 201.32846875, + "18": 203.226375, + "19": 193.799375, + "20": 196.3428671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 499894, + "answer_accuracy": 0.2055485197368421, + "avg_response_len": 169.65642064144737, + "resp_tokens_sum": 412604415, + "pass_at_k": { + "pass@1": 0.2055485197368421, + "pass@2": 0.26143210604019984, + "pass@4": 0.3183349632545875, + "pass@8": 0.3720750381747851, + "pass@16": 0.4210480826460184, + "pass@32": 0.46579232020733236, + "pass@64": 0.5081162761476423, + "pass@128": 0.5497368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1478984375, + "pass@2": 0.19923855807086627, + "pass@4": 0.25262500703037083, + "pass@8": 0.30650540226228934, + "pass@16": 0.35994912682498653, + "pass@32": 0.41290081123119726, + "pass@64": 0.4673808844065095, + "pass@128": 0.523 + }, + "2": { + "pass@1": 0.9587890625, + "pass@2": 0.9759959399606299, + "pass@4": 0.9859024143232096, + "pass@8": 0.9921983842033142, + "pass@16": 0.9959833778773702, + "pass@32": 0.9979885223455225, + "pass@64": 0.9988623805075786, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.5236328125, + "pass@2": 0.6070394931102369, + "pass@4": 0.6788192813085867, + "pass@8": 0.7424375231451571, + "pass@16": 0.7995801911522875, + "pass@32": 0.8495603915797477, + "pass@64": 0.8884488903916155, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.1328125, + "pass@2": 0.1752502460629921, + "pass@4": 0.2221315449943755, + "pass@8": 0.2728284436391267, + "pass@16": 0.3264936988844912, + "pass@32": 0.38123826139965056, + "pass@64": 0.43536047324376165, + "pass@128": 0.487 + }, + "5": { + "pass@1": 0.1868515625, + "pass@2": 0.24461724901574805, + "pass@4": 0.3004909116985375, + "pass@8": 0.35149775683040163, + "pass@16": 0.3993127693843177, + "pass@32": 0.4462225810693977, + "pass@64": 0.49302187438693307, + "pass@128": 0.538 + }, + "6": { + "pass@1": 0.1715625, + "pass@2": 0.22371198326771635, + "pass@4": 0.27955061801649767, + "pass@8": 0.34080354755334136, + "pass@16": 0.4062840574005593, + "pass@32": 0.47319170709674596, + "pass@64": 0.5371530434190255, + "pass@128": 0.596 + }, + "7": { + "pass@1": 0.1707265625, + "pass@2": 0.22783009350393704, + "pass@4": 0.2871581517622791, + "pass@8": 0.34684211803384596, + "pass@16": 0.40588472724342184, + "pass@32": 0.46268461053160126, + "pass@64": 0.5158797090415914, + "pass@128": 0.566 + }, + "8": { + "pass@1": 0.16203125, + "pass@2": 0.22241843011811016, + "pass@4": 0.28555184289463803, + "pass@8": 0.3488563330532779, + "pass@16": 0.410999142354151, + "pass@32": 0.470813795785859, + "pass@64": 0.5283060687421842, + "pass@128": 0.583 + }, + "9": { + "pass@1": 0.160328125, + "pass@2": 0.21869783464566916, + "pass@4": 0.27761340823022107, + "pass@8": 0.3340078325444927, + "pass@16": 0.3892774945499858, + "pass@32": 0.44521265742436017, + "pass@64": 0.5011826886679033, + "pass@128": 0.555 + }, + "11": { + "pass@1": 0.1494296875, + "pass@2": 0.2121313976377953, + "pass@4": 0.27820185901762257, + "pass@8": 0.3399698940576726, + "pass@16": 0.3939905585452765, + "pass@32": 0.4423371372495392, + "pass@64": 0.48918812515801696, + "pass@128": 0.537 + }, + "12": { + "pass@1": 0.160625, + "pass@2": 0.22663988681102348, + "pass@4": 0.2916212170978629, + "pass@8": 0.34822754981947357, + "pass@16": 0.3963911014309076, + "pass@32": 0.4398473857490613, + "pass@64": 0.4820694974925308, + "pass@128": 0.525 + }, + "13": { + "pass@1": 0.128828125, + "pass@2": 0.18485014763779528, + "pass@4": 0.24237966141732273, + "pass@8": 0.294481229789422, + "pass@16": 0.33967200111532, + "pass@32": 0.38147261968651247, + "pass@64": 0.4239452576013823, + "pass@128": 0.47 + }, + "14": { + "pass@1": 0.1373359375, + "pass@2": 0.19902374507874016, + "pass@4": 0.26239655764904346, + "pass@8": 0.3195219412709867, + "pass@16": 0.3674092323231115, + "pass@32": 0.40875126333943446, + "pass@64": 0.4485776234182972, + "pass@128": 0.486 + }, + "15": { + "pass@1": 0.1211796875, + "pass@2": 0.17508575295275589, + "pass@4": 0.2326256533558303, + "pass@8": 0.28774299435707185, + "pass@16": 0.3365395191749295, + "pass@32": 0.3781282218037708, + "pass@64": 0.41767681213174834, + "pass@128": 0.463 + }, + "16": { + "pass@1": 0.1435546875, + "pass@2": 0.20907381889763788, + "pass@4": 0.2768720108736405, + "pass@8": 0.33846787114434645, + "pass@16": 0.38942821097635366, + "pass@32": 0.4309488696250114, + "pass@64": 0.4682743075454802, + "pass@128": 0.505 + }, + "17": { + "pass@1": 0.128046875, + "pass@2": 0.18688361220472433, + "pass@4": 0.24821866788526425, + "pass@8": 0.3037966743819872, + "pass@16": 0.34922241180981833, + "pass@32": 0.3843204415947109, + "pass@64": 0.41371488728847383, + "pass@128": 0.444 + }, + "18": { + "pass@1": 0.1144609375, + "pass@2": 0.16997982283464572, + "pass@4": 0.22844402062242222, + "pass@8": 0.28092987058321883, + "pass@16": 0.3247783041463269, + "pass@32": 0.3622571724362258, + "pass@64": 0.39707136692912126, + "pass@128": 0.434 + }, + "19": { + "pass@1": 0.105515625, + "pass@2": 0.1550834153543307, + "pass@4": 0.2072736702287215, + "pass@8": 0.25674054817131314, + "pass@16": 0.3010017683454511, + "pass@32": 0.34017178013356214, + "pass@64": 0.3777464467051679, + "pass@128": 0.418 + }, + "20": { + "pass@1": 0.1018125, + "pass@2": 0.15365858759842518, + "pass@4": 0.210487803430821, + "pass@8": 0.26356981048007855, + "pass@16": 0.307715876734973, + "pass@32": 0.342005853857356, + "pass@64": 0.3703489097277736, + "pass@128": 0.402 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.csv b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8fc396db76424cb4637caa6414b65da8261cc37e --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18036,0.29051475694444445,0.12907890625,0.2055485197368421,0.0,0.0,0.0,142.94665972222222,193.69520546875,1152000,1280000,2432000 diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f82e952b117d1aa6de324cda25b2e29e86250c4e --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18036", + "id_acc": 0.29051475694444445, + "ood_acc": 0.12907890625, + "total_acc": 0.2055485197368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 142.94665972222222, + "ood_avg_resp_len": 193.69520546875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 499894, + "answer_accuracy": 0.2055485197368421, + "avg_response_len": 169.65642064144737, + "resp_tokens_sum": 412604415, + "pass_at_k": { + "pass@1": 0.2055485197368421, + "pass@2": 0.26143210604019984, + "pass@4": 0.3183349632545875, + "pass@8": 0.3720750381747851, + "pass@16": 0.4210480826460184, + "pass@32": 0.46579232020733236, + "pass@64": 0.5081162761476423, + "pass@128": 0.5497368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1478984375, + "pass@2": 0.19923855807086627, + "pass@4": 0.25262500703037083, + "pass@8": 0.30650540226228934, + "pass@16": 0.35994912682498653, + "pass@32": 0.41290081123119726, + "pass@64": 0.4673808844065095, + "pass@128": 0.523 + }, + "2": { + "pass@1": 0.9587890625, + "pass@2": 0.9759959399606299, + "pass@4": 0.9859024143232096, + "pass@8": 0.9921983842033142, + "pass@16": 0.9959833778773702, + "pass@32": 0.9979885223455225, + "pass@64": 0.9988623805075786, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.5236328125, + "pass@2": 0.6070394931102369, + "pass@4": 0.6788192813085867, + "pass@8": 0.7424375231451571, + "pass@16": 0.7995801911522875, + "pass@32": 0.8495603915797477, + "pass@64": 0.8884488903916155, + "pass@128": 0.914 + }, + "4": { + "pass@1": 0.1328125, + "pass@2": 0.1752502460629921, + "pass@4": 0.2221315449943755, + "pass@8": 0.2728284436391267, + "pass@16": 0.3264936988844912, + "pass@32": 0.38123826139965056, + "pass@64": 0.43536047324376165, + "pass@128": 0.487 + }, + "5": { + "pass@1": 0.1868515625, + "pass@2": 0.24461724901574805, + "pass@4": 0.3004909116985375, + "pass@8": 0.35149775683040163, + "pass@16": 0.3993127693843177, + "pass@32": 0.4462225810693977, + "pass@64": 0.49302187438693307, + "pass@128": 0.538 + }, + "6": { + "pass@1": 0.1715625, + "pass@2": 0.22371198326771635, + "pass@4": 0.27955061801649767, + "pass@8": 0.34080354755334136, + "pass@16": 0.4062840574005593, + "pass@32": 0.47319170709674596, + "pass@64": 0.5371530434190255, + "pass@128": 0.596 + }, + "7": { + "pass@1": 0.1707265625, + "pass@2": 0.22783009350393704, + "pass@4": 0.2871581517622791, + "pass@8": 0.34684211803384596, + "pass@16": 0.40588472724342184, + "pass@32": 0.46268461053160126, + "pass@64": 0.5158797090415914, + "pass@128": 0.566 + }, + "8": { + "pass@1": 0.16203125, + "pass@2": 0.22241843011811016, + "pass@4": 0.28555184289463803, + "pass@8": 0.3488563330532779, + "pass@16": 0.410999142354151, + "pass@32": 0.470813795785859, + "pass@64": 0.5283060687421842, + "pass@128": 0.583 + }, + "9": { + "pass@1": 0.160328125, + "pass@2": 0.21869783464566916, + "pass@4": 0.27761340823022107, + "pass@8": 0.3340078325444927, + "pass@16": 0.3892774945499858, + "pass@32": 0.44521265742436017, + "pass@64": 0.5011826886679033, + "pass@128": 0.555 + }, + "11": { + "pass@1": 0.1494296875, + "pass@2": 0.2121313976377953, + "pass@4": 0.27820185901762257, + "pass@8": 0.3399698940576726, + "pass@16": 0.3939905585452765, + "pass@32": 0.4423371372495392, + "pass@64": 0.48918812515801696, + "pass@128": 0.537 + }, + "12": { + "pass@1": 0.160625, + "pass@2": 0.22663988681102348, + "pass@4": 0.2916212170978629, + "pass@8": 0.34822754981947357, + "pass@16": 0.3963911014309076, + "pass@32": 0.4398473857490613, + "pass@64": 0.4820694974925308, + "pass@128": 0.525 + }, + "13": { + "pass@1": 0.128828125, + "pass@2": 0.18485014763779528, + "pass@4": 0.24237966141732273, + "pass@8": 0.294481229789422, + "pass@16": 0.33967200111532, + "pass@32": 0.38147261968651247, + "pass@64": 0.4239452576013823, + "pass@128": 0.47 + }, + "14": { + "pass@1": 0.1373359375, + "pass@2": 0.19902374507874016, + "pass@4": 0.26239655764904346, + "pass@8": 0.3195219412709867, + "pass@16": 0.3674092323231115, + "pass@32": 0.40875126333943446, + "pass@64": 0.4485776234182972, + "pass@128": 0.486 + }, + "15": { + "pass@1": 0.1211796875, + "pass@2": 0.17508575295275589, + "pass@4": 0.2326256533558303, + "pass@8": 0.28774299435707185, + "pass@16": 0.3365395191749295, + "pass@32": 0.3781282218037708, + "pass@64": 0.41767681213174834, + "pass@128": 0.463 + }, + "16": { + "pass@1": 0.1435546875, + "pass@2": 0.20907381889763788, + "pass@4": 0.2768720108736405, + "pass@8": 0.33846787114434645, + "pass@16": 0.38942821097635366, + "pass@32": 0.4309488696250114, + "pass@64": 0.4682743075454802, + "pass@128": 0.505 + }, + "17": { + "pass@1": 0.128046875, + "pass@2": 0.18688361220472433, + "pass@4": 0.24821866788526425, + "pass@8": 0.3037966743819872, + "pass@16": 0.34922241180981833, + "pass@32": 0.3843204415947109, + "pass@64": 0.41371488728847383, + "pass@128": 0.444 + }, + "18": { + "pass@1": 0.1144609375, + "pass@2": 0.16997982283464572, + "pass@4": 0.22844402062242222, + "pass@8": 0.28092987058321883, + "pass@16": 0.3247783041463269, + "pass@32": 0.3622571724362258, + "pass@64": 0.39707136692912126, + "pass@128": 0.434 + }, + "19": { + "pass@1": 0.105515625, + "pass@2": 0.1550834153543307, + "pass@4": 0.2072736702287215, + "pass@8": 0.25674054817131314, + "pass@16": 0.3010017683454511, + "pass@32": 0.34017178013356214, + "pass@64": 0.3777464467051679, + "pass@128": 0.418 + }, + "20": { + "pass@1": 0.1018125, + "pass@2": 0.15365858759842518, + "pass@4": 0.210487803430821, + "pass@8": 0.26356981048007855, + "pass@16": 0.307715876734973, + "pass@32": 0.342005853857356, + "pass@64": 0.3703489097277736, + "pass@128": 0.402 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18010_metrics.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18010_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0c3304715ad97859c06a7849e888931eaec3bc66 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18010_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18010", + "id": { + "answer_accuracy": 0.2905876736111111, + "avg_response_len": 212.98508333333334, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2905876736111111, + "pass@2": 0.3757110263560921, + "pass@4": 0.47081819943340814, + "pass@8": 0.5666408077816709, + "pass@16": 0.6582609157952408, + "pass@32": 0.7431476193952384, + "pass@64": 0.8161727129671587, + "pass@128": 0.8718888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1535625, + "pass@2": 0.23615563484251956, + "pass@4": 0.3313644041057365, + "pass@8": 0.4322543443704919, + "pass@16": 0.5397517873318299, + "pass@32": 0.6536812205750336, + "pass@64": 0.7635182573212812, + "pass@128": 0.854 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.3422890625, + "pass@2": 0.469228469488189, + "pass@4": 0.5868346908511433, + "pass@8": 0.6825482078242203, + "pass@16": 0.7552705310444486, + "pass@32": 0.8107429866510845, + "pass@64": 0.8542986323759144, + "pass@128": 0.891 + }, + "4": { + "pass@1": 0.20628125, + "pass@2": 0.31069180610236224, + "pass@4": 0.42973400562429676, + "pass@8": 0.5437283789685704, + "pass@16": 0.6373021804337103, + "pass@32": 0.7082035417675484, + "pass@64": 0.760672519399519, + "pass@128": 0.798 + }, + "5": { + "pass@1": 0.2379296875, + "pass@2": 0.3275477362204723, + "pass@4": 0.4267573031496054, + "pass@8": 0.5281339494014066, + "pass@16": 0.6227814416502605, + "pass@32": 0.7052228942192685, + "pass@64": 0.7743729632317604, + "pass@128": 0.831 + }, + "6": { + "pass@1": 0.188609375, + "pass@2": 0.29246530511811025, + "pass@4": 0.4121283263029615, + "pass@8": 0.5339728926680833, + "pass@16": 0.6504527221959205, + "pass@32": 0.7558571665056636, + "pass@64": 0.839577416192582, + "pass@128": 0.897 + }, + "7": { + "pass@1": 0.1702421875, + "pass@2": 0.2610345718503936, + "pass@4": 0.36890081514810585, + "pass@8": 0.4851046679913388, + "pass@16": 0.6031619242717101, + "pass@32": 0.7133174129702993, + "pass@64": 0.8032198822543871, + "pass@128": 0.865 + }, + "8": { + "pass@1": 0.17203125, + "pass@2": 0.260486466535433, + "pass@4": 0.3623691548556425, + "pass@8": 0.4698659992521673, + "pass@16": 0.579080865405446, + "pass@32": 0.6871811907288851, + "pass@64": 0.7839559840998817, + "pass@128": 0.857 + }, + "9": { + "pass@1": 0.14434375, + "pass@2": 0.22378924704724426, + "pass@4": 0.3192750948631419, + "pass@8": 0.4241588295586182, + "pass@16": 0.5365467898236371, + "pass@32": 0.6541221611393754, + "pass@64": 0.765938761829036, + "pass@128": 0.854 + } + }, + "per_op_accuracy": { + "10": 0.1535625, + "2": 1.0, + "3": 0.3422890625, + "4": 0.20628125, + "5": 0.2379296875, + "6": 0.188609375, + "7": 0.1702421875, + "8": 0.17203125, + "9": 0.14434375 + }, + "per_op_avg_response_len": { + "10": 291.2561484375, + "2": 107.38678125, + "3": 171.28271875, + "4": 156.9440625, + "5": 179.8192109375, + "6": 232.113046875, + "7": 234.457640625, + "8": 260.4949296875, + "9": 283.1112109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 291.2561484375, + "2": 107.38678125, + "3": 171.28271875, + "4": 156.9440625, + "5": 179.8192109375, + "6": 232.113046875, + "7": 234.457640625, + "8": 260.4949296875, + "9": 283.1112109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 334757, + "answer_accuracy": 0.2905876736111111, + "avg_response_len": 212.98508333333334, + "resp_tokens_sum": 245358816, + "pass_at_k": { + "pass@1": 0.2905876736111111, + "pass@2": 0.3757110263560921, + "pass@4": 0.47081819943340814, + "pass@8": 0.5666408077816709, + "pass@16": 0.6582609157952408, + "pass@32": 0.7431476193952384, + "pass@64": 0.8161727129671587, + "pass@128": 0.8718888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1535625, + "pass@2": 0.23615563484251956, + "pass@4": 0.3313644041057365, + "pass@8": 0.4322543443704919, + "pass@16": 0.5397517873318299, + "pass@32": 0.6536812205750336, + "pass@64": 0.7635182573212812, + "pass@128": 0.854 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.3422890625, + "pass@2": 0.469228469488189, + "pass@4": 0.5868346908511433, + "pass@8": 0.6825482078242203, + "pass@16": 0.7552705310444486, + "pass@32": 0.8107429866510845, + "pass@64": 0.8542986323759144, + "pass@128": 0.891 + }, + "4": { + "pass@1": 0.20628125, + "pass@2": 0.31069180610236224, + "pass@4": 0.42973400562429676, + "pass@8": 0.5437283789685704, + "pass@16": 0.6373021804337103, + "pass@32": 0.7082035417675484, + "pass@64": 0.760672519399519, + "pass@128": 0.798 + }, + "5": { + "pass@1": 0.2379296875, + "pass@2": 0.3275477362204723, + "pass@4": 0.4267573031496054, + "pass@8": 0.5281339494014066, + "pass@16": 0.6227814416502605, + "pass@32": 0.7052228942192685, + "pass@64": 0.7743729632317604, + "pass@128": 0.831 + }, + "6": { + "pass@1": 0.188609375, + "pass@2": 0.29246530511811025, + "pass@4": 0.4121283263029615, + "pass@8": 0.5339728926680833, + "pass@16": 0.6504527221959205, + "pass@32": 0.7558571665056636, + "pass@64": 0.839577416192582, + "pass@128": 0.897 + }, + "7": { + "pass@1": 0.1702421875, + "pass@2": 0.2610345718503936, + "pass@4": 0.36890081514810585, + "pass@8": 0.4851046679913388, + "pass@16": 0.6031619242717101, + "pass@32": 0.7133174129702993, + "pass@64": 0.8032198822543871, + "pass@128": 0.865 + }, + "8": { + "pass@1": 0.17203125, + "pass@2": 0.260486466535433, + "pass@4": 0.3623691548556425, + "pass@8": 0.4698659992521673, + "pass@16": 0.579080865405446, + "pass@32": 0.6871811907288851, + "pass@64": 0.7839559840998817, + "pass@128": 0.857 + }, + "9": { + "pass@1": 0.14434375, + "pass@2": 0.22378924704724426, + "pass@4": 0.3192750948631419, + "pass@8": 0.4241588295586182, + "pass@16": 0.5365467898236371, + "pass@32": 0.6541221611393754, + "pass@64": 0.765938761829036, + "pass@128": 0.854 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.12588515625, + "avg_response_len": 357.5629859375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.12588515625, + "pass@2": 0.19707919537401805, + "pass@4": 0.2791604231252401, + "pass@8": 0.3623333790243092, + "pass@16": 0.4452968000241374, + "pass@32": 0.534103302103374, + "pass@64": 0.6322614417868554, + "pass@128": 0.7329 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1424921875, + "pass@2": 0.22340280511811042, + "pass@4": 0.3188062520622418, + "pass@8": 0.41971003687558767, + "pass@16": 0.5244687780376375, + "pass@32": 0.6351116073592982, + "pass@64": 0.7457114417547469, + "pass@128": 0.841 + }, + "12": { + "pass@1": 0.1580390625, + "pass@2": 0.24750639763779528, + "pass@4": 0.34879692050993577, + "pass@8": 0.4489444014457862, + "pass@16": 0.5471894408362217, + "pass@32": 0.6482069592134363, + "pass@64": 0.7487004100526425, + "pass@128": 0.837 + }, + "13": { + "pass@1": 0.137203125, + "pass@2": 0.21484793307086603, + "pass@4": 0.30410517894638156, + "pass@8": 0.39436279262935253, + "pass@16": 0.4869214245024962, + "pass@32": 0.5885516578108125, + "pass@64": 0.6977312707591256, + "pass@128": 0.802 + }, + "14": { + "pass@1": 0.138421875, + "pass@2": 0.21568257874015784, + "pass@4": 0.30250095303712055, + "pass@8": 0.38742854884151073, + "pass@16": 0.471113950774246, + "pass@32": 0.5627169993779305, + "pass@64": 0.6662268699433128, + "pass@128": 0.772 + }, + "15": { + "pass@1": 0.1236796875, + "pass@2": 0.19075787401574845, + "pass@4": 0.2672861099550049, + "pass@8": 0.3457352960827552, + "pass@16": 0.42617246292005095, + "pass@32": 0.5152098938373147, + "pass@64": 0.6171748124143301, + "pass@128": 0.719 + }, + "16": { + "pass@1": 0.126515625, + "pass@2": 0.1987988435039372, + "pass@4": 0.2826862190663667, + "pass@8": 0.3678530617021002, + "pass@16": 0.45093558358661096, + "pass@32": 0.5351528343825194, + "pass@64": 0.6253042881510136, + "pass@128": 0.721 + }, + "17": { + "pass@1": 0.1192109375, + "pass@2": 0.18613152066929148, + "pass@4": 0.26321594891263583, + "pass@8": 0.3419057828768377, + "pass@16": 0.4220269175740973, + "pass@32": 0.5105044387235352, + "pass@64": 0.6118238153555224, + "pass@128": 0.72 + }, + "18": { + "pass@1": 0.107359375, + "pass@2": 0.16992679625984286, + "pass@4": 0.24431265832395888, + "pass@8": 0.3210678898460004, + "pass@16": 0.39598269900224564, + "pass@32": 0.47565861398601084, + "pass@64": 0.5682760133155218, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1059765625, + "pass@2": 0.1661788877952758, + "pass@4": 0.23450728430821097, + "pass@8": 0.30133843327893156, + "pass@16": 0.365023901924948, + "pass@32": 0.43474103369718325, + "pass@64": 0.5206322595279965, + "pass@128": 0.623 + }, + "20": { + "pass@1": 0.099953125, + "pass@2": 0.15755831692913397, + "pass@4": 0.22538670613048326, + "pass@8": 0.2949875466640804, + "pass@16": 0.3631328410827, + "pass@32": 0.43517898264575217, + "pass@64": 0.5210332365943791, + "pass@128": 0.624 + } + }, + "per_op_accuracy": { + "11": 0.1424921875, + "12": 0.1580390625, + "13": 0.137203125, + "14": 0.138421875, + "15": 0.1236796875, + "16": 0.126515625, + "17": 0.1192109375, + "18": 0.107359375, + "19": 0.1059765625, + "20": 0.099953125 + }, + "per_op_avg_response_len": { + "11": 309.7439453125, + "12": 306.5526171875, + "13": 313.8674609375, + "14": 344.5398203125, + "15": 362.3768671875, + "16": 376.48190625, + "17": 380.589515625, + "18": 386.76740625, + "19": 391.3802734375, + "20": 403.330046875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 309.7439453125, + "12": 306.5526171875, + "13": 313.8674609375, + "14": 344.5398203125, + "15": 362.3768671875, + "16": 376.48190625, + "17": 380.589515625, + "18": 386.76740625, + "19": 391.3802734375, + "20": 403.330046875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 161133, + "answer_accuracy": 0.12588515625, + "avg_response_len": 357.5629859375, + "resp_tokens_sum": 457680622, + "pass_at_k": { + "pass@1": 0.12588515625, + "pass@2": 0.19707919537401805, + "pass@4": 0.2791604231252401, + "pass@8": 0.3623333790243092, + "pass@16": 0.4452968000241374, + "pass@32": 0.534103302103374, + "pass@64": 0.6322614417868554, + "pass@128": 0.7329 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1424921875, + "pass@2": 0.22340280511811042, + "pass@4": 0.3188062520622418, + "pass@8": 0.41971003687558767, + "pass@16": 0.5244687780376375, + "pass@32": 0.6351116073592982, + "pass@64": 0.7457114417547469, + "pass@128": 0.841 + }, + "12": { + "pass@1": 0.1580390625, + "pass@2": 0.24750639763779528, + "pass@4": 0.34879692050993577, + "pass@8": 0.4489444014457862, + "pass@16": 0.5471894408362217, + "pass@32": 0.6482069592134363, + "pass@64": 0.7487004100526425, + "pass@128": 0.837 + }, + "13": { + "pass@1": 0.137203125, + "pass@2": 0.21484793307086603, + "pass@4": 0.30410517894638156, + "pass@8": 0.39436279262935253, + "pass@16": 0.4869214245024962, + "pass@32": 0.5885516578108125, + "pass@64": 0.6977312707591256, + "pass@128": 0.802 + }, + "14": { + "pass@1": 0.138421875, + "pass@2": 0.21568257874015784, + "pass@4": 0.30250095303712055, + "pass@8": 0.38742854884151073, + "pass@16": 0.471113950774246, + "pass@32": 0.5627169993779305, + "pass@64": 0.6662268699433128, + "pass@128": 0.772 + }, + "15": { + "pass@1": 0.1236796875, + "pass@2": 0.19075787401574845, + "pass@4": 0.2672861099550049, + "pass@8": 0.3457352960827552, + "pass@16": 0.42617246292005095, + "pass@32": 0.5152098938373147, + "pass@64": 0.6171748124143301, + "pass@128": 0.719 + }, + "16": { + "pass@1": 0.126515625, + "pass@2": 0.1987988435039372, + "pass@4": 0.2826862190663667, + "pass@8": 0.3678530617021002, + "pass@16": 0.45093558358661096, + "pass@32": 0.5351528343825194, + "pass@64": 0.6253042881510136, + "pass@128": 0.721 + }, + "17": { + "pass@1": 0.1192109375, + "pass@2": 0.18613152066929148, + "pass@4": 0.26321594891263583, + "pass@8": 0.3419057828768377, + "pass@16": 0.4220269175740973, + "pass@32": 0.5105044387235352, + "pass@64": 0.6118238153555224, + "pass@128": 0.72 + }, + "18": { + "pass@1": 0.107359375, + "pass@2": 0.16992679625984286, + "pass@4": 0.24431265832395888, + "pass@8": 0.3210678898460004, + "pass@16": 0.39598269900224564, + "pass@32": 0.47565861398601084, + "pass@64": 0.5682760133155218, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1059765625, + "pass@2": 0.1661788877952758, + "pass@4": 0.23450728430821097, + "pass@8": 0.30133843327893156, + "pass@16": 0.365023901924948, + "pass@32": 0.43474103369718325, + "pass@64": 0.5206322595279965, + "pass@128": 0.623 + }, + "20": { + "pass@1": 0.099953125, + "pass@2": 0.15755831692913397, + "pass@4": 0.22538670613048326, + "pass@8": 0.2949875466640804, + "pass@16": 0.3631328410827, + "pass@32": 0.43517898264575217, + "pass@64": 0.5210332365943791, + "pass@128": 0.624 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.20390213815789474, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.20390213815789474, + "pass@2": 0.28169427320761264, + "pass@4": 0.36994568558698104, + "pass@8": 0.45911058211989764, + "pass@16": 0.5461745390736306, + "pass@32": 0.6331242945048048, + "pass@64": 0.7193773070827906, + "pass@128": 0.7987368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1535625, + "pass@2": 0.23615563484251956, + "pass@4": 0.3313644041057365, + "pass@8": 0.4322543443704919, + "pass@16": 0.5397517873318299, + "pass@32": 0.6536812205750336, + "pass@64": 0.7635182573212812, + "pass@128": 0.854 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.3422890625, + "pass@2": 0.469228469488189, + "pass@4": 0.5868346908511433, + "pass@8": 0.6825482078242203, + "pass@16": 0.7552705310444486, + "pass@32": 0.8107429866510845, + "pass@64": 0.8542986323759144, + "pass@128": 0.891 + }, + "4": { + "pass@1": 0.20628125, + "pass@2": 0.31069180610236224, + "pass@4": 0.42973400562429676, + "pass@8": 0.5437283789685704, + "pass@16": 0.6373021804337103, + "pass@32": 0.7082035417675484, + "pass@64": 0.760672519399519, + "pass@128": 0.798 + }, + "5": { + "pass@1": 0.2379296875, + "pass@2": 0.3275477362204723, + "pass@4": 0.4267573031496054, + "pass@8": 0.5281339494014066, + "pass@16": 0.6227814416502605, + "pass@32": 0.7052228942192685, + "pass@64": 0.7743729632317604, + "pass@128": 0.831 + }, + "6": { + "pass@1": 0.188609375, + "pass@2": 0.29246530511811025, + "pass@4": 0.4121283263029615, + "pass@8": 0.5339728926680833, + "pass@16": 0.6504527221959205, + "pass@32": 0.7558571665056636, + "pass@64": 0.839577416192582, + "pass@128": 0.897 + }, + "7": { + "pass@1": 0.1702421875, + "pass@2": 0.2610345718503936, + "pass@4": 0.36890081514810585, + "pass@8": 0.4851046679913388, + "pass@16": 0.6031619242717101, + "pass@32": 0.7133174129702993, + "pass@64": 0.8032198822543871, + "pass@128": 0.865 + }, + "8": { + "pass@1": 0.17203125, + "pass@2": 0.260486466535433, + "pass@4": 0.3623691548556425, + "pass@8": 0.4698659992521673, + "pass@16": 0.579080865405446, + "pass@32": 0.6871811907288851, + "pass@64": 0.7839559840998817, + "pass@128": 0.857 + }, + "9": { + "pass@1": 0.14434375, + "pass@2": 0.22378924704724426, + "pass@4": 0.3192750948631419, + "pass@8": 0.4241588295586182, + "pass@16": 0.5365467898236371, + "pass@32": 0.6541221611393754, + "pass@64": 0.765938761829036, + "pass@128": 0.854 + }, + "11": { + "pass@1": 0.1424921875, + "pass@2": 0.22340280511811042, + "pass@4": 0.3188062520622418, + "pass@8": 0.41971003687558767, + "pass@16": 0.5244687780376375, + "pass@32": 0.6351116073592982, + "pass@64": 0.7457114417547469, + "pass@128": 0.841 + }, + "12": { + "pass@1": 0.1580390625, + "pass@2": 0.24750639763779528, + "pass@4": 0.34879692050993577, + "pass@8": 0.4489444014457862, + "pass@16": 0.5471894408362217, + "pass@32": 0.6482069592134363, + "pass@64": 0.7487004100526425, + "pass@128": 0.837 + }, + "13": { + "pass@1": 0.137203125, + "pass@2": 0.21484793307086603, + "pass@4": 0.30410517894638156, + "pass@8": 0.39436279262935253, + "pass@16": 0.4869214245024962, + "pass@32": 0.5885516578108125, + "pass@64": 0.6977312707591256, + "pass@128": 0.802 + }, + "14": { + "pass@1": 0.138421875, + "pass@2": 0.21568257874015784, + "pass@4": 0.30250095303712055, + "pass@8": 0.38742854884151073, + "pass@16": 0.471113950774246, + "pass@32": 0.5627169993779305, + "pass@64": 0.6662268699433128, + "pass@128": 0.772 + }, + "15": { + "pass@1": 0.1236796875, + "pass@2": 0.19075787401574845, + "pass@4": 0.2672861099550049, + "pass@8": 0.3457352960827552, + "pass@16": 0.42617246292005095, + "pass@32": 0.5152098938373147, + "pass@64": 0.6171748124143301, + "pass@128": 0.719 + }, + "16": { + "pass@1": 0.126515625, + "pass@2": 0.1987988435039372, + "pass@4": 0.2826862190663667, + "pass@8": 0.3678530617021002, + "pass@16": 0.45093558358661096, + "pass@32": 0.5351528343825194, + "pass@64": 0.6253042881510136, + "pass@128": 0.721 + }, + "17": { + "pass@1": 0.1192109375, + "pass@2": 0.18613152066929148, + "pass@4": 0.26321594891263583, + "pass@8": 0.3419057828768377, + "pass@16": 0.4220269175740973, + "pass@32": 0.5105044387235352, + "pass@64": 0.6118238153555224, + "pass@128": 0.72 + }, + "18": { + "pass@1": 0.107359375, + "pass@2": 0.16992679625984286, + "pass@4": 0.24431265832395888, + "pass@8": 0.3210678898460004, + "pass@16": 0.39598269900224564, + "pass@32": 0.47565861398601084, + "pass@64": 0.5682760133155218, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1059765625, + "pass@2": 0.1661788877952758, + "pass@4": 0.23450728430821097, + "pass@8": 0.30133843327893156, + "pass@16": 0.365023901924948, + "pass@32": 0.43474103369718325, + "pass@64": 0.5206322595279965, + "pass@128": 0.623 + }, + "20": { + "pass@1": 0.099953125, + "pass@2": 0.15755831692913397, + "pass@4": 0.22538670613048326, + "pass@8": 0.2949875466640804, + "pass@16": 0.3631328410827, + "pass@32": 0.43517898264575217, + "pass@64": 0.5210332365943791, + "pass@128": 0.624 + } + }, + "per_op_accuracy": { + "10": 0.1535625, + "2": 1.0, + "3": 0.3422890625, + "4": 0.20628125, + "5": 0.2379296875, + "6": 0.188609375, + "7": 0.1702421875, + "8": 0.17203125, + "9": 0.14434375, + "11": 0.1424921875, + "12": 0.1580390625, + "13": 0.137203125, + "14": 0.138421875, + "15": 0.1236796875, + "16": 0.126515625, + "17": 0.1192109375, + "18": 0.107359375, + "19": 0.1059765625, + "20": 0.099953125 + }, + "per_op_avg_response_len": { + "10": 291.2561484375, + "2": 107.38678125, + "3": 171.28271875, + "4": 156.9440625, + "5": 179.8192109375, + "6": 232.113046875, + "7": 234.457640625, + "8": 260.4949296875, + "9": 283.1112109375, + "11": 309.7439453125, + "12": 306.5526171875, + "13": 313.8674609375, + "14": 344.5398203125, + "15": 362.3768671875, + "16": 376.48190625, + "17": 380.589515625, + "18": 386.76740625, + "19": 391.3802734375, + "20": 403.330046875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 291.2561484375, + "2": 107.38678125, + "3": 171.28271875, + "4": 156.9440625, + "5": 179.8192109375, + "6": 232.113046875, + "7": 234.457640625, + "8": 260.4949296875, + "9": 283.1112109375, + "11": 309.7439453125, + "12": 306.5526171875, + "13": 313.8674609375, + "14": 344.5398203125, + "15": 362.3768671875, + "16": 376.48190625, + "17": 380.589515625, + "18": 386.76740625, + "19": 391.3802734375, + "20": 403.330046875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 495890, + "answer_accuracy": 0.20390213815789474, + "avg_response_len": 289.07871628289473, + "resp_tokens_sum": 703039438, + "pass_at_k": { + "pass@1": 0.20390213815789474, + "pass@2": 0.28169427320761264, + "pass@4": 0.36994568558698104, + "pass@8": 0.45911058211989764, + "pass@16": 0.5461745390736306, + "pass@32": 0.6331242945048048, + "pass@64": 0.7193773070827906, + "pass@128": 0.7987368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1535625, + "pass@2": 0.23615563484251956, + "pass@4": 0.3313644041057365, + "pass@8": 0.4322543443704919, + "pass@16": 0.5397517873318299, + "pass@32": 0.6536812205750336, + "pass@64": 0.7635182573212812, + "pass@128": 0.854 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.3422890625, + "pass@2": 0.469228469488189, + "pass@4": 0.5868346908511433, + "pass@8": 0.6825482078242203, + "pass@16": 0.7552705310444486, + "pass@32": 0.8107429866510845, + "pass@64": 0.8542986323759144, + "pass@128": 0.891 + }, + "4": { + "pass@1": 0.20628125, + "pass@2": 0.31069180610236224, + "pass@4": 0.42973400562429676, + "pass@8": 0.5437283789685704, + "pass@16": 0.6373021804337103, + "pass@32": 0.7082035417675484, + "pass@64": 0.760672519399519, + "pass@128": 0.798 + }, + "5": { + "pass@1": 0.2379296875, + "pass@2": 0.3275477362204723, + "pass@4": 0.4267573031496054, + "pass@8": 0.5281339494014066, + "pass@16": 0.6227814416502605, + "pass@32": 0.7052228942192685, + "pass@64": 0.7743729632317604, + "pass@128": 0.831 + }, + "6": { + "pass@1": 0.188609375, + "pass@2": 0.29246530511811025, + "pass@4": 0.4121283263029615, + "pass@8": 0.5339728926680833, + "pass@16": 0.6504527221959205, + "pass@32": 0.7558571665056636, + "pass@64": 0.839577416192582, + "pass@128": 0.897 + }, + "7": { + "pass@1": 0.1702421875, + "pass@2": 0.2610345718503936, + "pass@4": 0.36890081514810585, + "pass@8": 0.4851046679913388, + "pass@16": 0.6031619242717101, + "pass@32": 0.7133174129702993, + "pass@64": 0.8032198822543871, + "pass@128": 0.865 + }, + "8": { + "pass@1": 0.17203125, + "pass@2": 0.260486466535433, + "pass@4": 0.3623691548556425, + "pass@8": 0.4698659992521673, + "pass@16": 0.579080865405446, + "pass@32": 0.6871811907288851, + "pass@64": 0.7839559840998817, + "pass@128": 0.857 + }, + "9": { + "pass@1": 0.14434375, + "pass@2": 0.22378924704724426, + "pass@4": 0.3192750948631419, + "pass@8": 0.4241588295586182, + "pass@16": 0.5365467898236371, + "pass@32": 0.6541221611393754, + "pass@64": 0.765938761829036, + "pass@128": 0.854 + }, + "11": { + "pass@1": 0.1424921875, + "pass@2": 0.22340280511811042, + "pass@4": 0.3188062520622418, + "pass@8": 0.41971003687558767, + "pass@16": 0.5244687780376375, + "pass@32": 0.6351116073592982, + "pass@64": 0.7457114417547469, + "pass@128": 0.841 + }, + "12": { + "pass@1": 0.1580390625, + "pass@2": 0.24750639763779528, + "pass@4": 0.34879692050993577, + "pass@8": 0.4489444014457862, + "pass@16": 0.5471894408362217, + "pass@32": 0.6482069592134363, + "pass@64": 0.7487004100526425, + "pass@128": 0.837 + }, + "13": { + "pass@1": 0.137203125, + "pass@2": 0.21484793307086603, + "pass@4": 0.30410517894638156, + "pass@8": 0.39436279262935253, + "pass@16": 0.4869214245024962, + "pass@32": 0.5885516578108125, + "pass@64": 0.6977312707591256, + "pass@128": 0.802 + }, + "14": { + "pass@1": 0.138421875, + "pass@2": 0.21568257874015784, + "pass@4": 0.30250095303712055, + "pass@8": 0.38742854884151073, + "pass@16": 0.471113950774246, + "pass@32": 0.5627169993779305, + "pass@64": 0.6662268699433128, + "pass@128": 0.772 + }, + "15": { + "pass@1": 0.1236796875, + "pass@2": 0.19075787401574845, + "pass@4": 0.2672861099550049, + "pass@8": 0.3457352960827552, + "pass@16": 0.42617246292005095, + "pass@32": 0.5152098938373147, + "pass@64": 0.6171748124143301, + "pass@128": 0.719 + }, + "16": { + "pass@1": 0.126515625, + "pass@2": 0.1987988435039372, + "pass@4": 0.2826862190663667, + "pass@8": 0.3678530617021002, + "pass@16": 0.45093558358661096, + "pass@32": 0.5351528343825194, + "pass@64": 0.6253042881510136, + "pass@128": 0.721 + }, + "17": { + "pass@1": 0.1192109375, + "pass@2": 0.18613152066929148, + "pass@4": 0.26321594891263583, + "pass@8": 0.3419057828768377, + "pass@16": 0.4220269175740973, + "pass@32": 0.5105044387235352, + "pass@64": 0.6118238153555224, + "pass@128": 0.72 + }, + "18": { + "pass@1": 0.107359375, + "pass@2": 0.16992679625984286, + "pass@4": 0.24431265832395888, + "pass@8": 0.3210678898460004, + "pass@16": 0.39598269900224564, + "pass@32": 0.47565861398601084, + "pass@64": 0.5682760133155218, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1059765625, + "pass@2": 0.1661788877952758, + "pass@4": 0.23450728430821097, + "pass@8": 0.30133843327893156, + "pass@16": 0.365023901924948, + "pass@32": 0.43474103369718325, + "pass@64": 0.5206322595279965, + "pass@128": 0.623 + }, + "20": { + "pass@1": 0.099953125, + "pass@2": 0.15755831692913397, + "pass@4": 0.22538670613048326, + "pass@8": 0.2949875466640804, + "pass@16": 0.3631328410827, + "pass@32": 0.43517898264575217, + "pass@64": 0.5210332365943791, + "pass@128": 0.624 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..ec8fb4bd9c87feec33c7d8c1ed8d6987aa5ec0db --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive", "results_dir": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18010_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 133962, "value_mismatch": 706933, "dependency_mismatch": 786936, "answer_mismatch": 817231, "extra_nodes": 835359, "missing_nodes": 1016115, "perfect_with_extra": 2, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 706933, "dependency_mismatch": 786936, "answer_mismatch": 817231, "extra_nodes": 835359, "missing_nodes": 1016115, "perfect": 133962, "perfect_with_extra": 2}}, "results_path": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/checkpoint-18010_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18010_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 1035682, "dependency_mismatch": 1009419, "answer_mismatch": 1118862, "extra_nodes": 1229733, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 1035682, "dependency_mismatch": 1009419, "answer_mismatch": 1118862, "extra_nodes": 1229733, "missing_nodes": 1280000}}, "results_path": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/checkpoint-18010_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 133962, "value_mismatch": 1742615, "dependency_mismatch": 1796355, "answer_mismatch": 1936093, "extra_nodes": 2065092, "missing_nodes": 2296115, "perfect_with_extra": 2, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1742615, "dependency_mismatch": 1796355, "answer_mismatch": 1936093, "extra_nodes": 2065092, "missing_nodes": 2296115, "perfect": 133962, "perfect_with_extra": 2}}}} diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..df54b6fc3fa7fc10cf61adf31b43da080a65403f --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "checkpoint-18010_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.001, + "pass@2": 0.003, + "pass@4": 0.006, + "pass@8": 0.01, + "pass@16": 0.012, + "pass@32": 0.013, + "pass@64": 0.018, + "pass@128": 0.022 + }, + "4": { + "pass@1": 0.018, + "pass@2": 0.03, + "pass@4": 0.042, + "pass@8": 0.053, + "pass@16": 0.06, + "pass@32": 0.068, + "pass@64": 0.086, + "pass@128": 0.107 + }, + "5": { + "pass@1": 0.018, + "pass@2": 0.033, + "pass@4": 0.039, + "pass@8": 0.056, + "pass@16": 0.072, + "pass@32": 0.084, + "pass@64": 0.106, + "pass@128": 0.118 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.001 + }, + "7": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.001, + "pass@32": 0.002, + "pass@64": 0.002, + "pass@128": 0.002 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.001, + "4": 0.018, + "5": 0.018, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 1.0, + "3": 0.003, + "4": 0.03, + "5": 0.033, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 1.0, + "3": 0.006, + "4": 0.042, + "5": 0.039, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 1.0, + "3": 0.01, + "4": 0.053, + "5": 0.056, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 1.0, + "3": 0.012, + "4": 0.06, + "5": 0.072, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 1.0, + "3": 0.013, + "4": 0.068, + "5": 0.084, + "6": 0.0, + "7": 0.002, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 1.0, + "3": 0.018, + "4": 0.086, + "5": 0.106, + "6": 0.0, + "7": 0.002, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 1.0, + "3": 0.022, + "4": 0.107, + "5": 0.118, + "6": 0.001, + "7": 0.002, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11533333333333333, + "pass@2": 0.11855555555555555, + "pass@4": 0.12088888888888889, + "pass@8": 0.12444444444444444, + "pass@16": 0.1272222222222222, + "pass@32": 0.12966666666666668, + "pass@64": 0.13466666666666666, + "pass@128": 0.1388888888888889 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.001, + "pass@2": 0.003, + "pass@4": 0.006, + "pass@8": 0.01, + "pass@16": 0.012, + "pass@32": 0.013, + "pass@64": 0.018, + "pass@128": 0.022 + }, + "4": { + "pass@1": 0.018, + "pass@2": 0.03, + "pass@4": 0.042, + "pass@8": 0.053, + "pass@16": 0.06, + "pass@32": 0.068, + "pass@64": 0.086, + "pass@128": 0.107 + }, + "5": { + "pass@1": 0.018, + "pass@2": 0.033, + "pass@4": 0.039, + "pass@8": 0.056, + "pass@16": 0.072, + "pass@32": 0.084, + "pass@64": 0.106, + "pass@128": 0.118 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.001 + }, + "7": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.001, + "pass@32": 0.002, + "pass@64": 0.002, + "pass@128": 0.002 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.001, + "4": 0.018, + "5": 0.018, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 1.0, + "3": 0.003, + "4": 0.03, + "5": 0.033, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 1.0, + "3": 0.006, + "4": 0.042, + "5": 0.039, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 1.0, + "3": 0.01, + "4": 0.053, + "5": 0.056, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 1.0, + "3": 0.012, + "4": 0.06, + "5": 0.072, + "6": 0.0, + "7": 0.001, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 1.0, + "3": 0.013, + "4": 0.068, + "5": 0.084, + "6": 0.0, + "7": 0.002, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 1.0, + "3": 0.018, + "4": 0.086, + "5": 0.106, + "6": 0.0, + "7": 0.002, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 1.0, + "3": 0.022, + "4": 0.107, + "5": 0.118, + "6": 0.001, + "7": 0.002, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11533333333333333, + "pass@2": 0.11855555555555555, + "pass@4": 0.12088888888888889, + "pass@8": 0.12444444444444444, + "pass@16": 0.1272222222222222, + "pass@32": 0.12966666666666668, + "pass@64": 0.13466666666666666, + "pass@128": 0.1388888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-18010_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/summary.csv b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..056f1973e5c57c4af5d2ce685d3f53142608dc9b --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18010,0.2905876736111111,0.12588515625,0.20390213815789474,0.0,0.0,0.0,212.98508333333334,357.5629859375,1152000,1280000,2432000 diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/summary.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9854e8527ee4c5c6dbdee307e32c3e2af552a5af --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/autoregressive/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18010", + "id_acc": 0.2905876736111111, + "ood_acc": 0.12588515625, + "total_acc": 0.20390213815789474, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 212.98508333333334, + "ood_avg_resp_len": 357.5629859375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 495890, + "answer_accuracy": 0.20390213815789474, + "avg_response_len": 289.07871628289473, + "resp_tokens_sum": 703039438, + "pass_at_k": { + "pass@1": 0.20390213815789474, + "pass@2": 0.28169427320761264, + "pass@4": 0.36994568558698104, + "pass@8": 0.45911058211989764, + "pass@16": 0.5461745390736306, + "pass@32": 0.6331242945048048, + "pass@64": 0.7193773070827906, + "pass@128": 0.7987368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1535625, + "pass@2": 0.23615563484251956, + "pass@4": 0.3313644041057365, + "pass@8": 0.4322543443704919, + "pass@16": 0.5397517873318299, + "pass@32": 0.6536812205750336, + "pass@64": 0.7635182573212812, + "pass@128": 0.854 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.3422890625, + "pass@2": 0.469228469488189, + "pass@4": 0.5868346908511433, + "pass@8": 0.6825482078242203, + "pass@16": 0.7552705310444486, + "pass@32": 0.8107429866510845, + "pass@64": 0.8542986323759144, + "pass@128": 0.891 + }, + "4": { + "pass@1": 0.20628125, + "pass@2": 0.31069180610236224, + "pass@4": 0.42973400562429676, + "pass@8": 0.5437283789685704, + "pass@16": 0.6373021804337103, + "pass@32": 0.7082035417675484, + "pass@64": 0.760672519399519, + "pass@128": 0.798 + }, + "5": { + "pass@1": 0.2379296875, + "pass@2": 0.3275477362204723, + "pass@4": 0.4267573031496054, + "pass@8": 0.5281339494014066, + "pass@16": 0.6227814416502605, + "pass@32": 0.7052228942192685, + "pass@64": 0.7743729632317604, + "pass@128": 0.831 + }, + "6": { + "pass@1": 0.188609375, + "pass@2": 0.29246530511811025, + "pass@4": 0.4121283263029615, + "pass@8": 0.5339728926680833, + "pass@16": 0.6504527221959205, + "pass@32": 0.7558571665056636, + "pass@64": 0.839577416192582, + "pass@128": 0.897 + }, + "7": { + "pass@1": 0.1702421875, + "pass@2": 0.2610345718503936, + "pass@4": 0.36890081514810585, + "pass@8": 0.4851046679913388, + "pass@16": 0.6031619242717101, + "pass@32": 0.7133174129702993, + "pass@64": 0.8032198822543871, + "pass@128": 0.865 + }, + "8": { + "pass@1": 0.17203125, + "pass@2": 0.260486466535433, + "pass@4": 0.3623691548556425, + "pass@8": 0.4698659992521673, + "pass@16": 0.579080865405446, + "pass@32": 0.6871811907288851, + "pass@64": 0.7839559840998817, + "pass@128": 0.857 + }, + "9": { + "pass@1": 0.14434375, + "pass@2": 0.22378924704724426, + "pass@4": 0.3192750948631419, + "pass@8": 0.4241588295586182, + "pass@16": 0.5365467898236371, + "pass@32": 0.6541221611393754, + "pass@64": 0.765938761829036, + "pass@128": 0.854 + }, + "11": { + "pass@1": 0.1424921875, + "pass@2": 0.22340280511811042, + "pass@4": 0.3188062520622418, + "pass@8": 0.41971003687558767, + "pass@16": 0.5244687780376375, + "pass@32": 0.6351116073592982, + "pass@64": 0.7457114417547469, + "pass@128": 0.841 + }, + "12": { + "pass@1": 0.1580390625, + "pass@2": 0.24750639763779528, + "pass@4": 0.34879692050993577, + "pass@8": 0.4489444014457862, + "pass@16": 0.5471894408362217, + "pass@32": 0.6482069592134363, + "pass@64": 0.7487004100526425, + "pass@128": 0.837 + }, + "13": { + "pass@1": 0.137203125, + "pass@2": 0.21484793307086603, + "pass@4": 0.30410517894638156, + "pass@8": 0.39436279262935253, + "pass@16": 0.4869214245024962, + "pass@32": 0.5885516578108125, + "pass@64": 0.6977312707591256, + "pass@128": 0.802 + }, + "14": { + "pass@1": 0.138421875, + "pass@2": 0.21568257874015784, + "pass@4": 0.30250095303712055, + "pass@8": 0.38742854884151073, + "pass@16": 0.471113950774246, + "pass@32": 0.5627169993779305, + "pass@64": 0.6662268699433128, + "pass@128": 0.772 + }, + "15": { + "pass@1": 0.1236796875, + "pass@2": 0.19075787401574845, + "pass@4": 0.2672861099550049, + "pass@8": 0.3457352960827552, + "pass@16": 0.42617246292005095, + "pass@32": 0.5152098938373147, + "pass@64": 0.6171748124143301, + "pass@128": 0.719 + }, + "16": { + "pass@1": 0.126515625, + "pass@2": 0.1987988435039372, + "pass@4": 0.2826862190663667, + "pass@8": 0.3678530617021002, + "pass@16": 0.45093558358661096, + "pass@32": 0.5351528343825194, + "pass@64": 0.6253042881510136, + "pass@128": 0.721 + }, + "17": { + "pass@1": 0.1192109375, + "pass@2": 0.18613152066929148, + "pass@4": 0.26321594891263583, + "pass@8": 0.3419057828768377, + "pass@16": 0.4220269175740973, + "pass@32": 0.5105044387235352, + "pass@64": 0.6118238153555224, + "pass@128": 0.72 + }, + "18": { + "pass@1": 0.107359375, + "pass@2": 0.16992679625984286, + "pass@4": 0.24431265832395888, + "pass@8": 0.3210678898460004, + "pass@16": 0.39598269900224564, + "pass@32": 0.47565861398601084, + "pass@64": 0.5682760133155218, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1059765625, + "pass@2": 0.1661788877952758, + "pass@4": 0.23450728430821097, + "pass@8": 0.30133843327893156, + "pass@16": 0.365023901924948, + "pass@32": 0.43474103369718325, + "pass@64": 0.5206322595279965, + "pass@128": 0.623 + }, + "20": { + "pass@1": 0.099953125, + "pass@2": 0.15755831692913397, + "pass@4": 0.22538670613048326, + "pass@8": 0.2949875466640804, + "pass@16": 0.3631328410827, + "pass@32": 0.43517898264575217, + "pass@64": 0.5210332365943791, + "pass@128": 0.624 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/checkpoint-18010_metrics.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/checkpoint-18010_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e858a4a52d2096d90f819a587df555eb84763264 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/checkpoint-18010_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18010", + "id": { + "answer_accuracy": 0.28398524305555556, + "avg_response_len": 176.00885243055555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.28398524305555556, + "pass@2": 0.3522284011373653, + "pass@4": 0.4256367105049421, + "pass@8": 0.5001374387394704, + "pass@16": 0.5733062812429726, + "pass@32": 0.6413248869341965, + "pass@64": 0.6993228171085947, + "pass@128": 0.7461111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.152578125, + "pass@2": 0.22779023129921225, + "pass@4": 0.3139792260029995, + "pass@8": 0.40436801298890296, + "pass@16": 0.49550086207735394, + "pass@32": 0.5839780263217058, + "pass@64": 0.6638323088261202, + "pass@128": 0.734 + }, + "2": { + "pass@1": 0.9592265625, + "pass@2": 0.9772586122047247, + "pass@4": 0.9870330117172853, + "pass@8": 0.990970424750734, + "pass@16": 0.9926473174923504, + "pass@32": 0.9934851717255498, + "pass@64": 0.9938770845368372, + "pass@128": 0.994 + }, + "3": { + "pass@1": 0.459953125, + "pass@2": 0.5616037155511809, + "pass@4": 0.6435666486689159, + "pass@8": 0.7044025930160407, + "pass@16": 0.7511445535571443, + "pass@32": 0.7887028787756671, + "pass@64": 0.8182136826642157, + "pass@128": 0.842 + }, + "4": { + "pass@1": 0.171296875, + "pass@2": 0.23645841535433068, + "pass@4": 0.3053068216160478, + "pass@8": 0.36905284743028616, + "pass@16": 0.42313792247658066, + "pass@32": 0.46804808322507935, + "pass@64": 0.5041870246526634, + "pass@128": 0.531 + }, + "5": { + "pass@1": 0.1858046875, + "pass@2": 0.24397477854330707, + "pass@4": 0.3084486854143231, + "pass@8": 0.3761193180463611, + "pass@16": 0.4424574609942841, + "pass@32": 0.5033182989441415, + "pass@64": 0.5566335335375966, + "pass@128": 0.601 + }, + "6": { + "pass@1": 0.160296875, + "pass@2": 0.23703494094488164, + "pass@4": 0.3317535394638169, + "pass@8": 0.4385031130403318, + "pass@16": 0.5461951024846229, + "pass@32": 0.6421663495162627, + "pass@64": 0.7170787291465208, + "pass@128": 0.77 + }, + "7": { + "pass@1": 0.1524609375, + "pass@2": 0.2226471456692912, + "pass@4": 0.30402510170603647, + "pass@8": 0.3938039793827515, + "pass@16": 0.4905381693736334, + "pass@32": 0.5854082930981304, + "pass@64": 0.6648636050543109, + "pass@128": 0.725 + }, + "8": { + "pass@1": 0.1659296875, + "pass@2": 0.2433890255905511, + "pass@4": 0.33236180530558707, + "pass@8": 0.42672638252712924, + "pass@16": 0.521472596002238, + "pass@32": 0.6115113622261756, + "pass@64": 0.691711565222001, + "pass@128": 0.761 + }, + "9": { + "pass@1": 0.1483203125, + "pass@2": 0.21989874507874002, + "pass@4": 0.3042555546494189, + "pass@8": 0.3972902774725913, + "pass@16": 0.49666254672837173, + "pass@32": 0.5953055185750495, + "pass@64": 0.6835078203370488, + "pass@128": 0.757 + } + }, + "per_op_accuracy": { + "10": 0.152578125, + "2": 0.9592265625, + "3": 0.459953125, + "4": 0.171296875, + "5": 0.1858046875, + "6": 0.160296875, + "7": 0.1524609375, + "8": 0.1659296875, + "9": 0.1483203125 + }, + "per_op_avg_response_len": { + "10": 235.28671875, + "2": 106.5559609375, + "3": 136.3905859375, + "4": 135.8745234375, + "5": 155.457046875, + "6": 188.26746875, + "7": 188.7669921875, + "8": 209.9766171875, + "9": 227.5037578125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 235.28671875, + "2": 106.5559609375, + "3": 136.3905859375, + "4": 135.8745234375, + "5": 155.457046875, + "6": 188.26746875, + "7": 188.7669921875, + "8": 209.9766171875, + "9": 227.5037578125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 327151, + "answer_accuracy": 0.28398524305555556, + "avg_response_len": 176.00885243055555, + "resp_tokens_sum": 202762198, + "pass_at_k": { + "pass@1": 0.28398524305555556, + "pass@2": 0.3522284011373653, + "pass@4": 0.4256367105049421, + "pass@8": 0.5001374387394704, + "pass@16": 0.5733062812429726, + "pass@32": 0.6413248869341965, + "pass@64": 0.6993228171085947, + "pass@128": 0.7461111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.152578125, + "pass@2": 0.22779023129921225, + "pass@4": 0.3139792260029995, + "pass@8": 0.40436801298890296, + "pass@16": 0.49550086207735394, + "pass@32": 0.5839780263217058, + "pass@64": 0.6638323088261202, + "pass@128": 0.734 + }, + "2": { + "pass@1": 0.9592265625, + "pass@2": 0.9772586122047247, + "pass@4": 0.9870330117172853, + "pass@8": 0.990970424750734, + "pass@16": 0.9926473174923504, + "pass@32": 0.9934851717255498, + "pass@64": 0.9938770845368372, + "pass@128": 0.994 + }, + "3": { + "pass@1": 0.459953125, + "pass@2": 0.5616037155511809, + "pass@4": 0.6435666486689159, + "pass@8": 0.7044025930160407, + "pass@16": 0.7511445535571443, + "pass@32": 0.7887028787756671, + "pass@64": 0.8182136826642157, + "pass@128": 0.842 + }, + "4": { + "pass@1": 0.171296875, + "pass@2": 0.23645841535433068, + "pass@4": 0.3053068216160478, + "pass@8": 0.36905284743028616, + "pass@16": 0.42313792247658066, + "pass@32": 0.46804808322507935, + "pass@64": 0.5041870246526634, + "pass@128": 0.531 + }, + "5": { + "pass@1": 0.1858046875, + "pass@2": 0.24397477854330707, + "pass@4": 0.3084486854143231, + "pass@8": 0.3761193180463611, + "pass@16": 0.4424574609942841, + "pass@32": 0.5033182989441415, + "pass@64": 0.5566335335375966, + "pass@128": 0.601 + }, + "6": { + "pass@1": 0.160296875, + "pass@2": 0.23703494094488164, + "pass@4": 0.3317535394638169, + "pass@8": 0.4385031130403318, + "pass@16": 0.5461951024846229, + "pass@32": 0.6421663495162627, + "pass@64": 0.7170787291465208, + "pass@128": 0.77 + }, + "7": { + "pass@1": 0.1524609375, + "pass@2": 0.2226471456692912, + "pass@4": 0.30402510170603647, + "pass@8": 0.3938039793827515, + "pass@16": 0.4905381693736334, + "pass@32": 0.5854082930981304, + "pass@64": 0.6648636050543109, + "pass@128": 0.725 + }, + "8": { + "pass@1": 0.1659296875, + "pass@2": 0.2433890255905511, + "pass@4": 0.33236180530558707, + "pass@8": 0.42672638252712924, + "pass@16": 0.521472596002238, + "pass@32": 0.6115113622261756, + "pass@64": 0.691711565222001, + "pass@128": 0.761 + }, + "9": { + "pass@1": 0.1483203125, + "pass@2": 0.21989874507874002, + "pass@4": 0.3042555546494189, + "pass@8": 0.3972902774725913, + "pass@16": 0.49666254672837173, + "pass@32": 0.5953055185750495, + "pass@64": 0.6835078203370488, + "pass@128": 0.757 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.12071171875, + "avg_response_len": 296.813384375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.12071171875, + "pass@2": 0.18984655511811116, + "pass@4": 0.27028233612673946, + "pass@8": 0.3498542930184956, + "pass@16": 0.4233829034381145, + "pass@32": 0.4942663056433594, + "pass@64": 0.5663841008925269, + "pass@128": 0.6397 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1420703125, + "pass@2": 0.21616498523622027, + "pass@4": 0.30243917032245937, + "pass@8": 0.3926092870352065, + "pass@16": 0.484249329100816, + "pass@32": 0.5735155601485259, + "pass@64": 0.6529675754419116, + "pass@128": 0.719 + }, + "12": { + "pass@1": 0.15053125, + "pass@2": 0.22905511811023616, + "pass@4": 0.31439215288713934, + "pass@8": 0.39659629492888226, + "pass@16": 0.47593892996953874, + "pass@32": 0.5569127029398856, + "pass@64": 0.6386010774459661, + "pass@128": 0.713 + }, + "13": { + "pass@1": 0.1205078125, + "pass@2": 0.18958919783464573, + "pass@4": 0.2699959077615294, + "pass@8": 0.3499377130267689, + "pass@16": 0.4247099285646272, + "pass@32": 0.4994672172014515, + "pass@64": 0.5773712084384967, + "pass@128": 0.652 + }, + "14": { + "pass@1": 0.126421875, + "pass@2": 0.2015922736220474, + "pass@4": 0.28945480708661403, + "pass@8": 0.37579113501502814, + "pass@16": 0.45640708831824817, + "pass@32": 0.5372244048600412, + "pass@64": 0.6196751926235065, + "pass@128": 0.699 + }, + "15": { + "pass@1": 0.1149375, + "pass@2": 0.18155684055118107, + "pass@4": 0.2597222063179599, + "pass@8": 0.3384902243470089, + "pass@16": 0.41237075595856404, + "pass@32": 0.48508771063964434, + "pass@64": 0.5601345517764789, + "pass@128": 0.636 + }, + "16": { + "pass@1": 0.1270703125, + "pass@2": 0.19944611220472444, + "pass@4": 0.28334039998125204, + "pass@8": 0.3658349458965953, + "pass@16": 0.44124756629096407, + "pass@32": 0.5112130336255106, + "pass@64": 0.57938709860515, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.11515625, + "pass@2": 0.18289714566929116, + "pass@4": 0.2620662944319459, + "pass@8": 0.33891554447726824, + "pass@16": 0.4063832820638924, + "pass@32": 0.4688954151261894, + "pass@64": 0.5355758065020465, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.10871875, + "pass@2": 0.17352829724409446, + "pass@4": 0.2508396094863139, + "pass@8": 0.3269651830284786, + "pass@16": 0.395449007785804, + "pass@32": 0.46187129159386986, + "pass@64": 0.531695477409159, + "pass@128": 0.605 + }, + "19": { + "pass@1": 0.1031953125, + "pass@2": 0.16674864665354308, + "pass@4": 0.24180379208848854, + "pass@8": 0.31416790461546495, + "pass@16": 0.37557485028095305, + "pass@32": 0.43097992188912404, + "pass@64": 0.49095567424554715, + "pass@128": 0.56 + }, + "20": { + "pass@1": 0.0985078125, + "pass@2": 0.15788693405511822, + "pass@4": 0.22876902090363677, + "pass@8": 0.29923469781414813, + "pass@16": 0.36149829604766187, + "pass@32": 0.41749579840940637, + "pass@64": 0.47747734643701806, + "pass@128": 0.55 + } + }, + "per_op_accuracy": { + "11": 0.1420703125, + "12": 0.15053125, + "13": 0.1205078125, + "14": 0.126421875, + "15": 0.1149375, + "16": 0.1270703125, + "17": 0.11515625, + "18": 0.10871875, + "19": 0.1031953125, + "20": 0.0985078125 + }, + "per_op_avg_response_len": { + "11": 252.0244609375, + "12": 261.2362890625, + "13": 269.6761171875, + "14": 285.8716328125, + "15": 294.0159921875, + "16": 313.0270546875, + "17": 314.2615703125, + "18": 319.9344375, + "19": 324.1100234375, + "20": 333.976265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 252.0244609375, + "12": 261.2362890625, + "13": 269.6761171875, + "14": 285.8716328125, + "15": 294.0159921875, + "16": 313.0270546875, + "17": 314.2615703125, + "18": 319.9344375, + "19": 324.1100234375, + "20": 333.976265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 154511, + "answer_accuracy": 0.12071171875, + "avg_response_len": 296.813384375, + "resp_tokens_sum": 379921132, + "pass_at_k": { + "pass@1": 0.12071171875, + "pass@2": 0.18984655511811116, + "pass@4": 0.27028233612673946, + "pass@8": 0.3498542930184956, + "pass@16": 0.4233829034381145, + "pass@32": 0.4942663056433594, + "pass@64": 0.5663841008925269, + "pass@128": 0.6397 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1420703125, + "pass@2": 0.21616498523622027, + "pass@4": 0.30243917032245937, + "pass@8": 0.3926092870352065, + "pass@16": 0.484249329100816, + "pass@32": 0.5735155601485259, + "pass@64": 0.6529675754419116, + "pass@128": 0.719 + }, + "12": { + "pass@1": 0.15053125, + "pass@2": 0.22905511811023616, + "pass@4": 0.31439215288713934, + "pass@8": 0.39659629492888226, + "pass@16": 0.47593892996953874, + "pass@32": 0.5569127029398856, + "pass@64": 0.6386010774459661, + "pass@128": 0.713 + }, + "13": { + "pass@1": 0.1205078125, + "pass@2": 0.18958919783464573, + "pass@4": 0.2699959077615294, + "pass@8": 0.3499377130267689, + "pass@16": 0.4247099285646272, + "pass@32": 0.4994672172014515, + "pass@64": 0.5773712084384967, + "pass@128": 0.652 + }, + "14": { + "pass@1": 0.126421875, + "pass@2": 0.2015922736220474, + "pass@4": 0.28945480708661403, + "pass@8": 0.37579113501502814, + "pass@16": 0.45640708831824817, + "pass@32": 0.5372244048600412, + "pass@64": 0.6196751926235065, + "pass@128": 0.699 + }, + "15": { + "pass@1": 0.1149375, + "pass@2": 0.18155684055118107, + "pass@4": 0.2597222063179599, + "pass@8": 0.3384902243470089, + "pass@16": 0.41237075595856404, + "pass@32": 0.48508771063964434, + "pass@64": 0.5601345517764789, + "pass@128": 0.636 + }, + "16": { + "pass@1": 0.1270703125, + "pass@2": 0.19944611220472444, + "pass@4": 0.28334039998125204, + "pass@8": 0.3658349458965953, + "pass@16": 0.44124756629096407, + "pass@32": 0.5112130336255106, + "pass@64": 0.57938709860515, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.11515625, + "pass@2": 0.18289714566929116, + "pass@4": 0.2620662944319459, + "pass@8": 0.33891554447726824, + "pass@16": 0.4063832820638924, + "pass@32": 0.4688954151261894, + "pass@64": 0.5355758065020465, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.10871875, + "pass@2": 0.17352829724409446, + "pass@4": 0.2508396094863139, + "pass@8": 0.3269651830284786, + "pass@16": 0.395449007785804, + "pass@32": 0.46187129159386986, + "pass@64": 0.531695477409159, + "pass@128": 0.605 + }, + "19": { + "pass@1": 0.1031953125, + "pass@2": 0.16674864665354308, + "pass@4": 0.24180379208848854, + "pass@8": 0.31416790461546495, + "pass@16": 0.37557485028095305, + "pass@32": 0.43097992188912404, + "pass@64": 0.49095567424554715, + "pass@128": 0.56 + }, + "20": { + "pass@1": 0.0985078125, + "pass@2": 0.15788693405511822, + "pass@4": 0.22876902090363677, + "pass@8": 0.29923469781414813, + "pass@16": 0.36149829604766187, + "pass@32": 0.41749579840940637, + "pass@64": 0.47747734643701806, + "pass@128": 0.55 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.19805180921052631, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.19805180921052631, + "pass@2": 0.26676427165354244, + "pass@4": 0.3438712503058711, + "pass@8": 0.4210410462547424, + "pass@16": 0.4943992402930623, + "pass@32": 0.5639256336232448, + "pass@64": 0.6293550717317229, + "pass@128": 0.6901052631578948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.152578125, + "pass@2": 0.22779023129921225, + "pass@4": 0.3139792260029995, + "pass@8": 0.40436801298890296, + "pass@16": 0.49550086207735394, + "pass@32": 0.5839780263217058, + "pass@64": 0.6638323088261202, + "pass@128": 0.734 + }, + "2": { + "pass@1": 0.9592265625, + "pass@2": 0.9772586122047247, + "pass@4": 0.9870330117172853, + "pass@8": 0.990970424750734, + "pass@16": 0.9926473174923504, + "pass@32": 0.9934851717255498, + "pass@64": 0.9938770845368372, + "pass@128": 0.994 + }, + "3": { + "pass@1": 0.459953125, + "pass@2": 0.5616037155511809, + "pass@4": 0.6435666486689159, + "pass@8": 0.7044025930160407, + "pass@16": 0.7511445535571443, + "pass@32": 0.7887028787756671, + "pass@64": 0.8182136826642157, + "pass@128": 0.842 + }, + "4": { + "pass@1": 0.171296875, + "pass@2": 0.23645841535433068, + "pass@4": 0.3053068216160478, + "pass@8": 0.36905284743028616, + "pass@16": 0.42313792247658066, + "pass@32": 0.46804808322507935, + "pass@64": 0.5041870246526634, + "pass@128": 0.531 + }, + "5": { + "pass@1": 0.1858046875, + "pass@2": 0.24397477854330707, + "pass@4": 0.3084486854143231, + "pass@8": 0.3761193180463611, + "pass@16": 0.4424574609942841, + "pass@32": 0.5033182989441415, + "pass@64": 0.5566335335375966, + "pass@128": 0.601 + }, + "6": { + "pass@1": 0.160296875, + "pass@2": 0.23703494094488164, + "pass@4": 0.3317535394638169, + "pass@8": 0.4385031130403318, + "pass@16": 0.5461951024846229, + "pass@32": 0.6421663495162627, + "pass@64": 0.7170787291465208, + "pass@128": 0.77 + }, + "7": { + "pass@1": 0.1524609375, + "pass@2": 0.2226471456692912, + "pass@4": 0.30402510170603647, + "pass@8": 0.3938039793827515, + "pass@16": 0.4905381693736334, + "pass@32": 0.5854082930981304, + "pass@64": 0.6648636050543109, + "pass@128": 0.725 + }, + "8": { + "pass@1": 0.1659296875, + "pass@2": 0.2433890255905511, + "pass@4": 0.33236180530558707, + "pass@8": 0.42672638252712924, + "pass@16": 0.521472596002238, + "pass@32": 0.6115113622261756, + "pass@64": 0.691711565222001, + "pass@128": 0.761 + }, + "9": { + "pass@1": 0.1483203125, + "pass@2": 0.21989874507874002, + "pass@4": 0.3042555546494189, + "pass@8": 0.3972902774725913, + "pass@16": 0.49666254672837173, + "pass@32": 0.5953055185750495, + "pass@64": 0.6835078203370488, + "pass@128": 0.757 + }, + "11": { + "pass@1": 0.1420703125, + "pass@2": 0.21616498523622027, + "pass@4": 0.30243917032245937, + "pass@8": 0.3926092870352065, + "pass@16": 0.484249329100816, + "pass@32": 0.5735155601485259, + "pass@64": 0.6529675754419116, + "pass@128": 0.719 + }, + "12": { + "pass@1": 0.15053125, + "pass@2": 0.22905511811023616, + "pass@4": 0.31439215288713934, + "pass@8": 0.39659629492888226, + "pass@16": 0.47593892996953874, + "pass@32": 0.5569127029398856, + "pass@64": 0.6386010774459661, + "pass@128": 0.713 + }, + "13": { + "pass@1": 0.1205078125, + "pass@2": 0.18958919783464573, + "pass@4": 0.2699959077615294, + "pass@8": 0.3499377130267689, + "pass@16": 0.4247099285646272, + "pass@32": 0.4994672172014515, + "pass@64": 0.5773712084384967, + "pass@128": 0.652 + }, + "14": { + "pass@1": 0.126421875, + "pass@2": 0.2015922736220474, + "pass@4": 0.28945480708661403, + "pass@8": 0.37579113501502814, + "pass@16": 0.45640708831824817, + "pass@32": 0.5372244048600412, + "pass@64": 0.6196751926235065, + "pass@128": 0.699 + }, + "15": { + "pass@1": 0.1149375, + "pass@2": 0.18155684055118107, + "pass@4": 0.2597222063179599, + "pass@8": 0.3384902243470089, + "pass@16": 0.41237075595856404, + "pass@32": 0.48508771063964434, + "pass@64": 0.5601345517764789, + "pass@128": 0.636 + }, + "16": { + "pass@1": 0.1270703125, + "pass@2": 0.19944611220472444, + "pass@4": 0.28334039998125204, + "pass@8": 0.3658349458965953, + "pass@16": 0.44124756629096407, + "pass@32": 0.5112130336255106, + "pass@64": 0.57938709860515, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.11515625, + "pass@2": 0.18289714566929116, + "pass@4": 0.2620662944319459, + "pass@8": 0.33891554447726824, + "pass@16": 0.4063832820638924, + "pass@32": 0.4688954151261894, + "pass@64": 0.5355758065020465, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.10871875, + "pass@2": 0.17352829724409446, + "pass@4": 0.2508396094863139, + "pass@8": 0.3269651830284786, + "pass@16": 0.395449007785804, + "pass@32": 0.46187129159386986, + "pass@64": 0.531695477409159, + "pass@128": 0.605 + }, + "19": { + "pass@1": 0.1031953125, + "pass@2": 0.16674864665354308, + "pass@4": 0.24180379208848854, + "pass@8": 0.31416790461546495, + "pass@16": 0.37557485028095305, + "pass@32": 0.43097992188912404, + "pass@64": 0.49095567424554715, + "pass@128": 0.56 + }, + "20": { + "pass@1": 0.0985078125, + "pass@2": 0.15788693405511822, + "pass@4": 0.22876902090363677, + "pass@8": 0.29923469781414813, + "pass@16": 0.36149829604766187, + "pass@32": 0.41749579840940637, + "pass@64": 0.47747734643701806, + "pass@128": 0.55 + } + }, + "per_op_accuracy": { + "10": 0.152578125, + "2": 0.9592265625, + "3": 0.459953125, + "4": 0.171296875, + "5": 0.1858046875, + "6": 0.160296875, + "7": 0.1524609375, + "8": 0.1659296875, + "9": 0.1483203125, + "11": 0.1420703125, + "12": 0.15053125, + "13": 0.1205078125, + "14": 0.126421875, + "15": 0.1149375, + "16": 0.1270703125, + "17": 0.11515625, + "18": 0.10871875, + "19": 0.1031953125, + "20": 0.0985078125 + }, + "per_op_avg_response_len": { + "10": 235.28671875, + "2": 106.5559609375, + "3": 136.3905859375, + "4": 135.8745234375, + "5": 155.457046875, + "6": 188.26746875, + "7": 188.7669921875, + "8": 209.9766171875, + "9": 227.5037578125, + "11": 252.0244609375, + "12": 261.2362890625, + "13": 269.6761171875, + "14": 285.8716328125, + "15": 294.0159921875, + "16": 313.0270546875, + "17": 314.2615703125, + "18": 319.9344375, + "19": 324.1100234375, + "20": 333.976265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 235.28671875, + "2": 106.5559609375, + "3": 136.3905859375, + "4": 135.8745234375, + "5": 155.457046875, + "6": 188.26746875, + "7": 188.7669921875, + "8": 209.9766171875, + "9": 227.5037578125, + "11": 252.0244609375, + "12": 261.2362890625, + "13": 269.6761171875, + "14": 285.8716328125, + "15": 294.0159921875, + "16": 313.0270546875, + "17": 314.2615703125, + "18": 319.9344375, + "19": 324.1100234375, + "20": 333.976265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 481662, + "answer_accuracy": 0.19805180921052631, + "avg_response_len": 239.59018503289474, + "resp_tokens_sum": 582683330, + "pass_at_k": { + "pass@1": 0.19805180921052631, + "pass@2": 0.26676427165354244, + "pass@4": 0.3438712503058711, + "pass@8": 0.4210410462547424, + "pass@16": 0.4943992402930623, + "pass@32": 0.5639256336232448, + "pass@64": 0.6293550717317229, + "pass@128": 0.6901052631578948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.152578125, + "pass@2": 0.22779023129921225, + "pass@4": 0.3139792260029995, + "pass@8": 0.40436801298890296, + "pass@16": 0.49550086207735394, + "pass@32": 0.5839780263217058, + "pass@64": 0.6638323088261202, + "pass@128": 0.734 + }, + "2": { + "pass@1": 0.9592265625, + "pass@2": 0.9772586122047247, + "pass@4": 0.9870330117172853, + "pass@8": 0.990970424750734, + "pass@16": 0.9926473174923504, + "pass@32": 0.9934851717255498, + "pass@64": 0.9938770845368372, + "pass@128": 0.994 + }, + "3": { + "pass@1": 0.459953125, + "pass@2": 0.5616037155511809, + "pass@4": 0.6435666486689159, + "pass@8": 0.7044025930160407, + "pass@16": 0.7511445535571443, + "pass@32": 0.7887028787756671, + "pass@64": 0.8182136826642157, + "pass@128": 0.842 + }, + "4": { + "pass@1": 0.171296875, + "pass@2": 0.23645841535433068, + "pass@4": 0.3053068216160478, + "pass@8": 0.36905284743028616, + "pass@16": 0.42313792247658066, + "pass@32": 0.46804808322507935, + "pass@64": 0.5041870246526634, + "pass@128": 0.531 + }, + "5": { + "pass@1": 0.1858046875, + "pass@2": 0.24397477854330707, + "pass@4": 0.3084486854143231, + "pass@8": 0.3761193180463611, + "pass@16": 0.4424574609942841, + "pass@32": 0.5033182989441415, + "pass@64": 0.5566335335375966, + "pass@128": 0.601 + }, + "6": { + "pass@1": 0.160296875, + "pass@2": 0.23703494094488164, + "pass@4": 0.3317535394638169, + "pass@8": 0.4385031130403318, + "pass@16": 0.5461951024846229, + "pass@32": 0.6421663495162627, + "pass@64": 0.7170787291465208, + "pass@128": 0.77 + }, + "7": { + "pass@1": 0.1524609375, + "pass@2": 0.2226471456692912, + "pass@4": 0.30402510170603647, + "pass@8": 0.3938039793827515, + "pass@16": 0.4905381693736334, + "pass@32": 0.5854082930981304, + "pass@64": 0.6648636050543109, + "pass@128": 0.725 + }, + "8": { + "pass@1": 0.1659296875, + "pass@2": 0.2433890255905511, + "pass@4": 0.33236180530558707, + "pass@8": 0.42672638252712924, + "pass@16": 0.521472596002238, + "pass@32": 0.6115113622261756, + "pass@64": 0.691711565222001, + "pass@128": 0.761 + }, + "9": { + "pass@1": 0.1483203125, + "pass@2": 0.21989874507874002, + "pass@4": 0.3042555546494189, + "pass@8": 0.3972902774725913, + "pass@16": 0.49666254672837173, + "pass@32": 0.5953055185750495, + "pass@64": 0.6835078203370488, + "pass@128": 0.757 + }, + "11": { + "pass@1": 0.1420703125, + "pass@2": 0.21616498523622027, + "pass@4": 0.30243917032245937, + "pass@8": 0.3926092870352065, + "pass@16": 0.484249329100816, + "pass@32": 0.5735155601485259, + "pass@64": 0.6529675754419116, + "pass@128": 0.719 + }, + "12": { + "pass@1": 0.15053125, + "pass@2": 0.22905511811023616, + "pass@4": 0.31439215288713934, + "pass@8": 0.39659629492888226, + "pass@16": 0.47593892996953874, + "pass@32": 0.5569127029398856, + "pass@64": 0.6386010774459661, + "pass@128": 0.713 + }, + "13": { + "pass@1": 0.1205078125, + "pass@2": 0.18958919783464573, + "pass@4": 0.2699959077615294, + "pass@8": 0.3499377130267689, + "pass@16": 0.4247099285646272, + "pass@32": 0.4994672172014515, + "pass@64": 0.5773712084384967, + "pass@128": 0.652 + }, + "14": { + "pass@1": 0.126421875, + "pass@2": 0.2015922736220474, + "pass@4": 0.28945480708661403, + "pass@8": 0.37579113501502814, + "pass@16": 0.45640708831824817, + "pass@32": 0.5372244048600412, + "pass@64": 0.6196751926235065, + "pass@128": 0.699 + }, + "15": { + "pass@1": 0.1149375, + "pass@2": 0.18155684055118107, + "pass@4": 0.2597222063179599, + "pass@8": 0.3384902243470089, + "pass@16": 0.41237075595856404, + "pass@32": 0.48508771063964434, + "pass@64": 0.5601345517764789, + "pass@128": 0.636 + }, + "16": { + "pass@1": 0.1270703125, + "pass@2": 0.19944611220472444, + "pass@4": 0.28334039998125204, + "pass@8": 0.3658349458965953, + "pass@16": 0.44124756629096407, + "pass@32": 0.5112130336255106, + "pass@64": 0.57938709860515, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.11515625, + "pass@2": 0.18289714566929116, + "pass@4": 0.2620662944319459, + "pass@8": 0.33891554447726824, + "pass@16": 0.4063832820638924, + "pass@32": 0.4688954151261894, + "pass@64": 0.5355758065020465, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.10871875, + "pass@2": 0.17352829724409446, + "pass@4": 0.2508396094863139, + "pass@8": 0.3269651830284786, + "pass@16": 0.395449007785804, + "pass@32": 0.46187129159386986, + "pass@64": 0.531695477409159, + "pass@128": 0.605 + }, + "19": { + "pass@1": 0.1031953125, + "pass@2": 0.16674864665354308, + "pass@4": 0.24180379208848854, + "pass@8": 0.31416790461546495, + "pass@16": 0.37557485028095305, + "pass@32": 0.43097992188912404, + "pass@64": 0.49095567424554715, + "pass@128": 0.56 + }, + "20": { + "pass@1": 0.0985078125, + "pass@2": 0.15788693405511822, + "pass@4": 0.22876902090363677, + "pass@8": 0.29923469781414813, + "pass@16": 0.36149829604766187, + "pass@32": 0.41749579840940637, + "pass@64": 0.47747734643701806, + "pass@128": 0.55 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.csv b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d28720d8a73d58624b28a5d191342efd49234b33 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18010,0.28398524305555556,0.12071171875,0.19805180921052631,0.0,0.0,0.0,176.00885243055555,296.813384375,1152000,1280000,2432000 diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c94d241d16ec94e0e2e4d9dd6d4c751bfe156ea6 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18010", + "id_acc": 0.28398524305555556, + "ood_acc": 0.12071171875, + "total_acc": 0.19805180921052631, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 176.00885243055555, + "ood_avg_resp_len": 296.813384375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 481662, + "answer_accuracy": 0.19805180921052631, + "avg_response_len": 239.59018503289474, + "resp_tokens_sum": 582683330, + "pass_at_k": { + "pass@1": 0.19805180921052631, + "pass@2": 0.26676427165354244, + "pass@4": 0.3438712503058711, + "pass@8": 0.4210410462547424, + "pass@16": 0.4943992402930623, + "pass@32": 0.5639256336232448, + "pass@64": 0.6293550717317229, + "pass@128": 0.6901052631578948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.152578125, + "pass@2": 0.22779023129921225, + "pass@4": 0.3139792260029995, + "pass@8": 0.40436801298890296, + "pass@16": 0.49550086207735394, + "pass@32": 0.5839780263217058, + "pass@64": 0.6638323088261202, + "pass@128": 0.734 + }, + "2": { + "pass@1": 0.9592265625, + "pass@2": 0.9772586122047247, + "pass@4": 0.9870330117172853, + "pass@8": 0.990970424750734, + "pass@16": 0.9926473174923504, + "pass@32": 0.9934851717255498, + "pass@64": 0.9938770845368372, + "pass@128": 0.994 + }, + "3": { + "pass@1": 0.459953125, + "pass@2": 0.5616037155511809, + "pass@4": 0.6435666486689159, + "pass@8": 0.7044025930160407, + "pass@16": 0.7511445535571443, + "pass@32": 0.7887028787756671, + "pass@64": 0.8182136826642157, + "pass@128": 0.842 + }, + "4": { + "pass@1": 0.171296875, + "pass@2": 0.23645841535433068, + "pass@4": 0.3053068216160478, + "pass@8": 0.36905284743028616, + "pass@16": 0.42313792247658066, + "pass@32": 0.46804808322507935, + "pass@64": 0.5041870246526634, + "pass@128": 0.531 + }, + "5": { + "pass@1": 0.1858046875, + "pass@2": 0.24397477854330707, + "pass@4": 0.3084486854143231, + "pass@8": 0.3761193180463611, + "pass@16": 0.4424574609942841, + "pass@32": 0.5033182989441415, + "pass@64": 0.5566335335375966, + "pass@128": 0.601 + }, + "6": { + "pass@1": 0.160296875, + "pass@2": 0.23703494094488164, + "pass@4": 0.3317535394638169, + "pass@8": 0.4385031130403318, + "pass@16": 0.5461951024846229, + "pass@32": 0.6421663495162627, + "pass@64": 0.7170787291465208, + "pass@128": 0.77 + }, + "7": { + "pass@1": 0.1524609375, + "pass@2": 0.2226471456692912, + "pass@4": 0.30402510170603647, + "pass@8": 0.3938039793827515, + "pass@16": 0.4905381693736334, + "pass@32": 0.5854082930981304, + "pass@64": 0.6648636050543109, + "pass@128": 0.725 + }, + "8": { + "pass@1": 0.1659296875, + "pass@2": 0.2433890255905511, + "pass@4": 0.33236180530558707, + "pass@8": 0.42672638252712924, + "pass@16": 0.521472596002238, + "pass@32": 0.6115113622261756, + "pass@64": 0.691711565222001, + "pass@128": 0.761 + }, + "9": { + "pass@1": 0.1483203125, + "pass@2": 0.21989874507874002, + "pass@4": 0.3042555546494189, + "pass@8": 0.3972902774725913, + "pass@16": 0.49666254672837173, + "pass@32": 0.5953055185750495, + "pass@64": 0.6835078203370488, + "pass@128": 0.757 + }, + "11": { + "pass@1": 0.1420703125, + "pass@2": 0.21616498523622027, + "pass@4": 0.30243917032245937, + "pass@8": 0.3926092870352065, + "pass@16": 0.484249329100816, + "pass@32": 0.5735155601485259, + "pass@64": 0.6529675754419116, + "pass@128": 0.719 + }, + "12": { + "pass@1": 0.15053125, + "pass@2": 0.22905511811023616, + "pass@4": 0.31439215288713934, + "pass@8": 0.39659629492888226, + "pass@16": 0.47593892996953874, + "pass@32": 0.5569127029398856, + "pass@64": 0.6386010774459661, + "pass@128": 0.713 + }, + "13": { + "pass@1": 0.1205078125, + "pass@2": 0.18958919783464573, + "pass@4": 0.2699959077615294, + "pass@8": 0.3499377130267689, + "pass@16": 0.4247099285646272, + "pass@32": 0.4994672172014515, + "pass@64": 0.5773712084384967, + "pass@128": 0.652 + }, + "14": { + "pass@1": 0.126421875, + "pass@2": 0.2015922736220474, + "pass@4": 0.28945480708661403, + "pass@8": 0.37579113501502814, + "pass@16": 0.45640708831824817, + "pass@32": 0.5372244048600412, + "pass@64": 0.6196751926235065, + "pass@128": 0.699 + }, + "15": { + "pass@1": 0.1149375, + "pass@2": 0.18155684055118107, + "pass@4": 0.2597222063179599, + "pass@8": 0.3384902243470089, + "pass@16": 0.41237075595856404, + "pass@32": 0.48508771063964434, + "pass@64": 0.5601345517764789, + "pass@128": 0.636 + }, + "16": { + "pass@1": 0.1270703125, + "pass@2": 0.19944611220472444, + "pass@4": 0.28334039998125204, + "pass@8": 0.3658349458965953, + "pass@16": 0.44124756629096407, + "pass@32": 0.5112130336255106, + "pass@64": 0.57938709860515, + "pass@128": 0.651 + }, + "17": { + "pass@1": 0.11515625, + "pass@2": 0.18289714566929116, + "pass@4": 0.2620662944319459, + "pass@8": 0.33891554447726824, + "pass@16": 0.4063832820638924, + "pass@32": 0.4688954151261894, + "pass@64": 0.5355758065020465, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.10871875, + "pass@2": 0.17352829724409446, + "pass@4": 0.2508396094863139, + "pass@8": 0.3269651830284786, + "pass@16": 0.395449007785804, + "pass@32": 0.46187129159386986, + "pass@64": 0.531695477409159, + "pass@128": 0.605 + }, + "19": { + "pass@1": 0.1031953125, + "pass@2": 0.16674864665354308, + "pass@4": 0.24180379208848854, + "pass@8": 0.31416790461546495, + "pass@16": 0.37557485028095305, + "pass@32": 0.43097992188912404, + "pass@64": 0.49095567424554715, + "pass@128": 0.56 + }, + "20": { + "pass@1": 0.0985078125, + "pass@2": 0.15788693405511822, + "pass@4": 0.22876902090363677, + "pass@8": 0.29923469781414813, + "pass@16": 0.36149829604766187, + "pass@32": 0.41749579840940637, + "pass@64": 0.47747734643701806, + "pass@128": 0.55 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/checkpoint-18010_metrics.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/checkpoint-18010_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..30e16d1c7402fc0e3677441fa407806630d6156e --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/checkpoint-18010_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18010", + "id": { + "answer_accuracy": 0.2921935763888889, + "avg_response_len": 195.63549305555554, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2921935763888889, + "pass@2": 0.36882287565617583, + "pass@4": 0.4526185690017978, + "pass@8": 0.536893798424301, + "pass@16": 0.6185850036630156, + "pass@32": 0.6961531170835504, + "pass@64": 0.7660995847142775, + "pass@128": 0.825 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1622109375, + "pass@2": 0.24161417322834655, + "pass@4": 0.3307149086989128, + "pass@8": 0.4236988126526605, + "pass@16": 0.5207843498816409, + "pass@32": 0.6209528519354517, + "pass@64": 0.7174213762669805, + "pass@128": 0.8 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.37115625, + "pass@2": 0.4856058070866144, + "pass@4": 0.5916188328646416, + "pass@8": 0.6804346875022248, + "pass@16": 0.7496538496108398, + "pass@32": 0.8027586981636372, + "pass@64": 0.844341849254576, + "pass@128": 0.881 + }, + "4": { + "pass@1": 0.1880625, + "pass@2": 0.27276193405511834, + "pass@4": 0.3660262012560929, + "pass@8": 0.45623550422455117, + "pass@16": 0.5349092069078416, + "pass@32": 0.59883798686644, + "pass@64": 0.650161761781697, + "pass@128": 0.691 + }, + "5": { + "pass@1": 0.227375, + "pass@2": 0.3008003198818895, + "pass@4": 0.3795776673228347, + "pass@8": 0.4591488773493465, + "pass@16": 0.5361137406421513, + "pass@32": 0.6091381871643761, + "pass@64": 0.6740577725397885, + "pass@128": 0.727 + }, + "6": { + "pass@1": 0.1929296875, + "pass@2": 0.2899325787401573, + "pass@4": 0.3989696008623917, + "pass@8": 0.509233638224886, + "pass@16": 0.6144115416712757, + "pass@32": 0.7096447550064692, + "pass@64": 0.7888685721883177, + "pass@128": 0.853 + }, + "7": { + "pass@1": 0.169578125, + "pass@2": 0.2526342273622046, + "pass@4": 0.34885195500562394, + "pass@8": 0.4518169741025559, + "pass@16": 0.5571294807937225, + "pass@32": 0.6601633689120106, + "pass@64": 0.7537296424315669, + "pass@128": 0.831 + }, + "8": { + "pass@1": 0.1722421875, + "pass@2": 0.2528533464566928, + "pass@4": 0.34282652324709373, + "pass@8": 0.4365293284181813, + "pass@16": 0.5345421004294956, + "pass@32": 0.6363708779153936, + "pass@64": 0.7336411349123607, + "pass@128": 0.819 + }, + "9": { + "pass@1": 0.1461875, + "pass@2": 0.2232034940944881, + "pass@4": 0.3149814317585298, + "pass@8": 0.41494636334417995, + "pass@16": 0.5197207630300038, + "pass@32": 0.6275113277881846, + "pass@64": 0.7326741530531656, + "pass@128": 0.823 + } + }, + "per_op_accuracy": { + "10": 0.1622109375, + "2": 1.0, + "3": 0.37115625, + "4": 0.1880625, + "5": 0.227375, + "6": 0.1929296875, + "7": 0.169578125, + "8": 0.1722421875, + "9": 0.1461875 + }, + "per_op_avg_response_len": { + "10": 269.7712578125, + "2": 106.716015625, + "3": 153.8826875, + "4": 144.9838515625, + "5": 167.5860390625, + "6": 210.18378125, + "7": 210.9794609375, + "8": 236.959828125, + "9": 259.656515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 269.7712578125, + "2": 106.716015625, + "3": 153.8826875, + "4": 144.9838515625, + "5": 167.5860390625, + "6": 210.18378125, + "7": 210.9794609375, + "8": 236.959828125, + "9": 259.656515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 336607, + "answer_accuracy": 0.2921935763888889, + "avg_response_len": 195.63549305555554, + "resp_tokens_sum": 225372088, + "pass_at_k": { + "pass@1": 0.2921935763888889, + "pass@2": 0.36882287565617583, + "pass@4": 0.4526185690017978, + "pass@8": 0.536893798424301, + "pass@16": 0.6185850036630156, + "pass@32": 0.6961531170835504, + "pass@64": 0.7660995847142775, + "pass@128": 0.825 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1622109375, + "pass@2": 0.24161417322834655, + "pass@4": 0.3307149086989128, + "pass@8": 0.4236988126526605, + "pass@16": 0.5207843498816409, + "pass@32": 0.6209528519354517, + "pass@64": 0.7174213762669805, + "pass@128": 0.8 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.37115625, + "pass@2": 0.4856058070866144, + "pass@4": 0.5916188328646416, + "pass@8": 0.6804346875022248, + "pass@16": 0.7496538496108398, + "pass@32": 0.8027586981636372, + "pass@64": 0.844341849254576, + "pass@128": 0.881 + }, + "4": { + "pass@1": 0.1880625, + "pass@2": 0.27276193405511834, + "pass@4": 0.3660262012560929, + "pass@8": 0.45623550422455117, + "pass@16": 0.5349092069078416, + "pass@32": 0.59883798686644, + "pass@64": 0.650161761781697, + "pass@128": 0.691 + }, + "5": { + "pass@1": 0.227375, + "pass@2": 0.3008003198818895, + "pass@4": 0.3795776673228347, + "pass@8": 0.4591488773493465, + "pass@16": 0.5361137406421513, + "pass@32": 0.6091381871643761, + "pass@64": 0.6740577725397885, + "pass@128": 0.727 + }, + "6": { + "pass@1": 0.1929296875, + "pass@2": 0.2899325787401573, + "pass@4": 0.3989696008623917, + "pass@8": 0.509233638224886, + "pass@16": 0.6144115416712757, + "pass@32": 0.7096447550064692, + "pass@64": 0.7888685721883177, + "pass@128": 0.853 + }, + "7": { + "pass@1": 0.169578125, + "pass@2": 0.2526342273622046, + "pass@4": 0.34885195500562394, + "pass@8": 0.4518169741025559, + "pass@16": 0.5571294807937225, + "pass@32": 0.6601633689120106, + "pass@64": 0.7537296424315669, + "pass@128": 0.831 + }, + "8": { + "pass@1": 0.1722421875, + "pass@2": 0.2528533464566928, + "pass@4": 0.34282652324709373, + "pass@8": 0.4365293284181813, + "pass@16": 0.5345421004294956, + "pass@32": 0.6363708779153936, + "pass@64": 0.7336411349123607, + "pass@128": 0.819 + }, + "9": { + "pass@1": 0.1461875, + "pass@2": 0.2232034940944881, + "pass@4": 0.3149814317585298, + "pass@8": 0.41494636334417995, + "pass@16": 0.5197207630300038, + "pass@32": 0.6275113277881846, + "pass@64": 0.7326741530531656, + "pass@128": 0.823 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.12730234375, + "avg_response_len": 337.70623515625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.12730234375, + "pass@2": 0.1958882258858291, + "pass@4": 0.2734782902605979, + "pass@8": 0.3520871818086631, + "pass@16": 0.43198342659059147, + "pass@32": 0.5190308335042999, + "pass@64": 0.6153498764375804, + "pass@128": 0.7143 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.145515625, + "pass@2": 0.2225397391732285, + "pass@4": 0.30963146353580784, + "pass@8": 0.4004539560541219, + "pass@16": 0.496827531313926, + "pass@32": 0.6004886755011374, + "pass@64": 0.7041897723594484, + "pass@128": 0.795 + }, + "12": { + "pass@1": 0.16184375, + "pass@2": 0.24830142716535453, + "pass@4": 0.34490776284214447, + "pass@8": 0.439733104771824, + "pass@16": 0.5317119441161628, + "pass@32": 0.6263566180212361, + "pass@64": 0.721817743370228, + "pass@128": 0.807 + }, + "13": { + "pass@1": 0.137484375, + "pass@2": 0.20992384350393714, + "pass@4": 0.2901487774653163, + "pass@8": 0.37145564301962236, + "pass@16": 0.45743805575635343, + "pass@32": 0.5535878912154683, + "pass@64": 0.6564120511836622, + "pass@128": 0.755 + }, + "14": { + "pass@1": 0.1408671875, + "pass@2": 0.21532812500000004, + "pass@4": 0.2971830253093357, + "pass@8": 0.37665292392253863, + "pass@16": 0.45499388578911243, + "pass@32": 0.5399552650047299, + "pass@64": 0.633617410100413, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1237265625, + "pass@2": 0.18872797736220498, + "pass@4": 0.2621189280089984, + "pass@8": 0.337795945110374, + "pass@16": 0.41851257617682414, + "pass@32": 0.5107743418754581, + "pass@64": 0.614679699722663, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.127828125, + "pass@2": 0.19772268700787426, + "pass@4": 0.2779277995875511, + "pass@8": 0.35942340140632084, + "pass@16": 0.4407067475734005, + "pass@32": 0.5276160135040359, + "pass@64": 0.6251915684953607, + "pass@128": 0.73 + }, + "17": { + "pass@1": 0.12059375, + "pass@2": 0.18478432578740178, + "pass@4": 0.25755662776527904, + "pass@8": 0.33176436536861614, + "pass@16": 0.40637446113049275, + "pass@32": 0.4882567475586953, + "pass@64": 0.5826138214392281, + "pass@128": 0.684 + }, + "18": { + "pass@1": 0.1085078125, + "pass@2": 0.16963545767716542, + "pass@4": 0.24075687879639998, + "pass@8": 0.3144748140035013, + "pass@16": 0.3893986317330667, + "pass@32": 0.47082614055060545, + "pass@64": 0.5637040456169929, + "pass@128": 0.667 + }, + "19": { + "pass@1": 0.1057890625, + "pass@2": 0.16448658956692924, + "pass@4": 0.23051203233970705, + "pass@8": 0.29566219459132004, + "pass@16": 0.3603644254220861, + "pass@32": 0.43362825076212685, + "pass@64": 0.523283673915411, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.1008671875, + "pass@2": 0.15743208661417335, + "pass@4": 0.22403960695538017, + "pass@8": 0.29345546983824955, + "pass@16": 0.3635060068943852, + "pass@32": 0.43881839104954545, + "pass@64": 0.5279889781724307, + "pass@128": 0.628 + } + }, + "per_op_accuracy": { + "11": 0.145515625, + "12": 0.16184375, + "13": 0.137484375, + "14": 0.1408671875, + "15": 0.1237265625, + "16": 0.127828125, + "17": 0.12059375, + "18": 0.1085078125, + "19": 0.1057890625, + "20": 0.1008671875 + }, + "per_op_avg_response_len": { + "11": 287.6317890625, + "12": 288.762921875, + "13": 298.0527265625, + "14": 324.7144375, + "15": 340.4580546875, + "16": 358.46303125, + "17": 359.7728046875, + "18": 366.08284375, + "19": 370.4822265625, + "20": 382.641515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 287.6317890625, + "12": 288.762921875, + "13": 298.0527265625, + "14": 324.7144375, + "15": 340.4580546875, + "16": 358.46303125, + "17": 359.7728046875, + "18": 366.08284375, + "19": 370.4822265625, + "20": 382.641515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 162947, + "answer_accuracy": 0.12730234375, + "avg_response_len": 337.70623515625, + "resp_tokens_sum": 432263981, + "pass_at_k": { + "pass@1": 0.12730234375, + "pass@2": 0.1958882258858291, + "pass@4": 0.2734782902605979, + "pass@8": 0.3520871818086631, + "pass@16": 0.43198342659059147, + "pass@32": 0.5190308335042999, + "pass@64": 0.6153498764375804, + "pass@128": 0.7143 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.145515625, + "pass@2": 0.2225397391732285, + "pass@4": 0.30963146353580784, + "pass@8": 0.4004539560541219, + "pass@16": 0.496827531313926, + "pass@32": 0.6004886755011374, + "pass@64": 0.7041897723594484, + "pass@128": 0.795 + }, + "12": { + "pass@1": 0.16184375, + "pass@2": 0.24830142716535453, + "pass@4": 0.34490776284214447, + "pass@8": 0.439733104771824, + "pass@16": 0.5317119441161628, + "pass@32": 0.6263566180212361, + "pass@64": 0.721817743370228, + "pass@128": 0.807 + }, + "13": { + "pass@1": 0.137484375, + "pass@2": 0.20992384350393714, + "pass@4": 0.2901487774653163, + "pass@8": 0.37145564301962236, + "pass@16": 0.45743805575635343, + "pass@32": 0.5535878912154683, + "pass@64": 0.6564120511836622, + "pass@128": 0.755 + }, + "14": { + "pass@1": 0.1408671875, + "pass@2": 0.21532812500000004, + "pass@4": 0.2971830253093357, + "pass@8": 0.37665292392253863, + "pass@16": 0.45499388578911243, + "pass@32": 0.5399552650047299, + "pass@64": 0.633617410100413, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1237265625, + "pass@2": 0.18872797736220498, + "pass@4": 0.2621189280089984, + "pass@8": 0.337795945110374, + "pass@16": 0.41851257617682414, + "pass@32": 0.5107743418754581, + "pass@64": 0.614679699722663, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.127828125, + "pass@2": 0.19772268700787426, + "pass@4": 0.2779277995875511, + "pass@8": 0.35942340140632084, + "pass@16": 0.4407067475734005, + "pass@32": 0.5276160135040359, + "pass@64": 0.6251915684953607, + "pass@128": 0.73 + }, + "17": { + "pass@1": 0.12059375, + "pass@2": 0.18478432578740178, + "pass@4": 0.25755662776527904, + "pass@8": 0.33176436536861614, + "pass@16": 0.40637446113049275, + "pass@32": 0.4882567475586953, + "pass@64": 0.5826138214392281, + "pass@128": 0.684 + }, + "18": { + "pass@1": 0.1085078125, + "pass@2": 0.16963545767716542, + "pass@4": 0.24075687879639998, + "pass@8": 0.3144748140035013, + "pass@16": 0.3893986317330667, + "pass@32": 0.47082614055060545, + "pass@64": 0.5637040456169929, + "pass@128": 0.667 + }, + "19": { + "pass@1": 0.1057890625, + "pass@2": 0.16448658956692924, + "pass@4": 0.23051203233970705, + "pass@8": 0.29566219459132004, + "pass@16": 0.3603644254220861, + "pass@32": 0.43362825076212685, + "pass@64": 0.523283673915411, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.1008671875, + "pass@2": 0.15743208661417335, + "pass@4": 0.22403960695538017, + "pass@8": 0.29345546983824955, + "pass@16": 0.3635060068943852, + "pass@32": 0.43881839104954545, + "pass@64": 0.5279889781724307, + "pass@128": 0.628 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.20540871710526315, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.20540871710526315, + "pass@2": 0.27780463893492313, + "pass@4": 0.3583342117695657, + "pass@8": 0.43962715810027436, + "pass@16": 0.5203736473091285, + "pass@32": 0.602930862568176, + "pass@64": 0.6867576329897027, + "pass@128": 0.7667368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1622109375, + "pass@2": 0.24161417322834655, + "pass@4": 0.3307149086989128, + "pass@8": 0.4236988126526605, + "pass@16": 0.5207843498816409, + "pass@32": 0.6209528519354517, + "pass@64": 0.7174213762669805, + "pass@128": 0.8 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.37115625, + "pass@2": 0.4856058070866144, + "pass@4": 0.5916188328646416, + "pass@8": 0.6804346875022248, + "pass@16": 0.7496538496108398, + "pass@32": 0.8027586981636372, + "pass@64": 0.844341849254576, + "pass@128": 0.881 + }, + "4": { + "pass@1": 0.1880625, + "pass@2": 0.27276193405511834, + "pass@4": 0.3660262012560929, + "pass@8": 0.45623550422455117, + "pass@16": 0.5349092069078416, + "pass@32": 0.59883798686644, + "pass@64": 0.650161761781697, + "pass@128": 0.691 + }, + "5": { + "pass@1": 0.227375, + "pass@2": 0.3008003198818895, + "pass@4": 0.3795776673228347, + "pass@8": 0.4591488773493465, + "pass@16": 0.5361137406421513, + "pass@32": 0.6091381871643761, + "pass@64": 0.6740577725397885, + "pass@128": 0.727 + }, + "6": { + "pass@1": 0.1929296875, + "pass@2": 0.2899325787401573, + "pass@4": 0.3989696008623917, + "pass@8": 0.509233638224886, + "pass@16": 0.6144115416712757, + "pass@32": 0.7096447550064692, + "pass@64": 0.7888685721883177, + "pass@128": 0.853 + }, + "7": { + "pass@1": 0.169578125, + "pass@2": 0.2526342273622046, + "pass@4": 0.34885195500562394, + "pass@8": 0.4518169741025559, + "pass@16": 0.5571294807937225, + "pass@32": 0.6601633689120106, + "pass@64": 0.7537296424315669, + "pass@128": 0.831 + }, + "8": { + "pass@1": 0.1722421875, + "pass@2": 0.2528533464566928, + "pass@4": 0.34282652324709373, + "pass@8": 0.4365293284181813, + "pass@16": 0.5345421004294956, + "pass@32": 0.6363708779153936, + "pass@64": 0.7336411349123607, + "pass@128": 0.819 + }, + "9": { + "pass@1": 0.1461875, + "pass@2": 0.2232034940944881, + "pass@4": 0.3149814317585298, + "pass@8": 0.41494636334417995, + "pass@16": 0.5197207630300038, + "pass@32": 0.6275113277881846, + "pass@64": 0.7326741530531656, + "pass@128": 0.823 + }, + "11": { + "pass@1": 0.145515625, + "pass@2": 0.2225397391732285, + "pass@4": 0.30963146353580784, + "pass@8": 0.4004539560541219, + "pass@16": 0.496827531313926, + "pass@32": 0.6004886755011374, + "pass@64": 0.7041897723594484, + "pass@128": 0.795 + }, + "12": { + "pass@1": 0.16184375, + "pass@2": 0.24830142716535453, + "pass@4": 0.34490776284214447, + "pass@8": 0.439733104771824, + "pass@16": 0.5317119441161628, + "pass@32": 0.6263566180212361, + "pass@64": 0.721817743370228, + "pass@128": 0.807 + }, + "13": { + "pass@1": 0.137484375, + "pass@2": 0.20992384350393714, + "pass@4": 0.2901487774653163, + "pass@8": 0.37145564301962236, + "pass@16": 0.45743805575635343, + "pass@32": 0.5535878912154683, + "pass@64": 0.6564120511836622, + "pass@128": 0.755 + }, + "14": { + "pass@1": 0.1408671875, + "pass@2": 0.21532812500000004, + "pass@4": 0.2971830253093357, + "pass@8": 0.37665292392253863, + "pass@16": 0.45499388578911243, + "pass@32": 0.5399552650047299, + "pass@64": 0.633617410100413, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1237265625, + "pass@2": 0.18872797736220498, + "pass@4": 0.2621189280089984, + "pass@8": 0.337795945110374, + "pass@16": 0.41851257617682414, + "pass@32": 0.5107743418754581, + "pass@64": 0.614679699722663, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.127828125, + "pass@2": 0.19772268700787426, + "pass@4": 0.2779277995875511, + "pass@8": 0.35942340140632084, + "pass@16": 0.4407067475734005, + "pass@32": 0.5276160135040359, + "pass@64": 0.6251915684953607, + "pass@128": 0.73 + }, + "17": { + "pass@1": 0.12059375, + "pass@2": 0.18478432578740178, + "pass@4": 0.25755662776527904, + "pass@8": 0.33176436536861614, + "pass@16": 0.40637446113049275, + "pass@32": 0.4882567475586953, + "pass@64": 0.5826138214392281, + "pass@128": 0.684 + }, + "18": { + "pass@1": 0.1085078125, + "pass@2": 0.16963545767716542, + "pass@4": 0.24075687879639998, + "pass@8": 0.3144748140035013, + "pass@16": 0.3893986317330667, + "pass@32": 0.47082614055060545, + "pass@64": 0.5637040456169929, + "pass@128": 0.667 + }, + "19": { + "pass@1": 0.1057890625, + "pass@2": 0.16448658956692924, + "pass@4": 0.23051203233970705, + "pass@8": 0.29566219459132004, + "pass@16": 0.3603644254220861, + "pass@32": 0.43362825076212685, + "pass@64": 0.523283673915411, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.1008671875, + "pass@2": 0.15743208661417335, + "pass@4": 0.22403960695538017, + "pass@8": 0.29345546983824955, + "pass@16": 0.3635060068943852, + "pass@32": 0.43881839104954545, + "pass@64": 0.5279889781724307, + "pass@128": 0.628 + } + }, + "per_op_accuracy": { + "10": 0.1622109375, + "2": 1.0, + "3": 0.37115625, + "4": 0.1880625, + "5": 0.227375, + "6": 0.1929296875, + "7": 0.169578125, + "8": 0.1722421875, + "9": 0.1461875, + "11": 0.145515625, + "12": 0.16184375, + "13": 0.137484375, + "14": 0.1408671875, + "15": 0.1237265625, + "16": 0.127828125, + "17": 0.12059375, + "18": 0.1085078125, + "19": 0.1057890625, + "20": 0.1008671875 + }, + "per_op_avg_response_len": { + "10": 269.7712578125, + "2": 106.716015625, + "3": 153.8826875, + "4": 144.9838515625, + "5": 167.5860390625, + "6": 210.18378125, + "7": 210.9794609375, + "8": 236.959828125, + "9": 259.656515625, + "11": 287.6317890625, + "12": 288.762921875, + "13": 298.0527265625, + "14": 324.7144375, + "15": 340.4580546875, + "16": 358.46303125, + "17": 359.7728046875, + "18": 366.08284375, + "19": 370.4822265625, + "20": 382.641515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 269.7712578125, + "2": 106.716015625, + "3": 153.8826875, + "4": 144.9838515625, + "5": 167.5860390625, + "6": 210.18378125, + "7": 210.9794609375, + "8": 236.959828125, + "9": 259.656515625, + "11": 287.6317890625, + "12": 288.762921875, + "13": 298.0527265625, + "14": 324.7144375, + "15": 340.4580546875, + "16": 358.46303125, + "17": 359.7728046875, + "18": 366.08284375, + "19": 370.4822265625, + "20": 382.641515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 499554, + "answer_accuracy": 0.20540871710526315, + "avg_response_len": 270.40956784539475, + "resp_tokens_sum": 657636069, + "pass_at_k": { + "pass@1": 0.20540871710526315, + "pass@2": 0.27780463893492313, + "pass@4": 0.3583342117695657, + "pass@8": 0.43962715810027436, + "pass@16": 0.5203736473091285, + "pass@32": 0.602930862568176, + "pass@64": 0.6867576329897027, + "pass@128": 0.7667368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1622109375, + "pass@2": 0.24161417322834655, + "pass@4": 0.3307149086989128, + "pass@8": 0.4236988126526605, + "pass@16": 0.5207843498816409, + "pass@32": 0.6209528519354517, + "pass@64": 0.7174213762669805, + "pass@128": 0.8 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.37115625, + "pass@2": 0.4856058070866144, + "pass@4": 0.5916188328646416, + "pass@8": 0.6804346875022248, + "pass@16": 0.7496538496108398, + "pass@32": 0.8027586981636372, + "pass@64": 0.844341849254576, + "pass@128": 0.881 + }, + "4": { + "pass@1": 0.1880625, + "pass@2": 0.27276193405511834, + "pass@4": 0.3660262012560929, + "pass@8": 0.45623550422455117, + "pass@16": 0.5349092069078416, + "pass@32": 0.59883798686644, + "pass@64": 0.650161761781697, + "pass@128": 0.691 + }, + "5": { + "pass@1": 0.227375, + "pass@2": 0.3008003198818895, + "pass@4": 0.3795776673228347, + "pass@8": 0.4591488773493465, + "pass@16": 0.5361137406421513, + "pass@32": 0.6091381871643761, + "pass@64": 0.6740577725397885, + "pass@128": 0.727 + }, + "6": { + "pass@1": 0.1929296875, + "pass@2": 0.2899325787401573, + "pass@4": 0.3989696008623917, + "pass@8": 0.509233638224886, + "pass@16": 0.6144115416712757, + "pass@32": 0.7096447550064692, + "pass@64": 0.7888685721883177, + "pass@128": 0.853 + }, + "7": { + "pass@1": 0.169578125, + "pass@2": 0.2526342273622046, + "pass@4": 0.34885195500562394, + "pass@8": 0.4518169741025559, + "pass@16": 0.5571294807937225, + "pass@32": 0.6601633689120106, + "pass@64": 0.7537296424315669, + "pass@128": 0.831 + }, + "8": { + "pass@1": 0.1722421875, + "pass@2": 0.2528533464566928, + "pass@4": 0.34282652324709373, + "pass@8": 0.4365293284181813, + "pass@16": 0.5345421004294956, + "pass@32": 0.6363708779153936, + "pass@64": 0.7336411349123607, + "pass@128": 0.819 + }, + "9": { + "pass@1": 0.1461875, + "pass@2": 0.2232034940944881, + "pass@4": 0.3149814317585298, + "pass@8": 0.41494636334417995, + "pass@16": 0.5197207630300038, + "pass@32": 0.6275113277881846, + "pass@64": 0.7326741530531656, + "pass@128": 0.823 + }, + "11": { + "pass@1": 0.145515625, + "pass@2": 0.2225397391732285, + "pass@4": 0.30963146353580784, + "pass@8": 0.4004539560541219, + "pass@16": 0.496827531313926, + "pass@32": 0.6004886755011374, + "pass@64": 0.7041897723594484, + "pass@128": 0.795 + }, + "12": { + "pass@1": 0.16184375, + "pass@2": 0.24830142716535453, + "pass@4": 0.34490776284214447, + "pass@8": 0.439733104771824, + "pass@16": 0.5317119441161628, + "pass@32": 0.6263566180212361, + "pass@64": 0.721817743370228, + "pass@128": 0.807 + }, + "13": { + "pass@1": 0.137484375, + "pass@2": 0.20992384350393714, + "pass@4": 0.2901487774653163, + "pass@8": 0.37145564301962236, + "pass@16": 0.45743805575635343, + "pass@32": 0.5535878912154683, + "pass@64": 0.6564120511836622, + "pass@128": 0.755 + }, + "14": { + "pass@1": 0.1408671875, + "pass@2": 0.21532812500000004, + "pass@4": 0.2971830253093357, + "pass@8": 0.37665292392253863, + "pass@16": 0.45499388578911243, + "pass@32": 0.5399552650047299, + "pass@64": 0.633617410100413, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1237265625, + "pass@2": 0.18872797736220498, + "pass@4": 0.2621189280089984, + "pass@8": 0.337795945110374, + "pass@16": 0.41851257617682414, + "pass@32": 0.5107743418754581, + "pass@64": 0.614679699722663, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.127828125, + "pass@2": 0.19772268700787426, + "pass@4": 0.2779277995875511, + "pass@8": 0.35942340140632084, + "pass@16": 0.4407067475734005, + "pass@32": 0.5276160135040359, + "pass@64": 0.6251915684953607, + "pass@128": 0.73 + }, + "17": { + "pass@1": 0.12059375, + "pass@2": 0.18478432578740178, + "pass@4": 0.25755662776527904, + "pass@8": 0.33176436536861614, + "pass@16": 0.40637446113049275, + "pass@32": 0.4882567475586953, + "pass@64": 0.5826138214392281, + "pass@128": 0.684 + }, + "18": { + "pass@1": 0.1085078125, + "pass@2": 0.16963545767716542, + "pass@4": 0.24075687879639998, + "pass@8": 0.3144748140035013, + "pass@16": 0.3893986317330667, + "pass@32": 0.47082614055060545, + "pass@64": 0.5637040456169929, + "pass@128": 0.667 + }, + "19": { + "pass@1": 0.1057890625, + "pass@2": 0.16448658956692924, + "pass@4": 0.23051203233970705, + "pass@8": 0.29566219459132004, + "pass@16": 0.3603644254220861, + "pass@32": 0.43362825076212685, + "pass@64": 0.523283673915411, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.1008671875, + "pass@2": 0.15743208661417335, + "pass@4": 0.22403960695538017, + "pass@8": 0.29345546983824955, + "pass@16": 0.3635060068943852, + "pass@32": 0.43881839104954545, + "pass@64": 0.5279889781724307, + "pass@128": 0.628 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.csv b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..5f5f735a5ed06de936eef32ebd778d41d7e5ba96 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18010,0.2921935763888889,0.12730234375,0.20540871710526315,0.0,0.0,0.0,195.63549305555554,337.70623515625,1152000,1280000,2432000 diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..eca489455df1ea14dec1c3313b16ee8480267235 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18010", + "id_acc": 0.2921935763888889, + "ood_acc": 0.12730234375, + "total_acc": 0.20540871710526315, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 195.63549305555554, + "ood_avg_resp_len": 337.70623515625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 499554, + "answer_accuracy": 0.20540871710526315, + "avg_response_len": 270.40956784539475, + "resp_tokens_sum": 657636069, + "pass_at_k": { + "pass@1": 0.20540871710526315, + "pass@2": 0.27780463893492313, + "pass@4": 0.3583342117695657, + "pass@8": 0.43962715810027436, + "pass@16": 0.5203736473091285, + "pass@32": 0.602930862568176, + "pass@64": 0.6867576329897027, + "pass@128": 0.7667368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1622109375, + "pass@2": 0.24161417322834655, + "pass@4": 0.3307149086989128, + "pass@8": 0.4236988126526605, + "pass@16": 0.5207843498816409, + "pass@32": 0.6209528519354517, + "pass@64": 0.7174213762669805, + "pass@128": 0.8 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.37115625, + "pass@2": 0.4856058070866144, + "pass@4": 0.5916188328646416, + "pass@8": 0.6804346875022248, + "pass@16": 0.7496538496108398, + "pass@32": 0.8027586981636372, + "pass@64": 0.844341849254576, + "pass@128": 0.881 + }, + "4": { + "pass@1": 0.1880625, + "pass@2": 0.27276193405511834, + "pass@4": 0.3660262012560929, + "pass@8": 0.45623550422455117, + "pass@16": 0.5349092069078416, + "pass@32": 0.59883798686644, + "pass@64": 0.650161761781697, + "pass@128": 0.691 + }, + "5": { + "pass@1": 0.227375, + "pass@2": 0.3008003198818895, + "pass@4": 0.3795776673228347, + "pass@8": 0.4591488773493465, + "pass@16": 0.5361137406421513, + "pass@32": 0.6091381871643761, + "pass@64": 0.6740577725397885, + "pass@128": 0.727 + }, + "6": { + "pass@1": 0.1929296875, + "pass@2": 0.2899325787401573, + "pass@4": 0.3989696008623917, + "pass@8": 0.509233638224886, + "pass@16": 0.6144115416712757, + "pass@32": 0.7096447550064692, + "pass@64": 0.7888685721883177, + "pass@128": 0.853 + }, + "7": { + "pass@1": 0.169578125, + "pass@2": 0.2526342273622046, + "pass@4": 0.34885195500562394, + "pass@8": 0.4518169741025559, + "pass@16": 0.5571294807937225, + "pass@32": 0.6601633689120106, + "pass@64": 0.7537296424315669, + "pass@128": 0.831 + }, + "8": { + "pass@1": 0.1722421875, + "pass@2": 0.2528533464566928, + "pass@4": 0.34282652324709373, + "pass@8": 0.4365293284181813, + "pass@16": 0.5345421004294956, + "pass@32": 0.6363708779153936, + "pass@64": 0.7336411349123607, + "pass@128": 0.819 + }, + "9": { + "pass@1": 0.1461875, + "pass@2": 0.2232034940944881, + "pass@4": 0.3149814317585298, + "pass@8": 0.41494636334417995, + "pass@16": 0.5197207630300038, + "pass@32": 0.6275113277881846, + "pass@64": 0.7326741530531656, + "pass@128": 0.823 + }, + "11": { + "pass@1": 0.145515625, + "pass@2": 0.2225397391732285, + "pass@4": 0.30963146353580784, + "pass@8": 0.4004539560541219, + "pass@16": 0.496827531313926, + "pass@32": 0.6004886755011374, + "pass@64": 0.7041897723594484, + "pass@128": 0.795 + }, + "12": { + "pass@1": 0.16184375, + "pass@2": 0.24830142716535453, + "pass@4": 0.34490776284214447, + "pass@8": 0.439733104771824, + "pass@16": 0.5317119441161628, + "pass@32": 0.6263566180212361, + "pass@64": 0.721817743370228, + "pass@128": 0.807 + }, + "13": { + "pass@1": 0.137484375, + "pass@2": 0.20992384350393714, + "pass@4": 0.2901487774653163, + "pass@8": 0.37145564301962236, + "pass@16": 0.45743805575635343, + "pass@32": 0.5535878912154683, + "pass@64": 0.6564120511836622, + "pass@128": 0.755 + }, + "14": { + "pass@1": 0.1408671875, + "pass@2": 0.21532812500000004, + "pass@4": 0.2971830253093357, + "pass@8": 0.37665292392253863, + "pass@16": 0.45499388578911243, + "pass@32": 0.5399552650047299, + "pass@64": 0.633617410100413, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1237265625, + "pass@2": 0.18872797736220498, + "pass@4": 0.2621189280089984, + "pass@8": 0.337795945110374, + "pass@16": 0.41851257617682414, + "pass@32": 0.5107743418754581, + "pass@64": 0.614679699722663, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.127828125, + "pass@2": 0.19772268700787426, + "pass@4": 0.2779277995875511, + "pass@8": 0.35942340140632084, + "pass@16": 0.4407067475734005, + "pass@32": 0.5276160135040359, + "pass@64": 0.6251915684953607, + "pass@128": 0.73 + }, + "17": { + "pass@1": 0.12059375, + "pass@2": 0.18478432578740178, + "pass@4": 0.25755662776527904, + "pass@8": 0.33176436536861614, + "pass@16": 0.40637446113049275, + "pass@32": 0.4882567475586953, + "pass@64": 0.5826138214392281, + "pass@128": 0.684 + }, + "18": { + "pass@1": 0.1085078125, + "pass@2": 0.16963545767716542, + "pass@4": 0.24075687879639998, + "pass@8": 0.3144748140035013, + "pass@16": 0.3893986317330667, + "pass@32": 0.47082614055060545, + "pass@64": 0.5637040456169929, + "pass@128": 0.667 + }, + "19": { + "pass@1": 0.1057890625, + "pass@2": 0.16448658956692924, + "pass@4": 0.23051203233970705, + "pass@8": 0.29566219459132004, + "pass@16": 0.3603644254220861, + "pass@32": 0.43362825076212685, + "pass@64": 0.523283673915411, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.1008671875, + "pass@2": 0.15743208661417335, + "pass@4": 0.22403960695538017, + "pass@8": 0.29345546983824955, + "pass@16": 0.3635060068943852, + "pass@32": 0.43881839104954545, + "pass@64": 0.5279889781724307, + "pass@128": 0.628 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/checkpoint-18010_metrics.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/checkpoint-18010_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8dd763e1b4dcc597eeed7457f20e67d4ad0ba19e --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/checkpoint-18010_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18010", + "id": { + "answer_accuracy": 0.27102777777777776, + "avg_response_len": 173.61795225694445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.27102777777777776, + "pass@2": 0.33154647856518565, + "pass@4": 0.39621675500979503, + "pass@8": 0.46067744401801003, + "pass@16": 0.5225295895250077, + "pass@32": 0.5796312539473139, + "pass@64": 0.6304413211431196, + "pass@128": 0.6752222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.137609375, + "pass@2": 0.2044445127952755, + "pass@4": 0.2807508605174354, + "pass@8": 0.3593730437979568, + "pass@16": 0.43512775897758893, + "pass@32": 0.5042921162048943, + "pass@64": 0.5662955106188434, + "pass@128": 0.623 + }, + "2": { + "pass@1": 0.9689609375, + "pass@2": 0.9825099655511812, + "pass@4": 0.9885345863329588, + "pass@8": 0.9905474245953042, + "pass@16": 0.9913789876014476, + "pass@32": 0.9918568967077263, + "pass@64": 0.991996236389654, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.47971875, + "pass@2": 0.5780030757874023, + "pass@4": 0.6552130453693292, + "pass@8": 0.7088155395420423, + "pass@16": 0.7457569569827588, + "pass@32": 0.7748404435830744, + "pass@64": 0.7998995443105852, + "pass@128": 0.822 + }, + "4": { + "pass@1": 0.1451875, + "pass@2": 0.1972643946850394, + "pass@4": 0.25449041366704117, + "pass@8": 0.3098883354801711, + "pass@16": 0.35893551789667805, + "pass@32": 0.40036083151097207, + "pass@64": 0.4348911819551309, + "pass@128": 0.464 + }, + "5": { + "pass@1": 0.1498359375, + "pass@2": 0.1992218257874015, + "pass@4": 0.25530562307836485, + "pass@8": 0.31287900960176424, + "pass@16": 0.3692893207379819, + "pass@32": 0.4243511443374379, + "pass@64": 0.47744351587524025, + "pass@128": 0.525 + }, + "6": { + "pass@1": 0.1414921875, + "pass@2": 0.20634559547244066, + "pass@4": 0.2855413216160479, + "pass@8": 0.3759909813183259, + "pass@16": 0.47039010749595905, + "pass@32": 0.5583390689319193, + "pass@64": 0.6317356404851369, + "pass@128": 0.69 + }, + "7": { + "pass@1": 0.136390625, + "pass@2": 0.19957381889763773, + "pass@4": 0.27261641619797516, + "pass@8": 0.3518023343170404, + "pass@16": 0.4352175688724406, + "pass@32": 0.5175287112069517, + "pass@64": 0.5913243659581479, + "pass@128": 0.654 + }, + "8": { + "pass@1": 0.1460703125, + "pass@2": 0.2179701033464565, + "pass@4": 0.2997344593175849, + "pass@8": 0.3829834352426461, + "pass@16": 0.46184138214291204, + "pass@32": 0.5327699584678148, + "pass@64": 0.5953491007884835, + "pass@128": 0.651 + }, + "9": { + "pass@1": 0.133984375, + "pass@2": 0.19858501476377932, + "pass@4": 0.27376406899137595, + "pass@8": 0.353816892266726, + "pass@16": 0.4348287050171671, + "pass@32": 0.5123421145750472, + "pass@64": 0.5850367939068287, + "pass@128": 0.656 + } + }, + "per_op_accuracy": { + "10": 0.137609375, + "2": 0.9689609375, + "3": 0.47971875, + "4": 0.1451875, + "5": 0.1498359375, + "6": 0.1414921875, + "7": 0.136390625, + "8": 0.1460703125, + "9": 0.133984375 + }, + "per_op_avg_response_len": { + "10": 227.62675, + "2": 106.5008359375, + "3": 134.2596015625, + "4": 135.365046875, + "5": 156.17853125, + "6": 185.57253125, + "7": 187.2631796875, + "8": 207.2234453125, + "9": 222.5716484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 227.62675, + "2": 106.5008359375, + "3": 134.2596015625, + "4": 135.365046875, + "5": 156.17853125, + "6": 185.57253125, + "7": 187.2631796875, + "8": 207.2234453125, + "9": 222.5716484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 312224, + "answer_accuracy": 0.27102777777777776, + "avg_response_len": 173.61795225694445, + "resp_tokens_sum": 200007881, + "pass_at_k": { + "pass@1": 0.27102777777777776, + "pass@2": 0.33154647856518565, + "pass@4": 0.39621675500979503, + "pass@8": 0.46067744401801003, + "pass@16": 0.5225295895250077, + "pass@32": 0.5796312539473139, + "pass@64": 0.6304413211431196, + "pass@128": 0.6752222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.137609375, + "pass@2": 0.2044445127952755, + "pass@4": 0.2807508605174354, + "pass@8": 0.3593730437979568, + "pass@16": 0.43512775897758893, + "pass@32": 0.5042921162048943, + "pass@64": 0.5662955106188434, + "pass@128": 0.623 + }, + "2": { + "pass@1": 0.9689609375, + "pass@2": 0.9825099655511812, + "pass@4": 0.9885345863329588, + "pass@8": 0.9905474245953042, + "pass@16": 0.9913789876014476, + "pass@32": 0.9918568967077263, + "pass@64": 0.991996236389654, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.47971875, + "pass@2": 0.5780030757874023, + "pass@4": 0.6552130453693292, + "pass@8": 0.7088155395420423, + "pass@16": 0.7457569569827588, + "pass@32": 0.7748404435830744, + "pass@64": 0.7998995443105852, + "pass@128": 0.822 + }, + "4": { + "pass@1": 0.1451875, + "pass@2": 0.1972643946850394, + "pass@4": 0.25449041366704117, + "pass@8": 0.3098883354801711, + "pass@16": 0.35893551789667805, + "pass@32": 0.40036083151097207, + "pass@64": 0.4348911819551309, + "pass@128": 0.464 + }, + "5": { + "pass@1": 0.1498359375, + "pass@2": 0.1992218257874015, + "pass@4": 0.25530562307836485, + "pass@8": 0.31287900960176424, + "pass@16": 0.3692893207379819, + "pass@32": 0.4243511443374379, + "pass@64": 0.47744351587524025, + "pass@128": 0.525 + }, + "6": { + "pass@1": 0.1414921875, + "pass@2": 0.20634559547244066, + "pass@4": 0.2855413216160479, + "pass@8": 0.3759909813183259, + "pass@16": 0.47039010749595905, + "pass@32": 0.5583390689319193, + "pass@64": 0.6317356404851369, + "pass@128": 0.69 + }, + "7": { + "pass@1": 0.136390625, + "pass@2": 0.19957381889763773, + "pass@4": 0.27261641619797516, + "pass@8": 0.3518023343170404, + "pass@16": 0.4352175688724406, + "pass@32": 0.5175287112069517, + "pass@64": 0.5913243659581479, + "pass@128": 0.654 + }, + "8": { + "pass@1": 0.1460703125, + "pass@2": 0.2179701033464565, + "pass@4": 0.2997344593175849, + "pass@8": 0.3829834352426461, + "pass@16": 0.46184138214291204, + "pass@32": 0.5327699584678148, + "pass@64": 0.5953491007884835, + "pass@128": 0.651 + }, + "9": { + "pass@1": 0.133984375, + "pass@2": 0.19858501476377932, + "pass@4": 0.27376406899137595, + "pass@8": 0.353816892266726, + "pass@16": 0.4348287050171671, + "pass@32": 0.5123421145750472, + "pass@64": 0.5850367939068287, + "pass@128": 0.656 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.110734375, + "avg_response_len": 288.2031421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.110734375, + "pass@2": 0.17368340305118157, + "pass@4": 0.2467192759655073, + "pass@8": 0.31843268903885696, + "pass@16": 0.38080544587829585, + "pass@32": 0.4326664944122843, + "pass@64": 0.47895467061802144, + "pass@128": 0.5256 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.131359375, + "pass@2": 0.1979074803149605, + "pass@4": 0.2734353920134983, + "pass@8": 0.34991794185649616, + "pass@16": 0.4222118709001832, + "pass@32": 0.4868841744504942, + "pass@64": 0.5431513744419101, + "pass@128": 0.595 + }, + "12": { + "pass@1": 0.143828125, + "pass@2": 0.21519525098425205, + "pass@4": 0.2899552368766403, + "pass@8": 0.3594490123434772, + "pass@16": 0.4224746832075962, + "pass@32": 0.4809637642820015, + "pass@64": 0.5381737990170851, + "pass@128": 0.595 + }, + "13": { + "pass@1": 0.111484375, + "pass@2": 0.1739815452755905, + "pass@4": 0.24539893100862373, + "pass@8": 0.3153656153080575, + "pass@16": 0.3778591615177856, + "pass@32": 0.43372927559272895, + "pass@64": 0.4890995661569375, + "pass@128": 0.548 + }, + "14": { + "pass@1": 0.1206875, + "pass@2": 0.18854195374015764, + "pass@4": 0.26664505614923073, + "pass@8": 0.3413639269498822, + "pass@16": 0.4046785769193911, + "pass@32": 0.45851520728087153, + "pass@64": 0.509789165398226, + "pass@128": 0.56 + }, + "15": { + "pass@1": 0.1028203125, + "pass@2": 0.16334805610236225, + "pass@4": 0.23506728496437912, + "pass@8": 0.3068765586059638, + "pass@16": 0.37027187425270813, + "pass@32": 0.42434378259224537, + "pass@64": 0.4738853868332054, + "pass@128": 0.523 + }, + "16": { + "pass@1": 0.1196953125, + "pass@2": 0.1894175688976377, + "pass@4": 0.2691667745594299, + "pass@8": 0.3457092688172161, + "pass@16": 0.4096187459888607, + "pass@32": 0.4592911285191424, + "pass@64": 0.501203414513232, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.0991015625, + "pass@2": 0.16048720472440944, + "pass@4": 0.23582270894263194, + "pass@8": 0.31153919565936955, + "pass@16": 0.3757639244403764, + "pass@32": 0.4258215453216609, + "pass@64": 0.4667552886812957, + "pass@128": 0.507 + }, + "18": { + "pass@1": 0.09759375, + "pass@2": 0.15683747539370085, + "pass@4": 0.2280157783089612, + "pass@8": 0.2992512552701541, + "pass@16": 0.3611276837432124, + "pass@32": 0.41048525384187906, + "pass@64": 0.45148370151351497, + "pass@128": 0.495 + }, + "19": { + "pass@1": 0.09328125, + "pass@2": 0.1501445620078741, + "pass@4": 0.21755319610048723, + "pass@8": 0.2828352344004617, + "pass@16": 0.33638304684234605, + "pass@32": 0.37763774237042336, + "pass@64": 0.4131950503444665, + "pass@128": 0.45 + }, + "20": { + "pass@1": 0.0874921875, + "pass@2": 0.14097293307086609, + "pass@4": 0.20613240073115838, + "pass@8": 0.272018881177445, + "pass@16": 0.3276648909704376, + "pass@32": 0.3689930698714124, + "pass@64": 0.40280995928038527, + "pass@128": 0.441 + } + }, + "per_op_accuracy": { + "11": 0.131359375, + "12": 0.143828125, + "13": 0.111484375, + "14": 0.1206875, + "15": 0.1028203125, + "16": 0.1196953125, + "17": 0.0991015625, + "18": 0.09759375, + "19": 0.09328125, + "20": 0.0874921875 + }, + "per_op_avg_response_len": { + "11": 243.4620859375, + "12": 252.997296875, + "13": 263.408578125, + "14": 280.3458984375, + "15": 284.440109375, + "16": 303.5060703125, + "17": 308.352578125, + "18": 310.8587421875, + "19": 311.022609375, + "20": 323.637453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 243.4620859375, + "12": 252.997296875, + "13": 263.408578125, + "14": 280.3458984375, + "15": 284.440109375, + "16": 303.5060703125, + "17": 308.352578125, + "18": 310.8587421875, + "19": 311.022609375, + "20": 323.637453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 141740, + "answer_accuracy": 0.110734375, + "avg_response_len": 288.2031421875, + "resp_tokens_sum": 368900022, + "pass_at_k": { + "pass@1": 0.110734375, + "pass@2": 0.17368340305118157, + "pass@4": 0.2467192759655073, + "pass@8": 0.31843268903885696, + "pass@16": 0.38080544587829585, + "pass@32": 0.4326664944122843, + "pass@64": 0.47895467061802144, + "pass@128": 0.5256 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.131359375, + "pass@2": 0.1979074803149605, + "pass@4": 0.2734353920134983, + "pass@8": 0.34991794185649616, + "pass@16": 0.4222118709001832, + "pass@32": 0.4868841744504942, + "pass@64": 0.5431513744419101, + "pass@128": 0.595 + }, + "12": { + "pass@1": 0.143828125, + "pass@2": 0.21519525098425205, + "pass@4": 0.2899552368766403, + "pass@8": 0.3594490123434772, + "pass@16": 0.4224746832075962, + "pass@32": 0.4809637642820015, + "pass@64": 0.5381737990170851, + "pass@128": 0.595 + }, + "13": { + "pass@1": 0.111484375, + "pass@2": 0.1739815452755905, + "pass@4": 0.24539893100862373, + "pass@8": 0.3153656153080575, + "pass@16": 0.3778591615177856, + "pass@32": 0.43372927559272895, + "pass@64": 0.4890995661569375, + "pass@128": 0.548 + }, + "14": { + "pass@1": 0.1206875, + "pass@2": 0.18854195374015764, + "pass@4": 0.26664505614923073, + "pass@8": 0.3413639269498822, + "pass@16": 0.4046785769193911, + "pass@32": 0.45851520728087153, + "pass@64": 0.509789165398226, + "pass@128": 0.56 + }, + "15": { + "pass@1": 0.1028203125, + "pass@2": 0.16334805610236225, + "pass@4": 0.23506728496437912, + "pass@8": 0.3068765586059638, + "pass@16": 0.37027187425270813, + "pass@32": 0.42434378259224537, + "pass@64": 0.4738853868332054, + "pass@128": 0.523 + }, + "16": { + "pass@1": 0.1196953125, + "pass@2": 0.1894175688976377, + "pass@4": 0.2691667745594299, + "pass@8": 0.3457092688172161, + "pass@16": 0.4096187459888607, + "pass@32": 0.4592911285191424, + "pass@64": 0.501203414513232, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.0991015625, + "pass@2": 0.16048720472440944, + "pass@4": 0.23582270894263194, + "pass@8": 0.31153919565936955, + "pass@16": 0.3757639244403764, + "pass@32": 0.4258215453216609, + "pass@64": 0.4667552886812957, + "pass@128": 0.507 + }, + "18": { + "pass@1": 0.09759375, + "pass@2": 0.15683747539370085, + "pass@4": 0.2280157783089612, + "pass@8": 0.2992512552701541, + "pass@16": 0.3611276837432124, + "pass@32": 0.41048525384187906, + "pass@64": 0.45148370151351497, + "pass@128": 0.495 + }, + "19": { + "pass@1": 0.09328125, + "pass@2": 0.1501445620078741, + "pass@4": 0.21755319610048723, + "pass@8": 0.2828352344004617, + "pass@16": 0.33638304684234605, + "pass@32": 0.37763774237042336, + "pass@64": 0.4131950503444665, + "pass@128": 0.45 + }, + "20": { + "pass@1": 0.0874921875, + "pass@2": 0.14097293307086609, + "pass@4": 0.20613240073115838, + "pass@8": 0.272018881177445, + "pass@16": 0.3276648909704376, + "pass@32": 0.3689930698714124, + "pass@64": 0.40280995928038527, + "pass@128": 0.441 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.1866628289473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.1866628289473684, + "pass@2": 0.2484606493472894, + "pass@4": 0.31753387130226435, + "pass@8": 0.3858117835026649, + "pass@16": 0.44793793497411705, + "pass@32": 0.5022813805078344, + "pass@64": 0.5507115050772899, + "pass@128": 0.5964736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.137609375, + "pass@2": 0.2044445127952755, + "pass@4": 0.2807508605174354, + "pass@8": 0.3593730437979568, + "pass@16": 0.43512775897758893, + "pass@32": 0.5042921162048943, + "pass@64": 0.5662955106188434, + "pass@128": 0.623 + }, + "2": { + "pass@1": 0.9689609375, + "pass@2": 0.9825099655511812, + "pass@4": 0.9885345863329588, + "pass@8": 0.9905474245953042, + "pass@16": 0.9913789876014476, + "pass@32": 0.9918568967077263, + "pass@64": 0.991996236389654, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.47971875, + "pass@2": 0.5780030757874023, + "pass@4": 0.6552130453693292, + "pass@8": 0.7088155395420423, + "pass@16": 0.7457569569827588, + "pass@32": 0.7748404435830744, + "pass@64": 0.7998995443105852, + "pass@128": 0.822 + }, + "4": { + "pass@1": 0.1451875, + "pass@2": 0.1972643946850394, + "pass@4": 0.25449041366704117, + "pass@8": 0.3098883354801711, + "pass@16": 0.35893551789667805, + "pass@32": 0.40036083151097207, + "pass@64": 0.4348911819551309, + "pass@128": 0.464 + }, + "5": { + "pass@1": 0.1498359375, + "pass@2": 0.1992218257874015, + "pass@4": 0.25530562307836485, + "pass@8": 0.31287900960176424, + "pass@16": 0.3692893207379819, + "pass@32": 0.4243511443374379, + "pass@64": 0.47744351587524025, + "pass@128": 0.525 + }, + "6": { + "pass@1": 0.1414921875, + "pass@2": 0.20634559547244066, + "pass@4": 0.2855413216160479, + "pass@8": 0.3759909813183259, + "pass@16": 0.47039010749595905, + "pass@32": 0.5583390689319193, + "pass@64": 0.6317356404851369, + "pass@128": 0.69 + }, + "7": { + "pass@1": 0.136390625, + "pass@2": 0.19957381889763773, + "pass@4": 0.27261641619797516, + "pass@8": 0.3518023343170404, + "pass@16": 0.4352175688724406, + "pass@32": 0.5175287112069517, + "pass@64": 0.5913243659581479, + "pass@128": 0.654 + }, + "8": { + "pass@1": 0.1460703125, + "pass@2": 0.2179701033464565, + "pass@4": 0.2997344593175849, + "pass@8": 0.3829834352426461, + "pass@16": 0.46184138214291204, + "pass@32": 0.5327699584678148, + "pass@64": 0.5953491007884835, + "pass@128": 0.651 + }, + "9": { + "pass@1": 0.133984375, + "pass@2": 0.19858501476377932, + "pass@4": 0.27376406899137595, + "pass@8": 0.353816892266726, + "pass@16": 0.4348287050171671, + "pass@32": 0.5123421145750472, + "pass@64": 0.5850367939068287, + "pass@128": 0.656 + }, + "11": { + "pass@1": 0.131359375, + "pass@2": 0.1979074803149605, + "pass@4": 0.2734353920134983, + "pass@8": 0.34991794185649616, + "pass@16": 0.4222118709001832, + "pass@32": 0.4868841744504942, + "pass@64": 0.5431513744419101, + "pass@128": 0.595 + }, + "12": { + "pass@1": 0.143828125, + "pass@2": 0.21519525098425205, + "pass@4": 0.2899552368766403, + "pass@8": 0.3594490123434772, + "pass@16": 0.4224746832075962, + "pass@32": 0.4809637642820015, + "pass@64": 0.5381737990170851, + "pass@128": 0.595 + }, + "13": { + "pass@1": 0.111484375, + "pass@2": 0.1739815452755905, + "pass@4": 0.24539893100862373, + "pass@8": 0.3153656153080575, + "pass@16": 0.3778591615177856, + "pass@32": 0.43372927559272895, + "pass@64": 0.4890995661569375, + "pass@128": 0.548 + }, + "14": { + "pass@1": 0.1206875, + "pass@2": 0.18854195374015764, + "pass@4": 0.26664505614923073, + "pass@8": 0.3413639269498822, + "pass@16": 0.4046785769193911, + "pass@32": 0.45851520728087153, + "pass@64": 0.509789165398226, + "pass@128": 0.56 + }, + "15": { + "pass@1": 0.1028203125, + "pass@2": 0.16334805610236225, + "pass@4": 0.23506728496437912, + "pass@8": 0.3068765586059638, + "pass@16": 0.37027187425270813, + "pass@32": 0.42434378259224537, + "pass@64": 0.4738853868332054, + "pass@128": 0.523 + }, + "16": { + "pass@1": 0.1196953125, + "pass@2": 0.1894175688976377, + "pass@4": 0.2691667745594299, + "pass@8": 0.3457092688172161, + "pass@16": 0.4096187459888607, + "pass@32": 0.4592911285191424, + "pass@64": 0.501203414513232, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.0991015625, + "pass@2": 0.16048720472440944, + "pass@4": 0.23582270894263194, + "pass@8": 0.31153919565936955, + "pass@16": 0.3757639244403764, + "pass@32": 0.4258215453216609, + "pass@64": 0.4667552886812957, + "pass@128": 0.507 + }, + "18": { + "pass@1": 0.09759375, + "pass@2": 0.15683747539370085, + "pass@4": 0.2280157783089612, + "pass@8": 0.2992512552701541, + "pass@16": 0.3611276837432124, + "pass@32": 0.41048525384187906, + "pass@64": 0.45148370151351497, + "pass@128": 0.495 + }, + "19": { + "pass@1": 0.09328125, + "pass@2": 0.1501445620078741, + "pass@4": 0.21755319610048723, + "pass@8": 0.2828352344004617, + "pass@16": 0.33638304684234605, + "pass@32": 0.37763774237042336, + "pass@64": 0.4131950503444665, + "pass@128": 0.45 + }, + "20": { + "pass@1": 0.0874921875, + "pass@2": 0.14097293307086609, + "pass@4": 0.20613240073115838, + "pass@8": 0.272018881177445, + "pass@16": 0.3276648909704376, + "pass@32": 0.3689930698714124, + "pass@64": 0.40280995928038527, + "pass@128": 0.441 + } + }, + "per_op_accuracy": { + "10": 0.137609375, + "2": 0.9689609375, + "3": 0.47971875, + "4": 0.1451875, + "5": 0.1498359375, + "6": 0.1414921875, + "7": 0.136390625, + "8": 0.1460703125, + "9": 0.133984375, + "11": 0.131359375, + "12": 0.143828125, + "13": 0.111484375, + "14": 0.1206875, + "15": 0.1028203125, + "16": 0.1196953125, + "17": 0.0991015625, + "18": 0.09759375, + "19": 0.09328125, + "20": 0.0874921875 + }, + "per_op_avg_response_len": { + "10": 227.62675, + "2": 106.5008359375, + "3": 134.2596015625, + "4": 135.365046875, + "5": 156.17853125, + "6": 185.57253125, + "7": 187.2631796875, + "8": 207.2234453125, + "9": 222.5716484375, + "11": 243.4620859375, + "12": 252.997296875, + "13": 263.408578125, + "14": 280.3458984375, + "15": 284.440109375, + "16": 303.5060703125, + "17": 308.352578125, + "18": 310.8587421875, + "19": 311.022609375, + "20": 323.637453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 227.62675, + "2": 106.5008359375, + "3": 134.2596015625, + "4": 135.365046875, + "5": 156.17853125, + "6": 185.57253125, + "7": 187.2631796875, + "8": 207.2234453125, + "9": 222.5716484375, + "11": 243.4620859375, + "12": 252.997296875, + "13": 263.408578125, + "14": 280.3458984375, + "15": 284.440109375, + "16": 303.5060703125, + "17": 308.352578125, + "18": 310.8587421875, + "19": 311.022609375, + "20": 323.637453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 453964, + "answer_accuracy": 0.1866628289473684, + "avg_response_len": 233.92594695723685, + "resp_tokens_sum": 568907903, + "pass_at_k": { + "pass@1": 0.1866628289473684, + "pass@2": 0.2484606493472894, + "pass@4": 0.31753387130226435, + "pass@8": 0.3858117835026649, + "pass@16": 0.44793793497411705, + "pass@32": 0.5022813805078344, + "pass@64": 0.5507115050772899, + "pass@128": 0.5964736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.137609375, + "pass@2": 0.2044445127952755, + "pass@4": 0.2807508605174354, + "pass@8": 0.3593730437979568, + "pass@16": 0.43512775897758893, + "pass@32": 0.5042921162048943, + "pass@64": 0.5662955106188434, + "pass@128": 0.623 + }, + "2": { + "pass@1": 0.9689609375, + "pass@2": 0.9825099655511812, + "pass@4": 0.9885345863329588, + "pass@8": 0.9905474245953042, + "pass@16": 0.9913789876014476, + "pass@32": 0.9918568967077263, + "pass@64": 0.991996236389654, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.47971875, + "pass@2": 0.5780030757874023, + "pass@4": 0.6552130453693292, + "pass@8": 0.7088155395420423, + "pass@16": 0.7457569569827588, + "pass@32": 0.7748404435830744, + "pass@64": 0.7998995443105852, + "pass@128": 0.822 + }, + "4": { + "pass@1": 0.1451875, + "pass@2": 0.1972643946850394, + "pass@4": 0.25449041366704117, + "pass@8": 0.3098883354801711, + "pass@16": 0.35893551789667805, + "pass@32": 0.40036083151097207, + "pass@64": 0.4348911819551309, + "pass@128": 0.464 + }, + "5": { + "pass@1": 0.1498359375, + "pass@2": 0.1992218257874015, + "pass@4": 0.25530562307836485, + "pass@8": 0.31287900960176424, + "pass@16": 0.3692893207379819, + "pass@32": 0.4243511443374379, + "pass@64": 0.47744351587524025, + "pass@128": 0.525 + }, + "6": { + "pass@1": 0.1414921875, + "pass@2": 0.20634559547244066, + "pass@4": 0.2855413216160479, + "pass@8": 0.3759909813183259, + "pass@16": 0.47039010749595905, + "pass@32": 0.5583390689319193, + "pass@64": 0.6317356404851369, + "pass@128": 0.69 + }, + "7": { + "pass@1": 0.136390625, + "pass@2": 0.19957381889763773, + "pass@4": 0.27261641619797516, + "pass@8": 0.3518023343170404, + "pass@16": 0.4352175688724406, + "pass@32": 0.5175287112069517, + "pass@64": 0.5913243659581479, + "pass@128": 0.654 + }, + "8": { + "pass@1": 0.1460703125, + "pass@2": 0.2179701033464565, + "pass@4": 0.2997344593175849, + "pass@8": 0.3829834352426461, + "pass@16": 0.46184138214291204, + "pass@32": 0.5327699584678148, + "pass@64": 0.5953491007884835, + "pass@128": 0.651 + }, + "9": { + "pass@1": 0.133984375, + "pass@2": 0.19858501476377932, + "pass@4": 0.27376406899137595, + "pass@8": 0.353816892266726, + "pass@16": 0.4348287050171671, + "pass@32": 0.5123421145750472, + "pass@64": 0.5850367939068287, + "pass@128": 0.656 + }, + "11": { + "pass@1": 0.131359375, + "pass@2": 0.1979074803149605, + "pass@4": 0.2734353920134983, + "pass@8": 0.34991794185649616, + "pass@16": 0.4222118709001832, + "pass@32": 0.4868841744504942, + "pass@64": 0.5431513744419101, + "pass@128": 0.595 + }, + "12": { + "pass@1": 0.143828125, + "pass@2": 0.21519525098425205, + "pass@4": 0.2899552368766403, + "pass@8": 0.3594490123434772, + "pass@16": 0.4224746832075962, + "pass@32": 0.4809637642820015, + "pass@64": 0.5381737990170851, + "pass@128": 0.595 + }, + "13": { + "pass@1": 0.111484375, + "pass@2": 0.1739815452755905, + "pass@4": 0.24539893100862373, + "pass@8": 0.3153656153080575, + "pass@16": 0.3778591615177856, + "pass@32": 0.43372927559272895, + "pass@64": 0.4890995661569375, + "pass@128": 0.548 + }, + "14": { + "pass@1": 0.1206875, + "pass@2": 0.18854195374015764, + "pass@4": 0.26664505614923073, + "pass@8": 0.3413639269498822, + "pass@16": 0.4046785769193911, + "pass@32": 0.45851520728087153, + "pass@64": 0.509789165398226, + "pass@128": 0.56 + }, + "15": { + "pass@1": 0.1028203125, + "pass@2": 0.16334805610236225, + "pass@4": 0.23506728496437912, + "pass@8": 0.3068765586059638, + "pass@16": 0.37027187425270813, + "pass@32": 0.42434378259224537, + "pass@64": 0.4738853868332054, + "pass@128": 0.523 + }, + "16": { + "pass@1": 0.1196953125, + "pass@2": 0.1894175688976377, + "pass@4": 0.2691667745594299, + "pass@8": 0.3457092688172161, + "pass@16": 0.4096187459888607, + "pass@32": 0.4592911285191424, + "pass@64": 0.501203414513232, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.0991015625, + "pass@2": 0.16048720472440944, + "pass@4": 0.23582270894263194, + "pass@8": 0.31153919565936955, + "pass@16": 0.3757639244403764, + "pass@32": 0.4258215453216609, + "pass@64": 0.4667552886812957, + "pass@128": 0.507 + }, + "18": { + "pass@1": 0.09759375, + "pass@2": 0.15683747539370085, + "pass@4": 0.2280157783089612, + "pass@8": 0.2992512552701541, + "pass@16": 0.3611276837432124, + "pass@32": 0.41048525384187906, + "pass@64": 0.45148370151351497, + "pass@128": 0.495 + }, + "19": { + "pass@1": 0.09328125, + "pass@2": 0.1501445620078741, + "pass@4": 0.21755319610048723, + "pass@8": 0.2828352344004617, + "pass@16": 0.33638304684234605, + "pass@32": 0.37763774237042336, + "pass@64": 0.4131950503444665, + "pass@128": 0.45 + }, + "20": { + "pass@1": 0.0874921875, + "pass@2": 0.14097293307086609, + "pass@4": 0.20613240073115838, + "pass@8": 0.272018881177445, + "pass@16": 0.3276648909704376, + "pass@32": 0.3689930698714124, + "pass@64": 0.40280995928038527, + "pass@128": 0.441 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.csv b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..980382f8e40f4dc50beae854e3a25daa871e8f06 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18010,0.27102777777777776,0.110734375,0.1866628289473684,0.0,0.0,0.0,173.61795225694445,288.2031421875,1152000,1280000,2432000 diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f6df5e9b525e5376e787d871cb58831fbcc81e60 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18010", + "id_acc": 0.27102777777777776, + "ood_acc": 0.110734375, + "total_acc": 0.1866628289473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 173.61795225694445, + "ood_avg_resp_len": 288.2031421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 453964, + "answer_accuracy": 0.1866628289473684, + "avg_response_len": 233.92594695723685, + "resp_tokens_sum": 568907903, + "pass_at_k": { + "pass@1": 0.1866628289473684, + "pass@2": 0.2484606493472894, + "pass@4": 0.31753387130226435, + "pass@8": 0.3858117835026649, + "pass@16": 0.44793793497411705, + "pass@32": 0.5022813805078344, + "pass@64": 0.5507115050772899, + "pass@128": 0.5964736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.137609375, + "pass@2": 0.2044445127952755, + "pass@4": 0.2807508605174354, + "pass@8": 0.3593730437979568, + "pass@16": 0.43512775897758893, + "pass@32": 0.5042921162048943, + "pass@64": 0.5662955106188434, + "pass@128": 0.623 + }, + "2": { + "pass@1": 0.9689609375, + "pass@2": 0.9825099655511812, + "pass@4": 0.9885345863329588, + "pass@8": 0.9905474245953042, + "pass@16": 0.9913789876014476, + "pass@32": 0.9918568967077263, + "pass@64": 0.991996236389654, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.47971875, + "pass@2": 0.5780030757874023, + "pass@4": 0.6552130453693292, + "pass@8": 0.7088155395420423, + "pass@16": 0.7457569569827588, + "pass@32": 0.7748404435830744, + "pass@64": 0.7998995443105852, + "pass@128": 0.822 + }, + "4": { + "pass@1": 0.1451875, + "pass@2": 0.1972643946850394, + "pass@4": 0.25449041366704117, + "pass@8": 0.3098883354801711, + "pass@16": 0.35893551789667805, + "pass@32": 0.40036083151097207, + "pass@64": 0.4348911819551309, + "pass@128": 0.464 + }, + "5": { + "pass@1": 0.1498359375, + "pass@2": 0.1992218257874015, + "pass@4": 0.25530562307836485, + "pass@8": 0.31287900960176424, + "pass@16": 0.3692893207379819, + "pass@32": 0.4243511443374379, + "pass@64": 0.47744351587524025, + "pass@128": 0.525 + }, + "6": { + "pass@1": 0.1414921875, + "pass@2": 0.20634559547244066, + "pass@4": 0.2855413216160479, + "pass@8": 0.3759909813183259, + "pass@16": 0.47039010749595905, + "pass@32": 0.5583390689319193, + "pass@64": 0.6317356404851369, + "pass@128": 0.69 + }, + "7": { + "pass@1": 0.136390625, + "pass@2": 0.19957381889763773, + "pass@4": 0.27261641619797516, + "pass@8": 0.3518023343170404, + "pass@16": 0.4352175688724406, + "pass@32": 0.5175287112069517, + "pass@64": 0.5913243659581479, + "pass@128": 0.654 + }, + "8": { + "pass@1": 0.1460703125, + "pass@2": 0.2179701033464565, + "pass@4": 0.2997344593175849, + "pass@8": 0.3829834352426461, + "pass@16": 0.46184138214291204, + "pass@32": 0.5327699584678148, + "pass@64": 0.5953491007884835, + "pass@128": 0.651 + }, + "9": { + "pass@1": 0.133984375, + "pass@2": 0.19858501476377932, + "pass@4": 0.27376406899137595, + "pass@8": 0.353816892266726, + "pass@16": 0.4348287050171671, + "pass@32": 0.5123421145750472, + "pass@64": 0.5850367939068287, + "pass@128": 0.656 + }, + "11": { + "pass@1": 0.131359375, + "pass@2": 0.1979074803149605, + "pass@4": 0.2734353920134983, + "pass@8": 0.34991794185649616, + "pass@16": 0.4222118709001832, + "pass@32": 0.4868841744504942, + "pass@64": 0.5431513744419101, + "pass@128": 0.595 + }, + "12": { + "pass@1": 0.143828125, + "pass@2": 0.21519525098425205, + "pass@4": 0.2899552368766403, + "pass@8": 0.3594490123434772, + "pass@16": 0.4224746832075962, + "pass@32": 0.4809637642820015, + "pass@64": 0.5381737990170851, + "pass@128": 0.595 + }, + "13": { + "pass@1": 0.111484375, + "pass@2": 0.1739815452755905, + "pass@4": 0.24539893100862373, + "pass@8": 0.3153656153080575, + "pass@16": 0.3778591615177856, + "pass@32": 0.43372927559272895, + "pass@64": 0.4890995661569375, + "pass@128": 0.548 + }, + "14": { + "pass@1": 0.1206875, + "pass@2": 0.18854195374015764, + "pass@4": 0.26664505614923073, + "pass@8": 0.3413639269498822, + "pass@16": 0.4046785769193911, + "pass@32": 0.45851520728087153, + "pass@64": 0.509789165398226, + "pass@128": 0.56 + }, + "15": { + "pass@1": 0.1028203125, + "pass@2": 0.16334805610236225, + "pass@4": 0.23506728496437912, + "pass@8": 0.3068765586059638, + "pass@16": 0.37027187425270813, + "pass@32": 0.42434378259224537, + "pass@64": 0.4738853868332054, + "pass@128": 0.523 + }, + "16": { + "pass@1": 0.1196953125, + "pass@2": 0.1894175688976377, + "pass@4": 0.2691667745594299, + "pass@8": 0.3457092688172161, + "pass@16": 0.4096187459888607, + "pass@32": 0.4592911285191424, + "pass@64": 0.501203414513232, + "pass@128": 0.542 + }, + "17": { + "pass@1": 0.0991015625, + "pass@2": 0.16048720472440944, + "pass@4": 0.23582270894263194, + "pass@8": 0.31153919565936955, + "pass@16": 0.3757639244403764, + "pass@32": 0.4258215453216609, + "pass@64": 0.4667552886812957, + "pass@128": 0.507 + }, + "18": { + "pass@1": 0.09759375, + "pass@2": 0.15683747539370085, + "pass@4": 0.2280157783089612, + "pass@8": 0.2992512552701541, + "pass@16": 0.3611276837432124, + "pass@32": 0.41048525384187906, + "pass@64": 0.45148370151351497, + "pass@128": 0.495 + }, + "19": { + "pass@1": 0.09328125, + "pass@2": 0.1501445620078741, + "pass@4": 0.21755319610048723, + "pass@8": 0.2828352344004617, + "pass@16": 0.33638304684234605, + "pass@32": 0.37763774237042336, + "pass@64": 0.4131950503444665, + "pass@128": 0.45 + }, + "20": { + "pass@1": 0.0874921875, + "pass@2": 0.14097293307086609, + "pass@4": 0.20613240073115838, + "pass@8": 0.272018881177445, + "pass@16": 0.3276648909704376, + "pass@32": 0.3689930698714124, + "pass@64": 0.40280995928038527, + "pass@128": 0.441 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/checkpoint-18010_metrics.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/checkpoint-18010_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b088d8ca47b48d166115f78be5745a1700b05907 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/checkpoint-18010_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18010", + "id": { + "answer_accuracy": 0.3020729166666667, + "avg_response_len": 184.5305798611111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.3020729166666667, + "pass@2": 0.37382344980315735, + "pass@4": 0.45089398859518226, + "pass@8": 0.5283900751486241, + "pass@16": 0.6034240139015935, + "pass@32": 0.6738054655823494, + "pass@64": 0.7370642895117986, + "pass@128": 0.7895555555555556 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1616328125, + "pass@2": 0.2371321358267718, + "pass@4": 0.32437877737157794, + "pass@8": 0.41756130808955405, + "pass@16": 0.5143301666822244, + "pass@32": 0.6123601418094415, + "pass@64": 0.7076622565770715, + "pass@128": 0.792 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4157578125, + "pass@2": 0.5235864911417325, + "pass@4": 0.6176296038620174, + "pass@8": 0.6929616881022262, + "pass@16": 0.751709196930996, + "pass@32": 0.7987127188079279, + "pass@64": 0.8380727198564655, + "pass@128": 0.872 + }, + "4": { + "pass@1": 0.2006171875, + "pass@2": 0.275439345472441, + "pass@4": 0.35409392135358086, + "pass@8": 0.4294360103868058, + "pass@16": 0.49693970613616434, + "pass@32": 0.5541958542313058, + "pass@64": 0.6005014037992568, + "pass@128": 0.638 + }, + "5": { + "pass@1": 0.2241796875, + "pass@2": 0.28967876476377935, + "pass@4": 0.35803000056243006, + "pass@8": 0.4258272310696072, + "pass@16": 0.48969123652799784, + "pass@32": 0.5490053353819573, + "pass@64": 0.6045812801603323, + "pass@128": 0.654 + }, + "6": { + "pass@1": 0.2055625, + "pass@2": 0.297341658464567, + "pass@4": 0.39864485133108346, + "pass@8": 0.5011417308029112, + "pass@16": 0.5973344477607267, + "pass@32": 0.6824786499037648, + "pass@64": 0.7556893648624248, + "pass@128": 0.815 + }, + "7": { + "pass@1": 0.1781015625, + "pass@2": 0.25524581692913373, + "pass@4": 0.34333610948631355, + "pass@8": 0.43813205022352747, + "pass@16": 0.5350430251292626, + "pass@32": 0.6270678047943197, + "pass@64": 0.7081894460506878, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.18075, + "pass@2": 0.2597849409448822, + "pass@4": 0.3478030234345704, + "pass@8": 0.4401760208220928, + "pass@16": 0.5346522786047639, + "pass@32": 0.6280045987017412, + "pass@64": 0.7133960905497793, + "pass@128": 0.78 + }, + "9": { + "pass@1": 0.1520546875, + "pass@2": 0.22620189468503912, + "pass@4": 0.3141296099550055, + "pass@8": 0.4102746368407737, + "pass@16": 0.5111160673420707, + "pass@32": 0.6124240866107038, + "pass@64": 0.7054860437501366, + "pass@128": 0.781 + } + }, + "per_op_accuracy": { + "10": 0.1616328125, + "2": 1.0, + "3": 0.4157578125, + "4": 0.2006171875, + "5": 0.2241796875, + "6": 0.2055625, + "7": 0.1781015625, + "8": 0.18075, + "9": 0.1520546875 + }, + "per_op_avg_response_len": { + "10": 253.9462421875, + "2": 106.3366484375, + "3": 144.0999609375, + "4": 137.9358984375, + "5": 160.4995703125, + "6": 197.120046875, + "7": 194.91525, + "8": 223.8633984375, + "9": 242.058203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 253.9462421875, + "2": 106.3366484375, + "3": 144.0999609375, + "4": 137.9358984375, + "5": 160.4995703125, + "6": 197.120046875, + "7": 194.91525, + "8": 223.8633984375, + "9": 242.058203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 347988, + "answer_accuracy": 0.3020729166666667, + "avg_response_len": 184.5305798611111, + "resp_tokens_sum": 212579228, + "pass_at_k": { + "pass@1": 0.3020729166666667, + "pass@2": 0.37382344980315735, + "pass@4": 0.45089398859518226, + "pass@8": 0.5283900751486241, + "pass@16": 0.6034240139015935, + "pass@32": 0.6738054655823494, + "pass@64": 0.7370642895117986, + "pass@128": 0.7895555555555556 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1616328125, + "pass@2": 0.2371321358267718, + "pass@4": 0.32437877737157794, + "pass@8": 0.41756130808955405, + "pass@16": 0.5143301666822244, + "pass@32": 0.6123601418094415, + "pass@64": 0.7076622565770715, + "pass@128": 0.792 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4157578125, + "pass@2": 0.5235864911417325, + "pass@4": 0.6176296038620174, + "pass@8": 0.6929616881022262, + "pass@16": 0.751709196930996, + "pass@32": 0.7987127188079279, + "pass@64": 0.8380727198564655, + "pass@128": 0.872 + }, + "4": { + "pass@1": 0.2006171875, + "pass@2": 0.275439345472441, + "pass@4": 0.35409392135358086, + "pass@8": 0.4294360103868058, + "pass@16": 0.49693970613616434, + "pass@32": 0.5541958542313058, + "pass@64": 0.6005014037992568, + "pass@128": 0.638 + }, + "5": { + "pass@1": 0.2241796875, + "pass@2": 0.28967876476377935, + "pass@4": 0.35803000056243006, + "pass@8": 0.4258272310696072, + "pass@16": 0.48969123652799784, + "pass@32": 0.5490053353819573, + "pass@64": 0.6045812801603323, + "pass@128": 0.654 + }, + "6": { + "pass@1": 0.2055625, + "pass@2": 0.297341658464567, + "pass@4": 0.39864485133108346, + "pass@8": 0.5011417308029112, + "pass@16": 0.5973344477607267, + "pass@32": 0.6824786499037648, + "pass@64": 0.7556893648624248, + "pass@128": 0.815 + }, + "7": { + "pass@1": 0.1781015625, + "pass@2": 0.25524581692913373, + "pass@4": 0.34333610948631355, + "pass@8": 0.43813205022352747, + "pass@16": 0.5350430251292626, + "pass@32": 0.6270678047943197, + "pass@64": 0.7081894460506878, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.18075, + "pass@2": 0.2597849409448822, + "pass@4": 0.3478030234345704, + "pass@8": 0.4401760208220928, + "pass@16": 0.5346522786047639, + "pass@32": 0.6280045987017412, + "pass@64": 0.7133960905497793, + "pass@128": 0.78 + }, + "9": { + "pass@1": 0.1520546875, + "pass@2": 0.22620189468503912, + "pass@4": 0.3141296099550055, + "pass@8": 0.4102746368407737, + "pass@16": 0.5111160673420707, + "pass@32": 0.6124240866107038, + "pass@64": 0.7054860437501366, + "pass@128": 0.781 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.1320265625, + "avg_response_len": 321.86277578125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.1320265625, + "pass@2": 0.20114554625984432, + "pass@4": 0.27932025115298625, + "pass@8": 0.35878004351438375, + "pass@16": 0.4388273871149983, + "pass@32": 0.5240331764183352, + "pass@64": 0.6159384000798367, + "pass@128": 0.7073 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.153703125, + "pass@2": 0.23045374015748027, + "pass@4": 0.3178807728721409, + "pass@8": 0.41113749959262696, + "pass@16": 0.50993922657759, + "pass@32": 0.6130430482331137, + "pass@64": 0.7136666950165881, + "pass@128": 0.8 + }, + "12": { + "pass@1": 0.16615625, + "pass@2": 0.2501759350393704, + "pass@4": 0.34184892304086945, + "pass@8": 0.431253634251671, + "pass@16": 0.5167137869598212, + "pass@32": 0.6015680645863576, + "pass@64": 0.6854189174845233, + "pass@128": 0.761 + }, + "13": { + "pass@1": 0.139984375, + "pass@2": 0.21061601870078744, + "pass@4": 0.2901609191038619, + "pass@8": 0.3711063875047056, + "pass@16": 0.45246817091979913, + "pass@32": 0.5385821239880164, + "pass@64": 0.6288167618512827, + "pass@128": 0.715 + }, + "14": { + "pass@1": 0.1458671875, + "pass@2": 0.2221096210629923, + "pass@4": 0.30455538779527536, + "pass@8": 0.3838924102533045, + "pass@16": 0.4635731640624523, + "pass@32": 0.5518411727289914, + "pass@64": 0.6492694410903208, + "pass@128": 0.744 + }, + "15": { + "pass@1": 0.1281875, + "pass@2": 0.19309682578740164, + "pass@4": 0.26738501556055433, + "pass@8": 0.34488120939203987, + "pass@16": 0.42758087445597254, + "pass@32": 0.5213038492609346, + "pass@64": 0.624661080762865, + "pass@128": 0.728 + }, + "16": { + "pass@1": 0.1361796875, + "pass@2": 0.20873535925196865, + "pass@4": 0.2910645499625044, + "pass@8": 0.37335496399036194, + "pass@16": 0.45373865906218275, + "pass@32": 0.5380984104766606, + "pass@64": 0.6287105984766999, + "pass@128": 0.715 + }, + "17": { + "pass@1": 0.1215703125, + "pass@2": 0.18601218011811013, + "pass@4": 0.2600743533933257, + "pass@8": 0.3369977565249447, + "pass@16": 0.4147892254978584, + "pass@32": 0.4973943071874142, + "pass@64": 0.58699158220655, + "pass@128": 0.678 + }, + "18": { + "pass@1": 0.1148828125, + "pass@2": 0.17785629921259852, + "pass@4": 0.2509636291713533, + "pass@8": 0.3262295188419923, + "pass@16": 0.4010706780554508, + "pass@32": 0.47977257936756085, + "pass@64": 0.5678323121875622, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1098125, + "pass@2": 0.17042605807086622, + "pass@4": 0.23903213226471656, + "pass@8": 0.307595519268488, + "pass@16": 0.37571329356073757, + "pass@32": 0.4494958147506268, + "pass@64": 0.5342012914141494, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.103921875, + "pass@2": 0.16197342519685046, + "pass@4": 0.23023682836520395, + "pass@8": 0.30135153552356025, + "pass@16": 0.37268679199799903, + "pass@32": 0.4492323936037306, + "pass@64": 0.5398153203078361, + "pass@128": 0.645 + } + }, + "per_op_accuracy": { + "11": 0.153703125, + "12": 0.16615625, + "13": 0.139984375, + "14": 0.1458671875, + "15": 0.1281875, + "16": 0.1361796875, + "17": 0.1215703125, + "18": 0.1148828125, + "19": 0.1098125, + "20": 0.103921875 + }, + "per_op_avg_response_len": { + "11": 272.495640625, + "12": 276.4378203125, + "13": 285.053046875, + "14": 308.508671875, + "15": 322.4529375, + "16": 340.519890625, + "17": 343.37025, + "18": 349.88740625, + "19": 352.4517265625, + "20": 367.4503671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 272.495640625, + "12": 276.4378203125, + "13": 285.053046875, + "14": 308.508671875, + "15": 322.4529375, + "16": 340.519890625, + "17": 343.37025, + "18": 349.88740625, + "19": 352.4517265625, + "20": 367.4503671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 168994, + "answer_accuracy": 0.1320265625, + "avg_response_len": 321.86277578125, + "resp_tokens_sum": 411984353, + "pass_at_k": { + "pass@1": 0.1320265625, + "pass@2": 0.20114554625984432, + "pass@4": 0.27932025115298625, + "pass@8": 0.35878004351438375, + "pass@16": 0.4388273871149983, + "pass@32": 0.5240331764183352, + "pass@64": 0.6159384000798367, + "pass@128": 0.7073 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.153703125, + "pass@2": 0.23045374015748027, + "pass@4": 0.3178807728721409, + "pass@8": 0.41113749959262696, + "pass@16": 0.50993922657759, + "pass@32": 0.6130430482331137, + "pass@64": 0.7136666950165881, + "pass@128": 0.8 + }, + "12": { + "pass@1": 0.16615625, + "pass@2": 0.2501759350393704, + "pass@4": 0.34184892304086945, + "pass@8": 0.431253634251671, + "pass@16": 0.5167137869598212, + "pass@32": 0.6015680645863576, + "pass@64": 0.6854189174845233, + "pass@128": 0.761 + }, + "13": { + "pass@1": 0.139984375, + "pass@2": 0.21061601870078744, + "pass@4": 0.2901609191038619, + "pass@8": 0.3711063875047056, + "pass@16": 0.45246817091979913, + "pass@32": 0.5385821239880164, + "pass@64": 0.6288167618512827, + "pass@128": 0.715 + }, + "14": { + "pass@1": 0.1458671875, + "pass@2": 0.2221096210629923, + "pass@4": 0.30455538779527536, + "pass@8": 0.3838924102533045, + "pass@16": 0.4635731640624523, + "pass@32": 0.5518411727289914, + "pass@64": 0.6492694410903208, + "pass@128": 0.744 + }, + "15": { + "pass@1": 0.1281875, + "pass@2": 0.19309682578740164, + "pass@4": 0.26738501556055433, + "pass@8": 0.34488120939203987, + "pass@16": 0.42758087445597254, + "pass@32": 0.5213038492609346, + "pass@64": 0.624661080762865, + "pass@128": 0.728 + }, + "16": { + "pass@1": 0.1361796875, + "pass@2": 0.20873535925196865, + "pass@4": 0.2910645499625044, + "pass@8": 0.37335496399036194, + "pass@16": 0.45373865906218275, + "pass@32": 0.5380984104766606, + "pass@64": 0.6287105984766999, + "pass@128": 0.715 + }, + "17": { + "pass@1": 0.1215703125, + "pass@2": 0.18601218011811013, + "pass@4": 0.2600743533933257, + "pass@8": 0.3369977565249447, + "pass@16": 0.4147892254978584, + "pass@32": 0.4973943071874142, + "pass@64": 0.58699158220655, + "pass@128": 0.678 + }, + "18": { + "pass@1": 0.1148828125, + "pass@2": 0.17785629921259852, + "pass@4": 0.2509636291713533, + "pass@8": 0.3262295188419923, + "pass@16": 0.4010706780554508, + "pass@32": 0.47977257936756085, + "pass@64": 0.5678323121875622, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1098125, + "pass@2": 0.17042605807086622, + "pass@4": 0.23903213226471656, + "pass@8": 0.307595519268488, + "pass@16": 0.37571329356073757, + "pass@32": 0.4494958147506268, + "pass@64": 0.5342012914141494, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.103921875, + "pass@2": 0.16197342519685046, + "pass@4": 0.23023682836520395, + "pass@8": 0.30135153552356025, + "pass@16": 0.37268679199799903, + "pass@32": 0.4492323936037306, + "pass@64": 0.5398153203078361, + "pass@128": 0.645 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.2125748355263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.2125748355263158, + "pass@2": 0.2829403426750803, + "pass@4": 0.3605920215203193, + "pass@8": 0.4391216374463875, + "pass@16": 0.5167942103297193, + "pass@32": 0.5949779449697302, + "pass@64": 0.6733138213897139, + "pass@128": 0.7462631578947369 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1616328125, + "pass@2": 0.2371321358267718, + "pass@4": 0.32437877737157794, + "pass@8": 0.41756130808955405, + "pass@16": 0.5143301666822244, + "pass@32": 0.6123601418094415, + "pass@64": 0.7076622565770715, + "pass@128": 0.792 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4157578125, + "pass@2": 0.5235864911417325, + "pass@4": 0.6176296038620174, + "pass@8": 0.6929616881022262, + "pass@16": 0.751709196930996, + "pass@32": 0.7987127188079279, + "pass@64": 0.8380727198564655, + "pass@128": 0.872 + }, + "4": { + "pass@1": 0.2006171875, + "pass@2": 0.275439345472441, + "pass@4": 0.35409392135358086, + "pass@8": 0.4294360103868058, + "pass@16": 0.49693970613616434, + "pass@32": 0.5541958542313058, + "pass@64": 0.6005014037992568, + "pass@128": 0.638 + }, + "5": { + "pass@1": 0.2241796875, + "pass@2": 0.28967876476377935, + "pass@4": 0.35803000056243006, + "pass@8": 0.4258272310696072, + "pass@16": 0.48969123652799784, + "pass@32": 0.5490053353819573, + "pass@64": 0.6045812801603323, + "pass@128": 0.654 + }, + "6": { + "pass@1": 0.2055625, + "pass@2": 0.297341658464567, + "pass@4": 0.39864485133108346, + "pass@8": 0.5011417308029112, + "pass@16": 0.5973344477607267, + "pass@32": 0.6824786499037648, + "pass@64": 0.7556893648624248, + "pass@128": 0.815 + }, + "7": { + "pass@1": 0.1781015625, + "pass@2": 0.25524581692913373, + "pass@4": 0.34333610948631355, + "pass@8": 0.43813205022352747, + "pass@16": 0.5350430251292626, + "pass@32": 0.6270678047943197, + "pass@64": 0.7081894460506878, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.18075, + "pass@2": 0.2597849409448822, + "pass@4": 0.3478030234345704, + "pass@8": 0.4401760208220928, + "pass@16": 0.5346522786047639, + "pass@32": 0.6280045987017412, + "pass@64": 0.7133960905497793, + "pass@128": 0.78 + }, + "9": { + "pass@1": 0.1520546875, + "pass@2": 0.22620189468503912, + "pass@4": 0.3141296099550055, + "pass@8": 0.4102746368407737, + "pass@16": 0.5111160673420707, + "pass@32": 0.6124240866107038, + "pass@64": 0.7054860437501366, + "pass@128": 0.781 + }, + "11": { + "pass@1": 0.153703125, + "pass@2": 0.23045374015748027, + "pass@4": 0.3178807728721409, + "pass@8": 0.41113749959262696, + "pass@16": 0.50993922657759, + "pass@32": 0.6130430482331137, + "pass@64": 0.7136666950165881, + "pass@128": 0.8 + }, + "12": { + "pass@1": 0.16615625, + "pass@2": 0.2501759350393704, + "pass@4": 0.34184892304086945, + "pass@8": 0.431253634251671, + "pass@16": 0.5167137869598212, + "pass@32": 0.6015680645863576, + "pass@64": 0.6854189174845233, + "pass@128": 0.761 + }, + "13": { + "pass@1": 0.139984375, + "pass@2": 0.21061601870078744, + "pass@4": 0.2901609191038619, + "pass@8": 0.3711063875047056, + "pass@16": 0.45246817091979913, + "pass@32": 0.5385821239880164, + "pass@64": 0.6288167618512827, + "pass@128": 0.715 + }, + "14": { + "pass@1": 0.1458671875, + "pass@2": 0.2221096210629923, + "pass@4": 0.30455538779527536, + "pass@8": 0.3838924102533045, + "pass@16": 0.4635731640624523, + "pass@32": 0.5518411727289914, + "pass@64": 0.6492694410903208, + "pass@128": 0.744 + }, + "15": { + "pass@1": 0.1281875, + "pass@2": 0.19309682578740164, + "pass@4": 0.26738501556055433, + "pass@8": 0.34488120939203987, + "pass@16": 0.42758087445597254, + "pass@32": 0.5213038492609346, + "pass@64": 0.624661080762865, + "pass@128": 0.728 + }, + "16": { + "pass@1": 0.1361796875, + "pass@2": 0.20873535925196865, + "pass@4": 0.2910645499625044, + "pass@8": 0.37335496399036194, + "pass@16": 0.45373865906218275, + "pass@32": 0.5380984104766606, + "pass@64": 0.6287105984766999, + "pass@128": 0.715 + }, + "17": { + "pass@1": 0.1215703125, + "pass@2": 0.18601218011811013, + "pass@4": 0.2600743533933257, + "pass@8": 0.3369977565249447, + "pass@16": 0.4147892254978584, + "pass@32": 0.4973943071874142, + "pass@64": 0.58699158220655, + "pass@128": 0.678 + }, + "18": { + "pass@1": 0.1148828125, + "pass@2": 0.17785629921259852, + "pass@4": 0.2509636291713533, + "pass@8": 0.3262295188419923, + "pass@16": 0.4010706780554508, + "pass@32": 0.47977257936756085, + "pass@64": 0.5678323121875622, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1098125, + "pass@2": 0.17042605807086622, + "pass@4": 0.23903213226471656, + "pass@8": 0.307595519268488, + "pass@16": 0.37571329356073757, + "pass@32": 0.4494958147506268, + "pass@64": 0.5342012914141494, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.103921875, + "pass@2": 0.16197342519685046, + "pass@4": 0.23023682836520395, + "pass@8": 0.30135153552356025, + "pass@16": 0.37268679199799903, + "pass@32": 0.4492323936037306, + "pass@64": 0.5398153203078361, + "pass@128": 0.645 + } + }, + "per_op_accuracy": { + "10": 0.1616328125, + "2": 1.0, + "3": 0.4157578125, + "4": 0.2006171875, + "5": 0.2241796875, + "6": 0.2055625, + "7": 0.1781015625, + "8": 0.18075, + "9": 0.1520546875, + "11": 0.153703125, + "12": 0.16615625, + "13": 0.139984375, + "14": 0.1458671875, + "15": 0.1281875, + "16": 0.1361796875, + "17": 0.1215703125, + "18": 0.1148828125, + "19": 0.1098125, + "20": 0.103921875 + }, + "per_op_avg_response_len": { + "10": 253.9462421875, + "2": 106.3366484375, + "3": 144.0999609375, + "4": 137.9358984375, + "5": 160.4995703125, + "6": 197.120046875, + "7": 194.91525, + "8": 223.8633984375, + "9": 242.058203125, + "11": 272.495640625, + "12": 276.4378203125, + "13": 285.053046875, + "14": 308.508671875, + "15": 322.4529375, + "16": 340.519890625, + "17": 343.37025, + "18": 349.88740625, + "19": 352.4517265625, + "20": 367.4503671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 253.9462421875, + "2": 106.3366484375, + "3": 144.0999609375, + "4": 137.9358984375, + "5": 160.4995703125, + "6": 197.120046875, + "7": 194.91525, + "8": 223.8633984375, + "9": 242.058203125, + "11": 272.495640625, + "12": 276.4378203125, + "13": 285.053046875, + "14": 308.508671875, + "15": 322.4529375, + "16": 340.519890625, + "17": 343.37025, + "18": 349.88740625, + "19": 352.4517265625, + "20": 367.4503671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 516982, + "answer_accuracy": 0.2125748355263158, + "avg_response_len": 256.8106829769737, + "resp_tokens_sum": 624563581, + "pass_at_k": { + "pass@1": 0.2125748355263158, + "pass@2": 0.2829403426750803, + "pass@4": 0.3605920215203193, + "pass@8": 0.4391216374463875, + "pass@16": 0.5167942103297193, + "pass@32": 0.5949779449697302, + "pass@64": 0.6733138213897139, + "pass@128": 0.7462631578947369 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1616328125, + "pass@2": 0.2371321358267718, + "pass@4": 0.32437877737157794, + "pass@8": 0.41756130808955405, + "pass@16": 0.5143301666822244, + "pass@32": 0.6123601418094415, + "pass@64": 0.7076622565770715, + "pass@128": 0.792 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4157578125, + "pass@2": 0.5235864911417325, + "pass@4": 0.6176296038620174, + "pass@8": 0.6929616881022262, + "pass@16": 0.751709196930996, + "pass@32": 0.7987127188079279, + "pass@64": 0.8380727198564655, + "pass@128": 0.872 + }, + "4": { + "pass@1": 0.2006171875, + "pass@2": 0.275439345472441, + "pass@4": 0.35409392135358086, + "pass@8": 0.4294360103868058, + "pass@16": 0.49693970613616434, + "pass@32": 0.5541958542313058, + "pass@64": 0.6005014037992568, + "pass@128": 0.638 + }, + "5": { + "pass@1": 0.2241796875, + "pass@2": 0.28967876476377935, + "pass@4": 0.35803000056243006, + "pass@8": 0.4258272310696072, + "pass@16": 0.48969123652799784, + "pass@32": 0.5490053353819573, + "pass@64": 0.6045812801603323, + "pass@128": 0.654 + }, + "6": { + "pass@1": 0.2055625, + "pass@2": 0.297341658464567, + "pass@4": 0.39864485133108346, + "pass@8": 0.5011417308029112, + "pass@16": 0.5973344477607267, + "pass@32": 0.6824786499037648, + "pass@64": 0.7556893648624248, + "pass@128": 0.815 + }, + "7": { + "pass@1": 0.1781015625, + "pass@2": 0.25524581692913373, + "pass@4": 0.34333610948631355, + "pass@8": 0.43813205022352747, + "pass@16": 0.5350430251292626, + "pass@32": 0.6270678047943197, + "pass@64": 0.7081894460506878, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.18075, + "pass@2": 0.2597849409448822, + "pass@4": 0.3478030234345704, + "pass@8": 0.4401760208220928, + "pass@16": 0.5346522786047639, + "pass@32": 0.6280045987017412, + "pass@64": 0.7133960905497793, + "pass@128": 0.78 + }, + "9": { + "pass@1": 0.1520546875, + "pass@2": 0.22620189468503912, + "pass@4": 0.3141296099550055, + "pass@8": 0.4102746368407737, + "pass@16": 0.5111160673420707, + "pass@32": 0.6124240866107038, + "pass@64": 0.7054860437501366, + "pass@128": 0.781 + }, + "11": { + "pass@1": 0.153703125, + "pass@2": 0.23045374015748027, + "pass@4": 0.3178807728721409, + "pass@8": 0.41113749959262696, + "pass@16": 0.50993922657759, + "pass@32": 0.6130430482331137, + "pass@64": 0.7136666950165881, + "pass@128": 0.8 + }, + "12": { + "pass@1": 0.16615625, + "pass@2": 0.2501759350393704, + "pass@4": 0.34184892304086945, + "pass@8": 0.431253634251671, + "pass@16": 0.5167137869598212, + "pass@32": 0.6015680645863576, + "pass@64": 0.6854189174845233, + "pass@128": 0.761 + }, + "13": { + "pass@1": 0.139984375, + "pass@2": 0.21061601870078744, + "pass@4": 0.2901609191038619, + "pass@8": 0.3711063875047056, + "pass@16": 0.45246817091979913, + "pass@32": 0.5385821239880164, + "pass@64": 0.6288167618512827, + "pass@128": 0.715 + }, + "14": { + "pass@1": 0.1458671875, + "pass@2": 0.2221096210629923, + "pass@4": 0.30455538779527536, + "pass@8": 0.3838924102533045, + "pass@16": 0.4635731640624523, + "pass@32": 0.5518411727289914, + "pass@64": 0.6492694410903208, + "pass@128": 0.744 + }, + "15": { + "pass@1": 0.1281875, + "pass@2": 0.19309682578740164, + "pass@4": 0.26738501556055433, + "pass@8": 0.34488120939203987, + "pass@16": 0.42758087445597254, + "pass@32": 0.5213038492609346, + "pass@64": 0.624661080762865, + "pass@128": 0.728 + }, + "16": { + "pass@1": 0.1361796875, + "pass@2": 0.20873535925196865, + "pass@4": 0.2910645499625044, + "pass@8": 0.37335496399036194, + "pass@16": 0.45373865906218275, + "pass@32": 0.5380984104766606, + "pass@64": 0.6287105984766999, + "pass@128": 0.715 + }, + "17": { + "pass@1": 0.1215703125, + "pass@2": 0.18601218011811013, + "pass@4": 0.2600743533933257, + "pass@8": 0.3369977565249447, + "pass@16": 0.4147892254978584, + "pass@32": 0.4973943071874142, + "pass@64": 0.58699158220655, + "pass@128": 0.678 + }, + "18": { + "pass@1": 0.1148828125, + "pass@2": 0.17785629921259852, + "pass@4": 0.2509636291713533, + "pass@8": 0.3262295188419923, + "pass@16": 0.4010706780554508, + "pass@32": 0.47977257936756085, + "pass@64": 0.5678323121875622, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1098125, + "pass@2": 0.17042605807086622, + "pass@4": 0.23903213226471656, + "pass@8": 0.307595519268488, + "pass@16": 0.37571329356073757, + "pass@32": 0.4494958147506268, + "pass@64": 0.5342012914141494, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.103921875, + "pass@2": 0.16197342519685046, + "pass@4": 0.23023682836520395, + "pass@8": 0.30135153552356025, + "pass@16": 0.37268679199799903, + "pass@32": 0.4492323936037306, + "pass@64": 0.5398153203078361, + "pass@128": 0.645 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.csv b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..ac92f13d06bbae8fb11b33894d89fd9fda46e5aa --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18010,0.3020729166666667,0.1320265625,0.2125748355263158,0.0,0.0,0.0,184.5305798611111,321.86277578125,1152000,1280000,2432000 diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..38292a3c129c96e50857d4923bb9f14bc488be93 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block4/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18010", + "id_acc": 0.3020729166666667, + "ood_acc": 0.1320265625, + "total_acc": 0.2125748355263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 184.5305798611111, + "ood_avg_resp_len": 321.86277578125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 516982, + "answer_accuracy": 0.2125748355263158, + "avg_response_len": 256.8106829769737, + "resp_tokens_sum": 624563581, + "pass_at_k": { + "pass@1": 0.2125748355263158, + "pass@2": 0.2829403426750803, + "pass@4": 0.3605920215203193, + "pass@8": 0.4391216374463875, + "pass@16": 0.5167942103297193, + "pass@32": 0.5949779449697302, + "pass@64": 0.6733138213897139, + "pass@128": 0.7462631578947369 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1616328125, + "pass@2": 0.2371321358267718, + "pass@4": 0.32437877737157794, + "pass@8": 0.41756130808955405, + "pass@16": 0.5143301666822244, + "pass@32": 0.6123601418094415, + "pass@64": 0.7076622565770715, + "pass@128": 0.792 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4157578125, + "pass@2": 0.5235864911417325, + "pass@4": 0.6176296038620174, + "pass@8": 0.6929616881022262, + "pass@16": 0.751709196930996, + "pass@32": 0.7987127188079279, + "pass@64": 0.8380727198564655, + "pass@128": 0.872 + }, + "4": { + "pass@1": 0.2006171875, + "pass@2": 0.275439345472441, + "pass@4": 0.35409392135358086, + "pass@8": 0.4294360103868058, + "pass@16": 0.49693970613616434, + "pass@32": 0.5541958542313058, + "pass@64": 0.6005014037992568, + "pass@128": 0.638 + }, + "5": { + "pass@1": 0.2241796875, + "pass@2": 0.28967876476377935, + "pass@4": 0.35803000056243006, + "pass@8": 0.4258272310696072, + "pass@16": 0.48969123652799784, + "pass@32": 0.5490053353819573, + "pass@64": 0.6045812801603323, + "pass@128": 0.654 + }, + "6": { + "pass@1": 0.2055625, + "pass@2": 0.297341658464567, + "pass@4": 0.39864485133108346, + "pass@8": 0.5011417308029112, + "pass@16": 0.5973344477607267, + "pass@32": 0.6824786499037648, + "pass@64": 0.7556893648624248, + "pass@128": 0.815 + }, + "7": { + "pass@1": 0.1781015625, + "pass@2": 0.25524581692913373, + "pass@4": 0.34333610948631355, + "pass@8": 0.43813205022352747, + "pass@16": 0.5350430251292626, + "pass@32": 0.6270678047943197, + "pass@64": 0.7081894460506878, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.18075, + "pass@2": 0.2597849409448822, + "pass@4": 0.3478030234345704, + "pass@8": 0.4401760208220928, + "pass@16": 0.5346522786047639, + "pass@32": 0.6280045987017412, + "pass@64": 0.7133960905497793, + "pass@128": 0.78 + }, + "9": { + "pass@1": 0.1520546875, + "pass@2": 0.22620189468503912, + "pass@4": 0.3141296099550055, + "pass@8": 0.4102746368407737, + "pass@16": 0.5111160673420707, + "pass@32": 0.6124240866107038, + "pass@64": 0.7054860437501366, + "pass@128": 0.781 + }, + "11": { + "pass@1": 0.153703125, + "pass@2": 0.23045374015748027, + "pass@4": 0.3178807728721409, + "pass@8": 0.41113749959262696, + "pass@16": 0.50993922657759, + "pass@32": 0.6130430482331137, + "pass@64": 0.7136666950165881, + "pass@128": 0.8 + }, + "12": { + "pass@1": 0.16615625, + "pass@2": 0.2501759350393704, + "pass@4": 0.34184892304086945, + "pass@8": 0.431253634251671, + "pass@16": 0.5167137869598212, + "pass@32": 0.6015680645863576, + "pass@64": 0.6854189174845233, + "pass@128": 0.761 + }, + "13": { + "pass@1": 0.139984375, + "pass@2": 0.21061601870078744, + "pass@4": 0.2901609191038619, + "pass@8": 0.3711063875047056, + "pass@16": 0.45246817091979913, + "pass@32": 0.5385821239880164, + "pass@64": 0.6288167618512827, + "pass@128": 0.715 + }, + "14": { + "pass@1": 0.1458671875, + "pass@2": 0.2221096210629923, + "pass@4": 0.30455538779527536, + "pass@8": 0.3838924102533045, + "pass@16": 0.4635731640624523, + "pass@32": 0.5518411727289914, + "pass@64": 0.6492694410903208, + "pass@128": 0.744 + }, + "15": { + "pass@1": 0.1281875, + "pass@2": 0.19309682578740164, + "pass@4": 0.26738501556055433, + "pass@8": 0.34488120939203987, + "pass@16": 0.42758087445597254, + "pass@32": 0.5213038492609346, + "pass@64": 0.624661080762865, + "pass@128": 0.728 + }, + "16": { + "pass@1": 0.1361796875, + "pass@2": 0.20873535925196865, + "pass@4": 0.2910645499625044, + "pass@8": 0.37335496399036194, + "pass@16": 0.45373865906218275, + "pass@32": 0.5380984104766606, + "pass@64": 0.6287105984766999, + "pass@128": 0.715 + }, + "17": { + "pass@1": 0.1215703125, + "pass@2": 0.18601218011811013, + "pass@4": 0.2600743533933257, + "pass@8": 0.3369977565249447, + "pass@16": 0.4147892254978584, + "pass@32": 0.4973943071874142, + "pass@64": 0.58699158220655, + "pass@128": 0.678 + }, + "18": { + "pass@1": 0.1148828125, + "pass@2": 0.17785629921259852, + "pass@4": 0.2509636291713533, + "pass@8": 0.3262295188419923, + "pass@16": 0.4010706780554508, + "pass@32": 0.47977257936756085, + "pass@64": 0.5678323121875622, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1098125, + "pass@2": 0.17042605807086622, + "pass@4": 0.23903213226471656, + "pass@8": 0.307595519268488, + "pass@16": 0.37571329356073757, + "pass@32": 0.4494958147506268, + "pass@64": 0.5342012914141494, + "pass@128": 0.626 + }, + "20": { + "pass@1": 0.103921875, + "pass@2": 0.16197342519685046, + "pass@4": 0.23023682836520395, + "pass@8": 0.30135153552356025, + "pass@16": 0.37268679199799903, + "pass@32": 0.4492323936037306, + "pass@64": 0.5398153203078361, + "pass@128": 0.645 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/checkpoint-18010_metrics.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/checkpoint-18010_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e5d52c951ec7ab13811b8f2b46c72315e1a8effc --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/checkpoint-18010_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18010", + "id": { + "answer_accuracy": 0.29479340277777777, + "avg_response_len": 181.97637847222222, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.29479340277777777, + "pass@2": 0.36773587871829233, + "pass@4": 0.4462460452860125, + "pass@8": 0.5262009840352574, + "pass@16": 0.6044375616528503, + "pass@32": 0.6771280018515999, + "pass@64": 0.739727087862316, + "pass@128": 0.7886666666666666 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1554921875, + "pass@2": 0.23125332185039357, + "pass@4": 0.31928074625046815, + "pass@8": 0.4140100984508594, + "pass@16": 0.5149801083462835, + "pass@32": 0.6192187863921861, + "pass@64": 0.7151446923509687, + "pass@128": 0.792 + }, + "2": { + "pass@1": 0.9758125, + "pass@2": 0.9898241879921259, + "pass@4": 0.9966188771091112, + "pass@8": 0.9987892230151045, + "pass@16": 0.9996129477608403, + "pass@32": 0.9999621065354073, + "pass@64": 0.9999999081871598, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4396640625, + "pass@2": 0.5428911171259851, + "pass@4": 0.6294488910761153, + "pass@8": 0.6989894508662072, + "pass@16": 0.7554558408611288, + "pass@32": 0.8016163698395926, + "pass@64": 0.838380708283333, + "pass@128": 0.866 + }, + "4": { + "pass@1": 0.19096875, + "pass@2": 0.2649661663385828, + "pass@4": 0.34128241844769386, + "pass@8": 0.4126170280054176, + "pass@16": 0.4744368674880639, + "pass@32": 0.5248190384718068, + "pass@64": 0.5660038237491893, + "pass@128": 0.602 + }, + "5": { + "pass@1": 0.21321875, + "pass@2": 0.27896961122047237, + "pass@4": 0.34784114266966615, + "pass@8": 0.4178392897058929, + "pass@16": 0.4842250799508667, + "pass@32": 0.5435127339502186, + "pass@64": 0.5949294840510188, + "pass@128": 0.64 + }, + "6": { + "pass@1": 0.1857578125, + "pass@2": 0.2737919537401572, + "pass@4": 0.37808865532433394, + "pass@8": 0.4897762042687177, + "pass@16": 0.597658650681137, + "pass@32": 0.6932805935965465, + "pass@64": 0.7700153321893307, + "pass@128": 0.824 + }, + "7": { + "pass@1": 0.1685390625, + "pass@2": 0.24655708661417308, + "pass@4": 0.3367074599737531, + "pass@8": 0.4351236336050813, + "pass@16": 0.5374185904535973, + "pass@32": 0.6346027164019434, + "pass@64": 0.7164196376246919, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.172109375, + "pass@2": 0.2528591289370078, + "pass@4": 0.34520594394450665, + "pass@8": 0.4442619778086864, + "pass@16": 0.5455363352223178, + "pass@32": 0.6419740686617953, + "pass@64": 0.7269399396377242, + "pass@128": 0.795 + }, + "9": { + "pass@1": 0.151578125, + "pass@2": 0.22851033464566936, + "pass@4": 0.3217402727784024, + "pass@8": 0.42440195059123537, + "pass@16": 0.5306136341112656, + "pass@32": 0.6351656028149096, + "pass@64": 0.7297102646873684, + "pass@128": 0.805 + } + }, + "per_op_accuracy": { + "10": 0.1554921875, + "2": 0.9758125, + "3": 0.4396640625, + "4": 0.19096875, + "5": 0.21321875, + "6": 0.1857578125, + "7": 0.1685390625, + "8": 0.172109375, + "9": 0.151578125 + }, + "per_op_avg_response_len": { + "10": 246.290484375, + "2": 106.310828125, + "3": 140.529609375, + "4": 137.7806328125, + "5": 159.7301953125, + "6": 196.2060234375, + "7": 193.62515625, + "8": 218.4682265625, + "9": 238.84625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 246.290484375, + "2": 106.310828125, + "3": 140.529609375, + "4": 137.7806328125, + "5": 159.7301953125, + "6": 196.2060234375, + "7": 193.62515625, + "8": 218.4682265625, + "9": 238.84625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 339602, + "answer_accuracy": 0.29479340277777777, + "avg_response_len": 181.97637847222222, + "resp_tokens_sum": 209636788, + "pass_at_k": { + "pass@1": 0.29479340277777777, + "pass@2": 0.36773587871829233, + "pass@4": 0.4462460452860125, + "pass@8": 0.5262009840352574, + "pass@16": 0.6044375616528503, + "pass@32": 0.6771280018515999, + "pass@64": 0.739727087862316, + "pass@128": 0.7886666666666666 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1554921875, + "pass@2": 0.23125332185039357, + "pass@4": 0.31928074625046815, + "pass@8": 0.4140100984508594, + "pass@16": 0.5149801083462835, + "pass@32": 0.6192187863921861, + "pass@64": 0.7151446923509687, + "pass@128": 0.792 + }, + "2": { + "pass@1": 0.9758125, + "pass@2": 0.9898241879921259, + "pass@4": 0.9966188771091112, + "pass@8": 0.9987892230151045, + "pass@16": 0.9996129477608403, + "pass@32": 0.9999621065354073, + "pass@64": 0.9999999081871598, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4396640625, + "pass@2": 0.5428911171259851, + "pass@4": 0.6294488910761153, + "pass@8": 0.6989894508662072, + "pass@16": 0.7554558408611288, + "pass@32": 0.8016163698395926, + "pass@64": 0.838380708283333, + "pass@128": 0.866 + }, + "4": { + "pass@1": 0.19096875, + "pass@2": 0.2649661663385828, + "pass@4": 0.34128241844769386, + "pass@8": 0.4126170280054176, + "pass@16": 0.4744368674880639, + "pass@32": 0.5248190384718068, + "pass@64": 0.5660038237491893, + "pass@128": 0.602 + }, + "5": { + "pass@1": 0.21321875, + "pass@2": 0.27896961122047237, + "pass@4": 0.34784114266966615, + "pass@8": 0.4178392897058929, + "pass@16": 0.4842250799508667, + "pass@32": 0.5435127339502186, + "pass@64": 0.5949294840510188, + "pass@128": 0.64 + }, + "6": { + "pass@1": 0.1857578125, + "pass@2": 0.2737919537401572, + "pass@4": 0.37808865532433394, + "pass@8": 0.4897762042687177, + "pass@16": 0.597658650681137, + "pass@32": 0.6932805935965465, + "pass@64": 0.7700153321893307, + "pass@128": 0.824 + }, + "7": { + "pass@1": 0.1685390625, + "pass@2": 0.24655708661417308, + "pass@4": 0.3367074599737531, + "pass@8": 0.4351236336050813, + "pass@16": 0.5374185904535973, + "pass@32": 0.6346027164019434, + "pass@64": 0.7164196376246919, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.172109375, + "pass@2": 0.2528591289370078, + "pass@4": 0.34520594394450665, + "pass@8": 0.4442619778086864, + "pass@16": 0.5455363352223178, + "pass@32": 0.6419740686617953, + "pass@64": 0.7269399396377242, + "pass@128": 0.795 + }, + "9": { + "pass@1": 0.151578125, + "pass@2": 0.22851033464566936, + "pass@4": 0.3217402727784024, + "pass@8": 0.42440195059123537, + "pass@16": 0.5306136341112656, + "pass@32": 0.6351656028149096, + "pass@64": 0.7297102646873684, + "pass@128": 0.805 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.1294953125, + "avg_response_len": 312.61559296875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.1294953125, + "pass@2": 0.2012045521653559, + "pass@4": 0.28331449478815685, + "pass@8": 0.36547881851723363, + "pass@16": 0.44566474148877533, + "pass@32": 0.5298446124526037, + "pass@64": 0.6208295607729596, + "pass@128": 0.7115 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1496875, + "pass@2": 0.22677423720472428, + "pass@4": 0.31548245594300645, + "pass@8": 0.40792302869924524, + "pass@16": 0.5038893439918694, + "pass@32": 0.6056412796476469, + "pass@64": 0.705294172277044, + "pass@128": 0.788 + }, + "12": { + "pass@1": 0.1588671875, + "pass@2": 0.24316633858267728, + "pass@4": 0.33503074212598405, + "pass@8": 0.42464395485173545, + "pass@16": 0.5149391013811805, + "pass@32": 0.6107460586754517, + "pass@64": 0.7067709068632771, + "pass@128": 0.79 + }, + "13": { + "pass@1": 0.1332109375, + "pass@2": 0.20613201279527582, + "pass@4": 0.29053240851143575, + "pass@8": 0.37651105568431853, + "pass@16": 0.462729661322706, + "pass@32": 0.5542692252638413, + "pass@64": 0.6509012984770194, + "pass@128": 0.743 + }, + "14": { + "pass@1": 0.1372265625, + "pass@2": 0.21256791338582673, + "pass@4": 0.298558461942257, + "pass@8": 0.3844381214258216, + "pass@16": 0.46904045611242967, + "pass@32": 0.5576328586590467, + "pass@64": 0.650103354331368, + "pass@128": 0.735 + }, + "15": { + "pass@1": 0.1273828125, + "pass@2": 0.1969920029527562, + "pass@4": 0.27652288667041564, + "pass@8": 0.3570955498068145, + "pass@16": 0.4378015172981221, + "pass@32": 0.5256541836080247, + "pass@64": 0.6239085326704196, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.133984375, + "pass@2": 0.2085909202755905, + "pass@4": 0.2937789566929127, + "pass@8": 0.3780276907660483, + "pass@16": 0.4575149774892016, + "pass@32": 0.5376840163648835, + "pass@64": 0.6245121631743655, + "pass@128": 0.713 + }, + "17": { + "pass@1": 0.1233046875, + "pass@2": 0.19373646653543297, + "pass@4": 0.27461433539557545, + "pass@8": 0.3545213187799682, + "pass@16": 0.42935742908421576, + "pass@32": 0.5052463320432403, + "pass@64": 0.5884199239798831, + "pass@128": 0.675 + }, + "18": { + "pass@1": 0.1141171875, + "pass@2": 0.18002226870078727, + "pass@4": 0.25710795088113975, + "pass@8": 0.3341134516466114, + "pass@16": 0.4065938519271321, + "pass@32": 0.4823872609283686, + "pass@64": 0.5689302146297485, + "pass@128": 0.663 + }, + "19": { + "pass@1": 0.107390625, + "pass@2": 0.1714498031496063, + "pass@4": 0.24558504030746126, + "pass@8": 0.3178302257160668, + "pass@16": 0.3855530195651152, + "pass@32": 0.45745794591073363, + "pass@64": 0.5423795690102908, + "pass@128": 0.641 + }, + "20": { + "pass@1": 0.10978125, + "pass@2": 0.17261355807086629, + "pass@4": 0.24593170941132292, + "pass@8": 0.3196837877955671, + "pass@16": 0.38922805671566524, + "pass@32": 0.46172696342486735, + "pass@64": 0.5470754723162058, + "pass@128": 0.646 + } + }, + "per_op_accuracy": { + "11": 0.1496875, + "12": 0.1588671875, + "13": 0.1332109375, + "14": 0.1372265625, + "15": 0.1273828125, + "16": 0.133984375, + "17": 0.1233046875, + "18": 0.1141171875, + "19": 0.107390625, + "20": 0.10978125 + }, + "per_op_avg_response_len": { + "11": 263.8062578125, + "12": 271.610578125, + "13": 280.06665625, + "14": 299.74690625, + "15": 312.558453125, + "16": 329.6247109375, + "17": 333.3729375, + "18": 337.991953125, + "19": 343.2403046875, + "20": 354.137171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 263.8062578125, + "12": 271.610578125, + "13": 280.06665625, + "14": 299.74690625, + "15": 312.558453125, + "16": 329.6247109375, + "17": 333.3729375, + "18": 337.991953125, + "19": 343.2403046875, + "20": 354.137171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 165754, + "answer_accuracy": 0.1294953125, + "avg_response_len": 312.61559296875, + "resp_tokens_sum": 400147959, + "pass_at_k": { + "pass@1": 0.1294953125, + "pass@2": 0.2012045521653559, + "pass@4": 0.28331449478815685, + "pass@8": 0.36547881851723363, + "pass@16": 0.44566474148877533, + "pass@32": 0.5298446124526037, + "pass@64": 0.6208295607729596, + "pass@128": 0.7115 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1496875, + "pass@2": 0.22677423720472428, + "pass@4": 0.31548245594300645, + "pass@8": 0.40792302869924524, + "pass@16": 0.5038893439918694, + "pass@32": 0.6056412796476469, + "pass@64": 0.705294172277044, + "pass@128": 0.788 + }, + "12": { + "pass@1": 0.1588671875, + "pass@2": 0.24316633858267728, + "pass@4": 0.33503074212598405, + "pass@8": 0.42464395485173545, + "pass@16": 0.5149391013811805, + "pass@32": 0.6107460586754517, + "pass@64": 0.7067709068632771, + "pass@128": 0.79 + }, + "13": { + "pass@1": 0.1332109375, + "pass@2": 0.20613201279527582, + "pass@4": 0.29053240851143575, + "pass@8": 0.37651105568431853, + "pass@16": 0.462729661322706, + "pass@32": 0.5542692252638413, + "pass@64": 0.6509012984770194, + "pass@128": 0.743 + }, + "14": { + "pass@1": 0.1372265625, + "pass@2": 0.21256791338582673, + "pass@4": 0.298558461942257, + "pass@8": 0.3844381214258216, + "pass@16": 0.46904045611242967, + "pass@32": 0.5576328586590467, + "pass@64": 0.650103354331368, + "pass@128": 0.735 + }, + "15": { + "pass@1": 0.1273828125, + "pass@2": 0.1969920029527562, + "pass@4": 0.27652288667041564, + "pass@8": 0.3570955498068145, + "pass@16": 0.4378015172981221, + "pass@32": 0.5256541836080247, + "pass@64": 0.6239085326704196, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.133984375, + "pass@2": 0.2085909202755905, + "pass@4": 0.2937789566929127, + "pass@8": 0.3780276907660483, + "pass@16": 0.4575149774892016, + "pass@32": 0.5376840163648835, + "pass@64": 0.6245121631743655, + "pass@128": 0.713 + }, + "17": { + "pass@1": 0.1233046875, + "pass@2": 0.19373646653543297, + "pass@4": 0.27461433539557545, + "pass@8": 0.3545213187799682, + "pass@16": 0.42935742908421576, + "pass@32": 0.5052463320432403, + "pass@64": 0.5884199239798831, + "pass@128": 0.675 + }, + "18": { + "pass@1": 0.1141171875, + "pass@2": 0.18002226870078727, + "pass@4": 0.25710795088113975, + "pass@8": 0.3341134516466114, + "pass@16": 0.4065938519271321, + "pass@32": 0.4823872609283686, + "pass@64": 0.5689302146297485, + "pass@128": 0.663 + }, + "19": { + "pass@1": 0.107390625, + "pass@2": 0.1714498031496063, + "pass@4": 0.24558504030746126, + "pass@8": 0.3178302257160668, + "pass@16": 0.3855530195651152, + "pass@32": 0.45745794591073363, + "pass@64": 0.5423795690102908, + "pass@128": 0.641 + }, + "20": { + "pass@1": 0.10978125, + "pass@2": 0.17261355807086629, + "pass@4": 0.24593170941132292, + "pass@8": 0.3196837877955671, + "pass@16": 0.38922805671566524, + "pass@32": 0.46172696342486735, + "pass@64": 0.5470754723162058, + "pass@128": 0.646 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.20779440789473685, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.20779440789473685, + "pass@2": 0.28008781211146766, + "pass@4": 0.360492597655539, + "pass@8": 0.44161037060471475, + "pass@16": 0.5208729194612487, + "pass@32": 0.5996104284837255, + "pass@64": 0.6771494420258143, + "pass@128": 0.7480526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1554921875, + "pass@2": 0.23125332185039357, + "pass@4": 0.31928074625046815, + "pass@8": 0.4140100984508594, + "pass@16": 0.5149801083462835, + "pass@32": 0.6192187863921861, + "pass@64": 0.7151446923509687, + "pass@128": 0.792 + }, + "2": { + "pass@1": 0.9758125, + "pass@2": 0.9898241879921259, + "pass@4": 0.9966188771091112, + "pass@8": 0.9987892230151045, + "pass@16": 0.9996129477608403, + "pass@32": 0.9999621065354073, + "pass@64": 0.9999999081871598, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4396640625, + "pass@2": 0.5428911171259851, + "pass@4": 0.6294488910761153, + "pass@8": 0.6989894508662072, + "pass@16": 0.7554558408611288, + "pass@32": 0.8016163698395926, + "pass@64": 0.838380708283333, + "pass@128": 0.866 + }, + "4": { + "pass@1": 0.19096875, + "pass@2": 0.2649661663385828, + "pass@4": 0.34128241844769386, + "pass@8": 0.4126170280054176, + "pass@16": 0.4744368674880639, + "pass@32": 0.5248190384718068, + "pass@64": 0.5660038237491893, + "pass@128": 0.602 + }, + "5": { + "pass@1": 0.21321875, + "pass@2": 0.27896961122047237, + "pass@4": 0.34784114266966615, + "pass@8": 0.4178392897058929, + "pass@16": 0.4842250799508667, + "pass@32": 0.5435127339502186, + "pass@64": 0.5949294840510188, + "pass@128": 0.64 + }, + "6": { + "pass@1": 0.1857578125, + "pass@2": 0.2737919537401572, + "pass@4": 0.37808865532433394, + "pass@8": 0.4897762042687177, + "pass@16": 0.597658650681137, + "pass@32": 0.6932805935965465, + "pass@64": 0.7700153321893307, + "pass@128": 0.824 + }, + "7": { + "pass@1": 0.1685390625, + "pass@2": 0.24655708661417308, + "pass@4": 0.3367074599737531, + "pass@8": 0.4351236336050813, + "pass@16": 0.5374185904535973, + "pass@32": 0.6346027164019434, + "pass@64": 0.7164196376246919, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.172109375, + "pass@2": 0.2528591289370078, + "pass@4": 0.34520594394450665, + "pass@8": 0.4442619778086864, + "pass@16": 0.5455363352223178, + "pass@32": 0.6419740686617953, + "pass@64": 0.7269399396377242, + "pass@128": 0.795 + }, + "9": { + "pass@1": 0.151578125, + "pass@2": 0.22851033464566936, + "pass@4": 0.3217402727784024, + "pass@8": 0.42440195059123537, + "pass@16": 0.5306136341112656, + "pass@32": 0.6351656028149096, + "pass@64": 0.7297102646873684, + "pass@128": 0.805 + }, + "11": { + "pass@1": 0.1496875, + "pass@2": 0.22677423720472428, + "pass@4": 0.31548245594300645, + "pass@8": 0.40792302869924524, + "pass@16": 0.5038893439918694, + "pass@32": 0.6056412796476469, + "pass@64": 0.705294172277044, + "pass@128": 0.788 + }, + "12": { + "pass@1": 0.1588671875, + "pass@2": 0.24316633858267728, + "pass@4": 0.33503074212598405, + "pass@8": 0.42464395485173545, + "pass@16": 0.5149391013811805, + "pass@32": 0.6107460586754517, + "pass@64": 0.7067709068632771, + "pass@128": 0.79 + }, + "13": { + "pass@1": 0.1332109375, + "pass@2": 0.20613201279527582, + "pass@4": 0.29053240851143575, + "pass@8": 0.37651105568431853, + "pass@16": 0.462729661322706, + "pass@32": 0.5542692252638413, + "pass@64": 0.6509012984770194, + "pass@128": 0.743 + }, + "14": { + "pass@1": 0.1372265625, + "pass@2": 0.21256791338582673, + "pass@4": 0.298558461942257, + "pass@8": 0.3844381214258216, + "pass@16": 0.46904045611242967, + "pass@32": 0.5576328586590467, + "pass@64": 0.650103354331368, + "pass@128": 0.735 + }, + "15": { + "pass@1": 0.1273828125, + "pass@2": 0.1969920029527562, + "pass@4": 0.27652288667041564, + "pass@8": 0.3570955498068145, + "pass@16": 0.4378015172981221, + "pass@32": 0.5256541836080247, + "pass@64": 0.6239085326704196, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.133984375, + "pass@2": 0.2085909202755905, + "pass@4": 0.2937789566929127, + "pass@8": 0.3780276907660483, + "pass@16": 0.4575149774892016, + "pass@32": 0.5376840163648835, + "pass@64": 0.6245121631743655, + "pass@128": 0.713 + }, + "17": { + "pass@1": 0.1233046875, + "pass@2": 0.19373646653543297, + "pass@4": 0.27461433539557545, + "pass@8": 0.3545213187799682, + "pass@16": 0.42935742908421576, + "pass@32": 0.5052463320432403, + "pass@64": 0.5884199239798831, + "pass@128": 0.675 + }, + "18": { + "pass@1": 0.1141171875, + "pass@2": 0.18002226870078727, + "pass@4": 0.25710795088113975, + "pass@8": 0.3341134516466114, + "pass@16": 0.4065938519271321, + "pass@32": 0.4823872609283686, + "pass@64": 0.5689302146297485, + "pass@128": 0.663 + }, + "19": { + "pass@1": 0.107390625, + "pass@2": 0.1714498031496063, + "pass@4": 0.24558504030746126, + "pass@8": 0.3178302257160668, + "pass@16": 0.3855530195651152, + "pass@32": 0.45745794591073363, + "pass@64": 0.5423795690102908, + "pass@128": 0.641 + }, + "20": { + "pass@1": 0.10978125, + "pass@2": 0.17261355807086629, + "pass@4": 0.24593170941132292, + "pass@8": 0.3196837877955671, + "pass@16": 0.38922805671566524, + "pass@32": 0.46172696342486735, + "pass@64": 0.5470754723162058, + "pass@128": 0.646 + } + }, + "per_op_accuracy": { + "10": 0.1554921875, + "2": 0.9758125, + "3": 0.4396640625, + "4": 0.19096875, + "5": 0.21321875, + "6": 0.1857578125, + "7": 0.1685390625, + "8": 0.172109375, + "9": 0.151578125, + "11": 0.1496875, + "12": 0.1588671875, + "13": 0.1332109375, + "14": 0.1372265625, + "15": 0.1273828125, + "16": 0.133984375, + "17": 0.1233046875, + "18": 0.1141171875, + "19": 0.107390625, + "20": 0.10978125 + }, + "per_op_avg_response_len": { + "10": 246.290484375, + "2": 106.310828125, + "3": 140.529609375, + "4": 137.7806328125, + "5": 159.7301953125, + "6": 196.2060234375, + "7": 193.62515625, + "8": 218.4682265625, + "9": 238.84625, + "11": 263.8062578125, + "12": 271.610578125, + "13": 280.06665625, + "14": 299.74690625, + "15": 312.558453125, + "16": 329.6247109375, + "17": 333.3729375, + "18": 337.991953125, + "19": 343.2403046875, + "20": 354.137171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 246.290484375, + "2": 106.310828125, + "3": 140.529609375, + "4": 137.7806328125, + "5": 159.7301953125, + "6": 196.2060234375, + "7": 193.62515625, + "8": 218.4682265625, + "9": 238.84625, + "11": 263.8062578125, + "12": 271.610578125, + "13": 280.06665625, + "14": 299.74690625, + "15": 312.558453125, + "16": 329.6247109375, + "17": 333.3729375, + "18": 337.991953125, + "19": 343.2403046875, + "20": 354.137171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 505356, + "answer_accuracy": 0.20779440789473685, + "avg_response_len": 250.7338597861842, + "resp_tokens_sum": 609784747, + "pass_at_k": { + "pass@1": 0.20779440789473685, + "pass@2": 0.28008781211146766, + "pass@4": 0.360492597655539, + "pass@8": 0.44161037060471475, + "pass@16": 0.5208729194612487, + "pass@32": 0.5996104284837255, + "pass@64": 0.6771494420258143, + "pass@128": 0.7480526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1554921875, + "pass@2": 0.23125332185039357, + "pass@4": 0.31928074625046815, + "pass@8": 0.4140100984508594, + "pass@16": 0.5149801083462835, + "pass@32": 0.6192187863921861, + "pass@64": 0.7151446923509687, + "pass@128": 0.792 + }, + "2": { + "pass@1": 0.9758125, + "pass@2": 0.9898241879921259, + "pass@4": 0.9966188771091112, + "pass@8": 0.9987892230151045, + "pass@16": 0.9996129477608403, + "pass@32": 0.9999621065354073, + "pass@64": 0.9999999081871598, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4396640625, + "pass@2": 0.5428911171259851, + "pass@4": 0.6294488910761153, + "pass@8": 0.6989894508662072, + "pass@16": 0.7554558408611288, + "pass@32": 0.8016163698395926, + "pass@64": 0.838380708283333, + "pass@128": 0.866 + }, + "4": { + "pass@1": 0.19096875, + "pass@2": 0.2649661663385828, + "pass@4": 0.34128241844769386, + "pass@8": 0.4126170280054176, + "pass@16": 0.4744368674880639, + "pass@32": 0.5248190384718068, + "pass@64": 0.5660038237491893, + "pass@128": 0.602 + }, + "5": { + "pass@1": 0.21321875, + "pass@2": 0.27896961122047237, + "pass@4": 0.34784114266966615, + "pass@8": 0.4178392897058929, + "pass@16": 0.4842250799508667, + "pass@32": 0.5435127339502186, + "pass@64": 0.5949294840510188, + "pass@128": 0.64 + }, + "6": { + "pass@1": 0.1857578125, + "pass@2": 0.2737919537401572, + "pass@4": 0.37808865532433394, + "pass@8": 0.4897762042687177, + "pass@16": 0.597658650681137, + "pass@32": 0.6932805935965465, + "pass@64": 0.7700153321893307, + "pass@128": 0.824 + }, + "7": { + "pass@1": 0.1685390625, + "pass@2": 0.24655708661417308, + "pass@4": 0.3367074599737531, + "pass@8": 0.4351236336050813, + "pass@16": 0.5374185904535973, + "pass@32": 0.6346027164019434, + "pass@64": 0.7164196376246919, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.172109375, + "pass@2": 0.2528591289370078, + "pass@4": 0.34520594394450665, + "pass@8": 0.4442619778086864, + "pass@16": 0.5455363352223178, + "pass@32": 0.6419740686617953, + "pass@64": 0.7269399396377242, + "pass@128": 0.795 + }, + "9": { + "pass@1": 0.151578125, + "pass@2": 0.22851033464566936, + "pass@4": 0.3217402727784024, + "pass@8": 0.42440195059123537, + "pass@16": 0.5306136341112656, + "pass@32": 0.6351656028149096, + "pass@64": 0.7297102646873684, + "pass@128": 0.805 + }, + "11": { + "pass@1": 0.1496875, + "pass@2": 0.22677423720472428, + "pass@4": 0.31548245594300645, + "pass@8": 0.40792302869924524, + "pass@16": 0.5038893439918694, + "pass@32": 0.6056412796476469, + "pass@64": 0.705294172277044, + "pass@128": 0.788 + }, + "12": { + "pass@1": 0.1588671875, + "pass@2": 0.24316633858267728, + "pass@4": 0.33503074212598405, + "pass@8": 0.42464395485173545, + "pass@16": 0.5149391013811805, + "pass@32": 0.6107460586754517, + "pass@64": 0.7067709068632771, + "pass@128": 0.79 + }, + "13": { + "pass@1": 0.1332109375, + "pass@2": 0.20613201279527582, + "pass@4": 0.29053240851143575, + "pass@8": 0.37651105568431853, + "pass@16": 0.462729661322706, + "pass@32": 0.5542692252638413, + "pass@64": 0.6509012984770194, + "pass@128": 0.743 + }, + "14": { + "pass@1": 0.1372265625, + "pass@2": 0.21256791338582673, + "pass@4": 0.298558461942257, + "pass@8": 0.3844381214258216, + "pass@16": 0.46904045611242967, + "pass@32": 0.5576328586590467, + "pass@64": 0.650103354331368, + "pass@128": 0.735 + }, + "15": { + "pass@1": 0.1273828125, + "pass@2": 0.1969920029527562, + "pass@4": 0.27652288667041564, + "pass@8": 0.3570955498068145, + "pass@16": 0.4378015172981221, + "pass@32": 0.5256541836080247, + "pass@64": 0.6239085326704196, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.133984375, + "pass@2": 0.2085909202755905, + "pass@4": 0.2937789566929127, + "pass@8": 0.3780276907660483, + "pass@16": 0.4575149774892016, + "pass@32": 0.5376840163648835, + "pass@64": 0.6245121631743655, + "pass@128": 0.713 + }, + "17": { + "pass@1": 0.1233046875, + "pass@2": 0.19373646653543297, + "pass@4": 0.27461433539557545, + "pass@8": 0.3545213187799682, + "pass@16": 0.42935742908421576, + "pass@32": 0.5052463320432403, + "pass@64": 0.5884199239798831, + "pass@128": 0.675 + }, + "18": { + "pass@1": 0.1141171875, + "pass@2": 0.18002226870078727, + "pass@4": 0.25710795088113975, + "pass@8": 0.3341134516466114, + "pass@16": 0.4065938519271321, + "pass@32": 0.4823872609283686, + "pass@64": 0.5689302146297485, + "pass@128": 0.663 + }, + "19": { + "pass@1": 0.107390625, + "pass@2": 0.1714498031496063, + "pass@4": 0.24558504030746126, + "pass@8": 0.3178302257160668, + "pass@16": 0.3855530195651152, + "pass@32": 0.45745794591073363, + "pass@64": 0.5423795690102908, + "pass@128": 0.641 + }, + "20": { + "pass@1": 0.10978125, + "pass@2": 0.17261355807086629, + "pass@4": 0.24593170941132292, + "pass@8": 0.3196837877955671, + "pass@16": 0.38922805671566524, + "pass@32": 0.46172696342486735, + "pass@64": 0.5470754723162058, + "pass@128": 0.646 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.csv b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..cfbde1987045e0b03827c68c23b9b4fec1309828 --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18010,0.29479340277777777,0.1294953125,0.20779440789473685,0.0,0.0,0.0,181.97637847222222,312.61559296875,1152000,1280000,2432000 diff --git a/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.json b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2beb2deb8d297304419a7d6fe568a8f53a1b2a --- /dev/null +++ b/SDAR-100M-0.9zoo_op2-20+0.1teacher_op2/context/teachers_in_school/low_confidence_static/block8/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18010", + "id_acc": 0.29479340277777777, + "ood_acc": 0.1294953125, + "total_acc": 0.20779440789473685, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 181.97637847222222, + "ood_avg_resp_len": 312.61559296875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 505356, + "answer_accuracy": 0.20779440789473685, + "avg_response_len": 250.7338597861842, + "resp_tokens_sum": 609784747, + "pass_at_k": { + "pass@1": 0.20779440789473685, + "pass@2": 0.28008781211146766, + "pass@4": 0.360492597655539, + "pass@8": 0.44161037060471475, + "pass@16": 0.5208729194612487, + "pass@32": 0.5996104284837255, + "pass@64": 0.6771494420258143, + "pass@128": 0.7480526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1554921875, + "pass@2": 0.23125332185039357, + "pass@4": 0.31928074625046815, + "pass@8": 0.4140100984508594, + "pass@16": 0.5149801083462835, + "pass@32": 0.6192187863921861, + "pass@64": 0.7151446923509687, + "pass@128": 0.792 + }, + "2": { + "pass@1": 0.9758125, + "pass@2": 0.9898241879921259, + "pass@4": 0.9966188771091112, + "pass@8": 0.9987892230151045, + "pass@16": 0.9996129477608403, + "pass@32": 0.9999621065354073, + "pass@64": 0.9999999081871598, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.4396640625, + "pass@2": 0.5428911171259851, + "pass@4": 0.6294488910761153, + "pass@8": 0.6989894508662072, + "pass@16": 0.7554558408611288, + "pass@32": 0.8016163698395926, + "pass@64": 0.838380708283333, + "pass@128": 0.866 + }, + "4": { + "pass@1": 0.19096875, + "pass@2": 0.2649661663385828, + "pass@4": 0.34128241844769386, + "pass@8": 0.4126170280054176, + "pass@16": 0.4744368674880639, + "pass@32": 0.5248190384718068, + "pass@64": 0.5660038237491893, + "pass@128": 0.602 + }, + "5": { + "pass@1": 0.21321875, + "pass@2": 0.27896961122047237, + "pass@4": 0.34784114266966615, + "pass@8": 0.4178392897058929, + "pass@16": 0.4842250799508667, + "pass@32": 0.5435127339502186, + "pass@64": 0.5949294840510188, + "pass@128": 0.64 + }, + "6": { + "pass@1": 0.1857578125, + "pass@2": 0.2737919537401572, + "pass@4": 0.37808865532433394, + "pass@8": 0.4897762042687177, + "pass@16": 0.597658650681137, + "pass@32": 0.6932805935965465, + "pass@64": 0.7700153321893307, + "pass@128": 0.824 + }, + "7": { + "pass@1": 0.1685390625, + "pass@2": 0.24655708661417308, + "pass@4": 0.3367074599737531, + "pass@8": 0.4351236336050813, + "pass@16": 0.5374185904535973, + "pass@32": 0.6346027164019434, + "pass@64": 0.7164196376246919, + "pass@128": 0.774 + }, + "8": { + "pass@1": 0.172109375, + "pass@2": 0.2528591289370078, + "pass@4": 0.34520594394450665, + "pass@8": 0.4442619778086864, + "pass@16": 0.5455363352223178, + "pass@32": 0.6419740686617953, + "pass@64": 0.7269399396377242, + "pass@128": 0.795 + }, + "9": { + "pass@1": 0.151578125, + "pass@2": 0.22851033464566936, + "pass@4": 0.3217402727784024, + "pass@8": 0.42440195059123537, + "pass@16": 0.5306136341112656, + "pass@32": 0.6351656028149096, + "pass@64": 0.7297102646873684, + "pass@128": 0.805 + }, + "11": { + "pass@1": 0.1496875, + "pass@2": 0.22677423720472428, + "pass@4": 0.31548245594300645, + "pass@8": 0.40792302869924524, + "pass@16": 0.5038893439918694, + "pass@32": 0.6056412796476469, + "pass@64": 0.705294172277044, + "pass@128": 0.788 + }, + "12": { + "pass@1": 0.1588671875, + "pass@2": 0.24316633858267728, + "pass@4": 0.33503074212598405, + "pass@8": 0.42464395485173545, + "pass@16": 0.5149391013811805, + "pass@32": 0.6107460586754517, + "pass@64": 0.7067709068632771, + "pass@128": 0.79 + }, + "13": { + "pass@1": 0.1332109375, + "pass@2": 0.20613201279527582, + "pass@4": 0.29053240851143575, + "pass@8": 0.37651105568431853, + "pass@16": 0.462729661322706, + "pass@32": 0.5542692252638413, + "pass@64": 0.6509012984770194, + "pass@128": 0.743 + }, + "14": { + "pass@1": 0.1372265625, + "pass@2": 0.21256791338582673, + "pass@4": 0.298558461942257, + "pass@8": 0.3844381214258216, + "pass@16": 0.46904045611242967, + "pass@32": 0.5576328586590467, + "pass@64": 0.650103354331368, + "pass@128": 0.735 + }, + "15": { + "pass@1": 0.1273828125, + "pass@2": 0.1969920029527562, + "pass@4": 0.27652288667041564, + "pass@8": 0.3570955498068145, + "pass@16": 0.4378015172981221, + "pass@32": 0.5256541836080247, + "pass@64": 0.6239085326704196, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.133984375, + "pass@2": 0.2085909202755905, + "pass@4": 0.2937789566929127, + "pass@8": 0.3780276907660483, + "pass@16": 0.4575149774892016, + "pass@32": 0.5376840163648835, + "pass@64": 0.6245121631743655, + "pass@128": 0.713 + }, + "17": { + "pass@1": 0.1233046875, + "pass@2": 0.19373646653543297, + "pass@4": 0.27461433539557545, + "pass@8": 0.3545213187799682, + "pass@16": 0.42935742908421576, + "pass@32": 0.5052463320432403, + "pass@64": 0.5884199239798831, + "pass@128": 0.675 + }, + "18": { + "pass@1": 0.1141171875, + "pass@2": 0.18002226870078727, + "pass@4": 0.25710795088113975, + "pass@8": 0.3341134516466114, + "pass@16": 0.4065938519271321, + "pass@32": 0.4823872609283686, + "pass@64": 0.5689302146297485, + "pass@128": 0.663 + }, + "19": { + "pass@1": 0.107390625, + "pass@2": 0.1714498031496063, + "pass@4": 0.24558504030746126, + "pass@8": 0.3178302257160668, + "pass@16": 0.3855530195651152, + "pass@32": 0.45745794591073363, + "pass@64": 0.5423795690102908, + "pass@128": 0.641 + }, + "20": { + "pass@1": 0.10978125, + "pass@2": 0.17261355807086629, + "pass@4": 0.24593170941132292, + "pass@8": 0.3196837877955671, + "pass@16": 0.38922805671566524, + "pass@32": 0.46172696342486735, + "pass@64": 0.5470754723162058, + "pass@128": 0.646 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/autoregressive/checkpoint-21350_metrics.json b/SDAR-100M-ep1-block2/composition/autoregressive/checkpoint-21350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d09f398c720d5bffafd70cd5666aee3e9a64c0fa --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/autoregressive/checkpoint-21350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21350", + "id": { + "answer_accuracy": 0.9110954861111111, + "avg_response_len": 199.92764930555555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9110954861111111, + "pass@2": 0.9337374644575565, + "pass@4": 0.9486874978023538, + "pass@8": 0.9596063971459827, + "pass@16": 0.9682263933943015, + "pass@32": 0.9751791178195751, + "pass@64": 0.9806675537357076, + "pass@128": 0.9848888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.80821875, + "pass@2": 0.8611038385826778, + "pass@4": 0.893551319928759, + "pass@8": 0.9152204428848738, + "pass@16": 0.9319724334620572, + "pass@32": 0.9465334079108148, + "pass@64": 0.9587946628838566, + "pass@128": 0.968 + }, + "2": { + "pass@1": 0.9999453125, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998171875, + "pass@2": 0.9990873523622046, + "pass@4": 0.9995161725721784, + "pass@8": 0.9997889208055989, + "pass@16": 0.9999604948476541, + "pass@32": 0.9999991088495628, + "pass@64": 0.9999999999731547, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.976078125, + "pass@2": 0.9812984744094486, + "pass@4": 0.98483539848144, + "pass@8": 0.9875219111336576, + "pass@16": 0.989539551379835, + "pass@32": 0.9911053570965834, + "pass@64": 0.9924939351791154, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.8609609375, + "pass@2": 0.8890690206692917, + "pass@4": 0.9095421017060369, + "pass@8": 0.9254586695605911, + "pass@16": 0.9392363307896897, + "pass@32": 0.9515155455110998, + "pass@64": 0.9621924392468406, + "pass@128": 0.971 + }, + "6": { + "pass@1": 0.959984375, + "pass@2": 0.9701747047244096, + "pass@4": 0.9773701526059242, + "pass@8": 0.9827041747056208, + "pass@16": 0.9867273186619663, + "pass@32": 0.9897881513486811, + "pass@64": 0.9921325328624394, + "pass@128": 0.994 + }, + "7": { + "pass@1": 0.9054921875, + "pass@2": 0.9316528051181108, + "pass@4": 0.9482259476003001, + "pass@8": 0.9600481131322273, + "pass@16": 0.9692704251651028, + "pass@32": 0.9757960519218113, + "pass@64": 0.9797622739854073, + "pass@128": 0.982 + }, + "8": { + "pass@1": 0.8637265625, + "pass@2": 0.9008845964566933, + "pass@4": 0.9261050230596173, + "pass@8": 0.944664751294384, + "pass@16": 0.958480101133846, + "pass@32": 0.9681205229698595, + "pass@64": 0.9740102542149567, + "pass@128": 0.977 + }, + "9": { + "pass@1": 0.82728125, + "pass@2": 0.8703665108267724, + "pass@4": 0.899041364266967, + "pass@8": 0.9210505907968902, + "pass@16": 0.9388508851085343, + "pass@32": 0.953753914767747, + "pass@64": 0.9666218852755756, + "pass@128": 0.978 + } + }, + "per_op_accuracy": { + "10": 0.80821875, + "2": 0.9999453125, + "3": 0.998171875, + "4": 0.976078125, + "5": 0.8609609375, + "6": 0.959984375, + "7": 0.9054921875, + "8": 0.8637265625, + "9": 0.82728125 + }, + "per_op_avg_response_len": { + "10": 279.956171875, + "2": 115.5694375, + "3": 144.7300234375, + "4": 159.83084375, + "5": 173.175125, + "6": 199.1570859375, + "7": 223.978734375, + "8": 243.46984375, + "9": 259.481578125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.956171875, + "2": 115.5694375, + "3": 144.7300234375, + "4": 159.83084375, + "5": 173.175125, + "6": 199.1570859375, + "7": 223.978734375, + "8": 243.46984375, + "9": 259.481578125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 345770, + "answer_accuracy": 0.912302642688281, + "avg_response_len": 190.5781408308004, + "resp_tokens_sum": 72230640, + "pass_at_k": { + "pass@1": 0.912302642688281, + "pass@2": 0.935716271835704, + "pass@4": 0.9509430944992058, + "pass@8": 0.9618256655171101, + "pass@16": 0.9703981162132057, + "pass@32": 0.9772026155719008, + "pass@64": 0.9824919707333204, + "pass@128": 0.9861533265788585 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8190680309734514, + "pass@2": 0.8758038783592319, + "pass@4": 0.9097118844657693, + "pass@8": 0.9301279923292359, + "pass@16": 0.945463319652103, + "pass@32": 0.9597276818955355, + "pass@64": 0.971984946225041, + "pass@128": 0.9793510324483776 + }, + "2": { + "pass@1": 0.9998539719626168, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974197247706422, + "pass@2": 0.9979062071082859, + "pass@4": 0.9985759549551719, + "pass@8": 0.9993549275868031, + "pass@16": 0.9998791891503424, + "pass@32": 0.999997274769305, + "pass@64": 0.999999999917904, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9796875, + "pass@2": 0.9833226962352362, + "pass@4": 0.9855182915612108, + "pass@8": 0.9877684577554199, + "pass@16": 0.9898908624050529, + "pass@32": 0.9915912787745874, + "pass@64": 0.9929543613492345, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8453621031746031, + "pass@2": 0.878759686289214, + "pass@4": 0.9038273906237906, + "pass@8": 0.9236315572169654, + "pass@16": 0.9401206361112525, + "pass@32": 0.9527244490497119, + "pass@64": 0.9623853430676196, + "pass@128": 0.9682539682539683 + }, + "6": { + "pass@1": 0.9475580089820359, + "pass@2": 0.9595694192324015, + "pass@4": 0.9684871766590554, + "pass@8": 0.9760311732040033, + "pass@16": 0.9819037036422803, + "pass@32": 0.9859049285559663, + "pass@64": 0.9889859412882993, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9068047337278107, + "pass@2": 0.932142305945115, + "pass@4": 0.947185020234157, + "pass@8": 0.9582908289110658, + "pass@16": 0.9681797653463373, + "pass@32": 0.9767144305121371, + "pass@64": 0.9830296516855794, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8812349759615384, + "pass@2": 0.9134710024227741, + "pass@4": 0.9344750253574073, + "pass@8": 0.9487608950939497, + "pass@16": 0.9589510177536014, + "pass@32": 0.9657383512461981, + "pass@64": 0.9703603198365051, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.8417913732394366, + "pass@2": 0.8856545275590555, + "pass@4": 0.9142673059353488, + "pass@8": 0.9349954551526746, + "pass@16": 0.9509292121287449, + "pass@32": 0.9633821177411638, + "pass@64": 0.973017756199603, + "pass@128": 0.9802816901408451 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 354787, + "answer_accuracy": 0.9081826466251638, + "avg_response_len": 197.9078933895806, + "resp_tokens_sum": 77313906, + "pass_at_k": { + "pass@1": 0.9081826466251638, + "pass@2": 0.9301419747990238, + "pass@4": 0.9446857211533041, + "pass@8": 0.9555253435408846, + "pass@16": 0.9643569247979231, + "pass@32": 0.9715278350509495, + "pass@64": 0.9770850434722914, + "pass@128": 0.9819790301441678 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7672785659509203, + "pass@2": 0.8221017402540941, + "pass@4": 0.8567607347586148, + "pass@8": 0.8803886350813609, + "pass@16": 0.8990823514930147, + "pass@32": 0.9155576005344653, + "pass@64": 0.9292128475444902, + "pass@128": 0.9386503067484663 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975378787878788, + "pass@2": 0.999380368647101, + "pass@4": 0.9999463317085363, + "pass@8": 0.999999574463101, + "pass@16": 0.99999999998634, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9803069526627219, + "pass@2": 0.9856821186460419, + "pass@4": 0.9884984804968907, + "pass@8": 0.9901388748274397, + "pass@16": 0.9915489433224962, + "pass@32": 0.993055431506491, + "pass@64": 0.9948274051186248, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.8750452898550725, + "pass@2": 0.8991081108068009, + "pass@4": 0.91609362552507, + "pass@8": 0.9283145339273063, + "pass@16": 0.9387362873527924, + "pass@32": 0.9495945434273302, + "pass@64": 0.9611962026508659, + "pass@128": 0.9739130434782609 + }, + "6": { + "pass@1": 0.9597063037249284, + "pass@2": 0.9688247354646573, + "pass@4": 0.9755671450810769, + "pass@8": 0.9809583841351045, + "pass@16": 0.9856212779919914, + "pass@32": 0.9895090347929678, + "pass@64": 0.9923046542590954, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9184429190751445, + "pass@2": 0.9380771101679485, + "pass@4": 0.9515284368355686, + "pass@8": 0.9620686520464642, + "pass@16": 0.9694759351290418, + "pass@32": 0.9729727131601149, + "pass@64": 0.9739292569091923, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.8584507042253521, + "pass@2": 0.8973130614395034, + "pass@4": 0.9233392773966632, + "pass@8": 0.9433777973806029, + "pass@16": 0.9601626523567398, + "pass@32": 0.9730733897077097, + "pass@64": 0.9801915294569885, + "pass@128": 0.9830985915492958 + }, + "9": { + "pass@1": 0.8166401571856288, + "pass@2": 0.8604763897402046, + "pass@4": 0.8899476901090955, + "pass@8": 0.9137484797589711, + "pass@16": 0.9336248046731362, + "pass@32": 0.9489223240336991, + "pass@64": 0.9611196746977648, + "pass@128": 0.9760479041916168 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 349025, + "answer_accuracy": 0.912875062772012, + "avg_response_len": 211.2594838048209, + "resp_tokens_sum": 80772106, + "pass_at_k": { + "pass@1": 0.912875062772012, + "pass@2": 0.9354496123622329, + "pass@4": 0.9505403938564442, + "pass@8": 0.9615763074090742, + "pass@16": 0.9700272460522728, + "pass@32": 0.9769039715742426, + "pass@64": 0.9825194863081816, + "pass@128": 0.9866086374288584 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.837080223880597, + "pass@2": 0.8841825567046653, + "pass@4": 0.9129999748165799, + "pass@8": 0.9340309207425049, + "pass@16": 0.9503269298903634, + "pass@32": 0.9633252118626652, + "pass@64": 0.9742339636240699, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994989067055393, + "pass@2": 0.9999314897500058, + "pass@4": 0.9999986759125955, + "pass@8": 0.9999999997431218, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9685215643274854, + "pass@2": 0.9750720921397983, + "pass@4": 0.9805761952453314, + "pass@8": 0.9847048683048196, + "pass@16": 0.9872249490269424, + "pass@32": 0.988723427016145, + "pass@64": 0.9897569491732896, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.8611213235294117, + "pass@2": 0.8884335919407136, + "pass@4": 0.9081887436864504, + "pass@8": 0.924253573036253, + "pass@16": 0.9389244449350933, + "pass@32": 0.9523447840529764, + "pass@64": 0.9630246066646815, + "pass@128": 0.9705882352941176 + }, + "6": { + "pass@1": 0.9733832807570978, + "pass@2": 0.9828349716833503, + "pass@4": 0.9887145172508008, + "pass@8": 0.9916570561272308, + "pass@16": 0.9930273048146369, + "pass@32": 0.9941869150417751, + "pass@64": 0.9952583726678965, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8899080300632911, + "pass@2": 0.9240950176916176, + "pass@4": 0.945723327898886, + "pass@8": 0.9597153776968665, + "pass@16": 0.9702119965930145, + "pass@32": 0.9779051128332572, + "pass@64": 0.9826542051427243, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.8529466966966966, + "pass@2": 0.8928994200893807, + "pass@4": 0.9212113263432158, + "pass@8": 0.94219890076564, + "pass@16": 0.956245171195436, + "pass@32": 0.9650723844889141, + "pass@64": 0.9708403647709795, + "pass@128": 0.972972972972973 + }, + "9": { + "pass@1": 0.8221463022508039, + "pass@2": 0.8635371040838545, + "pass@4": 0.8914274667635996, + "pass@8": 0.9129749581292425, + "pass@16": 0.9306763023858616, + "pass@32": 0.9479524332552338, + "pass@64": 0.9652302587641902, + "pass@128": 0.977491961414791 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.35171796875, + "avg_response_len": 320.44338671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.35171796875, + "pass@2": 0.4300362204724456, + "pass@4": 0.49776747927446324, + "pass@8": 0.5545647302490223, + "pass@16": 0.6014491286908882, + "pass@32": 0.6401886832917174, + "pass@64": 0.6724773931175465, + "pass@128": 0.7007 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.751796875, + "pass@2": 0.8190654527559057, + "pass@4": 0.8630726945069366, + "pass@8": 0.894542210655404, + "pass@16": 0.9190403649249906, + "pass@32": 0.9374440363913676, + "pass@64": 0.9502156530360285, + "pass@128": 0.959 + }, + "12": { + "pass@1": 0.700140625, + "pass@2": 0.793962229330709, + "pass@4": 0.854157430914886, + "pass@8": 0.8934451628397935, + "pass@16": 0.9196320165231094, + "pass@32": 0.937486831105723, + "pass@64": 0.9503306681398772, + "pass@128": 0.96 + }, + "13": { + "pass@1": 0.5165546875, + "pass@2": 0.6264360236220475, + "pass@4": 0.7097452917135361, + "pass@8": 0.7716658762513716, + "pass@16": 0.8183161452392378, + "pass@32": 0.8543244745470427, + "pass@64": 0.8820446942550398, + "pass@128": 0.904 + }, + "14": { + "pass@1": 0.4000625, + "pass@2": 0.5102529527559055, + "pass@4": 0.6033366831646041, + "pass@8": 0.6760730679355115, + "pass@16": 0.7320432699894304, + "pass@32": 0.7757495079127603, + "pass@64": 0.8106559364195371, + "pass@128": 0.84 + }, + "15": { + "pass@1": 0.27609375, + "pass@2": 0.36719537401574803, + "pass@4": 0.45392199287589, + "pass@8": 0.5309015442986452, + "pass@16": 0.5955714301685737, + "pass@32": 0.648426830776277, + "pass@64": 0.6918340269168004, + "pass@128": 0.729 + }, + "16": { + "pass@1": 0.218125, + "pass@2": 0.29149446358267694, + "pass@4": 0.36224176481064857, + "pass@8": 0.4253586078796732, + "pass@16": 0.47872586508528225, + "pass@32": 0.5246915052521031, + "pass@64": 0.5674763848905444, + "pass@128": 0.61 + }, + "17": { + "pass@1": 0.1784453125, + "pass@2": 0.24156053149606296, + "pass@4": 0.30463912401574783, + "pass@8": 0.36292846531877865, + "pass@16": 0.4146627681853899, + "pass@32": 0.460885801867669, + "pass@64": 0.5021259015514952, + "pass@128": 0.54 + }, + "18": { + "pass@1": 0.172625, + "pass@2": 0.23304109251968474, + "pass@4": 0.29413246756655376, + "pass@8": 0.35290778519759475, + "pass@16": 0.40686077357526995, + "pass@32": 0.4537226532568914, + "pass@64": 0.493013939158898, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.15409375, + "pass@2": 0.21103358759842514, + "pass@4": 0.26828433352080966, + "pass@8": 0.3204884832435259, + "pass@16": 0.36592875798653585, + "pass@32": 0.40576208702195554, + "pass@64": 0.4402612131968013, + "pass@128": 0.473 + }, + "20": { + "pass@1": 0.1492421875, + "pass@2": 0.20632049704724434, + "pass@4": 0.26414300965504295, + "pass@8": 0.31733609886987624, + "pass@16": 0.3637098952309234, + "pass@32": 0.4033931047853745, + "pass@64": 0.43681551361036697, + "pass@128": 0.467 + } + }, + "per_op_accuracy": { + "11": 0.751796875, + "12": 0.700140625, + "13": 0.5165546875, + "14": 0.4000625, + "15": 0.27609375, + "16": 0.218125, + "17": 0.1784453125, + "18": 0.172625, + "19": 0.15409375, + "20": 0.1492421875 + }, + "per_op_avg_response_len": { + "11": 296.395796875, + "12": 304.7468984375, + "13": 316.2774765625, + "14": 323.9291875, + "15": 324.4436953125, + "16": 323.8881328125, + "17": 329.4645234375, + "18": 329.750609375, + "19": 325.5416171875, + "20": 329.9959296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 296.395796875, + "12": 304.7468984375, + "13": 316.2774765625, + "14": 323.9291875, + "15": 324.4436953125, + "16": 323.8881328125, + "17": 329.4645234375, + "18": 329.750609375, + "19": 325.5416171875, + "20": 329.9959296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 154139, + "answer_accuracy": 0.35786357726597323, + "avg_response_len": 305.4103129643388, + "resp_tokens_sum": 131546330, + "pass_at_k": { + "pass@1": 0.35786357726597323, + "pass@2": 0.43663691486001255, + "pass@4": 0.503037916731435, + "pass@8": 0.557868057110537, + "pass@16": 0.6032547579604092, + "pass@32": 0.6410760845408966, + "pass@64": 0.672396443552445, + "pass@128": 0.6989598811292719 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7452195467422096, + "pass@2": 0.8180329320113316, + "pass@4": 0.8652162037641893, + "pass@8": 0.8969106210163632, + "pass@16": 0.9210920894565872, + "pass@32": 0.9403227057391875, + "pass@64": 0.9548937056898483, + "pass@128": 0.9660056657223796 + }, + "12": { + "pass@1": 0.6948490466101694, + "pass@2": 0.7856954685484225, + "pass@4": 0.8424545043628017, + "pass@8": 0.8799306789509234, + "pass@16": 0.9062520096316939, + "pass@32": 0.924809518611065, + "pass@64": 0.938702946618994, + "pass@128": 0.9491525423728814 + }, + "13": { + "pass@1": 0.5179516065830722, + "pass@2": 0.6325184200133289, + "pass@4": 0.7205692302764656, + "pass@8": 0.7828447072240617, + "pass@16": 0.8262471661892159, + "pass@32": 0.8598153482049776, + "pass@64": 0.8862313514669503, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.3964417834890966, + "pass@2": 0.5047361376848922, + "pass@4": 0.5946405522323733, + "pass@8": 0.6680429180152108, + "pass@16": 0.7301454909544141, + "pass@32": 0.7804329625460306, + "pass@64": 0.8173909998798203, + "pass@128": 0.8411214953271028 + }, + "15": { + "pass@1": 0.29612987716763006, + "pass@2": 0.3869219653179187, + "pass@4": 0.46993725170191875, + "pass@8": 0.5421051675527985, + "pass@16": 0.6023771645269487, + "pass@32": 0.6529647288435331, + "pass@64": 0.6944795629555712, + "pass@128": 0.7283236994219653 + }, + "16": { + "pass@1": 0.24740423387096774, + "pass@2": 0.32904813309626596, + "pass@4": 0.40458356405046153, + "pass@8": 0.4702012137774771, + "pass@16": 0.5247288669662102, + "pass@32": 0.5706876527594704, + "pass@64": 0.6115942057418606, + "pass@128": 0.6451612903225806 + }, + "17": { + "pass@1": 0.16986542630057805, + "pass@2": 0.22985803729916698, + "pass@4": 0.28688736695124073, + "pass@8": 0.33843467932243226, + "pass@16": 0.38700181611143314, + "pass@32": 0.43410878184626833, + "pass@64": 0.47804135764926775, + "pass@128": 0.5202312138728323 + }, + "18": { + "pass@1": 0.1679017857142857, + "pass@2": 0.2257529527559055, + "pass@4": 0.2848023300658846, + "pass@8": 0.34394438573386016, + "pass@16": 0.3988663230247612, + "pass@32": 0.4435387309873241, + "pass@64": 0.47824795716449336, + "pass@128": 0.5085714285714286 + }, + "19": { + "pass@1": 0.16246715465465467, + "pass@2": 0.2220851510959779, + "pass@4": 0.2795086211070464, + "pass@8": 0.3279673225850375, + "pass@16": 0.36686001138513913, + "pass@32": 0.4010357183857945, + "pass@64": 0.4330668055174505, + "pass@128": 0.46546546546546547 + }, + "20": { + "pass@1": 0.15582770270270271, + "pass@2": 0.21083792946489788, + "pass@4": 0.2651404334593312, + "pass@8": 0.31510224505908696, + "pass@16": 0.3582141742580615, + "pass@32": 0.3942354522194376, + "pass@64": 0.4242069841105825, + "pass@128": 0.45345345345345345 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 152739, + "answer_accuracy": 0.3548240967885816, + "avg_response_len": 317.9495939265537, + "resp_tokens_sum": 136865854, + "pass_at_k": { + "pass@1": 0.3548240967885816, + "pass@2": 0.4348195377088804, + "pass@4": 0.5048561424246599, + "pass@8": 0.5638958078728147, + "pass@16": 0.6127901236084599, + "pass@32": 0.6531011736429899, + "pass@64": 0.68633018743688, + "pass@128": 0.7148379423134107 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.759765625, + "pass@2": 0.8237484880466749, + "pass@4": 0.8661105835038693, + "pass@8": 0.8974093782721269, + "pass@16": 0.9220586483640653, + "pass@32": 0.9400776540050912, + "pass@64": 0.9516687417338571, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.6983248432601881, + "pass@2": 0.792604765754202, + "pass@4": 0.8538179523210068, + "pass@8": 0.8929835881620504, + "pass@16": 0.9185320929903009, + "pass@32": 0.9352178761761952, + "pass@64": 0.9478804552757559, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5278361344537815, + "pass@2": 0.63536910827323, + "pass@4": 0.7126288210822387, + "pass@8": 0.7701978399669224, + "pass@16": 0.8166211744373886, + "pass@32": 0.8550839822738148, + "pass@64": 0.8865038689796082, + "pass@128": 0.9131652661064426 + }, + "14": { + "pass@1": 0.41721491228070173, + "pass@2": 0.5287199400239444, + "pass@4": 0.6211705283001904, + "pass@8": 0.6923516248529362, + "pass@16": 0.7462194130033619, + "pass@32": 0.7883736937027563, + "pass@64": 0.8222691206866892, + "pass@128": 0.8508771929824561 + }, + "15": { + "pass@1": 0.28927556818181815, + "pass@2": 0.38632150739680254, + "pass@4": 0.477739166411017, + "pass@8": 0.5566535711234109, + "pass@16": 0.6215893466194053, + "pass@32": 0.6719853954938052, + "pass@64": 0.7093504805012789, + "pass@128": 0.7393939393939394 + }, + "16": { + "pass@1": 0.21276264245014245, + "pass@2": 0.28995263813177213, + "pass@4": 0.3666882563824822, + "pass@8": 0.4346854612065274, + "pass@16": 0.490842220216734, + "pass@32": 0.5378512803345692, + "pass@64": 0.5810806305675018, + "pass@128": 0.6239316239316239 + }, + "17": { + "pass@1": 0.19004755434782608, + "pass@2": 0.2589851200665132, + "pass@4": 0.3332288832684735, + "pass@8": 0.4048084183288024, + "pass@16": 0.46686980557788155, + "pass@32": 0.5180633036985619, + "pass@64": 0.5611210047877057, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.17541360294117647, + "pass@2": 0.238078248031496, + "pass@4": 0.3021544674562736, + "pass@8": 0.36484250871024415, + "pass@16": 0.42552854294852027, + "pass@32": 0.4830248444609159, + "pass@64": 0.532842036580717, + "pass@128": 0.5705882352941176 + }, + "19": { + "pass@1": 0.14276413690476192, + "pass@2": 0.19880974350862385, + "pass@4": 0.25782905819808244, + "pass@8": 0.3139917285710976, + "pass@16": 0.3642751043986496, + "pass@32": 0.4084618548346345, + "pass@64": 0.44438723572602956, + "pass@128": 0.47619047619047616 + }, + "20": { + "pass@1": 0.14429734281437126, + "pass@2": 0.2033662743410817, + "pass@4": 0.26359721810597025, + "pass@8": 0.3163735906613064, + "pass@16": 0.359637297892788, + "pass@32": 0.395871809493765, + "pass@64": 0.42783318947128957, + "pass@128": 0.46107784431137727 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 143321, + "answer_accuracy": 0.3422051688569682, + "avg_response_len": 338.4668947700183, + "resp_tokens_sum": 141755351, + "pass_at_k": { + "pass@1": 0.3422051688569682, + "pass@2": 0.41833156506892255, + "pass@4": 0.4850614290859592, + "pass@8": 0.5415769218939334, + "pass@16": 0.5879357703779667, + "pass@32": 0.6260044503287424, + "pass@64": 0.6583225789337332, + "pass@128": 0.687958435207824 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7507688492063492, + "pass@2": 0.8152867610298712, + "pass@4": 0.8574687646187082, + "pass@8": 0.8888661836516896, + "pass@16": 0.9135599558410991, + "pass@32": 0.9314423496372826, + "pass@64": 0.9434417545138772, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.7076404816513762, + "pass@2": 0.8042358201497741, + "pass@4": 0.8671578274619333, + "pass@8": 0.9085258038760612, + "pass@16": 0.935189808714328, + "pass@32": 0.9534243395021395, + "pass@64": 0.9653087455773289, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5027488425925926, + "pass@2": 0.6106045555069503, + "pass@4": 0.6959111670647646, + "pass@8": 0.7622771166009414, + "pass@16": 0.8123751233047263, + "pass@32": 0.8480814715984645, + "pass@64": 0.8730092960227223, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.3861044139465875, + "pass@2": 0.4967668637117687, + "pass@4": 0.5935214991672034, + "pass@8": 0.6672018859137244, + "pass@16": 0.71946447521043, + "pass@32": 0.7584769248935936, + "pass@64": 0.7924551518791305, + "pass@128": 0.827893175074184 + }, + "15": { + "pass@1": 0.24127121913580246, + "pass@2": 0.32664900177408374, + "pass@4": 0.4125610458993551, + "pass@8": 0.49270826498349163, + "pass@16": 0.5618038483266853, + "pass@32": 0.6195859693933924, + "pass@64": 0.6711680233603406, + "pass@128": 0.7191358024691358 + }, + "16": { + "pass@1": 0.19690265486725664, + "pass@2": 0.2587497532111583, + "pass@4": 0.3189182358842312, + "pass@8": 0.374695087684852, + "pass@16": 0.4241129705890384, + "pass@32": 0.4690045235965585, + "pass@64": 0.5130468430129036, + "pass@128": 0.5634218289085545 + }, + "17": { + "pass@1": 0.17613422439759036, + "pass@2": 0.23685675279859578, + "pass@4": 0.2954108272234041, + "pass@8": 0.34783661316669684, + "pass@16": 0.3928556096830005, + "pass@32": 0.43333668541543363, + "pass@64": 0.4700079164554424, + "pass@128": 0.5060240963855421 + }, + "18": { + "pass@1": 0.1748991935483871, + "pass@2": 0.23574501524003058, + "pass@4": 0.2958681713172949, + "pass@8": 0.34993805557826135, + "pass@16": 0.395412438432603, + "pass@32": 0.43308274288586, + "pass@64": 0.4660027797221985, + "pass@128": 0.4935483870967742 + }, + "19": { + "pass@1": 0.15717050604229607, + "pass@2": 0.21232374158479428, + "pass@4": 0.2676054354610507, + "pass@8": 0.31955934750096615, + "pass@16": 0.3666705109285142, + "pass@32": 0.4077764639729569, + "pass@64": 0.44331074246388014, + "pass@128": 0.4773413897280967 + }, + "20": { + "pass@1": 0.14761636636636635, + "pass@2": 0.20476615887541083, + "pass@4": 0.2636930164134889, + "pass@8": 0.32053535130427724, + "pass@16": 0.3732904435639585, + "pass@32": 0.4200946391152685, + "pass@64": 0.4584333411955923, + "pass@128": 0.4864864864864865 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6166862664473685, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6166862664473685, + "pass@2": 0.6686315465706337, + "pass@4": 0.7113611722613452, + "pass@8": 0.7464265724633806, + "pass@16": 0.7751857277609306, + "pass@32": 0.7988683628049251, + "pass@64": 0.8184622060419368, + "pass@128": 0.8353157894736842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.80821875, + "pass@2": 0.8611038385826778, + "pass@4": 0.893551319928759, + "pass@8": 0.9152204428848738, + "pass@16": 0.9319724334620572, + "pass@32": 0.9465334079108148, + "pass@64": 0.9587946628838566, + "pass@128": 0.968 + }, + "2": { + "pass@1": 0.9999453125, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998171875, + "pass@2": 0.9990873523622046, + "pass@4": 0.9995161725721784, + "pass@8": 0.9997889208055989, + "pass@16": 0.9999604948476541, + "pass@32": 0.9999991088495628, + "pass@64": 0.9999999999731547, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.976078125, + "pass@2": 0.9812984744094486, + "pass@4": 0.98483539848144, + "pass@8": 0.9875219111336576, + "pass@16": 0.989539551379835, + "pass@32": 0.9911053570965834, + "pass@64": 0.9924939351791154, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.8609609375, + "pass@2": 0.8890690206692917, + "pass@4": 0.9095421017060369, + "pass@8": 0.9254586695605911, + "pass@16": 0.9392363307896897, + "pass@32": 0.9515155455110998, + "pass@64": 0.9621924392468406, + "pass@128": 0.971 + }, + "6": { + "pass@1": 0.959984375, + "pass@2": 0.9701747047244096, + "pass@4": 0.9773701526059242, + "pass@8": 0.9827041747056208, + "pass@16": 0.9867273186619663, + "pass@32": 0.9897881513486811, + "pass@64": 0.9921325328624394, + "pass@128": 0.994 + }, + "7": { + "pass@1": 0.9054921875, + "pass@2": 0.9316528051181108, + "pass@4": 0.9482259476003001, + "pass@8": 0.9600481131322273, + "pass@16": 0.9692704251651028, + "pass@32": 0.9757960519218113, + "pass@64": 0.9797622739854073, + "pass@128": 0.982 + }, + "8": { + "pass@1": 0.8637265625, + "pass@2": 0.9008845964566933, + "pass@4": 0.9261050230596173, + "pass@8": 0.944664751294384, + "pass@16": 0.958480101133846, + "pass@32": 0.9681205229698595, + "pass@64": 0.9740102542149567, + "pass@128": 0.977 + }, + "9": { + "pass@1": 0.82728125, + "pass@2": 0.8703665108267724, + "pass@4": 0.899041364266967, + "pass@8": 0.9210505907968902, + "pass@16": 0.9388508851085343, + "pass@32": 0.953753914767747, + "pass@64": 0.9666218852755756, + "pass@128": 0.978 + }, + "11": { + "pass@1": 0.751796875, + "pass@2": 0.8190654527559057, + "pass@4": 0.8630726945069366, + "pass@8": 0.894542210655404, + "pass@16": 0.9190403649249906, + "pass@32": 0.9374440363913676, + "pass@64": 0.9502156530360285, + "pass@128": 0.959 + }, + "12": { + "pass@1": 0.700140625, + "pass@2": 0.793962229330709, + "pass@4": 0.854157430914886, + "pass@8": 0.8934451628397935, + "pass@16": 0.9196320165231094, + "pass@32": 0.937486831105723, + "pass@64": 0.9503306681398772, + "pass@128": 0.96 + }, + "13": { + "pass@1": 0.5165546875, + "pass@2": 0.6264360236220475, + "pass@4": 0.7097452917135361, + "pass@8": 0.7716658762513716, + "pass@16": 0.8183161452392378, + "pass@32": 0.8543244745470427, + "pass@64": 0.8820446942550398, + "pass@128": 0.904 + }, + "14": { + "pass@1": 0.4000625, + "pass@2": 0.5102529527559055, + "pass@4": 0.6033366831646041, + "pass@8": 0.6760730679355115, + "pass@16": 0.7320432699894304, + "pass@32": 0.7757495079127603, + "pass@64": 0.8106559364195371, + "pass@128": 0.84 + }, + "15": { + "pass@1": 0.27609375, + "pass@2": 0.36719537401574803, + "pass@4": 0.45392199287589, + "pass@8": 0.5309015442986452, + "pass@16": 0.5955714301685737, + "pass@32": 0.648426830776277, + "pass@64": 0.6918340269168004, + "pass@128": 0.729 + }, + "16": { + "pass@1": 0.218125, + "pass@2": 0.29149446358267694, + "pass@4": 0.36224176481064857, + "pass@8": 0.4253586078796732, + "pass@16": 0.47872586508528225, + "pass@32": 0.5246915052521031, + "pass@64": 0.5674763848905444, + "pass@128": 0.61 + }, + "17": { + "pass@1": 0.1784453125, + "pass@2": 0.24156053149606296, + "pass@4": 0.30463912401574783, + "pass@8": 0.36292846531877865, + "pass@16": 0.4146627681853899, + "pass@32": 0.460885801867669, + "pass@64": 0.5021259015514952, + "pass@128": 0.54 + }, + "18": { + "pass@1": 0.172625, + "pass@2": 0.23304109251968474, + "pass@4": 0.29413246756655376, + "pass@8": 0.35290778519759475, + "pass@16": 0.40686077357526995, + "pass@32": 0.4537226532568914, + "pass@64": 0.493013939158898, + "pass@128": 0.525 + }, + "19": { + "pass@1": 0.15409375, + "pass@2": 0.21103358759842514, + "pass@4": 0.26828433352080966, + "pass@8": 0.3204884832435259, + "pass@16": 0.36592875798653585, + "pass@32": 0.40576208702195554, + "pass@64": 0.4402612131968013, + "pass@128": 0.473 + }, + "20": { + "pass@1": 0.1492421875, + "pass@2": 0.20632049704724434, + "pass@4": 0.26414300965504295, + "pass@8": 0.31733609886987624, + "pass@16": 0.3637098952309234, + "pass@32": 0.4033931047853745, + "pass@64": 0.43681551361036697, + "pass@128": 0.467 + } + }, + "per_op_accuracy": { + "10": 0.80821875, + "2": 0.9999453125, + "3": 0.998171875, + "4": 0.976078125, + "5": 0.8609609375, + "6": 0.959984375, + "7": 0.9054921875, + "8": 0.8637265625, + "9": 0.82728125, + "11": 0.751796875, + "12": 0.700140625, + "13": 0.5165546875, + "14": 0.4000625, + "15": 0.27609375, + "16": 0.218125, + "17": 0.1784453125, + "18": 0.172625, + "19": 0.15409375, + "20": 0.1492421875 + }, + "per_op_avg_response_len": { + "10": 279.956171875, + "2": 115.5694375, + "3": 144.7300234375, + "4": 159.83084375, + "5": 173.175125, + "6": 199.1570859375, + "7": 223.978734375, + "8": 243.46984375, + "9": 259.481578125, + "11": 296.395796875, + "12": 304.7468984375, + "13": 316.2774765625, + "14": 323.9291875, + "15": 324.4436953125, + "16": 323.8881328125, + "17": 329.4645234375, + "18": 329.750609375, + "19": 325.5416171875, + "20": 329.9959296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 279.956171875, + "2": 115.5694375, + "3": 144.7300234375, + "4": 159.83084375, + "5": 173.175125, + "6": 199.1570859375, + "7": 223.978734375, + "8": 243.46984375, + "9": 259.481578125, + "11": 296.395796875, + "12": 304.7468984375, + "13": 316.2774765625, + "14": 323.9291875, + "15": 324.4436953125, + "16": 323.8881328125, + "17": 329.4645234375, + "18": 329.750609375, + "19": 325.5416171875, + "20": 329.9959296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 499909, + "answer_accuracy": 0.617378922304774, + "avg_response_len": 251.66101456291494, + "resp_tokens_sum": 203776970, + "pass_at_k": { + "pass@1": 0.617378922304774, + "pass@2": 0.6702401358535359, + "pass@4": 0.7126881271914993, + "pass@8": 0.7469478039476964, + "pass@16": 0.7751029216952456, + "pass@32": 0.7984062550092523, + "pass@64": 0.8175423265721407, + "pass@128": 0.833386025924755 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8190680309734514, + "pass@2": 0.8758038783592319, + "pass@4": 0.9097118844657693, + "pass@8": 0.9301279923292359, + "pass@16": 0.945463319652103, + "pass@32": 0.9597276818955355, + "pass@64": 0.971984946225041, + "pass@128": 0.9793510324483776 + }, + "2": { + "pass@1": 0.9998539719626168, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974197247706422, + "pass@2": 0.9979062071082859, + "pass@4": 0.9985759549551719, + "pass@8": 0.9993549275868031, + "pass@16": 0.9998791891503424, + "pass@32": 0.999997274769305, + "pass@64": 0.999999999917904, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9796875, + "pass@2": 0.9833226962352362, + "pass@4": 0.9855182915612108, + "pass@8": 0.9877684577554199, + "pass@16": 0.9898908624050529, + "pass@32": 0.9915912787745874, + "pass@64": 0.9929543613492345, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8453621031746031, + "pass@2": 0.878759686289214, + "pass@4": 0.9038273906237906, + "pass@8": 0.9236315572169654, + "pass@16": 0.9401206361112525, + "pass@32": 0.9527244490497119, + "pass@64": 0.9623853430676196, + "pass@128": 0.9682539682539683 + }, + "6": { + "pass@1": 0.9475580089820359, + "pass@2": 0.9595694192324015, + "pass@4": 0.9684871766590554, + "pass@8": 0.9760311732040033, + "pass@16": 0.9819037036422803, + "pass@32": 0.9859049285559663, + "pass@64": 0.9889859412882993, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9068047337278107, + "pass@2": 0.932142305945115, + "pass@4": 0.947185020234157, + "pass@8": 0.9582908289110658, + "pass@16": 0.9681797653463373, + "pass@32": 0.9767144305121371, + "pass@64": 0.9830296516855794, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8812349759615384, + "pass@2": 0.9134710024227741, + "pass@4": 0.9344750253574073, + "pass@8": 0.9487608950939497, + "pass@16": 0.9589510177536014, + "pass@32": 0.9657383512461981, + "pass@64": 0.9703603198365051, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.8417913732394366, + "pass@2": 0.8856545275590555, + "pass@4": 0.9142673059353488, + "pass@8": 0.9349954551526746, + "pass@16": 0.9509292121287449, + "pass@32": 0.9633821177411638, + "pass@64": 0.973017756199603, + "pass@128": 0.9802816901408451 + }, + "11": { + "pass@1": 0.7452195467422096, + "pass@2": 0.8180329320113316, + "pass@4": 0.8652162037641893, + "pass@8": 0.8969106210163632, + "pass@16": 0.9210920894565872, + "pass@32": 0.9403227057391875, + "pass@64": 0.9548937056898483, + "pass@128": 0.9660056657223796 + }, + "12": { + "pass@1": 0.6948490466101694, + "pass@2": 0.7856954685484225, + "pass@4": 0.8424545043628017, + "pass@8": 0.8799306789509234, + "pass@16": 0.9062520096316939, + "pass@32": 0.924809518611065, + "pass@64": 0.938702946618994, + "pass@128": 0.9491525423728814 + }, + "13": { + "pass@1": 0.5179516065830722, + "pass@2": 0.6325184200133289, + "pass@4": 0.7205692302764656, + "pass@8": 0.7828447072240617, + "pass@16": 0.8262471661892159, + "pass@32": 0.8598153482049776, + "pass@64": 0.8862313514669503, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.3964417834890966, + "pass@2": 0.5047361376848922, + "pass@4": 0.5946405522323733, + "pass@8": 0.6680429180152108, + "pass@16": 0.7301454909544141, + "pass@32": 0.7804329625460306, + "pass@64": 0.8173909998798203, + "pass@128": 0.8411214953271028 + }, + "15": { + "pass@1": 0.29612987716763006, + "pass@2": 0.3869219653179187, + "pass@4": 0.46993725170191875, + "pass@8": 0.5421051675527985, + "pass@16": 0.6023771645269487, + "pass@32": 0.6529647288435331, + "pass@64": 0.6944795629555712, + "pass@128": 0.7283236994219653 + }, + "16": { + "pass@1": 0.24740423387096774, + "pass@2": 0.32904813309626596, + "pass@4": 0.40458356405046153, + "pass@8": 0.4702012137774771, + "pass@16": 0.5247288669662102, + "pass@32": 0.5706876527594704, + "pass@64": 0.6115942057418606, + "pass@128": 0.6451612903225806 + }, + "17": { + "pass@1": 0.16986542630057805, + "pass@2": 0.22985803729916698, + "pass@4": 0.28688736695124073, + "pass@8": 0.33843467932243226, + "pass@16": 0.38700181611143314, + "pass@32": 0.43410878184626833, + "pass@64": 0.47804135764926775, + "pass@128": 0.5202312138728323 + }, + "18": { + "pass@1": 0.1679017857142857, + "pass@2": 0.2257529527559055, + "pass@4": 0.2848023300658846, + "pass@8": 0.34394438573386016, + "pass@16": 0.3988663230247612, + "pass@32": 0.4435387309873241, + "pass@64": 0.47824795716449336, + "pass@128": 0.5085714285714286 + }, + "19": { + "pass@1": 0.16246715465465467, + "pass@2": 0.2220851510959779, + "pass@4": 0.2795086211070464, + "pass@8": 0.3279673225850375, + "pass@16": 0.36686001138513913, + "pass@32": 0.4010357183857945, + "pass@64": 0.4330668055174505, + "pass@128": 0.46546546546546547 + }, + "20": { + "pass@1": 0.15582770270270271, + "pass@2": 0.21083792946489788, + "pass@4": 0.2651404334593312, + "pass@8": 0.31510224505908696, + "pass@16": 0.3582141742580615, + "pass@32": 0.3942354522194376, + "pass@64": 0.4242069841105825, + "pass@128": 0.45345345345345345 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 507526, + "answer_accuracy": 0.6180899259547935, + "avg_response_len": 260.83856196414655, + "resp_tokens_sum": 214179760, + "pass_at_k": { + "pass@1": 0.6180899259547935, + "pass@2": 0.6704741094936216, + "pass@4": 0.714109435375526, + "pass@8": 0.7502174513426444, + "pass@16": 0.7800515230208226, + "pass@32": 0.804595822219313, + "pass@64": 0.8246596996146075, + "pass@128": 0.8419329696024942 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7672785659509203, + "pass@2": 0.8221017402540941, + "pass@4": 0.8567607347586148, + "pass@8": 0.8803886350813609, + "pass@16": 0.8990823514930147, + "pass@32": 0.9155576005344653, + "pass@64": 0.9292128475444902, + "pass@128": 0.9386503067484663 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975378787878788, + "pass@2": 0.999380368647101, + "pass@4": 0.9999463317085363, + "pass@8": 0.999999574463101, + "pass@16": 0.99999999998634, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9803069526627219, + "pass@2": 0.9856821186460419, + "pass@4": 0.9884984804968907, + "pass@8": 0.9901388748274397, + "pass@16": 0.9915489433224962, + "pass@32": 0.993055431506491, + "pass@64": 0.9948274051186248, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.8750452898550725, + "pass@2": 0.8991081108068009, + "pass@4": 0.91609362552507, + "pass@8": 0.9283145339273063, + "pass@16": 0.9387362873527924, + "pass@32": 0.9495945434273302, + "pass@64": 0.9611962026508659, + "pass@128": 0.9739130434782609 + }, + "6": { + "pass@1": 0.9597063037249284, + "pass@2": 0.9688247354646573, + "pass@4": 0.9755671450810769, + "pass@8": 0.9809583841351045, + "pass@16": 0.9856212779919914, + "pass@32": 0.9895090347929678, + "pass@64": 0.9923046542590954, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9184429190751445, + "pass@2": 0.9380771101679485, + "pass@4": 0.9515284368355686, + "pass@8": 0.9620686520464642, + "pass@16": 0.9694759351290418, + "pass@32": 0.9729727131601149, + "pass@64": 0.9739292569091923, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.8584507042253521, + "pass@2": 0.8973130614395034, + "pass@4": 0.9233392773966632, + "pass@8": 0.9433777973806029, + "pass@16": 0.9601626523567398, + "pass@32": 0.9730733897077097, + "pass@64": 0.9801915294569885, + "pass@128": 0.9830985915492958 + }, + "9": { + "pass@1": 0.8166401571856288, + "pass@2": 0.8604763897402046, + "pass@4": 0.8899476901090955, + "pass@8": 0.9137484797589711, + "pass@16": 0.9336248046731362, + "pass@32": 0.9489223240336991, + "pass@64": 0.9611196746977648, + "pass@128": 0.9760479041916168 + }, + "11": { + "pass@1": 0.759765625, + "pass@2": 0.8237484880466749, + "pass@4": 0.8661105835038693, + "pass@8": 0.8974093782721269, + "pass@16": 0.9220586483640653, + "pass@32": 0.9400776540050912, + "pass@64": 0.9516687417338571, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.6983248432601881, + "pass@2": 0.792604765754202, + "pass@4": 0.8538179523210068, + "pass@8": 0.8929835881620504, + "pass@16": 0.9185320929903009, + "pass@32": 0.9352178761761952, + "pass@64": 0.9478804552757559, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5278361344537815, + "pass@2": 0.63536910827323, + "pass@4": 0.7126288210822387, + "pass@8": 0.7701978399669224, + "pass@16": 0.8166211744373886, + "pass@32": 0.8550839822738148, + "pass@64": 0.8865038689796082, + "pass@128": 0.9131652661064426 + }, + "14": { + "pass@1": 0.41721491228070173, + "pass@2": 0.5287199400239444, + "pass@4": 0.6211705283001904, + "pass@8": 0.6923516248529362, + "pass@16": 0.7462194130033619, + "pass@32": 0.7883736937027563, + "pass@64": 0.8222691206866892, + "pass@128": 0.8508771929824561 + }, + "15": { + "pass@1": 0.28927556818181815, + "pass@2": 0.38632150739680254, + "pass@4": 0.477739166411017, + "pass@8": 0.5566535711234109, + "pass@16": 0.6215893466194053, + "pass@32": 0.6719853954938052, + "pass@64": 0.7093504805012789, + "pass@128": 0.7393939393939394 + }, + "16": { + "pass@1": 0.21276264245014245, + "pass@2": 0.28995263813177213, + "pass@4": 0.3666882563824822, + "pass@8": 0.4346854612065274, + "pass@16": 0.490842220216734, + "pass@32": 0.5378512803345692, + "pass@64": 0.5810806305675018, + "pass@128": 0.6239316239316239 + }, + "17": { + "pass@1": 0.19004755434782608, + "pass@2": 0.2589851200665132, + "pass@4": 0.3332288832684735, + "pass@8": 0.4048084183288024, + "pass@16": 0.46686980557788155, + "pass@32": 0.5180633036985619, + "pass@64": 0.5611210047877057, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.17541360294117647, + "pass@2": 0.238078248031496, + "pass@4": 0.3021544674562736, + "pass@8": 0.36484250871024415, + "pass@16": 0.42552854294852027, + "pass@32": 0.4830248444609159, + "pass@64": 0.532842036580717, + "pass@128": 0.5705882352941176 + }, + "19": { + "pass@1": 0.14276413690476192, + "pass@2": 0.19880974350862385, + "pass@4": 0.25782905819808244, + "pass@8": 0.3139917285710976, + "pass@16": 0.3642751043986496, + "pass@32": 0.4084618548346345, + "pass@64": 0.44438723572602956, + "pass@128": 0.47619047619047616 + }, + "20": { + "pass@1": 0.14429734281437126, + "pass@2": 0.2033662743410817, + "pass@4": 0.26359721810597025, + "pass@8": 0.3163735906613064, + "pass@16": 0.359637297892788, + "pass@32": 0.395871809493765, + "pass@64": 0.42783318947128957, + "pass@128": 0.46107784431137727 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 492346, + "answer_accuracy": 0.6145475515258029, + "avg_response_len": 277.75934778918355, + "resp_tokens_sum": 222527457, + "pass_at_k": { + "pass@1": 0.6145475515258029, + "pass@2": 0.66511725084383, + "pass@4": 0.7072032517044973, + "pass@8": 0.7420143982533723, + "pass@16": 0.7702823493585079, + "pass@32": 0.7934652060341785, + "pass@64": 0.8130399718603196, + "pass@128": 0.8304841028918357 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.837080223880597, + "pass@2": 0.8841825567046653, + "pass@4": 0.9129999748165799, + "pass@8": 0.9340309207425049, + "pass@16": 0.9503269298903634, + "pass@32": 0.9633252118626652, + "pass@64": 0.9742339636240699, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994989067055393, + "pass@2": 0.9999314897500058, + "pass@4": 0.9999986759125955, + "pass@8": 0.9999999997431218, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9685215643274854, + "pass@2": 0.9750720921397983, + "pass@4": 0.9805761952453314, + "pass@8": 0.9847048683048196, + "pass@16": 0.9872249490269424, + "pass@32": 0.988723427016145, + "pass@64": 0.9897569491732896, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.8611213235294117, + "pass@2": 0.8884335919407136, + "pass@4": 0.9081887436864504, + "pass@8": 0.924253573036253, + "pass@16": 0.9389244449350933, + "pass@32": 0.9523447840529764, + "pass@64": 0.9630246066646815, + "pass@128": 0.9705882352941176 + }, + "6": { + "pass@1": 0.9733832807570978, + "pass@2": 0.9828349716833503, + "pass@4": 0.9887145172508008, + "pass@8": 0.9916570561272308, + "pass@16": 0.9930273048146369, + "pass@32": 0.9941869150417751, + "pass@64": 0.9952583726678965, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8899080300632911, + "pass@2": 0.9240950176916176, + "pass@4": 0.945723327898886, + "pass@8": 0.9597153776968665, + "pass@16": 0.9702119965930145, + "pass@32": 0.9779051128332572, + "pass@64": 0.9826542051427243, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.8529466966966966, + "pass@2": 0.8928994200893807, + "pass@4": 0.9212113263432158, + "pass@8": 0.94219890076564, + "pass@16": 0.956245171195436, + "pass@32": 0.9650723844889141, + "pass@64": 0.9708403647709795, + "pass@128": 0.972972972972973 + }, + "9": { + "pass@1": 0.8221463022508039, + "pass@2": 0.8635371040838545, + "pass@4": 0.8914274667635996, + "pass@8": 0.9129749581292425, + "pass@16": 0.9306763023858616, + "pass@32": 0.9479524332552338, + "pass@64": 0.9652302587641902, + "pass@128": 0.977491961414791 + }, + "11": { + "pass@1": 0.7507688492063492, + "pass@2": 0.8152867610298712, + "pass@4": 0.8574687646187082, + "pass@8": 0.8888661836516896, + "pass@16": 0.9135599558410991, + "pass@32": 0.9314423496372826, + "pass@64": 0.9434417545138772, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.7076404816513762, + "pass@2": 0.8042358201497741, + "pass@4": 0.8671578274619333, + "pass@8": 0.9085258038760612, + "pass@16": 0.935189808714328, + "pass@32": 0.9534243395021395, + "pass@64": 0.9653087455773289, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5027488425925926, + "pass@2": 0.6106045555069503, + "pass@4": 0.6959111670647646, + "pass@8": 0.7622771166009414, + "pass@16": 0.8123751233047263, + "pass@32": 0.8480814715984645, + "pass@64": 0.8730092960227223, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.3861044139465875, + "pass@2": 0.4967668637117687, + "pass@4": 0.5935214991672034, + "pass@8": 0.6672018859137244, + "pass@16": 0.71946447521043, + "pass@32": 0.7584769248935936, + "pass@64": 0.7924551518791305, + "pass@128": 0.827893175074184 + }, + "15": { + "pass@1": 0.24127121913580246, + "pass@2": 0.32664900177408374, + "pass@4": 0.4125610458993551, + "pass@8": 0.49270826498349163, + "pass@16": 0.5618038483266853, + "pass@32": 0.6195859693933924, + "pass@64": 0.6711680233603406, + "pass@128": 0.7191358024691358 + }, + "16": { + "pass@1": 0.19690265486725664, + "pass@2": 0.2587497532111583, + "pass@4": 0.3189182358842312, + "pass@8": 0.374695087684852, + "pass@16": 0.4241129705890384, + "pass@32": 0.4690045235965585, + "pass@64": 0.5130468430129036, + "pass@128": 0.5634218289085545 + }, + "17": { + "pass@1": 0.17613422439759036, + "pass@2": 0.23685675279859578, + "pass@4": 0.2954108272234041, + "pass@8": 0.34783661316669684, + "pass@16": 0.3928556096830005, + "pass@32": 0.43333668541543363, + "pass@64": 0.4700079164554424, + "pass@128": 0.5060240963855421 + }, + "18": { + "pass@1": 0.1748991935483871, + "pass@2": 0.23574501524003058, + "pass@4": 0.2958681713172949, + "pass@8": 0.34993805557826135, + "pass@16": 0.395412438432603, + "pass@32": 0.43308274288586, + "pass@64": 0.4660027797221985, + "pass@128": 0.4935483870967742 + }, + "19": { + "pass@1": 0.15717050604229607, + "pass@2": 0.21232374158479428, + "pass@4": 0.2676054354610507, + "pass@8": 0.31955934750096615, + "pass@16": 0.3666705109285142, + "pass@32": 0.4077764639729569, + "pass@64": 0.44331074246388014, + "pass@128": 0.4773413897280967 + }, + "20": { + "pass@1": 0.14761636636636635, + "pass@2": 0.20476615887541083, + "pass@4": 0.2636930164134889, + "pass@8": 0.32053535130427724, + "pass@16": 0.3732904435639585, + "pass@32": 0.4200946391152685, + "pass@64": 0.4584333411955923, + "pass@128": 0.4864864864864865 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep1-block2/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6cfb3a9187cc823f71230252b4a631c123e8203f --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block2/composition/autoregressive", "results_dir": "results/SDAR-100M-ep1-block2/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block2/composition/autoregressive/checkpoint-21350_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 929469, "value_mismatch": 113840, "dependency_mismatch": 145716, "answer_mismatch": 102414, "extra_nodes": 20183, "missing_nodes": 16444, "perfect_with_extra": 636, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 306093, "value_mismatch": 37417, "answer_mismatch": 33239, "dependency_mismatch": 47983, "missing_nodes": 5587, "extra_nodes": 7421, "perfect_with_extra": 250}, "teachers_in_school": {"total": 390656, "perfect": 314949, "dependency_mismatch": 48955, "value_mismatch": 39230, "answer_mismatch": 35864, "extra_nodes": 6354, "missing_nodes": 5247, "perfect_with_extra": 199}, "movie_festival_awards": {"total": 382336, "perfect": 308427, "value_mismatch": 37193, "answer_mismatch": 33311, "dependency_mismatch": 48778, "missing_nodes": 5610, "extra_nodes": 6408, "perfect_with_extra": 187}}, "results_path": "results/SDAR-100M-ep1-block2/composition/autoregressive/graphs/checkpoint-21350_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block2/composition/autoregressive/checkpoint-21350_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 159866, "value_mismatch": 970061, "dependency_mismatch": 1047231, "answer_mismatch": 829835, "extra_nodes": 51859, "missing_nodes": 834642, "perfect_with_extra": 34, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 325506, "dependency_mismatch": 352515, "answer_mismatch": 276584, "extra_nodes": 18274, "missing_nodes": 281518, "perfect": 55530, "perfect_with_extra": 16}, "movie_festival_awards": {"total": 418816, "value_mismatch": 320111, "answer_mismatch": 275511, "perfect": 50514, "dependency_mismatch": 345916, "missing_nodes": 278336, "extra_nodes": 16092, "perfect_with_extra": 2}, "teachers_in_school": {"total": 430464, "perfect": 53822, "value_mismatch": 324444, "answer_mismatch": 277740, "dependency_mismatch": 348800, "extra_nodes": 17493, "missing_nodes": 274788, "perfect_with_extra": 16}}, "results_path": "results/SDAR-100M-ep1-block2/composition/autoregressive/graphs/checkpoint-21350_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1089335, "value_mismatch": 1083901, "dependency_mismatch": 1192947, "answer_mismatch": 932249, "extra_nodes": 72042, "missing_nodes": 851086, "perfect_with_extra": 670, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block2/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 361623, "value_mismatch": 362923, "answer_mismatch": 309823, "dependency_mismatch": 400498, "missing_nodes": 287105, "extra_nodes": 25695, "perfect_with_extra": 266}, "teachers_in_school": {"total": 821120, "perfect": 368771, "dependency_mismatch": 397755, "value_mismatch": 363674, "answer_mismatch": 313604, "extra_nodes": 23847, "missing_nodes": 280035, "perfect_with_extra": 215}, "movie_festival_awards": {"total": 801152, "perfect": 358941, "value_mismatch": 357304, "answer_mismatch": 308822, "dependency_mismatch": 394694, "missing_nodes": 283946, "extra_nodes": 22500, "perfect_with_extra": 189}}}} diff --git a/SDAR-100M-ep1-block2/composition/autoregressive/graphs/results.json b/SDAR-100M-ep1-block2/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..ab551281944fdef7d1c3972d22725ba7021b1004 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21350_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.971875, + "pass@2": 0.978125, + "pass@4": 0.984375, + "pass@8": 0.9875, + "pass@16": 0.990625, + "pass@32": 0.990625, + "pass@64": 0.99375, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8095238095238095, + "pass@2": 0.8476190476190476, + "pass@4": 0.8793650793650793, + "pass@8": 0.9206349206349206, + "pass@16": 0.9396825396825397, + "pass@32": 0.946031746031746, + "pass@64": 0.9587301587301588, + "pass@128": 0.9682539682539683 + }, + "6": { + "pass@1": 0.9101796407185628, + "pass@2": 0.937125748502994, + "pass@4": 0.9580838323353293, + "pass@8": 0.9820359281437125, + "pass@16": 0.9850299401197605, + "pass@32": 0.9880239520958084, + "pass@64": 0.9880239520958084, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.7840236686390533, + "pass@2": 0.8461538461538461, + "pass@4": 0.8964497041420119, + "pass@8": 0.9230769230769231, + "pass@16": 0.9526627218934911, + "pass@32": 0.9733727810650887, + "pass@64": 0.9763313609467456, + "pass@128": 0.9792899408284024 + }, + "8": { + "pass@1": 0.6955128205128205, + "pass@2": 0.7724358974358975, + "pass@4": 0.8557692307692307, + "pass@8": 0.9102564102564102, + "pass@16": 0.9391025641025641, + "pass@32": 0.9487179487179487, + "pass@64": 0.9551282051282052, + "pass@128": 0.9583333333333334 + }, + "9": { + "pass@1": 0.6309859154929578, + "pass@2": 0.704225352112676, + "pass@4": 0.7943661971830986, + "pass@8": 0.856338028169014, + "pass@16": 0.9042253521126761, + "pass@32": 0.9380281690140845, + "pass@64": 0.9577464788732394, + "pass@128": 0.971830985915493 + }, + "10": { + "pass@1": 0.5339233038348082, + "pass@2": 0.6135693215339233, + "pass@4": 0.6932153392330384, + "pass@8": 0.7905604719764012, + "pass@16": 0.8761061946902655, + "pass@32": 0.9026548672566371, + "pass@64": 0.9233038348082596, + "pass@128": 0.9380530973451328 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.971875, + "5": 0.8095238095238095, + "6": 0.9101796407185628, + "7": 0.7840236686390533, + "8": 0.6955128205128205, + "9": 0.6309859154929578, + "10": 0.5339233038348082 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.978125, + "5": 0.8476190476190476, + "6": 0.937125748502994, + "7": 0.8461538461538461, + "8": 0.7724358974358975, + "9": 0.704225352112676, + "10": 0.6135693215339233 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.984375, + "5": 0.8793650793650793, + "6": 0.9580838323353293, + "7": 0.8964497041420119, + "8": 0.8557692307692307, + "9": 0.7943661971830986, + "10": 0.6932153392330384 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9206349206349206, + "6": 0.9820359281437125, + "7": 0.9230769230769231, + "8": 0.9102564102564102, + "9": 0.856338028169014, + "10": 0.7905604719764012 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9396825396825397, + "6": 0.9850299401197605, + "7": 0.9526627218934911, + "8": 0.9391025641025641, + "9": 0.9042253521126761, + "10": 0.8761061946902655 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.946031746031746, + "6": 0.9880239520958084, + "7": 0.9733727810650887, + "8": 0.9487179487179487, + "9": 0.9380281690140845, + "10": 0.9026548672566371 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9587301587301588, + "6": 0.9880239520958084, + "7": 0.9763313609467456, + "8": 0.9551282051282052, + "9": 0.9577464788732394, + "10": 0.9233038348082596 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9682539682539683, + "6": 0.9910179640718563, + "7": 0.9792899408284024, + "8": 0.9583333333333334, + "9": 0.971830985915493, + "10": 0.9380530973451328 + } + }, + "total_pass_at_k": { + "pass@1": 0.811887875717663, + "pass@2": 0.8530901722391084, + "pass@4": 0.8939547450185749, + "pass@8": 0.9287402904424181, + "pass@16": 0.9533941236068896, + "pass@32": 0.9648767308341777, + "pass@64": 0.972306653157717, + "pass@128": 0.977710233029382 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9678362573099415, + "pass@2": 0.9766081871345029, + "pass@4": 0.9853801169590644, + "pass@8": 0.9883040935672515, + "pass@16": 0.9883040935672515, + "pass@32": 0.9883040935672515, + "pass@64": 0.9883040935672515, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.8264705882352941, + "pass@2": 0.8794117647058823, + "pass@4": 0.9117647058823529, + "pass@8": 0.9176470588235294, + "pass@16": 0.9264705882352942, + "pass@32": 0.9411764705882353, + "pass@64": 0.9588235294117647, + "pass@128": 0.9647058823529412 + }, + "6": { + "pass@1": 0.9242902208201893, + "pass@2": 0.9558359621451105, + "pass@4": 0.9747634069400631, + "pass@8": 0.9873817034700315, + "pass@16": 0.9936908517350158, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.759493670886076, + "pass@2": 0.8322784810126582, + "pass@4": 0.879746835443038, + "pass@8": 0.930379746835443, + "pass@16": 0.9588607594936709, + "pass@32": 0.9746835443037974, + "pass@64": 0.9810126582278481, + "pass@128": 0.9810126582278481 + }, + "8": { + "pass@1": 0.6846846846846847, + "pass@2": 0.8108108108108109, + "pass@4": 0.8558558558558559, + "pass@8": 0.9129129129129129, + "pass@16": 0.93993993993994, + "pass@32": 0.960960960960961, + "pass@64": 0.9669669669669669, + "pass@128": 0.972972972972973 + }, + "9": { + "pass@1": 0.594855305466238, + "pass@2": 0.6720257234726688, + "pass@4": 0.7717041800643086, + "pass@8": 0.8456591639871383, + "pass@16": 0.8938906752411575, + "pass@32": 0.9163987138263665, + "pass@64": 0.9421221864951769, + "pass@128": 0.9581993569131833 + }, + "10": { + "pass@1": 0.4746268656716418, + "pass@2": 0.5850746268656717, + "pass@4": 0.6716417910447762, + "pass@8": 0.746268656716418, + "pass@16": 0.835820895522388, + "pass@32": 0.8805970149253731, + "pass@64": 0.8985074626865671, + "pass@128": 0.9223880597014925 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9678362573099415, + "5": 0.8264705882352941, + "6": 0.9242902208201893, + "7": 0.759493670886076, + "8": 0.6846846846846847, + "9": 0.594855305466238, + "10": 0.4746268656716418 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9766081871345029, + "5": 0.8794117647058823, + "6": 0.9558359621451105, + "7": 0.8322784810126582, + "8": 0.8108108108108109, + "9": 0.6720257234726688, + "10": 0.5850746268656717 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9853801169590644, + "5": 0.9117647058823529, + "6": 0.9747634069400631, + "7": 0.879746835443038, + "8": 0.8558558558558559, + "9": 0.7717041800643086, + "10": 0.6716417910447762 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9176470588235294, + "6": 0.9873817034700315, + "7": 0.930379746835443, + "8": 0.9129129129129129, + "9": 0.8456591639871383, + "10": 0.746268656716418 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9264705882352942, + "6": 0.9936908517350158, + "7": 0.9588607594936709, + "8": 0.93993993993994, + "9": 0.8938906752411575, + "10": 0.835820895522388 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9411764705882353, + "6": 0.9968454258675079, + "7": 0.9746835443037974, + "8": 0.960960960960961, + "9": 0.9163987138263665, + "10": 0.8805970149253731 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9588235294117647, + "6": 0.9968454258675079, + "7": 0.9810126582278481, + "8": 0.9669669669669669, + "9": 0.9421221864951769, + "10": 0.8985074626865671 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9647058823529412, + "6": 0.9968454258675079, + "7": 0.9810126582278481, + "8": 0.972972972972973, + "9": 0.9581993569131833, + "10": 0.9223880597014925 + } + }, + "total_pass_at_k": { + "pass@1": 0.8068295949112823, + "pass@2": 0.8593906930030131, + "pass@4": 0.8962169400736525, + "pass@8": 0.9263475058587212, + "pass@16": 0.9491128222296619, + "pass@32": 0.9625041848008035, + "pass@64": 0.9705390023434884, + "pass@128": 0.9765651155005022 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9644970414201184, + "pass@2": 0.9733727810650887, + "pass@4": 0.9792899408284024, + "pass@8": 0.9881656804733728, + "pass@16": 0.9911242603550295, + "pass@32": 0.9940828402366864, + "pass@64": 0.9940828402366864, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.8376811594202899, + "pass@2": 0.8753623188405797, + "pass@4": 0.9101449275362319, + "pass@8": 0.9246376811594202, + "pass@16": 0.9333333333333333, + "pass@32": 0.9449275362318841, + "pass@64": 0.9565217391304348, + "pass@128": 0.9710144927536232 + }, + "6": { + "pass@1": 0.9369627507163324, + "pass@2": 0.9484240687679083, + "pass@4": 0.9713467048710601, + "pass@8": 0.9828080229226361, + "pass@16": 0.9914040114613181, + "pass@32": 0.9914040114613181, + "pass@64": 0.994269340974212, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.7861271676300579, + "pass@2": 0.869942196531792, + "pass@4": 0.9161849710982659, + "pass@8": 0.9450867052023122, + "pass@16": 0.9682080924855492, + "pass@32": 0.9739884393063584, + "pass@64": 0.9739884393063584, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.6845070422535211, + "pass@2": 0.7859154929577464, + "pass@4": 0.8366197183098592, + "pass@8": 0.9042253521126761, + "pass@16": 0.9323943661971831, + "pass@32": 0.9549295774647887, + "pass@64": 0.9633802816901409, + "pass@128": 0.9690140845070423 + }, + "9": { + "pass@1": 0.5658682634730539, + "pass@2": 0.6586826347305389, + "pass@4": 0.7574850299401198, + "pass@8": 0.8263473053892215, + "pass@16": 0.8892215568862275, + "pass@32": 0.9341317365269461, + "pass@64": 0.9580838323353293, + "pass@128": 0.9670658682634731 + }, + "10": { + "pass@1": 0.48466257668711654, + "pass@2": 0.5460122699386503, + "pass@4": 0.6595092024539877, + "pass@8": 0.745398773006135, + "pass@16": 0.8006134969325154, + "pass@32": 0.843558282208589, + "pass@64": 0.8742331288343558, + "pass@128": 0.8895705521472392 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9644970414201184, + "5": 0.8376811594202899, + "6": 0.9369627507163324, + "7": 0.7861271676300579, + "8": 0.6845070422535211, + "9": 0.5658682634730539, + "10": 0.48466257668711654 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9733727810650887, + "5": 0.8753623188405797, + "6": 0.9484240687679083, + "7": 0.869942196531792, + "8": 0.7859154929577464, + "9": 0.6586826347305389, + "10": 0.5460122699386503 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9792899408284024, + "5": 0.9101449275362319, + "6": 0.9713467048710601, + "7": 0.9161849710982659, + "8": 0.8366197183098592, + "9": 0.7574850299401198, + "10": 0.6595092024539877 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9246376811594202, + "6": 0.9828080229226361, + "7": 0.9450867052023122, + "8": 0.9042253521126761, + "9": 0.8263473053892215, + "10": 0.745398773006135 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9911242603550295, + "5": 0.9333333333333333, + "6": 0.9914040114613181, + "7": 0.9682080924855492, + "8": 0.9323943661971831, + "9": 0.8892215568862275, + "10": 0.8006134969325154 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9449275362318841, + "6": 0.9914040114613181, + "7": 0.9739884393063584, + "8": 0.9549295774647887, + "9": 0.9341317365269461, + "10": 0.843558282208589 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9565217391304348, + "6": 0.994269340974212, + "7": 0.9739884393063584, + "8": 0.9633802816901409, + "9": 0.9580838323353293, + "10": 0.8742331288343558 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9710144927536232, + "6": 0.994269340974212, + "7": 0.9739884393063584, + "8": 0.9690140845070423, + "9": 0.9670658682634731, + "10": 0.8895705521472392 + } + }, + "total_pass_at_k": { + "pass@1": 0.8066841415465269, + "pass@2": 0.851572739187418, + "pass@4": 0.8928571428571429, + "pass@8": 0.9246395806028833, + "pass@16": 0.9456094364351245, + "pass@32": 0.9600262123197904, + "pass@64": 0.9685452162516383, + "pass@128": 0.9737876802096985 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.968, + "pass@2": 0.976, + "pass@4": 0.983, + "pass@8": 0.988, + "pass@16": 0.99, + "pass@32": 0.991, + "pass@64": 0.992, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.825, + "pass@2": 0.868, + "pass@4": 0.901, + "pass@8": 0.921, + "pass@16": 0.933, + "pass@32": 0.944, + "pass@64": 0.958, + "pass@128": 0.968 + }, + "6": { + "pass@1": 0.924, + "pass@2": 0.947, + "pass@4": 0.968, + "pass@8": 0.984, + "pass@16": 0.99, + "pass@32": 0.992, + "pass@64": 0.993, + "pass@128": 0.994 + }, + "7": { + "pass@1": 0.777, + "pass@2": 0.85, + "pass@4": 0.898, + "pass@8": 0.933, + "pass@16": 0.96, + "pass@32": 0.974, + "pass@64": 0.977, + "pass@128": 0.978 + }, + "8": { + "pass@1": 0.688, + "pass@2": 0.79, + "pass@4": 0.849, + "pass@8": 0.909, + "pass@16": 0.937, + "pass@32": 0.955, + "pass@64": 0.962, + "pass@128": 0.967 + }, + "9": { + "pass@1": 0.598, + "pass@2": 0.679, + "pass@4": 0.775, + "pass@8": 0.843, + "pass@16": 0.896, + "pass@32": 0.93, + "pass@64": 0.953, + "pass@128": 0.966 + }, + "10": { + "pass@1": 0.498, + "pass@2": 0.582, + "pass@4": 0.675, + "pass@8": 0.761, + "pass@16": 0.838, + "pass@32": 0.876, + "pass@64": 0.899, + "pass@128": 0.917 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.998, + "4": 0.968, + "5": 0.825, + "6": 0.924, + "7": 0.777, + "8": 0.688, + "9": 0.598, + "10": 0.498 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.976, + "5": 0.868, + "6": 0.947, + "7": 0.85, + "8": 0.79, + "9": 0.679, + "10": 0.582 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.983, + "5": 0.901, + "6": 0.968, + "7": 0.898, + "8": 0.849, + "9": 0.775, + "10": 0.675 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.988, + "5": 0.921, + "6": 0.984, + "7": 0.933, + "8": 0.909, + "9": 0.843, + "10": 0.761 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.99, + "5": 0.933, + "6": 0.99, + "7": 0.96, + "8": 0.937, + "9": 0.896, + "10": 0.838 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.991, + "5": 0.944, + "6": 0.992, + "7": 0.974, + "8": 0.955, + "9": 0.93, + "10": 0.876 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.992, + "5": 0.958, + "6": 0.993, + "7": 0.977, + "8": 0.962, + "9": 0.953, + "10": 0.899 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.994, + "5": 0.968, + "6": 0.994, + "7": 0.978, + "8": 0.967, + "9": 0.966, + "10": 0.917 + } + }, + "total_pass_at_k": { + "pass@1": 0.8084444444444444, + "pass@2": 0.8546666666666667, + "pass@4": 0.8943333333333333, + "pass@8": 0.9265555555555556, + "pass@16": 0.9493333333333334, + "pass@32": 0.9624444444444444, + "pass@64": 0.9704444444444444, + "pass@128": 0.976 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21350_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3654390934844193, + "pass@2": 0.49291784702549574, + "pass@4": 0.5949008498583569, + "pass@8": 0.6968838526912181, + "pass@16": 0.773371104815864, + "pass@32": 0.8186968838526912, + "pass@64": 0.8470254957507082, + "pass@128": 0.8725212464589235 + }, + "12": { + "pass@1": 0.4406779661016949, + "pass@2": 0.53954802259887, + "pass@4": 0.6101694915254238, + "pass@8": 0.7062146892655368, + "pass@16": 0.7627118644067796, + "pass@32": 0.7824858757062146, + "pass@64": 0.8192090395480226, + "pass@128": 0.8389830508474576 + }, + "13": { + "pass@1": 0.2445141065830721, + "pass@2": 0.3072100313479624, + "pass@4": 0.3981191222570533, + "pass@8": 0.47648902821316613, + "pass@16": 0.5203761755485894, + "pass@32": 0.5736677115987461, + "pass@64": 0.6050156739811913, + "pass@128": 0.6520376175548589 + }, + "14": { + "pass@1": 0.11838006230529595, + "pass@2": 0.16199376947040497, + "pass@4": 0.21806853582554517, + "pass@8": 0.26791277258566976, + "pass@16": 0.3146417445482866, + "pass@32": 0.37383177570093457, + "pass@64": 0.40498442367601245, + "pass@128": 0.4672897196261682 + }, + "15": { + "pass@1": 0.07514450867052024, + "pass@2": 0.09826589595375723, + "pass@4": 0.12138728323699421, + "pass@8": 0.13872832369942195, + "pass@16": 0.1676300578034682, + "pass@32": 0.18786127167630057, + "pass@64": 0.20520231213872833, + "pass@128": 0.22254335260115607 + }, + "16": { + "pass@1": 0.012903225806451613, + "pass@2": 0.03870967741935484, + "pass@4": 0.05161290322580645, + "pass@8": 0.06774193548387097, + "pass@16": 0.09032258064516129, + "pass@32": 0.11935483870967742, + "pass@64": 0.13548387096774195, + "pass@128": 0.14516129032258066 + }, + "17": { + "pass@1": 0.002890173410404624, + "pass@2": 0.002890173410404624, + "pass@4": 0.008670520231213872, + "pass@8": 0.011560693641618497, + "pass@16": 0.011560693641618497, + "pass@32": 0.014450867052023121, + "pass@64": 0.017341040462427744, + "pass@128": 0.017341040462427744 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.002857142857142857, + "pass@16": 0.002857142857142857, + "pass@32": 0.002857142857142857, + "pass@64": 0.005714285714285714, + "pass@128": 0.005714285714285714 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.3654390934844193, + "12": 0.4406779661016949, + "13": 0.2445141065830721, + "14": 0.11838006230529595, + "15": 0.07514450867052024, + "16": 0.012903225806451613, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.49291784702549574, + "12": 0.53954802259887, + "13": 0.3072100313479624, + "14": 0.16199376947040497, + "15": 0.09826589595375723, + "16": 0.03870967741935484, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.5949008498583569, + "12": 0.6101694915254238, + "13": 0.3981191222570533, + "14": 0.21806853582554517, + "15": 0.12138728323699421, + "16": 0.05161290322580645, + "17": 0.008670520231213872, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.6968838526912181, + "12": 0.7062146892655368, + "13": 0.47648902821316613, + "14": 0.26791277258566976, + "15": 0.13872832369942195, + "16": 0.06774193548387097, + "17": 0.011560693641618497, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.773371104815864, + "12": 0.7627118644067796, + "13": 0.5203761755485894, + "14": 0.3146417445482866, + "15": 0.1676300578034682, + "16": 0.09032258064516129, + "17": 0.011560693641618497, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8186968838526912, + "12": 0.7824858757062146, + "13": 0.5736677115987461, + "14": 0.37383177570093457, + "15": 0.18786127167630057, + "16": 0.11935483870967742, + "17": 0.014450867052023121, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8470254957507082, + "12": 0.8192090395480226, + "13": 0.6050156739811913, + "14": 0.40498442367601245, + "15": 0.20520231213872833, + "16": 0.13548387096774195, + "17": 0.017341040462427744, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8725212464589235, + "12": 0.8389830508474576, + "13": 0.6520376175548589, + "14": 0.4672897196261682, + "15": 0.22254335260115607, + "16": 0.14516129032258066, + "17": 0.017341040462427744, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.12838038632986626, + "pass@2": 0.16701337295690935, + "pass@4": 0.20326894502228826, + "pass@8": 0.24011887072808322, + "pass@16": 0.2677563150074294, + "pass@32": 0.29034175334323925, + "pass@64": 0.3069836552748886, + "pass@128": 0.32481426448737 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4095238095238095, + "pass@2": 0.5301587301587302, + "pass@4": 0.638095238095238, + "pass@8": 0.7365079365079366, + "pass@16": 0.8095238095238095, + "pass@32": 0.8380952380952381, + "pass@64": 0.8761904761904762, + "pass@128": 0.8920634920634921 + }, + "12": { + "pass@1": 0.40672782874617736, + "pass@2": 0.5015290519877675, + "pass@4": 0.5932721712538226, + "pass@8": 0.6819571865443425, + "pass@16": 0.7492354740061162, + "pass@32": 0.7859327217125383, + "pass@64": 0.8195718654434251, + "pass@128": 0.8440366972477065 + }, + "13": { + "pass@1": 0.25, + "pass@2": 0.32407407407407407, + "pass@4": 0.37962962962962965, + "pass@8": 0.44135802469135804, + "pass@16": 0.5246913580246914, + "pass@32": 0.5679012345679012, + "pass@64": 0.6141975308641975, + "pass@128": 0.654320987654321 + }, + "14": { + "pass@1": 0.11572700296735905, + "pass@2": 0.1513353115727003, + "pass@4": 0.21364985163204747, + "pass@8": 0.2551928783382789, + "pass@16": 0.28486646884273, + "pass@32": 0.34124629080118696, + "pass@64": 0.3887240356083086, + "pass@128": 0.44510385756676557 + }, + "15": { + "pass@1": 0.05555555555555555, + "pass@2": 0.08024691358024691, + "pass@4": 0.09567901234567901, + "pass@8": 0.11419753086419752, + "pass@16": 0.13580246913580246, + "pass@32": 0.15123456790123457, + "pass@64": 0.1882716049382716, + "pass@128": 0.2222222222222222 + }, + "16": { + "pass@1": 0.011799410029498525, + "pass@2": 0.02359882005899705, + "pass@4": 0.02359882005899705, + "pass@8": 0.029498525073746312, + "pass@16": 0.035398230088495575, + "pass@32": 0.05309734513274336, + "pass@64": 0.06489675516224189, + "pass@128": 0.07964601769911504 + }, + "17": { + "pass@1": 0.0030120481927710845, + "pass@2": 0.0030120481927710845, + "pass@4": 0.006024096385542169, + "pass@8": 0.012048192771084338, + "pass@16": 0.018072289156626505, + "pass@32": 0.018072289156626505, + "pass@64": 0.024096385542168676, + "pass@128": 0.030120481927710843 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0032258064516129032, + "pass@4": 0.0032258064516129032, + "pass@8": 0.0032258064516129032, + "pass@16": 0.0032258064516129032, + "pass@32": 0.0064516129032258064, + "pass@64": 0.012903225806451613, + "pass@128": 0.01935483870967742 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.003003003003003003, + "pass@32": 0.003003003003003003, + "pass@64": 0.003003003003003003, + "pass@128": 0.003003003003003003 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4095238095238095, + "12": 0.40672782874617736, + "13": 0.25, + "14": 0.11572700296735905, + "15": 0.05555555555555555, + "16": 0.011799410029498525, + "17": 0.0030120481927710845, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5301587301587302, + "12": 0.5015290519877675, + "13": 0.32407407407407407, + "14": 0.1513353115727003, + "15": 0.08024691358024691, + "16": 0.02359882005899705, + "17": 0.0030120481927710845, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.638095238095238, + "12": 0.5932721712538226, + "13": 0.37962962962962965, + "14": 0.21364985163204747, + "15": 0.09567901234567901, + "16": 0.02359882005899705, + "17": 0.006024096385542169, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7365079365079366, + "12": 0.6819571865443425, + "13": 0.44135802469135804, + "14": 0.2551928783382789, + "15": 0.11419753086419752, + "16": 0.029498525073746312, + "17": 0.012048192771084338, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8095238095238095, + "12": 0.7492354740061162, + "13": 0.5246913580246914, + "14": 0.28486646884273, + "15": 0.13580246913580246, + "16": 0.035398230088495575, + "17": 0.018072289156626505, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.8380952380952381, + "12": 0.7859327217125383, + "13": 0.5679012345679012, + "14": 0.34124629080118696, + "15": 0.15123456790123457, + "16": 0.05309734513274336, + "17": 0.018072289156626505, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@64": { + "11": 0.8761904761904762, + "12": 0.8195718654434251, + "13": 0.6141975308641975, + "14": 0.3887240356083086, + "15": 0.1882716049382716, + "16": 0.06489675516224189, + "17": 0.024096385542168676, + "18": 0.012903225806451613, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@128": { + "11": 0.8920634920634921, + "12": 0.8440366972477065, + "13": 0.654320987654321, + "14": 0.44510385756676557, + "15": 0.2222222222222222, + "16": 0.07964601769911504, + "17": 0.030120481927710843, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.003003003003003003 + } + }, + "total_pass_at_k": { + "pass@1": 0.12377750611246943, + "pass@2": 0.15984107579462103, + "pass@4": 0.19315403422982885, + "pass@8": 0.22493887530562348, + "pass@16": 0.2536674816625917, + "pass@32": 0.27383863080684595, + "pass@64": 0.29645476772616136, + "pass@128": 0.316320293398533 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4397590361445783, + "pass@2": 0.5391566265060241, + "pass@4": 0.6265060240963856, + "pass@8": 0.7108433734939759, + "pass@16": 0.7981927710843374, + "pass@32": 0.8554216867469879, + "pass@64": 0.8765060240963856, + "pass@128": 0.9036144578313253 + }, + "12": { + "pass@1": 0.3573667711598746, + "pass@2": 0.43260188087774293, + "pass@4": 0.5141065830721003, + "pass@8": 0.6112852664576802, + "pass@16": 0.6833855799373041, + "pass@32": 0.7586206896551724, + "pass@64": 0.7931034482758621, + "pass@128": 0.8150470219435737 + }, + "13": { + "pass@1": 0.23249299719887956, + "pass@2": 0.3025210084033613, + "pass@4": 0.38095238095238093, + "pass@8": 0.484593837535014, + "pass@16": 0.5518207282913166, + "pass@32": 0.6050420168067226, + "pass@64": 0.6526610644257703, + "pass@128": 0.7058823529411765 + }, + "14": { + "pass@1": 0.10818713450292397, + "pass@2": 0.15204678362573099, + "pass@4": 0.2134502923976608, + "pass@8": 0.2631578947368421, + "pass@16": 0.3304093567251462, + "pass@32": 0.38596491228070173, + "pass@64": 0.4298245614035088, + "pass@128": 0.47076023391812866 + }, + "15": { + "pass@1": 0.06060606060606061, + "pass@2": 0.08484848484848485, + "pass@4": 0.09696969696969697, + "pass@8": 0.13333333333333333, + "pass@16": 0.1393939393939394, + "pass@32": 0.18181818181818182, + "pass@64": 0.24242424242424243, + "pass@128": 0.28484848484848485 + }, + "16": { + "pass@1": 0.005698005698005698, + "pass@2": 0.014245014245014245, + "pass@4": 0.03418803418803419, + "pass@8": 0.05982905982905983, + "pass@16": 0.08262108262108261, + "pass@32": 0.1111111111111111, + "pass@64": 0.1225071225071225, + "pass@128": 0.150997150997151 + }, + "17": { + "pass@1": 0.006211180124223602, + "pass@2": 0.015527950310559006, + "pass@4": 0.021739130434782608, + "pass@8": 0.021739130434782608, + "pass@16": 0.024844720496894408, + "pass@32": 0.034161490683229816, + "pass@64": 0.040372670807453416, + "pass@128": 0.049689440993788817 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0029411764705882353, + "pass@8": 0.0029411764705882353, + "pass@16": 0.011764705882352941, + "pass@32": 0.011764705882352941, + "pass@64": 0.011764705882352941, + "pass@128": 0.01764705882352941 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0029940119760479044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4397590361445783, + "12": 0.3573667711598746, + "13": 0.23249299719887956, + "14": 0.10818713450292397, + "15": 0.06060606060606061, + "16": 0.005698005698005698, + "17": 0.006211180124223602, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5391566265060241, + "12": 0.43260188087774293, + "13": 0.3025210084033613, + "14": 0.15204678362573099, + "15": 0.08484848484848485, + "16": 0.014245014245014245, + "17": 0.015527950310559006, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6265060240963856, + "12": 0.5141065830721003, + "13": 0.38095238095238093, + "14": 0.2134502923976608, + "15": 0.09696969696969697, + "16": 0.03418803418803419, + "17": 0.021739130434782608, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7108433734939759, + "12": 0.6112852664576802, + "13": 0.484593837535014, + "14": 0.2631578947368421, + "15": 0.13333333333333333, + "16": 0.05982905982905983, + "17": 0.021739130434782608, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.7981927710843374, + "12": 0.6833855799373041, + "13": 0.5518207282913166, + "14": 0.3304093567251462, + "15": 0.1393939393939394, + "16": 0.08262108262108261, + "17": 0.024844720496894408, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8554216867469879, + "12": 0.7586206896551724, + "13": 0.6050420168067226, + "14": 0.38596491228070173, + "15": 0.18181818181818182, + "16": 0.1111111111111111, + "17": 0.034161490683229816, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8765060240963856, + "12": 0.7931034482758621, + "13": 0.6526610644257703, + "14": 0.4298245614035088, + "15": 0.24242424242424243, + "16": 0.1225071225071225, + "17": 0.040372670807453416, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.9036144578313253, + "12": 0.8150470219435737, + "13": 0.7058823529411765, + "14": 0.47076023391812866, + "15": 0.28484848484848485, + "16": 0.150997150997151, + "17": 0.049689440993788817, + "18": 0.01764705882352941, + "19": 0.0, + "20": 0.0029940119760479044 + } + }, + "total_pass_at_k": { + "pass@1": 0.12013083556348499, + "pass@2": 0.15313707998810586, + "pass@4": 0.18822479928635147, + "pass@8": 0.22807017543859648, + "pass@16": 0.2616711269699673, + "pass@32": 0.2937853107344633, + "pass@64": 0.3163841807909605, + "pass@128": 0.33987511150758254 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.404, + "pass@2": 0.52, + "pass@4": 0.619, + "pass@8": 0.714, + "pass@16": 0.793, + "pass@32": 0.837, + "pass@64": 0.866, + "pass@128": 0.889 + }, + "12": { + "pass@1": 0.403, + "pass@2": 0.493, + "pass@4": 0.574, + "pass@8": 0.668, + "pass@16": 0.733, + "pass@32": 0.776, + "pass@64": 0.811, + "pass@128": 0.833 + }, + "13": { + "pass@1": 0.242, + "pass@2": 0.311, + "pass@4": 0.386, + "pass@8": 0.468, + "pass@16": 0.533, + "pass@32": 0.583, + "pass@64": 0.625, + "pass@128": 0.672 + }, + "14": { + "pass@1": 0.114, + "pass@2": 0.155, + "pass@4": 0.215, + "pass@8": 0.262, + "pass@16": 0.31, + "pass@32": 0.367, + "pass@64": 0.408, + "pass@128": 0.461 + }, + "15": { + "pass@1": 0.064, + "pass@2": 0.088, + "pass@4": 0.105, + "pass@8": 0.129, + "pass@16": 0.148, + "pass@32": 0.174, + "pass@64": 0.212, + "pass@128": 0.243 + }, + "16": { + "pass@1": 0.01, + "pass@2": 0.025, + "pass@4": 0.036, + "pass@8": 0.052, + "pass@16": 0.069, + "pass@32": 0.094, + "pass@64": 0.107, + "pass@128": 0.125 + }, + "17": { + "pass@1": 0.004, + "pass@2": 0.007, + "pass@4": 0.012, + "pass@8": 0.015, + "pass@16": 0.018, + "pass@32": 0.022, + "pass@64": 0.027, + "pass@128": 0.032 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.002, + "pass@8": 0.003, + "pass@16": 0.006, + "pass@32": 0.007, + "pass@64": 0.01, + "pass@128": 0.014 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.001, + "pass@32": 0.001, + "pass@64": 0.001, + "pass@128": 0.002 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.404, + "12": 0.403, + "13": 0.242, + "14": 0.114, + "15": 0.064, + "16": 0.01, + "17": 0.004, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.52, + "12": 0.493, + "13": 0.311, + "14": 0.155, + "15": 0.088, + "16": 0.025, + "17": 0.007, + "18": 0.001, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.619, + "12": 0.574, + "13": 0.386, + "14": 0.215, + "15": 0.105, + "16": 0.036, + "17": 0.012, + "18": 0.002, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.714, + "12": 0.668, + "13": 0.468, + "14": 0.262, + "15": 0.129, + "16": 0.052, + "17": 0.015, + "18": 0.003, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.793, + "12": 0.733, + "13": 0.533, + "14": 0.31, + "15": 0.148, + "16": 0.069, + "17": 0.018, + "18": 0.006, + "19": 0.0, + "20": 0.001 + }, + "pass@32": { + "11": 0.837, + "12": 0.776, + "13": 0.583, + "14": 0.367, + "15": 0.174, + "16": 0.094, + "17": 0.022, + "18": 0.007, + "19": 0.0, + "20": 0.001 + }, + "pass@64": { + "11": 0.866, + "12": 0.811, + "13": 0.625, + "14": 0.408, + "15": 0.212, + "16": 0.107, + "17": 0.027, + "18": 0.01, + "19": 0.0, + "20": 0.001 + }, + "pass@128": { + "11": 0.889, + "12": 0.833, + "13": 0.672, + "14": 0.461, + "15": 0.243, + "16": 0.125, + "17": 0.032, + "18": 0.014, + "19": 0.0, + "20": 0.002 + } + }, + "total_pass_at_k": { + "pass@1": 0.1241, + "pass@2": 0.16, + "pass@4": 0.1949, + "pass@8": 0.2311, + "pass@16": 0.2611, + "pass@32": 0.2861, + "pass@64": 0.3067, + "pass@128": 0.3271 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block2/composition/autoregressive/summary.csv b/SDAR-100M-ep1-block2/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..1ddf5901a510bad69621fad9f4053d877f9ca9b6 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21350,0.9110954861111111,0.35171796875,0.6166862664473685,0.0,0.0,0.0,199.92764930555555,320.44338671875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block2/composition/autoregressive/summary.json b/SDAR-100M-ep1-block2/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c17371e5fb3a85e0b10fe5d141e6fd0c8d9b36e6 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21350", + "id_acc": 0.9110954861111111, + "ood_acc": 0.35171796875, + "total_acc": 0.6166862664473685, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.92764930555555, + "ood_avg_resp_len": 320.44338671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 499909, + "answer_accuracy": 0.617378922304774, + "avg_response_len": 251.66101456291494, + "resp_tokens_sum": 203776970, + "pass_at_k": { + "pass@1": 0.617378922304774, + "pass@2": 0.6702401358535359, + "pass@4": 0.7126881271914993, + "pass@8": 0.7469478039476964, + "pass@16": 0.7751029216952456, + "pass@32": 0.7984062550092523, + "pass@64": 0.8175423265721407, + "pass@128": 0.833386025924755 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8190680309734514, + "pass@2": 0.8758038783592319, + "pass@4": 0.9097118844657693, + "pass@8": 0.9301279923292359, + "pass@16": 0.945463319652103, + "pass@32": 0.9597276818955355, + "pass@64": 0.971984946225041, + "pass@128": 0.9793510324483776 + }, + "2": { + "pass@1": 0.9998539719626168, + "pass@2": 0.9999996167243114, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974197247706422, + "pass@2": 0.9979062071082859, + "pass@4": 0.9985759549551719, + "pass@8": 0.9993549275868031, + "pass@16": 0.9998791891503424, + "pass@32": 0.999997274769305, + "pass@64": 0.999999999917904, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9796875, + "pass@2": 0.9833226962352362, + "pass@4": 0.9855182915612108, + "pass@8": 0.9877684577554199, + "pass@16": 0.9898908624050529, + "pass@32": 0.9915912787745874, + "pass@64": 0.9929543613492345, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8453621031746031, + "pass@2": 0.878759686289214, + "pass@4": 0.9038273906237906, + "pass@8": 0.9236315572169654, + "pass@16": 0.9401206361112525, + "pass@32": 0.9527244490497119, + "pass@64": 0.9623853430676196, + "pass@128": 0.9682539682539683 + }, + "6": { + "pass@1": 0.9475580089820359, + "pass@2": 0.9595694192324015, + "pass@4": 0.9684871766590554, + "pass@8": 0.9760311732040033, + "pass@16": 0.9819037036422803, + "pass@32": 0.9859049285559663, + "pass@64": 0.9889859412882993, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9068047337278107, + "pass@2": 0.932142305945115, + "pass@4": 0.947185020234157, + "pass@8": 0.9582908289110658, + "pass@16": 0.9681797653463373, + "pass@32": 0.9767144305121371, + "pass@64": 0.9830296516855794, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8812349759615384, + "pass@2": 0.9134710024227741, + "pass@4": 0.9344750253574073, + "pass@8": 0.9487608950939497, + "pass@16": 0.9589510177536014, + "pass@32": 0.9657383512461981, + "pass@64": 0.9703603198365051, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.8417913732394366, + "pass@2": 0.8856545275590555, + "pass@4": 0.9142673059353488, + "pass@8": 0.9349954551526746, + "pass@16": 0.9509292121287449, + "pass@32": 0.9633821177411638, + "pass@64": 0.973017756199603, + "pass@128": 0.9802816901408451 + }, + "11": { + "pass@1": 0.7452195467422096, + "pass@2": 0.8180329320113316, + "pass@4": 0.8652162037641893, + "pass@8": 0.8969106210163632, + "pass@16": 0.9210920894565872, + "pass@32": 0.9403227057391875, + "pass@64": 0.9548937056898483, + "pass@128": 0.9660056657223796 + }, + "12": { + "pass@1": 0.6948490466101694, + "pass@2": 0.7856954685484225, + "pass@4": 0.8424545043628017, + "pass@8": 0.8799306789509234, + "pass@16": 0.9062520096316939, + "pass@32": 0.924809518611065, + "pass@64": 0.938702946618994, + "pass@128": 0.9491525423728814 + }, + "13": { + "pass@1": 0.5179516065830722, + "pass@2": 0.6325184200133289, + "pass@4": 0.7205692302764656, + "pass@8": 0.7828447072240617, + "pass@16": 0.8262471661892159, + "pass@32": 0.8598153482049776, + "pass@64": 0.8862313514669503, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.3964417834890966, + "pass@2": 0.5047361376848922, + "pass@4": 0.5946405522323733, + "pass@8": 0.6680429180152108, + "pass@16": 0.7301454909544141, + "pass@32": 0.7804329625460306, + "pass@64": 0.8173909998798203, + "pass@128": 0.8411214953271028 + }, + "15": { + "pass@1": 0.29612987716763006, + "pass@2": 0.3869219653179187, + "pass@4": 0.46993725170191875, + "pass@8": 0.5421051675527985, + "pass@16": 0.6023771645269487, + "pass@32": 0.6529647288435331, + "pass@64": 0.6944795629555712, + "pass@128": 0.7283236994219653 + }, + "16": { + "pass@1": 0.24740423387096774, + "pass@2": 0.32904813309626596, + "pass@4": 0.40458356405046153, + "pass@8": 0.4702012137774771, + "pass@16": 0.5247288669662102, + "pass@32": 0.5706876527594704, + "pass@64": 0.6115942057418606, + "pass@128": 0.6451612903225806 + }, + "17": { + "pass@1": 0.16986542630057805, + "pass@2": 0.22985803729916698, + "pass@4": 0.28688736695124073, + "pass@8": 0.33843467932243226, + "pass@16": 0.38700181611143314, + "pass@32": 0.43410878184626833, + "pass@64": 0.47804135764926775, + "pass@128": 0.5202312138728323 + }, + "18": { + "pass@1": 0.1679017857142857, + "pass@2": 0.2257529527559055, + "pass@4": 0.2848023300658846, + "pass@8": 0.34394438573386016, + "pass@16": 0.3988663230247612, + "pass@32": 0.4435387309873241, + "pass@64": 0.47824795716449336, + "pass@128": 0.5085714285714286 + }, + "19": { + "pass@1": 0.16246715465465467, + "pass@2": 0.2220851510959779, + "pass@4": 0.2795086211070464, + "pass@8": 0.3279673225850375, + "pass@16": 0.36686001138513913, + "pass@32": 0.4010357183857945, + "pass@64": 0.4330668055174505, + "pass@128": 0.46546546546546547 + }, + "20": { + "pass@1": 0.15582770270270271, + "pass@2": 0.21083792946489788, + "pass@4": 0.2651404334593312, + "pass@8": 0.31510224505908696, + "pass@16": 0.3582141742580615, + "pass@32": 0.3942354522194376, + "pass@64": 0.4242069841105825, + "pass@128": 0.45345345345345345 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 507526, + "answer_accuracy": 0.6180899259547935, + "avg_response_len": 260.83856196414655, + "resp_tokens_sum": 214179760, + "pass_at_k": { + "pass@1": 0.6180899259547935, + "pass@2": 0.6704741094936216, + "pass@4": 0.714109435375526, + "pass@8": 0.7502174513426444, + "pass@16": 0.7800515230208226, + "pass@32": 0.804595822219313, + "pass@64": 0.8246596996146075, + "pass@128": 0.8419329696024942 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7672785659509203, + "pass@2": 0.8221017402540941, + "pass@4": 0.8567607347586148, + "pass@8": 0.8803886350813609, + "pass@16": 0.8990823514930147, + "pass@32": 0.9155576005344653, + "pass@64": 0.9292128475444902, + "pass@128": 0.9386503067484663 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9975378787878788, + "pass@2": 0.999380368647101, + "pass@4": 0.9999463317085363, + "pass@8": 0.999999574463101, + "pass@16": 0.99999999998634, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9803069526627219, + "pass@2": 0.9856821186460419, + "pass@4": 0.9884984804968907, + "pass@8": 0.9901388748274397, + "pass@16": 0.9915489433224962, + "pass@32": 0.993055431506491, + "pass@64": 0.9948274051186248, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.8750452898550725, + "pass@2": 0.8991081108068009, + "pass@4": 0.91609362552507, + "pass@8": 0.9283145339273063, + "pass@16": 0.9387362873527924, + "pass@32": 0.9495945434273302, + "pass@64": 0.9611962026508659, + "pass@128": 0.9739130434782609 + }, + "6": { + "pass@1": 0.9597063037249284, + "pass@2": 0.9688247354646573, + "pass@4": 0.9755671450810769, + "pass@8": 0.9809583841351045, + "pass@16": 0.9856212779919914, + "pass@32": 0.9895090347929678, + "pass@64": 0.9923046542590954, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9184429190751445, + "pass@2": 0.9380771101679485, + "pass@4": 0.9515284368355686, + "pass@8": 0.9620686520464642, + "pass@16": 0.9694759351290418, + "pass@32": 0.9729727131601149, + "pass@64": 0.9739292569091923, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.8584507042253521, + "pass@2": 0.8973130614395034, + "pass@4": 0.9233392773966632, + "pass@8": 0.9433777973806029, + "pass@16": 0.9601626523567398, + "pass@32": 0.9730733897077097, + "pass@64": 0.9801915294569885, + "pass@128": 0.9830985915492958 + }, + "9": { + "pass@1": 0.8166401571856288, + "pass@2": 0.8604763897402046, + "pass@4": 0.8899476901090955, + "pass@8": 0.9137484797589711, + "pass@16": 0.9336248046731362, + "pass@32": 0.9489223240336991, + "pass@64": 0.9611196746977648, + "pass@128": 0.9760479041916168 + }, + "11": { + "pass@1": 0.759765625, + "pass@2": 0.8237484880466749, + "pass@4": 0.8661105835038693, + "pass@8": 0.8974093782721269, + "pass@16": 0.9220586483640653, + "pass@32": 0.9400776540050912, + "pass@64": 0.9516687417338571, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.6983248432601881, + "pass@2": 0.792604765754202, + "pass@4": 0.8538179523210068, + "pass@8": 0.8929835881620504, + "pass@16": 0.9185320929903009, + "pass@32": 0.9352178761761952, + "pass@64": 0.9478804552757559, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5278361344537815, + "pass@2": 0.63536910827323, + "pass@4": 0.7126288210822387, + "pass@8": 0.7701978399669224, + "pass@16": 0.8166211744373886, + "pass@32": 0.8550839822738148, + "pass@64": 0.8865038689796082, + "pass@128": 0.9131652661064426 + }, + "14": { + "pass@1": 0.41721491228070173, + "pass@2": 0.5287199400239444, + "pass@4": 0.6211705283001904, + "pass@8": 0.6923516248529362, + "pass@16": 0.7462194130033619, + "pass@32": 0.7883736937027563, + "pass@64": 0.8222691206866892, + "pass@128": 0.8508771929824561 + }, + "15": { + "pass@1": 0.28927556818181815, + "pass@2": 0.38632150739680254, + "pass@4": 0.477739166411017, + "pass@8": 0.5566535711234109, + "pass@16": 0.6215893466194053, + "pass@32": 0.6719853954938052, + "pass@64": 0.7093504805012789, + "pass@128": 0.7393939393939394 + }, + "16": { + "pass@1": 0.21276264245014245, + "pass@2": 0.28995263813177213, + "pass@4": 0.3666882563824822, + "pass@8": 0.4346854612065274, + "pass@16": 0.490842220216734, + "pass@32": 0.5378512803345692, + "pass@64": 0.5810806305675018, + "pass@128": 0.6239316239316239 + }, + "17": { + "pass@1": 0.19004755434782608, + "pass@2": 0.2589851200665132, + "pass@4": 0.3332288832684735, + "pass@8": 0.4048084183288024, + "pass@16": 0.46686980557788155, + "pass@32": 0.5180633036985619, + "pass@64": 0.5611210047877057, + "pass@128": 0.5962732919254659 + }, + "18": { + "pass@1": 0.17541360294117647, + "pass@2": 0.238078248031496, + "pass@4": 0.3021544674562736, + "pass@8": 0.36484250871024415, + "pass@16": 0.42552854294852027, + "pass@32": 0.4830248444609159, + "pass@64": 0.532842036580717, + "pass@128": 0.5705882352941176 + }, + "19": { + "pass@1": 0.14276413690476192, + "pass@2": 0.19880974350862385, + "pass@4": 0.25782905819808244, + "pass@8": 0.3139917285710976, + "pass@16": 0.3642751043986496, + "pass@32": 0.4084618548346345, + "pass@64": 0.44438723572602956, + "pass@128": 0.47619047619047616 + }, + "20": { + "pass@1": 0.14429734281437126, + "pass@2": 0.2033662743410817, + "pass@4": 0.26359721810597025, + "pass@8": 0.3163735906613064, + "pass@16": 0.359637297892788, + "pass@32": 0.395871809493765, + "pass@64": 0.42783318947128957, + "pass@128": 0.46107784431137727 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 492346, + "answer_accuracy": 0.6145475515258029, + "avg_response_len": 277.75934778918355, + "resp_tokens_sum": 222527457, + "pass_at_k": { + "pass@1": 0.6145475515258029, + "pass@2": 0.66511725084383, + "pass@4": 0.7072032517044973, + "pass@8": 0.7420143982533723, + "pass@16": 0.7702823493585079, + "pass@32": 0.7934652060341785, + "pass@64": 0.8130399718603196, + "pass@128": 0.8304841028918357 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.837080223880597, + "pass@2": 0.8841825567046653, + "pass@4": 0.9129999748165799, + "pass@8": 0.9340309207425049, + "pass@16": 0.9503269298903634, + "pass@32": 0.9633252118626652, + "pass@64": 0.9742339636240699, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994989067055393, + "pass@2": 0.9999314897500058, + "pass@4": 0.9999986759125955, + "pass@8": 0.9999999997431218, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9685215643274854, + "pass@2": 0.9750720921397983, + "pass@4": 0.9805761952453314, + "pass@8": 0.9847048683048196, + "pass@16": 0.9872249490269424, + "pass@32": 0.988723427016145, + "pass@64": 0.9897569491732896, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.8611213235294117, + "pass@2": 0.8884335919407136, + "pass@4": 0.9081887436864504, + "pass@8": 0.924253573036253, + "pass@16": 0.9389244449350933, + "pass@32": 0.9523447840529764, + "pass@64": 0.9630246066646815, + "pass@128": 0.9705882352941176 + }, + "6": { + "pass@1": 0.9733832807570978, + "pass@2": 0.9828349716833503, + "pass@4": 0.9887145172508008, + "pass@8": 0.9916570561272308, + "pass@16": 0.9930273048146369, + "pass@32": 0.9941869150417751, + "pass@64": 0.9952583726678965, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8899080300632911, + "pass@2": 0.9240950176916176, + "pass@4": 0.945723327898886, + "pass@8": 0.9597153776968665, + "pass@16": 0.9702119965930145, + "pass@32": 0.9779051128332572, + "pass@64": 0.9826542051427243, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.8529466966966966, + "pass@2": 0.8928994200893807, + "pass@4": 0.9212113263432158, + "pass@8": 0.94219890076564, + "pass@16": 0.956245171195436, + "pass@32": 0.9650723844889141, + "pass@64": 0.9708403647709795, + "pass@128": 0.972972972972973 + }, + "9": { + "pass@1": 0.8221463022508039, + "pass@2": 0.8635371040838545, + "pass@4": 0.8914274667635996, + "pass@8": 0.9129749581292425, + "pass@16": 0.9306763023858616, + "pass@32": 0.9479524332552338, + "pass@64": 0.9652302587641902, + "pass@128": 0.977491961414791 + }, + "11": { + "pass@1": 0.7507688492063492, + "pass@2": 0.8152867610298712, + "pass@4": 0.8574687646187082, + "pass@8": 0.8888661836516896, + "pass@16": 0.9135599558410991, + "pass@32": 0.9314423496372826, + "pass@64": 0.9434417545138772, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.7076404816513762, + "pass@2": 0.8042358201497741, + "pass@4": 0.8671578274619333, + "pass@8": 0.9085258038760612, + "pass@16": 0.935189808714328, + "pass@32": 0.9534243395021395, + "pass@64": 0.9653087455773289, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.5027488425925926, + "pass@2": 0.6106045555069503, + "pass@4": 0.6959111670647646, + "pass@8": 0.7622771166009414, + "pass@16": 0.8123751233047263, + "pass@32": 0.8480814715984645, + "pass@64": 0.8730092960227223, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.3861044139465875, + "pass@2": 0.4967668637117687, + "pass@4": 0.5935214991672034, + "pass@8": 0.6672018859137244, + "pass@16": 0.71946447521043, + "pass@32": 0.7584769248935936, + "pass@64": 0.7924551518791305, + "pass@128": 0.827893175074184 + }, + "15": { + "pass@1": 0.24127121913580246, + "pass@2": 0.32664900177408374, + "pass@4": 0.4125610458993551, + "pass@8": 0.49270826498349163, + "pass@16": 0.5618038483266853, + "pass@32": 0.6195859693933924, + "pass@64": 0.6711680233603406, + "pass@128": 0.7191358024691358 + }, + "16": { + "pass@1": 0.19690265486725664, + "pass@2": 0.2587497532111583, + "pass@4": 0.3189182358842312, + "pass@8": 0.374695087684852, + "pass@16": 0.4241129705890384, + "pass@32": 0.4690045235965585, + "pass@64": 0.5130468430129036, + "pass@128": 0.5634218289085545 + }, + "17": { + "pass@1": 0.17613422439759036, + "pass@2": 0.23685675279859578, + "pass@4": 0.2954108272234041, + "pass@8": 0.34783661316669684, + "pass@16": 0.3928556096830005, + "pass@32": 0.43333668541543363, + "pass@64": 0.4700079164554424, + "pass@128": 0.5060240963855421 + }, + "18": { + "pass@1": 0.1748991935483871, + "pass@2": 0.23574501524003058, + "pass@4": 0.2958681713172949, + "pass@8": 0.34993805557826135, + "pass@16": 0.395412438432603, + "pass@32": 0.43308274288586, + "pass@64": 0.4660027797221985, + "pass@128": 0.4935483870967742 + }, + "19": { + "pass@1": 0.15717050604229607, + "pass@2": 0.21232374158479428, + "pass@4": 0.2676054354610507, + "pass@8": 0.31955934750096615, + "pass@16": 0.3666705109285142, + "pass@32": 0.4077764639729569, + "pass@64": 0.44331074246388014, + "pass@128": 0.4773413897280967 + }, + "20": { + "pass@1": 0.14761636636636635, + "pass@2": 0.20476615887541083, + "pass@4": 0.2636930164134889, + "pass@8": 0.32053535130427724, + "pass@16": 0.3732904435639585, + "pass@32": 0.4200946391152685, + "pass@64": 0.4584333411955923, + "pass@128": 0.4864864864864865 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/checkpoint-21350_metrics.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/checkpoint-21350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..a62a2d71792c449481966891f8aab0dd41531bf6 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/checkpoint-21350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21350", + "id": { + "answer_accuracy": 0.6244479166666667, + "avg_response_len": 198.69680642361112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6244479166666667, + "pass@2": 0.7214298993875694, + "pass@4": 0.8039659647127338, + "pass@8": 0.8679135716591414, + "pass@16": 0.9137113018057256, + "pass@32": 0.9445588004892217, + "pass@64": 0.9638695806078241, + "pass@128": 0.9754444444444444 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3711015625, + "pass@2": 0.5094146161417326, + "pass@4": 0.6451646847581556, + "pass@8": 0.7633579786765388, + "pass@16": 0.8536835013835412, + "pass@32": 0.9140677812702555, + "pass@64": 0.948798621951173, + "pass@128": 0.965 + }, + "2": { + "pass@1": 0.9466328125, + "pass@2": 0.9575182086614176, + "pass@4": 0.9661255690851143, + "pass@8": 0.9731577432639624, + "pass@16": 0.978962773776107, + "pass@32": 0.9835923465837139, + "pass@64": 0.9871631997620223, + "pass@128": 0.99 + }, + "3": { + "pass@1": 0.91284375, + "pass@2": 0.9340644685039373, + "pass@4": 0.9493579555680539, + "pass@8": 0.9614069006894743, + "pass@16": 0.9700490074503667, + "pass@32": 0.976047132172469, + "pass@64": 0.9812115837241149, + "pass@128": 0.987 + }, + "4": { + "pass@1": 0.8335078125, + "pass@2": 0.8884130167322835, + "pass@4": 0.9249312790588675, + "pass@8": 0.9500719540812473, + "pass@16": 0.9666378884533317, + "pass@32": 0.9766105929476998, + "pass@64": 0.9827645737628736, + "pass@128": 0.987 + }, + "5": { + "pass@1": 0.5989140625, + "pass@2": 0.7057034940944878, + "pass@4": 0.7913318609861272, + "pass@8": 0.8528965168786833, + "pass@16": 0.8953683039327304, + "pass@32": 0.9253480350020836, + "pass@64": 0.9447704900559682, + "pass@128": 0.955 + }, + "6": { + "pass@1": 0.6161953125, + "pass@2": 0.7436318897637799, + "pass@4": 0.8372246191413575, + "pass@8": 0.8982987699281322, + "pass@16": 0.9357811457552647, + "pass@32": 0.9595979539210537, + "pass@64": 0.9752454753353328, + "pass@128": 0.985 + }, + "7": { + "pass@1": 0.5113984375, + "pass@2": 0.6428337844488188, + "pass@4": 0.7535007716535433, + "pass@8": 0.8371692222528849, + "pass@16": 0.8948474045020977, + "pass@32": 0.9317821944074864, + "pass@64": 0.9546825627211082, + "pass@128": 0.969 + }, + "8": { + "pass@1": 0.4551875, + "pass@2": 0.598721579724409, + "pass@4": 0.7248052233783271, + "pass@8": 0.8209295762456776, + "pass@16": 0.8859572791048197, + "pass@32": 0.9272590131679226, + "pass@64": 0.9530154543949482, + "pass@128": 0.972 + }, + "9": { + "pass@1": 0.37425, + "pass@2": 0.5125680364173233, + "pass@4": 0.6432517187851521, + "pass@8": 0.7539334829157338, + "pass@16": 0.84211441189314, + "pass@32": 0.9067241549302479, + "pass@64": 0.9471742637627855, + "pass@128": 0.969 + } + }, + "per_op_accuracy": { + "10": 0.3711015625, + "2": 0.9466328125, + "3": 0.91284375, + "4": 0.8335078125, + "5": 0.5989140625, + "6": 0.6161953125, + "7": 0.5113984375, + "8": 0.4551875, + "9": 0.37425 + }, + "per_op_avg_response_len": { + "10": 274.2244765625, + "2": 115.141359375, + "3": 144.6123515625, + "4": 159.6056328125, + "5": 172.8479453125, + "6": 200.6065390625, + "7": 223.4283125, + "8": 242.0322109375, + "9": 255.7724296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.2244765625, + "2": 115.141359375, + "3": 144.6123515625, + "4": 159.6056328125, + "5": 172.8479453125, + "6": 200.6065390625, + "7": 223.4283125, + "8": 242.0322109375, + "9": 255.7724296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 239413, + "answer_accuracy": 0.6316832362377575, + "avg_response_len": 188.22033571850727, + "resp_tokens_sum": 71337013, + "pass_at_k": { + "pass@1": 0.6316832362377575, + "pass@2": 0.7281323046055446, + "pass@4": 0.8101128337301324, + "pass@8": 0.8736421992365726, + "pass@16": 0.9191508809990949, + "pass@32": 0.9494485649915887, + "pass@64": 0.9679151023059347, + "pass@128": 0.978047956771361 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.36114952064896755, + "pass@2": 0.5025727736743082, + "pass@4": 0.6456373453871359, + "pass@8": 0.7715342066613812, + "pass@16": 0.8645454062463671, + "pass@32": 0.9234167949172833, + "pass@64": 0.9545074396989176, + "pass@128": 0.9646017699115044 + }, + "2": { + "pass@1": 0.9437792056074766, + "pass@2": 0.9547834339048741, + "pass@4": 0.9639342809952494, + "pass@8": 0.9714543860845367, + "pass@16": 0.9780852797401268, + "pass@32": 0.9838669377540457, + "pass@64": 0.9889642323239781, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9333906727828746, + "pass@2": 0.9471856263093257, + "pass@4": 0.9564936748158777, + "pass@8": 0.9643997413788679, + "pass@16": 0.9714202643835527, + "pass@32": 0.9771898049285574, + "pass@64": 0.982185906508893, + "pass@128": 0.9877675840978594 + }, + "4": { + "pass@1": 0.846826171875, + "pass@2": 0.8998592827263778, + "pass@4": 0.9362568458239592, + "pass@8": 0.9610523510160874, + "pass@16": 0.9772996456506924, + "pass@32": 0.9865836582491718, + "pass@64": 0.9911079218798541, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.590203373015873, + "pass@2": 0.6948537682789656, + "pass@4": 0.7772942653001701, + "pass@8": 0.8367794936962248, + "pass@16": 0.8804916680795036, + "pass@32": 0.9143427896247542, + "pass@64": 0.9375744510001492, + "pass@128": 0.9523809523809523 + }, + "6": { + "pass@1": 0.6659571482035929, + "pass@2": 0.7866637541845445, + "pass@4": 0.8693523220712676, + "pass@8": 0.9201696057829176, + "pass@16": 0.9509777220112018, + "pass@32": 0.9715084608001282, + "pass@64": 0.9841061290050542, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.5229289940828402, + "pass@2": 0.6506014711363743, + "pass@4": 0.7598703474750563, + "pass@8": 0.843843292045879, + "pass@16": 0.9012108173980592, + "pass@32": 0.9370625269608213, + "pass@64": 0.960044360287011, + "pass@128": 0.9733727810650887 + }, + "8": { + "pass@1": 0.4805438701923077, + "pass@2": 0.626625435342217, + "pass@4": 0.7501883081922448, + "pass@8": 0.8398731877822811, + "pass@16": 0.8993453994361422, + "pass@32": 0.9372353301463613, + "pass@64": 0.9600822667646836, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.37691461267605636, + "pass@2": 0.5194996257069978, + "pass@4": 0.6534315924417888, + "pass@8": 0.7672971099907365, + "pass@16": 0.856289359327201, + "pass@32": 0.9170800615077654, + "pass@64": 0.9537005371934129, + "pass@128": 0.971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 238017, + "answer_accuracy": 0.6092751679226737, + "avg_response_len": 197.77667052342727, + "resp_tokens_sum": 77262643, + "pass_at_k": { + "pass@1": 0.6092751679226737, + "pass@2": 0.7094336697376729, + "pass@4": 0.7944722058006206, + "pass@8": 0.8601878551381394, + "pass@16": 0.9073736021041413, + "pass@32": 0.9396363926159387, + "pass@64": 0.9606613132157711, + "pass@128": 0.9747706422018348 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3624664493865031, + "pass@2": 0.4975967344572724, + "pass@4": 0.6321679976927728, + "pass@8": 0.7529265654036871, + "pass@16": 0.8500658601313674, + "pass@32": 0.917990814785285, + "pass@64": 0.9590980548606445, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.9269329407294833, + "pass@2": 0.9438456579230784, + "pass@4": 0.9553312027106032, + "pass@8": 0.9639957189471136, + "pass@16": 0.9706872409005898, + "pass@32": 0.9753433482006632, + "pass@64": 0.9778610939418738, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.881155303030303, + "pass@2": 0.9059744840133619, + "pass@4": 0.9232438743452522, + "pass@8": 0.9368277008371412, + "pass@16": 0.9468825073475181, + "pass@32": 0.9552734657415504, + "pass@64": 0.9645054655673708, + "pass@128": 0.9757575757575757 + }, + "4": { + "pass@1": 0.8148345044378699, + "pass@2": 0.8734522783394674, + "pass@4": 0.9115685165833552, + "pass@8": 0.9380823088154047, + "pass@16": 0.956343947805061, + "pass@32": 0.9685328260459767, + "pass@64": 0.9768866649882043, + "pass@128": 0.9822485207100592 + }, + "5": { + "pass@1": 0.6144021739130435, + "pass@2": 0.7229213026360837, + "pass@4": 0.8079759513756427, + "pass@8": 0.8658693081389128, + "pass@16": 0.902718231976262, + "pass@32": 0.9294425193718734, + "pass@64": 0.949326977779095, + "pass@128": 0.9594202898550724 + }, + "6": { + "pass@1": 0.5740732449856734, + "pass@2": 0.7082373288134823, + "pass@4": 0.8135206498184862, + "pass@8": 0.8863266213359048, + "pass@16": 0.9320016713493857, + "pass@32": 0.9581798591821608, + "pass@64": 0.9729124958024994, + "pass@128": 0.9828080229226361 + }, + "7": { + "pass@1": 0.49701950867052025, + "pass@2": 0.6282756246870876, + "pass@4": 0.7404991438931393, + "pass@8": 0.8254763539044299, + "pass@16": 0.8839952514643473, + "pass@32": 0.9212432414261512, + "pass@64": 0.944875483972104, + "pass@128": 0.9653179190751445 + }, + "8": { + "pass@1": 0.43058978873239434, + "pass@2": 0.5753673616502167, + "pass@4": 0.7030660641011416, + "pass@8": 0.8001631029757508, + "pass@16": 0.8668395813069671, + "pass@32": 0.9117980434621504, + "pass@64": 0.9437722729837049, + "pass@128": 0.971830985915493 + }, + "9": { + "pass@1": 0.3983205464071856, + "pass@2": 0.5390116666273753, + "pass@4": 0.6679228606903187, + "pass@8": 0.774652988649831, + "pass@16": 0.8587257409476783, + "pass@32": 0.9206947819975753, + "pass@64": 0.9582371946330216, + "pass@128": 0.9760479041916168 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 241934, + "answer_accuracy": 0.6327784984934717, + "avg_response_len": 210.02224483177102, + "resp_tokens_sum": 80299065, + "pass_at_k": { + "pass@1": 0.6327784984934717, + "pass@2": 0.7270431136631497, + "pass@4": 0.8075729526736866, + "pass@8": 0.8701286438270098, + "pass@16": 0.9147946849652837, + "pass@32": 0.9447411225975829, + "pass@64": 0.9631373550746194, + "pass@128": 0.9735520589219954 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.38957555970149255, + "pass@2": 0.5278385386061816, + "pass@4": 0.6573339027584231, + "pass@8": 0.7652352904379972, + "pass@16": 0.8462123530244711, + "pass@32": 0.9007894990247578, + "pass@64": 0.9329989074885636, + "pass@128": 0.9492537313432836 + }, + "2": { + "pass@1": 0.9677678571428572, + "pass@2": 0.9728785854893138, + "pass@4": 0.9782819976967164, + "pass@8": 0.9833322679920727, + "pass@16": 0.987546562066378, + "pass@32": 0.9910945657332776, + "pass@64": 0.9942553750832824, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9237427113702624, + "pass@2": 0.9485807258786534, + "pass@4": 0.9676794325636404, + "pass@8": 0.9822012944091195, + "pass@16": 0.9910301853418775, + "pass@32": 0.9949440882394144, + "pass@64": 0.9963556520655231, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.8395010964912281, + "pass@2": 0.8924888192429892, + "pass@4": 0.9275407303801936, + "pass@8": 0.9516473139663537, + "pass@16": 0.9668355189678384, + "pass@32": 0.9752623599544581, + "pass@64": 0.980767093553529, + "pass@128": 0.9853801169590644 + }, + "5": { + "pass@1": 0.5912683823529412, + "pass@2": 0.6982844343446042, + "pass@4": 0.7874484240940461, + "pass@8": 0.8546648972248505, + "pass@16": 0.9016930542819938, + "pass@32": 0.9313893737852658, + "pass@64": 0.9468139136974493, + "pass@128": 0.9529411764705882 + }, + "6": { + "pass@1": 0.6101389984227129, + "pass@2": 0.7372598363098934, + "pass@4": 0.8294707784949593, + "pass@8": 0.8884357436921327, + "pass@16": 0.9239306097873485, + "pass@32": 0.9486099594928609, + "pass@64": 0.9684781300081131, + "pass@128": 0.9810725552050473 + }, + "7": { + "pass@1": 0.5148091376582279, + "pass@2": 0.6504655729841523, + "pass@4": 0.7609237038605933, + "pass@8": 0.8428334528178655, + "pass@16": 0.8999233899205381, + "pass@32": 0.9376737302572167, + "pass@64": 0.9596856059802206, + "pass@128": 0.9683544303797469 + }, + "8": { + "pass@1": 0.45765296546546547, + "pass@2": 0.5974745661015346, + "pass@4": 0.7241982536529772, + "pass@8": 0.8253190393429889, + "pass@16": 0.8937941835338469, + "pass@32": 0.9343942484480311, + "pass@64": 0.9562481389043579, + "pass@128": 0.96996996996997 + }, + "9": { + "pass@1": 0.3453577170418006, + "pass@2": 0.4762565036585059, + "pass@4": 0.6051359099606117, + "pass@8": 0.716427365466166, + "pass@16": 0.8080941860304234, + "pass@32": 0.8798992794463033, + "pass@64": 0.9278435692980506, + "pass@128": 0.9581993569131833 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.19060625, + "avg_response_len": 306.29846640625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.19060625, + "pass@2": 0.2811587598425232, + "pass@4": 0.376613723190856, + "pass@8": 0.4657296956639886, + "pass@16": 0.544274404282556, + "pass@32": 0.6124379917005603, + "pass@64": 0.6709115673104772, + "pass@128": 0.7199 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.31721875, + "pass@2": 0.44937032480314976, + "pass@4": 0.576589917041619, + "pass@8": 0.686107046815521, + "pass@16": 0.7765809836060162, + "pass@32": 0.8485191952750573, + "pass@64": 0.9026797909967988, + "pass@128": 0.939 + }, + "12": { + "pass@1": 0.2670859375, + "pass@2": 0.39264936023622077, + "pass@4": 0.5219323790776147, + "pass@8": 0.6397001286593244, + "pass@16": 0.7413914304266788, + "pass@32": 0.8231341171072257, + "pass@64": 0.8826855332519571, + "pass@128": 0.923 + }, + "13": { + "pass@1": 0.231421875, + "pass@2": 0.3360574557086615, + "pass@4": 0.44430182330333673, + "pass@8": 0.5460734279268706, + "pass@16": 0.6389725336518018, + "pass@32": 0.7231192113101128, + "pass@64": 0.7984372082685508, + "pass@128": 0.865 + }, + "14": { + "pass@1": 0.2080625, + "pass@2": 0.3065164862204723, + "pass@4": 0.4100432375328085, + "pass@8": 0.5061367289738223, + "pass@16": 0.591757256651165, + "pass@32": 0.668878756428832, + "pass@64": 0.7371932633272651, + "pass@128": 0.794 + }, + "15": { + "pass@1": 0.1604140625, + "pass@2": 0.24454429133858263, + "pass@4": 0.3389893219910011, + "pass@8": 0.43129429840224176, + "pass@16": 0.5138607178535096, + "pass@32": 0.5868449063692801, + "pass@64": 0.651710448194384, + "pass@128": 0.708 + }, + "16": { + "pass@1": 0.1695859375, + "pass@2": 0.2486421013779527, + "pass@4": 0.33098634655043074, + "pass@8": 0.4086920060007447, + "pass@16": 0.48057115442715015, + "pass@32": 0.5488996076362931, + "pass@64": 0.6147793077497744, + "pass@128": 0.676 + }, + "17": { + "pass@1": 0.1491953125, + "pass@2": 0.2257424950787403, + "pass@4": 0.30925168944506903, + "pass@8": 0.3883682710519393, + "pass@16": 0.45788349585230737, + "pass@32": 0.5192731522506836, + "pass@64": 0.5757310959401232, + "pass@128": 0.629 + }, + "18": { + "pass@1": 0.1405859375, + "pass@2": 0.21206016240157474, + "pass@4": 0.29123473068991335, + "pass@8": 0.36771355333853034, + "pass@16": 0.43525967595166964, + "pass@32": 0.4928083538337913, + "pass@64": 0.5412831740975808, + "pass@128": 0.584 + }, + "19": { + "pass@1": 0.1368671875, + "pass@2": 0.20502522145669289, + "pass@4": 0.27736697187851495, + "pass@8": 0.34449878786327726, + "pass@16": 0.4046973511279526, + "pass@32": 0.45896898311479345, + "pass@64": 0.5058269952007296, + "pass@128": 0.545 + }, + "20": { + "pass@1": 0.125625, + "pass@2": 0.19097969980314963, + "pass@4": 0.2654408143981999, + "pass@8": 0.3387127076074899, + "pass@16": 0.4017694432771526, + "pass@32": 0.4539336336795431, + "pass@64": 0.498788856077583, + "pass@128": 0.536 + } + }, + "per_op_accuracy": { + "11": 0.31721875, + "12": 0.2670859375, + "13": 0.231421875, + "14": 0.2080625, + "15": 0.1604140625, + "16": 0.1695859375, + "17": 0.1491953125, + "18": 0.1405859375, + "19": 0.1368671875, + "20": 0.125625 + }, + "per_op_avg_response_len": { + "11": 286.93871875, + "12": 292.962890625, + "13": 304.0875078125, + "14": 308.417375, + "15": 306.567859375, + "16": 311.959078125, + "17": 314.6864453125, + "18": 312.17496875, + "19": 311.87471875, + "20": 313.3151015625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 286.93871875, + "12": 292.962890625, + "13": 304.0875078125, + "14": 308.417375, + "15": 306.567859375, + "16": 311.959078125, + "17": 314.6864453125, + "18": 312.17496875, + "19": 311.87471875, + "20": 313.3151015625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 84416, + "answer_accuracy": 0.19598811292719168, + "avg_response_len": 292.80113763001486, + "resp_tokens_sum": 126115306, + "pass_at_k": { + "pass@1": 0.19598811292719168, + "pass@2": 0.28887930555393043, + "pass@4": 0.38516765661536106, + "pass@8": 0.4733597390316806, + "pass@16": 0.5504625411400456, + "pass@32": 0.6174461201162061, + "pass@64": 0.6750922465165412, + "pass@128": 0.7242199108469539 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3295414305949009, + "pass@2": 0.4673450709330597, + "pass@4": 0.5997836868302225, + "pass@8": 0.7113282977329743, + "pass@16": 0.7996943213346072, + "pass@32": 0.8639812545790467, + "pass@64": 0.9078105095951646, + "pass@128": 0.9348441926345609 + }, + "12": { + "pass@1": 0.2501765536723164, + "pass@2": 0.3753962031229148, + "pass@4": 0.5053718790977395, + "pass@8": 0.6213835158955956, + "pass@16": 0.7194175565450607, + "pass@32": 0.8002603777466087, + "pass@64": 0.863669554304307, + "pass@128": 0.9067796610169492 + }, + "13": { + "pass@1": 0.24708561912225704, + "pass@2": 0.3578045164515093, + "pass@4": 0.4703984194373826, + "pass@8": 0.5756403999380573, + "pass@16": 0.6725806558811607, + "pass@32": 0.7608977414595537, + "pass@64": 0.8395201593334661, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.2110105140186916, + "pass@2": 0.314535577182525, + "pass@4": 0.42529508636186836, + "pass@8": 0.5282425677297654, + "pass@16": 0.6195701240372914, + "pass@32": 0.7011807434818168, + "pass@64": 0.7712958911735933, + "pass@128": 0.8286604361370716 + }, + "15": { + "pass@1": 0.174516799132948, + "pass@2": 0.26026530710937146, + "pass@4": 0.3516182513854841, + "pass@8": 0.4383474727139665, + "pass@16": 0.5179759923282242, + "pass@32": 0.592788340884816, + "pass@64": 0.6628031726152839, + "pass@128": 0.7254335260115607 + }, + "16": { + "pass@1": 0.17643649193548386, + "pass@2": 0.256484950469901, + "pass@4": 0.3398621137075606, + "pass@8": 0.4208168272956852, + "pass@16": 0.49688550526352426, + "pass@32": 0.5672181973038024, + "pass@64": 0.6332357350352547, + "pass@128": 0.7 + }, + "17": { + "pass@1": 0.1539694725433526, + "pass@2": 0.2301268575622411, + "pass@4": 0.3082558133990478, + "pass@8": 0.3769771190958141, + "pass@16": 0.4362476355948833, + "pass@32": 0.49174618367651685, + "pass@64": 0.5481902103997353, + "pass@128": 0.6098265895953757 + }, + "18": { + "pass@1": 0.14058035714285713, + "pass@2": 0.21339742688413954, + "pass@4": 0.2935735435749102, + "pass@8": 0.37007484386718004, + "pass@16": 0.43467331394651476, + "pass@32": 0.48700053725037323, + "pass@64": 0.5293119476612531, + "pass@128": 0.5685714285714286 + }, + "19": { + "pass@1": 0.1365897147147147, + "pass@2": 0.2067106476555297, + "pass@4": 0.28071525490620003, + "pass@8": 0.348540194766144, + "pass@16": 0.40951359580054114, + "pass@32": 0.4652272616603531, + "pass@64": 0.5140670759360153, + "pass@128": 0.5555555555555556 + }, + "20": { + "pass@1": 0.13518205705705705, + "pass@2": 0.19940080336241758, + "pass@4": 0.26722059320287667, + "pass@8": 0.33167000256711, + "pass@16": 0.3880052020125598, + "pass@32": 0.4359332930515247, + "pass@64": 0.47551369120416725, + "pass@128": 0.5045045045045045 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 80961, + "answer_accuracy": 0.18807844558429973, + "avg_response_len": 303.51602921498665, + "resp_tokens_sum": 130652724, + "pass_at_k": { + "pass@1": 0.18807844558429973, + "pass@2": 0.27781448357648425, + "pass@4": 0.37280390979785005, + "pass@8": 0.4623306125280471, + "pass@16": 0.5424125258522479, + "pass@32": 0.6126339883150821, + "pass@64": 0.6726584284111549, + "pass@128": 0.72019030627416 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.29918109939759036, + "pass@2": 0.4210441668247794, + "pass@4": 0.5394813751631948, + "pass@8": 0.6456911887862564, + "pass@16": 0.7395810077099041, + "pass@32": 0.8209640589794067, + "pass@64": 0.887567370372437, + "pass@128": 0.9307228915662651 + }, + "12": { + "pass@1": 0.28950333072100315, + "pass@2": 0.41677362821810293, + "pass@4": 0.5441571644257631, + "pass@8": 0.6591465794235026, + "pass@16": 0.7601174561613118, + "pass@32": 0.8408860799581142, + "pass@64": 0.8961900157902011, + "pass@128": 0.9341692789968652 + }, + "13": { + "pass@1": 0.21962535014005602, + "pass@2": 0.32394020600366136, + "pass@4": 0.4328213529506295, + "pass@8": 0.5350798081641757, + "pass@16": 0.6285083457669532, + "pass@32": 0.714978858889809, + "pass@64": 0.7926042040647505, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.20360471491228072, + "pass@2": 0.2982582049776674, + "pass@4": 0.39714510000942854, + "pass@8": 0.491090990747798, + "pass@16": 0.5775013194565408, + "pass@32": 0.6566763801674114, + "pass@64": 0.727151911027562, + "pass@128": 0.783625730994152 + }, + "15": { + "pass@1": 0.15722064393939394, + "pass@2": 0.2387232313290383, + "pass@4": 0.32951538074786096, + "pass@8": 0.41899566652474257, + "pass@16": 0.5013357299356879, + "pass@32": 0.5751957873786209, + "pass@64": 0.639451132054347, + "pass@128": 0.6909090909090909 + }, + "16": { + "pass@1": 0.16415153133903135, + "pass@2": 0.24567567074948962, + "pass@4": 0.33242294766573, + "pass@8": 0.41320291592274866, + "pass@16": 0.48563958155008496, + "pass@32": 0.5530816266561013, + "pass@64": 0.6152416612226966, + "pass@128": 0.6638176638176638 + }, + "17": { + "pass@1": 0.16037461180124224, + "pass@2": 0.24287831038783184, + "pass@4": 0.33337522590343915, + "pass@8": 0.4201866766107831, + "pass@16": 0.49633960489111445, + "pass@32": 0.5619214680954321, + "pass@64": 0.6206053593384304, + "pass@128": 0.6708074534161491 + }, + "18": { + "pass@1": 0.1348345588235294, + "pass@2": 0.20740287748957853, + "pass@4": 0.28915632430137395, + "pass@8": 0.37054043763698213, + "pass@16": 0.4462654321967189, + "pass@32": 0.5117364598301063, + "pass@64": 0.5669886295013616, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.13102213541666666, + "pass@2": 0.19896023739220098, + "pass@4": 0.27425777079204383, + "pass@8": 0.3457286602947428, + "pass@16": 0.408312844903427, + "pass@32": 0.4615735021120364, + "pass@64": 0.5057884604835114, + "pass@128": 0.5446428571428571 + }, + "20": { + "pass@1": 0.1250935628742515, + "pass@2": 0.18957017068225754, + "pass@4": 0.2618971112892326, + "pass@8": 0.3306961799461177, + "pass@16": 0.38798512293652654, + "pass@32": 0.4361992048056054, + "pass@64": 0.48030832094747544, + "pass@128": 0.5209580838323353 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 78599, + "answer_accuracy": 0.1876695255195599, + "avg_response_len": 323.0392511269865, + "resp_tokens_sum": 135294007, + "pass_at_k": { + "pass@1": 0.1876695255195599, + "pass@2": 0.27665605958695144, + "pass@4": 0.37173243238009135, + "pass@8": 0.46137640124276474, + "pass@16": 0.5398240426185903, + "pass@32": 0.6070860697160496, + "pass@64": 0.6648166163905422, + "pass@128": 0.715158924205379 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.32242063492063494, + "pass@2": 0.45908206786651673, + "pass@4": 0.5897093937662551, + "pass@8": 0.7004402636150577, + "pass@16": 0.7896761702070251, + "pass@32": 0.8602341105634611, + "pass@64": 0.9128581401462108, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.2635225535168196, + "pass@2": 0.3877930181319079, + "pass@4": 0.5181792000770544, + "pass@8": 0.6405584256764115, + "pass@16": 0.7469117642026594, + "pass@32": 0.8305788497806939, + "pass@64": 0.8900975412573625, + "pass@128": 0.9296636085626911 + }, + "13": { + "pass@1": 0.22899787808641975, + "pass@2": 0.3279974117818605, + "pass@4": 0.431257729936536, + "pass@8": 0.529076076642035, + "pass@16": 0.6174130400830578, + "pass@32": 0.6948931455581896, + "pass@64": 0.7644153598458592, + "pass@128": 0.8364197530864198 + }, + "14": { + "pass@1": 0.2097783753709199, + "pass@2": 0.3072589312834414, + "pass@4": 0.40860501663924065, + "pass@8": 0.5003493943525822, + "pass@16": 0.5797323311010636, + "pass@32": 0.6504938152935147, + "pass@64": 0.714900085130906, + "pass@128": 0.771513353115727 + }, + "15": { + "pass@1": 0.1486062885802469, + "pass@2": 0.23368465660542423, + "pass@4": 0.3351522572988561, + "pass@8": 0.4362885891668056, + "pass@16": 0.5222229432999017, + "pass@32": 0.5923627487289759, + "pass@64": 0.65235085460358, + "pass@128": 0.7067901234567902 + }, + "16": { + "pass@1": 0.16894819321533924, + "pass@2": 0.24454161150210196, + "pass@4": 0.3213824090572752, + "pass@8": 0.39293382315692404, + "pass@16": 0.4604045860512616, + "pass@32": 0.5278180398696859, + "pass@64": 0.5974230289076049, + "pass@128": 0.6666666666666666 + }, + "17": { + "pass@1": 0.13337725903614459, + "pass@2": 0.20455357354615303, + "pass@4": 0.28689263634967316, + "pass@8": 0.3693797529401078, + "pass@16": 0.44313391916126493, + "pass@32": 0.5065971685899989, + "pass@64": 0.5609107151651793, + "pass@128": 0.608433734939759 + }, + "18": { + "pass@1": 0.14690020161290324, + "pass@2": 0.21565833756667505, + "pass@4": 0.2908736779878321, + "pass@8": 0.3619471264143349, + "pass@16": 0.42385086814033907, + "pass@32": 0.47860570791588575, + "pass@64": 0.5266059947925121, + "pass@128": 0.5709677419354838 + }, + "19": { + "pass@1": 0.14307968277945618, + "pass@2": 0.20948621155172828, + "pass@4": 0.2771546254036975, + "pass@8": 0.3391845110184821, + "pass@16": 0.39618189679402127, + "pass@32": 0.45002902801284544, + "pass@64": 0.4975762422706243, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.11660097597597598, + "pass@2": 0.18397235818495664, + "pass@4": 0.2672153804535694, + "pass@8": 0.3537960139658834, + "pass@16": 0.4293593992377488, + "pass@32": 0.4897216597246643, + "pass@64": 0.5406000531835978, + "pass@128": 0.5825825825825826 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.39611019736842107, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.39611019736842107, + "pass@2": 0.48970824699542026, + "pass@4": 0.5790437323327701, + "pass@8": 0.6562378474511611, + "pass@16": 0.7192708294251269, + "pass@32": 0.7697583748110065, + "pass@64": 0.8096811525565941, + "pass@128": 0.8409473684210527 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3711015625, + "pass@2": 0.5094146161417326, + "pass@4": 0.6451646847581556, + "pass@8": 0.7633579786765388, + "pass@16": 0.8536835013835412, + "pass@32": 0.9140677812702555, + "pass@64": 0.948798621951173, + "pass@128": 0.965 + }, + "2": { + "pass@1": 0.9466328125, + "pass@2": 0.9575182086614176, + "pass@4": 0.9661255690851143, + "pass@8": 0.9731577432639624, + "pass@16": 0.978962773776107, + "pass@32": 0.9835923465837139, + "pass@64": 0.9871631997620223, + "pass@128": 0.99 + }, + "3": { + "pass@1": 0.91284375, + "pass@2": 0.9340644685039373, + "pass@4": 0.9493579555680539, + "pass@8": 0.9614069006894743, + "pass@16": 0.9700490074503667, + "pass@32": 0.976047132172469, + "pass@64": 0.9812115837241149, + "pass@128": 0.987 + }, + "4": { + "pass@1": 0.8335078125, + "pass@2": 0.8884130167322835, + "pass@4": 0.9249312790588675, + "pass@8": 0.9500719540812473, + "pass@16": 0.9666378884533317, + "pass@32": 0.9766105929476998, + "pass@64": 0.9827645737628736, + "pass@128": 0.987 + }, + "5": { + "pass@1": 0.5989140625, + "pass@2": 0.7057034940944878, + "pass@4": 0.7913318609861272, + "pass@8": 0.8528965168786833, + "pass@16": 0.8953683039327304, + "pass@32": 0.9253480350020836, + "pass@64": 0.9447704900559682, + "pass@128": 0.955 + }, + "6": { + "pass@1": 0.6161953125, + "pass@2": 0.7436318897637799, + "pass@4": 0.8372246191413575, + "pass@8": 0.8982987699281322, + "pass@16": 0.9357811457552647, + "pass@32": 0.9595979539210537, + "pass@64": 0.9752454753353328, + "pass@128": 0.985 + }, + "7": { + "pass@1": 0.5113984375, + "pass@2": 0.6428337844488188, + "pass@4": 0.7535007716535433, + "pass@8": 0.8371692222528849, + "pass@16": 0.8948474045020977, + "pass@32": 0.9317821944074864, + "pass@64": 0.9546825627211082, + "pass@128": 0.969 + }, + "8": { + "pass@1": 0.4551875, + "pass@2": 0.598721579724409, + "pass@4": 0.7248052233783271, + "pass@8": 0.8209295762456776, + "pass@16": 0.8859572791048197, + "pass@32": 0.9272590131679226, + "pass@64": 0.9530154543949482, + "pass@128": 0.972 + }, + "9": { + "pass@1": 0.37425, + "pass@2": 0.5125680364173233, + "pass@4": 0.6432517187851521, + "pass@8": 0.7539334829157338, + "pass@16": 0.84211441189314, + "pass@32": 0.9067241549302479, + "pass@64": 0.9471742637627855, + "pass@128": 0.969 + }, + "11": { + "pass@1": 0.31721875, + "pass@2": 0.44937032480314976, + "pass@4": 0.576589917041619, + "pass@8": 0.686107046815521, + "pass@16": 0.7765809836060162, + "pass@32": 0.8485191952750573, + "pass@64": 0.9026797909967988, + "pass@128": 0.939 + }, + "12": { + "pass@1": 0.2670859375, + "pass@2": 0.39264936023622077, + "pass@4": 0.5219323790776147, + "pass@8": 0.6397001286593244, + "pass@16": 0.7413914304266788, + "pass@32": 0.8231341171072257, + "pass@64": 0.8826855332519571, + "pass@128": 0.923 + }, + "13": { + "pass@1": 0.231421875, + "pass@2": 0.3360574557086615, + "pass@4": 0.44430182330333673, + "pass@8": 0.5460734279268706, + "pass@16": 0.6389725336518018, + "pass@32": 0.7231192113101128, + "pass@64": 0.7984372082685508, + "pass@128": 0.865 + }, + "14": { + "pass@1": 0.2080625, + "pass@2": 0.3065164862204723, + "pass@4": 0.4100432375328085, + "pass@8": 0.5061367289738223, + "pass@16": 0.591757256651165, + "pass@32": 0.668878756428832, + "pass@64": 0.7371932633272651, + "pass@128": 0.794 + }, + "15": { + "pass@1": 0.1604140625, + "pass@2": 0.24454429133858263, + "pass@4": 0.3389893219910011, + "pass@8": 0.43129429840224176, + "pass@16": 0.5138607178535096, + "pass@32": 0.5868449063692801, + "pass@64": 0.651710448194384, + "pass@128": 0.708 + }, + "16": { + "pass@1": 0.1695859375, + "pass@2": 0.2486421013779527, + "pass@4": 0.33098634655043074, + "pass@8": 0.4086920060007447, + "pass@16": 0.48057115442715015, + "pass@32": 0.5488996076362931, + "pass@64": 0.6147793077497744, + "pass@128": 0.676 + }, + "17": { + "pass@1": 0.1491953125, + "pass@2": 0.2257424950787403, + "pass@4": 0.30925168944506903, + "pass@8": 0.3883682710519393, + "pass@16": 0.45788349585230737, + "pass@32": 0.5192731522506836, + "pass@64": 0.5757310959401232, + "pass@128": 0.629 + }, + "18": { + "pass@1": 0.1405859375, + "pass@2": 0.21206016240157474, + "pass@4": 0.29123473068991335, + "pass@8": 0.36771355333853034, + "pass@16": 0.43525967595166964, + "pass@32": 0.4928083538337913, + "pass@64": 0.5412831740975808, + "pass@128": 0.584 + }, + "19": { + "pass@1": 0.1368671875, + "pass@2": 0.20502522145669289, + "pass@4": 0.27736697187851495, + "pass@8": 0.34449878786327726, + "pass@16": 0.4046973511279526, + "pass@32": 0.45896898311479345, + "pass@64": 0.5058269952007296, + "pass@128": 0.545 + }, + "20": { + "pass@1": 0.125625, + "pass@2": 0.19097969980314963, + "pass@4": 0.2654408143981999, + "pass@8": 0.3387127076074899, + "pass@16": 0.4017694432771526, + "pass@32": 0.4539336336795431, + "pass@64": 0.498788856077583, + "pass@128": 0.536 + } + }, + "per_op_accuracy": { + "10": 0.3711015625, + "2": 0.9466328125, + "3": 0.91284375, + "4": 0.8335078125, + "5": 0.5989140625, + "6": 0.6161953125, + "7": 0.5113984375, + "8": 0.4551875, + "9": 0.37425, + "11": 0.31721875, + "12": 0.2670859375, + "13": 0.231421875, + "14": 0.2080625, + "15": 0.1604140625, + "16": 0.1695859375, + "17": 0.1491953125, + "18": 0.1405859375, + "19": 0.1368671875, + "20": 0.125625 + }, + "per_op_avg_response_len": { + "10": 274.2244765625, + "2": 115.141359375, + "3": 144.6123515625, + "4": 159.6056328125, + "5": 172.8479453125, + "6": 200.6065390625, + "7": 223.4283125, + "8": 242.0322109375, + "9": 255.7724296875, + "11": 286.93871875, + "12": 292.962890625, + "13": 304.0875078125, + "14": 308.417375, + "15": 306.567859375, + "16": 311.959078125, + "17": 314.6864453125, + "18": 312.17496875, + "19": 311.87471875, + "20": 313.3151015625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.2244765625, + "2": 115.141359375, + "3": 144.6123515625, + "4": 159.6056328125, + "5": 172.8479453125, + "6": 200.6065390625, + "7": 223.4283125, + "8": 242.0322109375, + "9": 255.7724296875, + "11": 286.93871875, + "12": 292.962890625, + "13": 304.0875078125, + "14": 308.417375, + "15": 306.567859375, + "16": 311.959078125, + "17": 314.6864453125, + "18": 312.17496875, + "19": 311.87471875, + "20": 313.3151015625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 323829, + "answer_accuracy": 0.39992318408156813, + "avg_response_len": 243.85018055445778, + "resp_tokens_sum": 197452319, + "pass_at_k": { + "pass@1": 0.39992318408156813, + "pass@2": 0.4944797055210301, + "pass@4": 0.5840710188405998, + "pass@8": 0.660719265536069, + "pass@16": 0.7230338617727872, + "pass@32": 0.7728459366315423, + "pass@64": 0.8121533397812276, + "pass@128": 0.8430287701549162 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.36114952064896755, + "pass@2": 0.5025727736743082, + "pass@4": 0.6456373453871359, + "pass@8": 0.7715342066613812, + "pass@16": 0.8645454062463671, + "pass@32": 0.9234167949172833, + "pass@64": 0.9545074396989176, + "pass@128": 0.9646017699115044 + }, + "2": { + "pass@1": 0.9437792056074766, + "pass@2": 0.9547834339048741, + "pass@4": 0.9639342809952494, + "pass@8": 0.9714543860845367, + "pass@16": 0.9780852797401268, + "pass@32": 0.9838669377540457, + "pass@64": 0.9889642323239781, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9333906727828746, + "pass@2": 0.9471856263093257, + "pass@4": 0.9564936748158777, + "pass@8": 0.9643997413788679, + "pass@16": 0.9714202643835527, + "pass@32": 0.9771898049285574, + "pass@64": 0.982185906508893, + "pass@128": 0.9877675840978594 + }, + "4": { + "pass@1": 0.846826171875, + "pass@2": 0.8998592827263778, + "pass@4": 0.9362568458239592, + "pass@8": 0.9610523510160874, + "pass@16": 0.9772996456506924, + "pass@32": 0.9865836582491718, + "pass@64": 0.9911079218798541, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.590203373015873, + "pass@2": 0.6948537682789656, + "pass@4": 0.7772942653001701, + "pass@8": 0.8367794936962248, + "pass@16": 0.8804916680795036, + "pass@32": 0.9143427896247542, + "pass@64": 0.9375744510001492, + "pass@128": 0.9523809523809523 + }, + "6": { + "pass@1": 0.6659571482035929, + "pass@2": 0.7866637541845445, + "pass@4": 0.8693523220712676, + "pass@8": 0.9201696057829176, + "pass@16": 0.9509777220112018, + "pass@32": 0.9715084608001282, + "pass@64": 0.9841061290050542, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.5229289940828402, + "pass@2": 0.6506014711363743, + "pass@4": 0.7598703474750563, + "pass@8": 0.843843292045879, + "pass@16": 0.9012108173980592, + "pass@32": 0.9370625269608213, + "pass@64": 0.960044360287011, + "pass@128": 0.9733727810650887 + }, + "8": { + "pass@1": 0.4805438701923077, + "pass@2": 0.626625435342217, + "pass@4": 0.7501883081922448, + "pass@8": 0.8398731877822811, + "pass@16": 0.8993453994361422, + "pass@32": 0.9372353301463613, + "pass@64": 0.9600822667646836, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.37691461267605636, + "pass@2": 0.5194996257069978, + "pass@4": 0.6534315924417888, + "pass@8": 0.7672971099907365, + "pass@16": 0.856289359327201, + "pass@32": 0.9170800615077654, + "pass@64": 0.9537005371934129, + "pass@128": 0.971830985915493 + }, + "11": { + "pass@1": 0.3295414305949009, + "pass@2": 0.4673450709330597, + "pass@4": 0.5997836868302225, + "pass@8": 0.7113282977329743, + "pass@16": 0.7996943213346072, + "pass@32": 0.8639812545790467, + "pass@64": 0.9078105095951646, + "pass@128": 0.9348441926345609 + }, + "12": { + "pass@1": 0.2501765536723164, + "pass@2": 0.3753962031229148, + "pass@4": 0.5053718790977395, + "pass@8": 0.6213835158955956, + "pass@16": 0.7194175565450607, + "pass@32": 0.8002603777466087, + "pass@64": 0.863669554304307, + "pass@128": 0.9067796610169492 + }, + "13": { + "pass@1": 0.24708561912225704, + "pass@2": 0.3578045164515093, + "pass@4": 0.4703984194373826, + "pass@8": 0.5756403999380573, + "pass@16": 0.6725806558811607, + "pass@32": 0.7608977414595537, + "pass@64": 0.8395201593334661, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.2110105140186916, + "pass@2": 0.314535577182525, + "pass@4": 0.42529508636186836, + "pass@8": 0.5282425677297654, + "pass@16": 0.6195701240372914, + "pass@32": 0.7011807434818168, + "pass@64": 0.7712958911735933, + "pass@128": 0.8286604361370716 + }, + "15": { + "pass@1": 0.174516799132948, + "pass@2": 0.26026530710937146, + "pass@4": 0.3516182513854841, + "pass@8": 0.4383474727139665, + "pass@16": 0.5179759923282242, + "pass@32": 0.592788340884816, + "pass@64": 0.6628031726152839, + "pass@128": 0.7254335260115607 + }, + "16": { + "pass@1": 0.17643649193548386, + "pass@2": 0.256484950469901, + "pass@4": 0.3398621137075606, + "pass@8": 0.4208168272956852, + "pass@16": 0.49688550526352426, + "pass@32": 0.5672181973038024, + "pass@64": 0.6332357350352547, + "pass@128": 0.7 + }, + "17": { + "pass@1": 0.1539694725433526, + "pass@2": 0.2301268575622411, + "pass@4": 0.3082558133990478, + "pass@8": 0.3769771190958141, + "pass@16": 0.4362476355948833, + "pass@32": 0.49174618367651685, + "pass@64": 0.5481902103997353, + "pass@128": 0.6098265895953757 + }, + "18": { + "pass@1": 0.14058035714285713, + "pass@2": 0.21339742688413954, + "pass@4": 0.2935735435749102, + "pass@8": 0.37007484386718004, + "pass@16": 0.43467331394651476, + "pass@32": 0.48700053725037323, + "pass@64": 0.5293119476612531, + "pass@128": 0.5685714285714286 + }, + "19": { + "pass@1": 0.1365897147147147, + "pass@2": 0.2067106476555297, + "pass@4": 0.28071525490620003, + "pass@8": 0.348540194766144, + "pass@16": 0.40951359580054114, + "pass@32": 0.4652272616603531, + "pass@64": 0.5140670759360153, + "pass@128": 0.5555555555555556 + }, + "20": { + "pass@1": 0.13518205705705705, + "pass@2": 0.19940080336241758, + "pass@4": 0.26722059320287667, + "pass@8": 0.33167000256711, + "pass@16": 0.3880052020125598, + "pass@32": 0.4359332930515247, + "pass@64": 0.47551369120416725, + "pass@128": 0.5045045045045045 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 318978, + "answer_accuracy": 0.38846697194076385, + "avg_response_len": 253.20947851714732, + "resp_tokens_sum": 207915367, + "pass_at_k": { + "pass@1": 0.38846697194076385, + "pass@2": 0.4831616006714161, + "pass@4": 0.5734167920114881, + "pass@8": 0.6516151494643043, + "pass@16": 0.7160463847331318, + "pass@32": 0.7682086317953974, + "pass@64": 0.8096786629277101, + "pass@128": 0.8413094310210444 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3624664493865031, + "pass@2": 0.4975967344572724, + "pass@4": 0.6321679976927728, + "pass@8": 0.7529265654036871, + "pass@16": 0.8500658601313674, + "pass@32": 0.917990814785285, + "pass@64": 0.9590980548606445, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.9269329407294833, + "pass@2": 0.9438456579230784, + "pass@4": 0.9553312027106032, + "pass@8": 0.9639957189471136, + "pass@16": 0.9706872409005898, + "pass@32": 0.9753433482006632, + "pass@64": 0.9778610939418738, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.881155303030303, + "pass@2": 0.9059744840133619, + "pass@4": 0.9232438743452522, + "pass@8": 0.9368277008371412, + "pass@16": 0.9468825073475181, + "pass@32": 0.9552734657415504, + "pass@64": 0.9645054655673708, + "pass@128": 0.9757575757575757 + }, + "4": { + "pass@1": 0.8148345044378699, + "pass@2": 0.8734522783394674, + "pass@4": 0.9115685165833552, + "pass@8": 0.9380823088154047, + "pass@16": 0.956343947805061, + "pass@32": 0.9685328260459767, + "pass@64": 0.9768866649882043, + "pass@128": 0.9822485207100592 + }, + "5": { + "pass@1": 0.6144021739130435, + "pass@2": 0.7229213026360837, + "pass@4": 0.8079759513756427, + "pass@8": 0.8658693081389128, + "pass@16": 0.902718231976262, + "pass@32": 0.9294425193718734, + "pass@64": 0.949326977779095, + "pass@128": 0.9594202898550724 + }, + "6": { + "pass@1": 0.5740732449856734, + "pass@2": 0.7082373288134823, + "pass@4": 0.8135206498184862, + "pass@8": 0.8863266213359048, + "pass@16": 0.9320016713493857, + "pass@32": 0.9581798591821608, + "pass@64": 0.9729124958024994, + "pass@128": 0.9828080229226361 + }, + "7": { + "pass@1": 0.49701950867052025, + "pass@2": 0.6282756246870876, + "pass@4": 0.7404991438931393, + "pass@8": 0.8254763539044299, + "pass@16": 0.8839952514643473, + "pass@32": 0.9212432414261512, + "pass@64": 0.944875483972104, + "pass@128": 0.9653179190751445 + }, + "8": { + "pass@1": 0.43058978873239434, + "pass@2": 0.5753673616502167, + "pass@4": 0.7030660641011416, + "pass@8": 0.8001631029757508, + "pass@16": 0.8668395813069671, + "pass@32": 0.9117980434621504, + "pass@64": 0.9437722729837049, + "pass@128": 0.971830985915493 + }, + "9": { + "pass@1": 0.3983205464071856, + "pass@2": 0.5390116666273753, + "pass@4": 0.6679228606903187, + "pass@8": 0.774652988649831, + "pass@16": 0.8587257409476783, + "pass@32": 0.9206947819975753, + "pass@64": 0.9582371946330216, + "pass@128": 0.9760479041916168 + }, + "11": { + "pass@1": 0.29918109939759036, + "pass@2": 0.4210441668247794, + "pass@4": 0.5394813751631948, + "pass@8": 0.6456911887862564, + "pass@16": 0.7395810077099041, + "pass@32": 0.8209640589794067, + "pass@64": 0.887567370372437, + "pass@128": 0.9307228915662651 + }, + "12": { + "pass@1": 0.28950333072100315, + "pass@2": 0.41677362821810293, + "pass@4": 0.5441571644257631, + "pass@8": 0.6591465794235026, + "pass@16": 0.7601174561613118, + "pass@32": 0.8408860799581142, + "pass@64": 0.8961900157902011, + "pass@128": 0.9341692789968652 + }, + "13": { + "pass@1": 0.21962535014005602, + "pass@2": 0.32394020600366136, + "pass@4": 0.4328213529506295, + "pass@8": 0.5350798081641757, + "pass@16": 0.6285083457669532, + "pass@32": 0.714978858889809, + "pass@64": 0.7926042040647505, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.20360471491228072, + "pass@2": 0.2982582049776674, + "pass@4": 0.39714510000942854, + "pass@8": 0.491090990747798, + "pass@16": 0.5775013194565408, + "pass@32": 0.6566763801674114, + "pass@64": 0.727151911027562, + "pass@128": 0.783625730994152 + }, + "15": { + "pass@1": 0.15722064393939394, + "pass@2": 0.2387232313290383, + "pass@4": 0.32951538074786096, + "pass@8": 0.41899566652474257, + "pass@16": 0.5013357299356879, + "pass@32": 0.5751957873786209, + "pass@64": 0.639451132054347, + "pass@128": 0.6909090909090909 + }, + "16": { + "pass@1": 0.16415153133903135, + "pass@2": 0.24567567074948962, + "pass@4": 0.33242294766573, + "pass@8": 0.41320291592274866, + "pass@16": 0.48563958155008496, + "pass@32": 0.5530816266561013, + "pass@64": 0.6152416612226966, + "pass@128": 0.6638176638176638 + }, + "17": { + "pass@1": 0.16037461180124224, + "pass@2": 0.24287831038783184, + "pass@4": 0.33337522590343915, + "pass@8": 0.4201866766107831, + "pass@16": 0.49633960489111445, + "pass@32": 0.5619214680954321, + "pass@64": 0.6206053593384304, + "pass@128": 0.6708074534161491 + }, + "18": { + "pass@1": 0.1348345588235294, + "pass@2": 0.20740287748957853, + "pass@4": 0.28915632430137395, + "pass@8": 0.37054043763698213, + "pass@16": 0.4462654321967189, + "pass@32": 0.5117364598301063, + "pass@64": 0.5669886295013616, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.13102213541666666, + "pass@2": 0.19896023739220098, + "pass@4": 0.27425777079204383, + "pass@8": 0.3457286602947428, + "pass@16": 0.408312844903427, + "pass@32": 0.4615735021120364, + "pass@64": 0.5057884604835114, + "pass@128": 0.5446428571428571 + }, + "20": { + "pass@1": 0.1250935628742515, + "pass@2": 0.18957017068225754, + "pass@4": 0.2618971112892326, + "pass@8": 0.3306961799461177, + "pass@16": 0.38798512293652654, + "pass@32": 0.4361992048056054, + "pass@64": 0.48030832094747544, + "pass@128": 0.5209580838323353 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 320533, + "answer_accuracy": 0.4000901202268733, + "avg_response_len": 269.1038304841029, + "resp_tokens_sum": 215593072, + "pass_at_k": { + "pass@1": 0.4000901202268733, + "pass@2": 0.4915955276370618, + "pass@4": 0.5797298176040884, + "pass@8": 0.656446372260374, + "pass@16": 0.7187723264801746, + "pass@32": 0.7682261309010882, + "pass@64": 0.8071850532733251, + "pass@128": 0.8384725994567822 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.38957555970149255, + "pass@2": 0.5278385386061816, + "pass@4": 0.6573339027584231, + "pass@8": 0.7652352904379972, + "pass@16": 0.8462123530244711, + "pass@32": 0.9007894990247578, + "pass@64": 0.9329989074885636, + "pass@128": 0.9492537313432836 + }, + "2": { + "pass@1": 0.9677678571428572, + "pass@2": 0.9728785854893138, + "pass@4": 0.9782819976967164, + "pass@8": 0.9833322679920727, + "pass@16": 0.987546562066378, + "pass@32": 0.9910945657332776, + "pass@64": 0.9942553750832824, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9237427113702624, + "pass@2": 0.9485807258786534, + "pass@4": 0.9676794325636404, + "pass@8": 0.9822012944091195, + "pass@16": 0.9910301853418775, + "pass@32": 0.9949440882394144, + "pass@64": 0.9963556520655231, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.8395010964912281, + "pass@2": 0.8924888192429892, + "pass@4": 0.9275407303801936, + "pass@8": 0.9516473139663537, + "pass@16": 0.9668355189678384, + "pass@32": 0.9752623599544581, + "pass@64": 0.980767093553529, + "pass@128": 0.9853801169590644 + }, + "5": { + "pass@1": 0.5912683823529412, + "pass@2": 0.6982844343446042, + "pass@4": 0.7874484240940461, + "pass@8": 0.8546648972248505, + "pass@16": 0.9016930542819938, + "pass@32": 0.9313893737852658, + "pass@64": 0.9468139136974493, + "pass@128": 0.9529411764705882 + }, + "6": { + "pass@1": 0.6101389984227129, + "pass@2": 0.7372598363098934, + "pass@4": 0.8294707784949593, + "pass@8": 0.8884357436921327, + "pass@16": 0.9239306097873485, + "pass@32": 0.9486099594928609, + "pass@64": 0.9684781300081131, + "pass@128": 0.9810725552050473 + }, + "7": { + "pass@1": 0.5148091376582279, + "pass@2": 0.6504655729841523, + "pass@4": 0.7609237038605933, + "pass@8": 0.8428334528178655, + "pass@16": 0.8999233899205381, + "pass@32": 0.9376737302572167, + "pass@64": 0.9596856059802206, + "pass@128": 0.9683544303797469 + }, + "8": { + "pass@1": 0.45765296546546547, + "pass@2": 0.5974745661015346, + "pass@4": 0.7241982536529772, + "pass@8": 0.8253190393429889, + "pass@16": 0.8937941835338469, + "pass@32": 0.9343942484480311, + "pass@64": 0.9562481389043579, + "pass@128": 0.96996996996997 + }, + "9": { + "pass@1": 0.3453577170418006, + "pass@2": 0.4762565036585059, + "pass@4": 0.6051359099606117, + "pass@8": 0.716427365466166, + "pass@16": 0.8080941860304234, + "pass@32": 0.8798992794463033, + "pass@64": 0.9278435692980506, + "pass@128": 0.9581993569131833 + }, + "11": { + "pass@1": 0.32242063492063494, + "pass@2": 0.45908206786651673, + "pass@4": 0.5897093937662551, + "pass@8": 0.7004402636150577, + "pass@16": 0.7896761702070251, + "pass@32": 0.8602341105634611, + "pass@64": 0.9128581401462108, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.2635225535168196, + "pass@2": 0.3877930181319079, + "pass@4": 0.5181792000770544, + "pass@8": 0.6405584256764115, + "pass@16": 0.7469117642026594, + "pass@32": 0.8305788497806939, + "pass@64": 0.8900975412573625, + "pass@128": 0.9296636085626911 + }, + "13": { + "pass@1": 0.22899787808641975, + "pass@2": 0.3279974117818605, + "pass@4": 0.431257729936536, + "pass@8": 0.529076076642035, + "pass@16": 0.6174130400830578, + "pass@32": 0.6948931455581896, + "pass@64": 0.7644153598458592, + "pass@128": 0.8364197530864198 + }, + "14": { + "pass@1": 0.2097783753709199, + "pass@2": 0.3072589312834414, + "pass@4": 0.40860501663924065, + "pass@8": 0.5003493943525822, + "pass@16": 0.5797323311010636, + "pass@32": 0.6504938152935147, + "pass@64": 0.714900085130906, + "pass@128": 0.771513353115727 + }, + "15": { + "pass@1": 0.1486062885802469, + "pass@2": 0.23368465660542423, + "pass@4": 0.3351522572988561, + "pass@8": 0.4362885891668056, + "pass@16": 0.5222229432999017, + "pass@32": 0.5923627487289759, + "pass@64": 0.65235085460358, + "pass@128": 0.7067901234567902 + }, + "16": { + "pass@1": 0.16894819321533924, + "pass@2": 0.24454161150210196, + "pass@4": 0.3213824090572752, + "pass@8": 0.39293382315692404, + "pass@16": 0.4604045860512616, + "pass@32": 0.5278180398696859, + "pass@64": 0.5974230289076049, + "pass@128": 0.6666666666666666 + }, + "17": { + "pass@1": 0.13337725903614459, + "pass@2": 0.20455357354615303, + "pass@4": 0.28689263634967316, + "pass@8": 0.3693797529401078, + "pass@16": 0.44313391916126493, + "pass@32": 0.5065971685899989, + "pass@64": 0.5609107151651793, + "pass@128": 0.608433734939759 + }, + "18": { + "pass@1": 0.14690020161290324, + "pass@2": 0.21565833756667505, + "pass@4": 0.2908736779878321, + "pass@8": 0.3619471264143349, + "pass@16": 0.42385086814033907, + "pass@32": 0.47860570791588575, + "pass@64": 0.5266059947925121, + "pass@128": 0.5709677419354838 + }, + "19": { + "pass@1": 0.14307968277945618, + "pass@2": 0.20948621155172828, + "pass@4": 0.2771546254036975, + "pass@8": 0.3391845110184821, + "pass@16": 0.39618189679402127, + "pass@32": 0.45002902801284544, + "pass@64": 0.4975762422706243, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.11660097597597598, + "pass@2": 0.18397235818495664, + "pass@4": 0.2672153804535694, + "pass@8": 0.3537960139658834, + "pass@16": 0.4293593992377488, + "pass@32": 0.4897216597246643, + "pass@64": 0.5406000531835978, + "pass@128": 0.5825825825825826 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/summary.csv b/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8a6abfe6bdd00829b3496be99e01bfc1d52e3872 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21350,0.6244479166666667,0.19060625,0.39611019736842107,0.0,0.0,0.0,198.69680642361112,306.29846640625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/summary.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b6879e7da20e4b949ae791f96afde7ebd18332aa --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block16/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21350", + "id_acc": 0.6244479166666667, + "ood_acc": 0.19060625, + "total_acc": 0.39611019736842107, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 198.69680642361112, + "ood_avg_resp_len": 306.29846640625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 323829, + "answer_accuracy": 0.39992318408156813, + "avg_response_len": 243.85018055445778, + "resp_tokens_sum": 197452319, + "pass_at_k": { + "pass@1": 0.39992318408156813, + "pass@2": 0.4944797055210301, + "pass@4": 0.5840710188405998, + "pass@8": 0.660719265536069, + "pass@16": 0.7230338617727872, + "pass@32": 0.7728459366315423, + "pass@64": 0.8121533397812276, + "pass@128": 0.8430287701549162 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.36114952064896755, + "pass@2": 0.5025727736743082, + "pass@4": 0.6456373453871359, + "pass@8": 0.7715342066613812, + "pass@16": 0.8645454062463671, + "pass@32": 0.9234167949172833, + "pass@64": 0.9545074396989176, + "pass@128": 0.9646017699115044 + }, + "2": { + "pass@1": 0.9437792056074766, + "pass@2": 0.9547834339048741, + "pass@4": 0.9639342809952494, + "pass@8": 0.9714543860845367, + "pass@16": 0.9780852797401268, + "pass@32": 0.9838669377540457, + "pass@64": 0.9889642323239781, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9333906727828746, + "pass@2": 0.9471856263093257, + "pass@4": 0.9564936748158777, + "pass@8": 0.9643997413788679, + "pass@16": 0.9714202643835527, + "pass@32": 0.9771898049285574, + "pass@64": 0.982185906508893, + "pass@128": 0.9877675840978594 + }, + "4": { + "pass@1": 0.846826171875, + "pass@2": 0.8998592827263778, + "pass@4": 0.9362568458239592, + "pass@8": 0.9610523510160874, + "pass@16": 0.9772996456506924, + "pass@32": 0.9865836582491718, + "pass@64": 0.9911079218798541, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.590203373015873, + "pass@2": 0.6948537682789656, + "pass@4": 0.7772942653001701, + "pass@8": 0.8367794936962248, + "pass@16": 0.8804916680795036, + "pass@32": 0.9143427896247542, + "pass@64": 0.9375744510001492, + "pass@128": 0.9523809523809523 + }, + "6": { + "pass@1": 0.6659571482035929, + "pass@2": 0.7866637541845445, + "pass@4": 0.8693523220712676, + "pass@8": 0.9201696057829176, + "pass@16": 0.9509777220112018, + "pass@32": 0.9715084608001282, + "pass@64": 0.9841061290050542, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.5229289940828402, + "pass@2": 0.6506014711363743, + "pass@4": 0.7598703474750563, + "pass@8": 0.843843292045879, + "pass@16": 0.9012108173980592, + "pass@32": 0.9370625269608213, + "pass@64": 0.960044360287011, + "pass@128": 0.9733727810650887 + }, + "8": { + "pass@1": 0.4805438701923077, + "pass@2": 0.626625435342217, + "pass@4": 0.7501883081922448, + "pass@8": 0.8398731877822811, + "pass@16": 0.8993453994361422, + "pass@32": 0.9372353301463613, + "pass@64": 0.9600822667646836, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.37691461267605636, + "pass@2": 0.5194996257069978, + "pass@4": 0.6534315924417888, + "pass@8": 0.7672971099907365, + "pass@16": 0.856289359327201, + "pass@32": 0.9170800615077654, + "pass@64": 0.9537005371934129, + "pass@128": 0.971830985915493 + }, + "11": { + "pass@1": 0.3295414305949009, + "pass@2": 0.4673450709330597, + "pass@4": 0.5997836868302225, + "pass@8": 0.7113282977329743, + "pass@16": 0.7996943213346072, + "pass@32": 0.8639812545790467, + "pass@64": 0.9078105095951646, + "pass@128": 0.9348441926345609 + }, + "12": { + "pass@1": 0.2501765536723164, + "pass@2": 0.3753962031229148, + "pass@4": 0.5053718790977395, + "pass@8": 0.6213835158955956, + "pass@16": 0.7194175565450607, + "pass@32": 0.8002603777466087, + "pass@64": 0.863669554304307, + "pass@128": 0.9067796610169492 + }, + "13": { + "pass@1": 0.24708561912225704, + "pass@2": 0.3578045164515093, + "pass@4": 0.4703984194373826, + "pass@8": 0.5756403999380573, + "pass@16": 0.6725806558811607, + "pass@32": 0.7608977414595537, + "pass@64": 0.8395201593334661, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.2110105140186916, + "pass@2": 0.314535577182525, + "pass@4": 0.42529508636186836, + "pass@8": 0.5282425677297654, + "pass@16": 0.6195701240372914, + "pass@32": 0.7011807434818168, + "pass@64": 0.7712958911735933, + "pass@128": 0.8286604361370716 + }, + "15": { + "pass@1": 0.174516799132948, + "pass@2": 0.26026530710937146, + "pass@4": 0.3516182513854841, + "pass@8": 0.4383474727139665, + "pass@16": 0.5179759923282242, + "pass@32": 0.592788340884816, + "pass@64": 0.6628031726152839, + "pass@128": 0.7254335260115607 + }, + "16": { + "pass@1": 0.17643649193548386, + "pass@2": 0.256484950469901, + "pass@4": 0.3398621137075606, + "pass@8": 0.4208168272956852, + "pass@16": 0.49688550526352426, + "pass@32": 0.5672181973038024, + "pass@64": 0.6332357350352547, + "pass@128": 0.7 + }, + "17": { + "pass@1": 0.1539694725433526, + "pass@2": 0.2301268575622411, + "pass@4": 0.3082558133990478, + "pass@8": 0.3769771190958141, + "pass@16": 0.4362476355948833, + "pass@32": 0.49174618367651685, + "pass@64": 0.5481902103997353, + "pass@128": 0.6098265895953757 + }, + "18": { + "pass@1": 0.14058035714285713, + "pass@2": 0.21339742688413954, + "pass@4": 0.2935735435749102, + "pass@8": 0.37007484386718004, + "pass@16": 0.43467331394651476, + "pass@32": 0.48700053725037323, + "pass@64": 0.5293119476612531, + "pass@128": 0.5685714285714286 + }, + "19": { + "pass@1": 0.1365897147147147, + "pass@2": 0.2067106476555297, + "pass@4": 0.28071525490620003, + "pass@8": 0.348540194766144, + "pass@16": 0.40951359580054114, + "pass@32": 0.4652272616603531, + "pass@64": 0.5140670759360153, + "pass@128": 0.5555555555555556 + }, + "20": { + "pass@1": 0.13518205705705705, + "pass@2": 0.19940080336241758, + "pass@4": 0.26722059320287667, + "pass@8": 0.33167000256711, + "pass@16": 0.3880052020125598, + "pass@32": 0.4359332930515247, + "pass@64": 0.47551369120416725, + "pass@128": 0.5045045045045045 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 318978, + "answer_accuracy": 0.38846697194076385, + "avg_response_len": 253.20947851714732, + "resp_tokens_sum": 207915367, + "pass_at_k": { + "pass@1": 0.38846697194076385, + "pass@2": 0.4831616006714161, + "pass@4": 0.5734167920114881, + "pass@8": 0.6516151494643043, + "pass@16": 0.7160463847331318, + "pass@32": 0.7682086317953974, + "pass@64": 0.8096786629277101, + "pass@128": 0.8413094310210444 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3624664493865031, + "pass@2": 0.4975967344572724, + "pass@4": 0.6321679976927728, + "pass@8": 0.7529265654036871, + "pass@16": 0.8500658601313674, + "pass@32": 0.917990814785285, + "pass@64": 0.9590980548606445, + "pass@128": 0.9815950920245399 + }, + "2": { + "pass@1": 0.9269329407294833, + "pass@2": 0.9438456579230784, + "pass@4": 0.9553312027106032, + "pass@8": 0.9639957189471136, + "pass@16": 0.9706872409005898, + "pass@32": 0.9753433482006632, + "pass@64": 0.9778610939418738, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.881155303030303, + "pass@2": 0.9059744840133619, + "pass@4": 0.9232438743452522, + "pass@8": 0.9368277008371412, + "pass@16": 0.9468825073475181, + "pass@32": 0.9552734657415504, + "pass@64": 0.9645054655673708, + "pass@128": 0.9757575757575757 + }, + "4": { + "pass@1": 0.8148345044378699, + "pass@2": 0.8734522783394674, + "pass@4": 0.9115685165833552, + "pass@8": 0.9380823088154047, + "pass@16": 0.956343947805061, + "pass@32": 0.9685328260459767, + "pass@64": 0.9768866649882043, + "pass@128": 0.9822485207100592 + }, + "5": { + "pass@1": 0.6144021739130435, + "pass@2": 0.7229213026360837, + "pass@4": 0.8079759513756427, + "pass@8": 0.8658693081389128, + "pass@16": 0.902718231976262, + "pass@32": 0.9294425193718734, + "pass@64": 0.949326977779095, + "pass@128": 0.9594202898550724 + }, + "6": { + "pass@1": 0.5740732449856734, + "pass@2": 0.7082373288134823, + "pass@4": 0.8135206498184862, + "pass@8": 0.8863266213359048, + "pass@16": 0.9320016713493857, + "pass@32": 0.9581798591821608, + "pass@64": 0.9729124958024994, + "pass@128": 0.9828080229226361 + }, + "7": { + "pass@1": 0.49701950867052025, + "pass@2": 0.6282756246870876, + "pass@4": 0.7404991438931393, + "pass@8": 0.8254763539044299, + "pass@16": 0.8839952514643473, + "pass@32": 0.9212432414261512, + "pass@64": 0.944875483972104, + "pass@128": 0.9653179190751445 + }, + "8": { + "pass@1": 0.43058978873239434, + "pass@2": 0.5753673616502167, + "pass@4": 0.7030660641011416, + "pass@8": 0.8001631029757508, + "pass@16": 0.8668395813069671, + "pass@32": 0.9117980434621504, + "pass@64": 0.9437722729837049, + "pass@128": 0.971830985915493 + }, + "9": { + "pass@1": 0.3983205464071856, + "pass@2": 0.5390116666273753, + "pass@4": 0.6679228606903187, + "pass@8": 0.774652988649831, + "pass@16": 0.8587257409476783, + "pass@32": 0.9206947819975753, + "pass@64": 0.9582371946330216, + "pass@128": 0.9760479041916168 + }, + "11": { + "pass@1": 0.29918109939759036, + "pass@2": 0.4210441668247794, + "pass@4": 0.5394813751631948, + "pass@8": 0.6456911887862564, + "pass@16": 0.7395810077099041, + "pass@32": 0.8209640589794067, + "pass@64": 0.887567370372437, + "pass@128": 0.9307228915662651 + }, + "12": { + "pass@1": 0.28950333072100315, + "pass@2": 0.41677362821810293, + "pass@4": 0.5441571644257631, + "pass@8": 0.6591465794235026, + "pass@16": 0.7601174561613118, + "pass@32": 0.8408860799581142, + "pass@64": 0.8961900157902011, + "pass@128": 0.9341692789968652 + }, + "13": { + "pass@1": 0.21962535014005602, + "pass@2": 0.32394020600366136, + "pass@4": 0.4328213529506295, + "pass@8": 0.5350798081641757, + "pass@16": 0.6285083457669532, + "pass@32": 0.714978858889809, + "pass@64": 0.7926042040647505, + "pass@128": 0.8543417366946778 + }, + "14": { + "pass@1": 0.20360471491228072, + "pass@2": 0.2982582049776674, + "pass@4": 0.39714510000942854, + "pass@8": 0.491090990747798, + "pass@16": 0.5775013194565408, + "pass@32": 0.6566763801674114, + "pass@64": 0.727151911027562, + "pass@128": 0.783625730994152 + }, + "15": { + "pass@1": 0.15722064393939394, + "pass@2": 0.2387232313290383, + "pass@4": 0.32951538074786096, + "pass@8": 0.41899566652474257, + "pass@16": 0.5013357299356879, + "pass@32": 0.5751957873786209, + "pass@64": 0.639451132054347, + "pass@128": 0.6909090909090909 + }, + "16": { + "pass@1": 0.16415153133903135, + "pass@2": 0.24567567074948962, + "pass@4": 0.33242294766573, + "pass@8": 0.41320291592274866, + "pass@16": 0.48563958155008496, + "pass@32": 0.5530816266561013, + "pass@64": 0.6152416612226966, + "pass@128": 0.6638176638176638 + }, + "17": { + "pass@1": 0.16037461180124224, + "pass@2": 0.24287831038783184, + "pass@4": 0.33337522590343915, + "pass@8": 0.4201866766107831, + "pass@16": 0.49633960489111445, + "pass@32": 0.5619214680954321, + "pass@64": 0.6206053593384304, + "pass@128": 0.6708074534161491 + }, + "18": { + "pass@1": 0.1348345588235294, + "pass@2": 0.20740287748957853, + "pass@4": 0.28915632430137395, + "pass@8": 0.37054043763698213, + "pass@16": 0.4462654321967189, + "pass@32": 0.5117364598301063, + "pass@64": 0.5669886295013616, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.13102213541666666, + "pass@2": 0.19896023739220098, + "pass@4": 0.27425777079204383, + "pass@8": 0.3457286602947428, + "pass@16": 0.408312844903427, + "pass@32": 0.4615735021120364, + "pass@64": 0.5057884604835114, + "pass@128": 0.5446428571428571 + }, + "20": { + "pass@1": 0.1250935628742515, + "pass@2": 0.18957017068225754, + "pass@4": 0.2618971112892326, + "pass@8": 0.3306961799461177, + "pass@16": 0.38798512293652654, + "pass@32": 0.4361992048056054, + "pass@64": 0.48030832094747544, + "pass@128": 0.5209580838323353 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 320533, + "answer_accuracy": 0.4000901202268733, + "avg_response_len": 269.1038304841029, + "resp_tokens_sum": 215593072, + "pass_at_k": { + "pass@1": 0.4000901202268733, + "pass@2": 0.4915955276370618, + "pass@4": 0.5797298176040884, + "pass@8": 0.656446372260374, + "pass@16": 0.7187723264801746, + "pass@32": 0.7682261309010882, + "pass@64": 0.8071850532733251, + "pass@128": 0.8384725994567822 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.38957555970149255, + "pass@2": 0.5278385386061816, + "pass@4": 0.6573339027584231, + "pass@8": 0.7652352904379972, + "pass@16": 0.8462123530244711, + "pass@32": 0.9007894990247578, + "pass@64": 0.9329989074885636, + "pass@128": 0.9492537313432836 + }, + "2": { + "pass@1": 0.9677678571428572, + "pass@2": 0.9728785854893138, + "pass@4": 0.9782819976967164, + "pass@8": 0.9833322679920727, + "pass@16": 0.987546562066378, + "pass@32": 0.9910945657332776, + "pass@64": 0.9942553750832824, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9237427113702624, + "pass@2": 0.9485807258786534, + "pass@4": 0.9676794325636404, + "pass@8": 0.9822012944091195, + "pass@16": 0.9910301853418775, + "pass@32": 0.9949440882394144, + "pass@64": 0.9963556520655231, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.8395010964912281, + "pass@2": 0.8924888192429892, + "pass@4": 0.9275407303801936, + "pass@8": 0.9516473139663537, + "pass@16": 0.9668355189678384, + "pass@32": 0.9752623599544581, + "pass@64": 0.980767093553529, + "pass@128": 0.9853801169590644 + }, + "5": { + "pass@1": 0.5912683823529412, + "pass@2": 0.6982844343446042, + "pass@4": 0.7874484240940461, + "pass@8": 0.8546648972248505, + "pass@16": 0.9016930542819938, + "pass@32": 0.9313893737852658, + "pass@64": 0.9468139136974493, + "pass@128": 0.9529411764705882 + }, + "6": { + "pass@1": 0.6101389984227129, + "pass@2": 0.7372598363098934, + "pass@4": 0.8294707784949593, + "pass@8": 0.8884357436921327, + "pass@16": 0.9239306097873485, + "pass@32": 0.9486099594928609, + "pass@64": 0.9684781300081131, + "pass@128": 0.9810725552050473 + }, + "7": { + "pass@1": 0.5148091376582279, + "pass@2": 0.6504655729841523, + "pass@4": 0.7609237038605933, + "pass@8": 0.8428334528178655, + "pass@16": 0.8999233899205381, + "pass@32": 0.9376737302572167, + "pass@64": 0.9596856059802206, + "pass@128": 0.9683544303797469 + }, + "8": { + "pass@1": 0.45765296546546547, + "pass@2": 0.5974745661015346, + "pass@4": 0.7241982536529772, + "pass@8": 0.8253190393429889, + "pass@16": 0.8937941835338469, + "pass@32": 0.9343942484480311, + "pass@64": 0.9562481389043579, + "pass@128": 0.96996996996997 + }, + "9": { + "pass@1": 0.3453577170418006, + "pass@2": 0.4762565036585059, + "pass@4": 0.6051359099606117, + "pass@8": 0.716427365466166, + "pass@16": 0.8080941860304234, + "pass@32": 0.8798992794463033, + "pass@64": 0.9278435692980506, + "pass@128": 0.9581993569131833 + }, + "11": { + "pass@1": 0.32242063492063494, + "pass@2": 0.45908206786651673, + "pass@4": 0.5897093937662551, + "pass@8": 0.7004402636150577, + "pass@16": 0.7896761702070251, + "pass@32": 0.8602341105634611, + "pass@64": 0.9128581401462108, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.2635225535168196, + "pass@2": 0.3877930181319079, + "pass@4": 0.5181792000770544, + "pass@8": 0.6405584256764115, + "pass@16": 0.7469117642026594, + "pass@32": 0.8305788497806939, + "pass@64": 0.8900975412573625, + "pass@128": 0.9296636085626911 + }, + "13": { + "pass@1": 0.22899787808641975, + "pass@2": 0.3279974117818605, + "pass@4": 0.431257729936536, + "pass@8": 0.529076076642035, + "pass@16": 0.6174130400830578, + "pass@32": 0.6948931455581896, + "pass@64": 0.7644153598458592, + "pass@128": 0.8364197530864198 + }, + "14": { + "pass@1": 0.2097783753709199, + "pass@2": 0.3072589312834414, + "pass@4": 0.40860501663924065, + "pass@8": 0.5003493943525822, + "pass@16": 0.5797323311010636, + "pass@32": 0.6504938152935147, + "pass@64": 0.714900085130906, + "pass@128": 0.771513353115727 + }, + "15": { + "pass@1": 0.1486062885802469, + "pass@2": 0.23368465660542423, + "pass@4": 0.3351522572988561, + "pass@8": 0.4362885891668056, + "pass@16": 0.5222229432999017, + "pass@32": 0.5923627487289759, + "pass@64": 0.65235085460358, + "pass@128": 0.7067901234567902 + }, + "16": { + "pass@1": 0.16894819321533924, + "pass@2": 0.24454161150210196, + "pass@4": 0.3213824090572752, + "pass@8": 0.39293382315692404, + "pass@16": 0.4604045860512616, + "pass@32": 0.5278180398696859, + "pass@64": 0.5974230289076049, + "pass@128": 0.6666666666666666 + }, + "17": { + "pass@1": 0.13337725903614459, + "pass@2": 0.20455357354615303, + "pass@4": 0.28689263634967316, + "pass@8": 0.3693797529401078, + "pass@16": 0.44313391916126493, + "pass@32": 0.5065971685899989, + "pass@64": 0.5609107151651793, + "pass@128": 0.608433734939759 + }, + "18": { + "pass@1": 0.14690020161290324, + "pass@2": 0.21565833756667505, + "pass@4": 0.2908736779878321, + "pass@8": 0.3619471264143349, + "pass@16": 0.42385086814033907, + "pass@32": 0.47860570791588575, + "pass@64": 0.5266059947925121, + "pass@128": 0.5709677419354838 + }, + "19": { + "pass@1": 0.14307968277945618, + "pass@2": 0.20948621155172828, + "pass@4": 0.2771546254036975, + "pass@8": 0.3391845110184821, + "pass@16": 0.39618189679402127, + "pass@32": 0.45002902801284544, + "pass@64": 0.4975762422706243, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.11660097597597598, + "pass@2": 0.18397235818495664, + "pass@4": 0.2672153804535694, + "pass@8": 0.3537960139658834, + "pass@16": 0.4293593992377488, + "pass@32": 0.4897216597246643, + "pass@64": 0.5406000531835978, + "pass@128": 0.5825825825825826 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/checkpoint-21350_metrics.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/checkpoint-21350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..f2b62c78d594a1d40e3587cf8e6812d59ca94293 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/checkpoint-21350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21350", + "id": { + "answer_accuracy": 0.9463585069444445, + "avg_response_len": 197.93385069444443, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9463585069444445, + "pass@2": 0.9596882655293003, + "pass@4": 0.9683837043390396, + "pass@8": 0.9747526385084491, + "pass@16": 0.9796784191169823, + "pass@32": 0.9836883253406419, + "pass@64": 0.9870538876300826, + "pass@128": 0.9897777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.868234375, + "pass@2": 0.8999448818897641, + "pass@4": 0.9197073342707163, + "pass@8": 0.9331076441166724, + "pass@16": 0.9433611017222235, + "pass@32": 0.9523665996932675, + "pass@64": 0.9610741606668138, + "pass@128": 0.968 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983046875, + "pass@2": 0.9991064222440943, + "pass@4": 0.999448690944882, + "pass@8": 0.9997133661121024, + "pass@16": 0.9999252105409085, + "pass@32": 0.9999963932374032, + "pass@64": 0.9999999990502599, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.986625, + "pass@2": 0.9891411171259843, + "pass@4": 0.9909370606486688, + "pass@8": 0.9925176607283016, + "pass@16": 0.9939879059023505, + "pass@32": 0.995160827004678, + "pass@64": 0.996128323786748, + "pass@128": 0.997 + }, + "5": { + "pass@1": 0.9240625, + "pass@2": 0.9398949311023624, + "pass@4": 0.9511819646606674, + "pass@8": 0.9597021804111605, + "pass@16": 0.9663716821807865, + "pass@32": 0.9720694340633057, + "pass@64": 0.9778673595311561, + "pass@128": 0.984 + }, + "6": { + "pass@1": 0.9748984375, + "pass@2": 0.9809261811023623, + "pass@4": 0.9849990272778404, + "pass@8": 0.9877118907748831, + "pass@16": 0.9895177450928163, + "pass@32": 0.9911213266355339, + "pass@64": 0.9929305146553141, + "pass@128": 0.995 + }, + "7": { + "pass@1": 0.9504453125, + "pass@2": 0.965091658464567, + "pass@4": 0.9748439168541432, + "pass@8": 0.9818050618635551, + "pass@16": 0.9865700446035913, + "pass@32": 0.9900657689345447, + "pass@64": 0.9931611428840618, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.92240625, + "pass@2": 0.9430205462598424, + "pass@4": 0.9563017823397072, + "pass@8": 0.9664568236870597, + "pass@16": 0.9744903411611363, + "pass@32": 0.9806094062516463, + "pass@64": 0.9847184264373984, + "pass@128": 0.987 + }, + "9": { + "pass@1": 0.8922578125, + "pass@2": 0.9200686515748038, + "pass@4": 0.9380335620547431, + "pass@8": 0.9517591188823109, + "pass@16": 0.9628817408490075, + "pass@32": 0.9718051722453913, + "pass@64": 0.9776050616589868, + "pass@128": 0.981 + } + }, + "per_op_accuracy": { + "10": 0.868234375, + "2": 0.9999921875, + "3": 0.9983046875, + "4": 0.986625, + "5": 0.9240625, + "6": 0.9748984375, + "7": 0.9504453125, + "8": 0.92240625, + "9": 0.8922578125 + }, + "per_op_avg_response_len": { + "10": 276.837875, + "2": 114.8861875, + "3": 143.5635390625, + "4": 158.611890625, + "5": 171.5188203125, + "6": 197.0850859375, + "7": 221.6113671875, + "8": 240.8029140625, + "9": 256.4869765625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.837875, + "2": 114.8861875, + "3": 143.5635390625, + "4": 158.611890625, + "5": 171.5188203125, + "6": 197.0850859375, + "7": 221.6113671875, + "8": 240.8029140625, + "9": 256.4869765625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 359587, + "answer_accuracy": 0.9487583375548801, + "avg_response_len": 188.60273134076326, + "resp_tokens_sum": 71481944, + "pass_at_k": { + "pass@1": 0.9487583375548801, + "pass@2": 0.9628412056870569, + "pass@4": 0.9716166803870911, + "pass@8": 0.9776510646285631, + "pass@16": 0.9821713776881223, + "pass@32": 0.9858318204777058, + "pass@64": 0.9889798130482067, + "pass@128": 0.9915569064505235 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8773276179941003, + "pass@2": 0.9119181590133094, + "pass@4": 0.9311756113450421, + "pass@8": 0.9420426122301978, + "pass@16": 0.9502590673394334, + "pass@32": 0.9584931653088372, + "pass@64": 0.9669784816191926, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973241590214067, + "pass@2": 0.9977444225721784, + "pass@4": 0.9983450506186725, + "pass@8": 0.9991236482754671, + "pass@16": 0.9997712860612012, + "pass@32": 0.9999889701449637, + "pass@64": 0.9999999970955962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9881591796875, + "pass@2": 0.9902486005167324, + "pass@4": 0.9916557953693287, + "pass@8": 0.9928006480587112, + "pass@16": 0.9937693424773156, + "pass@32": 0.994500319642821, + "pass@64": 0.9953124622828344, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9253224206349207, + "pass@2": 0.9414018560179978, + "pass@4": 0.9528689018039411, + "pass@8": 0.9625002308833911, + "pass@16": 0.9709636195225438, + "pass@32": 0.9777918358496315, + "pass@64": 0.9838997800481912, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9671594311377245, + "pass@2": 0.9752820883822906, + "pass@4": 0.9815461927663233, + "pass@8": 0.9861414925202345, + "pass@16": 0.9886085746496264, + "pass@32": 0.9900551906526195, + "pass@64": 0.9917723246221735, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9505593565088757, + "pass@2": 0.9665288810977031, + "pass@4": 0.976226311115696, + "pass@8": 0.9825814576886557, + "pass@16": 0.9870875795976058, + "pass@32": 0.9910536770529155, + "pass@64": 0.9947454835702054, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9407301682692307, + "pass@2": 0.9576164382697357, + "pass@4": 0.9672972564126601, + "pass@8": 0.9735731917238386, + "pass@16": 0.977776925424273, + "pass@32": 0.980567025073276, + "pass@64": 0.9823063355516893, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.8992077464788732, + "pass@2": 0.9294131917489195, + "pass@4": 0.9485870498582041, + "pass@8": 0.9623563850919464, + "pass@16": 0.9729279398669448, + "pass@32": 0.9810885217868446, + "pass@64": 0.9864213997414775, + "pass@128": 0.9887323943661972 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 368669, + "answer_accuracy": 0.943717746559633, + "avg_response_len": 195.85829732552423, + "resp_tokens_sum": 76513219, + "pass_at_k": { + "pass@1": 0.943717746559633, + "pass@2": 0.9567327010944173, + "pass@4": 0.9655748257241944, + "pass@8": 0.972221249560269, + "pass@16": 0.9774107239324217, + "pass@32": 0.9816549821121477, + "pass@64": 0.9851665059461757, + "pass@128": 0.9882044560943644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8393884202453987, + "pass@2": 0.8698545662045313, + "pass@4": 0.8904956702114691, + "pass@8": 0.9055833604644317, + "pass@16": 0.9173558658878211, + "pass@32": 0.9274155966730994, + "pass@64": 0.9375057974027622, + "pass@128": 0.9478527607361963 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9979876893939394, + "pass@2": 0.9995783375089478, + "pass@4": 0.9999699469384509, + "pass@8": 0.9999997974031156, + "pass@16": 0.9999999999966537, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9895062869822485, + "pass@2": 0.9918151295252295, + "pass@4": 0.9931505192768064, + "pass@8": 0.9945047565814593, + "pass@16": 0.9959301126815046, + "pass@32": 0.9968252000189942, + "pass@64": 0.9970371391134153, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.9276947463768116, + "pass@2": 0.941486933698505, + "pass@4": 0.9521146527879665, + "pass@8": 0.9596253376638073, + "pass@16": 0.9648710403519182, + "pass@32": 0.9689627240711924, + "pass@64": 0.9727993021286935, + "pass@128": 0.9768115942028985 + }, + "6": { + "pass@1": 0.973047994269341, + "pass@2": 0.9776248364280394, + "pass@4": 0.9810814086097404, + "pass@8": 0.9838584328812273, + "pass@16": 0.9865986488348304, + "pass@32": 0.9898077163259399, + "pass@64": 0.9933483043882753, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.949534862716763, + "pass@2": 0.9634724466341997, + "pass@4": 0.9741216728328035, + "pass@8": 0.9818345949858238, + "pass@16": 0.9860409160803892, + "pass@32": 0.9882156768256282, + "pass@64": 0.9906125875539492, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9240096830985915, + "pass@2": 0.94706491904181, + "pass@4": 0.9618155671562175, + "pass@8": 0.972741330632731, + "pass@16": 0.9807618398300412, + "pass@32": 0.9868507906518273, + "pass@64": 0.9914611399873605, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.8909758607784432, + "pass@2": 0.918286458578905, + "pass@4": 0.9358988911939895, + "pass@8": 0.9502432948194586, + "pass@16": 0.9635661425413976, + "pass@32": 0.9753201066438478, + "pass@64": 0.9822986698173887, + "pass@128": 0.9850299401197605 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 361949, + "answer_accuracy": 0.9466777912621359, + "avg_response_len": 209.3044677979578, + "resp_tokens_sum": 80024633, + "pass_at_k": { + "pass@1": 0.9466777912621359, + "pass@2": 0.9595826501453824, + "pass@4": 0.9680488718831473, + "pass@8": 0.9744659158195264, + "pass@16": 0.9795242026369446, + "pass@32": 0.983641078749584, + "pass@64": 0.9870731791386894, + "pass@128": 0.9896216940073652 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8871035447761194, + "pass@2": 0.917110559407686, + "pass@4": 0.9365289956292769, + "pass@8": 0.95085081510218, + "pass@16": 0.9616873600439593, + "pass@32": 0.9704475586213182, + "pass@64": 0.9780345237152435, + "pass@128": 0.982089552238806 + }, + "2": { + "pass@1": 0.9999776785714286, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995444606413995, + "pass@2": 0.9999508591400565, + "pass@4": 0.9999993495710995, + "pass@8": 0.9999999999504271, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9823419225146199, + "pass@2": 0.9854621379564396, + "pass@4": 0.9880769901021142, + "pass@8": 0.9902890222952649, + "pass@16": 0.9922729187814656, + "pass@32": 0.9941339389256002, + "pass@64": 0.9959935170640539, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9192095588235294, + "pass@2": 0.9368833950903197, + "pass@4": 0.9486726628840507, + "pass@8": 0.957187841731995, + "pass@16": 0.963640097381688, + "pass@32": 0.9699201940179724, + "pass@64": 0.9774210870046359, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9850897082018928, + "pass@2": 0.9905075573163766, + "pass@4": 0.9929501176311952, + "pass@8": 0.9936089564592303, + "pass@16": 0.9936894407460112, + "pass@32": 0.9936908516082206, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9513202136075949, + "pass@2": 0.9653273105003486, + "pass@4": 0.9741560914458472, + "pass@8": 0.9809422762648561, + "pass@16": 0.9865958409360319, + "pass@32": 0.9910348161676964, + "pass@64": 0.9942570067837525, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9035285285285285, + "pass@2": 0.9250335473268545, + "pass@4": 0.9401216576531538, + "pass@8": 0.9530895360198267, + "pass@16": 0.9647251870543512, + "pass@32": 0.9739953866287852, + "pass@64": 0.9797902253746495, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.8857013665594855, + "pass@2": 0.9113160493202016, + "pass@4": 0.9282795167083213, + "pass@8": 0.9412904877973327, + "pass@16": 0.9506791980302753, + "pass@32": 0.9574335414534275, + "pass@64": 0.962500672127828, + "pass@128": 0.9678456591639871 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.3788546875, + "avg_response_len": 316.5756515625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.3788546875, + "pass@2": 0.4468145054133882, + "pass@4": 0.5052995829583767, + "pass@8": 0.5549551204811086, + "pass@16": 0.5968148364108548, + "pass@32": 0.6325811412160591, + "pass@64": 0.6635066827237466, + "pass@128": 0.6911 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8291484375, + "pass@2": 0.873491018700788, + "pass@4": 0.9010196287963997, + "pass@8": 0.9201213309779261, + "pass@16": 0.9355011309270999, + "pass@32": 0.9491665247467566, + "pass@64": 0.9605726109540238, + "pass@128": 0.969 + }, + "12": { + "pass@1": 0.7599296875, + "pass@2": 0.8307192421259846, + "pass@4": 0.8765910849268834, + "pass@8": 0.9067747904417431, + "pass@16": 0.9269737822770376, + "pass@32": 0.9421725813666925, + "pass@64": 0.9550685629642635, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.5739453125, + "pass@2": 0.6672687007874021, + "pass@4": 0.7365055441507309, + "pass@8": 0.7895033392107424, + "pass@16": 0.8321358451700712, + "pass@32": 0.8673482450962469, + "pass@64": 0.8947274314200474, + "pass@128": 0.915 + }, + "14": { + "pass@1": 0.4429765625, + "pass@2": 0.5411972194881896, + "pass@4": 0.6223649927821522, + "pass@8": 0.6864600896229462, + "pass@16": 0.734990186345135, + "pass@32": 0.7720859734078392, + "pass@64": 0.8021589021822163, + "pass@128": 0.829 + }, + "15": { + "pass@1": 0.2981171875, + "pass@2": 0.3796037155511811, + "pass@4": 0.454164298181477, + "pass@8": 0.5216627779720748, + "pass@16": 0.5819837911464046, + "pass@32": 0.6335288623305411, + "pass@64": 0.6771801624126927, + "pass@128": 0.718 + }, + "16": { + "pass@1": 0.2247421875, + "pass@2": 0.2948574064960629, + "pass@4": 0.3609774574428192, + "pass@8": 0.4196990046322681, + "pass@16": 0.4703733132845017, + "pass@32": 0.5154560906917757, + "pass@64": 0.5580749266719405, + "pass@128": 0.601 + }, + "17": { + "pass@1": 0.177125, + "pass@2": 0.2343399360236221, + "pass@4": 0.29065695088113985, + "pass@8": 0.34292267929622505, + "pass@16": 0.39018242306353984, + "pass@32": 0.43289689874886583, + "pass@64": 0.47146846566711437, + "pass@128": 0.506 + }, + "18": { + "pass@1": 0.1759375, + "pass@2": 0.23381606791338574, + "pass@4": 0.2916848028683913, + "pass@8": 0.3458914790449526, + "pass@16": 0.3942252943790366, + "pass@32": 0.43722673231514386, + "pass@64": 0.4755549722411987, + "pass@128": 0.511 + }, + "19": { + "pass@1": 0.15396875, + "pass@2": 0.20712869094488198, + "pass@4": 0.26051826546681645, + "pass@8": 0.3095364327763067, + "pass@16": 0.35240874208040834, + "pass@32": 0.3901755959020489, + "pass@64": 0.4230897825923825, + "pass@128": 0.452 + }, + "20": { + "pass@1": 0.15265625, + "pass@2": 0.20572305610236225, + "pass@4": 0.25851280408698907, + "pass@8": 0.3069792808358564, + "pass@16": 0.349373855435176, + "pass@32": 0.38575390755468886, + "pass@64": 0.41717101013153834, + "pass@128": 0.445 + } + }, + "per_op_accuracy": { + "11": 0.8291484375, + "12": 0.7599296875, + "13": 0.5739453125, + "14": 0.4429765625, + "15": 0.2981171875, + "16": 0.2247421875, + "17": 0.177125, + "18": 0.1759375, + "19": 0.15396875, + "20": 0.15265625 + }, + "per_op_avg_response_len": { + "11": 293.0501484375, + "12": 301.278390625, + "13": 312.555453125, + "14": 320.473265625, + "15": 321.0238828125, + "16": 319.6802578125, + "17": 325.398703125, + "18": 325.6219765625, + "19": 321.5646015625, + "20": 325.1098359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 293.0501484375, + "12": 301.278390625, + "13": 312.555453125, + "14": 320.473265625, + "15": 321.0238828125, + "16": 319.6802578125, + "17": 325.398703125, + "18": 325.6219765625, + "19": 321.5646015625, + "20": 325.1098359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 166059, + "answer_accuracy": 0.38553816864784546, + "avg_response_len": 301.7915722511144, + "resp_tokens_sum": 129987666, + "pass_at_k": { + "pass@1": 0.38553816864784546, + "pass@2": 0.45339526710814393, + "pass@4": 0.5109667024627136, + "pass@8": 0.5596039271446713, + "pass@16": 0.6013694017394466, + "pass@32": 0.6374170076422611, + "pass@64": 0.6678700712730753, + "pass@128": 0.6936106983655275 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8281471317280453, + "pass@2": 0.8772393126408069, + "pass@4": 0.9072971293779494, + "pass@8": 0.9263560752155664, + "pass@16": 0.9417339425971359, + "pass@32": 0.9566709407386268, + "pass@64": 0.9699104428716377, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.7567973163841808, + "pass@2": 0.8258520452422264, + "pass@4": 0.8681805065680348, + "pass@8": 0.8951861291592782, + "pass@16": 0.9141290331047311, + "pass@32": 0.9301895220722955, + "pass@64": 0.9461462846873937, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.5737656739811913, + "pass@2": 0.6663702083281907, + "pass@4": 0.7377947493044555, + "pass@8": 0.7931345691897921, + "pass@16": 0.8379584677978166, + "pass@32": 0.8751047840970735, + "pass@64": 0.9032302694399705, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.45030179127725856, + "pass@2": 0.5457960789363944, + "pass@4": 0.6231636244067621, + "pass@8": 0.6869051629447177, + "pass@16": 0.7392870707923059, + "pass@32": 0.7804201059407638, + "pass@64": 0.8141525302007103, + "pass@128": 0.8473520249221184 + }, + "15": { + "pass@1": 0.3126354768786127, + "pass@2": 0.3937431017022439, + "pass@4": 0.4662351267688358, + "pass@8": 0.5297946130837053, + "pass@16": 0.586091346781155, + "pass@32": 0.6351826501037516, + "pass@64": 0.6761407868785784, + "pass@128": 0.7109826589595376 + }, + "16": { + "pass@1": 0.25042842741935484, + "pass@2": 0.32574811087122146, + "pass@4": 0.39351368578927626, + "pass@8": 0.45303587701396264, + "pass@16": 0.5062086173550845, + "pass@32": 0.5539039754509739, + "pass@64": 0.5955508562334999, + "pass@128": 0.632258064516129 + }, + "17": { + "pass@1": 0.16848807803468208, + "pass@2": 0.22524364503208774, + "pass@4": 0.2802942645175134, + "pass@8": 0.330783163112241, + "pass@16": 0.37907349032212845, + "pass@32": 0.42469351653658793, + "pass@64": 0.4638600016363443, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.1693080357142857, + "pass@2": 0.22570620078740164, + "pass@4": 0.28333418795864795, + "pass@8": 0.33963163946315195, + "pass@16": 0.3904539856568932, + "pass@32": 0.4325615854877157, + "pass@64": 0.464574906335151, + "pass@128": 0.49142857142857144 + }, + "19": { + "pass@1": 0.16042605105105104, + "pass@2": 0.21579022427939754, + "pass@4": 0.2713904286738932, + "pass@8": 0.320673574822643, + "pass@16": 0.36145502547813574, + "pass@32": 0.3960321250148641, + "pass@64": 0.42531332447168435, + "pass@128": 0.45045045045045046 + }, + "20": { + "pass@1": 0.15836148648648649, + "pass@2": 0.20984591875339906, + "pass@4": 0.25997215860404826, + "pass@8": 0.3057199370131288, + "pass@16": 0.34552328217296413, + "pass@32": 0.38000113840819033, + "pass@64": 0.41232314729781633, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 163915, + "answer_accuracy": 0.3807867789176331, + "avg_response_len": 313.8555790960452, + "resp_tokens_sum": 135103528, + "pass_at_k": { + "pass@1": 0.3807867789176331, + "pass@2": 0.4495632327014004, + "pass@4": 0.5093539215269832, + "pass@8": 0.5604206578674388, + "pass@16": 0.6035574413237094, + "pass@32": 0.6408628361842268, + "pass@64": 0.6737987491809093, + "pass@128": 0.7032411537317871 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8322195030120482, + "pass@2": 0.8740064836116118, + "pass@4": 0.9007651367562485, + "pass@8": 0.9202482006306049, + "pass@16": 0.9352773543837686, + "pass@32": 0.9477013646227804, + "pass@64": 0.9572351213137299, + "pass@128": 0.963855421686747 + }, + "12": { + "pass@1": 0.7488489420062696, + "pass@2": 0.8215862038111226, + "pass@4": 0.8709659145271413, + "pass@8": 0.9032511169652427, + "pass@16": 0.9243224931700824, + "pass@32": 0.9391710642481247, + "pass@64": 0.9510478149135736, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5842086834733894, + "pass@2": 0.676471966739452, + "pass@4": 0.741411255840919, + "pass@8": 0.7904271208733067, + "pass@16": 0.8313294364350797, + "pass@32": 0.8672863371588099, + "pass@64": 0.8973304964641204, + "pass@128": 0.9215686274509803 + }, + "14": { + "pass@1": 0.45575201023391815, + "pass@2": 0.5523883938849747, + "pass@4": 0.6318049690060672, + "pass@8": 0.6928149400614, + "pass@16": 0.7359096694993563, + "pass@32": 0.7666387497632563, + "pass@64": 0.790515190000481, + "pass@128": 0.8099415204678363 + }, + "15": { + "pass@1": 0.3084280303030303, + "pass@2": 0.3932630935337627, + "pass@4": 0.471590225085501, + "pass@8": 0.5436062297147697, + "pass@16": 0.6081874040818871, + "pass@32": 0.6620940150440326, + "pass@64": 0.7063833432601488, + "pass@128": 0.7454545454545455 + }, + "16": { + "pass@1": 0.22760861823361822, + "pass@2": 0.3023122209884019, + "pass@4": 0.3737860908412092, + "pass@8": 0.43597528456928797, + "pass@16": 0.4874427513369775, + "pass@32": 0.532987304252585, + "pass@64": 0.5746519986523746, + "pass@128": 0.6096866096866097 + }, + "17": { + "pass@1": 0.19157608695652173, + "pass@2": 0.2501818726463538, + "pass@4": 0.3103001718147032, + "pass@8": 0.36859091329527444, + "pass@16": 0.4233694246594647, + "pass@32": 0.47383565180387, + "pass@64": 0.5219014440208865, + "pass@128": 0.5714285714285714 + }, + "18": { + "pass@1": 0.1758501838235294, + "pass@2": 0.2325841680176007, + "pass@4": 0.2908814795209421, + "pass@8": 0.3483809368179013, + "pass@16": 0.4025550346930342, + "pass@32": 0.45392796747630026, + "pass@64": 0.5037675550562966, + "pass@128": 0.5529411764705883 + }, + "19": { + "pass@1": 0.14306640625, + "pass@2": 0.19530993684383194, + "pass@4": 0.2483651784040386, + "pass@8": 0.29777947891872014, + "pass@16": 0.34304739154710734, + "pass@32": 0.3863642388713654, + "pass@64": 0.4263632707832371, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.14897548652694612, + "pass@2": 0.20376631088217265, + "pass@4": 0.25812340644919385, + "pass@8": 0.30629608870839276, + "pass@16": 0.3463328241220228, + "pass@32": 0.3798774817729502, + "pass@64": 0.40926661845536116, + "pass@128": 0.437125748502994 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 154960, + "answer_accuracy": 0.36999541564792177, + "avg_response_len": 334.57566091075796, + "resp_tokens_sum": 140125640, + "pass_at_k": { + "pass@1": 0.36999541564792177, + "pass@2": 0.43722152467608877, + "pass@4": 0.4953042902510825, + "pass@8": 0.5445566373964037, + "pass@16": 0.5852006882895915, + "pass@32": 0.6190958017594159, + "pass@64": 0.6484409669645834, + "pass@128": 0.676039119804401 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8270337301587302, + "pass@2": 0.8687472659667539, + "pass@4": 0.8942530689616175, + "pass@8": 0.913000742277685, + "pass@16": 0.9287522778885744, + "pass@32": 0.9423010463849473, + "pass@64": 0.9536259185529427, + "pass@128": 0.9619047619047619 + }, + "12": { + "pass@1": 0.7741303516819572, + "pass@2": 0.844897917720147, + "pass@4": 0.8911836662619007, + "pass@8": 0.9227577810625305, + "pass@16": 0.9434655328339684, + "pass@32": 0.9580731530824094, + "pass@64": 0.9686499242431075, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5628134645061729, + "pass@2": 0.6580126919898899, + "pass@4": 0.7298308666277817, + "pass@8": 0.7849102761957668, + "pass@16": 0.8272916207877943, + "pass@32": 0.8597796191468667, + "pass@64": 0.8834876179969318, + "pass@128": 0.9012345679012346 + }, + "14": { + "pass@1": 0.42303412462908013, + "pass@2": 0.5254594879553262, + "pass@4": 0.612024243167675, + "pass@8": 0.6795870113254974, + "pass@16": 0.7299641829436971, + "pass@32": 0.769675569678991, + "pass@64": 0.802551172129445, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.2721113040123457, + "pass@2": 0.3505919181977249, + "pass@4": 0.42352521617668165, + "pass@8": 0.49062896925691024, + "pass@16": 0.5509084623861222, + "pass@32": 0.6026686433028146, + "pass@64": 0.648546132335972, + "pass@128": 0.6975308641975309 + }, + "16": { + "pass@1": 0.1982853982300885, + "pass@2": 0.25889056802081145, + "pass@4": 0.3179625279140993, + "pass@8": 0.3723615276522692, + "pass@16": 0.41992990025116966, + "pass@32": 0.46214547052895755, + "pass@64": 0.5066410316005072, + "pass@128": 0.5634218289085545 + }, + "17": { + "pass@1": 0.17211031626506024, + "pass@2": 0.2284550356939569, + "pass@4": 0.2824050603426078, + "pass@8": 0.3306790083684093, + "pass@16": 0.3695724116618554, + "pass@32": 0.40174054863361564, + "pass@64": 0.4304838558018472, + "pass@128": 0.4578313253012048 + }, + "18": { + "pass@1": 0.18351814516129034, + "pass@2": 0.2443234855219709, + "pass@4": 0.3019939807927236, + "pass@8": 0.35022866682181714, + "pass@16": 0.38934737936610586, + "pass@32": 0.4241763498467773, + "pass@64": 0.457008988079855, + "pass@128": 0.4870967741935484 + }, + "19": { + "pass@1": 0.15853946374622357, + "pass@2": 0.21041210719128384, + "pass@4": 0.26191707786904267, + "pass@8": 0.3102665481682076, + "pass@16": 0.35281055902223907, + "pass@32": 0.38815261030610226, + "pass@64": 0.4175298687617645, + "pass@128": 0.44410876132930516 + }, + "20": { + "pass@1": 0.15064283033033032, + "pass@2": 0.20356281478328725, + "pass@4": 0.2574440165700009, + "pass@8": 0.30892386841405955, + "pass@16": 0.35627459223670677, + "pass@32": 0.3974007494071951, + "pass@64": 0.4299470015533774, + "pass@128": 0.45345345345345345 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6476722861842106, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6476722861842106, + "pass@2": 0.6897547075735369, + "pass@4": 0.7246552194018351, + "pass@8": 0.7538065763888013, + "pass@16": 0.7781712703242901, + "pass@32": 0.7988950705382448, + "pass@64": 0.8167658850478021, + "pass@128": 0.8325789473684211 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.868234375, + "pass@2": 0.8999448818897641, + "pass@4": 0.9197073342707163, + "pass@8": 0.9331076441166724, + "pass@16": 0.9433611017222235, + "pass@32": 0.9523665996932675, + "pass@64": 0.9610741606668138, + "pass@128": 0.968 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983046875, + "pass@2": 0.9991064222440943, + "pass@4": 0.999448690944882, + "pass@8": 0.9997133661121024, + "pass@16": 0.9999252105409085, + "pass@32": 0.9999963932374032, + "pass@64": 0.9999999990502599, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.986625, + "pass@2": 0.9891411171259843, + "pass@4": 0.9909370606486688, + "pass@8": 0.9925176607283016, + "pass@16": 0.9939879059023505, + "pass@32": 0.995160827004678, + "pass@64": 0.996128323786748, + "pass@128": 0.997 + }, + "5": { + "pass@1": 0.9240625, + "pass@2": 0.9398949311023624, + "pass@4": 0.9511819646606674, + "pass@8": 0.9597021804111605, + "pass@16": 0.9663716821807865, + "pass@32": 0.9720694340633057, + "pass@64": 0.9778673595311561, + "pass@128": 0.984 + }, + "6": { + "pass@1": 0.9748984375, + "pass@2": 0.9809261811023623, + "pass@4": 0.9849990272778404, + "pass@8": 0.9877118907748831, + "pass@16": 0.9895177450928163, + "pass@32": 0.9911213266355339, + "pass@64": 0.9929305146553141, + "pass@128": 0.995 + }, + "7": { + "pass@1": 0.9504453125, + "pass@2": 0.965091658464567, + "pass@4": 0.9748439168541432, + "pass@8": 0.9818050618635551, + "pass@16": 0.9865700446035913, + "pass@32": 0.9900657689345447, + "pass@64": 0.9931611428840618, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.92240625, + "pass@2": 0.9430205462598424, + "pass@4": 0.9563017823397072, + "pass@8": 0.9664568236870597, + "pass@16": 0.9744903411611363, + "pass@32": 0.9806094062516463, + "pass@64": 0.9847184264373984, + "pass@128": 0.987 + }, + "9": { + "pass@1": 0.8922578125, + "pass@2": 0.9200686515748038, + "pass@4": 0.9380335620547431, + "pass@8": 0.9517591188823109, + "pass@16": 0.9628817408490075, + "pass@32": 0.9718051722453913, + "pass@64": 0.9776050616589868, + "pass@128": 0.981 + }, + "11": { + "pass@1": 0.8291484375, + "pass@2": 0.873491018700788, + "pass@4": 0.9010196287963997, + "pass@8": 0.9201213309779261, + "pass@16": 0.9355011309270999, + "pass@32": 0.9491665247467566, + "pass@64": 0.9605726109540238, + "pass@128": 0.969 + }, + "12": { + "pass@1": 0.7599296875, + "pass@2": 0.8307192421259846, + "pass@4": 0.8765910849268834, + "pass@8": 0.9067747904417431, + "pass@16": 0.9269737822770376, + "pass@32": 0.9421725813666925, + "pass@64": 0.9550685629642635, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.5739453125, + "pass@2": 0.6672687007874021, + "pass@4": 0.7365055441507309, + "pass@8": 0.7895033392107424, + "pass@16": 0.8321358451700712, + "pass@32": 0.8673482450962469, + "pass@64": 0.8947274314200474, + "pass@128": 0.915 + }, + "14": { + "pass@1": 0.4429765625, + "pass@2": 0.5411972194881896, + "pass@4": 0.6223649927821522, + "pass@8": 0.6864600896229462, + "pass@16": 0.734990186345135, + "pass@32": 0.7720859734078392, + "pass@64": 0.8021589021822163, + "pass@128": 0.829 + }, + "15": { + "pass@1": 0.2981171875, + "pass@2": 0.3796037155511811, + "pass@4": 0.454164298181477, + "pass@8": 0.5216627779720748, + "pass@16": 0.5819837911464046, + "pass@32": 0.6335288623305411, + "pass@64": 0.6771801624126927, + "pass@128": 0.718 + }, + "16": { + "pass@1": 0.2247421875, + "pass@2": 0.2948574064960629, + "pass@4": 0.3609774574428192, + "pass@8": 0.4196990046322681, + "pass@16": 0.4703733132845017, + "pass@32": 0.5154560906917757, + "pass@64": 0.5580749266719405, + "pass@128": 0.601 + }, + "17": { + "pass@1": 0.177125, + "pass@2": 0.2343399360236221, + "pass@4": 0.29065695088113985, + "pass@8": 0.34292267929622505, + "pass@16": 0.39018242306353984, + "pass@32": 0.43289689874886583, + "pass@64": 0.47146846566711437, + "pass@128": 0.506 + }, + "18": { + "pass@1": 0.1759375, + "pass@2": 0.23381606791338574, + "pass@4": 0.2916848028683913, + "pass@8": 0.3458914790449526, + "pass@16": 0.3942252943790366, + "pass@32": 0.43722673231514386, + "pass@64": 0.4755549722411987, + "pass@128": 0.511 + }, + "19": { + "pass@1": 0.15396875, + "pass@2": 0.20712869094488198, + "pass@4": 0.26051826546681645, + "pass@8": 0.3095364327763067, + "pass@16": 0.35240874208040834, + "pass@32": 0.3901755959020489, + "pass@64": 0.4230897825923825, + "pass@128": 0.452 + }, + "20": { + "pass@1": 0.15265625, + "pass@2": 0.20572305610236225, + "pass@4": 0.25851280408698907, + "pass@8": 0.3069792808358564, + "pass@16": 0.349373855435176, + "pass@32": 0.38575390755468886, + "pass@64": 0.41717101013153834, + "pass@128": 0.445 + } + }, + "per_op_accuracy": { + "10": 0.868234375, + "2": 0.9999921875, + "3": 0.9983046875, + "4": 0.986625, + "5": 0.9240625, + "6": 0.9748984375, + "7": 0.9504453125, + "8": 0.92240625, + "9": 0.8922578125, + "11": 0.8291484375, + "12": 0.7599296875, + "13": 0.5739453125, + "14": 0.4429765625, + "15": 0.2981171875, + "16": 0.2247421875, + "17": 0.177125, + "18": 0.1759375, + "19": 0.15396875, + "20": 0.15265625 + }, + "per_op_avg_response_len": { + "10": 276.837875, + "2": 114.8861875, + "3": 143.5635390625, + "4": 158.611890625, + "5": 171.5188203125, + "6": 197.0850859375, + "7": 221.6113671875, + "8": 240.8029140625, + "9": 256.4869765625, + "11": 293.0501484375, + "12": 301.278390625, + "13": 312.555453125, + "14": 320.473265625, + "15": 321.0238828125, + "16": 319.6802578125, + "17": 325.398703125, + "18": 325.6219765625, + "19": 321.5646015625, + "20": 325.1098359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.837875, + "2": 114.8861875, + "3": 143.5635390625, + "4": 158.611890625, + "5": 171.5188203125, + "6": 197.0850859375, + "7": 221.6113671875, + "8": 240.8029140625, + "9": 256.4869765625, + "11": 293.0501484375, + "12": 301.278390625, + "13": 312.555453125, + "14": 320.473265625, + "15": 321.0238828125, + "16": 319.6802578125, + "17": 325.398703125, + "18": 325.6219765625, + "19": 321.5646015625, + "20": 325.1098359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 525646, + "answer_accuracy": 0.6491636697755295, + "avg_response_len": 248.8114650845716, + "resp_tokens_sum": 201469610, + "pass_at_k": { + "pass@1": 0.6491636697755295, + "pass@2": 0.6918507562216686, + "pass@4": 0.7265823497333529, + "pass@8": 0.7552785357582993, + "pass@16": 0.779610731297473, + "pass@32": 0.8004989331569241, + "pass@64": 0.8181713588791755, + "pass@128": 0.8330698703762252 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8773276179941003, + "pass@2": 0.9119181590133094, + "pass@4": 0.9311756113450421, + "pass@8": 0.9420426122301978, + "pass@16": 0.9502590673394334, + "pass@32": 0.9584931653088372, + "pass@64": 0.9669784816191926, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973241590214067, + "pass@2": 0.9977444225721784, + "pass@4": 0.9983450506186725, + "pass@8": 0.9991236482754671, + "pass@16": 0.9997712860612012, + "pass@32": 0.9999889701449637, + "pass@64": 0.9999999970955962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9881591796875, + "pass@2": 0.9902486005167324, + "pass@4": 0.9916557953693287, + "pass@8": 0.9928006480587112, + "pass@16": 0.9937693424773156, + "pass@32": 0.994500319642821, + "pass@64": 0.9953124622828344, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9253224206349207, + "pass@2": 0.9414018560179978, + "pass@4": 0.9528689018039411, + "pass@8": 0.9625002308833911, + "pass@16": 0.9709636195225438, + "pass@32": 0.9777918358496315, + "pass@64": 0.9838997800481912, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9671594311377245, + "pass@2": 0.9752820883822906, + "pass@4": 0.9815461927663233, + "pass@8": 0.9861414925202345, + "pass@16": 0.9886085746496264, + "pass@32": 0.9900551906526195, + "pass@64": 0.9917723246221735, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9505593565088757, + "pass@2": 0.9665288810977031, + "pass@4": 0.976226311115696, + "pass@8": 0.9825814576886557, + "pass@16": 0.9870875795976058, + "pass@32": 0.9910536770529155, + "pass@64": 0.9947454835702054, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9407301682692307, + "pass@2": 0.9576164382697357, + "pass@4": 0.9672972564126601, + "pass@8": 0.9735731917238386, + "pass@16": 0.977776925424273, + "pass@32": 0.980567025073276, + "pass@64": 0.9823063355516893, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.8992077464788732, + "pass@2": 0.9294131917489195, + "pass@4": 0.9485870498582041, + "pass@8": 0.9623563850919464, + "pass@16": 0.9729279398669448, + "pass@32": 0.9810885217868446, + "pass@64": 0.9864213997414775, + "pass@128": 0.9887323943661972 + }, + "11": { + "pass@1": 0.8281471317280453, + "pass@2": 0.8772393126408069, + "pass@4": 0.9072971293779494, + "pass@8": 0.9263560752155664, + "pass@16": 0.9417339425971359, + "pass@32": 0.9566709407386268, + "pass@64": 0.9699104428716377, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.7567973163841808, + "pass@2": 0.8258520452422264, + "pass@4": 0.8681805065680348, + "pass@8": 0.8951861291592782, + "pass@16": 0.9141290331047311, + "pass@32": 0.9301895220722955, + "pass@64": 0.9461462846873937, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.5737656739811913, + "pass@2": 0.6663702083281907, + "pass@4": 0.7377947493044555, + "pass@8": 0.7931345691897921, + "pass@16": 0.8379584677978166, + "pass@32": 0.8751047840970735, + "pass@64": 0.9032302694399705, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.45030179127725856, + "pass@2": 0.5457960789363944, + "pass@4": 0.6231636244067621, + "pass@8": 0.6869051629447177, + "pass@16": 0.7392870707923059, + "pass@32": 0.7804201059407638, + "pass@64": 0.8141525302007103, + "pass@128": 0.8473520249221184 + }, + "15": { + "pass@1": 0.3126354768786127, + "pass@2": 0.3937431017022439, + "pass@4": 0.4662351267688358, + "pass@8": 0.5297946130837053, + "pass@16": 0.586091346781155, + "pass@32": 0.6351826501037516, + "pass@64": 0.6761407868785784, + "pass@128": 0.7109826589595376 + }, + "16": { + "pass@1": 0.25042842741935484, + "pass@2": 0.32574811087122146, + "pass@4": 0.39351368578927626, + "pass@8": 0.45303587701396264, + "pass@16": 0.5062086173550845, + "pass@32": 0.5539039754509739, + "pass@64": 0.5955508562334999, + "pass@128": 0.632258064516129 + }, + "17": { + "pass@1": 0.16848807803468208, + "pass@2": 0.22524364503208774, + "pass@4": 0.2802942645175134, + "pass@8": 0.330783163112241, + "pass@16": 0.37907349032212845, + "pass@32": 0.42469351653658793, + "pass@64": 0.4638600016363443, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.1693080357142857, + "pass@2": 0.22570620078740164, + "pass@4": 0.28333418795864795, + "pass@8": 0.33963163946315195, + "pass@16": 0.3904539856568932, + "pass@32": 0.4325615854877157, + "pass@64": 0.464574906335151, + "pass@128": 0.49142857142857144 + }, + "19": { + "pass@1": 0.16042605105105104, + "pass@2": 0.21579022427939754, + "pass@4": 0.2713904286738932, + "pass@8": 0.320673574822643, + "pass@16": 0.36145502547813574, + "pass@32": 0.3960321250148641, + "pass@64": 0.42531332447168435, + "pass@128": 0.45045045045045046 + }, + "20": { + "pass@1": 0.15836148648648649, + "pass@2": 0.20984591875339906, + "pass@4": 0.25997215860404826, + "pass@8": 0.3057199370131288, + "pass@16": 0.34552328217296413, + "pass@32": 0.38000113840819033, + "pass@64": 0.41232314729781633, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 532584, + "answer_accuracy": 0.6486067809820732, + "avg_response_len": 257.71719967848793, + "resp_tokens_sum": 211616747, + "pass_at_k": { + "pass@1": 0.6486067809820732, + "pass@2": 0.6908541473600844, + "pass@4": 0.7264055504607114, + "pass@8": 0.7563388816938654, + "pass@16": 0.7814218557464435, + "pass@32": 0.8029980862811904, + "pass@64": 0.821935053724575, + "pass@128": 0.8388152766952455 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8393884202453987, + "pass@2": 0.8698545662045313, + "pass@4": 0.8904956702114691, + "pass@8": 0.9055833604644317, + "pass@16": 0.9173558658878211, + "pass@32": 0.9274155966730994, + "pass@64": 0.9375057974027622, + "pass@128": 0.9478527607361963 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9979876893939394, + "pass@2": 0.9995783375089478, + "pass@4": 0.9999699469384509, + "pass@8": 0.9999997974031156, + "pass@16": 0.9999999999966537, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9895062869822485, + "pass@2": 0.9918151295252295, + "pass@4": 0.9931505192768064, + "pass@8": 0.9945047565814593, + "pass@16": 0.9959301126815046, + "pass@32": 0.9968252000189942, + "pass@64": 0.9970371391134153, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.9276947463768116, + "pass@2": 0.941486933698505, + "pass@4": 0.9521146527879665, + "pass@8": 0.9596253376638073, + "pass@16": 0.9648710403519182, + "pass@32": 0.9689627240711924, + "pass@64": 0.9727993021286935, + "pass@128": 0.9768115942028985 + }, + "6": { + "pass@1": 0.973047994269341, + "pass@2": 0.9776248364280394, + "pass@4": 0.9810814086097404, + "pass@8": 0.9838584328812273, + "pass@16": 0.9865986488348304, + "pass@32": 0.9898077163259399, + "pass@64": 0.9933483043882753, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.949534862716763, + "pass@2": 0.9634724466341997, + "pass@4": 0.9741216728328035, + "pass@8": 0.9818345949858238, + "pass@16": 0.9860409160803892, + "pass@32": 0.9882156768256282, + "pass@64": 0.9906125875539492, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9240096830985915, + "pass@2": 0.94706491904181, + "pass@4": 0.9618155671562175, + "pass@8": 0.972741330632731, + "pass@16": 0.9807618398300412, + "pass@32": 0.9868507906518273, + "pass@64": 0.9914611399873605, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.8909758607784432, + "pass@2": 0.918286458578905, + "pass@4": 0.9358988911939895, + "pass@8": 0.9502432948194586, + "pass@16": 0.9635661425413976, + "pass@32": 0.9753201066438478, + "pass@64": 0.9822986698173887, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.8322195030120482, + "pass@2": 0.8740064836116118, + "pass@4": 0.9007651367562485, + "pass@8": 0.9202482006306049, + "pass@16": 0.9352773543837686, + "pass@32": 0.9477013646227804, + "pass@64": 0.9572351213137299, + "pass@128": 0.963855421686747 + }, + "12": { + "pass@1": 0.7488489420062696, + "pass@2": 0.8215862038111226, + "pass@4": 0.8709659145271413, + "pass@8": 0.9032511169652427, + "pass@16": 0.9243224931700824, + "pass@32": 0.9391710642481247, + "pass@64": 0.9510478149135736, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5842086834733894, + "pass@2": 0.676471966739452, + "pass@4": 0.741411255840919, + "pass@8": 0.7904271208733067, + "pass@16": 0.8313294364350797, + "pass@32": 0.8672863371588099, + "pass@64": 0.8973304964641204, + "pass@128": 0.9215686274509803 + }, + "14": { + "pass@1": 0.45575201023391815, + "pass@2": 0.5523883938849747, + "pass@4": 0.6318049690060672, + "pass@8": 0.6928149400614, + "pass@16": 0.7359096694993563, + "pass@32": 0.7666387497632563, + "pass@64": 0.790515190000481, + "pass@128": 0.8099415204678363 + }, + "15": { + "pass@1": 0.3084280303030303, + "pass@2": 0.3932630935337627, + "pass@4": 0.471590225085501, + "pass@8": 0.5436062297147697, + "pass@16": 0.6081874040818871, + "pass@32": 0.6620940150440326, + "pass@64": 0.7063833432601488, + "pass@128": 0.7454545454545455 + }, + "16": { + "pass@1": 0.22760861823361822, + "pass@2": 0.3023122209884019, + "pass@4": 0.3737860908412092, + "pass@8": 0.43597528456928797, + "pass@16": 0.4874427513369775, + "pass@32": 0.532987304252585, + "pass@64": 0.5746519986523746, + "pass@128": 0.6096866096866097 + }, + "17": { + "pass@1": 0.19157608695652173, + "pass@2": 0.2501818726463538, + "pass@4": 0.3103001718147032, + "pass@8": 0.36859091329527444, + "pass@16": 0.4233694246594647, + "pass@32": 0.47383565180387, + "pass@64": 0.5219014440208865, + "pass@128": 0.5714285714285714 + }, + "18": { + "pass@1": 0.1758501838235294, + "pass@2": 0.2325841680176007, + "pass@4": 0.2908814795209421, + "pass@8": 0.3483809368179013, + "pass@16": 0.4025550346930342, + "pass@32": 0.45392796747630026, + "pass@64": 0.5037675550562966, + "pass@128": 0.5529411764705883 + }, + "19": { + "pass@1": 0.14306640625, + "pass@2": 0.19530993684383194, + "pass@4": 0.2483651784040386, + "pass@8": 0.29777947891872014, + "pass@16": 0.34304739154710734, + "pass@32": 0.3863642388713654, + "pass@64": 0.4263632707832371, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.14897548652694612, + "pass@2": 0.20376631088217265, + "pass@4": 0.25812340644919385, + "pass@8": 0.30629608870839276, + "pass@16": 0.3463328241220228, + "pass@32": 0.3798774817729502, + "pass@64": 0.40926661845536116, + "pass@128": 0.437125748502994 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 516909, + "answer_accuracy": 0.6452071517015497, + "avg_response_len": 274.7921405675827, + "resp_tokens_sum": 220150273, + "pass_at_k": { + "pass@1": 0.6452071517015497, + "pass@2": 0.6865093792497867, + "pass@4": 0.7209135034376871, + "pass@8": 0.7497234395452865, + "pass@16": 0.7733847971497279, + "pass@32": 0.7930687594794426, + "pass@64": 0.810047360600001, + "pass@128": 0.8256910049528678 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8871035447761194, + "pass@2": 0.917110559407686, + "pass@4": 0.9365289956292769, + "pass@8": 0.95085081510218, + "pass@16": 0.9616873600439593, + "pass@32": 0.9704475586213182, + "pass@64": 0.9780345237152435, + "pass@128": 0.982089552238806 + }, + "2": { + "pass@1": 0.9999776785714286, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995444606413995, + "pass@2": 0.9999508591400565, + "pass@4": 0.9999993495710995, + "pass@8": 0.9999999999504271, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9823419225146199, + "pass@2": 0.9854621379564396, + "pass@4": 0.9880769901021142, + "pass@8": 0.9902890222952649, + "pass@16": 0.9922729187814656, + "pass@32": 0.9941339389256002, + "pass@64": 0.9959935170640539, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9192095588235294, + "pass@2": 0.9368833950903197, + "pass@4": 0.9486726628840507, + "pass@8": 0.957187841731995, + "pass@16": 0.963640097381688, + "pass@32": 0.9699201940179724, + "pass@64": 0.9774210870046359, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9850897082018928, + "pass@2": 0.9905075573163766, + "pass@4": 0.9929501176311952, + "pass@8": 0.9936089564592303, + "pass@16": 0.9936894407460112, + "pass@32": 0.9936908516082206, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9513202136075949, + "pass@2": 0.9653273105003486, + "pass@4": 0.9741560914458472, + "pass@8": 0.9809422762648561, + "pass@16": 0.9865958409360319, + "pass@32": 0.9910348161676964, + "pass@64": 0.9942570067837525, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9035285285285285, + "pass@2": 0.9250335473268545, + "pass@4": 0.9401216576531538, + "pass@8": 0.9530895360198267, + "pass@16": 0.9647251870543512, + "pass@32": 0.9739953866287852, + "pass@64": 0.9797902253746495, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.8857013665594855, + "pass@2": 0.9113160493202016, + "pass@4": 0.9282795167083213, + "pass@8": 0.9412904877973327, + "pass@16": 0.9506791980302753, + "pass@32": 0.9574335414534275, + "pass@64": 0.962500672127828, + "pass@128": 0.9678456591639871 + }, + "11": { + "pass@1": 0.8270337301587302, + "pass@2": 0.8687472659667539, + "pass@4": 0.8942530689616175, + "pass@8": 0.913000742277685, + "pass@16": 0.9287522778885744, + "pass@32": 0.9423010463849473, + "pass@64": 0.9536259185529427, + "pass@128": 0.9619047619047619 + }, + "12": { + "pass@1": 0.7741303516819572, + "pass@2": 0.844897917720147, + "pass@4": 0.8911836662619007, + "pass@8": 0.9227577810625305, + "pass@16": 0.9434655328339684, + "pass@32": 0.9580731530824094, + "pass@64": 0.9686499242431075, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5628134645061729, + "pass@2": 0.6580126919898899, + "pass@4": 0.7298308666277817, + "pass@8": 0.7849102761957668, + "pass@16": 0.8272916207877943, + "pass@32": 0.8597796191468667, + "pass@64": 0.8834876179969318, + "pass@128": 0.9012345679012346 + }, + "14": { + "pass@1": 0.42303412462908013, + "pass@2": 0.5254594879553262, + "pass@4": 0.612024243167675, + "pass@8": 0.6795870113254974, + "pass@16": 0.7299641829436971, + "pass@32": 0.769675569678991, + "pass@64": 0.802551172129445, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.2721113040123457, + "pass@2": 0.3505919181977249, + "pass@4": 0.42352521617668165, + "pass@8": 0.49062896925691024, + "pass@16": 0.5509084623861222, + "pass@32": 0.6026686433028146, + "pass@64": 0.648546132335972, + "pass@128": 0.6975308641975309 + }, + "16": { + "pass@1": 0.1982853982300885, + "pass@2": 0.25889056802081145, + "pass@4": 0.3179625279140993, + "pass@8": 0.3723615276522692, + "pass@16": 0.41992990025116966, + "pass@32": 0.46214547052895755, + "pass@64": 0.5066410316005072, + "pass@128": 0.5634218289085545 + }, + "17": { + "pass@1": 0.17211031626506024, + "pass@2": 0.2284550356939569, + "pass@4": 0.2824050603426078, + "pass@8": 0.3306790083684093, + "pass@16": 0.3695724116618554, + "pass@32": 0.40174054863361564, + "pass@64": 0.4304838558018472, + "pass@128": 0.4578313253012048 + }, + "18": { + "pass@1": 0.18351814516129034, + "pass@2": 0.2443234855219709, + "pass@4": 0.3019939807927236, + "pass@8": 0.35022866682181714, + "pass@16": 0.38934737936610586, + "pass@32": 0.4241763498467773, + "pass@64": 0.457008988079855, + "pass@128": 0.4870967741935484 + }, + "19": { + "pass@1": 0.15853946374622357, + "pass@2": 0.21041210719128384, + "pass@4": 0.26191707786904267, + "pass@8": 0.3102665481682076, + "pass@16": 0.35281055902223907, + "pass@32": 0.38815261030610226, + "pass@64": 0.4175298687617645, + "pass@128": 0.44410876132930516 + }, + "20": { + "pass@1": 0.15064283033033032, + "pass@2": 0.20356281478328725, + "pass@4": 0.2574440165700009, + "pass@8": 0.30892386841405955, + "pass@16": 0.35627459223670677, + "pass@32": 0.3974007494071951, + "pass@64": 0.4299470015533774, + "pass@128": 0.45345345345345345 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/summary.csv b/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..b1e746c8432dd68c95b461f58d9e9b091de3b985 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21350,0.9463585069444445,0.3788546875,0.6476722861842106,0.0,0.0,0.0,197.93385069444443,316.5756515625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/summary.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0b7b6257c675ebb77ecc2dc1c67cd3724828662e --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block2/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21350", + "id_acc": 0.9463585069444445, + "ood_acc": 0.3788546875, + "total_acc": 0.6476722861842106, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.93385069444443, + "ood_avg_resp_len": 316.5756515625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 525646, + "answer_accuracy": 0.6491636697755295, + "avg_response_len": 248.8114650845716, + "resp_tokens_sum": 201469610, + "pass_at_k": { + "pass@1": 0.6491636697755295, + "pass@2": 0.6918507562216686, + "pass@4": 0.7265823497333529, + "pass@8": 0.7552785357582993, + "pass@16": 0.779610731297473, + "pass@32": 0.8004989331569241, + "pass@64": 0.8181713588791755, + "pass@128": 0.8330698703762252 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8773276179941003, + "pass@2": 0.9119181590133094, + "pass@4": 0.9311756113450421, + "pass@8": 0.9420426122301978, + "pass@16": 0.9502590673394334, + "pass@32": 0.9584931653088372, + "pass@64": 0.9669784816191926, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9973241590214067, + "pass@2": 0.9977444225721784, + "pass@4": 0.9983450506186725, + "pass@8": 0.9991236482754671, + "pass@16": 0.9997712860612012, + "pass@32": 0.9999889701449637, + "pass@64": 0.9999999970955962, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9881591796875, + "pass@2": 0.9902486005167324, + "pass@4": 0.9916557953693287, + "pass@8": 0.9928006480587112, + "pass@16": 0.9937693424773156, + "pass@32": 0.994500319642821, + "pass@64": 0.9953124622828344, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9253224206349207, + "pass@2": 0.9414018560179978, + "pass@4": 0.9528689018039411, + "pass@8": 0.9625002308833911, + "pass@16": 0.9709636195225438, + "pass@32": 0.9777918358496315, + "pass@64": 0.9838997800481912, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9671594311377245, + "pass@2": 0.9752820883822906, + "pass@4": 0.9815461927663233, + "pass@8": 0.9861414925202345, + "pass@16": 0.9886085746496264, + "pass@32": 0.9900551906526195, + "pass@64": 0.9917723246221735, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9505593565088757, + "pass@2": 0.9665288810977031, + "pass@4": 0.976226311115696, + "pass@8": 0.9825814576886557, + "pass@16": 0.9870875795976058, + "pass@32": 0.9910536770529155, + "pass@64": 0.9947454835702054, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9407301682692307, + "pass@2": 0.9576164382697357, + "pass@4": 0.9672972564126601, + "pass@8": 0.9735731917238386, + "pass@16": 0.977776925424273, + "pass@32": 0.980567025073276, + "pass@64": 0.9823063355516893, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.8992077464788732, + "pass@2": 0.9294131917489195, + "pass@4": 0.9485870498582041, + "pass@8": 0.9623563850919464, + "pass@16": 0.9729279398669448, + "pass@32": 0.9810885217868446, + "pass@64": 0.9864213997414775, + "pass@128": 0.9887323943661972 + }, + "11": { + "pass@1": 0.8281471317280453, + "pass@2": 0.8772393126408069, + "pass@4": 0.9072971293779494, + "pass@8": 0.9263560752155664, + "pass@16": 0.9417339425971359, + "pass@32": 0.9566709407386268, + "pass@64": 0.9699104428716377, + "pass@128": 0.9801699716713881 + }, + "12": { + "pass@1": 0.7567973163841808, + "pass@2": 0.8258520452422264, + "pass@4": 0.8681805065680348, + "pass@8": 0.8951861291592782, + "pass@16": 0.9141290331047311, + "pass@32": 0.9301895220722955, + "pass@64": 0.9461462846873937, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.5737656739811913, + "pass@2": 0.6663702083281907, + "pass@4": 0.7377947493044555, + "pass@8": 0.7931345691897921, + "pass@16": 0.8379584677978166, + "pass@32": 0.8751047840970735, + "pass@64": 0.9032302694399705, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.45030179127725856, + "pass@2": 0.5457960789363944, + "pass@4": 0.6231636244067621, + "pass@8": 0.6869051629447177, + "pass@16": 0.7392870707923059, + "pass@32": 0.7804201059407638, + "pass@64": 0.8141525302007103, + "pass@128": 0.8473520249221184 + }, + "15": { + "pass@1": 0.3126354768786127, + "pass@2": 0.3937431017022439, + "pass@4": 0.4662351267688358, + "pass@8": 0.5297946130837053, + "pass@16": 0.586091346781155, + "pass@32": 0.6351826501037516, + "pass@64": 0.6761407868785784, + "pass@128": 0.7109826589595376 + }, + "16": { + "pass@1": 0.25042842741935484, + "pass@2": 0.32574811087122146, + "pass@4": 0.39351368578927626, + "pass@8": 0.45303587701396264, + "pass@16": 0.5062086173550845, + "pass@32": 0.5539039754509739, + "pass@64": 0.5955508562334999, + "pass@128": 0.632258064516129 + }, + "17": { + "pass@1": 0.16848807803468208, + "pass@2": 0.22524364503208774, + "pass@4": 0.2802942645175134, + "pass@8": 0.330783163112241, + "pass@16": 0.37907349032212845, + "pass@32": 0.42469351653658793, + "pass@64": 0.4638600016363443, + "pass@128": 0.4913294797687861 + }, + "18": { + "pass@1": 0.1693080357142857, + "pass@2": 0.22570620078740164, + "pass@4": 0.28333418795864795, + "pass@8": 0.33963163946315195, + "pass@16": 0.3904539856568932, + "pass@32": 0.4325615854877157, + "pass@64": 0.464574906335151, + "pass@128": 0.49142857142857144 + }, + "19": { + "pass@1": 0.16042605105105104, + "pass@2": 0.21579022427939754, + "pass@4": 0.2713904286738932, + "pass@8": 0.320673574822643, + "pass@16": 0.36145502547813574, + "pass@32": 0.3960321250148641, + "pass@64": 0.42531332447168435, + "pass@128": 0.45045045045045046 + }, + "20": { + "pass@1": 0.15836148648648649, + "pass@2": 0.20984591875339906, + "pass@4": 0.25997215860404826, + "pass@8": 0.3057199370131288, + "pass@16": 0.34552328217296413, + "pass@32": 0.38000113840819033, + "pass@64": 0.41232314729781633, + "pass@128": 0.4444444444444444 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 532584, + "answer_accuracy": 0.6486067809820732, + "avg_response_len": 257.71719967848793, + "resp_tokens_sum": 211616747, + "pass_at_k": { + "pass@1": 0.6486067809820732, + "pass@2": 0.6908541473600844, + "pass@4": 0.7264055504607114, + "pass@8": 0.7563388816938654, + "pass@16": 0.7814218557464435, + "pass@32": 0.8029980862811904, + "pass@64": 0.821935053724575, + "pass@128": 0.8388152766952455 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8393884202453987, + "pass@2": 0.8698545662045313, + "pass@4": 0.8904956702114691, + "pass@8": 0.9055833604644317, + "pass@16": 0.9173558658878211, + "pass@32": 0.9274155966730994, + "pass@64": 0.9375057974027622, + "pass@128": 0.9478527607361963 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9979876893939394, + "pass@2": 0.9995783375089478, + "pass@4": 0.9999699469384509, + "pass@8": 0.9999997974031156, + "pass@16": 0.9999999999966537, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9895062869822485, + "pass@2": 0.9918151295252295, + "pass@4": 0.9931505192768064, + "pass@8": 0.9945047565814593, + "pass@16": 0.9959301126815046, + "pass@32": 0.9968252000189942, + "pass@64": 0.9970371391134153, + "pass@128": 0.9970414201183432 + }, + "5": { + "pass@1": 0.9276947463768116, + "pass@2": 0.941486933698505, + "pass@4": 0.9521146527879665, + "pass@8": 0.9596253376638073, + "pass@16": 0.9648710403519182, + "pass@32": 0.9689627240711924, + "pass@64": 0.9727993021286935, + "pass@128": 0.9768115942028985 + }, + "6": { + "pass@1": 0.973047994269341, + "pass@2": 0.9776248364280394, + "pass@4": 0.9810814086097404, + "pass@8": 0.9838584328812273, + "pass@16": 0.9865986488348304, + "pass@32": 0.9898077163259399, + "pass@64": 0.9933483043882753, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.949534862716763, + "pass@2": 0.9634724466341997, + "pass@4": 0.9741216728328035, + "pass@8": 0.9818345949858238, + "pass@16": 0.9860409160803892, + "pass@32": 0.9882156768256282, + "pass@64": 0.9906125875539492, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9240096830985915, + "pass@2": 0.94706491904181, + "pass@4": 0.9618155671562175, + "pass@8": 0.972741330632731, + "pass@16": 0.9807618398300412, + "pass@32": 0.9868507906518273, + "pass@64": 0.9914611399873605, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.8909758607784432, + "pass@2": 0.918286458578905, + "pass@4": 0.9358988911939895, + "pass@8": 0.9502432948194586, + "pass@16": 0.9635661425413976, + "pass@32": 0.9753201066438478, + "pass@64": 0.9822986698173887, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.8322195030120482, + "pass@2": 0.8740064836116118, + "pass@4": 0.9007651367562485, + "pass@8": 0.9202482006306049, + "pass@16": 0.9352773543837686, + "pass@32": 0.9477013646227804, + "pass@64": 0.9572351213137299, + "pass@128": 0.963855421686747 + }, + "12": { + "pass@1": 0.7488489420062696, + "pass@2": 0.8215862038111226, + "pass@4": 0.8709659145271413, + "pass@8": 0.9032511169652427, + "pass@16": 0.9243224931700824, + "pass@32": 0.9391710642481247, + "pass@64": 0.9510478149135736, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.5842086834733894, + "pass@2": 0.676471966739452, + "pass@4": 0.741411255840919, + "pass@8": 0.7904271208733067, + "pass@16": 0.8313294364350797, + "pass@32": 0.8672863371588099, + "pass@64": 0.8973304964641204, + "pass@128": 0.9215686274509803 + }, + "14": { + "pass@1": 0.45575201023391815, + "pass@2": 0.5523883938849747, + "pass@4": 0.6318049690060672, + "pass@8": 0.6928149400614, + "pass@16": 0.7359096694993563, + "pass@32": 0.7666387497632563, + "pass@64": 0.790515190000481, + "pass@128": 0.8099415204678363 + }, + "15": { + "pass@1": 0.3084280303030303, + "pass@2": 0.3932630935337627, + "pass@4": 0.471590225085501, + "pass@8": 0.5436062297147697, + "pass@16": 0.6081874040818871, + "pass@32": 0.6620940150440326, + "pass@64": 0.7063833432601488, + "pass@128": 0.7454545454545455 + }, + "16": { + "pass@1": 0.22760861823361822, + "pass@2": 0.3023122209884019, + "pass@4": 0.3737860908412092, + "pass@8": 0.43597528456928797, + "pass@16": 0.4874427513369775, + "pass@32": 0.532987304252585, + "pass@64": 0.5746519986523746, + "pass@128": 0.6096866096866097 + }, + "17": { + "pass@1": 0.19157608695652173, + "pass@2": 0.2501818726463538, + "pass@4": 0.3103001718147032, + "pass@8": 0.36859091329527444, + "pass@16": 0.4233694246594647, + "pass@32": 0.47383565180387, + "pass@64": 0.5219014440208865, + "pass@128": 0.5714285714285714 + }, + "18": { + "pass@1": 0.1758501838235294, + "pass@2": 0.2325841680176007, + "pass@4": 0.2908814795209421, + "pass@8": 0.3483809368179013, + "pass@16": 0.4025550346930342, + "pass@32": 0.45392796747630026, + "pass@64": 0.5037675550562966, + "pass@128": 0.5529411764705883 + }, + "19": { + "pass@1": 0.14306640625, + "pass@2": 0.19530993684383194, + "pass@4": 0.2483651784040386, + "pass@8": 0.29777947891872014, + "pass@16": 0.34304739154710734, + "pass@32": 0.3863642388713654, + "pass@64": 0.4263632707832371, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.14897548652694612, + "pass@2": 0.20376631088217265, + "pass@4": 0.25812340644919385, + "pass@8": 0.30629608870839276, + "pass@16": 0.3463328241220228, + "pass@32": 0.3798774817729502, + "pass@64": 0.40926661845536116, + "pass@128": 0.437125748502994 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 516909, + "answer_accuracy": 0.6452071517015497, + "avg_response_len": 274.7921405675827, + "resp_tokens_sum": 220150273, + "pass_at_k": { + "pass@1": 0.6452071517015497, + "pass@2": 0.6865093792497867, + "pass@4": 0.7209135034376871, + "pass@8": 0.7497234395452865, + "pass@16": 0.7733847971497279, + "pass@32": 0.7930687594794426, + "pass@64": 0.810047360600001, + "pass@128": 0.8256910049528678 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8871035447761194, + "pass@2": 0.917110559407686, + "pass@4": 0.9365289956292769, + "pass@8": 0.95085081510218, + "pass@16": 0.9616873600439593, + "pass@32": 0.9704475586213182, + "pass@64": 0.9780345237152435, + "pass@128": 0.982089552238806 + }, + "2": { + "pass@1": 0.9999776785714286, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995444606413995, + "pass@2": 0.9999508591400565, + "pass@4": 0.9999993495710995, + "pass@8": 0.9999999999504271, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9823419225146199, + "pass@2": 0.9854621379564396, + "pass@4": 0.9880769901021142, + "pass@8": 0.9902890222952649, + "pass@16": 0.9922729187814656, + "pass@32": 0.9941339389256002, + "pass@64": 0.9959935170640539, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9192095588235294, + "pass@2": 0.9368833950903197, + "pass@4": 0.9486726628840507, + "pass@8": 0.957187841731995, + "pass@16": 0.963640097381688, + "pass@32": 0.9699201940179724, + "pass@64": 0.9774210870046359, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9850897082018928, + "pass@2": 0.9905075573163766, + "pass@4": 0.9929501176311952, + "pass@8": 0.9936089564592303, + "pass@16": 0.9936894407460112, + "pass@32": 0.9936908516082206, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9513202136075949, + "pass@2": 0.9653273105003486, + "pass@4": 0.9741560914458472, + "pass@8": 0.9809422762648561, + "pass@16": 0.9865958409360319, + "pass@32": 0.9910348161676964, + "pass@64": 0.9942570067837525, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9035285285285285, + "pass@2": 0.9250335473268545, + "pass@4": 0.9401216576531538, + "pass@8": 0.9530895360198267, + "pass@16": 0.9647251870543512, + "pass@32": 0.9739953866287852, + "pass@64": 0.9797902253746495, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.8857013665594855, + "pass@2": 0.9113160493202016, + "pass@4": 0.9282795167083213, + "pass@8": 0.9412904877973327, + "pass@16": 0.9506791980302753, + "pass@32": 0.9574335414534275, + "pass@64": 0.962500672127828, + "pass@128": 0.9678456591639871 + }, + "11": { + "pass@1": 0.8270337301587302, + "pass@2": 0.8687472659667539, + "pass@4": 0.8942530689616175, + "pass@8": 0.913000742277685, + "pass@16": 0.9287522778885744, + "pass@32": 0.9423010463849473, + "pass@64": 0.9536259185529427, + "pass@128": 0.9619047619047619 + }, + "12": { + "pass@1": 0.7741303516819572, + "pass@2": 0.844897917720147, + "pass@4": 0.8911836662619007, + "pass@8": 0.9227577810625305, + "pass@16": 0.9434655328339684, + "pass@32": 0.9580731530824094, + "pass@64": 0.9686499242431075, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5628134645061729, + "pass@2": 0.6580126919898899, + "pass@4": 0.7298308666277817, + "pass@8": 0.7849102761957668, + "pass@16": 0.8272916207877943, + "pass@32": 0.8597796191468667, + "pass@64": 0.8834876179969318, + "pass@128": 0.9012345679012346 + }, + "14": { + "pass@1": 0.42303412462908013, + "pass@2": 0.5254594879553262, + "pass@4": 0.612024243167675, + "pass@8": 0.6795870113254974, + "pass@16": 0.7299641829436971, + "pass@32": 0.769675569678991, + "pass@64": 0.802551172129445, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.2721113040123457, + "pass@2": 0.3505919181977249, + "pass@4": 0.42352521617668165, + "pass@8": 0.49062896925691024, + "pass@16": 0.5509084623861222, + "pass@32": 0.6026686433028146, + "pass@64": 0.648546132335972, + "pass@128": 0.6975308641975309 + }, + "16": { + "pass@1": 0.1982853982300885, + "pass@2": 0.25889056802081145, + "pass@4": 0.3179625279140993, + "pass@8": 0.3723615276522692, + "pass@16": 0.41992990025116966, + "pass@32": 0.46214547052895755, + "pass@64": 0.5066410316005072, + "pass@128": 0.5634218289085545 + }, + "17": { + "pass@1": 0.17211031626506024, + "pass@2": 0.2284550356939569, + "pass@4": 0.2824050603426078, + "pass@8": 0.3306790083684093, + "pass@16": 0.3695724116618554, + "pass@32": 0.40174054863361564, + "pass@64": 0.4304838558018472, + "pass@128": 0.4578313253012048 + }, + "18": { + "pass@1": 0.18351814516129034, + "pass@2": 0.2443234855219709, + "pass@4": 0.3019939807927236, + "pass@8": 0.35022866682181714, + "pass@16": 0.38934737936610586, + "pass@32": 0.4241763498467773, + "pass@64": 0.457008988079855, + "pass@128": 0.4870967741935484 + }, + "19": { + "pass@1": 0.15853946374622357, + "pass@2": 0.21041210719128384, + "pass@4": 0.26191707786904267, + "pass@8": 0.3102665481682076, + "pass@16": 0.35281055902223907, + "pass@32": 0.38815261030610226, + "pass@64": 0.4175298687617645, + "pass@128": 0.44410876132930516 + }, + "20": { + "pass@1": 0.15064283033033032, + "pass@2": 0.20356281478328725, + "pass@4": 0.2574440165700009, + "pass@8": 0.30892386841405955, + "pass@16": 0.35627459223670677, + "pass@32": 0.3974007494071951, + "pass@64": 0.4299470015533774, + "pass@128": 0.45345345345345345 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/checkpoint-21350_metrics.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/checkpoint-21350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..07d176bd182cb0de91e16c03f26392a373048599 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/checkpoint-21350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21350", + "id": { + "answer_accuracy": 0.509234375, + "avg_response_len": 202.11198611111112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.509234375, + "pass@2": 0.6027151410761119, + "pass@4": 0.6920740703661933, + "pass@8": 0.7690246436234909, + "pass@16": 0.8301603489437002, + "pass@32": 0.8776945143314032, + "pass@64": 0.9145530180698405, + "pass@128": 0.942 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2391328125, + "pass@2": 0.35308710629921264, + "pass@4": 0.4779890196850392, + "pass@8": 0.5983404366426991, + "pass@16": 0.7025556939003653, + "pass@32": 0.7880716543980395, + "pass@64": 0.8550543114097935, + "pass@128": 0.904 + }, + "2": { + "pass@1": 0.93296875, + "pass@2": 0.946091289370079, + "pass@4": 0.9574039729096363, + "pass@8": 0.9673537985153713, + "pass@16": 0.9754377893321514, + "pass@32": 0.9820102065156547, + "pass@64": 0.9870498115345813, + "pass@128": 0.99 + }, + "3": { + "pass@1": 0.8902109375, + "pass@2": 0.9252320374015754, + "pass@4": 0.9462774102924636, + "pass@8": 0.9600688463402408, + "pass@16": 0.9694946858975106, + "pass@32": 0.9758968136385496, + "pass@64": 0.9802805104126652, + "pass@128": 0.984 + }, + "4": { + "pass@1": 0.7475546875, + "pass@2": 0.8143569143700784, + "pass@4": 0.8639035401199855, + "pass@8": 0.8996917929775318, + "pass@16": 0.9250908501267543, + "pass@32": 0.9425543170446925, + "pass@64": 0.9550058423117043, + "pass@128": 0.965 + }, + "5": { + "pass@1": 0.4376171875, + "pass@2": 0.5431691683070871, + "pass@4": 0.6393361189538813, + "pass@8": 0.7236498280843922, + "pass@16": 0.7927134685726412, + "pass@32": 0.8462263961232798, + "pass@64": 0.8856200302156424, + "pass@128": 0.913 + }, + "6": { + "pass@1": 0.436140625, + "pass@2": 0.5749576771653546, + "pass@4": 0.7034071109861268, + "pass@8": 0.8033494150190057, + "pass@16": 0.8716152333594489, + "pass@32": 0.916058813354604, + "pass@64": 0.9450113977230314, + "pass@128": 0.965 + }, + "7": { + "pass@1": 0.363453125, + "pass@2": 0.48999187992125987, + "pass@4": 0.6112021976940375, + "pass@8": 0.7143600169659958, + "pass@16": 0.7957880248776545, + "pass@32": 0.8596348255159127, + "pass@64": 0.9114422391992211, + "pass@128": 0.951 + }, + "8": { + "pass@1": 0.29865625, + "pass@2": 0.4261337352362206, + "pass@4": 0.5547488435508061, + "pass@8": 0.6675436813877224, + "pass@16": 0.7569429013867229, + "pass@32": 0.824615286256769, + "pass@64": 0.8754762742085503, + "pass@128": 0.915 + }, + "9": { + "pass@1": 0.237375, + "pass@2": 0.35141646161417367, + "pass@4": 0.47439841910386227, + "pass@8": 0.5868639766784868, + "pass@16": 0.6818044930399056, + "pass@32": 0.7641823161351303, + "pass@64": 0.8360367456132549, + "pass@128": 0.891 + } + }, + "per_op_accuracy": { + "10": 0.2391328125, + "2": 0.93296875, + "3": 0.8902109375, + "4": 0.7475546875, + "5": 0.4376171875, + "6": 0.436140625, + "7": 0.363453125, + "8": 0.29865625, + "9": 0.237375 + }, + "per_op_avg_response_len": { + "10": 273.950546875, + "2": 116.4940703125, + "3": 148.770578125, + "4": 162.255390625, + "5": 175.85075, + "6": 207.5196875, + "7": 229.7950859375, + "8": 245.6833828125, + "9": 258.6883828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.950546875, + "2": 116.4940703125, + "3": 148.770578125, + "4": 162.255390625, + "5": 175.85075, + "6": 207.5196875, + "7": 229.7950859375, + "8": 245.6833828125, + "9": 258.6883828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 195558, + "answer_accuracy": 0.5159732776089159, + "avg_response_len": 190.8523012706856, + "resp_tokens_sum": 72334549, + "pass_at_k": { + "pass@1": 0.5159732776089159, + "pass@2": 0.608439611604401, + "pass@4": 0.6962606503594382, + "pass@8": 0.771608355479979, + "pass@16": 0.8315340689454365, + "pass@32": 0.8785152501474103, + "pass@64": 0.9155450230501024, + "pass@128": 0.9425869638635596 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2340523598820059, + "pass@2": 0.3490363621582701, + "pass@4": 0.47703086422825464, + "pass@8": 0.601095237968881, + "pass@16": 0.7081131385563241, + "pass@32": 0.7955993715227827, + "pass@64": 0.8638342379925815, + "pass@128": 0.911504424778761 + }, + "2": { + "pass@1": 0.9397390965732088, + "pass@2": 0.9527060796722839, + "pass@4": 0.9634816159078247, + "pass@8": 0.97184488771153, + "pass@16": 0.9779010470744166, + "pass@32": 0.9834545773716056, + "pass@64": 0.9890862112444283, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9171683868501529, + "pass@2": 0.9446715698668398, + "pass@4": 0.9611175976856101, + "pass@8": 0.9714190867433976, + "pass@16": 0.9792196020103332, + "pass@32": 0.9848030044380172, + "pass@64": 0.9877250665899934, + "pass@128": 0.9908256880733946 + }, + "4": { + "pass@1": 0.777978515625, + "pass@2": 0.8460526113435035, + "pass@4": 0.8948536686820393, + "pass@8": 0.9276525455102214, + "pass@16": 0.9486645850423827, + "pass@32": 0.9630959185836433, + "pass@64": 0.9733635031273581, + "pass@128": 0.978125 + }, + "5": { + "pass@1": 0.42544642857142856, + "pass@2": 0.5282945100612425, + "pass@4": 0.6184357633867192, + "pass@8": 0.6987714941711862, + "pass@16": 0.7683401121051054, + "pass@32": 0.8247387249798265, + "pass@64": 0.8683272151365441, + "pass@128": 0.8984126984126984 + }, + "6": { + "pass@1": 0.4719311377245509, + "pass@2": 0.6100170475977178, + "pass@4": 0.7327381440718109, + "pass@8": 0.8212352253202607, + "pass@16": 0.8766424175779357, + "pass@32": 0.9137167660624057, + "pass@64": 0.9411640584150373, + "pass@128": 0.9610778443113772 + }, + "7": { + "pass@1": 0.36415957840236685, + "pass@2": 0.48880959383590367, + "pass@4": 0.6072677575805979, + "pass@8": 0.7089181989531216, + "pass@16": 0.7888269810111848, + "pass@32": 0.8487011389145268, + "pass@64": 0.8983636353290456, + "pass@128": 0.9437869822485208 + }, + "8": { + "pass@1": 0.3150040064102564, + "pass@2": 0.43504725671310324, + "pass@4": 0.5564820518228488, + "pass@8": 0.6692766640774171, + "pass@16": 0.7639736770960015, + "pass@32": 0.8368242107183997, + "pass@64": 0.8889416956818764, + "pass@128": 0.9230769230769231 + }, + "9": { + "pass@1": 0.23921654929577466, + "pass@2": 0.3568800598868798, + "pass@4": 0.4833165853388044, + "pass@8": 0.5962302356670168, + "pass@16": 0.6890966235919163, + "pass@32": 0.7682846086626541, + "pass@64": 0.8373558560961706, + "pass@128": 0.8873239436619719 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 192431, + "answer_accuracy": 0.49258426851245085, + "avg_response_len": 201.7485639539646, + "resp_tokens_sum": 78814287, + "pass_at_k": { + "pass@1": 0.49258426851245085, + "pass@2": 0.5909627641871619, + "pass@4": 0.6861983368457577, + "pass@8": 0.7678727497235465, + "pass@16": 0.8307868706503942, + "pass@32": 0.8773797914386009, + "pass@64": 0.9123962489141342, + "pass@128": 0.9390563564875491 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.22354294478527606, + "pass@2": 0.3360058088981208, + "pass@4": 0.46179167040469643, + "pass@8": 0.5850830032020309, + "pass@16": 0.6922560235978716, + "pass@32": 0.779310015924251, + "pass@64": 0.8474751226546069, + "pass@128": 0.8987730061349694 + }, + "2": { + "pass@1": 0.8912186550151976, + "pass@2": 0.9116783440633751, + "pass@4": 0.9292722276546739, + "pass@8": 0.9451709089562008, + "pass@16": 0.9578299275067208, + "pass@32": 0.967487067518195, + "pass@64": 0.9747033816302779, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.848626893939394, + "pass@2": 0.8895933995466476, + "pass@4": 0.9138172435831877, + "pass@8": 0.9303741145081666, + "pass@16": 0.9424221740566364, + "pass@32": 0.9518335581963728, + "pass@64": 0.9600967272076638, + "pass@128": 0.9666666666666667 + }, + "4": { + "pass@1": 0.725938424556213, + "pass@2": 0.7984489295531847, + "pass@4": 0.8548653993139906, + "pass@8": 0.8967305917882408, + "pass@16": 0.9268802741732332, + "pass@32": 0.945689901633452, + "pass@64": 0.95690937985168, + "pass@128": 0.9674556213017751 + }, + "5": { + "pass@1": 0.4459239130434783, + "pass@2": 0.5601492068926165, + "pass@4": 0.6647551846780018, + "pass@8": 0.7498131323981773, + "pass@16": 0.8119176290092686, + "pass@32": 0.8557433256995092, + "pass@64": 0.8852069468068376, + "pass@128": 0.9072463768115943 + }, + "6": { + "pass@1": 0.42238986389684813, + "pass@2": 0.560813164158563, + "pass@4": 0.6923920269278656, + "pass@8": 0.796637969019255, + "pass@16": 0.8679993396877942, + "pass@32": 0.9132326395237702, + "pass@64": 0.9407776590161546, + "pass@128": 0.9598853868194842 + }, + "7": { + "pass@1": 0.3537978684971098, + "pass@2": 0.48195561763233363, + "pass@4": 0.6084160362360775, + "pass@8": 0.717139403598144, + "pass@16": 0.8003843219004848, + "pass@32": 0.8634777033100696, + "pass@64": 0.915658946495517, + "pass@128": 0.9566473988439307 + }, + "8": { + "pass@1": 0.28818221830985913, + "pass@2": 0.41921301430631, + "pass@4": 0.5555384356004792, + "pass@8": 0.6756259834552851, + "pass@16": 0.7670445900118551, + "pass@32": 0.8316106463558992, + "pass@64": 0.8783679240426463, + "pass@128": 0.9154929577464789 + }, + "9": { + "pass@1": 0.25715755988023953, + "pass@2": 0.37767722429157424, + "pass@4": 0.5053444070801026, + "pass@8": 0.6198841515478999, + "pass@16": 0.7131677032398571, + "pass@32": 0.7895863734944806, + "pass@64": 0.8534291934377334, + "pass@128": 0.9011976047904192 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 198649, + "answer_accuracy": 0.5195665592567794, + "avg_response_len": 213.64499288583863, + "resp_tokens_sum": 81684172, + "pass_at_k": { + "pass@1": 0.5195665592567794, + "pass@2": 0.6090486184885173, + "pass@4": 0.6939275271269908, + "pass@8": 0.7676403816133269, + "pass@16": 0.8281584309075226, + "pass@32": 0.8772024940828665, + "pass@64": 0.9157733504155053, + "pass@128": 0.9444258453297623 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.25944496268656714, + "pass@2": 0.37380861440827357, + "pass@4": 0.4947208124171043, + "pass@8": 0.6084540057533921, + "pass@16": 0.7069548544384052, + "pass@32": 0.7889803052552564, + "pass@64": 0.8535451186415071, + "pass@128": 0.9014925373134328 + }, + "2": { + "pass@1": 0.9660044642857143, + "pass@2": 0.9723727502812146, + "pass@4": 0.9782737465852483, + "pass@8": 0.9840867443239417, + "pass@16": 0.9897300202044366, + "pass@32": 0.9943372627596654, + "pass@64": 0.9967877861964524, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9045189504373178, + "pass@2": 0.9409872075939486, + "pass@4": 0.963359374538824, + "pass@8": 0.9778173387389365, + "pass@16": 0.9862698501499768, + "pass@32": 0.9905573089869245, + "pass@64": 0.9926020223300518, + "pass@128": 0.9941690962099126 + }, + "4": { + "pass@1": 0.7404513888888888, + "pass@2": 0.8004220483952662, + "pass@4": 0.8438767870573194, + "pass@8": 0.8764562526018601, + "pass@16": 0.9012650591890069, + "pass@32": 0.9202351939936253, + "pass@64": 0.9359478097107635, + "pass@128": 0.9502923976608187 + }, + "5": { + "pass@1": 0.44046415441176473, + "pass@2": 0.539720356646595, + "pass@4": 0.632906808156333, + "pass@8": 0.7201508139502907, + "pass@16": 0.7958080919156965, + "pass@32": 0.8564772070244233, + "pass@64": 0.9020604729390345, + "pass@128": 0.9323529411764706 + }, + "6": { + "pass@1": 0.4135695977917981, + "pass@2": 0.5535905015027699, + "pass@4": 0.6846301371240265, + "pass@8": 0.7918933708969056, + "pass@16": 0.870299357531166, + "pass@32": 0.921637925223863, + "pass@64": 0.953726180491391, + "pass@128": 0.9747634069400631 + }, + "7": { + "pass@1": 0.3732693829113924, + "pass@2": 0.5000556756453705, + "pass@4": 0.6184612249813708, + "pass@8": 0.71713744327495, + "pass@16": 0.7982010440452766, + "pass@32": 0.8671220101820218, + "pass@64": 0.9208143511726397, + "pass@128": 0.9525316455696202 + }, + "8": { + "pass@1": 0.29450544294294295, + "pass@2": 0.4251602734624386, + "pass@4": 0.5522831794111326, + "pass@8": 0.6573037179247492, + "pass@16": 0.739586440476168, + "pass@32": 0.8057187779468007, + "pass@64": 0.859777333695693, + "pass@128": 0.9069069069069069 + }, + "9": { + "pass@1": 0.21402733118971062, + "pass@2": 0.3169770014178291, + "pass@4": 0.43098392071971686, + "pass@8": 0.540710406429895, + "pass@16": 0.6397980025166145, + "pass@32": 0.732216820941264, + "pass@64": 0.8158523025431184, + "pass@128": 0.8842443729903537 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.1478125, + "avg_response_len": 301.16630703125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.1478125, + "pass@2": 0.23208056102362476, + "pass@4": 0.33033442000375174, + "pass@8": 0.42624948311094624, + "pass@16": 0.5056970481626066, + "pass@32": 0.5657901758875066, + "pass@64": 0.6132016856723344, + "pass@128": 0.6555 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.209125, + "pass@2": 0.31599544783464584, + "pass@4": 0.43296036661042375, + "pass@8": 0.5408760189531607, + "pass@16": 0.6317585985341309, + "pass@32": 0.709115666263266, + "pass@64": 0.7757584862400542, + "pass@128": 0.833 + }, + "12": { + "pass@1": 0.1938203125, + "pass@2": 0.29579306102362213, + "pass@4": 0.4067105079677537, + "pass@8": 0.5103219976245189, + "pass@16": 0.6014493312628362, + "pass@32": 0.6817909519123444, + "pass@64": 0.7529548902131044, + "pass@128": 0.814 + }, + "13": { + "pass@1": 0.1741484375, + "pass@2": 0.266900098425197, + "pass@4": 0.3699952282527184, + "pass@8": 0.4666827988048414, + "pass@16": 0.5471934854287409, + "pass@32": 0.6156571698381453, + "pass@64": 0.6795958515434805, + "pass@128": 0.742 + }, + "14": { + "pass@1": 0.1625, + "pass@2": 0.25185617618110245, + "pass@4": 0.351397114173228, + "pass@8": 0.4426129085951435, + "pass@16": 0.5147836072916462, + "pass@32": 0.5720964848126953, + "pass@64": 0.6258006465476725, + "pass@128": 0.683 + }, + "15": { + "pass@1": 0.1331484375, + "pass@2": 0.21419598917322838, + "pass@4": 0.3120670732095988, + "pass@8": 0.4095027876840067, + "pass@16": 0.4914640443783659, + "pass@32": 0.5562901367992357, + "pass@64": 0.6112821863894128, + "pass@128": 0.663 + }, + "16": { + "pass@1": 0.1392421875, + "pass@2": 0.2179140009842518, + "pass@4": 0.3086317001312334, + "pass@8": 0.39754894747440916, + "pass@16": 0.47080147456801014, + "pass@32": 0.5244014677734099, + "pass@64": 0.566191618835557, + "pass@128": 0.605 + }, + "17": { + "pass@1": 0.120296875, + "pass@2": 0.1974090797244096, + "pass@4": 0.29445156242969633, + "pass@8": 0.39433446661852467, + "pass@16": 0.47551720066461045, + "pass@32": 0.529413186972841, + "pass@64": 0.5662209874364034, + "pass@128": 0.6 + }, + "18": { + "pass@1": 0.1258828125, + "pass@2": 0.20046678149606292, + "pass@4": 0.28968374540682396, + "pass@8": 0.37826731331014896, + "pass@16": 0.4498736332368763, + "pass@32": 0.4986247511474188, + "pass@64": 0.5301998629824372, + "pass@128": 0.557 + }, + "19": { + "pass@1": 0.1133125, + "pass@2": 0.18398240649606296, + "pass@4": 0.2718294195725533, + "pass@8": 0.36243743513663734, + "pass@16": 0.4369511701806545, + "pass@32": 0.4842971990798993, + "pass@64": 0.5100450168165499, + "pass@128": 0.527 + }, + "20": { + "pass@1": 0.1066484375, + "pass@2": 0.17629256889763756, + "pass@4": 0.26561748228346427, + "pass@8": 0.3599101569079771, + "pass@16": 0.4371779360800682, + "pass@32": 0.486214744275793, + "pass@64": 0.5139673097186235, + "pass@128": 0.531 + } + }, + "per_op_accuracy": { + "11": 0.209125, + "12": 0.1938203125, + "13": 0.1741484375, + "14": 0.1625, + "15": 0.1331484375, + "16": 0.1392421875, + "17": 0.120296875, + "18": 0.1258828125, + "19": 0.1133125, + "20": 0.1066484375 + }, + "per_op_avg_response_len": { + "11": 283.8235859375, + "12": 287.7186640625, + "13": 297.058125, + "14": 303.3533515625, + "15": 300.442515625, + "16": 306.6927109375, + "17": 309.658453125, + "18": 306.358734375, + "19": 307.3152265625, + "20": 309.241703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 283.8235859375, + "12": 287.7186640625, + "13": 297.058125, + "14": 303.3533515625, + "15": 300.442515625, + "16": 306.6927109375, + "17": 309.658453125, + "18": 306.358734375, + "19": 307.3152265625, + "20": 309.241703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 64778, + "answer_accuracy": 0.15039468796433877, + "avg_response_len": 287.0356078194651, + "resp_tokens_sum": 123631977, + "pass_at_k": { + "pass@1": 0.15039468796433877, + "pass@2": 0.23519856150039184, + "pass@4": 0.3328943195714957, + "pass@8": 0.4274612364455714, + "pass@16": 0.5060689793996103, + "pass@32": 0.5659251684578299, + "pass@64": 0.611620349397551, + "pass@128": 0.649925705794948 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.21746635977337112, + "pass@2": 0.3272846913965782, + "pass@4": 0.44665189983334197, + "pass@8": 0.5553283625674275, + "pass@16": 0.6448423394148377, + "pass@32": 0.7179749189998534, + "pass@64": 0.777423089022195, + "pass@128": 0.8243626062322946 + }, + "12": { + "pass@1": 0.18498411016949154, + "pass@2": 0.2839403582232306, + "pass@4": 0.3926915541807275, + "pass@8": 0.494440493747792, + "pass@16": 0.5843288803240919, + "pass@32": 0.6652268589464182, + "pass@64": 0.735752190146036, + "pass@128": 0.7937853107344632 + }, + "13": { + "pass@1": 0.18647139498432602, + "pass@2": 0.2839080974008344, + "pass@4": 0.38877480009356197, + "pass@8": 0.4846227863805249, + "pass@16": 0.5638918640501261, + "pass@32": 0.6319251904268449, + "pass@64": 0.6916457926217192, + "pass@128": 0.7429467084639498 + }, + "14": { + "pass@1": 0.16109326323987538, + "pass@2": 0.25444944746976716, + "pass@4": 0.3592573200546192, + "pass@8": 0.4555773271112732, + "pass@16": 0.5327569175982596, + "pass@32": 0.5947606734348753, + "pass@64": 0.6511536361169645, + "pass@128": 0.7102803738317757 + }, + "15": { + "pass@1": 0.14234104046242774, + "pass@2": 0.2236904612898822, + "pass@4": 0.31877335600390994, + "pass@8": 0.4121823992074167, + "pass@16": 0.49071930914631096, + "pass@32": 0.5529727050149277, + "pass@64": 0.6061524184861755, + "pass@128": 0.6560693641618497 + }, + "16": { + "pass@1": 0.14392641129032258, + "pass@2": 0.22453287719075446, + "pass@4": 0.3174367695973486, + "pass@8": 0.4091923824210985, + "pass@16": 0.48592865391022005, + "pass@32": 0.5402647597178885, + "pass@64": 0.5777915977164921, + "pass@128": 0.6096774193548387 + }, + "17": { + "pass@1": 0.11477149566473989, + "pass@2": 0.18487011287606392, + "pass@4": 0.27174867872368563, + "pass@8": 0.3607717520233434, + "pass@16": 0.43365161352207626, + "pass@32": 0.482409295928184, + "pass@64": 0.5150428298119281, + "pass@128": 0.546242774566474 + }, + "18": { + "pass@1": 0.12763392857142858, + "pass@2": 0.2035601799775029, + "pass@4": 0.2933536606138519, + "pass@8": 0.3812815069370258, + "pass@16": 0.45235960903548417, + "pass@32": 0.501900940088412, + "pass@64": 0.5345649813685637, + "pass@128": 0.56 + }, + "19": { + "pass@1": 0.11732826576576577, + "pass@2": 0.18959707739235296, + "pass@4": 0.2780769194391241, + "pass@8": 0.3682895575262585, + "pass@16": 0.44274252772123246, + "pass@32": 0.4905652476370396, + "pass@64": 0.5157753123580949, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.10604354354354355, + "pass@2": 0.17358192345889195, + "pass@4": 0.25876359036201546, + "pass@8": 0.3487342283712976, + "pass@16": 0.4243679405449727, + "pass@32": 0.47493279620096895, + "pass@64": 0.5034854391857208, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 63975, + "answer_accuracy": 0.14861869982158787, + "avg_response_len": 298.66589308281294, + "resp_tokens_sum": 128564915, + "pass_at_k": { + "pass@1": 0.14861869982158787, + "pass@2": 0.23458868628263632, + "pass@4": 0.3349682185690229, + "pass@8": 0.4325489498936321, + "pass@16": 0.5127752176553522, + "pass@32": 0.5725923594707951, + "pass@64": 0.6203321387557768, + "pass@128": 0.664287838239667 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1899472891566265, + "pass@2": 0.29249144708756303, + "pass@4": 0.4056045608192049, + "pass@8": 0.5118326321094175, + "pass@16": 0.6064758426416722, + "pass@32": 0.691489508808827, + "pass@64": 0.767702363812573, + "pass@128": 0.8373493975903614 + }, + "12": { + "pass@1": 0.21497844827586207, + "pass@2": 0.3253735683607731, + "pass@4": 0.44188718318517406, + "pass@8": 0.547290961654598, + "pass@16": 0.6366487702338203, + "pass@32": 0.7110514969364836, + "pass@64": 0.7759151125242277, + "pass@128": 0.8307210031347962 + }, + "13": { + "pass@1": 0.16281512605042017, + "pass@2": 0.25351139471536654, + "pass@4": 0.3567216876251812, + "pass@8": 0.45448099582903717, + "pass@16": 0.5343941235795181, + "pass@32": 0.5985284837482586, + "pass@64": 0.6597611542390495, + "pass@128": 0.7282913165266106 + }, + "14": { + "pass@1": 0.16132127192982457, + "pass@2": 0.24900207786526687, + "pass@4": 0.34654788052809166, + "pass@8": 0.43636616447770876, + "pass@16": 0.5097873294578236, + "pass@32": 0.5690943424021969, + "pass@64": 0.6227856094077184, + "pass@128": 0.6754385964912281 + }, + "15": { + "pass@1": 0.13856534090909092, + "pass@2": 0.22398182116439982, + "pass@4": 0.32483539131472194, + "pass@8": 0.4210097549573201, + "pass@16": 0.49911894400022283, + "pass@32": 0.5627604370589941, + "pass@64": 0.6207404034236885, + "pass@128": 0.6727272727272727 + }, + "16": { + "pass@1": 0.13837695868945868, + "pass@2": 0.21823368833703485, + "pass@4": 0.3100300085352579, + "pass@8": 0.39765295225678154, + "pass@16": 0.46724374386741413, + "pass@32": 0.5191649589933541, + "pass@64": 0.5639517418293533, + "pass@128": 0.6068376068376068 + }, + "17": { + "pass@1": 0.1343652950310559, + "pass@2": 0.2217845221792929, + "pass@4": 0.3313873420131489, + "pass@8": 0.44396581707866756, + "pass@16": 0.5348156239501506, + "pass@32": 0.5924449604975142, + "pass@64": 0.6296611056432868, + "pass@128": 0.6645962732919255 + }, + "18": { + "pass@1": 0.13033088235294119, + "pass@2": 0.2081334674617878, + "pass@4": 0.302660001599065, + "pass@8": 0.39741988375269416, + "pass@16": 0.4747293286693812, + "pass@32": 0.5260981930569909, + "pass@64": 0.5574985566060943, + "pass@128": 0.5852941176470589 + }, + "19": { + "pass@1": 0.11130487351190477, + "pass@2": 0.1813290184430071, + "pass@4": 0.2708650369596659, + "pass@8": 0.3643281841752635, + "pass@16": 0.4387280487490098, + "pass@32": 0.4830895220764344, + "pass@64": 0.506235461720585, + "pass@128": 0.5208333333333334 + }, + "20": { + "pass@1": 0.10656811377245509, + "pass@2": 0.17586099945777736, + "pass@4": 0.2645760077146045, + "pass@8": 0.3582559738491696, + "pass@16": 0.4345901738446862, + "pass@32": 0.48211568706570346, + "pass@64": 0.5091873400019051, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 60447, + "answer_accuracy": 0.14432829691320292, + "avg_response_len": 318.2685976658007, + "resp_tokens_sum": 133295981, + "pass_at_k": { + "pass@1": 0.14432829691320292, + "pass@2": 0.22629605709624068, + "pass@4": 0.3229390879070295, + "pass@8": 0.41852862224257886, + "pass@16": 0.4980395198567529, + "pass@32": 0.5586599823087265, + "pass@64": 0.607499204879244, + "pass@128": 0.652200488997555 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.21999007936507936, + "pass@2": 0.32811679790026244, + "pass@4": 0.4464493072294533, + "pass@8": 0.5552910257350235, + "pass@16": 0.6437437236941529, + "pass@32": 0.7177650886723408, + "pass@64": 0.7823839715220456, + "pass@128": 0.8380952380952381 + }, + "12": { + "pass@1": 0.18274560397553516, + "pass@2": 0.2797676021575284, + "pass@4": 0.38757090627433033, + "pass@8": 0.4914502937919992, + "pass@16": 0.5856451067997539, + "pass@32": 0.6711780315675037, + "pass@64": 0.7491794923736346, + "pass@128": 0.8195718654434251 + }, + "13": { + "pass@1": 0.17450327932098766, + "pass@2": 0.26490693654612624, + "pass@4": 0.3661309399403777, + "pass@8": 0.4624642420940969, + "pass@16": 0.5448557983915544, + "pass@32": 0.6185134734378167, + "pass@64": 0.6895867643018863, + "pass@128": 0.7561728395061729 + }, + "14": { + "pass@1": 0.1650361646884273, + "pass@2": 0.2522824715530736, + "pass@4": 0.3488312738726651, + "pass@8": 0.43660343721379263, + "pass@16": 0.5027340358101761, + "pass@32": 0.5535550253369403, + "pass@64": 0.6047111303759262, + "pass@128": 0.6646884272997032 + }, + "15": { + "pass@1": 0.11781442901234568, + "pass@2": 0.19408977957616405, + "pass@4": 0.2919006570937892, + "pass@8": 0.3949211679701401, + "pass@16": 0.48446269102984507, + "pass@32": 0.5532427056623543, + "pass@64": 0.6071269026647454, + "pass@128": 0.6604938271604939 + }, + "16": { + "pass@1": 0.1358545353982301, + "pass@2": 0.21153033760713535, + "pass@4": 0.2991320606494985, + "pass@8": 0.3867938722175179, + "pass@16": 0.46065202878578043, + "pass@32": 0.5153170845256568, + "pass@64": 0.5579031332192951, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.11241057981927711, + "pass@2": 0.18683555580590078, + "pass@4": 0.2822885405514067, + "pass@8": 0.3811760461419187, + "pass@16": 0.4616358779338103, + "pass@32": 0.5172659436791867, + "pass@64": 0.5580279888684868, + "pass@128": 0.5933734939759037 + }, + "18": { + "pass@1": 0.11902721774193549, + "pass@2": 0.18856561150622297, + "pass@4": 0.2713082698332063, + "pass@8": 0.3538581464718504, + "pass@16": 0.41980580105441206, + "pass@32": 0.4647936660551533, + "pass@64": 0.49533100083021947, + "pass@128": 0.5225806451612903 + }, + "19": { + "pass@1": 0.11131042296072508, + "pass@2": 0.18102727953945336, + "pass@4": 0.26652312078814944, + "pass@8": 0.3546306422885341, + "pass@16": 0.42932109990881356, + "pass@32": 0.4792171969760829, + "pass@64": 0.5081471983782093, + "pass@128": 0.5287009063444109 + }, + "20": { + "pass@1": 0.1073338963963964, + "pass@2": 0.17943607978056803, + "pass@4": 0.2735159763250315, + "pass@8": 0.3727452360201581, + "pass@16": 0.45258346490831136, + "pass@32": 0.5016080590418188, + "pass@64": 0.5292435042316577, + "pass@128": 0.5465465465465466 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.3190123355263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.3190123355263158, + "pass@2": 0.4076443094695297, + "pass@4": 0.501684780701738, + "pass@8": 0.5886166644063656, + "pass@16": 0.6593901906378742, + "pass@32": 0.7135343362030585, + "pass@64": 0.7559470536501054, + "pass@128": 0.7912105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2391328125, + "pass@2": 0.35308710629921264, + "pass@4": 0.4779890196850392, + "pass@8": 0.5983404366426991, + "pass@16": 0.7025556939003653, + "pass@32": 0.7880716543980395, + "pass@64": 0.8550543114097935, + "pass@128": 0.904 + }, + "2": { + "pass@1": 0.93296875, + "pass@2": 0.946091289370079, + "pass@4": 0.9574039729096363, + "pass@8": 0.9673537985153713, + "pass@16": 0.9754377893321514, + "pass@32": 0.9820102065156547, + "pass@64": 0.9870498115345813, + "pass@128": 0.99 + }, + "3": { + "pass@1": 0.8902109375, + "pass@2": 0.9252320374015754, + "pass@4": 0.9462774102924636, + "pass@8": 0.9600688463402408, + "pass@16": 0.9694946858975106, + "pass@32": 0.9758968136385496, + "pass@64": 0.9802805104126652, + "pass@128": 0.984 + }, + "4": { + "pass@1": 0.7475546875, + "pass@2": 0.8143569143700784, + "pass@4": 0.8639035401199855, + "pass@8": 0.8996917929775318, + "pass@16": 0.9250908501267543, + "pass@32": 0.9425543170446925, + "pass@64": 0.9550058423117043, + "pass@128": 0.965 + }, + "5": { + "pass@1": 0.4376171875, + "pass@2": 0.5431691683070871, + "pass@4": 0.6393361189538813, + "pass@8": 0.7236498280843922, + "pass@16": 0.7927134685726412, + "pass@32": 0.8462263961232798, + "pass@64": 0.8856200302156424, + "pass@128": 0.913 + }, + "6": { + "pass@1": 0.436140625, + "pass@2": 0.5749576771653546, + "pass@4": 0.7034071109861268, + "pass@8": 0.8033494150190057, + "pass@16": 0.8716152333594489, + "pass@32": 0.916058813354604, + "pass@64": 0.9450113977230314, + "pass@128": 0.965 + }, + "7": { + "pass@1": 0.363453125, + "pass@2": 0.48999187992125987, + "pass@4": 0.6112021976940375, + "pass@8": 0.7143600169659958, + "pass@16": 0.7957880248776545, + "pass@32": 0.8596348255159127, + "pass@64": 0.9114422391992211, + "pass@128": 0.951 + }, + "8": { + "pass@1": 0.29865625, + "pass@2": 0.4261337352362206, + "pass@4": 0.5547488435508061, + "pass@8": 0.6675436813877224, + "pass@16": 0.7569429013867229, + "pass@32": 0.824615286256769, + "pass@64": 0.8754762742085503, + "pass@128": 0.915 + }, + "9": { + "pass@1": 0.237375, + "pass@2": 0.35141646161417367, + "pass@4": 0.47439841910386227, + "pass@8": 0.5868639766784868, + "pass@16": 0.6818044930399056, + "pass@32": 0.7641823161351303, + "pass@64": 0.8360367456132549, + "pass@128": 0.891 + }, + "11": { + "pass@1": 0.209125, + "pass@2": 0.31599544783464584, + "pass@4": 0.43296036661042375, + "pass@8": 0.5408760189531607, + "pass@16": 0.6317585985341309, + "pass@32": 0.709115666263266, + "pass@64": 0.7757584862400542, + "pass@128": 0.833 + }, + "12": { + "pass@1": 0.1938203125, + "pass@2": 0.29579306102362213, + "pass@4": 0.4067105079677537, + "pass@8": 0.5103219976245189, + "pass@16": 0.6014493312628362, + "pass@32": 0.6817909519123444, + "pass@64": 0.7529548902131044, + "pass@128": 0.814 + }, + "13": { + "pass@1": 0.1741484375, + "pass@2": 0.266900098425197, + "pass@4": 0.3699952282527184, + "pass@8": 0.4666827988048414, + "pass@16": 0.5471934854287409, + "pass@32": 0.6156571698381453, + "pass@64": 0.6795958515434805, + "pass@128": 0.742 + }, + "14": { + "pass@1": 0.1625, + "pass@2": 0.25185617618110245, + "pass@4": 0.351397114173228, + "pass@8": 0.4426129085951435, + "pass@16": 0.5147836072916462, + "pass@32": 0.5720964848126953, + "pass@64": 0.6258006465476725, + "pass@128": 0.683 + }, + "15": { + "pass@1": 0.1331484375, + "pass@2": 0.21419598917322838, + "pass@4": 0.3120670732095988, + "pass@8": 0.4095027876840067, + "pass@16": 0.4914640443783659, + "pass@32": 0.5562901367992357, + "pass@64": 0.6112821863894128, + "pass@128": 0.663 + }, + "16": { + "pass@1": 0.1392421875, + "pass@2": 0.2179140009842518, + "pass@4": 0.3086317001312334, + "pass@8": 0.39754894747440916, + "pass@16": 0.47080147456801014, + "pass@32": 0.5244014677734099, + "pass@64": 0.566191618835557, + "pass@128": 0.605 + }, + "17": { + "pass@1": 0.120296875, + "pass@2": 0.1974090797244096, + "pass@4": 0.29445156242969633, + "pass@8": 0.39433446661852467, + "pass@16": 0.47551720066461045, + "pass@32": 0.529413186972841, + "pass@64": 0.5662209874364034, + "pass@128": 0.6 + }, + "18": { + "pass@1": 0.1258828125, + "pass@2": 0.20046678149606292, + "pass@4": 0.28968374540682396, + "pass@8": 0.37826731331014896, + "pass@16": 0.4498736332368763, + "pass@32": 0.4986247511474188, + "pass@64": 0.5301998629824372, + "pass@128": 0.557 + }, + "19": { + "pass@1": 0.1133125, + "pass@2": 0.18398240649606296, + "pass@4": 0.2718294195725533, + "pass@8": 0.36243743513663734, + "pass@16": 0.4369511701806545, + "pass@32": 0.4842971990798993, + "pass@64": 0.5100450168165499, + "pass@128": 0.527 + }, + "20": { + "pass@1": 0.1066484375, + "pass@2": 0.17629256889763756, + "pass@4": 0.26561748228346427, + "pass@8": 0.3599101569079771, + "pass@16": 0.4371779360800682, + "pass@32": 0.486214744275793, + "pass@64": 0.5139673097186235, + "pass@128": 0.531 + } + }, + "per_op_accuracy": { + "10": 0.2391328125, + "2": 0.93296875, + "3": 0.8902109375, + "4": 0.7475546875, + "5": 0.4376171875, + "6": 0.436140625, + "7": 0.363453125, + "8": 0.29865625, + "9": 0.237375, + "11": 0.209125, + "12": 0.1938203125, + "13": 0.1741484375, + "14": 0.1625, + "15": 0.1331484375, + "16": 0.1392421875, + "17": 0.120296875, + "18": 0.1258828125, + "19": 0.1133125, + "20": 0.1066484375 + }, + "per_op_avg_response_len": { + "10": 273.950546875, + "2": 116.4940703125, + "3": 148.770578125, + "4": 162.255390625, + "5": 175.85075, + "6": 207.5196875, + "7": 229.7950859375, + "8": 245.6833828125, + "9": 258.6883828125, + "11": 283.8235859375, + "12": 287.7186640625, + "13": 297.058125, + "14": 303.3533515625, + "15": 300.442515625, + "16": 306.6927109375, + "17": 309.658453125, + "18": 306.358734375, + "19": 307.3152265625, + "20": 309.241703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.950546875, + "2": 116.4940703125, + "3": 148.770578125, + "4": 162.255390625, + "5": 175.85075, + "6": 207.5196875, + "7": 229.7950859375, + "8": 245.6833828125, + "9": 258.6883828125, + "11": 283.8235859375, + "12": 287.7186640625, + "13": 297.058125, + "14": 303.3533515625, + "15": 300.442515625, + "16": 306.6927109375, + "17": 309.658453125, + "18": 306.358734375, + "19": 307.3152265625, + "20": 309.241703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 260336, + "answer_accuracy": 0.32151043313310146, + "avg_response_len": 242.01525203525134, + "resp_tokens_sum": 195966526, + "pass_at_k": { + "pass@1": 0.32151043313310146, + "pass@2": 0.4099008614305211, + "pass@4": 0.5029745765210857, + "pass@8": 0.5885455898222649, + "pass@16": 0.6584088671873528, + "pass@32": 0.71223867333972, + "pass@64": 0.7538778515608776, + "pass@128": 0.786911160290863 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2340523598820059, + "pass@2": 0.3490363621582701, + "pass@4": 0.47703086422825464, + "pass@8": 0.601095237968881, + "pass@16": 0.7081131385563241, + "pass@32": 0.7955993715227827, + "pass@64": 0.8638342379925815, + "pass@128": 0.911504424778761 + }, + "2": { + "pass@1": 0.9397390965732088, + "pass@2": 0.9527060796722839, + "pass@4": 0.9634816159078247, + "pass@8": 0.97184488771153, + "pass@16": 0.9779010470744166, + "pass@32": 0.9834545773716056, + "pass@64": 0.9890862112444283, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9171683868501529, + "pass@2": 0.9446715698668398, + "pass@4": 0.9611175976856101, + "pass@8": 0.9714190867433976, + "pass@16": 0.9792196020103332, + "pass@32": 0.9848030044380172, + "pass@64": 0.9877250665899934, + "pass@128": 0.9908256880733946 + }, + "4": { + "pass@1": 0.777978515625, + "pass@2": 0.8460526113435035, + "pass@4": 0.8948536686820393, + "pass@8": 0.9276525455102214, + "pass@16": 0.9486645850423827, + "pass@32": 0.9630959185836433, + "pass@64": 0.9733635031273581, + "pass@128": 0.978125 + }, + "5": { + "pass@1": 0.42544642857142856, + "pass@2": 0.5282945100612425, + "pass@4": 0.6184357633867192, + "pass@8": 0.6987714941711862, + "pass@16": 0.7683401121051054, + "pass@32": 0.8247387249798265, + "pass@64": 0.8683272151365441, + "pass@128": 0.8984126984126984 + }, + "6": { + "pass@1": 0.4719311377245509, + "pass@2": 0.6100170475977178, + "pass@4": 0.7327381440718109, + "pass@8": 0.8212352253202607, + "pass@16": 0.8766424175779357, + "pass@32": 0.9137167660624057, + "pass@64": 0.9411640584150373, + "pass@128": 0.9610778443113772 + }, + "7": { + "pass@1": 0.36415957840236685, + "pass@2": 0.48880959383590367, + "pass@4": 0.6072677575805979, + "pass@8": 0.7089181989531216, + "pass@16": 0.7888269810111848, + "pass@32": 0.8487011389145268, + "pass@64": 0.8983636353290456, + "pass@128": 0.9437869822485208 + }, + "8": { + "pass@1": 0.3150040064102564, + "pass@2": 0.43504725671310324, + "pass@4": 0.5564820518228488, + "pass@8": 0.6692766640774171, + "pass@16": 0.7639736770960015, + "pass@32": 0.8368242107183997, + "pass@64": 0.8889416956818764, + "pass@128": 0.9230769230769231 + }, + "9": { + "pass@1": 0.23921654929577466, + "pass@2": 0.3568800598868798, + "pass@4": 0.4833165853388044, + "pass@8": 0.5962302356670168, + "pass@16": 0.6890966235919163, + "pass@32": 0.7682846086626541, + "pass@64": 0.8373558560961706, + "pass@128": 0.8873239436619719 + }, + "11": { + "pass@1": 0.21746635977337112, + "pass@2": 0.3272846913965782, + "pass@4": 0.44665189983334197, + "pass@8": 0.5553283625674275, + "pass@16": 0.6448423394148377, + "pass@32": 0.7179749189998534, + "pass@64": 0.777423089022195, + "pass@128": 0.8243626062322946 + }, + "12": { + "pass@1": 0.18498411016949154, + "pass@2": 0.2839403582232306, + "pass@4": 0.3926915541807275, + "pass@8": 0.494440493747792, + "pass@16": 0.5843288803240919, + "pass@32": 0.6652268589464182, + "pass@64": 0.735752190146036, + "pass@128": 0.7937853107344632 + }, + "13": { + "pass@1": 0.18647139498432602, + "pass@2": 0.2839080974008344, + "pass@4": 0.38877480009356197, + "pass@8": 0.4846227863805249, + "pass@16": 0.5638918640501261, + "pass@32": 0.6319251904268449, + "pass@64": 0.6916457926217192, + "pass@128": 0.7429467084639498 + }, + "14": { + "pass@1": 0.16109326323987538, + "pass@2": 0.25444944746976716, + "pass@4": 0.3592573200546192, + "pass@8": 0.4555773271112732, + "pass@16": 0.5327569175982596, + "pass@32": 0.5947606734348753, + "pass@64": 0.6511536361169645, + "pass@128": 0.7102803738317757 + }, + "15": { + "pass@1": 0.14234104046242774, + "pass@2": 0.2236904612898822, + "pass@4": 0.31877335600390994, + "pass@8": 0.4121823992074167, + "pass@16": 0.49071930914631096, + "pass@32": 0.5529727050149277, + "pass@64": 0.6061524184861755, + "pass@128": 0.6560693641618497 + }, + "16": { + "pass@1": 0.14392641129032258, + "pass@2": 0.22453287719075446, + "pass@4": 0.3174367695973486, + "pass@8": 0.4091923824210985, + "pass@16": 0.48592865391022005, + "pass@32": 0.5402647597178885, + "pass@64": 0.5777915977164921, + "pass@128": 0.6096774193548387 + }, + "17": { + "pass@1": 0.11477149566473989, + "pass@2": 0.18487011287606392, + "pass@4": 0.27174867872368563, + "pass@8": 0.3607717520233434, + "pass@16": 0.43365161352207626, + "pass@32": 0.482409295928184, + "pass@64": 0.5150428298119281, + "pass@128": 0.546242774566474 + }, + "18": { + "pass@1": 0.12763392857142858, + "pass@2": 0.2035601799775029, + "pass@4": 0.2933536606138519, + "pass@8": 0.3812815069370258, + "pass@16": 0.45235960903548417, + "pass@32": 0.501900940088412, + "pass@64": 0.5345649813685637, + "pass@128": 0.56 + }, + "19": { + "pass@1": 0.11732826576576577, + "pass@2": 0.18959707739235296, + "pass@4": 0.2780769194391241, + "pass@8": 0.3682895575262585, + "pass@16": 0.44274252772123246, + "pass@32": 0.4905652476370396, + "pass@64": 0.5157753123580949, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.10604354354354355, + "pass@2": 0.17358192345889195, + "pass@4": 0.25876359036201546, + "pass@8": 0.3487342283712976, + "pass@16": 0.4243679405449727, + "pass@32": 0.47493279620096895, + "pass@64": 0.5034854391857208, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 256406, + "answer_accuracy": 0.31226373733437257, + "avg_response_len": 252.55651061964147, + "resp_tokens_sum": 207379202, + "pass_at_k": { + "pass@1": 0.31226373733437257, + "pass@2": 0.4041371953651974, + "pass@4": 0.5020694377398108, + "pass@8": 0.5920825799919917, + "pass@16": 0.6640724218550299, + "pass@32": 0.7175980090991305, + "pass@64": 0.7592845415933916, + "pass@128": 0.7950116913484022 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.22354294478527606, + "pass@2": 0.3360058088981208, + "pass@4": 0.46179167040469643, + "pass@8": 0.5850830032020309, + "pass@16": 0.6922560235978716, + "pass@32": 0.779310015924251, + "pass@64": 0.8474751226546069, + "pass@128": 0.8987730061349694 + }, + "2": { + "pass@1": 0.8912186550151976, + "pass@2": 0.9116783440633751, + "pass@4": 0.9292722276546739, + "pass@8": 0.9451709089562008, + "pass@16": 0.9578299275067208, + "pass@32": 0.967487067518195, + "pass@64": 0.9747033816302779, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.848626893939394, + "pass@2": 0.8895933995466476, + "pass@4": 0.9138172435831877, + "pass@8": 0.9303741145081666, + "pass@16": 0.9424221740566364, + "pass@32": 0.9518335581963728, + "pass@64": 0.9600967272076638, + "pass@128": 0.9666666666666667 + }, + "4": { + "pass@1": 0.725938424556213, + "pass@2": 0.7984489295531847, + "pass@4": 0.8548653993139906, + "pass@8": 0.8967305917882408, + "pass@16": 0.9268802741732332, + "pass@32": 0.945689901633452, + "pass@64": 0.95690937985168, + "pass@128": 0.9674556213017751 + }, + "5": { + "pass@1": 0.4459239130434783, + "pass@2": 0.5601492068926165, + "pass@4": 0.6647551846780018, + "pass@8": 0.7498131323981773, + "pass@16": 0.8119176290092686, + "pass@32": 0.8557433256995092, + "pass@64": 0.8852069468068376, + "pass@128": 0.9072463768115943 + }, + "6": { + "pass@1": 0.42238986389684813, + "pass@2": 0.560813164158563, + "pass@4": 0.6923920269278656, + "pass@8": 0.796637969019255, + "pass@16": 0.8679993396877942, + "pass@32": 0.9132326395237702, + "pass@64": 0.9407776590161546, + "pass@128": 0.9598853868194842 + }, + "7": { + "pass@1": 0.3537978684971098, + "pass@2": 0.48195561763233363, + "pass@4": 0.6084160362360775, + "pass@8": 0.717139403598144, + "pass@16": 0.8003843219004848, + "pass@32": 0.8634777033100696, + "pass@64": 0.915658946495517, + "pass@128": 0.9566473988439307 + }, + "8": { + "pass@1": 0.28818221830985913, + "pass@2": 0.41921301430631, + "pass@4": 0.5555384356004792, + "pass@8": 0.6756259834552851, + "pass@16": 0.7670445900118551, + "pass@32": 0.8316106463558992, + "pass@64": 0.8783679240426463, + "pass@128": 0.9154929577464789 + }, + "9": { + "pass@1": 0.25715755988023953, + "pass@2": 0.37767722429157424, + "pass@4": 0.5053444070801026, + "pass@8": 0.6198841515478999, + "pass@16": 0.7131677032398571, + "pass@32": 0.7895863734944806, + "pass@64": 0.8534291934377334, + "pass@128": 0.9011976047904192 + }, + "11": { + "pass@1": 0.1899472891566265, + "pass@2": 0.29249144708756303, + "pass@4": 0.4056045608192049, + "pass@8": 0.5118326321094175, + "pass@16": 0.6064758426416722, + "pass@32": 0.691489508808827, + "pass@64": 0.767702363812573, + "pass@128": 0.8373493975903614 + }, + "12": { + "pass@1": 0.21497844827586207, + "pass@2": 0.3253735683607731, + "pass@4": 0.44188718318517406, + "pass@8": 0.547290961654598, + "pass@16": 0.6366487702338203, + "pass@32": 0.7110514969364836, + "pass@64": 0.7759151125242277, + "pass@128": 0.8307210031347962 + }, + "13": { + "pass@1": 0.16281512605042017, + "pass@2": 0.25351139471536654, + "pass@4": 0.3567216876251812, + "pass@8": 0.45448099582903717, + "pass@16": 0.5343941235795181, + "pass@32": 0.5985284837482586, + "pass@64": 0.6597611542390495, + "pass@128": 0.7282913165266106 + }, + "14": { + "pass@1": 0.16132127192982457, + "pass@2": 0.24900207786526687, + "pass@4": 0.34654788052809166, + "pass@8": 0.43636616447770876, + "pass@16": 0.5097873294578236, + "pass@32": 0.5690943424021969, + "pass@64": 0.6227856094077184, + "pass@128": 0.6754385964912281 + }, + "15": { + "pass@1": 0.13856534090909092, + "pass@2": 0.22398182116439982, + "pass@4": 0.32483539131472194, + "pass@8": 0.4210097549573201, + "pass@16": 0.49911894400022283, + "pass@32": 0.5627604370589941, + "pass@64": 0.6207404034236885, + "pass@128": 0.6727272727272727 + }, + "16": { + "pass@1": 0.13837695868945868, + "pass@2": 0.21823368833703485, + "pass@4": 0.3100300085352579, + "pass@8": 0.39765295225678154, + "pass@16": 0.46724374386741413, + "pass@32": 0.5191649589933541, + "pass@64": 0.5639517418293533, + "pass@128": 0.6068376068376068 + }, + "17": { + "pass@1": 0.1343652950310559, + "pass@2": 0.2217845221792929, + "pass@4": 0.3313873420131489, + "pass@8": 0.44396581707866756, + "pass@16": 0.5348156239501506, + "pass@32": 0.5924449604975142, + "pass@64": 0.6296611056432868, + "pass@128": 0.6645962732919255 + }, + "18": { + "pass@1": 0.13033088235294119, + "pass@2": 0.2081334674617878, + "pass@4": 0.302660001599065, + "pass@8": 0.39741988375269416, + "pass@16": 0.4747293286693812, + "pass@32": 0.5260981930569909, + "pass@64": 0.5574985566060943, + "pass@128": 0.5852941176470589 + }, + "19": { + "pass@1": 0.11130487351190477, + "pass@2": 0.1813290184430071, + "pass@4": 0.2708650369596659, + "pass@8": 0.3643281841752635, + "pass@16": 0.4387280487490098, + "pass@32": 0.4830895220764344, + "pass@64": 0.506235461720585, + "pass@128": 0.5208333333333334 + }, + "20": { + "pass@1": 0.10656811377245509, + "pass@2": 0.17586099945777736, + "pass@4": 0.2645760077146045, + "pass@8": 0.3582559738491696, + "pass@16": 0.4345901738446862, + "pass@32": 0.48211568706570346, + "pass@64": 0.5091873400019051, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 259096, + "answer_accuracy": 0.32340429781115193, + "avg_response_len": 268.33878340190125, + "resp_tokens_sum": 214980153, + "pass_at_k": { + "pass@1": 0.32340429781115193, + "pass@2": 0.4089581278549485, + "pass@4": 0.4999869338808328, + "pass@8": 0.5851361993699952, + "pass@16": 0.655583087089333, + "pass@32": 0.7106789122766686, + "pass@64": 0.75461773383224, + "pass@128": 0.7916600095861959 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.25944496268656714, + "pass@2": 0.37380861440827357, + "pass@4": 0.4947208124171043, + "pass@8": 0.6084540057533921, + "pass@16": 0.7069548544384052, + "pass@32": 0.7889803052552564, + "pass@64": 0.8535451186415071, + "pass@128": 0.9014925373134328 + }, + "2": { + "pass@1": 0.9660044642857143, + "pass@2": 0.9723727502812146, + "pass@4": 0.9782737465852483, + "pass@8": 0.9840867443239417, + "pass@16": 0.9897300202044366, + "pass@32": 0.9943372627596654, + "pass@64": 0.9967877861964524, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9045189504373178, + "pass@2": 0.9409872075939486, + "pass@4": 0.963359374538824, + "pass@8": 0.9778173387389365, + "pass@16": 0.9862698501499768, + "pass@32": 0.9905573089869245, + "pass@64": 0.9926020223300518, + "pass@128": 0.9941690962099126 + }, + "4": { + "pass@1": 0.7404513888888888, + "pass@2": 0.8004220483952662, + "pass@4": 0.8438767870573194, + "pass@8": 0.8764562526018601, + "pass@16": 0.9012650591890069, + "pass@32": 0.9202351939936253, + "pass@64": 0.9359478097107635, + "pass@128": 0.9502923976608187 + }, + "5": { + "pass@1": 0.44046415441176473, + "pass@2": 0.539720356646595, + "pass@4": 0.632906808156333, + "pass@8": 0.7201508139502907, + "pass@16": 0.7958080919156965, + "pass@32": 0.8564772070244233, + "pass@64": 0.9020604729390345, + "pass@128": 0.9323529411764706 + }, + "6": { + "pass@1": 0.4135695977917981, + "pass@2": 0.5535905015027699, + "pass@4": 0.6846301371240265, + "pass@8": 0.7918933708969056, + "pass@16": 0.870299357531166, + "pass@32": 0.921637925223863, + "pass@64": 0.953726180491391, + "pass@128": 0.9747634069400631 + }, + "7": { + "pass@1": 0.3732693829113924, + "pass@2": 0.5000556756453705, + "pass@4": 0.6184612249813708, + "pass@8": 0.71713744327495, + "pass@16": 0.7982010440452766, + "pass@32": 0.8671220101820218, + "pass@64": 0.9208143511726397, + "pass@128": 0.9525316455696202 + }, + "8": { + "pass@1": 0.29450544294294295, + "pass@2": 0.4251602734624386, + "pass@4": 0.5522831794111326, + "pass@8": 0.6573037179247492, + "pass@16": 0.739586440476168, + "pass@32": 0.8057187779468007, + "pass@64": 0.859777333695693, + "pass@128": 0.9069069069069069 + }, + "9": { + "pass@1": 0.21402733118971062, + "pass@2": 0.3169770014178291, + "pass@4": 0.43098392071971686, + "pass@8": 0.540710406429895, + "pass@16": 0.6397980025166145, + "pass@32": 0.732216820941264, + "pass@64": 0.8158523025431184, + "pass@128": 0.8842443729903537 + }, + "11": { + "pass@1": 0.21999007936507936, + "pass@2": 0.32811679790026244, + "pass@4": 0.4464493072294533, + "pass@8": 0.5552910257350235, + "pass@16": 0.6437437236941529, + "pass@32": 0.7177650886723408, + "pass@64": 0.7823839715220456, + "pass@128": 0.8380952380952381 + }, + "12": { + "pass@1": 0.18274560397553516, + "pass@2": 0.2797676021575284, + "pass@4": 0.38757090627433033, + "pass@8": 0.4914502937919992, + "pass@16": 0.5856451067997539, + "pass@32": 0.6711780315675037, + "pass@64": 0.7491794923736346, + "pass@128": 0.8195718654434251 + }, + "13": { + "pass@1": 0.17450327932098766, + "pass@2": 0.26490693654612624, + "pass@4": 0.3661309399403777, + "pass@8": 0.4624642420940969, + "pass@16": 0.5448557983915544, + "pass@32": 0.6185134734378167, + "pass@64": 0.6895867643018863, + "pass@128": 0.7561728395061729 + }, + "14": { + "pass@1": 0.1650361646884273, + "pass@2": 0.2522824715530736, + "pass@4": 0.3488312738726651, + "pass@8": 0.43660343721379263, + "pass@16": 0.5027340358101761, + "pass@32": 0.5535550253369403, + "pass@64": 0.6047111303759262, + "pass@128": 0.6646884272997032 + }, + "15": { + "pass@1": 0.11781442901234568, + "pass@2": 0.19408977957616405, + "pass@4": 0.2919006570937892, + "pass@8": 0.3949211679701401, + "pass@16": 0.48446269102984507, + "pass@32": 0.5532427056623543, + "pass@64": 0.6071269026647454, + "pass@128": 0.6604938271604939 + }, + "16": { + "pass@1": 0.1358545353982301, + "pass@2": 0.21153033760713535, + "pass@4": 0.2991320606494985, + "pass@8": 0.3867938722175179, + "pass@16": 0.46065202878578043, + "pass@32": 0.5153170845256568, + "pass@64": 0.5579031332192951, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.11241057981927711, + "pass@2": 0.18683555580590078, + "pass@4": 0.2822885405514067, + "pass@8": 0.3811760461419187, + "pass@16": 0.4616358779338103, + "pass@32": 0.5172659436791867, + "pass@64": 0.5580279888684868, + "pass@128": 0.5933734939759037 + }, + "18": { + "pass@1": 0.11902721774193549, + "pass@2": 0.18856561150622297, + "pass@4": 0.2713082698332063, + "pass@8": 0.3538581464718504, + "pass@16": 0.41980580105441206, + "pass@32": 0.4647936660551533, + "pass@64": 0.49533100083021947, + "pass@128": 0.5225806451612903 + }, + "19": { + "pass@1": 0.11131042296072508, + "pass@2": 0.18102727953945336, + "pass@4": 0.26652312078814944, + "pass@8": 0.3546306422885341, + "pass@16": 0.42932109990881356, + "pass@32": 0.4792171969760829, + "pass@64": 0.5081471983782093, + "pass@128": 0.5287009063444109 + }, + "20": { + "pass@1": 0.1073338963963964, + "pass@2": 0.17943607978056803, + "pass@4": 0.2735159763250315, + "pass@8": 0.3727452360201581, + "pass@16": 0.45258346490831136, + "pass@32": 0.5016080590418188, + "pass@64": 0.5292435042316577, + "pass@128": 0.5465465465465466 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/summary.csv b/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..02f6592c1e790e5eceb364c7539e411e90c60e63 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21350,0.509234375,0.1478125,0.3190123355263158,0.0,0.0,0.0,202.11198611111112,301.16630703125,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/summary.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2837d5130b41ea1ee694187cd9701a56351006a2 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block32/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21350", + "id_acc": 0.509234375, + "ood_acc": 0.1478125, + "total_acc": 0.3190123355263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 202.11198611111112, + "ood_avg_resp_len": 301.16630703125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 260336, + "answer_accuracy": 0.32151043313310146, + "avg_response_len": 242.01525203525134, + "resp_tokens_sum": 195966526, + "pass_at_k": { + "pass@1": 0.32151043313310146, + "pass@2": 0.4099008614305211, + "pass@4": 0.5029745765210857, + "pass@8": 0.5885455898222649, + "pass@16": 0.6584088671873528, + "pass@32": 0.71223867333972, + "pass@64": 0.7538778515608776, + "pass@128": 0.786911160290863 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2340523598820059, + "pass@2": 0.3490363621582701, + "pass@4": 0.47703086422825464, + "pass@8": 0.601095237968881, + "pass@16": 0.7081131385563241, + "pass@32": 0.7955993715227827, + "pass@64": 0.8638342379925815, + "pass@128": 0.911504424778761 + }, + "2": { + "pass@1": 0.9397390965732088, + "pass@2": 0.9527060796722839, + "pass@4": 0.9634816159078247, + "pass@8": 0.97184488771153, + "pass@16": 0.9779010470744166, + "pass@32": 0.9834545773716056, + "pass@64": 0.9890862112444283, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9171683868501529, + "pass@2": 0.9446715698668398, + "pass@4": 0.9611175976856101, + "pass@8": 0.9714190867433976, + "pass@16": 0.9792196020103332, + "pass@32": 0.9848030044380172, + "pass@64": 0.9877250665899934, + "pass@128": 0.9908256880733946 + }, + "4": { + "pass@1": 0.777978515625, + "pass@2": 0.8460526113435035, + "pass@4": 0.8948536686820393, + "pass@8": 0.9276525455102214, + "pass@16": 0.9486645850423827, + "pass@32": 0.9630959185836433, + "pass@64": 0.9733635031273581, + "pass@128": 0.978125 + }, + "5": { + "pass@1": 0.42544642857142856, + "pass@2": 0.5282945100612425, + "pass@4": 0.6184357633867192, + "pass@8": 0.6987714941711862, + "pass@16": 0.7683401121051054, + "pass@32": 0.8247387249798265, + "pass@64": 0.8683272151365441, + "pass@128": 0.8984126984126984 + }, + "6": { + "pass@1": 0.4719311377245509, + "pass@2": 0.6100170475977178, + "pass@4": 0.7327381440718109, + "pass@8": 0.8212352253202607, + "pass@16": 0.8766424175779357, + "pass@32": 0.9137167660624057, + "pass@64": 0.9411640584150373, + "pass@128": 0.9610778443113772 + }, + "7": { + "pass@1": 0.36415957840236685, + "pass@2": 0.48880959383590367, + "pass@4": 0.6072677575805979, + "pass@8": 0.7089181989531216, + "pass@16": 0.7888269810111848, + "pass@32": 0.8487011389145268, + "pass@64": 0.8983636353290456, + "pass@128": 0.9437869822485208 + }, + "8": { + "pass@1": 0.3150040064102564, + "pass@2": 0.43504725671310324, + "pass@4": 0.5564820518228488, + "pass@8": 0.6692766640774171, + "pass@16": 0.7639736770960015, + "pass@32": 0.8368242107183997, + "pass@64": 0.8889416956818764, + "pass@128": 0.9230769230769231 + }, + "9": { + "pass@1": 0.23921654929577466, + "pass@2": 0.3568800598868798, + "pass@4": 0.4833165853388044, + "pass@8": 0.5962302356670168, + "pass@16": 0.6890966235919163, + "pass@32": 0.7682846086626541, + "pass@64": 0.8373558560961706, + "pass@128": 0.8873239436619719 + }, + "11": { + "pass@1": 0.21746635977337112, + "pass@2": 0.3272846913965782, + "pass@4": 0.44665189983334197, + "pass@8": 0.5553283625674275, + "pass@16": 0.6448423394148377, + "pass@32": 0.7179749189998534, + "pass@64": 0.777423089022195, + "pass@128": 0.8243626062322946 + }, + "12": { + "pass@1": 0.18498411016949154, + "pass@2": 0.2839403582232306, + "pass@4": 0.3926915541807275, + "pass@8": 0.494440493747792, + "pass@16": 0.5843288803240919, + "pass@32": 0.6652268589464182, + "pass@64": 0.735752190146036, + "pass@128": 0.7937853107344632 + }, + "13": { + "pass@1": 0.18647139498432602, + "pass@2": 0.2839080974008344, + "pass@4": 0.38877480009356197, + "pass@8": 0.4846227863805249, + "pass@16": 0.5638918640501261, + "pass@32": 0.6319251904268449, + "pass@64": 0.6916457926217192, + "pass@128": 0.7429467084639498 + }, + "14": { + "pass@1": 0.16109326323987538, + "pass@2": 0.25444944746976716, + "pass@4": 0.3592573200546192, + "pass@8": 0.4555773271112732, + "pass@16": 0.5327569175982596, + "pass@32": 0.5947606734348753, + "pass@64": 0.6511536361169645, + "pass@128": 0.7102803738317757 + }, + "15": { + "pass@1": 0.14234104046242774, + "pass@2": 0.2236904612898822, + "pass@4": 0.31877335600390994, + "pass@8": 0.4121823992074167, + "pass@16": 0.49071930914631096, + "pass@32": 0.5529727050149277, + "pass@64": 0.6061524184861755, + "pass@128": 0.6560693641618497 + }, + "16": { + "pass@1": 0.14392641129032258, + "pass@2": 0.22453287719075446, + "pass@4": 0.3174367695973486, + "pass@8": 0.4091923824210985, + "pass@16": 0.48592865391022005, + "pass@32": 0.5402647597178885, + "pass@64": 0.5777915977164921, + "pass@128": 0.6096774193548387 + }, + "17": { + "pass@1": 0.11477149566473989, + "pass@2": 0.18487011287606392, + "pass@4": 0.27174867872368563, + "pass@8": 0.3607717520233434, + "pass@16": 0.43365161352207626, + "pass@32": 0.482409295928184, + "pass@64": 0.5150428298119281, + "pass@128": 0.546242774566474 + }, + "18": { + "pass@1": 0.12763392857142858, + "pass@2": 0.2035601799775029, + "pass@4": 0.2933536606138519, + "pass@8": 0.3812815069370258, + "pass@16": 0.45235960903548417, + "pass@32": 0.501900940088412, + "pass@64": 0.5345649813685637, + "pass@128": 0.56 + }, + "19": { + "pass@1": 0.11732826576576577, + "pass@2": 0.18959707739235296, + "pass@4": 0.2780769194391241, + "pass@8": 0.3682895575262585, + "pass@16": 0.44274252772123246, + "pass@32": 0.4905652476370396, + "pass@64": 0.5157753123580949, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.10604354354354355, + "pass@2": 0.17358192345889195, + "pass@4": 0.25876359036201546, + "pass@8": 0.3487342283712976, + "pass@16": 0.4243679405449727, + "pass@32": 0.47493279620096895, + "pass@64": 0.5034854391857208, + "pass@128": 0.5165165165165165 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 256406, + "answer_accuracy": 0.31226373733437257, + "avg_response_len": 252.55651061964147, + "resp_tokens_sum": 207379202, + "pass_at_k": { + "pass@1": 0.31226373733437257, + "pass@2": 0.4041371953651974, + "pass@4": 0.5020694377398108, + "pass@8": 0.5920825799919917, + "pass@16": 0.6640724218550299, + "pass@32": 0.7175980090991305, + "pass@64": 0.7592845415933916, + "pass@128": 0.7950116913484022 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.22354294478527606, + "pass@2": 0.3360058088981208, + "pass@4": 0.46179167040469643, + "pass@8": 0.5850830032020309, + "pass@16": 0.6922560235978716, + "pass@32": 0.779310015924251, + "pass@64": 0.8474751226546069, + "pass@128": 0.8987730061349694 + }, + "2": { + "pass@1": 0.8912186550151976, + "pass@2": 0.9116783440633751, + "pass@4": 0.9292722276546739, + "pass@8": 0.9451709089562008, + "pass@16": 0.9578299275067208, + "pass@32": 0.967487067518195, + "pass@64": 0.9747033816302779, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.848626893939394, + "pass@2": 0.8895933995466476, + "pass@4": 0.9138172435831877, + "pass@8": 0.9303741145081666, + "pass@16": 0.9424221740566364, + "pass@32": 0.9518335581963728, + "pass@64": 0.9600967272076638, + "pass@128": 0.9666666666666667 + }, + "4": { + "pass@1": 0.725938424556213, + "pass@2": 0.7984489295531847, + "pass@4": 0.8548653993139906, + "pass@8": 0.8967305917882408, + "pass@16": 0.9268802741732332, + "pass@32": 0.945689901633452, + "pass@64": 0.95690937985168, + "pass@128": 0.9674556213017751 + }, + "5": { + "pass@1": 0.4459239130434783, + "pass@2": 0.5601492068926165, + "pass@4": 0.6647551846780018, + "pass@8": 0.7498131323981773, + "pass@16": 0.8119176290092686, + "pass@32": 0.8557433256995092, + "pass@64": 0.8852069468068376, + "pass@128": 0.9072463768115943 + }, + "6": { + "pass@1": 0.42238986389684813, + "pass@2": 0.560813164158563, + "pass@4": 0.6923920269278656, + "pass@8": 0.796637969019255, + "pass@16": 0.8679993396877942, + "pass@32": 0.9132326395237702, + "pass@64": 0.9407776590161546, + "pass@128": 0.9598853868194842 + }, + "7": { + "pass@1": 0.3537978684971098, + "pass@2": 0.48195561763233363, + "pass@4": 0.6084160362360775, + "pass@8": 0.717139403598144, + "pass@16": 0.8003843219004848, + "pass@32": 0.8634777033100696, + "pass@64": 0.915658946495517, + "pass@128": 0.9566473988439307 + }, + "8": { + "pass@1": 0.28818221830985913, + "pass@2": 0.41921301430631, + "pass@4": 0.5555384356004792, + "pass@8": 0.6756259834552851, + "pass@16": 0.7670445900118551, + "pass@32": 0.8316106463558992, + "pass@64": 0.8783679240426463, + "pass@128": 0.9154929577464789 + }, + "9": { + "pass@1": 0.25715755988023953, + "pass@2": 0.37767722429157424, + "pass@4": 0.5053444070801026, + "pass@8": 0.6198841515478999, + "pass@16": 0.7131677032398571, + "pass@32": 0.7895863734944806, + "pass@64": 0.8534291934377334, + "pass@128": 0.9011976047904192 + }, + "11": { + "pass@1": 0.1899472891566265, + "pass@2": 0.29249144708756303, + "pass@4": 0.4056045608192049, + "pass@8": 0.5118326321094175, + "pass@16": 0.6064758426416722, + "pass@32": 0.691489508808827, + "pass@64": 0.767702363812573, + "pass@128": 0.8373493975903614 + }, + "12": { + "pass@1": 0.21497844827586207, + "pass@2": 0.3253735683607731, + "pass@4": 0.44188718318517406, + "pass@8": 0.547290961654598, + "pass@16": 0.6366487702338203, + "pass@32": 0.7110514969364836, + "pass@64": 0.7759151125242277, + "pass@128": 0.8307210031347962 + }, + "13": { + "pass@1": 0.16281512605042017, + "pass@2": 0.25351139471536654, + "pass@4": 0.3567216876251812, + "pass@8": 0.45448099582903717, + "pass@16": 0.5343941235795181, + "pass@32": 0.5985284837482586, + "pass@64": 0.6597611542390495, + "pass@128": 0.7282913165266106 + }, + "14": { + "pass@1": 0.16132127192982457, + "pass@2": 0.24900207786526687, + "pass@4": 0.34654788052809166, + "pass@8": 0.43636616447770876, + "pass@16": 0.5097873294578236, + "pass@32": 0.5690943424021969, + "pass@64": 0.6227856094077184, + "pass@128": 0.6754385964912281 + }, + "15": { + "pass@1": 0.13856534090909092, + "pass@2": 0.22398182116439982, + "pass@4": 0.32483539131472194, + "pass@8": 0.4210097549573201, + "pass@16": 0.49911894400022283, + "pass@32": 0.5627604370589941, + "pass@64": 0.6207404034236885, + "pass@128": 0.6727272727272727 + }, + "16": { + "pass@1": 0.13837695868945868, + "pass@2": 0.21823368833703485, + "pass@4": 0.3100300085352579, + "pass@8": 0.39765295225678154, + "pass@16": 0.46724374386741413, + "pass@32": 0.5191649589933541, + "pass@64": 0.5639517418293533, + "pass@128": 0.6068376068376068 + }, + "17": { + "pass@1": 0.1343652950310559, + "pass@2": 0.2217845221792929, + "pass@4": 0.3313873420131489, + "pass@8": 0.44396581707866756, + "pass@16": 0.5348156239501506, + "pass@32": 0.5924449604975142, + "pass@64": 0.6296611056432868, + "pass@128": 0.6645962732919255 + }, + "18": { + "pass@1": 0.13033088235294119, + "pass@2": 0.2081334674617878, + "pass@4": 0.302660001599065, + "pass@8": 0.39741988375269416, + "pass@16": 0.4747293286693812, + "pass@32": 0.5260981930569909, + "pass@64": 0.5574985566060943, + "pass@128": 0.5852941176470589 + }, + "19": { + "pass@1": 0.11130487351190477, + "pass@2": 0.1813290184430071, + "pass@4": 0.2708650369596659, + "pass@8": 0.3643281841752635, + "pass@16": 0.4387280487490098, + "pass@32": 0.4830895220764344, + "pass@64": 0.506235461720585, + "pass@128": 0.5208333333333334 + }, + "20": { + "pass@1": 0.10656811377245509, + "pass@2": 0.17586099945777736, + "pass@4": 0.2645760077146045, + "pass@8": 0.3582559738491696, + "pass@16": 0.4345901738446862, + "pass@32": 0.48211568706570346, + "pass@64": 0.5091873400019051, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 259096, + "answer_accuracy": 0.32340429781115193, + "avg_response_len": 268.33878340190125, + "resp_tokens_sum": 214980153, + "pass_at_k": { + "pass@1": 0.32340429781115193, + "pass@2": 0.4089581278549485, + "pass@4": 0.4999869338808328, + "pass@8": 0.5851361993699952, + "pass@16": 0.655583087089333, + "pass@32": 0.7106789122766686, + "pass@64": 0.75461773383224, + "pass@128": 0.7916600095861959 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.25944496268656714, + "pass@2": 0.37380861440827357, + "pass@4": 0.4947208124171043, + "pass@8": 0.6084540057533921, + "pass@16": 0.7069548544384052, + "pass@32": 0.7889803052552564, + "pass@64": 0.8535451186415071, + "pass@128": 0.9014925373134328 + }, + "2": { + "pass@1": 0.9660044642857143, + "pass@2": 0.9723727502812146, + "pass@4": 0.9782737465852483, + "pass@8": 0.9840867443239417, + "pass@16": 0.9897300202044366, + "pass@32": 0.9943372627596654, + "pass@64": 0.9967877861964524, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9045189504373178, + "pass@2": 0.9409872075939486, + "pass@4": 0.963359374538824, + "pass@8": 0.9778173387389365, + "pass@16": 0.9862698501499768, + "pass@32": 0.9905573089869245, + "pass@64": 0.9926020223300518, + "pass@128": 0.9941690962099126 + }, + "4": { + "pass@1": 0.7404513888888888, + "pass@2": 0.8004220483952662, + "pass@4": 0.8438767870573194, + "pass@8": 0.8764562526018601, + "pass@16": 0.9012650591890069, + "pass@32": 0.9202351939936253, + "pass@64": 0.9359478097107635, + "pass@128": 0.9502923976608187 + }, + "5": { + "pass@1": 0.44046415441176473, + "pass@2": 0.539720356646595, + "pass@4": 0.632906808156333, + "pass@8": 0.7201508139502907, + "pass@16": 0.7958080919156965, + "pass@32": 0.8564772070244233, + "pass@64": 0.9020604729390345, + "pass@128": 0.9323529411764706 + }, + "6": { + "pass@1": 0.4135695977917981, + "pass@2": 0.5535905015027699, + "pass@4": 0.6846301371240265, + "pass@8": 0.7918933708969056, + "pass@16": 0.870299357531166, + "pass@32": 0.921637925223863, + "pass@64": 0.953726180491391, + "pass@128": 0.9747634069400631 + }, + "7": { + "pass@1": 0.3732693829113924, + "pass@2": 0.5000556756453705, + "pass@4": 0.6184612249813708, + "pass@8": 0.71713744327495, + "pass@16": 0.7982010440452766, + "pass@32": 0.8671220101820218, + "pass@64": 0.9208143511726397, + "pass@128": 0.9525316455696202 + }, + "8": { + "pass@1": 0.29450544294294295, + "pass@2": 0.4251602734624386, + "pass@4": 0.5522831794111326, + "pass@8": 0.6573037179247492, + "pass@16": 0.739586440476168, + "pass@32": 0.8057187779468007, + "pass@64": 0.859777333695693, + "pass@128": 0.9069069069069069 + }, + "9": { + "pass@1": 0.21402733118971062, + "pass@2": 0.3169770014178291, + "pass@4": 0.43098392071971686, + "pass@8": 0.540710406429895, + "pass@16": 0.6397980025166145, + "pass@32": 0.732216820941264, + "pass@64": 0.8158523025431184, + "pass@128": 0.8842443729903537 + }, + "11": { + "pass@1": 0.21999007936507936, + "pass@2": 0.32811679790026244, + "pass@4": 0.4464493072294533, + "pass@8": 0.5552910257350235, + "pass@16": 0.6437437236941529, + "pass@32": 0.7177650886723408, + "pass@64": 0.7823839715220456, + "pass@128": 0.8380952380952381 + }, + "12": { + "pass@1": 0.18274560397553516, + "pass@2": 0.2797676021575284, + "pass@4": 0.38757090627433033, + "pass@8": 0.4914502937919992, + "pass@16": 0.5856451067997539, + "pass@32": 0.6711780315675037, + "pass@64": 0.7491794923736346, + "pass@128": 0.8195718654434251 + }, + "13": { + "pass@1": 0.17450327932098766, + "pass@2": 0.26490693654612624, + "pass@4": 0.3661309399403777, + "pass@8": 0.4624642420940969, + "pass@16": 0.5448557983915544, + "pass@32": 0.6185134734378167, + "pass@64": 0.6895867643018863, + "pass@128": 0.7561728395061729 + }, + "14": { + "pass@1": 0.1650361646884273, + "pass@2": 0.2522824715530736, + "pass@4": 0.3488312738726651, + "pass@8": 0.43660343721379263, + "pass@16": 0.5027340358101761, + "pass@32": 0.5535550253369403, + "pass@64": 0.6047111303759262, + "pass@128": 0.6646884272997032 + }, + "15": { + "pass@1": 0.11781442901234568, + "pass@2": 0.19408977957616405, + "pass@4": 0.2919006570937892, + "pass@8": 0.3949211679701401, + "pass@16": 0.48446269102984507, + "pass@32": 0.5532427056623543, + "pass@64": 0.6071269026647454, + "pass@128": 0.6604938271604939 + }, + "16": { + "pass@1": 0.1358545353982301, + "pass@2": 0.21153033760713535, + "pass@4": 0.2991320606494985, + "pass@8": 0.3867938722175179, + "pass@16": 0.46065202878578043, + "pass@32": 0.5153170845256568, + "pass@64": 0.5579031332192951, + "pass@128": 0.5988200589970502 + }, + "17": { + "pass@1": 0.11241057981927711, + "pass@2": 0.18683555580590078, + "pass@4": 0.2822885405514067, + "pass@8": 0.3811760461419187, + "pass@16": 0.4616358779338103, + "pass@32": 0.5172659436791867, + "pass@64": 0.5580279888684868, + "pass@128": 0.5933734939759037 + }, + "18": { + "pass@1": 0.11902721774193549, + "pass@2": 0.18856561150622297, + "pass@4": 0.2713082698332063, + "pass@8": 0.3538581464718504, + "pass@16": 0.41980580105441206, + "pass@32": 0.4647936660551533, + "pass@64": 0.49533100083021947, + "pass@128": 0.5225806451612903 + }, + "19": { + "pass@1": 0.11131042296072508, + "pass@2": 0.18102727953945336, + "pass@4": 0.26652312078814944, + "pass@8": 0.3546306422885341, + "pass@16": 0.42932109990881356, + "pass@32": 0.4792171969760829, + "pass@64": 0.5081471983782093, + "pass@128": 0.5287009063444109 + }, + "20": { + "pass@1": 0.1073338963963964, + "pass@2": 0.17943607978056803, + "pass@4": 0.2735159763250315, + "pass@8": 0.3727452360201581, + "pass@16": 0.45258346490831136, + "pass@32": 0.5016080590418188, + "pass@64": 0.5292435042316577, + "pass@128": 0.5465465465465466 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/checkpoint-21350_metrics.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/checkpoint-21350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..49e8735f786294ce4029ec558501cf53d4a78e4b --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/checkpoint-21350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21350", + "id": { + "answer_accuracy": 0.9823862847222222, + "avg_response_len": 197.05145833333333, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9823862847222222, + "pass@2": 0.9875863407698983, + "pass@4": 0.9904172675498891, + "pass@8": 0.9922989949739147, + "pass@16": 0.9937167300175361, + "pass@32": 0.9948233786809566, + "pass@64": 0.9956022580423095, + "pass@128": 0.9962222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9247734375, + "pass@2": 0.9439683809055122, + "pass@4": 0.9562506626359206, + "pass@8": 0.963894185093944, + "pass@16": 0.9690634148740488, + "pass@32": 0.9732302849431483, + "pass@64": 0.9765343228436548, + "pass@128": 0.979 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9986171875, + "pass@2": 0.9992669783464567, + "pass@4": 0.999595402137233, + "pass@8": 0.9998465879068722, + "pass@16": 0.9999796800606743, + "pass@32": 0.999999793590006, + "pass@64": 0.9999999999993952, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9988515625, + "pass@2": 0.9989880659448819, + "pass@4": 0.9989999060742407, + "pass@8": 0.9989999999978996, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9846328125, + "pass@2": 0.9880631151574807, + "pass@4": 0.9899134654105735, + "pass@8": 0.9911568007077315, + "pass@16": 0.9922402977900743, + "pass@32": 0.993272704498571, + "pass@64": 0.9941906591276778, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.9973984375, + "pass@2": 0.9981446850393701, + "pass@4": 0.9984959660667417, + "pass@8": 0.9988935782393594, + "pass@16": 0.9994010868705335, + "pass@32": 0.9998376651396068, + "pass@64": 0.9999931225037461, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9865078125, + "pass@2": 0.9903694635826774, + "pass@4": 0.9924604293213348, + "pass@8": 0.9943845859818515, + "pass@16": 0.9963773520655672, + "pass@32": 0.9978226010458, + "pass@64": 0.9984801674769502, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.979640625, + "pass@2": 0.9879799458661419, + "pass@4": 0.9915468292088483, + "pass@8": 0.9935623370467179, + "pass@16": 0.994786978981977, + "pass@32": 0.995685358786083, + "pass@64": 0.9963741882064558, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9710546875, + "pass@2": 0.9814964320866143, + "pass@4": 0.9864927470941129, + "pass@8": 0.9899528797908489, + "pass@16": 0.9926017595149479, + "pass@32": 0.9945620001253924, + "pass@64": 0.995847862222904, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.9247734375, + "2": 1.0, + "3": 0.9986171875, + "4": 0.9988515625, + "5": 0.9846328125, + "6": 0.9973984375, + "7": 0.9865078125, + "8": 0.979640625, + "9": 0.9710546875 + }, + "per_op_avg_response_len": { + "10": 275.375984375, + "2": 114.54525, + "3": 142.971828125, + "4": 158.007921875, + "5": 170.8766796875, + "6": 196.06684375, + "7": 220.5956171875, + "8": 239.678515625, + "9": 255.344484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 275.375984375, + "2": 114.54525, + "3": 142.971828125, + "4": 158.007921875, + "5": 170.8766796875, + "6": 196.06684375, + "7": 220.5956171875, + "8": 239.678515625, + "9": 255.344484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371949, + "answer_accuracy": 0.9813750633232016, + "avg_response_len": 187.6097364699426, + "resp_tokens_sum": 71105591, + "pass_at_k": { + "pass@1": 0.9813750633232016, + "pass@2": 0.9875192213340365, + "pass@4": 0.9906089075175131, + "pass@8": 0.9924568104767721, + "pass@16": 0.9938435017335243, + "pass@32": 0.9948168678638909, + "pass@64": 0.99533565447748, + "pass@128": 0.9956095913542722 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9228890117994101, + "pass@2": 0.9435989507119136, + "pass@4": 0.9556432768912737, + "pass@8": 0.9621737909507128, + "pass@16": 0.9668383416955398, + "pass@32": 0.9704871371383337, + "pass@64": 0.972724070641377, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974675076452599, + "pass@2": 0.9980608430253559, + "pass@4": 0.9987818108172257, + "pass@8": 0.9995309324501443, + "pass@16": 0.999937859513187, + "pass@32": 0.9999993687767768, + "pass@64": 0.9999999999981503, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99677734375, + "pass@2": 0.9968726931594489, + "pass@4": 0.9968749997070677, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9848958333333333, + "pass@2": 0.9886990688663916, + "pass@4": 0.9909946988769263, + "pass@8": 0.9925728488812268, + "pass@16": 0.9940615103834403, + "pass@32": 0.9956209222387189, + "pass@64": 0.9966317339943471, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9984562125748503, + "pass@2": 0.9996393777405818, + "pass@4": 0.9999618897750056, + "pass@8": 0.9999996288482736, + "pass@16": 0.9999999999866336, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9819018121301775, + "pass@2": 0.9881998963332246, + "pass@4": 0.9908525150147651, + "pass@8": 0.992921263269219, + "pass@16": 0.9953543225386476, + "pass@32": 0.9973493282786543, + "pass@64": 0.9985121256129645, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9764372996794872, + "pass@2": 0.9853718705834849, + "pass@4": 0.9896878509176735, + "pass@8": 0.9922069966524123, + "pass@16": 0.99332484085682, + "pass@32": 0.9935781371491155, + "pass@64": 0.9935897405457036, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9763204225352112, + "pass@2": 0.9888221553731841, + "pass@4": 0.9937658911474093, + "pass@8": 0.9966332180416213, + "pass@16": 0.9987499819877586, + "pass@32": 0.9998218884821831, + "pass@64": 0.9999980823812246, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383761, + "answer_accuracy": 0.9823502006880734, + "avg_response_len": 195.06704363941677, + "resp_tokens_sum": 76204111, + "pass_at_k": { + "pass@1": 0.9823502006880734, + "pass@2": 0.9868652187541933, + "pass@4": 0.9895015308948114, + "pass@8": 0.9913981195079338, + "pass@16": 0.992894385941712, + "pass@32": 0.9942000910661344, + "pass@64": 0.9953393783642251, + "pass@128": 0.9963958060288336 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9202214340490797, + "pass@2": 0.9360723062895516, + "pass@4": 0.9465917125581695, + "pass@8": 0.9537278944651678, + "pass@16": 0.959031497936665, + "pass@32": 0.9641086020857509, + "pass@64": 0.9686631103780736, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985795454545454, + "pass@2": 0.9997170275590551, + "pass@4": 0.999981120257695, + "pass@8": 0.9999999181688074, + "pass@16": 0.9999999999995821, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991304347826087, + "pass@2": 0.9935520797672031, + "pass@4": 0.994298946055656, + "pass@8": 0.9945538934608948, + "pass@16": 0.9948847424451219, + "pass@32": 0.9954752938491385, + "pass@64": 0.996382517402716, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9971570558739254, + "pass@2": 0.9976052923989802, + "pass@4": 0.9980087450565815, + "pass@8": 0.9986333194949814, + "pass@16": 0.999382080743133, + "pass@32": 0.9998961597167226, + "pass@64": 0.9999991244302526, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9897940751445087, + "pass@2": 0.9925530386646032, + "pass@4": 0.9948359450010945, + "pass@8": 0.997232438770044, + "pass@16": 0.9991944808243975, + "pass@32": 0.9999485653009884, + "pass@64": 0.9999999670026358, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9779709507042254, + "pass@2": 0.9868096373516692, + "pass@4": 0.9909287493992825, + "pass@8": 0.9933035854016835, + "pass@16": 0.9949393747728471, + "pass@32": 0.9965682470179622, + "pass@64": 0.9982458732971077, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9639549026946108, + "pass@2": 0.9735508068508654, + "pass@4": 0.9791762661403849, + "pass@8": 0.9836002268475236, + "pass@16": 0.9872007390250478, + "pass@32": 0.990516785177483, + "pass@64": 0.9936008151224044, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375999, + "answer_accuracy": 0.9834255733177101, + "avg_response_len": 208.4385932792099, + "resp_tokens_sum": 79693578, + "pass_at_k": { + "pass@1": 0.9834255733177101, + "pass@2": 0.9883896902983804, + "pass@4": 0.9911629596580815, + "pass@8": 0.9930630325427812, + "pass@16": 0.9944313008472564, + "pass@32": 0.9954666837796396, + "pass@64": 0.996135141180231, + "pass@128": 0.9966521593572146 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9311100746268657, + "pass@2": 0.9520261634739686, + "pass@4": 0.9662647566442252, + "pass@8": 0.9755282876597258, + "pass@16": 0.9810774588415156, + "pass@32": 0.9848828094725318, + "pass@64": 0.9880498176506736, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997494533527697, + "pass@2": 0.9999838588416243, + "pass@4": 0.9999999426092147, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996801900584795, + "pass@2": 0.9999672635492931, + "pass@4": 0.9999997256373656, + "pass@8": 0.9999999999938582, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9776194852941177, + "pass@2": 0.9819042380731822, + "pass@4": 0.9844617613974724, + "pass@8": 0.9863978237239305, + "pass@16": 0.9878696643403634, + "pass@32": 0.988862169457123, + "pass@64": 0.9897049835750625, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9965496845425867, + "pass@2": 0.9971636776869769, + "pass@4": 0.9974878323569174, + "pass@8": 0.99801474353403, + "pass@16": 0.9987909801124433, + "pass@32": 0.9996022252317682, + "pass@64": 0.9999792683835579, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9878362341772152, + "pass@2": 0.9902992001395396, + "pass@4": 0.9915792477087515, + "pass@8": 0.9928315670329757, + "pass@16": 0.9943870274755153, + "pass@32": 0.9960010268780787, + "pass@64": 0.9967819001166339, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9844219219219219, + "pass@2": 0.9916711741268824, + "pass@4": 0.9939474885459137, + "pass@8": 0.9951080518725748, + "pass@16": 0.9959944462170828, + "pass@32": 0.9967184753879357, + "pass@64": 0.9969877001072768, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9726688102893891, + "pass@2": 0.9816675158872827, + "pass@4": 0.9860481763533578, + "pass@8": 0.9891498765884253, + "pass@16": 0.9913841449354583, + "pass@32": 0.9929023262538196, + "pass@64": 0.9935237000858083, + "pass@128": 0.9935691318327974 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.3994484375, + "avg_response_len": 314.61243671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.3994484375, + "pass@2": 0.4580676796259855, + "pass@4": 0.5077439735939214, + "pass@8": 0.5504779734481006, + "pass@16": 0.5876872622271497, + "pass@32": 0.6206564295933461, + "pass@64": 0.6503660963988431, + "pass@128": 0.6775 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.896671875, + "pass@2": 0.9257250246062998, + "pass@4": 0.9424503491751033, + "pass@8": 0.9534902449246606, + "pass@16": 0.9617341647619115, + "pass@32": 0.9685526691343028, + "pass@64": 0.9739862456805812, + "pass@128": 0.979 + }, + "12": { + "pass@1": 0.80815625, + "pass@2": 0.8617167814960643, + "pass@4": 0.8952266491376072, + "pass@8": 0.9174555970412555, + "pass@16": 0.9328540335518769, + "pass@32": 0.9449988518049591, + "pass@64": 0.9558885648361812, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.611390625, + "pass@2": 0.6904826525590564, + "pass@4": 0.7472079101049863, + "pass@8": 0.7914406927782185, + "pass@16": 0.8288341851671736, + "pass@32": 0.861509523658841, + "pass@64": 0.8889878970233329, + "pass@128": 0.908 + }, + "14": { + "pass@1": 0.471796875, + "pass@2": 0.5570604084645676, + "pass@4": 0.6261572565616799, + "pass@8": 0.683053618225761, + "pass@16": 0.7289388120489303, + "pass@32": 0.7649940912758662, + "pass@64": 0.7953310389712002, + "pass@128": 0.821 + }, + "15": { + "pass@1": 0.318984375, + "pass@2": 0.39327522145669336, + "pass@4": 0.4584166613235846, + "pass@8": 0.5164938821750149, + "pass@16": 0.5686359339713938, + "pass@32": 0.6147548772017475, + "pass@64": 0.6561541463069708, + "pass@128": 0.696 + }, + "16": { + "pass@1": 0.22421875, + "pass@2": 0.28558993602362204, + "pass@4": 0.3436635868016498, + "pass@8": 0.3963355521106936, + "pass@16": 0.4432359851509301, + "pass@32": 0.4863019464805994, + "pass@64": 0.527861398191278, + "pass@128": 0.569 + }, + "17": { + "pass@1": 0.1804296875, + "pass@2": 0.2315994094488188, + "pass@4": 0.28011776031121094, + "pass@8": 0.32552405623661257, + "pass@16": 0.3681552857648949, + "pass@32": 0.40843428995183584, + "pass@64": 0.4482001769556677, + "pass@128": 0.492 + }, + "18": { + "pass@1": 0.1810703125, + "pass@2": 0.23492507381889757, + "pass@4": 0.2869294929696286, + "pass@8": 0.3331809278116836, + "pass@16": 0.3739857580462215, + "pass@32": 0.4127878310891122, + "pass@64": 0.44893216254517043, + "pass@128": 0.479 + }, + "19": { + "pass@1": 0.15225, + "pass@2": 0.20230487204724387, + "pass@4": 0.25132070678665164, + "pass@8": 0.296715079215693, + "pass@16": 0.3375134586858961, + "pass@32": 0.3732706964877968, + "pass@64": 0.4044542672025891, + "pass@128": 0.434 + }, + "20": { + "pass@1": 0.149515625, + "pass@2": 0.1979974163385825, + "pass@4": 0.24594936276715385, + "pass@8": 0.29109008396138786, + "pass@16": 0.33298500512216034, + "pass@32": 0.3709595188483823, + "pass@64": 0.40386506627542906, + "pass@128": 0.43 + } + }, + "per_op_accuracy": { + "11": 0.896671875, + "12": 0.80815625, + "13": 0.611390625, + "14": 0.471796875, + "15": 0.318984375, + "16": 0.22421875, + "17": 0.1804296875, + "18": 0.1810703125, + "19": 0.15225, + "20": 0.149515625 + }, + "per_op_avg_response_len": { + "11": 291.5578359375, + "12": 299.696359375, + "13": 310.6432578125, + "14": 318.5055390625, + "15": 319.774390625, + "16": 317.105671875, + "17": 323.4047265625, + "18": 323.4605390625, + "19": 319.3578984375, + "20": 322.6181484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 291.5578359375, + "12": 299.696359375, + "13": 310.6432578125, + "14": 318.5055390625, + "15": 319.774390625, + "16": 317.105671875, + "17": 323.4047265625, + "18": 323.4605390625, + "19": 319.3578984375, + "20": 322.6181484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 174598, + "answer_accuracy": 0.4053631129271917, + "avg_response_len": 300.27192143387816, + "resp_tokens_sum": 129333122, + "pass_at_k": { + "pass@1": 0.4053631129271917, + "pass@2": 0.46600484374817347, + "pass@4": 0.5168705684913475, + "pass@8": 0.5601349722073556, + "pass@16": 0.5976063230770273, + "pass@32": 0.6308083131898564, + "pass@64": 0.6610245225245038, + "pass@128": 0.6882615156017831 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8874601628895185, + "pass@2": 0.9190749007383285, + "pass@4": 0.9379697790963095, + "pass@8": 0.9499868038901691, + "pass@16": 0.958601731222962, + "pass@32": 0.9659637243092551, + "pass@64": 0.9738305540462164, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.797801906779661, + "pass@2": 0.8524762000088971, + "pass@4": 0.8844551449077335, + "pass@8": 0.9041495170093695, + "pass@16": 0.9175524746444642, + "pass@32": 0.9286310489660135, + "pass@64": 0.9398662723540929, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.6121424373040752, + "pass@2": 0.6958881254165329, + "pass@4": 0.7590125529959217, + "pass@8": 0.8064926681604645, + "pass@16": 0.8440489366042815, + "pass@32": 0.8767117774857617, + "pass@64": 0.9045546474803343, + "pass@128": 0.9247648902821317 + }, + "14": { + "pass@1": 0.4789476246105919, + "pass@2": 0.5674093782961706, + "pass@4": 0.6386814332203795, + "pass@8": 0.699237479706666, + "pass@16": 0.749983464432357, + "pass@32": 0.7912858007134622, + "pass@64": 0.8267369070020342, + "pass@128": 0.8535825545171339 + }, + "15": { + "pass@1": 0.3295926661849711, + "pass@2": 0.4034149418551725, + "pass@4": 0.4654226851520722, + "pass@8": 0.5206955188961653, + "pass@16": 0.5724889979926816, + "pass@32": 0.6207619416448199, + "pass@64": 0.6668436877333662, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2501260080645161, + "pass@2": 0.3187746062992125, + "pass@4": 0.3809942934552536, + "pass@8": 0.43700423620641776, + "pass@16": 0.48804601495912603, + "pass@32": 0.5356938779931577, + "pass@64": 0.5791105338215622, + "pass@128": 0.6129032258064516 + }, + "17": { + "pass@1": 0.17241690751445088, + "pass@2": 0.2274190979017795, + "pass@4": 0.27921864887915004, + "pass@8": 0.32568698302405064, + "pass@16": 0.3676062818519709, + "pass@32": 0.4061846663090747, + "pass@64": 0.44490948479031045, + "pass@128": 0.4884393063583815 + }, + "18": { + "pass@1": 0.17680803571428572, + "pass@2": 0.2299905089988751, + "pass@4": 0.2822858747120897, + "pass@8": 0.33053922417649595, + "pass@16": 0.3754431840988442, + "pass@32": 0.4195090787833997, + "pass@64": 0.45981741367360784, + "pass@128": 0.49142857142857144 + }, + "19": { + "pass@1": 0.15892454954954954, + "pass@2": 0.21205235747558576, + "pass@4": 0.2627885883633915, + "pass@8": 0.30732869937392576, + "pass@16": 0.34537864261948176, + "pass@32": 0.37712504930713686, + "pass@64": 0.40499634950234564, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.16005067567567569, + "pass@2": 0.2089211504811898, + "pass@4": 0.2578512809772651, + "pass@8": 0.30439842135885803, + "pass@16": 0.34464130107609786, + "pass@32": 0.37680522912423126, + "pass@64": 0.4022415923768691, + "pass@128": 0.42042042042042044 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 173648, + "answer_accuracy": 0.40339726434730894, + "avg_response_len": 311.89995911388644, + "resp_tokens_sum": 134261704, + "pass_at_k": { + "pass@1": 0.40339726434730894, + "pass@2": 0.46212806660485584, + "pass@4": 0.5114594105710031, + "pass@8": 0.5535051441701547, + "pass@16": 0.590053018585874, + "pass@32": 0.6226786507905985, + "pass@64": 0.6524727746706163, + "pass@128": 0.6806422836752899 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9093561746987951, + "pass@2": 0.9346984394269996, + "pass@4": 0.9480427477627043, + "pass@8": 0.9566085144358756, + "pass@16": 0.9621387470688224, + "pass@32": 0.9660424121012906, + "pass@64": 0.9682693163975254, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.8195532915360502, + "pass@2": 0.8715327487473156, + "pass@4": 0.9028534996996849, + "pass@8": 0.9224261190045239, + "pass@16": 0.9357743493401, + "pass@32": 0.9465884364452, + "pass@64": 0.9558694712461404, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.6233368347338936, + "pass@2": 0.7012609177529281, + "pass@4": 0.752510632074352, + "pass@8": 0.7909648856753403, + "pass@16": 0.825167615439675, + "pass@32": 0.856456023376782, + "pass@64": 0.8833283014284938, + "pass@128": 0.9019607843137255 + }, + "14": { + "pass@1": 0.484078033625731, + "pass@2": 0.5705107174103236, + "pass@4": 0.6397225598993105, + "pass@8": 0.6942476151472764, + "pass@16": 0.735376785198006, + "pass@32": 0.7658550027742763, + "pass@64": 0.7913686429763481, + "pass@128": 0.8128654970760234 + }, + "15": { + "pass@1": 0.332717803030303, + "pass@2": 0.4114877863278452, + "pass@4": 0.4812904997670743, + "pass@8": 0.5437209905176837, + "pass@16": 0.5981587135360744, + "pass@32": 0.6433320324324602, + "pass@64": 0.6817106148445318, + "pass@128": 0.7181818181818181 + }, + "16": { + "pass@1": 0.22081997863247863, + "pass@2": 0.2871509550889474, + "pass@4": 0.3512183749467215, + "pass@8": 0.4076236800542498, + "pass@16": 0.4546544514041406, + "pass@32": 0.4964979651009504, + "pass@64": 0.5380652864044299, + "pass@128": 0.5811965811965812 + }, + "17": { + "pass@1": 0.18730590062111802, + "pass@2": 0.23642870897931226, + "pass@4": 0.2845957679203141, + "pass@8": 0.3326317237752575, + "pass@16": 0.38087277944396986, + "pass@32": 0.42657265186991555, + "pass@64": 0.46936251256103506, + "pass@128": 0.5186335403726708 + }, + "18": { + "pass@1": 0.18276654411764706, + "pass@2": 0.2352152327466419, + "pass@4": 0.2849862334303801, + "pass@8": 0.3288812893596294, + "pass@16": 0.3685753443013062, + "pass@32": 0.40794190067953773, + "pass@64": 0.4466121363445853, + "pass@128": 0.4823529411764706 + }, + "19": { + "pass@1": 0.14267113095238096, + "pass@2": 0.19041760404949387, + "pass@4": 0.2374986904538717, + "pass@8": 0.28168255992725094, + "pass@16": 0.3226338126666716, + "pass@32": 0.3610534610549758, + "pass@64": 0.39768084772182344, + "pass@128": 0.43452380952380953 + }, + "20": { + "pass@1": 0.14221556886227546, + "pass@2": 0.19062441062756372, + "pass@4": 0.2377481939944634, + "pass@8": 0.280558264608587, + "pass@16": 0.32031383167398947, + "pass@32": 0.35856987955107245, + "pass@64": 0.3935714032182891, + "pass@128": 0.4221556886227545 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 163048, + "answer_accuracy": 0.38930699877750613, + "avg_response_len": 332.1484685398839, + "resp_tokens_sum": 139109093, + "pass_at_k": { + "pass@1": 0.38930699877750613, + "pass@2": 0.4457316042344509, + "pass@4": 0.49453920391674794, + "pass@8": 0.53743513238356, + "pass@16": 0.5750547199305907, + "pass@32": 0.6081375365039013, + "pass@64": 0.6372394573582372, + "pass@128": 0.66320293398533 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.893625992063492, + "pass@2": 0.923719691288589, + "pass@4": 0.9415772218948818, + "pass@8": 0.9541297662181599, + "pass@16": 0.9648180622646247, + "pass@32": 0.9740996623349433, + "pass@64": 0.9801861811374231, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8082473241590215, + "pass@2": 0.8621445255122921, + "pass@4": 0.8994472826791146, + "pass@8": 0.9270114252522809, + "pass@16": 0.9465701531750612, + "pass@32": 0.9611674288837075, + "pass@64": 0.9732524253679298, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5974874614197531, + "pass@2": 0.6732845460289684, + "pass@4": 0.7297426236072337, + "pass@8": 0.7771452699041174, + "pass@16": 0.8178942459211255, + "pass@32": 0.8521100811585546, + "pass@64": 0.879897410083746, + "pass@128": 0.8981481481481481 + }, + "14": { + "pass@1": 0.45252225519287836, + "pass@2": 0.5335529159559803, + "pass@4": 0.6004611306005145, + "pass@8": 0.6562780500283442, + "pass@16": 0.7023598202623946, + "pass@32": 0.7390769682434493, + "pass@64": 0.7694374359811129, + "pass@128": 0.798219584569733 + }, + "15": { + "pass@1": 0.29366801697530864, + "pass@2": 0.3638971669339944, + "pass@4": 0.42763749178343446, + "pass@8": 0.48427563508057453, + "pass@16": 0.534451744441426, + "pass@32": 0.5792335638578042, + "pass@64": 0.6187090353473133, + "pass@128": 0.654320987654321 + }, + "16": { + "pass@1": 0.20404682890855458, + "pass@2": 0.253627795972406, + "pass@4": 0.30170414815404706, + "pass@8": 0.34745819229398955, + "pass@16": 0.3904366019786063, + "pass@32": 0.43057834381205445, + "pass@64": 0.4704313191110292, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.18211125753012047, + "pass@2": 0.23127216049236302, + "pass@4": 0.27671165821892746, + "pass@8": 0.3184606778754773, + "pass@16": 0.3563930038046845, + "pass@32": 0.3931867515264538, + "pass@64": 0.43110471739026307, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.18402217741935484, + "pass@2": 0.24017811785623552, + "pass@4": 0.29430360469054273, + "pass@8": 0.3408792289278583, + "pass@16": 0.3782742791909101, + "pass@32": 0.41051421704477314, + "pass@64": 0.43918690774918834, + "pass@128": 0.4612903225806452 + }, + "19": { + "pass@1": 0.15525868580060423, + "pass@2": 0.20456532340557113, + "pass@4": 0.2538143409943849, + "pass@8": 0.301296925041148, + "pass@16": 0.3447051650683001, + "pass@32": 0.3817948403445567, + "pass@64": 0.41078464647666285, + "pass@128": 0.43202416918429004 + }, + "20": { + "pass@1": 0.14630255255255256, + "pass@2": 0.19446882906528568, + "pass@4": 0.24227324146418622, + "pass@8": 0.2883451930018625, + "pass@16": 0.3340379341883104, + "pass@32": 0.37754065399385844, + "pass@64": 0.41581311513220076, + "pass@128": 0.44744744744744747 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6755768914473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6755768914473684, + "pass@2": 0.7088923085888719, + "pass@4": 0.7363786917835755, + "pass@8": 0.7597616152234937, + "pass@16": 0.7800170101278652, + "pass@32": 0.7978934054769612, + "pass@64": 0.8138990150720641, + "pass@128": 0.8284736842105264 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9247734375, + "pass@2": 0.9439683809055122, + "pass@4": 0.9562506626359206, + "pass@8": 0.963894185093944, + "pass@16": 0.9690634148740488, + "pass@32": 0.9732302849431483, + "pass@64": 0.9765343228436548, + "pass@128": 0.979 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9986171875, + "pass@2": 0.9992669783464567, + "pass@4": 0.999595402137233, + "pass@8": 0.9998465879068722, + "pass@16": 0.9999796800606743, + "pass@32": 0.999999793590006, + "pass@64": 0.9999999999993952, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9988515625, + "pass@2": 0.9989880659448819, + "pass@4": 0.9989999060742407, + "pass@8": 0.9989999999978996, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9846328125, + "pass@2": 0.9880631151574807, + "pass@4": 0.9899134654105735, + "pass@8": 0.9911568007077315, + "pass@16": 0.9922402977900743, + "pass@32": 0.993272704498571, + "pass@64": 0.9941906591276778, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.9973984375, + "pass@2": 0.9981446850393701, + "pass@4": 0.9984959660667417, + "pass@8": 0.9988935782393594, + "pass@16": 0.9994010868705335, + "pass@32": 0.9998376651396068, + "pass@64": 0.9999931225037461, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9865078125, + "pass@2": 0.9903694635826774, + "pass@4": 0.9924604293213348, + "pass@8": 0.9943845859818515, + "pass@16": 0.9963773520655672, + "pass@32": 0.9978226010458, + "pass@64": 0.9984801674769502, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.979640625, + "pass@2": 0.9879799458661419, + "pass@4": 0.9915468292088483, + "pass@8": 0.9935623370467179, + "pass@16": 0.994786978981977, + "pass@32": 0.995685358786083, + "pass@64": 0.9963741882064558, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.9710546875, + "pass@2": 0.9814964320866143, + "pass@4": 0.9864927470941129, + "pass@8": 0.9899528797908489, + "pass@16": 0.9926017595149479, + "pass@32": 0.9945620001253924, + "pass@64": 0.995847862222904, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.896671875, + "pass@2": 0.9257250246062998, + "pass@4": 0.9424503491751033, + "pass@8": 0.9534902449246606, + "pass@16": 0.9617341647619115, + "pass@32": 0.9685526691343028, + "pass@64": 0.9739862456805812, + "pass@128": 0.979 + }, + "12": { + "pass@1": 0.80815625, + "pass@2": 0.8617167814960643, + "pass@4": 0.8952266491376072, + "pass@8": 0.9174555970412555, + "pass@16": 0.9328540335518769, + "pass@32": 0.9449988518049591, + "pass@64": 0.9558885648361812, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.611390625, + "pass@2": 0.6904826525590564, + "pass@4": 0.7472079101049863, + "pass@8": 0.7914406927782185, + "pass@16": 0.8288341851671736, + "pass@32": 0.861509523658841, + "pass@64": 0.8889878970233329, + "pass@128": 0.908 + }, + "14": { + "pass@1": 0.471796875, + "pass@2": 0.5570604084645676, + "pass@4": 0.6261572565616799, + "pass@8": 0.683053618225761, + "pass@16": 0.7289388120489303, + "pass@32": 0.7649940912758662, + "pass@64": 0.7953310389712002, + "pass@128": 0.821 + }, + "15": { + "pass@1": 0.318984375, + "pass@2": 0.39327522145669336, + "pass@4": 0.4584166613235846, + "pass@8": 0.5164938821750149, + "pass@16": 0.5686359339713938, + "pass@32": 0.6147548772017475, + "pass@64": 0.6561541463069708, + "pass@128": 0.696 + }, + "16": { + "pass@1": 0.22421875, + "pass@2": 0.28558993602362204, + "pass@4": 0.3436635868016498, + "pass@8": 0.3963355521106936, + "pass@16": 0.4432359851509301, + "pass@32": 0.4863019464805994, + "pass@64": 0.527861398191278, + "pass@128": 0.569 + }, + "17": { + "pass@1": 0.1804296875, + "pass@2": 0.2315994094488188, + "pass@4": 0.28011776031121094, + "pass@8": 0.32552405623661257, + "pass@16": 0.3681552857648949, + "pass@32": 0.40843428995183584, + "pass@64": 0.4482001769556677, + "pass@128": 0.492 + }, + "18": { + "pass@1": 0.1810703125, + "pass@2": 0.23492507381889757, + "pass@4": 0.2869294929696286, + "pass@8": 0.3331809278116836, + "pass@16": 0.3739857580462215, + "pass@32": 0.4127878310891122, + "pass@64": 0.44893216254517043, + "pass@128": 0.479 + }, + "19": { + "pass@1": 0.15225, + "pass@2": 0.20230487204724387, + "pass@4": 0.25132070678665164, + "pass@8": 0.296715079215693, + "pass@16": 0.3375134586858961, + "pass@32": 0.3732706964877968, + "pass@64": 0.4044542672025891, + "pass@128": 0.434 + }, + "20": { + "pass@1": 0.149515625, + "pass@2": 0.1979974163385825, + "pass@4": 0.24594936276715385, + "pass@8": 0.29109008396138786, + "pass@16": 0.33298500512216034, + "pass@32": 0.3709595188483823, + "pass@64": 0.40386506627542906, + "pass@128": 0.43 + } + }, + "per_op_accuracy": { + "10": 0.9247734375, + "2": 1.0, + "3": 0.9986171875, + "4": 0.9988515625, + "5": 0.9846328125, + "6": 0.9973984375, + "7": 0.9865078125, + "8": 0.979640625, + "9": 0.9710546875, + "11": 0.896671875, + "12": 0.80815625, + "13": 0.611390625, + "14": 0.471796875, + "15": 0.318984375, + "16": 0.22421875, + "17": 0.1804296875, + "18": 0.1810703125, + "19": 0.15225, + "20": 0.149515625 + }, + "per_op_avg_response_len": { + "10": 275.375984375, + "2": 114.54525, + "3": 142.971828125, + "4": 158.007921875, + "5": 170.8766796875, + "6": 196.06684375, + "7": 220.5956171875, + "8": 239.678515625, + "9": 255.344484375, + "11": 291.5578359375, + "12": 299.696359375, + "13": 310.6432578125, + "14": 318.5055390625, + "15": 319.774390625, + "16": 317.105671875, + "17": 323.4047265625, + "18": 323.4605390625, + "19": 319.3578984375, + "20": 322.6181484375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 275.375984375, + "2": 114.54525, + "3": 142.971828125, + "4": 158.007921875, + "5": 170.8766796875, + "6": 196.06684375, + "7": 220.5956171875, + "8": 239.678515625, + "9": 255.344484375, + "11": 291.5578359375, + "12": 299.696359375, + "13": 310.6432578125, + "14": 318.5055390625, + "15": 319.774390625, + "16": 317.105671875, + "17": 323.4047265625, + "18": 323.4605390625, + "19": 319.3578984375, + "20": 322.6181484375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 546547, + "answer_accuracy": 0.674976041337338, + "avg_response_len": 247.53832521538098, + "resp_tokens_sum": 200438713, + "pass_at_k": { + "pass@1": 0.674976041337338, + "pass@2": 0.7101091864658026, + "pass@4": 0.7386124625565469, + "pass@8": 0.762491115602194, + "pass@16": 0.7830723815661093, + "pass@32": 0.8011891747753505, + "pass@64": 0.817504962251469, + "pass@128": 0.8321214037306355 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9228890117994101, + "pass@2": 0.9435989507119136, + "pass@4": 0.9556432768912737, + "pass@8": 0.9621737909507128, + "pass@16": 0.9668383416955398, + "pass@32": 0.9704871371383337, + "pass@64": 0.972724070641377, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974675076452599, + "pass@2": 0.9980608430253559, + "pass@4": 0.9987818108172257, + "pass@8": 0.9995309324501443, + "pass@16": 0.999937859513187, + "pass@32": 0.9999993687767768, + "pass@64": 0.9999999999981503, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99677734375, + "pass@2": 0.9968726931594489, + "pass@4": 0.9968749997070677, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9848958333333333, + "pass@2": 0.9886990688663916, + "pass@4": 0.9909946988769263, + "pass@8": 0.9925728488812268, + "pass@16": 0.9940615103834403, + "pass@32": 0.9956209222387189, + "pass@64": 0.9966317339943471, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9984562125748503, + "pass@2": 0.9996393777405818, + "pass@4": 0.9999618897750056, + "pass@8": 0.9999996288482736, + "pass@16": 0.9999999999866336, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9819018121301775, + "pass@2": 0.9881998963332246, + "pass@4": 0.9908525150147651, + "pass@8": 0.992921263269219, + "pass@16": 0.9953543225386476, + "pass@32": 0.9973493282786543, + "pass@64": 0.9985121256129645, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9764372996794872, + "pass@2": 0.9853718705834849, + "pass@4": 0.9896878509176735, + "pass@8": 0.9922069966524123, + "pass@16": 0.99332484085682, + "pass@32": 0.9935781371491155, + "pass@64": 0.9935897405457036, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9763204225352112, + "pass@2": 0.9888221553731841, + "pass@4": 0.9937658911474093, + "pass@8": 0.9966332180416213, + "pass@16": 0.9987499819877586, + "pass@32": 0.9998218884821831, + "pass@64": 0.9999980823812246, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8874601628895185, + "pass@2": 0.9190749007383285, + "pass@4": 0.9379697790963095, + "pass@8": 0.9499868038901691, + "pass@16": 0.958601731222962, + "pass@32": 0.9659637243092551, + "pass@64": 0.9738305540462164, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.797801906779661, + "pass@2": 0.8524762000088971, + "pass@4": 0.8844551449077335, + "pass@8": 0.9041495170093695, + "pass@16": 0.9175524746444642, + "pass@32": 0.9286310489660135, + "pass@64": 0.9398662723540929, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.6121424373040752, + "pass@2": 0.6958881254165329, + "pass@4": 0.7590125529959217, + "pass@8": 0.8064926681604645, + "pass@16": 0.8440489366042815, + "pass@32": 0.8767117774857617, + "pass@64": 0.9045546474803343, + "pass@128": 0.9247648902821317 + }, + "14": { + "pass@1": 0.4789476246105919, + "pass@2": 0.5674093782961706, + "pass@4": 0.6386814332203795, + "pass@8": 0.699237479706666, + "pass@16": 0.749983464432357, + "pass@32": 0.7912858007134622, + "pass@64": 0.8267369070020342, + "pass@128": 0.8535825545171339 + }, + "15": { + "pass@1": 0.3295926661849711, + "pass@2": 0.4034149418551725, + "pass@4": 0.4654226851520722, + "pass@8": 0.5206955188961653, + "pass@16": 0.5724889979926816, + "pass@32": 0.6207619416448199, + "pass@64": 0.6668436877333662, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2501260080645161, + "pass@2": 0.3187746062992125, + "pass@4": 0.3809942934552536, + "pass@8": 0.43700423620641776, + "pass@16": 0.48804601495912603, + "pass@32": 0.5356938779931577, + "pass@64": 0.5791105338215622, + "pass@128": 0.6129032258064516 + }, + "17": { + "pass@1": 0.17241690751445088, + "pass@2": 0.2274190979017795, + "pass@4": 0.27921864887915004, + "pass@8": 0.32568698302405064, + "pass@16": 0.3676062818519709, + "pass@32": 0.4061846663090747, + "pass@64": 0.44490948479031045, + "pass@128": 0.4884393063583815 + }, + "18": { + "pass@1": 0.17680803571428572, + "pass@2": 0.2299905089988751, + "pass@4": 0.2822858747120897, + "pass@8": 0.33053922417649595, + "pass@16": 0.3754431840988442, + "pass@32": 0.4195090787833997, + "pass@64": 0.45981741367360784, + "pass@128": 0.49142857142857144 + }, + "19": { + "pass@1": 0.15892454954954954, + "pass@2": 0.21205235747558576, + "pass@4": 0.2627885883633915, + "pass@8": 0.30732869937392576, + "pass@16": 0.34537864261948176, + "pass@32": 0.37712504930713686, + "pass@64": 0.40499634950234564, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.16005067567567569, + "pass@2": 0.2089211504811898, + "pass@4": 0.2578512809772651, + "pass@8": 0.30439842135885803, + "pass@16": 0.34464130107609786, + "pass@32": 0.37680522912423126, + "pass@64": 0.4022415923768691, + "pass@128": 0.42042042042042044 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 557409, + "answer_accuracy": 0.6788398772408418, + "avg_response_len": 256.3155385327358, + "resp_tokens_sum": 210465815, + "pass_at_k": { + "pass@1": 0.6788398772408418, + "pass@2": 0.7117769813920345, + "pass@4": 0.7388926999284842, + "pass@8": 0.7618370788125385, + "pass@16": 0.7817088024003805, + "pass@32": 0.7994336680502945, + "pass@64": 0.8155949686648367, + "pass@128": 0.8308651597817615 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9202214340490797, + "pass@2": 0.9360723062895516, + "pass@4": 0.9465917125581695, + "pass@8": 0.9537278944651678, + "pass@16": 0.959031497936665, + "pass@32": 0.9641086020857509, + "pass@64": 0.9686631103780736, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985795454545454, + "pass@2": 0.9997170275590551, + "pass@4": 0.999981120257695, + "pass@8": 0.9999999181688074, + "pass@16": 0.9999999999995821, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991304347826087, + "pass@2": 0.9935520797672031, + "pass@4": 0.994298946055656, + "pass@8": 0.9945538934608948, + "pass@16": 0.9948847424451219, + "pass@32": 0.9954752938491385, + "pass@64": 0.996382517402716, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9971570558739254, + "pass@2": 0.9976052923989802, + "pass@4": 0.9980087450565815, + "pass@8": 0.9986333194949814, + "pass@16": 0.999382080743133, + "pass@32": 0.9998961597167226, + "pass@64": 0.9999991244302526, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9897940751445087, + "pass@2": 0.9925530386646032, + "pass@4": 0.9948359450010945, + "pass@8": 0.997232438770044, + "pass@16": 0.9991944808243975, + "pass@32": 0.9999485653009884, + "pass@64": 0.9999999670026358, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9779709507042254, + "pass@2": 0.9868096373516692, + "pass@4": 0.9909287493992825, + "pass@8": 0.9933035854016835, + "pass@16": 0.9949393747728471, + "pass@32": 0.9965682470179622, + "pass@64": 0.9982458732971077, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9639549026946108, + "pass@2": 0.9735508068508654, + "pass@4": 0.9791762661403849, + "pass@8": 0.9836002268475236, + "pass@16": 0.9872007390250478, + "pass@32": 0.990516785177483, + "pass@64": 0.9936008151224044, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9093561746987951, + "pass@2": 0.9346984394269996, + "pass@4": 0.9480427477627043, + "pass@8": 0.9566085144358756, + "pass@16": 0.9621387470688224, + "pass@32": 0.9660424121012906, + "pass@64": 0.9682693163975254, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.8195532915360502, + "pass@2": 0.8715327487473156, + "pass@4": 0.9028534996996849, + "pass@8": 0.9224261190045239, + "pass@16": 0.9357743493401, + "pass@32": 0.9465884364452, + "pass@64": 0.9558694712461404, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.6233368347338936, + "pass@2": 0.7012609177529281, + "pass@4": 0.752510632074352, + "pass@8": 0.7909648856753403, + "pass@16": 0.825167615439675, + "pass@32": 0.856456023376782, + "pass@64": 0.8833283014284938, + "pass@128": 0.9019607843137255 + }, + "14": { + "pass@1": 0.484078033625731, + "pass@2": 0.5705107174103236, + "pass@4": 0.6397225598993105, + "pass@8": 0.6942476151472764, + "pass@16": 0.735376785198006, + "pass@32": 0.7658550027742763, + "pass@64": 0.7913686429763481, + "pass@128": 0.8128654970760234 + }, + "15": { + "pass@1": 0.332717803030303, + "pass@2": 0.4114877863278452, + "pass@4": 0.4812904997670743, + "pass@8": 0.5437209905176837, + "pass@16": 0.5981587135360744, + "pass@32": 0.6433320324324602, + "pass@64": 0.6817106148445318, + "pass@128": 0.7181818181818181 + }, + "16": { + "pass@1": 0.22081997863247863, + "pass@2": 0.2871509550889474, + "pass@4": 0.3512183749467215, + "pass@8": 0.4076236800542498, + "pass@16": 0.4546544514041406, + "pass@32": 0.4964979651009504, + "pass@64": 0.5380652864044299, + "pass@128": 0.5811965811965812 + }, + "17": { + "pass@1": 0.18730590062111802, + "pass@2": 0.23642870897931226, + "pass@4": 0.2845957679203141, + "pass@8": 0.3326317237752575, + "pass@16": 0.38087277944396986, + "pass@32": 0.42657265186991555, + "pass@64": 0.46936251256103506, + "pass@128": 0.5186335403726708 + }, + "18": { + "pass@1": 0.18276654411764706, + "pass@2": 0.2352152327466419, + "pass@4": 0.2849862334303801, + "pass@8": 0.3288812893596294, + "pass@16": 0.3685753443013062, + "pass@32": 0.40794190067953773, + "pass@64": 0.4466121363445853, + "pass@128": 0.4823529411764706 + }, + "19": { + "pass@1": 0.14267113095238096, + "pass@2": 0.19041760404949387, + "pass@4": 0.2374986904538717, + "pass@8": 0.28168255992725094, + "pass@16": 0.3226338126666716, + "pass@32": 0.3610534610549758, + "pass@64": 0.39768084772182344, + "pass@128": 0.43452380952380953 + }, + "20": { + "pass@1": 0.14221556886227546, + "pass@2": 0.19062441062756372, + "pass@4": 0.2377481939944634, + "pass@8": 0.280558264608587, + "pass@16": 0.32031383167398947, + "pass@32": 0.35856987955107245, + "pass@64": 0.3935714032182891, + "pass@128": 0.4221556886227545 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 539047, + "answer_accuracy": 0.672839860600735, + "avg_response_len": 273.110060263221, + "resp_tokens_sum": 218802671, + "pass_at_k": { + "pass@1": 0.672839860600735, + "pass@2": 0.7047058338354943, + "pass@4": 0.7315443418620015, + "pass@8": 0.7548757040045201, + "pass@16": 0.7751949735171257, + "pass@32": 0.7929837040886027, + "pass@64": 0.8085162439976868, + "pass@128": 0.8223358363955904 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9311100746268657, + "pass@2": 0.9520261634739686, + "pass@4": 0.9662647566442252, + "pass@8": 0.9755282876597258, + "pass@16": 0.9810774588415156, + "pass@32": 0.9848828094725318, + "pass@64": 0.9880498176506736, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997494533527697, + "pass@2": 0.9999838588416243, + "pass@4": 0.9999999426092147, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996801900584795, + "pass@2": 0.9999672635492931, + "pass@4": 0.9999997256373656, + "pass@8": 0.9999999999938582, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9776194852941177, + "pass@2": 0.9819042380731822, + "pass@4": 0.9844617613974724, + "pass@8": 0.9863978237239305, + "pass@16": 0.9878696643403634, + "pass@32": 0.988862169457123, + "pass@64": 0.9897049835750625, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9965496845425867, + "pass@2": 0.9971636776869769, + "pass@4": 0.9974878323569174, + "pass@8": 0.99801474353403, + "pass@16": 0.9987909801124433, + "pass@32": 0.9996022252317682, + "pass@64": 0.9999792683835579, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9878362341772152, + "pass@2": 0.9902992001395396, + "pass@4": 0.9915792477087515, + "pass@8": 0.9928315670329757, + "pass@16": 0.9943870274755153, + "pass@32": 0.9960010268780787, + "pass@64": 0.9967819001166339, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9844219219219219, + "pass@2": 0.9916711741268824, + "pass@4": 0.9939474885459137, + "pass@8": 0.9951080518725748, + "pass@16": 0.9959944462170828, + "pass@32": 0.9967184753879357, + "pass@64": 0.9969877001072768, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9726688102893891, + "pass@2": 0.9816675158872827, + "pass@4": 0.9860481763533578, + "pass@8": 0.9891498765884253, + "pass@16": 0.9913841449354583, + "pass@32": 0.9929023262538196, + "pass@64": 0.9935237000858083, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.893625992063492, + "pass@2": 0.923719691288589, + "pass@4": 0.9415772218948818, + "pass@8": 0.9541297662181599, + "pass@16": 0.9648180622646247, + "pass@32": 0.9740996623349433, + "pass@64": 0.9801861811374231, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8082473241590215, + "pass@2": 0.8621445255122921, + "pass@4": 0.8994472826791146, + "pass@8": 0.9270114252522809, + "pass@16": 0.9465701531750612, + "pass@32": 0.9611674288837075, + "pass@64": 0.9732524253679298, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5974874614197531, + "pass@2": 0.6732845460289684, + "pass@4": 0.7297426236072337, + "pass@8": 0.7771452699041174, + "pass@16": 0.8178942459211255, + "pass@32": 0.8521100811585546, + "pass@64": 0.879897410083746, + "pass@128": 0.8981481481481481 + }, + "14": { + "pass@1": 0.45252225519287836, + "pass@2": 0.5335529159559803, + "pass@4": 0.6004611306005145, + "pass@8": 0.6562780500283442, + "pass@16": 0.7023598202623946, + "pass@32": 0.7390769682434493, + "pass@64": 0.7694374359811129, + "pass@128": 0.798219584569733 + }, + "15": { + "pass@1": 0.29366801697530864, + "pass@2": 0.3638971669339944, + "pass@4": 0.42763749178343446, + "pass@8": 0.48427563508057453, + "pass@16": 0.534451744441426, + "pass@32": 0.5792335638578042, + "pass@64": 0.6187090353473133, + "pass@128": 0.654320987654321 + }, + "16": { + "pass@1": 0.20404682890855458, + "pass@2": 0.253627795972406, + "pass@4": 0.30170414815404706, + "pass@8": 0.34745819229398955, + "pass@16": 0.3904366019786063, + "pass@32": 0.43057834381205445, + "pass@64": 0.4704313191110292, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.18211125753012047, + "pass@2": 0.23127216049236302, + "pass@4": 0.27671165821892746, + "pass@8": 0.3184606778754773, + "pass@16": 0.3563930038046845, + "pass@32": 0.3931867515264538, + "pass@64": 0.43110471739026307, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.18402217741935484, + "pass@2": 0.24017811785623552, + "pass@4": 0.29430360469054273, + "pass@8": 0.3408792289278583, + "pass@16": 0.3782742791909101, + "pass@32": 0.41051421704477314, + "pass@64": 0.43918690774918834, + "pass@128": 0.4612903225806452 + }, + "19": { + "pass@1": 0.15525868580060423, + "pass@2": 0.20456532340557113, + "pass@4": 0.2538143409943849, + "pass@8": 0.301296925041148, + "pass@16": 0.3447051650683001, + "pass@32": 0.3817948403445567, + "pass@64": 0.41078464647666285, + "pass@128": 0.43202416918429004 + }, + "20": { + "pass@1": 0.14630255255255256, + "pass@2": 0.19446882906528568, + "pass@4": 0.24227324146418622, + "pass@8": 0.2883451930018625, + "pass@16": 0.3340379341883104, + "pass@32": 0.37754065399385844, + "pass@64": 0.41581311513220076, + "pass@128": 0.44744744744744747 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/summary.csv b/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..56babee5a688d7dd98653ed0a659bbbddb52622b --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21350,0.9823862847222222,0.3994484375,0.6755768914473684,0.0,0.0,0.0,197.05145833333333,314.61243671875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/summary.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4e6a866b44a0d94ab3e1bc537880f8f987c14721 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block4/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21350", + "id_acc": 0.9823862847222222, + "ood_acc": 0.3994484375, + "total_acc": 0.6755768914473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.05145833333333, + "ood_avg_resp_len": 314.61243671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 546547, + "answer_accuracy": 0.674976041337338, + "avg_response_len": 247.53832521538098, + "resp_tokens_sum": 200438713, + "pass_at_k": { + "pass@1": 0.674976041337338, + "pass@2": 0.7101091864658026, + "pass@4": 0.7386124625565469, + "pass@8": 0.762491115602194, + "pass@16": 0.7830723815661093, + "pass@32": 0.8011891747753505, + "pass@64": 0.817504962251469, + "pass@128": 0.8321214037306355 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9228890117994101, + "pass@2": 0.9435989507119136, + "pass@4": 0.9556432768912737, + "pass@8": 0.9621737909507128, + "pass@16": 0.9668383416955398, + "pass@32": 0.9704871371383337, + "pass@64": 0.972724070641377, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9974675076452599, + "pass@2": 0.9980608430253559, + "pass@4": 0.9987818108172257, + "pass@8": 0.9995309324501443, + "pass@16": 0.999937859513187, + "pass@32": 0.9999993687767768, + "pass@64": 0.9999999999981503, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99677734375, + "pass@2": 0.9968726931594489, + "pass@4": 0.9968749997070677, + "pass@8": 0.996875, + "pass@16": 0.996875, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9848958333333333, + "pass@2": 0.9886990688663916, + "pass@4": 0.9909946988769263, + "pass@8": 0.9925728488812268, + "pass@16": 0.9940615103834403, + "pass@32": 0.9956209222387189, + "pass@64": 0.9966317339943471, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9984562125748503, + "pass@2": 0.9996393777405818, + "pass@4": 0.9999618897750056, + "pass@8": 0.9999996288482736, + "pass@16": 0.9999999999866336, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9819018121301775, + "pass@2": 0.9881998963332246, + "pass@4": 0.9908525150147651, + "pass@8": 0.992921263269219, + "pass@16": 0.9953543225386476, + "pass@32": 0.9973493282786543, + "pass@64": 0.9985121256129645, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9764372996794872, + "pass@2": 0.9853718705834849, + "pass@4": 0.9896878509176735, + "pass@8": 0.9922069966524123, + "pass@16": 0.99332484085682, + "pass@32": 0.9935781371491155, + "pass@64": 0.9935897405457036, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9763204225352112, + "pass@2": 0.9888221553731841, + "pass@4": 0.9937658911474093, + "pass@8": 0.9966332180416213, + "pass@16": 0.9987499819877586, + "pass@32": 0.9998218884821831, + "pass@64": 0.9999980823812246, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8874601628895185, + "pass@2": 0.9190749007383285, + "pass@4": 0.9379697790963095, + "pass@8": 0.9499868038901691, + "pass@16": 0.958601731222962, + "pass@32": 0.9659637243092551, + "pass@64": 0.9738305540462164, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.797801906779661, + "pass@2": 0.8524762000088971, + "pass@4": 0.8844551449077335, + "pass@8": 0.9041495170093695, + "pass@16": 0.9175524746444642, + "pass@32": 0.9286310489660135, + "pass@64": 0.9398662723540929, + "pass@128": 0.9519774011299436 + }, + "13": { + "pass@1": 0.6121424373040752, + "pass@2": 0.6958881254165329, + "pass@4": 0.7590125529959217, + "pass@8": 0.8064926681604645, + "pass@16": 0.8440489366042815, + "pass@32": 0.8767117774857617, + "pass@64": 0.9045546474803343, + "pass@128": 0.9247648902821317 + }, + "14": { + "pass@1": 0.4789476246105919, + "pass@2": 0.5674093782961706, + "pass@4": 0.6386814332203795, + "pass@8": 0.699237479706666, + "pass@16": 0.749983464432357, + "pass@32": 0.7912858007134622, + "pass@64": 0.8267369070020342, + "pass@128": 0.8535825545171339 + }, + "15": { + "pass@1": 0.3295926661849711, + "pass@2": 0.4034149418551725, + "pass@4": 0.4654226851520722, + "pass@8": 0.5206955188961653, + "pass@16": 0.5724889979926816, + "pass@32": 0.6207619416448199, + "pass@64": 0.6668436877333662, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.2501260080645161, + "pass@2": 0.3187746062992125, + "pass@4": 0.3809942934552536, + "pass@8": 0.43700423620641776, + "pass@16": 0.48804601495912603, + "pass@32": 0.5356938779931577, + "pass@64": 0.5791105338215622, + "pass@128": 0.6129032258064516 + }, + "17": { + "pass@1": 0.17241690751445088, + "pass@2": 0.2274190979017795, + "pass@4": 0.27921864887915004, + "pass@8": 0.32568698302405064, + "pass@16": 0.3676062818519709, + "pass@32": 0.4061846663090747, + "pass@64": 0.44490948479031045, + "pass@128": 0.4884393063583815 + }, + "18": { + "pass@1": 0.17680803571428572, + "pass@2": 0.2299905089988751, + "pass@4": 0.2822858747120897, + "pass@8": 0.33053922417649595, + "pass@16": 0.3754431840988442, + "pass@32": 0.4195090787833997, + "pass@64": 0.45981741367360784, + "pass@128": 0.49142857142857144 + }, + "19": { + "pass@1": 0.15892454954954954, + "pass@2": 0.21205235747558576, + "pass@4": 0.2627885883633915, + "pass@8": 0.30732869937392576, + "pass@16": 0.34537864261948176, + "pass@32": 0.37712504930713686, + "pass@64": 0.40499634950234564, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.16005067567567569, + "pass@2": 0.2089211504811898, + "pass@4": 0.2578512809772651, + "pass@8": 0.30439842135885803, + "pass@16": 0.34464130107609786, + "pass@32": 0.37680522912423126, + "pass@64": 0.4022415923768691, + "pass@128": 0.42042042042042044 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 557409, + "answer_accuracy": 0.6788398772408418, + "avg_response_len": 256.3155385327358, + "resp_tokens_sum": 210465815, + "pass_at_k": { + "pass@1": 0.6788398772408418, + "pass@2": 0.7117769813920345, + "pass@4": 0.7388926999284842, + "pass@8": 0.7618370788125385, + "pass@16": 0.7817088024003805, + "pass@32": 0.7994336680502945, + "pass@64": 0.8155949686648367, + "pass@128": 0.8308651597817615 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9202214340490797, + "pass@2": 0.9360723062895516, + "pass@4": 0.9465917125581695, + "pass@8": 0.9537278944651678, + "pass@16": 0.959031497936665, + "pass@32": 0.9641086020857509, + "pass@64": 0.9686631103780736, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9985795454545454, + "pass@2": 0.9997170275590551, + "pass@4": 0.999981120257695, + "pass@8": 0.9999999181688074, + "pass@16": 0.9999999999995821, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991304347826087, + "pass@2": 0.9935520797672031, + "pass@4": 0.994298946055656, + "pass@8": 0.9945538934608948, + "pass@16": 0.9948847424451219, + "pass@32": 0.9954752938491385, + "pass@64": 0.996382517402716, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9971570558739254, + "pass@2": 0.9976052923989802, + "pass@4": 0.9980087450565815, + "pass@8": 0.9986333194949814, + "pass@16": 0.999382080743133, + "pass@32": 0.9998961597167226, + "pass@64": 0.9999991244302526, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9897940751445087, + "pass@2": 0.9925530386646032, + "pass@4": 0.9948359450010945, + "pass@8": 0.997232438770044, + "pass@16": 0.9991944808243975, + "pass@32": 0.9999485653009884, + "pass@64": 0.9999999670026358, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9779709507042254, + "pass@2": 0.9868096373516692, + "pass@4": 0.9909287493992825, + "pass@8": 0.9933035854016835, + "pass@16": 0.9949393747728471, + "pass@32": 0.9965682470179622, + "pass@64": 0.9982458732971077, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9639549026946108, + "pass@2": 0.9735508068508654, + "pass@4": 0.9791762661403849, + "pass@8": 0.9836002268475236, + "pass@16": 0.9872007390250478, + "pass@32": 0.990516785177483, + "pass@64": 0.9936008151224044, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.9093561746987951, + "pass@2": 0.9346984394269996, + "pass@4": 0.9480427477627043, + "pass@8": 0.9566085144358756, + "pass@16": 0.9621387470688224, + "pass@32": 0.9660424121012906, + "pass@64": 0.9682693163975254, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.8195532915360502, + "pass@2": 0.8715327487473156, + "pass@4": 0.9028534996996849, + "pass@8": 0.9224261190045239, + "pass@16": 0.9357743493401, + "pass@32": 0.9465884364452, + "pass@64": 0.9558694712461404, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.6233368347338936, + "pass@2": 0.7012609177529281, + "pass@4": 0.752510632074352, + "pass@8": 0.7909648856753403, + "pass@16": 0.825167615439675, + "pass@32": 0.856456023376782, + "pass@64": 0.8833283014284938, + "pass@128": 0.9019607843137255 + }, + "14": { + "pass@1": 0.484078033625731, + "pass@2": 0.5705107174103236, + "pass@4": 0.6397225598993105, + "pass@8": 0.6942476151472764, + "pass@16": 0.735376785198006, + "pass@32": 0.7658550027742763, + "pass@64": 0.7913686429763481, + "pass@128": 0.8128654970760234 + }, + "15": { + "pass@1": 0.332717803030303, + "pass@2": 0.4114877863278452, + "pass@4": 0.4812904997670743, + "pass@8": 0.5437209905176837, + "pass@16": 0.5981587135360744, + "pass@32": 0.6433320324324602, + "pass@64": 0.6817106148445318, + "pass@128": 0.7181818181818181 + }, + "16": { + "pass@1": 0.22081997863247863, + "pass@2": 0.2871509550889474, + "pass@4": 0.3512183749467215, + "pass@8": 0.4076236800542498, + "pass@16": 0.4546544514041406, + "pass@32": 0.4964979651009504, + "pass@64": 0.5380652864044299, + "pass@128": 0.5811965811965812 + }, + "17": { + "pass@1": 0.18730590062111802, + "pass@2": 0.23642870897931226, + "pass@4": 0.2845957679203141, + "pass@8": 0.3326317237752575, + "pass@16": 0.38087277944396986, + "pass@32": 0.42657265186991555, + "pass@64": 0.46936251256103506, + "pass@128": 0.5186335403726708 + }, + "18": { + "pass@1": 0.18276654411764706, + "pass@2": 0.2352152327466419, + "pass@4": 0.2849862334303801, + "pass@8": 0.3288812893596294, + "pass@16": 0.3685753443013062, + "pass@32": 0.40794190067953773, + "pass@64": 0.4466121363445853, + "pass@128": 0.4823529411764706 + }, + "19": { + "pass@1": 0.14267113095238096, + "pass@2": 0.19041760404949387, + "pass@4": 0.2374986904538717, + "pass@8": 0.28168255992725094, + "pass@16": 0.3226338126666716, + "pass@32": 0.3610534610549758, + "pass@64": 0.39768084772182344, + "pass@128": 0.43452380952380953 + }, + "20": { + "pass@1": 0.14221556886227546, + "pass@2": 0.19062441062756372, + "pass@4": 0.2377481939944634, + "pass@8": 0.280558264608587, + "pass@16": 0.32031383167398947, + "pass@32": 0.35856987955107245, + "pass@64": 0.3935714032182891, + "pass@128": 0.4221556886227545 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 539047, + "answer_accuracy": 0.672839860600735, + "avg_response_len": 273.110060263221, + "resp_tokens_sum": 218802671, + "pass_at_k": { + "pass@1": 0.672839860600735, + "pass@2": 0.7047058338354943, + "pass@4": 0.7315443418620015, + "pass@8": 0.7548757040045201, + "pass@16": 0.7751949735171257, + "pass@32": 0.7929837040886027, + "pass@64": 0.8085162439976868, + "pass@128": 0.8223358363955904 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9311100746268657, + "pass@2": 0.9520261634739686, + "pass@4": 0.9662647566442252, + "pass@8": 0.9755282876597258, + "pass@16": 0.9810774588415156, + "pass@32": 0.9848828094725318, + "pass@64": 0.9880498176506736, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997494533527697, + "pass@2": 0.9999838588416243, + "pass@4": 0.9999999426092147, + "pass@8": 0.9999999999999082, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996801900584795, + "pass@2": 0.9999672635492931, + "pass@4": 0.9999997256373656, + "pass@8": 0.9999999999938582, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9776194852941177, + "pass@2": 0.9819042380731822, + "pass@4": 0.9844617613974724, + "pass@8": 0.9863978237239305, + "pass@16": 0.9878696643403634, + "pass@32": 0.988862169457123, + "pass@64": 0.9897049835750625, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9965496845425867, + "pass@2": 0.9971636776869769, + "pass@4": 0.9974878323569174, + "pass@8": 0.99801474353403, + "pass@16": 0.9987909801124433, + "pass@32": 0.9996022252317682, + "pass@64": 0.9999792683835579, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9878362341772152, + "pass@2": 0.9902992001395396, + "pass@4": 0.9915792477087515, + "pass@8": 0.9928315670329757, + "pass@16": 0.9943870274755153, + "pass@32": 0.9960010268780787, + "pass@64": 0.9967819001166339, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9844219219219219, + "pass@2": 0.9916711741268824, + "pass@4": 0.9939474885459137, + "pass@8": 0.9951080518725748, + "pass@16": 0.9959944462170828, + "pass@32": 0.9967184753879357, + "pass@64": 0.9969877001072768, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.9726688102893891, + "pass@2": 0.9816675158872827, + "pass@4": 0.9860481763533578, + "pass@8": 0.9891498765884253, + "pass@16": 0.9913841449354583, + "pass@32": 0.9929023262538196, + "pass@64": 0.9935237000858083, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.893625992063492, + "pass@2": 0.923719691288589, + "pass@4": 0.9415772218948818, + "pass@8": 0.9541297662181599, + "pass@16": 0.9648180622646247, + "pass@32": 0.9740996623349433, + "pass@64": 0.9801861811374231, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8082473241590215, + "pass@2": 0.8621445255122921, + "pass@4": 0.8994472826791146, + "pass@8": 0.9270114252522809, + "pass@16": 0.9465701531750612, + "pass@32": 0.9611674288837075, + "pass@64": 0.9732524253679298, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5974874614197531, + "pass@2": 0.6732845460289684, + "pass@4": 0.7297426236072337, + "pass@8": 0.7771452699041174, + "pass@16": 0.8178942459211255, + "pass@32": 0.8521100811585546, + "pass@64": 0.879897410083746, + "pass@128": 0.8981481481481481 + }, + "14": { + "pass@1": 0.45252225519287836, + "pass@2": 0.5335529159559803, + "pass@4": 0.6004611306005145, + "pass@8": 0.6562780500283442, + "pass@16": 0.7023598202623946, + "pass@32": 0.7390769682434493, + "pass@64": 0.7694374359811129, + "pass@128": 0.798219584569733 + }, + "15": { + "pass@1": 0.29366801697530864, + "pass@2": 0.3638971669339944, + "pass@4": 0.42763749178343446, + "pass@8": 0.48427563508057453, + "pass@16": 0.534451744441426, + "pass@32": 0.5792335638578042, + "pass@64": 0.6187090353473133, + "pass@128": 0.654320987654321 + }, + "16": { + "pass@1": 0.20404682890855458, + "pass@2": 0.253627795972406, + "pass@4": 0.30170414815404706, + "pass@8": 0.34745819229398955, + "pass@16": 0.3904366019786063, + "pass@32": 0.43057834381205445, + "pass@64": 0.4704313191110292, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.18211125753012047, + "pass@2": 0.23127216049236302, + "pass@4": 0.27671165821892746, + "pass@8": 0.3184606778754773, + "pass@16": 0.3563930038046845, + "pass@32": 0.3931867515264538, + "pass@64": 0.43110471739026307, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.18402217741935484, + "pass@2": 0.24017811785623552, + "pass@4": 0.29430360469054273, + "pass@8": 0.3408792289278583, + "pass@16": 0.3782742791909101, + "pass@32": 0.41051421704477314, + "pass@64": 0.43918690774918834, + "pass@128": 0.4612903225806452 + }, + "19": { + "pass@1": 0.15525868580060423, + "pass@2": 0.20456532340557113, + "pass@4": 0.2538143409943849, + "pass@8": 0.301296925041148, + "pass@16": 0.3447051650683001, + "pass@32": 0.3817948403445567, + "pass@64": 0.41078464647666285, + "pass@128": 0.43202416918429004 + }, + "20": { + "pass@1": 0.14630255255255256, + "pass@2": 0.19446882906528568, + "pass@4": 0.24227324146418622, + "pass@8": 0.2883451930018625, + "pass@16": 0.3340379341883104, + "pass@32": 0.37754065399385844, + "pass@64": 0.41581311513220076, + "pass@128": 0.44744744744744747 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/checkpoint-21350_metrics.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/checkpoint-21350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..29390c80029a52caada215a67f4f95dd6cd0c461 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/checkpoint-21350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21350", + "id": { + "answer_accuracy": 0.7816371527777778, + "avg_response_len": 196.91984982638888, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.7816371527777778, + "pass@2": 0.8681601733377071, + "pass@4": 0.9254738087009939, + "pass@8": 0.9569689165351943, + "pass@16": 0.973540318152665, + "pass@32": 0.9832439559317901, + "pass@64": 0.9893019488767522, + "pass@128": 0.9932222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6076640625, + "pass@2": 0.756191067913386, + "pass@4": 0.8603429181664787, + "pass@8": 0.9199172971920299, + "pass@16": 0.9515354728881059, + "pass@32": 0.9687067881816586, + "pass@64": 0.9787306082691241, + "pass@128": 0.987 + }, + "2": { + "pass@1": 0.9796953125, + "pass@2": 0.9826592027559057, + "pass@4": 0.9850796374203223, + "pass@8": 0.9875326505398578, + "pass@16": 0.9906439412196009, + "pass@32": 0.9940873178573064, + "pass@64": 0.9966575695273527, + "pass@128": 0.998 + }, + "3": { + "pass@1": 0.9749140625, + "pass@2": 0.9810580708661415, + "pass@4": 0.985971505718035, + "pass@8": 0.9898390963793305, + "pass@16": 0.9924167713774642, + "pass@32": 0.9942863589815633, + "pass@64": 0.9960664597579397, + "pass@128": 0.998 + }, + "4": { + "pass@1": 0.91559375, + "pass@2": 0.949925196850393, + "pass@4": 0.9694112275965503, + "pass@8": 0.9798026126919146, + "pass@16": 0.9855816778143771, + "pass@32": 0.9890364734039345, + "pass@64": 0.9914867091300696, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.7999453125, + "pass@2": 0.88533341535433, + "pass@4": 0.9336656406074244, + "pass@8": 0.9568312556526607, + "pass@16": 0.969011778418327, + "pass@32": 0.9767837698692393, + "pass@64": 0.9822871781103928, + "pass@128": 0.987 + }, + "6": { + "pass@1": 0.77846875, + "pass@2": 0.8735273129921275, + "pass@4": 0.9312027098800149, + "pass@8": 0.9614433644622998, + "pass@16": 0.9768078281288376, + "pass@32": 0.9860289444598231, + "pass@64": 0.9921008351872295, + "pass@128": 0.995 + }, + "7": { + "pass@1": 0.7226875, + "pass@2": 0.8443891486220463, + "pass@4": 0.9200709941882256, + "pass@8": 0.9567265116477753, + "pass@16": 0.9742473535227066, + "pass@32": 0.9843899238601288, + "pass@64": 0.9905993308577857, + "pass@128": 0.994 + }, + "8": { + "pass@1": 0.669109375, + "pass@2": 0.8043056102362194, + "pass@4": 0.8959773600487436, + "pass@8": 0.9454055296272024, + "pass@16": 0.9687693078025161, + "pass@32": 0.9815056127748554, + "pass@64": 0.9894645631281762, + "pass@128": 0.993 + }, + "9": { + "pass@1": 0.58665625, + "pass@2": 0.7360525344488175, + "pass@4": 0.8475422846831647, + "pass@8": 0.9152219306237759, + "pass@16": 0.9528487322020358, + "pass@32": 0.9743704139975835, + "pass@64": 0.9863242859226867, + "pass@128": 0.993 + } + }, + "per_op_accuracy": { + "10": 0.6076640625, + "2": 0.9796953125, + "3": 0.9749140625, + "4": 0.91559375, + "5": 0.7999453125, + "6": 0.77846875, + "7": 0.7226875, + "8": 0.669109375, + "9": 0.58665625 + }, + "per_op_avg_response_len": { + "10": 274.77615625, + "2": 114.3989375, + "3": 142.7864609375, + "4": 158.1644296875, + "5": 171.5203046875, + "6": 195.9111484375, + "7": 220.8932890625, + "8": 239.4595546875, + "9": 254.3683671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.77615625, + "2": 114.3989375, + "3": 142.7864609375, + "4": 158.1644296875, + "5": 171.5203046875, + "6": 195.9111484375, + "7": 220.8932890625, + "8": 239.4595546875, + "9": 254.3683671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 296172, + "answer_accuracy": 0.7814399696048632, + "avg_response_len": 187.40295191658223, + "resp_tokens_sum": 71027218, + "pass_at_k": { + "pass@1": 0.7814399696048632, + "pass@2": 0.8704980693902684, + "pass@4": 0.9291449392660818, + "pass@8": 0.9606268503492171, + "pass@16": 0.9765422448293186, + "pass@32": 0.9854953546865458, + "pass@64": 0.9910011067869215, + "pass@128": 0.994596420128335 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6108038348082596, + "pass@2": 0.7592836155436323, + "pass@4": 0.8628990543770518, + "pass@8": 0.9209853506323293, + "pass@16": 0.9502138983934935, + "pass@32": 0.9653789425226263, + "pass@64": 0.9738636888533403, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9687013239875389, + "pass@2": 0.9735359635244192, + "pass@4": 0.9774219598134347, + "pass@8": 0.9811756048228187, + "pass@16": 0.9857846921623187, + "pass@32": 0.9912202144423357, + "pass@64": 0.9954408818642898, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9708046636085627, + "pass@2": 0.9782207914950999, + "pass@4": 0.9856412343525864, + "pass@8": 0.9913135287729292, + "pass@16": 0.9943854355053758, + "pass@32": 0.9964695421928315, + "pass@64": 0.9982793427520417, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.933056640625, + "pass@2": 0.9640221302903547, + "pass@4": 0.9808446167861831, + "pass@8": 0.9889389221699499, + "pass@16": 0.9928154480539686, + "pass@32": 0.9944218943628333, + "pass@64": 0.9953122612788613, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.7997519841269841, + "pass@2": 0.8826568553930755, + "pass@4": 0.930272895352366, + "pass@8": 0.9548178801748886, + "pass@16": 0.9687459281478877, + "pass@32": 0.9774046788113472, + "pass@64": 0.983474310194418, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.7953078218562875, + "pass@2": 0.8896447999669947, + "pass@4": 0.944976489933518, + "pass@8": 0.973097387972258, + "pass@16": 0.9870541545418998, + "pass@32": 0.9947098289309901, + "pass@64": 0.9987991381053104, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.7229105029585798, + "pass@2": 0.8472880655080834, + "pass@4": 0.9232964889189142, + "pass@8": 0.9585954889402899, + "pass@16": 0.9758673121900198, + "pass@32": 0.987287560047621, + "pass@64": 0.9952321084717118, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6659405048076923, + "pass@2": 0.8077088696244699, + "pass@4": 0.9056807724755556, + "pass@8": 0.9572339732355738, + "pass@16": 0.9783949645645401, + "pass@32": 0.986614119977316, + "pass@64": 0.9907329976438498, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.5919014084507043, + "pass@2": 0.7484803010979263, + "pass@4": 0.8604035258796869, + "pass@8": 0.9244515130120431, + "pass@16": 0.958264149196147, + "pass@32": 0.9771934956760493, + "pass@64": 0.9883104989872438, + "pass@128": 0.9943661971830986 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 300368, + "answer_accuracy": 0.7688810615989515, + "avg_response_len": 195.37704015809305, + "resp_tokens_sum": 76325213, + "pass_at_k": { + "pass@1": 0.7688810615989515, + "pass@2": 0.8554261807798705, + "pass@4": 0.9153715333113704, + "pass@8": 0.9497989502405041, + "pass@16": 0.9680435044407102, + "pass@32": 0.9788862971619176, + "pass@64": 0.9862395830157321, + "pass@128": 0.991480996068152 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5747699386503068, + "pass@2": 0.7293609004395927, + "pass@4": 0.842726695972819, + "pass@8": 0.9087806167016811, + "pass@16": 0.9435410406326107, + "pass@32": 0.962740051480476, + "pass@64": 0.9756515561073661, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9805043693009119, + "pass@2": 0.9843484491300291, + "pass@4": 0.9870946238103215, + "pass@8": 0.9895161087507777, + "pass@16": 0.9925479984428305, + "pass@32": 0.9957528327388283, + "pass@64": 0.9980910863778228, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9767282196969697, + "pass@2": 0.9832457199952278, + "pass@4": 0.9873704749974436, + "pass@8": 0.9902525452585946, + "pass@16": 0.9920598769834958, + "pass@32": 0.9934207656439095, + "pass@64": 0.9950847041173666, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 0.890625, + "pass@2": 0.927779783814006, + "pass@4": 0.9517953019371097, + "pass@8": 0.9651948360744051, + "pass@16": 0.9715446905626869, + "pass@32": 0.9753629793049493, + "pass@64": 0.9796651628850817, + "pass@128": 0.985207100591716 + }, + "5": { + "pass@1": 0.8036005434782608, + "pass@2": 0.8880306401917152, + "pass@4": 0.9356044589534999, + "pass@8": 0.9566631468858801, + "pass@16": 0.9677299365458314, + "pass@32": 0.9762975400418303, + "pass@64": 0.9827398189807199, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.7473809097421203, + "pass@2": 0.8443866615526928, + "pass@4": 0.9105473168826673, + "pass@8": 0.9485723437526672, + "pass@16": 0.9680171909631179, + "pass@32": 0.9798515662831963, + "pass@64": 0.9887143445075527, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.7122245303468208, + "pass@2": 0.82878353852806, + "pass@4": 0.9054310449271872, + "pass@8": 0.9459152039364331, + "pass@16": 0.9658799168085859, + "pass@32": 0.9764867382386906, + "pass@64": 0.9835947173089279, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.6647887323943662, + "pass@2": 0.7976325967616721, + "pass@4": 0.8875285859514035, + "pass@8": 0.9386763687150315, + "pass@16": 0.9646758684012232, + "pass@32": 0.979982181086267, + "pass@64": 0.9892714095496256, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.5772595434131736, + "pass@2": 0.7188549819652033, + "pass@4": 0.8316630995601596, + "pass@8": 0.9051994481455924, + "pass@16": 0.9467216399752423, + "pass@32": 0.9702232540521561, + "pass@64": 0.9833604457495104, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 303906, + "answer_accuracy": 0.7948662956143288, + "avg_response_len": 207.930291680616, + "resp_tokens_sum": 79499236, + "pass_at_k": { + "pass@1": 0.7948662956143288, + "pass@2": 0.8788537237477939, + "pass@4": 0.9321567437147036, + "pass@8": 0.9606688144622707, + "pass@16": 0.9761809510818038, + "pass@32": 0.9854646398463527, + "pass@64": 0.9907465869871617, + "pass@128": 0.9936391027787077 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6364972014925373, + "pass@2": 0.779170951345635, + "pass@4": 0.8748992114791617, + "pass@8": 0.9296739769639495, + "pass@16": 0.9606524838402445, + "pass@32": 0.9778808056234413, + "pass@64": 0.9866519715129541, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9890178571428572, + "pass@2": 0.9894386248593927, + "pass@4": 0.9902087345331835, + "pass@8": 0.9914985188935059, + "pass@16": 0.9933107529937302, + "pass@32": 0.9951512772864062, + "pass@64": 0.9964259400874629, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9770863702623906, + "pass@2": 0.9816582636991802, + "pass@4": 0.9849404237189008, + "pass@8": 0.9880356633680653, + "pass@16": 0.9908833077628361, + "pass@32": 0.9930378018134619, + "pass@64": 0.994901347869653, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9239309210526315, + "pass@2": 0.9586214860477973, + "pass@4": 0.976123211024061, + "pass@8": 0.9856909444572529, + "pass@16": 0.992686049786314, + "pass@32": 0.9975110532244272, + "pass@64": 0.9995905276774154, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7964154411764706, + "pass@2": 0.8850762795275591, + "pass@4": 0.9348415889190315, + "pass@8": 0.9588671697704182, + "pass@16": 0.9705587733630365, + "pass@32": 0.9767018962624524, + "pass@64": 0.9807280377964777, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.7949526813880127, + "pass@2": 0.8886277757768444, + "pass@4": 0.9394307528396486, + "pass@8": 0.9633346653308688, + "pass@16": 0.9756900342767074, + "pass@32": 0.9836834856909692, + "pass@64": 0.9887716619145747, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.7339052610759493, + "pass@2": 0.858375563764577, + "pass@4": 0.9326507575595763, + "pass@8": 0.9665651133669365, + "pass@16": 0.9816764265402234, + "pass@32": 0.989944041561538, + "pass@64": 0.9933136265995508, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.676684496996997, + "pass@2": 0.8082308440330094, + "pass@4": 0.8958928258967639, + "pass@8": 0.9414967540056054, + "pass@16": 0.9641145512791177, + "pass@32": 0.9783433425114311, + "pass@64": 0.9884820344539859, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.5907606511254019, + "pass@2": 0.740335895764235, + "pass@4": 0.8499149766648948, + "pass@8": 0.9154502502375335, + "pass@16": 0.9532473681211369, + "pass@32": 0.9756017883574458, + "pass@64": 0.9872400961475196, + "pass@128": 0.9935691318327974 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.2690828125, + "avg_response_len": 310.56489765625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2690828125, + "pass@2": 0.36646231545276375, + "pass@4": 0.4582728507967779, + "pass@8": 0.5363188239957953, + "pass@16": 0.5998365913507534, + "pass@32": 0.6518350419923482, + "pass@64": 0.6950708203399962, + "pass@128": 0.7318 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5480859375, + "pass@2": 0.6981151574803157, + "pass@4": 0.811993447131609, + "pass@8": 0.8848340644892468, + "pass@16": 0.9271581231461244, + "pass@32": 0.952261953106656, + "pass@64": 0.9674929330726126, + "pass@128": 0.977 + }, + "12": { + "pass@1": 0.454984375, + "pass@2": 0.6140105807086608, + "pass@4": 0.7477736696662932, + "pass@8": 0.8419166090235274, + "pass@16": 0.9015706080771255, + "pass@32": 0.9376095239781851, + "pass@64": 0.9585292949447553, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.3558125, + "pass@2": 0.4849186761811016, + "pass@4": 0.6067286934758146, + "pass@8": 0.7085479191323254, + "pass@16": 0.7870774809041959, + "pass@32": 0.844301207277016, + "pass@64": 0.8846046024653631, + "pass@128": 0.913 + }, + "14": { + "pass@1": 0.286828125, + "pass@2": 0.39775233759842554, + "pass@4": 0.5078328182414698, + "pass@8": 0.6052508839410764, + "pass@16": 0.6860653986220863, + "pass@32": 0.7516146910750364, + "pass@64": 0.8037517223816558, + "pass@128": 0.845 + }, + "15": { + "pass@1": 0.21540625, + "pass@2": 0.3088097933070866, + "pass@4": 0.40559388863891943, + "pass@8": 0.4958354264948282, + "pass@16": 0.5750020189177119, + "pass@32": 0.6415148083776369, + "pass@64": 0.6943959361981233, + "pass@128": 0.736 + }, + "16": { + "pass@1": 0.1929765625, + "pass@2": 0.2658702017716534, + "pass@4": 0.3404780056242968, + "pass@8": 0.41142052836791654, + "pass@16": 0.47782751781461763, + "pass@32": 0.5408739161283275, + "pass@64": 0.5993847202323961, + "pass@128": 0.652 + }, + "17": { + "pass@1": 0.17515625, + "pass@2": 0.24412610728346473, + "pass@4": 0.31504707508436386, + "pass@8": 0.38360111671883745, + "pass@16": 0.4473836350458656, + "pass@32": 0.505540861947074, + "pass@64": 0.5591098731736599, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.1674921875, + "pass@2": 0.23586060531496067, + "pass@4": 0.3069344478815142, + "pass@8": 0.3743178318371625, + "pass@16": 0.4354929268152162, + "pass@32": 0.4908378367051541, + "pass@64": 0.5416621528832821, + "pass@128": 0.587 + }, + "19": { + "pass@1": 0.1531484375, + "pass@2": 0.2136903297244094, + "pass@4": 0.27431024484439437, + "pass@8": 0.32973523949412276, + "pass@16": 0.3796167540585565, + "pass@32": 0.42665328161645505, + "pass@64": 0.4712488579834778, + "pass@128": 0.511 + }, + "20": { + "pass@1": 0.1409375, + "pass@2": 0.20146936515748032, + "pass@4": 0.2660362173790774, + "pass@8": 0.3277286204588297, + "pass@16": 0.38117145010586473, + "pass@32": 0.4271423397119454, + "pass@64": 0.4705281100645885, + "pass@128": 0.515 + } + }, + "per_op_accuracy": { + "11": 0.5480859375, + "12": 0.454984375, + "13": 0.3558125, + "14": 0.286828125, + "15": 0.21540625, + "16": 0.1929765625, + "17": 0.17515625, + "18": 0.1674921875, + "19": 0.1531484375, + "20": 0.1409375 + }, + "per_op_avg_response_len": { + "11": 289.850421875, + "12": 297.486453125, + "13": 308.000375, + "14": 312.466140625, + "15": 313.4066796875, + "16": 313.8389921875, + "17": 319.033359375, + "18": 318.24709375, + "19": 314.953140625, + "20": 318.3663203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 289.850421875, + "12": 297.486453125, + "13": 308.000375, + "14": 312.466140625, + "15": 313.4066796875, + "16": 313.8389921875, + "17": 319.033359375, + "18": 318.24709375, + "19": 314.953140625, + "20": 318.3663203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 118323, + "answer_accuracy": 0.2747097882615156, + "avg_response_len": 296.7311292719168, + "resp_tokens_sum": 127808032, + "pass_at_k": { + "pass@1": 0.2747097882615156, + "pass@2": 0.3733014341121551, + "pass@4": 0.465171150309408, + "pass@8": 0.5425877333436215, + "pass@16": 0.6051977961257701, + "pass@32": 0.6558609144833849, + "pass@64": 0.6976752948696923, + "pass@128": 0.7340267459138187 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5599327195467422, + "pass@2": 0.7109233844884119, + "pass@4": 0.8225052578413533, + "pass@8": 0.8926580349921708, + "pass@16": 0.9333295570967041, + "pass@32": 0.9564493119764775, + "pass@64": 0.9692695987042779, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.4427966101694915, + "pass@2": 0.6038222479870099, + "pass@4": 0.7427874291137334, + "pass@8": 0.8402630444157572, + "pass@16": 0.8997258866173762, + "pass@32": 0.9344561909630209, + "pass@64": 0.9530677128620351, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.3682650862068966, + "pass@2": 0.4979925425172167, + "pass@4": 0.619347059673967, + "pass@8": 0.7220487867666285, + "pass@16": 0.8013165423964927, + "pass@32": 0.8591777688713896, + "pass@64": 0.9028755230719813, + "pass@128": 0.9341692789968652 + }, + "14": { + "pass@1": 0.2775506230529595, + "pass@2": 0.3937211010130745, + "pass@4": 0.5110733629090756, + "pass@8": 0.615459827824061, + "pass@16": 0.7029316216951226, + "pass@32": 0.7736901410312667, + "pass@64": 0.8274568239163917, + "pass@128": 0.8660436137071651 + }, + "15": { + "pass@1": 0.23317828757225434, + "pass@2": 0.3282679441081425, + "pass@4": 0.42495304768623543, + "pass@8": 0.5137240855394527, + "pass@16": 0.5889845728337569, + "pass@32": 0.6469725361731052, + "pass@64": 0.689443755215235, + "pass@128": 0.7254335260115607 + }, + "16": { + "pass@1": 0.1934475806451613, + "pass@2": 0.26473877635255283, + "pass@4": 0.33628472217182526, + "pass@8": 0.40642216848629104, + "pass@16": 0.4755940708933289, + "pass@32": 0.5403641008495248, + "pass@64": 0.5982561508228122, + "pass@128": 0.6483870967741936 + }, + "17": { + "pass@1": 0.17131051300578035, + "pass@2": 0.2387941775522277, + "pass@4": 0.30575373544347406, + "pass@8": 0.36986780070560576, + "pass@16": 0.4334840695506441, + "pass@32": 0.49633664095597957, + "pass@64": 0.554674308041247, + "pass@128": 0.6098265895953757 + }, + "18": { + "pass@1": 0.1675669642857143, + "pass@2": 0.23335911136107987, + "pass@4": 0.30120476270823276, + "pass@8": 0.3655080978889007, + "pass@16": 0.42347287095849856, + "pass@32": 0.4755511329959968, + "pass@64": 0.5251996675191798, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.1631709834834835, + "pass@2": 0.22588434891584497, + "pass@4": 0.286252485049729, + "pass@8": 0.33920095575428966, + "pass@16": 0.38482372915001195, + "pass@32": 0.42880135905591316, + "pass@64": 0.47486273358836156, + "pass@128": 0.5195195195195195 + }, + "20": { + "pass@1": 0.15169857357357358, + "pass@2": 0.21332885543023333, + "pass@4": 0.2779789730675557, + "pass@8": 0.3386028033542325, + "pass@16": 0.3894628828004633, + "pass@32": 0.4317839018100493, + "pass@64": 0.4703749719074299, + "pass@128": 0.5105105105105106 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 114581, + "answer_accuracy": 0.2661802148379423, + "avg_response_len": 307.6496129757657, + "resp_tokens_sum": 132432083, + "pass_at_k": { + "pass@1": 0.2661802148379423, + "pass@2": 0.3643672691002839, + "pass@4": 0.4576664330596068, + "pass@8": 0.5380332151354551, + "pass@16": 0.6045268659168196, + "pass@32": 0.6591300322751201, + "pass@64": 0.7029869925007359, + "pass@128": 0.7374368123699078 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5197900978915663, + "pass@2": 0.6667180533156247, + "pass@4": 0.7842412309530583, + "pass@8": 0.8631492332252853, + "pass@16": 0.9100323170688028, + "pass@32": 0.939724543524283, + "pass@64": 0.9587396589529518, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.47910952194357365, + "pass@2": 0.6399678035445416, + "pass@4": 0.7682812988893628, + "pass@8": 0.8561805492242159, + "pass@16": 0.9122820941511327, + "pass@32": 0.9466209121984869, + "pass@64": 0.9664428205027745, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.3403361344537815, + "pass@2": 0.47338797723813925, + "pass@4": 0.599396724831665, + "pass@8": 0.7050868992294267, + "pass@16": 0.789229855222965, + "pass@32": 0.850339928979183, + "pass@64": 0.8882099270210574, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.2902960526315789, + "pass@2": 0.3985098079845278, + "pass@4": 0.5051356878087606, + "pass@8": 0.5994879403043188, + "pass@16": 0.6772260970805483, + "pass@32": 0.7417045218997341, + "pass@64": 0.7956760258214428, + "pass@128": 0.8391812865497076 + }, + "15": { + "pass@1": 0.21420454545454545, + "pass@2": 0.3055867483894059, + "pass@4": 0.39845808222835793, + "pass@8": 0.4871670652592622, + "pass@16": 0.5689720699097469, + "pass@32": 0.6398546008577851, + "pass@64": 0.6962629638631309, + "pass@128": 0.7393939393939394 + }, + "16": { + "pass@1": 0.19827279202279202, + "pass@2": 0.27872031821342824, + "pass@4": 0.3629984417011977, + "pass@8": 0.4408281715911366, + "pass@16": 0.508026552191099, + "pass@32": 0.5674690802927236, + "pass@64": 0.6205853526518682, + "pass@128": 0.6695156695156695 + }, + "17": { + "pass@1": 0.18997476708074534, + "pass@2": 0.2610380648750425, + "pass@4": 0.3368291220973155, + "pass@8": 0.4132398731867813, + "pass@16": 0.4834337259752426, + "pass@32": 0.5428640234414458, + "pass@64": 0.5920497781538735, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.1624310661764706, + "pass@2": 0.23409311313107897, + "pass@4": 0.30889321582963913, + "pass@8": 0.38150896761898023, + "pass@16": 0.45005568993156586, + "pass@32": 0.5123031105086977, + "pass@64": 0.5669113473578711, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.1396484375, + "pass@2": 0.20062848589238844, + "pass@4": 0.26531595074053244, + "pass@8": 0.3268746347503379, + "pass@16": 0.38500503796087715, + "pass@32": 0.4401306384131243, + "pass@64": 0.48860559547150584, + "pass@128": 0.5267857142857143 + }, + "20": { + "pass@1": 0.13585329341317365, + "pass@2": 0.19414848943844587, + "pass@4": 0.2557152845602383, + "pass@8": 0.31399903611220836, + "pass@16": 0.3664202373537723, + "pass@32": 0.41394175111035364, + "pass@64": 0.458630781103217, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 111522, + "answer_accuracy": 0.26627922524449876, + "avg_response_len": 327.7882268108191, + "resp_tokens_sum": 137282954, + "pass_at_k": { + "pass@1": 0.26627922524449876, + "pass@2": 0.36158212186916516, + "pass@4": 0.4518017643023103, + "pass@8": 0.5281096622115071, + "pass@16": 0.5895022859003298, + "pass@32": 0.6401968655701813, + "pass@64": 0.6842559842247575, + "pass@128": 0.7237163814180929 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5646329365079366, + "pass@2": 0.71685336207974, + "pass@4": 0.82946349980062, + "pass@8": 0.8989213736705283, + "pass@16": 0.9382922546798307, + "pass@32": 0.9607834842187218, + "pass@64": 0.9747276125638802, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.4446435397553517, + "pass@2": 0.5997179681668231, + "pass@4": 0.7331657046309579, + "pass@8": 0.8297917311860121, + "pass@16": 0.8931182146188502, + "pass@32": 0.9322322978163884, + "pass@64": 0.9567219108599694, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.36060474537037035, + "pass@2": 0.48475168902498333, + "pass@4": 0.6023837983793693, + "pass@8": 0.6990689294100764, + "pass@16": 0.7706865603861656, + "pass@32": 0.8230004455601055, + "pass@64": 0.8626431069718627, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.29214577151335314, + "pass@2": 0.4008234713427883, + "pass@4": 0.5074832745313363, + "pass@8": 0.601375084942065, + "pass@16": 0.6789703942326578, + "pass@32": 0.7406445380246015, + "pass@64": 0.789367599624807, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.1976514274691358, + "pass@2": 0.29131314097890537, + "pass@4": 0.3921881697658164, + "pass@8": 0.4855609918599434, + "pass@16": 0.5662116470586928, + "pass@32": 0.6373774462304737, + "pass@64": 0.6977827741321524, + "pass@128": 0.7438271604938271 + }, + "16": { + "pass@1": 0.18706213126843657, + "pass@2": 0.25359985076533575, + "pass@4": 0.3209949519584388, + "pass@8": 0.38554267819668736, + "pass@16": 0.4486018761610928, + "pass@32": 0.5138035329859271, + "pass@64": 0.5784656480723253, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.16479198042168675, + "pass@2": 0.23328031614647562, + "pass@4": 0.30360634128188774, + "pass@8": 0.3691675256281784, + "pass@16": 0.42690508197986565, + "pass@32": 0.47893424285590325, + "pass@64": 0.5317847410416872, + "pass@128": 0.5933734939759037 + }, + "18": { + "pass@1": 0.17295866935483872, + "pass@2": 0.24062341249682495, + "pass@4": 0.3112551211340518, + "pass@8": 0.37637725350191814, + "pass@16": 0.4330918948484191, + "pass@32": 0.48455445994709084, + "pass@64": 0.5325561649996549, + "pass@128": 0.5709677419354838 + }, + "19": { + "pass@1": 0.1567692598187311, + "pass@2": 0.21468178330994123, + "pass@4": 0.2714260056616788, + "pass@8": 0.3231161448695808, + "pass@16": 0.36890863896298454, + "pass@32": 0.4108112826103503, + "pass@64": 0.449994222417303, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.13527590090090091, + "pass@2": 0.1969527352155305, + "pass@4": 0.26444538830168735, + "pass@8": 0.330625251893072, + "pass@16": 0.3876755280995512, + "pass@32": 0.43574100762264556, + "pass@64": 0.48261430489771634, + "pass@128": 0.5345345345345346 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5118717105263157, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5118717105263157, + "pass@2": 0.6041086691877158, + "pass@4": 0.6795785676987479, + "pass@8": 0.7355741309881468, + "pass@16": 0.7768541461516653, + "pass@32": 0.8088182117531557, + "pass@64": 0.8344434601731968, + "pass@128": 0.8556315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6076640625, + "pass@2": 0.756191067913386, + "pass@4": 0.8603429181664787, + "pass@8": 0.9199172971920299, + "pass@16": 0.9515354728881059, + "pass@32": 0.9687067881816586, + "pass@64": 0.9787306082691241, + "pass@128": 0.987 + }, + "2": { + "pass@1": 0.9796953125, + "pass@2": 0.9826592027559057, + "pass@4": 0.9850796374203223, + "pass@8": 0.9875326505398578, + "pass@16": 0.9906439412196009, + "pass@32": 0.9940873178573064, + "pass@64": 0.9966575695273527, + "pass@128": 0.998 + }, + "3": { + "pass@1": 0.9749140625, + "pass@2": 0.9810580708661415, + "pass@4": 0.985971505718035, + "pass@8": 0.9898390963793305, + "pass@16": 0.9924167713774642, + "pass@32": 0.9942863589815633, + "pass@64": 0.9960664597579397, + "pass@128": 0.998 + }, + "4": { + "pass@1": 0.91559375, + "pass@2": 0.949925196850393, + "pass@4": 0.9694112275965503, + "pass@8": 0.9798026126919146, + "pass@16": 0.9855816778143771, + "pass@32": 0.9890364734039345, + "pass@64": 0.9914867091300696, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.7999453125, + "pass@2": 0.88533341535433, + "pass@4": 0.9336656406074244, + "pass@8": 0.9568312556526607, + "pass@16": 0.969011778418327, + "pass@32": 0.9767837698692393, + "pass@64": 0.9822871781103928, + "pass@128": 0.987 + }, + "6": { + "pass@1": 0.77846875, + "pass@2": 0.8735273129921275, + "pass@4": 0.9312027098800149, + "pass@8": 0.9614433644622998, + "pass@16": 0.9768078281288376, + "pass@32": 0.9860289444598231, + "pass@64": 0.9921008351872295, + "pass@128": 0.995 + }, + "7": { + "pass@1": 0.7226875, + "pass@2": 0.8443891486220463, + "pass@4": 0.9200709941882256, + "pass@8": 0.9567265116477753, + "pass@16": 0.9742473535227066, + "pass@32": 0.9843899238601288, + "pass@64": 0.9905993308577857, + "pass@128": 0.994 + }, + "8": { + "pass@1": 0.669109375, + "pass@2": 0.8043056102362194, + "pass@4": 0.8959773600487436, + "pass@8": 0.9454055296272024, + "pass@16": 0.9687693078025161, + "pass@32": 0.9815056127748554, + "pass@64": 0.9894645631281762, + "pass@128": 0.993 + }, + "9": { + "pass@1": 0.58665625, + "pass@2": 0.7360525344488175, + "pass@4": 0.8475422846831647, + "pass@8": 0.9152219306237759, + "pass@16": 0.9528487322020358, + "pass@32": 0.9743704139975835, + "pass@64": 0.9863242859226867, + "pass@128": 0.993 + }, + "11": { + "pass@1": 0.5480859375, + "pass@2": 0.6981151574803157, + "pass@4": 0.811993447131609, + "pass@8": 0.8848340644892468, + "pass@16": 0.9271581231461244, + "pass@32": 0.952261953106656, + "pass@64": 0.9674929330726126, + "pass@128": 0.977 + }, + "12": { + "pass@1": 0.454984375, + "pass@2": 0.6140105807086608, + "pass@4": 0.7477736696662932, + "pass@8": 0.8419166090235274, + "pass@16": 0.9015706080771255, + "pass@32": 0.9376095239781851, + "pass@64": 0.9585292949447553, + "pass@128": 0.97 + }, + "13": { + "pass@1": 0.3558125, + "pass@2": 0.4849186761811016, + "pass@4": 0.6067286934758146, + "pass@8": 0.7085479191323254, + "pass@16": 0.7870774809041959, + "pass@32": 0.844301207277016, + "pass@64": 0.8846046024653631, + "pass@128": 0.913 + }, + "14": { + "pass@1": 0.286828125, + "pass@2": 0.39775233759842554, + "pass@4": 0.5078328182414698, + "pass@8": 0.6052508839410764, + "pass@16": 0.6860653986220863, + "pass@32": 0.7516146910750364, + "pass@64": 0.8037517223816558, + "pass@128": 0.845 + }, + "15": { + "pass@1": 0.21540625, + "pass@2": 0.3088097933070866, + "pass@4": 0.40559388863891943, + "pass@8": 0.4958354264948282, + "pass@16": 0.5750020189177119, + "pass@32": 0.6415148083776369, + "pass@64": 0.6943959361981233, + "pass@128": 0.736 + }, + "16": { + "pass@1": 0.1929765625, + "pass@2": 0.2658702017716534, + "pass@4": 0.3404780056242968, + "pass@8": 0.41142052836791654, + "pass@16": 0.47782751781461763, + "pass@32": 0.5408739161283275, + "pass@64": 0.5993847202323961, + "pass@128": 0.652 + }, + "17": { + "pass@1": 0.17515625, + "pass@2": 0.24412610728346473, + "pass@4": 0.31504707508436386, + "pass@8": 0.38360111671883745, + "pass@16": 0.4473836350458656, + "pass@32": 0.505540861947074, + "pass@64": 0.5591098731736599, + "pass@128": 0.612 + }, + "18": { + "pass@1": 0.1674921875, + "pass@2": 0.23586060531496067, + "pass@4": 0.3069344478815142, + "pass@8": 0.3743178318371625, + "pass@16": 0.4354929268152162, + "pass@32": 0.4908378367051541, + "pass@64": 0.5416621528832821, + "pass@128": 0.587 + }, + "19": { + "pass@1": 0.1531484375, + "pass@2": 0.2136903297244094, + "pass@4": 0.27431024484439437, + "pass@8": 0.32973523949412276, + "pass@16": 0.3796167540585565, + "pass@32": 0.42665328161645505, + "pass@64": 0.4712488579834778, + "pass@128": 0.511 + }, + "20": { + "pass@1": 0.1409375, + "pass@2": 0.20146936515748032, + "pass@4": 0.2660362173790774, + "pass@8": 0.3277286204588297, + "pass@16": 0.38117145010586473, + "pass@32": 0.4271423397119454, + "pass@64": 0.4705281100645885, + "pass@128": 0.515 + } + }, + "per_op_accuracy": { + "10": 0.6076640625, + "2": 0.9796953125, + "3": 0.9749140625, + "4": 0.91559375, + "5": 0.7999453125, + "6": 0.77846875, + "7": 0.7226875, + "8": 0.669109375, + "9": 0.58665625, + "11": 0.5480859375, + "12": 0.454984375, + "13": 0.3558125, + "14": 0.286828125, + "15": 0.21540625, + "16": 0.1929765625, + "17": 0.17515625, + "18": 0.1674921875, + "19": 0.1531484375, + "20": 0.1409375 + }, + "per_op_avg_response_len": { + "10": 274.77615625, + "2": 114.3989375, + "3": 142.7864609375, + "4": 158.1644296875, + "5": 171.5203046875, + "6": 195.9111484375, + "7": 220.8932890625, + "8": 239.4595546875, + "9": 254.3683671875, + "11": 289.850421875, + "12": 297.486453125, + "13": 308.000375, + "14": 312.466140625, + "15": 313.4066796875, + "16": 313.8389921875, + "17": 319.033359375, + "18": 318.24709375, + "19": 314.953140625, + "20": 318.3663203125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.77615625, + "2": 114.3989375, + "3": 142.7864609375, + "4": 158.1644296875, + "5": 171.5203046875, + "6": 195.9111484375, + "7": 220.8932890625, + "8": 239.4595546875, + "9": 254.3683671875, + "11": 289.850421875, + "12": 297.486453125, + "13": 308.000375, + "14": 312.466140625, + "15": 313.4066796875, + "16": 313.8389921875, + "17": 319.033359375, + "18": 318.24709375, + "19": 314.953140625, + "20": 318.3663203125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 414495, + "answer_accuracy": 0.5118941175308251, + "avg_response_len": 245.55807629228582, + "resp_tokens_sum": 198835250, + "pass_at_k": { + "pass@1": 0.5118941175308251, + "pass@2": 0.6060234127809088, + "pass@4": 0.6823425681248854, + "pass@8": 0.7382585878256954, + "pass@16": 0.7790123570823432, + "pass@32": 0.8101523431020332, + "pass@64": 0.834971805948881, + "pass@128": 0.8559911476446411 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6108038348082596, + "pass@2": 0.7592836155436323, + "pass@4": 0.8628990543770518, + "pass@8": 0.9209853506323293, + "pass@16": 0.9502138983934935, + "pass@32": 0.9653789425226263, + "pass@64": 0.9738636888533403, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9687013239875389, + "pass@2": 0.9735359635244192, + "pass@4": 0.9774219598134347, + "pass@8": 0.9811756048228187, + "pass@16": 0.9857846921623187, + "pass@32": 0.9912202144423357, + "pass@64": 0.9954408818642898, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9708046636085627, + "pass@2": 0.9782207914950999, + "pass@4": 0.9856412343525864, + "pass@8": 0.9913135287729292, + "pass@16": 0.9943854355053758, + "pass@32": 0.9964695421928315, + "pass@64": 0.9982793427520417, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.933056640625, + "pass@2": 0.9640221302903547, + "pass@4": 0.9808446167861831, + "pass@8": 0.9889389221699499, + "pass@16": 0.9928154480539686, + "pass@32": 0.9944218943628333, + "pass@64": 0.9953122612788613, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.7997519841269841, + "pass@2": 0.8826568553930755, + "pass@4": 0.930272895352366, + "pass@8": 0.9548178801748886, + "pass@16": 0.9687459281478877, + "pass@32": 0.9774046788113472, + "pass@64": 0.983474310194418, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.7953078218562875, + "pass@2": 0.8896447999669947, + "pass@4": 0.944976489933518, + "pass@8": 0.973097387972258, + "pass@16": 0.9870541545418998, + "pass@32": 0.9947098289309901, + "pass@64": 0.9987991381053104, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.7229105029585798, + "pass@2": 0.8472880655080834, + "pass@4": 0.9232964889189142, + "pass@8": 0.9585954889402899, + "pass@16": 0.9758673121900198, + "pass@32": 0.987287560047621, + "pass@64": 0.9952321084717118, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6659405048076923, + "pass@2": 0.8077088696244699, + "pass@4": 0.9056807724755556, + "pass@8": 0.9572339732355738, + "pass@16": 0.9783949645645401, + "pass@32": 0.986614119977316, + "pass@64": 0.9907329976438498, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.5919014084507043, + "pass@2": 0.7484803010979263, + "pass@4": 0.8604035258796869, + "pass@8": 0.9244515130120431, + "pass@16": 0.958264149196147, + "pass@32": 0.9771934956760493, + "pass@64": 0.9883104989872438, + "pass@128": 0.9943661971830986 + }, + "11": { + "pass@1": 0.5599327195467422, + "pass@2": 0.7109233844884119, + "pass@4": 0.8225052578413533, + "pass@8": 0.8926580349921708, + "pass@16": 0.9333295570967041, + "pass@32": 0.9564493119764775, + "pass@64": 0.9692695987042779, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.4427966101694915, + "pass@2": 0.6038222479870099, + "pass@4": 0.7427874291137334, + "pass@8": 0.8402630444157572, + "pass@16": 0.8997258866173762, + "pass@32": 0.9344561909630209, + "pass@64": 0.9530677128620351, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.3682650862068966, + "pass@2": 0.4979925425172167, + "pass@4": 0.619347059673967, + "pass@8": 0.7220487867666285, + "pass@16": 0.8013165423964927, + "pass@32": 0.8591777688713896, + "pass@64": 0.9028755230719813, + "pass@128": 0.9341692789968652 + }, + "14": { + "pass@1": 0.2775506230529595, + "pass@2": 0.3937211010130745, + "pass@4": 0.5110733629090756, + "pass@8": 0.615459827824061, + "pass@16": 0.7029316216951226, + "pass@32": 0.7736901410312667, + "pass@64": 0.8274568239163917, + "pass@128": 0.8660436137071651 + }, + "15": { + "pass@1": 0.23317828757225434, + "pass@2": 0.3282679441081425, + "pass@4": 0.42495304768623543, + "pass@8": 0.5137240855394527, + "pass@16": 0.5889845728337569, + "pass@32": 0.6469725361731052, + "pass@64": 0.689443755215235, + "pass@128": 0.7254335260115607 + }, + "16": { + "pass@1": 0.1934475806451613, + "pass@2": 0.26473877635255283, + "pass@4": 0.33628472217182526, + "pass@8": 0.40642216848629104, + "pass@16": 0.4755940708933289, + "pass@32": 0.5403641008495248, + "pass@64": 0.5982561508228122, + "pass@128": 0.6483870967741936 + }, + "17": { + "pass@1": 0.17131051300578035, + "pass@2": 0.2387941775522277, + "pass@4": 0.30575373544347406, + "pass@8": 0.36986780070560576, + "pass@16": 0.4334840695506441, + "pass@32": 0.49633664095597957, + "pass@64": 0.554674308041247, + "pass@128": 0.6098265895953757 + }, + "18": { + "pass@1": 0.1675669642857143, + "pass@2": 0.23335911136107987, + "pass@4": 0.30120476270823276, + "pass@8": 0.3655080978889007, + "pass@16": 0.42347287095849856, + "pass@32": 0.4755511329959968, + "pass@64": 0.5251996675191798, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.1631709834834835, + "pass@2": 0.22588434891584497, + "pass@4": 0.286252485049729, + "pass@8": 0.33920095575428966, + "pass@16": 0.38482372915001195, + "pass@32": 0.42880135905591316, + "pass@64": 0.47486273358836156, + "pass@128": 0.5195195195195195 + }, + "20": { + "pass@1": 0.15169857357357358, + "pass@2": 0.21332885543023333, + "pass@4": 0.2779789730675557, + "pass@8": 0.3386028033542325, + "pass@16": 0.3894628828004633, + "pass@32": 0.4317839018100493, + "pass@64": 0.4703749719074299, + "pass@128": 0.5105105105105106 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 414949, + "answer_accuracy": 0.5053451383476227, + "avg_response_len": 254.23482073265782, + "resp_tokens_sum": 208757296, + "pass_at_k": { + "pass@1": 0.5053451383476227, + "pass@2": 0.5979934263015504, + "pass@4": 0.6754241830157071, + "pass@8": 0.733934855593854, + "pass@16": 0.777473519194293, + "pass@32": 0.811257252919628, + "pass@64": 0.8377472273022628, + "pass@128": 0.8583008573655495 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5747699386503068, + "pass@2": 0.7293609004395927, + "pass@4": 0.842726695972819, + "pass@8": 0.9087806167016811, + "pass@16": 0.9435410406326107, + "pass@32": 0.962740051480476, + "pass@64": 0.9756515561073661, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9805043693009119, + "pass@2": 0.9843484491300291, + "pass@4": 0.9870946238103215, + "pass@8": 0.9895161087507777, + "pass@16": 0.9925479984428305, + "pass@32": 0.9957528327388283, + "pass@64": 0.9980910863778228, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9767282196969697, + "pass@2": 0.9832457199952278, + "pass@4": 0.9873704749974436, + "pass@8": 0.9902525452585946, + "pass@16": 0.9920598769834958, + "pass@32": 0.9934207656439095, + "pass@64": 0.9950847041173666, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 0.890625, + "pass@2": 0.927779783814006, + "pass@4": 0.9517953019371097, + "pass@8": 0.9651948360744051, + "pass@16": 0.9715446905626869, + "pass@32": 0.9753629793049493, + "pass@64": 0.9796651628850817, + "pass@128": 0.985207100591716 + }, + "5": { + "pass@1": 0.8036005434782608, + "pass@2": 0.8880306401917152, + "pass@4": 0.9356044589534999, + "pass@8": 0.9566631468858801, + "pass@16": 0.9677299365458314, + "pass@32": 0.9762975400418303, + "pass@64": 0.9827398189807199, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.7473809097421203, + "pass@2": 0.8443866615526928, + "pass@4": 0.9105473168826673, + "pass@8": 0.9485723437526672, + "pass@16": 0.9680171909631179, + "pass@32": 0.9798515662831963, + "pass@64": 0.9887143445075527, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.7122245303468208, + "pass@2": 0.82878353852806, + "pass@4": 0.9054310449271872, + "pass@8": 0.9459152039364331, + "pass@16": 0.9658799168085859, + "pass@32": 0.9764867382386906, + "pass@64": 0.9835947173089279, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.6647887323943662, + "pass@2": 0.7976325967616721, + "pass@4": 0.8875285859514035, + "pass@8": 0.9386763687150315, + "pass@16": 0.9646758684012232, + "pass@32": 0.979982181086267, + "pass@64": 0.9892714095496256, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.5772595434131736, + "pass@2": 0.7188549819652033, + "pass@4": 0.8316630995601596, + "pass@8": 0.9051994481455924, + "pass@16": 0.9467216399752423, + "pass@32": 0.9702232540521561, + "pass@64": 0.9833604457495104, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.5197900978915663, + "pass@2": 0.6667180533156247, + "pass@4": 0.7842412309530583, + "pass@8": 0.8631492332252853, + "pass@16": 0.9100323170688028, + "pass@32": 0.939724543524283, + "pass@64": 0.9587396589529518, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.47910952194357365, + "pass@2": 0.6399678035445416, + "pass@4": 0.7682812988893628, + "pass@8": 0.8561805492242159, + "pass@16": 0.9122820941511327, + "pass@32": 0.9466209121984869, + "pass@64": 0.9664428205027745, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.3403361344537815, + "pass@2": 0.47338797723813925, + "pass@4": 0.599396724831665, + "pass@8": 0.7050868992294267, + "pass@16": 0.789229855222965, + "pass@32": 0.850339928979183, + "pass@64": 0.8882099270210574, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.2902960526315789, + "pass@2": 0.3985098079845278, + "pass@4": 0.5051356878087606, + "pass@8": 0.5994879403043188, + "pass@16": 0.6772260970805483, + "pass@32": 0.7417045218997341, + "pass@64": 0.7956760258214428, + "pass@128": 0.8391812865497076 + }, + "15": { + "pass@1": 0.21420454545454545, + "pass@2": 0.3055867483894059, + "pass@4": 0.39845808222835793, + "pass@8": 0.4871670652592622, + "pass@16": 0.5689720699097469, + "pass@32": 0.6398546008577851, + "pass@64": 0.6962629638631309, + "pass@128": 0.7393939393939394 + }, + "16": { + "pass@1": 0.19827279202279202, + "pass@2": 0.27872031821342824, + "pass@4": 0.3629984417011977, + "pass@8": 0.4408281715911366, + "pass@16": 0.508026552191099, + "pass@32": 0.5674690802927236, + "pass@64": 0.6205853526518682, + "pass@128": 0.6695156695156695 + }, + "17": { + "pass@1": 0.18997476708074534, + "pass@2": 0.2610380648750425, + "pass@4": 0.3368291220973155, + "pass@8": 0.4132398731867813, + "pass@16": 0.4834337259752426, + "pass@32": 0.5428640234414458, + "pass@64": 0.5920497781538735, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.1624310661764706, + "pass@2": 0.23409311313107897, + "pass@4": 0.30889321582963913, + "pass@8": 0.38150896761898023, + "pass@16": 0.45005568993156586, + "pass@32": 0.5123031105086977, + "pass@64": 0.5669113473578711, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.1396484375, + "pass@2": 0.20062848589238844, + "pass@4": 0.26531595074053244, + "pass@8": 0.3268746347503379, + "pass@16": 0.38500503796087715, + "pass@32": 0.4401306384131243, + "pass@64": 0.48860559547150584, + "pass@128": 0.5267857142857143 + }, + "20": { + "pass@1": 0.13585329341317365, + "pass@2": 0.19414848943844587, + "pass@4": 0.2557152845602383, + "pass@8": 0.31399903611220836, + "pass@16": 0.3664202373537723, + "pass@32": 0.41394175111035364, + "pass@64": 0.458630781103217, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 415428, + "answer_accuracy": 0.5185383048410289, + "avg_response_len": 270.5880906494648, + "resp_tokens_sum": 216782190, + "pass_at_k": { + "pass@1": 0.5185383048410289, + "pass@2": 0.6084410889264427, + "pass@4": 0.6810429088149833, + "pass@8": 0.7345410710265036, + "pass@16": 0.7740380221037376, + "pass@32": 0.8049699669862138, + "pass@64": 0.830523348092998, + "pass@128": 0.852532353411088 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6364972014925373, + "pass@2": 0.779170951345635, + "pass@4": 0.8748992114791617, + "pass@8": 0.9296739769639495, + "pass@16": 0.9606524838402445, + "pass@32": 0.9778808056234413, + "pass@64": 0.9866519715129541, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9890178571428572, + "pass@2": 0.9894386248593927, + "pass@4": 0.9902087345331835, + "pass@8": 0.9914985188935059, + "pass@16": 0.9933107529937302, + "pass@32": 0.9951512772864062, + "pass@64": 0.9964259400874629, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9770863702623906, + "pass@2": 0.9816582636991802, + "pass@4": 0.9849404237189008, + "pass@8": 0.9880356633680653, + "pass@16": 0.9908833077628361, + "pass@32": 0.9930378018134619, + "pass@64": 0.994901347869653, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9239309210526315, + "pass@2": 0.9586214860477973, + "pass@4": 0.976123211024061, + "pass@8": 0.9856909444572529, + "pass@16": 0.992686049786314, + "pass@32": 0.9975110532244272, + "pass@64": 0.9995905276774154, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7964154411764706, + "pass@2": 0.8850762795275591, + "pass@4": 0.9348415889190315, + "pass@8": 0.9588671697704182, + "pass@16": 0.9705587733630365, + "pass@32": 0.9767018962624524, + "pass@64": 0.9807280377964777, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.7949526813880127, + "pass@2": 0.8886277757768444, + "pass@4": 0.9394307528396486, + "pass@8": 0.9633346653308688, + "pass@16": 0.9756900342767074, + "pass@32": 0.9836834856909692, + "pass@64": 0.9887716619145747, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.7339052610759493, + "pass@2": 0.858375563764577, + "pass@4": 0.9326507575595763, + "pass@8": 0.9665651133669365, + "pass@16": 0.9816764265402234, + "pass@32": 0.989944041561538, + "pass@64": 0.9933136265995508, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.676684496996997, + "pass@2": 0.8082308440330094, + "pass@4": 0.8958928258967639, + "pass@8": 0.9414967540056054, + "pass@16": 0.9641145512791177, + "pass@32": 0.9783433425114311, + "pass@64": 0.9884820344539859, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.5907606511254019, + "pass@2": 0.740335895764235, + "pass@4": 0.8499149766648948, + "pass@8": 0.9154502502375335, + "pass@16": 0.9532473681211369, + "pass@32": 0.9756017883574458, + "pass@64": 0.9872400961475196, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.5646329365079366, + "pass@2": 0.71685336207974, + "pass@4": 0.82946349980062, + "pass@8": 0.8989213736705283, + "pass@16": 0.9382922546798307, + "pass@32": 0.9607834842187218, + "pass@64": 0.9747276125638802, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.4446435397553517, + "pass@2": 0.5997179681668231, + "pass@4": 0.7331657046309579, + "pass@8": 0.8297917311860121, + "pass@16": 0.8931182146188502, + "pass@32": 0.9322322978163884, + "pass@64": 0.9567219108599694, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.36060474537037035, + "pass@2": 0.48475168902498333, + "pass@4": 0.6023837983793693, + "pass@8": 0.6990689294100764, + "pass@16": 0.7706865603861656, + "pass@32": 0.8230004455601055, + "pass@64": 0.8626431069718627, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.29214577151335314, + "pass@2": 0.4008234713427883, + "pass@4": 0.5074832745313363, + "pass@8": 0.601375084942065, + "pass@16": 0.6789703942326578, + "pass@32": 0.7406445380246015, + "pass@64": 0.789367599624807, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.1976514274691358, + "pass@2": 0.29131314097890537, + "pass@4": 0.3921881697658164, + "pass@8": 0.4855609918599434, + "pass@16": 0.5662116470586928, + "pass@32": 0.6373774462304737, + "pass@64": 0.6977827741321524, + "pass@128": 0.7438271604938271 + }, + "16": { + "pass@1": 0.18706213126843657, + "pass@2": 0.25359985076533575, + "pass@4": 0.3209949519584388, + "pass@8": 0.38554267819668736, + "pass@16": 0.4486018761610928, + "pass@32": 0.5138035329859271, + "pass@64": 0.5784656480723253, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.16479198042168675, + "pass@2": 0.23328031614647562, + "pass@4": 0.30360634128188774, + "pass@8": 0.3691675256281784, + "pass@16": 0.42690508197986565, + "pass@32": 0.47893424285590325, + "pass@64": 0.5317847410416872, + "pass@128": 0.5933734939759037 + }, + "18": { + "pass@1": 0.17295866935483872, + "pass@2": 0.24062341249682495, + "pass@4": 0.3112551211340518, + "pass@8": 0.37637725350191814, + "pass@16": 0.4330918948484191, + "pass@32": 0.48455445994709084, + "pass@64": 0.5325561649996549, + "pass@128": 0.5709677419354838 + }, + "19": { + "pass@1": 0.1567692598187311, + "pass@2": 0.21468178330994123, + "pass@4": 0.2714260056616788, + "pass@8": 0.3231161448695808, + "pass@16": 0.36890863896298454, + "pass@32": 0.4108112826103503, + "pass@64": 0.449994222417303, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.13527590090090091, + "pass@2": 0.1969527352155305, + "pass@4": 0.26444538830168735, + "pass@8": 0.330625251893072, + "pass@16": 0.3876755280995512, + "pass@32": 0.43574100762264556, + "pass@64": 0.48261430489771634, + "pass@128": 0.5345345345345346 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/summary.csv b/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d544104abbe634c417b6ca71aa38f3cd4fe51b13 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21350,0.7816371527777778,0.2690828125,0.5118717105263157,0.0,0.0,0.0,196.91984982638888,310.56489765625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/summary.json b/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5319f3184913a7d35e4ce806b095321e7e4f4d88 --- /dev/null +++ b/SDAR-100M-ep1-block2/composition/low_confidence_static/block8/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21350", + "id_acc": 0.7816371527777778, + "ood_acc": 0.2690828125, + "total_acc": 0.5118717105263157, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.91984982638888, + "ood_avg_resp_len": 310.56489765625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 414495, + "answer_accuracy": 0.5118941175308251, + "avg_response_len": 245.55807629228582, + "resp_tokens_sum": 198835250, + "pass_at_k": { + "pass@1": 0.5118941175308251, + "pass@2": 0.6060234127809088, + "pass@4": 0.6823425681248854, + "pass@8": 0.7382585878256954, + "pass@16": 0.7790123570823432, + "pass@32": 0.8101523431020332, + "pass@64": 0.834971805948881, + "pass@128": 0.8559911476446411 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6108038348082596, + "pass@2": 0.7592836155436323, + "pass@4": 0.8628990543770518, + "pass@8": 0.9209853506323293, + "pass@16": 0.9502138983934935, + "pass@32": 0.9653789425226263, + "pass@64": 0.9738636888533403, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9687013239875389, + "pass@2": 0.9735359635244192, + "pass@4": 0.9774219598134347, + "pass@8": 0.9811756048228187, + "pass@16": 0.9857846921623187, + "pass@32": 0.9912202144423357, + "pass@64": 0.9954408818642898, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9708046636085627, + "pass@2": 0.9782207914950999, + "pass@4": 0.9856412343525864, + "pass@8": 0.9913135287729292, + "pass@16": 0.9943854355053758, + "pass@32": 0.9964695421928315, + "pass@64": 0.9982793427520417, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.933056640625, + "pass@2": 0.9640221302903547, + "pass@4": 0.9808446167861831, + "pass@8": 0.9889389221699499, + "pass@16": 0.9928154480539686, + "pass@32": 0.9944218943628333, + "pass@64": 0.9953122612788613, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.7997519841269841, + "pass@2": 0.8826568553930755, + "pass@4": 0.930272895352366, + "pass@8": 0.9548178801748886, + "pass@16": 0.9687459281478877, + "pass@32": 0.9774046788113472, + "pass@64": 0.983474310194418, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.7953078218562875, + "pass@2": 0.8896447999669947, + "pass@4": 0.944976489933518, + "pass@8": 0.973097387972258, + "pass@16": 0.9870541545418998, + "pass@32": 0.9947098289309901, + "pass@64": 0.9987991381053104, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.7229105029585798, + "pass@2": 0.8472880655080834, + "pass@4": 0.9232964889189142, + "pass@8": 0.9585954889402899, + "pass@16": 0.9758673121900198, + "pass@32": 0.987287560047621, + "pass@64": 0.9952321084717118, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6659405048076923, + "pass@2": 0.8077088696244699, + "pass@4": 0.9056807724755556, + "pass@8": 0.9572339732355738, + "pass@16": 0.9783949645645401, + "pass@32": 0.986614119977316, + "pass@64": 0.9907329976438498, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.5919014084507043, + "pass@2": 0.7484803010979263, + "pass@4": 0.8604035258796869, + "pass@8": 0.9244515130120431, + "pass@16": 0.958264149196147, + "pass@32": 0.9771934956760493, + "pass@64": 0.9883104989872438, + "pass@128": 0.9943661971830986 + }, + "11": { + "pass@1": 0.5599327195467422, + "pass@2": 0.7109233844884119, + "pass@4": 0.8225052578413533, + "pass@8": 0.8926580349921708, + "pass@16": 0.9333295570967041, + "pass@32": 0.9564493119764775, + "pass@64": 0.9692695987042779, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.4427966101694915, + "pass@2": 0.6038222479870099, + "pass@4": 0.7427874291137334, + "pass@8": 0.8402630444157572, + "pass@16": 0.8997258866173762, + "pass@32": 0.9344561909630209, + "pass@64": 0.9530677128620351, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.3682650862068966, + "pass@2": 0.4979925425172167, + "pass@4": 0.619347059673967, + "pass@8": 0.7220487867666285, + "pass@16": 0.8013165423964927, + "pass@32": 0.8591777688713896, + "pass@64": 0.9028755230719813, + "pass@128": 0.9341692789968652 + }, + "14": { + "pass@1": 0.2775506230529595, + "pass@2": 0.3937211010130745, + "pass@4": 0.5110733629090756, + "pass@8": 0.615459827824061, + "pass@16": 0.7029316216951226, + "pass@32": 0.7736901410312667, + "pass@64": 0.8274568239163917, + "pass@128": 0.8660436137071651 + }, + "15": { + "pass@1": 0.23317828757225434, + "pass@2": 0.3282679441081425, + "pass@4": 0.42495304768623543, + "pass@8": 0.5137240855394527, + "pass@16": 0.5889845728337569, + "pass@32": 0.6469725361731052, + "pass@64": 0.689443755215235, + "pass@128": 0.7254335260115607 + }, + "16": { + "pass@1": 0.1934475806451613, + "pass@2": 0.26473877635255283, + "pass@4": 0.33628472217182526, + "pass@8": 0.40642216848629104, + "pass@16": 0.4755940708933289, + "pass@32": 0.5403641008495248, + "pass@64": 0.5982561508228122, + "pass@128": 0.6483870967741936 + }, + "17": { + "pass@1": 0.17131051300578035, + "pass@2": 0.2387941775522277, + "pass@4": 0.30575373544347406, + "pass@8": 0.36986780070560576, + "pass@16": 0.4334840695506441, + "pass@32": 0.49633664095597957, + "pass@64": 0.554674308041247, + "pass@128": 0.6098265895953757 + }, + "18": { + "pass@1": 0.1675669642857143, + "pass@2": 0.23335911136107987, + "pass@4": 0.30120476270823276, + "pass@8": 0.3655080978889007, + "pass@16": 0.42347287095849856, + "pass@32": 0.4755511329959968, + "pass@64": 0.5251996675191798, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.1631709834834835, + "pass@2": 0.22588434891584497, + "pass@4": 0.286252485049729, + "pass@8": 0.33920095575428966, + "pass@16": 0.38482372915001195, + "pass@32": 0.42880135905591316, + "pass@64": 0.47486273358836156, + "pass@128": 0.5195195195195195 + }, + "20": { + "pass@1": 0.15169857357357358, + "pass@2": 0.21332885543023333, + "pass@4": 0.2779789730675557, + "pass@8": 0.3386028033542325, + "pass@16": 0.3894628828004633, + "pass@32": 0.4317839018100493, + "pass@64": 0.4703749719074299, + "pass@128": 0.5105105105105106 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 414949, + "answer_accuracy": 0.5053451383476227, + "avg_response_len": 254.23482073265782, + "resp_tokens_sum": 208757296, + "pass_at_k": { + "pass@1": 0.5053451383476227, + "pass@2": 0.5979934263015504, + "pass@4": 0.6754241830157071, + "pass@8": 0.733934855593854, + "pass@16": 0.777473519194293, + "pass@32": 0.811257252919628, + "pass@64": 0.8377472273022628, + "pass@128": 0.8583008573655495 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5747699386503068, + "pass@2": 0.7293609004395927, + "pass@4": 0.842726695972819, + "pass@8": 0.9087806167016811, + "pass@16": 0.9435410406326107, + "pass@32": 0.962740051480476, + "pass@64": 0.9756515561073661, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9805043693009119, + "pass@2": 0.9843484491300291, + "pass@4": 0.9870946238103215, + "pass@8": 0.9895161087507777, + "pass@16": 0.9925479984428305, + "pass@32": 0.9957528327388283, + "pass@64": 0.9980910863778228, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9767282196969697, + "pass@2": 0.9832457199952278, + "pass@4": 0.9873704749974436, + "pass@8": 0.9902525452585946, + "pass@16": 0.9920598769834958, + "pass@32": 0.9934207656439095, + "pass@64": 0.9950847041173666, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 0.890625, + "pass@2": 0.927779783814006, + "pass@4": 0.9517953019371097, + "pass@8": 0.9651948360744051, + "pass@16": 0.9715446905626869, + "pass@32": 0.9753629793049493, + "pass@64": 0.9796651628850817, + "pass@128": 0.985207100591716 + }, + "5": { + "pass@1": 0.8036005434782608, + "pass@2": 0.8880306401917152, + "pass@4": 0.9356044589534999, + "pass@8": 0.9566631468858801, + "pass@16": 0.9677299365458314, + "pass@32": 0.9762975400418303, + "pass@64": 0.9827398189807199, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.7473809097421203, + "pass@2": 0.8443866615526928, + "pass@4": 0.9105473168826673, + "pass@8": 0.9485723437526672, + "pass@16": 0.9680171909631179, + "pass@32": 0.9798515662831963, + "pass@64": 0.9887143445075527, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.7122245303468208, + "pass@2": 0.82878353852806, + "pass@4": 0.9054310449271872, + "pass@8": 0.9459152039364331, + "pass@16": 0.9658799168085859, + "pass@32": 0.9764867382386906, + "pass@64": 0.9835947173089279, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.6647887323943662, + "pass@2": 0.7976325967616721, + "pass@4": 0.8875285859514035, + "pass@8": 0.9386763687150315, + "pass@16": 0.9646758684012232, + "pass@32": 0.979982181086267, + "pass@64": 0.9892714095496256, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.5772595434131736, + "pass@2": 0.7188549819652033, + "pass@4": 0.8316630995601596, + "pass@8": 0.9051994481455924, + "pass@16": 0.9467216399752423, + "pass@32": 0.9702232540521561, + "pass@64": 0.9833604457495104, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.5197900978915663, + "pass@2": 0.6667180533156247, + "pass@4": 0.7842412309530583, + "pass@8": 0.8631492332252853, + "pass@16": 0.9100323170688028, + "pass@32": 0.939724543524283, + "pass@64": 0.9587396589529518, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.47910952194357365, + "pass@2": 0.6399678035445416, + "pass@4": 0.7682812988893628, + "pass@8": 0.8561805492242159, + "pass@16": 0.9122820941511327, + "pass@32": 0.9466209121984869, + "pass@64": 0.9664428205027745, + "pass@128": 0.9749216300940439 + }, + "13": { + "pass@1": 0.3403361344537815, + "pass@2": 0.47338797723813925, + "pass@4": 0.599396724831665, + "pass@8": 0.7050868992294267, + "pass@16": 0.789229855222965, + "pass@32": 0.850339928979183, + "pass@64": 0.8882099270210574, + "pass@128": 0.9103641456582633 + }, + "14": { + "pass@1": 0.2902960526315789, + "pass@2": 0.3985098079845278, + "pass@4": 0.5051356878087606, + "pass@8": 0.5994879403043188, + "pass@16": 0.6772260970805483, + "pass@32": 0.7417045218997341, + "pass@64": 0.7956760258214428, + "pass@128": 0.8391812865497076 + }, + "15": { + "pass@1": 0.21420454545454545, + "pass@2": 0.3055867483894059, + "pass@4": 0.39845808222835793, + "pass@8": 0.4871670652592622, + "pass@16": 0.5689720699097469, + "pass@32": 0.6398546008577851, + "pass@64": 0.6962629638631309, + "pass@128": 0.7393939393939394 + }, + "16": { + "pass@1": 0.19827279202279202, + "pass@2": 0.27872031821342824, + "pass@4": 0.3629984417011977, + "pass@8": 0.4408281715911366, + "pass@16": 0.508026552191099, + "pass@32": 0.5674690802927236, + "pass@64": 0.6205853526518682, + "pass@128": 0.6695156695156695 + }, + "17": { + "pass@1": 0.18997476708074534, + "pass@2": 0.2610380648750425, + "pass@4": 0.3368291220973155, + "pass@8": 0.4132398731867813, + "pass@16": 0.4834337259752426, + "pass@32": 0.5428640234414458, + "pass@64": 0.5920497781538735, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.1624310661764706, + "pass@2": 0.23409311313107897, + "pass@4": 0.30889321582963913, + "pass@8": 0.38150896761898023, + "pass@16": 0.45005568993156586, + "pass@32": 0.5123031105086977, + "pass@64": 0.5669113473578711, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.1396484375, + "pass@2": 0.20062848589238844, + "pass@4": 0.26531595074053244, + "pass@8": 0.3268746347503379, + "pass@16": 0.38500503796087715, + "pass@32": 0.4401306384131243, + "pass@64": 0.48860559547150584, + "pass@128": 0.5267857142857143 + }, + "20": { + "pass@1": 0.13585329341317365, + "pass@2": 0.19414848943844587, + "pass@4": 0.2557152845602383, + "pass@8": 0.31399903611220836, + "pass@16": 0.3664202373537723, + "pass@32": 0.41394175111035364, + "pass@64": 0.458630781103217, + "pass@128": 0.5 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 415428, + "answer_accuracy": 0.5185383048410289, + "avg_response_len": 270.5880906494648, + "resp_tokens_sum": 216782190, + "pass_at_k": { + "pass@1": 0.5185383048410289, + "pass@2": 0.6084410889264427, + "pass@4": 0.6810429088149833, + "pass@8": 0.7345410710265036, + "pass@16": 0.7740380221037376, + "pass@32": 0.8049699669862138, + "pass@64": 0.830523348092998, + "pass@128": 0.852532353411088 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6364972014925373, + "pass@2": 0.779170951345635, + "pass@4": 0.8748992114791617, + "pass@8": 0.9296739769639495, + "pass@16": 0.9606524838402445, + "pass@32": 0.9778808056234413, + "pass@64": 0.9866519715129541, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9890178571428572, + "pass@2": 0.9894386248593927, + "pass@4": 0.9902087345331835, + "pass@8": 0.9914985188935059, + "pass@16": 0.9933107529937302, + "pass@32": 0.9951512772864062, + "pass@64": 0.9964259400874629, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9770863702623906, + "pass@2": 0.9816582636991802, + "pass@4": 0.9849404237189008, + "pass@8": 0.9880356633680653, + "pass@16": 0.9908833077628361, + "pass@32": 0.9930378018134619, + "pass@64": 0.994901347869653, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9239309210526315, + "pass@2": 0.9586214860477973, + "pass@4": 0.976123211024061, + "pass@8": 0.9856909444572529, + "pass@16": 0.992686049786314, + "pass@32": 0.9975110532244272, + "pass@64": 0.9995905276774154, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7964154411764706, + "pass@2": 0.8850762795275591, + "pass@4": 0.9348415889190315, + "pass@8": 0.9588671697704182, + "pass@16": 0.9705587733630365, + "pass@32": 0.9767018962624524, + "pass@64": 0.9807280377964777, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.7949526813880127, + "pass@2": 0.8886277757768444, + "pass@4": 0.9394307528396486, + "pass@8": 0.9633346653308688, + "pass@16": 0.9756900342767074, + "pass@32": 0.9836834856909692, + "pass@64": 0.9887716619145747, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.7339052610759493, + "pass@2": 0.858375563764577, + "pass@4": 0.9326507575595763, + "pass@8": 0.9665651133669365, + "pass@16": 0.9816764265402234, + "pass@32": 0.989944041561538, + "pass@64": 0.9933136265995508, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.676684496996997, + "pass@2": 0.8082308440330094, + "pass@4": 0.8958928258967639, + "pass@8": 0.9414967540056054, + "pass@16": 0.9641145512791177, + "pass@32": 0.9783433425114311, + "pass@64": 0.9884820344539859, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.5907606511254019, + "pass@2": 0.740335895764235, + "pass@4": 0.8499149766648948, + "pass@8": 0.9154502502375335, + "pass@16": 0.9532473681211369, + "pass@32": 0.9756017883574458, + "pass@64": 0.9872400961475196, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.5646329365079366, + "pass@2": 0.71685336207974, + "pass@4": 0.82946349980062, + "pass@8": 0.8989213736705283, + "pass@16": 0.9382922546798307, + "pass@32": 0.9607834842187218, + "pass@64": 0.9747276125638802, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.4446435397553517, + "pass@2": 0.5997179681668231, + "pass@4": 0.7331657046309579, + "pass@8": 0.8297917311860121, + "pass@16": 0.8931182146188502, + "pass@32": 0.9322322978163884, + "pass@64": 0.9567219108599694, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.36060474537037035, + "pass@2": 0.48475168902498333, + "pass@4": 0.6023837983793693, + "pass@8": 0.6990689294100764, + "pass@16": 0.7706865603861656, + "pass@32": 0.8230004455601055, + "pass@64": 0.8626431069718627, + "pass@128": 0.8950617283950617 + }, + "14": { + "pass@1": 0.29214577151335314, + "pass@2": 0.4008234713427883, + "pass@4": 0.5074832745313363, + "pass@8": 0.601375084942065, + "pass@16": 0.6789703942326578, + "pass@32": 0.7406445380246015, + "pass@64": 0.789367599624807, + "pass@128": 0.8308605341246291 + }, + "15": { + "pass@1": 0.1976514274691358, + "pass@2": 0.29131314097890537, + "pass@4": 0.3921881697658164, + "pass@8": 0.4855609918599434, + "pass@16": 0.5662116470586928, + "pass@32": 0.6373774462304737, + "pass@64": 0.6977827741321524, + "pass@128": 0.7438271604938271 + }, + "16": { + "pass@1": 0.18706213126843657, + "pass@2": 0.25359985076533575, + "pass@4": 0.3209949519584388, + "pass@8": 0.38554267819668736, + "pass@16": 0.4486018761610928, + "pass@32": 0.5138035329859271, + "pass@64": 0.5784656480723253, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.16479198042168675, + "pass@2": 0.23328031614647562, + "pass@4": 0.30360634128188774, + "pass@8": 0.3691675256281784, + "pass@16": 0.42690508197986565, + "pass@32": 0.47893424285590325, + "pass@64": 0.5317847410416872, + "pass@128": 0.5933734939759037 + }, + "18": { + "pass@1": 0.17295866935483872, + "pass@2": 0.24062341249682495, + "pass@4": 0.3112551211340518, + "pass@8": 0.37637725350191814, + "pass@16": 0.4330918948484191, + "pass@32": 0.48455445994709084, + "pass@64": 0.5325561649996549, + "pass@128": 0.5709677419354838 + }, + "19": { + "pass@1": 0.1567692598187311, + "pass@2": 0.21468178330994123, + "pass@4": 0.2714260056616788, + "pass@8": 0.3231161448695808, + "pass@16": 0.36890863896298454, + "pass@32": 0.4108112826103503, + "pass@64": 0.449994222417303, + "pass@128": 0.486404833836858 + }, + "20": { + "pass@1": 0.13527590090090091, + "pass@2": 0.1969527352155305, + "pass@4": 0.26444538830168735, + "pass@8": 0.330625251893072, + "pass@16": 0.3876755280995512, + "pass@32": 0.43574100762264556, + "pass@64": 0.48261430489771634, + "pass@128": 0.5345345345345346 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep2-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fa1f8f67391136d450dc2797c829546e155ec7c7 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep2-block4/composition/autoregressive", "results_dir": "results/SDAR-100M-ep2-block4/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep2-block4/composition/autoregressive/checkpoint-42828_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1020120, "value_mismatch": 13592, "dependency_mismatch": 129311, "answer_mismatch": 12116, "extra_nodes": 11386, "missing_nodes": 10062, "perfect_with_extra": 553, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 336159, "dependency_mismatch": 41915, "value_mismatch": 4624, "answer_mismatch": 4063, "extra_nodes": 3548, "missing_nodes": 3197, "perfect_with_extra": 84}, "teachers_in_school": {"total": 390656, "perfect": 345709, "value_mismatch": 4598, "answer_mismatch": 4267, "dependency_mismatch": 44045, "missing_nodes": 3532, "extra_nodes": 3866, "perfect_with_extra": 146}, "movie_festival_awards": {"total": 382336, "perfect": 338252, "dependency_mismatch": 43351, "value_mismatch": 4370, "answer_mismatch": 3786, "extra_nodes": 3972, "perfect_with_extra": 323, "missing_nodes": 3333}}, "results_path": "results/SDAR-100M-ep2-block4/composition/autoregressive/graphs/checkpoint-42828_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep2-block4/composition/autoregressive/checkpoint-42828_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 270566, "value_mismatch": 788805, "dependency_mismatch": 965724, "answer_mismatch": 677721, "extra_nodes": 60932, "missing_nodes": 604989, "perfect_with_extra": 73, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 268626, "dependency_mismatch": 326724, "answer_mismatch": 230895, "extra_nodes": 24840, "missing_nodes": 215645, "perfect": 90318, "perfect_with_extra": 43}, "movie_festival_awards": {"total": 418816, "perfect": 87458, "value_mismatch": 258859, "dependency_mismatch": 317412, "answer_mismatch": 222740, "extra_nodes": 17921, "missing_nodes": 191148, "perfect_with_extra": 13}, "teachers_in_school": {"total": 430464, "perfect": 92790, "dependency_mismatch": 321588, "value_mismatch": 261320, "answer_mismatch": 224086, "extra_nodes": 18171, "missing_nodes": 198196, "perfect_with_extra": 17}}, "results_path": "results/SDAR-100M-ep2-block4/composition/autoregressive/graphs/checkpoint-42828_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1290686, "value_mismatch": 802397, "dependency_mismatch": 1095035, "answer_mismatch": 689837, "extra_nodes": 72318, "missing_nodes": 615051, "perfect_with_extra": 626, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep2-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 426477, "dependency_mismatch": 368639, "value_mismatch": 273250, "answer_mismatch": 234958, "extra_nodes": 28388, "missing_nodes": 218842, "perfect_with_extra": 127}, "teachers_in_school": {"total": 821120, "perfect": 438499, "value_mismatch": 265918, "answer_mismatch": 228353, "dependency_mismatch": 365633, "missing_nodes": 201728, "extra_nodes": 22037, "perfect_with_extra": 163}, "movie_festival_awards": {"total": 801152, "perfect": 425710, "dependency_mismatch": 360763, "value_mismatch": 263229, "answer_mismatch": 226526, "extra_nodes": 21893, "perfect_with_extra": 336, "missing_nodes": 194481}}}} diff --git a/SDAR-100M-ep2-block4/composition/autoregressive/graphs/results.json b/SDAR-100M-ep2-block4/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..cf8972c1f5ed68d0d946a8b3deb693a1b51c9640 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-42828_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875, + "pass@2": 0.99375, + "pass@4": 0.996875, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9555555555555556, + "pass@2": 0.9841269841269841, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9670658682634731, + "pass@2": 0.9880239520958084, + "pass@4": 0.9910179640718563, + "pass@8": 0.9940119760479041, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.878698224852071, + "pass@2": 0.9378698224852071, + "pass@4": 0.9585798816568047, + "pass@8": 0.985207100591716, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8205128205128205, + "pass@2": 0.8717948717948718, + "pass@4": 0.9487179487179487, + "pass@8": 0.9775641025641025, + "pass@16": 0.9967948717948718, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7323943661971831, + "pass@2": 0.8253521126760563, + "pass@4": 0.9154929577464789, + "pass@8": 0.9690140845070423, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6312684365781711, + "pass@2": 0.6902654867256637, + "pass@4": 0.7817109144542773, + "pass@8": 0.8525073746312685, + "pass@16": 0.8967551622418879, + "pass@32": 0.9203539823008849, + "pass@64": 0.9351032448377581, + "pass@128": 0.9380530973451328 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9555555555555556, + "6": 0.9670658682634731, + "7": 0.878698224852071, + "8": 0.8205128205128205, + "9": 0.7323943661971831, + "10": 0.6312684365781711 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9841269841269841, + "6": 0.9880239520958084, + "7": 0.9378698224852071, + "8": 0.8717948717948718, + "9": 0.8253521126760563, + "10": 0.6902654867256637 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 0.9910179640718563, + "7": 0.9585798816568047, + "8": 0.9487179487179487, + "9": 0.9154929577464789, + "10": 0.7817109144542773 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9940119760479041, + "7": 0.985207100591716, + "8": 0.9775641025641025, + "9": 0.9690140845070423, + "10": 0.8525073746312685 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 0.9887323943661972, + "10": 0.8967551622418879 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9203539823008849 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9351032448377581 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9380530973451328 + } + }, + "total_pass_at_k": { + "pass@1": 0.8831475852752448, + "pass@2": 0.9192840256670044, + "pass@4": 0.9533941236068896, + "pass@8": 0.9746707193515705, + "pass@16": 0.9858156028368794, + "pass@32": 0.9905437352245863, + "pass@64": 0.9925700776764607, + "pass@128": 0.9929078014184397 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9970845481049563, + "pass@2": 0.9970845481049563, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9470588235294117, + "pass@2": 0.9647058823529412, + "pass@4": 0.9852941176470589, + "pass@8": 0.9970588235294118, + "pass@16": 0.9970588235294118, + "pass@32": 0.9970588235294118, + "pass@64": 0.9970588235294118, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9779179810725552, + "pass@2": 0.9810725552050473, + "pass@4": 0.9905362776025236, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8860759493670886, + "pass@2": 0.9177215189873418, + "pass@4": 0.9493670886075949, + "pass@8": 0.9810126582278481, + "pass@16": 0.9968354430379747, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7897897897897898, + "pass@2": 0.8798798798798799, + "pass@4": 0.948948948948949, + "pass@8": 0.972972972972973, + "pass@16": 0.990990990990991, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7684887459807074, + "pass@2": 0.819935691318328, + "pass@4": 0.8906752411575563, + "pass@8": 0.9485530546623794, + "pass@16": 0.9935691318327974, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6895522388059702, + "pass@4": 0.7671641791044777, + "pass@8": 0.835820895522388, + "pass@16": 0.8925373134328358, + "pass@32": 0.9283582089552239, + "pass@64": 0.9582089552238806, + "pass@128": 0.9641791044776119 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9883040935672515, + "5": 0.9470588235294117, + "6": 0.9779179810725552, + "7": 0.8860759493670886, + "8": 0.7897897897897898, + "9": 0.7684887459807074, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 1.0, + "3": 0.9970845481049563, + "4": 0.9941520467836257, + "5": 0.9647058823529412, + "6": 0.9810725552050473, + "7": 0.9177215189873418, + "8": 0.8798798798798799, + "9": 0.819935691318328, + "10": 0.6895522388059702 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9852941176470589, + "6": 0.9905362776025236, + "7": 0.9493670886075949, + "8": 0.948948948948949, + "9": 0.8906752411575563, + "10": 0.7671641791044777 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 0.9968454258675079, + "7": 0.9810126582278481, + "8": 0.972972972972973, + "9": 0.9485530546623794, + "10": 0.835820895522388 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 0.9968354430379747, + "8": 0.990990990990991, + "9": 0.9935691318327974, + "10": 0.8925373134328358 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9283582089552239 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9970588235294118, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9582089552238806 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9641791044776119 + } + }, + "total_pass_at_k": { + "pass@1": 0.884164713759625, + "pass@2": 0.9173083361232005, + "pass@4": 0.9484432541011047, + "pass@8": 0.9705390023434884, + "pass@16": 0.9856042852360227, + "pass@32": 0.9916303983930365, + "pass@64": 0.9949782390358219, + "pass@128": 0.9959825912286575 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9763313609467456, + "pass@2": 0.9881656804733728, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.936231884057971, + "pass@2": 0.9826086956521739, + "pass@4": 0.991304347826087, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9770773638968482, + "pass@2": 0.9799426934097422, + "pass@4": 0.9856733524355301, + "pass@8": 0.9885386819484241, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8786127167630058, + "pass@2": 0.9190751445086706, + "pass@4": 0.9624277456647399, + "pass@8": 0.9855491329479769, + "pass@16": 0.9942196531791907, + "pass@32": 0.9971098265895953, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7915492957746478, + "pass@2": 0.8732394366197183, + "pass@4": 0.9267605633802817, + "pass@8": 0.9774647887323944, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7125748502994012, + "pass@2": 0.8143712574850299, + "pass@4": 0.8772455089820359, + "pass@8": 0.9431137724550899, + "pass@16": 0.9760479041916168, + "pass@32": 0.9910179640718563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6380368098159509, + "pass@2": 0.7085889570552147, + "pass@4": 0.7668711656441718, + "pass@8": 0.8404907975460123, + "pass@16": 0.9049079754601227, + "pass@32": 0.9202453987730062, + "pass@64": 0.9478527607361963, + "pass@128": 0.9631901840490797 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9763313609467456, + "5": 0.936231884057971, + "6": 0.9770773638968482, + "7": 0.8786127167630058, + "8": 0.7915492957746478, + "9": 0.7125748502994012, + "10": 0.6380368098159509 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9881656804733728, + "5": 0.9826086956521739, + "6": 0.9799426934097422, + "7": 0.9190751445086706, + "8": 0.8732394366197183, + "9": 0.8143712574850299, + "10": 0.7085889570552147 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.991304347826087, + "6": 0.9856733524355301, + "7": 0.9624277456647399, + "8": 0.9267605633802817, + "9": 0.8772455089820359, + "10": 0.7668711656441718 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9885386819484241, + "7": 0.9855491329479769, + "8": 0.9774647887323944, + "9": 0.9431137724550899, + "10": 0.8404907975460123 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 1.0, + "9": 0.9760479041916168, + "10": 0.9049079754601227 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9971098265895953, + "8": 1.0, + "9": 0.9910179640718563, + "10": 0.9202453987730062 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9478527607361963 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9631901840490797 + } + }, + "total_pass_at_k": { + "pass@1": 0.8794233289646134, + "pass@2": 0.9190694626474443, + "pass@4": 0.9459370904325033, + "pass@8": 0.9711664482306684, + "pass@16": 0.9862385321100917, + "pass@32": 0.990170380078637, + "pass@64": 0.994429882044561, + "pass@128": 0.9960681520314548 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.984, + "pass@2": 0.992, + "pass@4": 0.997, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.946, + "pass@2": 0.977, + "pass@4": 0.992, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.974, + "pass@2": 0.983, + "pass@4": 0.989, + "pass@8": 0.993, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.881, + "pass@2": 0.925, + "pass@4": 0.957, + "pass@8": 0.984, + "pass@16": 0.996, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8, + "pass@2": 0.875, + "pass@4": 0.941, + "pass@8": 0.976, + "pass@16": 0.996, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.737, + "pass@2": 0.82, + "pass@4": 0.895, + "pass@8": 0.954, + "pass@16": 0.986, + "pass@32": 0.996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.619, + "pass@2": 0.696, + "pass@4": 0.772, + "pass@8": 0.843, + "pass@16": 0.898, + "pass@32": 0.923, + "pass@64": 0.947, + "pass@128": 0.955 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.999, + "4": 0.984, + "5": 0.946, + "6": 0.974, + "7": 0.881, + "8": 0.8, + "9": 0.737, + "10": 0.619 + }, + "pass@2": { + "2": 1.0, + "3": 0.999, + "4": 0.992, + "5": 0.977, + "6": 0.983, + "7": 0.925, + "8": 0.875, + "9": 0.82, + "10": 0.696 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.992, + "6": 0.989, + "7": 0.957, + "8": 0.941, + "9": 0.895, + "10": 0.772 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.993, + "7": 0.984, + "8": 0.976, + "9": 0.954, + "10": 0.843 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 0.998, + "7": 0.996, + "8": 0.996, + "9": 0.986, + "10": 0.898 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.996, + "10": 0.923 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.947 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.955 + } + }, + "total_pass_at_k": { + "pass@1": 0.8822222222222222, + "pass@2": 0.9185555555555556, + "pass@4": 0.9492222222222222, + "pass@8": 0.9721111111111111, + "pass@16": 0.9858888888888889, + "pass@32": 0.9907777777777778, + "pass@64": 0.994, + "pass@128": 0.995 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-42828_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5439093484419264, + "pass@2": 0.623229461756374, + "pass@4": 0.7025495750708215, + "pass@8": 0.7932011331444759, + "pass@16": 0.8526912181303116, + "pass@32": 0.9008498583569405, + "pass@64": 0.9320113314447592, + "pass@128": 0.9490084985835694 + }, + "12": { + "pass@1": 0.5423728813559322, + "pass@2": 0.6016949152542372, + "pass@4": 0.6694915254237288, + "pass@8": 0.748587570621469, + "pass@16": 0.7909604519774012, + "pass@32": 0.847457627118644, + "pass@64": 0.867231638418079, + "pass@128": 0.8813559322033898 + }, + "13": { + "pass@1": 0.329153605015674, + "pass@2": 0.45454545454545453, + "pass@4": 0.5078369905956113, + "pass@8": 0.5893416927899686, + "pass@16": 0.6363636363636364, + "pass@32": 0.7021943573667712, + "pass@64": 0.7304075235109718, + "pass@128": 0.7711598746081505 + }, + "14": { + "pass@1": 0.2803738317757009, + "pass@2": 0.35514018691588783, + "pass@4": 0.3956386292834891, + "pass@8": 0.4672897196261682, + "pass@16": 0.5202492211838006, + "pass@32": 0.5700934579439252, + "pass@64": 0.6199376947040498, + "pass@128": 0.7009345794392523 + }, + "15": { + "pass@1": 0.18208092485549132, + "pass@2": 0.23410404624277456, + "pass@4": 0.28901734104046245, + "pass@8": 0.3352601156069364, + "pass@16": 0.3872832369942196, + "pass@32": 0.44508670520231214, + "pass@64": 0.5057803468208093, + "pass@128": 0.5635838150289018 + }, + "16": { + "pass@1": 0.08387096774193549, + "pass@2": 0.1032258064516129, + "pass@4": 0.14838709677419354, + "pass@8": 0.18064516129032257, + "pass@16": 0.21935483870967742, + "pass@32": 0.24193548387096775, + "pass@64": 0.29354838709677417, + "pass@128": 0.3258064516129032 + }, + "17": { + "pass@1": 0.049132947976878616, + "pass@2": 0.08092485549132948, + "pass@4": 0.08959537572254335, + "pass@8": 0.11271676300578035, + "pass@16": 0.1416184971098266, + "pass@32": 0.16473988439306358, + "pass@64": 0.1907514450867052, + "pass@128": 0.22254335260115607 + }, + "18": { + "pass@1": 0.022857142857142857, + "pass@2": 0.03142857142857143, + "pass@4": 0.045714285714285714, + "pass@8": 0.04857142857142857, + "pass@16": 0.06571428571428571, + "pass@32": 0.08, + "pass@64": 0.10571428571428572, + "pass@128": 0.13142857142857142 + }, + "19": { + "pass@1": 0.009009009009009009, + "pass@2": 0.009009009009009009, + "pass@4": 0.015015015015015015, + "pass@8": 0.021021021021021023, + "pass@16": 0.021021021021021023, + "pass@32": 0.02702702702702703, + "pass@64": 0.03003003003003003, + "pass@128": 0.04504504504504504 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.006006006006006006, + "pass@4": 0.006006006006006006, + "pass@8": 0.009009009009009009, + "pass@16": 0.015015015015015015, + "pass@32": 0.018018018018018018, + "pass@64": 0.024024024024024024, + "pass@128": 0.03303303303303303 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5439093484419264, + "12": 0.5423728813559322, + "13": 0.329153605015674, + "14": 0.2803738317757009, + "15": 0.18208092485549132, + "16": 0.08387096774193549, + "17": 0.049132947976878616, + "18": 0.022857142857142857, + "19": 0.009009009009009009, + "20": 0.0 + }, + "pass@2": { + "11": 0.623229461756374, + "12": 0.6016949152542372, + "13": 0.45454545454545453, + "14": 0.35514018691588783, + "15": 0.23410404624277456, + "16": 0.1032258064516129, + "17": 0.08092485549132948, + "18": 0.03142857142857143, + "19": 0.009009009009009009, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.7025495750708215, + "12": 0.6694915254237288, + "13": 0.5078369905956113, + "14": 0.3956386292834891, + "15": 0.28901734104046245, + "16": 0.14838709677419354, + "17": 0.08959537572254335, + "18": 0.045714285714285714, + "19": 0.015015015015015015, + "20": 0.006006006006006006 + }, + "pass@8": { + "11": 0.7932011331444759, + "12": 0.748587570621469, + "13": 0.5893416927899686, + "14": 0.4672897196261682, + "15": 0.3352601156069364, + "16": 0.18064516129032257, + "17": 0.11271676300578035, + "18": 0.04857142857142857, + "19": 0.021021021021021023, + "20": 0.009009009009009009 + }, + "pass@16": { + "11": 0.8526912181303116, + "12": 0.7909604519774012, + "13": 0.6363636363636364, + "14": 0.5202492211838006, + "15": 0.3872832369942196, + "16": 0.21935483870967742, + "17": 0.1416184971098266, + "18": 0.06571428571428571, + "19": 0.021021021021021023, + "20": 0.015015015015015015 + }, + "pass@32": { + "11": 0.9008498583569405, + "12": 0.847457627118644, + "13": 0.7021943573667712, + "14": 0.5700934579439252, + "15": 0.44508670520231214, + "16": 0.24193548387096775, + "17": 0.16473988439306358, + "18": 0.08, + "19": 0.02702702702702703, + "20": 0.018018018018018018 + }, + "pass@64": { + "11": 0.9320113314447592, + "12": 0.867231638418079, + "13": 0.7304075235109718, + "14": 0.6199376947040498, + "15": 0.5057803468208093, + "16": 0.29354838709677417, + "17": 0.1907514450867052, + "18": 0.10571428571428572, + "19": 0.03003003003003003, + "20": 0.024024024024024024 + }, + "pass@128": { + "11": 0.9490084985835694, + "12": 0.8813559322033898, + "13": 0.7711598746081505, + "14": 0.7009345794392523, + "15": 0.5635838150289018, + "16": 0.3258064516129032, + "17": 0.22254335260115607, + "18": 0.13142857142857142, + "19": 0.04504504504504504, + "20": 0.03303303303303303 + } + }, + "total_pass_at_k": { + "pass@1": 0.20683506686478453, + "pass@2": 0.2523031203566122, + "pass@4": 0.2894502228826152, + "pass@8": 0.33313521545319463, + "pass@16": 0.36760772659732543, + "pass@32": 0.4023774145616642, + "pass@64": 0.4323922734026746, + "pass@128": 0.4644873699851412 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5619047619047619, + "pass@2": 0.6476190476190476, + "pass@4": 0.746031746031746, + "pass@8": 0.8380952380952381, + "pass@16": 0.9142857142857143, + "pass@32": 0.9492063492063492, + "pass@64": 0.9650793650793651, + "pass@128": 0.9809523809523809 + }, + "12": { + "pass@1": 0.5259938837920489, + "pass@2": 0.5932721712538226, + "pass@4": 0.6666666666666666, + "pass@8": 0.7675840978593272, + "pass@16": 0.8409785932721713, + "pass@32": 0.8685015290519877, + "pass@64": 0.8837920489296636, + "pass@128": 0.8990825688073395 + }, + "13": { + "pass@1": 0.3611111111111111, + "pass@2": 0.4104938271604938, + "pass@4": 0.4845679012345679, + "pass@8": 0.5462962962962963, + "pass@16": 0.6234567901234568, + "pass@32": 0.6666666666666666, + "pass@64": 0.7253086419753086, + "pass@128": 0.7407407407407407 + }, + "14": { + "pass@1": 0.27002967359050445, + "pass@2": 0.3264094955489614, + "pass@4": 0.3887240356083086, + "pass@8": 0.4807121661721068, + "pass@16": 0.5341246290801187, + "pass@32": 0.6231454005934718, + "pass@64": 0.6765578635014837, + "pass@128": 0.7299703264094956 + }, + "15": { + "pass@1": 0.19444444444444445, + "pass@2": 0.24382716049382716, + "pass@4": 0.2932098765432099, + "pass@8": 0.31790123456790126, + "pass@16": 0.3734567901234568, + "pass@32": 0.4506172839506173, + "pass@64": 0.5123456790123457, + "pass@128": 0.5802469135802469 + }, + "16": { + "pass@1": 0.07964601769911504, + "pass@2": 0.12979351032448377, + "pass@4": 0.16224188790560473, + "pass@8": 0.17699115044247787, + "pass@16": 0.24188790560471976, + "pass@32": 0.2949852507374631, + "pass@64": 0.3185840707964602, + "pass@128": 0.3687315634218289 + }, + "17": { + "pass@1": 0.060240963855421686, + "pass@2": 0.06927710843373494, + "pass@4": 0.09939759036144578, + "pass@8": 0.11144578313253012, + "pass@16": 0.1355421686746988, + "pass@32": 0.15963855421686746, + "pass@64": 0.17771084337349397, + "pass@128": 0.22590361445783133 + }, + "18": { + "pass@1": 0.04838709677419355, + "pass@2": 0.06774193548387097, + "pass@4": 0.07419354838709677, + "pass@8": 0.1032258064516129, + "pass@16": 0.11935483870967742, + "pass@32": 0.13548387096774195, + "pass@64": 0.15161290322580645, + "pass@128": 0.1774193548387097 + }, + "19": { + "pass@1": 0.006042296072507553, + "pass@2": 0.012084592145015106, + "pass@4": 0.027190332326283987, + "pass@8": 0.030211480362537766, + "pass@16": 0.030211480362537766, + "pass@32": 0.03625377643504532, + "pass@64": 0.04229607250755287, + "pass@128": 0.0513595166163142 + }, + "20": { + "pass@1": 0.006006006006006006, + "pass@2": 0.006006006006006006, + "pass@4": 0.006006006006006006, + "pass@8": 0.018018018018018018, + "pass@16": 0.021021021021021023, + "pass@32": 0.03303303303303303, + "pass@64": 0.057057057057057055, + "pass@128": 0.06006006006006006 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5619047619047619, + "12": 0.5259938837920489, + "13": 0.3611111111111111, + "14": 0.27002967359050445, + "15": 0.19444444444444445, + "16": 0.07964601769911504, + "17": 0.060240963855421686, + "18": 0.04838709677419355, + "19": 0.006042296072507553, + "20": 0.006006006006006006 + }, + "pass@2": { + "11": 0.6476190476190476, + "12": 0.5932721712538226, + "13": 0.4104938271604938, + "14": 0.3264094955489614, + "15": 0.24382716049382716, + "16": 0.12979351032448377, + "17": 0.06927710843373494, + "18": 0.06774193548387097, + "19": 0.012084592145015106, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.746031746031746, + "12": 0.6666666666666666, + "13": 0.4845679012345679, + "14": 0.3887240356083086, + "15": 0.2932098765432099, + "16": 0.16224188790560473, + "17": 0.09939759036144578, + "18": 0.07419354838709677, + "19": 0.027190332326283987, + "20": 0.006006006006006006 + }, + "pass@8": { + "11": 0.8380952380952381, + "12": 0.7675840978593272, + "13": 0.5462962962962963, + "14": 0.4807121661721068, + "15": 0.31790123456790126, + "16": 0.17699115044247787, + "17": 0.11144578313253012, + "18": 0.1032258064516129, + "19": 0.030211480362537766, + "20": 0.018018018018018018 + }, + "pass@16": { + "11": 0.9142857142857143, + "12": 0.8409785932721713, + "13": 0.6234567901234568, + "14": 0.5341246290801187, + "15": 0.3734567901234568, + "16": 0.24188790560471976, + "17": 0.1355421686746988, + "18": 0.11935483870967742, + "19": 0.030211480362537766, + "20": 0.021021021021021023 + }, + "pass@32": { + "11": 0.9492063492063492, + "12": 0.8685015290519877, + "13": 0.6666666666666666, + "14": 0.6231454005934718, + "15": 0.4506172839506173, + "16": 0.2949852507374631, + "17": 0.15963855421686746, + "18": 0.13548387096774195, + "19": 0.03625377643504532, + "20": 0.03303303303303303 + }, + "pass@64": { + "11": 0.9650793650793651, + "12": 0.8837920489296636, + "13": 0.7253086419753086, + "14": 0.6765578635014837, + "15": 0.5123456790123457, + "16": 0.3185840707964602, + "17": 0.17771084337349397, + "18": 0.15161290322580645, + "19": 0.04229607250755287, + "20": 0.057057057057057055 + }, + "pass@128": { + "11": 0.9809523809523809, + "12": 0.8990825688073395, + "13": 0.7407407407407407, + "14": 0.7299703264094956, + "15": 0.5802469135802469, + "16": 0.3687315634218289, + "17": 0.22590361445783133, + "18": 0.1774193548387097, + "19": 0.0513595166163142, + "20": 0.06006006006006006 + } + }, + "total_pass_at_k": { + "pass@1": 0.20965770171149145, + "pass@2": 0.24877750611246943, + "pass@4": 0.2927872860635697, + "pass@8": 0.3367970660146699, + "pass@16": 0.3811124694376528, + "pass@32": 0.4196210268948655, + "pass@64": 0.44896088019559904, + "pass@128": 0.4795232273838631 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6174698795180723, + "pass@2": 0.677710843373494, + "pass@4": 0.7650602409638554, + "pass@8": 0.8403614457831325, + "pass@16": 0.8975903614457831, + "pass@32": 0.9216867469879518, + "pass@64": 0.9548192771084337, + "pass@128": 0.9608433734939759 + }, + "12": { + "pass@1": 0.5172413793103449, + "pass@2": 0.5893416927899686, + "pass@4": 0.6677115987460815, + "pass@8": 0.7586206896551724, + "pass@16": 0.7899686520376176, + "pass@32": 0.8369905956112853, + "pass@64": 0.8557993730407524, + "pass@128": 0.877742946708464 + }, + "13": { + "pass@1": 0.39775910364145656, + "pass@2": 0.4957983193277311, + "pass@4": 0.5742296918767507, + "pass@8": 0.6498599439775911, + "pass@16": 0.7198879551820728, + "pass@32": 0.773109243697479, + "pass@64": 0.8123249299719888, + "pass@128": 0.8403361344537815 + }, + "14": { + "pass@1": 0.2573099415204678, + "pass@2": 0.2982456140350877, + "pass@4": 0.3742690058479532, + "pass@8": 0.4444444444444444, + "pass@16": 0.5233918128654971, + "pass@32": 0.5906432748538012, + "pass@64": 0.6461988304093568, + "pass@128": 0.7046783625730995 + }, + "15": { + "pass@1": 0.20303030303030303, + "pass@2": 0.24242424242424243, + "pass@4": 0.31212121212121213, + "pass@8": 0.37272727272727274, + "pass@16": 0.4303030303030303, + "pass@32": 0.49393939393939396, + "pass@64": 0.5272727272727272, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.10826210826210826, + "pass@2": 0.13105413105413105, + "pass@4": 0.150997150997151, + "pass@8": 0.18518518518518517, + "pass@16": 0.2336182336182336, + "pass@32": 0.2934472934472934, + "pass@64": 0.33903133903133903, + "pass@128": 0.3646723646723647 + }, + "17": { + "pass@1": 0.059006211180124224, + "pass@2": 0.07763975155279502, + "pass@4": 0.09316770186335403, + "pass@8": 0.10869565217391304, + "pass@16": 0.15217391304347827, + "pass@32": 0.18633540372670807, + "pass@64": 0.2453416149068323, + "pass@128": 0.2701863354037267 + }, + "18": { + "pass@1": 0.03235294117647059, + "pass@2": 0.05, + "pass@4": 0.06764705882352941, + "pass@8": 0.1, + "pass@16": 0.12058823529411765, + "pass@32": 0.13823529411764707, + "pass@64": 0.15, + "pass@128": 0.18235294117647058 + }, + "19": { + "pass@1": 0.011904761904761904, + "pass@2": 0.011904761904761904, + "pass@4": 0.017857142857142856, + "pass@8": 0.026785714285714284, + "pass@16": 0.02976190476190476, + "pass@32": 0.03571428571428571, + "pass@64": 0.03869047619047619, + "pass@128": 0.05357142857142857 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.005988023952095809, + "pass@4": 0.005988023952095809, + "pass@8": 0.014970059880239521, + "pass@16": 0.020958083832335328, + "pass@32": 0.023952095808383235, + "pass@64": 0.03293413173652695, + "pass@128": 0.03592814371257485 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6174698795180723, + "12": 0.5172413793103449, + "13": 0.39775910364145656, + "14": 0.2573099415204678, + "15": 0.20303030303030303, + "16": 0.10826210826210826, + "17": 0.059006211180124224, + "18": 0.03235294117647059, + "19": 0.011904761904761904, + "20": 0.0 + }, + "pass@2": { + "11": 0.677710843373494, + "12": 0.5893416927899686, + "13": 0.4957983193277311, + "14": 0.2982456140350877, + "15": 0.24242424242424243, + "16": 0.13105413105413105, + "17": 0.07763975155279502, + "18": 0.05, + "19": 0.011904761904761904, + "20": 0.005988023952095809 + }, + "pass@4": { + "11": 0.7650602409638554, + "12": 0.6677115987460815, + "13": 0.5742296918767507, + "14": 0.3742690058479532, + "15": 0.31212121212121213, + "16": 0.150997150997151, + "17": 0.09316770186335403, + "18": 0.06764705882352941, + "19": 0.017857142857142856, + "20": 0.005988023952095809 + }, + "pass@8": { + "11": 0.8403614457831325, + "12": 0.7586206896551724, + "13": 0.6498599439775911, + "14": 0.4444444444444444, + "15": 0.37272727272727274, + "16": 0.18518518518518517, + "17": 0.10869565217391304, + "18": 0.1, + "19": 0.026785714285714284, + "20": 0.014970059880239521 + }, + "pass@16": { + "11": 0.8975903614457831, + "12": 0.7899686520376176, + "13": 0.7198879551820728, + "14": 0.5233918128654971, + "15": 0.4303030303030303, + "16": 0.2336182336182336, + "17": 0.15217391304347827, + "18": 0.12058823529411765, + "19": 0.02976190476190476, + "20": 0.020958083832335328 + }, + "pass@32": { + "11": 0.9216867469879518, + "12": 0.8369905956112853, + "13": 0.773109243697479, + "14": 0.5906432748538012, + "15": 0.49393939393939396, + "16": 0.2934472934472934, + "17": 0.18633540372670807, + "18": 0.13823529411764707, + "19": 0.03571428571428571, + "20": 0.023952095808383235 + }, + "pass@64": { + "11": 0.9548192771084337, + "12": 0.8557993730407524, + "13": 0.8123249299719888, + "14": 0.6461988304093568, + "15": 0.5272727272727272, + "16": 0.33903133903133903, + "17": 0.2453416149068323, + "18": 0.15, + "19": 0.03869047619047619, + "20": 0.03293413173652695 + }, + "pass@128": { + "11": 0.9608433734939759, + "12": 0.877742946708464, + "13": 0.8403361344537815, + "14": 0.7046783625730995, + "15": 0.5727272727272728, + "16": 0.3646723646723647, + "17": 0.2701863354037267, + "18": 0.18235294117647058, + "19": 0.05357142857142857, + "20": 0.03592814371257485 + } + }, + "total_pass_at_k": { + "pass@1": 0.21974427594409754, + "pass@2": 0.2575081772227178, + "pass@4": 0.3024085637823372, + "pass@8": 0.3496877787689563, + "pass@16": 0.39161462979482603, + "pass@32": 0.4293785310734463, + "pass@64": 0.46030330062444247, + "pass@128": 0.4864704133214392 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.574, + "pass@2": 0.649, + "pass@4": 0.737, + "pass@8": 0.823, + "pass@16": 0.887, + "pass@32": 0.923, + "pass@64": 0.95, + "pass@128": 0.963 + }, + "12": { + "pass@1": 0.529, + "pass@2": 0.595, + "pass@4": 0.668, + "pass@8": 0.758, + "pass@16": 0.807, + "pass@32": 0.851, + "pass@64": 0.869, + "pass@128": 0.886 + }, + "13": { + "pass@1": 0.364, + "pass@2": 0.455, + "pass@4": 0.524, + "pass@8": 0.597, + "pass@16": 0.662, + "pass@32": 0.716, + "pass@64": 0.758, + "pass@128": 0.786 + }, + "14": { + "pass@1": 0.269, + "pass@2": 0.326, + "pass@4": 0.386, + "pass@8": 0.464, + "pass@16": 0.526, + "pass@32": 0.595, + "pass@64": 0.648, + "pass@128": 0.712 + }, + "15": { + "pass@1": 0.193, + "pass@2": 0.24, + "pass@4": 0.298, + "pass@8": 0.342, + "pass@16": 0.397, + "pass@32": 0.463, + "pass@64": 0.515, + "pass@128": 0.572 + }, + "16": { + "pass@1": 0.091, + "pass@2": 0.122, + "pass@4": 0.154, + "pass@8": 0.181, + "pass@16": 0.232, + "pass@32": 0.278, + "pass@64": 0.318, + "pass@128": 0.354 + }, + "17": { + "pass@1": 0.056, + "pass@2": 0.076, + "pass@4": 0.094, + "pass@8": 0.111, + "pass@16": 0.143, + "pass@32": 0.17, + "pass@64": 0.204, + "pass@128": 0.239 + }, + "18": { + "pass@1": 0.034, + "pass@2": 0.049, + "pass@4": 0.062, + "pass@8": 0.083, + "pass@16": 0.101, + "pass@32": 0.117, + "pass@64": 0.135, + "pass@128": 0.163 + }, + "19": { + "pass@1": 0.009, + "pass@2": 0.011, + "pass@4": 0.02, + "pass@8": 0.026, + "pass@16": 0.027, + "pass@32": 0.033, + "pass@64": 0.037, + "pass@128": 0.05 + }, + "20": { + "pass@1": 0.002, + "pass@2": 0.006, + "pass@4": 0.006, + "pass@8": 0.014, + "pass@16": 0.019, + "pass@32": 0.025, + "pass@64": 0.038, + "pass@128": 0.043 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.574, + "12": 0.529, + "13": 0.364, + "14": 0.269, + "15": 0.193, + "16": 0.091, + "17": 0.056, + "18": 0.034, + "19": 0.009, + "20": 0.002 + }, + "pass@2": { + "11": 0.649, + "12": 0.595, + "13": 0.455, + "14": 0.326, + "15": 0.24, + "16": 0.122, + "17": 0.076, + "18": 0.049, + "19": 0.011, + "20": 0.006 + }, + "pass@4": { + "11": 0.737, + "12": 0.668, + "13": 0.524, + "14": 0.386, + "15": 0.298, + "16": 0.154, + "17": 0.094, + "18": 0.062, + "19": 0.02, + "20": 0.006 + }, + "pass@8": { + "11": 0.823, + "12": 0.758, + "13": 0.597, + "14": 0.464, + "15": 0.342, + "16": 0.181, + "17": 0.111, + "18": 0.083, + "19": 0.026, + "20": 0.014 + }, + "pass@16": { + "11": 0.887, + "12": 0.807, + "13": 0.662, + "14": 0.526, + "15": 0.397, + "16": 0.232, + "17": 0.143, + "18": 0.101, + "19": 0.027, + "20": 0.019 + }, + "pass@32": { + "11": 0.923, + "12": 0.851, + "13": 0.716, + "14": 0.595, + "15": 0.463, + "16": 0.278, + "17": 0.17, + "18": 0.117, + "19": 0.033, + "20": 0.025 + }, + "pass@64": { + "11": 0.95, + "12": 0.869, + "13": 0.758, + "14": 0.648, + "15": 0.515, + "16": 0.318, + "17": 0.204, + "18": 0.135, + "19": 0.037, + "20": 0.038 + }, + "pass@128": { + "11": 0.963, + "12": 0.886, + "13": 0.786, + "14": 0.712, + "15": 0.572, + "16": 0.354, + "17": 0.239, + "18": 0.163, + "19": 0.05, + "20": 0.043 + } + }, + "total_pass_at_k": { + "pass@1": 0.2121, + "pass@2": 0.2529, + "pass@4": 0.2949, + "pass@8": 0.3399, + "pass@16": 0.3801, + "pass@32": 0.4171, + "pass@64": 0.4472, + "pass@128": 0.4768 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep2-block4/composition/autoregressive/summary.csv b/SDAR-100M-ep2-block4/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..92148b713c8dc99d728acd522d213c49dac7c206 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-42828,0.9894826388888889,0.470528125,0.7163486842105263,0.0,0.0,0.0,200.09417708333334,344.22949609375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep2-block4/composition/autoregressive/summary.json b/SDAR-100M-ep2-block4/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a1db0a2eb7e2c6f89f9d8bceed960bc59df2982f --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-42828", + "id_acc": 0.9894826388888889, + "ood_acc": 0.470528125, + "total_acc": 0.7163486842105263, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.09417708333334, + "ood_avg_resp_len": 344.22949609375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 574771, + "answer_accuracy": 0.7098321905627569, + "avg_response_len": 262.7198664242807, + "resp_tokens_sum": 212731632, + "pass_at_k": { + "pass@1": 0.7098321905627569, + "pass@2": 0.7544343266197929, + "pass@4": 0.7921442557488494, + "pass@8": 0.8234180639046024, + "pass@16": 0.8490259583479542, + "pass@32": 0.8706260639313723, + "pass@64": 0.8894478974698937, + "pass@128": 0.9054694909895669 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.938191371681416, + "pass@2": 0.9507075581260308, + "pass@4": 0.9574170305592333, + "pass@8": 0.9626028420434352, + "pass@16": 0.9676380108556495, + "pass@32": 0.9733761722067171, + "pass@64": 0.9794348064762847, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999044342507645, + "pass@2": 0.9999988712706783, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9985107421875, + "pass@2": 0.999341781496063, + "pass@4": 0.9998666789698163, + "pass@8": 0.9999951531193496, + "pass@16": 0.9999999963269612, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.994469246031746, + "pass@2": 0.9972573740782403, + "pass@4": 0.9988838163086756, + "pass@8": 0.999703452073703, + "pass@16": 0.999977476461648, + "pass@32": 0.9999999361951407, + "pass@64": 0.999999999999996, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9982924775449101, + "pass@2": 0.9991354643076052, + "pass@4": 0.9997573747330986, + "pass@8": 0.9999824898608188, + "pass@16": 0.9999999386330103, + "pass@32": 0.9999999999999043, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.993805473372781, + "pass@2": 0.9953204351674976, + "pass@4": 0.9963636679290828, + "pass@8": 0.9978022115058525, + "pass@16": 0.9992398726779238, + "pass@32": 0.9999300060627638, + "pass@64": 0.9999998828759263, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918619791666666, + "pass@2": 0.9953259861447606, + "pass@4": 0.9971175333852498, + "pass@8": 0.9986976735909107, + "pass@16": 0.9997214713252663, + "pass@32": 0.9999882289469381, + "pass@64": 0.999999996955866, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9906029929577465, + "pass@2": 0.9971175973161807, + "pass@4": 0.9994117220382662, + "pass@8": 0.9999626653484882, + "pass@16": 0.9999997920087084, + "pass@32": 0.9999999999982944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.91983888101983, + "pass@2": 0.946985972318262, + "pass@4": 0.9615740933410231, + "pass@8": 0.9704805355870788, + "pass@16": 0.9768131739959114, + "pass@32": 0.9819557012317468, + "pass@64": 0.98653133809928, + "pass@128": 0.9915014164305949 + }, + "12": { + "pass@1": 0.822607697740113, + "pass@2": 0.8753218281507184, + "pass@4": 0.9018579549907951, + "pass@8": 0.9178689747222858, + "pass@16": 0.9318606981543764, + "pass@32": 0.9459486582670955, + "pass@64": 0.9575915981731089, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.6809855015673981, + "pass@2": 0.7670173000024677, + "pass@4": 0.8201065466346483, + "pass@8": 0.8561603281573825, + "pass@16": 0.882208233859429, + "pass@32": 0.899328313039419, + "pass@64": 0.9099842471218675, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.5971816588785047, + "pass@2": 0.6971129375475261, + "pass@4": 0.7666536472660535, + "pass@8": 0.8149757096073096, + "pass@16": 0.8522813198771078, + "pass@32": 0.8827592151983151, + "pass@64": 0.907156078560458, + "pass@128": 0.9252336448598131 + }, + "15": { + "pass@1": 0.49001986994219654, + "pass@2": 0.5955823158936784, + "pass@4": 0.682237448552312, + "pass@8": 0.7510216556505724, + "pass@16": 0.8021241041232691, + "pass@32": 0.840262532442047, + "pass@64": 0.871512978537945, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.3142641129032258, + "pass@2": 0.42235323533147057, + "pass@4": 0.5219397541033175, + "pass@8": 0.6054150479280793, + "pass@16": 0.6731704410957603, + "pass@32": 0.7300474628311022, + "pass@64": 0.7812425186016846, + "pass@128": 0.8258064516129032 + }, + "17": { + "pass@1": 0.2594608020231214, + "pass@2": 0.3552075747576351, + "pass@4": 0.44889092467126573, + "pass@8": 0.5298944076343943, + "pass@16": 0.5938477057646299, + "pass@32": 0.6458432672898637, + "pass@64": 0.6926852959730377, + "pass@128": 0.7312138728323699 + }, + "18": { + "pass@1": 0.20560267857142858, + "pass@2": 0.2975752249718784, + "pass@4": 0.3935471034870643, + "pass@8": 0.479720452534764, + "pass@16": 0.5518197621634817, + "pass@32": 0.6137227694722202, + "pass@64": 0.6675575336255497, + "pass@128": 0.7114285714285714 + }, + "19": { + "pass@1": 0.16469594594594594, + "pass@2": 0.2434220637960795, + "pass@4": 0.3257808131528604, + "pass@8": 0.4000305781451452, + "pass@16": 0.46376379967114273, + "pass@32": 0.5206510590669114, + "pass@64": 0.571185394124095, + "pass@128": 0.6126126126126126 + }, + "20": { + "pass@1": 0.15843186936936937, + "pass@2": 0.22814768508666133, + "pass@4": 0.30495890800812053, + "pass@8": 0.383387374477363, + "pass@16": 0.4573577428718796, + "pass@32": 0.5261218888926433, + "pass@64": 0.5905595183935609, + "pass@128": 0.6516516516516516 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 592767, + "answer_accuracy": 0.7219005748246298, + "avg_response_len": 273.3977908222915, + "resp_tokens_sum": 224492394, + "pass_at_k": { + "pass@1": 0.7219005748246298, + "pass@2": 0.7684856788653494, + "pass@4": 0.8070809353721593, + "pass@8": 0.8377910566802691, + "pass@16": 0.8615161766040658, + "pass@32": 0.8800848254091402, + "pass@64": 0.89531877294161, + "pass@128": 0.9078721745908028 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9266919095092024, + "pass@2": 0.9459019944688664, + "pass@4": 0.9580926648931151, + "pass@8": 0.9655181079663312, + "pass@16": 0.9705065116715573, + "pass@32": 0.9745265481735442, + "pass@64": 0.9789238933060727, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9993134469696969, + "pass@2": 0.9998590730136006, + "pass@4": 0.9999941839656407, + "pass@8": 0.9999999934122665, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967391304347826, + "pass@2": 0.9973970814789457, + "pass@4": 0.9976917420648507, + "pass@8": 0.9981758657979531, + "pass@16": 0.9988891034656363, + "pass@32": 0.9996345084013638, + "pass@64": 0.99998095094953, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9988135744985673, + "pass@2": 0.9997299652550595, + "pass@4": 0.9999778987153827, + "pass@8": 0.9999998766975703, + "pass@16": 0.9999999999988423, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9965227601156069, + "pass@2": 0.9971318726958264, + "pass@4": 0.9972870163267165, + "pass@8": 0.9974611417052052, + "pass@16": 0.9977897000591688, + "pass@32": 0.9983785444449502, + "pass@64": 0.9992831459651359, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9943221830985915, + "pass@2": 0.9966639403349228, + "pass@4": 0.9979784254080916, + "pass@8": 0.9990666990790485, + "pass@16": 0.9997528296434962, + "pass@32": 0.9999857246359625, + "pass@64": 0.9999999936816814, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9866672904191617, + "pass@2": 0.9917241059220143, + "pass@4": 0.9938411520603335, + "pass@8": 0.9956474719604435, + "pass@16": 0.9972573036700675, + "pass@32": 0.998297458460058, + "pass@64": 0.9992573828256583, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9343232304216867, + "pass@2": 0.9600106874347788, + "pass@4": 0.9705750945288463, + "pass@8": 0.9756324867411611, + "pass@16": 0.9785410759514516, + "pass@32": 0.9804200441280092, + "pass@64": 0.9815556641572076, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8473011363636364, + "pass@2": 0.8946973810875521, + "pass@4": 0.9175820971046328, + "pass@8": 0.9309735895345088, + "pass@16": 0.9410367488667444, + "pass@32": 0.948965953999329, + "pass@64": 0.9554065505058231, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6850709033613446, + "pass@2": 0.7874598166038067, + "pass@4": 0.8496326567372344, + "pass@8": 0.8870374883664758, + "pass@16": 0.9118338376968488, + "pass@32": 0.9300755296021455, + "pass@64": 0.9456425249067281, + "pass@128": 0.957983193277311 + }, + "14": { + "pass@1": 0.6223958333333334, + "pass@2": 0.7197378493806696, + "pass@4": 0.7886473072444895, + "pass@8": 0.8338802937283072, + "pass@16": 0.8612657118281954, + "pass@32": 0.8797224978366543, + "pass@64": 0.8969039766342238, + "pass@128": 0.9152046783625731 + }, + "15": { + "pass@1": 0.5188920454545455, + "pass@2": 0.6297918903602958, + "pass@4": 0.7113841423231179, + "pass@8": 0.7704131884610596, + "pass@16": 0.8132045442176113, + "pass@32": 0.8445315913779556, + "pass@64": 0.8667790210259716, + "pass@128": 0.8818181818181818 + }, + "16": { + "pass@1": 0.31828703703703703, + "pass@2": 0.4308475362855286, + "pass@4": 0.5321735987595564, + "pass@8": 0.6133575183418821, + "pass@16": 0.6771684148710823, + "pass@32": 0.7313473883636334, + "pass@64": 0.7795209648610576, + "pass@128": 0.8176638176638177 + }, + "17": { + "pass@1": 0.29660811335403725, + "pass@2": 0.404082047488629, + "pass@4": 0.5058153114206997, + "pass@8": 0.59019338398362, + "pass@16": 0.6588107355969026, + "pass@32": 0.7140254963144298, + "pass@64": 0.755545829423683, + "pass@128": 0.7888198757763976 + }, + "18": { + "pass@1": 0.25089613970588237, + "pass@2": 0.3484751331635016, + "pass@4": 0.4468376891307704, + "pass@8": 0.5344262099305134, + "pass@16": 0.602894653464828, + "pass@32": 0.652252172748721, + "pass@64": 0.6898146169752933, + "pass@128": 0.7176470588235294 + }, + "19": { + "pass@1": 0.1790829613095238, + "pass@2": 0.2575305528215224, + "pass@4": 0.3426924809622011, + "pass@8": 0.4214982239741903, + "pass@16": 0.48576795980818427, + "pass@32": 0.5364442340319796, + "pass@64": 0.5787094815498106, + "pass@128": 0.6160714285714286 + }, + "20": { + "pass@1": 0.15112743263473055, + "pass@2": 0.22712756082323543, + "pass@4": 0.311789831192508, + "pass@8": 0.3931681982311315, + "pass@16": 0.4636711942826472, + "pass@32": 0.5235392610475729, + "pass@64": 0.5746665246604618, + "pass@128": 0.6167664670658682 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 574622, + "answer_accuracy": 0.7172446676785429, + "avg_response_len": 291.95236484462373, + "resp_tokens_sum": 233898221, + "pass_at_k": { + "pass@1": 0.7172446676785429, + "pass@2": 0.762473200009304, + "pass@4": 0.8005009265997003, + "pass@8": 0.8318729010553301, + "pass@16": 0.8572073607122934, + "pass@32": 0.8776230304314351, + "pass@64": 0.8943118328037649, + "pass@128": 0.9079725195718166 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9461054104477612, + "pass@2": 0.9629205106357972, + "pass@4": 0.9724614089283613, + "pass@8": 0.9784954370979474, + "pass@16": 0.9833977623356791, + "pass@32": 0.9880483413656285, + "pass@64": 0.9915697437769286, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999543128654971, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9866957720588235, + "pass@2": 0.991383091130153, + "pass@4": 0.9954316220582722, + "pass@8": 0.998009907801721, + "pass@16": 0.9992383013303037, + "pass@32": 0.9998536614272977, + "pass@64": 0.9999980360722474, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.999728903785489, + "pass@2": 0.9999992237760501, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9992335838607594, + "pass@2": 0.9999707994866939, + "pass@4": 0.999999947791336, + "pass@8": 0.99999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920701951951952, + "pass@2": 0.99714071847438, + "pass@4": 0.9993788327134785, + "pass@8": 0.9999618621335632, + "pass@16": 0.9999998522509821, + "pass@32": 0.9999999999994185, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9870880225080386, + "pass@2": 0.9951408493050107, + "pass@4": 0.9981870624290932, + "pass@8": 0.9994770626159226, + "pass@16": 0.9999345493968065, + "pass@32": 0.9999993363019254, + "pass@64": 0.9999999999980551, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9361111111111111, + "pass@2": 0.964721597300337, + "pass@4": 0.9788161914284527, + "pass@8": 0.9862816248572933, + "pass@16": 0.9897694199344508, + "pass@32": 0.9919058827484885, + "pass@64": 0.9933702121063085, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.8474053899082569, + "pass@2": 0.9069223464326136, + "pass@4": 0.9367416807990738, + "pass@8": 0.9538386375839678, + "pass@16": 0.9652243644935586, + "pass@32": 0.9741221163718324, + "pass@64": 0.9806778665797814, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.6504870756172839, + "pass@2": 0.737235102556625, + "pass@4": 0.7968340239298788, + "pass@8": 0.8394478736620564, + "pass@16": 0.8707955001207655, + "pass@32": 0.8962127601427606, + "pass@64": 0.9199947039243642, + "pass@128": 0.9444444444444444 + }, + "14": { + "pass@1": 0.5979460311572701, + "pass@2": 0.6981321847473071, + "pass@4": 0.7671013725287299, + "pass@8": 0.8156496766290819, + "pass@16": 0.8534954255336188, + "pass@32": 0.8846802898454802, + "pass@64": 0.9111804207009937, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.5047019675925926, + "pass@2": 0.6032917760279963, + "pass@4": 0.6773077448073619, + "pass@8": 0.7323858349826511, + "pass@16": 0.7744864686864456, + "pass@32": 0.8079447450839151, + "pass@64": 0.8351820095707687, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.32107300884955753, + "pass@2": 0.4297624439644157, + "pass@4": 0.5362244711114648, + "pass@8": 0.6299412145504063, + "pass@16": 0.7046635110365608, + "pass@32": 0.7583830063556092, + "pass@64": 0.7949665227100015, + "pass@128": 0.8200589970501475 + }, + "17": { + "pass@1": 0.252941453313253, + "pass@2": 0.34630216238971623, + "pass@4": 0.4382089559023495, + "pass@8": 0.5172900182449583, + "pass@16": 0.5816590133785813, + "pass@32": 0.6347724043312906, + "pass@64": 0.6815166946995262, + "pass@128": 0.7228915662650602 + }, + "18": { + "pass@1": 0.24987399193548387, + "pass@2": 0.3411508604267206, + "pass@4": 0.42806011889642825, + "pass@8": 0.5009790895393157, + "pass@16": 0.5617827302678425, + "pass@32": 0.6148278041185289, + "pass@64": 0.6618773335461938, + "pass@128": 0.7 + }, + "19": { + "pass@1": 0.18797205438066467, + "pass@2": 0.26968615445916705, + "pass@4": 0.3539405954165095, + "pass@8": 0.43323175405120734, + "pass@16": 0.5015421311382682, + "pass@32": 0.5558973363316422, + "pass@64": 0.5980831369200011, + "pass@128": 0.6374622356495468 + }, + "20": { + "pass@1": 0.1547954204204204, + "pass@2": 0.2324537874488661, + "pass@4": 0.32128977881143245, + "pass@8": 0.4114801380719811, + "pass@16": 0.49201842931368517, + "pass@32": 0.5597262016579201, + "pass@64": 0.6158544846756309, + "pass@128": 0.6606606606606606 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/checkpoint-42828_metrics.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/checkpoint-42828_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..696770d4038f3386db7131f6171208d8ebfda169 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/checkpoint-42828_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-42828", + "id": { + "answer_accuracy": 0.5884739583333334, + "avg_response_len": 196.50156336805554, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5884739583333334, + "pass@2": 0.6947505331364885, + "pass@4": 0.789893223555378, + "pass@8": 0.8652182970477127, + "pass@16": 0.9174590080894933, + "pass@32": 0.9499212500312727, + "pass@64": 0.9692888310961021, + "pass@128": 0.9812222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3162578125, + "pass@2": 0.45835113188976345, + "pass@4": 0.605407840269966, + "pass@8": 0.7346850838514688, + "pass@16": 0.8314864758890895, + "pass@32": 0.8961760938604171, + "pass@64": 0.9363183423146224, + "pass@128": 0.959 + }, + "2": { + "pass@1": 0.96496875, + "pass@2": 0.9756861466535435, + "pass@4": 0.9817775052493437, + "pass@8": 0.9855409672046594, + "pass@16": 0.988243151429112, + "pass@32": 0.9905674299207347, + "pass@64": 0.9928421193376944, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.938078125, + "pass@2": 0.9617782972440957, + "pass@4": 0.9719853221784777, + "pass@8": 0.9780014606598325, + "pass@16": 0.9828907921768589, + "pass@32": 0.9873451095130987, + "pass@64": 0.9909909439624601, + "pass@128": 0.994 + }, + "4": { + "pass@1": 0.7827578125, + "pass@2": 0.8596659694881891, + "pass@4": 0.9139369010123735, + "pass@8": 0.9497228241119576, + "pass@16": 0.9708192659905702, + "pass@32": 0.9818875683979876, + "pass@64": 0.9880706917246304, + "pass@128": 0.991 + }, + "5": { + "pass@1": 0.6006328125, + "pass@2": 0.7053149606299219, + "pass@4": 0.7868438413010889, + "pass@8": 0.8490633293541885, + "pass@16": 0.8957431185434932, + "pass@32": 0.9294812687833187, + "pass@64": 0.9532782525266222, + "pass@128": 0.969 + }, + "6": { + "pass@1": 0.5610703125, + "pass@2": 0.7107727608267694, + "pass@4": 0.8281742008811401, + "pass@8": 0.9045137915028677, + "pass@16": 0.9480918197484305, + "pass@32": 0.9703859865544925, + "pass@64": 0.9815268172963851, + "pass@128": 0.989 + }, + "7": { + "pass@1": 0.4492734375, + "pass@2": 0.5993241879921253, + "pass@4": 0.7357537199100115, + "pass@8": 0.8429689295485534, + "pass@16": 0.9134422184403561, + "pass@32": 0.9523973907052001, + "pass@64": 0.9735939345300991, + "pass@128": 0.987 + }, + "8": { + "pass@1": 0.38553125, + "pass@2": 0.5402993356299214, + "pass@4": 0.6903833682039752, + "pass@8": 0.8103299521170219, + "pass@16": 0.8898116146886553, + "pass@32": 0.937163412625617, + "pass@64": 0.9642082110969803, + "pass@128": 0.979 + }, + "9": { + "pass@1": 0.2976953125, + "pass@2": 0.4415620078740162, + "pass@4": 0.5947763129921264, + "pass@8": 0.7321383350788808, + "pass@16": 0.8366026158987336, + "pass@32": 0.903886989920488, + "pass@64": 0.9427701670753371, + "pass@128": 0.968 + } + }, + "per_op_accuracy": { + "10": 0.3162578125, + "2": 0.96496875, + "3": 0.938078125, + "4": 0.7827578125, + "5": 0.6006328125, + "6": 0.5610703125, + "7": 0.4492734375, + "8": 0.38553125, + "9": 0.2976953125 + }, + "per_op_avg_response_len": { + "10": 274.1716015625, + "2": 116.9542890625, + "3": 145.395640625, + "4": 157.3481796875, + "5": 170.938171875, + "6": 194.9085625, + "7": 219.56721875, + "8": 239.2271796875, + "9": 250.0032265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.1716015625, + "2": 116.9542890625, + "3": 145.395640625, + "4": 157.3481796875, + "5": 170.938171875, + "6": 194.9085625, + "7": 219.56721875, + "8": 239.2271796875, + "9": 250.0032265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 222673, + "answer_accuracy": 0.5875153031070585, + "avg_response_len": 186.00562521107733, + "resp_tokens_sum": 70497620, + "pass_at_k": { + "pass@1": 0.5875153031070585, + "pass@2": 0.6937834565892051, + "pass@4": 0.7898807007343437, + "pass@8": 0.8661809256029457, + "pass@16": 0.91858197984295, + "pass@32": 0.9504379995240725, + "pass@64": 0.9686830369994636, + "pass@128": 0.9793988517392773 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3160951327433628, + "pass@2": 0.45978539532669027, + "pass@4": 0.6087440955942451, + "pass@8": 0.7389912618384603, + "pass@16": 0.8347761227732956, + "pass@32": 0.8974145781361296, + "pass@64": 0.9361195557565894, + "pass@128": 0.9587020648967551 + }, + "2": { + "pass@1": 0.9389846183800623, + "pass@2": 0.9547148275566021, + "pass@4": 0.964830180398011, + "pass@8": 0.9716501249659655, + "pass@16": 0.9762703677512582, + "pass@32": 0.9794593284358751, + "pass@64": 0.9826803268884213, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.93623375382263, + "pass@2": 0.9602559356112597, + "pass@4": 0.971118965347221, + "pass@8": 0.9775815501353607, + "pass@16": 0.9825540775856214, + "pass@32": 0.9868925189336957, + "pass@64": 0.9907274512321347, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.782666015625, + "pass@2": 0.8574395607775591, + "pass@4": 0.9108361469464749, + "pass@8": 0.9460693707480754, + "pass@16": 0.9668682851949002, + "pass@32": 0.9775023168825377, + "pass@64": 0.9821243598725908, + "pass@128": 0.984375 + }, + "5": { + "pass@1": 0.6209325396825397, + "pass@2": 0.7310359642544682, + "pass@4": 0.8163355446640592, + "pass@8": 0.8783539103209079, + "pass@16": 0.922046305462945, + "pass@32": 0.9516874532916285, + "pass@64": 0.971409019971419, + "pass@128": 0.9841269841269841 + }, + "6": { + "pass@1": 0.5845808383233533, + "pass@2": 0.723798343274082, + "pass@4": 0.834097648067195, + "pass@8": 0.908153197796081, + "pass@16": 0.9503107359238322, + "pass@32": 0.9694990410164247, + "pass@64": 0.9774287469166411, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.45092917899408286, + "pass@2": 0.596248121767693, + "pass@4": 0.7315876857848386, + "pass@8": 0.8403357342652344, + "pass@16": 0.9115535229290597, + "pass@32": 0.9505698802282272, + "pass@64": 0.9706700595276606, + "pass@128": 0.9822485207100592 + }, + "8": { + "pass@1": 0.40384615384615385, + "pass@2": 0.5536141290631941, + "pass@4": 0.698270090036822, + "pass@8": 0.8140536634874682, + "pass@16": 0.8919098750755075, + "pass@32": 0.9385195819134277, + "pass@64": 0.9639372810558046, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.29634683098591547, + "pass@2": 0.44307973827215225, + "pass@4": 0.5996262900236059, + "pass@8": 0.7377739613933347, + "pass@16": 0.8412027330164041, + "pass@32": 0.9081616296352407, + "pass@64": 0.9463322026470657, + "pass@128": 0.9690140845070423 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 230595, + "answer_accuracy": 0.5902763556684142, + "avg_response_len": 194.26658236402358, + "resp_tokens_sum": 75891406, + "pass_at_k": { + "pass@1": 0.5902763556684142, + "pass@2": 0.6967596116139171, + "pass@4": 0.7908906759463917, + "pass@8": 0.8650238624157042, + "pass@16": 0.9162825752797982, + "pass@32": 0.9482979401687237, + "pass@64": 0.9684873279473752, + "pass@128": 0.981651376146789 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.305382476993865, + "pass@2": 0.4384706656683249, + "pass@4": 0.5788130030525324, + "pass@8": 0.7079080797271436, + "pass@16": 0.809303783504082, + "pass@32": 0.8795306867277652, + "pass@64": 0.9246891544210146, + "pass@128": 0.950920245398773 + }, + "2": { + "pass@1": 0.9775598404255319, + "pass@2": 0.9882914402986861, + "pass@4": 0.9931788543529323, + "pass@8": 0.9950785758504687, + "pass@16": 0.9963275595235965, + "pass@32": 0.9977752173693791, + "pass@64": 0.9992040875534045, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9306344696969697, + "pass@2": 0.9533270699117159, + "pass@4": 0.9638342272556834, + "pass@8": 0.9706880776824892, + "pass@16": 0.9762976075504467, + "pass@32": 0.981515194409324, + "pass@64": 0.9865150313128073, + "pass@128": 0.990909090909091 + }, + "4": { + "pass@1": 0.7786150147928994, + "pass@2": 0.8566846870195216, + "pass@4": 0.9127427596883223, + "pass@8": 0.9497142611938738, + "pass@16": 0.9711949174245315, + "pass@32": 0.9826570262446811, + "pass@64": 0.9899957012280247, + "pass@128": 0.9940828402366864 + }, + "5": { + "pass@1": 0.6168025362318841, + "pass@2": 0.7120884685609948, + "pass@4": 0.7810124440966606, + "pass@8": 0.8333119147189844, + "pass@16": 0.8745230907515112, + "pass@32": 0.9086414545020511, + "pass@64": 0.9394193940381265, + "pass@128": 0.9652173913043478 + }, + "6": { + "pass@1": 0.5708945200573066, + "pass@2": 0.7201273182095074, + "pass@4": 0.8332363168966347, + "pass@8": 0.9048154542138614, + "pass@16": 0.9454304796626805, + "pass@32": 0.9676635934894852, + "pass@64": 0.9803148386384075, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.45682803468208094, + "pass@2": 0.6100648297756136, + "pass@4": 0.7466018149465419, + "pass@8": 0.8507837490523493, + "pass@16": 0.9185662560649683, + "pass@32": 0.9563321743967517, + "pass@64": 0.9785556722654779, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.3819762323943662, + "pass@2": 0.5406409421093487, + "pass@4": 0.693883219263085, + "pass@8": 0.8154527976781428, + "pass@16": 0.8940305609873201, + "pass@32": 0.9388662926400119, + "pass@64": 0.9641705344024881, + "pass@128": 0.9802816901408451 + }, + "9": { + "pass@1": 0.3124766092814371, + "pass@2": 0.4618525007072463, + "pass@4": 0.6193879476704632, + "pass@8": 0.7587824319807451, + "pass@16": 0.8609492491217673, + "pass@32": 0.9215203946762859, + "pass@64": 0.9532264564553681, + "pass@128": 0.9700598802395209 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 224654, + "answer_accuracy": 0.5875826498158687, + "avg_response_len": 209.18975717693337, + "resp_tokens_sum": 79980775, + "pass_at_k": { + "pass@1": 0.5875826498158687, + "pass@2": 0.6936563939143117, + "pass@4": 0.7888864794562237, + "pass@8": 0.8644627132997738, + "pass@16": 0.9175478442371732, + "pass@32": 0.9510676331756316, + "pass@64": 0.9707082967572729, + "pass@128": 0.9825912286575159 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3270055970149254, + "pass@2": 0.47624610706311005, + "pass@4": 0.6279120981146009, + "pass@8": 0.7563851107378542, + "pass@16": 0.8497442890943676, + "pass@32": 0.9111210388030382, + "pass@64": 0.9478362643936907, + "pass@128": 0.9671641791044776 + }, + "2": { + "pass@1": 0.9769642857142857, + "pass@2": 0.9830708661417321, + "pass@4": 0.9866033550270502, + "pass@8": 0.9893155018165151, + "pass@16": 0.9916245608505572, + "pass@32": 0.9939798256522657, + "pass@64": 0.9961816845755459, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9469979956268222, + "pass@2": 0.9713605633479483, + "pass@4": 0.9806534271590681, + "pass@8": 0.9854379828873127, + "pass@16": 0.989555097127444, + "pass@32": 0.9933855442177933, + "pass@64": 0.9955484171321441, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.7869380482456141, + "pass@2": 0.8646955725928998, + "pass@4": 0.9180183661252861, + "pass@8": 0.9531497227749841, + "pass@16": 0.9741448322769339, + "pass@32": 0.9852302693709742, + "pass@64": 0.9917320162290298, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.5654181985294118, + "pass@2": 0.6746120889300605, + "pass@4": 0.7654377985840002, + "pass@8": 0.8379095206619214, + "pass@16": 0.8929060765099897, + "pass@32": 0.9300541740977897, + "pass@64": 0.9505432949778564, + "pass@128": 0.9588235294117647 + }, + "6": { + "pass@1": 0.5254830441640379, + "pass@2": 0.6867497795523986, + "pass@4": 0.8163599742264075, + "pass@8": 0.9003470975341901, + "pass@16": 0.9486839007810618, + "pass@32": 0.9743177057639598, + "pass@64": 0.9871789815186821, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.4392306170886076, + "pass@2": 0.5908540877603908, + "pass@4": 0.7283318169089623, + "pass@8": 0.8372287158063058, + "pass@16": 0.9098519085185298, + "pass@32": 0.9500437938189338, + "pass@64": 0.971288581664221, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.37216122372372373, + "pass@2": 0.5274600387789361, + "pass@4": 0.6792629347570285, + "pass@8": 0.8013797775771526, + "pass@16": 0.8833480015453465, + "pass@32": 0.9340773849291986, + "pass@64": 0.9645022213053622, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.28336012861736337, + "pass@2": 0.4180384744410967, + "pass@4": 0.5628083778514825, + "pass@8": 0.6970908247674539, + "pass@16": 0.8052044902612292, + "pass@32": 0.8800700951064251, + "pass@64": 0.9274745938248696, + "pass@128": 0.9646302250803859 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.1697921875, + "avg_response_len": 322.145346875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.1697921875, + "pass@2": 0.2654636072834687, + "pass@4": 0.3762926771184901, + "pass@8": 0.4857105529458426, + "pass@16": 0.5812994858308275, + "pass@32": 0.6596549969146106, + "pass@64": 0.7230964336400634, + "pass@128": 0.7757 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.2570078125, + "pass@2": 0.38598166830708713, + "pass@4": 0.5238880150918627, + "pass@8": 0.6522282735164434, + "pass@16": 0.760711084477594, + "pass@32": 0.8437929754442984, + "pass@64": 0.8998519201186355, + "pass@128": 0.935 + }, + "12": { + "pass@1": 0.2289453125, + "pass@2": 0.3513831200787408, + "pass@4": 0.4888043504874388, + "pass@8": 0.6208747007609436, + "pass@16": 0.734419131861814, + "pass@32": 0.8242462716782151, + "pass@64": 0.8897463763053745, + "pass@128": 0.935 + }, + "13": { + "pass@1": 0.2053359375, + "pass@2": 0.3149776082677165, + "pass@4": 0.43664417997750254, + "pass@8": 0.5540695498325362, + "pass@16": 0.6583834428382952, + "pass@32": 0.7483065056537443, + "pass@64": 0.8232374532827101, + "pass@128": 0.881 + }, + "14": { + "pass@1": 0.1828984375, + "pass@2": 0.28725012303149605, + "pass@4": 0.4080562338770153, + "pass@8": 0.5257933530059418, + "pass@16": 0.6295265083104246, + "pass@32": 0.7193081362823707, + "pass@64": 0.7937083391766463, + "pass@128": 0.852 + }, + "15": { + "pass@1": 0.1443828125, + "pass@2": 0.2320948572834645, + "pass@4": 0.3411599497562799, + "pass@8": 0.45857131926512257, + "pass@16": 0.5694939223397415, + "pass@32": 0.6654687670420999, + "pass@64": 0.745631044604045, + "pass@128": 0.813 + }, + "16": { + "pass@1": 0.1585, + "pass@2": 0.24985494586614163, + "pass@4": 0.3558414847206595, + "pass@8": 0.45910886786215366, + "pass@16": 0.549541252597057, + "pass@32": 0.6299680907133983, + "pass@64": 0.7055871644045559, + "pass@128": 0.777 + }, + "17": { + "pass@1": 0.14559375, + "pass@2": 0.23222207185039417, + "pass@4": 0.3345885004686911, + "pass@8": 0.43526238781629567, + "pass@16": 0.5214875283206494, + "pass@32": 0.5922619878497463, + "pass@64": 0.6515677542306969, + "pass@128": 0.701 + }, + "18": { + "pass@1": 0.1288828125, + "pass@2": 0.20564665354330708, + "pass@4": 0.29888706674165694, + "pass@8": 0.3948386743891329, + "pass@16": 0.4796908314713499, + "pass@32": 0.5486896211975746, + "pass@64": 0.6071358961452984, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1318984375, + "pass@2": 0.2093880413385825, + "pass@4": 0.3012156912260963, + "pass@8": 0.3910865165786873, + "pass@16": 0.4643179810887953, + "pass@32": 0.518873028817878, + "pass@64": 0.5639701245868507, + "pass@128": 0.612 + }, + "20": { + "pass@1": 0.1144765625, + "pass@2": 0.18583698326771636, + "pass@4": 0.2738412988376452, + "pass@8": 0.36527188643102915, + "pass@16": 0.44542317500235973, + "pass@32": 0.5056345844667983, + "pass@64": 0.5505282635457379, + "pass@128": 0.59 + } + }, + "per_op_accuracy": { + "11": 0.2570078125, + "12": 0.2289453125, + "13": 0.2053359375, + "14": 0.1828984375, + "15": 0.1443828125, + "16": 0.1585, + "17": 0.14559375, + "18": 0.1288828125, + "19": 0.1318984375, + "20": 0.1144765625 + }, + "per_op_avg_response_len": { + "11": 286.8010859375, + "12": 299.0134375, + "13": 312.4879140625, + "14": 322.02253125, + "15": 324.1684765625, + "16": 331.384828125, + "17": 338.6601796875, + "18": 335.31759375, + "19": 334.377515625, + "20": 337.21990625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 286.8010859375, + "12": 299.0134375, + "13": 312.4879140625, + "14": 322.02253125, + "15": 324.1684765625, + "16": 331.384828125, + "17": 338.6601796875, + "18": 335.31759375, + "19": 334.377515625, + "20": 337.21990625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 74837, + "answer_accuracy": 0.17374860698365527, + "avg_response_len": 305.5321879643388, + "resp_tokens_sum": 131598824, + "pass_at_k": { + "pass@1": 0.17374860698365527, + "pass@2": 0.270465530706322, + "pass@4": 0.3811262522905307, + "pass@8": 0.48929743238835643, + "pass@16": 0.583492852871176, + "pass@32": 0.6611911151891285, + "pass@64": 0.7247972610300216, + "pass@128": 0.7777117384843982 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.27554001416430596, + "pass@2": 0.4100284262006203, + "pass@4": 0.5497061987294078, + "pass@8": 0.6745030060620455, + "pass@16": 0.7762085776756984, + "pass@32": 0.8539760738175253, + "pass@64": 0.9082416689117168, + "pass@128": 0.9405099150141643 + }, + "12": { + "pass@1": 0.223583156779661, + "pass@2": 0.34553187141331937, + "pass@4": 0.48248001150279896, + "pass@8": 0.6133563555890079, + "pass@16": 0.7254988286208823, + "pass@32": 0.8126269158410524, + "pass@64": 0.8750599826992427, + "pass@128": 0.9180790960451978 + }, + "13": { + "pass@1": 0.21137833072100312, + "pass@2": 0.32505769752918806, + "pass@4": 0.44905429150901616, + "pass@8": 0.5689478723786576, + "pass@16": 0.6776168936361864, + "pass@32": 0.7707110571183069, + "pass@64": 0.8439511414301138, + "pass@128": 0.8934169278996865 + }, + "14": { + "pass@1": 0.17978485202492211, + "pass@2": 0.2873168708759536, + "pass@4": 0.4125795875982794, + "pass@8": 0.5333225333050318, + "pass@16": 0.6392014624846123, + "pass@32": 0.7319459986189009, + "pass@64": 0.8117486314877205, + "pass@128": 0.8785046728971962 + }, + "15": { + "pass@1": 0.14981484826589594, + "pass@2": 0.23933430715488574, + "pass@4": 0.34942827206209043, + "pass@8": 0.46691425776949924, + "pass@16": 0.577039877850708, + "pass@32": 0.6734734398150024, + "pass@64": 0.7581490937591103, + "pass@128": 0.8352601156069365 + }, + "16": { + "pass@1": 0.1690272177419355, + "pass@2": 0.26243729362458734, + "pass@4": 0.3701715785526809, + "pass@8": 0.4754923682022537, + "pass@16": 0.566165648570034, + "pass@32": 0.6434866743114298, + "pass@64": 0.715818663351052, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.14773753612716764, + "pass@2": 0.22860994322060887, + "pass@4": 0.3213330353214517, + "pass@8": 0.4127466446487047, + "pass@16": 0.4945081321294641, + "pass@32": 0.5670904861099084, + "pass@64": 0.6320432166581963, + "pass@128": 0.684971098265896 + }, + "18": { + "pass@1": 0.12301339285714286, + "pass@2": 0.19675583520809908, + "pass@4": 0.2861904290535112, + "pass@8": 0.37906513974828127, + "pass@16": 0.46258529242199625, + "pass@32": 0.531878886537751, + "pass@64": 0.5906816065671889, + "pass@128": 0.6457142857142857 + }, + "19": { + "pass@1": 0.13278903903903905, + "pass@2": 0.21219681788087297, + "pass@4": 0.30740728129704514, + "pass@8": 0.3992844439193679, + "pass@16": 0.47047243675247624, + "pass@32": 0.522112980009493, + "pass@64": 0.5637430472466359, + "pass@128": 0.6066066066066066 + }, + "20": { + "pass@1": 0.1215746996996997, + "pass@2": 0.19323038589770877, + "pass@4": 0.27804619692808696, + "pass@8": 0.36386612424102277, + "pass@16": 0.4397676932029267, + "pass@32": 0.4992069295218254, + "pass@64": 0.5447303107088249, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 72813, + "answer_accuracy": 0.16915003345227475, + "avg_response_len": 319.9735099799286, + "resp_tokens_sum": 137737077, + "pass_at_k": { + "pass@1": 0.16915003345227475, + "pass@2": 0.2649153830124497, + "pass@4": 0.37650637487218835, + "pass@8": 0.48709837988254956, + "pass@16": 0.583529698949973, + "pass@32": 0.6618418367631765, + "pass@64": 0.7239107672350203, + "pass@128": 0.7749033600951531 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.23966961596385541, + "pass@2": 0.3581350500426905, + "pass@4": 0.48620031052217855, + "pass@8": 0.6102333477314572, + "pass@16": 0.7208873268789171, + "pass@32": 0.8102303148700188, + "pass@64": 0.8733357401088951, + "pass@128": 0.9156626506024096 + }, + "12": { + "pass@1": 0.2491673197492163, + "pass@2": 0.376715884407474, + "pass@4": 0.5163546439532046, + "pass@8": 0.6470903372450432, + "pass@16": 0.7569027766764611, + "pass@32": 0.8431432973967836, + "pass@64": 0.904642073000038, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.195750175070028, + "pass@2": 0.3036820535300734, + "pass@4": 0.4266344887036182, + "pass@8": 0.5461250500217757, + "pass@16": 0.6500571912155733, + "pass@32": 0.7377077535289895, + "pass@64": 0.8120524459657263, + "pass@128": 0.876750700280112 + }, + "14": { + "pass@1": 0.18718019005847952, + "pass@2": 0.29091444778284303, + "pass@4": 0.40853745721477797, + "pass@8": 0.5212011661278351, + "pass@16": 0.6195737300108569, + "pass@32": 0.7043303525635856, + "pass@64": 0.7735331168668783, + "pass@128": 0.8245614035087719 + }, + "15": { + "pass@1": 0.15082859848484848, + "pass@2": 0.24180013719875898, + "pass@4": 0.3517029428139666, + "pass@8": 0.46704428304724294, + "pass@16": 0.5760473201704117, + "pass@32": 0.6708225663764403, + "pass@64": 0.7498293734022303, + "pass@128": 0.8121212121212121 + }, + "16": { + "pass@1": 0.14872685185185186, + "pass@2": 0.237111136348341, + "pass@4": 0.34124766161708425, + "pass@8": 0.44297967345870165, + "pass@16": 0.5312036383809488, + "pass@32": 0.6088643968988856, + "pass@64": 0.6800786909386469, + "pass@128": 0.7464387464387464 + }, + "17": { + "pass@1": 0.15285326086956522, + "pass@2": 0.2479397955690318, + "pass@4": 0.3625688723109922, + "pass@8": 0.47662920580834106, + "pass@16": 0.5736009919599262, + "pass@32": 0.6496201555232083, + "pass@64": 0.7054333096266076, + "pass@128": 0.7422360248447205 + }, + "18": { + "pass@1": 0.13494944852941176, + "pass@2": 0.21811783522464095, + "pass@4": 0.3204172084188005, + "pass@8": 0.4256938916457332, + "pass@16": 0.5168048555096408, + "pass@32": 0.5864634897053611, + "pass@64": 0.6395187651433718, + "pass@128": 0.6852941176470588 + }, + "19": { + "pass@1": 0.12423270089285714, + "pass@2": 0.19673212235189355, + "pass@4": 0.2854409744540861, + "pass@8": 0.3764069622534746, + "pass@16": 0.4543999639029509, + "pass@32": 0.5144732309561025, + "pass@64": 0.5651312688961807, + "pass@128": 0.6220238095238095 + }, + "20": { + "pass@1": 0.1110123502994012, + "pass@2": 0.18206966971568675, + "pass@4": 0.27122720307416665, + "pass@8": 0.364016093733274, + "pass@16": 0.44317220201732704, + "pass@32": 0.5001531127134062, + "pass@64": 0.5416504433836719, + "pass@128": 0.5808383233532934 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 69684, + "answer_accuracy": 0.1663833282396088, + "avg_response_len": 341.46294076635087, + "resp_tokens_sum": 143010143, + "pass_at_k": { + "pass@1": 0.1663833282396088, + "pass@2": 0.2608829856164451, + "pass@4": 0.37110207626285624, + "pass@8": 0.48059529887727503, + "pass@16": 0.5767515375390598, + "pass@32": 0.6558275579156807, + "pass@64": 0.7205102820364083, + "pass@128": 0.7744498777506112 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.2545138888888889, + "pass@2": 0.3883834833145856, + "pass@4": 0.5346769011016481, + "pass@8": 0.6715277489831641, + "pass@16": 0.7853170286802313, + "pass@32": 0.8677554185709992, + "pass@64": 0.9183972865925268, + "pass@128": 0.9492063492063492 + }, + "12": { + "pass@1": 0.21502293577981652, + "pass@2": 0.3330044968576177, + "pass@4": 0.4687746024292833, + "pass@8": 0.6034395513800196, + "pass@16": 0.7221423876765495, + "pass@32": 0.8183903107061395, + "pass@64": 0.8911141319352575, + "pass@128": 0.9418960244648318 + }, + "13": { + "pass@1": 0.2099488811728395, + "pass@2": 0.3174991038446582, + "pass@4": 0.4354547793794296, + "pass@8": 0.5481744928579311, + "pass@16": 0.6486211003222007, + "pass@32": 0.7379259580961539, + "pass@64": 0.8151676418417848, + "pass@128": 0.8734567901234568 + }, + "14": { + "pass@1": 0.1815189169139466, + "pass@2": 0.28346785263674384, + "pass@4": 0.4032592756395063, + "pass@8": 0.523281961422275, + "pass@16": 0.6304113447749283, + "pass@32": 0.7224702971779134, + "pass@64": 0.7969991172125063, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.1320167824074074, + "pass@2": 0.2144788325070477, + "pass@4": 0.3215919644882351, + "pass@8": 0.44103201441754786, + "pass@16": 0.5547608300838373, + "pass@32": 0.651467623339146, + "pass@64": 0.7279869288291854, + "pass@128": 0.7901234567901234 + }, + "16": { + "pass@1": 0.15899244100294985, + "pass@2": 0.2515438819594455, + "pass@4": 0.3578476877337234, + "pass@8": 0.4608270452373182, + "pass@16": 0.5533257358956741, + "pass@32": 0.6394566913432015, + "pass@64": 0.7226423547087432, + "pass@128": 0.799410029498525 + }, + "17": { + "pass@1": 0.1363187123493976, + "pass@2": 0.22074222085191147, + "pass@4": 0.3212653414558117, + "pass@8": 0.4186067605348141, + "pass@16": 0.49906082889385794, + "pass@32": 0.5628644264374904, + "pass@64": 0.6196725167686506, + "pass@128": 0.677710843373494 + }, + "18": { + "pass@1": 0.12885584677419354, + "pass@2": 0.202006604013208, + "pass@4": 0.2896082764856006, + "pass@8": 0.3788062977989847, + "pass@16": 0.4582978330657227, + "pass@32": 0.5262400787404502, + "pass@64": 0.5901966248323723, + "pass@128": 0.6516129032258065 + }, + "19": { + "pass@1": 0.13878398791540786, + "pass@2": 0.2194093893474797, + "pass@4": 0.31099969527978194, + "pass@8": 0.3977403547926579, + "pass@16": 0.4681941745583987, + "pass@32": 0.5200797609470273, + "pass@64": 0.5630198897420078, + "pass@128": 0.6072507552870091 + }, + "20": { + "pass@1": 0.11085304054054054, + "pass@2": 0.18222220744366419, + "pass@4": 0.2722583466481104, + "pass@8": 0.36793721246803396, + "pass@16": 0.4533363894654612, + "pass@32": 0.5175601720413012, + "pass@64": 0.5652306966653234, + "pass@128": 0.6036036036036037 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.3681151315789474, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.3681151315789474, + "pass@2": 0.46881004584540903, + "pass@4": 0.5722087254306757, + "pass@8": 0.6654773790993593, + "pass@16": 0.7405329437428465, + "pass@32": 0.7971495378646412, + "pass@64": 0.8397138850666145, + "pass@128": 0.8730526315789474 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3162578125, + "pass@2": 0.45835113188976345, + "pass@4": 0.605407840269966, + "pass@8": 0.7346850838514688, + "pass@16": 0.8314864758890895, + "pass@32": 0.8961760938604171, + "pass@64": 0.9363183423146224, + "pass@128": 0.959 + }, + "2": { + "pass@1": 0.96496875, + "pass@2": 0.9756861466535435, + "pass@4": 0.9817775052493437, + "pass@8": 0.9855409672046594, + "pass@16": 0.988243151429112, + "pass@32": 0.9905674299207347, + "pass@64": 0.9928421193376944, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.938078125, + "pass@2": 0.9617782972440957, + "pass@4": 0.9719853221784777, + "pass@8": 0.9780014606598325, + "pass@16": 0.9828907921768589, + "pass@32": 0.9873451095130987, + "pass@64": 0.9909909439624601, + "pass@128": 0.994 + }, + "4": { + "pass@1": 0.7827578125, + "pass@2": 0.8596659694881891, + "pass@4": 0.9139369010123735, + "pass@8": 0.9497228241119576, + "pass@16": 0.9708192659905702, + "pass@32": 0.9818875683979876, + "pass@64": 0.9880706917246304, + "pass@128": 0.991 + }, + "5": { + "pass@1": 0.6006328125, + "pass@2": 0.7053149606299219, + "pass@4": 0.7868438413010889, + "pass@8": 0.8490633293541885, + "pass@16": 0.8957431185434932, + "pass@32": 0.9294812687833187, + "pass@64": 0.9532782525266222, + "pass@128": 0.969 + }, + "6": { + "pass@1": 0.5610703125, + "pass@2": 0.7107727608267694, + "pass@4": 0.8281742008811401, + "pass@8": 0.9045137915028677, + "pass@16": 0.9480918197484305, + "pass@32": 0.9703859865544925, + "pass@64": 0.9815268172963851, + "pass@128": 0.989 + }, + "7": { + "pass@1": 0.4492734375, + "pass@2": 0.5993241879921253, + "pass@4": 0.7357537199100115, + "pass@8": 0.8429689295485534, + "pass@16": 0.9134422184403561, + "pass@32": 0.9523973907052001, + "pass@64": 0.9735939345300991, + "pass@128": 0.987 + }, + "8": { + "pass@1": 0.38553125, + "pass@2": 0.5402993356299214, + "pass@4": 0.6903833682039752, + "pass@8": 0.8103299521170219, + "pass@16": 0.8898116146886553, + "pass@32": 0.937163412625617, + "pass@64": 0.9642082110969803, + "pass@128": 0.979 + }, + "9": { + "pass@1": 0.2976953125, + "pass@2": 0.4415620078740162, + "pass@4": 0.5947763129921264, + "pass@8": 0.7321383350788808, + "pass@16": 0.8366026158987336, + "pass@32": 0.903886989920488, + "pass@64": 0.9427701670753371, + "pass@128": 0.968 + }, + "11": { + "pass@1": 0.2570078125, + "pass@2": 0.38598166830708713, + "pass@4": 0.5238880150918627, + "pass@8": 0.6522282735164434, + "pass@16": 0.760711084477594, + "pass@32": 0.8437929754442984, + "pass@64": 0.8998519201186355, + "pass@128": 0.935 + }, + "12": { + "pass@1": 0.2289453125, + "pass@2": 0.3513831200787408, + "pass@4": 0.4888043504874388, + "pass@8": 0.6208747007609436, + "pass@16": 0.734419131861814, + "pass@32": 0.8242462716782151, + "pass@64": 0.8897463763053745, + "pass@128": 0.935 + }, + "13": { + "pass@1": 0.2053359375, + "pass@2": 0.3149776082677165, + "pass@4": 0.43664417997750254, + "pass@8": 0.5540695498325362, + "pass@16": 0.6583834428382952, + "pass@32": 0.7483065056537443, + "pass@64": 0.8232374532827101, + "pass@128": 0.881 + }, + "14": { + "pass@1": 0.1828984375, + "pass@2": 0.28725012303149605, + "pass@4": 0.4080562338770153, + "pass@8": 0.5257933530059418, + "pass@16": 0.6295265083104246, + "pass@32": 0.7193081362823707, + "pass@64": 0.7937083391766463, + "pass@128": 0.852 + }, + "15": { + "pass@1": 0.1443828125, + "pass@2": 0.2320948572834645, + "pass@4": 0.3411599497562799, + "pass@8": 0.45857131926512257, + "pass@16": 0.5694939223397415, + "pass@32": 0.6654687670420999, + "pass@64": 0.745631044604045, + "pass@128": 0.813 + }, + "16": { + "pass@1": 0.1585, + "pass@2": 0.24985494586614163, + "pass@4": 0.3558414847206595, + "pass@8": 0.45910886786215366, + "pass@16": 0.549541252597057, + "pass@32": 0.6299680907133983, + "pass@64": 0.7055871644045559, + "pass@128": 0.777 + }, + "17": { + "pass@1": 0.14559375, + "pass@2": 0.23222207185039417, + "pass@4": 0.3345885004686911, + "pass@8": 0.43526238781629567, + "pass@16": 0.5214875283206494, + "pass@32": 0.5922619878497463, + "pass@64": 0.6515677542306969, + "pass@128": 0.701 + }, + "18": { + "pass@1": 0.1288828125, + "pass@2": 0.20564665354330708, + "pass@4": 0.29888706674165694, + "pass@8": 0.3948386743891329, + "pass@16": 0.4796908314713499, + "pass@32": 0.5486896211975746, + "pass@64": 0.6071358961452984, + "pass@128": 0.661 + }, + "19": { + "pass@1": 0.1318984375, + "pass@2": 0.2093880413385825, + "pass@4": 0.3012156912260963, + "pass@8": 0.3910865165786873, + "pass@16": 0.4643179810887953, + "pass@32": 0.518873028817878, + "pass@64": 0.5639701245868507, + "pass@128": 0.612 + }, + "20": { + "pass@1": 0.1144765625, + "pass@2": 0.18583698326771636, + "pass@4": 0.2738412988376452, + "pass@8": 0.36527188643102915, + "pass@16": 0.44542317500235973, + "pass@32": 0.5056345844667983, + "pass@64": 0.5505282635457379, + "pass@128": 0.59 + } + }, + "per_op_accuracy": { + "10": 0.3162578125, + "2": 0.96496875, + "3": 0.938078125, + "4": 0.7827578125, + "5": 0.6006328125, + "6": 0.5610703125, + "7": 0.4492734375, + "8": 0.38553125, + "9": 0.2976953125, + "11": 0.2570078125, + "12": 0.2289453125, + "13": 0.2053359375, + "14": 0.1828984375, + "15": 0.1443828125, + "16": 0.1585, + "17": 0.14559375, + "18": 0.1288828125, + "19": 0.1318984375, + "20": 0.1144765625 + }, + "per_op_avg_response_len": { + "10": 274.1716015625, + "2": 116.9542890625, + "3": 145.395640625, + "4": 157.3481796875, + "5": 170.938171875, + "6": 194.9085625, + "7": 219.56721875, + "8": 239.2271796875, + "9": 250.0032265625, + "11": 286.8010859375, + "12": 299.0134375, + "13": 312.4879140625, + "14": 322.02253125, + "15": 324.1684765625, + "16": 331.384828125, + "17": 338.6601796875, + "18": 335.31759375, + "19": 334.377515625, + "20": 337.21990625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.1716015625, + "2": 116.9542890625, + "3": 145.395640625, + "4": 157.3481796875, + "5": 170.938171875, + "6": 194.9085625, + "7": 219.56721875, + "8": 239.2271796875, + "9": 250.0032265625, + "11": 286.8010859375, + "12": 299.0134375, + "13": 312.4879140625, + "14": 322.02253125, + "15": 324.1684765625, + "16": 331.384828125, + "17": 338.6601796875, + "18": 335.31759375, + "19": 334.377515625, + "20": 337.21990625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 297510, + "answer_accuracy": 0.3674196767309516, + "avg_response_len": 249.58559417483403, + "resp_tokens_sum": 202096444, + "pass_at_k": { + "pass@1": 0.3674196767309516, + "pass@2": 0.4686072282307062, + "pass@4": 0.5724512478394008, + "pass@8": 0.6657046444352267, + "pass@16": 0.7403374473959198, + "pass@32": 0.7965784096114769, + "pass@64": 0.8389524590454404, + "pass@128": 0.8721150806196649 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3160951327433628, + "pass@2": 0.45978539532669027, + "pass@4": 0.6087440955942451, + "pass@8": 0.7389912618384603, + "pass@16": 0.8347761227732956, + "pass@32": 0.8974145781361296, + "pass@64": 0.9361195557565894, + "pass@128": 0.9587020648967551 + }, + "2": { + "pass@1": 0.9389846183800623, + "pass@2": 0.9547148275566021, + "pass@4": 0.964830180398011, + "pass@8": 0.9716501249659655, + "pass@16": 0.9762703677512582, + "pass@32": 0.9794593284358751, + "pass@64": 0.9826803268884213, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.93623375382263, + "pass@2": 0.9602559356112597, + "pass@4": 0.971118965347221, + "pass@8": 0.9775815501353607, + "pass@16": 0.9825540775856214, + "pass@32": 0.9868925189336957, + "pass@64": 0.9907274512321347, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.782666015625, + "pass@2": 0.8574395607775591, + "pass@4": 0.9108361469464749, + "pass@8": 0.9460693707480754, + "pass@16": 0.9668682851949002, + "pass@32": 0.9775023168825377, + "pass@64": 0.9821243598725908, + "pass@128": 0.984375 + }, + "5": { + "pass@1": 0.6209325396825397, + "pass@2": 0.7310359642544682, + "pass@4": 0.8163355446640592, + "pass@8": 0.8783539103209079, + "pass@16": 0.922046305462945, + "pass@32": 0.9516874532916285, + "pass@64": 0.971409019971419, + "pass@128": 0.9841269841269841 + }, + "6": { + "pass@1": 0.5845808383233533, + "pass@2": 0.723798343274082, + "pass@4": 0.834097648067195, + "pass@8": 0.908153197796081, + "pass@16": 0.9503107359238322, + "pass@32": 0.9694990410164247, + "pass@64": 0.9774287469166411, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.45092917899408286, + "pass@2": 0.596248121767693, + "pass@4": 0.7315876857848386, + "pass@8": 0.8403357342652344, + "pass@16": 0.9115535229290597, + "pass@32": 0.9505698802282272, + "pass@64": 0.9706700595276606, + "pass@128": 0.9822485207100592 + }, + "8": { + "pass@1": 0.40384615384615385, + "pass@2": 0.5536141290631941, + "pass@4": 0.698270090036822, + "pass@8": 0.8140536634874682, + "pass@16": 0.8919098750755075, + "pass@32": 0.9385195819134277, + "pass@64": 0.9639372810558046, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.29634683098591547, + "pass@2": 0.44307973827215225, + "pass@4": 0.5996262900236059, + "pass@8": 0.7377739613933347, + "pass@16": 0.8412027330164041, + "pass@32": 0.9081616296352407, + "pass@64": 0.9463322026470657, + "pass@128": 0.9690140845070423 + }, + "11": { + "pass@1": 0.27554001416430596, + "pass@2": 0.4100284262006203, + "pass@4": 0.5497061987294078, + "pass@8": 0.6745030060620455, + "pass@16": 0.7762085776756984, + "pass@32": 0.8539760738175253, + "pass@64": 0.9082416689117168, + "pass@128": 0.9405099150141643 + }, + "12": { + "pass@1": 0.223583156779661, + "pass@2": 0.34553187141331937, + "pass@4": 0.48248001150279896, + "pass@8": 0.6133563555890079, + "pass@16": 0.7254988286208823, + "pass@32": 0.8126269158410524, + "pass@64": 0.8750599826992427, + "pass@128": 0.9180790960451978 + }, + "13": { + "pass@1": 0.21137833072100312, + "pass@2": 0.32505769752918806, + "pass@4": 0.44905429150901616, + "pass@8": 0.5689478723786576, + "pass@16": 0.6776168936361864, + "pass@32": 0.7707110571183069, + "pass@64": 0.8439511414301138, + "pass@128": 0.8934169278996865 + }, + "14": { + "pass@1": 0.17978485202492211, + "pass@2": 0.2873168708759536, + "pass@4": 0.4125795875982794, + "pass@8": 0.5333225333050318, + "pass@16": 0.6392014624846123, + "pass@32": 0.7319459986189009, + "pass@64": 0.8117486314877205, + "pass@128": 0.8785046728971962 + }, + "15": { + "pass@1": 0.14981484826589594, + "pass@2": 0.23933430715488574, + "pass@4": 0.34942827206209043, + "pass@8": 0.46691425776949924, + "pass@16": 0.577039877850708, + "pass@32": 0.6734734398150024, + "pass@64": 0.7581490937591103, + "pass@128": 0.8352601156069365 + }, + "16": { + "pass@1": 0.1690272177419355, + "pass@2": 0.26243729362458734, + "pass@4": 0.3701715785526809, + "pass@8": 0.4754923682022537, + "pass@16": 0.566165648570034, + "pass@32": 0.6434866743114298, + "pass@64": 0.715818663351052, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.14773753612716764, + "pass@2": 0.22860994322060887, + "pass@4": 0.3213330353214517, + "pass@8": 0.4127466446487047, + "pass@16": 0.4945081321294641, + "pass@32": 0.5670904861099084, + "pass@64": 0.6320432166581963, + "pass@128": 0.684971098265896 + }, + "18": { + "pass@1": 0.12301339285714286, + "pass@2": 0.19675583520809908, + "pass@4": 0.2861904290535112, + "pass@8": 0.37906513974828127, + "pass@16": 0.46258529242199625, + "pass@32": 0.531878886537751, + "pass@64": 0.5906816065671889, + "pass@128": 0.6457142857142857 + }, + "19": { + "pass@1": 0.13278903903903905, + "pass@2": 0.21219681788087297, + "pass@4": 0.30740728129704514, + "pass@8": 0.3992844439193679, + "pass@16": 0.47047243675247624, + "pass@32": 0.522112980009493, + "pass@64": 0.5637430472466359, + "pass@128": 0.6066066066066066 + }, + "20": { + "pass@1": 0.1215746996996997, + "pass@2": 0.19323038589770877, + "pass@4": 0.27804619692808696, + "pass@8": 0.36386612424102277, + "pass@16": 0.4397676932029267, + "pass@32": 0.4992069295218254, + "pass@64": 0.5447303107088249, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 303408, + "answer_accuracy": 0.3695050662509743, + "avg_response_len": 260.1671899356976, + "resp_tokens_sum": 213628483, + "pass_at_k": { + "pass@1": 0.3695050662509743, + "pass@2": 0.47036956628473514, + "pass@4": 0.573653824112794, + "pass@8": 0.6669001838874253, + "pass@16": 0.7418401866442392, + "pass@32": 0.7981261746577575, + "pass@64": 0.8402704965092456, + "pass@128": 0.873265783320343 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.305382476993865, + "pass@2": 0.4384706656683249, + "pass@4": 0.5788130030525324, + "pass@8": 0.7079080797271436, + "pass@16": 0.809303783504082, + "pass@32": 0.8795306867277652, + "pass@64": 0.9246891544210146, + "pass@128": 0.950920245398773 + }, + "2": { + "pass@1": 0.9775598404255319, + "pass@2": 0.9882914402986861, + "pass@4": 0.9931788543529323, + "pass@8": 0.9950785758504687, + "pass@16": 0.9963275595235965, + "pass@32": 0.9977752173693791, + "pass@64": 0.9992040875534045, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9306344696969697, + "pass@2": 0.9533270699117159, + "pass@4": 0.9638342272556834, + "pass@8": 0.9706880776824892, + "pass@16": 0.9762976075504467, + "pass@32": 0.981515194409324, + "pass@64": 0.9865150313128073, + "pass@128": 0.990909090909091 + }, + "4": { + "pass@1": 0.7786150147928994, + "pass@2": 0.8566846870195216, + "pass@4": 0.9127427596883223, + "pass@8": 0.9497142611938738, + "pass@16": 0.9711949174245315, + "pass@32": 0.9826570262446811, + "pass@64": 0.9899957012280247, + "pass@128": 0.9940828402366864 + }, + "5": { + "pass@1": 0.6168025362318841, + "pass@2": 0.7120884685609948, + "pass@4": 0.7810124440966606, + "pass@8": 0.8333119147189844, + "pass@16": 0.8745230907515112, + "pass@32": 0.9086414545020511, + "pass@64": 0.9394193940381265, + "pass@128": 0.9652173913043478 + }, + "6": { + "pass@1": 0.5708945200573066, + "pass@2": 0.7201273182095074, + "pass@4": 0.8332363168966347, + "pass@8": 0.9048154542138614, + "pass@16": 0.9454304796626805, + "pass@32": 0.9676635934894852, + "pass@64": 0.9803148386384075, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.45682803468208094, + "pass@2": 0.6100648297756136, + "pass@4": 0.7466018149465419, + "pass@8": 0.8507837490523493, + "pass@16": 0.9185662560649683, + "pass@32": 0.9563321743967517, + "pass@64": 0.9785556722654779, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.3819762323943662, + "pass@2": 0.5406409421093487, + "pass@4": 0.693883219263085, + "pass@8": 0.8154527976781428, + "pass@16": 0.8940305609873201, + "pass@32": 0.9388662926400119, + "pass@64": 0.9641705344024881, + "pass@128": 0.9802816901408451 + }, + "9": { + "pass@1": 0.3124766092814371, + "pass@2": 0.4618525007072463, + "pass@4": 0.6193879476704632, + "pass@8": 0.7587824319807451, + "pass@16": 0.8609492491217673, + "pass@32": 0.9215203946762859, + "pass@64": 0.9532264564553681, + "pass@128": 0.9700598802395209 + }, + "11": { + "pass@1": 0.23966961596385541, + "pass@2": 0.3581350500426905, + "pass@4": 0.48620031052217855, + "pass@8": 0.6102333477314572, + "pass@16": 0.7208873268789171, + "pass@32": 0.8102303148700188, + "pass@64": 0.8733357401088951, + "pass@128": 0.9156626506024096 + }, + "12": { + "pass@1": 0.2491673197492163, + "pass@2": 0.376715884407474, + "pass@4": 0.5163546439532046, + "pass@8": 0.6470903372450432, + "pass@16": 0.7569027766764611, + "pass@32": 0.8431432973967836, + "pass@64": 0.904642073000038, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.195750175070028, + "pass@2": 0.3036820535300734, + "pass@4": 0.4266344887036182, + "pass@8": 0.5461250500217757, + "pass@16": 0.6500571912155733, + "pass@32": 0.7377077535289895, + "pass@64": 0.8120524459657263, + "pass@128": 0.876750700280112 + }, + "14": { + "pass@1": 0.18718019005847952, + "pass@2": 0.29091444778284303, + "pass@4": 0.40853745721477797, + "pass@8": 0.5212011661278351, + "pass@16": 0.6195737300108569, + "pass@32": 0.7043303525635856, + "pass@64": 0.7735331168668783, + "pass@128": 0.8245614035087719 + }, + "15": { + "pass@1": 0.15082859848484848, + "pass@2": 0.24180013719875898, + "pass@4": 0.3517029428139666, + "pass@8": 0.46704428304724294, + "pass@16": 0.5760473201704117, + "pass@32": 0.6708225663764403, + "pass@64": 0.7498293734022303, + "pass@128": 0.8121212121212121 + }, + "16": { + "pass@1": 0.14872685185185186, + "pass@2": 0.237111136348341, + "pass@4": 0.34124766161708425, + "pass@8": 0.44297967345870165, + "pass@16": 0.5312036383809488, + "pass@32": 0.6088643968988856, + "pass@64": 0.6800786909386469, + "pass@128": 0.7464387464387464 + }, + "17": { + "pass@1": 0.15285326086956522, + "pass@2": 0.2479397955690318, + "pass@4": 0.3625688723109922, + "pass@8": 0.47662920580834106, + "pass@16": 0.5736009919599262, + "pass@32": 0.6496201555232083, + "pass@64": 0.7054333096266076, + "pass@128": 0.7422360248447205 + }, + "18": { + "pass@1": 0.13494944852941176, + "pass@2": 0.21811783522464095, + "pass@4": 0.3204172084188005, + "pass@8": 0.4256938916457332, + "pass@16": 0.5168048555096408, + "pass@32": 0.5864634897053611, + "pass@64": 0.6395187651433718, + "pass@128": 0.6852941176470588 + }, + "19": { + "pass@1": 0.12423270089285714, + "pass@2": 0.19673212235189355, + "pass@4": 0.2854409744540861, + "pass@8": 0.3764069622534746, + "pass@16": 0.4543999639029509, + "pass@32": 0.5144732309561025, + "pass@64": 0.5651312688961807, + "pass@128": 0.6220238095238095 + }, + "20": { + "pass@1": 0.1110123502994012, + "pass@2": 0.18206966971568675, + "pass@4": 0.27122720307416665, + "pass@8": 0.364016093733274, + "pass@16": 0.44317220201732704, + "pass@32": 0.5001531127134062, + "pass@64": 0.5416504433836719, + "pass@128": 0.5808383233532934 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 294338, + "answer_accuracy": 0.36739345342706503, + "avg_response_len": 278.3378410089471, + "resp_tokens_sum": 222990918, + "pass_at_k": { + "pass@1": 0.36739345342706503, + "pass@2": 0.4674166444414597, + "pass@4": 0.5704824904406169, + "pass@8": 0.6637894140522359, + "pass@16": 0.7393907080307304, + "pass@32": 0.7967258012135678, + "pass@64": 0.8399129773505546, + "pass@128": 0.8737817542738456 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3270055970149254, + "pass@2": 0.47624610706311005, + "pass@4": 0.6279120981146009, + "pass@8": 0.7563851107378542, + "pass@16": 0.8497442890943676, + "pass@32": 0.9111210388030382, + "pass@64": 0.9478362643936907, + "pass@128": 0.9671641791044776 + }, + "2": { + "pass@1": 0.9769642857142857, + "pass@2": 0.9830708661417321, + "pass@4": 0.9866033550270502, + "pass@8": 0.9893155018165151, + "pass@16": 0.9916245608505572, + "pass@32": 0.9939798256522657, + "pass@64": 0.9961816845755459, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9469979956268222, + "pass@2": 0.9713605633479483, + "pass@4": 0.9806534271590681, + "pass@8": 0.9854379828873127, + "pass@16": 0.989555097127444, + "pass@32": 0.9933855442177933, + "pass@64": 0.9955484171321441, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.7869380482456141, + "pass@2": 0.8646955725928998, + "pass@4": 0.9180183661252861, + "pass@8": 0.9531497227749841, + "pass@16": 0.9741448322769339, + "pass@32": 0.9852302693709742, + "pass@64": 0.9917320162290298, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.5654181985294118, + "pass@2": 0.6746120889300605, + "pass@4": 0.7654377985840002, + "pass@8": 0.8379095206619214, + "pass@16": 0.8929060765099897, + "pass@32": 0.9300541740977897, + "pass@64": 0.9505432949778564, + "pass@128": 0.9588235294117647 + }, + "6": { + "pass@1": 0.5254830441640379, + "pass@2": 0.6867497795523986, + "pass@4": 0.8163599742264075, + "pass@8": 0.9003470975341901, + "pass@16": 0.9486839007810618, + "pass@32": 0.9743177057639598, + "pass@64": 0.9871789815186821, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.4392306170886076, + "pass@2": 0.5908540877603908, + "pass@4": 0.7283318169089623, + "pass@8": 0.8372287158063058, + "pass@16": 0.9098519085185298, + "pass@32": 0.9500437938189338, + "pass@64": 0.971288581664221, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.37216122372372373, + "pass@2": 0.5274600387789361, + "pass@4": 0.6792629347570285, + "pass@8": 0.8013797775771526, + "pass@16": 0.8833480015453465, + "pass@32": 0.9340773849291986, + "pass@64": 0.9645022213053622, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.28336012861736337, + "pass@2": 0.4180384744410967, + "pass@4": 0.5628083778514825, + "pass@8": 0.6970908247674539, + "pass@16": 0.8052044902612292, + "pass@32": 0.8800700951064251, + "pass@64": 0.9274745938248696, + "pass@128": 0.9646302250803859 + }, + "11": { + "pass@1": 0.2545138888888889, + "pass@2": 0.3883834833145856, + "pass@4": 0.5346769011016481, + "pass@8": 0.6715277489831641, + "pass@16": 0.7853170286802313, + "pass@32": 0.8677554185709992, + "pass@64": 0.9183972865925268, + "pass@128": 0.9492063492063492 + }, + "12": { + "pass@1": 0.21502293577981652, + "pass@2": 0.3330044968576177, + "pass@4": 0.4687746024292833, + "pass@8": 0.6034395513800196, + "pass@16": 0.7221423876765495, + "pass@32": 0.8183903107061395, + "pass@64": 0.8911141319352575, + "pass@128": 0.9418960244648318 + }, + "13": { + "pass@1": 0.2099488811728395, + "pass@2": 0.3174991038446582, + "pass@4": 0.4354547793794296, + "pass@8": 0.5481744928579311, + "pass@16": 0.6486211003222007, + "pass@32": 0.7379259580961539, + "pass@64": 0.8151676418417848, + "pass@128": 0.8734567901234568 + }, + "14": { + "pass@1": 0.1815189169139466, + "pass@2": 0.28346785263674384, + "pass@4": 0.4032592756395063, + "pass@8": 0.523281961422275, + "pass@16": 0.6304113447749283, + "pass@32": 0.7224702971779134, + "pass@64": 0.7969991172125063, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.1320167824074074, + "pass@2": 0.2144788325070477, + "pass@4": 0.3215919644882351, + "pass@8": 0.44103201441754786, + "pass@16": 0.5547608300838373, + "pass@32": 0.651467623339146, + "pass@64": 0.7279869288291854, + "pass@128": 0.7901234567901234 + }, + "16": { + "pass@1": 0.15899244100294985, + "pass@2": 0.2515438819594455, + "pass@4": 0.3578476877337234, + "pass@8": 0.4608270452373182, + "pass@16": 0.5533257358956741, + "pass@32": 0.6394566913432015, + "pass@64": 0.7226423547087432, + "pass@128": 0.799410029498525 + }, + "17": { + "pass@1": 0.1363187123493976, + "pass@2": 0.22074222085191147, + "pass@4": 0.3212653414558117, + "pass@8": 0.4186067605348141, + "pass@16": 0.49906082889385794, + "pass@32": 0.5628644264374904, + "pass@64": 0.6196725167686506, + "pass@128": 0.677710843373494 + }, + "18": { + "pass@1": 0.12885584677419354, + "pass@2": 0.202006604013208, + "pass@4": 0.2896082764856006, + "pass@8": 0.3788062977989847, + "pass@16": 0.4582978330657227, + "pass@32": 0.5262400787404502, + "pass@64": 0.5901966248323723, + "pass@128": 0.6516129032258065 + }, + "19": { + "pass@1": 0.13878398791540786, + "pass@2": 0.2194093893474797, + "pass@4": 0.31099969527978194, + "pass@8": 0.3977403547926579, + "pass@16": 0.4681941745583987, + "pass@32": 0.5200797609470273, + "pass@64": 0.5630198897420078, + "pass@128": 0.6072507552870091 + }, + "20": { + "pass@1": 0.11085304054054054, + "pass@2": 0.18222220744366419, + "pass@4": 0.2722583466481104, + "pass@8": 0.36793721246803396, + "pass@16": 0.4533363894654612, + "pass@32": 0.5175601720413012, + "pass@64": 0.5652306966653234, + "pass@128": 0.6036036036036037 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/summary.csv b/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..983e313818f51b4fd56b7ef909dc832d708343bd --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-42828,0.5884739583333334,0.1697921875,0.3681151315789474,0.0,0.0,0.0,196.50156336805554,322.145346875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/summary.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7940f73cabf5840f4d4fcc7e134763dbb0b3b4db --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block16/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-42828", + "id_acc": 0.5884739583333334, + "ood_acc": 0.1697921875, + "total_acc": 0.3681151315789474, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.50156336805554, + "ood_avg_resp_len": 322.145346875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 297510, + "answer_accuracy": 0.3674196767309516, + "avg_response_len": 249.58559417483403, + "resp_tokens_sum": 202096444, + "pass_at_k": { + "pass@1": 0.3674196767309516, + "pass@2": 0.4686072282307062, + "pass@4": 0.5724512478394008, + "pass@8": 0.6657046444352267, + "pass@16": 0.7403374473959198, + "pass@32": 0.7965784096114769, + "pass@64": 0.8389524590454404, + "pass@128": 0.8721150806196649 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3160951327433628, + "pass@2": 0.45978539532669027, + "pass@4": 0.6087440955942451, + "pass@8": 0.7389912618384603, + "pass@16": 0.8347761227732956, + "pass@32": 0.8974145781361296, + "pass@64": 0.9361195557565894, + "pass@128": 0.9587020648967551 + }, + "2": { + "pass@1": 0.9389846183800623, + "pass@2": 0.9547148275566021, + "pass@4": 0.964830180398011, + "pass@8": 0.9716501249659655, + "pass@16": 0.9762703677512582, + "pass@32": 0.9794593284358751, + "pass@64": 0.9826803268884213, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.93623375382263, + "pass@2": 0.9602559356112597, + "pass@4": 0.971118965347221, + "pass@8": 0.9775815501353607, + "pass@16": 0.9825540775856214, + "pass@32": 0.9868925189336957, + "pass@64": 0.9907274512321347, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.782666015625, + "pass@2": 0.8574395607775591, + "pass@4": 0.9108361469464749, + "pass@8": 0.9460693707480754, + "pass@16": 0.9668682851949002, + "pass@32": 0.9775023168825377, + "pass@64": 0.9821243598725908, + "pass@128": 0.984375 + }, + "5": { + "pass@1": 0.6209325396825397, + "pass@2": 0.7310359642544682, + "pass@4": 0.8163355446640592, + "pass@8": 0.8783539103209079, + "pass@16": 0.922046305462945, + "pass@32": 0.9516874532916285, + "pass@64": 0.971409019971419, + "pass@128": 0.9841269841269841 + }, + "6": { + "pass@1": 0.5845808383233533, + "pass@2": 0.723798343274082, + "pass@4": 0.834097648067195, + "pass@8": 0.908153197796081, + "pass@16": 0.9503107359238322, + "pass@32": 0.9694990410164247, + "pass@64": 0.9774287469166411, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.45092917899408286, + "pass@2": 0.596248121767693, + "pass@4": 0.7315876857848386, + "pass@8": 0.8403357342652344, + "pass@16": 0.9115535229290597, + "pass@32": 0.9505698802282272, + "pass@64": 0.9706700595276606, + "pass@128": 0.9822485207100592 + }, + "8": { + "pass@1": 0.40384615384615385, + "pass@2": 0.5536141290631941, + "pass@4": 0.698270090036822, + "pass@8": 0.8140536634874682, + "pass@16": 0.8919098750755075, + "pass@32": 0.9385195819134277, + "pass@64": 0.9639372810558046, + "pass@128": 0.9743589743589743 + }, + "9": { + "pass@1": 0.29634683098591547, + "pass@2": 0.44307973827215225, + "pass@4": 0.5996262900236059, + "pass@8": 0.7377739613933347, + "pass@16": 0.8412027330164041, + "pass@32": 0.9081616296352407, + "pass@64": 0.9463322026470657, + "pass@128": 0.9690140845070423 + }, + "11": { + "pass@1": 0.27554001416430596, + "pass@2": 0.4100284262006203, + "pass@4": 0.5497061987294078, + "pass@8": 0.6745030060620455, + "pass@16": 0.7762085776756984, + "pass@32": 0.8539760738175253, + "pass@64": 0.9082416689117168, + "pass@128": 0.9405099150141643 + }, + "12": { + "pass@1": 0.223583156779661, + "pass@2": 0.34553187141331937, + "pass@4": 0.48248001150279896, + "pass@8": 0.6133563555890079, + "pass@16": 0.7254988286208823, + "pass@32": 0.8126269158410524, + "pass@64": 0.8750599826992427, + "pass@128": 0.9180790960451978 + }, + "13": { + "pass@1": 0.21137833072100312, + "pass@2": 0.32505769752918806, + "pass@4": 0.44905429150901616, + "pass@8": 0.5689478723786576, + "pass@16": 0.6776168936361864, + "pass@32": 0.7707110571183069, + "pass@64": 0.8439511414301138, + "pass@128": 0.8934169278996865 + }, + "14": { + "pass@1": 0.17978485202492211, + "pass@2": 0.2873168708759536, + "pass@4": 0.4125795875982794, + "pass@8": 0.5333225333050318, + "pass@16": 0.6392014624846123, + "pass@32": 0.7319459986189009, + "pass@64": 0.8117486314877205, + "pass@128": 0.8785046728971962 + }, + "15": { + "pass@1": 0.14981484826589594, + "pass@2": 0.23933430715488574, + "pass@4": 0.34942827206209043, + "pass@8": 0.46691425776949924, + "pass@16": 0.577039877850708, + "pass@32": 0.6734734398150024, + "pass@64": 0.7581490937591103, + "pass@128": 0.8352601156069365 + }, + "16": { + "pass@1": 0.1690272177419355, + "pass@2": 0.26243729362458734, + "pass@4": 0.3701715785526809, + "pass@8": 0.4754923682022537, + "pass@16": 0.566165648570034, + "pass@32": 0.6434866743114298, + "pass@64": 0.715818663351052, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.14773753612716764, + "pass@2": 0.22860994322060887, + "pass@4": 0.3213330353214517, + "pass@8": 0.4127466446487047, + "pass@16": 0.4945081321294641, + "pass@32": 0.5670904861099084, + "pass@64": 0.6320432166581963, + "pass@128": 0.684971098265896 + }, + "18": { + "pass@1": 0.12301339285714286, + "pass@2": 0.19675583520809908, + "pass@4": 0.2861904290535112, + "pass@8": 0.37906513974828127, + "pass@16": 0.46258529242199625, + "pass@32": 0.531878886537751, + "pass@64": 0.5906816065671889, + "pass@128": 0.6457142857142857 + }, + "19": { + "pass@1": 0.13278903903903905, + "pass@2": 0.21219681788087297, + "pass@4": 0.30740728129704514, + "pass@8": 0.3992844439193679, + "pass@16": 0.47047243675247624, + "pass@32": 0.522112980009493, + "pass@64": 0.5637430472466359, + "pass@128": 0.6066066066066066 + }, + "20": { + "pass@1": 0.1215746996996997, + "pass@2": 0.19323038589770877, + "pass@4": 0.27804619692808696, + "pass@8": 0.36386612424102277, + "pass@16": 0.4397676932029267, + "pass@32": 0.4992069295218254, + "pass@64": 0.5447303107088249, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 303408, + "answer_accuracy": 0.3695050662509743, + "avg_response_len": 260.1671899356976, + "resp_tokens_sum": 213628483, + "pass_at_k": { + "pass@1": 0.3695050662509743, + "pass@2": 0.47036956628473514, + "pass@4": 0.573653824112794, + "pass@8": 0.6669001838874253, + "pass@16": 0.7418401866442392, + "pass@32": 0.7981261746577575, + "pass@64": 0.8402704965092456, + "pass@128": 0.873265783320343 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.305382476993865, + "pass@2": 0.4384706656683249, + "pass@4": 0.5788130030525324, + "pass@8": 0.7079080797271436, + "pass@16": 0.809303783504082, + "pass@32": 0.8795306867277652, + "pass@64": 0.9246891544210146, + "pass@128": 0.950920245398773 + }, + "2": { + "pass@1": 0.9775598404255319, + "pass@2": 0.9882914402986861, + "pass@4": 0.9931788543529323, + "pass@8": 0.9950785758504687, + "pass@16": 0.9963275595235965, + "pass@32": 0.9977752173693791, + "pass@64": 0.9992040875534045, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9306344696969697, + "pass@2": 0.9533270699117159, + "pass@4": 0.9638342272556834, + "pass@8": 0.9706880776824892, + "pass@16": 0.9762976075504467, + "pass@32": 0.981515194409324, + "pass@64": 0.9865150313128073, + "pass@128": 0.990909090909091 + }, + "4": { + "pass@1": 0.7786150147928994, + "pass@2": 0.8566846870195216, + "pass@4": 0.9127427596883223, + "pass@8": 0.9497142611938738, + "pass@16": 0.9711949174245315, + "pass@32": 0.9826570262446811, + "pass@64": 0.9899957012280247, + "pass@128": 0.9940828402366864 + }, + "5": { + "pass@1": 0.6168025362318841, + "pass@2": 0.7120884685609948, + "pass@4": 0.7810124440966606, + "pass@8": 0.8333119147189844, + "pass@16": 0.8745230907515112, + "pass@32": 0.9086414545020511, + "pass@64": 0.9394193940381265, + "pass@128": 0.9652173913043478 + }, + "6": { + "pass@1": 0.5708945200573066, + "pass@2": 0.7201273182095074, + "pass@4": 0.8332363168966347, + "pass@8": 0.9048154542138614, + "pass@16": 0.9454304796626805, + "pass@32": 0.9676635934894852, + "pass@64": 0.9803148386384075, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.45682803468208094, + "pass@2": 0.6100648297756136, + "pass@4": 0.7466018149465419, + "pass@8": 0.8507837490523493, + "pass@16": 0.9185662560649683, + "pass@32": 0.9563321743967517, + "pass@64": 0.9785556722654779, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.3819762323943662, + "pass@2": 0.5406409421093487, + "pass@4": 0.693883219263085, + "pass@8": 0.8154527976781428, + "pass@16": 0.8940305609873201, + "pass@32": 0.9388662926400119, + "pass@64": 0.9641705344024881, + "pass@128": 0.9802816901408451 + }, + "9": { + "pass@1": 0.3124766092814371, + "pass@2": 0.4618525007072463, + "pass@4": 0.6193879476704632, + "pass@8": 0.7587824319807451, + "pass@16": 0.8609492491217673, + "pass@32": 0.9215203946762859, + "pass@64": 0.9532264564553681, + "pass@128": 0.9700598802395209 + }, + "11": { + "pass@1": 0.23966961596385541, + "pass@2": 0.3581350500426905, + "pass@4": 0.48620031052217855, + "pass@8": 0.6102333477314572, + "pass@16": 0.7208873268789171, + "pass@32": 0.8102303148700188, + "pass@64": 0.8733357401088951, + "pass@128": 0.9156626506024096 + }, + "12": { + "pass@1": 0.2491673197492163, + "pass@2": 0.376715884407474, + "pass@4": 0.5163546439532046, + "pass@8": 0.6470903372450432, + "pass@16": 0.7569027766764611, + "pass@32": 0.8431432973967836, + "pass@64": 0.904642073000038, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.195750175070028, + "pass@2": 0.3036820535300734, + "pass@4": 0.4266344887036182, + "pass@8": 0.5461250500217757, + "pass@16": 0.6500571912155733, + "pass@32": 0.7377077535289895, + "pass@64": 0.8120524459657263, + "pass@128": 0.876750700280112 + }, + "14": { + "pass@1": 0.18718019005847952, + "pass@2": 0.29091444778284303, + "pass@4": 0.40853745721477797, + "pass@8": 0.5212011661278351, + "pass@16": 0.6195737300108569, + "pass@32": 0.7043303525635856, + "pass@64": 0.7735331168668783, + "pass@128": 0.8245614035087719 + }, + "15": { + "pass@1": 0.15082859848484848, + "pass@2": 0.24180013719875898, + "pass@4": 0.3517029428139666, + "pass@8": 0.46704428304724294, + "pass@16": 0.5760473201704117, + "pass@32": 0.6708225663764403, + "pass@64": 0.7498293734022303, + "pass@128": 0.8121212121212121 + }, + "16": { + "pass@1": 0.14872685185185186, + "pass@2": 0.237111136348341, + "pass@4": 0.34124766161708425, + "pass@8": 0.44297967345870165, + "pass@16": 0.5312036383809488, + "pass@32": 0.6088643968988856, + "pass@64": 0.6800786909386469, + "pass@128": 0.7464387464387464 + }, + "17": { + "pass@1": 0.15285326086956522, + "pass@2": 0.2479397955690318, + "pass@4": 0.3625688723109922, + "pass@8": 0.47662920580834106, + "pass@16": 0.5736009919599262, + "pass@32": 0.6496201555232083, + "pass@64": 0.7054333096266076, + "pass@128": 0.7422360248447205 + }, + "18": { + "pass@1": 0.13494944852941176, + "pass@2": 0.21811783522464095, + "pass@4": 0.3204172084188005, + "pass@8": 0.4256938916457332, + "pass@16": 0.5168048555096408, + "pass@32": 0.5864634897053611, + "pass@64": 0.6395187651433718, + "pass@128": 0.6852941176470588 + }, + "19": { + "pass@1": 0.12423270089285714, + "pass@2": 0.19673212235189355, + "pass@4": 0.2854409744540861, + "pass@8": 0.3764069622534746, + "pass@16": 0.4543999639029509, + "pass@32": 0.5144732309561025, + "pass@64": 0.5651312688961807, + "pass@128": 0.6220238095238095 + }, + "20": { + "pass@1": 0.1110123502994012, + "pass@2": 0.18206966971568675, + "pass@4": 0.27122720307416665, + "pass@8": 0.364016093733274, + "pass@16": 0.44317220201732704, + "pass@32": 0.5001531127134062, + "pass@64": 0.5416504433836719, + "pass@128": 0.5808383233532934 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 294338, + "answer_accuracy": 0.36739345342706503, + "avg_response_len": 278.3378410089471, + "resp_tokens_sum": 222990918, + "pass_at_k": { + "pass@1": 0.36739345342706503, + "pass@2": 0.4674166444414597, + "pass@4": 0.5704824904406169, + "pass@8": 0.6637894140522359, + "pass@16": 0.7393907080307304, + "pass@32": 0.7967258012135678, + "pass@64": 0.8399129773505546, + "pass@128": 0.8737817542738456 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3270055970149254, + "pass@2": 0.47624610706311005, + "pass@4": 0.6279120981146009, + "pass@8": 0.7563851107378542, + "pass@16": 0.8497442890943676, + "pass@32": 0.9111210388030382, + "pass@64": 0.9478362643936907, + "pass@128": 0.9671641791044776 + }, + "2": { + "pass@1": 0.9769642857142857, + "pass@2": 0.9830708661417321, + "pass@4": 0.9866033550270502, + "pass@8": 0.9893155018165151, + "pass@16": 0.9916245608505572, + "pass@32": 0.9939798256522657, + "pass@64": 0.9961816845755459, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9469979956268222, + "pass@2": 0.9713605633479483, + "pass@4": 0.9806534271590681, + "pass@8": 0.9854379828873127, + "pass@16": 0.989555097127444, + "pass@32": 0.9933855442177933, + "pass@64": 0.9955484171321441, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.7869380482456141, + "pass@2": 0.8646955725928998, + "pass@4": 0.9180183661252861, + "pass@8": 0.9531497227749841, + "pass@16": 0.9741448322769339, + "pass@32": 0.9852302693709742, + "pass@64": 0.9917320162290298, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.5654181985294118, + "pass@2": 0.6746120889300605, + "pass@4": 0.7654377985840002, + "pass@8": 0.8379095206619214, + "pass@16": 0.8929060765099897, + "pass@32": 0.9300541740977897, + "pass@64": 0.9505432949778564, + "pass@128": 0.9588235294117647 + }, + "6": { + "pass@1": 0.5254830441640379, + "pass@2": 0.6867497795523986, + "pass@4": 0.8163599742264075, + "pass@8": 0.9003470975341901, + "pass@16": 0.9486839007810618, + "pass@32": 0.9743177057639598, + "pass@64": 0.9871789815186821, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.4392306170886076, + "pass@2": 0.5908540877603908, + "pass@4": 0.7283318169089623, + "pass@8": 0.8372287158063058, + "pass@16": 0.9098519085185298, + "pass@32": 0.9500437938189338, + "pass@64": 0.971288581664221, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.37216122372372373, + "pass@2": 0.5274600387789361, + "pass@4": 0.6792629347570285, + "pass@8": 0.8013797775771526, + "pass@16": 0.8833480015453465, + "pass@32": 0.9340773849291986, + "pass@64": 0.9645022213053622, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.28336012861736337, + "pass@2": 0.4180384744410967, + "pass@4": 0.5628083778514825, + "pass@8": 0.6970908247674539, + "pass@16": 0.8052044902612292, + "pass@32": 0.8800700951064251, + "pass@64": 0.9274745938248696, + "pass@128": 0.9646302250803859 + }, + "11": { + "pass@1": 0.2545138888888889, + "pass@2": 0.3883834833145856, + "pass@4": 0.5346769011016481, + "pass@8": 0.6715277489831641, + "pass@16": 0.7853170286802313, + "pass@32": 0.8677554185709992, + "pass@64": 0.9183972865925268, + "pass@128": 0.9492063492063492 + }, + "12": { + "pass@1": 0.21502293577981652, + "pass@2": 0.3330044968576177, + "pass@4": 0.4687746024292833, + "pass@8": 0.6034395513800196, + "pass@16": 0.7221423876765495, + "pass@32": 0.8183903107061395, + "pass@64": 0.8911141319352575, + "pass@128": 0.9418960244648318 + }, + "13": { + "pass@1": 0.2099488811728395, + "pass@2": 0.3174991038446582, + "pass@4": 0.4354547793794296, + "pass@8": 0.5481744928579311, + "pass@16": 0.6486211003222007, + "pass@32": 0.7379259580961539, + "pass@64": 0.8151676418417848, + "pass@128": 0.8734567901234568 + }, + "14": { + "pass@1": 0.1815189169139466, + "pass@2": 0.28346785263674384, + "pass@4": 0.4032592756395063, + "pass@8": 0.523281961422275, + "pass@16": 0.6304113447749283, + "pass@32": 0.7224702971779134, + "pass@64": 0.7969991172125063, + "pass@128": 0.8545994065281899 + }, + "15": { + "pass@1": 0.1320167824074074, + "pass@2": 0.2144788325070477, + "pass@4": 0.3215919644882351, + "pass@8": 0.44103201441754786, + "pass@16": 0.5547608300838373, + "pass@32": 0.651467623339146, + "pass@64": 0.7279869288291854, + "pass@128": 0.7901234567901234 + }, + "16": { + "pass@1": 0.15899244100294985, + "pass@2": 0.2515438819594455, + "pass@4": 0.3578476877337234, + "pass@8": 0.4608270452373182, + "pass@16": 0.5533257358956741, + "pass@32": 0.6394566913432015, + "pass@64": 0.7226423547087432, + "pass@128": 0.799410029498525 + }, + "17": { + "pass@1": 0.1363187123493976, + "pass@2": 0.22074222085191147, + "pass@4": 0.3212653414558117, + "pass@8": 0.4186067605348141, + "pass@16": 0.49906082889385794, + "pass@32": 0.5628644264374904, + "pass@64": 0.6196725167686506, + "pass@128": 0.677710843373494 + }, + "18": { + "pass@1": 0.12885584677419354, + "pass@2": 0.202006604013208, + "pass@4": 0.2896082764856006, + "pass@8": 0.3788062977989847, + "pass@16": 0.4582978330657227, + "pass@32": 0.5262400787404502, + "pass@64": 0.5901966248323723, + "pass@128": 0.6516129032258065 + }, + "19": { + "pass@1": 0.13878398791540786, + "pass@2": 0.2194093893474797, + "pass@4": 0.31099969527978194, + "pass@8": 0.3977403547926579, + "pass@16": 0.4681941745583987, + "pass@32": 0.5200797609470273, + "pass@64": 0.5630198897420078, + "pass@128": 0.6072507552870091 + }, + "20": { + "pass@1": 0.11085304054054054, + "pass@2": 0.18222220744366419, + "pass@4": 0.2722583466481104, + "pass@8": 0.36793721246803396, + "pass@16": 0.4533363894654612, + "pass@32": 0.5175601720413012, + "pass@64": 0.5652306966653234, + "pass@128": 0.6036036036036037 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/checkpoint-42828_metrics.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/checkpoint-42828_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..54765c0e6bbfa52100ae811dde8e41514aeb08de --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/checkpoint-42828_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-42828", + "id": { + "answer_accuracy": 0.9903732638888889, + "avg_response_len": 197.94452256944444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9903732638888889, + "pass@2": 0.9931025945975482, + "pass@4": 0.9946375277569469, + "pass@8": 0.9956594653678896, + "pass@16": 0.9963844192588985, + "pass@32": 0.9969947370095827, + "pass@64": 0.9976140419540213, + "pass@128": 0.9982222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.940046875, + "pass@2": 0.9539087106299211, + "pass@4": 0.9620897415635549, + "pass@8": 0.9673531227365778, + "pass@16": 0.9712878106066547, + "pass@32": 0.9753845854705726, + "pass@64": 0.9801630592581841, + "pass@128": 0.985 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999765625, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9931875, + "pass@2": 0.9951967273622047, + "pass@4": 0.9966725070303712, + "pass@8": 0.9976910659892855, + "pass@16": 0.9982521531431331, + "pass@32": 0.9985788431070177, + "pass@64": 0.9988779523587306, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.998921875, + "pass@2": 0.9995840305118111, + "pass@4": 0.9998836683539557, + "pass@8": 0.9999881979994633, + "pass@16": 0.999999908401855, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977109375, + "pass@2": 0.9981526820866141, + "pass@4": 0.998311847487814, + "pass@8": 0.9985508396667573, + "pass@16": 0.998875518129056, + "pass@32": 0.999200274993652, + "pass@64": 0.9994993322645678, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.99365625, + "pass@2": 0.9964869586614173, + "pass@4": 0.9981354686914137, + "pass@8": 0.999296527211222, + "pass@16": 0.9998734507384869, + "pass@32": 0.9999957517907246, + "pass@64": 0.9999999990455016, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.989875, + "pass@2": 0.9945944881889768, + "pass@4": 0.9966445166854141, + "pass@8": 0.9980554347076996, + "pass@16": 0.9991709323109028, + "pass@32": 0.9997931777253324, + "pass@64": 0.9999860346592059, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.940046875, + "2": 1.0, + "3": 0.9999765625, + "4": 0.999984375, + "5": 0.9931875, + "6": 0.998921875, + "7": 0.9977109375, + "8": 0.99365625, + "9": 0.989875 + }, + "per_op_avg_response_len": { + "10": 277.4631171875, + "2": 114.9146171875, + "3": 143.5135, + "4": 158.5818984375, + "5": 171.3797890625, + "6": 196.8831171875, + "7": 221.55025, + "8": 240.8111328125, + "9": 256.40328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.4631171875, + "2": 114.9146171875, + "3": 143.5135, + "4": 158.5818984375, + "5": 171.3797890625, + "6": 196.8831171875, + "7": 221.55025, + "8": 240.8111328125, + "9": 256.40328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375292, + "answer_accuracy": 0.9901954576156704, + "avg_response_len": 188.44878999915568, + "resp_tokens_sum": 71423599, + "pass_at_k": { + "pass@1": 0.9901954576156704, + "pass@2": 0.9927189121306648, + "pass@4": 0.9941063069054056, + "pass@8": 0.9950618164347469, + "pass@16": 0.9957664547644377, + "pass@32": 0.996444313619182, + "pass@64": 0.9973445209589374, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391362463126843, + "pass@2": 0.9502618865119741, + "pass@4": 0.9565611159777595, + "pass@8": 0.960697067871047, + "pass@16": 0.9641969779408918, + "pass@32": 0.9691040751361097, + "pass@64": 0.9768076556801932, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9950892857142857, + "pass@2": 0.9976178758905138, + "pass@4": 0.9989978952035756, + "pass@8": 0.9997296666578273, + "pass@16": 0.9999812306164019, + "pass@32": 0.9999999574634271, + "pass@64": 0.9999999999999986, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981989146706587, + "pass@2": 0.9989778071809138, + "pass@4": 0.9996588796535164, + "pass@8": 0.9999646694808768, + "pass@16": 0.9999997257540574, + "pass@32": 0.9999999999968382, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969720784023669, + "pass@2": 0.9974847703023807, + "pass@4": 0.9978717981997812, + "pass@8": 0.9984862120344895, + "pass@16": 0.9992618879557866, + "pass@32": 0.9998528845966035, + "pass@64": 0.999998024451384, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927884615384616, + "pass@2": 0.9952408104936402, + "pass@4": 0.9968641546008672, + "pass@8": 0.9985039928739285, + "pass@16": 0.9996691654274897, + "pass@32": 0.9999870535678677, + "pass@64": 0.9999999969426491, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916593309859155, + "pass@2": 0.9964192636131751, + "pass@4": 0.9983151256621091, + "pass@8": 0.9993721194559138, + "pass@16": 0.9998884598509388, + "pass@32": 0.9999974897141034, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386651, + "answer_accuracy": 0.9897480135976409, + "avg_response_len": 195.91925632781783, + "resp_tokens_sum": 76537033, + "pass_at_k": { + "pass@1": 0.9897480135976409, + "pass@2": 0.9923977661479242, + "pass@4": 0.9939334393079635, + "pass@8": 0.9949619900509088, + "pass@16": 0.9957299659571438, + "pass@32": 0.9963873581330077, + "pass@64": 0.9969199601830231, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9294478527607362, + "pass@2": 0.9466284841312018, + "pass@4": 0.9571646012614984, + "pass@8": 0.9635862044908389, + "pass@16": 0.9679916416901503, + "pass@32": 0.9721634565198133, + "pass@64": 0.9758088391108484, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968976449275362, + "pass@2": 0.9970971699189775, + "pass@4": 0.9971014479168364, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9988359598853869, + "pass@2": 0.9997881320082124, + "pass@4": 0.9999931305148461, + "pass@8": 0.999999995394987, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967711343930635, + "pass@2": 0.997127250125165, + "pass@4": 0.9971999508009476, + "pass@8": 0.997290462425144, + "pass@16": 0.997471098265896, + "pass@32": 0.9978323699421965, + "pass@64": 0.9985549132947977, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954445422535211, + "pass@2": 0.9973179133858266, + "pass@4": 0.9986211822113786, + "pass@8": 0.9995469673676058, + "pass@16": 0.9999426641178323, + "pass@32": 0.9999994185630952, + "pass@64": 0.9999999999982961, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9878134356287425, + "pass@2": 0.9916725358338442, + "pass@4": 0.9937421916571807, + "pass@8": 0.9957880813290706, + "pass@16": 0.9978980094262961, + "pass@32": 0.9993986125945581, + "pass@64": 0.9999581943983665, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 378967, + "answer_accuracy": 0.9911883788081687, + "avg_response_len": 209.42693860897222, + "resp_tokens_sum": 80071458, + "pass_at_k": { + "pass@1": 0.9911883788081687, + "pass@2": 0.9942031035405396, + "pass@4": 0.9958835347498204, + "pass@8": 0.9969645651865903, + "pass@16": 0.9976656995886801, + "pass@32": 0.998160965329061, + "pass@64": 0.9985904025939704, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9512826492537313, + "pass@2": 0.9646837172405689, + "pass@4": 0.972477203689091, + "pass@8": 0.9777543703411323, + "pass@16": 0.9816709250558314, + "pass@32": 0.9848746781312615, + "pass@64": 0.98779576845511, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99995444606414, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9876608455882353, + "pass@2": 0.9910252142195461, + "pass@4": 0.9940828544409888, + "pass@8": 0.9964006499766762, + "pass@16": 0.9978178397028429, + "pass@32": 0.998761342664818, + "pass@64": 0.9996410363492089, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997781940063092, + "pass@2": 0.9999980594401252, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995302610759493, + "pass@2": 0.9999898771553873, + "pass@4": 0.9999999896175953, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9925628753753754, + "pass@2": 0.9967686682745736, + "pass@4": 0.9988088071198306, + "pass@8": 0.9997720961533224, + "pass@16": 0.9999910671569961, + "pass@32": 0.9999999924554084, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9900522508038585, + "pass@2": 0.995649590475226, + "pass@4": 0.9978545178838177, + "pass@8": 0.9989875020480404, + "pass@16": 0.9997189514965811, + "pass@32": 0.9999837050490139, + "pass@64": 0.9999999927877713, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.47812890625, + "avg_response_len": 338.39947109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.47812890625, + "pass@2": 0.5536532111220396, + "pass@4": 0.6183275945725415, + "pass@8": 0.6718912925779685, + "pass@16": 0.7151089123573282, + "pass@32": 0.7503878728849743, + "pass@64": 0.7807164015152299, + "pass@128": 0.8075 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9383125, + "pass@2": 0.9590136564960636, + "pass@4": 0.9696847320959882, + "pass@8": 0.9761633096527691, + "pass@16": 0.9803984161140732, + "pass@32": 0.983386489904395, + "pass@64": 0.9857262741232489, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8476796875, + "pass@2": 0.8924173228346463, + "pass@4": 0.9171074446944133, + "pass@8": 0.9333752530484688, + "pass@16": 0.9455672808407929, + "pass@32": 0.9549591513160477, + "pass@64": 0.9623154103917709, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.6962109375, + "pass@2": 0.7726345964566936, + "pass@4": 0.8222474234158222, + "pass@8": 0.856545671556595, + "pass@16": 0.8809665507559866, + "pass@32": 0.8985424480805302, + "pass@64": 0.913224331651731, + "pass@128": 0.927 + }, + "14": { + "pass@1": 0.62440625, + "pass@2": 0.7120086122047234, + "pass@4": 0.7731541090176218, + "pass@8": 0.8170347055909208, + "pass@16": 0.8494115376758603, + "pass@32": 0.8745452275237636, + "pass@64": 0.8956640884547219, + "pass@128": 0.913 + }, + "15": { + "pass@1": 0.51234375, + "pass@2": 0.6071980807086611, + "pass@4": 0.6804598734533189, + "pass@8": 0.7365722276708908, + "pass@16": 0.7799610394318404, + "pass@32": 0.8152456205013187, + "pass@64": 0.8447109414185273, + "pass@128": 0.869 + }, + "16": { + "pass@1": 0.3223359375, + "pass@2": 0.4263375984251966, + "pass@4": 0.5234660494000746, + "pass@8": 0.6059840191843239, + "pass@16": 0.6727891764989107, + "pass@32": 0.7272819482386783, + "pass@64": 0.7738442731749547, + "pass@128": 0.815 + }, + "17": { + "pass@1": 0.271390625, + "pass@2": 0.3647472933070873, + "pass@4": 0.4529848034308209, + "pass@8": 0.5279137253955453, + "pass@16": 0.5893852204022109, + "pass@32": 0.6408462552365666, + "pass@64": 0.6855945715322403, + "pass@128": 0.721 + }, + "18": { + "pass@1": 0.236875, + "pass@2": 0.3236471456692916, + "pass@4": 0.4089280459317586, + "pass@8": 0.48372288955778087, + "pass@16": 0.5449902991093204, + "pass@32": 0.5944102663571567, + "pass@64": 0.635087373223452, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1762421875, + "pass@2": 0.2509820374015752, + "pass@4": 0.3285974938132734, + "pass@8": 0.3995420743766352, + "pass@16": 0.4599080269284726, + "pass@32": 0.5104871992005041, + "pass@64": 0.5556715591845206, + "pass@128": 0.6 + }, + "20": { + "pass@1": 0.1554921875, + "pass@2": 0.22754576771653515, + "pass@4": 0.30664597047244085, + "pass@8": 0.38205904974572824, + "pass@16": 0.4477115758156405, + "pass@32": 0.5041741224907627, + "pass@64": 0.5553251919970433, + "pass@128": 0.605 + } + }, + "per_op_accuracy": { + "11": 0.9383125, + "12": 0.8476796875, + "13": 0.6962109375, + "14": 0.62440625, + "15": 0.51234375, + "16": 0.3223359375, + "17": 0.271390625, + "18": 0.236875, + "19": 0.1762421875, + "20": 0.1554921875 + }, + "per_op_avg_response_len": { + "11": 295.2929453125, + "12": 304.2809140625, + "13": 319.5819140625, + "14": 339.468, + "15": 348.131703125, + "16": 348.175515625, + "17": 358.687390625, + "18": 359.1170078125, + "19": 351.3666875, + "20": 359.8926328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 295.2929453125, + "12": 304.2809140625, + "13": 319.5819140625, + "14": 339.468, + "15": 348.131703125, + "16": 348.175515625, + "17": 358.687390625, + "18": 359.1170078125, + "19": 351.3666875, + "20": 359.8926328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 201868, + "answer_accuracy": 0.468675705794948, + "avg_response_len": 320.8595607355126, + "resp_tokens_sum": 138200630, + "pass_at_k": { + "pass@1": 0.468675705794948, + "pass@2": 0.5427983614325342, + "pass@4": 0.6071605365729772, + "pass@8": 0.6612218025483215, + "pass@16": 0.7053437738399166, + "pass@32": 0.7415817685374643, + "pass@64": 0.7733845150010682, + "pass@128": 0.8026745913818722 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9259693696883853, + "pass@2": 0.947184635073944, + "pass@4": 0.9595871744785438, + "pass@8": 0.9678898043700129, + "pass@16": 0.9737432000120178, + "pass@32": 0.9782974664456189, + "pass@64": 0.9823106914719336, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8313691737288136, + "pass@2": 0.8758886766537666, + "pass@4": 0.9012220606958026, + "pass@8": 0.918590237623188, + "pass@16": 0.9326627176304815, + "pass@32": 0.9442766994564674, + "pass@64": 0.9530374167763507, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.7017535266457681, + "pass@2": 0.7727542702342459, + "pass@4": 0.8188653515685144, + "pass@8": 0.8522624263765721, + "pass@16": 0.8756735247780262, + "pass@32": 0.8917525794424754, + "pass@64": 0.9062939546723526, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.6107135903426791, + "pass@2": 0.6995911981504643, + "pass@4": 0.7615712451831373, + "pass@8": 0.8077732564202801, + "pass@16": 0.8452598509358014, + "pass@32": 0.8754393054155718, + "pass@64": 0.8993014806085093, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.4949421965317919, + "pass@2": 0.5938915217787082, + "pass@4": 0.6744975657847677, + "pass@8": 0.737580929276051, + "pass@16": 0.785904780521841, + "pass@32": 0.8238610481001554, + "pass@64": 0.8551167092765369, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.3142641129032258, + "pass@2": 0.4167489522479044, + "pass@4": 0.5133889400921661, + "pass@8": 0.595774204307558, + "pass@16": 0.6626249637042165, + "pass@32": 0.7184575003185609, + "pass@64": 0.7686606169806237, + "pass@128": 0.8161290322580645 + }, + "17": { + "pass@1": 0.2604768786127168, + "pass@2": 0.3510835305630146, + "pass@4": 0.4378864013927445, + "pass@8": 0.5130409664619432, + "pass@16": 0.5733728302206639, + "pass@32": 0.6215663020835889, + "pass@64": 0.663231219544036, + "pass@128": 0.6965317919075145 + }, + "18": { + "pass@1": 0.20383928571428572, + "pass@2": 0.2901342800899887, + "pass@4": 0.37793249933044054, + "pass@8": 0.4567220739411263, + "pass@16": 0.5217816873300685, + "pass@32": 0.5729463732998272, + "pass@64": 0.6138528084122289, + "pass@128": 0.6485714285714286 + }, + "19": { + "pass@1": 0.16096565315315314, + "pass@2": 0.23618795961315653, + "pass@4": 0.31359704361279167, + "pass@8": 0.3814168055094861, + "pass@16": 0.4386634635102604, + "pass@32": 0.4892526277950988, + "pass@64": 0.5395220286288696, + "pass@128": 0.5945945945945946 + }, + "20": { + "pass@1": 0.15700075075075076, + "pass@2": 0.22422582523468362, + "pass@4": 0.2970158927206171, + "pass@8": 0.3685546240663337, + "pass@16": 0.4339257200950346, + "pass@32": 0.4925902703065918, + "pass@64": 0.547693493756059, + "pass@128": 0.6036036036036037 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 209981, + "answer_accuracy": 0.4878015350877193, + "avg_response_len": 335.6290467960155, + "resp_tokens_sum": 144476222, + "pass_at_k": { + "pass@1": 0.4878015350877193, + "pass@2": 0.5652740072020451, + "pass@4": 0.6306018598422313, + "pass@8": 0.6835428224508729, + "pass@16": 0.725319418079389, + "pass@32": 0.7591858497962812, + "pass@64": 0.7882248536446625, + "pass@128": 0.8138566755872733 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9418768825301205, + "pass@2": 0.9599654770657432, + "pass@4": 0.9683396273507975, + "pass@8": 0.9735100751334598, + "pass@16": 0.9769297712624254, + "pass@32": 0.9790286004592866, + "pass@64": 0.9803791219396885, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8510726880877743, + "pass@2": 0.8924041357095258, + "pass@4": 0.9151241544454282, + "pass@8": 0.9299759245700525, + "pass@16": 0.9410987595365371, + "pass@32": 0.9503070478915217, + "pass@64": 0.9591426372189908, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7162114845938375, + "pass@2": 0.7996261496724675, + "pass@4": 0.8501936338839996, + "pass@8": 0.8820293064882619, + "pass@16": 0.9039283198592027, + "pass@32": 0.9203175166364791, + "pass@64": 0.9332433569571942, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6420184576023392, + "pass@2": 0.728438190703136, + "pass@4": 0.7885472805482644, + "pass@8": 0.8277271874588323, + "pass@16": 0.8511195248343622, + "pass@32": 0.8669647904579663, + "pass@64": 0.8803333166221328, + "pass@128": 0.8918128654970761 + }, + "15": { + "pass@1": 0.5269412878787879, + "pass@2": 0.6264215730136006, + "pass@4": 0.6986491589119537, + "pass@8": 0.7511878067235214, + "pass@16": 0.7906412093161832, + "pass@32": 0.8247690661576631, + "pass@64": 0.856615497975502, + "pass@128": 0.8848484848484849 + }, + "16": { + "pass@1": 0.32289440883190884, + "pass@2": 0.42797890447988907, + "pass@4": 0.5235572764942842, + "pass@8": 0.6020844538967673, + "pass@16": 0.6653369741453462, + "pass@32": 0.7188300176879009, + "pass@64": 0.7655181009036083, + "pass@128": 0.8034188034188035 + }, + "17": { + "pass@1": 0.3027950310559006, + "pass@2": 0.4041748942387634, + "pass@4": 0.498130968101037, + "pass@8": 0.5755826570354862, + "pass@16": 0.6377447403306611, + "pass@32": 0.6878838678929854, + "pass@64": 0.7274527788066238, + "pass@128": 0.7608695652173914 + }, + "18": { + "pass@1": 0.2521599264705882, + "pass@2": 0.3417890950671606, + "pass@4": 0.4293763864994814, + "pass@8": 0.5076022232730653, + "pass@16": 0.5732797195419035, + "pass@32": 0.6261815943845891, + "pass@64": 0.6687111040670097, + "pass@128": 0.7058823529411765 + }, + "19": { + "pass@1": 0.17959449404761904, + "pass@2": 0.2535843908183353, + "pass@4": 0.3336726154208402, + "pass@8": 0.4090762902920383, + "pass@16": 0.4727559068833661, + "pass@32": 0.5258139147075629, + "pass@64": 0.574969613587171, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.1503555389221557, + "pass@2": 0.22317950221604052, + "pass@4": 0.30321984265439883, + "pass@8": 0.3782157543112791, + "pass@16": 0.4411765096518501, + "pass@32": 0.4920488219301042, + "pass@64": 0.5357323720323429, + "pass@128": 0.5748502994011976 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 200156, + "answer_accuracy": 0.4779091534229829, + "avg_response_len": 359.28539263065403, + "resp_tokens_sum": 150474471, + "pass_at_k": { + "pass@1": 0.4779091534229829, + "pass@2": 0.5528725974265066, + "pass@4": 0.6171964197763032, + "pass@8": 0.6708884621950805, + "pass@16": 0.7146571276286162, + "pass@32": 0.7504016090636411, + "pass@64": 0.7805394068968754, + "pass@128": 0.8059290953545232 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9483878968253968, + "pass@2": 0.9712664823147109, + "pass@4": 0.9824181054749105, + "pass@8": 0.9882313135423663, + "pass@16": 0.9915123252403345, + "pass@32": 0.993682504433677, + "pass@64": 0.9951896239672378, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.8620269495412844, + "pass@2": 0.9103235841219393, + "pass@4": 0.9362392352905429, + "pass@8": 0.9526972140428558, + "pass@16": 0.9638965581267493, + "pass@32": 0.9710619309818433, + "pass@64": 0.9754546286852732, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6687162422839507, + "pass@2": 0.7427760766015359, + "pass@4": 0.7947847190397491, + "pass@8": 0.8326836269325879, + "pass@16": 0.8608774264261173, + "pass@32": 0.8812346351825862, + "pass@64": 0.897989758263987, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.6195752967359051, + "pass@2": 0.7071631346526789, + "pass@4": 0.7685654877339142, + "pass@8": 0.8150053180091682, + "pass@16": 0.8516327834780938, + "pass@32": 0.8813865048924064, + "pass@64": 0.9077576228327011, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.5160590277777778, + "pass@2": 0.6018286885146302, + "pass@4": 0.668300905095196, + "pass@8": 0.720608796057457, + "pass@16": 0.7627357601757521, + "pass@32": 0.7963453889710991, + "pass@64": 0.8214735977682994, + "pass@128": 0.8395061728395061 + }, + "16": { + "pass@1": 0.32913901179941, + "pass@2": 0.43340657155134343, + "pass@4": 0.5325866487374916, + "pass@8": 0.6193580310655307, + "pass@16": 0.6897998814914108, + "pass@32": 0.7441026163170258, + "pass@64": 0.7872053937870015, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.25230609939759036, + "pass@2": 0.3407472162271132, + "pass@4": 0.4249335446397511, + "pass@8": 0.4971805284165239, + "pass@16": 0.5591699239740667, + "pass@32": 0.6153184013677834, + "pass@64": 0.6683035385369585, + "pass@128": 0.7078313253012049 + }, + "18": { + "pass@1": 0.25740927419354837, + "pass@2": 0.34158663004826023, + "pass@4": 0.4214958056654211, + "pass@8": 0.4880174444049805, + "pass@16": 0.5401664641920964, + "pass@32": 0.5837977213272798, + "pass@64": 0.622184241601249, + "pass@128": 0.6548387096774193 + }, + "19": { + "pass@1": 0.18820808157099697, + "pass@2": 0.26322384149677674, + "pass@4": 0.3385367960991281, + "pass@8": 0.4080986241808218, + "pass@16": 0.4682389995974202, + "pass@32": 0.5162918392839718, + "pass@64": 0.5523291041867587, + "pass@128": 0.5800604229607251 + }, + "20": { + "pass@1": 0.1591356981981982, + "pass@2": 0.23524508760729218, + "pass@4": 0.31971246471443326, + "pass@8": 0.3994183122873029, + "pass@16": 0.46805212252335276, + "pass@32": 0.5279196875495892, + "pass@64": 0.582608547499917, + "pass@128": 0.6366366366366366 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7207709703947368, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7207709703947368, + "pass@2": 0.7618134453999001, + "pass@4": 0.7965796681861933, + "pass@8": 0.8252551638995197, + "pass@16": 0.8483446787843888, + "pass@32": 0.8672016506282205, + "pass@64": 0.8834573890914952, + "pass@128": 0.8978421052631579 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.940046875, + "pass@2": 0.9539087106299211, + "pass@4": 0.9620897415635549, + "pass@8": 0.9673531227365778, + "pass@16": 0.9712878106066547, + "pass@32": 0.9753845854705726, + "pass@64": 0.9801630592581841, + "pass@128": 0.985 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999765625, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9931875, + "pass@2": 0.9951967273622047, + "pass@4": 0.9966725070303712, + "pass@8": 0.9976910659892855, + "pass@16": 0.9982521531431331, + "pass@32": 0.9985788431070177, + "pass@64": 0.9988779523587306, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.998921875, + "pass@2": 0.9995840305118111, + "pass@4": 0.9998836683539557, + "pass@8": 0.9999881979994633, + "pass@16": 0.999999908401855, + "pass@32": 0.999999999998944, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9977109375, + "pass@2": 0.9981526820866141, + "pass@4": 0.998311847487814, + "pass@8": 0.9985508396667573, + "pass@16": 0.998875518129056, + "pass@32": 0.999200274993652, + "pass@64": 0.9994993322645678, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.99365625, + "pass@2": 0.9964869586614173, + "pass@4": 0.9981354686914137, + "pass@8": 0.999296527211222, + "pass@16": 0.9998734507384869, + "pass@32": 0.9999957517907246, + "pass@64": 0.9999999990455016, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.989875, + "pass@2": 0.9945944881889768, + "pass@4": 0.9966445166854141, + "pass@8": 0.9980554347076996, + "pass@16": 0.9991709323109028, + "pass@32": 0.9997931777253324, + "pass@64": 0.9999860346592059, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9383125, + "pass@2": 0.9590136564960636, + "pass@4": 0.9696847320959882, + "pass@8": 0.9761633096527691, + "pass@16": 0.9803984161140732, + "pass@32": 0.983386489904395, + "pass@64": 0.9857262741232489, + "pass@128": 0.988 + }, + "12": { + "pass@1": 0.8476796875, + "pass@2": 0.8924173228346463, + "pass@4": 0.9171074446944133, + "pass@8": 0.9333752530484688, + "pass@16": 0.9455672808407929, + "pass@32": 0.9549591513160477, + "pass@64": 0.9623154103917709, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.6962109375, + "pass@2": 0.7726345964566936, + "pass@4": 0.8222474234158222, + "pass@8": 0.856545671556595, + "pass@16": 0.8809665507559866, + "pass@32": 0.8985424480805302, + "pass@64": 0.913224331651731, + "pass@128": 0.927 + }, + "14": { + "pass@1": 0.62440625, + "pass@2": 0.7120086122047234, + "pass@4": 0.7731541090176218, + "pass@8": 0.8170347055909208, + "pass@16": 0.8494115376758603, + "pass@32": 0.8745452275237636, + "pass@64": 0.8956640884547219, + "pass@128": 0.913 + }, + "15": { + "pass@1": 0.51234375, + "pass@2": 0.6071980807086611, + "pass@4": 0.6804598734533189, + "pass@8": 0.7365722276708908, + "pass@16": 0.7799610394318404, + "pass@32": 0.8152456205013187, + "pass@64": 0.8447109414185273, + "pass@128": 0.869 + }, + "16": { + "pass@1": 0.3223359375, + "pass@2": 0.4263375984251966, + "pass@4": 0.5234660494000746, + "pass@8": 0.6059840191843239, + "pass@16": 0.6727891764989107, + "pass@32": 0.7272819482386783, + "pass@64": 0.7738442731749547, + "pass@128": 0.815 + }, + "17": { + "pass@1": 0.271390625, + "pass@2": 0.3647472933070873, + "pass@4": 0.4529848034308209, + "pass@8": 0.5279137253955453, + "pass@16": 0.5893852204022109, + "pass@32": 0.6408462552365666, + "pass@64": 0.6855945715322403, + "pass@128": 0.721 + }, + "18": { + "pass@1": 0.236875, + "pass@2": 0.3236471456692916, + "pass@4": 0.4089280459317586, + "pass@8": 0.48372288955778087, + "pass@16": 0.5449902991093204, + "pass@32": 0.5944102663571567, + "pass@64": 0.635087373223452, + "pass@128": 0.67 + }, + "19": { + "pass@1": 0.1762421875, + "pass@2": 0.2509820374015752, + "pass@4": 0.3285974938132734, + "pass@8": 0.3995420743766352, + "pass@16": 0.4599080269284726, + "pass@32": 0.5104871992005041, + "pass@64": 0.5556715591845206, + "pass@128": 0.6 + }, + "20": { + "pass@1": 0.1554921875, + "pass@2": 0.22754576771653515, + "pass@4": 0.30664597047244085, + "pass@8": 0.38205904974572824, + "pass@16": 0.4477115758156405, + "pass@32": 0.5041741224907627, + "pass@64": 0.5553251919970433, + "pass@128": 0.605 + } + }, + "per_op_accuracy": { + "10": 0.940046875, + "2": 1.0, + "3": 0.9999765625, + "4": 0.999984375, + "5": 0.9931875, + "6": 0.998921875, + "7": 0.9977109375, + "8": 0.99365625, + "9": 0.989875, + "11": 0.9383125, + "12": 0.8476796875, + "13": 0.6962109375, + "14": 0.62440625, + "15": 0.51234375, + "16": 0.3223359375, + "17": 0.271390625, + "18": 0.236875, + "19": 0.1762421875, + "20": 0.1554921875 + }, + "per_op_avg_response_len": { + "10": 277.4631171875, + "2": 114.9146171875, + "3": 143.5135, + "4": 158.5818984375, + "5": 171.3797890625, + "6": 196.8831171875, + "7": 221.55025, + "8": 240.8111328125, + "9": 256.40328125, + "11": 295.2929453125, + "12": 304.2809140625, + "13": 319.5819140625, + "14": 339.468, + "15": 348.131703125, + "16": 348.175515625, + "17": 358.687390625, + "18": 359.1170078125, + "19": 351.3666875, + "20": 359.8926328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.4631171875, + "2": 114.9146171875, + "3": 143.5135, + "4": 158.5818984375, + "5": 171.3797890625, + "6": 196.8831171875, + "7": 221.55025, + "8": 240.8111328125, + "9": 256.40328125, + "11": 295.2929453125, + "12": 304.2809140625, + "13": 319.5819140625, + "14": 339.468, + "15": 348.131703125, + "16": 348.175515625, + "17": 358.687390625, + "18": 359.1170078125, + "19": 351.3666875, + "20": 359.8926328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 577160, + "answer_accuracy": 0.7127825640214985, + "avg_response_len": 258.88227775252926, + "resp_tokens_sum": 209624229, + "pass_at_k": { + "pass@1": 0.7127825640214985, + "pass@2": 0.7533919040530093, + "pass@4": 0.7882775814598341, + "pass@8": 0.8174817268476742, + "pass@16": 0.8412814213608717, + "pass@32": 0.8608748440965824, + "pass@64": 0.8782130919282408, + "pass@128": 0.8942459690167562 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391362463126843, + "pass@2": 0.9502618865119741, + "pass@4": 0.9565611159777595, + "pass@8": 0.960697067871047, + "pass@16": 0.9641969779408918, + "pass@32": 0.9691040751361097, + "pass@64": 0.9768076556801932, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9950892857142857, + "pass@2": 0.9976178758905138, + "pass@4": 0.9989978952035756, + "pass@8": 0.9997296666578273, + "pass@16": 0.9999812306164019, + "pass@32": 0.9999999574634271, + "pass@64": 0.9999999999999986, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981989146706587, + "pass@2": 0.9989778071809138, + "pass@4": 0.9996588796535164, + "pass@8": 0.9999646694808768, + "pass@16": 0.9999997257540574, + "pass@32": 0.9999999999968382, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969720784023669, + "pass@2": 0.9974847703023807, + "pass@4": 0.9978717981997812, + "pass@8": 0.9984862120344895, + "pass@16": 0.9992618879557866, + "pass@32": 0.9998528845966035, + "pass@64": 0.999998024451384, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927884615384616, + "pass@2": 0.9952408104936402, + "pass@4": 0.9968641546008672, + "pass@8": 0.9985039928739285, + "pass@16": 0.9996691654274897, + "pass@32": 0.9999870535678677, + "pass@64": 0.9999999969426491, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916593309859155, + "pass@2": 0.9964192636131751, + "pass@4": 0.9983151256621091, + "pass@8": 0.9993721194559138, + "pass@16": 0.9998884598509388, + "pass@32": 0.9999974897141034, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9259693696883853, + "pass@2": 0.947184635073944, + "pass@4": 0.9595871744785438, + "pass@8": 0.9678898043700129, + "pass@16": 0.9737432000120178, + "pass@32": 0.9782974664456189, + "pass@64": 0.9823106914719336, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8313691737288136, + "pass@2": 0.8758886766537666, + "pass@4": 0.9012220606958026, + "pass@8": 0.918590237623188, + "pass@16": 0.9326627176304815, + "pass@32": 0.9442766994564674, + "pass@64": 0.9530374167763507, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.7017535266457681, + "pass@2": 0.7727542702342459, + "pass@4": 0.8188653515685144, + "pass@8": 0.8522624263765721, + "pass@16": 0.8756735247780262, + "pass@32": 0.8917525794424754, + "pass@64": 0.9062939546723526, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.6107135903426791, + "pass@2": 0.6995911981504643, + "pass@4": 0.7615712451831373, + "pass@8": 0.8077732564202801, + "pass@16": 0.8452598509358014, + "pass@32": 0.8754393054155718, + "pass@64": 0.8993014806085093, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.4949421965317919, + "pass@2": 0.5938915217787082, + "pass@4": 0.6744975657847677, + "pass@8": 0.737580929276051, + "pass@16": 0.785904780521841, + "pass@32": 0.8238610481001554, + "pass@64": 0.8551167092765369, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.3142641129032258, + "pass@2": 0.4167489522479044, + "pass@4": 0.5133889400921661, + "pass@8": 0.595774204307558, + "pass@16": 0.6626249637042165, + "pass@32": 0.7184575003185609, + "pass@64": 0.7686606169806237, + "pass@128": 0.8161290322580645 + }, + "17": { + "pass@1": 0.2604768786127168, + "pass@2": 0.3510835305630146, + "pass@4": 0.4378864013927445, + "pass@8": 0.5130409664619432, + "pass@16": 0.5733728302206639, + "pass@32": 0.6215663020835889, + "pass@64": 0.663231219544036, + "pass@128": 0.6965317919075145 + }, + "18": { + "pass@1": 0.20383928571428572, + "pass@2": 0.2901342800899887, + "pass@4": 0.37793249933044054, + "pass@8": 0.4567220739411263, + "pass@16": 0.5217816873300685, + "pass@32": 0.5729463732998272, + "pass@64": 0.6138528084122289, + "pass@128": 0.6485714285714286 + }, + "19": { + "pass@1": 0.16096565315315314, + "pass@2": 0.23618795961315653, + "pass@4": 0.31359704361279167, + "pass@8": 0.3814168055094861, + "pass@16": 0.4386634635102604, + "pass@32": 0.4892526277950988, + "pass@64": 0.5395220286288696, + "pass@128": 0.5945945945945946 + }, + "20": { + "pass@1": 0.15700075075075076, + "pass@2": 0.22422582523468362, + "pass@4": 0.2970158927206171, + "pass@8": 0.3685546240663337, + "pass@16": 0.4339257200950346, + "pass@32": 0.4925902703065918, + "pass@64": 0.547693493756059, + "pass@128": 0.6036036036036037 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 596632, + "answer_accuracy": 0.7266075604053001, + "avg_response_len": 269.16072559431024, + "resp_tokens_sum": 221013255, + "pass_at_k": { + "pass@1": 0.7266075604053001, + "pass@2": 0.7684823801253143, + "pass@4": 0.8034604694337151, + "pass@8": 0.8317035862100778, + "pass@16": 0.8539699234765759, + "pass@32": 0.8720368246121353, + "pass@64": 0.887513624518416, + "pass@128": 0.9011691348402182 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9294478527607362, + "pass@2": 0.9466284841312018, + "pass@4": 0.9571646012614984, + "pass@8": 0.9635862044908389, + "pass@16": 0.9679916416901503, + "pass@32": 0.9721634565198133, + "pass@64": 0.9758088391108484, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968976449275362, + "pass@2": 0.9970971699189775, + "pass@4": 0.9971014479168364, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9988359598853869, + "pass@2": 0.9997881320082124, + "pass@4": 0.9999931305148461, + "pass@8": 0.999999995394987, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967711343930635, + "pass@2": 0.997127250125165, + "pass@4": 0.9971999508009476, + "pass@8": 0.997290462425144, + "pass@16": 0.997471098265896, + "pass@32": 0.9978323699421965, + "pass@64": 0.9985549132947977, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954445422535211, + "pass@2": 0.9973179133858266, + "pass@4": 0.9986211822113786, + "pass@8": 0.9995469673676058, + "pass@16": 0.9999426641178323, + "pass@32": 0.9999994185630952, + "pass@64": 0.9999999999982961, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9878134356287425, + "pass@2": 0.9916725358338442, + "pass@4": 0.9937421916571807, + "pass@8": 0.9957880813290706, + "pass@16": 0.9978980094262961, + "pass@32": 0.9993986125945581, + "pass@64": 0.9999581943983665, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9418768825301205, + "pass@2": 0.9599654770657432, + "pass@4": 0.9683396273507975, + "pass@8": 0.9735100751334598, + "pass@16": 0.9769297712624254, + "pass@32": 0.9790286004592866, + "pass@64": 0.9803791219396885, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8510726880877743, + "pass@2": 0.8924041357095258, + "pass@4": 0.9151241544454282, + "pass@8": 0.9299759245700525, + "pass@16": 0.9410987595365371, + "pass@32": 0.9503070478915217, + "pass@64": 0.9591426372189908, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7162114845938375, + "pass@2": 0.7996261496724675, + "pass@4": 0.8501936338839996, + "pass@8": 0.8820293064882619, + "pass@16": 0.9039283198592027, + "pass@32": 0.9203175166364791, + "pass@64": 0.9332433569571942, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6420184576023392, + "pass@2": 0.728438190703136, + "pass@4": 0.7885472805482644, + "pass@8": 0.8277271874588323, + "pass@16": 0.8511195248343622, + "pass@32": 0.8669647904579663, + "pass@64": 0.8803333166221328, + "pass@128": 0.8918128654970761 + }, + "15": { + "pass@1": 0.5269412878787879, + "pass@2": 0.6264215730136006, + "pass@4": 0.6986491589119537, + "pass@8": 0.7511878067235214, + "pass@16": 0.7906412093161832, + "pass@32": 0.8247690661576631, + "pass@64": 0.856615497975502, + "pass@128": 0.8848484848484849 + }, + "16": { + "pass@1": 0.32289440883190884, + "pass@2": 0.42797890447988907, + "pass@4": 0.5235572764942842, + "pass@8": 0.6020844538967673, + "pass@16": 0.6653369741453462, + "pass@32": 0.7188300176879009, + "pass@64": 0.7655181009036083, + "pass@128": 0.8034188034188035 + }, + "17": { + "pass@1": 0.3027950310559006, + "pass@2": 0.4041748942387634, + "pass@4": 0.498130968101037, + "pass@8": 0.5755826570354862, + "pass@16": 0.6377447403306611, + "pass@32": 0.6878838678929854, + "pass@64": 0.7274527788066238, + "pass@128": 0.7608695652173914 + }, + "18": { + "pass@1": 0.2521599264705882, + "pass@2": 0.3417890950671606, + "pass@4": 0.4293763864994814, + "pass@8": 0.5076022232730653, + "pass@16": 0.5732797195419035, + "pass@32": 0.6261815943845891, + "pass@64": 0.6687111040670097, + "pass@128": 0.7058823529411765 + }, + "19": { + "pass@1": 0.17959449404761904, + "pass@2": 0.2535843908183353, + "pass@4": 0.3336726154208402, + "pass@8": 0.4090762902920383, + "pass@16": 0.4727559068833661, + "pass@32": 0.5258139147075629, + "pass@64": 0.574969613587171, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.1503555389221557, + "pass@2": 0.22317950221604052, + "pass@4": 0.30321984265439883, + "pass@8": 0.3782157543112791, + "pass@16": 0.4411765096518501, + "pass@32": 0.4920488219301042, + "pass@64": 0.5357323720323429, + "pass@128": 0.5748502994011976 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 579123, + "answer_accuracy": 0.7228628275283592, + "avg_response_len": 287.76802529357724, + "resp_tokens_sum": 230545929, + "pass_at_k": { + "pass@1": 0.7228628275283592, + "pass@2": 0.7634899838720332, + "pass@4": 0.7979183262191598, + "pass@8": 0.826502668879155, + "pass@16": 0.8497180965445399, + "pass@32": 0.8686404966119444, + "pass@64": 0.8846004907996181, + "pass@128": 0.8980667838312829 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9512826492537313, + "pass@2": 0.9646837172405689, + "pass@4": 0.972477203689091, + "pass@8": 0.9777543703411323, + "pass@16": 0.9816709250558314, + "pass@32": 0.9848746781312615, + "pass@64": 0.98779576845511, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99995444606414, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9876608455882353, + "pass@2": 0.9910252142195461, + "pass@4": 0.9940828544409888, + "pass@8": 0.9964006499766762, + "pass@16": 0.9978178397028429, + "pass@32": 0.998761342664818, + "pass@64": 0.9996410363492089, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997781940063092, + "pass@2": 0.9999980594401252, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995302610759493, + "pass@2": 0.9999898771553873, + "pass@4": 0.9999999896175953, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9925628753753754, + "pass@2": 0.9967686682745736, + "pass@4": 0.9988088071198306, + "pass@8": 0.9997720961533224, + "pass@16": 0.9999910671569961, + "pass@32": 0.9999999924554084, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9900522508038585, + "pass@2": 0.995649590475226, + "pass@4": 0.9978545178838177, + "pass@8": 0.9989875020480404, + "pass@16": 0.9997189514965811, + "pass@32": 0.9999837050490139, + "pass@64": 0.9999999927877713, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9483878968253968, + "pass@2": 0.9712664823147109, + "pass@4": 0.9824181054749105, + "pass@8": 0.9882313135423663, + "pass@16": 0.9915123252403345, + "pass@32": 0.993682504433677, + "pass@64": 0.9951896239672378, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.8620269495412844, + "pass@2": 0.9103235841219393, + "pass@4": 0.9362392352905429, + "pass@8": 0.9526972140428558, + "pass@16": 0.9638965581267493, + "pass@32": 0.9710619309818433, + "pass@64": 0.9754546286852732, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6687162422839507, + "pass@2": 0.7427760766015359, + "pass@4": 0.7947847190397491, + "pass@8": 0.8326836269325879, + "pass@16": 0.8608774264261173, + "pass@32": 0.8812346351825862, + "pass@64": 0.897989758263987, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.6195752967359051, + "pass@2": 0.7071631346526789, + "pass@4": 0.7685654877339142, + "pass@8": 0.8150053180091682, + "pass@16": 0.8516327834780938, + "pass@32": 0.8813865048924064, + "pass@64": 0.9077576228327011, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.5160590277777778, + "pass@2": 0.6018286885146302, + "pass@4": 0.668300905095196, + "pass@8": 0.720608796057457, + "pass@16": 0.7627357601757521, + "pass@32": 0.7963453889710991, + "pass@64": 0.8214735977682994, + "pass@128": 0.8395061728395061 + }, + "16": { + "pass@1": 0.32913901179941, + "pass@2": 0.43340657155134343, + "pass@4": 0.5325866487374916, + "pass@8": 0.6193580310655307, + "pass@16": 0.6897998814914108, + "pass@32": 0.7441026163170258, + "pass@64": 0.7872053937870015, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.25230609939759036, + "pass@2": 0.3407472162271132, + "pass@4": 0.4249335446397511, + "pass@8": 0.4971805284165239, + "pass@16": 0.5591699239740667, + "pass@32": 0.6153184013677834, + "pass@64": 0.6683035385369585, + "pass@128": 0.7078313253012049 + }, + "18": { + "pass@1": 0.25740927419354837, + "pass@2": 0.34158663004826023, + "pass@4": 0.4214958056654211, + "pass@8": 0.4880174444049805, + "pass@16": 0.5401664641920964, + "pass@32": 0.5837977213272798, + "pass@64": 0.622184241601249, + "pass@128": 0.6548387096774193 + }, + "19": { + "pass@1": 0.18820808157099697, + "pass@2": 0.26322384149677674, + "pass@4": 0.3385367960991281, + "pass@8": 0.4080986241808218, + "pass@16": 0.4682389995974202, + "pass@32": 0.5162918392839718, + "pass@64": 0.5523291041867587, + "pass@128": 0.5800604229607251 + }, + "20": { + "pass@1": 0.1591356981981982, + "pass@2": 0.23524508760729218, + "pass@4": 0.31971246471443326, + "pass@8": 0.3994183122873029, + "pass@16": 0.46805212252335276, + "pass@32": 0.5279196875495892, + "pass@64": 0.582608547499917, + "pass@128": 0.6366366366366366 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/summary.csv b/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..c4a783abddd8992c06e5f3390a786aa99c688948 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-42828,0.9903732638888889,0.47812890625,0.7207709703947368,0.0,0.0,0.0,197.94452256944444,338.39947109375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/summary.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..24e643e7954248418a801b0dd1b3f7e3f90b5ac5 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block2/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-42828", + "id_acc": 0.9903732638888889, + "ood_acc": 0.47812890625, + "total_acc": 0.7207709703947368, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.94452256944444, + "ood_avg_resp_len": 338.39947109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 577160, + "answer_accuracy": 0.7127825640214985, + "avg_response_len": 258.88227775252926, + "resp_tokens_sum": 209624229, + "pass_at_k": { + "pass@1": 0.7127825640214985, + "pass@2": 0.7533919040530093, + "pass@4": 0.7882775814598341, + "pass@8": 0.8174817268476742, + "pass@16": 0.8412814213608717, + "pass@32": 0.8608748440965824, + "pass@64": 0.8782130919282408, + "pass@128": 0.8942459690167562 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9391362463126843, + "pass@2": 0.9502618865119741, + "pass@4": 0.9565611159777595, + "pass@8": 0.960697067871047, + "pass@16": 0.9641969779408918, + "pass@32": 0.9691040751361097, + "pass@64": 0.9768076556801932, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 0.9999996155265748, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9950892857142857, + "pass@2": 0.9976178758905138, + "pass@4": 0.9989978952035756, + "pass@8": 0.9997296666578273, + "pass@16": 0.9999812306164019, + "pass@32": 0.9999999574634271, + "pass@64": 0.9999999999999986, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9981989146706587, + "pass@2": 0.9989778071809138, + "pass@4": 0.9996588796535164, + "pass@8": 0.9999646694808768, + "pass@16": 0.9999997257540574, + "pass@32": 0.9999999999968382, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9969720784023669, + "pass@2": 0.9974847703023807, + "pass@4": 0.9978717981997812, + "pass@8": 0.9984862120344895, + "pass@16": 0.9992618879557866, + "pass@32": 0.9998528845966035, + "pass@64": 0.999998024451384, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9927884615384616, + "pass@2": 0.9952408104936402, + "pass@4": 0.9968641546008672, + "pass@8": 0.9985039928739285, + "pass@16": 0.9996691654274897, + "pass@32": 0.9999870535678677, + "pass@64": 0.9999999969426491, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9916593309859155, + "pass@2": 0.9964192636131751, + "pass@4": 0.9983151256621091, + "pass@8": 0.9993721194559138, + "pass@16": 0.9998884598509388, + "pass@32": 0.9999974897141034, + "pass@64": 0.9999999999243792, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9259693696883853, + "pass@2": 0.947184635073944, + "pass@4": 0.9595871744785438, + "pass@8": 0.9678898043700129, + "pass@16": 0.9737432000120178, + "pass@32": 0.9782974664456189, + "pass@64": 0.9823106914719336, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.8313691737288136, + "pass@2": 0.8758886766537666, + "pass@4": 0.9012220606958026, + "pass@8": 0.918590237623188, + "pass@16": 0.9326627176304815, + "pass@32": 0.9442766994564674, + "pass@64": 0.9530374167763507, + "pass@128": 0.9576271186440678 + }, + "13": { + "pass@1": 0.7017535266457681, + "pass@2": 0.7727542702342459, + "pass@4": 0.8188653515685144, + "pass@8": 0.8522624263765721, + "pass@16": 0.8756735247780262, + "pass@32": 0.8917525794424754, + "pass@64": 0.9062939546723526, + "pass@128": 0.9216300940438872 + }, + "14": { + "pass@1": 0.6107135903426791, + "pass@2": 0.6995911981504643, + "pass@4": 0.7615712451831373, + "pass@8": 0.8077732564202801, + "pass@16": 0.8452598509358014, + "pass@32": 0.8754393054155718, + "pass@64": 0.8993014806085093, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.4949421965317919, + "pass@2": 0.5938915217787082, + "pass@4": 0.6744975657847677, + "pass@8": 0.737580929276051, + "pass@16": 0.785904780521841, + "pass@32": 0.8238610481001554, + "pass@64": 0.8551167092765369, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.3142641129032258, + "pass@2": 0.4167489522479044, + "pass@4": 0.5133889400921661, + "pass@8": 0.595774204307558, + "pass@16": 0.6626249637042165, + "pass@32": 0.7184575003185609, + "pass@64": 0.7686606169806237, + "pass@128": 0.8161290322580645 + }, + "17": { + "pass@1": 0.2604768786127168, + "pass@2": 0.3510835305630146, + "pass@4": 0.4378864013927445, + "pass@8": 0.5130409664619432, + "pass@16": 0.5733728302206639, + "pass@32": 0.6215663020835889, + "pass@64": 0.663231219544036, + "pass@128": 0.6965317919075145 + }, + "18": { + "pass@1": 0.20383928571428572, + "pass@2": 0.2901342800899887, + "pass@4": 0.37793249933044054, + "pass@8": 0.4567220739411263, + "pass@16": 0.5217816873300685, + "pass@32": 0.5729463732998272, + "pass@64": 0.6138528084122289, + "pass@128": 0.6485714285714286 + }, + "19": { + "pass@1": 0.16096565315315314, + "pass@2": 0.23618795961315653, + "pass@4": 0.31359704361279167, + "pass@8": 0.3814168055094861, + "pass@16": 0.4386634635102604, + "pass@32": 0.4892526277950988, + "pass@64": 0.5395220286288696, + "pass@128": 0.5945945945945946 + }, + "20": { + "pass@1": 0.15700075075075076, + "pass@2": 0.22422582523468362, + "pass@4": 0.2970158927206171, + "pass@8": 0.3685546240663337, + "pass@16": 0.4339257200950346, + "pass@32": 0.4925902703065918, + "pass@64": 0.547693493756059, + "pass@128": 0.6036036036036037 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 596632, + "answer_accuracy": 0.7266075604053001, + "avg_response_len": 269.16072559431024, + "resp_tokens_sum": 221013255, + "pass_at_k": { + "pass@1": 0.7266075604053001, + "pass@2": 0.7684823801253143, + "pass@4": 0.8034604694337151, + "pass@8": 0.8317035862100778, + "pass@16": 0.8539699234765759, + "pass@32": 0.8720368246121353, + "pass@64": 0.887513624518416, + "pass@128": 0.9011691348402182 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9294478527607362, + "pass@2": 0.9466284841312018, + "pass@4": 0.9571646012614984, + "pass@8": 0.9635862044908389, + "pass@16": 0.9679916416901503, + "pass@32": 0.9721634565198133, + "pass@64": 0.9758088391108484, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9968976449275362, + "pass@2": 0.9970971699189775, + "pass@4": 0.9971014479168364, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9988359598853869, + "pass@2": 0.9997881320082124, + "pass@4": 0.9999931305148461, + "pass@8": 0.999999995394987, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967711343930635, + "pass@2": 0.997127250125165, + "pass@4": 0.9971999508009476, + "pass@8": 0.997290462425144, + "pass@16": 0.997471098265896, + "pass@32": 0.9978323699421965, + "pass@64": 0.9985549132947977, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9954445422535211, + "pass@2": 0.9973179133858266, + "pass@4": 0.9986211822113786, + "pass@8": 0.9995469673676058, + "pass@16": 0.9999426641178323, + "pass@32": 0.9999994185630952, + "pass@64": 0.9999999999982961, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9878134356287425, + "pass@2": 0.9916725358338442, + "pass@4": 0.9937421916571807, + "pass@8": 0.9957880813290706, + "pass@16": 0.9978980094262961, + "pass@32": 0.9993986125945581, + "pass@64": 0.9999581943983665, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9418768825301205, + "pass@2": 0.9599654770657432, + "pass@4": 0.9683396273507975, + "pass@8": 0.9735100751334598, + "pass@16": 0.9769297712624254, + "pass@32": 0.9790286004592866, + "pass@64": 0.9803791219396885, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8510726880877743, + "pass@2": 0.8924041357095258, + "pass@4": 0.9151241544454282, + "pass@8": 0.9299759245700525, + "pass@16": 0.9410987595365371, + "pass@32": 0.9503070478915217, + "pass@64": 0.9591426372189908, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7162114845938375, + "pass@2": 0.7996261496724675, + "pass@4": 0.8501936338839996, + "pass@8": 0.8820293064882619, + "pass@16": 0.9039283198592027, + "pass@32": 0.9203175166364791, + "pass@64": 0.9332433569571942, + "pass@128": 0.9439775910364145 + }, + "14": { + "pass@1": 0.6420184576023392, + "pass@2": 0.728438190703136, + "pass@4": 0.7885472805482644, + "pass@8": 0.8277271874588323, + "pass@16": 0.8511195248343622, + "pass@32": 0.8669647904579663, + "pass@64": 0.8803333166221328, + "pass@128": 0.8918128654970761 + }, + "15": { + "pass@1": 0.5269412878787879, + "pass@2": 0.6264215730136006, + "pass@4": 0.6986491589119537, + "pass@8": 0.7511878067235214, + "pass@16": 0.7906412093161832, + "pass@32": 0.8247690661576631, + "pass@64": 0.856615497975502, + "pass@128": 0.8848484848484849 + }, + "16": { + "pass@1": 0.32289440883190884, + "pass@2": 0.42797890447988907, + "pass@4": 0.5235572764942842, + "pass@8": 0.6020844538967673, + "pass@16": 0.6653369741453462, + "pass@32": 0.7188300176879009, + "pass@64": 0.7655181009036083, + "pass@128": 0.8034188034188035 + }, + "17": { + "pass@1": 0.3027950310559006, + "pass@2": 0.4041748942387634, + "pass@4": 0.498130968101037, + "pass@8": 0.5755826570354862, + "pass@16": 0.6377447403306611, + "pass@32": 0.6878838678929854, + "pass@64": 0.7274527788066238, + "pass@128": 0.7608695652173914 + }, + "18": { + "pass@1": 0.2521599264705882, + "pass@2": 0.3417890950671606, + "pass@4": 0.4293763864994814, + "pass@8": 0.5076022232730653, + "pass@16": 0.5732797195419035, + "pass@32": 0.6261815943845891, + "pass@64": 0.6687111040670097, + "pass@128": 0.7058823529411765 + }, + "19": { + "pass@1": 0.17959449404761904, + "pass@2": 0.2535843908183353, + "pass@4": 0.3336726154208402, + "pass@8": 0.4090762902920383, + "pass@16": 0.4727559068833661, + "pass@32": 0.5258139147075629, + "pass@64": 0.574969613587171, + "pass@128": 0.625 + }, + "20": { + "pass@1": 0.1503555389221557, + "pass@2": 0.22317950221604052, + "pass@4": 0.30321984265439883, + "pass@8": 0.3782157543112791, + "pass@16": 0.4411765096518501, + "pass@32": 0.4920488219301042, + "pass@64": 0.5357323720323429, + "pass@128": 0.5748502994011976 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 579123, + "answer_accuracy": 0.7228628275283592, + "avg_response_len": 287.76802529357724, + "resp_tokens_sum": 230545929, + "pass_at_k": { + "pass@1": 0.7228628275283592, + "pass@2": 0.7634899838720332, + "pass@4": 0.7979183262191598, + "pass@8": 0.826502668879155, + "pass@16": 0.8497180965445399, + "pass@32": 0.8686404966119444, + "pass@64": 0.8846004907996181, + "pass@128": 0.8980667838312829 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9512826492537313, + "pass@2": 0.9646837172405689, + "pass@4": 0.972477203689091, + "pass@8": 0.9777543703411323, + "pass@16": 0.9816709250558314, + "pass@32": 0.9848746781312615, + "pass@64": 0.98779576845511, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.99995444606414, + "pass@2": 0.9999996413075917, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9876608455882353, + "pass@2": 0.9910252142195461, + "pass@4": 0.9940828544409888, + "pass@8": 0.9964006499766762, + "pass@16": 0.9978178397028429, + "pass@32": 0.998761342664818, + "pass@64": 0.9996410363492089, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9997781940063092, + "pass@2": 0.9999980594401252, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995302610759493, + "pass@2": 0.9999898771553873, + "pass@4": 0.9999999896175953, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9925628753753754, + "pass@2": 0.9967686682745736, + "pass@4": 0.9988088071198306, + "pass@8": 0.9997720961533224, + "pass@16": 0.9999910671569961, + "pass@32": 0.9999999924554084, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9900522508038585, + "pass@2": 0.995649590475226, + "pass@4": 0.9978545178838177, + "pass@8": 0.9989875020480404, + "pass@16": 0.9997189514965811, + "pass@32": 0.9999837050490139, + "pass@64": 0.9999999927877713, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9483878968253968, + "pass@2": 0.9712664823147109, + "pass@4": 0.9824181054749105, + "pass@8": 0.9882313135423663, + "pass@16": 0.9915123252403345, + "pass@32": 0.993682504433677, + "pass@64": 0.9951896239672378, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.8620269495412844, + "pass@2": 0.9103235841219393, + "pass@4": 0.9362392352905429, + "pass@8": 0.9526972140428558, + "pass@16": 0.9638965581267493, + "pass@32": 0.9710619309818433, + "pass@64": 0.9754546286852732, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6687162422839507, + "pass@2": 0.7427760766015359, + "pass@4": 0.7947847190397491, + "pass@8": 0.8326836269325879, + "pass@16": 0.8608774264261173, + "pass@32": 0.8812346351825862, + "pass@64": 0.897989758263987, + "pass@128": 0.9135802469135802 + }, + "14": { + "pass@1": 0.6195752967359051, + "pass@2": 0.7071631346526789, + "pass@4": 0.7685654877339142, + "pass@8": 0.8150053180091682, + "pass@16": 0.8516327834780938, + "pass@32": 0.8813865048924064, + "pass@64": 0.9077576228327011, + "pass@128": 0.9287833827893175 + }, + "15": { + "pass@1": 0.5160590277777778, + "pass@2": 0.6018286885146302, + "pass@4": 0.668300905095196, + "pass@8": 0.720608796057457, + "pass@16": 0.7627357601757521, + "pass@32": 0.7963453889710991, + "pass@64": 0.8214735977682994, + "pass@128": 0.8395061728395061 + }, + "16": { + "pass@1": 0.32913901179941, + "pass@2": 0.43340657155134343, + "pass@4": 0.5325866487374916, + "pass@8": 0.6193580310655307, + "pass@16": 0.6897998814914108, + "pass@32": 0.7441026163170258, + "pass@64": 0.7872053937870015, + "pass@128": 0.8259587020648967 + }, + "17": { + "pass@1": 0.25230609939759036, + "pass@2": 0.3407472162271132, + "pass@4": 0.4249335446397511, + "pass@8": 0.4971805284165239, + "pass@16": 0.5591699239740667, + "pass@32": 0.6153184013677834, + "pass@64": 0.6683035385369585, + "pass@128": 0.7078313253012049 + }, + "18": { + "pass@1": 0.25740927419354837, + "pass@2": 0.34158663004826023, + "pass@4": 0.4214958056654211, + "pass@8": 0.4880174444049805, + "pass@16": 0.5401664641920964, + "pass@32": 0.5837977213272798, + "pass@64": 0.622184241601249, + "pass@128": 0.6548387096774193 + }, + "19": { + "pass@1": 0.18820808157099697, + "pass@2": 0.26322384149677674, + "pass@4": 0.3385367960991281, + "pass@8": 0.4080986241808218, + "pass@16": 0.4682389995974202, + "pass@32": 0.5162918392839718, + "pass@64": 0.5523291041867587, + "pass@128": 0.5800604229607251 + }, + "20": { + "pass@1": 0.1591356981981982, + "pass@2": 0.23524508760729218, + "pass@4": 0.31971246471443326, + "pass@8": 0.3994183122873029, + "pass@16": 0.46805212252335276, + "pass@32": 0.5279196875495892, + "pass@64": 0.582608547499917, + "pass@128": 0.6366366366366366 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/checkpoint-42828_metrics.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/checkpoint-42828_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..3fc37cd567fa5ac6ef54c92c57dff943a0a6f179 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/checkpoint-42828_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-42828", + "id": { + "answer_accuracy": 0.4487638888888889, + "avg_response_len": 204.87644010416668, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4487638888888889, + "pass@2": 0.5513008803587074, + "pass@4": 0.6537630185601645, + "pass@8": 0.746476880135296, + "pass@16": 0.8224654638528102, + "pass@32": 0.8801672376660384, + "pass@64": 0.921617372955124, + "pass@128": 0.9501111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1992890625, + "pass@2": 0.3125543799212598, + "pass@4": 0.44401627090363704, + "pass@8": 0.5753215603405694, + "pass@16": 0.6912148385299013, + "pass@32": 0.7852131040043056, + "pass@64": 0.8587442727751192, + "pass@128": 0.918 + }, + "2": { + "pass@1": 0.8721953125, + "pass@2": 0.9042281003937006, + "pass@4": 0.9299732025684282, + "pass@8": 0.9501093154237494, + "pass@16": 0.9651901174725063, + "pass@32": 0.9757265157459862, + "pass@64": 0.9828679096028664, + "pass@128": 0.988 + }, + "3": { + "pass@1": 0.8510546875, + "pass@2": 0.9055986712598424, + "pass@4": 0.9355771586051743, + "pass@8": 0.9540556049457282, + "pass@16": 0.967113004694048, + "pass@32": 0.9766052540263035, + "pass@64": 0.983570469094579, + "pass@128": 0.99 + }, + "4": { + "pass@1": 0.5731328125, + "pass@2": 0.6732614419291336, + "pass@4": 0.7590569236970385, + "pass@8": 0.8279303949256342, + "pass@16": 0.8804485147259381, + "pass@32": 0.9167334784533838, + "pass@64": 0.9386469535016336, + "pass@128": 0.953 + }, + "5": { + "pass@1": 0.3436015625, + "pass@2": 0.44655191929133936, + "pass@4": 0.5452499143232092, + "pass@8": 0.6376423926636297, + "pass@16": 0.7235327055370553, + "pass@32": 0.7990005413771109, + "pass@64": 0.8569880485324983, + "pass@128": 0.896 + }, + "6": { + "pass@1": 0.431078125, + "pass@2": 0.5777931840551189, + "pass@4": 0.7124146303899507, + "pass@8": 0.816428467947096, + "pass@16": 0.8877456246882576, + "pass@32": 0.9339198246266077, + "pass@64": 0.9623973979117663, + "pass@128": 0.978 + }, + "7": { + "pass@1": 0.31109375, + "pass@2": 0.44415428149606295, + "pass@4": 0.583285361454818, + "pass@8": 0.7091024975808733, + "pass@16": 0.8077639294200822, + "pass@32": 0.878038327242287, + "pass@64": 0.9256090060941099, + "pass@128": 0.954 + }, + "8": { + "pass@1": 0.25571875, + "pass@2": 0.38235063976377903, + "pass@4": 0.5241426136107981, + "pass@8": 0.6603608218326873, + "pass@16": 0.7722822147287928, + "pass@32": 0.8555920506067689, + "pass@64": 0.9152873405211892, + "pass@128": 0.956 + }, + "9": { + "pass@1": 0.2017109375, + "pass@2": 0.31521530511811047, + "pass@4": 0.450151091488563, + "pass@8": 0.5873408655576899, + "pass@16": 0.7068982248785488, + "pass@32": 0.80067604291157, + "pass@64": 0.8704449585622345, + "pass@128": 0.918 + } + }, + "per_op_accuracy": { + "10": 0.1992890625, + "2": 0.8721953125, + "3": 0.8510546875, + "4": 0.5731328125, + "5": 0.3436015625, + "6": 0.431078125, + "7": 0.31109375, + "8": 0.25571875, + "9": 0.2017109375 + }, + "per_op_avg_response_len": { + "10": 277.849953125, + "2": 130.1738359375, + "3": 152.65065625, + "4": 167.6288359375, + "5": 175.914, + "6": 207.139859375, + "7": 229.352875, + "8": 246.1938203125, + "9": 256.984125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.849953125, + "2": 130.1738359375, + "3": 152.65065625, + "4": 167.6288359375, + "5": 175.914, + "6": 207.139859375, + "7": 229.352875, + "8": 246.1938203125, + "9": 256.984125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 165267, + "answer_accuracy": 0.4360514817629179, + "avg_response_len": 193.85048600557244, + "resp_tokens_sum": 73470885, + "pass_at_k": { + "pass@1": 0.4360514817629179, + "pass@2": 0.542099998470937, + "pass@4": 0.6488104161118705, + "pass@8": 0.7452283840235931, + "pass@16": 0.8232380799383711, + "pass@32": 0.880749456160684, + "pass@64": 0.9215806843305637, + "pass@128": 0.950016886187099 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.19429848820058998, + "pass@2": 0.304934107379277, + "pass@4": 0.434951352319898, + "pass@8": 0.5684199490328546, + "pass@16": 0.6891009886552397, + "pass@32": 0.7857931447451506, + "pass@64": 0.860108943525792, + "pass@128": 0.9233038348082596 + }, + "2": { + "pass@1": 0.831386292834891, + "pass@2": 0.8734914268893955, + "pass@4": 0.9105459002204167, + "pass@8": 0.9401008638276085, + "pass@16": 0.9611783814748307, + "pass@32": 0.9741949948603676, + "pass@64": 0.9817242745943495, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.8260225535168195, + "pass@2": 0.8906170988947479, + "pass@4": 0.9285593713056506, + "pass@8": 0.9526553375068405, + "pass@16": 0.9699066685395095, + "pass@32": 0.9816788551557367, + "pass@64": 0.988712453516909, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.5572509765625, + "pass@2": 0.6628467950295271, + "pass@4": 0.7544799485025304, + "pass@8": 0.8284263532654119, + "pass@16": 0.884775967340774, + "pass@32": 0.9236644996257792, + "pass@64": 0.948151095592842, + "pass@128": 0.965625 + }, + "5": { + "pass@1": 0.3347718253968254, + "pass@2": 0.4497898700162479, + "pass@4": 0.5641970530469402, + "pass@8": 0.6677945563217111, + "pass@16": 0.7584282589253484, + "pass@32": 0.8321889687382951, + "pass@64": 0.8851659220568333, + "pass@128": 0.9174603174603174 + }, + "6": { + "pass@1": 0.4252432634730539, + "pass@2": 0.5709287625536326, + "pass@4": 0.7067664802565847, + "pass@8": 0.8127587788249542, + "pass@16": 0.8845402446547445, + "pass@32": 0.9276708113313906, + "pass@64": 0.9525643996495025, + "pass@128": 0.9670658682634731 + }, + "7": { + "pass@1": 0.3097956730769231, + "pass@2": 0.4431132210082467, + "pass@4": 0.5825773815070447, + "pass@8": 0.7077649850344365, + "pass@16": 0.804085063866593, + "pass@32": 0.8732138745287983, + "pass@64": 0.9242260451555102, + "pass@128": 0.9526627218934911 + }, + "8": { + "pass@1": 0.27226061698717946, + "pass@2": 0.3969216888754288, + "pass@4": 0.5352359978440191, + "pass@8": 0.668389018888514, + "pass@16": 0.777516858096813, + "pass@32": 0.8581637138287497, + "pass@64": 0.916168827905763, + "pass@128": 0.9551282051282052 + }, + "9": { + "pass@1": 0.20517165492957745, + "pass@2": 0.3156790645447484, + "pass@4": 0.44685785738050343, + "pass@8": 0.5801721703370608, + "pass@16": 0.6942797587019922, + "pass@32": 0.7812453529979867, + "pass@64": 0.8455093152480154, + "pass@128": 0.8929577464788733 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 176547, + "answer_accuracy": 0.45192445527522934, + "avg_response_len": 200.86459698558323, + "resp_tokens_sum": 78468960, + "pass_at_k": { + "pass@1": 0.45192445527522934, + "pass@2": 0.5524496328727283, + "pass@4": 0.65324320523377, + "pass@8": 0.7449309526307615, + "pass@16": 0.8208160277054444, + "pass@32": 0.878958968600913, + "pass@64": 0.9203503768450284, + "pass@128": 0.9475753604193972 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1877396472392638, + "pass@2": 0.2979068040191296, + "pass@4": 0.42743402032567995, + "pass@8": 0.5592994362559849, + "pass@16": 0.6798039506691061, + "pass@32": 0.7821547762425294, + "pass@64": 0.8609090332078575, + "pass@128": 0.911042944785276 + }, + "2": { + "pass@1": 0.8750237462006079, + "pass@2": 0.9054504822535481, + "pass@4": 0.9290282496982711, + "pass@8": 0.9469597285175716, + "pass@16": 0.9616501693834948, + "pass@32": 0.9731913245513274, + "pass@64": 0.982534585697019, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.8613162878787879, + "pass@2": 0.9110038624433314, + "pass@4": 0.9384561071911464, + "pass@8": 0.9562286327789103, + "pass@16": 0.9687754714930751, + "pass@32": 0.9774223917946637, + "pass@64": 0.9834020912247246, + "pass@128": 0.9878787878787879 + }, + "4": { + "pass@1": 0.5816383136094675, + "pass@2": 0.6702755905511808, + "pass@4": 0.7480943742054436, + "pass@8": 0.8105270906947715, + "pass@16": 0.8574687882367842, + "pass@32": 0.8923947023004354, + "pass@64": 0.9173499844593105, + "pass@128": 0.9349112426035503 + }, + "5": { + "pass@1": 0.35183423913043477, + "pass@2": 0.44896653543307047, + "pass@4": 0.5367537307836514, + "pass@8": 0.6184585802569412, + "pass@16": 0.6973476991211023, + "pass@32": 0.7694325969464841, + "pass@64": 0.8289443827711489, + "pass@128": 0.8753623188405797 + }, + "6": { + "pass@1": 0.44206661891117477, + "pass@2": 0.5813061926088036, + "pass@4": 0.7101583094555867, + "pass@8": 0.8137002926251504, + "pass@16": 0.8888854739106302, + "pass@32": 0.9397605232654371, + "pass@64": 0.9715154997083257, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.31550307080924855, + "pass@2": 0.4467138500750988, + "pass@4": 0.5860489259651785, + "pass@8": 0.7147282946538364, + "pass@16": 0.8169303698336912, + "pass@32": 0.8862546718118719, + "pass@64": 0.9261973005402598, + "pass@128": 0.9450867052023122 + }, + "8": { + "pass@1": 0.25985915492957745, + "pass@2": 0.3925993955861151, + "pass@4": 0.5404817329171882, + "pass@8": 0.6789521589715024, + "pass@16": 0.78910166835959, + "pass@32": 0.8686986141098911, + "pass@64": 0.923139404679525, + "pass@128": 0.9605633802816902 + }, + "9": { + "pass@1": 0.21641092814371257, + "pass@2": 0.33585609293224555, + "pass@4": 0.47451746422789937, + "pass@8": 0.6121577136393171, + "pass@16": 0.730587365488769, + "pass@32": 0.8226909510349335, + "pass@64": 0.889786424123189, + "pass@128": 0.9341317365269461 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 175162, + "answer_accuracy": 0.4581362989621694, + "avg_response_len": 219.90556473886843, + "resp_tokens_sum": 84077814, + "pass_at_k": { + "pass@1": 0.4581362989621694, + "pass@2": 0.5592479237456827, + "pass@4": 0.6592036366123012, + "pass@8": 0.749294077232924, + "pass@16": 0.8233849023838853, + "pass@32": 0.8808246491906658, + "pass@64": 0.9229483093947489, + "pass@128": 0.9527954469367258 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2155783582089552, + "pass@2": 0.33451969972969814, + "pass@4": 0.4693261846000589, + "pass@8": 0.5978972579074031, + "pass@16": 0.7044582550974517, + "pass@32": 0.7876023011362222, + "pass@64": 0.8552567047585439, + "pass@128": 0.9194029850746268 + }, + "2": { + "pass@1": 0.9069642857142857, + "pass@2": 0.9312689820022496, + "pass@4": 0.9486790698484115, + "pass@8": 0.9622491070080178, + "pass@16": 0.9721970036911877, + "pass@32": 0.9795142189097774, + "pass@64": 0.984230110753602, + "pass@128": 0.9857142857142858 + }, + "3": { + "pass@1": 0.8650464650145773, + "pass@2": 0.9146810650581944, + "pass@4": 0.9394977516476619, + "pass@8": 0.9532998856091857, + "pass@16": 0.9628501996761378, + "pass@32": 0.9709821548050687, + "pass@64": 0.97883034020522, + "pass@128": 0.9883381924198251 + }, + "4": { + "pass@1": 0.5795869883040936, + "pass@2": 0.685957069922181, + "pass@4": 0.7741738061251112, + "pass@8": 0.8446660971516667, + "pass@16": 0.8991103940142082, + "pass@32": 0.9343024245485025, + "pass@64": 0.9508020706569525, + "pass@128": 0.9590643274853801 + }, + "5": { + "pass@1": 0.3434283088235294, + "pass@2": 0.4411019279759142, + "pass@4": 0.5363170749795979, + "pass@8": 0.6291732271577856, + "pass@16": 0.7177730817023343, + "pass@32": 0.7982552067000316, + "pass@64": 0.8593381500839656, + "pass@128": 0.8970588235294118 + }, + "6": { + "pass@1": 0.4251281545741325, + "pass@2": 0.5811580950843291, + "pass@4": 0.7208497665118351, + "pass@8": 0.8232985290012133, + "pass@16": 0.8898679892074591, + "pass@32": 0.9340736625308671, + "pass@64": 0.9627191767527687, + "pass@128": 0.9779179810725552 + }, + "7": { + "pass@1": 0.3076542721518987, + "pass@2": 0.44246525528256747, + "pass@4": 0.5810167029160911, + "pass@8": 0.7043732363576187, + "pass@16": 0.8016622464263247, + "pass@32": 0.8742022822931801, + "pass@64": 0.9264441039386613, + "pass@128": 0.9651898734177216 + }, + "8": { + "pass@1": 0.2358061186186186, + "pass@2": 0.35777263483956373, + "pass@4": 0.4963302315251132, + "pass@8": 0.6330193138275914, + "pass@16": 0.7494470352400429, + "pass@32": 0.8392101017513182, + "pass@64": 0.9060906232833575, + "pass@128": 0.9519519519519519 + }, + "9": { + "pass@1": 0.18197347266881028, + "pass@2": 0.2925186564802387, + "pass@4": 0.42774186194973196, + "pass@8": 0.5688716036414828, + "pass@16": 0.6958608696659061, + "pass@32": 0.7992127490405743, + "pass@64": 0.8781366109068879, + "pass@128": 0.9292604501607717 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.13439375, + "avg_response_len": 314.86191484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.13439375, + "pass@2": 0.21813414124015973, + "pass@4": 0.3208465112767184, + "pass@8": 0.4249888943504032, + "pass@16": 0.5127451917284023, + "pass@32": 0.5799013169671484, + "pass@64": 0.6343530893093835, + "pass@128": 0.6828 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1766796875, + "pass@2": 0.2829890501968507, + "pass@4": 0.40945695716160463, + "pass@8": 0.5329312864370122, + "pass@16": 0.6371944487073617, + "pass@32": 0.7241935581498754, + "pass@64": 0.7981385594046095, + "pass@128": 0.858 + }, + "12": { + "pass@1": 0.1753515625, + "pass@2": 0.2753619586614175, + "pass@4": 0.39047123725159316, + "pass@8": 0.5019142941805507, + "pass@16": 0.5979309507247363, + "pass@32": 0.6811670116285944, + "pass@64": 0.756459062205653, + "pass@128": 0.822 + }, + "13": { + "pass@1": 0.154375, + "pass@2": 0.24649532480314956, + "pass@4": 0.35418161295463046, + "pass@8": 0.45912232438049866, + "pass@16": 0.5503354766041044, + "pass@32": 0.6289536652219929, + "pass@64": 0.6996460973814737, + "pass@128": 0.764 + }, + "14": { + "pass@1": 0.143828125, + "pass@2": 0.23223781988188993, + "pass@4": 0.33858406627296594, + "pass@8": 0.4436302506510994, + "pass@16": 0.5317393843456101, + "pass@32": 0.6033135063862696, + "pass@64": 0.668302539245179, + "pass@128": 0.733 + }, + "15": { + "pass@1": 0.114609375, + "pass@2": 0.19118270177165375, + "pass@4": 0.29116556224221957, + "pass@8": 0.4007695494746514, + "pass@16": 0.4987306356065984, + "pass@32": 0.5733699532200323, + "pass@64": 0.6320358741620058, + "pass@128": 0.684 + }, + "16": { + "pass@1": 0.13715625, + "pass@2": 0.2218093011811022, + "pass@4": 0.3238864452568426, + "pass@8": 0.423411652608963, + "pass@16": 0.5025843150798188, + "pass@32": 0.5620127629555011, + "pass@64": 0.6132447054060225, + "pass@128": 0.662 + }, + "17": { + "pass@1": 0.1210546875, + "pass@2": 0.19964087106299203, + "pass@4": 0.2984382436257967, + "pass@8": 0.39926686487304824, + "pass@16": 0.4813704932570885, + "pass@32": 0.5397169982998381, + "pass@64": 0.5854133678331674, + "pass@128": 0.629 + }, + "18": { + "pass@1": 0.114875, + "pass@2": 0.1887636564960629, + "pass@4": 0.28218380474315724, + "pass@8": 0.37921148825376216, + "pass@16": 0.4583536184720494, + "pass@32": 0.5111680380493269, + "pass@64": 0.5482389252583693, + "pass@128": 0.581 + }, + "19": { + "pass@1": 0.1104609375, + "pass@2": 0.18176230314960617, + "pass@4": 0.2717745817397823, + "pass@8": 0.36498519472469115, + "pass@16": 0.44052881370250735, + "pass@32": 0.48934163598074065, + "pass@64": 0.5207733839924187, + "pass@128": 0.546 + }, + "20": { + "pass@1": 0.095546875, + "pass@2": 0.1610984251968504, + "pass@4": 0.24832260151856003, + "pass@8": 0.34464603791964593, + "pass@16": 0.42868378078398867, + "pass@32": 0.48577603977929185, + "pass@64": 0.5212783782048667, + "pass@128": 0.549 + } + }, + "per_op_accuracy": { + "11": 0.1766796875, + "12": 0.1753515625, + "13": 0.154375, + "14": 0.143828125, + "15": 0.114609375, + "16": 0.13715625, + "17": 0.1210546875, + "18": 0.114875, + "19": 0.1104609375, + "20": 0.095546875 + }, + "per_op_avg_response_len": { + "11": 290.6091171875, + "12": 297.0828515625, + "13": 305.3002734375, + "14": 315.76828125, + "15": 315.9274140625, + "16": 320.9783359375, + "17": 327.2103515625, + "18": 325.39678125, + "19": 323.3549140625, + "20": 326.990828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 290.6091171875, + "12": 297.0828515625, + "13": 305.3002734375, + "14": 315.76828125, + "15": 315.9274140625, + "16": 320.9783359375, + "17": 327.2103515625, + "18": 325.39678125, + "19": 323.3549140625, + "20": 326.990828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 58986, + "answer_accuracy": 0.13694743684992572, + "avg_response_len": 298.4180163447251, + "resp_tokens_sum": 128534608, + "pass_at_k": { + "pass@1": 0.13694743684992572, + "pass@2": 0.2211374325794715, + "pass@4": 0.32335033965850984, + "pass@8": 0.42691769754304526, + "pass@16": 0.5154425319145856, + "pass@32": 0.5843046812435945, + "pass@64": 0.6395969674055325, + "pass@128": 0.687072808320951 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1927452195467422, + "pass@2": 0.3026219301376279, + "pass@4": 0.431418140455531, + "pass@8": 0.5596011558602536, + "pass@16": 0.6691124471752867, + "pass@32": 0.7558813134298097, + "pass@64": 0.821310706535779, + "pass@128": 0.8640226628895185 + }, + "12": { + "pass@1": 0.16684322033898305, + "pass@2": 0.264343248142711, + "pass@4": 0.3767520347032891, + "pass@8": 0.4853319445797364, + "pass@16": 0.5806646367336986, + "pass@32": 0.6630849144980867, + "pass@64": 0.7337648428979751, + "pass@128": 0.7909604519774012 + }, + "13": { + "pass@1": 0.16465027429467086, + "pass@2": 0.25840046713400666, + "pass@4": 0.3634469285579116, + "pass@8": 0.46413359882342115, + "pass@16": 0.5558490901969614, + "pass@32": 0.6407655550139999, + "pass@64": 0.7175643568137752, + "pass@128": 0.780564263322884 + }, + "14": { + "pass@1": 0.14057632398753894, + "pass@2": 0.23038280042681575, + "pass@4": 0.3386714943342362, + "pass@8": 0.4454643097630123, + "pass@16": 0.5379967864190127, + "pass@32": 0.6166861756042253, + "pass@64": 0.689484801493184, + "pass@128": 0.7632398753894081 + }, + "15": { + "pass@1": 0.11599078757225434, + "pass@2": 0.193432891652633, + "pass@4": 0.2944971575084906, + "pass@8": 0.40651592812419884, + "pass@16": 0.5089160038620619, + "pass@32": 0.5890422875655702, + "pass@64": 0.6509615060463998, + "pass@128": 0.708092485549133 + }, + "16": { + "pass@1": 0.1469758064516129, + "pass@2": 0.23466432245364482, + "pass@4": 0.3392853496135562, + "pass@8": 0.441151099642495, + "pass@16": 0.5216397014566494, + "pass@32": 0.5830837413676024, + "pass@64": 0.6388692766638284, + "pass@128": 0.6935483870967742 + }, + "17": { + "pass@1": 0.11547145953757225, + "pass@2": 0.1877869549633608, + "pass@4": 0.2775737121552002, + "pass@8": 0.36958965536460775, + "pass@16": 0.4467152912085064, + "pass@32": 0.5056999218405851, + "pass@64": 0.5550444893472826, + "pass@128": 0.6011560693641619 + }, + "18": { + "pass@1": 0.11703125, + "pass@2": 0.1914370078740158, + "pass@4": 0.2851955653757565, + "pass@8": 0.3820999147912633, + "pass@16": 0.46089162638401937, + "pass@32": 0.5131813025594367, + "pass@64": 0.5494175506884521, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.11031343843843844, + "pass@2": 0.1848742196921331, + "pass@4": 0.2798845093687614, + "pass@8": 0.3767028297947531, + "pass@16": 0.45188701937385706, + "pass@32": 0.4971710103722426, + "pass@64": 0.5240927048521262, + "pass@128": 0.5435435435435435 + }, + "20": { + "pass@1": 0.09830142642642643, + "pass@2": 0.1625456657444847, + "pass@4": 0.24512006128738417, + "pass@8": 0.3354723396365215, + "pass@16": 0.41607247504734884, + "pass@32": 0.47314849238736145, + "pass@64": 0.5110935339188466, + "pass@128": 0.5405405405405406 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 58243, + "answer_accuracy": 0.13530283600951531, + "avg_response_len": 310.7512172911091, + "resp_tokens_sum": 133767212, + "pass_at_k": { + "pass@1": 0.13530283600951531, + "pass@2": 0.21915030051439846, + "pass@4": 0.32221819693925546, + "pass@8": 0.42671241617379874, + "pass@16": 0.5140266691859038, + "pass@32": 0.579820396352385, + "pass@64": 0.6330446750137837, + "pass@128": 0.6812369907820398 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.16799228162650603, + "pass@2": 0.2671806813869653, + "pass@4": 0.3836231012350866, + "pass@8": 0.49547203715780913, + "pass@16": 0.5913990690849538, + "pass@32": 0.6777404005188927, + "pass@64": 0.7607330836250256, + "pass@128": 0.8403614457831325 + }, + "12": { + "pass@1": 0.19803095611285265, + "pass@2": 0.3043818496531975, + "pass@4": 0.4234678795048738, + "pass@8": 0.5373117655964851, + "pass@16": 0.633112223326805, + "pass@32": 0.7141837879299113, + "pass@64": 0.787705464533175, + "pass@128": 0.8526645768025078 + }, + "13": { + "pass@1": 0.14377626050420167, + "pass@2": 0.2336692058713247, + "pass@4": 0.3419758614731982, + "pass@8": 0.44945168699137733, + "pass@16": 0.5414633284811603, + "pass@32": 0.6177322308195665, + "pass@64": 0.686910562781833, + "pass@128": 0.7563025210084033 + }, + "14": { + "pass@1": 0.14761513157894737, + "pass@2": 0.2372986168669705, + "pass@4": 0.3444328192528565, + "pass@8": 0.44664556769660785, + "pass@16": 0.5269598579960679, + "pass@32": 0.5898055394127177, + "pass@64": 0.6485638274319744, + "pass@128": 0.7076023391812866 + }, + "15": { + "pass@1": 0.12208806818181818, + "pass@2": 0.19942510737294203, + "pass@4": 0.2963789963754532, + "pass@8": 0.3982598483278521, + "pass@16": 0.4878949107801932, + "pass@32": 0.5587040931915987, + "pass@64": 0.6197747324074336, + "pass@128": 0.6727272727272727 + }, + "16": { + "pass@1": 0.1268028846153846, + "pass@2": 0.2082698897413465, + "pass@4": 0.31007912766246093, + "pass@8": 0.4124197783852443, + "pass@16": 0.4938064368685502, + "pass@32": 0.5518641332595097, + "pass@64": 0.5992631357352116, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.13327348602484473, + "pass@2": 0.22033259769159275, + "pass@4": 0.32994341555519846, + "pass@8": 0.4416166613059523, + "pass@16": 0.5330099260532796, + "pass@32": 0.597999239811453, + "pass@64": 0.6442029397091859, + "pass@128": 0.6832298136645962 + }, + "18": { + "pass@1": 0.11691176470588235, + "pass@2": 0.19327307202408514, + "pass@4": 0.29130406658726493, + "pass@8": 0.39555143896715, + "pass@16": 0.48208167098196425, + "pass@32": 0.5367870440715797, + "pass@64": 0.569770393006611, + "pass@128": 0.5970588235294118 + }, + "19": { + "pass@1": 0.10398065476190477, + "pass@2": 0.16996581482002251, + "pass@4": 0.25625598474297845, + "pass@8": 0.35065165885240784, + "pass@16": 0.4305015663253454, + "pass@32": 0.4829005293848349, + "pass@64": 0.5146124285334518, + "pass@128": 0.5357142857142857 + }, + "20": { + "pass@1": 0.09541074101796407, + "pass@2": 0.1615064654156255, + "pass@4": 0.24927919302003418, + "pass@8": 0.3450166044834941, + "pass@16": 0.4261222996576307, + "pass@32": 0.4774343147460568, + "pass@64": 0.5063468164477533, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 54795, + "answer_accuracy": 0.1308331104828851, + "avg_response_len": 335.99822117588633, + "resp_tokens_sum": 140721431, + "pass_at_k": { + "pass@1": 0.1308331104828851, + "pass@2": 0.21400106697485713, + "pass@4": 0.3168616820017009, + "pass@8": 0.42123381286035466, + "pass@16": 0.5086540675180571, + "pass@32": 0.5754559671007563, + "pass@64": 0.6303049681243292, + "pass@128": 0.6800122249388753 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.16783234126984126, + "pass@2": 0.2776492782152234, + "pass@4": 0.41207455228810697, + "pass@8": 0.5425249589903164, + "pass@16": 0.6496929013278826, + "pass@32": 0.7376433064662824, + "pass@64": 0.8115952261395818, + "pass@128": 0.8698412698412699 + }, + "12": { + "pass@1": 0.16243788226299694, + "pass@2": 0.25898054672638415, + "pass@4": 0.37313383304151193, + "pass@8": 0.4853343503793426, + "pass@16": 0.5823023549839643, + "pass@32": 0.6685330995309777, + "pass@64": 0.7505451517849687, + "pass@128": 0.8256880733944955 + }, + "13": { + "pass@1": 0.15593653549382716, + "pass@2": 0.2489063867016624, + "pass@4": 0.35850821048989273, + "pass@8": 0.46484399413550886, + "pass@16": 0.554682742479938, + "pass@32": 0.6296884159566101, + "pass@64": 0.6960370266813714, + "pass@128": 0.7561728395061729 + }, + "14": { + "pass@1": 0.14308234421364985, + "pass@2": 0.2288688842029019, + "pass@4": 0.3325652593685433, + "pass@8": 0.43882321384252904, + "pass@16": 0.530629508814399, + "pass@32": 0.6042841232586478, + "pass@64": 0.6681575340775399, + "pass@128": 0.7299703264094956 + }, + "15": { + "pass@1": 0.10551697530864197, + "pass@2": 0.18038467847769024, + "pass@4": 0.28229776833451387, + "pass@8": 0.3971891617144702, + "pass@16": 0.49889005466932873, + "pass@32": 0.5715708980528333, + "pass@64": 0.6243133684428923, + "pass@128": 0.6697530864197531 + }, + "16": { + "pass@1": 0.13889657079646017, + "pass@2": 0.22407265463498482, + "pass@4": 0.32410092350181907, + "pass@8": 0.41857070650905354, + "pass@16": 0.49424763506606695, + "pass@32": 0.5532521898450039, + "pass@64": 0.6042886991067128, + "pass@128": 0.6519174041297935 + }, + "17": { + "pass@1": 0.11502259036144578, + "pass@2": 0.19192616924390482, + "pass@4": 0.28962638376904687, + "pass@8": 0.3891213228204151, + "pass@16": 0.4674030310535856, + "pass@32": 0.5186417774810506, + "pass@64": 0.560044060038099, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.11020665322580646, + "pass@2": 0.18079954597409195, + "pass@4": 0.2687805620063621, + "pass@8": 0.35802912525157776, + "pass@16": 0.429463810012179, + "pass@32": 0.48079673280382973, + "pass@64": 0.5232930609521405, + "pass@128": 0.5612903225806452 + }, + "19": { + "pass@1": 0.1171875, + "pass@2": 0.19060626650331858, + "pass@4": 0.2793686684179584, + "pass@8": 0.36774681277531496, + "pass@16": 0.43928069476041537, + "pass@32": 0.48800335846972764, + "pass@64": 0.5236880280647143, + "pass@128": 0.5589123867069486 + }, + "20": { + "pass@1": 0.09292886636636637, + "pass@2": 0.15924191908443877, + "pass@4": 0.25056567760111054, + "pass@8": 0.35344805682641933, + "pass@16": 0.44386425978250205, + "pass@32": 0.506770362429784, + "pass@64": 0.5464396237727965, + "pass@128": 0.5765765765765766 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.2833059210526316, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.2833059210526316, + "pass@2": 0.3759499650331434, + "pass@4": 0.47854380420043746, + "pass@8": 0.5772726770906215, + "pass@16": 0.6594547943136616, + "pass@32": 0.7221325425613815, + "pass@64": 0.7704256447205274, + "pass@128": 0.8094210526315789 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1992890625, + "pass@2": 0.3125543799212598, + "pass@4": 0.44401627090363704, + "pass@8": 0.5753215603405694, + "pass@16": 0.6912148385299013, + "pass@32": 0.7852131040043056, + "pass@64": 0.8587442727751192, + "pass@128": 0.918 + }, + "2": { + "pass@1": 0.8721953125, + "pass@2": 0.9042281003937006, + "pass@4": 0.9299732025684282, + "pass@8": 0.9501093154237494, + "pass@16": 0.9651901174725063, + "pass@32": 0.9757265157459862, + "pass@64": 0.9828679096028664, + "pass@128": 0.988 + }, + "3": { + "pass@1": 0.8510546875, + "pass@2": 0.9055986712598424, + "pass@4": 0.9355771586051743, + "pass@8": 0.9540556049457282, + "pass@16": 0.967113004694048, + "pass@32": 0.9766052540263035, + "pass@64": 0.983570469094579, + "pass@128": 0.99 + }, + "4": { + "pass@1": 0.5731328125, + "pass@2": 0.6732614419291336, + "pass@4": 0.7590569236970385, + "pass@8": 0.8279303949256342, + "pass@16": 0.8804485147259381, + "pass@32": 0.9167334784533838, + "pass@64": 0.9386469535016336, + "pass@128": 0.953 + }, + "5": { + "pass@1": 0.3436015625, + "pass@2": 0.44655191929133936, + "pass@4": 0.5452499143232092, + "pass@8": 0.6376423926636297, + "pass@16": 0.7235327055370553, + "pass@32": 0.7990005413771109, + "pass@64": 0.8569880485324983, + "pass@128": 0.896 + }, + "6": { + "pass@1": 0.431078125, + "pass@2": 0.5777931840551189, + "pass@4": 0.7124146303899507, + "pass@8": 0.816428467947096, + "pass@16": 0.8877456246882576, + "pass@32": 0.9339198246266077, + "pass@64": 0.9623973979117663, + "pass@128": 0.978 + }, + "7": { + "pass@1": 0.31109375, + "pass@2": 0.44415428149606295, + "pass@4": 0.583285361454818, + "pass@8": 0.7091024975808733, + "pass@16": 0.8077639294200822, + "pass@32": 0.878038327242287, + "pass@64": 0.9256090060941099, + "pass@128": 0.954 + }, + "8": { + "pass@1": 0.25571875, + "pass@2": 0.38235063976377903, + "pass@4": 0.5241426136107981, + "pass@8": 0.6603608218326873, + "pass@16": 0.7722822147287928, + "pass@32": 0.8555920506067689, + "pass@64": 0.9152873405211892, + "pass@128": 0.956 + }, + "9": { + "pass@1": 0.2017109375, + "pass@2": 0.31521530511811047, + "pass@4": 0.450151091488563, + "pass@8": 0.5873408655576899, + "pass@16": 0.7068982248785488, + "pass@32": 0.80067604291157, + "pass@64": 0.8704449585622345, + "pass@128": 0.918 + }, + "11": { + "pass@1": 0.1766796875, + "pass@2": 0.2829890501968507, + "pass@4": 0.40945695716160463, + "pass@8": 0.5329312864370122, + "pass@16": 0.6371944487073617, + "pass@32": 0.7241935581498754, + "pass@64": 0.7981385594046095, + "pass@128": 0.858 + }, + "12": { + "pass@1": 0.1753515625, + "pass@2": 0.2753619586614175, + "pass@4": 0.39047123725159316, + "pass@8": 0.5019142941805507, + "pass@16": 0.5979309507247363, + "pass@32": 0.6811670116285944, + "pass@64": 0.756459062205653, + "pass@128": 0.822 + }, + "13": { + "pass@1": 0.154375, + "pass@2": 0.24649532480314956, + "pass@4": 0.35418161295463046, + "pass@8": 0.45912232438049866, + "pass@16": 0.5503354766041044, + "pass@32": 0.6289536652219929, + "pass@64": 0.6996460973814737, + "pass@128": 0.764 + }, + "14": { + "pass@1": 0.143828125, + "pass@2": 0.23223781988188993, + "pass@4": 0.33858406627296594, + "pass@8": 0.4436302506510994, + "pass@16": 0.5317393843456101, + "pass@32": 0.6033135063862696, + "pass@64": 0.668302539245179, + "pass@128": 0.733 + }, + "15": { + "pass@1": 0.114609375, + "pass@2": 0.19118270177165375, + "pass@4": 0.29116556224221957, + "pass@8": 0.4007695494746514, + "pass@16": 0.4987306356065984, + "pass@32": 0.5733699532200323, + "pass@64": 0.6320358741620058, + "pass@128": 0.684 + }, + "16": { + "pass@1": 0.13715625, + "pass@2": 0.2218093011811022, + "pass@4": 0.3238864452568426, + "pass@8": 0.423411652608963, + "pass@16": 0.5025843150798188, + "pass@32": 0.5620127629555011, + "pass@64": 0.6132447054060225, + "pass@128": 0.662 + }, + "17": { + "pass@1": 0.1210546875, + "pass@2": 0.19964087106299203, + "pass@4": 0.2984382436257967, + "pass@8": 0.39926686487304824, + "pass@16": 0.4813704932570885, + "pass@32": 0.5397169982998381, + "pass@64": 0.5854133678331674, + "pass@128": 0.629 + }, + "18": { + "pass@1": 0.114875, + "pass@2": 0.1887636564960629, + "pass@4": 0.28218380474315724, + "pass@8": 0.37921148825376216, + "pass@16": 0.4583536184720494, + "pass@32": 0.5111680380493269, + "pass@64": 0.5482389252583693, + "pass@128": 0.581 + }, + "19": { + "pass@1": 0.1104609375, + "pass@2": 0.18176230314960617, + "pass@4": 0.2717745817397823, + "pass@8": 0.36498519472469115, + "pass@16": 0.44052881370250735, + "pass@32": 0.48934163598074065, + "pass@64": 0.5207733839924187, + "pass@128": 0.546 + }, + "20": { + "pass@1": 0.095546875, + "pass@2": 0.1610984251968504, + "pass@4": 0.24832260151856003, + "pass@8": 0.34464603791964593, + "pass@16": 0.42868378078398867, + "pass@32": 0.48577603977929185, + "pass@64": 0.5212783782048667, + "pass@128": 0.549 + } + }, + "per_op_accuracy": { + "10": 0.1992890625, + "2": 0.8721953125, + "3": 0.8510546875, + "4": 0.5731328125, + "5": 0.3436015625, + "6": 0.431078125, + "7": 0.31109375, + "8": 0.25571875, + "9": 0.2017109375, + "11": 0.1766796875, + "12": 0.1753515625, + "13": 0.154375, + "14": 0.143828125, + "15": 0.114609375, + "16": 0.13715625, + "17": 0.1210546875, + "18": 0.114875, + "19": 0.1104609375, + "20": 0.095546875 + }, + "per_op_avg_response_len": { + "10": 277.849953125, + "2": 130.1738359375, + "3": 152.65065625, + "4": 167.6288359375, + "5": 175.914, + "6": 207.139859375, + "7": 229.352875, + "8": 246.1938203125, + "9": 256.984125, + "11": 290.6091171875, + "12": 297.0828515625, + "13": 305.3002734375, + "14": 315.76828125, + "15": 315.9274140625, + "16": 320.9783359375, + "17": 327.2103515625, + "18": 325.39678125, + "19": 323.3549140625, + "20": 326.990828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.849953125, + "2": 130.1738359375, + "3": 152.65065625, + "4": 167.6288359375, + "5": 175.914, + "6": 207.139859375, + "7": 229.352875, + "8": 246.1938203125, + "9": 256.984125, + "11": 290.6091171875, + "12": 297.0828515625, + "13": 305.3002734375, + "14": 315.76828125, + "15": 315.9274140625, + "16": 320.9783359375, + "17": 327.2103515625, + "18": 325.39678125, + "19": 323.3549140625, + "20": 326.990828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 224253, + "answer_accuracy": 0.2769485555643377, + "avg_response_len": 249.4732712713405, + "resp_tokens_sum": 202005493, + "pass_at_k": { + "pass@1": 0.2769485555643377, + "pass@2": 0.3713698318214332, + "pass@4": 0.47568788097662684, + "pass@8": 0.5759088361249274, + "pass@16": 0.6595118676241185, + "pass@32": 0.7230610799994464, + "pass@64": 0.7715846034812562, + "pass@128": 0.8101485931078091 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.19429848820058998, + "pass@2": 0.304934107379277, + "pass@4": 0.434951352319898, + "pass@8": 0.5684199490328546, + "pass@16": 0.6891009886552397, + "pass@32": 0.7857931447451506, + "pass@64": 0.860108943525792, + "pass@128": 0.9233038348082596 + }, + "2": { + "pass@1": 0.831386292834891, + "pass@2": 0.8734914268893955, + "pass@4": 0.9105459002204167, + "pass@8": 0.9401008638276085, + "pass@16": 0.9611783814748307, + "pass@32": 0.9741949948603676, + "pass@64": 0.9817242745943495, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.8260225535168195, + "pass@2": 0.8906170988947479, + "pass@4": 0.9285593713056506, + "pass@8": 0.9526553375068405, + "pass@16": 0.9699066685395095, + "pass@32": 0.9816788551557367, + "pass@64": 0.988712453516909, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.5572509765625, + "pass@2": 0.6628467950295271, + "pass@4": 0.7544799485025304, + "pass@8": 0.8284263532654119, + "pass@16": 0.884775967340774, + "pass@32": 0.9236644996257792, + "pass@64": 0.948151095592842, + "pass@128": 0.965625 + }, + "5": { + "pass@1": 0.3347718253968254, + "pass@2": 0.4497898700162479, + "pass@4": 0.5641970530469402, + "pass@8": 0.6677945563217111, + "pass@16": 0.7584282589253484, + "pass@32": 0.8321889687382951, + "pass@64": 0.8851659220568333, + "pass@128": 0.9174603174603174 + }, + "6": { + "pass@1": 0.4252432634730539, + "pass@2": 0.5709287625536326, + "pass@4": 0.7067664802565847, + "pass@8": 0.8127587788249542, + "pass@16": 0.8845402446547445, + "pass@32": 0.9276708113313906, + "pass@64": 0.9525643996495025, + "pass@128": 0.9670658682634731 + }, + "7": { + "pass@1": 0.3097956730769231, + "pass@2": 0.4431132210082467, + "pass@4": 0.5825773815070447, + "pass@8": 0.7077649850344365, + "pass@16": 0.804085063866593, + "pass@32": 0.8732138745287983, + "pass@64": 0.9242260451555102, + "pass@128": 0.9526627218934911 + }, + "8": { + "pass@1": 0.27226061698717946, + "pass@2": 0.3969216888754288, + "pass@4": 0.5352359978440191, + "pass@8": 0.668389018888514, + "pass@16": 0.777516858096813, + "pass@32": 0.8581637138287497, + "pass@64": 0.916168827905763, + "pass@128": 0.9551282051282052 + }, + "9": { + "pass@1": 0.20517165492957745, + "pass@2": 0.3156790645447484, + "pass@4": 0.44685785738050343, + "pass@8": 0.5801721703370608, + "pass@16": 0.6942797587019922, + "pass@32": 0.7812453529979867, + "pass@64": 0.8455093152480154, + "pass@128": 0.8929577464788733 + }, + "11": { + "pass@1": 0.1927452195467422, + "pass@2": 0.3026219301376279, + "pass@4": 0.431418140455531, + "pass@8": 0.5596011558602536, + "pass@16": 0.6691124471752867, + "pass@32": 0.7558813134298097, + "pass@64": 0.821310706535779, + "pass@128": 0.8640226628895185 + }, + "12": { + "pass@1": 0.16684322033898305, + "pass@2": 0.264343248142711, + "pass@4": 0.3767520347032891, + "pass@8": 0.4853319445797364, + "pass@16": 0.5806646367336986, + "pass@32": 0.6630849144980867, + "pass@64": 0.7337648428979751, + "pass@128": 0.7909604519774012 + }, + "13": { + "pass@1": 0.16465027429467086, + "pass@2": 0.25840046713400666, + "pass@4": 0.3634469285579116, + "pass@8": 0.46413359882342115, + "pass@16": 0.5558490901969614, + "pass@32": 0.6407655550139999, + "pass@64": 0.7175643568137752, + "pass@128": 0.780564263322884 + }, + "14": { + "pass@1": 0.14057632398753894, + "pass@2": 0.23038280042681575, + "pass@4": 0.3386714943342362, + "pass@8": 0.4454643097630123, + "pass@16": 0.5379967864190127, + "pass@32": 0.6166861756042253, + "pass@64": 0.689484801493184, + "pass@128": 0.7632398753894081 + }, + "15": { + "pass@1": 0.11599078757225434, + "pass@2": 0.193432891652633, + "pass@4": 0.2944971575084906, + "pass@8": 0.40651592812419884, + "pass@16": 0.5089160038620619, + "pass@32": 0.5890422875655702, + "pass@64": 0.6509615060463998, + "pass@128": 0.708092485549133 + }, + "16": { + "pass@1": 0.1469758064516129, + "pass@2": 0.23466432245364482, + "pass@4": 0.3392853496135562, + "pass@8": 0.441151099642495, + "pass@16": 0.5216397014566494, + "pass@32": 0.5830837413676024, + "pass@64": 0.6388692766638284, + "pass@128": 0.6935483870967742 + }, + "17": { + "pass@1": 0.11547145953757225, + "pass@2": 0.1877869549633608, + "pass@4": 0.2775737121552002, + "pass@8": 0.36958965536460775, + "pass@16": 0.4467152912085064, + "pass@32": 0.5056999218405851, + "pass@64": 0.5550444893472826, + "pass@128": 0.6011560693641619 + }, + "18": { + "pass@1": 0.11703125, + "pass@2": 0.1914370078740158, + "pass@4": 0.2851955653757565, + "pass@8": 0.3820999147912633, + "pass@16": 0.46089162638401937, + "pass@32": 0.5131813025594367, + "pass@64": 0.5494175506884521, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.11031343843843844, + "pass@2": 0.1848742196921331, + "pass@4": 0.2798845093687614, + "pass@8": 0.3767028297947531, + "pass@16": 0.45188701937385706, + "pass@32": 0.4971710103722426, + "pass@64": 0.5240927048521262, + "pass@128": 0.5435435435435435 + }, + "20": { + "pass@1": 0.09830142642642643, + "pass@2": 0.1625456657444847, + "pass@4": 0.24512006128738417, + "pass@8": 0.3354723396365215, + "pass@16": 0.41607247504734884, + "pass@32": 0.47314849238736145, + "pass@64": 0.5110935339188466, + "pass@128": 0.5405405405405406 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 234790, + "answer_accuracy": 0.2859387178487919, + "avg_response_len": 258.47156566640683, + "resp_tokens_sum": 212236172, + "pass_at_k": { + "pass@1": 0.2859387178487919, + "pass@2": 0.3777207700947009, + "pass@4": 0.4797066342447682, + "pass@8": 0.5781080472364224, + "pass@16": 0.659984755265672, + "pass@32": 0.7221385448329016, + "pass@64": 0.7697332178023997, + "pass@128": 0.8079501169134841 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1877396472392638, + "pass@2": 0.2979068040191296, + "pass@4": 0.42743402032567995, + "pass@8": 0.5592994362559849, + "pass@16": 0.6798039506691061, + "pass@32": 0.7821547762425294, + "pass@64": 0.8609090332078575, + "pass@128": 0.911042944785276 + }, + "2": { + "pass@1": 0.8750237462006079, + "pass@2": 0.9054504822535481, + "pass@4": 0.9290282496982711, + "pass@8": 0.9469597285175716, + "pass@16": 0.9616501693834948, + "pass@32": 0.9731913245513274, + "pass@64": 0.982534585697019, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.8613162878787879, + "pass@2": 0.9110038624433314, + "pass@4": 0.9384561071911464, + "pass@8": 0.9562286327789103, + "pass@16": 0.9687754714930751, + "pass@32": 0.9774223917946637, + "pass@64": 0.9834020912247246, + "pass@128": 0.9878787878787879 + }, + "4": { + "pass@1": 0.5816383136094675, + "pass@2": 0.6702755905511808, + "pass@4": 0.7480943742054436, + "pass@8": 0.8105270906947715, + "pass@16": 0.8574687882367842, + "pass@32": 0.8923947023004354, + "pass@64": 0.9173499844593105, + "pass@128": 0.9349112426035503 + }, + "5": { + "pass@1": 0.35183423913043477, + "pass@2": 0.44896653543307047, + "pass@4": 0.5367537307836514, + "pass@8": 0.6184585802569412, + "pass@16": 0.6973476991211023, + "pass@32": 0.7694325969464841, + "pass@64": 0.8289443827711489, + "pass@128": 0.8753623188405797 + }, + "6": { + "pass@1": 0.44206661891117477, + "pass@2": 0.5813061926088036, + "pass@4": 0.7101583094555867, + "pass@8": 0.8137002926251504, + "pass@16": 0.8888854739106302, + "pass@32": 0.9397605232654371, + "pass@64": 0.9715154997083257, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.31550307080924855, + "pass@2": 0.4467138500750988, + "pass@4": 0.5860489259651785, + "pass@8": 0.7147282946538364, + "pass@16": 0.8169303698336912, + "pass@32": 0.8862546718118719, + "pass@64": 0.9261973005402598, + "pass@128": 0.9450867052023122 + }, + "8": { + "pass@1": 0.25985915492957745, + "pass@2": 0.3925993955861151, + "pass@4": 0.5404817329171882, + "pass@8": 0.6789521589715024, + "pass@16": 0.78910166835959, + "pass@32": 0.8686986141098911, + "pass@64": 0.923139404679525, + "pass@128": 0.9605633802816902 + }, + "9": { + "pass@1": 0.21641092814371257, + "pass@2": 0.33585609293224555, + "pass@4": 0.47451746422789937, + "pass@8": 0.6121577136393171, + "pass@16": 0.730587365488769, + "pass@32": 0.8226909510349335, + "pass@64": 0.889786424123189, + "pass@128": 0.9341317365269461 + }, + "11": { + "pass@1": 0.16799228162650603, + "pass@2": 0.2671806813869653, + "pass@4": 0.3836231012350866, + "pass@8": 0.49547203715780913, + "pass@16": 0.5913990690849538, + "pass@32": 0.6777404005188927, + "pass@64": 0.7607330836250256, + "pass@128": 0.8403614457831325 + }, + "12": { + "pass@1": 0.19803095611285265, + "pass@2": 0.3043818496531975, + "pass@4": 0.4234678795048738, + "pass@8": 0.5373117655964851, + "pass@16": 0.633112223326805, + "pass@32": 0.7141837879299113, + "pass@64": 0.787705464533175, + "pass@128": 0.8526645768025078 + }, + "13": { + "pass@1": 0.14377626050420167, + "pass@2": 0.2336692058713247, + "pass@4": 0.3419758614731982, + "pass@8": 0.44945168699137733, + "pass@16": 0.5414633284811603, + "pass@32": 0.6177322308195665, + "pass@64": 0.686910562781833, + "pass@128": 0.7563025210084033 + }, + "14": { + "pass@1": 0.14761513157894737, + "pass@2": 0.2372986168669705, + "pass@4": 0.3444328192528565, + "pass@8": 0.44664556769660785, + "pass@16": 0.5269598579960679, + "pass@32": 0.5898055394127177, + "pass@64": 0.6485638274319744, + "pass@128": 0.7076023391812866 + }, + "15": { + "pass@1": 0.12208806818181818, + "pass@2": 0.19942510737294203, + "pass@4": 0.2963789963754532, + "pass@8": 0.3982598483278521, + "pass@16": 0.4878949107801932, + "pass@32": 0.5587040931915987, + "pass@64": 0.6197747324074336, + "pass@128": 0.6727272727272727 + }, + "16": { + "pass@1": 0.1268028846153846, + "pass@2": 0.2082698897413465, + "pass@4": 0.31007912766246093, + "pass@8": 0.4124197783852443, + "pass@16": 0.4938064368685502, + "pass@32": 0.5518641332595097, + "pass@64": 0.5992631357352116, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.13327348602484473, + "pass@2": 0.22033259769159275, + "pass@4": 0.32994341555519846, + "pass@8": 0.4416166613059523, + "pass@16": 0.5330099260532796, + "pass@32": 0.597999239811453, + "pass@64": 0.6442029397091859, + "pass@128": 0.6832298136645962 + }, + "18": { + "pass@1": 0.11691176470588235, + "pass@2": 0.19327307202408514, + "pass@4": 0.29130406658726493, + "pass@8": 0.39555143896715, + "pass@16": 0.48208167098196425, + "pass@32": 0.5367870440715797, + "pass@64": 0.569770393006611, + "pass@128": 0.5970588235294118 + }, + "19": { + "pass@1": 0.10398065476190477, + "pass@2": 0.16996581482002251, + "pass@4": 0.25625598474297845, + "pass@8": 0.35065165885240784, + "pass@16": 0.4305015663253454, + "pass@32": 0.4829005293848349, + "pass@64": 0.5146124285334518, + "pass@128": 0.5357142857142857 + }, + "20": { + "pass@1": 0.09541074101796407, + "pass@2": 0.1615064654156255, + "pass@4": 0.24927919302003418, + "pass@8": 0.3450166044834941, + "pass@16": 0.4261222996576307, + "pass@32": 0.4774343147460568, + "pass@64": 0.5063468164477533, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 229957, + "answer_accuracy": 0.2870329225914683, + "avg_response_len": 280.59499945079085, + "resp_tokens_sum": 224799245, + "pass_at_k": { + "pass@1": 0.2870329225914683, + "pass@2": 0.3787641858715626, + "pass@4": 0.4802384863509366, + "pass@8": 0.5777949264058008, + "pass@16": 0.6588539403003395, + "pass@32": 0.7211879136421478, + "pass@64": 0.7699639648290318, + "pass@128": 0.8101933216168717 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2155783582089552, + "pass@2": 0.33451969972969814, + "pass@4": 0.4693261846000589, + "pass@8": 0.5978972579074031, + "pass@16": 0.7044582550974517, + "pass@32": 0.7876023011362222, + "pass@64": 0.8552567047585439, + "pass@128": 0.9194029850746268 + }, + "2": { + "pass@1": 0.9069642857142857, + "pass@2": 0.9312689820022496, + "pass@4": 0.9486790698484115, + "pass@8": 0.9622491070080178, + "pass@16": 0.9721970036911877, + "pass@32": 0.9795142189097774, + "pass@64": 0.984230110753602, + "pass@128": 0.9857142857142858 + }, + "3": { + "pass@1": 0.8650464650145773, + "pass@2": 0.9146810650581944, + "pass@4": 0.9394977516476619, + "pass@8": 0.9532998856091857, + "pass@16": 0.9628501996761378, + "pass@32": 0.9709821548050687, + "pass@64": 0.97883034020522, + "pass@128": 0.9883381924198251 + }, + "4": { + "pass@1": 0.5795869883040936, + "pass@2": 0.685957069922181, + "pass@4": 0.7741738061251112, + "pass@8": 0.8446660971516667, + "pass@16": 0.8991103940142082, + "pass@32": 0.9343024245485025, + "pass@64": 0.9508020706569525, + "pass@128": 0.9590643274853801 + }, + "5": { + "pass@1": 0.3434283088235294, + "pass@2": 0.4411019279759142, + "pass@4": 0.5363170749795979, + "pass@8": 0.6291732271577856, + "pass@16": 0.7177730817023343, + "pass@32": 0.7982552067000316, + "pass@64": 0.8593381500839656, + "pass@128": 0.8970588235294118 + }, + "6": { + "pass@1": 0.4251281545741325, + "pass@2": 0.5811580950843291, + "pass@4": 0.7208497665118351, + "pass@8": 0.8232985290012133, + "pass@16": 0.8898679892074591, + "pass@32": 0.9340736625308671, + "pass@64": 0.9627191767527687, + "pass@128": 0.9779179810725552 + }, + "7": { + "pass@1": 0.3076542721518987, + "pass@2": 0.44246525528256747, + "pass@4": 0.5810167029160911, + "pass@8": 0.7043732363576187, + "pass@16": 0.8016622464263247, + "pass@32": 0.8742022822931801, + "pass@64": 0.9264441039386613, + "pass@128": 0.9651898734177216 + }, + "8": { + "pass@1": 0.2358061186186186, + "pass@2": 0.35777263483956373, + "pass@4": 0.4963302315251132, + "pass@8": 0.6330193138275914, + "pass@16": 0.7494470352400429, + "pass@32": 0.8392101017513182, + "pass@64": 0.9060906232833575, + "pass@128": 0.9519519519519519 + }, + "9": { + "pass@1": 0.18197347266881028, + "pass@2": 0.2925186564802387, + "pass@4": 0.42774186194973196, + "pass@8": 0.5688716036414828, + "pass@16": 0.6958608696659061, + "pass@32": 0.7992127490405743, + "pass@64": 0.8781366109068879, + "pass@128": 0.9292604501607717 + }, + "11": { + "pass@1": 0.16783234126984126, + "pass@2": 0.2776492782152234, + "pass@4": 0.41207455228810697, + "pass@8": 0.5425249589903164, + "pass@16": 0.6496929013278826, + "pass@32": 0.7376433064662824, + "pass@64": 0.8115952261395818, + "pass@128": 0.8698412698412699 + }, + "12": { + "pass@1": 0.16243788226299694, + "pass@2": 0.25898054672638415, + "pass@4": 0.37313383304151193, + "pass@8": 0.4853343503793426, + "pass@16": 0.5823023549839643, + "pass@32": 0.6685330995309777, + "pass@64": 0.7505451517849687, + "pass@128": 0.8256880733944955 + }, + "13": { + "pass@1": 0.15593653549382716, + "pass@2": 0.2489063867016624, + "pass@4": 0.35850821048989273, + "pass@8": 0.46484399413550886, + "pass@16": 0.554682742479938, + "pass@32": 0.6296884159566101, + "pass@64": 0.6960370266813714, + "pass@128": 0.7561728395061729 + }, + "14": { + "pass@1": 0.14308234421364985, + "pass@2": 0.2288688842029019, + "pass@4": 0.3325652593685433, + "pass@8": 0.43882321384252904, + "pass@16": 0.530629508814399, + "pass@32": 0.6042841232586478, + "pass@64": 0.6681575340775399, + "pass@128": 0.7299703264094956 + }, + "15": { + "pass@1": 0.10551697530864197, + "pass@2": 0.18038467847769024, + "pass@4": 0.28229776833451387, + "pass@8": 0.3971891617144702, + "pass@16": 0.49889005466932873, + "pass@32": 0.5715708980528333, + "pass@64": 0.6243133684428923, + "pass@128": 0.6697530864197531 + }, + "16": { + "pass@1": 0.13889657079646017, + "pass@2": 0.22407265463498482, + "pass@4": 0.32410092350181907, + "pass@8": 0.41857070650905354, + "pass@16": 0.49424763506606695, + "pass@32": 0.5532521898450039, + "pass@64": 0.6042886991067128, + "pass@128": 0.6519174041297935 + }, + "17": { + "pass@1": 0.11502259036144578, + "pass@2": 0.19192616924390482, + "pass@4": 0.28962638376904687, + "pass@8": 0.3891213228204151, + "pass@16": 0.4674030310535856, + "pass@32": 0.5186417774810506, + "pass@64": 0.560044060038099, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.11020665322580646, + "pass@2": 0.18079954597409195, + "pass@4": 0.2687805620063621, + "pass@8": 0.35802912525157776, + "pass@16": 0.429463810012179, + "pass@32": 0.48079673280382973, + "pass@64": 0.5232930609521405, + "pass@128": 0.5612903225806452 + }, + "19": { + "pass@1": 0.1171875, + "pass@2": 0.19060626650331858, + "pass@4": 0.2793686684179584, + "pass@8": 0.36774681277531496, + "pass@16": 0.43928069476041537, + "pass@32": 0.48800335846972764, + "pass@64": 0.5236880280647143, + "pass@128": 0.5589123867069486 + }, + "20": { + "pass@1": 0.09292886636636637, + "pass@2": 0.15924191908443877, + "pass@4": 0.25056567760111054, + "pass@8": 0.35344805682641933, + "pass@16": 0.44386425978250205, + "pass@32": 0.506770362429784, + "pass@64": 0.5464396237727965, + "pass@128": 0.5765765765765766 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/summary.csv b/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..5bf71e21434c7fa94925fc917b66a9ad30843f9e --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-42828,0.4487638888888889,0.13439375,0.2833059210526316,0.0,0.0,0.0,204.87644010416668,314.86191484375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/summary.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..99f1d9201bf210c0eec6a9b01dd0e82890748934 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block32/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-42828", + "id_acc": 0.4487638888888889, + "ood_acc": 0.13439375, + "total_acc": 0.2833059210526316, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 204.87644010416668, + "ood_avg_resp_len": 314.86191484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 224253, + "answer_accuracy": 0.2769485555643377, + "avg_response_len": 249.4732712713405, + "resp_tokens_sum": 202005493, + "pass_at_k": { + "pass@1": 0.2769485555643377, + "pass@2": 0.3713698318214332, + "pass@4": 0.47568788097662684, + "pass@8": 0.5759088361249274, + "pass@16": 0.6595118676241185, + "pass@32": 0.7230610799994464, + "pass@64": 0.7715846034812562, + "pass@128": 0.8101485931078091 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.19429848820058998, + "pass@2": 0.304934107379277, + "pass@4": 0.434951352319898, + "pass@8": 0.5684199490328546, + "pass@16": 0.6891009886552397, + "pass@32": 0.7857931447451506, + "pass@64": 0.860108943525792, + "pass@128": 0.9233038348082596 + }, + "2": { + "pass@1": 0.831386292834891, + "pass@2": 0.8734914268893955, + "pass@4": 0.9105459002204167, + "pass@8": 0.9401008638276085, + "pass@16": 0.9611783814748307, + "pass@32": 0.9741949948603676, + "pass@64": 0.9817242745943495, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.8260225535168195, + "pass@2": 0.8906170988947479, + "pass@4": 0.9285593713056506, + "pass@8": 0.9526553375068405, + "pass@16": 0.9699066685395095, + "pass@32": 0.9816788551557367, + "pass@64": 0.988712453516909, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.5572509765625, + "pass@2": 0.6628467950295271, + "pass@4": 0.7544799485025304, + "pass@8": 0.8284263532654119, + "pass@16": 0.884775967340774, + "pass@32": 0.9236644996257792, + "pass@64": 0.948151095592842, + "pass@128": 0.965625 + }, + "5": { + "pass@1": 0.3347718253968254, + "pass@2": 0.4497898700162479, + "pass@4": 0.5641970530469402, + "pass@8": 0.6677945563217111, + "pass@16": 0.7584282589253484, + "pass@32": 0.8321889687382951, + "pass@64": 0.8851659220568333, + "pass@128": 0.9174603174603174 + }, + "6": { + "pass@1": 0.4252432634730539, + "pass@2": 0.5709287625536326, + "pass@4": 0.7067664802565847, + "pass@8": 0.8127587788249542, + "pass@16": 0.8845402446547445, + "pass@32": 0.9276708113313906, + "pass@64": 0.9525643996495025, + "pass@128": 0.9670658682634731 + }, + "7": { + "pass@1": 0.3097956730769231, + "pass@2": 0.4431132210082467, + "pass@4": 0.5825773815070447, + "pass@8": 0.7077649850344365, + "pass@16": 0.804085063866593, + "pass@32": 0.8732138745287983, + "pass@64": 0.9242260451555102, + "pass@128": 0.9526627218934911 + }, + "8": { + "pass@1": 0.27226061698717946, + "pass@2": 0.3969216888754288, + "pass@4": 0.5352359978440191, + "pass@8": 0.668389018888514, + "pass@16": 0.777516858096813, + "pass@32": 0.8581637138287497, + "pass@64": 0.916168827905763, + "pass@128": 0.9551282051282052 + }, + "9": { + "pass@1": 0.20517165492957745, + "pass@2": 0.3156790645447484, + "pass@4": 0.44685785738050343, + "pass@8": 0.5801721703370608, + "pass@16": 0.6942797587019922, + "pass@32": 0.7812453529979867, + "pass@64": 0.8455093152480154, + "pass@128": 0.8929577464788733 + }, + "11": { + "pass@1": 0.1927452195467422, + "pass@2": 0.3026219301376279, + "pass@4": 0.431418140455531, + "pass@8": 0.5596011558602536, + "pass@16": 0.6691124471752867, + "pass@32": 0.7558813134298097, + "pass@64": 0.821310706535779, + "pass@128": 0.8640226628895185 + }, + "12": { + "pass@1": 0.16684322033898305, + "pass@2": 0.264343248142711, + "pass@4": 0.3767520347032891, + "pass@8": 0.4853319445797364, + "pass@16": 0.5806646367336986, + "pass@32": 0.6630849144980867, + "pass@64": 0.7337648428979751, + "pass@128": 0.7909604519774012 + }, + "13": { + "pass@1": 0.16465027429467086, + "pass@2": 0.25840046713400666, + "pass@4": 0.3634469285579116, + "pass@8": 0.46413359882342115, + "pass@16": 0.5558490901969614, + "pass@32": 0.6407655550139999, + "pass@64": 0.7175643568137752, + "pass@128": 0.780564263322884 + }, + "14": { + "pass@1": 0.14057632398753894, + "pass@2": 0.23038280042681575, + "pass@4": 0.3386714943342362, + "pass@8": 0.4454643097630123, + "pass@16": 0.5379967864190127, + "pass@32": 0.6166861756042253, + "pass@64": 0.689484801493184, + "pass@128": 0.7632398753894081 + }, + "15": { + "pass@1": 0.11599078757225434, + "pass@2": 0.193432891652633, + "pass@4": 0.2944971575084906, + "pass@8": 0.40651592812419884, + "pass@16": 0.5089160038620619, + "pass@32": 0.5890422875655702, + "pass@64": 0.6509615060463998, + "pass@128": 0.708092485549133 + }, + "16": { + "pass@1": 0.1469758064516129, + "pass@2": 0.23466432245364482, + "pass@4": 0.3392853496135562, + "pass@8": 0.441151099642495, + "pass@16": 0.5216397014566494, + "pass@32": 0.5830837413676024, + "pass@64": 0.6388692766638284, + "pass@128": 0.6935483870967742 + }, + "17": { + "pass@1": 0.11547145953757225, + "pass@2": 0.1877869549633608, + "pass@4": 0.2775737121552002, + "pass@8": 0.36958965536460775, + "pass@16": 0.4467152912085064, + "pass@32": 0.5056999218405851, + "pass@64": 0.5550444893472826, + "pass@128": 0.6011560693641619 + }, + "18": { + "pass@1": 0.11703125, + "pass@2": 0.1914370078740158, + "pass@4": 0.2851955653757565, + "pass@8": 0.3820999147912633, + "pass@16": 0.46089162638401937, + "pass@32": 0.5131813025594367, + "pass@64": 0.5494175506884521, + "pass@128": 0.5828571428571429 + }, + "19": { + "pass@1": 0.11031343843843844, + "pass@2": 0.1848742196921331, + "pass@4": 0.2798845093687614, + "pass@8": 0.3767028297947531, + "pass@16": 0.45188701937385706, + "pass@32": 0.4971710103722426, + "pass@64": 0.5240927048521262, + "pass@128": 0.5435435435435435 + }, + "20": { + "pass@1": 0.09830142642642643, + "pass@2": 0.1625456657444847, + "pass@4": 0.24512006128738417, + "pass@8": 0.3354723396365215, + "pass@16": 0.41607247504734884, + "pass@32": 0.47314849238736145, + "pass@64": 0.5110935339188466, + "pass@128": 0.5405405405405406 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 234790, + "answer_accuracy": 0.2859387178487919, + "avg_response_len": 258.47156566640683, + "resp_tokens_sum": 212236172, + "pass_at_k": { + "pass@1": 0.2859387178487919, + "pass@2": 0.3777207700947009, + "pass@4": 0.4797066342447682, + "pass@8": 0.5781080472364224, + "pass@16": 0.659984755265672, + "pass@32": 0.7221385448329016, + "pass@64": 0.7697332178023997, + "pass@128": 0.8079501169134841 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1877396472392638, + "pass@2": 0.2979068040191296, + "pass@4": 0.42743402032567995, + "pass@8": 0.5592994362559849, + "pass@16": 0.6798039506691061, + "pass@32": 0.7821547762425294, + "pass@64": 0.8609090332078575, + "pass@128": 0.911042944785276 + }, + "2": { + "pass@1": 0.8750237462006079, + "pass@2": 0.9054504822535481, + "pass@4": 0.9290282496982711, + "pass@8": 0.9469597285175716, + "pass@16": 0.9616501693834948, + "pass@32": 0.9731913245513274, + "pass@64": 0.982534585697019, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.8613162878787879, + "pass@2": 0.9110038624433314, + "pass@4": 0.9384561071911464, + "pass@8": 0.9562286327789103, + "pass@16": 0.9687754714930751, + "pass@32": 0.9774223917946637, + "pass@64": 0.9834020912247246, + "pass@128": 0.9878787878787879 + }, + "4": { + "pass@1": 0.5816383136094675, + "pass@2": 0.6702755905511808, + "pass@4": 0.7480943742054436, + "pass@8": 0.8105270906947715, + "pass@16": 0.8574687882367842, + "pass@32": 0.8923947023004354, + "pass@64": 0.9173499844593105, + "pass@128": 0.9349112426035503 + }, + "5": { + "pass@1": 0.35183423913043477, + "pass@2": 0.44896653543307047, + "pass@4": 0.5367537307836514, + "pass@8": 0.6184585802569412, + "pass@16": 0.6973476991211023, + "pass@32": 0.7694325969464841, + "pass@64": 0.8289443827711489, + "pass@128": 0.8753623188405797 + }, + "6": { + "pass@1": 0.44206661891117477, + "pass@2": 0.5813061926088036, + "pass@4": 0.7101583094555867, + "pass@8": 0.8137002926251504, + "pass@16": 0.8888854739106302, + "pass@32": 0.9397605232654371, + "pass@64": 0.9715154997083257, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.31550307080924855, + "pass@2": 0.4467138500750988, + "pass@4": 0.5860489259651785, + "pass@8": 0.7147282946538364, + "pass@16": 0.8169303698336912, + "pass@32": 0.8862546718118719, + "pass@64": 0.9261973005402598, + "pass@128": 0.9450867052023122 + }, + "8": { + "pass@1": 0.25985915492957745, + "pass@2": 0.3925993955861151, + "pass@4": 0.5404817329171882, + "pass@8": 0.6789521589715024, + "pass@16": 0.78910166835959, + "pass@32": 0.8686986141098911, + "pass@64": 0.923139404679525, + "pass@128": 0.9605633802816902 + }, + "9": { + "pass@1": 0.21641092814371257, + "pass@2": 0.33585609293224555, + "pass@4": 0.47451746422789937, + "pass@8": 0.6121577136393171, + "pass@16": 0.730587365488769, + "pass@32": 0.8226909510349335, + "pass@64": 0.889786424123189, + "pass@128": 0.9341317365269461 + }, + "11": { + "pass@1": 0.16799228162650603, + "pass@2": 0.2671806813869653, + "pass@4": 0.3836231012350866, + "pass@8": 0.49547203715780913, + "pass@16": 0.5913990690849538, + "pass@32": 0.6777404005188927, + "pass@64": 0.7607330836250256, + "pass@128": 0.8403614457831325 + }, + "12": { + "pass@1": 0.19803095611285265, + "pass@2": 0.3043818496531975, + "pass@4": 0.4234678795048738, + "pass@8": 0.5373117655964851, + "pass@16": 0.633112223326805, + "pass@32": 0.7141837879299113, + "pass@64": 0.787705464533175, + "pass@128": 0.8526645768025078 + }, + "13": { + "pass@1": 0.14377626050420167, + "pass@2": 0.2336692058713247, + "pass@4": 0.3419758614731982, + "pass@8": 0.44945168699137733, + "pass@16": 0.5414633284811603, + "pass@32": 0.6177322308195665, + "pass@64": 0.686910562781833, + "pass@128": 0.7563025210084033 + }, + "14": { + "pass@1": 0.14761513157894737, + "pass@2": 0.2372986168669705, + "pass@4": 0.3444328192528565, + "pass@8": 0.44664556769660785, + "pass@16": 0.5269598579960679, + "pass@32": 0.5898055394127177, + "pass@64": 0.6485638274319744, + "pass@128": 0.7076023391812866 + }, + "15": { + "pass@1": 0.12208806818181818, + "pass@2": 0.19942510737294203, + "pass@4": 0.2963789963754532, + "pass@8": 0.3982598483278521, + "pass@16": 0.4878949107801932, + "pass@32": 0.5587040931915987, + "pass@64": 0.6197747324074336, + "pass@128": 0.6727272727272727 + }, + "16": { + "pass@1": 0.1268028846153846, + "pass@2": 0.2082698897413465, + "pass@4": 0.31007912766246093, + "pass@8": 0.4124197783852443, + "pass@16": 0.4938064368685502, + "pass@32": 0.5518641332595097, + "pass@64": 0.5992631357352116, + "pass@128": 0.6438746438746439 + }, + "17": { + "pass@1": 0.13327348602484473, + "pass@2": 0.22033259769159275, + "pass@4": 0.32994341555519846, + "pass@8": 0.4416166613059523, + "pass@16": 0.5330099260532796, + "pass@32": 0.597999239811453, + "pass@64": 0.6442029397091859, + "pass@128": 0.6832298136645962 + }, + "18": { + "pass@1": 0.11691176470588235, + "pass@2": 0.19327307202408514, + "pass@4": 0.29130406658726493, + "pass@8": 0.39555143896715, + "pass@16": 0.48208167098196425, + "pass@32": 0.5367870440715797, + "pass@64": 0.569770393006611, + "pass@128": 0.5970588235294118 + }, + "19": { + "pass@1": 0.10398065476190477, + "pass@2": 0.16996581482002251, + "pass@4": 0.25625598474297845, + "pass@8": 0.35065165885240784, + "pass@16": 0.4305015663253454, + "pass@32": 0.4829005293848349, + "pass@64": 0.5146124285334518, + "pass@128": 0.5357142857142857 + }, + "20": { + "pass@1": 0.09541074101796407, + "pass@2": 0.1615064654156255, + "pass@4": 0.24927919302003418, + "pass@8": 0.3450166044834941, + "pass@16": 0.4261222996576307, + "pass@32": 0.4774343147460568, + "pass@64": 0.5063468164477533, + "pass@128": 0.5299401197604791 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 229957, + "answer_accuracy": 0.2870329225914683, + "avg_response_len": 280.59499945079085, + "resp_tokens_sum": 224799245, + "pass_at_k": { + "pass@1": 0.2870329225914683, + "pass@2": 0.3787641858715626, + "pass@4": 0.4802384863509366, + "pass@8": 0.5777949264058008, + "pass@16": 0.6588539403003395, + "pass@32": 0.7211879136421478, + "pass@64": 0.7699639648290318, + "pass@128": 0.8101933216168717 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2155783582089552, + "pass@2": 0.33451969972969814, + "pass@4": 0.4693261846000589, + "pass@8": 0.5978972579074031, + "pass@16": 0.7044582550974517, + "pass@32": 0.7876023011362222, + "pass@64": 0.8552567047585439, + "pass@128": 0.9194029850746268 + }, + "2": { + "pass@1": 0.9069642857142857, + "pass@2": 0.9312689820022496, + "pass@4": 0.9486790698484115, + "pass@8": 0.9622491070080178, + "pass@16": 0.9721970036911877, + "pass@32": 0.9795142189097774, + "pass@64": 0.984230110753602, + "pass@128": 0.9857142857142858 + }, + "3": { + "pass@1": 0.8650464650145773, + "pass@2": 0.9146810650581944, + "pass@4": 0.9394977516476619, + "pass@8": 0.9532998856091857, + "pass@16": 0.9628501996761378, + "pass@32": 0.9709821548050687, + "pass@64": 0.97883034020522, + "pass@128": 0.9883381924198251 + }, + "4": { + "pass@1": 0.5795869883040936, + "pass@2": 0.685957069922181, + "pass@4": 0.7741738061251112, + "pass@8": 0.8446660971516667, + "pass@16": 0.8991103940142082, + "pass@32": 0.9343024245485025, + "pass@64": 0.9508020706569525, + "pass@128": 0.9590643274853801 + }, + "5": { + "pass@1": 0.3434283088235294, + "pass@2": 0.4411019279759142, + "pass@4": 0.5363170749795979, + "pass@8": 0.6291732271577856, + "pass@16": 0.7177730817023343, + "pass@32": 0.7982552067000316, + "pass@64": 0.8593381500839656, + "pass@128": 0.8970588235294118 + }, + "6": { + "pass@1": 0.4251281545741325, + "pass@2": 0.5811580950843291, + "pass@4": 0.7208497665118351, + "pass@8": 0.8232985290012133, + "pass@16": 0.8898679892074591, + "pass@32": 0.9340736625308671, + "pass@64": 0.9627191767527687, + "pass@128": 0.9779179810725552 + }, + "7": { + "pass@1": 0.3076542721518987, + "pass@2": 0.44246525528256747, + "pass@4": 0.5810167029160911, + "pass@8": 0.7043732363576187, + "pass@16": 0.8016622464263247, + "pass@32": 0.8742022822931801, + "pass@64": 0.9264441039386613, + "pass@128": 0.9651898734177216 + }, + "8": { + "pass@1": 0.2358061186186186, + "pass@2": 0.35777263483956373, + "pass@4": 0.4963302315251132, + "pass@8": 0.6330193138275914, + "pass@16": 0.7494470352400429, + "pass@32": 0.8392101017513182, + "pass@64": 0.9060906232833575, + "pass@128": 0.9519519519519519 + }, + "9": { + "pass@1": 0.18197347266881028, + "pass@2": 0.2925186564802387, + "pass@4": 0.42774186194973196, + "pass@8": 0.5688716036414828, + "pass@16": 0.6958608696659061, + "pass@32": 0.7992127490405743, + "pass@64": 0.8781366109068879, + "pass@128": 0.9292604501607717 + }, + "11": { + "pass@1": 0.16783234126984126, + "pass@2": 0.2776492782152234, + "pass@4": 0.41207455228810697, + "pass@8": 0.5425249589903164, + "pass@16": 0.6496929013278826, + "pass@32": 0.7376433064662824, + "pass@64": 0.8115952261395818, + "pass@128": 0.8698412698412699 + }, + "12": { + "pass@1": 0.16243788226299694, + "pass@2": 0.25898054672638415, + "pass@4": 0.37313383304151193, + "pass@8": 0.4853343503793426, + "pass@16": 0.5823023549839643, + "pass@32": 0.6685330995309777, + "pass@64": 0.7505451517849687, + "pass@128": 0.8256880733944955 + }, + "13": { + "pass@1": 0.15593653549382716, + "pass@2": 0.2489063867016624, + "pass@4": 0.35850821048989273, + "pass@8": 0.46484399413550886, + "pass@16": 0.554682742479938, + "pass@32": 0.6296884159566101, + "pass@64": 0.6960370266813714, + "pass@128": 0.7561728395061729 + }, + "14": { + "pass@1": 0.14308234421364985, + "pass@2": 0.2288688842029019, + "pass@4": 0.3325652593685433, + "pass@8": 0.43882321384252904, + "pass@16": 0.530629508814399, + "pass@32": 0.6042841232586478, + "pass@64": 0.6681575340775399, + "pass@128": 0.7299703264094956 + }, + "15": { + "pass@1": 0.10551697530864197, + "pass@2": 0.18038467847769024, + "pass@4": 0.28229776833451387, + "pass@8": 0.3971891617144702, + "pass@16": 0.49889005466932873, + "pass@32": 0.5715708980528333, + "pass@64": 0.6243133684428923, + "pass@128": 0.6697530864197531 + }, + "16": { + "pass@1": 0.13889657079646017, + "pass@2": 0.22407265463498482, + "pass@4": 0.32410092350181907, + "pass@8": 0.41857070650905354, + "pass@16": 0.49424763506606695, + "pass@32": 0.5532521898450039, + "pass@64": 0.6042886991067128, + "pass@128": 0.6519174041297935 + }, + "17": { + "pass@1": 0.11502259036144578, + "pass@2": 0.19192616924390482, + "pass@4": 0.28962638376904687, + "pass@8": 0.3891213228204151, + "pass@16": 0.4674030310535856, + "pass@32": 0.5186417774810506, + "pass@64": 0.560044060038099, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.11020665322580646, + "pass@2": 0.18079954597409195, + "pass@4": 0.2687805620063621, + "pass@8": 0.35802912525157776, + "pass@16": 0.429463810012179, + "pass@32": 0.48079673280382973, + "pass@64": 0.5232930609521405, + "pass@128": 0.5612903225806452 + }, + "19": { + "pass@1": 0.1171875, + "pass@2": 0.19060626650331858, + "pass@4": 0.2793686684179584, + "pass@8": 0.36774681277531496, + "pass@16": 0.43928069476041537, + "pass@32": 0.48800335846972764, + "pass@64": 0.5236880280647143, + "pass@128": 0.5589123867069486 + }, + "20": { + "pass@1": 0.09292886636636637, + "pass@2": 0.15924191908443877, + "pass@4": 0.25056567760111054, + "pass@8": 0.35344805682641933, + "pass@16": 0.44386425978250205, + "pass@32": 0.506770362429784, + "pass@64": 0.5464396237727965, + "pass@128": 0.5765765765765766 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/checkpoint-42828_metrics.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/checkpoint-42828_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..5550961f72c21335701fd3d010fc8ae759ea262d --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/checkpoint-42828_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-42828", + "id": { + "answer_accuracy": 0.9906362847222222, + "avg_response_len": 197.01928211805554, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9906362847222222, + "pass@2": 0.9930506616360439, + "pass@4": 0.9944965327459065, + "pass@8": 0.9954734625347186, + "pass@16": 0.9960879399665757, + "pass@32": 0.9965444605960161, + "pass@64": 0.9970008450638569, + "pass@128": 0.9974444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.941546875, + "pass@2": 0.9541322588582679, + "pass@4": 0.9617565804274469, + "pass@8": 0.9666136161576501, + "pass@16": 0.969640538408961, + "pass@32": 0.9721651374786244, + "pass@64": 0.9750316280539475, + "pass@128": 0.978 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99321875, + "pass@2": 0.995060531496063, + "pass@4": 0.9966759176040496, + "pass@8": 0.9978702653844793, + "pass@16": 0.9984946364476459, + "pass@32": 0.9988272520477863, + "pass@64": 0.9989861761371662, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.999265625, + "pass@2": 0.999728715551181, + "pass@4": 0.9999324100112487, + "pass@8": 0.9999959826805417, + "pass@16": 0.9999999908357302, + "pass@32": 0.9999999999999952, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976171875, + "pass@2": 0.9980301427165353, + "pass@4": 0.9980929569741283, + "pass@8": 0.9981840551177641, + "pass@16": 0.9983602362204724, + "pass@32": 0.9986889763779527, + "pass@64": 0.9992519685039369, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936796875, + "pass@2": 0.9962599655511811, + "pass@4": 0.9978180674915634, + "pass@8": 0.9989961770682471, + "pass@16": 0.9996979232775005, + "pass@32": 0.9999615188082255, + "pass@64": 0.9999996942580379, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9904140625, + "pass@2": 0.9942443405511814, + "pass@4": 0.9961928622047245, + "pass@8": 0.9976010664037822, + "pass@16": 0.9985981345088686, + "pass@32": 0.9992572606515581, + "pass@64": 0.9997381386216251, + "pass@128": 1.0 + } + }, + "per_op_accuracy": { + "10": 0.941546875, + "2": 1.0, + "3": 0.999984375, + "4": 1.0, + "5": 0.99321875, + "6": 0.999265625, + "7": 0.9976171875, + "8": 0.9936796875, + "9": 0.9904140625 + }, + "per_op_avg_response_len": { + "10": 275.93503125, + "2": 114.55378125, + "3": 142.871921875, + "4": 157.9089921875, + "5": 170.6686015625, + "6": 195.9776171875, + "7": 220.5018984375, + "8": 239.625765625, + "9": 255.1299296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 275.93503125, + "2": 114.55378125, + "3": 142.871921875, + "4": 157.9089921875, + "5": 170.6686015625, + "6": 195.9776171875, + "7": 220.5018984375, + "8": 239.625765625, + "9": 255.1299296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 375325, + "answer_accuracy": 0.9902825270178993, + "avg_response_len": 187.5023060199257, + "resp_tokens_sum": 71064874, + "pass_at_k": { + "pass@1": 0.9902825270178993, + "pass@2": 0.9926270020768687, + "pass@4": 0.9938936876811371, + "pass@8": 0.9946718905736202, + "pass@16": 0.9951802607735272, + "pass@32": 0.9956186903808396, + "pass@64": 0.9962024990239703, + "pass@128": 0.9969604863221885 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9403346238938053, + "pass@2": 0.9509430962999099, + "pass@4": 0.9565902188111443, + "pass@8": 0.9595371010736143, + "pass@16": 0.9614777765607152, + "pass@32": 0.9640615389013077, + "pass@64": 0.9683065059344581, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9942212301587302, + "pass@2": 0.9963039776277965, + "pass@4": 0.9979051413216204, + "pass@8": 0.9991469167239885, + "pass@16": 0.9998257477435663, + "pass@32": 0.999994264397576, + "pass@64": 0.9999999994794706, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9983626497005988, + "pass@2": 0.9992098725776793, + "pass@4": 0.9997977273798858, + "pass@8": 0.9999879720977759, + "pass@16": 0.9999999725620663, + "pass@32": 0.9999999999999857, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966715976331361, + "pass@2": 0.9970745439826679, + "pass@4": 0.997133861873035, + "pass@8": 0.9972263313609467, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9923377403846154, + "pass@2": 0.9947195039622452, + "pass@4": 0.9961377279763107, + "pass@8": 0.9976819311023585, + "pass@16": 0.9991340276202069, + "pass@32": 0.9998780546502605, + "pass@64": 0.9999990200711261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925176056338029, + "pass@2": 0.9967980619940113, + "pass@4": 0.998693938433752, + "pass@8": 0.9996446951320699, + "pass@16": 0.9999658250813326, + "pass@32": 0.9999998146310128, + "pass@64": 0.9999999999999163, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 386827, + "answer_accuracy": 0.9901985378440367, + "avg_response_len": 194.99581473214286, + "resp_tokens_sum": 76176285, + "pass_at_k": { + "pass@1": 0.9901985378440367, + "pass@2": 0.9923994189301456, + "pass@4": 0.9937562865462589, + "pass@8": 0.9946553458656318, + "pass@16": 0.9952522574514109, + "pass@32": 0.9958403603462688, + "pass@64": 0.9964923601717215, + "pass@128": 0.997051114023591 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9299750766871165, + "pass@2": 0.9450815328969617, + "pass@4": 0.9548209196243105, + "pass@8": 0.9609402334817647, + "pass@16": 0.9646918199515472, + "pass@32": 0.9680913386644568, + "pass@64": 0.9717931593100866, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9970108695652173, + "pass@2": 0.9971003794362662, + "pass@4": 0.9971014492753624, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9996194484240688, + "pass@2": 0.9999802585565056, + "pass@4": 0.9999999113649476, + "pass@8": 0.9999999999996694, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9971763204906468, + "pass@4": 0.9972881948282477, + "pass@8": 0.9974611419588555, + "pass@16": 0.9977897000591688, + "pass@32": 0.9983785444449502, + "pass@64": 0.9992831459651359, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.997819743262726, + "pass@4": 0.9991270069234303, + "pass@8": 0.9998443747974352, + "pass@16": 0.9999946157287308, + "pass@32": 0.9999999970491635, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9898016467065869, + "pass@2": 0.9924092513791315, + "pass@4": 0.9937748347511949, + "pass@8": 0.9950757224434577, + "pass@16": 0.9963683581836216, + "pass@32": 0.9978084671909977, + "pass@64": 0.9992160024103857, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379061, + "answer_accuracy": 0.9914342358553733, + "avg_response_len": 208.5209187730164, + "resp_tokens_sum": 79725054, + "pass_at_k": { + "pass@1": 0.9914342358553733, + "pass@2": 0.9941360478741217, + "pass@4": 0.9958504850854136, + "pass@8": 0.99710397698094, + "pass@16": 0.9978415861423028, + "pass@32": 0.9981815947002559, + "pass@64": 0.998311792005572, + "pass@128": 0.9983260796786073 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9540345149253732, + "pass@2": 0.9661670730990719, + "pass@4": 0.9737339595610252, + "pass@8": 0.9792955903839986, + "pass@16": 0.9827165338826096, + "pass@32": 0.9843298489148303, + "pass@64": 0.9849885152450082, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9884420955882353, + "pass@2": 0.9918386695229272, + "pass@4": 0.9951052884933501, + "pass@8": 0.9974676076953616, + "pass@16": 0.9986750762012426, + "pass@32": 0.9994972316545585, + "pass@64": 0.9999593420621558, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998274842271293, + "pass@2": 0.9999984475521002, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995302610759493, + "pass@2": 0.9999871517741453, + "pass@4": 0.9999999881343946, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9923986486486487, + "pass@2": 0.9960404548249037, + "pass@4": 0.9979970237391498, + "pass@8": 0.9993233078415068, + "pass@16": 0.9999099641930826, + "pass@32": 0.9999986991137875, + "pass@64": 0.9999999999875272, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9886706189710611, + "pass@2": 0.9933001240600552, + "pass@4": 0.9959347885975668, + "pass@8": 0.9979804126552496, + "pass@16": 0.999431623703106, + "pass@32": 0.9999655897612711, + "pass@64": 0.9999999801176399, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.482271875, + "avg_response_len": 336.27141796875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.482271875, + "pass@2": 0.551825282972435, + "pass@4": 0.6112077546212886, + "pass@8": 0.6608506904596572, + "pass@16": 0.7017576692555063, + "pass@32": 0.735935275975569, + "pass@64": 0.7653365429030543, + "pass@128": 0.7915 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.942671875, + "pass@2": 0.960434178149607, + "pass@4": 0.9696364513498306, + "pass@8": 0.9749565103869903, + "pass@16": 0.9783121675393608, + "pass@32": 0.9807151258502091, + "pass@64": 0.9829278991632607, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.849421875, + "pass@2": 0.8900904281496061, + "pass@4": 0.9141636468878885, + "pass@8": 0.9296551819081322, + "pass@16": 0.9409815058963098, + "pass@32": 0.9505703005005227, + "pass@64": 0.9587466850730288, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.704140625, + "pass@2": 0.772272637795276, + "pass@4": 0.8158114396325459, + "pass@8": 0.8471688419253586, + "pass@16": 0.8718156843098835, + "pass@32": 0.8913931930461154, + "pass@64": 0.9073516238668969, + "pass@128": 0.923 + }, + "14": { + "pass@1": 0.6336484375, + "pass@2": 0.711968996062992, + "pass@4": 0.7677791620734906, + "pass@8": 0.8101158297551516, + "pass@16": 0.842159818968282, + "pass@32": 0.8659369733768657, + "pass@64": 0.8849830454988323, + "pass@128": 0.9 + }, + "15": { + "pass@1": 0.5167734375, + "pass@2": 0.6035274360236225, + "pass@4": 0.6720056540119984, + "pass@8": 0.7259832347190448, + "pass@16": 0.7688914252521226, + "pass@32": 0.8038843254483445, + "pass@64": 0.8337489515843769, + "pass@128": 0.858 + }, + "16": { + "pass@1": 0.327734375, + "pass@2": 0.4263244340551183, + "pass@4": 0.5173127898387698, + "pass@8": 0.5945853815198193, + "pass@16": 0.657150364268373, + "pass@32": 0.7078959901208094, + "pass@64": 0.7504828173849925, + "pass@128": 0.789 + }, + "17": { + "pass@1": 0.270984375, + "pass@2": 0.3586905757874019, + "pass@4": 0.4406256098612674, + "pass@8": 0.5099975618175794, + "pass@16": 0.5683759263622997, + "pass@32": 0.6187283717082966, + "pass@64": 0.6615651633205979, + "pass@128": 0.7 + }, + "18": { + "pass@1": 0.2395859375, + "pass@2": 0.3193051181102364, + "pass@4": 0.39681272290963565, + "pass@8": 0.4657354658247915, + "pass@16": 0.5226772675992768, + "pass@32": 0.5702328270253328, + "pass@64": 0.6120179403738408, + "pass@128": 0.651 + }, + "19": { + "pass@1": 0.1775234375, + "pass@2": 0.24752140748031498, + "pass@4": 0.31807677634045733, + "pass@8": 0.38242877616280324, + "pass@16": 0.43912582734264327, + "pass@32": 0.48954767168615276, + "pass@64": 0.533963244980295, + "pass@128": 0.573 + }, + "20": { + "pass@1": 0.160234375, + "pass@2": 0.22811761811023612, + "pass@4": 0.299853293307086, + "pass@8": 0.3678801205769079, + "pass@16": 0.4280867050163702, + "pass@32": 0.4804479809930262, + "pass@64": 0.5275780577843395, + "pass@128": 0.571 + } + }, + "per_op_accuracy": { + "11": 0.942671875, + "12": 0.849421875, + "13": 0.704140625, + "14": 0.6336484375, + "15": 0.5167734375, + "16": 0.327734375, + "17": 0.270984375, + "18": 0.2395859375, + "19": 0.1775234375, + "20": 0.160234375 + }, + "per_op_avg_response_len": { + "11": 293.8241484375, + "12": 302.7433515625, + "13": 318.103125, + "14": 337.8135078125, + "15": 346.3200859375, + "16": 345.770484375, + "17": 356.0385703125, + "18": 357.017890625, + "19": 348.2607265625, + "20": 356.8222890625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 293.8241484375, + "12": 302.7433515625, + "13": 318.103125, + "14": 337.8135078125, + "15": 346.3200859375, + "16": 345.770484375, + "17": 356.0385703125, + "18": 357.017890625, + "19": 348.2607265625, + "20": 356.8222890625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 203577, + "answer_accuracy": 0.4726434806835067, + "avg_response_len": 318.5935387258544, + "resp_tokens_sum": 137224609, + "pass_at_k": { + "pass@1": 0.4726434806835067, + "pass@2": 0.5419173974213498, + "pass@4": 0.6018438352523943, + "pass@8": 0.6518511736833396, + "pass@16": 0.6931041229571772, + "pass@32": 0.7284729355490037, + "pass@64": 0.7601472808469933, + "pass@128": 0.7895988112927191 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9313473796033994, + "pass@2": 0.9494340551181104, + "pass@4": 0.9594840918433353, + "pass@8": 0.9656293634436289, + "pass@16": 0.9699144094325778, + "pass@32": 0.9738884592658764, + "pass@64": 0.9786071170542627, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8317002118644068, + "pass@2": 0.8737276319008856, + "pass@4": 0.8998880984686232, + "pass@8": 0.9179671212613043, + "pass@16": 0.9325065069008127, + "pass@32": 0.9452756292304754, + "pass@64": 0.9557585734361933, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.7110109717868338, + "pass@2": 0.7740023264137438, + "pass@4": 0.8133029303938883, + "pass@8": 0.8424745465775301, + "pass@16": 0.8657274734514422, + "pass@32": 0.883408745813764, + "pass@64": 0.8965575001318108, + "pass@128": 0.9090909090909091 + }, + "14": { + "pass@1": 0.6173091900311527, + "pass@2": 0.6968503937007875, + "pass@4": 0.755002373827103, + "pass@8": 0.8013279156768645, + "pass@16": 0.8399074981743214, + "pass@32": 0.8702874981026382, + "pass@64": 0.8921680924393293, + "pass@128": 0.9034267912772586 + }, + "15": { + "pass@1": 0.4992097182080925, + "pass@2": 0.5920616949615403, + "pass@4": 0.6689661642728186, + "pass@8": 0.7308171485662596, + "pass@16": 0.7809849819776262, + "pass@32": 0.8218656781610865, + "pass@64": 0.8537746171902934, + "pass@128": 0.8786127167630058 + }, + "16": { + "pass@1": 0.3229586693548387, + "pass@2": 0.42145907416814815, + "pass@4": 0.5108256428631905, + "pass@8": 0.5849112297661468, + "pass@16": 0.6464023440647372, + "pass@32": 0.7016135981530378, + "pass@64": 0.7537077441857907, + "pass@128": 0.8032258064516129 + }, + "17": { + "pass@1": 0.25645773121387283, + "pass@2": 0.3438627196076645, + "pass@4": 0.42732528484517446, + "pass@8": 0.4971233048281026, + "pass@16": 0.5524004532069977, + "pass@32": 0.5981978550504864, + "pass@64": 0.6375194136444414, + "pass@128": 0.6734104046242775 + }, + "18": { + "pass@1": 0.20622767857142857, + "pass@2": 0.28793061023622046, + "pass@4": 0.3699408173978249, + "pass@8": 0.44184211629728665, + "pass@16": 0.4989635409044062, + "pass@32": 0.5479358885222821, + "pass@64": 0.5944936051842626, + "pass@128": 0.6428571428571429 + }, + "19": { + "pass@1": 0.16169294294294295, + "pass@2": 0.23235661842945318, + "pass@4": 0.30415678501898985, + "pass@8": 0.3676896908172229, + "pass@16": 0.4226789379330411, + "pass@32": 0.47296546637220566, + "pass@64": 0.5220829393578104, + "pass@128": 0.5705705705705706 + }, + "20": { + "pass@1": 0.16422672672672672, + "pass@2": 0.2278495276772835, + "pass@4": 0.29342507918492167, + "pass@8": 0.3555577960470933, + "pass@16": 0.4111276274809168, + "pass@32": 0.4613041839190401, + "pass@64": 0.5109832928872265, + "pass@128": 0.5645645645645646 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 211435, + "answer_accuracy": 0.491179285608088, + "avg_response_len": 333.55958918748144, + "resp_tokens_sum": 143585395, + "pass_at_k": { + "pass@1": 0.491179285608088, + "pass@2": 0.5624001626664444, + "pass@4": 0.6226630204899697, + "pass@8": 0.6724817983901821, + "pass@16": 0.7129479140920036, + "pass@32": 0.7459789678040957, + "pass@64": 0.7735737375817594, + "pass@128": 0.7972048765982753 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9445359563253012, + "pass@2": 0.960735165069728, + "pass@4": 0.9692484578358429, + "pass@8": 0.9744137411819622, + "pass@16": 0.9774220287888713, + "pass@32": 0.9791276976341268, + "pass@64": 0.9803800331296995, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8528605015673981, + "pass@2": 0.8893576598869498, + "pass@4": 0.9101586607120814, + "pass@8": 0.9224588859306808, + "pass@16": 0.9310670110053562, + "pass@32": 0.939270483161552, + "pass@64": 0.9473029142231101, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7229297969187675, + "pass@2": 0.7983782587838287, + "pass@4": 0.8442330002867283, + "pass@8": 0.8754003024489906, + "pass@16": 0.8995318657586917, + "pass@32": 0.9187236439009093, + "pass@64": 0.9338720887441335, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6482775950292398, + "pass@2": 0.7274618530413961, + "pass@4": 0.7836484260739333, + "pass@8": 0.8230514235910104, + "pass@16": 0.8486700010918876, + "pass@32": 0.8653014217564913, + "pass@64": 0.8789849352201726, + "pass@128": 0.8918128654970761 + }, + "15": { + "pass@1": 0.5319365530303031, + "pass@2": 0.6206174689811499, + "pass@4": 0.6868207306473046, + "pass@8": 0.7367855877874147, + "pass@16": 0.7752766588096213, + "pass@32": 0.806337169755552, + "pass@64": 0.8352289076502452, + "pass@128": 0.8606060606060606 + }, + "16": { + "pass@1": 0.32307247150997154, + "pass@2": 0.42357676043699655, + "pass@4": 0.5158068435889954, + "pass@8": 0.5908012273052607, + "pass@16": 0.6477279950868452, + "pass@32": 0.6922022409236969, + "pass@64": 0.7305376737607546, + "pass@128": 0.7663817663817664 + }, + "17": { + "pass@1": 0.3075504658385093, + "pass@2": 0.40160269538318555, + "pass@4": 0.48713984820220435, + "pass@8": 0.5596349682703199, + "pass@16": 0.6222102488736226, + "pass@32": 0.6744260839145514, + "pass@64": 0.7144136967050305, + "pass@128": 0.7484472049689441 + }, + "18": { + "pass@1": 0.25282628676470587, + "pass@2": 0.33365743689207966, + "pass@4": 0.4128379664858067, + "pass@8": 0.48697562456319365, + "pass@16": 0.552866663966737, + "pass@32": 0.6086135381508534, + "pass@64": 0.6542544109542544, + "pass@128": 0.6911764705882353 + }, + "19": { + "pass@1": 0.17978050595238096, + "pass@2": 0.24771110154199458, + "pass@4": 0.3194336779331155, + "pass@8": 0.386929726317321, + "pass@16": 0.446349013761704, + "pass@32": 0.5002298543667417, + "pass@64": 0.5475897643199032, + "pass@128": 0.5863095238095238 + }, + "20": { + "pass@1": 0.15585235778443113, + "pass@2": 0.2257480609646848, + "pass@4": 0.30009878920449323, + "pass@8": 0.3697365614620309, + "pass@16": 0.4289157795835793, + "pass@32": 0.47570921025814755, + "pass@64": 0.5129501839362879, + "pass@128": 0.5449101796407185 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 202296, + "answer_accuracy": 0.4830187958435208, + "avg_response_len": 357.23900471806235, + "resp_tokens_sum": 149617411, + "pass_at_k": { + "pass@1": 0.4830187958435208, + "pass@2": 0.5511457947293222, + "pass@4": 0.6090639678120178, + "pass@8": 0.6581514111142969, + "pass@16": 0.6991557101811252, + "pass@32": 0.7332866940428104, + "pass@64": 0.7622070139036811, + "pass@128": 0.7875916870415648 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9533978174603175, + "pass@2": 0.9724440694913137, + "pass@4": 0.9814224727861399, + "pass@8": 0.9859808984729452, + "pass@16": 0.9886611665135104, + "pass@32": 0.9900384257613478, + "pass@64": 0.9904552883937954, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8652522935779816, + "pass@2": 0.9085191221796822, + "pass@4": 0.9335249151768868, + "pass@8": 0.9493285515894258, + "pass@16": 0.9598282138920922, + "pass@32": 0.9673255156709447, + "pass@64": 0.9731453224447834, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6766734182098766, + "pass@2": 0.7418051150724216, + "pass@4": 0.7869648880695465, + "pass@8": 0.8206838073852996, + "pass@16": 0.8472708154414211, + "pass@32": 0.8691403156138889, + "pass@64": 0.8887575482814627, + "pass@128": 0.9104938271604939 + }, + "14": { + "pass@1": 0.6343657270029673, + "pass@2": 0.710647079955139, + "pass@4": 0.76384462420684, + "pass@8": 0.8053589968983749, + "pass@16": 0.8376984322875355, + "pass@32": 0.8624379829234405, + "pass@64": 0.8842262313961666, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.5200858410493827, + "pass@2": 0.5983652000097209, + "pass@4": 0.660162098950594, + "pass@8": 0.7098186955100998, + "pass@16": 0.7494732224712638, + "pass@32": 0.782183749337275, + "pass@64": 0.8108561559010913, + "pass@128": 0.8333333333333334 + }, + "16": { + "pass@1": 0.3369284660766962, + "pass@2": 0.4336185196153577, + "pass@4": 0.5248042432195976, + "pass@8": 0.6073500575462162, + "pass@16": 0.6767348416897403, + "pass@32": 0.7298902304694688, + "pass@64": 0.7681849356854631, + "pass@128": 0.799410029498525 + }, + "17": { + "pass@1": 0.2506588855421687, + "pass@2": 0.3325241171378427, + "pass@4": 0.4093735848907438, + "pass@8": 0.47527240531329096, + "pass@16": 0.5328122572752206, + "pass@32": 0.5861045627721161, + "pass@64": 0.6353681803030131, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.26272681451612906, + "pass@2": 0.33898669672339343, + "pass@4": 0.40957589746846157, + "pass@8": 0.46941617022340476, + "pass@16": 0.516339879142077, + "pass@32": 0.5533118163588501, + "pass@64": 0.585479609144845, + "pass@128": 0.6161290322580645 + }, + "19": { + "pass@1": 0.19115842145015105, + "pass@2": 0.26258526714560976, + "pass@4": 0.3307034777752478, + "pass@8": 0.3926879790877576, + "pass@16": 0.44833979573114197, + "pass@32": 0.49538652663741045, + "pass@64": 0.5320829769264548, + "pass@128": 0.5619335347432024 + }, + "20": { + "pass@1": 0.1606371996996997, + "pass@2": 0.230762381475964, + "pass@4": 0.3060352743069278, + "pass@8": 0.3783404293240473, + "pass@16": 0.44421421827119956, + "pass@32": 0.5043447793446393, + "pass@64": 0.558844624078598, + "pass@128": 0.6036036036036037 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7230760690789474, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7230760690789474, + "pass@2": 0.7608267781288642, + "pass@4": 0.7927655968908314, + "pass@8": 0.8193562140741684, + "pass@16": 0.8411772711712796, + "pass@32": 0.8593817318484243, + "pass@64": 0.8750722649792195, + "pass@128": 0.8890526315789473 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.941546875, + "pass@2": 0.9541322588582679, + "pass@4": 0.9617565804274469, + "pass@8": 0.9666136161576501, + "pass@16": 0.969640538408961, + "pass@32": 0.9721651374786244, + "pass@64": 0.9750316280539475, + "pass@128": 0.978 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.99321875, + "pass@2": 0.995060531496063, + "pass@4": 0.9966759176040496, + "pass@8": 0.9978702653844793, + "pass@16": 0.9984946364476459, + "pass@32": 0.9988272520477863, + "pass@64": 0.9989861761371662, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.999265625, + "pass@2": 0.999728715551181, + "pass@4": 0.9999324100112487, + "pass@8": 0.9999959826805417, + "pass@16": 0.9999999908357302, + "pass@32": 0.9999999999999952, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9976171875, + "pass@2": 0.9980301427165353, + "pass@4": 0.9980929569741283, + "pass@8": 0.9981840551177641, + "pass@16": 0.9983602362204724, + "pass@32": 0.9986889763779527, + "pass@64": 0.9992519685039369, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9936796875, + "pass@2": 0.9962599655511811, + "pass@4": 0.9978180674915634, + "pass@8": 0.9989961770682471, + "pass@16": 0.9996979232775005, + "pass@32": 0.9999615188082255, + "pass@64": 0.9999996942580379, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9904140625, + "pass@2": 0.9942443405511814, + "pass@4": 0.9961928622047245, + "pass@8": 0.9976010664037822, + "pass@16": 0.9985981345088686, + "pass@32": 0.9992572606515581, + "pass@64": 0.9997381386216251, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.942671875, + "pass@2": 0.960434178149607, + "pass@4": 0.9696364513498306, + "pass@8": 0.9749565103869903, + "pass@16": 0.9783121675393608, + "pass@32": 0.9807151258502091, + "pass@64": 0.9829278991632607, + "pass@128": 0.985 + }, + "12": { + "pass@1": 0.849421875, + "pass@2": 0.8900904281496061, + "pass@4": 0.9141636468878885, + "pass@8": 0.9296551819081322, + "pass@16": 0.9409815058963098, + "pass@32": 0.9505703005005227, + "pass@64": 0.9587466850730288, + "pass@128": 0.965 + }, + "13": { + "pass@1": 0.704140625, + "pass@2": 0.772272637795276, + "pass@4": 0.8158114396325459, + "pass@8": 0.8471688419253586, + "pass@16": 0.8718156843098835, + "pass@32": 0.8913931930461154, + "pass@64": 0.9073516238668969, + "pass@128": 0.923 + }, + "14": { + "pass@1": 0.6336484375, + "pass@2": 0.711968996062992, + "pass@4": 0.7677791620734906, + "pass@8": 0.8101158297551516, + "pass@16": 0.842159818968282, + "pass@32": 0.8659369733768657, + "pass@64": 0.8849830454988323, + "pass@128": 0.9 + }, + "15": { + "pass@1": 0.5167734375, + "pass@2": 0.6035274360236225, + "pass@4": 0.6720056540119984, + "pass@8": 0.7259832347190448, + "pass@16": 0.7688914252521226, + "pass@32": 0.8038843254483445, + "pass@64": 0.8337489515843769, + "pass@128": 0.858 + }, + "16": { + "pass@1": 0.327734375, + "pass@2": 0.4263244340551183, + "pass@4": 0.5173127898387698, + "pass@8": 0.5945853815198193, + "pass@16": 0.657150364268373, + "pass@32": 0.7078959901208094, + "pass@64": 0.7504828173849925, + "pass@128": 0.789 + }, + "17": { + "pass@1": 0.270984375, + "pass@2": 0.3586905757874019, + "pass@4": 0.4406256098612674, + "pass@8": 0.5099975618175794, + "pass@16": 0.5683759263622997, + "pass@32": 0.6187283717082966, + "pass@64": 0.6615651633205979, + "pass@128": 0.7 + }, + "18": { + "pass@1": 0.2395859375, + "pass@2": 0.3193051181102364, + "pass@4": 0.39681272290963565, + "pass@8": 0.4657354658247915, + "pass@16": 0.5226772675992768, + "pass@32": 0.5702328270253328, + "pass@64": 0.6120179403738408, + "pass@128": 0.651 + }, + "19": { + "pass@1": 0.1775234375, + "pass@2": 0.24752140748031498, + "pass@4": 0.31807677634045733, + "pass@8": 0.38242877616280324, + "pass@16": 0.43912582734264327, + "pass@32": 0.48954767168615276, + "pass@64": 0.533963244980295, + "pass@128": 0.573 + }, + "20": { + "pass@1": 0.160234375, + "pass@2": 0.22811761811023612, + "pass@4": 0.299853293307086, + "pass@8": 0.3678801205769079, + "pass@16": 0.4280867050163702, + "pass@32": 0.4804479809930262, + "pass@64": 0.5275780577843395, + "pass@128": 0.571 + } + }, + "per_op_accuracy": { + "10": 0.941546875, + "2": 1.0, + "3": 0.999984375, + "4": 1.0, + "5": 0.99321875, + "6": 0.999265625, + "7": 0.9976171875, + "8": 0.9936796875, + "9": 0.9904140625, + "11": 0.942671875, + "12": 0.849421875, + "13": 0.704140625, + "14": 0.6336484375, + "15": 0.5167734375, + "16": 0.327734375, + "17": 0.270984375, + "18": 0.2395859375, + "19": 0.1775234375, + "20": 0.160234375 + }, + "per_op_avg_response_len": { + "10": 275.93503125, + "2": 114.55378125, + "3": 142.871921875, + "4": 157.9089921875, + "5": 170.6686015625, + "6": 195.9776171875, + "7": 220.5018984375, + "8": 239.625765625, + "9": 255.1299296875, + "11": 293.8241484375, + "12": 302.7433515625, + "13": 318.103125, + "14": 337.8135078125, + "15": 346.3200859375, + "16": 345.770484375, + "17": 356.0385703125, + "18": 357.017890625, + "19": 348.2607265625, + "20": 356.8222890625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 275.93503125, + "2": 114.55378125, + "3": 142.871921875, + "4": 157.9089921875, + "5": 170.6686015625, + "6": 195.9776171875, + "7": 220.5018984375, + "8": 239.625765625, + "9": 255.1299296875, + "11": 293.8241484375, + "12": 302.7433515625, + "13": 318.103125, + "14": 337.8135078125, + "15": 346.3200859375, + "16": 345.770484375, + "17": 356.0385703125, + "18": 357.017890625, + "19": 348.2607265625, + "20": 356.8222890625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 578902, + "answer_accuracy": 0.7149339037306355, + "avg_response_len": 257.2338896518337, + "resp_tokens_sum": 208289483, + "pass_at_k": { + "pass@1": 0.7149339037306355, + "pass@2": 0.7528802711780616, + "pass@4": 0.7853499391160436, + "pass@8": 0.8123146802770955, + "pass@16": 0.8344963841133981, + "pass@32": 0.8535153920866395, + "pass@64": 0.8706372430698933, + "pass@128": 0.8866582358520392 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9403346238938053, + "pass@2": 0.9509430962999099, + "pass@4": 0.9565902188111443, + "pass@8": 0.9595371010736143, + "pass@16": 0.9614777765607152, + "pass@32": 0.9640615389013077, + "pass@64": 0.9683065059344581, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9942212301587302, + "pass@2": 0.9963039776277965, + "pass@4": 0.9979051413216204, + "pass@8": 0.9991469167239885, + "pass@16": 0.9998257477435663, + "pass@32": 0.999994264397576, + "pass@64": 0.9999999994794706, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9983626497005988, + "pass@2": 0.9992098725776793, + "pass@4": 0.9997977273798858, + "pass@8": 0.9999879720977759, + "pass@16": 0.9999999725620663, + "pass@32": 0.9999999999999857, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966715976331361, + "pass@2": 0.9970745439826679, + "pass@4": 0.997133861873035, + "pass@8": 0.9972263313609467, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9923377403846154, + "pass@2": 0.9947195039622452, + "pass@4": 0.9961377279763107, + "pass@8": 0.9976819311023585, + "pass@16": 0.9991340276202069, + "pass@32": 0.9998780546502605, + "pass@64": 0.9999990200711261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925176056338029, + "pass@2": 0.9967980619940113, + "pass@4": 0.998693938433752, + "pass@8": 0.9996446951320699, + "pass@16": 0.9999658250813326, + "pass@32": 0.9999998146310128, + "pass@64": 0.9999999999999163, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9313473796033994, + "pass@2": 0.9494340551181104, + "pass@4": 0.9594840918433353, + "pass@8": 0.9656293634436289, + "pass@16": 0.9699144094325778, + "pass@32": 0.9738884592658764, + "pass@64": 0.9786071170542627, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8317002118644068, + "pass@2": 0.8737276319008856, + "pass@4": 0.8998880984686232, + "pass@8": 0.9179671212613043, + "pass@16": 0.9325065069008127, + "pass@32": 0.9452756292304754, + "pass@64": 0.9557585734361933, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.7110109717868338, + "pass@2": 0.7740023264137438, + "pass@4": 0.8133029303938883, + "pass@8": 0.8424745465775301, + "pass@16": 0.8657274734514422, + "pass@32": 0.883408745813764, + "pass@64": 0.8965575001318108, + "pass@128": 0.9090909090909091 + }, + "14": { + "pass@1": 0.6173091900311527, + "pass@2": 0.6968503937007875, + "pass@4": 0.755002373827103, + "pass@8": 0.8013279156768645, + "pass@16": 0.8399074981743214, + "pass@32": 0.8702874981026382, + "pass@64": 0.8921680924393293, + "pass@128": 0.9034267912772586 + }, + "15": { + "pass@1": 0.4992097182080925, + "pass@2": 0.5920616949615403, + "pass@4": 0.6689661642728186, + "pass@8": 0.7308171485662596, + "pass@16": 0.7809849819776262, + "pass@32": 0.8218656781610865, + "pass@64": 0.8537746171902934, + "pass@128": 0.8786127167630058 + }, + "16": { + "pass@1": 0.3229586693548387, + "pass@2": 0.42145907416814815, + "pass@4": 0.5108256428631905, + "pass@8": 0.5849112297661468, + "pass@16": 0.6464023440647372, + "pass@32": 0.7016135981530378, + "pass@64": 0.7537077441857907, + "pass@128": 0.8032258064516129 + }, + "17": { + "pass@1": 0.25645773121387283, + "pass@2": 0.3438627196076645, + "pass@4": 0.42732528484517446, + "pass@8": 0.4971233048281026, + "pass@16": 0.5524004532069977, + "pass@32": 0.5981978550504864, + "pass@64": 0.6375194136444414, + "pass@128": 0.6734104046242775 + }, + "18": { + "pass@1": 0.20622767857142857, + "pass@2": 0.28793061023622046, + "pass@4": 0.3699408173978249, + "pass@8": 0.44184211629728665, + "pass@16": 0.4989635409044062, + "pass@32": 0.5479358885222821, + "pass@64": 0.5944936051842626, + "pass@128": 0.6428571428571429 + }, + "19": { + "pass@1": 0.16169294294294295, + "pass@2": 0.23235661842945318, + "pass@4": 0.30415678501898985, + "pass@8": 0.3676896908172229, + "pass@16": 0.4226789379330411, + "pass@32": 0.47296546637220566, + "pass@64": 0.5220829393578104, + "pass@128": 0.5705705705705706 + }, + "20": { + "pass@1": 0.16422672672672672, + "pass@2": 0.2278495276772835, + "pass@4": 0.29342507918492167, + "pass@8": 0.3555577960470933, + "pass@16": 0.4111276274809168, + "pass@32": 0.4613041839190401, + "pass@64": 0.5109832928872265, + "pass@128": 0.5645645645645646 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 598262, + "answer_accuracy": 0.7285926539360873, + "avg_response_len": 267.6364964925955, + "resp_tokens_sum": 219761680, + "pass_at_k": { + "pass@1": 0.7285926539360873, + "pass@2": 0.7669765820143414, + "pass@4": 0.7992143296098074, + "pass@8": 0.8257590652483338, + "pass@16": 0.8472570108859162, + "pass@32": 0.8648530083401398, + "pass@64": 0.8796294875653305, + "pass@128": 0.8922837100545596 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9299750766871165, + "pass@2": 0.9450815328969617, + "pass@4": 0.9548209196243105, + "pass@8": 0.9609402334817647, + "pass@16": 0.9646918199515472, + "pass@32": 0.9680913386644568, + "pass@64": 0.9717931593100866, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9970108695652173, + "pass@2": 0.9971003794362662, + "pass@4": 0.9971014492753624, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9996194484240688, + "pass@2": 0.9999802585565056, + "pass@4": 0.9999999113649476, + "pass@8": 0.9999999999996694, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9971763204906468, + "pass@4": 0.9972881948282477, + "pass@8": 0.9974611419588555, + "pass@16": 0.9977897000591688, + "pass@32": 0.9983785444449502, + "pass@64": 0.9992831459651359, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.997819743262726, + "pass@4": 0.9991270069234303, + "pass@8": 0.9998443747974352, + "pass@16": 0.9999946157287308, + "pass@32": 0.9999999970491635, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9898016467065869, + "pass@2": 0.9924092513791315, + "pass@4": 0.9937748347511949, + "pass@8": 0.9950757224434577, + "pass@16": 0.9963683581836216, + "pass@32": 0.9978084671909977, + "pass@64": 0.9992160024103857, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9445359563253012, + "pass@2": 0.960735165069728, + "pass@4": 0.9692484578358429, + "pass@8": 0.9744137411819622, + "pass@16": 0.9774220287888713, + "pass@32": 0.9791276976341268, + "pass@64": 0.9803800331296995, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8528605015673981, + "pass@2": 0.8893576598869498, + "pass@4": 0.9101586607120814, + "pass@8": 0.9224588859306808, + "pass@16": 0.9310670110053562, + "pass@32": 0.939270483161552, + "pass@64": 0.9473029142231101, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7229297969187675, + "pass@2": 0.7983782587838287, + "pass@4": 0.8442330002867283, + "pass@8": 0.8754003024489906, + "pass@16": 0.8995318657586917, + "pass@32": 0.9187236439009093, + "pass@64": 0.9338720887441335, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6482775950292398, + "pass@2": 0.7274618530413961, + "pass@4": 0.7836484260739333, + "pass@8": 0.8230514235910104, + "pass@16": 0.8486700010918876, + "pass@32": 0.8653014217564913, + "pass@64": 0.8789849352201726, + "pass@128": 0.8918128654970761 + }, + "15": { + "pass@1": 0.5319365530303031, + "pass@2": 0.6206174689811499, + "pass@4": 0.6868207306473046, + "pass@8": 0.7367855877874147, + "pass@16": 0.7752766588096213, + "pass@32": 0.806337169755552, + "pass@64": 0.8352289076502452, + "pass@128": 0.8606060606060606 + }, + "16": { + "pass@1": 0.32307247150997154, + "pass@2": 0.42357676043699655, + "pass@4": 0.5158068435889954, + "pass@8": 0.5908012273052607, + "pass@16": 0.6477279950868452, + "pass@32": 0.6922022409236969, + "pass@64": 0.7305376737607546, + "pass@128": 0.7663817663817664 + }, + "17": { + "pass@1": 0.3075504658385093, + "pass@2": 0.40160269538318555, + "pass@4": 0.48713984820220435, + "pass@8": 0.5596349682703199, + "pass@16": 0.6222102488736226, + "pass@32": 0.6744260839145514, + "pass@64": 0.7144136967050305, + "pass@128": 0.7484472049689441 + }, + "18": { + "pass@1": 0.25282628676470587, + "pass@2": 0.33365743689207966, + "pass@4": 0.4128379664858067, + "pass@8": 0.48697562456319365, + "pass@16": 0.552866663966737, + "pass@32": 0.6086135381508534, + "pass@64": 0.6542544109542544, + "pass@128": 0.6911764705882353 + }, + "19": { + "pass@1": 0.17978050595238096, + "pass@2": 0.24771110154199458, + "pass@4": 0.3194336779331155, + "pass@8": 0.386929726317321, + "pass@16": 0.446349013761704, + "pass@32": 0.5002298543667417, + "pass@64": 0.5475897643199032, + "pass@128": 0.5863095238095238 + }, + "20": { + "pass@1": 0.15585235778443113, + "pass@2": 0.2257480609646848, + "pass@4": 0.30009878920449323, + "pass@8": 0.3697365614620309, + "pass@16": 0.4289157795835793, + "pass@32": 0.47570921025814755, + "pass@64": 0.5129501839362879, + "pass@128": 0.5449101796407185 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581357, + "answer_accuracy": 0.7256513121105608, + "avg_response_len": 286.265858413884, + "resp_tokens_sum": 229342465, + "pass_at_k": { + "pass@1": 0.7256513121105608, + "pass@2": 0.7625552668723906, + "pass@4": 0.7936511745695828, + "pass@8": 0.8199106880345169, + "pass@16": 0.8416984025434961, + "pass@32": 0.859703225160211, + "pass@64": 0.8748839546594552, + "pass@128": 0.8881610480907494 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9540345149253732, + "pass@2": 0.9661670730990719, + "pass@4": 0.9737339595610252, + "pass@8": 0.9792955903839986, + "pass@16": 0.9827165338826096, + "pass@32": 0.9843298489148303, + "pass@64": 0.9849885152450082, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9884420955882353, + "pass@2": 0.9918386695229272, + "pass@4": 0.9951052884933501, + "pass@8": 0.9974676076953616, + "pass@16": 0.9986750762012426, + "pass@32": 0.9994972316545585, + "pass@64": 0.9999593420621558, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998274842271293, + "pass@2": 0.9999984475521002, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995302610759493, + "pass@2": 0.9999871517741453, + "pass@4": 0.9999999881343946, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9923986486486487, + "pass@2": 0.9960404548249037, + "pass@4": 0.9979970237391498, + "pass@8": 0.9993233078415068, + "pass@16": 0.9999099641930826, + "pass@32": 0.9999986991137875, + "pass@64": 0.9999999999875272, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9886706189710611, + "pass@2": 0.9933001240600552, + "pass@4": 0.9959347885975668, + "pass@8": 0.9979804126552496, + "pass@16": 0.999431623703106, + "pass@32": 0.9999655897612711, + "pass@64": 0.9999999801176399, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9533978174603175, + "pass@2": 0.9724440694913137, + "pass@4": 0.9814224727861399, + "pass@8": 0.9859808984729452, + "pass@16": 0.9886611665135104, + "pass@32": 0.9900384257613478, + "pass@64": 0.9904552883937954, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8652522935779816, + "pass@2": 0.9085191221796822, + "pass@4": 0.9335249151768868, + "pass@8": 0.9493285515894258, + "pass@16": 0.9598282138920922, + "pass@32": 0.9673255156709447, + "pass@64": 0.9731453224447834, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6766734182098766, + "pass@2": 0.7418051150724216, + "pass@4": 0.7869648880695465, + "pass@8": 0.8206838073852996, + "pass@16": 0.8472708154414211, + "pass@32": 0.8691403156138889, + "pass@64": 0.8887575482814627, + "pass@128": 0.9104938271604939 + }, + "14": { + "pass@1": 0.6343657270029673, + "pass@2": 0.710647079955139, + "pass@4": 0.76384462420684, + "pass@8": 0.8053589968983749, + "pass@16": 0.8376984322875355, + "pass@32": 0.8624379829234405, + "pass@64": 0.8842262313961666, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.5200858410493827, + "pass@2": 0.5983652000097209, + "pass@4": 0.660162098950594, + "pass@8": 0.7098186955100998, + "pass@16": 0.7494732224712638, + "pass@32": 0.782183749337275, + "pass@64": 0.8108561559010913, + "pass@128": 0.8333333333333334 + }, + "16": { + "pass@1": 0.3369284660766962, + "pass@2": 0.4336185196153577, + "pass@4": 0.5248042432195976, + "pass@8": 0.6073500575462162, + "pass@16": 0.6767348416897403, + "pass@32": 0.7298902304694688, + "pass@64": 0.7681849356854631, + "pass@128": 0.799410029498525 + }, + "17": { + "pass@1": 0.2506588855421687, + "pass@2": 0.3325241171378427, + "pass@4": 0.4093735848907438, + "pass@8": 0.47527240531329096, + "pass@16": 0.5328122572752206, + "pass@32": 0.5861045627721161, + "pass@64": 0.6353681803030131, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.26272681451612906, + "pass@2": 0.33898669672339343, + "pass@4": 0.40957589746846157, + "pass@8": 0.46941617022340476, + "pass@16": 0.516339879142077, + "pass@32": 0.5533118163588501, + "pass@64": 0.585479609144845, + "pass@128": 0.6161290322580645 + }, + "19": { + "pass@1": 0.19115842145015105, + "pass@2": 0.26258526714560976, + "pass@4": 0.3307034777752478, + "pass@8": 0.3926879790877576, + "pass@16": 0.44833979573114197, + "pass@32": 0.49538652663741045, + "pass@64": 0.5320829769264548, + "pass@128": 0.5619335347432024 + }, + "20": { + "pass@1": 0.1606371996996997, + "pass@2": 0.230762381475964, + "pass@4": 0.3060352743069278, + "pass@8": 0.3783404293240473, + "pass@16": 0.44421421827119956, + "pass@32": 0.5043447793446393, + "pass@64": 0.558844624078598, + "pass@128": 0.6036036036036037 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/summary.csv b/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..5df6f69685fb20fa7ed321de42e501b03e8562f2 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-42828,0.9906362847222222,0.482271875,0.7230760690789474,0.0,0.0,0.0,197.01928211805554,336.27141796875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/summary.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3c442ec635aa5ad0c17c3e1f41812f0697682aa5 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block4/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-42828", + "id_acc": 0.9906362847222222, + "ood_acc": 0.482271875, + "total_acc": 0.7230760690789474, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.01928211805554, + "ood_avg_resp_len": 336.27141796875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 578902, + "answer_accuracy": 0.7149339037306355, + "avg_response_len": 257.2338896518337, + "resp_tokens_sum": 208289483, + "pass_at_k": { + "pass@1": 0.7149339037306355, + "pass@2": 0.7528802711780616, + "pass@4": 0.7853499391160436, + "pass@8": 0.8123146802770955, + "pass@16": 0.8344963841133981, + "pass@32": 0.8535153920866395, + "pass@64": 0.8706372430698933, + "pass@128": 0.8866582358520392 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9403346238938053, + "pass@2": 0.9509430962999099, + "pass@4": 0.9565902188111443, + "pass@8": 0.9595371010736143, + "pass@16": 0.9614777765607152, + "pass@32": 0.9640615389013077, + "pass@64": 0.9683065059344581, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999761085626911, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9942212301587302, + "pass@2": 0.9963039776277965, + "pass@4": 0.9979051413216204, + "pass@8": 0.9991469167239885, + "pass@16": 0.9998257477435663, + "pass@32": 0.999994264397576, + "pass@64": 0.9999999994794706, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9983626497005988, + "pass@2": 0.9992098725776793, + "pass@4": 0.9997977273798858, + "pass@8": 0.9999879720977759, + "pass@16": 0.9999999725620663, + "pass@32": 0.9999999999999857, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9966715976331361, + "pass@2": 0.9970745439826679, + "pass@4": 0.997133861873035, + "pass@8": 0.9972263313609467, + "pass@16": 0.9974112426035503, + "pass@32": 0.9977810650887574, + "pass@64": 0.9985207100591716, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9923377403846154, + "pass@2": 0.9947195039622452, + "pass@4": 0.9961377279763107, + "pass@8": 0.9976819311023585, + "pass@16": 0.9991340276202069, + "pass@32": 0.9998780546502605, + "pass@64": 0.9999990200711261, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9925176056338029, + "pass@2": 0.9967980619940113, + "pass@4": 0.998693938433752, + "pass@8": 0.9996446951320699, + "pass@16": 0.9999658250813326, + "pass@32": 0.9999998146310128, + "pass@64": 0.9999999999999163, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9313473796033994, + "pass@2": 0.9494340551181104, + "pass@4": 0.9594840918433353, + "pass@8": 0.9656293634436289, + "pass@16": 0.9699144094325778, + "pass@32": 0.9738884592658764, + "pass@64": 0.9786071170542627, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.8317002118644068, + "pass@2": 0.8737276319008856, + "pass@4": 0.8998880984686232, + "pass@8": 0.9179671212613043, + "pass@16": 0.9325065069008127, + "pass@32": 0.9452756292304754, + "pass@64": 0.9557585734361933, + "pass@128": 0.963276836158192 + }, + "13": { + "pass@1": 0.7110109717868338, + "pass@2": 0.7740023264137438, + "pass@4": 0.8133029303938883, + "pass@8": 0.8424745465775301, + "pass@16": 0.8657274734514422, + "pass@32": 0.883408745813764, + "pass@64": 0.8965575001318108, + "pass@128": 0.9090909090909091 + }, + "14": { + "pass@1": 0.6173091900311527, + "pass@2": 0.6968503937007875, + "pass@4": 0.755002373827103, + "pass@8": 0.8013279156768645, + "pass@16": 0.8399074981743214, + "pass@32": 0.8702874981026382, + "pass@64": 0.8921680924393293, + "pass@128": 0.9034267912772586 + }, + "15": { + "pass@1": 0.4992097182080925, + "pass@2": 0.5920616949615403, + "pass@4": 0.6689661642728186, + "pass@8": 0.7308171485662596, + "pass@16": 0.7809849819776262, + "pass@32": 0.8218656781610865, + "pass@64": 0.8537746171902934, + "pass@128": 0.8786127167630058 + }, + "16": { + "pass@1": 0.3229586693548387, + "pass@2": 0.42145907416814815, + "pass@4": 0.5108256428631905, + "pass@8": 0.5849112297661468, + "pass@16": 0.6464023440647372, + "pass@32": 0.7016135981530378, + "pass@64": 0.7537077441857907, + "pass@128": 0.8032258064516129 + }, + "17": { + "pass@1": 0.25645773121387283, + "pass@2": 0.3438627196076645, + "pass@4": 0.42732528484517446, + "pass@8": 0.4971233048281026, + "pass@16": 0.5524004532069977, + "pass@32": 0.5981978550504864, + "pass@64": 0.6375194136444414, + "pass@128": 0.6734104046242775 + }, + "18": { + "pass@1": 0.20622767857142857, + "pass@2": 0.28793061023622046, + "pass@4": 0.3699408173978249, + "pass@8": 0.44184211629728665, + "pass@16": 0.4989635409044062, + "pass@32": 0.5479358885222821, + "pass@64": 0.5944936051842626, + "pass@128": 0.6428571428571429 + }, + "19": { + "pass@1": 0.16169294294294295, + "pass@2": 0.23235661842945318, + "pass@4": 0.30415678501898985, + "pass@8": 0.3676896908172229, + "pass@16": 0.4226789379330411, + "pass@32": 0.47296546637220566, + "pass@64": 0.5220829393578104, + "pass@128": 0.5705705705705706 + }, + "20": { + "pass@1": 0.16422672672672672, + "pass@2": 0.2278495276772835, + "pass@4": 0.29342507918492167, + "pass@8": 0.3555577960470933, + "pass@16": 0.4111276274809168, + "pass@32": 0.4613041839190401, + "pass@64": 0.5109832928872265, + "pass@128": 0.5645645645645646 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 598262, + "answer_accuracy": 0.7285926539360873, + "avg_response_len": 267.6364964925955, + "resp_tokens_sum": 219761680, + "pass_at_k": { + "pass@1": 0.7285926539360873, + "pass@2": 0.7669765820143414, + "pass@4": 0.7992143296098074, + "pass@8": 0.8257590652483338, + "pass@16": 0.8472570108859162, + "pass@32": 0.8648530083401398, + "pass@64": 0.8796294875653305, + "pass@128": 0.8922837100545596 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9299750766871165, + "pass@2": 0.9450815328969617, + "pass@4": 0.9548209196243105, + "pass@8": 0.9609402334817647, + "pass@16": 0.9646918199515472, + "pass@32": 0.9680913386644568, + "pass@64": 0.9717931593100866, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9970108695652173, + "pass@2": 0.9971003794362662, + "pass@4": 0.9971014492753624, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9996194484240688, + "pass@2": 0.9999802585565056, + "pass@4": 0.9999999113649476, + "pass@8": 0.9999999999996694, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9967937138728323, + "pass@2": 0.9971763204906468, + "pass@4": 0.9972881948282477, + "pass@8": 0.9974611419588555, + "pass@16": 0.9977897000591688, + "pass@32": 0.9983785444449502, + "pass@64": 0.9992831459651359, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9960607394366198, + "pass@2": 0.997819743262726, + "pass@4": 0.9991270069234303, + "pass@8": 0.9998443747974352, + "pass@16": 0.9999946157287308, + "pass@32": 0.9999999970491635, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9898016467065869, + "pass@2": 0.9924092513791315, + "pass@4": 0.9937748347511949, + "pass@8": 0.9950757224434577, + "pass@16": 0.9963683581836216, + "pass@32": 0.9978084671909977, + "pass@64": 0.9992160024103857, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9445359563253012, + "pass@2": 0.960735165069728, + "pass@4": 0.9692484578358429, + "pass@8": 0.9744137411819622, + "pass@16": 0.9774220287888713, + "pass@32": 0.9791276976341268, + "pass@64": 0.9803800331296995, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.8528605015673981, + "pass@2": 0.8893576598869498, + "pass@4": 0.9101586607120814, + "pass@8": 0.9224588859306808, + "pass@16": 0.9310670110053562, + "pass@32": 0.939270483161552, + "pass@64": 0.9473029142231101, + "pass@128": 0.9529780564263323 + }, + "13": { + "pass@1": 0.7229297969187675, + "pass@2": 0.7983782587838287, + "pass@4": 0.8442330002867283, + "pass@8": 0.8754003024489906, + "pass@16": 0.8995318657586917, + "pass@32": 0.9187236439009093, + "pass@64": 0.9338720887441335, + "pass@128": 0.9467787114845938 + }, + "14": { + "pass@1": 0.6482775950292398, + "pass@2": 0.7274618530413961, + "pass@4": 0.7836484260739333, + "pass@8": 0.8230514235910104, + "pass@16": 0.8486700010918876, + "pass@32": 0.8653014217564913, + "pass@64": 0.8789849352201726, + "pass@128": 0.8918128654970761 + }, + "15": { + "pass@1": 0.5319365530303031, + "pass@2": 0.6206174689811499, + "pass@4": 0.6868207306473046, + "pass@8": 0.7367855877874147, + "pass@16": 0.7752766588096213, + "pass@32": 0.806337169755552, + "pass@64": 0.8352289076502452, + "pass@128": 0.8606060606060606 + }, + "16": { + "pass@1": 0.32307247150997154, + "pass@2": 0.42357676043699655, + "pass@4": 0.5158068435889954, + "pass@8": 0.5908012273052607, + "pass@16": 0.6477279950868452, + "pass@32": 0.6922022409236969, + "pass@64": 0.7305376737607546, + "pass@128": 0.7663817663817664 + }, + "17": { + "pass@1": 0.3075504658385093, + "pass@2": 0.40160269538318555, + "pass@4": 0.48713984820220435, + "pass@8": 0.5596349682703199, + "pass@16": 0.6222102488736226, + "pass@32": 0.6744260839145514, + "pass@64": 0.7144136967050305, + "pass@128": 0.7484472049689441 + }, + "18": { + "pass@1": 0.25282628676470587, + "pass@2": 0.33365743689207966, + "pass@4": 0.4128379664858067, + "pass@8": 0.48697562456319365, + "pass@16": 0.552866663966737, + "pass@32": 0.6086135381508534, + "pass@64": 0.6542544109542544, + "pass@128": 0.6911764705882353 + }, + "19": { + "pass@1": 0.17978050595238096, + "pass@2": 0.24771110154199458, + "pass@4": 0.3194336779331155, + "pass@8": 0.386929726317321, + "pass@16": 0.446349013761704, + "pass@32": 0.5002298543667417, + "pass@64": 0.5475897643199032, + "pass@128": 0.5863095238095238 + }, + "20": { + "pass@1": 0.15585235778443113, + "pass@2": 0.2257480609646848, + "pass@4": 0.30009878920449323, + "pass@8": 0.3697365614620309, + "pass@16": 0.4289157795835793, + "pass@32": 0.47570921025814755, + "pass@64": 0.5129501839362879, + "pass@128": 0.5449101796407185 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581357, + "answer_accuracy": 0.7256513121105608, + "avg_response_len": 286.265858413884, + "resp_tokens_sum": 229342465, + "pass_at_k": { + "pass@1": 0.7256513121105608, + "pass@2": 0.7625552668723906, + "pass@4": 0.7936511745695828, + "pass@8": 0.8199106880345169, + "pass@16": 0.8416984025434961, + "pass@32": 0.859703225160211, + "pass@64": 0.8748839546594552, + "pass@128": 0.8881610480907494 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9540345149253732, + "pass@2": 0.9661670730990719, + "pass@4": 0.9737339595610252, + "pass@8": 0.9792955903839986, + "pass@16": 0.9827165338826096, + "pass@32": 0.9843298489148303, + "pass@64": 0.9849885152450082, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9884420955882353, + "pass@2": 0.9918386695229272, + "pass@4": 0.9951052884933501, + "pass@8": 0.9974676076953616, + "pass@16": 0.9986750762012426, + "pass@32": 0.9994972316545585, + "pass@64": 0.9999593420621558, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998274842271293, + "pass@2": 0.9999984475521002, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995302610759493, + "pass@2": 0.9999871517741453, + "pass@4": 0.9999999881343946, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9923986486486487, + "pass@2": 0.9960404548249037, + "pass@4": 0.9979970237391498, + "pass@8": 0.9993233078415068, + "pass@16": 0.9999099641930826, + "pass@32": 0.9999986991137875, + "pass@64": 0.9999999999875272, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9886706189710611, + "pass@2": 0.9933001240600552, + "pass@4": 0.9959347885975668, + "pass@8": 0.9979804126552496, + "pass@16": 0.999431623703106, + "pass@32": 0.9999655897612711, + "pass@64": 0.9999999801176399, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9533978174603175, + "pass@2": 0.9724440694913137, + "pass@4": 0.9814224727861399, + "pass@8": 0.9859808984729452, + "pass@16": 0.9886611665135104, + "pass@32": 0.9900384257613478, + "pass@64": 0.9904552883937954, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.8652522935779816, + "pass@2": 0.9085191221796822, + "pass@4": 0.9335249151768868, + "pass@8": 0.9493285515894258, + "pass@16": 0.9598282138920922, + "pass@32": 0.9673255156709447, + "pass@64": 0.9731453224447834, + "pass@128": 0.9785932721712538 + }, + "13": { + "pass@1": 0.6766734182098766, + "pass@2": 0.7418051150724216, + "pass@4": 0.7869648880695465, + "pass@8": 0.8206838073852996, + "pass@16": 0.8472708154414211, + "pass@32": 0.8691403156138889, + "pass@64": 0.8887575482814627, + "pass@128": 0.9104938271604939 + }, + "14": { + "pass@1": 0.6343657270029673, + "pass@2": 0.710647079955139, + "pass@4": 0.76384462420684, + "pass@8": 0.8053589968983749, + "pass@16": 0.8376984322875355, + "pass@32": 0.8624379829234405, + "pass@64": 0.8842262313961666, + "pass@128": 0.9050445103857567 + }, + "15": { + "pass@1": 0.5200858410493827, + "pass@2": 0.5983652000097209, + "pass@4": 0.660162098950594, + "pass@8": 0.7098186955100998, + "pass@16": 0.7494732224712638, + "pass@32": 0.782183749337275, + "pass@64": 0.8108561559010913, + "pass@128": 0.8333333333333334 + }, + "16": { + "pass@1": 0.3369284660766962, + "pass@2": 0.4336185196153577, + "pass@4": 0.5248042432195976, + "pass@8": 0.6073500575462162, + "pass@16": 0.6767348416897403, + "pass@32": 0.7298902304694688, + "pass@64": 0.7681849356854631, + "pass@128": 0.799410029498525 + }, + "17": { + "pass@1": 0.2506588855421687, + "pass@2": 0.3325241171378427, + "pass@4": 0.4093735848907438, + "pass@8": 0.47527240531329096, + "pass@16": 0.5328122572752206, + "pass@32": 0.5861045627721161, + "pass@64": 0.6353681803030131, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.26272681451612906, + "pass@2": 0.33898669672339343, + "pass@4": 0.40957589746846157, + "pass@8": 0.46941617022340476, + "pass@16": 0.516339879142077, + "pass@32": 0.5533118163588501, + "pass@64": 0.585479609144845, + "pass@128": 0.6161290322580645 + }, + "19": { + "pass@1": 0.19115842145015105, + "pass@2": 0.26258526714560976, + "pass@4": 0.3307034777752478, + "pass@8": 0.3926879790877576, + "pass@16": 0.44833979573114197, + "pass@32": 0.49538652663741045, + "pass@64": 0.5320829769264548, + "pass@128": 0.5619335347432024 + }, + "20": { + "pass@1": 0.1606371996996997, + "pass@2": 0.230762381475964, + "pass@4": 0.3060352743069278, + "pass@8": 0.3783404293240473, + "pass@16": 0.44421421827119956, + "pass@32": 0.5043447793446393, + "pass@64": 0.558844624078598, + "pass@128": 0.6036036036036037 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/checkpoint-42828_metrics.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/checkpoint-42828_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d13eb73104461c58ec0053ae063081d1488113a3 --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/checkpoint-42828_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-42828", + "id": { + "answer_accuracy": 0.7638064236111111, + "avg_response_len": 194.81892274305557, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.7638064236111111, + "pass@2": 0.8661242481408598, + "pass@4": 0.9357514556930356, + "pass@8": 0.9717067898069933, + "pass@16": 0.9868487021205053, + "pass@32": 0.9930371992473789, + "pass@64": 0.9960234337719428, + "pass@128": 0.9975555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.576046875, + "pass@2": 0.7446422244094478, + "pass@4": 0.868429268278965, + "pass@8": 0.9363018045335371, + "pass@16": 0.9657945219740212, + "pass@32": 0.9774312362222521, + "pass@64": 0.983277959536309, + "pass@128": 0.987 + }, + "2": { + "pass@1": 0.993375, + "pass@2": 0.9950381397637795, + "pass@4": 0.9964497278777653, + "pass@8": 0.9977168988758309, + "pass@16": 0.9987192176170508, + "pass@32": 0.9994457239751973, + "pass@64": 0.999911609813559, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9967265625, + "pass@2": 0.9980157480314962, + "pass@4": 0.9983646218597675, + "pass@8": 0.9986813607432419, + "pass@16": 0.9991295720608329, + "pass@32": 0.999620590510498, + "pass@64": 0.9999389962082194, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9175703125, + "pass@2": 0.9612010334645666, + "pass@4": 0.9835383036182974, + "pass@8": 0.9922453872595064, + "pass@16": 0.9954632512708923, + "pass@32": 0.9973113309439379, + "pass@64": 0.9985483811738365, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.7889453125, + "pass@2": 0.8904751476377938, + "pass@4": 0.9478738757967761, + "pass@8": 0.9733710237356877, + "pass@16": 0.9838328222600065, + "pass@32": 0.9896174313872915, + "pass@64": 0.9940076841983323, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.7451875, + "pass@2": 0.8649803149606291, + "pass@4": 0.9412136158605187, + "pass@8": 0.9774166083823232, + "pass@16": 0.9915435397530497, + "pass@32": 0.9967424179158043, + "pass@64": 0.9985807341070502, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.6773671875, + "pass@2": 0.8233677411417315, + "pass@4": 0.9168870138732652, + "pass@8": 0.9622512981360754, + "pass@16": 0.9816290232259682, + "pass@32": 0.9909437069667945, + "pass@64": 0.9959782455605222, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.626453125, + "pass@2": 0.7906162647637783, + "pass@4": 0.9066801547619057, + "pass@8": 0.9661962591578691, + "pass@16": 0.9892341158572059, + "pass@32": 0.9965846773764292, + "pass@64": 0.9988797194841242, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5525859375, + "pass@2": 0.726781619094487, + "pass@4": 0.8623265193100866, + "pass@8": 0.9411804674389743, + "pass@16": 0.9762922550654889, + "pass@32": 0.9896376779281962, + "pass@64": 0.995087573865492, + "pass@128": 0.998 + } + }, + "per_op_accuracy": { + "10": 0.576046875, + "2": 0.993375, + "3": 0.9967265625, + "4": 0.9175703125, + "5": 0.7889453125, + "6": 0.7451875, + "7": 0.6773671875, + "8": 0.626453125, + "9": 0.5525859375 + }, + "per_op_avg_response_len": { + "10": 273.33903125, + "2": 114.38990625, + "3": 142.7707890625, + "4": 157.1316015625, + "5": 170.9126875, + "6": 189.2881640625, + "7": 218.0550234375, + "8": 236.542890625, + "9": 250.9402109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.33903125, + "2": 114.38990625, + "3": 142.7707890625, + "4": 157.1316015625, + "5": 170.9126875, + "6": 189.2881640625, + "7": 218.0550234375, + "8": 236.542890625, + "9": 250.9402109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 290059, + "answer_accuracy": 0.7653110224586288, + "avg_response_len": 185.16307835190813, + "resp_tokens_sum": 70178288, + "pass_at_k": { + "pass@1": 0.7653110224586288, + "pass@2": 0.8674335006661428, + "pass@4": 0.9364385199697068, + "pass@8": 0.9720504149505815, + "pass@16": 0.9871926513633674, + "pass@32": 0.9933026265180284, + "pass@64": 0.9960397407080228, + "pass@128": 0.9972982100641675 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5961928466076696, + "pass@2": 0.7596196838780109, + "pass@4": 0.8756493018571788, + "pass@8": 0.9365228324321908, + "pass@16": 0.9606329559490022, + "pass@32": 0.9688884108408146, + "pass@64": 0.9737911492704813, + "pass@128": 0.9793510324483776 + }, + "2": { + "pass@1": 0.9976878894080997, + "pass@2": 0.9985906952927612, + "pass@4": 0.9993798701564174, + "pass@8": 0.9998843836043162, + "pass@16": 0.9999967738107939, + "pass@32": 0.9999999991592076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983514908256881, + "pass@2": 0.9998521364588602, + "pass@4": 0.9999973724614697, + "pass@8": 0.99999999928849, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9142578125, + "pass@2": 0.9533737543061023, + "pass@4": 0.9760740093035247, + "pass@8": 0.9864437669797045, + "pass@16": 0.9911445712028863, + "pass@32": 0.9944248711665763, + "pass@64": 0.9964253512379372, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.794890873015873, + "pass@2": 0.894600674915636, + "pass@4": 0.94974954797317, + "pass@8": 0.9755730371991035, + "pass@16": 0.9874647018460322, + "pass@32": 0.9937251962720715, + "pass@64": 0.9976983546461972, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.770279752994012, + "pass@2": 0.8859394891319728, + "pass@4": 0.9532805356665737, + "pass@8": 0.9830399819257302, + "pass@16": 0.9944878419475502, + "pass@32": 0.9986150659952652, + "pass@64": 0.999872031232074, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.6746486686390533, + "pass@2": 0.8198156420351302, + "pass@4": 0.9145076452166845, + "pass@8": 0.9619153600090276, + "pass@16": 0.9826155678869178, + "pass@32": 0.992917007771169, + "pass@64": 0.9985178232847783, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6353415464743589, + "pass@2": 0.7960370450989301, + "pass@4": 0.9076204095281363, + "pass@8": 0.9645666794146627, + "pass@16": 0.9884791332293995, + "pass@32": 0.9974799702985278, + "pass@64": 0.9998132509778702, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5373899647887324, + "pass@2": 0.7189194715537314, + "pass@4": 0.8618524614880887, + "pass@8": 0.9448509571689906, + "pass@16": 0.9817401961596698, + "pass@32": 0.9947024763761073, + "pass@64": 0.998955947935806, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 299897, + "answer_accuracy": 0.7676753972804718, + "avg_response_len": 192.93668854439713, + "resp_tokens_sum": 75371875, + "pass_at_k": { + "pass@1": 0.7676753972804718, + "pass@2": 0.8690829993369552, + "pass@4": 0.9370937943966886, + "pass@8": 0.9714742574722954, + "pass@16": 0.9856879061513478, + "pass@32": 0.9918636033163504, + "pass@64": 0.9954028685265931, + "pass@128": 0.9973787680209698 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5438075153374233, + "pass@2": 0.7155738249359938, + "pass@4": 0.8482569538853645, + "pass@8": 0.9247060702289117, + "pass@16": 0.9606764223030863, + "pass@32": 0.9775510243733617, + "pass@64": 0.9867887531474895, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.992686170212766, + "pass@2": 0.9948932580235983, + "pass@4": 0.996548313679635, + "pass@8": 0.9981614876492373, + "pass@16": 0.9994732641458764, + "pass@32": 0.9999644243028596, + "pass@64": 0.9999999628162317, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9937973484848485, + "pass@2": 0.9945269625387737, + "pass@4": 0.995079583233914, + "pass@8": 0.9960043744311183, + "pass@16": 0.997362339583687, + "pass@32": 0.998850274274236, + "pass@64": 0.9998151400249071, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.933385724852071, + "pass@2": 0.9726342280902018, + "pass@4": 0.9899115271463842, + "pass@8": 0.9961884271819588, + "pass@16": 0.9981460451848981, + "pass@32": 0.9990776110322851, + "pass@64": 0.999823789772943, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8190896739130434, + "pass@2": 0.9131440431359124, + "pass@4": 0.9604721708699447, + "pass@8": 0.9786022704945484, + "pass@16": 0.9854326450754342, + "pass@32": 0.9896840841129063, + "pass@64": 0.9937767545499087, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.7318230659025788, + "pass@2": 0.853087350247051, + "pass@4": 0.9329505317028773, + "pass@8": 0.9718729806164039, + "pass@16": 0.9874429073861202, + "pass@32": 0.9933592781253554, + "pass@64": 0.996150603505014, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.6908417630057804, + "pass@2": 0.8355122234991585, + "pass@4": 0.9259327110845249, + "pass@8": 0.9657232728447545, + "pass@16": 0.9799154383926086, + "pass@32": 0.987047671764001, + "pass@64": 0.9922131555149458, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.6376320422535211, + "pass@2": 0.8039948846623045, + "pass@4": 0.9164696530891375, + "pass@8": 0.969848565415213, + "pass@16": 0.9884038113446275, + "pass@32": 0.9941602181010866, + "pass@64": 0.9971833475810209, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5755988023952096, + "pass@2": 0.7414146853929929, + "pass@4": 0.8681606595582738, + "pass@8": 0.941494152221592, + "pass@16": 0.9739024093533677, + "pass@32": 0.9868868137718635, + "pass@64": 0.9927593268163736, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 289949, + "answer_accuracy": 0.758361755105457, + "avg_response_len": 206.31391237027117, + "resp_tokens_sum": 78881236, + "pass_at_k": { + "pass@1": 0.758361755105457, + "pass@2": 0.8618032553796139, + "pass@4": 0.9336988226007271, + "pass@8": 0.9716037481717249, + "pass@16": 0.9876938027531671, + "pass@32": 0.9939732168681006, + "pass@64": 0.9966413381211285, + "pass@128": 0.9979912956143288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5870335820895523, + "pass@2": 0.7577733870019981, + "pass@4": 0.880753396963439, + "pass@8": 0.9473623446101491, + "pass@16": 0.9759983171238966, + "pass@32": 0.9859594955268647, + "pass@64": 0.9894615713657725, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9900669642857143, + "pass@2": 0.991916127671541, + "pass@4": 0.9936696981627297, + "pass@8": 0.9953110922921324, + "pass@16": 0.9968387123422647, + "pass@32": 0.9984497961412884, + "pass@64": 0.9997474915629107, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9979956268221575, + "pass@2": 0.999621579509194, + "pass@4": 0.999968567340161, + "pass@8": 0.99999975922343, + "pass@16": 0.9999999999947993, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9050392909356725, + "pass@2": 0.9572253303863328, + "pass@4": 0.9842237557476368, + "pass@8": 0.9937768813991182, + "pass@16": 0.9968527053025525, + "pass@32": 0.9982664901804696, + "pass@64": 0.9992743503931047, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7528492647058823, + "pass@2": 0.863650706345531, + "pass@4": 0.9333525565738104, + "pass@8": 0.9660227461686778, + "pass@16": 0.9788446430220072, + "pass@32": 0.9857440751253994, + "pass@64": 0.9908227122501815, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.7334631309148265, + "pass@2": 0.85599066357833, + "pass@4": 0.9375967866989801, + "pass@8": 0.9775949027886589, + "pass@16": 0.992955917554611, + "pass@32": 0.9984939993931751, + "pass@64": 0.9998956310797713, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.6655211629746836, + "pass@2": 0.8138696987441444, + "pass@4": 0.9095275688442417, + "pass@8": 0.9588090317998087, + "pass@16": 0.9824500620137617, + "pass@32": 0.993098936423399, + "pass@64": 0.9973843970952397, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6062077702702703, + "pass@2": 0.7712748427561416, + "pass@4": 0.8953629433708176, + "pass@8": 0.9638294728471273, + "pass@16": 0.9908266464333062, + "pass@32": 0.9983304781303394, + "pass@64": 0.9998135038671645, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5452170418006431, + "pass@2": 0.7200408417094971, + "pass@4": 0.8566020424094902, + "pass@8": 0.9366537967909059, + "pass@16": 0.9726401308835425, + "pass@32": 0.9868106206266033, + "pass@64": 0.993172338236693, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.26778984375, + "avg_response_len": 329.1969828125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.26778984375, + "pass@2": 0.38627690698819594, + "pass@4": 0.5058358295650495, + "pass@8": 0.6088323620178936, + "pass@16": 0.6877922888891399, + "pass@32": 0.7459494502149341, + "pass@64": 0.7906824528742201, + "pass@128": 0.8274 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4993671875, + "pass@2": 0.6709565698818899, + "pass@4": 0.812126751968504, + "pass@8": 0.9048149163498116, + "pass@16": 0.9543744542602351, + "pass@32": 0.9764558964190443, + "pass@64": 0.9854394494253862, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.432859375, + "pass@2": 0.6020659448818889, + "pass@4": 0.7518307778402696, + "pass@8": 0.858328467080128, + "pass@16": 0.9203534910477849, + "pass@32": 0.9528836641976156, + "pass@64": 0.9700422665939142, + "pass@128": 0.98 + }, + "13": { + "pass@1": 0.358328125, + "pass@2": 0.5068014271653548, + "pass@4": 0.6499801903824524, + "pass@8": 0.7665844993592499, + "pass@16": 0.8464375312854919, + "pass@32": 0.8955056932273469, + "pass@64": 0.9264540088812818, + "pass@128": 0.948 + }, + "14": { + "pass@1": 0.31334375, + "pass@2": 0.4546761811023625, + "pass@4": 0.5970082288151483, + "pass@8": 0.7164840994233646, + "pass@16": 0.8028474777370648, + "pass@32": 0.860503981485152, + "pass@64": 0.9001973957184077, + "pass@128": 0.933 + }, + "15": { + "pass@1": 0.2424765625, + "pass@2": 0.36315243602362246, + "pass@4": 0.4933400735845513, + "pass@8": 0.6128165801382486, + "pass@16": 0.7098067839815075, + "pass@32": 0.7826892275286861, + "pass@64": 0.835406118391729, + "pass@128": 0.874 + }, + "16": { + "pass@1": 0.2068046875, + "pass@2": 0.3097512303149608, + "pass@4": 0.4220277742782148, + "pass@8": 0.5286350534590336, + "pass@16": 0.622969998355859, + "pass@32": 0.7059370467967937, + "pass@64": 0.7792968114066574, + "pass@128": 0.84 + }, + "17": { + "pass@1": 0.1795625, + "pass@2": 0.2758592519685042, + "pass@4": 0.3841129922197223, + "pass@8": 0.4865529977293074, + "pass@16": 0.5724467022703361, + "pass@32": 0.6425452462892906, + "pass@64": 0.7020462088224163, + "pass@128": 0.754 + }, + "18": { + "pass@1": 0.1645546875, + "pass@2": 0.2505367864173227, + "pass@4": 0.34793365063742043, + "pass@8": 0.4433056314972349, + "pass@16": 0.5269794941315902, + "pass@32": 0.597290354517864, + "pass@64": 0.6571631471849959, + "pass@128": 0.709 + }, + "19": { + "pass@1": 0.14834375, + "pass@2": 0.225490280511811, + "pass@4": 0.31258315194975617, + "pass@8": 0.3972803173380285, + "pass@16": 0.47102807244583395, + "pass@32": 0.5322369052965162, + "pass@64": 0.5837977418712871, + "pass@128": 0.629 + }, + "20": { + "pass@1": 0.1322578125, + "pass@2": 0.20347896161417325, + "pass@4": 0.287414703974503, + "pass@8": 0.37352105780445216, + "pass@16": 0.45067888337547946, + "pass@32": 0.5134464863910018, + "pass@64": 0.5669813804460146, + "pass@128": 0.617 + } + }, + "per_op_accuracy": { + "11": 0.4993671875, + "12": 0.432859375, + "13": 0.358328125, + "14": 0.31334375, + "15": 0.2424765625, + "16": 0.2068046875, + "17": 0.1795625, + "18": 0.1645546875, + "19": 0.14834375, + "20": 0.1322578125 + }, + "per_op_avg_response_len": { + "11": 289.572828125, + "12": 299.9657265625, + "13": 315.501671875, + "14": 330.3920625, + "15": 335.3429453125, + "16": 339.6034765625, + "17": 347.273046875, + "18": 346.6276171875, + "19": 340.3072734375, + "20": 347.3831796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 289.572828125, + "12": 299.9657265625, + "13": 315.501671875, + "14": 330.3920625, + "15": 335.3429453125, + "16": 339.6034765625, + "17": 347.273046875, + "18": 346.6276171875, + "19": 340.3072734375, + "20": 347.3831796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 116888, + "answer_accuracy": 0.2713781575037147, + "avg_response_len": 311.5246099554235, + "resp_tokens_sum": 134179880, + "pass_at_k": { + "pass@1": 0.2713781575037147, + "pass@2": 0.3902723950228745, + "pass@4": 0.5084496030678227, + "pass@8": 0.6088428585505664, + "pass@16": 0.6855959020736752, + "pass@32": 0.7425598187879399, + "pass@64": 0.7869272607766493, + "pass@128": 0.8237741456166419 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5171963526912181, + "pass@2": 0.6901484882112825, + "pass@4": 0.8268923876548013, + "pass@8": 0.9124617472551908, + "pass@16": 0.9559764167265765, + "pass@32": 0.9757553214879822, + "pass@64": 0.9844442937777405, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.435204802259887, + "pass@2": 0.6030044708394502, + "pass@4": 0.748447401966703, + "pass@8": 0.8497352014720185, + "pass@16": 0.9089616085075132, + "pass@32": 0.9431542535728017, + "pass@64": 0.9651132899120077, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.3713019200626959, + "pass@2": 0.523809101399551, + "pass@4": 0.6686971683398507, + "pass@8": 0.7846809994799563, + "pass@16": 0.8617996533225794, + "pass@32": 0.9064250879752345, + "pass@64": 0.9331490503349423, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.30415206386292837, + "pass@2": 0.44828767753329857, + "pass@4": 0.5943040942896155, + "pass@8": 0.716445667438578, + "pass@16": 0.8051037312119722, + "pass@32": 0.8657429264678448, + "pass@64": 0.9090951189790747, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.24954841040462428, + "pass@2": 0.37298704826817164, + "pass@4": 0.5029955495989294, + "pass@8": 0.6201905378692797, + "pass@16": 0.7157051200779855, + "pass@32": 0.7883710262863396, + "pass@64": 0.8416705019935963, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.21121471774193548, + "pass@2": 0.3187492062484124, + "pass@4": 0.4356203729574124, + "pass@8": 0.545153874059209, + "pass@16": 0.6407560148339182, + "pass@32": 0.7208517883776376, + "pass@64": 0.7881858215754862, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.17341040462427745, + "pass@2": 0.26089931045468995, + "pass@4": 0.35955679374976973, + "pass@8": 0.45545195127969673, + "pass@16": 0.5399904831688518, + "pass@32": 0.6110066282578624, + "pass@64": 0.6704129618632997, + "pass@128": 0.7225433526011561 + }, + "18": { + "pass@1": 0.15582589285714285, + "pass@2": 0.23676708380202446, + "pass@4": 0.3290234993840056, + "pass@8": 0.42050899990842666, + "pass@16": 0.5023571496853358, + "pass@32": 0.5734286017598653, + "pass@64": 0.6377678650847342, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.14161036036036037, + "pass@2": 0.22125718237922962, + "pass@4": 0.3123289304490093, + "pass@8": 0.3986272218604655, + "pass@16": 0.47052017433388255, + "pass@32": 0.527380938890095, + "pass@64": 0.5741233473287567, + "pass@128": 0.6156156156156156 + }, + "20": { + "pass@1": 0.14102383633633633, + "pass@2": 0.21171503688728094, + "pass@4": 0.2923553308651733, + "pass@8": 0.37377671407570623, + "pass@16": 0.4469443695731371, + "pass@32": 0.5086362984246378, + "pass@64": 0.5628735775041814, + "pass@128": 0.6126126126126126 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 115204, + "answer_accuracy": 0.2676274903360095, + "avg_response_len": 326.81290653806127, + "resp_tokens_sum": 140681191, + "pass_at_k": { + "pass@1": 0.2676274903360095, + "pass@2": 0.38766536633021326, + "pass@4": 0.5099157097445827, + "pass@8": 0.61549870202102, + "pass@16": 0.6952147101095503, + "pass@32": 0.7526077179643443, + "pass@64": 0.7969522875247215, + "pass@128": 0.8343740707701457 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4614787274096386, + "pass@2": 0.6289481251778768, + "pass@4": 0.7768584167942858, + "pass@8": 0.8823223386877882, + "pass@16": 0.9427012275107226, + "pass@32": 0.9688437100059571, + "pass@64": 0.9783544805563124, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.4521698667711599, + "pass@2": 0.6230932046503593, + "pass@4": 0.7703201394496527, + "pass@8": 0.8716570043484047, + "pass@16": 0.9280077925551187, + "pass@32": 0.9550018756047801, + "pass@64": 0.9660625310485854, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.3567270658263305, + "pass@2": 0.5099872626215837, + "pass@4": 0.6578685560838504, + "pass@8": 0.7772300365473884, + "pass@16": 0.8555928488258188, + "pass@32": 0.9011742109700471, + "pass@64": 0.9303704417034507, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.3323967470760234, + "pass@2": 0.4757476861444953, + "pass@4": 0.6150902066518003, + "pass@8": 0.7283701824607279, + "pass@16": 0.8069417810438068, + "pass@32": 0.8571822437247654, + "pass@64": 0.8945694038536045, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.2434185606060606, + "pass@2": 0.36151984908136464, + "pass@4": 0.4901963163695448, + "pass@8": 0.6128158772102645, + "pass@16": 0.7157703886521702, + "pass@32": 0.7941160880333268, + "pass@64": 0.8492354664830497, + "pass@128": 0.8848484848484849 + }, + "16": { + "pass@1": 0.1988292378917379, + "pass@2": 0.30013200473338286, + "pass@4": 0.4110917996895686, + "pass@8": 0.5151463559434297, + "pass@16": 0.6051339051747506, + "pass@32": 0.6838217166573823, + "pass@64": 0.7560222940461221, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.1983695652173913, + "pass@2": 0.30658913899349527, + "pass@4": 0.42618183159946577, + "pass@8": 0.5340680663265758, + "pass@16": 0.618370732659288, + "pass@32": 0.6845467437656324, + "pass@64": 0.7440009642404236, + "pass@128": 0.8012422360248447 + }, + "18": { + "pass@1": 0.16353400735294119, + "pass@2": 0.25522702929597024, + "pass@4": 0.3619508745230373, + "pass@8": 0.46639634535611824, + "pass@16": 0.5540833993978276, + "pass@32": 0.6223032123495336, + "pass@64": 0.6775643958034884, + "pass@128": 0.7264705882352941 + }, + "19": { + "pass@1": 0.14448474702380953, + "pass@2": 0.21798508096644167, + "pass@4": 0.30426176778125946, + "pass@8": 0.39128929215247166, + "pass@16": 0.4700711724164494, + "pass@32": 0.5399918946084133, + "pass@64": 0.6015360772969983, + "pass@128": 0.6547619047619048 + }, + "20": { + "pass@1": 0.12995883233532934, + "pass@2": 0.20353350876986184, + "pass@4": 0.29159784620859525, + "pass@8": 0.3811237163449579, + "pass@16": 0.45964420951180207, + "pass@32": 0.5219112961870748, + "pass@64": 0.5730840359592941, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 110679, + "answer_accuracy": 0.2642664081601467, + "avg_response_len": 349.82203879507944, + "resp_tokens_sum": 146511067, + "pass_at_k": { + "pass@1": 0.2642664081601467, + "pass@2": 0.38074078045886883, + "pass@4": 0.49895441609301033, + "pass@8": 0.6019698246514387, + "pass@16": 0.6824222519299701, + "pass@32": 0.7425919793440597, + "pass@64": 0.7881001690962655, + "pass@128": 0.823960880195599 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5193204365079365, + "pass@2": 0.6937250031246089, + "pass@4": 0.8327515706370029, + "pass@8": 0.9199520734107461, + "pass@16": 0.9648824495942682, + "pass@32": 0.9852639879739948, + "pass@64": 0.9940219878639607, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.4114822247706422, + "pass@2": 0.5805370945604271, + "pass@4": 0.7374564925370567, + "pass@8": 0.8546288115348385, + "pass@16": 0.9252189474343845, + "pass@32": 0.9613500309324735, + "pass@64": 0.9792605949864354, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.3473186728395062, + "pass@2": 0.48654589834256845, + "pass@4": 0.6228602443213108, + "pass@8": 0.7370374613510134, + "pass@16": 0.8212246754468296, + "pass@32": 0.8785089408856174, + "pass@64": 0.9155469572108085, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.302763353115727, + "pass@2": 0.4393772342811747, + "pass@4": 0.5812337206921829, + "pass@8": 0.7044582723264429, + "pass@16": 0.7965432964422573, + "pass@32": 0.8588847914988257, + "pass@64": 0.8974336095198548, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.23396508487654322, + "pass@2": 0.3543128614999515, + "pass@4": 0.4862309537696679, + "pass@8": 0.6049426375187965, + "pass@16": 0.6974339017879962, + "pass@32": 0.7649831585883168, + "pass@64": 0.8146309282794365, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.21102968289085547, + "pass@2": 0.31148272187768566, + "pass@4": 0.42092105300996674, + "pass@8": 0.5274955208393359, + "pass@16": 0.6251729588230321, + "pass@32": 0.7151963712477419, + "pass@64": 0.7952666121181906, + "pass@128": 0.855457227138643 + }, + "17": { + "pass@1": 0.16773343373493976, + "pass@2": 0.2616457463713118, + "pass@4": 0.3689029873713071, + "pass@8": 0.4728816422571547, + "pass@16": 0.5617307806554981, + "pass@32": 0.6346774139142679, + "pass@64": 0.6943223299165555, + "pass@128": 0.7409638554216867 + }, + "18": { + "pass@1": 0.17552923387096775, + "pass@2": 0.26093908750317496, + "pass@4": 0.35391009198446943, + "pass@8": 0.4437184648651771, + "pass@16": 0.5250520514402012, + "pass@32": 0.5967975861389322, + "pass@64": 0.6566854833295221, + "pass@128": 0.7032258064516129 + }, + "19": { + "pass@1": 0.15903512084592145, + "pass@2": 0.2373675274163237, + "pass@4": 0.32128599436777355, + "pass@8": 0.40200679847511533, + "pass@16": 0.47251039414116286, + "pass@32": 0.5292500787241331, + "pass@64": 0.575524305858063, + "pass@128": 0.6163141993957704 + }, + "20": { + "pass@1": 0.12579767267267267, + "pass@2": 0.1951881753801045, + "pass@4": 0.2782783728610501, + "pass@8": 0.36563991218626535, + "pass@16": 0.4454211481401893, + "pass@32": 0.5097664447120562, + "pass@64": 0.5649682015817358, + "pass@128": 0.6246246246246246 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5027450657894736, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5027450657894736, + "pass@2": 0.6135730159552273, + "pass@4": 0.709480073520385, + "pass@8": 0.780720248865373, + "pass@16": 0.8294505898934772, + "pass@32": 0.8629910155461137, + "pass@64": 0.8879492332994562, + "pass@128": 0.908 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.576046875, + "pass@2": 0.7446422244094478, + "pass@4": 0.868429268278965, + "pass@8": 0.9363018045335371, + "pass@16": 0.9657945219740212, + "pass@32": 0.9774312362222521, + "pass@64": 0.983277959536309, + "pass@128": 0.987 + }, + "2": { + "pass@1": 0.993375, + "pass@2": 0.9950381397637795, + "pass@4": 0.9964497278777653, + "pass@8": 0.9977168988758309, + "pass@16": 0.9987192176170508, + "pass@32": 0.9994457239751973, + "pass@64": 0.999911609813559, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9967265625, + "pass@2": 0.9980157480314962, + "pass@4": 0.9983646218597675, + "pass@8": 0.9986813607432419, + "pass@16": 0.9991295720608329, + "pass@32": 0.999620590510498, + "pass@64": 0.9999389962082194, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9175703125, + "pass@2": 0.9612010334645666, + "pass@4": 0.9835383036182974, + "pass@8": 0.9922453872595064, + "pass@16": 0.9954632512708923, + "pass@32": 0.9973113309439379, + "pass@64": 0.9985483811738365, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.7889453125, + "pass@2": 0.8904751476377938, + "pass@4": 0.9478738757967761, + "pass@8": 0.9733710237356877, + "pass@16": 0.9838328222600065, + "pass@32": 0.9896174313872915, + "pass@64": 0.9940076841983323, + "pass@128": 0.997 + }, + "6": { + "pass@1": 0.7451875, + "pass@2": 0.8649803149606291, + "pass@4": 0.9412136158605187, + "pass@8": 0.9774166083823232, + "pass@16": 0.9915435397530497, + "pass@32": 0.9967424179158043, + "pass@64": 0.9985807341070502, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.6773671875, + "pass@2": 0.8233677411417315, + "pass@4": 0.9168870138732652, + "pass@8": 0.9622512981360754, + "pass@16": 0.9816290232259682, + "pass@32": 0.9909437069667945, + "pass@64": 0.9959782455605222, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.626453125, + "pass@2": 0.7906162647637783, + "pass@4": 0.9066801547619057, + "pass@8": 0.9661962591578691, + "pass@16": 0.9892341158572059, + "pass@32": 0.9965846773764292, + "pass@64": 0.9988797194841242, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5525859375, + "pass@2": 0.726781619094487, + "pass@4": 0.8623265193100866, + "pass@8": 0.9411804674389743, + "pass@16": 0.9762922550654889, + "pass@32": 0.9896376779281962, + "pass@64": 0.995087573865492, + "pass@128": 0.998 + }, + "11": { + "pass@1": 0.4993671875, + "pass@2": 0.6709565698818899, + "pass@4": 0.812126751968504, + "pass@8": 0.9048149163498116, + "pass@16": 0.9543744542602351, + "pass@32": 0.9764558964190443, + "pass@64": 0.9854394494253862, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.432859375, + "pass@2": 0.6020659448818889, + "pass@4": 0.7518307778402696, + "pass@8": 0.858328467080128, + "pass@16": 0.9203534910477849, + "pass@32": 0.9528836641976156, + "pass@64": 0.9700422665939142, + "pass@128": 0.98 + }, + "13": { + "pass@1": 0.358328125, + "pass@2": 0.5068014271653548, + "pass@4": 0.6499801903824524, + "pass@8": 0.7665844993592499, + "pass@16": 0.8464375312854919, + "pass@32": 0.8955056932273469, + "pass@64": 0.9264540088812818, + "pass@128": 0.948 + }, + "14": { + "pass@1": 0.31334375, + "pass@2": 0.4546761811023625, + "pass@4": 0.5970082288151483, + "pass@8": 0.7164840994233646, + "pass@16": 0.8028474777370648, + "pass@32": 0.860503981485152, + "pass@64": 0.9001973957184077, + "pass@128": 0.933 + }, + "15": { + "pass@1": 0.2424765625, + "pass@2": 0.36315243602362246, + "pass@4": 0.4933400735845513, + "pass@8": 0.6128165801382486, + "pass@16": 0.7098067839815075, + "pass@32": 0.7826892275286861, + "pass@64": 0.835406118391729, + "pass@128": 0.874 + }, + "16": { + "pass@1": 0.2068046875, + "pass@2": 0.3097512303149608, + "pass@4": 0.4220277742782148, + "pass@8": 0.5286350534590336, + "pass@16": 0.622969998355859, + "pass@32": 0.7059370467967937, + "pass@64": 0.7792968114066574, + "pass@128": 0.84 + }, + "17": { + "pass@1": 0.1795625, + "pass@2": 0.2758592519685042, + "pass@4": 0.3841129922197223, + "pass@8": 0.4865529977293074, + "pass@16": 0.5724467022703361, + "pass@32": 0.6425452462892906, + "pass@64": 0.7020462088224163, + "pass@128": 0.754 + }, + "18": { + "pass@1": 0.1645546875, + "pass@2": 0.2505367864173227, + "pass@4": 0.34793365063742043, + "pass@8": 0.4433056314972349, + "pass@16": 0.5269794941315902, + "pass@32": 0.597290354517864, + "pass@64": 0.6571631471849959, + "pass@128": 0.709 + }, + "19": { + "pass@1": 0.14834375, + "pass@2": 0.225490280511811, + "pass@4": 0.31258315194975617, + "pass@8": 0.3972803173380285, + "pass@16": 0.47102807244583395, + "pass@32": 0.5322369052965162, + "pass@64": 0.5837977418712871, + "pass@128": 0.629 + }, + "20": { + "pass@1": 0.1322578125, + "pass@2": 0.20347896161417325, + "pass@4": 0.287414703974503, + "pass@8": 0.37352105780445216, + "pass@16": 0.45067888337547946, + "pass@32": 0.5134464863910018, + "pass@64": 0.5669813804460146, + "pass@128": 0.617 + } + }, + "per_op_accuracy": { + "10": 0.576046875, + "2": 0.993375, + "3": 0.9967265625, + "4": 0.9175703125, + "5": 0.7889453125, + "6": 0.7451875, + "7": 0.6773671875, + "8": 0.626453125, + "9": 0.5525859375, + "11": 0.4993671875, + "12": 0.432859375, + "13": 0.358328125, + "14": 0.31334375, + "15": 0.2424765625, + "16": 0.2068046875, + "17": 0.1795625, + "18": 0.1645546875, + "19": 0.14834375, + "20": 0.1322578125 + }, + "per_op_avg_response_len": { + "10": 273.33903125, + "2": 114.38990625, + "3": 142.7707890625, + "4": 157.1316015625, + "5": 170.9126875, + "6": 189.2881640625, + "7": 218.0550234375, + "8": 236.542890625, + "9": 250.9402109375, + "11": 289.572828125, + "12": 299.9657265625, + "13": 315.501671875, + "14": 330.3920625, + "15": 335.3429453125, + "16": 339.6034765625, + "17": 347.273046875, + "18": 346.6276171875, + "19": 340.3072734375, + "20": 347.3831796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.33903125, + "2": 114.38990625, + "3": 142.7707890625, + "4": 157.1316015625, + "5": 170.9126875, + "6": 189.2881640625, + "7": 218.0550234375, + "8": 236.542890625, + "9": 250.9402109375, + "11": 289.572828125, + "12": 299.9657265625, + "13": 315.501671875, + "14": 330.3920625, + "15": 335.3429453125, + "16": 339.6034765625, + "17": 347.273046875, + "18": 346.6276171875, + "19": 340.3072734375, + "20": 347.3831796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 406947, + "answer_accuracy": 0.5025724687796396, + "avg_response_len": 252.37878398672146, + "resp_tokens_sum": 204358168, + "pass_at_k": { + "pass@1": 0.5025724687796396, + "pass@2": 0.6136163776042443, + "pass@4": 0.7087776433691901, + "pass@8": 0.778848798243971, + "pass@16": 0.8267637766621762, + "pass@32": 0.8599245759312861, + "pass@64": 0.8848061815918338, + "pass@128": 0.904995257666772 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5961928466076696, + "pass@2": 0.7596196838780109, + "pass@4": 0.8756493018571788, + "pass@8": 0.9365228324321908, + "pass@16": 0.9606329559490022, + "pass@32": 0.9688884108408146, + "pass@64": 0.9737911492704813, + "pass@128": 0.9793510324483776 + }, + "2": { + "pass@1": 0.9976878894080997, + "pass@2": 0.9985906952927612, + "pass@4": 0.9993798701564174, + "pass@8": 0.9998843836043162, + "pass@16": 0.9999967738107939, + "pass@32": 0.9999999991592076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983514908256881, + "pass@2": 0.9998521364588602, + "pass@4": 0.9999973724614697, + "pass@8": 0.99999999928849, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9142578125, + "pass@2": 0.9533737543061023, + "pass@4": 0.9760740093035247, + "pass@8": 0.9864437669797045, + "pass@16": 0.9911445712028863, + "pass@32": 0.9944248711665763, + "pass@64": 0.9964253512379372, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.794890873015873, + "pass@2": 0.894600674915636, + "pass@4": 0.94974954797317, + "pass@8": 0.9755730371991035, + "pass@16": 0.9874647018460322, + "pass@32": 0.9937251962720715, + "pass@64": 0.9976983546461972, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.770279752994012, + "pass@2": 0.8859394891319728, + "pass@4": 0.9532805356665737, + "pass@8": 0.9830399819257302, + "pass@16": 0.9944878419475502, + "pass@32": 0.9986150659952652, + "pass@64": 0.999872031232074, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.6746486686390533, + "pass@2": 0.8198156420351302, + "pass@4": 0.9145076452166845, + "pass@8": 0.9619153600090276, + "pass@16": 0.9826155678869178, + "pass@32": 0.992917007771169, + "pass@64": 0.9985178232847783, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6353415464743589, + "pass@2": 0.7960370450989301, + "pass@4": 0.9076204095281363, + "pass@8": 0.9645666794146627, + "pass@16": 0.9884791332293995, + "pass@32": 0.9974799702985278, + "pass@64": 0.9998132509778702, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5373899647887324, + "pass@2": 0.7189194715537314, + "pass@4": 0.8618524614880887, + "pass@8": 0.9448509571689906, + "pass@16": 0.9817401961596698, + "pass@32": 0.9947024763761073, + "pass@64": 0.998955947935806, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.5171963526912181, + "pass@2": 0.6901484882112825, + "pass@4": 0.8268923876548013, + "pass@8": 0.9124617472551908, + "pass@16": 0.9559764167265765, + "pass@32": 0.9757553214879822, + "pass@64": 0.9844442937777405, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.435204802259887, + "pass@2": 0.6030044708394502, + "pass@4": 0.748447401966703, + "pass@8": 0.8497352014720185, + "pass@16": 0.9089616085075132, + "pass@32": 0.9431542535728017, + "pass@64": 0.9651132899120077, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.3713019200626959, + "pass@2": 0.523809101399551, + "pass@4": 0.6686971683398507, + "pass@8": 0.7846809994799563, + "pass@16": 0.8617996533225794, + "pass@32": 0.9064250879752345, + "pass@64": 0.9331490503349423, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.30415206386292837, + "pass@2": 0.44828767753329857, + "pass@4": 0.5943040942896155, + "pass@8": 0.716445667438578, + "pass@16": 0.8051037312119722, + "pass@32": 0.8657429264678448, + "pass@64": 0.9090951189790747, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.24954841040462428, + "pass@2": 0.37298704826817164, + "pass@4": 0.5029955495989294, + "pass@8": 0.6201905378692797, + "pass@16": 0.7157051200779855, + "pass@32": 0.7883710262863396, + "pass@64": 0.8416705019935963, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.21121471774193548, + "pass@2": 0.3187492062484124, + "pass@4": 0.4356203729574124, + "pass@8": 0.545153874059209, + "pass@16": 0.6407560148339182, + "pass@32": 0.7208517883776376, + "pass@64": 0.7881858215754862, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.17341040462427745, + "pass@2": 0.26089931045468995, + "pass@4": 0.35955679374976973, + "pass@8": 0.45545195127969673, + "pass@16": 0.5399904831688518, + "pass@32": 0.6110066282578624, + "pass@64": 0.6704129618632997, + "pass@128": 0.7225433526011561 + }, + "18": { + "pass@1": 0.15582589285714285, + "pass@2": 0.23676708380202446, + "pass@4": 0.3290234993840056, + "pass@8": 0.42050899990842666, + "pass@16": 0.5023571496853358, + "pass@32": 0.5734286017598653, + "pass@64": 0.6377678650847342, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.14161036036036037, + "pass@2": 0.22125718237922962, + "pass@4": 0.3123289304490093, + "pass@8": 0.3986272218604655, + "pass@16": 0.47052017433388255, + "pass@32": 0.527380938890095, + "pass@64": 0.5741233473287567, + "pass@128": 0.6156156156156156 + }, + "20": { + "pass@1": 0.14102383633633633, + "pass@2": 0.21171503688728094, + "pass@4": 0.2923553308651733, + "pass@8": 0.37377671407570623, + "pass@16": 0.4469443695731371, + "pass@32": 0.5086362984246378, + "pass@64": 0.5628735775041814, + "pass@128": 0.6126126126126126 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 415101, + "answer_accuracy": 0.5055302513639907, + "avg_response_len": 263.1199654130943, + "resp_tokens_sum": 216053066, + "pass_at_k": { + "pass@1": 0.5055302513639907, + "pass@2": 0.6167045893912595, + "pass@4": 0.7131499286624569, + "pass@8": 0.7848576100860732, + "pass@16": 0.8334102197462872, + "pass@32": 0.8664360830608916, + "pass@64": 0.8913671235680202, + "pass@128": 0.911925175370226 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5438075153374233, + "pass@2": 0.7155738249359938, + "pass@4": 0.8482569538853645, + "pass@8": 0.9247060702289117, + "pass@16": 0.9606764223030863, + "pass@32": 0.9775510243733617, + "pass@64": 0.9867887531474895, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.992686170212766, + "pass@2": 0.9948932580235983, + "pass@4": 0.996548313679635, + "pass@8": 0.9981614876492373, + "pass@16": 0.9994732641458764, + "pass@32": 0.9999644243028596, + "pass@64": 0.9999999628162317, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9937973484848485, + "pass@2": 0.9945269625387737, + "pass@4": 0.995079583233914, + "pass@8": 0.9960043744311183, + "pass@16": 0.997362339583687, + "pass@32": 0.998850274274236, + "pass@64": 0.9998151400249071, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.933385724852071, + "pass@2": 0.9726342280902018, + "pass@4": 0.9899115271463842, + "pass@8": 0.9961884271819588, + "pass@16": 0.9981460451848981, + "pass@32": 0.9990776110322851, + "pass@64": 0.999823789772943, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8190896739130434, + "pass@2": 0.9131440431359124, + "pass@4": 0.9604721708699447, + "pass@8": 0.9786022704945484, + "pass@16": 0.9854326450754342, + "pass@32": 0.9896840841129063, + "pass@64": 0.9937767545499087, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.7318230659025788, + "pass@2": 0.853087350247051, + "pass@4": 0.9329505317028773, + "pass@8": 0.9718729806164039, + "pass@16": 0.9874429073861202, + "pass@32": 0.9933592781253554, + "pass@64": 0.996150603505014, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.6908417630057804, + "pass@2": 0.8355122234991585, + "pass@4": 0.9259327110845249, + "pass@8": 0.9657232728447545, + "pass@16": 0.9799154383926086, + "pass@32": 0.987047671764001, + "pass@64": 0.9922131555149458, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.6376320422535211, + "pass@2": 0.8039948846623045, + "pass@4": 0.9164696530891375, + "pass@8": 0.969848565415213, + "pass@16": 0.9884038113446275, + "pass@32": 0.9941602181010866, + "pass@64": 0.9971833475810209, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5755988023952096, + "pass@2": 0.7414146853929929, + "pass@4": 0.8681606595582738, + "pass@8": 0.941494152221592, + "pass@16": 0.9739024093533677, + "pass@32": 0.9868868137718635, + "pass@64": 0.9927593268163736, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.4614787274096386, + "pass@2": 0.6289481251778768, + "pass@4": 0.7768584167942858, + "pass@8": 0.8823223386877882, + "pass@16": 0.9427012275107226, + "pass@32": 0.9688437100059571, + "pass@64": 0.9783544805563124, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.4521698667711599, + "pass@2": 0.6230932046503593, + "pass@4": 0.7703201394496527, + "pass@8": 0.8716570043484047, + "pass@16": 0.9280077925551187, + "pass@32": 0.9550018756047801, + "pass@64": 0.9660625310485854, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.3567270658263305, + "pass@2": 0.5099872626215837, + "pass@4": 0.6578685560838504, + "pass@8": 0.7772300365473884, + "pass@16": 0.8555928488258188, + "pass@32": 0.9011742109700471, + "pass@64": 0.9303704417034507, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.3323967470760234, + "pass@2": 0.4757476861444953, + "pass@4": 0.6150902066518003, + "pass@8": 0.7283701824607279, + "pass@16": 0.8069417810438068, + "pass@32": 0.8571822437247654, + "pass@64": 0.8945694038536045, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.2434185606060606, + "pass@2": 0.36151984908136464, + "pass@4": 0.4901963163695448, + "pass@8": 0.6128158772102645, + "pass@16": 0.7157703886521702, + "pass@32": 0.7941160880333268, + "pass@64": 0.8492354664830497, + "pass@128": 0.8848484848484849 + }, + "16": { + "pass@1": 0.1988292378917379, + "pass@2": 0.30013200473338286, + "pass@4": 0.4110917996895686, + "pass@8": 0.5151463559434297, + "pass@16": 0.6051339051747506, + "pass@32": 0.6838217166573823, + "pass@64": 0.7560222940461221, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.1983695652173913, + "pass@2": 0.30658913899349527, + "pass@4": 0.42618183159946577, + "pass@8": 0.5340680663265758, + "pass@16": 0.618370732659288, + "pass@32": 0.6845467437656324, + "pass@64": 0.7440009642404236, + "pass@128": 0.8012422360248447 + }, + "18": { + "pass@1": 0.16353400735294119, + "pass@2": 0.25522702929597024, + "pass@4": 0.3619508745230373, + "pass@8": 0.46639634535611824, + "pass@16": 0.5540833993978276, + "pass@32": 0.6223032123495336, + "pass@64": 0.6775643958034884, + "pass@128": 0.7264705882352941 + }, + "19": { + "pass@1": 0.14448474702380953, + "pass@2": 0.21798508096644167, + "pass@4": 0.30426176778125946, + "pass@8": 0.39128929215247166, + "pass@16": 0.4700711724164494, + "pass@32": 0.5399918946084133, + "pass@64": 0.6015360772969983, + "pass@128": 0.6547619047619048 + }, + "20": { + "pass@1": 0.12995883233532934, + "pass@2": 0.20353350876986184, + "pass@4": 0.29159784620859525, + "pass@8": 0.3811237163449579, + "pass@16": 0.45964420951180207, + "pass@32": 0.5219112961870748, + "pass@64": 0.5730840359592941, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 400628, + "answer_accuracy": 0.5000649065345902, + "avg_response_len": 281.3352559813868, + "resp_tokens_sum": 225392303, + "pass_at_k": { + "pass@1": 0.5000649065345902, + "pass@2": 0.6103195650232244, + "pass@4": 0.7064286998825201, + "pass@8": 0.7783712513258505, + "pass@16": 0.8281078442464707, + "pass@32": 0.8625593473715938, + "pass@64": 0.8876228519333529, + "pass@128": 0.907013899984023 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5870335820895523, + "pass@2": 0.7577733870019981, + "pass@4": 0.880753396963439, + "pass@8": 0.9473623446101491, + "pass@16": 0.9759983171238966, + "pass@32": 0.9859594955268647, + "pass@64": 0.9894615713657725, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9900669642857143, + "pass@2": 0.991916127671541, + "pass@4": 0.9936696981627297, + "pass@8": 0.9953110922921324, + "pass@16": 0.9968387123422647, + "pass@32": 0.9984497961412884, + "pass@64": 0.9997474915629107, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9979956268221575, + "pass@2": 0.999621579509194, + "pass@4": 0.999968567340161, + "pass@8": 0.99999975922343, + "pass@16": 0.9999999999947993, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9050392909356725, + "pass@2": 0.9572253303863328, + "pass@4": 0.9842237557476368, + "pass@8": 0.9937768813991182, + "pass@16": 0.9968527053025525, + "pass@32": 0.9982664901804696, + "pass@64": 0.9992743503931047, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7528492647058823, + "pass@2": 0.863650706345531, + "pass@4": 0.9333525565738104, + "pass@8": 0.9660227461686778, + "pass@16": 0.9788446430220072, + "pass@32": 0.9857440751253994, + "pass@64": 0.9908227122501815, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.7334631309148265, + "pass@2": 0.85599066357833, + "pass@4": 0.9375967866989801, + "pass@8": 0.9775949027886589, + "pass@16": 0.992955917554611, + "pass@32": 0.9984939993931751, + "pass@64": 0.9998956310797713, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.6655211629746836, + "pass@2": 0.8138696987441444, + "pass@4": 0.9095275688442417, + "pass@8": 0.9588090317998087, + "pass@16": 0.9824500620137617, + "pass@32": 0.993098936423399, + "pass@64": 0.9973843970952397, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6062077702702703, + "pass@2": 0.7712748427561416, + "pass@4": 0.8953629433708176, + "pass@8": 0.9638294728471273, + "pass@16": 0.9908266464333062, + "pass@32": 0.9983304781303394, + "pass@64": 0.9998135038671645, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5452170418006431, + "pass@2": 0.7200408417094971, + "pass@4": 0.8566020424094902, + "pass@8": 0.9366537967909059, + "pass@16": 0.9726401308835425, + "pass@32": 0.9868106206266033, + "pass@64": 0.993172338236693, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.5193204365079365, + "pass@2": 0.6937250031246089, + "pass@4": 0.8327515706370029, + "pass@8": 0.9199520734107461, + "pass@16": 0.9648824495942682, + "pass@32": 0.9852639879739948, + "pass@64": 0.9940219878639607, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.4114822247706422, + "pass@2": 0.5805370945604271, + "pass@4": 0.7374564925370567, + "pass@8": 0.8546288115348385, + "pass@16": 0.9252189474343845, + "pass@32": 0.9613500309324735, + "pass@64": 0.9792605949864354, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.3473186728395062, + "pass@2": 0.48654589834256845, + "pass@4": 0.6228602443213108, + "pass@8": 0.7370374613510134, + "pass@16": 0.8212246754468296, + "pass@32": 0.8785089408856174, + "pass@64": 0.9155469572108085, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.302763353115727, + "pass@2": 0.4393772342811747, + "pass@4": 0.5812337206921829, + "pass@8": 0.7044582723264429, + "pass@16": 0.7965432964422573, + "pass@32": 0.8588847914988257, + "pass@64": 0.8974336095198548, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.23396508487654322, + "pass@2": 0.3543128614999515, + "pass@4": 0.4862309537696679, + "pass@8": 0.6049426375187965, + "pass@16": 0.6974339017879962, + "pass@32": 0.7649831585883168, + "pass@64": 0.8146309282794365, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.21102968289085547, + "pass@2": 0.31148272187768566, + "pass@4": 0.42092105300996674, + "pass@8": 0.5274955208393359, + "pass@16": 0.6251729588230321, + "pass@32": 0.7151963712477419, + "pass@64": 0.7952666121181906, + "pass@128": 0.855457227138643 + }, + "17": { + "pass@1": 0.16773343373493976, + "pass@2": 0.2616457463713118, + "pass@4": 0.3689029873713071, + "pass@8": 0.4728816422571547, + "pass@16": 0.5617307806554981, + "pass@32": 0.6346774139142679, + "pass@64": 0.6943223299165555, + "pass@128": 0.7409638554216867 + }, + "18": { + "pass@1": 0.17552923387096775, + "pass@2": 0.26093908750317496, + "pass@4": 0.35391009198446943, + "pass@8": 0.4437184648651771, + "pass@16": 0.5250520514402012, + "pass@32": 0.5967975861389322, + "pass@64": 0.6566854833295221, + "pass@128": 0.7032258064516129 + }, + "19": { + "pass@1": 0.15903512084592145, + "pass@2": 0.2373675274163237, + "pass@4": 0.32128599436777355, + "pass@8": 0.40200679847511533, + "pass@16": 0.47251039414116286, + "pass@32": 0.5292500787241331, + "pass@64": 0.575524305858063, + "pass@128": 0.6163141993957704 + }, + "20": { + "pass@1": 0.12579767267267267, + "pass@2": 0.1951881753801045, + "pass@4": 0.2782783728610501, + "pass@8": 0.36563991218626535, + "pass@16": 0.4454211481401893, + "pass@32": 0.5097664447120562, + "pass@64": 0.5649682015817358, + "pass@128": 0.6246246246246246 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/summary.csv b/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..cbe86cb6e3068f68e05f497a10f6f3170f7ba2ac --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-42828,0.7638064236111111,0.26778984375,0.5027450657894736,0.0,0.0,0.0,194.81892274305557,329.1969828125,1152000,1280000,2432000 diff --git a/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/summary.json b/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b94a368b463197ab19be41350f11fdc4ed8e221b --- /dev/null +++ b/SDAR-100M-ep2-block4/composition/low_confidence_static/block8/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-42828", + "id_acc": 0.7638064236111111, + "ood_acc": 0.26778984375, + "total_acc": 0.5027450657894736, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 194.81892274305557, + "ood_avg_resp_len": 329.1969828125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 406947, + "answer_accuracy": 0.5025724687796396, + "avg_response_len": 252.37878398672146, + "resp_tokens_sum": 204358168, + "pass_at_k": { + "pass@1": 0.5025724687796396, + "pass@2": 0.6136163776042443, + "pass@4": 0.7087776433691901, + "pass@8": 0.778848798243971, + "pass@16": 0.8267637766621762, + "pass@32": 0.8599245759312861, + "pass@64": 0.8848061815918338, + "pass@128": 0.904995257666772 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5961928466076696, + "pass@2": 0.7596196838780109, + "pass@4": 0.8756493018571788, + "pass@8": 0.9365228324321908, + "pass@16": 0.9606329559490022, + "pass@32": 0.9688884108408146, + "pass@64": 0.9737911492704813, + "pass@128": 0.9793510324483776 + }, + "2": { + "pass@1": 0.9976878894080997, + "pass@2": 0.9985906952927612, + "pass@4": 0.9993798701564174, + "pass@8": 0.9998843836043162, + "pass@16": 0.9999967738107939, + "pass@32": 0.9999999991592076, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983514908256881, + "pass@2": 0.9998521364588602, + "pass@4": 0.9999973724614697, + "pass@8": 0.99999999928849, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9142578125, + "pass@2": 0.9533737543061023, + "pass@4": 0.9760740093035247, + "pass@8": 0.9864437669797045, + "pass@16": 0.9911445712028863, + "pass@32": 0.9944248711665763, + "pass@64": 0.9964253512379372, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.794890873015873, + "pass@2": 0.894600674915636, + "pass@4": 0.94974954797317, + "pass@8": 0.9755730371991035, + "pass@16": 0.9874647018460322, + "pass@32": 0.9937251962720715, + "pass@64": 0.9976983546461972, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.770279752994012, + "pass@2": 0.8859394891319728, + "pass@4": 0.9532805356665737, + "pass@8": 0.9830399819257302, + "pass@16": 0.9944878419475502, + "pass@32": 0.9986150659952652, + "pass@64": 0.999872031232074, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.6746486686390533, + "pass@2": 0.8198156420351302, + "pass@4": 0.9145076452166845, + "pass@8": 0.9619153600090276, + "pass@16": 0.9826155678869178, + "pass@32": 0.992917007771169, + "pass@64": 0.9985178232847783, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6353415464743589, + "pass@2": 0.7960370450989301, + "pass@4": 0.9076204095281363, + "pass@8": 0.9645666794146627, + "pass@16": 0.9884791332293995, + "pass@32": 0.9974799702985278, + "pass@64": 0.9998132509778702, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5373899647887324, + "pass@2": 0.7189194715537314, + "pass@4": 0.8618524614880887, + "pass@8": 0.9448509571689906, + "pass@16": 0.9817401961596698, + "pass@32": 0.9947024763761073, + "pass@64": 0.998955947935806, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.5171963526912181, + "pass@2": 0.6901484882112825, + "pass@4": 0.8268923876548013, + "pass@8": 0.9124617472551908, + "pass@16": 0.9559764167265765, + "pass@32": 0.9757553214879822, + "pass@64": 0.9844442937777405, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.435204802259887, + "pass@2": 0.6030044708394502, + "pass@4": 0.748447401966703, + "pass@8": 0.8497352014720185, + "pass@16": 0.9089616085075132, + "pass@32": 0.9431542535728017, + "pass@64": 0.9651132899120077, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.3713019200626959, + "pass@2": 0.523809101399551, + "pass@4": 0.6686971683398507, + "pass@8": 0.7846809994799563, + "pass@16": 0.8617996533225794, + "pass@32": 0.9064250879752345, + "pass@64": 0.9331490503349423, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.30415206386292837, + "pass@2": 0.44828767753329857, + "pass@4": 0.5943040942896155, + "pass@8": 0.716445667438578, + "pass@16": 0.8051037312119722, + "pass@32": 0.8657429264678448, + "pass@64": 0.9090951189790747, + "pass@128": 0.9470404984423676 + }, + "15": { + "pass@1": 0.24954841040462428, + "pass@2": 0.37298704826817164, + "pass@4": 0.5029955495989294, + "pass@8": 0.6201905378692797, + "pass@16": 0.7157051200779855, + "pass@32": 0.7883710262863396, + "pass@64": 0.8416705019935963, + "pass@128": 0.8815028901734104 + }, + "16": { + "pass@1": 0.21121471774193548, + "pass@2": 0.3187492062484124, + "pass@4": 0.4356203729574124, + "pass@8": 0.545153874059209, + "pass@16": 0.6407560148339182, + "pass@32": 0.7208517883776376, + "pass@64": 0.7881858215754862, + "pass@128": 0.8419354838709677 + }, + "17": { + "pass@1": 0.17341040462427745, + "pass@2": 0.26089931045468995, + "pass@4": 0.35955679374976973, + "pass@8": 0.45545195127969673, + "pass@16": 0.5399904831688518, + "pass@32": 0.6110066282578624, + "pass@64": 0.6704129618632997, + "pass@128": 0.7225433526011561 + }, + "18": { + "pass@1": 0.15582589285714285, + "pass@2": 0.23676708380202446, + "pass@4": 0.3290234993840056, + "pass@8": 0.42050899990842666, + "pass@16": 0.5023571496853358, + "pass@32": 0.5734286017598653, + "pass@64": 0.6377678650847342, + "pass@128": 0.6971428571428572 + }, + "19": { + "pass@1": 0.14161036036036037, + "pass@2": 0.22125718237922962, + "pass@4": 0.3123289304490093, + "pass@8": 0.3986272218604655, + "pass@16": 0.47052017433388255, + "pass@32": 0.527380938890095, + "pass@64": 0.5741233473287567, + "pass@128": 0.6156156156156156 + }, + "20": { + "pass@1": 0.14102383633633633, + "pass@2": 0.21171503688728094, + "pass@4": 0.2923553308651733, + "pass@8": 0.37377671407570623, + "pass@16": 0.4469443695731371, + "pass@32": 0.5086362984246378, + "pass@64": 0.5628735775041814, + "pass@128": 0.6126126126126126 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 415101, + "answer_accuracy": 0.5055302513639907, + "avg_response_len": 263.1199654130943, + "resp_tokens_sum": 216053066, + "pass_at_k": { + "pass@1": 0.5055302513639907, + "pass@2": 0.6167045893912595, + "pass@4": 0.7131499286624569, + "pass@8": 0.7848576100860732, + "pass@16": 0.8334102197462872, + "pass@32": 0.8664360830608916, + "pass@64": 0.8913671235680202, + "pass@128": 0.911925175370226 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5438075153374233, + "pass@2": 0.7155738249359938, + "pass@4": 0.8482569538853645, + "pass@8": 0.9247060702289117, + "pass@16": 0.9606764223030863, + "pass@32": 0.9775510243733617, + "pass@64": 0.9867887531474895, + "pass@128": 0.99079754601227 + }, + "2": { + "pass@1": 0.992686170212766, + "pass@2": 0.9948932580235983, + "pass@4": 0.996548313679635, + "pass@8": 0.9981614876492373, + "pass@16": 0.9994732641458764, + "pass@32": 0.9999644243028596, + "pass@64": 0.9999999628162317, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9937973484848485, + "pass@2": 0.9945269625387737, + "pass@4": 0.995079583233914, + "pass@8": 0.9960043744311183, + "pass@16": 0.997362339583687, + "pass@32": 0.998850274274236, + "pass@64": 0.9998151400249071, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.933385724852071, + "pass@2": 0.9726342280902018, + "pass@4": 0.9899115271463842, + "pass@8": 0.9961884271819588, + "pass@16": 0.9981460451848981, + "pass@32": 0.9990776110322851, + "pass@64": 0.999823789772943, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8190896739130434, + "pass@2": 0.9131440431359124, + "pass@4": 0.9604721708699447, + "pass@8": 0.9786022704945484, + "pass@16": 0.9854326450754342, + "pass@32": 0.9896840841129063, + "pass@64": 0.9937767545499087, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.7318230659025788, + "pass@2": 0.853087350247051, + "pass@4": 0.9329505317028773, + "pass@8": 0.9718729806164039, + "pass@16": 0.9874429073861202, + "pass@32": 0.9933592781253554, + "pass@64": 0.996150603505014, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.6908417630057804, + "pass@2": 0.8355122234991585, + "pass@4": 0.9259327110845249, + "pass@8": 0.9657232728447545, + "pass@16": 0.9799154383926086, + "pass@32": 0.987047671764001, + "pass@64": 0.9922131555149458, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.6376320422535211, + "pass@2": 0.8039948846623045, + "pass@4": 0.9164696530891375, + "pass@8": 0.969848565415213, + "pass@16": 0.9884038113446275, + "pass@32": 0.9941602181010866, + "pass@64": 0.9971833475810209, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5755988023952096, + "pass@2": 0.7414146853929929, + "pass@4": 0.8681606595582738, + "pass@8": 0.941494152221592, + "pass@16": 0.9739024093533677, + "pass@32": 0.9868868137718635, + "pass@64": 0.9927593268163736, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.4614787274096386, + "pass@2": 0.6289481251778768, + "pass@4": 0.7768584167942858, + "pass@8": 0.8823223386877882, + "pass@16": 0.9427012275107226, + "pass@32": 0.9688437100059571, + "pass@64": 0.9783544805563124, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.4521698667711599, + "pass@2": 0.6230932046503593, + "pass@4": 0.7703201394496527, + "pass@8": 0.8716570043484047, + "pass@16": 0.9280077925551187, + "pass@32": 0.9550018756047801, + "pass@64": 0.9660625310485854, + "pass@128": 0.9717868338557993 + }, + "13": { + "pass@1": 0.3567270658263305, + "pass@2": 0.5099872626215837, + "pass@4": 0.6578685560838504, + "pass@8": 0.7772300365473884, + "pass@16": 0.8555928488258188, + "pass@32": 0.9011742109700471, + "pass@64": 0.9303704417034507, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.3323967470760234, + "pass@2": 0.4757476861444953, + "pass@4": 0.6150902066518003, + "pass@8": 0.7283701824607279, + "pass@16": 0.8069417810438068, + "pass@32": 0.8571822437247654, + "pass@64": 0.8945694038536045, + "pass@128": 0.9298245614035088 + }, + "15": { + "pass@1": 0.2434185606060606, + "pass@2": 0.36151984908136464, + "pass@4": 0.4901963163695448, + "pass@8": 0.6128158772102645, + "pass@16": 0.7157703886521702, + "pass@32": 0.7941160880333268, + "pass@64": 0.8492354664830497, + "pass@128": 0.8848484848484849 + }, + "16": { + "pass@1": 0.1988292378917379, + "pass@2": 0.30013200473338286, + "pass@4": 0.4110917996895686, + "pass@8": 0.5151463559434297, + "pass@16": 0.6051339051747506, + "pass@32": 0.6838217166573823, + "pass@64": 0.7560222940461221, + "pass@128": 0.8233618233618234 + }, + "17": { + "pass@1": 0.1983695652173913, + "pass@2": 0.30658913899349527, + "pass@4": 0.42618183159946577, + "pass@8": 0.5340680663265758, + "pass@16": 0.618370732659288, + "pass@32": 0.6845467437656324, + "pass@64": 0.7440009642404236, + "pass@128": 0.8012422360248447 + }, + "18": { + "pass@1": 0.16353400735294119, + "pass@2": 0.25522702929597024, + "pass@4": 0.3619508745230373, + "pass@8": 0.46639634535611824, + "pass@16": 0.5540833993978276, + "pass@32": 0.6223032123495336, + "pass@64": 0.6775643958034884, + "pass@128": 0.7264705882352941 + }, + "19": { + "pass@1": 0.14448474702380953, + "pass@2": 0.21798508096644167, + "pass@4": 0.30426176778125946, + "pass@8": 0.39128929215247166, + "pass@16": 0.4700711724164494, + "pass@32": 0.5399918946084133, + "pass@64": 0.6015360772969983, + "pass@128": 0.6547619047619048 + }, + "20": { + "pass@1": 0.12995883233532934, + "pass@2": 0.20353350876986184, + "pass@4": 0.29159784620859525, + "pass@8": 0.3811237163449579, + "pass@16": 0.45964420951180207, + "pass@32": 0.5219112961870748, + "pass@64": 0.5730840359592941, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 400628, + "answer_accuracy": 0.5000649065345902, + "avg_response_len": 281.3352559813868, + "resp_tokens_sum": 225392303, + "pass_at_k": { + "pass@1": 0.5000649065345902, + "pass@2": 0.6103195650232244, + "pass@4": 0.7064286998825201, + "pass@8": 0.7783712513258505, + "pass@16": 0.8281078442464707, + "pass@32": 0.8625593473715938, + "pass@64": 0.8876228519333529, + "pass@128": 0.907013899984023 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5870335820895523, + "pass@2": 0.7577733870019981, + "pass@4": 0.880753396963439, + "pass@8": 0.9473623446101491, + "pass@16": 0.9759983171238966, + "pass@32": 0.9859594955268647, + "pass@64": 0.9894615713657725, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9900669642857143, + "pass@2": 0.991916127671541, + "pass@4": 0.9936696981627297, + "pass@8": 0.9953110922921324, + "pass@16": 0.9968387123422647, + "pass@32": 0.9984497961412884, + "pass@64": 0.9997474915629107, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9979956268221575, + "pass@2": 0.999621579509194, + "pass@4": 0.999968567340161, + "pass@8": 0.99999975922343, + "pass@16": 0.9999999999947993, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9050392909356725, + "pass@2": 0.9572253303863328, + "pass@4": 0.9842237557476368, + "pass@8": 0.9937768813991182, + "pass@16": 0.9968527053025525, + "pass@32": 0.9982664901804696, + "pass@64": 0.9992743503931047, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7528492647058823, + "pass@2": 0.863650706345531, + "pass@4": 0.9333525565738104, + "pass@8": 0.9660227461686778, + "pass@16": 0.9788446430220072, + "pass@32": 0.9857440751253994, + "pass@64": 0.9908227122501815, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.7334631309148265, + "pass@2": 0.85599066357833, + "pass@4": 0.9375967866989801, + "pass@8": 0.9775949027886589, + "pass@16": 0.992955917554611, + "pass@32": 0.9984939993931751, + "pass@64": 0.9998956310797713, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.6655211629746836, + "pass@2": 0.8138696987441444, + "pass@4": 0.9095275688442417, + "pass@8": 0.9588090317998087, + "pass@16": 0.9824500620137617, + "pass@32": 0.993098936423399, + "pass@64": 0.9973843970952397, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6062077702702703, + "pass@2": 0.7712748427561416, + "pass@4": 0.8953629433708176, + "pass@8": 0.9638294728471273, + "pass@16": 0.9908266464333062, + "pass@32": 0.9983304781303394, + "pass@64": 0.9998135038671645, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5452170418006431, + "pass@2": 0.7200408417094971, + "pass@4": 0.8566020424094902, + "pass@8": 0.9366537967909059, + "pass@16": 0.9726401308835425, + "pass@32": 0.9868106206266033, + "pass@64": 0.993172338236693, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.5193204365079365, + "pass@2": 0.6937250031246089, + "pass@4": 0.8327515706370029, + "pass@8": 0.9199520734107461, + "pass@16": 0.9648824495942682, + "pass@32": 0.9852639879739948, + "pass@64": 0.9940219878639607, + "pass@128": 0.9968253968253968 + }, + "12": { + "pass@1": 0.4114822247706422, + "pass@2": 0.5805370945604271, + "pass@4": 0.7374564925370567, + "pass@8": 0.8546288115348385, + "pass@16": 0.9252189474343845, + "pass@32": 0.9613500309324735, + "pass@64": 0.9792605949864354, + "pass@128": 0.9908256880733946 + }, + "13": { + "pass@1": 0.3473186728395062, + "pass@2": 0.48654589834256845, + "pass@4": 0.6228602443213108, + "pass@8": 0.7370374613510134, + "pass@16": 0.8212246754468296, + "pass@32": 0.8785089408856174, + "pass@64": 0.9155469572108085, + "pass@128": 0.9382716049382716 + }, + "14": { + "pass@1": 0.302763353115727, + "pass@2": 0.4393772342811747, + "pass@4": 0.5812337206921829, + "pass@8": 0.7044582723264429, + "pass@16": 0.7965432964422573, + "pass@32": 0.8588847914988257, + "pass@64": 0.8974336095198548, + "pass@128": 0.9228486646884273 + }, + "15": { + "pass@1": 0.23396508487654322, + "pass@2": 0.3543128614999515, + "pass@4": 0.4862309537696679, + "pass@8": 0.6049426375187965, + "pass@16": 0.6974339017879962, + "pass@32": 0.7649831585883168, + "pass@64": 0.8146309282794365, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.21102968289085547, + "pass@2": 0.31148272187768566, + "pass@4": 0.42092105300996674, + "pass@8": 0.5274955208393359, + "pass@16": 0.6251729588230321, + "pass@32": 0.7151963712477419, + "pass@64": 0.7952666121181906, + "pass@128": 0.855457227138643 + }, + "17": { + "pass@1": 0.16773343373493976, + "pass@2": 0.2616457463713118, + "pass@4": 0.3689029873713071, + "pass@8": 0.4728816422571547, + "pass@16": 0.5617307806554981, + "pass@32": 0.6346774139142679, + "pass@64": 0.6943223299165555, + "pass@128": 0.7409638554216867 + }, + "18": { + "pass@1": 0.17552923387096775, + "pass@2": 0.26093908750317496, + "pass@4": 0.35391009198446943, + "pass@8": 0.4437184648651771, + "pass@16": 0.5250520514402012, + "pass@32": 0.5967975861389322, + "pass@64": 0.6566854833295221, + "pass@128": 0.7032258064516129 + }, + "19": { + "pass@1": 0.15903512084592145, + "pass@2": 0.2373675274163237, + "pass@4": 0.32128599436777355, + "pass@8": 0.40200679847511533, + "pass@16": 0.47251039414116286, + "pass@32": 0.5292500787241331, + "pass@64": 0.575524305858063, + "pass@128": 0.6163141993957704 + }, + "20": { + "pass@1": 0.12579767267267267, + "pass@2": 0.1951881753801045, + "pass@4": 0.2782783728610501, + "pass@8": 0.36563991218626535, + "pass@16": 0.4454211481401893, + "pass@32": 0.5097664447120562, + "pass@64": 0.5649682015817358, + "pass@128": 0.6246246246246246 + } + } + } + } + } +] \ No newline at end of file