diff --git a/Qwen3-100M/composition/checkpoint-21414_metrics.json b/Qwen3-100M/composition/checkpoint-21414_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..24b5edb207d71fcb71cbe8d186cec844b7ba4837 --- /dev/null +++ b/Qwen3-100M/composition/checkpoint-21414_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21414", + "id": { + "answer_accuracy": 0.9793385416666667, + "avg_response_len": 200.03130208333334, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9793385416666667, + "pass@2": 0.9886347741688425, + "pass@4": 0.9926702687476551, + "pass@8": 0.9948221440951345, + "pass@16": 0.9960548904667301, + "pass@32": 0.996865920477279, + "pass@64": 0.9975009157807829, + "pass@128": 0.998 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.908453125, + "pass@2": 0.9377813730314967, + "pass@4": 0.9537591940382449, + "pass@8": 0.963215894411251, + "pass@16": 0.969382031030337, + "pass@32": 0.9745295017538527, + "pass@64": 0.9791577949664569, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997890625, + "pass@2": 0.9999781003937008, + "pass@4": 0.9999996365766779, + "pass@8": 0.9999999999471345, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998828125, + "pass@2": 0.9999969242125986, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991546875, + "pass@2": 0.9942684547244096, + "pass@4": 0.996191978440195, + "pass@8": 0.9979975101125733, + "pass@16": 0.999352327635449, + "pass@32": 0.9999106357214738, + "pass@64": 0.9999985265375396, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99834375, + "pass@2": 0.9998756151574807, + "pass@4": 0.9999940906449193, + "pass@8": 0.9999999830235391, + "pass@16": 0.9999999999999566, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99253125, + "pass@2": 0.9984806840551185, + "pass@4": 0.9997116128608922, + "pass@8": 0.9999809029763909, + "pass@16": 0.9999999143341539, + "pass@32": 0.9999999999995973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.972359375, + "pass@2": 0.9884831446850403, + "pass@4": 0.9945377080989877, + "pass@8": 0.9975546230392116, + "pass@16": 0.9990330155955035, + "pass@32": 0.9996555346296846, + "pass@64": 0.9999403002469388, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9511484375, + "pass@2": 0.9788486712598438, + "pass@4": 0.9898382013498308, + "pass@8": 0.9946503833461049, + "pass@16": 0.9967267256051743, + "pass@32": 0.9976976121909005, + "pass@64": 0.9984116202761056, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.908453125, + "2": 0.9999921875, + "3": 0.9997890625, + "4": 0.9998828125, + "5": 0.991546875, + "6": 0.99834375, + "7": 0.99253125, + "8": 0.972359375, + "9": 0.9511484375 + }, + "per_op_avg_response_len": { + "10": 280.38803125, + "2": 115.6685859375, + "3": 144.8313828125, + "4": 159.95446875, + "5": 173.216546875, + "6": 199.033796875, + "7": 224.100125, + "8": 243.532375, + "9": 259.55640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.38803125, + "2": 115.6685859375, + "3": 144.8313828125, + "4": 159.95446875, + "5": 173.216546875, + "6": 199.033796875, + "7": 224.100125, + "8": 243.532375, + "9": 259.55640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 370276, + "answer_accuracy": 0.9769609084768659, + "avg_response_len": 190.56355802516043, + "resp_tokens_sum": 72225113, + "pass_at_k": { + "pass@1": 0.9769609084768659, + "pass@2": 0.9874733078577961, + "pass@4": 0.992029703575555, + "pass@8": 0.9943640874763533, + "pass@16": 0.9957032884186743, + "pass@32": 0.9966601676290162, + "pass@64": 0.9974816352955542, + "pass@128": 0.9983113812901047 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9032079646017699, + "pass@2": 0.9346350718881379, + "pass@4": 0.9515867239493292, + "pass@8": 0.9609780321826099, + "pass@16": 0.9675171858317083, + "pass@32": 0.9737561988324965, + "pass@64": 0.9795632535184104, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.9999356624286643, + "pass@4": 0.9999988889003553, + "pass@8": 0.9999999998383318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998046875, + "pass@2": 0.999991926058071, + "pass@4": 0.9999999897473753, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9921378968253968, + "pass@2": 0.9949963285839271, + "pass@4": 0.9968516524720126, + "pass@8": 0.9985472791795658, + "pass@16": 0.9997014768693816, + "pass@32": 0.9999914354263738, + "pass@64": 0.9999999993942469, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990877619760479, + "pass@2": 0.9999871074779575, + "pass@4": 0.9999999969128112, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9929271449704142, + "pass@2": 0.9985600218981505, + "pass@4": 0.9997228755244352, + "pass@8": 0.9999804198863331, + "pass@16": 0.9999999213915781, + "pass@32": 0.9999999999998258, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9703024839743589, + "pass@2": 0.9894890880526952, + "pass@4": 0.996452376445732, + "pass@8": 0.9993380349504346, + "pass@16": 0.9999672539462268, + "pass@32": 0.9999999033301368, + "pass@64": 0.9999999999999883, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9415272887323943, + "pass@2": 0.9730630337140954, + "pass@4": 0.9859286664871115, + "pass@8": 0.9921444180077915, + "pass@16": 0.9954743355924334, + "pass@32": 0.9972116421043642, + "pass@64": 0.9985103643893272, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383082, + "answer_accuracy": 0.9806120986238532, + "avg_response_len": 198.05634368856488, + "resp_tokens_sum": 77371899, + "pass_at_k": { + "pass@1": 0.9806120986238532, + "pass@2": 0.9890831315595312, + "pass@4": 0.9926247708142001, + "pass@8": 0.9944516578234645, + "pass@16": 0.9954495834165552, + "pass@32": 0.9961014230827924, + "pass@64": 0.9966596565288138, + "pass@128": 0.997051114023591 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9058186349693251, + "pass@2": 0.9336452647215113, + "pass@4": 0.9483575641043333, + "pass@8": 0.9574015649119085, + "pass@16": 0.9631325453396871, + "pass@32": 0.9676096905416598, + "pass@64": 0.9719784332346865, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998342803030303, + "pass@2": 0.9999973902409927, + "pass@4": 0.9999999997159447, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996580615942029, + "pass@2": 0.9986808883943853, + "pass@4": 0.9996463534449496, + "pass@8": 0.9999656710759401, + "pass@16": 0.99999973449696, + "pass@32": 0.999999999996939, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974256805157593, + "pass@2": 0.9996788490174402, + "pass@4": 0.9999830779569457, + "pass@8": 0.9999999513568459, + "pass@16": 0.9999999999998754, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9918939667630058, + "pass@2": 0.9981001234581949, + "pass@4": 0.999588065491091, + "pass@8": 0.9999698934790995, + "pass@16": 0.9999998354810945, + "pass@32": 0.9999999999990067, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9757702464788732, + "pass@2": 0.9883116613064215, + "pass@4": 0.992730978257999, + "pass@8": 0.995465999524856, + "pass@16": 0.9976442421491534, + "pass@32": 0.9990449383281566, + "pass@64": 0.9998318380043506, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9553938997005988, + "pass@2": 0.9811025095478334, + "pass@4": 0.9915484450109904, + "pass@8": 0.995764661664974, + "pass@16": 0.9969082675621918, + "pass@32": 0.9970054821167854, + "pass@64": 0.9970059880236158, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374840, + "answer_accuracy": 0.980394208235688, + "avg_response_len": 211.43457063943757, + "resp_tokens_sum": 80839048, + "pass_at_k": { + "pass@1": 0.980394208235688, + "pass@2": 0.9893280165494057, + "pass@4": 0.9933517462058133, + "pass@8": 0.9956547620225993, + "pass@16": 0.9970219111501694, + "pass@32": 0.997851015968299, + "pass@64": 0.9983795943056459, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163246268656716, + "pass@2": 0.9449902309319544, + "pass@4": 0.9612141155862982, + "pass@8": 0.9711386010151163, + "pass@16": 0.9773507320974638, + "pass@32": 0.982045943979269, + "pass@64": 0.9857339784752468, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999772230320699, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999086257309941, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.985891544117647, + "pass@2": 0.9891167786012042, + "pass@4": 0.992075635214716, + "pass@8": 0.9954910607347378, + "pass@16": 0.9983719235886552, + "pass@32": 0.9997450987153587, + "pass@64": 0.9999956668480934, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9999747727216277, + "pass@4": 0.9999999920159823, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928055775316456, + "pass@2": 0.9988125124588857, + "pass@4": 0.9998348426383409, + "pass@8": 0.999993474402031, + "pass@16": 0.9999999931244994, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9706503378378378, + "pass@2": 0.9877234517982552, + "pass@4": 0.9946698779769646, + "pass@8": 0.9981102891992556, + "pass@16": 0.9996382174214158, + "pass@32": 0.9999838188714301, + "pass@64": 0.999999993259454, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9575713424437299, + "pass@2": 0.9830323347342834, + "pass@4": 0.9924641289814653, + "pass@8": 0.9963142056502817, + "pass@16": 0.997961334739834, + "pass@32": 0.9989956534303698, + "pass@64": 0.9998084916977711, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.37216171875, + "avg_response_len": 331.76342890625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.37216171875, + "pass@2": 0.4571696235236222, + "pass@4": 0.5325279767997672, + "pass@8": 0.5963750938383446, + "pass@16": 0.6492850309208061, + "pass@32": 0.6932886692210227, + "pass@64": 0.7308038684519446, + "pass@128": 0.7646 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8564375, + "pass@2": 0.915875861220472, + "pass@4": 0.9491968083989503, + "pass@8": 0.9674503762819611, + "pass@16": 0.9773159411789815, + "pass@32": 0.9830697728152279, + "pass@64": 0.9868645321047694, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.757046875, + "pass@2": 0.8480180856299216, + "pass@4": 0.8975600645856767, + "pass@8": 0.9257965967458588, + "pass@16": 0.9430769081652128, + "pass@32": 0.9541065205194755, + "pass@64": 0.961485327247675, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.51134375, + "pass@2": 0.6262690698818901, + "pass@4": 0.7164490299025117, + "pass@8": 0.7830122785915437, + "pass@16": 0.8316874283536161, + "pass@32": 0.8687514328627973, + "pass@64": 0.8990914429073045, + "pass@128": 0.926 + }, + "14": { + "pass@1": 0.4388125, + "pass@2": 0.5559735482283463, + "pass@4": 0.6556419095425574, + "pass@8": 0.733001266325819, + "pass@16": 0.7905868447924808, + "pass@32": 0.8328817730210621, + "pass@64": 0.864709437516425, + "pass@128": 0.893 + }, + "15": { + "pass@1": 0.3053984375, + "pass@2": 0.41739616141732294, + "pass@4": 0.5248345971128606, + "pass@8": 0.6183824732977962, + "pass@16": 0.6946463607359336, + "pass@32": 0.7542230487007859, + "pass@64": 0.8020780470956491, + "pass@128": 0.846 + }, + "16": { + "pass@1": 0.2159453125, + "pass@2": 0.2953483021653543, + "pass@4": 0.3731902739970001, + "pass@8": 0.4437958185733525, + "pass@16": 0.5077149682684065, + "pass@32": 0.5663136787097907, + "pass@64": 0.6201853308182402, + "pass@128": 0.669 + }, + "17": { + "pass@1": 0.179109375, + "pass@2": 0.25765046751968507, + "pass@4": 0.3408265441507307, + "pass@8": 0.41935008501751736, + "pass@16": 0.4893052229018614, + "pass@32": 0.5510861203973568, + "pass@64": 0.6064906874648237, + "pass@128": 0.66 + }, + "18": { + "pass@1": 0.1731171875, + "pass@2": 0.24430081200787396, + "pass@4": 0.31992820331833505, + "pass@8": 0.3939542185834461, + "pass@16": 0.4626459430303022, + "pass@32": 0.5240476052499612, + "pass@64": 0.5780989350713831, + "pass@128": 0.627 + }, + "19": { + "pass@1": 0.1513515625, + "pass@2": 0.21474963090551194, + "pass@4": 0.28110088142107253, + "pass@8": 0.3440299513276521, + "pass@16": 0.4013259396224595, + "pass@32": 0.4521014908968373, + "pass@64": 0.494970877378985, + "pass@128": 0.531 + }, + "20": { + "pass@1": 0.1330546875, + "pass@2": 0.1961142962598426, + "pass@4": 0.26655145556805393, + "pass@8": 0.33497787363845366, + "pass@16": 0.3945447521586475, + "pass@32": 0.4463052490369157, + "pass@64": 0.4940640669141259, + "pass@128": 0.537 + } + }, + "per_op_accuracy": { + "11": 0.8564375, + "12": 0.757046875, + "13": 0.51134375, + "14": 0.4388125, + "15": 0.3053984375, + "16": 0.2159453125, + "17": 0.179109375, + "18": 0.1731171875, + "19": 0.1513515625, + "20": 0.1330546875 + }, + "per_op_avg_response_len": { + "11": 297.664234375, + "12": 305.9078671875, + "13": 318.4241796875, + "14": 337.5991015625, + "15": 343.0364296875, + "16": 336.486890625, + "17": 346.3351953125, + "18": 347.1025078125, + "19": 339.1710859375, + "20": 345.906796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.664234375, + "12": 305.9078671875, + "13": 318.4241796875, + "14": 337.5991015625, + "15": 343.0364296875, + "16": 336.486890625, + "17": 346.3351953125, + "18": 347.1025078125, + "19": 339.1710859375, + "20": 345.906796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 159033, + "answer_accuracy": 0.3692259472511144, + "avg_response_len": 313.6137955052006, + "resp_tokens_sum": 135079734, + "pass_at_k": { + "pass@1": 0.3692259472511144, + "pass@2": 0.4521837450714288, + "pass@4": 0.5254462381002534, + "pass@8": 0.5882488534440974, + "pass@16": 0.6413566478424377, + "pass@32": 0.6861532875635945, + "pass@64": 0.7244193691692493, + "pass@128": 0.7589895988112927 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8610127478753541, + "pass@2": 0.9223810672302646, + "pass@4": 0.9552548427905423, + "pass@8": 0.9721816695259764, + "pass@16": 0.9802649889123723, + "pass@32": 0.9843935336083315, + "pass@64": 0.9871840886324861, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.7441516596045198, + "pass@2": 0.8418833272165134, + "pass@4": 0.8947573865237176, + "pass@8": 0.9246933072119006, + "pass@16": 0.9426086944518492, + "pass@32": 0.9545316421674195, + "pass@64": 0.9634646705384856, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.5101391065830722, + "pass@2": 0.6230156832374795, + "pass@4": 0.7152372407210855, + "pass@8": 0.7878959307878005, + "pass@16": 0.8409354625756095, + "pass@32": 0.8770589761241073, + "pass@64": 0.9035716421830617, + "pass@128": 0.9247648902821317 + }, + "14": { + "pass@1": 0.4194168613707165, + "pass@2": 0.5439467737385627, + "pass@4": 0.6519504974261392, + "pass@8": 0.7363008770588754, + "pass@16": 0.7986098817967806, + "pass@32": 0.8427948549317736, + "pass@64": 0.8734223469389563, + "pass@128": 0.897196261682243 + }, + "15": { + "pass@1": 0.29874909682080925, + "pass@2": 0.40230872513768134, + "pass@4": 0.5039451926348798, + "pass@8": 0.59629899102927, + "pass@16": 0.6766542211141445, + "pass@32": 0.7433549386887425, + "pass@64": 0.7957277053375337, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.21708669354838708, + "pass@2": 0.2930705486410974, + "pass@4": 0.36808121061480226, + "pass@8": 0.43515866962214683, + "pass@16": 0.4970340465228494, + "pass@32": 0.5566325896582562, + "pass@64": 0.614041673094276, + "pass@128": 0.6709677419354839 + }, + "17": { + "pass@1": 0.16415281791907516, + "pass@2": 0.2352422653725365, + "pass@4": 0.31182571181492447, + "pass@8": 0.38722358317358735, + "pass@16": 0.4585706153324987, + "pass@32": 0.5251923400842742, + "pass@64": 0.5866486952857641, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.1639732142857143, + "pass@2": 0.22608408323959514, + "pass@4": 0.29194012319888574, + "pass@8": 0.3600449101090104, + "pass@16": 0.42628749539285904, + "pass@32": 0.4863983571547588, + "pass@64": 0.5410503666444886, + "pass@128": 0.5942857142857143 + }, + "19": { + "pass@1": 0.14604448198198197, + "pass@2": 0.212031667494266, + "pass@4": 0.27985416518881095, + "pass@8": 0.34056775486808916, + "pass@16": 0.3952738230697976, + "pass@32": 0.4443278308301486, + "pass@64": 0.4845923777911046, + "pass@128": 0.5165165165165165 + }, + "20": { + "pass@1": 0.1328125, + "pass@2": 0.19051556477737575, + "pass@4": 0.2552802974515573, + "pass@8": 0.32018519255319294, + "pass@16": 0.37880108752861386, + "pass@32": 0.4309840186897329, + "pass@64": 0.480867056960796, + "pass@128": 0.5285285285285285 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 160807, + "answer_accuracy": 0.3735666629497473, + "avg_response_len": 327.4918901464466, + "resp_tokens_sum": 140973469, + "pass_at_k": { + "pass@1": 0.3735666629497473, + "pass@2": 0.45976946904830635, + "pass@4": 0.5364624337769576, + "pass@8": 0.6011613680377044, + "pass@16": 0.6542217010649847, + "pass@32": 0.6980003309310308, + "pass@64": 0.735189438041135, + "pass@128": 0.768361581920904 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8523390436746988, + "pass@2": 0.9107528786405467, + "pass@4": 0.9440548679532523, + "pass@8": 0.9625706848908397, + "pass@16": 0.9727083458666396, + "pass@32": 0.9791950380886896, + "pass@64": 0.9838954980871961, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.7638616771159875, + "pass@2": 0.8491753418655734, + "pass@4": 0.8971176635835875, + "pass@8": 0.9248675858291987, + "pass@16": 0.9404593997412108, + "pass@32": 0.9482462710786644, + "pass@64": 0.9524548178315899, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.5103947829131653, + "pass@2": 0.6257864366218929, + "pass@4": 0.7116970704292209, + "pass@8": 0.7704481704228001, + "pass@16": 0.813783721572076, + "pass@32": 0.8494450983678067, + "pass@64": 0.879012062296556, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.4642955043859649, + "pass@2": 0.5800334271538425, + "pass@4": 0.6762907079049327, + "pass@8": 0.7515076239679204, + "pass@16": 0.8083146186308952, + "pass@32": 0.8498004110358647, + "pass@64": 0.8794711389108484, + "pass@128": 0.9064327485380117 + }, + "15": { + "pass@1": 0.30433238636363635, + "pass@2": 0.4224077636602243, + "pass@4": 0.53449844621695, + "pass@8": 0.6291558085078369, + "pass@16": 0.7018289838535104, + "pass@32": 0.7557897028251086, + "pass@64": 0.8011498464693166, + "pass@128": 0.8454545454545455 + }, + "16": { + "pass@1": 0.20570690883190884, + "pass@2": 0.28798027850685337, + "pass@4": 0.3726769952367064, + "pass@8": 0.4499992465609035, + "pass@16": 0.5174759311156972, + "pass@32": 0.577641908620851, + "pass@64": 0.6325486367079364, + "pass@128": 0.6780626780626781 + }, + "17": { + "pass@1": 0.19570069875776397, + "pass@2": 0.28435444380593705, + "pass@4": 0.3767762825260196, + "pass@8": 0.4607843191671588, + "pass@16": 0.5321952328155982, + "pass@32": 0.5925545825114374, + "pass@64": 0.6454274279444601, + "pass@128": 0.6956521739130435 + }, + "18": { + "pass@1": 0.18129595588235295, + "pass@2": 0.2583194621352477, + "pass@4": 0.3406034090407815, + "pass@8": 0.4206929253875019, + "pass@16": 0.4939028233168417, + "pass@32": 0.5584202430963165, + "pass@64": 0.6135477816987461, + "pass@128": 0.6588235294117647 + }, + "19": { + "pass@1": 0.14936755952380953, + "pass@2": 0.20650763381139856, + "pass@4": 0.2671933831931724, + "pass@8": 0.3285557423083966, + "pass@16": 0.38803556694750535, + "pass@32": 0.44266199120033145, + "pass@64": 0.4902872097304205, + "pass@128": 0.5357142857142857 + }, + "20": { + "pass@1": 0.12329247754491018, + "pass@2": 0.1853823111650714, + "pass@4": 0.25464349198116704, + "pass@8": 0.3223347965266385, + "pass@16": 0.38121581221973927, + "pass@32": 0.43217173648806084, + "pass@64": 0.4784276984049925, + "pass@128": 0.5179640718562875 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 156527, + "answer_accuracy": 0.37373691549511, + "avg_response_len": 354.81926669468214, + "resp_tokens_sum": 148603986, + "pass_at_k": { + "pass@1": 0.37373691549511, + "pass@2": 0.4596250637718283, + "pass@4": 0.5357671185814513, + "pass@8": 0.5998129174306907, + "pass@16": 0.6523647947850137, + "pass@32": 0.6957841585078304, + "pass@64": 0.7328622943558968, + "pass@128": 0.7665036674816625 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8556299603174603, + "pass@2": 0.913985361204849, + "pass@4": 0.9478274055028832, + "pass@8": 0.9672913636683549, + "pass@16": 0.9788673944926667, + "pass@32": 0.9856701676382292, + "pass@64": 0.9896356935001666, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.76435875382263, + "pass@2": 0.8535304395723473, + "pass@4": 0.9010257341685495, + "pass@8": 0.9278972664016272, + "pass@16": 0.9461372532471328, + "pass@32": 0.9593631765079959, + "pass@64": 0.9681521314641406, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5135754243827161, + "pass@2": 0.630004040293574, + "pass@4": 0.7228780739791779, + "pass@8": 0.792047777250913, + "pass@16": 0.842309343181378, + "pass@32": 0.8818448128453698, + "pass@64": 0.9168048852192489, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.43142618694362017, + "pass@2": 0.543012468165144, + "pass@4": 0.6382029013806506, + "pass@8": 0.7110773809878095, + "pass@16": 0.7649539263025206, + "pass@32": 0.806269685500585, + "pass@64": 0.8414294796483958, + "pass@128": 0.8753709198813057 + }, + "15": { + "pass@1": 0.3135850694444444, + "pass@2": 0.4284036435549723, + "pass@4": 0.5372996086716009, + "pass@8": 0.6309926098582802, + "pass@16": 0.7065445541937744, + "pass@32": 0.7642334505623332, + "pass@64": 0.8098049744258834, + "pass@128": 0.8518518518518519 + }, + "16": { + "pass@1": 0.22550239675516223, + "pass@2": 0.3050600422734767, + "pass@4": 0.3783937267797276, + "pass@8": 0.4452710780165477, + "pass@16": 0.5073756991879474, + "pass@32": 0.5634373628018063, + "pass@64": 0.6130024798658662, + "pass@128": 0.6578171091445427 + }, + "17": { + "pass@1": 0.1786050451807229, + "pass@2": 0.2551039542026372, + "pass@4": 0.33618338822105065, + "pass@8": 0.41264510381817004, + "pass@16": 0.4797377259945632, + "pass@32": 0.5378523950587828, + "pass@64": 0.5894054430657593, + "pass@128": 0.6385542168674698 + }, + "18": { + "pass@1": 0.17447076612903226, + "pass@2": 0.24949279273558542, + "pass@4": 0.32885161653180445, + "pass@8": 0.40291259810820174, + "pass@16": 0.4694140635970205, + "pass@32": 0.5288557986872517, + "pass@64": 0.5810485837685092, + "pass@128": 0.6290322580645161 + }, + "19": { + "pass@1": 0.15870468277945618, + "pass@2": 0.22585051561719435, + "pass@4": 0.2964727119646298, + "pass@8": 0.3632210258034961, + "pass@16": 0.42090575844669265, + "pass@32": 0.4695042120455788, + "pass@64": 0.5101665048493216, + "pass@128": 0.540785498489426 + }, + "20": { + "pass@1": 0.14308840090090091, + "pass@2": 0.21247724102054807, + "pass@4": 0.28976633692184883, + "pass@8": 0.36245159903406554, + "pass@16": 0.42365738357425387, + "pass@32": 0.4758024349135808, + "pass@64": 0.5229444014381782, + "pass@128": 0.5645645645645646 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6597717927631579, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6597717927631579, + "pass@2": 0.7089162738292367, + "pass@4": 0.7504901150908506, + "pass@8": 0.7851131702757833, + "pass@16": 0.8135444380741469, + "pass@32": 0.8370884198161082, + "pass@64": 0.8571340487656051, + "pass@128": 0.8751578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.908453125, + "pass@2": 0.9377813730314967, + "pass@4": 0.9537591940382449, + "pass@8": 0.963215894411251, + "pass@16": 0.969382031030337, + "pass@32": 0.9745295017538527, + "pass@64": 0.9791577949664569, + "pass@128": 0.983 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997890625, + "pass@2": 0.9999781003937008, + "pass@4": 0.9999996365766779, + "pass@8": 0.9999999999471345, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998828125, + "pass@2": 0.9999969242125986, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.991546875, + "pass@2": 0.9942684547244096, + "pass@4": 0.996191978440195, + "pass@8": 0.9979975101125733, + "pass@16": 0.999352327635449, + "pass@32": 0.9999106357214738, + "pass@64": 0.9999985265375396, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.99834375, + "pass@2": 0.9998756151574807, + "pass@4": 0.9999940906449193, + "pass@8": 0.9999999830235391, + "pass@16": 0.9999999999999566, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.99253125, + "pass@2": 0.9984806840551185, + "pass@4": 0.9997116128608922, + "pass@8": 0.9999809029763909, + "pass@16": 0.9999999143341539, + "pass@32": 0.9999999999995973, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.972359375, + "pass@2": 0.9884831446850403, + "pass@4": 0.9945377080989877, + "pass@8": 0.9975546230392116, + "pass@16": 0.9990330155955035, + "pass@32": 0.9996555346296846, + "pass@64": 0.9999403002469388, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9511484375, + "pass@2": 0.9788486712598438, + "pass@4": 0.9898382013498308, + "pass@8": 0.9946503833461049, + "pass@16": 0.9967267256051743, + "pass@32": 0.9976976121909005, + "pass@64": 0.9984116202761056, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.8564375, + "pass@2": 0.915875861220472, + "pass@4": 0.9491968083989503, + "pass@8": 0.9674503762819611, + "pass@16": 0.9773159411789815, + "pass@32": 0.9830697728152279, + "pass@64": 0.9868645321047694, + "pass@128": 0.99 + }, + "12": { + "pass@1": 0.757046875, + "pass@2": 0.8480180856299216, + "pass@4": 0.8975600645856767, + "pass@8": 0.9257965967458588, + "pass@16": 0.9430769081652128, + "pass@32": 0.9541065205194755, + "pass@64": 0.961485327247675, + "pass@128": 0.967 + }, + "13": { + "pass@1": 0.51134375, + "pass@2": 0.6262690698818901, + "pass@4": 0.7164490299025117, + "pass@8": 0.7830122785915437, + "pass@16": 0.8316874283536161, + "pass@32": 0.8687514328627973, + "pass@64": 0.8990914429073045, + "pass@128": 0.926 + }, + "14": { + "pass@1": 0.4388125, + "pass@2": 0.5559735482283463, + "pass@4": 0.6556419095425574, + "pass@8": 0.733001266325819, + "pass@16": 0.7905868447924808, + "pass@32": 0.8328817730210621, + "pass@64": 0.864709437516425, + "pass@128": 0.893 + }, + "15": { + "pass@1": 0.3053984375, + "pass@2": 0.41739616141732294, + "pass@4": 0.5248345971128606, + "pass@8": 0.6183824732977962, + "pass@16": 0.6946463607359336, + "pass@32": 0.7542230487007859, + "pass@64": 0.8020780470956491, + "pass@128": 0.846 + }, + "16": { + "pass@1": 0.2159453125, + "pass@2": 0.2953483021653543, + "pass@4": 0.3731902739970001, + "pass@8": 0.4437958185733525, + "pass@16": 0.5077149682684065, + "pass@32": 0.5663136787097907, + "pass@64": 0.6201853308182402, + "pass@128": 0.669 + }, + "17": { + "pass@1": 0.179109375, + "pass@2": 0.25765046751968507, + "pass@4": 0.3408265441507307, + "pass@8": 0.41935008501751736, + "pass@16": 0.4893052229018614, + "pass@32": 0.5510861203973568, + "pass@64": 0.6064906874648237, + "pass@128": 0.66 + }, + "18": { + "pass@1": 0.1731171875, + "pass@2": 0.24430081200787396, + "pass@4": 0.31992820331833505, + "pass@8": 0.3939542185834461, + "pass@16": 0.4626459430303022, + "pass@32": 0.5240476052499612, + "pass@64": 0.5780989350713831, + "pass@128": 0.627 + }, + "19": { + "pass@1": 0.1513515625, + "pass@2": 0.21474963090551194, + "pass@4": 0.28110088142107253, + "pass@8": 0.3440299513276521, + "pass@16": 0.4013259396224595, + "pass@32": 0.4521014908968373, + "pass@64": 0.494970877378985, + "pass@128": 0.531 + }, + "20": { + "pass@1": 0.1330546875, + "pass@2": 0.1961142962598426, + "pass@4": 0.26655145556805393, + "pass@8": 0.33497787363845366, + "pass@16": 0.3945447521586475, + "pass@32": 0.4463052490369157, + "pass@64": 0.4940640669141259, + "pass@128": 0.537 + } + }, + "per_op_accuracy": { + "10": 0.908453125, + "2": 0.9999921875, + "3": 0.9997890625, + "4": 0.9998828125, + "5": 0.991546875, + "6": 0.99834375, + "7": 0.99253125, + "8": 0.972359375, + "9": 0.9511484375, + "11": 0.8564375, + "12": 0.757046875, + "13": 0.51134375, + "14": 0.4388125, + "15": 0.3053984375, + "16": 0.2159453125, + "17": 0.179109375, + "18": 0.1731171875, + "19": 0.1513515625, + "20": 0.1330546875 + }, + "per_op_avg_response_len": { + "10": 280.38803125, + "2": 115.6685859375, + "3": 144.8313828125, + "4": 159.95446875, + "5": 173.216546875, + "6": 199.033796875, + "7": 224.100125, + "8": 243.532375, + "9": 259.55640625, + "11": 297.664234375, + "12": 305.9078671875, + "13": 318.4241796875, + "14": 337.5991015625, + "15": 343.0364296875, + "16": 336.486890625, + "17": 346.3351953125, + "18": 347.1025078125, + "19": 339.1710859375, + "20": 345.906796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.38803125, + "2": 115.6685859375, + "3": 144.8313828125, + "4": 159.95446875, + "5": 173.216546875, + "6": 199.033796875, + "7": 224.100125, + "8": 243.532375, + "9": 259.55640625, + "11": 297.664234375, + "12": 305.9078671875, + "13": 318.4241796875, + "14": 337.5991015625, + "15": 343.0364296875, + "16": 336.486890625, + "17": 346.3351953125, + "18": 347.1025078125, + "19": 339.1710859375, + "20": 345.906796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 529309, + "answer_accuracy": 0.653687411081252, + "avg_response_len": 256.0178813132311, + "resp_tokens_sum": 207304847, + "pass_at_k": { + "pass@1": 0.653687411081252, + "pass@2": 0.7027358151647587, + "pass@4": 0.7438391627402035, + "pass@8": 0.7783385164174618, + "pass@16": 0.8072150738219351, + "pass@32": 0.8314917118243812, + "pass@64": 0.8522311570288807, + "pass@128": 0.8710085361998103 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9032079646017699, + "pass@2": 0.9346350718881379, + "pass@4": 0.9515867239493292, + "pass@8": 0.9609780321826099, + "pass@16": 0.9675171858317083, + "pass@32": 0.9737561988324965, + "pass@64": 0.9795632535184104, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995460626911316, + "pass@2": 0.9999356624286643, + "pass@4": 0.9999988889003553, + "pass@8": 0.9999999998383318, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998046875, + "pass@2": 0.999991926058071, + "pass@4": 0.9999999897473753, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9921378968253968, + "pass@2": 0.9949963285839271, + "pass@4": 0.9968516524720126, + "pass@8": 0.9985472791795658, + "pass@16": 0.9997014768693816, + "pass@32": 0.9999914354263738, + "pass@64": 0.9999999993942469, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9990877619760479, + "pass@2": 0.9999871074779575, + "pass@4": 0.9999999969128112, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9929271449704142, + "pass@2": 0.9985600218981505, + "pass@4": 0.9997228755244352, + "pass@8": 0.9999804198863331, + "pass@16": 0.9999999213915781, + "pass@32": 0.9999999999998258, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9703024839743589, + "pass@2": 0.9894890880526952, + "pass@4": 0.996452376445732, + "pass@8": 0.9993380349504346, + "pass@16": 0.9999672539462268, + "pass@32": 0.9999999033301368, + "pass@64": 0.9999999999999883, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9415272887323943, + "pass@2": 0.9730630337140954, + "pass@4": 0.9859286664871115, + "pass@8": 0.9921444180077915, + "pass@16": 0.9954743355924334, + "pass@32": 0.9972116421043642, + "pass@64": 0.9985103643893272, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8610127478753541, + "pass@2": 0.9223810672302646, + "pass@4": 0.9552548427905423, + "pass@8": 0.9721816695259764, + "pass@16": 0.9802649889123723, + "pass@32": 0.9843935336083315, + "pass@64": 0.9871840886324861, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.7441516596045198, + "pass@2": 0.8418833272165134, + "pass@4": 0.8947573865237176, + "pass@8": 0.9246933072119006, + "pass@16": 0.9426086944518492, + "pass@32": 0.9545316421674195, + "pass@64": 0.9634646705384856, + "pass@128": 0.9689265536723164 + }, + "13": { + "pass@1": 0.5101391065830722, + "pass@2": 0.6230156832374795, + "pass@4": 0.7152372407210855, + "pass@8": 0.7878959307878005, + "pass@16": 0.8409354625756095, + "pass@32": 0.8770589761241073, + "pass@64": 0.9035716421830617, + "pass@128": 0.9247648902821317 + }, + "14": { + "pass@1": 0.4194168613707165, + "pass@2": 0.5439467737385627, + "pass@4": 0.6519504974261392, + "pass@8": 0.7363008770588754, + "pass@16": 0.7986098817967806, + "pass@32": 0.8427948549317736, + "pass@64": 0.8734223469389563, + "pass@128": 0.897196261682243 + }, + "15": { + "pass@1": 0.29874909682080925, + "pass@2": 0.40230872513768134, + "pass@4": 0.5039451926348798, + "pass@8": 0.59629899102927, + "pass@16": 0.6766542211141445, + "pass@32": 0.7433549386887425, + "pass@64": 0.7957277053375337, + "pass@128": 0.8410404624277457 + }, + "16": { + "pass@1": 0.21708669354838708, + "pass@2": 0.2930705486410974, + "pass@4": 0.36808121061480226, + "pass@8": 0.43515866962214683, + "pass@16": 0.4970340465228494, + "pass@32": 0.5566325896582562, + "pass@64": 0.614041673094276, + "pass@128": 0.6709677419354839 + }, + "17": { + "pass@1": 0.16415281791907516, + "pass@2": 0.2352422653725365, + "pass@4": 0.31182571181492447, + "pass@8": 0.38722358317358735, + "pass@16": 0.4585706153324987, + "pass@32": 0.5251923400842742, + "pass@64": 0.5866486952857641, + "pass@128": 0.6473988439306358 + }, + "18": { + "pass@1": 0.1639732142857143, + "pass@2": 0.22608408323959514, + "pass@4": 0.29194012319888574, + "pass@8": 0.3600449101090104, + "pass@16": 0.42628749539285904, + "pass@32": 0.4863983571547588, + "pass@64": 0.5410503666444886, + "pass@128": 0.5942857142857143 + }, + "19": { + "pass@1": 0.14604448198198197, + "pass@2": 0.212031667494266, + "pass@4": 0.27985416518881095, + "pass@8": 0.34056775486808916, + "pass@16": 0.3952738230697976, + "pass@32": 0.4443278308301486, + "pass@64": 0.4845923777911046, + "pass@128": 0.5165165165165165 + }, + "20": { + "pass@1": 0.1328125, + "pass@2": 0.19051556477737575, + "pass@4": 0.2552802974515573, + "pass@8": 0.32018519255319294, + "pass@16": 0.37880108752861386, + "pass@32": 0.4309840186897329, + "pass@64": 0.480867056960796, + "pass@128": 0.5285285285285285 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 543889, + "answer_accuracy": 0.662374561574435, + "avg_response_len": 265.9116426344505, + "resp_tokens_sum": 218345368, + "pass_at_k": { + "pass@1": 0.662374561574435, + "pass@2": 0.7115957041198928, + "pass@4": 0.7534861988023069, + "pass@8": 0.7882731317830092, + "pass@16": 0.8165642570957005, + "pass@32": 0.8398248879454048, + "pass@64": 0.8595864928851629, + "pass@128": 0.8771628994544037 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9058186349693251, + "pass@2": 0.9336452647215113, + "pass@4": 0.9483575641043333, + "pass@8": 0.9574015649119085, + "pass@16": 0.9631325453396871, + "pass@32": 0.9676096905416598, + "pass@64": 0.9719784332346865, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 0.9999762537993921, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998342803030303, + "pass@2": 0.9999973902409927, + "pass@4": 0.9999999997159447, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999306582840237, + "pass@2": 0.999999636001491, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996580615942029, + "pass@2": 0.9986808883943853, + "pass@4": 0.9996463534449496, + "pass@8": 0.9999656710759401, + "pass@16": 0.99999973449696, + "pass@32": 0.999999999996939, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9974256805157593, + "pass@2": 0.9996788490174402, + "pass@4": 0.9999830779569457, + "pass@8": 0.9999999513568459, + "pass@16": 0.9999999999998754, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9918939667630058, + "pass@2": 0.9981001234581949, + "pass@4": 0.999588065491091, + "pass@8": 0.9999698934790995, + "pass@16": 0.9999998354810945, + "pass@32": 0.9999999999990067, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9757702464788732, + "pass@2": 0.9883116613064215, + "pass@4": 0.992730978257999, + "pass@8": 0.995465999524856, + "pass@16": 0.9976442421491534, + "pass@32": 0.9990449383281566, + "pass@64": 0.9998318380043506, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9553938997005988, + "pass@2": 0.9811025095478334, + "pass@4": 0.9915484450109904, + "pass@8": 0.995764661664974, + "pass@16": 0.9969082675621918, + "pass@32": 0.9970054821167854, + "pass@64": 0.9970059880236158, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.8523390436746988, + "pass@2": 0.9107528786405467, + "pass@4": 0.9440548679532523, + "pass@8": 0.9625706848908397, + "pass@16": 0.9727083458666396, + "pass@32": 0.9791950380886896, + "pass@64": 0.9838954980871961, + "pass@128": 0.9879518072289156 + }, + "12": { + "pass@1": 0.7638616771159875, + "pass@2": 0.8491753418655734, + "pass@4": 0.8971176635835875, + "pass@8": 0.9248675858291987, + "pass@16": 0.9404593997412108, + "pass@32": 0.9482462710786644, + "pass@64": 0.9524548178315899, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.5103947829131653, + "pass@2": 0.6257864366218929, + "pass@4": 0.7116970704292209, + "pass@8": 0.7704481704228001, + "pass@16": 0.813783721572076, + "pass@32": 0.8494450983678067, + "pass@64": 0.879012062296556, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.4642955043859649, + "pass@2": 0.5800334271538425, + "pass@4": 0.6762907079049327, + "pass@8": 0.7515076239679204, + "pass@16": 0.8083146186308952, + "pass@32": 0.8498004110358647, + "pass@64": 0.8794711389108484, + "pass@128": 0.9064327485380117 + }, + "15": { + "pass@1": 0.30433238636363635, + "pass@2": 0.4224077636602243, + "pass@4": 0.53449844621695, + "pass@8": 0.6291558085078369, + "pass@16": 0.7018289838535104, + "pass@32": 0.7557897028251086, + "pass@64": 0.8011498464693166, + "pass@128": 0.8454545454545455 + }, + "16": { + "pass@1": 0.20570690883190884, + "pass@2": 0.28798027850685337, + "pass@4": 0.3726769952367064, + "pass@8": 0.4499992465609035, + "pass@16": 0.5174759311156972, + "pass@32": 0.577641908620851, + "pass@64": 0.6325486367079364, + "pass@128": 0.6780626780626781 + }, + "17": { + "pass@1": 0.19570069875776397, + "pass@2": 0.28435444380593705, + "pass@4": 0.3767762825260196, + "pass@8": 0.4607843191671588, + "pass@16": 0.5321952328155982, + "pass@32": 0.5925545825114374, + "pass@64": 0.6454274279444601, + "pass@128": 0.6956521739130435 + }, + "18": { + "pass@1": 0.18129595588235295, + "pass@2": 0.2583194621352477, + "pass@4": 0.3406034090407815, + "pass@8": 0.4206929253875019, + "pass@16": 0.4939028233168417, + "pass@32": 0.5584202430963165, + "pass@64": 0.6135477816987461, + "pass@128": 0.6588235294117647 + }, + "19": { + "pass@1": 0.14936755952380953, + "pass@2": 0.20650763381139856, + "pass@4": 0.2671933831931724, + "pass@8": 0.3285557423083966, + "pass@16": 0.38803556694750535, + "pass@32": 0.44266199120033145, + "pass@64": 0.4902872097304205, + "pass@128": 0.5357142857142857 + }, + "20": { + "pass@1": 0.12329247754491018, + "pass@2": 0.1853823111650714, + "pass@4": 0.25464349198116704, + "pass@8": 0.3223347965266385, + "pass@16": 0.38121581221973927, + "pass@32": 0.43217173648806084, + "pass@64": 0.4784276984049925, + "pass@128": 0.5179640718562875 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 531367, + "answer_accuracy": 0.6632536647227991, + "avg_response_len": 286.391388899984, + "resp_tokens_sum": 229443034, + "pass_at_k": { + "pass@1": 0.6632536647227991, + "pass@2": 0.7124166790373029, + "pass@4": 0.7541415046996705, + "pass@8": 0.7887216232616626, + "pass@16": 0.8168464702256237, + "pass@32": 0.8399403660864274, + "pass@64": 0.8595758548208159, + "pass@128": 0.8772966927624221 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9163246268656716, + "pass@2": 0.9449902309319544, + "pass@4": 0.9612141155862982, + "pass@8": 0.9711386010151163, + "pass@16": 0.9773507320974638, + "pass@32": 0.982045943979269, + "pass@64": 0.9857339784752468, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999772230320699, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999086257309941, + "pass@2": 0.9999989207763503, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.985891544117647, + "pass@2": 0.9891167786012042, + "pass@4": 0.992075635214716, + "pass@8": 0.9954910607347378, + "pass@16": 0.9983719235886552, + "pass@32": 0.9997450987153587, + "pass@64": 0.9999956668480934, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9985705835962145, + "pass@2": 0.9999747727216277, + "pass@4": 0.9999999920159823, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928055775316456, + "pass@2": 0.9988125124588857, + "pass@4": 0.9998348426383409, + "pass@8": 0.999993474402031, + "pass@16": 0.9999999931244994, + "pass@32": 0.9999999999999997, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9706503378378378, + "pass@2": 0.9877234517982552, + "pass@4": 0.9946698779769646, + "pass@8": 0.9981102891992556, + "pass@16": 0.9996382174214158, + "pass@32": 0.9999838188714301, + "pass@64": 0.999999993259454, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9575713424437299, + "pass@2": 0.9830323347342834, + "pass@4": 0.9924641289814653, + "pass@8": 0.9963142056502817, + "pass@16": 0.997961334739834, + "pass@32": 0.9989956534303698, + "pass@64": 0.9998084916977711, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8556299603174603, + "pass@2": 0.913985361204849, + "pass@4": 0.9478274055028832, + "pass@8": 0.9672913636683549, + "pass@16": 0.9788673944926667, + "pass@32": 0.9856701676382292, + "pass@64": 0.9896356935001666, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.76435875382263, + "pass@2": 0.8535304395723473, + "pass@4": 0.9010257341685495, + "pass@8": 0.9278972664016272, + "pass@16": 0.9461372532471328, + "pass@32": 0.9593631765079959, + "pass@64": 0.9681521314641406, + "pass@128": 0.9755351681957186 + }, + "13": { + "pass@1": 0.5135754243827161, + "pass@2": 0.630004040293574, + "pass@4": 0.7228780739791779, + "pass@8": 0.792047777250913, + "pass@16": 0.842309343181378, + "pass@32": 0.8818448128453698, + "pass@64": 0.9168048852192489, + "pass@128": 0.9506172839506173 + }, + "14": { + "pass@1": 0.43142618694362017, + "pass@2": 0.543012468165144, + "pass@4": 0.6382029013806506, + "pass@8": 0.7110773809878095, + "pass@16": 0.7649539263025206, + "pass@32": 0.806269685500585, + "pass@64": 0.8414294796483958, + "pass@128": 0.8753709198813057 + }, + "15": { + "pass@1": 0.3135850694444444, + "pass@2": 0.4284036435549723, + "pass@4": 0.5372996086716009, + "pass@8": 0.6309926098582802, + "pass@16": 0.7065445541937744, + "pass@32": 0.7642334505623332, + "pass@64": 0.8098049744258834, + "pass@128": 0.8518518518518519 + }, + "16": { + "pass@1": 0.22550239675516223, + "pass@2": 0.3050600422734767, + "pass@4": 0.3783937267797276, + "pass@8": 0.4452710780165477, + "pass@16": 0.5073756991879474, + "pass@32": 0.5634373628018063, + "pass@64": 0.6130024798658662, + "pass@128": 0.6578171091445427 + }, + "17": { + "pass@1": 0.1786050451807229, + "pass@2": 0.2551039542026372, + "pass@4": 0.33618338822105065, + "pass@8": 0.41264510381817004, + "pass@16": 0.4797377259945632, + "pass@32": 0.5378523950587828, + "pass@64": 0.5894054430657593, + "pass@128": 0.6385542168674698 + }, + "18": { + "pass@1": 0.17447076612903226, + "pass@2": 0.24949279273558542, + "pass@4": 0.32885161653180445, + "pass@8": 0.40291259810820174, + "pass@16": 0.4694140635970205, + "pass@32": 0.5288557986872517, + "pass@64": 0.5810485837685092, + "pass@128": 0.6290322580645161 + }, + "19": { + "pass@1": 0.15870468277945618, + "pass@2": 0.22585051561719435, + "pass@4": 0.2964727119646298, + "pass@8": 0.3632210258034961, + "pass@16": 0.42090575844669265, + "pass@32": 0.4695042120455788, + "pass@64": 0.5101665048493216, + "pass@128": 0.540785498489426 + }, + "20": { + "pass@1": 0.14308840090090091, + "pass@2": 0.21247724102054807, + "pass@4": 0.28976633692184883, + "pass@8": 0.36245159903406554, + "pass@16": 0.42365738357425387, + "pass@32": 0.4758024349135808, + "pass@64": 0.5229444014381782, + "pass@128": 0.5645645645645646 + } + } + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M/composition/graphs/composition_score_report.jsonl b/Qwen3-100M/composition/graphs/composition_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6618fd6f98c94ea9fb30d14a4ca7ffe573703fcc --- /dev/null +++ b/Qwen3-100M/composition/graphs/composition_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M/composition", "results_dir": "results/Qwen3-100M/composition/graphs", "per_file": [{"file": "results/Qwen3-100M/composition/checkpoint-21414_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1000024, "value_mismatch": 26024, "dependency_mismatch": 149100, "answer_mismatch": 23802, "extra_nodes": 11190, "missing_nodes": 10769, "perfect_with_extra": 313, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 329006, "dependency_mismatch": 49118, "value_mismatch": 9389, "answer_mismatch": 8732, "missing_nodes": 3471, "extra_nodes": 3651, "perfect_with_extra": 103}, "teachers_in_school": {"total": 390656, "perfect": 339613, "dependency_mismatch": 50020, "value_mismatch": 8367, "answer_mismatch": 7574, "extra_nodes": 3807, "missing_nodes": 3843, "perfect_with_extra": 47}, "movie_festival_awards": {"total": 382336, "perfect": 331405, "value_mismatch": 8268, "dependency_mismatch": 49962, "answer_mismatch": 7496, "extra_nodes": 3732, "missing_nodes": 3455, "perfect_with_extra": 163}}, "results_path": "results/Qwen3-100M/composition/graphs/checkpoint-21414_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M/composition/checkpoint-21414_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 177121, "value_mismatch": 938826, "dependency_mismatch": 1078759, "answer_mismatch": 803644, "extra_nodes": 52171, "missing_nodes": 681350, "perfect_with_extra": 60, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 316566, "dependency_mismatch": 365270, "answer_mismatch": 271691, "extra_nodes": 18289, "perfect": 58140, "missing_nodes": 241666, "perfect_with_extra": 16}, "movie_festival_awards": {"total": 418816, "dependency_mismatch": 352010, "value_mismatch": 305912, "answer_mismatch": 262289, "perfect": 58778, "missing_nodes": 214731, "extra_nodes": 18046, "perfect_with_extra": 2}, "teachers_in_school": {"total": 430464, "perfect": 60203, "dependency_mismatch": 361479, "value_mismatch": 316348, "answer_mismatch": 269664, "extra_nodes": 15836, "missing_nodes": 224953, "perfect_with_extra": 42}}, "results_path": "results/Qwen3-100M/composition/graphs/checkpoint-21414_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1177145, "value_mismatch": 964850, "dependency_mismatch": 1227859, "answer_mismatch": 827446, "extra_nodes": 63361, "missing_nodes": 692119, "perfect_with_extra": 373, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M/composition/graphs/composition_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 387146, "dependency_mismatch": 414388, "value_mismatch": 325955, "answer_mismatch": 280423, "missing_nodes": 245137, "extra_nodes": 21940, "perfect_with_extra": 119}, "teachers_in_school": {"total": 821120, "perfect": 399816, "dependency_mismatch": 411499, "value_mismatch": 324715, "answer_mismatch": 277238, "extra_nodes": 19643, "missing_nodes": 228796, "perfect_with_extra": 89}, "movie_festival_awards": {"total": 801152, "perfect": 390183, "value_mismatch": 314180, "dependency_mismatch": 401972, "answer_mismatch": 269785, "extra_nodes": 21778, "missing_nodes": 218186, "perfect_with_extra": 165}}}} diff --git a/Qwen3-100M/composition/graphs/results.json b/Qwen3-100M/composition/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..2c56a40e48b657277c0d461b33d95278d59e487b --- /dev/null +++ b/Qwen3-100M/composition/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21414_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.990625, + "pass@2": 0.990625, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9587301587301588, + "pass@2": 0.9746031746031746, + "pass@4": 0.9873015873015873, + "pass@8": 0.9936507936507937, + "pass@16": 0.9936507936507937, + "pass@32": 0.9936507936507937, + "pass@64": 0.9968253968253968, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9610778443113772, + "pass@2": 0.9700598802395209, + "pass@4": 0.9820359281437125, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8816568047337278, + "pass@2": 0.9437869822485208, + "pass@4": 0.9644970414201184, + "pass@8": 0.9881656804733728, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7852564102564102, + "pass@2": 0.8717948717948718, + "pass@4": 0.9391025641025641, + "pass@8": 0.9839743589743589, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.6732394366197183, + "pass@2": 0.7915492957746478, + "pass@4": 0.8676056338028169, + "pass@8": 0.9492957746478873, + "pass@16": 0.9802816901408451, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5693215339233039, + "pass@2": 0.6696165191740413, + "pass@4": 0.7463126843657817, + "pass@8": 0.831858407079646, + "pass@16": 0.8731563421828908, + "pass@32": 0.9056047197640118, + "pass@64": 0.9262536873156342, + "pass@128": 0.9321533923303835 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9587301587301588, + "6": 0.9610778443113772, + "7": 0.8816568047337278, + "8": 0.7852564102564102, + "9": 0.6732394366197183, + "10": 0.5693215339233039 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9746031746031746, + "6": 0.9700598802395209, + "7": 0.9437869822485208, + "8": 0.8717948717948718, + "9": 0.7915492957746478, + "10": 0.6696165191740413 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9820359281437125, + "7": 0.9644970414201184, + "8": 0.9391025641025641, + "9": 0.8676056338028169, + "10": 0.7463126843657817 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 0.9970059880239521, + "7": 0.9881656804733728, + "8": 0.9839743589743589, + "9": 0.9492957746478873, + "10": 0.831858407079646 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 0.9802816901408451, + "10": 0.8731563421828908 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9936507936507937, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9943661971830986, + "10": 0.9056047197640118 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9262536873156342 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9968253968253968, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9321533923303835 + } + }, + "total_pass_at_k": { + "pass@1": 0.8655859506923337, + "pass@2": 0.9101654846335697, + "pass@4": 0.9412360688956434, + "pass@8": 0.9706180344478217, + "pass@16": 0.9821006416751098, + "pass@32": 0.9878419452887538, + "pass@64": 0.9908814589665653, + "pass@128": 0.9918946301925026 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.95, + "pass@2": 0.9794117647058823, + "pass@4": 0.9882352941176471, + "pass@8": 0.9941176470588236, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9842271293375394, + "pass@4": 0.9936908517350158, + "pass@8": 0.9968454258675079, + "pass@16": 0.9968454258675079, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8639240506329114, + "pass@2": 0.9177215189873418, + "pass@4": 0.9620253164556962, + "pass@8": 0.9778481012658228, + "pass@16": 0.9936708860759493, + "pass@32": 0.9968354430379747, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7567567567567568, + "pass@2": 0.8288288288288288, + "pass@4": 0.9159159159159159, + "pass@8": 0.975975975975976, + "pass@16": 0.996996996996997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.729903536977492, + "pass@2": 0.819935691318328, + "pass@4": 0.8713826366559485, + "pass@8": 0.9228295819935691, + "pass@16": 0.9710610932475884, + "pass@32": 0.9903536977491961, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5552238805970149, + "pass@2": 0.6507462686567164, + "pass@4": 0.7432835820895523, + "pass@8": 0.8477611940298507, + "pass@16": 0.8865671641791045, + "pass@32": 0.9283582089552239, + "pass@64": 0.9462686567164179, + "pass@128": 0.9522388059701492 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.95, + "6": 0.9621451104100947, + "7": 0.8639240506329114, + "8": 0.7567567567567568, + "9": 0.729903536977492, + "10": 0.5552238805970149 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9794117647058823, + "6": 0.9842271293375394, + "7": 0.9177215189873418, + "8": 0.8288288288288288, + "9": 0.819935691318328, + "10": 0.6507462686567164 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9936908517350158, + "7": 0.9620253164556962, + "8": 0.9159159159159159, + "9": 0.8713826366559485, + "10": 0.7432835820895523 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9941176470588236, + "6": 0.9968454258675079, + "7": 0.9778481012658228, + "8": 0.975975975975976, + "9": 0.9228295819935691, + "10": 0.8477611940298507 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9710610932475884, + "10": 0.8865671641791045 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 0.9968354430379747, + "8": 1.0, + "9": 0.9903536977491961, + "10": 0.9283582089552239 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9462686567164179 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9522388059701492 + } + }, + "total_pass_at_k": { + "pass@1": 0.8694342149313693, + "pass@2": 0.9096083026447941, + "pass@4": 0.942417140944091, + "pass@8": 0.9688650820220958, + "pass@16": 0.9829260127217945, + "pass@32": 0.9906260462002009, + "pass@64": 0.9939738868429863, + "pass@128": 0.9946434549715434 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9940828402366864, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9478260869565217, + "pass@2": 0.9855072463768116, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9627507163323782, + "pass@2": 0.9885386819484241, + "pass@4": 0.9885386819484241, + "pass@8": 0.994269340974212, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8352601156069365, + "pass@2": 0.930635838150289, + "pass@4": 0.9682080924855492, + "pass@8": 0.9971098265895953, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7690140845070422, + "pass@2": 0.8535211267605634, + "pass@4": 0.9323943661971831, + "pass@8": 0.9661971830985916, + "pass@16": 0.9887323943661972, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.6796407185628742, + "pass@2": 0.7664670658682635, + "pass@4": 0.8502994011976048, + "pass@8": 0.9401197604790419, + "pass@16": 0.9820359281437125, + "pass@32": 0.9940119760479041, + "pass@64": 0.9970059880239521, + "pass@128": 0.9970059880239521 + }, + "10": { + "pass@1": 0.6319018404907976, + "pass@2": 0.7147239263803681, + "pass@4": 0.7883435582822086, + "pass@8": 0.8374233128834356, + "pass@16": 0.8834355828220859, + "pass@32": 0.9049079754601227, + "pass@64": 0.9325153374233128, + "pass@128": 0.9447852760736196 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.985207100591716, + "5": 0.9478260869565217, + "6": 0.9627507163323782, + "7": 0.8352601156069365, + "8": 0.7690140845070422, + "9": 0.6796407185628742, + "10": 0.6319018404907976 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9855072463768116, + "6": 0.9885386819484241, + "7": 0.930635838150289, + "8": 0.8535211267605634, + "9": 0.7664670658682635, + "10": 0.7147239263803681 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9885386819484241, + "7": 0.9682080924855492, + "8": 0.9323943661971831, + "9": 0.8502994011976048, + "10": 0.7883435582822086 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.994269340974212, + "7": 0.9971098265895953, + "8": 0.9661971830985916, + "9": 0.9401197604790419, + "10": 0.8374233128834356 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9887323943661972, + "9": 0.9820359281437125, + "10": 0.8834355828220859 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.9049079754601227 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9325153374233128 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9447852760736196 + } + }, + "total_pass_at_k": { + "pass@1": 0.8679554390563565, + "pass@2": 0.9154652686762779, + "pass@4": 0.9482306684141546, + "pass@8": 0.9711664482306684, + "pass@16": 0.9842726081258192, + "pass@32": 0.9891874180865007, + "pass@64": 0.9924639580602883, + "pass@128": 0.9937745740498034 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.988, + "pass@2": 0.993, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.952, + "pass@2": 0.98, + "pass@4": 0.992, + "pass@8": 0.996, + "pass@16": 0.998, + "pass@32": 0.998, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "6": { + "pass@1": 0.962, + "pass@2": 0.981, + "pass@4": 0.988, + "pass@8": 0.996, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.86, + "pass@2": 0.931, + "pass@4": 0.965, + "pass@8": 0.988, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.77, + "pass@2": 0.851, + "pass@4": 0.929, + "pass@8": 0.975, + "pass@16": 0.995, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.693, + "pass@2": 0.792, + "pass@4": 0.863, + "pass@8": 0.938, + "pass@16": 0.978, + "pass@32": 0.993, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.585, + "pass@2": 0.678, + "pass@4": 0.759, + "pass@8": 0.839, + "pass@16": 0.881, + "pass@32": 0.913, + "pass@64": 0.935, + "pass@128": 0.943 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.999, + "4": 0.988, + "5": 0.952, + "6": 0.962, + "7": 0.86, + "8": 0.77, + "9": 0.693, + "10": 0.585 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.993, + "5": 0.98, + "6": 0.981, + "7": 0.931, + "8": 0.851, + "9": 0.792, + "10": 0.678 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.992, + "6": 0.988, + "7": 0.965, + "8": 0.929, + "9": 0.863, + "10": 0.759 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.996, + "6": 0.996, + "7": 0.988, + "8": 0.975, + "9": 0.938, + "10": 0.839 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 0.999, + "7": 0.997, + "8": 0.995, + "9": 0.978, + "10": 0.881 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.998, + "6": 1.0, + "7": 0.999, + "8": 1.0, + "9": 0.993, + "10": 0.913 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.998, + "10": 0.935 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.999, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.999, + "10": 0.943 + } + }, + "total_pass_at_k": { + "pass@1": 0.8676666666666667, + "pass@2": 0.9117777777777778, + "pass@4": 0.944, + "pass@8": 0.9702222222222222, + "pass@16": 0.9831111111111112, + "pass@32": 0.9892222222222222, + "pass@64": 0.9924444444444445, + "pass@128": 0.9934444444444445 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21414_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4475920679886686, + "pass@2": 0.5694050991501416, + "pass@4": 0.6713881019830028, + "pass@8": 0.7620396600566572, + "pass@16": 0.8470254957507082, + "pass@32": 0.8980169971671388, + "pass@64": 0.9235127478753541, + "pass@128": 0.9291784702549575 + }, + "12": { + "pass@1": 0.3898305084745763, + "pass@2": 0.5084745762711864, + "pass@4": 0.5847457627118644, + "pass@8": 0.6864406779661016, + "pass@16": 0.748587570621469, + "pass@32": 0.788135593220339, + "pass@64": 0.8163841807909604, + "pass@128": 0.8333333333333334 + }, + "13": { + "pass@1": 0.22884012539184953, + "pass@2": 0.30407523510971785, + "pass@4": 0.3761755485893417, + "pass@8": 0.45454545454545453, + "pass@16": 0.54858934169279, + "pass@32": 0.6144200626959248, + "pass@64": 0.658307210031348, + "pass@128": 0.6739811912225705 + }, + "14": { + "pass@1": 0.12149532710280374, + "pass@2": 0.1806853582554517, + "pass@4": 0.2554517133956386, + "pass@8": 0.32710280373831774, + "pass@16": 0.3956386292834891, + "pass@32": 0.4797507788161994, + "pass@64": 0.5202492211838006, + "pass@128": 0.5451713395638629 + }, + "15": { + "pass@1": 0.06936416184971098, + "pass@2": 0.11271676300578035, + "pass@4": 0.15028901734104047, + "pass@8": 0.17052023121387283, + "pass@16": 0.20809248554913296, + "pass@32": 0.2630057803468208, + "pass@64": 0.29190751445086704, + "pass@128": 0.3786127167630058 + }, + "16": { + "pass@1": 0.012903225806451613, + "pass@2": 0.025806451612903226, + "pass@4": 0.03870967741935484, + "pass@8": 0.05161290322580645, + "pass@16": 0.05806451612903226, + "pass@32": 0.06451612903225806, + "pass@64": 0.08064516129032258, + "pass@128": 0.1064516129032258 + }, + "17": { + "pass@1": 0.002890173410404624, + "pass@2": 0.008670520231213872, + "pass@4": 0.017341040462427744, + "pass@8": 0.023121387283236993, + "pass@16": 0.031791907514450865, + "pass@32": 0.046242774566473986, + "pass@64": 0.057803468208092484, + "pass@128": 0.07225433526011561 + }, + "18": { + "pass@1": 0.002857142857142857, + "pass@2": 0.002857142857142857, + "pass@4": 0.002857142857142857, + "pass@8": 0.005714285714285714, + "pass@16": 0.008571428571428572, + "pass@32": 0.011428571428571429, + "pass@64": 0.017142857142857144, + "pass@128": 0.022857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.003003003003003003, + "pass@64": 0.006006006006006006, + "pass@128": 0.009009009009009009 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.009009009009009009 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4475920679886686, + "12": 0.3898305084745763, + "13": 0.22884012539184953, + "14": 0.12149532710280374, + "15": 0.06936416184971098, + "16": 0.012903225806451613, + "17": 0.002890173410404624, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5694050991501416, + "12": 0.5084745762711864, + "13": 0.30407523510971785, + "14": 0.1806853582554517, + "15": 0.11271676300578035, + "16": 0.025806451612903226, + "17": 0.008670520231213872, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6713881019830028, + "12": 0.5847457627118644, + "13": 0.3761755485893417, + "14": 0.2554517133956386, + "15": 0.15028901734104047, + "16": 0.03870967741935484, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@8": { + "11": 0.7620396600566572, + "12": 0.6864406779661016, + "13": 0.45454545454545453, + "14": 0.32710280373831774, + "15": 0.17052023121387283, + "16": 0.05161290322580645, + "17": 0.023121387283236993, + "18": 0.005714285714285714, + "19": 0.003003003003003003, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.8470254957507082, + "12": 0.748587570621469, + "13": 0.54858934169279, + "14": 0.3956386292834891, + "15": 0.20809248554913296, + "16": 0.05806451612903226, + "17": 0.031791907514450865, + "18": 0.008571428571428572, + "19": 0.003003003003003003, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.8980169971671388, + "12": 0.788135593220339, + "13": 0.6144200626959248, + "14": 0.4797507788161994, + "15": 0.2630057803468208, + "16": 0.06451612903225806, + "17": 0.046242774566473986, + "18": 0.011428571428571429, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.9235127478753541, + "12": 0.8163841807909604, + "13": 0.658307210031348, + "14": 0.5202492211838006, + "15": 0.29190751445086704, + "16": 0.08064516129032258, + "17": 0.057803468208092484, + "18": 0.017142857142857144, + "19": 0.006006006006006006, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9291784702549575, + "12": 0.8333333333333334, + "13": 0.6739811912225705, + "14": 0.5451713395638629, + "15": 0.3786127167630058, + "16": 0.1064516129032258, + "17": 0.07225433526011561, + "18": 0.022857142857142857, + "19": 0.009009009009009009, + "20": 0.009009009009009009 + } + }, + "total_pass_at_k": { + "pass@1": 0.13016344725111442, + "pass@2": 0.17444279346210995, + "pass@4": 0.21337295690936106, + "pass@8": 0.2523031203566122, + "pass@16": 0.2888558692421991, + "pass@32": 0.3209509658246657, + "pass@64": 0.3411589895988113, + "pass@128": 0.3613670133729569 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5015873015873016, + "pass@2": 0.6253968253968254, + "pass@4": 0.7396825396825397, + "pass@8": 0.8285714285714286, + "pass@16": 0.8793650793650793, + "pass@32": 0.9142857142857143, + "pass@64": 0.946031746031746, + "pass@128": 0.9619047619047619 + }, + "12": { + "pass@1": 0.41896024464831805, + "pass@2": 0.5412844036697247, + "pass@4": 0.6391437308868502, + "pass@8": 0.7155963302752294, + "pass@16": 0.7706422018348624, + "pass@32": 0.8287461773700305, + "pass@64": 0.8470948012232415, + "pass@128": 0.8654434250764526 + }, + "13": { + "pass@1": 0.2345679012345679, + "pass@2": 0.3395061728395062, + "pass@4": 0.41975308641975306, + "pass@8": 0.4876543209876543, + "pass@16": 0.5462962962962963, + "pass@32": 0.5833333333333334, + "pass@64": 0.6327160493827161, + "pass@128": 0.6666666666666666 + }, + "14": { + "pass@1": 0.14540059347181009, + "pass@2": 0.19881305637982197, + "pass@4": 0.23738872403560832, + "pass@8": 0.2818991097922849, + "pass@16": 0.3353115727002967, + "pass@32": 0.4035608308605341, + "pass@64": 0.45103857566765576, + "pass@128": 0.5103857566765578 + }, + "15": { + "pass@1": 0.08024691358024691, + "pass@2": 0.14506172839506173, + "pass@4": 0.1697530864197531, + "pass@8": 0.20987654320987653, + "pass@16": 0.26851851851851855, + "pass@32": 0.3117283950617284, + "pass@64": 0.37962962962962965, + "pass@128": 0.4228395061728395 + }, + "16": { + "pass@1": 0.038348082595870206, + "pass@2": 0.0471976401179941, + "pass@4": 0.06489675516224189, + "pass@8": 0.07079646017699115, + "pass@16": 0.07964601769911504, + "pass@32": 0.08849557522123894, + "pass@64": 0.12389380530973451, + "pass@128": 0.15339233038348082 + }, + "17": { + "pass@1": 0.015060240963855422, + "pass@2": 0.024096385542168676, + "pass@4": 0.03313253012048193, + "pass@8": 0.045180722891566265, + "pass@16": 0.05120481927710843, + "pass@32": 0.060240963855421686, + "pass@64": 0.06325301204819277, + "pass@128": 0.06927710843373494 + }, + "18": { + "pass@1": 0.012903225806451613, + "pass@2": 0.016129032258064516, + "pass@4": 0.01935483870967742, + "pass@8": 0.02258064516129032, + "pass@16": 0.03225806451612903, + "pass@32": 0.03225806451612903, + "pass@64": 0.03870967741935484, + "pass@128": 0.06129032258064516 + }, + "19": { + "pass@1": 0.0030211480362537764, + "pass@2": 0.0030211480362537764, + "pass@4": 0.0030211480362537764, + "pass@8": 0.006042296072507553, + "pass@16": 0.012084592145015106, + "pass@32": 0.012084592145015106, + "pass@64": 0.012084592145015106, + "pass@128": 0.012084592145015106 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.003003003003003003, + "pass@16": 0.009009009009009009, + "pass@32": 0.009009009009009009, + "pass@64": 0.009009009009009009, + "pass@128": 0.012012012012012012 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5015873015873016, + "12": 0.41896024464831805, + "13": 0.2345679012345679, + "14": 0.14540059347181009, + "15": 0.08024691358024691, + "16": 0.038348082595870206, + "17": 0.015060240963855422, + "18": 0.012903225806451613, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@2": { + "11": 0.6253968253968254, + "12": 0.5412844036697247, + "13": 0.3395061728395062, + "14": 0.19881305637982197, + "15": 0.14506172839506173, + "16": 0.0471976401179941, + "17": 0.024096385542168676, + "18": 0.016129032258064516, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@4": { + "11": 0.7396825396825397, + "12": 0.6391437308868502, + "13": 0.41975308641975306, + "14": 0.23738872403560832, + "15": 0.1697530864197531, + "16": 0.06489675516224189, + "17": 0.03313253012048193, + "18": 0.01935483870967742, + "19": 0.0030211480362537764, + "20": 0.003003003003003003 + }, + "pass@8": { + "11": 0.8285714285714286, + "12": 0.7155963302752294, + "13": 0.4876543209876543, + "14": 0.2818991097922849, + "15": 0.20987654320987653, + "16": 0.07079646017699115, + "17": 0.045180722891566265, + "18": 0.02258064516129032, + "19": 0.006042296072507553, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.8793650793650793, + "12": 0.7706422018348624, + "13": 0.5462962962962963, + "14": 0.3353115727002967, + "15": 0.26851851851851855, + "16": 0.07964601769911504, + "17": 0.05120481927710843, + "18": 0.03225806451612903, + "19": 0.012084592145015106, + "20": 0.009009009009009009 + }, + "pass@32": { + "11": 0.9142857142857143, + "12": 0.8287461773700305, + "13": 0.5833333333333334, + "14": 0.4035608308605341, + "15": 0.3117283950617284, + "16": 0.08849557522123894, + "17": 0.060240963855421686, + "18": 0.03225806451612903, + "19": 0.012084592145015106, + "20": 0.009009009009009009 + }, + "pass@64": { + "11": 0.946031746031746, + "12": 0.8470948012232415, + "13": 0.6327160493827161, + "14": 0.45103857566765576, + "15": 0.37962962962962965, + "16": 0.12389380530973451, + "17": 0.06325301204819277, + "18": 0.03870967741935484, + "19": 0.012084592145015106, + "20": 0.009009009009009009 + }, + "pass@128": { + "11": 0.9619047619047619, + "12": 0.8654434250764526, + "13": 0.6666666666666666, + "14": 0.5103857566765578, + "15": 0.4228395061728395, + "16": 0.15339233038348082, + "17": 0.06927710843373494, + "18": 0.06129032258064516, + "19": 0.012084592145015106, + "20": 0.012012012012012012 + } + }, + "total_pass_at_k": { + "pass@1": 0.14333740831295844, + "pass@2": 0.19193154034229828, + "pass@4": 0.230440097799511, + "pass@8": 0.2643643031784841, + "pass@16": 0.29553789731051344, + "pass@32": 0.3215158924205379, + "pass@64": 0.3474938875305624, + "pass@128": 0.370721271393643 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5421686746987951, + "pass@2": 0.6295180722891566, + "pass@4": 0.7289156626506024, + "pass@8": 0.8042168674698795, + "pass@16": 0.8704819277108434, + "pass@32": 0.9246987951807228, + "pass@64": 0.9487951807228916, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.3949843260188088, + "pass@2": 0.4890282131661442, + "pass@4": 0.5987460815047022, + "pass@8": 0.6677115987460815, + "pass@16": 0.7429467084639498, + "pass@32": 0.7836990595611285, + "pass@64": 0.8119122257053292, + "pass@128": 0.8369905956112853 + }, + "13": { + "pass@1": 0.23809523809523808, + "pass@2": 0.31092436974789917, + "pass@4": 0.4257703081232493, + "pass@8": 0.5126050420168067, + "pass@16": 0.5574229691876751, + "pass@32": 0.6274509803921569, + "pass@64": 0.6554621848739496, + "pass@128": 0.6974789915966386 + }, + "14": { + "pass@1": 0.1783625730994152, + "pass@2": 0.23976608187134502, + "pass@4": 0.28362573099415206, + "pass@8": 0.34502923976608185, + "pass@16": 0.3830409356725146, + "pass@32": 0.4590643274853801, + "pass@64": 0.5058479532163743, + "pass@128": 0.5467836257309941 + }, + "15": { + "pass@1": 0.06060606060606061, + "pass@2": 0.08181818181818182, + "pass@4": 0.1393939393939394, + "pass@8": 0.18787878787878787, + "pass@16": 0.23333333333333334, + "pass@32": 0.2787878787878788, + "pass@64": 0.3212121212121212, + "pass@128": 0.38484848484848483 + }, + "16": { + "pass@1": 0.017094017094017096, + "pass@2": 0.02564102564102564, + "pass@4": 0.02564102564102564, + "pass@8": 0.04843304843304843, + "pass@16": 0.06552706552706553, + "pass@32": 0.07692307692307693, + "pass@64": 0.09116809116809117, + "pass@128": 0.11965811965811966 + }, + "17": { + "pass@1": 0.012422360248447204, + "pass@2": 0.018633540372670808, + "pass@4": 0.021739130434782608, + "pass@8": 0.027950310559006212, + "pass@16": 0.040372670807453416, + "pass@32": 0.055900621118012424, + "pass@64": 0.06832298136645963, + "pass@128": 0.08385093167701864 + }, + "18": { + "pass@1": 0.0058823529411764705, + "pass@2": 0.011764705882352941, + "pass@4": 0.014705882352941176, + "pass@8": 0.023529411764705882, + "pass@16": 0.023529411764705882, + "pass@32": 0.03235294117647059, + "pass@64": 0.047058823529411764, + "pass@128": 0.058823529411764705 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.011904761904761904 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0029940119760479044, + "pass@32": 0.0029940119760479044, + "pass@64": 0.0029940119760479044, + "pass@128": 0.0029940119760479044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5421686746987951, + "12": 0.3949843260188088, + "13": 0.23809523809523808, + "14": 0.1783625730994152, + "15": 0.06060606060606061, + "16": 0.017094017094017096, + "17": 0.012422360248447204, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6295180722891566, + "12": 0.4890282131661442, + "13": 0.31092436974789917, + "14": 0.23976608187134502, + "15": 0.08181818181818182, + "16": 0.02564102564102564, + "17": 0.018633540372670808, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.7289156626506024, + "12": 0.5987460815047022, + "13": 0.4257703081232493, + "14": 0.28362573099415206, + "15": 0.1393939393939394, + "16": 0.02564102564102564, + "17": 0.021739130434782608, + "18": 0.014705882352941176, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.8042168674698795, + "12": 0.6677115987460815, + "13": 0.5126050420168067, + "14": 0.34502923976608185, + "15": 0.18787878787878787, + "16": 0.04843304843304843, + "17": 0.027950310559006212, + "18": 0.023529411764705882, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8704819277108434, + "12": 0.7429467084639498, + "13": 0.5574229691876751, + "14": 0.3830409356725146, + "15": 0.23333333333333334, + "16": 0.06552706552706553, + "17": 0.040372670807453416, + "18": 0.023529411764705882, + "19": 0.0, + "20": 0.0029940119760479044 + }, + "pass@32": { + "11": 0.9246987951807228, + "12": 0.7836990595611285, + "13": 0.6274509803921569, + "14": 0.4590643274853801, + "15": 0.2787878787878788, + "16": 0.07692307692307693, + "17": 0.055900621118012424, + "18": 0.03235294117647059, + "19": 0.0, + "20": 0.0029940119760479044 + }, + "pass@64": { + "11": 0.9487951807228916, + "12": 0.8119122257053292, + "13": 0.6554621848739496, + "14": 0.5058479532163743, + "15": 0.3212121212121212, + "16": 0.09116809116809117, + "17": 0.06832298136645963, + "18": 0.047058823529411764, + "19": 0.0, + "20": 0.0029940119760479044 + }, + "pass@128": { + "11": 0.9668674698795181, + "12": 0.8369905956112853, + "13": 0.6974789915966386, + "14": 0.5467836257309941, + "15": 0.38484848484848483, + "16": 0.11965811965811966, + "17": 0.08385093167701864, + "18": 0.058823529411764705, + "19": 0.011904761904761904, + "20": 0.0029940119760479044 + } + }, + "total_pass_at_k": { + "pass@1": 0.14391911983348202, + "pass@2": 0.17960154623847754, + "pass@4": 0.22271781147784717, + "pass@8": 0.2607790663098424, + "pass@16": 0.29081177520071366, + "pass@32": 0.3232233125185846, + "pass@64": 0.34433541480820695, + "pass@128": 0.3702051739518287 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.496, + "pass@2": 0.607, + "pass@4": 0.712, + "pass@8": 0.797, + "pass@16": 0.865, + "pass@32": 0.912, + "pass@64": 0.939, + "pass@128": 0.952 + }, + "12": { + "pass@1": 0.401, + "pass@2": 0.513, + "pass@4": 0.607, + "pass@8": 0.69, + "pass@16": 0.754, + "pass@32": 0.8, + "pass@64": 0.825, + "pass@128": 0.845 + }, + "13": { + "pass@1": 0.234, + "pass@2": 0.318, + "pass@4": 0.408, + "pass@8": 0.486, + "pass@16": 0.551, + "pass@32": 0.609, + "pass@64": 0.649, + "pass@128": 0.68 + }, + "14": { + "pass@1": 0.149, + "pass@2": 0.207, + "pass@4": 0.259, + "pass@8": 0.318, + "pass@16": 0.371, + "pass@32": 0.447, + "pass@64": 0.492, + "pass@128": 0.534 + }, + "15": { + "pass@1": 0.07, + "pass@2": 0.113, + "pass@4": 0.153, + "pass@8": 0.189, + "pass@16": 0.236, + "pass@32": 0.284, + "pass@64": 0.33, + "pass@128": 0.395 + }, + "16": { + "pass@1": 0.023, + "pass@2": 0.033, + "pass@4": 0.043, + "pass@8": 0.057, + "pass@16": 0.068, + "pass@32": 0.077, + "pass@64": 0.099, + "pass@128": 0.127 + }, + "17": { + "pass@1": 0.01, + "pass@2": 0.017, + "pass@4": 0.024, + "pass@8": 0.032, + "pass@16": 0.041, + "pass@32": 0.054, + "pass@64": 0.063, + "pass@128": 0.075 + }, + "18": { + "pass@1": 0.007, + "pass@2": 0.01, + "pass@4": 0.012, + "pass@8": 0.017, + "pass@16": 0.021, + "pass@32": 0.025, + "pass@64": 0.034, + "pass@128": 0.047 + }, + "19": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.003, + "pass@16": 0.005, + "pass@32": 0.005, + "pass@64": 0.006, + "pass@128": 0.011 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.002, + "pass@8": 0.002, + "pass@16": 0.005, + "pass@32": 0.006, + "pass@64": 0.006, + "pass@128": 0.008 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.496, + "12": 0.401, + "13": 0.234, + "14": 0.149, + "15": 0.07, + "16": 0.023, + "17": 0.01, + "18": 0.007, + "19": 0.001, + "20": 0.0 + }, + "pass@2": { + "11": 0.607, + "12": 0.513, + "13": 0.318, + "14": 0.207, + "15": 0.113, + "16": 0.033, + "17": 0.017, + "18": 0.01, + "19": 0.001, + "20": 0.0 + }, + "pass@4": { + "11": 0.712, + "12": 0.607, + "13": 0.408, + "14": 0.259, + "15": 0.153, + "16": 0.043, + "17": 0.024, + "18": 0.012, + "19": 0.001, + "20": 0.002 + }, + "pass@8": { + "11": 0.797, + "12": 0.69, + "13": 0.486, + "14": 0.318, + "15": 0.189, + "16": 0.057, + "17": 0.032, + "18": 0.017, + "19": 0.003, + "20": 0.002 + }, + "pass@16": { + "11": 0.865, + "12": 0.754, + "13": 0.551, + "14": 0.371, + "15": 0.236, + "16": 0.068, + "17": 0.041, + "18": 0.021, + "19": 0.005, + "20": 0.005 + }, + "pass@32": { + "11": 0.912, + "12": 0.8, + "13": 0.609, + "14": 0.447, + "15": 0.284, + "16": 0.077, + "17": 0.054, + "18": 0.025, + "19": 0.005, + "20": 0.006 + }, + "pass@64": { + "11": 0.939, + "12": 0.825, + "13": 0.649, + "14": 0.492, + "15": 0.33, + "16": 0.099, + "17": 0.063, + "18": 0.034, + "19": 0.006, + "20": 0.006 + }, + "pass@128": { + "11": 0.952, + "12": 0.845, + "13": 0.68, + "14": 0.534, + "15": 0.395, + "16": 0.127, + "17": 0.075, + "18": 0.047, + "19": 0.011, + "20": 0.008 + } + }, + "total_pass_at_k": { + "pass@1": 0.1391, + "pass@2": 0.1819, + "pass@4": 0.2221, + "pass@8": 0.2591, + "pass@16": 0.2917, + "pass@32": 0.3219, + "pass@64": 0.3443, + "pass@128": 0.3674 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/checkpoint-17961_metrics.json b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/checkpoint-17961_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..4defbcf8c63f12139b4b702c9642ac72cf3398c4 --- /dev/null +++ b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/checkpoint-17961_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-17961", + "id": { + "answer_accuracy": 0.28959288194444444, + "avg_response_len": 156.12507638888889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.28959288194444444, + "pass@2": 0.34262635334646274, + "pass@4": 0.3948685813648338, + "pass@8": 0.4460377999799963, + "pass@16": 0.4961945316624793, + "pass@32": 0.5455726761815762, + "pass@64": 0.5936100661350605, + "pass@128": 0.6386666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1421484375, + "pass@2": 0.2019319635826771, + "pass@4": 0.2675311035808021, + "pass@8": 0.3354918311154123, + "pass@16": 0.4035763828069189, + "pass@32": 0.4722130277954208, + "pass@64": 0.5407401882605127, + "pass@128": 0.607 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999965551181103, + "pass@4": 0.9999999934383201, + "pass@8": 0.9999999999999993, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.6639921875, + "pass@2": 0.7774766240157491, + "pass@4": 0.8526536640419953, + "pass@8": 0.9029886475424349, + "pass@16": 0.937476737230175, + "pass@32": 0.9604196881305793, + "pass@64": 0.9749770567942502, + "pass@128": 0.985 + }, + "4": { + "pass@1": 0.1276171875, + "pass@2": 0.15832997047244102, + "pass@4": 0.1872849082302211, + "pass@8": 0.21563903290402045, + "pass@16": 0.24367596289131252, + "pass@32": 0.27073871545235473, + "pass@64": 0.29676134644022606, + "pass@128": 0.323 + }, + "5": { + "pass@1": 0.1388984375, + "pass@2": 0.18880118110236213, + "pass@4": 0.24112863357705266, + "pass@8": 0.29174905163792975, + "pass@16": 0.3374785038753412, + "pass@32": 0.37881865292092876, + "pass@64": 0.41741142906331263, + "pass@128": 0.45 + }, + "6": { + "pass@1": 0.11321875, + "pass@2": 0.16583169291338565, + "pass@4": 0.22559863198350183, + "pass@8": 0.29129661794694, + "pass@16": 0.36286936822505034, + "pass@32": 0.4390410688234122, + "pass@64": 0.515177409352268, + "pass@128": 0.584 + }, + "7": { + "pass@1": 0.1404140625, + "pass@2": 0.19221210629921273, + "pass@4": 0.24894790298087704, + "pass@8": 0.3090416719025815, + "pass@16": 0.37351155811611225, + "pass@32": 0.44140390476043995, + "pass@64": 0.5086901963961028, + "pass@128": 0.574 + }, + "8": { + "pass@1": 0.1426796875, + "pass@2": 0.20215809547244082, + "pass@4": 0.26888194516310454, + "pass@8": 0.3391558825231013, + "pass@16": 0.41011030518414526, + "pass@32": 0.4814887601621789, + "pass@64": 0.553404039866597, + "pass@128": 0.622 + }, + "9": { + "pass@1": 0.1374296875, + "pass@2": 0.19689899114173212, + "pass@4": 0.2617904492875886, + "pass@8": 0.32897746424743896, + "pass@16": 0.39705196663315656, + "pass@32": 0.46603026758888305, + "pass@64": 0.5353289290422811, + "pass@128": 0.603 + } + }, + "per_op_accuracy": { + "10": 0.1421484375, + "2": 0.9999375, + "3": 0.6639921875, + "4": 0.1276171875, + "5": 0.1388984375, + "6": 0.11321875, + "7": 0.1404140625, + "8": 0.1426796875, + "9": 0.1374296875 + }, + "per_op_avg_response_len": { + "10": 202.869234375, + "2": 107.42871875, + "3": 134.1750703125, + "4": 126.277578125, + "5": 146.054453125, + "6": 159.2597421875, + "7": 158.127859375, + "8": 178.793859375, + "9": 192.139171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 202.869234375, + "2": 107.42871875, + "3": 134.1750703125, + "4": 126.277578125, + "5": 146.054453125, + "6": 159.2597421875, + "7": 158.127859375, + "8": 178.793859375, + "9": 192.139171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 333611, + "answer_accuracy": 0.28959288194444444, + "avg_response_len": 156.12507638888889, + "resp_tokens_sum": 179856088, + "pass_at_k": { + "pass@1": 0.28959288194444444, + "pass@2": 0.34262635334646274, + "pass@4": 0.3948685813648338, + "pass@8": 0.4460377999799963, + "pass@16": 0.4961945316624793, + "pass@32": 0.5455726761815762, + "pass@64": 0.5936100661350605, + "pass@128": 0.6386666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1421484375, + "pass@2": 0.2019319635826771, + "pass@4": 0.2675311035808021, + "pass@8": 0.3354918311154123, + "pass@16": 0.4035763828069189, + "pass@32": 0.4722130277954208, + "pass@64": 0.5407401882605127, + "pass@128": 0.607 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999965551181103, + "pass@4": 0.9999999934383201, + "pass@8": 0.9999999999999993, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.6639921875, + "pass@2": 0.7774766240157491, + "pass@4": 0.8526536640419953, + "pass@8": 0.9029886475424349, + "pass@16": 0.937476737230175, + "pass@32": 0.9604196881305793, + "pass@64": 0.9749770567942502, + "pass@128": 0.985 + }, + "4": { + "pass@1": 0.1276171875, + "pass@2": 0.15832997047244102, + "pass@4": 0.1872849082302211, + "pass@8": 0.21563903290402045, + "pass@16": 0.24367596289131252, + "pass@32": 0.27073871545235473, + "pass@64": 0.29676134644022606, + "pass@128": 0.323 + }, + "5": { + "pass@1": 0.1388984375, + "pass@2": 0.18880118110236213, + "pass@4": 0.24112863357705266, + "pass@8": 0.29174905163792975, + "pass@16": 0.3374785038753412, + "pass@32": 0.37881865292092876, + "pass@64": 0.41741142906331263, + "pass@128": 0.45 + }, + "6": { + "pass@1": 0.11321875, + "pass@2": 0.16583169291338565, + "pass@4": 0.22559863198350183, + "pass@8": 0.29129661794694, + "pass@16": 0.36286936822505034, + "pass@32": 0.4390410688234122, + "pass@64": 0.515177409352268, + "pass@128": 0.584 + }, + "7": { + "pass@1": 0.1404140625, + "pass@2": 0.19221210629921273, + "pass@4": 0.24894790298087704, + "pass@8": 0.3090416719025815, + "pass@16": 0.37351155811611225, + "pass@32": 0.44140390476043995, + "pass@64": 0.5086901963961028, + "pass@128": 0.574 + }, + "8": { + "pass@1": 0.1426796875, + "pass@2": 0.20215809547244082, + "pass@4": 0.26888194516310454, + "pass@8": 0.3391558825231013, + "pass@16": 0.41011030518414526, + "pass@32": 0.4814887601621789, + "pass@64": 0.553404039866597, + "pass@128": 0.622 + }, + "9": { + "pass@1": 0.1374296875, + "pass@2": 0.19689899114173212, + "pass@4": 0.2617904492875886, + "pass@8": 0.32897746424743896, + "pass@16": 0.39705196663315656, + "pass@32": 0.46603026758888305, + "pass@64": 0.5353289290422811, + "pass@128": 0.603 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.10608828125, + "avg_response_len": 243.1132, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.10608828125, + "pass@2": 0.1592413877952765, + "pass@4": 0.2193713709786299, + "pass@8": 0.2798159978123999, + "pass@16": 0.3359345002967184, + "pass@32": 0.3873317531008492, + "pass@64": 0.43674999999745034, + "pass@128": 0.4865 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1257734375, + "pass@2": 0.1826700295275591, + "pass@4": 0.24666055193100825, + "pass@8": 0.3133372568044674, + "pass@16": 0.3800405689609709, + "pass@32": 0.4472412037392085, + "pass@64": 0.515816287026656, + "pass@128": 0.584 + }, + "12": { + "pass@1": 0.1406953125, + "pass@2": 0.19972293307086605, + "pass@4": 0.26151849718785136, + "pass@8": 0.32349719362794194, + "pass@16": 0.3854197876880773, + "pass@32": 0.44788263681730694, + "pass@64": 0.5095937035969575, + "pass@128": 0.565 + }, + "13": { + "pass@1": 0.1142734375, + "pass@2": 0.16852608267716554, + "pass@4": 0.22744862317210304, + "pass@8": 0.2846946144650094, + "pass@16": 0.3376277142016649, + "pass@32": 0.3894711349860603, + "pass@64": 0.4446024136889778, + "pass@128": 0.503 + }, + "14": { + "pass@1": 0.10925, + "pass@2": 0.16707775590551177, + "pass@4": 0.23257601977877743, + "pass@8": 0.29608074918542804, + "pass@16": 0.35325993392771365, + "pass@32": 0.4066002848127786, + "pass@64": 0.4614076166012878, + "pass@128": 0.518 + }, + "15": { + "pass@1": 0.0967421875, + "pass@2": 0.14799999999999985, + "pass@4": 0.20718057827146588, + "pass@8": 0.2672984495327348, + "pass@16": 0.32402858429363574, + "pass@32": 0.37904297506011464, + "pass@64": 0.433966666677465, + "pass@128": 0.487 + }, + "16": { + "pass@1": 0.1063125, + "pass@2": 0.16356815944881878, + "pass@4": 0.23028677905886746, + "pass@8": 0.2974334275274117, + "pass@16": 0.3579154279358942, + "pass@32": 0.4108232654174191, + "pass@64": 0.4614986546159244, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.0986328125, + "pass@2": 0.1506205708661416, + "pass@4": 0.20998232386576665, + "pass@8": 0.26870551219352806, + "pass@16": 0.3212954583569121, + "pass@32": 0.3670934345768469, + "pass@64": 0.40888332493563667, + "pass@128": 0.452 + }, + "18": { + "pass@1": 0.0931171875, + "pass@2": 0.1426550196850393, + "pass@4": 0.20064130183727028, + "pass@8": 0.25970073893490137, + "pass@16": 0.3130741493245069, + "pass@32": 0.3591104656288232, + "pass@64": 0.4007521196596996, + "pass@128": 0.444 + }, + "19": { + "pass@1": 0.0888359375, + "pass@2": 0.1349997539370076, + "pass@4": 0.18865937242219707, + "pass@8": 0.24352683545687182, + "pass@16": 0.29326001992536843, + "pass@32": 0.33419395607073804, + "pass@64": 0.36847114696644606, + "pass@128": 0.403 + }, + "20": { + "pass@1": 0.08725, + "pass@2": 0.13457357283464566, + "pass@4": 0.1887596622609673, + "pass@8": 0.24388520039567826, + "pass@16": 0.29342335835240535, + "pass@32": 0.3318581738992238, + "pass@64": 0.3625080662055037, + "pass@128": 0.394 + } + }, + "per_op_accuracy": { + "11": 0.1257734375, + "12": 0.1406953125, + "13": 0.1142734375, + "14": 0.10925, + "15": 0.0967421875, + "16": 0.1063125, + "17": 0.0986328125, + "18": 0.0931171875, + "19": 0.0888359375, + "20": 0.08725 + }, + "per_op_avg_response_len": { + "11": 216.623453125, + "12": 226.7830234375, + "13": 236.461453125, + "14": 252.041734375, + "15": 247.1566171875, + "16": 257.9425078125, + "17": 248.749, + "18": 252.7748359375, + "19": 246.3945390625, + "20": 246.2048359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 216.623453125, + "12": 226.7830234375, + "13": 236.461453125, + "14": 252.041734375, + "15": 247.1566171875, + "16": 257.9425078125, + "17": 248.749, + "18": 252.7748359375, + "19": 246.3945390625, + "20": 246.2048359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 135793, + "answer_accuracy": 0.10608828125, + "avg_response_len": 243.1132, + "resp_tokens_sum": 311184896, + "pass_at_k": { + "pass@1": 0.10608828125, + "pass@2": 0.1592413877952765, + "pass@4": 0.2193713709786299, + "pass@8": 0.2798159978123999, + "pass@16": 0.3359345002967184, + "pass@32": 0.3873317531008492, + "pass@64": 0.43674999999745034, + "pass@128": 0.4865 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1257734375, + "pass@2": 0.1826700295275591, + "pass@4": 0.24666055193100825, + "pass@8": 0.3133372568044674, + "pass@16": 0.3800405689609709, + "pass@32": 0.4472412037392085, + "pass@64": 0.515816287026656, + "pass@128": 0.584 + }, + "12": { + "pass@1": 0.1406953125, + "pass@2": 0.19972293307086605, + "pass@4": 0.26151849718785136, + "pass@8": 0.32349719362794194, + "pass@16": 0.3854197876880773, + "pass@32": 0.44788263681730694, + "pass@64": 0.5095937035969575, + "pass@128": 0.565 + }, + "13": { + "pass@1": 0.1142734375, + "pass@2": 0.16852608267716554, + "pass@4": 0.22744862317210304, + "pass@8": 0.2846946144650094, + "pass@16": 0.3376277142016649, + "pass@32": 0.3894711349860603, + "pass@64": 0.4446024136889778, + "pass@128": 0.503 + }, + "14": { + "pass@1": 0.10925, + "pass@2": 0.16707775590551177, + "pass@4": 0.23257601977877743, + "pass@8": 0.29608074918542804, + "pass@16": 0.35325993392771365, + "pass@32": 0.4066002848127786, + "pass@64": 0.4614076166012878, + "pass@128": 0.518 + }, + "15": { + "pass@1": 0.0967421875, + "pass@2": 0.14799999999999985, + "pass@4": 0.20718057827146588, + "pass@8": 0.2672984495327348, + "pass@16": 0.32402858429363574, + "pass@32": 0.37904297506011464, + "pass@64": 0.433966666677465, + "pass@128": 0.487 + }, + "16": { + "pass@1": 0.1063125, + "pass@2": 0.16356815944881878, + "pass@4": 0.23028677905886746, + "pass@8": 0.2974334275274117, + "pass@16": 0.3579154279358942, + "pass@32": 0.4108232654174191, + "pass@64": 0.4614986546159244, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.0986328125, + "pass@2": 0.1506205708661416, + "pass@4": 0.20998232386576665, + "pass@8": 0.26870551219352806, + "pass@16": 0.3212954583569121, + "pass@32": 0.3670934345768469, + "pass@64": 0.40888332493563667, + "pass@128": 0.452 + }, + "18": { + "pass@1": 0.0931171875, + "pass@2": 0.1426550196850393, + "pass@4": 0.20064130183727028, + "pass@8": 0.25970073893490137, + "pass@16": 0.3130741493245069, + "pass@32": 0.3591104656288232, + "pass@64": 0.4007521196596996, + "pass@128": 0.444 + }, + "19": { + "pass@1": 0.0888359375, + "pass@2": 0.1349997539370076, + "pass@4": 0.18865937242219707, + "pass@8": 0.24352683545687182, + "pass@16": 0.29326001992536843, + "pass@32": 0.33419395607073804, + "pass@64": 0.36847114696644606, + "pass@128": 0.403 + }, + "20": { + "pass@1": 0.08725, + "pass@2": 0.13457357283464566, + "pass@4": 0.1887596622609673, + "pass@8": 0.24388520039567826, + "pass@16": 0.29342335835240535, + "pass@32": 0.3318581738992238, + "pass@64": 0.3625080662055037, + "pass@128": 0.394 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.19301151315789475, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.19301151315789475, + "pass@2": 0.24610795042478487, + "pass@4": 0.3025016285299793, + "pass@8": 0.35855264094441963, + "pass@16": 0.41184714673314216, + "pass@32": 0.4622879798233027, + "pass@64": 0.511052136588958, + "pass@128": 0.5585789473684211 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1421484375, + "pass@2": 0.2019319635826771, + "pass@4": 0.2675311035808021, + "pass@8": 0.3354918311154123, + "pass@16": 0.4035763828069189, + "pass@32": 0.4722130277954208, + "pass@64": 0.5407401882605127, + "pass@128": 0.607 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999965551181103, + "pass@4": 0.9999999934383201, + "pass@8": 0.9999999999999993, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.6639921875, + "pass@2": 0.7774766240157491, + "pass@4": 0.8526536640419953, + "pass@8": 0.9029886475424349, + "pass@16": 0.937476737230175, + "pass@32": 0.9604196881305793, + "pass@64": 0.9749770567942502, + "pass@128": 0.985 + }, + "4": { + "pass@1": 0.1276171875, + "pass@2": 0.15832997047244102, + "pass@4": 0.1872849082302211, + "pass@8": 0.21563903290402045, + "pass@16": 0.24367596289131252, + "pass@32": 0.27073871545235473, + "pass@64": 0.29676134644022606, + "pass@128": 0.323 + }, + "5": { + "pass@1": 0.1388984375, + "pass@2": 0.18880118110236213, + "pass@4": 0.24112863357705266, + "pass@8": 0.29174905163792975, + "pass@16": 0.3374785038753412, + "pass@32": 0.37881865292092876, + "pass@64": 0.41741142906331263, + "pass@128": 0.45 + }, + "6": { + "pass@1": 0.11321875, + "pass@2": 0.16583169291338565, + "pass@4": 0.22559863198350183, + "pass@8": 0.29129661794694, + "pass@16": 0.36286936822505034, + "pass@32": 0.4390410688234122, + "pass@64": 0.515177409352268, + "pass@128": 0.584 + }, + "7": { + "pass@1": 0.1404140625, + "pass@2": 0.19221210629921273, + "pass@4": 0.24894790298087704, + "pass@8": 0.3090416719025815, + "pass@16": 0.37351155811611225, + "pass@32": 0.44140390476043995, + "pass@64": 0.5086901963961028, + "pass@128": 0.574 + }, + "8": { + "pass@1": 0.1426796875, + "pass@2": 0.20215809547244082, + "pass@4": 0.26888194516310454, + "pass@8": 0.3391558825231013, + "pass@16": 0.41011030518414526, + "pass@32": 0.4814887601621789, + "pass@64": 0.553404039866597, + "pass@128": 0.622 + }, + "9": { + "pass@1": 0.1374296875, + "pass@2": 0.19689899114173212, + "pass@4": 0.2617904492875886, + "pass@8": 0.32897746424743896, + "pass@16": 0.39705196663315656, + "pass@32": 0.46603026758888305, + "pass@64": 0.5353289290422811, + "pass@128": 0.603 + }, + "11": { + "pass@1": 0.1257734375, + "pass@2": 0.1826700295275591, + "pass@4": 0.24666055193100825, + "pass@8": 0.3133372568044674, + "pass@16": 0.3800405689609709, + "pass@32": 0.4472412037392085, + "pass@64": 0.515816287026656, + "pass@128": 0.584 + }, + "12": { + "pass@1": 0.1406953125, + "pass@2": 0.19972293307086605, + "pass@4": 0.26151849718785136, + "pass@8": 0.32349719362794194, + "pass@16": 0.3854197876880773, + "pass@32": 0.44788263681730694, + "pass@64": 0.5095937035969575, + "pass@128": 0.565 + }, + "13": { + "pass@1": 0.1142734375, + "pass@2": 0.16852608267716554, + "pass@4": 0.22744862317210304, + "pass@8": 0.2846946144650094, + "pass@16": 0.3376277142016649, + "pass@32": 0.3894711349860603, + "pass@64": 0.4446024136889778, + "pass@128": 0.503 + }, + "14": { + "pass@1": 0.10925, + "pass@2": 0.16707775590551177, + "pass@4": 0.23257601977877743, + "pass@8": 0.29608074918542804, + "pass@16": 0.35325993392771365, + "pass@32": 0.4066002848127786, + "pass@64": 0.4614076166012878, + "pass@128": 0.518 + }, + "15": { + "pass@1": 0.0967421875, + "pass@2": 0.14799999999999985, + "pass@4": 0.20718057827146588, + "pass@8": 0.2672984495327348, + "pass@16": 0.32402858429363574, + "pass@32": 0.37904297506011464, + "pass@64": 0.433966666677465, + "pass@128": 0.487 + }, + "16": { + "pass@1": 0.1063125, + "pass@2": 0.16356815944881878, + "pass@4": 0.23028677905886746, + "pass@8": 0.2974334275274117, + "pass@16": 0.3579154279358942, + "pass@32": 0.4108232654174191, + "pass@64": 0.4614986546159244, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.0986328125, + "pass@2": 0.1506205708661416, + "pass@4": 0.20998232386576665, + "pass@8": 0.26870551219352806, + "pass@16": 0.3212954583569121, + "pass@32": 0.3670934345768469, + "pass@64": 0.40888332493563667, + "pass@128": 0.452 + }, + "18": { + "pass@1": 0.0931171875, + "pass@2": 0.1426550196850393, + "pass@4": 0.20064130183727028, + "pass@8": 0.25970073893490137, + "pass@16": 0.3130741493245069, + "pass@32": 0.3591104656288232, + "pass@64": 0.4007521196596996, + "pass@128": 0.444 + }, + "19": { + "pass@1": 0.0888359375, + "pass@2": 0.1349997539370076, + "pass@4": 0.18865937242219707, + "pass@8": 0.24352683545687182, + "pass@16": 0.29326001992536843, + "pass@32": 0.33419395607073804, + "pass@64": 0.36847114696644606, + "pass@128": 0.403 + }, + "20": { + "pass@1": 0.08725, + "pass@2": 0.13457357283464566, + "pass@4": 0.1887596622609673, + "pass@8": 0.24388520039567826, + "pass@16": 0.29342335835240535, + "pass@32": 0.3318581738992238, + "pass@64": 0.3625080662055037, + "pass@128": 0.394 + } + }, + "per_op_accuracy": { + "10": 0.1421484375, + "2": 0.9999375, + "3": 0.6639921875, + "4": 0.1276171875, + "5": 0.1388984375, + "6": 0.11321875, + "7": 0.1404140625, + "8": 0.1426796875, + "9": 0.1374296875, + "11": 0.1257734375, + "12": 0.1406953125, + "13": 0.1142734375, + "14": 0.10925, + "15": 0.0967421875, + "16": 0.1063125, + "17": 0.0986328125, + "18": 0.0931171875, + "19": 0.0888359375, + "20": 0.08725 + }, + "per_op_avg_response_len": { + "10": 202.869234375, + "2": 107.42871875, + "3": 134.1750703125, + "4": 126.277578125, + "5": 146.054453125, + "6": 159.2597421875, + "7": 158.127859375, + "8": 178.793859375, + "9": 192.139171875, + "11": 216.623453125, + "12": 226.7830234375, + "13": 236.461453125, + "14": 252.041734375, + "15": 247.1566171875, + "16": 257.9425078125, + "17": 248.749, + "18": 252.7748359375, + "19": 246.3945390625, + "20": 246.2048359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 202.869234375, + "2": 107.42871875, + "3": 134.1750703125, + "4": 126.277578125, + "5": 146.054453125, + "6": 159.2597421875, + "7": 158.127859375, + "8": 178.793859375, + "9": 192.139171875, + "11": 216.623453125, + "12": 226.7830234375, + "13": 236.461453125, + "14": 252.041734375, + "15": 247.1566171875, + "16": 257.9425078125, + "17": 248.749, + "18": 252.7748359375, + "19": 246.3945390625, + "20": 246.2048359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 469404, + "answer_accuracy": 0.19301151315789475, + "avg_response_len": 201.90829934210527, + "resp_tokens_sum": 491040984, + "pass_at_k": { + "pass@1": 0.19301151315789475, + "pass@2": 0.24610795042478487, + "pass@4": 0.3025016285299793, + "pass@8": 0.35855264094441963, + "pass@16": 0.41184714673314216, + "pass@32": 0.4622879798233027, + "pass@64": 0.511052136588958, + "pass@128": 0.5585789473684211 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1421484375, + "pass@2": 0.2019319635826771, + "pass@4": 0.2675311035808021, + "pass@8": 0.3354918311154123, + "pass@16": 0.4035763828069189, + "pass@32": 0.4722130277954208, + "pass@64": 0.5407401882605127, + "pass@128": 0.607 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999965551181103, + "pass@4": 0.9999999934383201, + "pass@8": 0.9999999999999993, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.6639921875, + "pass@2": 0.7774766240157491, + "pass@4": 0.8526536640419953, + "pass@8": 0.9029886475424349, + "pass@16": 0.937476737230175, + "pass@32": 0.9604196881305793, + "pass@64": 0.9749770567942502, + "pass@128": 0.985 + }, + "4": { + "pass@1": 0.1276171875, + "pass@2": 0.15832997047244102, + "pass@4": 0.1872849082302211, + "pass@8": 0.21563903290402045, + "pass@16": 0.24367596289131252, + "pass@32": 0.27073871545235473, + "pass@64": 0.29676134644022606, + "pass@128": 0.323 + }, + "5": { + "pass@1": 0.1388984375, + "pass@2": 0.18880118110236213, + "pass@4": 0.24112863357705266, + "pass@8": 0.29174905163792975, + "pass@16": 0.3374785038753412, + "pass@32": 0.37881865292092876, + "pass@64": 0.41741142906331263, + "pass@128": 0.45 + }, + "6": { + "pass@1": 0.11321875, + "pass@2": 0.16583169291338565, + "pass@4": 0.22559863198350183, + "pass@8": 0.29129661794694, + "pass@16": 0.36286936822505034, + "pass@32": 0.4390410688234122, + "pass@64": 0.515177409352268, + "pass@128": 0.584 + }, + "7": { + "pass@1": 0.1404140625, + "pass@2": 0.19221210629921273, + "pass@4": 0.24894790298087704, + "pass@8": 0.3090416719025815, + "pass@16": 0.37351155811611225, + "pass@32": 0.44140390476043995, + "pass@64": 0.5086901963961028, + "pass@128": 0.574 + }, + "8": { + "pass@1": 0.1426796875, + "pass@2": 0.20215809547244082, + "pass@4": 0.26888194516310454, + "pass@8": 0.3391558825231013, + "pass@16": 0.41011030518414526, + "pass@32": 0.4814887601621789, + "pass@64": 0.553404039866597, + "pass@128": 0.622 + }, + "9": { + "pass@1": 0.1374296875, + "pass@2": 0.19689899114173212, + "pass@4": 0.2617904492875886, + "pass@8": 0.32897746424743896, + "pass@16": 0.39705196663315656, + "pass@32": 0.46603026758888305, + "pass@64": 0.5353289290422811, + "pass@128": 0.603 + }, + "11": { + "pass@1": 0.1257734375, + "pass@2": 0.1826700295275591, + "pass@4": 0.24666055193100825, + "pass@8": 0.3133372568044674, + "pass@16": 0.3800405689609709, + "pass@32": 0.4472412037392085, + "pass@64": 0.515816287026656, + "pass@128": 0.584 + }, + "12": { + "pass@1": 0.1406953125, + "pass@2": 0.19972293307086605, + "pass@4": 0.26151849718785136, + "pass@8": 0.32349719362794194, + "pass@16": 0.3854197876880773, + "pass@32": 0.44788263681730694, + "pass@64": 0.5095937035969575, + "pass@128": 0.565 + }, + "13": { + "pass@1": 0.1142734375, + "pass@2": 0.16852608267716554, + "pass@4": 0.22744862317210304, + "pass@8": 0.2846946144650094, + "pass@16": 0.3376277142016649, + "pass@32": 0.3894711349860603, + "pass@64": 0.4446024136889778, + "pass@128": 0.503 + }, + "14": { + "pass@1": 0.10925, + "pass@2": 0.16707775590551177, + "pass@4": 0.23257601977877743, + "pass@8": 0.29608074918542804, + "pass@16": 0.35325993392771365, + "pass@32": 0.4066002848127786, + "pass@64": 0.4614076166012878, + "pass@128": 0.518 + }, + "15": { + "pass@1": 0.0967421875, + "pass@2": 0.14799999999999985, + "pass@4": 0.20718057827146588, + "pass@8": 0.2672984495327348, + "pass@16": 0.32402858429363574, + "pass@32": 0.37904297506011464, + "pass@64": 0.433966666677465, + "pass@128": 0.487 + }, + "16": { + "pass@1": 0.1063125, + "pass@2": 0.16356815944881878, + "pass@4": 0.23028677905886746, + "pass@8": 0.2974334275274117, + "pass@16": 0.3579154279358942, + "pass@32": 0.4108232654174191, + "pass@64": 0.4614986546159244, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.0986328125, + "pass@2": 0.1506205708661416, + "pass@4": 0.20998232386576665, + "pass@8": 0.26870551219352806, + "pass@16": 0.3212954583569121, + "pass@32": 0.3670934345768469, + "pass@64": 0.40888332493563667, + "pass@128": 0.452 + }, + "18": { + "pass@1": 0.0931171875, + "pass@2": 0.1426550196850393, + "pass@4": 0.20064130183727028, + "pass@8": 0.25970073893490137, + "pass@16": 0.3130741493245069, + "pass@32": 0.3591104656288232, + "pass@64": 0.4007521196596996, + "pass@128": 0.444 + }, + "19": { + "pass@1": 0.0888359375, + "pass@2": 0.1349997539370076, + "pass@4": 0.18865937242219707, + "pass@8": 0.24352683545687182, + "pass@16": 0.29326001992536843, + "pass@32": 0.33419395607073804, + "pass@64": 0.36847114696644606, + "pass@128": 0.403 + }, + "20": { + "pass@1": 0.08725, + "pass@2": 0.13457357283464566, + "pass@4": 0.1887596622609673, + "pass@8": 0.24388520039567826, + "pass@16": 0.29342335835240535, + "pass@32": 0.3318581738992238, + "pass@64": 0.3625080662055037, + "pass@128": 0.394 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..be811ad4887fb4ab5447572983c029b4b2d0493b --- /dev/null +++ b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2", "results_dir": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs", "per_file": [{"file": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/checkpoint-17961_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 131529, "value_mismatch": 689249, "dependency_mismatch": 762310, "answer_mismatch": 818386, "extra_nodes": 517504, "missing_nodes": 1018962, "perfect_with_extra": 69, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 689249, "dependency_mismatch": 762310, "answer_mismatch": 818386, "extra_nodes": 517504, "missing_nodes": 1018962, "perfect": 131529, "perfect_with_extra": 69}}, "results_path": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/checkpoint-17961_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/checkpoint-17961_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 979532, "dependency_mismatch": 1006638, "answer_mismatch": 1144210, "extra_nodes": 915564, "missing_nodes": 1279996, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "dependency_mismatch": 1006638, "extra_nodes": 915564, "missing_nodes": 1279996, "value_mismatch": 979532, "answer_mismatch": 1144210}}, "results_path": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/checkpoint-17961_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 131529, "value_mismatch": 1668781, "dependency_mismatch": 1768948, "answer_mismatch": 1962596, "extra_nodes": 1433068, "missing_nodes": 2298958, "perfect_with_extra": 69, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/0.99zoo_op2-20+0.01teacher_op2_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1668781, "dependency_mismatch": 1768948, "answer_mismatch": 1962596, "extra_nodes": 1433068, "missing_nodes": 2298958, "perfect": 131529, "perfect_with_extra": 69}}}} diff --git a/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..af90aa6f3b83bbfea820639d5ce8c2769c712d7d --- /dev/null +++ b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "checkpoint-17961_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.025, + "pass@2": 0.042, + "pass@4": 0.054, + "pass@8": 0.066, + "pass@16": 0.081, + "pass@32": 0.1, + "pass@64": 0.129, + "pass@128": 0.144 + }, + "4": { + "pass@1": 0.003, + "pass@2": 0.004, + "pass@4": 0.006, + "pass@8": 0.007, + "pass@16": 0.013, + "pass@32": 0.014, + "pass@64": 0.016, + "pass@128": 0.019 + }, + "5": { + "pass@1": 0.002, + "pass@2": 0.003, + "pass@4": 0.004, + "pass@8": 0.006, + "pass@16": 0.007, + "pass@32": 0.012, + "pass@64": 0.013, + "pass@128": 0.017 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.002, + "pass@128": 0.002 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.025, + "4": 0.003, + "5": 0.002, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 1.0, + "3": 0.042, + "4": 0.004, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 1.0, + "3": 0.054, + "4": 0.006, + "5": 0.004, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 1.0, + "3": 0.066, + "4": 0.007, + "5": 0.006, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 1.0, + "3": 0.081, + "4": 0.013, + "5": 0.007, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 1.0, + "3": 0.1, + "4": 0.014, + "5": 0.012, + "6": 0.001, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 1.0, + "3": 0.129, + "4": 0.016, + "5": 0.013, + "6": 0.002, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 1.0, + "3": 0.144, + "4": 0.019, + "5": 0.017, + "6": 0.002, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11444444444444445, + "pass@2": 0.11655555555555555, + "pass@4": 0.11822222222222223, + "pass@8": 0.11988888888888889, + "pass@16": 0.12233333333333334, + "pass@32": 0.12522222222222223, + "pass@64": 0.1288888888888889, + "pass@128": 0.13133333333333333 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.025, + "pass@2": 0.042, + "pass@4": 0.054, + "pass@8": 0.066, + "pass@16": 0.081, + "pass@32": 0.1, + "pass@64": 0.129, + "pass@128": 0.144 + }, + "4": { + "pass@1": 0.003, + "pass@2": 0.004, + "pass@4": 0.006, + "pass@8": 0.007, + "pass@16": 0.013, + "pass@32": 0.014, + "pass@64": 0.016, + "pass@128": 0.019 + }, + "5": { + "pass@1": 0.002, + "pass@2": 0.003, + "pass@4": 0.004, + "pass@8": 0.006, + "pass@16": 0.007, + "pass@32": 0.012, + "pass@64": 0.013, + "pass@128": 0.017 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.001, + "pass@64": 0.002, + "pass@128": 0.002 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.025, + "4": 0.003, + "5": 0.002, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 1.0, + "3": 0.042, + "4": 0.004, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 1.0, + "3": 0.054, + "4": 0.006, + "5": 0.004, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 1.0, + "3": 0.066, + "4": 0.007, + "5": 0.006, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 1.0, + "3": 0.081, + "4": 0.013, + "5": 0.007, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 1.0, + "3": 0.1, + "4": 0.014, + "5": 0.012, + "6": 0.001, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 1.0, + "3": 0.129, + "4": 0.016, + "5": 0.013, + "6": 0.002, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 1.0, + "3": 0.144, + "4": 0.019, + "5": 0.017, + "6": 0.002, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11444444444444445, + "pass@2": 0.11655555555555555, + "pass@4": 0.11822222222222223, + "pass@8": 0.11988888888888889, + "pass@16": 0.12233333333333334, + "pass@32": 0.12522222222222223, + "pass@64": 0.1288888888888889, + "pass@128": 0.13133333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-17961_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..2eb77d04a60bac902e8ffb47152eb723034b7018 --- /dev/null +++ b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-17961,0.28959288194444444,0.10608828125,0.19301151315789475,0.0,0.0,0.0,156.12507638888889,243.1132,1152000,1280000,2432000 diff --git a/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/summary.json b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..574b57541ce3ff7b1bad38de3bf1fb52d8e69320 --- /dev/null +++ b/Qwen3-100M/context/0.99zoo_op2-20+0.01teacher_op2/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-17961", + "id_acc": 0.28959288194444444, + "ood_acc": 0.10608828125, + "total_acc": 0.19301151315789475, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 156.12507638888889, + "ood_avg_resp_len": 243.1132, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 469404, + "answer_accuracy": 0.19301151315789475, + "avg_response_len": 201.90829934210527, + "resp_tokens_sum": 491040984, + "pass_at_k": { + "pass@1": 0.19301151315789475, + "pass@2": 0.24610795042478487, + "pass@4": 0.3025016285299793, + "pass@8": 0.35855264094441963, + "pass@16": 0.41184714673314216, + "pass@32": 0.4622879798233027, + "pass@64": 0.511052136588958, + "pass@128": 0.5585789473684211 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1421484375, + "pass@2": 0.2019319635826771, + "pass@4": 0.2675311035808021, + "pass@8": 0.3354918311154123, + "pass@16": 0.4035763828069189, + "pass@32": 0.4722130277954208, + "pass@64": 0.5407401882605127, + "pass@128": 0.607 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999965551181103, + "pass@4": 0.9999999934383201, + "pass@8": 0.9999999999999993, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.6639921875, + "pass@2": 0.7774766240157491, + "pass@4": 0.8526536640419953, + "pass@8": 0.9029886475424349, + "pass@16": 0.937476737230175, + "pass@32": 0.9604196881305793, + "pass@64": 0.9749770567942502, + "pass@128": 0.985 + }, + "4": { + "pass@1": 0.1276171875, + "pass@2": 0.15832997047244102, + "pass@4": 0.1872849082302211, + "pass@8": 0.21563903290402045, + "pass@16": 0.24367596289131252, + "pass@32": 0.27073871545235473, + "pass@64": 0.29676134644022606, + "pass@128": 0.323 + }, + "5": { + "pass@1": 0.1388984375, + "pass@2": 0.18880118110236213, + "pass@4": 0.24112863357705266, + "pass@8": 0.29174905163792975, + "pass@16": 0.3374785038753412, + "pass@32": 0.37881865292092876, + "pass@64": 0.41741142906331263, + "pass@128": 0.45 + }, + "6": { + "pass@1": 0.11321875, + "pass@2": 0.16583169291338565, + "pass@4": 0.22559863198350183, + "pass@8": 0.29129661794694, + "pass@16": 0.36286936822505034, + "pass@32": 0.4390410688234122, + "pass@64": 0.515177409352268, + "pass@128": 0.584 + }, + "7": { + "pass@1": 0.1404140625, + "pass@2": 0.19221210629921273, + "pass@4": 0.24894790298087704, + "pass@8": 0.3090416719025815, + "pass@16": 0.37351155811611225, + "pass@32": 0.44140390476043995, + "pass@64": 0.5086901963961028, + "pass@128": 0.574 + }, + "8": { + "pass@1": 0.1426796875, + "pass@2": 0.20215809547244082, + "pass@4": 0.26888194516310454, + "pass@8": 0.3391558825231013, + "pass@16": 0.41011030518414526, + "pass@32": 0.4814887601621789, + "pass@64": 0.553404039866597, + "pass@128": 0.622 + }, + "9": { + "pass@1": 0.1374296875, + "pass@2": 0.19689899114173212, + "pass@4": 0.2617904492875886, + "pass@8": 0.32897746424743896, + "pass@16": 0.39705196663315656, + "pass@32": 0.46603026758888305, + "pass@64": 0.5353289290422811, + "pass@128": 0.603 + }, + "11": { + "pass@1": 0.1257734375, + "pass@2": 0.1826700295275591, + "pass@4": 0.24666055193100825, + "pass@8": 0.3133372568044674, + "pass@16": 0.3800405689609709, + "pass@32": 0.4472412037392085, + "pass@64": 0.515816287026656, + "pass@128": 0.584 + }, + "12": { + "pass@1": 0.1406953125, + "pass@2": 0.19972293307086605, + "pass@4": 0.26151849718785136, + "pass@8": 0.32349719362794194, + "pass@16": 0.3854197876880773, + "pass@32": 0.44788263681730694, + "pass@64": 0.5095937035969575, + "pass@128": 0.565 + }, + "13": { + "pass@1": 0.1142734375, + "pass@2": 0.16852608267716554, + "pass@4": 0.22744862317210304, + "pass@8": 0.2846946144650094, + "pass@16": 0.3376277142016649, + "pass@32": 0.3894711349860603, + "pass@64": 0.4446024136889778, + "pass@128": 0.503 + }, + "14": { + "pass@1": 0.10925, + "pass@2": 0.16707775590551177, + "pass@4": 0.23257601977877743, + "pass@8": 0.29608074918542804, + "pass@16": 0.35325993392771365, + "pass@32": 0.4066002848127786, + "pass@64": 0.4614076166012878, + "pass@128": 0.518 + }, + "15": { + "pass@1": 0.0967421875, + "pass@2": 0.14799999999999985, + "pass@4": 0.20718057827146588, + "pass@8": 0.2672984495327348, + "pass@16": 0.32402858429363574, + "pass@32": 0.37904297506011464, + "pass@64": 0.433966666677465, + "pass@128": 0.487 + }, + "16": { + "pass@1": 0.1063125, + "pass@2": 0.16356815944881878, + "pass@4": 0.23028677905886746, + "pass@8": 0.2974334275274117, + "pass@16": 0.3579154279358942, + "pass@32": 0.4108232654174191, + "pass@64": 0.4614986546159244, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.0986328125, + "pass@2": 0.1506205708661416, + "pass@4": 0.20998232386576665, + "pass@8": 0.26870551219352806, + "pass@16": 0.3212954583569121, + "pass@32": 0.3670934345768469, + "pass@64": 0.40888332493563667, + "pass@128": 0.452 + }, + "18": { + "pass@1": 0.0931171875, + "pass@2": 0.1426550196850393, + "pass@4": 0.20064130183727028, + "pass@8": 0.25970073893490137, + "pass@16": 0.3130741493245069, + "pass@32": 0.3591104656288232, + "pass@64": 0.4007521196596996, + "pass@128": 0.444 + }, + "19": { + "pass@1": 0.0888359375, + "pass@2": 0.1349997539370076, + "pass@4": 0.18865937242219707, + "pass@8": 0.24352683545687182, + "pass@16": 0.29326001992536843, + "pass@32": 0.33419395607073804, + "pass@64": 0.36847114696644606, + "pass@128": 0.403 + }, + "20": { + "pass@1": 0.08725, + "pass@2": 0.13457357283464566, + "pass@4": 0.1887596622609673, + "pass@8": 0.24388520039567826, + "pass@16": 0.29342335835240535, + "pass@32": 0.3318581738992238, + "pass@64": 0.3625080662055037, + "pass@128": 0.394 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18039_metrics.json b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18039_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..ef5462fcea4bb3835580efcc48f5baf37644421d --- /dev/null +++ b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18039_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18039", + "id": { + "answer_accuracy": 0.2678029513888889, + "avg_response_len": 143.94357899305555, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2678029513888889, + "pass@2": 0.32717231244532596, + "pass@4": 0.39137174454235363, + "pass@8": 0.45450245549583085, + "pass@16": 0.5114464065629598, + "pass@32": 0.560969880201785, + "pass@64": 0.6049211110910968, + "pass@128": 0.6444444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.120171875, + "pass@2": 0.1845765255905512, + "pass@4": 0.25638461661042333, + "pass@8": 0.32468271814781274, + "pass@16": 0.3813800131130348, + "pass@32": 0.42692040797937, + "pass@64": 0.46937992101561565, + "pass@128": 0.515 + }, + "2": { + "pass@1": 0.999734375, + "pass@2": 0.9999965551181104, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.505421875, + "pass@2": 0.6108905019685037, + "pass@4": 0.7027766107986495, + "pass@8": 0.7832486371327387, + "pass@16": 0.8504031585351777, + "pass@32": 0.9030809264372809, + "pass@64": 0.9419219140067047, + "pass@128": 0.968 + }, + "4": { + "pass@1": 0.1529609375, + "pass@2": 0.20825799704724401, + "pass@4": 0.26808056074240705, + "pass@8": 0.32692932890817733, + "pass@16": 0.3807442844868449, + "pass@32": 0.427772580334604, + "pass@64": 0.46742228572792166, + "pass@128": 0.5 + }, + "5": { + "pass@1": 0.125765625, + "pass@2": 0.17454441437007864, + "pass@4": 0.22857620350581168, + "pass@8": 0.2852151041284449, + "pass@16": 0.34054437677879673, + "pass@32": 0.39037834154612494, + "pass@64": 0.43415115133440446, + "pass@128": 0.472 + }, + "6": { + "pass@1": 0.1275625, + "pass@2": 0.19318749999999962, + "pass@4": 0.2716715278402696, + "pass@8": 0.3551362466803586, + "pass@16": 0.4359238966217402, + "pass@32": 0.5110290756429484, + "pass@64": 0.5806887807601362, + "pass@128": 0.644 + }, + "7": { + "pass@1": 0.122828125, + "pass@2": 0.1829392224409449, + "pass@4": 0.25282114135733, + "pass@8": 0.3241583608664076, + "pass@16": 0.39027791997284617, + "pass@32": 0.45005677808798544, + "pass@64": 0.5046726186675862, + "pass@128": 0.553 + }, + "8": { + "pass@1": 0.1354921875, + "pass@2": 0.20422428641732263, + "pass@4": 0.2816454187289086, + "pass@8": 0.3586060594021748, + "pass@16": 0.42847405183941994, + "pass@32": 0.49112355691398996, + "pass@64": 0.5491285932650029, + "pass@128": 0.602 + }, + "9": { + "pass@1": 0.1202890625, + "pass@2": 0.1859338090551182, + "pass@4": 0.26038962457817744, + "pass@8": 0.3325456441962499, + "pass@16": 0.3952699577186529, + "pass@32": 0.44836725487375767, + "pass@64": 0.4969247350424958, + "pass@128": 0.546 + } + }, + "per_op_accuracy": { + "10": 0.120171875, + "2": 0.999734375, + "3": 0.505421875, + "4": 0.1529609375, + "5": 0.125765625, + "6": 0.1275625, + "7": 0.122828125, + "8": 0.1354921875, + "9": 0.1202890625 + }, + "per_op_avg_response_len": { + "10": 169.3360078125, + "2": 107.5119921875, + "3": 117.0876875, + "4": 123.8172890625, + "5": 140.26375, + "6": 151.924515625, + "7": 148.0960625, + "8": 164.6956953125, + "9": 172.7592109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 169.3360078125, + "2": 107.5119921875, + "3": 117.0876875, + "4": 123.8172890625, + "5": 140.26375, + "6": 151.924515625, + "7": 148.0960625, + "8": 164.6956953125, + "9": 172.7592109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 308509, + "answer_accuracy": 0.2678029513888889, + "avg_response_len": 143.94357899305555, + "resp_tokens_sum": 165823003, + "pass_at_k": { + "pass@1": 0.2678029513888889, + "pass@2": 0.32717231244532596, + "pass@4": 0.39137174454235363, + "pass@8": 0.45450245549583085, + "pass@16": 0.5114464065629598, + "pass@32": 0.560969880201785, + "pass@64": 0.6049211110910968, + "pass@128": 0.6444444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.120171875, + "pass@2": 0.1845765255905512, + "pass@4": 0.25638461661042333, + "pass@8": 0.32468271814781274, + "pass@16": 0.3813800131130348, + "pass@32": 0.42692040797937, + "pass@64": 0.46937992101561565, + "pass@128": 0.515 + }, + "2": { + "pass@1": 0.999734375, + "pass@2": 0.9999965551181104, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.505421875, + "pass@2": 0.6108905019685037, + "pass@4": 0.7027766107986495, + "pass@8": 0.7832486371327387, + "pass@16": 0.8504031585351777, + "pass@32": 0.9030809264372809, + "pass@64": 0.9419219140067047, + "pass@128": 0.968 + }, + "4": { + "pass@1": 0.1529609375, + "pass@2": 0.20825799704724401, + "pass@4": 0.26808056074240705, + "pass@8": 0.32692932890817733, + "pass@16": 0.3807442844868449, + "pass@32": 0.427772580334604, + "pass@64": 0.46742228572792166, + "pass@128": 0.5 + }, + "5": { + "pass@1": 0.125765625, + "pass@2": 0.17454441437007864, + "pass@4": 0.22857620350581168, + "pass@8": 0.2852151041284449, + "pass@16": 0.34054437677879673, + "pass@32": 0.39037834154612494, + "pass@64": 0.43415115133440446, + "pass@128": 0.472 + }, + "6": { + "pass@1": 0.1275625, + "pass@2": 0.19318749999999962, + "pass@4": 0.2716715278402696, + "pass@8": 0.3551362466803586, + "pass@16": 0.4359238966217402, + "pass@32": 0.5110290756429484, + "pass@64": 0.5806887807601362, + "pass@128": 0.644 + }, + "7": { + "pass@1": 0.122828125, + "pass@2": 0.1829392224409449, + "pass@4": 0.25282114135733, + "pass@8": 0.3241583608664076, + "pass@16": 0.39027791997284617, + "pass@32": 0.45005677808798544, + "pass@64": 0.5046726186675862, + "pass@128": 0.553 + }, + "8": { + "pass@1": 0.1354921875, + "pass@2": 0.20422428641732263, + "pass@4": 0.2816454187289086, + "pass@8": 0.3586060594021748, + "pass@16": 0.42847405183941994, + "pass@32": 0.49112355691398996, + "pass@64": 0.5491285932650029, + "pass@128": 0.602 + }, + "9": { + "pass@1": 0.1202890625, + "pass@2": 0.1859338090551182, + "pass@4": 0.26038962457817744, + "pass@8": 0.3325456441962499, + "pass@16": 0.3952699577186529, + "pass@32": 0.44836725487375767, + "pass@64": 0.4969247350424958, + "pass@128": 0.546 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.107984375, + "avg_response_len": 195.5853921875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.107984375, + "pass@2": 0.17416117125984262, + "pass@4": 0.25207665444319693, + "pass@8": 0.32351274484049874, + "pass@16": 0.3735105708064023, + "pass@32": 0.4036616220582521, + "pass@64": 0.4255359850374452, + "pass@128": 0.4473 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1277890625, + "pass@2": 0.19893959153543297, + "pass@4": 0.27852251134233186, + "pass@8": 0.35055435913210575, + "pass@16": 0.40497469620793947, + "pass@32": 0.4440919079127353, + "pass@64": 0.47735138850224945, + "pass@128": 0.511 + }, + "12": { + "pass@1": 0.131921875, + "pass@2": 0.20371678149606307, + "pass@4": 0.2822928592050986, + "pass@8": 0.35083657381639893, + "pass@16": 0.4006997280008851, + "pass@32": 0.4350507594503491, + "pass@64": 0.4625301729624927, + "pass@128": 0.49 + }, + "13": { + "pass@1": 0.10671875, + "pass@2": 0.16880167322834636, + "pass@4": 0.23927782817772775, + "pass@8": 0.30379911705615276, + "pass@16": 0.35148905071127345, + "pass@32": 0.3824129713136272, + "pass@64": 0.40608314530182293, + "pass@128": 0.431 + }, + "14": { + "pass@1": 0.1090546875, + "pass@2": 0.17565354330708663, + "pass@4": 0.25505934842519645, + "pass@8": 0.32899526995383227, + "pass@16": 0.3814744099856488, + "pass@32": 0.4126708516412164, + "pass@64": 0.43439935075478797, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1031640625, + "pass@2": 0.16833280019685046, + "pass@4": 0.24538877174728124, + "pass@8": 0.3161733998486826, + "pass@16": 0.3670585687351444, + "pass@32": 0.40074092351864204, + "pass@64": 0.42732743991054156, + "pass@128": 0.453 + }, + "16": { + "pass@1": 0.11321875, + "pass@2": 0.18650147637795272, + "pass@4": 0.27419264566929097, + "pass@8": 0.3548485199416686, + "pass@16": 0.41054390776905425, + "pass@32": 0.4420058000016596, + "pass@64": 0.4614154287718065, + "pass@128": 0.477 + }, + "17": { + "pass@1": 0.1094140625, + "pass@2": 0.17641646161417313, + "pass@4": 0.255266340551181, + "pass@8": 0.32690535029086165, + "pass@16": 0.3747943480731019, + "pass@32": 0.4017334323764408, + "pass@64": 0.4209359657143077, + "pass@128": 0.441 + }, + "18": { + "pass@1": 0.0980546875, + "pass@2": 0.1624542322834646, + "pass@4": 0.24187763113985722, + "pass@8": 0.31670832960549516, + "pass@16": 0.367754836030914, + "pass@32": 0.3958085143317505, + "pass@64": 0.4149853012051455, + "pass@128": 0.435 + }, + "19": { + "pass@1": 0.0902265625, + "pass@2": 0.15009645669291344, + "pass@4": 0.22361137842144718, + "pass@8": 0.29180410029151527, + "pass@16": 0.3368515905100927, + "pass@32": 0.360478942488469, + "pass@64": 0.3760498303132275, + "pass@128": 0.393 + }, + "20": { + "pass@1": 0.09028125, + "pass@2": 0.1506986958661418, + "pass@4": 0.2252772297525307, + "pass@8": 0.29450242846822444, + "pass@16": 0.3394645720399114, + "pass@32": 0.3616221175476457, + "pass@64": 0.374281826938126, + "pass@128": 0.386 + } + }, + "per_op_accuracy": { + "11": 0.1277890625, + "12": 0.131921875, + "13": 0.10671875, + "14": 0.1090546875, + "15": 0.1031640625, + "16": 0.11321875, + "17": 0.1094140625, + "18": 0.0980546875, + "19": 0.0902265625, + "20": 0.09028125 + }, + "per_op_avg_response_len": { + "11": 179.072234375, + "12": 180.3420390625, + "13": 178.3294921875, + "14": 190.645171875, + "15": 193.7206875, + "16": 210.2710234375, + "17": 208.2497421875, + "18": 209.6898828125, + "19": 199.3242734375, + "20": 206.209375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 179.072234375, + "12": 180.3420390625, + "13": 178.3294921875, + "14": 190.645171875, + "15": 193.7206875, + "16": 210.2710234375, + "17": 208.2497421875, + "18": 209.6898828125, + "19": 199.3242734375, + "20": 206.209375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 138220, + "answer_accuracy": 0.107984375, + "avg_response_len": 195.5853921875, + "resp_tokens_sum": 250349302, + "pass_at_k": { + "pass@1": 0.107984375, + "pass@2": 0.17416117125984262, + "pass@4": 0.25207665444319693, + "pass@8": 0.32351274484049874, + "pass@16": 0.3735105708064023, + "pass@32": 0.4036616220582521, + "pass@64": 0.4255359850374452, + "pass@128": 0.4473 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1277890625, + "pass@2": 0.19893959153543297, + "pass@4": 0.27852251134233186, + "pass@8": 0.35055435913210575, + "pass@16": 0.40497469620793947, + "pass@32": 0.4440919079127353, + "pass@64": 0.47735138850224945, + "pass@128": 0.511 + }, + "12": { + "pass@1": 0.131921875, + "pass@2": 0.20371678149606307, + "pass@4": 0.2822928592050986, + "pass@8": 0.35083657381639893, + "pass@16": 0.4006997280008851, + "pass@32": 0.4350507594503491, + "pass@64": 0.4625301729624927, + "pass@128": 0.49 + }, + "13": { + "pass@1": 0.10671875, + "pass@2": 0.16880167322834636, + "pass@4": 0.23927782817772775, + "pass@8": 0.30379911705615276, + "pass@16": 0.35148905071127345, + "pass@32": 0.3824129713136272, + "pass@64": 0.40608314530182293, + "pass@128": 0.431 + }, + "14": { + "pass@1": 0.1090546875, + "pass@2": 0.17565354330708663, + "pass@4": 0.25505934842519645, + "pass@8": 0.32899526995383227, + "pass@16": 0.3814744099856488, + "pass@32": 0.4126708516412164, + "pass@64": 0.43439935075478797, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1031640625, + "pass@2": 0.16833280019685046, + "pass@4": 0.24538877174728124, + "pass@8": 0.3161733998486826, + "pass@16": 0.3670585687351444, + "pass@32": 0.40074092351864204, + "pass@64": 0.42732743991054156, + "pass@128": 0.453 + }, + "16": { + "pass@1": 0.11321875, + "pass@2": 0.18650147637795272, + "pass@4": 0.27419264566929097, + "pass@8": 0.3548485199416686, + "pass@16": 0.41054390776905425, + "pass@32": 0.4420058000016596, + "pass@64": 0.4614154287718065, + "pass@128": 0.477 + }, + "17": { + "pass@1": 0.1094140625, + "pass@2": 0.17641646161417313, + "pass@4": 0.255266340551181, + "pass@8": 0.32690535029086165, + "pass@16": 0.3747943480731019, + "pass@32": 0.4017334323764408, + "pass@64": 0.4209359657143077, + "pass@128": 0.441 + }, + "18": { + "pass@1": 0.0980546875, + "pass@2": 0.1624542322834646, + "pass@4": 0.24187763113985722, + "pass@8": 0.31670832960549516, + "pass@16": 0.367754836030914, + "pass@32": 0.3958085143317505, + "pass@64": 0.4149853012051455, + "pass@128": 0.435 + }, + "19": { + "pass@1": 0.0902265625, + "pass@2": 0.15009645669291344, + "pass@4": 0.22361137842144718, + "pass@8": 0.29180410029151527, + "pass@16": 0.3368515905100927, + "pass@32": 0.360478942488469, + "pass@64": 0.3760498303132275, + "pass@128": 0.393 + }, + "20": { + "pass@1": 0.09028125, + "pass@2": 0.1506986958661418, + "pass@4": 0.2252772297525307, + "pass@8": 0.29450242846822444, + "pass@16": 0.3394645720399114, + "pass@32": 0.3616221175476457, + "pass@64": 0.374281826938126, + "pass@128": 0.386 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.18368791118421052, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.18368791118421052, + "pass@2": 0.24664013287402164, + "pass@4": 0.3180585392269993, + "pass@8": 0.3855605025193377, + "pass@16": 0.4388485982700423, + "pass@32": 0.47817606012624636, + "pass@64": 0.5105078868523447, + "pass@128": 0.5406842105263158 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.120171875, + "pass@2": 0.1845765255905512, + "pass@4": 0.25638461661042333, + "pass@8": 0.32468271814781274, + "pass@16": 0.3813800131130348, + "pass@32": 0.42692040797937, + "pass@64": 0.46937992101561565, + "pass@128": 0.515 + }, + "2": { + "pass@1": 0.999734375, + "pass@2": 0.9999965551181104, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.505421875, + "pass@2": 0.6108905019685037, + "pass@4": 0.7027766107986495, + "pass@8": 0.7832486371327387, + "pass@16": 0.8504031585351777, + "pass@32": 0.9030809264372809, + "pass@64": 0.9419219140067047, + "pass@128": 0.968 + }, + "4": { + "pass@1": 0.1529609375, + "pass@2": 0.20825799704724401, + "pass@4": 0.26808056074240705, + "pass@8": 0.32692932890817733, + "pass@16": 0.3807442844868449, + "pass@32": 0.427772580334604, + "pass@64": 0.46742228572792166, + "pass@128": 0.5 + }, + "5": { + "pass@1": 0.125765625, + "pass@2": 0.17454441437007864, + "pass@4": 0.22857620350581168, + "pass@8": 0.2852151041284449, + "pass@16": 0.34054437677879673, + "pass@32": 0.39037834154612494, + "pass@64": 0.43415115133440446, + "pass@128": 0.472 + }, + "6": { + "pass@1": 0.1275625, + "pass@2": 0.19318749999999962, + "pass@4": 0.2716715278402696, + "pass@8": 0.3551362466803586, + "pass@16": 0.4359238966217402, + "pass@32": 0.5110290756429484, + "pass@64": 0.5806887807601362, + "pass@128": 0.644 + }, + "7": { + "pass@1": 0.122828125, + "pass@2": 0.1829392224409449, + "pass@4": 0.25282114135733, + "pass@8": 0.3241583608664076, + "pass@16": 0.39027791997284617, + "pass@32": 0.45005677808798544, + "pass@64": 0.5046726186675862, + "pass@128": 0.553 + }, + "8": { + "pass@1": 0.1354921875, + "pass@2": 0.20422428641732263, + "pass@4": 0.2816454187289086, + "pass@8": 0.3586060594021748, + "pass@16": 0.42847405183941994, + "pass@32": 0.49112355691398996, + "pass@64": 0.5491285932650029, + "pass@128": 0.602 + }, + "9": { + "pass@1": 0.1202890625, + "pass@2": 0.1859338090551182, + "pass@4": 0.26038962457817744, + "pass@8": 0.3325456441962499, + "pass@16": 0.3952699577186529, + "pass@32": 0.44836725487375767, + "pass@64": 0.4969247350424958, + "pass@128": 0.546 + }, + "11": { + "pass@1": 0.1277890625, + "pass@2": 0.19893959153543297, + "pass@4": 0.27852251134233186, + "pass@8": 0.35055435913210575, + "pass@16": 0.40497469620793947, + "pass@32": 0.4440919079127353, + "pass@64": 0.47735138850224945, + "pass@128": 0.511 + }, + "12": { + "pass@1": 0.131921875, + "pass@2": 0.20371678149606307, + "pass@4": 0.2822928592050986, + "pass@8": 0.35083657381639893, + "pass@16": 0.4006997280008851, + "pass@32": 0.4350507594503491, + "pass@64": 0.4625301729624927, + "pass@128": 0.49 + }, + "13": { + "pass@1": 0.10671875, + "pass@2": 0.16880167322834636, + "pass@4": 0.23927782817772775, + "pass@8": 0.30379911705615276, + "pass@16": 0.35148905071127345, + "pass@32": 0.3824129713136272, + "pass@64": 0.40608314530182293, + "pass@128": 0.431 + }, + "14": { + "pass@1": 0.1090546875, + "pass@2": 0.17565354330708663, + "pass@4": 0.25505934842519645, + "pass@8": 0.32899526995383227, + "pass@16": 0.3814744099856488, + "pass@32": 0.4126708516412164, + "pass@64": 0.43439935075478797, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1031640625, + "pass@2": 0.16833280019685046, + "pass@4": 0.24538877174728124, + "pass@8": 0.3161733998486826, + "pass@16": 0.3670585687351444, + "pass@32": 0.40074092351864204, + "pass@64": 0.42732743991054156, + "pass@128": 0.453 + }, + "16": { + "pass@1": 0.11321875, + "pass@2": 0.18650147637795272, + "pass@4": 0.27419264566929097, + "pass@8": 0.3548485199416686, + "pass@16": 0.41054390776905425, + "pass@32": 0.4420058000016596, + "pass@64": 0.4614154287718065, + "pass@128": 0.477 + }, + "17": { + "pass@1": 0.1094140625, + "pass@2": 0.17641646161417313, + "pass@4": 0.255266340551181, + "pass@8": 0.32690535029086165, + "pass@16": 0.3747943480731019, + "pass@32": 0.4017334323764408, + "pass@64": 0.4209359657143077, + "pass@128": 0.441 + }, + "18": { + "pass@1": 0.0980546875, + "pass@2": 0.1624542322834646, + "pass@4": 0.24187763113985722, + "pass@8": 0.31670832960549516, + "pass@16": 0.367754836030914, + "pass@32": 0.3958085143317505, + "pass@64": 0.4149853012051455, + "pass@128": 0.435 + }, + "19": { + "pass@1": 0.0902265625, + "pass@2": 0.15009645669291344, + "pass@4": 0.22361137842144718, + "pass@8": 0.29180410029151527, + "pass@16": 0.3368515905100927, + "pass@32": 0.360478942488469, + "pass@64": 0.3760498303132275, + "pass@128": 0.393 + }, + "20": { + "pass@1": 0.09028125, + "pass@2": 0.1506986958661418, + "pass@4": 0.2252772297525307, + "pass@8": 0.29450242846822444, + "pass@16": 0.3394645720399114, + "pass@32": 0.3616221175476457, + "pass@64": 0.374281826938126, + "pass@128": 0.386 + } + }, + "per_op_accuracy": { + "10": 0.120171875, + "2": 0.999734375, + "3": 0.505421875, + "4": 0.1529609375, + "5": 0.125765625, + "6": 0.1275625, + "7": 0.122828125, + "8": 0.1354921875, + "9": 0.1202890625, + "11": 0.1277890625, + "12": 0.131921875, + "13": 0.10671875, + "14": 0.1090546875, + "15": 0.1031640625, + "16": 0.11321875, + "17": 0.1094140625, + "18": 0.0980546875, + "19": 0.0902265625, + "20": 0.09028125 + }, + "per_op_avg_response_len": { + "10": 169.3360078125, + "2": 107.5119921875, + "3": 117.0876875, + "4": 123.8172890625, + "5": 140.26375, + "6": 151.924515625, + "7": 148.0960625, + "8": 164.6956953125, + "9": 172.7592109375, + "11": 179.072234375, + "12": 180.3420390625, + "13": 178.3294921875, + "14": 190.645171875, + "15": 193.7206875, + "16": 210.2710234375, + "17": 208.2497421875, + "18": 209.6898828125, + "19": 199.3242734375, + "20": 206.209375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 169.3360078125, + "2": 107.5119921875, + "3": 117.0876875, + "4": 123.8172890625, + "5": 140.26375, + "6": 151.924515625, + "7": 148.0960625, + "8": 164.6956953125, + "9": 172.7592109375, + "11": 179.072234375, + "12": 180.3420390625, + "13": 178.3294921875, + "14": 190.645171875, + "15": 193.7206875, + "16": 210.2710234375, + "17": 208.2497421875, + "18": 209.6898828125, + "19": 199.3242734375, + "20": 206.209375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 446729, + "answer_accuracy": 0.18368791118421052, + "avg_response_len": 171.12348067434212, + "resp_tokens_sum": 416172305, + "pass_at_k": { + "pass@1": 0.18368791118421052, + "pass@2": 0.24664013287402164, + "pass@4": 0.3180585392269993, + "pass@8": 0.3855605025193377, + "pass@16": 0.4388485982700423, + "pass@32": 0.47817606012624636, + "pass@64": 0.5105078868523447, + "pass@128": 0.5406842105263158 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.120171875, + "pass@2": 0.1845765255905512, + "pass@4": 0.25638461661042333, + "pass@8": 0.32468271814781274, + "pass@16": 0.3813800131130348, + "pass@32": 0.42692040797937, + "pass@64": 0.46937992101561565, + "pass@128": 0.515 + }, + "2": { + "pass@1": 0.999734375, + "pass@2": 0.9999965551181104, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.505421875, + "pass@2": 0.6108905019685037, + "pass@4": 0.7027766107986495, + "pass@8": 0.7832486371327387, + "pass@16": 0.8504031585351777, + "pass@32": 0.9030809264372809, + "pass@64": 0.9419219140067047, + "pass@128": 0.968 + }, + "4": { + "pass@1": 0.1529609375, + "pass@2": 0.20825799704724401, + "pass@4": 0.26808056074240705, + "pass@8": 0.32692932890817733, + "pass@16": 0.3807442844868449, + "pass@32": 0.427772580334604, + "pass@64": 0.46742228572792166, + "pass@128": 0.5 + }, + "5": { + "pass@1": 0.125765625, + "pass@2": 0.17454441437007864, + "pass@4": 0.22857620350581168, + "pass@8": 0.2852151041284449, + "pass@16": 0.34054437677879673, + "pass@32": 0.39037834154612494, + "pass@64": 0.43415115133440446, + "pass@128": 0.472 + }, + "6": { + "pass@1": 0.1275625, + "pass@2": 0.19318749999999962, + "pass@4": 0.2716715278402696, + "pass@8": 0.3551362466803586, + "pass@16": 0.4359238966217402, + "pass@32": 0.5110290756429484, + "pass@64": 0.5806887807601362, + "pass@128": 0.644 + }, + "7": { + "pass@1": 0.122828125, + "pass@2": 0.1829392224409449, + "pass@4": 0.25282114135733, + "pass@8": 0.3241583608664076, + "pass@16": 0.39027791997284617, + "pass@32": 0.45005677808798544, + "pass@64": 0.5046726186675862, + "pass@128": 0.553 + }, + "8": { + "pass@1": 0.1354921875, + "pass@2": 0.20422428641732263, + "pass@4": 0.2816454187289086, + "pass@8": 0.3586060594021748, + "pass@16": 0.42847405183941994, + "pass@32": 0.49112355691398996, + "pass@64": 0.5491285932650029, + "pass@128": 0.602 + }, + "9": { + "pass@1": 0.1202890625, + "pass@2": 0.1859338090551182, + "pass@4": 0.26038962457817744, + "pass@8": 0.3325456441962499, + "pass@16": 0.3952699577186529, + "pass@32": 0.44836725487375767, + "pass@64": 0.4969247350424958, + "pass@128": 0.546 + }, + "11": { + "pass@1": 0.1277890625, + "pass@2": 0.19893959153543297, + "pass@4": 0.27852251134233186, + "pass@8": 0.35055435913210575, + "pass@16": 0.40497469620793947, + "pass@32": 0.4440919079127353, + "pass@64": 0.47735138850224945, + "pass@128": 0.511 + }, + "12": { + "pass@1": 0.131921875, + "pass@2": 0.20371678149606307, + "pass@4": 0.2822928592050986, + "pass@8": 0.35083657381639893, + "pass@16": 0.4006997280008851, + "pass@32": 0.4350507594503491, + "pass@64": 0.4625301729624927, + "pass@128": 0.49 + }, + "13": { + "pass@1": 0.10671875, + "pass@2": 0.16880167322834636, + "pass@4": 0.23927782817772775, + "pass@8": 0.30379911705615276, + "pass@16": 0.35148905071127345, + "pass@32": 0.3824129713136272, + "pass@64": 0.40608314530182293, + "pass@128": 0.431 + }, + "14": { + "pass@1": 0.1090546875, + "pass@2": 0.17565354330708663, + "pass@4": 0.25505934842519645, + "pass@8": 0.32899526995383227, + "pass@16": 0.3814744099856488, + "pass@32": 0.4126708516412164, + "pass@64": 0.43439935075478797, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1031640625, + "pass@2": 0.16833280019685046, + "pass@4": 0.24538877174728124, + "pass@8": 0.3161733998486826, + "pass@16": 0.3670585687351444, + "pass@32": 0.40074092351864204, + "pass@64": 0.42732743991054156, + "pass@128": 0.453 + }, + "16": { + "pass@1": 0.11321875, + "pass@2": 0.18650147637795272, + "pass@4": 0.27419264566929097, + "pass@8": 0.3548485199416686, + "pass@16": 0.41054390776905425, + "pass@32": 0.4420058000016596, + "pass@64": 0.4614154287718065, + "pass@128": 0.477 + }, + "17": { + "pass@1": 0.1094140625, + "pass@2": 0.17641646161417313, + "pass@4": 0.255266340551181, + "pass@8": 0.32690535029086165, + "pass@16": 0.3747943480731019, + "pass@32": 0.4017334323764408, + "pass@64": 0.4209359657143077, + "pass@128": 0.441 + }, + "18": { + "pass@1": 0.0980546875, + "pass@2": 0.1624542322834646, + "pass@4": 0.24187763113985722, + "pass@8": 0.31670832960549516, + "pass@16": 0.367754836030914, + "pass@32": 0.3958085143317505, + "pass@64": 0.4149853012051455, + "pass@128": 0.435 + }, + "19": { + "pass@1": 0.0902265625, + "pass@2": 0.15009645669291344, + "pass@4": 0.22361137842144718, + "pass@8": 0.29180410029151527, + "pass@16": 0.3368515905100927, + "pass@32": 0.360478942488469, + "pass@64": 0.3760498303132275, + "pass@128": 0.393 + }, + "20": { + "pass@1": 0.09028125, + "pass@2": 0.1506986958661418, + "pass@4": 0.2252772297525307, + "pass@8": 0.29450242846822444, + "pass@16": 0.3394645720399114, + "pass@32": 0.3616221175476457, + "pass@64": 0.374281826938126, + "pass@128": 0.386 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..12562baafdeb4f2e4a864f08a4c21317872ca965 --- /dev/null +++ b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/context/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive", "results_dir": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18039_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 127867, "value_mismatch": 761658, "dependency_mismatch": 730495, "answer_mismatch": 843472, "extra_nodes": 602833, "missing_nodes": 1022686, "perfect_with_extra": 0, "unused_gold_entries": 10000, "template_stats": {"teachers_in_school": {"total": 1152000, "value_mismatch": 761658, "dependency_mismatch": 730495, "answer_mismatch": 843472, "missing_nodes": 1022686, "extra_nodes": 602833, "perfect": 127867}}, "results_path": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/checkpoint-18039_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/checkpoint-18039_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 0, "value_mismatch": 1064053, "dependency_mismatch": 941293, "answer_mismatch": 1141790, "extra_nodes": 946600, "missing_nodes": 1280000, "perfect_with_extra": 0, "unused_gold_entries": 9000, "template_stats": {"teachers_in_school": {"total": 1280000, "value_mismatch": 1064053, "dependency_mismatch": 941293, "answer_mismatch": 1141790, "extra_nodes": 946600, "missing_nodes": 1280000}}, "results_path": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/checkpoint-18039_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 127867, "value_mismatch": 1825711, "dependency_mismatch": 1671788, "answer_mismatch": 1985262, "extra_nodes": 1549433, "missing_nodes": 2302686, "perfect_with_extra": 0, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"teachers_in_school": {"total": 2432000, "value_mismatch": 1825711, "dependency_mismatch": 1671788, "answer_mismatch": 1985262, "missing_nodes": 2302686, "extra_nodes": 1549433, "perfect": 127867}}}} diff --git a/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..f08a97a8e2adf354195e6a65cf72f68b90d9465c --- /dev/null +++ b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/graphs/results.json @@ -0,0 +1,910 @@ +{ + "files": { + "checkpoint-18039_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.003, + "pass@8": 0.003, + "pass@16": 0.005, + "pass@32": 0.006, + "pass@64": 0.01, + "pass@128": 0.012 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.002, + "pass@16": 0.004, + "pass@32": 0.006, + "pass@64": 0.009, + "pass@128": 0.012 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.003, + "pass@16": 0.003, + "pass@32": 0.006, + "pass@64": 0.008, + "pass@128": 0.01 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 1.0, + "3": 0.001, + "4": 0.0, + "5": 0.001, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 1.0, + "3": 0.003, + "4": 0.0, + "5": 0.001, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 1.0, + "3": 0.003, + "4": 0.002, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 1.0, + "3": 0.005, + "4": 0.004, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 1.0, + "3": 0.006, + "4": 0.006, + "5": 0.006, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 1.0, + "3": 0.01, + "4": 0.009, + "5": 0.008, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 1.0, + "3": 0.012, + "4": 0.012, + "5": 0.01, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.111, + "pass@2": 0.11133333333333334, + "pass@4": 0.11155555555555556, + "pass@8": 0.112, + "pass@16": 0.11244444444444444, + "pass@32": 0.1131111111111111, + "pass@64": 0.11411111111111111, + "pass@128": 0.11488888888888889 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.003, + "pass@8": 0.003, + "pass@16": 0.005, + "pass@32": 0.006, + "pass@64": 0.01, + "pass@128": 0.012 + }, + "4": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.002, + "pass@16": 0.004, + "pass@32": 0.006, + "pass@64": 0.009, + "pass@128": 0.012 + }, + "5": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.003, + "pass@16": 0.003, + "pass@32": 0.006, + "pass@64": 0.008, + "pass@128": 0.01 + }, + "6": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "7": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "8": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "9": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "10": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.0, + "4": 0.0, + "5": 0.0, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@2": { + "2": 1.0, + "3": 0.001, + "4": 0.0, + "5": 0.001, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@4": { + "2": 1.0, + "3": 0.003, + "4": 0.0, + "5": 0.001, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@8": { + "2": 1.0, + "3": 0.003, + "4": 0.002, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@16": { + "2": 1.0, + "3": 0.005, + "4": 0.004, + "5": 0.003, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@32": { + "2": 1.0, + "3": 0.006, + "4": 0.006, + "5": 0.006, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@64": { + "2": 1.0, + "3": 0.01, + "4": 0.009, + "5": 0.008, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + }, + "pass@128": { + "2": 1.0, + "3": 0.012, + "4": 0.012, + "5": 0.01, + "6": 0.0, + "7": 0.0, + "8": 0.0, + "9": 0.0, + "10": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.111, + "pass@2": 0.11133333333333334, + "pass@4": 0.11155555555555556, + "pass@8": 0.112, + "pass@16": 0.11244444444444444, + "pass@32": 0.1131111111111111, + "pass@64": 0.11411111111111111, + "pass@128": 0.11488888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-18039_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "12": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "13": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "14": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.0, + "12": 0.0, + "13": 0.0, + "14": 0.0, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "teachers_in_school": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/summary.csv b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..6119c0d0e276f5eedab7926c9fc9c8928802ce7f --- /dev/null +++ b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18039,0.2678029513888889,0.107984375,0.18368791118421052,0.0,0.0,0.0,143.94357899305555,195.5853921875,1152000,1280000,2432000 diff --git a/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/summary.json b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c5430afbb1fee7fbf734e360201c9ccc9d0e9580 --- /dev/null +++ b/SDAR-100M-0.999zoo_op2-20+0.001teacher_op2/context/teachers_in_school/autoregressive/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18039", + "id_acc": 0.2678029513888889, + "ood_acc": 0.107984375, + "total_acc": 0.18368791118421052, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 143.94357899305555, + "ood_avg_resp_len": 195.5853921875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 446729, + "answer_accuracy": 0.18368791118421052, + "avg_response_len": 171.12348067434212, + "resp_tokens_sum": 416172305, + "pass_at_k": { + "pass@1": 0.18368791118421052, + "pass@2": 0.24664013287402164, + "pass@4": 0.3180585392269993, + "pass@8": 0.3855605025193377, + "pass@16": 0.4388485982700423, + "pass@32": 0.47817606012624636, + "pass@64": 0.5105078868523447, + "pass@128": 0.5406842105263158 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.120171875, + "pass@2": 0.1845765255905512, + "pass@4": 0.25638461661042333, + "pass@8": 0.32468271814781274, + "pass@16": 0.3813800131130348, + "pass@32": 0.42692040797937, + "pass@64": 0.46937992101561565, + "pass@128": 0.515 + }, + "2": { + "pass@1": 0.999734375, + "pass@2": 0.9999965551181104, + "pass@4": 0.9999999967191601, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.505421875, + "pass@2": 0.6108905019685037, + "pass@4": 0.7027766107986495, + "pass@8": 0.7832486371327387, + "pass@16": 0.8504031585351777, + "pass@32": 0.9030809264372809, + "pass@64": 0.9419219140067047, + "pass@128": 0.968 + }, + "4": { + "pass@1": 0.1529609375, + "pass@2": 0.20825799704724401, + "pass@4": 0.26808056074240705, + "pass@8": 0.32692932890817733, + "pass@16": 0.3807442844868449, + "pass@32": 0.427772580334604, + "pass@64": 0.46742228572792166, + "pass@128": 0.5 + }, + "5": { + "pass@1": 0.125765625, + "pass@2": 0.17454441437007864, + "pass@4": 0.22857620350581168, + "pass@8": 0.2852151041284449, + "pass@16": 0.34054437677879673, + "pass@32": 0.39037834154612494, + "pass@64": 0.43415115133440446, + "pass@128": 0.472 + }, + "6": { + "pass@1": 0.1275625, + "pass@2": 0.19318749999999962, + "pass@4": 0.2716715278402696, + "pass@8": 0.3551362466803586, + "pass@16": 0.4359238966217402, + "pass@32": 0.5110290756429484, + "pass@64": 0.5806887807601362, + "pass@128": 0.644 + }, + "7": { + "pass@1": 0.122828125, + "pass@2": 0.1829392224409449, + "pass@4": 0.25282114135733, + "pass@8": 0.3241583608664076, + "pass@16": 0.39027791997284617, + "pass@32": 0.45005677808798544, + "pass@64": 0.5046726186675862, + "pass@128": 0.553 + }, + "8": { + "pass@1": 0.1354921875, + "pass@2": 0.20422428641732263, + "pass@4": 0.2816454187289086, + "pass@8": 0.3586060594021748, + "pass@16": 0.42847405183941994, + "pass@32": 0.49112355691398996, + "pass@64": 0.5491285932650029, + "pass@128": 0.602 + }, + "9": { + "pass@1": 0.1202890625, + "pass@2": 0.1859338090551182, + "pass@4": 0.26038962457817744, + "pass@8": 0.3325456441962499, + "pass@16": 0.3952699577186529, + "pass@32": 0.44836725487375767, + "pass@64": 0.4969247350424958, + "pass@128": 0.546 + }, + "11": { + "pass@1": 0.1277890625, + "pass@2": 0.19893959153543297, + "pass@4": 0.27852251134233186, + "pass@8": 0.35055435913210575, + "pass@16": 0.40497469620793947, + "pass@32": 0.4440919079127353, + "pass@64": 0.47735138850224945, + "pass@128": 0.511 + }, + "12": { + "pass@1": 0.131921875, + "pass@2": 0.20371678149606307, + "pass@4": 0.2822928592050986, + "pass@8": 0.35083657381639893, + "pass@16": 0.4006997280008851, + "pass@32": 0.4350507594503491, + "pass@64": 0.4625301729624927, + "pass@128": 0.49 + }, + "13": { + "pass@1": 0.10671875, + "pass@2": 0.16880167322834636, + "pass@4": 0.23927782817772775, + "pass@8": 0.30379911705615276, + "pass@16": 0.35148905071127345, + "pass@32": 0.3824129713136272, + "pass@64": 0.40608314530182293, + "pass@128": 0.431 + }, + "14": { + "pass@1": 0.1090546875, + "pass@2": 0.17565354330708663, + "pass@4": 0.25505934842519645, + "pass@8": 0.32899526995383227, + "pass@16": 0.3814744099856488, + "pass@32": 0.4126708516412164, + "pass@64": 0.43439935075478797, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1031640625, + "pass@2": 0.16833280019685046, + "pass@4": 0.24538877174728124, + "pass@8": 0.3161733998486826, + "pass@16": 0.3670585687351444, + "pass@32": 0.40074092351864204, + "pass@64": 0.42732743991054156, + "pass@128": 0.453 + }, + "16": { + "pass@1": 0.11321875, + "pass@2": 0.18650147637795272, + "pass@4": 0.27419264566929097, + "pass@8": 0.3548485199416686, + "pass@16": 0.41054390776905425, + "pass@32": 0.4420058000016596, + "pass@64": 0.4614154287718065, + "pass@128": 0.477 + }, + "17": { + "pass@1": 0.1094140625, + "pass@2": 0.17641646161417313, + "pass@4": 0.255266340551181, + "pass@8": 0.32690535029086165, + "pass@16": 0.3747943480731019, + "pass@32": 0.4017334323764408, + "pass@64": 0.4209359657143077, + "pass@128": 0.441 + }, + "18": { + "pass@1": 0.0980546875, + "pass@2": 0.1624542322834646, + "pass@4": 0.24187763113985722, + "pass@8": 0.31670832960549516, + "pass@16": 0.367754836030914, + "pass@32": 0.3958085143317505, + "pass@64": 0.4149853012051455, + "pass@128": 0.435 + }, + "19": { + "pass@1": 0.0902265625, + "pass@2": 0.15009645669291344, + "pass@4": 0.22361137842144718, + "pass@8": 0.29180410029151527, + "pass@16": 0.3368515905100927, + "pass@32": 0.360478942488469, + "pass@64": 0.3760498303132275, + "pass@128": 0.393 + }, + "20": { + "pass@1": 0.09028125, + "pass@2": 0.1506986958661418, + "pass@4": 0.2252772297525307, + "pass@8": 0.29450242846822444, + "pass@16": 0.3394645720399114, + "pass@32": 0.3616221175476457, + "pass@64": 0.374281826938126, + "pass@128": 0.386 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.csv b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..04eb00f857d1a5579d23e9a0029bb57a974ff900 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18036,0.2539678819444444,0.11763046875,0.18221134868421052,0.0,0.0,0.0,140.28355381944445,186.45809296875,1152000,1280000,2432000 diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/checkpoint-18036_metrics.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/checkpoint-18036_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e71e62e6a0211197ac819b679a9f1c960fd6c200 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/checkpoint-18036_metrics.json @@ -0,0 +1,1077 @@ +{ + "checkpoint": "checkpoint-18036", + "id": { + "answer_accuracy": 0.22641059027777777, + "avg_response_len": 147.4383689236111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.22641059027777777, + "pass@2": 0.28549320592738875, + "pass@4": 0.3468693685476867, + "pass@8": 0.40666053721811896, + "pass@16": 0.46284297688975007, + "pass@32": 0.5152149850418628, + "pass@64": 0.5643908677368596, + "pass@128": 0.6104444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1275546875, + "pass@2": 0.18397231791338578, + "pass@4": 0.24460194375703018, + "pass@8": 0.30093062803178483, + "pass@16": 0.34952821660283057, + "pass@32": 0.3917943201510353, + "pass@64": 0.43101287281811457, + "pass@128": 0.471 + }, + "2": { + "pass@1": 0.6855703125, + "pass@2": 0.7616323818897633, + "pass@4": 0.8226437450318711, + "pass@8": 0.8693866936995356, + "pass@16": 0.9040634940436155, + "pass@32": 0.9282689793861032, + "pass@64": 0.9445755234826567, + "pass@128": 0.957 + }, + "3": { + "pass@1": 0.4090234375, + "pass@2": 0.49490834153543284, + "pass@4": 0.5742481702287209, + "pass@8": 0.6476560954329591, + "pass@16": 0.7155708340968276, + "pass@32": 0.7762121165323362, + "pass@64": 0.8259984159839241, + "pass@128": 0.863 + }, + "4": { + "pass@1": 0.1388984375, + "pass@2": 0.1856339812992125, + "pass@4": 0.2416480807086611, + "pass@8": 0.303264455265656, + "pass@16": 0.36506400174384634, + "pass@32": 0.42329004323340164, + "pass@64": 0.4791230660786076, + "pass@128": 0.534 + }, + "5": { + "pass@1": 0.1397734375, + "pass@2": 0.18694069881889755, + "pass@4": 0.23781398397075337, + "pass@8": 0.29042003946289713, + "pass@16": 0.34542889984414304, + "pass@32": 0.4017893143845517, + "pass@64": 0.45632773040525376, + "pass@128": 0.507 + }, + "6": { + "pass@1": 0.138859375, + "pass@2": 0.1920521653543307, + "pass@4": 0.25324392566554155, + "pass@8": 0.3193882041292773, + "pass@16": 0.3863620472995072, + "pass@32": 0.45335867606520763, + "pass@64": 0.5216536578003536, + "pass@128": 0.587 + }, + "7": { + "pass@1": 0.1370859375, + "pass@2": 0.1885483513779525, + "pass@4": 0.24427601331083604, + "pass@8": 0.30058732946938577, + "pass@16": 0.35691304048000494, + "pass@32": 0.4138330364039067, + "pass@64": 0.47147496388776383, + "pass@128": 0.528 + }, + "8": { + "pass@1": 0.1301484375, + "pass@2": 0.189478838582677, + "pass@4": 0.25686761707911476, + "pass@8": 0.3236102771095384, + "pass@16": 0.38469261291165585, + "pass@32": 0.4408883638976847, + "pass@64": 0.49421998397292644, + "pass@128": 0.545 + }, + "9": { + "pass@1": 0.13078125, + "pass@2": 0.1862717765748031, + "pass@4": 0.24648083717660285, + "pass@8": 0.3047011123619361, + "pass@16": 0.3579636449852364, + "pass@32": 0.4075000153225741, + "pass@64": 0.45513159520215657, + "pass@128": 0.502 + } + }, + "per_op_accuracy": { + "10": 0.1275546875, + "2": 0.6855703125, + "3": 0.4090234375, + "4": 0.1388984375, + "5": 0.1397734375, + "6": 0.138859375, + "7": 0.1370859375, + "8": 0.1301484375, + "9": 0.13078125 + }, + "per_op_avg_response_len": { + "10": 166.1975859375, + "2": 126.634703125, + "3": 122.5899609375, + "4": 134.656765625, + "5": 149.1991796875, + "6": 152.527109375, + "7": 147.051921875, + "8": 160.6266171875, + "9": 167.4614765625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 166.1975859375, + "2": 126.634703125, + "3": 122.5899609375, + "4": 134.656765625, + "5": 149.1991796875, + "6": 152.527109375, + "7": 147.051921875, + "8": 160.6266171875, + "9": 167.4614765625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1152000, + "correct": 260825, + "answer_accuracy": 0.22641059027777777, + "avg_response_len": 147.4383689236111, + "resp_tokens_sum": 169849001, + "pass_at_k": { + "pass@1": 0.22641059027777777, + "pass@2": 0.28549320592738875, + "pass@4": 0.3468693685476867, + "pass@8": 0.40666053721811896, + "pass@16": 0.46284297688975007, + "pass@32": 0.5152149850418628, + "pass@64": 0.5643908677368596, + "pass@128": 0.6104444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1275546875, + "pass@2": 0.18397231791338578, + "pass@4": 0.24460194375703018, + "pass@8": 0.30093062803178483, + "pass@16": 0.34952821660283057, + "pass@32": 0.3917943201510353, + "pass@64": 0.43101287281811457, + "pass@128": 0.471 + }, + "2": { + "pass@1": 0.6855703125, + "pass@2": 0.7616323818897633, + "pass@4": 0.8226437450318711, + "pass@8": 0.8693866936995356, + "pass@16": 0.9040634940436155, + "pass@32": 0.9282689793861032, + "pass@64": 0.9445755234826567, + "pass@128": 0.957 + }, + "3": { + "pass@1": 0.4090234375, + "pass@2": 0.49490834153543284, + "pass@4": 0.5742481702287209, + "pass@8": 0.6476560954329591, + "pass@16": 0.7155708340968276, + "pass@32": 0.7762121165323362, + "pass@64": 0.8259984159839241, + "pass@128": 0.863 + }, + "4": { + "pass@1": 0.1388984375, + "pass@2": 0.1856339812992125, + "pass@4": 0.2416480807086611, + "pass@8": 0.303264455265656, + "pass@16": 0.36506400174384634, + "pass@32": 0.42329004323340164, + "pass@64": 0.4791230660786076, + "pass@128": 0.534 + }, + "5": { + "pass@1": 0.1397734375, + "pass@2": 0.18694069881889755, + "pass@4": 0.23781398397075337, + "pass@8": 0.29042003946289713, + "pass@16": 0.34542889984414304, + "pass@32": 0.4017893143845517, + "pass@64": 0.45632773040525376, + "pass@128": 0.507 + }, + "6": { + "pass@1": 0.138859375, + "pass@2": 0.1920521653543307, + "pass@4": 0.25324392566554155, + "pass@8": 0.3193882041292773, + "pass@16": 0.3863620472995072, + "pass@32": 0.45335867606520763, + "pass@64": 0.5216536578003536, + "pass@128": 0.587 + }, + "7": { + "pass@1": 0.1370859375, + "pass@2": 0.1885483513779525, + "pass@4": 0.24427601331083604, + "pass@8": 0.30058732946938577, + "pass@16": 0.35691304048000494, + "pass@32": 0.4138330364039067, + "pass@64": 0.47147496388776383, + "pass@128": 0.528 + }, + "8": { + "pass@1": 0.1301484375, + "pass@2": 0.189478838582677, + "pass@4": 0.25686761707911476, + "pass@8": 0.3236102771095384, + "pass@16": 0.38469261291165585, + "pass@32": 0.4408883638976847, + "pass@64": 0.49421998397292644, + "pass@128": 0.545 + }, + "9": { + "pass@1": 0.13078125, + "pass@2": 0.1862717765748031, + "pass@4": 0.24648083717660285, + "pass@8": 0.3047011123619361, + "pass@16": 0.3579636449852364, + "pass@32": 0.4075000153225741, + "pass@64": 0.45513159520215657, + "pass@128": 0.502 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "ood": { + "answer_accuracy": 0.10658125, + "avg_response_len": 195.21404765625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.10658125, + "pass@2": 0.16503981299212636, + "pass@4": 0.23195330954255863, + "pass@8": 0.29479746300791754, + "pass@16": 0.34454023209895407, + "pass@32": 0.38039439387720164, + "pass@64": 0.40849044375408883, + "pass@128": 0.436 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.124390625, + "pass@2": 0.18436478838582684, + "pass@4": 0.2500079065429317, + "pass@8": 0.31106538435639236, + "pass@16": 0.3623097871847289, + "pass@32": 0.4058911350924241, + "pass@64": 0.44593339172384244, + "pass@128": 0.482 + }, + "12": { + "pass@1": 0.1319765625, + "pass@2": 0.1991973425196851, + "pass@4": 0.27057843203974474, + "pass@8": 0.33285091912765663, + "pass@16": 0.3811495948226996, + "pass@32": 0.4176612500666168, + "pass@64": 0.44872255990349574, + "pass@128": 0.482 + }, + "13": { + "pass@1": 0.1016640625, + "pass@2": 0.1597054625984254, + "pass@4": 0.22568774906261685, + "pass@8": 0.28777537852588486, + "pass@16": 0.3385163532829712, + "pass@32": 0.37733183761275724, + "pass@64": 0.40994087419578623, + "pass@128": 0.442 + }, + "14": { + "pass@1": 0.1083828125, + "pass@2": 0.16907812500000005, + "pass@4": 0.2387201989126356, + "pass@8": 0.30363066018614365, + "pass@16": 0.35460814100983196, + "pass@32": 0.3919011519565949, + "pass@64": 0.42349025773371596, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1035, + "pass@2": 0.1568507627952755, + "pass@4": 0.2188536396700412, + "pass@8": 0.2797044652411689, + "pass@16": 0.3300338701996676, + "pass@32": 0.3668407469411609, + "pass@64": 0.39452554117651084, + "pass@128": 0.421 + }, + "16": { + "pass@1": 0.119, + "pass@2": 0.18299101870078735, + "pass@4": 0.2547764849081364, + "pass@8": 0.32078744379541735, + "pass@16": 0.3720685714165167, + "pass@32": 0.4080461099795796, + "pass@64": 0.4356798482727351, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.100953125, + "pass@2": 0.1606283218503936, + "pass@4": 0.23170640616797888, + "pass@8": 0.2990199972412359, + "pass@16": 0.3491571986709382, + "pass@32": 0.3812644281454258, + "pass@64": 0.4038042343416932, + "pass@128": 0.425 + }, + "18": { + "pass@1": 0.1005703125, + "pass@2": 0.15684867125984242, + "pass@4": 0.22264220828646397, + "pass@8": 0.2856716837373378, + "pass@16": 0.3351695143493097, + "pass@32": 0.36871204272318947, + "pass@64": 0.39286218477287355, + "pass@128": 0.416 + }, + "19": { + "pass@1": 0.084859375, + "pass@2": 0.1370867372047243, + "pass@4": 0.1991671542932132, + "pass@8": 0.25901242025944826, + "pass@16": 0.3074748369099441, + "pass@32": 0.34126734759969835, + "pass@64": 0.36446943391923176, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.090515625, + "pass@2": 0.1436468996062992, + "pass@4": 0.20739291554180722, + "pass@8": 0.2684562776084627, + "pass@16": 0.314914453142897, + "pass@32": 0.3450278886545889, + "pass@64": 0.36547611150106074, + "pass@128": 0.383 + } + }, + "per_op_accuracy": { + "11": 0.124390625, + "12": 0.1319765625, + "13": 0.1016640625, + "14": 0.1083828125, + "15": 0.1035, + "16": 0.119, + "17": 0.100953125, + "18": 0.1005703125, + "19": 0.084859375, + "20": 0.090515625 + }, + "per_op_avg_response_len": { + "11": 177.9458046875, + "12": 189.344203125, + "13": 191.698328125, + "14": 195.127484375, + "15": 191.3649140625, + "16": 206.090328125, + "17": 202.41746875, + "18": 204.03959375, + "19": 197.69428125, + "20": 196.4180703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 177.9458046875, + "12": 189.344203125, + "13": 191.698328125, + "14": 195.127484375, + "15": 191.3649140625, + "16": 206.090328125, + "17": 202.41746875, + "18": 204.03959375, + "19": 197.69428125, + "20": 196.4180703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 1280000, + "correct": 136424, + "answer_accuracy": 0.10658125, + "avg_response_len": 195.21404765625, + "resp_tokens_sum": 249873981, + "pass_at_k": { + "pass@1": 0.10658125, + "pass@2": 0.16503981299212636, + "pass@4": 0.23195330954255863, + "pass@8": 0.29479746300791754, + "pass@16": 0.34454023209895407, + "pass@32": 0.38039439387720164, + "pass@64": 0.40849044375408883, + "pass@128": 0.436 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.124390625, + "pass@2": 0.18436478838582684, + "pass@4": 0.2500079065429317, + "pass@8": 0.31106538435639236, + "pass@16": 0.3623097871847289, + "pass@32": 0.4058911350924241, + "pass@64": 0.44593339172384244, + "pass@128": 0.482 + }, + "12": { + "pass@1": 0.1319765625, + "pass@2": 0.1991973425196851, + "pass@4": 0.27057843203974474, + "pass@8": 0.33285091912765663, + "pass@16": 0.3811495948226996, + "pass@32": 0.4176612500666168, + "pass@64": 0.44872255990349574, + "pass@128": 0.482 + }, + "13": { + "pass@1": 0.1016640625, + "pass@2": 0.1597054625984254, + "pass@4": 0.22568774906261685, + "pass@8": 0.28777537852588486, + "pass@16": 0.3385163532829712, + "pass@32": 0.37733183761275724, + "pass@64": 0.40994087419578623, + "pass@128": 0.442 + }, + "14": { + "pass@1": 0.1083828125, + "pass@2": 0.16907812500000005, + "pass@4": 0.2387201989126356, + "pass@8": 0.30363066018614365, + "pass@16": 0.35460814100983196, + "pass@32": 0.3919011519565949, + "pass@64": 0.42349025773371596, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1035, + "pass@2": 0.1568507627952755, + "pass@4": 0.2188536396700412, + "pass@8": 0.2797044652411689, + "pass@16": 0.3300338701996676, + "pass@32": 0.3668407469411609, + "pass@64": 0.39452554117651084, + "pass@128": 0.421 + }, + "16": { + "pass@1": 0.119, + "pass@2": 0.18299101870078735, + "pass@4": 0.2547764849081364, + "pass@8": 0.32078744379541735, + "pass@16": 0.3720685714165167, + "pass@32": 0.4080461099795796, + "pass@64": 0.4356798482727351, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.100953125, + "pass@2": 0.1606283218503936, + "pass@4": 0.23170640616797888, + "pass@8": 0.2990199972412359, + "pass@16": 0.3491571986709382, + "pass@32": 0.3812644281454258, + "pass@64": 0.4038042343416932, + "pass@128": 0.425 + }, + "18": { + "pass@1": 0.1005703125, + "pass@2": 0.15684867125984242, + "pass@4": 0.22264220828646397, + "pass@8": 0.2856716837373378, + "pass@16": 0.3351695143493097, + "pass@32": 0.36871204272318947, + "pass@64": 0.39286218477287355, + "pass@128": 0.416 + }, + "19": { + "pass@1": 0.084859375, + "pass@2": 0.1370867372047243, + "pass@4": 0.1991671542932132, + "pass@8": 0.25901242025944826, + "pass@16": 0.3074748369099441, + "pass@32": 0.34126734759969835, + "pass@64": 0.36446943391923176, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.090515625, + "pass@2": 0.1436468996062992, + "pass@4": 0.20739291554180722, + "pass@8": 0.2684562776084627, + "pass@16": 0.314914453142897, + "pass@32": 0.3450278886545889, + "pass@64": 0.36547611150106074, + "pass@128": 0.383 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + }, + "total": { + "answer_accuracy": 0.1633425164473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.1633425164473684, + "pass@2": 0.22209668332988525, + "pass@4": 0.28638723222919216, + "pass@8": 0.3477852350022235, + "pass@16": 0.4005783743682867, + "pass@32": 0.44425677916572875, + "pass@64": 0.4823380130090965, + "pass@128": 0.5186315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1275546875, + "pass@2": 0.18397231791338578, + "pass@4": 0.24460194375703018, + "pass@8": 0.30093062803178483, + "pass@16": 0.34952821660283057, + "pass@32": 0.3917943201510353, + "pass@64": 0.43101287281811457, + "pass@128": 0.471 + }, + "2": { + "pass@1": 0.6855703125, + "pass@2": 0.7616323818897633, + "pass@4": 0.8226437450318711, + "pass@8": 0.8693866936995356, + "pass@16": 0.9040634940436155, + "pass@32": 0.9282689793861032, + "pass@64": 0.9445755234826567, + "pass@128": 0.957 + }, + "3": { + "pass@1": 0.4090234375, + "pass@2": 0.49490834153543284, + "pass@4": 0.5742481702287209, + "pass@8": 0.6476560954329591, + "pass@16": 0.7155708340968276, + "pass@32": 0.7762121165323362, + "pass@64": 0.8259984159839241, + "pass@128": 0.863 + }, + "4": { + "pass@1": 0.1388984375, + "pass@2": 0.1856339812992125, + "pass@4": 0.2416480807086611, + "pass@8": 0.303264455265656, + "pass@16": 0.36506400174384634, + "pass@32": 0.42329004323340164, + "pass@64": 0.4791230660786076, + "pass@128": 0.534 + }, + "5": { + "pass@1": 0.1397734375, + "pass@2": 0.18694069881889755, + "pass@4": 0.23781398397075337, + "pass@8": 0.29042003946289713, + "pass@16": 0.34542889984414304, + "pass@32": 0.4017893143845517, + "pass@64": 0.45632773040525376, + "pass@128": 0.507 + }, + "6": { + "pass@1": 0.138859375, + "pass@2": 0.1920521653543307, + "pass@4": 0.25324392566554155, + "pass@8": 0.3193882041292773, + "pass@16": 0.3863620472995072, + "pass@32": 0.45335867606520763, + "pass@64": 0.5216536578003536, + "pass@128": 0.587 + }, + "7": { + "pass@1": 0.1370859375, + "pass@2": 0.1885483513779525, + "pass@4": 0.24427601331083604, + "pass@8": 0.30058732946938577, + "pass@16": 0.35691304048000494, + "pass@32": 0.4138330364039067, + "pass@64": 0.47147496388776383, + "pass@128": 0.528 + }, + "8": { + "pass@1": 0.1301484375, + "pass@2": 0.189478838582677, + "pass@4": 0.25686761707911476, + "pass@8": 0.3236102771095384, + "pass@16": 0.38469261291165585, + "pass@32": 0.4408883638976847, + "pass@64": 0.49421998397292644, + "pass@128": 0.545 + }, + "9": { + "pass@1": 0.13078125, + "pass@2": 0.1862717765748031, + "pass@4": 0.24648083717660285, + "pass@8": 0.3047011123619361, + "pass@16": 0.3579636449852364, + "pass@32": 0.4075000153225741, + "pass@64": 0.45513159520215657, + "pass@128": 0.502 + }, + "11": { + "pass@1": 0.124390625, + "pass@2": 0.18436478838582684, + "pass@4": 0.2500079065429317, + "pass@8": 0.31106538435639236, + "pass@16": 0.3623097871847289, + "pass@32": 0.4058911350924241, + "pass@64": 0.44593339172384244, + "pass@128": 0.482 + }, + "12": { + "pass@1": 0.1319765625, + "pass@2": 0.1991973425196851, + "pass@4": 0.27057843203974474, + "pass@8": 0.33285091912765663, + "pass@16": 0.3811495948226996, + "pass@32": 0.4176612500666168, + "pass@64": 0.44872255990349574, + "pass@128": 0.482 + }, + "13": { + "pass@1": 0.1016640625, + "pass@2": 0.1597054625984254, + "pass@4": 0.22568774906261685, + "pass@8": 0.28777537852588486, + "pass@16": 0.3385163532829712, + "pass@32": 0.37733183761275724, + "pass@64": 0.40994087419578623, + "pass@128": 0.442 + }, + "14": { + "pass@1": 0.1083828125, + "pass@2": 0.16907812500000005, + "pass@4": 0.2387201989126356, + "pass@8": 0.30363066018614365, + "pass@16": 0.35460814100983196, + "pass@32": 0.3919011519565949, + "pass@64": 0.42349025773371596, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1035, + "pass@2": 0.1568507627952755, + "pass@4": 0.2188536396700412, + "pass@8": 0.2797044652411689, + "pass@16": 0.3300338701996676, + "pass@32": 0.3668407469411609, + "pass@64": 0.39452554117651084, + "pass@128": 0.421 + }, + "16": { + "pass@1": 0.119, + "pass@2": 0.18299101870078735, + "pass@4": 0.2547764849081364, + "pass@8": 0.32078744379541735, + "pass@16": 0.3720685714165167, + "pass@32": 0.4080461099795796, + "pass@64": 0.4356798482727351, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.100953125, + "pass@2": 0.1606283218503936, + "pass@4": 0.23170640616797888, + "pass@8": 0.2990199972412359, + "pass@16": 0.3491571986709382, + "pass@32": 0.3812644281454258, + "pass@64": 0.4038042343416932, + "pass@128": 0.425 + }, + "18": { + "pass@1": 0.1005703125, + "pass@2": 0.15684867125984242, + "pass@4": 0.22264220828646397, + "pass@8": 0.2856716837373378, + "pass@16": 0.3351695143493097, + "pass@32": 0.36871204272318947, + "pass@64": 0.39286218477287355, + "pass@128": 0.416 + }, + "19": { + "pass@1": 0.084859375, + "pass@2": 0.1370867372047243, + "pass@4": 0.1991671542932132, + "pass@8": 0.25901242025944826, + "pass@16": 0.3074748369099441, + "pass@32": 0.34126734759969835, + "pass@64": 0.36446943391923176, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.090515625, + "pass@2": 0.1436468996062992, + "pass@4": 0.20739291554180722, + "pass@8": 0.2684562776084627, + "pass@16": 0.314914453142897, + "pass@32": 0.3450278886545889, + "pass@64": 0.36547611150106074, + "pass@128": 0.383 + } + }, + "per_op_accuracy": { + "10": 0.1275546875, + "2": 0.6855703125, + "3": 0.4090234375, + "4": 0.1388984375, + "5": 0.1397734375, + "6": 0.138859375, + "7": 0.1370859375, + "8": 0.1301484375, + "9": 0.13078125, + "11": 0.124390625, + "12": 0.1319765625, + "13": 0.1016640625, + "14": 0.1083828125, + "15": 0.1035, + "16": 0.119, + "17": 0.100953125, + "18": 0.1005703125, + "19": 0.084859375, + "20": 0.090515625 + }, + "per_op_avg_response_len": { + "10": 166.1975859375, + "2": 126.634703125, + "3": 122.5899609375, + "4": 134.656765625, + "5": 149.1991796875, + "6": 152.527109375, + "7": 147.051921875, + "8": 160.6266171875, + "9": 167.4614765625, + "11": 177.9458046875, + "12": 189.344203125, + "13": 191.698328125, + "14": 195.127484375, + "15": 191.3649140625, + "16": 206.090328125, + "17": 202.41746875, + "18": 204.03959375, + "19": 197.69428125, + "20": 196.4180703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 166.1975859375, + "2": 126.634703125, + "3": 122.5899609375, + "4": 134.656765625, + "5": 149.1991796875, + "6": 152.527109375, + "7": 147.051921875, + "8": 160.6266171875, + "9": 167.4614765625, + "11": 177.9458046875, + "12": 189.344203125, + "13": 191.698328125, + "14": 195.127484375, + "15": 191.3649140625, + "16": 206.090328125, + "17": 202.41746875, + "18": 204.03959375, + "19": 197.69428125, + "20": 196.4180703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 397249, + "answer_accuracy": 0.1633425164473684, + "avg_response_len": 172.58346299342105, + "resp_tokens_sum": 419722982, + "pass_at_k": { + "pass@1": 0.1633425164473684, + "pass@2": 0.22209668332988525, + "pass@4": 0.28638723222919216, + "pass@8": 0.3477852350022235, + "pass@16": 0.4005783743682867, + "pass@32": 0.44425677916572875, + "pass@64": 0.4823380130090965, + "pass@128": 0.5186315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1275546875, + "pass@2": 0.18397231791338578, + "pass@4": 0.24460194375703018, + "pass@8": 0.30093062803178483, + "pass@16": 0.34952821660283057, + "pass@32": 0.3917943201510353, + "pass@64": 0.43101287281811457, + "pass@128": 0.471 + }, + "2": { + "pass@1": 0.6855703125, + "pass@2": 0.7616323818897633, + "pass@4": 0.8226437450318711, + "pass@8": 0.8693866936995356, + "pass@16": 0.9040634940436155, + "pass@32": 0.9282689793861032, + "pass@64": 0.9445755234826567, + "pass@128": 0.957 + }, + "3": { + "pass@1": 0.4090234375, + "pass@2": 0.49490834153543284, + "pass@4": 0.5742481702287209, + "pass@8": 0.6476560954329591, + "pass@16": 0.7155708340968276, + "pass@32": 0.7762121165323362, + "pass@64": 0.8259984159839241, + "pass@128": 0.863 + }, + "4": { + "pass@1": 0.1388984375, + "pass@2": 0.1856339812992125, + "pass@4": 0.2416480807086611, + "pass@8": 0.303264455265656, + "pass@16": 0.36506400174384634, + "pass@32": 0.42329004323340164, + "pass@64": 0.4791230660786076, + "pass@128": 0.534 + }, + "5": { + "pass@1": 0.1397734375, + "pass@2": 0.18694069881889755, + "pass@4": 0.23781398397075337, + "pass@8": 0.29042003946289713, + "pass@16": 0.34542889984414304, + "pass@32": 0.4017893143845517, + "pass@64": 0.45632773040525376, + "pass@128": 0.507 + }, + "6": { + "pass@1": 0.138859375, + "pass@2": 0.1920521653543307, + "pass@4": 0.25324392566554155, + "pass@8": 0.3193882041292773, + "pass@16": 0.3863620472995072, + "pass@32": 0.45335867606520763, + "pass@64": 0.5216536578003536, + "pass@128": 0.587 + }, + "7": { + "pass@1": 0.1370859375, + "pass@2": 0.1885483513779525, + "pass@4": 0.24427601331083604, + "pass@8": 0.30058732946938577, + "pass@16": 0.35691304048000494, + "pass@32": 0.4138330364039067, + "pass@64": 0.47147496388776383, + "pass@128": 0.528 + }, + "8": { + "pass@1": 0.1301484375, + "pass@2": 0.189478838582677, + "pass@4": 0.25686761707911476, + "pass@8": 0.3236102771095384, + "pass@16": 0.38469261291165585, + "pass@32": 0.4408883638976847, + "pass@64": 0.49421998397292644, + "pass@128": 0.545 + }, + "9": { + "pass@1": 0.13078125, + "pass@2": 0.1862717765748031, + "pass@4": 0.24648083717660285, + "pass@8": 0.3047011123619361, + "pass@16": 0.3579636449852364, + "pass@32": 0.4075000153225741, + "pass@64": 0.45513159520215657, + "pass@128": 0.502 + }, + "11": { + "pass@1": 0.124390625, + "pass@2": 0.18436478838582684, + "pass@4": 0.2500079065429317, + "pass@8": 0.31106538435639236, + "pass@16": 0.3623097871847289, + "pass@32": 0.4058911350924241, + "pass@64": 0.44593339172384244, + "pass@128": 0.482 + }, + "12": { + "pass@1": 0.1319765625, + "pass@2": 0.1991973425196851, + "pass@4": 0.27057843203974474, + "pass@8": 0.33285091912765663, + "pass@16": 0.3811495948226996, + "pass@32": 0.4176612500666168, + "pass@64": 0.44872255990349574, + "pass@128": 0.482 + }, + "13": { + "pass@1": 0.1016640625, + "pass@2": 0.1597054625984254, + "pass@4": 0.22568774906261685, + "pass@8": 0.28777537852588486, + "pass@16": 0.3385163532829712, + "pass@32": 0.37733183761275724, + "pass@64": 0.40994087419578623, + "pass@128": 0.442 + }, + "14": { + "pass@1": 0.1083828125, + "pass@2": 0.16907812500000005, + "pass@4": 0.2387201989126356, + "pass@8": 0.30363066018614365, + "pass@16": 0.35460814100983196, + "pass@32": 0.3919011519565949, + "pass@64": 0.42349025773371596, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1035, + "pass@2": 0.1568507627952755, + "pass@4": 0.2188536396700412, + "pass@8": 0.2797044652411689, + "pass@16": 0.3300338701996676, + "pass@32": 0.3668407469411609, + "pass@64": 0.39452554117651084, + "pass@128": 0.421 + }, + "16": { + "pass@1": 0.119, + "pass@2": 0.18299101870078735, + "pass@4": 0.2547764849081364, + "pass@8": 0.32078744379541735, + "pass@16": 0.3720685714165167, + "pass@32": 0.4080461099795796, + "pass@64": 0.4356798482727351, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.100953125, + "pass@2": 0.1606283218503936, + "pass@4": 0.23170640616797888, + "pass@8": 0.2990199972412359, + "pass@16": 0.3491571986709382, + "pass@32": 0.3812644281454258, + "pass@64": 0.4038042343416932, + "pass@128": 0.425 + }, + "18": { + "pass@1": 0.1005703125, + "pass@2": 0.15684867125984242, + "pass@4": 0.22264220828646397, + "pass@8": 0.2856716837373378, + "pass@16": 0.3351695143493097, + "pass@32": 0.36871204272318947, + "pass@64": 0.39286218477287355, + "pass@128": 0.416 + }, + "19": { + "pass@1": 0.084859375, + "pass@2": 0.1370867372047243, + "pass@4": 0.1991671542932132, + "pass@8": 0.25901242025944826, + "pass@16": 0.3074748369099441, + "pass@32": 0.34126734759969835, + "pass@64": 0.36446943391923176, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.090515625, + "pass@2": 0.1436468996062992, + "pass@4": 0.20739291554180722, + "pass@8": 0.2684562776084627, + "pass@16": 0.314914453142897, + "pass@32": 0.3450278886545889, + "pass@64": 0.36547611150106074, + "pass@128": 0.383 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.csv b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc8985a3e3a8c6ab43574043a1eb359f2a177b84 --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-18036,0.22641059027777777,0.10658125,0.1633425164473684,0.0,0.0,0.0,147.4383689236111,195.21404765625,1152000,1280000,2432000 diff --git a/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.json b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1c7f316b33cff202ce9a18f82f4736ade0161acc --- /dev/null +++ b/SDAR-100M-0.99zoo_op2-20+0.01teacher_op2/context/teachers_in_school/low_confidence_static/block32/summary.json @@ -0,0 +1,245 @@ +[ + { + "checkpoint": "checkpoint-18036", + "id_acc": 0.22641059027777777, + "ood_acc": 0.10658125, + "total_acc": 0.1633425164473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 147.4383689236111, + "ood_avg_resp_len": 195.21404765625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + }, + "teachers_in_school": { + "count": 2432000, + "correct": 397249, + "answer_accuracy": 0.1633425164473684, + "avg_response_len": 172.58346299342105, + "resp_tokens_sum": 419722982, + "pass_at_k": { + "pass@1": 0.1633425164473684, + "pass@2": 0.22209668332988525, + "pass@4": 0.28638723222919216, + "pass@8": 0.3477852350022235, + "pass@16": 0.4005783743682867, + "pass@32": 0.44425677916572875, + "pass@64": 0.4823380130090965, + "pass@128": 0.5186315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1275546875, + "pass@2": 0.18397231791338578, + "pass@4": 0.24460194375703018, + "pass@8": 0.30093062803178483, + "pass@16": 0.34952821660283057, + "pass@32": 0.3917943201510353, + "pass@64": 0.43101287281811457, + "pass@128": 0.471 + }, + "2": { + "pass@1": 0.6855703125, + "pass@2": 0.7616323818897633, + "pass@4": 0.8226437450318711, + "pass@8": 0.8693866936995356, + "pass@16": 0.9040634940436155, + "pass@32": 0.9282689793861032, + "pass@64": 0.9445755234826567, + "pass@128": 0.957 + }, + "3": { + "pass@1": 0.4090234375, + "pass@2": 0.49490834153543284, + "pass@4": 0.5742481702287209, + "pass@8": 0.6476560954329591, + "pass@16": 0.7155708340968276, + "pass@32": 0.7762121165323362, + "pass@64": 0.8259984159839241, + "pass@128": 0.863 + }, + "4": { + "pass@1": 0.1388984375, + "pass@2": 0.1856339812992125, + "pass@4": 0.2416480807086611, + "pass@8": 0.303264455265656, + "pass@16": 0.36506400174384634, + "pass@32": 0.42329004323340164, + "pass@64": 0.4791230660786076, + "pass@128": 0.534 + }, + "5": { + "pass@1": 0.1397734375, + "pass@2": 0.18694069881889755, + "pass@4": 0.23781398397075337, + "pass@8": 0.29042003946289713, + "pass@16": 0.34542889984414304, + "pass@32": 0.4017893143845517, + "pass@64": 0.45632773040525376, + "pass@128": 0.507 + }, + "6": { + "pass@1": 0.138859375, + "pass@2": 0.1920521653543307, + "pass@4": 0.25324392566554155, + "pass@8": 0.3193882041292773, + "pass@16": 0.3863620472995072, + "pass@32": 0.45335867606520763, + "pass@64": 0.5216536578003536, + "pass@128": 0.587 + }, + "7": { + "pass@1": 0.1370859375, + "pass@2": 0.1885483513779525, + "pass@4": 0.24427601331083604, + "pass@8": 0.30058732946938577, + "pass@16": 0.35691304048000494, + "pass@32": 0.4138330364039067, + "pass@64": 0.47147496388776383, + "pass@128": 0.528 + }, + "8": { + "pass@1": 0.1301484375, + "pass@2": 0.189478838582677, + "pass@4": 0.25686761707911476, + "pass@8": 0.3236102771095384, + "pass@16": 0.38469261291165585, + "pass@32": 0.4408883638976847, + "pass@64": 0.49421998397292644, + "pass@128": 0.545 + }, + "9": { + "pass@1": 0.13078125, + "pass@2": 0.1862717765748031, + "pass@4": 0.24648083717660285, + "pass@8": 0.3047011123619361, + "pass@16": 0.3579636449852364, + "pass@32": 0.4075000153225741, + "pass@64": 0.45513159520215657, + "pass@128": 0.502 + }, + "11": { + "pass@1": 0.124390625, + "pass@2": 0.18436478838582684, + "pass@4": 0.2500079065429317, + "pass@8": 0.31106538435639236, + "pass@16": 0.3623097871847289, + "pass@32": 0.4058911350924241, + "pass@64": 0.44593339172384244, + "pass@128": 0.482 + }, + "12": { + "pass@1": 0.1319765625, + "pass@2": 0.1991973425196851, + "pass@4": 0.27057843203974474, + "pass@8": 0.33285091912765663, + "pass@16": 0.3811495948226996, + "pass@32": 0.4176612500666168, + "pass@64": 0.44872255990349574, + "pass@128": 0.482 + }, + "13": { + "pass@1": 0.1016640625, + "pass@2": 0.1597054625984254, + "pass@4": 0.22568774906261685, + "pass@8": 0.28777537852588486, + "pass@16": 0.3385163532829712, + "pass@32": 0.37733183761275724, + "pass@64": 0.40994087419578623, + "pass@128": 0.442 + }, + "14": { + "pass@1": 0.1083828125, + "pass@2": 0.16907812500000005, + "pass@4": 0.2387201989126356, + "pass@8": 0.30363066018614365, + "pass@16": 0.35460814100983196, + "pass@32": 0.3919011519565949, + "pass@64": 0.42349025773371596, + "pass@128": 0.456 + }, + "15": { + "pass@1": 0.1035, + "pass@2": 0.1568507627952755, + "pass@4": 0.2188536396700412, + "pass@8": 0.2797044652411689, + "pass@16": 0.3300338701996676, + "pass@32": 0.3668407469411609, + "pass@64": 0.39452554117651084, + "pass@128": 0.421 + }, + "16": { + "pass@1": 0.119, + "pass@2": 0.18299101870078735, + "pass@4": 0.2547764849081364, + "pass@8": 0.32078744379541735, + "pass@16": 0.3720685714165167, + "pass@32": 0.4080461099795796, + "pass@64": 0.4356798482727351, + "pass@128": 0.465 + }, + "17": { + "pass@1": 0.100953125, + "pass@2": 0.1606283218503936, + "pass@4": 0.23170640616797888, + "pass@8": 0.2990199972412359, + "pass@16": 0.3491571986709382, + "pass@32": 0.3812644281454258, + "pass@64": 0.4038042343416932, + "pass@128": 0.425 + }, + "18": { + "pass@1": 0.1005703125, + "pass@2": 0.15684867125984242, + "pass@4": 0.22264220828646397, + "pass@8": 0.2856716837373378, + "pass@16": 0.3351695143493097, + "pass@32": 0.36871204272318947, + "pass@64": 0.39286218477287355, + "pass@128": 0.416 + }, + "19": { + "pass@1": 0.084859375, + "pass@2": 0.1370867372047243, + "pass@4": 0.1991671542932132, + "pass@8": 0.25901242025944826, + "pass@16": 0.3074748369099441, + "pass@32": 0.34126734759969835, + "pass@64": 0.36446943391923176, + "pass@128": 0.388 + }, + "20": { + "pass@1": 0.090515625, + "pass@2": 0.1436468996062992, + "pass@4": 0.20739291554180722, + "pass@8": 0.2684562776084627, + "pass@16": 0.314914453142897, + "pass@32": 0.3450278886545889, + "pass@64": 0.36547611150106074, + "pass@128": 0.383 + } + } + }, + "movie_festival_awards": { + "count": 0, + "correct": 0, + "answer_accuracy": 0.0, + "avg_response_len": 0.0, + "resp_tokens_sum": 0, + "pass_at_k": {}, + "per_op_pass_at_k": {} + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/autoregressive/checkpoint-21808_metrics.json b/SDAR-100M-ep1-block16/composition/autoregressive/checkpoint-21808_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..62f527c70d97418fc62935182e4f9d38ad84464d --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/autoregressive/checkpoint-21808_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21808", + "id": { + "answer_accuracy": 0.9135225694444444, + "avg_response_len": 199.60449913194444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9135225694444444, + "pass@2": 0.9486792979002415, + "pass@4": 0.9674514434445672, + "pass@8": 0.9781444420481766, + "pass@16": 0.98473471954891, + "pass@32": 0.9891270976970611, + "pass@64": 0.9922526565051226, + "pass@128": 0.9946666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8130390625, + "pass@2": 0.8855469980314973, + "pass@4": 0.92615446559805, + "pass@8": 0.9501353146499899, + "pass@16": 0.9657012336439675, + "pass@32": 0.9770663746047966, + "pass@64": 0.9851435119560961, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.9385703125, + "pass@2": 0.9525967027559057, + "pass@4": 0.9638686646981628, + "pass@8": 0.9724943090518903, + "pass@16": 0.9790929620706803, + "pass@32": 0.9842960053580423, + "pass@64": 0.9882386855842575, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.980015625, + "pass@2": 0.9875548720472446, + "pass@4": 0.9916484928758906, + "pass@8": 0.9943460459073127, + "pass@16": 0.9961729071719045, + "pass@32": 0.9973141929536077, + "pass@64": 0.9978679671606755, + "pass@128": 0.998 + }, + "4": { + "pass@1": 0.9676171875, + "pass@2": 0.9803442421259845, + "pass@4": 0.9866892795275589, + "pass@8": 0.9907940834376028, + "pass@16": 0.9937288393186484, + "pass@32": 0.9959362702515183, + "pass@64": 0.9979969035840609, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9206328125, + "pass@2": 0.9421621555118115, + "pass@4": 0.9567363708286464, + "pass@8": 0.9672069341201128, + "pass@16": 0.9743872248750242, + "pass@32": 0.9796198926321592, + "pass@64": 0.9843256419706993, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.9333359375, + "pass@2": 0.9661230314960632, + "pass@4": 0.9814899094488188, + "pass@8": 0.9888695765997886, + "pass@16": 0.9930217167612667, + "pass@32": 0.9952694212303389, + "pass@64": 0.9959625787840237, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.9184453125, + "pass@2": 0.9520457677165358, + "pass@4": 0.9696526289838768, + "pass@8": 0.97916936059381, + "pass@16": 0.9845091635229746, + "pass@32": 0.9878582864806053, + "pass@64": 0.9906888510786144, + "pass@128": 0.993 + }, + "8": { + "pass@1": 0.891296875, + "pass@2": 0.9444388533464576, + "pass@4": 0.9712997254405698, + "pass@8": 0.9844960086691613, + "pass@16": 0.9911105470974821, + "pass@32": 0.9946645302609637, + "pass@64": 0.9966357214747974, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.85875, + "pass@2": 0.9273010580708668, + "pass@4": 0.9595234535995493, + "pass@8": 0.9757883454039419, + "pass@16": 0.9848878814782323, + "pass@32": 0.9901189055015083, + "pass@64": 0.9934140469528695, + "pass@128": 0.996 + } + }, + "per_op_accuracy": { + "10": 0.8130390625, + "2": 0.9385703125, + "3": 0.980015625, + "4": 0.9676171875, + "5": 0.9206328125, + "6": 0.9333359375, + "7": 0.9184453125, + "8": 0.891296875, + "9": 0.85875 + }, + "per_op_avg_response_len": { + "10": 275.31103125, + "2": 120.48475, + "3": 145.7017421875, + "4": 159.9137265625, + "5": 173.2708828125, + "6": 199.5473984375, + "7": 222.4298828125, + "8": 242.818015625, + "9": 256.9630625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 275.31103125, + "2": 120.48475, + "3": 145.7017421875, + "4": 159.9137265625, + "5": 173.2708828125, + "6": 199.5473984375, + "7": 222.4298828125, + "8": 242.818015625, + "9": 256.9630625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 341891, + "answer_accuracy": 0.9020680302262749, + "avg_response_len": 189.47420107227288, + "resp_tokens_sum": 71812238, + "pass_at_k": { + "pass@1": 0.9020680302262749, + "pass@2": 0.940811753583998, + "pass@4": 0.9616757204361651, + "pass@8": 0.9733680390472118, + "pass@16": 0.9805197850736754, + "pass@32": 0.9854645629154367, + "pass@64": 0.9891140826326636, + "pass@128": 0.9918946301925026 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.811992994100295, + "pass@2": 0.8843032134810588, + "pass@4": 0.923425883711439, + "pass@8": 0.9457277067863578, + "pass@16": 0.9606133820258961, + "pass@32": 0.971941456242833, + "pass@64": 0.979374562311218, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9364047897196262, + "pass@2": 0.9491822429906543, + "pass@4": 0.9604580686175911, + "pass@8": 0.9694622504718368, + "pass@16": 0.9762818313978732, + "pass@32": 0.9814895602246809, + "pass@64": 0.985727686903887, + "pass@128": 0.9906542056074766 + }, + "3": { + "pass@1": 0.9801462155963303, + "pass@2": 0.9896420272580605, + "pass@4": 0.9937778259368953, + "pass@8": 0.9962077916624781, + "pass@16": 0.9981311364256852, + "pass@32": 0.9995033618733702, + "pass@64": 0.999978967901273, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.969677734375, + "pass@2": 0.9814676119586613, + "pass@4": 0.986887528707349, + "pass@8": 0.9899307200629559, + "pass@16": 0.9917621691650451, + "pass@32": 0.993528368509755, + "pass@64": 0.9960998638576377, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8992063492063492, + "pass@2": 0.9267841519810028, + "pass@4": 0.9439647365507879, + "pass@8": 0.9548172712010686, + "pass@16": 0.9623409287636865, + "pass@32": 0.9688380283149466, + "pass@64": 0.9755950258820365, + "pass@128": 0.9809523809523809 + }, + "6": { + "pass@1": 0.9132204341317365, + "pass@2": 0.9474187697439762, + "pass@4": 0.965994401803816, + "pass@8": 0.9784493955374186, + "pass@16": 0.9872882423827155, + "pass@32": 0.9922681964206024, + "pass@64": 0.9939103062084376, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.8838063979289941, + "pass@2": 0.9277586719004803, + "pass@4": 0.9530765176172504, + "pass@8": 0.9664595878668178, + "pass@16": 0.9740977457411283, + "pass@32": 0.9795563479381585, + "pass@64": 0.9843567124179412, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8821864983974359, + "pass@2": 0.9410403101655567, + "pass@4": 0.9686909091291468, + "pass@8": 0.9800378182202505, + "pass@16": 0.9851975708878595, + "pass@32": 0.988658383641544, + "pass@64": 0.9911660518873375, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.8510783450704226, + "pass@2": 0.924036195519574, + "pass@4": 0.9606805254096751, + "pass@8": 0.9797217797223858, + "pass@16": 0.9887705852950439, + "pass@32": 0.9929424597619723, + "pass@64": 0.9953811357885473, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 358464, + "answer_accuracy": 0.9175950196592398, + "avg_response_len": 197.47346258600916, + "resp_tokens_sum": 77144193, + "pass_at_k": { + "pass@1": 0.9175950196592398, + "pass@2": 0.9521344593967092, + "pass@4": 0.9705430778701007, + "pass@8": 0.9811392976093394, + "pass@16": 0.9876874367520381, + "pass@32": 0.9917055334180617, + "pass@64": 0.9941915822813107, + "pass@128": 0.9960681520314548 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8102233512269938, + "pass@2": 0.8818788192599386, + "pass@4": 0.9230943886998785, + "pass@8": 0.9486758986818827, + "pass@16": 0.9665855700413544, + "pass@32": 0.9798571294366684, + "pass@64": 0.9889181243773738, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9407057370820668, + "pass@2": 0.9543108142067347, + "pass@4": 0.9648635555244045, + "pass@8": 0.9735475712944428, + "pass@16": 0.980713903139753, + "pass@32": 0.9855875985518405, + "pass@64": 0.9885083794652192, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.9831912878787878, + "pass@2": 0.9890449773323791, + "pass@4": 0.9925742770790013, + "pass@8": 0.9949117732864091, + "pass@16": 0.9968032945575152, + "pass@32": 0.998445165474543, + "pass@64": 0.999620777962612, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9603365384615384, + "pass@2": 0.9773203448958673, + "pass@4": 0.9862346477215493, + "pass@8": 0.9916871794012287, + "pass@16": 0.995383380617976, + "pass@32": 0.9977698233271339, + "pass@64": 0.9994625180003253, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9483016304347827, + "pass@2": 0.9635705523222642, + "pass@4": 0.9734722803671282, + "pass@8": 0.9809383138264673, + "pass@16": 0.9851226251370658, + "pass@32": 0.9872274839565104, + "pass@64": 0.9891271668301083, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9358882521489972, + "pass@2": 0.9721159161157868, + "pass@4": 0.9889131951271127, + "pass@8": 0.9951489089035775, + "pass@16": 0.9979499235991857, + "pass@32": 0.9995757636157672, + "pass@64": 0.999990075961284, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9294391257225434, + "pass@2": 0.9595930431022713, + "pass@4": 0.9755940045969683, + "pass@8": 0.9847787879592633, + "pass@16": 0.9892688554315738, + "pass@32": 0.9908053653135379, + "pass@64": 0.9912895111919918, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.8965448943661972, + "pass@2": 0.9502491821004772, + "pass@4": 0.977070321931589, + "pass@8": 0.9902363110097167, + "pass@16": 0.9961681334129603, + "pass@32": 0.9986011198470229, + "pass@64": 0.999793253291699, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8508373877245509, + "pass@2": 0.9181932640624261, + "pass@4": 0.9504120931365609, + "pass@8": 0.9681510539987431, + "pass@16": 0.9796319664627892, + "pass@32": 0.9865058729633788, + "pass@64": 0.9904633864127251, + "pass@128": 0.9940119760479041 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 352023, + "answer_accuracy": 0.920716333277536, + "avg_response_len": 211.8240291262136, + "resp_tokens_sum": 80987952, + "pass_at_k": { + "pass@1": 0.920716333277536, + "pass@2": 0.9529480109345247, + "pass@4": 0.9700179809608686, + "pass@8": 0.9798192428895547, + "pass@16": 0.9858959944324802, + "pass@32": 0.9901232073950488, + "pass@64": 0.9933827923495863, + "pass@128": 0.9959825912286575 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8168376865671642, + "pass@2": 0.8903752644259023, + "pass@4": 0.9318934934439158, + "pass@8": 0.9560157584451383, + "pass@16": 0.9699892576230075, + "pass@32": 0.9795367066929017, + "pass@64": 0.9873081396584162, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9385491071428571, + "pass@2": 0.9541169853768277, + "pass@4": 0.9660614711553913, + "pass@8": 0.9742850734130257, + "pass@16": 0.9801474858828125, + "pass@32": 0.9856558188638964, + "pass@64": 0.990288117840151, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9768358236151603, + "pass@2": 0.9841314478547326, + "pass@4": 0.9887277911325226, + "pass@8": 0.9920268596185634, + "pass@16": 0.9936995287368084, + "pass@32": 0.9941390350274529, + "pass@64": 0.9941690610183596, + "pass@128": 0.9941690962099126 + }, + "4": { + "pass@1": 0.9728846856725146, + "pass@2": 0.9822816658608464, + "pass@4": 0.9869530976939285, + "pass@8": 0.9907192584205899, + "pass@16": 0.9939338085875972, + "pass@32": 0.9963771697187885, + "pass@64": 0.9983234387880316, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9124080882352941, + "pass@2": 0.9346861973135713, + "pass@4": 0.9515868884955558, + "pass@8": 0.9647522806518978, + "pass@16": 0.9746544901240447, + "pass@32": 0.9818895051410447, + "pass@64": 0.987542165474913, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9517202287066246, + "pass@2": 0.9792325163317526, + "pass@4": 0.9896437670251783, + "pass@8": 0.992935360514013, + "pass@16": 0.9936369730893512, + "pass@32": 0.9936905429777773, + "pass@64": 0.9936908517347554, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9434582674050633, + "pass@2": 0.9697599484202134, + "pass@4": 0.9808773431604276, + "pass@8": 0.9866220229775958, + "pass@16": 0.990433865453002, + "pass@32": 0.9935113432880496, + "pass@64": 0.9968041500282304, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.894237987987988, + "pass@2": 0.9414288796670683, + "pass@4": 0.9675921847043887, + "pass@8": 0.9825535104384212, + "pass@16": 0.9912589117683784, + "pass@32": 0.9960952463036303, + "pass@64": 0.9983943194216062, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8760048231511254, + "pass@2": 0.940809191204396, + "pass@4": 0.967987871291007, + "pass@8": 0.9795005195077631, + "pass@16": 0.9861004723470128, + "pass@32": 0.9907761116277802, + "pass@64": 0.9943375327526845, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.2918375, + "avg_response_len": 312.06185859375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2918375, + "pass@2": 0.38549048966536076, + "pass@4": 0.4709890827240346, + "pass@8": 0.5418854141993076, + "pass@16": 0.5983291260391027, + "pass@32": 0.6432064182525502, + "pass@64": 0.6797531535624188, + "pass@128": 0.7113 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7227734375, + "pass@2": 0.8238348917322844, + "pass@4": 0.8843667961192356, + "pass@8": 0.919460194088572, + "pass@16": 0.9415317183254884, + "pass@32": 0.9565976455684818, + "pass@64": 0.9671927002884151, + "pass@128": 0.975 + }, + "12": { + "pass@1": 0.5365, + "pass@2": 0.6742450787401567, + "pass@4": 0.7776323629546307, + "pass@8": 0.8463525823436129, + "pass@16": 0.8909261922144747, + "pass@32": 0.9206554040031512, + "pass@64": 0.9398806436368472, + "pass@128": 0.953 + }, + "13": { + "pass@1": 0.3344609375, + "pass@2": 0.44971481299212634, + "pass@4": 0.5544226151106105, + "pass@8": 0.6415505086285688, + "pass@16": 0.7111350744435204, + "pass@32": 0.7666698924273481, + "pass@64": 0.8116365000384063, + "pass@128": 0.846 + }, + "14": { + "pass@1": 0.2862890625, + "pass@2": 0.38968171751968494, + "pass@4": 0.48735889698162665, + "pass@8": 0.571221938992996, + "pass@16": 0.6421740318475855, + "pass@32": 0.7043120405178619, + "pass@64": 0.759916295963575, + "pass@128": 0.811 + }, + "15": { + "pass@1": 0.21346875, + "pass@2": 0.30841227854330716, + "pass@4": 0.4049719622234717, + "pass@8": 0.49117825934757464, + "pass@16": 0.5634527503442983, + "pass@32": 0.6227830148090836, + "pass@64": 0.6709739915681013, + "pass@128": 0.713 + }, + "16": { + "pass@1": 0.189796875, + "pass@2": 0.2703700787401575, + "pass@4": 0.3502655314960627, + "pass@8": 0.4213255040033605, + "pass@16": 0.48112732277263875, + "pass@32": 0.5305128166850834, + "pass@64": 0.5745087040366486, + "pass@128": 0.617 + }, + "17": { + "pass@1": 0.1787109375, + "pass@2": 0.25942790354330686, + "pass@4": 0.34142654199475053, + "pass@8": 0.4153820807769305, + "pass@16": 0.47790914635349946, + "pass@32": 0.5294888282400934, + "pass@64": 0.572868264153356, + "pass@128": 0.61 + }, + "18": { + "pass@1": 0.1653359375, + "pass@2": 0.24202337598425205, + "pass@4": 0.3200914730033739, + "pass@8": 0.38850015803167415, + "pass@16": 0.44363779653733026, + "pass@32": 0.4890084030206471, + "pass@64": 0.5291521300832082, + "pass@128": 0.568 + }, + "19": { + "pass@1": 0.147171875, + "pass@2": 0.21987709153543314, + "pass@4": 0.29461646522309676, + "pass@8": 0.3602151732688555, + "pass@16": 0.4130222240557081, + "pass@32": 0.4532058939682465, + "pass@64": 0.48334476130869763, + "pass@128": 0.508 + }, + "20": { + "pass@1": 0.1438671875, + "pass@2": 0.21731766732283428, + "pass@4": 0.2947381821334833, + "pass@8": 0.3636677425108799, + "pass@16": 0.4183750034963259, + "pass@32": 0.45883024328549243, + "pass@64": 0.4880575445468837, + "pass@128": 0.512 + } + }, + "per_op_accuracy": { + "11": 0.7227734375, + "12": 0.5365, + "13": 0.3344609375, + "14": 0.2862890625, + "15": 0.21346875, + "16": 0.189796875, + "17": 0.1787109375, + "18": 0.1653359375, + "19": 0.147171875, + "20": 0.1438671875 + }, + "per_op_avg_response_len": { + "11": 289.669125, + "12": 294.913328125, + "13": 302.4557421875, + "14": 312.470046875, + "15": 315.0388203125, + "16": 318.04046875, + "17": 322.45125, + "18": 321.5861328125, + "19": 321.1727265625, + "20": 322.8209453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 289.669125, + "12": 294.913328125, + "13": 302.4557421875, + "14": 312.470046875, + "15": 315.0388203125, + "16": 318.04046875, + "17": 322.45125, + "18": 321.5861328125, + "19": 321.1727265625, + "20": 322.8209453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 125688, + "answer_accuracy": 0.29180906389301636, + "avg_response_len": 294.47536914933136, + "resp_tokens_sum": 126836431, + "pass_at_k": { + "pass@1": 0.29180906389301636, + "pass@2": 0.3851575388143349, + "pass@4": 0.46899535537255743, + "pass@8": 0.5371794669428832, + "pass@16": 0.5902631858026006, + "pass@32": 0.6315234636832018, + "pass@64": 0.6656499070104925, + "pass@128": 0.6974739970282318 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7112694759206799, + "pass@2": 0.8131120206999624, + "pass@4": 0.8741029357874173, + "pass@8": 0.9131565853749971, + "pass@16": 0.9409259508048508, + "pass@32": 0.9607761804421557, + "pass@64": 0.9752001653135165, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.5198181497175142, + "pass@2": 0.652291444236843, + "pass@4": 0.7531961534469203, + "pass@8": 0.8229850744031951, + "pass@16": 0.8703747593722103, + "pass@32": 0.9035354750046218, + "pass@64": 0.92679855938621, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.34842770376175547, + "pass@2": 0.4644909504356623, + "pass@4": 0.5666685317352571, + "pass@8": 0.6515867742597433, + "pass@16": 0.7209558398105108, + "pass@32": 0.774390651037868, + "pass@64": 0.8159555350904601, + "pass@128": 0.8495297805642633 + }, + "14": { + "pass@1": 0.2620959890965732, + "pass@2": 0.36483859494198756, + "pass@4": 0.4648828545964464, + "pass@8": 0.5501300452769146, + "pass@16": 0.6169298321030059, + "pass@32": 0.6706355826772032, + "pass@64": 0.7209786790660362, + "pass@128": 0.7757009345794392 + }, + "15": { + "pass@1": 0.2202176661849711, + "pass@2": 0.3138636583450913, + "pass@4": 0.4027050682067921, + "pass@8": 0.4772871808592236, + "pass@16": 0.5401813834656242, + "pass@32": 0.5942484724698351, + "pass@64": 0.6404521097017122, + "pass@128": 0.684971098265896 + }, + "16": { + "pass@1": 0.1792086693548387, + "pass@2": 0.25795577216154436, + "pass@4": 0.33871481397486597, + "pass@8": 0.4115471071605108, + "pass@16": 0.47119022128456445, + "pass@32": 0.5159831622872939, + "pass@64": 0.551697044512079, + "pass@128": 0.5870967741935483 + }, + "17": { + "pass@1": 0.17575867052023122, + "pass@2": 0.2527504295434891, + "pass@4": 0.3287763653820727, + "pass@8": 0.3946413381092982, + "pass@16": 0.4472001973243355, + "pass@32": 0.48985344057501407, + "pass@64": 0.5292538770652234, + "pass@128": 0.5664739884393064 + }, + "18": { + "pass@1": 0.16078125, + "pass@2": 0.2357448678290212, + "pass@4": 0.31116799641116283, + "pass@8": 0.3780366912754487, + "pass@16": 0.4329714410571362, + "pass@32": 0.47825937552060704, + "pass@64": 0.5182588254375425, + "pass@128": 0.5571428571428572 + }, + "19": { + "pass@1": 0.15373967717717718, + "pass@2": 0.23123566479865715, + "pass@4": 0.30846989959588395, + "pass@8": 0.37090323389681995, + "pass@16": 0.4176420067808336, + "pass@32": 0.4519367816266829, + "pass@64": 0.4781589939232686, + "pass@128": 0.5045045045045045 + }, + "20": { + "pass@1": 0.1547484984984985, + "pass@2": 0.23214011255349842, + "pass@4": 0.3100945782678066, + "pass@8": 0.3735420424835841, + "pass@16": 0.4193360888579724, + "pass@32": 0.45293241378176113, + "pass@64": 0.47896014395846753, + "pass@128": 0.5015015015015015 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 124067, + "answer_accuracy": 0.28821690083258994, + "avg_response_len": 308.98490466101697, + "resp_tokens_sum": 133006878, + "pass_at_k": { + "pass@1": 0.28821690083258994, + "pass@2": 0.38068187472049964, + "pass@4": 0.4662712473998315, + "pass@8": 0.5383907563834387, + "pass@16": 0.5971019047548034, + "pass@32": 0.6451374930605417, + "pass@64": 0.6846334342529395, + "pass@128": 0.7169194171870353 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.719714796686747, + "pass@2": 0.8202420904088796, + "pass@4": 0.8819906120545167, + "pass@8": 0.9167660961161666, + "pass@16": 0.9370201729227511, + "pass@32": 0.9507746557206528, + "pass@64": 0.9612102815531016, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5531690830721003, + "pass@2": 0.691132707402562, + "pass@4": 0.7918788569336356, + "pass@8": 0.8558394828962391, + "pass@16": 0.8961570326737536, + "pass@32": 0.9242513072207585, + "pass@64": 0.9447949934870175, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.31256565126050423, + "pass@2": 0.4224401867046028, + "pass@4": 0.5283050091427646, + "pass@8": 0.6220202116228649, + "pass@16": 0.6990070859405011, + "pass@32": 0.7615370265201747, + "pass@64": 0.8117002962134, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.28723501461988304, + "pass@2": 0.3853191767969794, + "pass@4": 0.4756250548177094, + "pass@8": 0.5538791634637721, + "pass@16": 0.6247279946652008, + "pass@32": 0.6916342968490513, + "pass@64": 0.7535727398766265, + "pass@128": 0.8070175438596491 + }, + "15": { + "pass@1": 0.19509943181818182, + "pass@2": 0.2874798675733715, + "pass@4": 0.38628292741816334, + "pass@8": 0.47756921040509814, + "pass@16": 0.5566065517223067, + "pass@32": 0.6230961707223709, + "pass@64": 0.6746829994145453, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.19831730769230768, + "pass@2": 0.27961203367207305, + "pass@4": 0.35857375119776697, + "pass@8": 0.4255978477392451, + "pass@16": 0.47931263574118715, + "pass@32": 0.5250995747508337, + "pass@64": 0.5695799608843882, + "pass@128": 0.6125356125356125 + }, + "17": { + "pass@1": 0.19613742236024845, + "pass@2": 0.2840029252457574, + "pass@4": 0.3722219300421298, + "pass@8": 0.45339473578248296, + "pass@16": 0.5245596190614837, + "pass@32": 0.5828725152246444, + "pass@64": 0.6277436674236548, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.16300551470588234, + "pass@2": 0.24151480720240845, + "pass@4": 0.32298752040186174, + "pass@8": 0.39488191333318684, + "pass@16": 0.4550903465829013, + "pass@32": 0.5079746597085013, + "pass@64": 0.5556431661505973, + "pass@128": 0.5970588235294118 + }, + "19": { + "pass@1": 0.13816034226190477, + "pass@2": 0.20664604424446936, + "pass@4": 0.27967271836556157, + "pass@8": 0.34818998174106214, + "pass@16": 0.4053455576540456, + "pass@32": 0.4497394448722171, + "pass@64": 0.4830647705525789, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.1348942739520958, + "pass@2": 0.20611348660945836, + "pass@4": 0.2826266614689631, + "pass@8": 0.35183716167265444, + "pass@16": 0.4075823091006288, + "pass@32": 0.44687399315461174, + "pass@64": 0.474497256958554, + "pass@128": 0.5029940119760479 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 123797, + "answer_accuracy": 0.2955880386613692, + "avg_response_len": 333.31073788966995, + "resp_tokens_sum": 139595870, + "pass_at_k": { + "pass@1": 0.2955880386613692, + "pass@2": 0.390775254846659, + "pass@4": 0.47788852426836287, + "pass@8": 0.5503169688302919, + "pass@16": 0.6078856767954486, + "pass@32": 0.6532366558890249, + "pass@64": 0.6892412466812912, + "pass@128": 0.7197432762836186 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7388888888888889, + "pass@2": 0.8396380139982507, + "pass@4": 0.8983732589973874, + "pass@8": 0.9293637318750155, + "pass@16": 0.9469655881937245, + "pass@32": 0.958052279914744, + "pass@64": 0.9645245347844873, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5382979740061162, + "pass@2": 0.6815369229935708, + "pass@4": 0.7901882852372809, + "pass@8": 0.8623947124188998, + "pass@16": 0.9080716635283972, + "pass@32": 0.9356809750706235, + "pass@64": 0.9492487788739141, + "pass@128": 0.9571865443425076 + }, + "13": { + "pass@1": 0.3448350694444444, + "pass@2": 0.4652192998444639, + "pass@4": 0.571143411182398, + "pass@8": 0.6531886175627993, + "pass@16": 0.7148291106889169, + "pass@32": 0.7647239391313766, + "pass@64": 0.8073138228282857, + "pass@128": 0.8395061728395061 + }, + "14": { + "pass@1": 0.30837351632047477, + "pass@2": 0.41777261150961487, + "pass@4": 0.5206757623397962, + "pass@8": 0.6089125239005821, + "pass@16": 0.6839245743828597, + "pass@32": 0.7492554568430402, + "pass@64": 0.8034429167524338, + "pass@128": 0.8486646884272997 + }, + "15": { + "pass@1": 0.22497106481481483, + "pass@2": 0.32391079153300256, + "pass@4": 0.4264279091849627, + "pass@8": 0.5198736275821021, + "pass@16": 0.5952772518420743, + "pass@32": 0.6529361327041301, + "pass@64": 0.6997906537176186, + "pass@128": 0.7407407407407407 + }, + "16": { + "pass@1": 0.19065726401179942, + "pass@2": 0.27215329071144856, + "pass@4": 0.35222581856471474, + "pass@8": 0.4258438236788404, + "pass@16": 0.49209327147276405, + "pass@32": 0.5494043824143943, + "pass@64": 0.6004721355972974, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.16488610692771086, + "pass@2": 0.24255214756664453, + "pass@4": 0.3247423436716496, + "pass@8": 0.40012973755769266, + "pass@16": 0.4646677130767527, + "pass@32": 0.5190198430686861, + "pass@64": 0.5650992824649722, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.1730342741935484, + "pass@2": 0.24966979933959876, + "pass@4": 0.32699005588011176, + "pass@8": 0.39331440500639975, + "pass@16": 0.4431195946101498, + "pass@32": 0.48034270092756165, + "pass@64": 0.5123963377060167, + "pass@128": 0.5483870967741935 + }, + "19": { + "pass@1": 0.14971204682779457, + "pass@2": 0.2218807984632585, + "pass@4": 0.2958488075131092, + "pass@8": 0.36166937316077835, + "pass@16": 0.41616715536535226, + "pass@32": 0.4580014870377535, + "pass@64": 0.4888460828597657, + "pass@128": 0.5166163141993958 + }, + "20": { + "pass@1": 0.14198573573573572, + "pass@2": 0.2137330489938758, + "pass@4": 0.2915296775966066, + "pass@8": 0.36565955064618744, + "pass@16": 0.4282390230240574, + "pass@32": 0.4767202275751524, + "pass@64": 0.5107559543079784, + "pass@128": 0.5315315315315315 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5863199013157895, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5863199013157895, + "pass@2": 0.6522641356713357, + "pass@4": 0.7061554641179485, + "pass@8": 0.7485344273908813, + "pass@16": 0.781363354543752, + "pass@32": 0.8070635821999685, + "pass@64": 0.8277792339037006, + "pass@128": 0.8455263157894737 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8130390625, + "pass@2": 0.8855469980314973, + "pass@4": 0.92615446559805, + "pass@8": 0.9501353146499899, + "pass@16": 0.9657012336439675, + "pass@32": 0.9770663746047966, + "pass@64": 0.9851435119560961, + "pass@128": 0.991 + }, + "2": { + "pass@1": 0.9385703125, + "pass@2": 0.9525967027559057, + "pass@4": 0.9638686646981628, + "pass@8": 0.9724943090518903, + "pass@16": 0.9790929620706803, + "pass@32": 0.9842960053580423, + "pass@64": 0.9882386855842575, + "pass@128": 0.992 + }, + "3": { + "pass@1": 0.980015625, + "pass@2": 0.9875548720472446, + "pass@4": 0.9916484928758906, + "pass@8": 0.9943460459073127, + "pass@16": 0.9961729071719045, + "pass@32": 0.9973141929536077, + "pass@64": 0.9978679671606755, + "pass@128": 0.998 + }, + "4": { + "pass@1": 0.9676171875, + "pass@2": 0.9803442421259845, + "pass@4": 0.9866892795275589, + "pass@8": 0.9907940834376028, + "pass@16": 0.9937288393186484, + "pass@32": 0.9959362702515183, + "pass@64": 0.9979969035840609, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9206328125, + "pass@2": 0.9421621555118115, + "pass@4": 0.9567363708286464, + "pass@8": 0.9672069341201128, + "pass@16": 0.9743872248750242, + "pass@32": 0.9796198926321592, + "pass@64": 0.9843256419706993, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.9333359375, + "pass@2": 0.9661230314960632, + "pass@4": 0.9814899094488188, + "pass@8": 0.9888695765997886, + "pass@16": 0.9930217167612667, + "pass@32": 0.9952694212303389, + "pass@64": 0.9959625787840237, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.9184453125, + "pass@2": 0.9520457677165358, + "pass@4": 0.9696526289838768, + "pass@8": 0.97916936059381, + "pass@16": 0.9845091635229746, + "pass@32": 0.9878582864806053, + "pass@64": 0.9906888510786144, + "pass@128": 0.993 + }, + "8": { + "pass@1": 0.891296875, + "pass@2": 0.9444388533464576, + "pass@4": 0.9712997254405698, + "pass@8": 0.9844960086691613, + "pass@16": 0.9911105470974821, + "pass@32": 0.9946645302609637, + "pass@64": 0.9966357214747974, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.85875, + "pass@2": 0.9273010580708668, + "pass@4": 0.9595234535995493, + "pass@8": 0.9757883454039419, + "pass@16": 0.9848878814782323, + "pass@32": 0.9901189055015083, + "pass@64": 0.9934140469528695, + "pass@128": 0.996 + }, + "11": { + "pass@1": 0.7227734375, + "pass@2": 0.8238348917322844, + "pass@4": 0.8843667961192356, + "pass@8": 0.919460194088572, + "pass@16": 0.9415317183254884, + "pass@32": 0.9565976455684818, + "pass@64": 0.9671927002884151, + "pass@128": 0.975 + }, + "12": { + "pass@1": 0.5365, + "pass@2": 0.6742450787401567, + "pass@4": 0.7776323629546307, + "pass@8": 0.8463525823436129, + "pass@16": 0.8909261922144747, + "pass@32": 0.9206554040031512, + "pass@64": 0.9398806436368472, + "pass@128": 0.953 + }, + "13": { + "pass@1": 0.3344609375, + "pass@2": 0.44971481299212634, + "pass@4": 0.5544226151106105, + "pass@8": 0.6415505086285688, + "pass@16": 0.7111350744435204, + "pass@32": 0.7666698924273481, + "pass@64": 0.8116365000384063, + "pass@128": 0.846 + }, + "14": { + "pass@1": 0.2862890625, + "pass@2": 0.38968171751968494, + "pass@4": 0.48735889698162665, + "pass@8": 0.571221938992996, + "pass@16": 0.6421740318475855, + "pass@32": 0.7043120405178619, + "pass@64": 0.759916295963575, + "pass@128": 0.811 + }, + "15": { + "pass@1": 0.21346875, + "pass@2": 0.30841227854330716, + "pass@4": 0.4049719622234717, + "pass@8": 0.49117825934757464, + "pass@16": 0.5634527503442983, + "pass@32": 0.6227830148090836, + "pass@64": 0.6709739915681013, + "pass@128": 0.713 + }, + "16": { + "pass@1": 0.189796875, + "pass@2": 0.2703700787401575, + "pass@4": 0.3502655314960627, + "pass@8": 0.4213255040033605, + "pass@16": 0.48112732277263875, + "pass@32": 0.5305128166850834, + "pass@64": 0.5745087040366486, + "pass@128": 0.617 + }, + "17": { + "pass@1": 0.1787109375, + "pass@2": 0.25942790354330686, + "pass@4": 0.34142654199475053, + "pass@8": 0.4153820807769305, + "pass@16": 0.47790914635349946, + "pass@32": 0.5294888282400934, + "pass@64": 0.572868264153356, + "pass@128": 0.61 + }, + "18": { + "pass@1": 0.1653359375, + "pass@2": 0.24202337598425205, + "pass@4": 0.3200914730033739, + "pass@8": 0.38850015803167415, + "pass@16": 0.44363779653733026, + "pass@32": 0.4890084030206471, + "pass@64": 0.5291521300832082, + "pass@128": 0.568 + }, + "19": { + "pass@1": 0.147171875, + "pass@2": 0.21987709153543314, + "pass@4": 0.29461646522309676, + "pass@8": 0.3602151732688555, + "pass@16": 0.4130222240557081, + "pass@32": 0.4532058939682465, + "pass@64": 0.48334476130869763, + "pass@128": 0.508 + }, + "20": { + "pass@1": 0.1438671875, + "pass@2": 0.21731766732283428, + "pass@4": 0.2947381821334833, + "pass@8": 0.3636677425108799, + "pass@16": 0.4183750034963259, + "pass@32": 0.45883024328549243, + "pass@64": 0.4880575445468837, + "pass@128": 0.512 + } + }, + "per_op_accuracy": { + "10": 0.8130390625, + "2": 0.9385703125, + "3": 0.980015625, + "4": 0.9676171875, + "5": 0.9206328125, + "6": 0.9333359375, + "7": 0.9184453125, + "8": 0.891296875, + "9": 0.85875, + "11": 0.7227734375, + "12": 0.5365, + "13": 0.3344609375, + "14": 0.2862890625, + "15": 0.21346875, + "16": 0.189796875, + "17": 0.1787109375, + "18": 0.1653359375, + "19": 0.147171875, + "20": 0.1438671875 + }, + "per_op_avg_response_len": { + "10": 275.31103125, + "2": 120.48475, + "3": 145.7017421875, + "4": 159.9137265625, + "5": 173.2708828125, + "6": 199.5473984375, + "7": 222.4298828125, + "8": 242.818015625, + "9": 256.9630625, + "11": 289.669125, + "12": 294.913328125, + "13": 302.4557421875, + "14": 312.470046875, + "15": 315.0388203125, + "16": 318.04046875, + "17": 322.45125, + "18": 321.5861328125, + "19": 321.1727265625, + "20": 322.8209453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 275.31103125, + "2": 120.48475, + "3": 145.7017421875, + "4": 159.9137265625, + "5": 173.2708828125, + "6": 199.5473984375, + "7": 222.4298828125, + "8": 242.818015625, + "9": 256.9630625, + "11": 289.669125, + "12": 294.913328125, + "13": 302.4557421875, + "14": 312.470046875, + "15": 315.0388203125, + "16": 318.04046875, + "17": 322.45125, + "18": 321.5861328125, + "19": 321.1727265625, + "20": 322.8209453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 467579, + "answer_accuracy": 0.5774519344767626, + "avg_response_len": 245.32765200165983, + "resp_tokens_sum": 198648669, + "pass_at_k": { + "pass@1": 0.5774519344767626, + "pass@2": 0.6452416567297637, + "pass@4": 0.6996034111666334, + "pass@8": 0.7413455058301636, + "pass@16": 0.7729299247279426, + "pass@32": 0.7971920686194446, + "pass@64": 0.8170532304403487, + "pass@128": 0.8352829592159342 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.811992994100295, + "pass@2": 0.8843032134810588, + "pass@4": 0.923425883711439, + "pass@8": 0.9457277067863578, + "pass@16": 0.9606133820258961, + "pass@32": 0.971941456242833, + "pass@64": 0.979374562311218, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9364047897196262, + "pass@2": 0.9491822429906543, + "pass@4": 0.9604580686175911, + "pass@8": 0.9694622504718368, + "pass@16": 0.9762818313978732, + "pass@32": 0.9814895602246809, + "pass@64": 0.985727686903887, + "pass@128": 0.9906542056074766 + }, + "3": { + "pass@1": 0.9801462155963303, + "pass@2": 0.9896420272580605, + "pass@4": 0.9937778259368953, + "pass@8": 0.9962077916624781, + "pass@16": 0.9981311364256852, + "pass@32": 0.9995033618733702, + "pass@64": 0.999978967901273, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.969677734375, + "pass@2": 0.9814676119586613, + "pass@4": 0.986887528707349, + "pass@8": 0.9899307200629559, + "pass@16": 0.9917621691650451, + "pass@32": 0.993528368509755, + "pass@64": 0.9960998638576377, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8992063492063492, + "pass@2": 0.9267841519810028, + "pass@4": 0.9439647365507879, + "pass@8": 0.9548172712010686, + "pass@16": 0.9623409287636865, + "pass@32": 0.9688380283149466, + "pass@64": 0.9755950258820365, + "pass@128": 0.9809523809523809 + }, + "6": { + "pass@1": 0.9132204341317365, + "pass@2": 0.9474187697439762, + "pass@4": 0.965994401803816, + "pass@8": 0.9784493955374186, + "pass@16": 0.9872882423827155, + "pass@32": 0.9922681964206024, + "pass@64": 0.9939103062084376, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.8838063979289941, + "pass@2": 0.9277586719004803, + "pass@4": 0.9530765176172504, + "pass@8": 0.9664595878668178, + "pass@16": 0.9740977457411283, + "pass@32": 0.9795563479381585, + "pass@64": 0.9843567124179412, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8821864983974359, + "pass@2": 0.9410403101655567, + "pass@4": 0.9686909091291468, + "pass@8": 0.9800378182202505, + "pass@16": 0.9851975708878595, + "pass@32": 0.988658383641544, + "pass@64": 0.9911660518873375, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.8510783450704226, + "pass@2": 0.924036195519574, + "pass@4": 0.9606805254096751, + "pass@8": 0.9797217797223858, + "pass@16": 0.9887705852950439, + "pass@32": 0.9929424597619723, + "pass@64": 0.9953811357885473, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.7112694759206799, + "pass@2": 0.8131120206999624, + "pass@4": 0.8741029357874173, + "pass@8": 0.9131565853749971, + "pass@16": 0.9409259508048508, + "pass@32": 0.9607761804421557, + "pass@64": 0.9752001653135165, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.5198181497175142, + "pass@2": 0.652291444236843, + "pass@4": 0.7531961534469203, + "pass@8": 0.8229850744031951, + "pass@16": 0.8703747593722103, + "pass@32": 0.9035354750046218, + "pass@64": 0.92679855938621, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.34842770376175547, + "pass@2": 0.4644909504356623, + "pass@4": 0.5666685317352571, + "pass@8": 0.6515867742597433, + "pass@16": 0.7209558398105108, + "pass@32": 0.774390651037868, + "pass@64": 0.8159555350904601, + "pass@128": 0.8495297805642633 + }, + "14": { + "pass@1": 0.2620959890965732, + "pass@2": 0.36483859494198756, + "pass@4": 0.4648828545964464, + "pass@8": 0.5501300452769146, + "pass@16": 0.6169298321030059, + "pass@32": 0.6706355826772032, + "pass@64": 0.7209786790660362, + "pass@128": 0.7757009345794392 + }, + "15": { + "pass@1": 0.2202176661849711, + "pass@2": 0.3138636583450913, + "pass@4": 0.4027050682067921, + "pass@8": 0.4772871808592236, + "pass@16": 0.5401813834656242, + "pass@32": 0.5942484724698351, + "pass@64": 0.6404521097017122, + "pass@128": 0.684971098265896 + }, + "16": { + "pass@1": 0.1792086693548387, + "pass@2": 0.25795577216154436, + "pass@4": 0.33871481397486597, + "pass@8": 0.4115471071605108, + "pass@16": 0.47119022128456445, + "pass@32": 0.5159831622872939, + "pass@64": 0.551697044512079, + "pass@128": 0.5870967741935483 + }, + "17": { + "pass@1": 0.17575867052023122, + "pass@2": 0.2527504295434891, + "pass@4": 0.3287763653820727, + "pass@8": 0.3946413381092982, + "pass@16": 0.4472001973243355, + "pass@32": 0.48985344057501407, + "pass@64": 0.5292538770652234, + "pass@128": 0.5664739884393064 + }, + "18": { + "pass@1": 0.16078125, + "pass@2": 0.2357448678290212, + "pass@4": 0.31116799641116283, + "pass@8": 0.3780366912754487, + "pass@16": 0.4329714410571362, + "pass@32": 0.47825937552060704, + "pass@64": 0.5182588254375425, + "pass@128": 0.5571428571428572 + }, + "19": { + "pass@1": 0.15373967717717718, + "pass@2": 0.23123566479865715, + "pass@4": 0.30846989959588395, + "pass@8": 0.37090323389681995, + "pass@16": 0.4176420067808336, + "pass@32": 0.4519367816266829, + "pass@64": 0.4781589939232686, + "pass@128": 0.5045045045045045 + }, + "20": { + "pass@1": 0.1547484984984985, + "pass@2": 0.23214011255349842, + "pass@4": 0.3100945782678066, + "pass@8": 0.3735420424835841, + "pass@16": 0.4193360888579724, + "pass@32": 0.45293241378176113, + "pass@64": 0.47896014395846753, + "pass@128": 0.5015015015015015 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 482531, + "answer_accuracy": 0.5876497954014029, + "avg_response_len": 255.93222793257988, + "resp_tokens_sum": 210151071, + "pass_at_k": { + "pass@1": 0.5876497954014029, + "pass@2": 0.6525561207737851, + "pass@4": 0.7061835820210686, + "pass@8": 0.7490327747499966, + "pass@16": 0.7829268531033039, + "pass@32": 0.8100206823311829, + "pass@64": 0.8319087994567788, + "pass@128": 0.8497272018706158 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8102233512269938, + "pass@2": 0.8818788192599386, + "pass@4": 0.9230943886998785, + "pass@8": 0.9486758986818827, + "pass@16": 0.9665855700413544, + "pass@32": 0.9798571294366684, + "pass@64": 0.9889181243773738, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9407057370820668, + "pass@2": 0.9543108142067347, + "pass@4": 0.9648635555244045, + "pass@8": 0.9735475712944428, + "pass@16": 0.980713903139753, + "pass@32": 0.9855875985518405, + "pass@64": 0.9885083794652192, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.9831912878787878, + "pass@2": 0.9890449773323791, + "pass@4": 0.9925742770790013, + "pass@8": 0.9949117732864091, + "pass@16": 0.9968032945575152, + "pass@32": 0.998445165474543, + "pass@64": 0.999620777962612, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9603365384615384, + "pass@2": 0.9773203448958673, + "pass@4": 0.9862346477215493, + "pass@8": 0.9916871794012287, + "pass@16": 0.995383380617976, + "pass@32": 0.9977698233271339, + "pass@64": 0.9994625180003253, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9483016304347827, + "pass@2": 0.9635705523222642, + "pass@4": 0.9734722803671282, + "pass@8": 0.9809383138264673, + "pass@16": 0.9851226251370658, + "pass@32": 0.9872274839565104, + "pass@64": 0.9891271668301083, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9358882521489972, + "pass@2": 0.9721159161157868, + "pass@4": 0.9889131951271127, + "pass@8": 0.9951489089035775, + "pass@16": 0.9979499235991857, + "pass@32": 0.9995757636157672, + "pass@64": 0.999990075961284, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9294391257225434, + "pass@2": 0.9595930431022713, + "pass@4": 0.9755940045969683, + "pass@8": 0.9847787879592633, + "pass@16": 0.9892688554315738, + "pass@32": 0.9908053653135379, + "pass@64": 0.9912895111919918, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.8965448943661972, + "pass@2": 0.9502491821004772, + "pass@4": 0.977070321931589, + "pass@8": 0.9902363110097167, + "pass@16": 0.9961681334129603, + "pass@32": 0.9986011198470229, + "pass@64": 0.999793253291699, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8508373877245509, + "pass@2": 0.9181932640624261, + "pass@4": 0.9504120931365609, + "pass@8": 0.9681510539987431, + "pass@16": 0.9796319664627892, + "pass@32": 0.9865058729633788, + "pass@64": 0.9904633864127251, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.719714796686747, + "pass@2": 0.8202420904088796, + "pass@4": 0.8819906120545167, + "pass@8": 0.9167660961161666, + "pass@16": 0.9370201729227511, + "pass@32": 0.9507746557206528, + "pass@64": 0.9612102815531016, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5531690830721003, + "pass@2": 0.691132707402562, + "pass@4": 0.7918788569336356, + "pass@8": 0.8558394828962391, + "pass@16": 0.8961570326737536, + "pass@32": 0.9242513072207585, + "pass@64": 0.9447949934870175, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.31256565126050423, + "pass@2": 0.4224401867046028, + "pass@4": 0.5283050091427646, + "pass@8": 0.6220202116228649, + "pass@16": 0.6990070859405011, + "pass@32": 0.7615370265201747, + "pass@64": 0.8117002962134, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.28723501461988304, + "pass@2": 0.3853191767969794, + "pass@4": 0.4756250548177094, + "pass@8": 0.5538791634637721, + "pass@16": 0.6247279946652008, + "pass@32": 0.6916342968490513, + "pass@64": 0.7535727398766265, + "pass@128": 0.8070175438596491 + }, + "15": { + "pass@1": 0.19509943181818182, + "pass@2": 0.2874798675733715, + "pass@4": 0.38628292741816334, + "pass@8": 0.47756921040509814, + "pass@16": 0.5566065517223067, + "pass@32": 0.6230961707223709, + "pass@64": 0.6746829994145453, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.19831730769230768, + "pass@2": 0.27961203367207305, + "pass@4": 0.35857375119776697, + "pass@8": 0.4255978477392451, + "pass@16": 0.47931263574118715, + "pass@32": 0.5250995747508337, + "pass@64": 0.5695799608843882, + "pass@128": 0.6125356125356125 + }, + "17": { + "pass@1": 0.19613742236024845, + "pass@2": 0.2840029252457574, + "pass@4": 0.3722219300421298, + "pass@8": 0.45339473578248296, + "pass@16": 0.5245596190614837, + "pass@32": 0.5828725152246444, + "pass@64": 0.6277436674236548, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.16300551470588234, + "pass@2": 0.24151480720240845, + "pass@4": 0.32298752040186174, + "pass@8": 0.39488191333318684, + "pass@16": 0.4550903465829013, + "pass@32": 0.5079746597085013, + "pass@64": 0.5556431661505973, + "pass@128": 0.5970588235294118 + }, + "19": { + "pass@1": 0.13816034226190477, + "pass@2": 0.20664604424446936, + "pass@4": 0.27967271836556157, + "pass@8": 0.34818998174106214, + "pass@16": 0.4053455576540456, + "pass@32": 0.4497394448722171, + "pass@64": 0.4830647705525789, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.1348942739520958, + "pass@2": 0.20611348660945836, + "pass@4": 0.2826266614689631, + "pass@8": 0.35183716167265444, + "pass@16": 0.4075823091006288, + "pass@32": 0.44687399315461174, + "pass@64": 0.474497256958554, + "pass@128": 0.5029940119760479 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 475820, + "answer_accuracy": 0.5939197555520052, + "avg_response_len": 275.3332975515258, + "resp_tokens_sum": 220583822, + "pass_at_k": { + "pass@1": 0.5939197555520052, + "pass@2": 0.6590625247674902, + "pass@4": 0.7127488353628675, + "pass@8": 0.7552895351531898, + "pass@16": 0.7882845933606959, + "pass@32": 0.8140099630225133, + "pass@64": 0.8343875634908813, + "pass@128": 0.8515737338232945 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8168376865671642, + "pass@2": 0.8903752644259023, + "pass@4": 0.9318934934439158, + "pass@8": 0.9560157584451383, + "pass@16": 0.9699892576230075, + "pass@32": 0.9795367066929017, + "pass@64": 0.9873081396584162, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9385491071428571, + "pass@2": 0.9541169853768277, + "pass@4": 0.9660614711553913, + "pass@8": 0.9742850734130257, + "pass@16": 0.9801474858828125, + "pass@32": 0.9856558188638964, + "pass@64": 0.990288117840151, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9768358236151603, + "pass@2": 0.9841314478547326, + "pass@4": 0.9887277911325226, + "pass@8": 0.9920268596185634, + "pass@16": 0.9936995287368084, + "pass@32": 0.9941390350274529, + "pass@64": 0.9941690610183596, + "pass@128": 0.9941690962099126 + }, + "4": { + "pass@1": 0.9728846856725146, + "pass@2": 0.9822816658608464, + "pass@4": 0.9869530976939285, + "pass@8": 0.9907192584205899, + "pass@16": 0.9939338085875972, + "pass@32": 0.9963771697187885, + "pass@64": 0.9983234387880316, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9124080882352941, + "pass@2": 0.9346861973135713, + "pass@4": 0.9515868884955558, + "pass@8": 0.9647522806518978, + "pass@16": 0.9746544901240447, + "pass@32": 0.9818895051410447, + "pass@64": 0.987542165474913, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9517202287066246, + "pass@2": 0.9792325163317526, + "pass@4": 0.9896437670251783, + "pass@8": 0.992935360514013, + "pass@16": 0.9936369730893512, + "pass@32": 0.9936905429777773, + "pass@64": 0.9936908517347554, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9434582674050633, + "pass@2": 0.9697599484202134, + "pass@4": 0.9808773431604276, + "pass@8": 0.9866220229775958, + "pass@16": 0.990433865453002, + "pass@32": 0.9935113432880496, + "pass@64": 0.9968041500282304, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.894237987987988, + "pass@2": 0.9414288796670683, + "pass@4": 0.9675921847043887, + "pass@8": 0.9825535104384212, + "pass@16": 0.9912589117683784, + "pass@32": 0.9960952463036303, + "pass@64": 0.9983943194216062, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8760048231511254, + "pass@2": 0.940809191204396, + "pass@4": 0.967987871291007, + "pass@8": 0.9795005195077631, + "pass@16": 0.9861004723470128, + "pass@32": 0.9907761116277802, + "pass@64": 0.9943375327526845, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.7388888888888889, + "pass@2": 0.8396380139982507, + "pass@4": 0.8983732589973874, + "pass@8": 0.9293637318750155, + "pass@16": 0.9469655881937245, + "pass@32": 0.958052279914744, + "pass@64": 0.9645245347844873, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5382979740061162, + "pass@2": 0.6815369229935708, + "pass@4": 0.7901882852372809, + "pass@8": 0.8623947124188998, + "pass@16": 0.9080716635283972, + "pass@32": 0.9356809750706235, + "pass@64": 0.9492487788739141, + "pass@128": 0.9571865443425076 + }, + "13": { + "pass@1": 0.3448350694444444, + "pass@2": 0.4652192998444639, + "pass@4": 0.571143411182398, + "pass@8": 0.6531886175627993, + "pass@16": 0.7148291106889169, + "pass@32": 0.7647239391313766, + "pass@64": 0.8073138228282857, + "pass@128": 0.8395061728395061 + }, + "14": { + "pass@1": 0.30837351632047477, + "pass@2": 0.41777261150961487, + "pass@4": 0.5206757623397962, + "pass@8": 0.6089125239005821, + "pass@16": 0.6839245743828597, + "pass@32": 0.7492554568430402, + "pass@64": 0.8034429167524338, + "pass@128": 0.8486646884272997 + }, + "15": { + "pass@1": 0.22497106481481483, + "pass@2": 0.32391079153300256, + "pass@4": 0.4264279091849627, + "pass@8": 0.5198736275821021, + "pass@16": 0.5952772518420743, + "pass@32": 0.6529361327041301, + "pass@64": 0.6997906537176186, + "pass@128": 0.7407407407407407 + }, + "16": { + "pass@1": 0.19065726401179942, + "pass@2": 0.27215329071144856, + "pass@4": 0.35222581856471474, + "pass@8": 0.4258438236788404, + "pass@16": 0.49209327147276405, + "pass@32": 0.5494043824143943, + "pass@64": 0.6004721355972974, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.16488610692771086, + "pass@2": 0.24255214756664453, + "pass@4": 0.3247423436716496, + "pass@8": 0.40012973755769266, + "pass@16": 0.4646677130767527, + "pass@32": 0.5190198430686861, + "pass@64": 0.5650992824649722, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.1730342741935484, + "pass@2": 0.24966979933959876, + "pass@4": 0.32699005588011176, + "pass@8": 0.39331440500639975, + "pass@16": 0.4431195946101498, + "pass@32": 0.48034270092756165, + "pass@64": 0.5123963377060167, + "pass@128": 0.5483870967741935 + }, + "19": { + "pass@1": 0.14971204682779457, + "pass@2": 0.2218807984632585, + "pass@4": 0.2958488075131092, + "pass@8": 0.36166937316077835, + "pass@16": 0.41616715536535226, + "pass@32": 0.4580014870377535, + "pass@64": 0.4888460828597657, + "pass@128": 0.5166163141993958 + }, + "20": { + "pass@1": 0.14198573573573572, + "pass@2": 0.2137330489938758, + "pass@4": 0.2915296775966066, + "pass@8": 0.36565955064618744, + "pass@16": 0.4282390230240574, + "pass@32": 0.4767202275751524, + "pass@64": 0.5107559543079784, + "pass@128": 0.5315315315315315 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep1-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..795d56c5c0ee67421c760c6d077a227b81009def --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block16/composition/autoregressive", "results_dir": "results/SDAR-100M-ep1-block16/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block16/composition/autoregressive/checkpoint-21808_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 897867, "value_mismatch": 122548, "dependency_mismatch": 204852, "answer_mismatch": 99624, "extra_nodes": 88483, "missing_nodes": 91920, "perfect_with_extra": 7064, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "value_mismatch": 45953, "answer_mismatch": 37117, "perfect": 289803, "dependency_mismatch": 71587, "missing_nodes": 35185, "extra_nodes": 31774, "perfect_with_extra": 1611}, "teachers_in_school": {"total": 390656, "perfect": 306398, "value_mismatch": 38713, "dependency_mismatch": 68024, "answer_mismatch": 32194, "extra_nodes": 28648, "missing_nodes": 29422, "perfect_with_extra": 2670}, "movie_festival_awards": {"total": 382336, "perfect": 301666, "dependency_mismatch": 65241, "value_mismatch": 37882, "answer_mismatch": 30313, "missing_nodes": 27313, "extra_nodes": 28061, "perfect_with_extra": 2783}}, "results_path": "results/SDAR-100M-ep1-block16/composition/autoregressive/graphs/checkpoint-21808_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block16/composition/autoregressive/checkpoint-21808_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 98089, "value_mismatch": 1067430, "dependency_mismatch": 1127456, "answer_mismatch": 906471, "extra_nodes": 193658, "missing_nodes": 899585, "perfect_with_extra": 70, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 359196, "dependency_mismatch": 381674, "answer_mismatch": 305029, "missing_nodes": 314003, "extra_nodes": 84347, "perfect": 31404, "perfect_with_extra": 22}, "movie_festival_awards": {"total": 418816, "dependency_mismatch": 368683, "perfect": 33353, "value_mismatch": 348122, "answer_mismatch": 295033, "missing_nodes": 290587, "extra_nodes": 55821, "perfect_with_extra": 19}, "teachers_in_school": {"total": 430464, "perfect": 33332, "value_mismatch": 360112, "dependency_mismatch": 377099, "answer_mismatch": 306409, "missing_nodes": 294995, "extra_nodes": 53490, "perfect_with_extra": 29}}, "results_path": "results/SDAR-100M-ep1-block16/composition/autoregressive/graphs/checkpoint-21808_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 995956, "value_mismatch": 1189978, "dependency_mismatch": 1332308, "answer_mismatch": 1006095, "extra_nodes": 282141, "missing_nodes": 991505, "perfect_with_extra": 7134, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block16/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "value_mismatch": 405149, "answer_mismatch": 342146, "perfect": 321207, "dependency_mismatch": 453261, "missing_nodes": 349188, "extra_nodes": 116121, "perfect_with_extra": 1633}, "teachers_in_school": {"total": 821120, "perfect": 339730, "value_mismatch": 398825, "dependency_mismatch": 445123, "answer_mismatch": 338603, "extra_nodes": 82138, "missing_nodes": 324417, "perfect_with_extra": 2699}, "movie_festival_awards": {"total": 801152, "perfect": 335019, "dependency_mismatch": 433924, "value_mismatch": 386004, "answer_mismatch": 325346, "missing_nodes": 317900, "extra_nodes": 83882, "perfect_with_extra": 2802}}}} diff --git a/SDAR-100M-ep1-block16/composition/autoregressive/graphs/results.json b/SDAR-100M-ep1-block16/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..6eb22f4b218d3d7a5bd5ca206368ab1b3dc8c12a --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21808_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9221183800623053, + "pass@2": 0.9470404984423676, + "pass@4": 0.9532710280373832, + "pass@8": 0.956386292834891, + "pass@16": 0.956386292834891, + "pass@32": 0.9626168224299065, + "pass@64": 0.9657320872274143, + "pass@128": 0.9688473520249221 + }, + "3": { + "pass@1": 0.9602446483180428, + "pass@2": 0.9755351681957186, + "pass@4": 0.9785932721712538, + "pass@8": 0.981651376146789, + "pass@16": 0.981651376146789, + "pass@32": 0.9847094801223242, + "pass@64": 0.9847094801223242, + "pass@128": 0.9908256880733946 + }, + "4": { + "pass@1": 0.95625, + "pass@2": 0.96875, + "pass@4": 0.978125, + "pass@8": 0.978125, + "pass@16": 0.978125, + "pass@32": 0.984375, + "pass@64": 0.984375, + "pass@128": 0.9875 + }, + "5": { + "pass@1": 0.8063492063492064, + "pass@2": 0.8507936507936508, + "pass@4": 0.8857142857142857, + "pass@8": 0.9015873015873016, + "pass@16": 0.9079365079365079, + "pass@32": 0.9238095238095239, + "pass@64": 0.9333333333333333, + "pass@128": 0.946031746031746 + }, + "6": { + "pass@1": 0.8233532934131736, + "pass@2": 0.8862275449101796, + "pass@4": 0.9281437125748503, + "pass@8": 0.9431137724550899, + "pass@16": 0.9670658682634731, + "pass@32": 0.9700598802395209, + "pass@64": 0.9820359281437125, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.7307692307692307, + "pass@2": 0.8165680473372781, + "pass@4": 0.8579881656804734, + "pass@8": 0.8727810650887574, + "pass@16": 0.9023668639053254, + "pass@32": 0.9171597633136095, + "pass@64": 0.9260355029585798, + "pass@128": 0.9349112426035503 + }, + "8": { + "pass@1": 0.6282051282051282, + "pass@2": 0.7243589743589743, + "pass@4": 0.8397435897435898, + "pass@8": 0.8974358974358975, + "pass@16": 0.9391025641025641, + "pass@32": 0.9487179487179487, + "pass@64": 0.9583333333333334, + "pass@128": 0.9647435897435898 + }, + "9": { + "pass@1": 0.6028169014084507, + "pass@2": 0.7126760563380282, + "pass@4": 0.7774647887323943, + "pass@8": 0.8422535211267606, + "pass@16": 0.8985915492957747, + "pass@32": 0.9154929577464789, + "pass@64": 0.9492957746478873, + "pass@128": 0.9633802816901409 + }, + "10": { + "pass@1": 0.5014749262536873, + "pass@2": 0.6047197640117994, + "pass@4": 0.6814159292035398, + "pass@8": 0.775811209439528, + "pass@16": 0.8141592920353983, + "pass@32": 0.8495575221238938, + "pass@64": 0.8731563421828908, + "pass@128": 0.887905604719764 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9221183800623053, + "3": 0.9602446483180428, + "4": 0.95625, + "5": 0.8063492063492064, + "6": 0.8233532934131736, + "7": 0.7307692307692307, + "8": 0.6282051282051282, + "9": 0.6028169014084507, + "10": 0.5014749262536873 + }, + "pass@2": { + "2": 0.9470404984423676, + "3": 0.9755351681957186, + "4": 0.96875, + "5": 0.8507936507936508, + "6": 0.8862275449101796, + "7": 0.8165680473372781, + "8": 0.7243589743589743, + "9": 0.7126760563380282, + "10": 0.6047197640117994 + }, + "pass@4": { + "2": 0.9532710280373832, + "3": 0.9785932721712538, + "4": 0.978125, + "5": 0.8857142857142857, + "6": 0.9281437125748503, + "7": 0.8579881656804734, + "8": 0.8397435897435898, + "9": 0.7774647887323943, + "10": 0.6814159292035398 + }, + "pass@8": { + "2": 0.956386292834891, + "3": 0.981651376146789, + "4": 0.978125, + "5": 0.9015873015873016, + "6": 0.9431137724550899, + "7": 0.8727810650887574, + "8": 0.8974358974358975, + "9": 0.8422535211267606, + "10": 0.775811209439528 + }, + "pass@16": { + "2": 0.956386292834891, + "3": 0.981651376146789, + "4": 0.978125, + "5": 0.9079365079365079, + "6": 0.9670658682634731, + "7": 0.9023668639053254, + "8": 0.9391025641025641, + "9": 0.8985915492957747, + "10": 0.8141592920353983 + }, + "pass@32": { + "2": 0.9626168224299065, + "3": 0.9847094801223242, + "4": 0.984375, + "5": 0.9238095238095239, + "6": 0.9700598802395209, + "7": 0.9171597633136095, + "8": 0.9487179487179487, + "9": 0.9154929577464789, + "10": 0.8495575221238938 + }, + "pass@64": { + "2": 0.9657320872274143, + "3": 0.9847094801223242, + "4": 0.984375, + "5": 0.9333333333333333, + "6": 0.9820359281437125, + "7": 0.9260355029585798, + "8": 0.9583333333333334, + "9": 0.9492957746478873, + "10": 0.8731563421828908 + }, + "pass@128": { + "2": 0.9688473520249221, + "3": 0.9908256880733946, + "4": 0.9875, + "5": 0.946031746031746, + "6": 0.9820359281437125, + "7": 0.9349112426035503, + "8": 0.9647435897435898, + "9": 0.9633802816901409, + "10": 0.887905604719764 + } + }, + "total_pass_at_k": { + "pass@1": 0.7673083417764269, + "pass@2": 0.8297872340425532, + "pass@4": 0.8736913204998311, + "pass@8": 0.9040864572779467, + "pass@16": 0.9263762242485647, + "pass@32": 0.9388720027017899, + "pass@64": 0.950354609929078, + "pass@128": 0.9581222559945964 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.92, + "pass@2": 0.9342857142857143, + "pass@4": 0.9428571428571428, + "pass@8": 0.9571428571428572, + "pass@16": 0.9628571428571429, + "pass@32": 0.9714285714285714, + "pass@64": 0.9714285714285714, + "pass@128": 0.9714285714285714 + }, + "3": { + "pass@1": 0.9650145772594753, + "pass@2": 0.9737609329446064, + "pass@4": 0.9825072886297376, + "pass@8": 0.9825072886297376, + "pass@16": 0.9854227405247813, + "pass@32": 0.9883381924198251, + "pass@64": 0.9883381924198251, + "pass@128": 0.9912536443148688 + }, + "4": { + "pass@1": 0.935672514619883, + "pass@2": 0.9590643274853801, + "pass@4": 0.9649122807017544, + "pass@8": 0.9766081871345029, + "pass@16": 0.9853801169590644, + "pass@32": 0.9883040935672515, + "pass@64": 0.9883040935672515, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.8323529411764706, + "pass@2": 0.8735294117647059, + "pass@4": 0.9058823529411765, + "pass@8": 0.9117647058823529, + "pass@16": 0.9235294117647059, + "pass@32": 0.9235294117647059, + "pass@64": 0.9294117647058824, + "pass@128": 0.9382352941176471 + }, + "6": { + "pass@1": 0.8832807570977917, + "pass@2": 0.9305993690851735, + "pass@4": 0.9589905362776026, + "pass@8": 0.9779179810725552, + "pass@16": 0.9810725552050473, + "pass@32": 0.9842271293375394, + "pass@64": 0.9905362776025236, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.7689873417721519, + "pass@2": 0.8354430379746836, + "pass@4": 0.8860759493670886, + "pass@8": 0.9145569620253164, + "pass@16": 0.9462025316455697, + "pass@32": 0.9556962025316456, + "pass@64": 0.9620253164556962, + "pass@128": 0.9651898734177216 + }, + "8": { + "pass@1": 0.6786786786786787, + "pass@2": 0.7777777777777778, + "pass@4": 0.8468468468468469, + "pass@8": 0.9129129129129129, + "pass@16": 0.9279279279279279, + "pass@32": 0.954954954954955, + "pass@64": 0.9669669669669669, + "pass@128": 0.972972972972973 + }, + "9": { + "pass@1": 0.5819935691318328, + "pass@2": 0.7331189710610932, + "pass@4": 0.797427652733119, + "pass@8": 0.8810289389067524, + "pass@16": 0.9356913183279743, + "pass@32": 0.9614147909967846, + "pass@64": 0.977491961414791, + "pass@128": 0.977491961414791 + }, + "10": { + "pass@1": 0.46567164179104475, + "pass@2": 0.5462686567164179, + "pass@4": 0.6119402985074627, + "pass@8": 0.7044776119402985, + "pass@16": 0.7701492537313432, + "pass@32": 0.8149253731343283, + "pass@64": 0.8626865671641791, + "pass@128": 0.8805970149253731 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.92, + "3": 0.9650145772594753, + "4": 0.935672514619883, + "5": 0.8323529411764706, + "6": 0.8832807570977917, + "7": 0.7689873417721519, + "8": 0.6786786786786787, + "9": 0.5819935691318328, + "10": 0.46567164179104475 + }, + "pass@2": { + "2": 0.9342857142857143, + "3": 0.9737609329446064, + "4": 0.9590643274853801, + "5": 0.8735294117647059, + "6": 0.9305993690851735, + "7": 0.8354430379746836, + "8": 0.7777777777777778, + "9": 0.7331189710610932, + "10": 0.5462686567164179 + }, + "pass@4": { + "2": 0.9428571428571428, + "3": 0.9825072886297376, + "4": 0.9649122807017544, + "5": 0.9058823529411765, + "6": 0.9589905362776026, + "7": 0.8860759493670886, + "8": 0.8468468468468469, + "9": 0.797427652733119, + "10": 0.6119402985074627 + }, + "pass@8": { + "2": 0.9571428571428572, + "3": 0.9825072886297376, + "4": 0.9766081871345029, + "5": 0.9117647058823529, + "6": 0.9779179810725552, + "7": 0.9145569620253164, + "8": 0.9129129129129129, + "9": 0.8810289389067524, + "10": 0.7044776119402985 + }, + "pass@16": { + "2": 0.9628571428571429, + "3": 0.9854227405247813, + "4": 0.9853801169590644, + "5": 0.9235294117647059, + "6": 0.9810725552050473, + "7": 0.9462025316455697, + "8": 0.9279279279279279, + "9": 0.9356913183279743, + "10": 0.7701492537313432 + }, + "pass@32": { + "2": 0.9714285714285714, + "3": 0.9883381924198251, + "4": 0.9883040935672515, + "5": 0.9235294117647059, + "6": 0.9842271293375394, + "7": 0.9556962025316456, + "8": 0.954954954954955, + "9": 0.9614147909967846, + "10": 0.8149253731343283 + }, + "pass@64": { + "2": 0.9714285714285714, + "3": 0.9883381924198251, + "4": 0.9883040935672515, + "5": 0.9294117647058824, + "6": 0.9905362776025236, + "7": 0.9620253164556962, + "8": 0.9669669669669669, + "9": 0.977491961414791, + "10": 0.8626865671641791 + }, + "pass@128": { + "2": 0.9714285714285714, + "3": 0.9912536443148688, + "4": 0.9883040935672515, + "5": 0.9382352941176471, + "6": 0.9905362776025236, + "7": 0.9651898734177216, + "8": 0.972972972972973, + "9": 0.977491961414791, + "10": 0.8805970149253731 + } + }, + "total_pass_at_k": { + "pass@1": 0.7840642785403414, + "pass@2": 0.841981921660529, + "pass@4": 0.8784733846668898, + "pass@8": 0.9136257114161366, + "pass@16": 0.9353866755942417, + "pass@32": 0.9491128222296619, + "pass@64": 0.9594911282222967, + "pass@128": 0.9638433210579176 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9270516717325228, + "pass@2": 0.9422492401215805, + "pass@4": 0.9544072948328267, + "pass@8": 0.9604863221884499, + "pass@16": 0.9696048632218845, + "pass@32": 0.9726443768996961, + "pass@64": 0.9756838905775076, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.9666666666666667, + "pass@2": 0.9787878787878788, + "pass@4": 0.9848484848484849, + "pass@8": 0.990909090909091, + "pass@16": 0.9939393939393939, + "pass@32": 0.9939393939393939, + "pass@64": 0.9939393939393939, + "pass@128": 0.9939393939393939 + }, + "4": { + "pass@1": 0.9053254437869822, + "pass@2": 0.9230769230769231, + "pass@4": 0.9437869822485208, + "pass@8": 0.9526627218934911, + "pass@16": 0.9585798816568047, + "pass@32": 0.9644970414201184, + "pass@64": 0.9644970414201184, + "pass@128": 0.9644970414201184 + }, + "5": { + "pass@1": 0.8840579710144928, + "pass@2": 0.9188405797101449, + "pass@4": 0.9420289855072463, + "pass@8": 0.9478260869565217, + "pass@16": 0.9507246376811594, + "pass@32": 0.9565217391304348, + "pass@64": 0.9594202898550724, + "pass@128": 0.9594202898550724 + }, + "6": { + "pass@1": 0.8595988538681948, + "pass@2": 0.9111747851002865, + "pass@4": 0.9455587392550143, + "pass@8": 0.9713467048710601, + "pass@16": 0.9828080229226361, + "pass@32": 0.9885386819484241, + "pass@64": 0.9914040114613181, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.7630057803468208, + "pass@2": 0.8352601156069365, + "pass@4": 0.8872832369942196, + "pass@8": 0.930635838150289, + "pass@16": 0.9595375722543352, + "pass@32": 0.9710982658959537, + "pass@64": 0.976878612716763, + "pass@128": 0.976878612716763 + }, + "8": { + "pass@1": 0.6676056338028169, + "pass@2": 0.7690140845070422, + "pass@4": 0.8788732394366198, + "pass@8": 0.9352112676056338, + "pass@16": 0.9690140845070423, + "pass@32": 0.9774647887323944, + "pass@64": 0.9802816901408451, + "pass@128": 0.9859154929577465 + }, + "9": { + "pass@1": 0.562874251497006, + "pass@2": 0.6826347305389222, + "pass@4": 0.7904191616766467, + "pass@8": 0.8562874251497006, + "pass@16": 0.9011976047904192, + "pass@32": 0.9341317365269461, + "pass@64": 0.9461077844311377, + "pass@128": 0.9550898203592815 + }, + "10": { + "pass@1": 0.5153374233128835, + "pass@2": 0.6012269938650306, + "pass@4": 0.6871165644171779, + "pass@8": 0.745398773006135, + "pass@16": 0.8067484662576687, + "pass@32": 0.8466257668711656, + "pass@64": 0.8650306748466258, + "pass@128": 0.8865030674846626 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9270516717325228, + "3": 0.9666666666666667, + "4": 0.9053254437869822, + "5": 0.8840579710144928, + "6": 0.8595988538681948, + "7": 0.7630057803468208, + "8": 0.6676056338028169, + "9": 0.562874251497006, + "10": 0.5153374233128835 + }, + "pass@2": { + "2": 0.9422492401215805, + "3": 0.9787878787878788, + "4": 0.9230769230769231, + "5": 0.9188405797101449, + "6": 0.9111747851002865, + "7": 0.8352601156069365, + "8": 0.7690140845070422, + "9": 0.6826347305389222, + "10": 0.6012269938650306 + }, + "pass@4": { + "2": 0.9544072948328267, + "3": 0.9848484848484849, + "4": 0.9437869822485208, + "5": 0.9420289855072463, + "6": 0.9455587392550143, + "7": 0.8872832369942196, + "8": 0.8788732394366198, + "9": 0.7904191616766467, + "10": 0.6871165644171779 + }, + "pass@8": { + "2": 0.9604863221884499, + "3": 0.990909090909091, + "4": 0.9526627218934911, + "5": 0.9478260869565217, + "6": 0.9713467048710601, + "7": 0.930635838150289, + "8": 0.9352112676056338, + "9": 0.8562874251497006, + "10": 0.745398773006135 + }, + "pass@16": { + "2": 0.9696048632218845, + "3": 0.9939393939393939, + "4": 0.9585798816568047, + "5": 0.9507246376811594, + "6": 0.9828080229226361, + "7": 0.9595375722543352, + "8": 0.9690140845070423, + "9": 0.9011976047904192, + "10": 0.8067484662576687 + }, + "pass@32": { + "2": 0.9726443768996961, + "3": 0.9939393939393939, + "4": 0.9644970414201184, + "5": 0.9565217391304348, + "6": 0.9885386819484241, + "7": 0.9710982658959537, + "8": 0.9774647887323944, + "9": 0.9341317365269461, + "10": 0.8466257668711656 + }, + "pass@64": { + "2": 0.9756838905775076, + "3": 0.9939393939393939, + "4": 0.9644970414201184, + "5": 0.9594202898550724, + "6": 0.9914040114613181, + "7": 0.976878612716763, + "8": 0.9802816901408451, + "9": 0.9461077844311377, + "10": 0.8650306748466258 + }, + "pass@128": { + "2": 0.9787234042553191, + "3": 0.9939393939393939, + "4": 0.9644970414201184, + "5": 0.9594202898550724, + "6": 0.9914040114613181, + "7": 0.976878612716763, + "8": 0.9859154929577465, + "9": 0.9550898203592815, + "10": 0.8865030674846626 + } + }, + "total_pass_at_k": { + "pass@1": 0.783748361730013, + "pass@2": 0.8407601572739187, + "pass@4": 0.891218872870249, + "pass@8": 0.9220183486238532, + "pass@16": 0.9442988204456094, + "pass@32": 0.9567496723460026, + "pass@64": 0.9619921363040629, + "pass@128": 0.966251638269987 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.923, + "pass@2": 0.941, + "pass@4": 0.95, + "pass@8": 0.958, + "pass@16": 0.963, + "pass@32": 0.969, + "pass@64": 0.971, + "pass@128": 0.973 + }, + "3": { + "pass@1": 0.964, + "pass@2": 0.976, + "pass@4": 0.982, + "pass@8": 0.985, + "pass@16": 0.987, + "pass@32": 0.989, + "pass@64": 0.989, + "pass@128": 0.992 + }, + "4": { + "pass@1": 0.932, + "pass@2": 0.95, + "pass@4": 0.962, + "pass@8": 0.969, + "pass@16": 0.974, + "pass@32": 0.979, + "pass@64": 0.979, + "pass@128": 0.98 + }, + "5": { + "pass@1": 0.842, + "pass@2": 0.882, + "pass@4": 0.912, + "pass@8": 0.921, + "pass@16": 0.928, + "pass@32": 0.935, + "pass@64": 0.941, + "pass@128": 0.948 + }, + "6": { + "pass@1": 0.855, + "pass@2": 0.909, + "pass@4": 0.944, + "pass@8": 0.964, + "pass@16": 0.977, + "pass@32": 0.981, + "pass@64": 0.988, + "pass@128": 0.988 + }, + "7": { + "pass@1": 0.754, + "pass@2": 0.829, + "pass@4": 0.877, + "pass@8": 0.906, + "pass@16": 0.936, + "pass@32": 0.948, + "pass@64": 0.955, + "pass@128": 0.959 + }, + "8": { + "pass@1": 0.659, + "pass@2": 0.758, + "pass@4": 0.856, + "pass@8": 0.916, + "pass@16": 0.946, + "pass@32": 0.961, + "pass@64": 0.969, + "pass@128": 0.975 + }, + "9": { + "pass@1": 0.583, + "pass@2": 0.709, + "pass@4": 0.788, + "pass@8": 0.859, + "pass@16": 0.911, + "pass@32": 0.936, + "pass@64": 0.957, + "pass@128": 0.965 + }, + "10": { + "pass@1": 0.494, + "pass@2": 0.584, + "pass@4": 0.66, + "pass@8": 0.742, + "pass@16": 0.797, + "pass@32": 0.837, + "pass@64": 0.867, + "pass@128": 0.885 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.923, + "3": 0.964, + "4": 0.932, + "5": 0.842, + "6": 0.855, + "7": 0.754, + "8": 0.659, + "9": 0.583, + "10": 0.494 + }, + "pass@2": { + "2": 0.941, + "3": 0.976, + "4": 0.95, + "5": 0.882, + "6": 0.909, + "7": 0.829, + "8": 0.758, + "9": 0.709, + "10": 0.584 + }, + "pass@4": { + "2": 0.95, + "3": 0.982, + "4": 0.962, + "5": 0.912, + "6": 0.944, + "7": 0.877, + "8": 0.856, + "9": 0.788, + "10": 0.66 + }, + "pass@8": { + "2": 0.958, + "3": 0.985, + "4": 0.969, + "5": 0.921, + "6": 0.964, + "7": 0.906, + "8": 0.916, + "9": 0.859, + "10": 0.742 + }, + "pass@16": { + "2": 0.963, + "3": 0.987, + "4": 0.974, + "5": 0.928, + "6": 0.977, + "7": 0.936, + "8": 0.946, + "9": 0.911, + "10": 0.797 + }, + "pass@32": { + "2": 0.969, + "3": 0.989, + "4": 0.979, + "5": 0.935, + "6": 0.981, + "7": 0.948, + "8": 0.961, + "9": 0.936, + "10": 0.837 + }, + "pass@64": { + "2": 0.971, + "3": 0.989, + "4": 0.979, + "5": 0.941, + "6": 0.988, + "7": 0.955, + "8": 0.969, + "9": 0.957, + "10": 0.867 + }, + "pass@128": { + "2": 0.973, + "3": 0.992, + "4": 0.98, + "5": 0.948, + "6": 0.988, + "7": 0.959, + "8": 0.975, + "9": 0.965, + "10": 0.885 + } + }, + "total_pass_at_k": { + "pass@1": 0.7784444444444445, + "pass@2": 0.8375555555555556, + "pass@4": 0.8812222222222222, + "pass@8": 0.9133333333333333, + "pass@16": 0.9354444444444444, + "pass@32": 0.9483333333333334, + "pass@64": 0.9573333333333334, + "pass@128": 0.9627777777777777 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21808_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.34277620396600567, + "pass@2": 0.42492917847025496, + "pass@4": 0.5354107648725213, + "pass@8": 0.6147308781869688, + "pass@16": 0.7053824362606232, + "pass@32": 0.7592067988668555, + "pass@64": 0.8101983002832861, + "pass@128": 0.8470254957507082 + }, + "12": { + "pass@1": 0.2344632768361582, + "pass@2": 0.3220338983050847, + "pass@4": 0.4435028248587571, + "pass@8": 0.5254237288135594, + "pass@16": 0.5847457627118644, + "pass@32": 0.6299435028248588, + "pass@64": 0.6666666666666666, + "pass@128": 0.7062146892655368 + }, + "13": { + "pass@1": 0.06583072100313479, + "pass@2": 0.10031347962382445, + "pass@4": 0.15047021943573669, + "pass@8": 0.21630094043887146, + "pass@16": 0.25391849529780564, + "pass@32": 0.32601880877742945, + "pass@64": 0.4012539184952978, + "pass@128": 0.4231974921630094 + }, + "14": { + "pass@1": 0.037383177570093455, + "pass@2": 0.04984423676012461, + "pass@4": 0.08411214953271028, + "pass@8": 0.102803738317757, + "pass@16": 0.1308411214953271, + "pass@32": 0.14953271028037382, + "pass@64": 0.19003115264797507, + "pass@128": 0.21495327102803738 + }, + "15": { + "pass@1": 0.005780346820809248, + "pass@2": 0.008670520231213872, + "pass@4": 0.02023121387283237, + "pass@8": 0.031791907514450865, + "pass@16": 0.04335260115606936, + "pass@32": 0.049132947976878616, + "pass@64": 0.057803468208092484, + "pass@128": 0.0838150289017341 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0032258064516129032, + "pass@8": 0.0032258064516129032, + "pass@16": 0.0064516129032258064, + "pass@32": 0.00967741935483871, + "pass@64": 0.012903225806451613, + "pass@128": 0.016129032258064516 + }, + "17": { + "pass@1": 0.002890173410404624, + "pass@2": 0.002890173410404624, + "pass@4": 0.005780346820809248, + "pass@8": 0.008670520231213872, + "pass@16": 0.011560693641618497, + "pass@32": 0.014450867052023121, + "pass@64": 0.017341040462427744, + "pass@128": 0.017341040462427744 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.002857142857142857, + "pass@128": 0.002857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.34277620396600567, + "12": 0.2344632768361582, + "13": 0.06583072100313479, + "14": 0.037383177570093455, + "15": 0.005780346820809248, + "16": 0.0, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.42492917847025496, + "12": 0.3220338983050847, + "13": 0.10031347962382445, + "14": 0.04984423676012461, + "15": 0.008670520231213872, + "16": 0.0, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.5354107648725213, + "12": 0.4435028248587571, + "13": 0.15047021943573669, + "14": 0.08411214953271028, + "15": 0.02023121387283237, + "16": 0.0032258064516129032, + "17": 0.005780346820809248, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.6147308781869688, + "12": 0.5254237288135594, + "13": 0.21630094043887146, + "14": 0.102803738317757, + "15": 0.031791907514450865, + "16": 0.0032258064516129032, + "17": 0.008670520231213872, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.7053824362606232, + "12": 0.5847457627118644, + "13": 0.25391849529780564, + "14": 0.1308411214953271, + "15": 0.04335260115606936, + "16": 0.0064516129032258064, + "17": 0.011560693641618497, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.7592067988668555, + "12": 0.6299435028248588, + "13": 0.32601880877742945, + "14": 0.14953271028037382, + "15": 0.049132947976878616, + "16": 0.00967741935483871, + "17": 0.014450867052023121, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8101983002832861, + "12": 0.6666666666666666, + "13": 0.4012539184952978, + "14": 0.19003115264797507, + "15": 0.057803468208092484, + "16": 0.012903225806451613, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8470254957507082, + "12": 0.7062146892655368, + "13": 0.4231974921630094, + "14": 0.21495327102803738, + "15": 0.0838150289017341, + "16": 0.016129032258064516, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0713224368499257, + "pass@2": 0.09390787518573551, + "pass@4": 0.12808320950965824, + "pass@8": 0.15453194650817237, + "pass@16": 0.17830609212481427, + "pass@32": 0.19851411589895987, + "pass@64": 0.22050520059435363, + "pass@128": 0.23595839524517087 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.35873015873015873, + "pass@2": 0.4603174603174603, + "pass@4": 0.5619047619047619, + "pass@8": 0.6603174603174603, + "pass@16": 0.7301587301587301, + "pass@32": 0.7873015873015873, + "pass@64": 0.8285714285714286, + "pass@128": 0.8444444444444444 + }, + "12": { + "pass@1": 0.25382262996941896, + "pass@2": 0.3363914373088685, + "pass@4": 0.45565749235474007, + "pass@8": 0.5443425076452599, + "pass@16": 0.6055045871559633, + "pass@32": 0.6605504587155964, + "pass@64": 0.709480122324159, + "pass@128": 0.746177370030581 + }, + "13": { + "pass@1": 0.1111111111111111, + "pass@2": 0.14814814814814814, + "pass@4": 0.19135802469135801, + "pass@8": 0.23765432098765432, + "pass@16": 0.28703703703703703, + "pass@32": 0.33024691358024694, + "pass@64": 0.3765432098765432, + "pass@128": 0.4228395061728395 + }, + "14": { + "pass@1": 0.05934718100890208, + "pass@2": 0.0830860534124629, + "pass@4": 0.11869436201780416, + "pass@8": 0.1572700296735905, + "pass@16": 0.17804154302670624, + "pass@32": 0.2166172106824926, + "pass@64": 0.2433234421364985, + "pass@128": 0.29080118694362017 + }, + "15": { + "pass@1": 0.030864197530864196, + "pass@2": 0.046296296296296294, + "pass@4": 0.05246913580246913, + "pass@8": 0.06790123456790123, + "pass@16": 0.08950617283950617, + "pass@32": 0.12037037037037036, + "pass@64": 0.14506172839506173, + "pass@128": 0.1697530864197531 + }, + "16": { + "pass@1": 0.008849557522123894, + "pass@2": 0.014749262536873156, + "pass@4": 0.014749262536873156, + "pass@8": 0.014749262536873156, + "pass@16": 0.02359882005899705, + "pass@32": 0.032448377581120944, + "pass@64": 0.035398230088495575, + "pass@128": 0.04424778761061947 + }, + "17": { + "pass@1": 0.0030120481927710845, + "pass@2": 0.0030120481927710845, + "pass@4": 0.006024096385542169, + "pass@8": 0.012048192771084338, + "pass@16": 0.015060240963855422, + "pass@32": 0.018072289156626505, + "pass@64": 0.02710843373493976, + "pass@128": 0.02710843373493976 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0032258064516129032, + "pass@32": 0.0032258064516129032, + "pass@64": 0.0064516129032258064, + "pass@128": 0.012903225806451613 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.35873015873015873, + "12": 0.25382262996941896, + "13": 0.1111111111111111, + "14": 0.05934718100890208, + "15": 0.030864197530864196, + "16": 0.008849557522123894, + "17": 0.0030120481927710845, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.4603174603174603, + "12": 0.3363914373088685, + "13": 0.14814814814814814, + "14": 0.0830860534124629, + "15": 0.046296296296296294, + "16": 0.014749262536873156, + "17": 0.0030120481927710845, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.5619047619047619, + "12": 0.45565749235474007, + "13": 0.19135802469135801, + "14": 0.11869436201780416, + "15": 0.05246913580246913, + "16": 0.014749262536873156, + "17": 0.006024096385542169, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.6603174603174603, + "12": 0.5443425076452599, + "13": 0.23765432098765432, + "14": 0.1572700296735905, + "15": 0.06790123456790123, + "16": 0.014749262536873156, + "17": 0.012048192771084338, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.7301587301587301, + "12": 0.6055045871559633, + "13": 0.28703703703703703, + "14": 0.17804154302670624, + "15": 0.08950617283950617, + "16": 0.02359882005899705, + "17": 0.015060240963855422, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.7873015873015873, + "12": 0.6605504587155964, + "13": 0.33024691358024694, + "14": 0.2166172106824926, + "15": 0.12037037037037036, + "16": 0.032448377581120944, + "17": 0.018072289156626505, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8285714285714286, + "12": 0.709480122324159, + "13": 0.3765432098765432, + "14": 0.2433234421364985, + "15": 0.14506172839506173, + "16": 0.035398230088495575, + "17": 0.02710843373493976, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8444444444444444, + "12": 0.746177370030581, + "13": 0.4228395061728395, + "14": 0.29080118694362017, + "15": 0.1697530864197531, + "16": 0.04424778761061947, + "17": 0.02710843373493976, + "18": 0.012903225806451613, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0812958435207824, + "pass@2": 0.10757946210268948, + "pass@4": 0.13814180929095354, + "pass@8": 0.1671760391198044, + "pass@16": 0.19070904645476772, + "pass@32": 0.21424205378973105, + "pass@64": 0.23441320293398532, + "pass@128": 0.2530562347188264 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.41566265060240964, + "pass@2": 0.48493975903614456, + "pass@4": 0.5933734939759037, + "pass@8": 0.677710843373494, + "pass@16": 0.7530120481927711, + "pass@32": 0.8132530120481928, + "pass@64": 0.8433734939759037, + "pass@128": 0.8554216867469879 + }, + "12": { + "pass@1": 0.23510971786833856, + "pass@2": 0.3072100313479624, + "pass@4": 0.3981191222570533, + "pass@8": 0.48589341692789967, + "pass@16": 0.54858934169279, + "pass@32": 0.6112852664576802, + "pass@64": 0.6802507836990596, + "pass@128": 0.7021943573667712 + }, + "13": { + "pass@1": 0.08123249299719888, + "pass@2": 0.11484593837535013, + "pass@4": 0.17086834733893558, + "pass@8": 0.2184873949579832, + "pass@16": 0.3165266106442577, + "pass@32": 0.3585434173669468, + "pass@64": 0.43137254901960786, + "pass@128": 0.4957983193277311 + }, + "14": { + "pass@1": 0.04678362573099415, + "pass@2": 0.07017543859649122, + "pass@4": 0.0847953216374269, + "pass@8": 0.11988304093567251, + "pass@16": 0.15789473684210525, + "pass@32": 0.18128654970760233, + "pass@64": 0.21637426900584794, + "pass@128": 0.2543859649122807 + }, + "15": { + "pass@1": 0.00909090909090909, + "pass@2": 0.012121212121212121, + "pass@4": 0.01818181818181818, + "pass@8": 0.03636363636363636, + "pass@16": 0.048484848484848485, + "pass@32": 0.07272727272727272, + "pass@64": 0.08181818181818182, + "pass@128": 0.09090909090909091 + }, + "16": { + "pass@1": 0.008547008547008548, + "pass@2": 0.011396011396011397, + "pass@4": 0.019943019943019943, + "pass@8": 0.019943019943019943, + "pass@16": 0.022792022792022793, + "pass@32": 0.02849002849002849, + "pass@64": 0.03133903133903134, + "pass@128": 0.03418803418803419 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.003105590062111801, + "pass@16": 0.003105590062111801, + "pass@32": 0.006211180124223602, + "pass@64": 0.006211180124223602, + "pass@128": 0.012422360248447204 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0058823529411764705, + "pass@128": 0.0058823529411764705 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.41566265060240964, + "12": 0.23510971786833856, + "13": 0.08123249299719888, + "14": 0.04678362573099415, + "15": 0.00909090909090909, + "16": 0.008547008547008548, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.48493975903614456, + "12": 0.3072100313479624, + "13": 0.11484593837535013, + "14": 0.07017543859649122, + "15": 0.012121212121212121, + "16": 0.011396011396011397, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.5933734939759037, + "12": 0.3981191222570533, + "13": 0.17086834733893558, + "14": 0.0847953216374269, + "15": 0.01818181818181818, + "16": 0.019943019943019943, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.677710843373494, + "12": 0.48589341692789967, + "13": 0.2184873949579832, + "14": 0.11988304093567251, + "15": 0.03636363636363636, + "16": 0.019943019943019943, + "17": 0.003105590062111801, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.7530120481927711, + "12": 0.54858934169279, + "13": 0.3165266106442577, + "14": 0.15789473684210525, + "15": 0.048484848484848485, + "16": 0.022792022792022793, + "17": 0.003105590062111801, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8132530120481928, + "12": 0.6112852664576802, + "13": 0.3585434173669468, + "14": 0.18128654970760233, + "15": 0.07272727272727272, + "16": 0.02849002849002849, + "17": 0.006211180124223602, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8433734939759037, + "12": 0.6802507836990596, + "13": 0.43137254901960786, + "14": 0.21637426900584794, + "15": 0.08181818181818182, + "16": 0.03133903133903134, + "17": 0.006211180124223602, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8554216867469879, + "12": 0.7021943573667712, + "13": 0.4957983193277311, + "14": 0.2543859649122807, + "15": 0.09090909090909091, + "16": 0.03418803418803419, + "17": 0.012422360248447204, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.07850133809099019, + "pass@2": 0.09872137972048765, + "pass@4": 0.12696996729110913, + "pass@8": 0.1543264942016057, + "pass@16": 0.18346714243235207, + "pass@32": 0.20547130538209932, + "pass@64": 0.22807017543859648, + "pass@128": 0.2438299137674695 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.372, + "pass@2": 0.456, + "pass@4": 0.563, + "pass@8": 0.65, + "pass@16": 0.729, + "pass@32": 0.786, + "pass@64": 0.827, + "pass@128": 0.849 + }, + "12": { + "pass@1": 0.241, + "pass@2": 0.322, + "pass@4": 0.433, + "pass@8": 0.519, + "pass@16": 0.58, + "pass@32": 0.634, + "pass@64": 0.685, + "pass@128": 0.718 + }, + "13": { + "pass@1": 0.086, + "pass@2": 0.121, + "pass@4": 0.171, + "pass@8": 0.224, + "pass@16": 0.287, + "pass@32": 0.339, + "pass@64": 0.404, + "pass@128": 0.449 + }, + "14": { + "pass@1": 0.048, + "pass@2": 0.068, + "pass@4": 0.096, + "pass@8": 0.127, + "pass@16": 0.156, + "pass@32": 0.183, + "pass@64": 0.217, + "pass@128": 0.254 + }, + "15": { + "pass@1": 0.015, + "pass@2": 0.022, + "pass@4": 0.03, + "pass@8": 0.045, + "pass@16": 0.06, + "pass@32": 0.08, + "pass@64": 0.094, + "pass@128": 0.114 + }, + "16": { + "pass@1": 0.006, + "pass@2": 0.009, + "pass@4": 0.013, + "pass@8": 0.013, + "pass@16": 0.018, + "pass@32": 0.024, + "pass@64": 0.027, + "pass@128": 0.032 + }, + "17": { + "pass@1": 0.002, + "pass@2": 0.002, + "pass@4": 0.004, + "pass@8": 0.008, + "pass@16": 0.01, + "pass@32": 0.013, + "pass@64": 0.017, + "pass@128": 0.019 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.001, + "pass@32": 0.001, + "pass@64": 0.005, + "pass@128": 0.007 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.372, + "12": 0.241, + "13": 0.086, + "14": 0.048, + "15": 0.015, + "16": 0.006, + "17": 0.002, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.456, + "12": 0.322, + "13": 0.121, + "14": 0.068, + "15": 0.022, + "16": 0.009, + "17": 0.002, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.563, + "12": 0.433, + "13": 0.171, + "14": 0.096, + "15": 0.03, + "16": 0.013, + "17": 0.004, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.65, + "12": 0.519, + "13": 0.224, + "14": 0.127, + "15": 0.045, + "16": 0.013, + "17": 0.008, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.729, + "12": 0.58, + "13": 0.287, + "14": 0.156, + "15": 0.06, + "16": 0.018, + "17": 0.01, + "18": 0.001, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.786, + "12": 0.634, + "13": 0.339, + "14": 0.183, + "15": 0.08, + "16": 0.024, + "17": 0.013, + "18": 0.001, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.827, + "12": 0.685, + "13": 0.404, + "14": 0.217, + "15": 0.094, + "16": 0.027, + "17": 0.017, + "18": 0.005, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.849, + "12": 0.718, + "13": 0.449, + "14": 0.254, + "15": 0.114, + "16": 0.032, + "17": 0.019, + "18": 0.007, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.077, + "pass@2": 0.1, + "pass@4": 0.131, + "pass@8": 0.1586, + "pass@16": 0.1841, + "pass@32": 0.206, + "pass@64": 0.2276, + "pass@128": 0.2442 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block16/composition/autoregressive/summary.csv b/SDAR-100M-ep1-block16/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..785517eb7ffee29a4f84d5db882238c3634e0cf1 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21808,0.9135225694444444,0.2918375,0.5863199013157895,0.0,0.0,0.0,199.60449913194444,312.06185859375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block16/composition/autoregressive/summary.json b/SDAR-100M-ep1-block16/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..20ae842d9febfa761538479e35768830752d7d8d --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21808", + "id_acc": 0.9135225694444444, + "ood_acc": 0.2918375, + "total_acc": 0.5863199013157895, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.60449913194444, + "ood_avg_resp_len": 312.06185859375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 467579, + "answer_accuracy": 0.5774519344767626, + "avg_response_len": 245.32765200165983, + "resp_tokens_sum": 198648669, + "pass_at_k": { + "pass@1": 0.5774519344767626, + "pass@2": 0.6452416567297637, + "pass@4": 0.6996034111666334, + "pass@8": 0.7413455058301636, + "pass@16": 0.7729299247279426, + "pass@32": 0.7971920686194446, + "pass@64": 0.8170532304403487, + "pass@128": 0.8352829592159342 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.811992994100295, + "pass@2": 0.8843032134810588, + "pass@4": 0.923425883711439, + "pass@8": 0.9457277067863578, + "pass@16": 0.9606133820258961, + "pass@32": 0.971941456242833, + "pass@64": 0.979374562311218, + "pass@128": 0.9823008849557522 + }, + "2": { + "pass@1": 0.9364047897196262, + "pass@2": 0.9491822429906543, + "pass@4": 0.9604580686175911, + "pass@8": 0.9694622504718368, + "pass@16": 0.9762818313978732, + "pass@32": 0.9814895602246809, + "pass@64": 0.985727686903887, + "pass@128": 0.9906542056074766 + }, + "3": { + "pass@1": 0.9801462155963303, + "pass@2": 0.9896420272580605, + "pass@4": 0.9937778259368953, + "pass@8": 0.9962077916624781, + "pass@16": 0.9981311364256852, + "pass@32": 0.9995033618733702, + "pass@64": 0.999978967901273, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.969677734375, + "pass@2": 0.9814676119586613, + "pass@4": 0.986887528707349, + "pass@8": 0.9899307200629559, + "pass@16": 0.9917621691650451, + "pass@32": 0.993528368509755, + "pass@64": 0.9960998638576377, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8992063492063492, + "pass@2": 0.9267841519810028, + "pass@4": 0.9439647365507879, + "pass@8": 0.9548172712010686, + "pass@16": 0.9623409287636865, + "pass@32": 0.9688380283149466, + "pass@64": 0.9755950258820365, + "pass@128": 0.9809523809523809 + }, + "6": { + "pass@1": 0.9132204341317365, + "pass@2": 0.9474187697439762, + "pass@4": 0.965994401803816, + "pass@8": 0.9784493955374186, + "pass@16": 0.9872882423827155, + "pass@32": 0.9922681964206024, + "pass@64": 0.9939103062084376, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.8838063979289941, + "pass@2": 0.9277586719004803, + "pass@4": 0.9530765176172504, + "pass@8": 0.9664595878668178, + "pass@16": 0.9740977457411283, + "pass@32": 0.9795563479381585, + "pass@64": 0.9843567124179412, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8821864983974359, + "pass@2": 0.9410403101655567, + "pass@4": 0.9686909091291468, + "pass@8": 0.9800378182202505, + "pass@16": 0.9851975708878595, + "pass@32": 0.988658383641544, + "pass@64": 0.9911660518873375, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.8510783450704226, + "pass@2": 0.924036195519574, + "pass@4": 0.9606805254096751, + "pass@8": 0.9797217797223858, + "pass@16": 0.9887705852950439, + "pass@32": 0.9929424597619723, + "pass@64": 0.9953811357885473, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.7112694759206799, + "pass@2": 0.8131120206999624, + "pass@4": 0.8741029357874173, + "pass@8": 0.9131565853749971, + "pass@16": 0.9409259508048508, + "pass@32": 0.9607761804421557, + "pass@64": 0.9752001653135165, + "pass@128": 0.9858356940509915 + }, + "12": { + "pass@1": 0.5198181497175142, + "pass@2": 0.652291444236843, + "pass@4": 0.7531961534469203, + "pass@8": 0.8229850744031951, + "pass@16": 0.8703747593722103, + "pass@32": 0.9035354750046218, + "pass@64": 0.92679855938621, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.34842770376175547, + "pass@2": 0.4644909504356623, + "pass@4": 0.5666685317352571, + "pass@8": 0.6515867742597433, + "pass@16": 0.7209558398105108, + "pass@32": 0.774390651037868, + "pass@64": 0.8159555350904601, + "pass@128": 0.8495297805642633 + }, + "14": { + "pass@1": 0.2620959890965732, + "pass@2": 0.36483859494198756, + "pass@4": 0.4648828545964464, + "pass@8": 0.5501300452769146, + "pass@16": 0.6169298321030059, + "pass@32": 0.6706355826772032, + "pass@64": 0.7209786790660362, + "pass@128": 0.7757009345794392 + }, + "15": { + "pass@1": 0.2202176661849711, + "pass@2": 0.3138636583450913, + "pass@4": 0.4027050682067921, + "pass@8": 0.4772871808592236, + "pass@16": 0.5401813834656242, + "pass@32": 0.5942484724698351, + "pass@64": 0.6404521097017122, + "pass@128": 0.684971098265896 + }, + "16": { + "pass@1": 0.1792086693548387, + "pass@2": 0.25795577216154436, + "pass@4": 0.33871481397486597, + "pass@8": 0.4115471071605108, + "pass@16": 0.47119022128456445, + "pass@32": 0.5159831622872939, + "pass@64": 0.551697044512079, + "pass@128": 0.5870967741935483 + }, + "17": { + "pass@1": 0.17575867052023122, + "pass@2": 0.2527504295434891, + "pass@4": 0.3287763653820727, + "pass@8": 0.3946413381092982, + "pass@16": 0.4472001973243355, + "pass@32": 0.48985344057501407, + "pass@64": 0.5292538770652234, + "pass@128": 0.5664739884393064 + }, + "18": { + "pass@1": 0.16078125, + "pass@2": 0.2357448678290212, + "pass@4": 0.31116799641116283, + "pass@8": 0.3780366912754487, + "pass@16": 0.4329714410571362, + "pass@32": 0.47825937552060704, + "pass@64": 0.5182588254375425, + "pass@128": 0.5571428571428572 + }, + "19": { + "pass@1": 0.15373967717717718, + "pass@2": 0.23123566479865715, + "pass@4": 0.30846989959588395, + "pass@8": 0.37090323389681995, + "pass@16": 0.4176420067808336, + "pass@32": 0.4519367816266829, + "pass@64": 0.4781589939232686, + "pass@128": 0.5045045045045045 + }, + "20": { + "pass@1": 0.1547484984984985, + "pass@2": 0.23214011255349842, + "pass@4": 0.3100945782678066, + "pass@8": 0.3735420424835841, + "pass@16": 0.4193360888579724, + "pass@32": 0.45293241378176113, + "pass@64": 0.47896014395846753, + "pass@128": 0.5015015015015015 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 482531, + "answer_accuracy": 0.5876497954014029, + "avg_response_len": 255.93222793257988, + "resp_tokens_sum": 210151071, + "pass_at_k": { + "pass@1": 0.5876497954014029, + "pass@2": 0.6525561207737851, + "pass@4": 0.7061835820210686, + "pass@8": 0.7490327747499966, + "pass@16": 0.7829268531033039, + "pass@32": 0.8100206823311829, + "pass@64": 0.8319087994567788, + "pass@128": 0.8497272018706158 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8102233512269938, + "pass@2": 0.8818788192599386, + "pass@4": 0.9230943886998785, + "pass@8": 0.9486758986818827, + "pass@16": 0.9665855700413544, + "pass@32": 0.9798571294366684, + "pass@64": 0.9889181243773738, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 0.9407057370820668, + "pass@2": 0.9543108142067347, + "pass@4": 0.9648635555244045, + "pass@8": 0.9735475712944428, + "pass@16": 0.980713903139753, + "pass@32": 0.9855875985518405, + "pass@64": 0.9885083794652192, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.9831912878787878, + "pass@2": 0.9890449773323791, + "pass@4": 0.9925742770790013, + "pass@8": 0.9949117732864091, + "pass@16": 0.9968032945575152, + "pass@32": 0.998445165474543, + "pass@64": 0.999620777962612, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9603365384615384, + "pass@2": 0.9773203448958673, + "pass@4": 0.9862346477215493, + "pass@8": 0.9916871794012287, + "pass@16": 0.995383380617976, + "pass@32": 0.9977698233271339, + "pass@64": 0.9994625180003253, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9483016304347827, + "pass@2": 0.9635705523222642, + "pass@4": 0.9734722803671282, + "pass@8": 0.9809383138264673, + "pass@16": 0.9851226251370658, + "pass@32": 0.9872274839565104, + "pass@64": 0.9891271668301083, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9358882521489972, + "pass@2": 0.9721159161157868, + "pass@4": 0.9889131951271127, + "pass@8": 0.9951489089035775, + "pass@16": 0.9979499235991857, + "pass@32": 0.9995757636157672, + "pass@64": 0.999990075961284, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9294391257225434, + "pass@2": 0.9595930431022713, + "pass@4": 0.9755940045969683, + "pass@8": 0.9847787879592633, + "pass@16": 0.9892688554315738, + "pass@32": 0.9908053653135379, + "pass@64": 0.9912895111919918, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.8965448943661972, + "pass@2": 0.9502491821004772, + "pass@4": 0.977070321931589, + "pass@8": 0.9902363110097167, + "pass@16": 0.9961681334129603, + "pass@32": 0.9986011198470229, + "pass@64": 0.999793253291699, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8508373877245509, + "pass@2": 0.9181932640624261, + "pass@4": 0.9504120931365609, + "pass@8": 0.9681510539987431, + "pass@16": 0.9796319664627892, + "pass@32": 0.9865058729633788, + "pass@64": 0.9904633864127251, + "pass@128": 0.9940119760479041 + }, + "11": { + "pass@1": 0.719714796686747, + "pass@2": 0.8202420904088796, + "pass@4": 0.8819906120545167, + "pass@8": 0.9167660961161666, + "pass@16": 0.9370201729227511, + "pass@32": 0.9507746557206528, + "pass@64": 0.9612102815531016, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.5531690830721003, + "pass@2": 0.691132707402562, + "pass@4": 0.7918788569336356, + "pass@8": 0.8558394828962391, + "pass@16": 0.8961570326737536, + "pass@32": 0.9242513072207585, + "pass@64": 0.9447949934870175, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.31256565126050423, + "pass@2": 0.4224401867046028, + "pass@4": 0.5283050091427646, + "pass@8": 0.6220202116228649, + "pass@16": 0.6990070859405011, + "pass@32": 0.7615370265201747, + "pass@64": 0.8117002962134, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.28723501461988304, + "pass@2": 0.3853191767969794, + "pass@4": 0.4756250548177094, + "pass@8": 0.5538791634637721, + "pass@16": 0.6247279946652008, + "pass@32": 0.6916342968490513, + "pass@64": 0.7535727398766265, + "pass@128": 0.8070175438596491 + }, + "15": { + "pass@1": 0.19509943181818182, + "pass@2": 0.2874798675733715, + "pass@4": 0.38628292741816334, + "pass@8": 0.47756921040509814, + "pass@16": 0.5566065517223067, + "pass@32": 0.6230961707223709, + "pass@64": 0.6746829994145453, + "pass@128": 0.7151515151515152 + }, + "16": { + "pass@1": 0.19831730769230768, + "pass@2": 0.27961203367207305, + "pass@4": 0.35857375119776697, + "pass@8": 0.4255978477392451, + "pass@16": 0.47931263574118715, + "pass@32": 0.5250995747508337, + "pass@64": 0.5695799608843882, + "pass@128": 0.6125356125356125 + }, + "17": { + "pass@1": 0.19613742236024845, + "pass@2": 0.2840029252457574, + "pass@4": 0.3722219300421298, + "pass@8": 0.45339473578248296, + "pass@16": 0.5245596190614837, + "pass@32": 0.5828725152246444, + "pass@64": 0.6277436674236548, + "pass@128": 0.6614906832298136 + }, + "18": { + "pass@1": 0.16300551470588234, + "pass@2": 0.24151480720240845, + "pass@4": 0.32298752040186174, + "pass@8": 0.39488191333318684, + "pass@16": 0.4550903465829013, + "pass@32": 0.5079746597085013, + "pass@64": 0.5556431661505973, + "pass@128": 0.5970588235294118 + }, + "19": { + "pass@1": 0.13816034226190477, + "pass@2": 0.20664604424446936, + "pass@4": 0.27967271836556157, + "pass@8": 0.34818998174106214, + "pass@16": 0.4053455576540456, + "pass@32": 0.4497394448722171, + "pass@64": 0.4830647705525789, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.1348942739520958, + "pass@2": 0.20611348660945836, + "pass@4": 0.2826266614689631, + "pass@8": 0.35183716167265444, + "pass@16": 0.4075823091006288, + "pass@32": 0.44687399315461174, + "pass@64": 0.474497256958554, + "pass@128": 0.5029940119760479 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 475820, + "answer_accuracy": 0.5939197555520052, + "avg_response_len": 275.3332975515258, + "resp_tokens_sum": 220583822, + "pass_at_k": { + "pass@1": 0.5939197555520052, + "pass@2": 0.6590625247674902, + "pass@4": 0.7127488353628675, + "pass@8": 0.7552895351531898, + "pass@16": 0.7882845933606959, + "pass@32": 0.8140099630225133, + "pass@64": 0.8343875634908813, + "pass@128": 0.8515737338232945 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8168376865671642, + "pass@2": 0.8903752644259023, + "pass@4": 0.9318934934439158, + "pass@8": 0.9560157584451383, + "pass@16": 0.9699892576230075, + "pass@32": 0.9795367066929017, + "pass@64": 0.9873081396584162, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 0.9385491071428571, + "pass@2": 0.9541169853768277, + "pass@4": 0.9660614711553913, + "pass@8": 0.9742850734130257, + "pass@16": 0.9801474858828125, + "pass@32": 0.9856558188638964, + "pass@64": 0.990288117840151, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9768358236151603, + "pass@2": 0.9841314478547326, + "pass@4": 0.9887277911325226, + "pass@8": 0.9920268596185634, + "pass@16": 0.9936995287368084, + "pass@32": 0.9941390350274529, + "pass@64": 0.9941690610183596, + "pass@128": 0.9941690962099126 + }, + "4": { + "pass@1": 0.9728846856725146, + "pass@2": 0.9822816658608464, + "pass@4": 0.9869530976939285, + "pass@8": 0.9907192584205899, + "pass@16": 0.9939338085875972, + "pass@32": 0.9963771697187885, + "pass@64": 0.9983234387880316, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9124080882352941, + "pass@2": 0.9346861973135713, + "pass@4": 0.9515868884955558, + "pass@8": 0.9647522806518978, + "pass@16": 0.9746544901240447, + "pass@32": 0.9818895051410447, + "pass@64": 0.987542165474913, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9517202287066246, + "pass@2": 0.9792325163317526, + "pass@4": 0.9896437670251783, + "pass@8": 0.992935360514013, + "pass@16": 0.9936369730893512, + "pass@32": 0.9936905429777773, + "pass@64": 0.9936908517347554, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9434582674050633, + "pass@2": 0.9697599484202134, + "pass@4": 0.9808773431604276, + "pass@8": 0.9866220229775958, + "pass@16": 0.990433865453002, + "pass@32": 0.9935113432880496, + "pass@64": 0.9968041500282304, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.894237987987988, + "pass@2": 0.9414288796670683, + "pass@4": 0.9675921847043887, + "pass@8": 0.9825535104384212, + "pass@16": 0.9912589117683784, + "pass@32": 0.9960952463036303, + "pass@64": 0.9983943194216062, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.8760048231511254, + "pass@2": 0.940809191204396, + "pass@4": 0.967987871291007, + "pass@8": 0.9795005195077631, + "pass@16": 0.9861004723470128, + "pass@32": 0.9907761116277802, + "pass@64": 0.9943375327526845, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.7388888888888889, + "pass@2": 0.8396380139982507, + "pass@4": 0.8983732589973874, + "pass@8": 0.9293637318750155, + "pass@16": 0.9469655881937245, + "pass@32": 0.958052279914744, + "pass@64": 0.9645245347844873, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5382979740061162, + "pass@2": 0.6815369229935708, + "pass@4": 0.7901882852372809, + "pass@8": 0.8623947124188998, + "pass@16": 0.9080716635283972, + "pass@32": 0.9356809750706235, + "pass@64": 0.9492487788739141, + "pass@128": 0.9571865443425076 + }, + "13": { + "pass@1": 0.3448350694444444, + "pass@2": 0.4652192998444639, + "pass@4": 0.571143411182398, + "pass@8": 0.6531886175627993, + "pass@16": 0.7148291106889169, + "pass@32": 0.7647239391313766, + "pass@64": 0.8073138228282857, + "pass@128": 0.8395061728395061 + }, + "14": { + "pass@1": 0.30837351632047477, + "pass@2": 0.41777261150961487, + "pass@4": 0.5206757623397962, + "pass@8": 0.6089125239005821, + "pass@16": 0.6839245743828597, + "pass@32": 0.7492554568430402, + "pass@64": 0.8034429167524338, + "pass@128": 0.8486646884272997 + }, + "15": { + "pass@1": 0.22497106481481483, + "pass@2": 0.32391079153300256, + "pass@4": 0.4264279091849627, + "pass@8": 0.5198736275821021, + "pass@16": 0.5952772518420743, + "pass@32": 0.6529361327041301, + "pass@64": 0.6997906537176186, + "pass@128": 0.7407407407407407 + }, + "16": { + "pass@1": 0.19065726401179942, + "pass@2": 0.27215329071144856, + "pass@4": 0.35222581856471474, + "pass@8": 0.4258438236788404, + "pass@16": 0.49209327147276405, + "pass@32": 0.5494043824143943, + "pass@64": 0.6004721355972974, + "pass@128": 0.6489675516224189 + }, + "17": { + "pass@1": 0.16488610692771086, + "pass@2": 0.24255214756664453, + "pass@4": 0.3247423436716496, + "pass@8": 0.40012973755769266, + "pass@16": 0.4646677130767527, + "pass@32": 0.5190198430686861, + "pass@64": 0.5650992824649722, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.1730342741935484, + "pass@2": 0.24966979933959876, + "pass@4": 0.32699005588011176, + "pass@8": 0.39331440500639975, + "pass@16": 0.4431195946101498, + "pass@32": 0.48034270092756165, + "pass@64": 0.5123963377060167, + "pass@128": 0.5483870967741935 + }, + "19": { + "pass@1": 0.14971204682779457, + "pass@2": 0.2218807984632585, + "pass@4": 0.2958488075131092, + "pass@8": 0.36166937316077835, + "pass@16": 0.41616715536535226, + "pass@32": 0.4580014870377535, + "pass@64": 0.4888460828597657, + "pass@128": 0.5166163141993958 + }, + "20": { + "pass@1": 0.14198573573573572, + "pass@2": 0.2137330489938758, + "pass@4": 0.2915296775966066, + "pass@8": 0.36565955064618744, + "pass@16": 0.4282390230240574, + "pass@32": 0.4767202275751524, + "pass@64": 0.5107559543079784, + "pass@128": 0.5315315315315315 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/checkpoint-21808_metrics.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/checkpoint-21808_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..87e55ca88fddfb1625c77e60a9fba8ad5bb03b7b --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/checkpoint-21808_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21808", + "id": { + "answer_accuracy": 0.9772300347222223, + "avg_response_len": 196.0645373263889, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9772300347222223, + "pass@2": 0.9823446932414661, + "pass@4": 0.9853120525038529, + "pass@8": 0.9874025448227628, + "pass@16": 0.9890813494453924, + "pass@32": 0.9905469111678545, + "pass@64": 0.9918145325835068, + "pass@128": 0.9927777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.90171875, + "pass@2": 0.9185543799212602, + "pass@4": 0.9303104290401201, + "pass@8": 0.93989990916447, + "pass@16": 0.9483905366285567, + "pass@32": 0.9562143070532549, + "pass@64": 0.9627343606118026, + "pass@128": 0.967 + }, + "2": { + "pass@1": 0.9990546875, + "pass@2": 0.9991067913385827, + "pass@4": 0.999203654855643, + "pass@8": 0.9993706737002938, + "pass@16": 0.9996167406956444, + "pass@32": 0.9998739053984704, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998828125, + "pass@2": 0.9990104576771653, + "pass@4": 0.9990614735658042, + "pass@8": 0.9991215550875039, + "pass@16": 0.9992352362204724, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992421875, + "pass@2": 0.9998128690944881, + "pass@4": 0.9999874877202849, + "pass@8": 0.9999999401771543, + "pass@16": 0.9999999999993234, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.97690625, + "pass@2": 0.9797711614173229, + "pass@4": 0.9822365332770904, + "pass@8": 0.9839197808629503, + "pass@16": 0.9851091800687043, + "pass@32": 0.9864141944813757, + "pass@64": 0.9875902077971946, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.9890859375, + "pass@2": 0.9915361712598425, + "pass@4": 0.9926196254218225, + "pass@8": 0.9933268121438683, + "pass@16": 0.9938099796924188, + "pass@32": 0.9939878373429921, + "pass@64": 0.9939999925233229, + "pass@128": 0.994 + }, + "7": { + "pass@1": 0.986375, + "pass@2": 0.9904328248031498, + "pass@4": 0.9927345397450318, + "pass@8": 0.994424364831558, + "pass@16": 0.9957573771225494, + "pass@32": 0.9967679662033939, + "pass@64": 0.9976010722035833, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.978515625, + "pass@2": 0.9858937007874016, + "pass@4": 0.9891826092988377, + "pass@8": 0.990850334423763, + "pass@16": 0.9918520165660328, + "pass@32": 0.9926379936242916, + "pass@64": 0.9934968420841649, + "pass@128": 0.995 + }, + "9": { + "pass@1": 0.96534375, + "pass@2": 0.9769838828740166, + "pass@4": 0.9824721196100491, + "pass@8": 0.9857095330132968, + "pass@16": 0.9879610780148321, + "pass@32": 0.9895870200289643, + "pass@64": 0.9911629214499705, + "pass@128": 0.993 + } + }, + "per_op_accuracy": { + "10": 0.90171875, + "2": 0.9990546875, + "3": 0.998828125, + "4": 0.9992421875, + "5": 0.97690625, + "6": 0.9890859375, + "7": 0.986375, + "8": 0.978515625, + "9": 0.96534375 + }, + "per_op_avg_response_len": { + "10": 273.5842265625, + "2": 114.487234375, + "3": 142.4478046875, + "4": 157.4185625, + "5": 170.3610546875, + "6": 195.172, + "7": 219.4658671875, + "8": 238.2007890625, + "9": 253.443296875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.5842265625, + "2": 114.487234375, + "3": 142.4478046875, + "4": 157.4185625, + "5": 170.3610546875, + "6": 195.172, + "7": 219.4658671875, + "8": 238.2007890625, + "9": 253.443296875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 368901, + "answer_accuracy": 0.9733330167173252, + "avg_response_len": 186.53898862293144, + "resp_tokens_sum": 70699769, + "pass_at_k": { + "pass@1": 0.9733330167173252, + "pass@2": 0.9804567314723968, + "pass@4": 0.984037224608576, + "pass@8": 0.9863265427413677, + "pass@16": 0.9881481678952296, + "pass@32": 0.9898705666230594, + "pass@64": 0.991468838289892, + "pass@128": 0.9925700776764607 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9000046091445427, + "pass@2": 0.9150345939888044, + "pass@4": 0.9236329891838736, + "pass@8": 0.9308381201660656, + "pass@16": 0.9381075792957397, + "pass@32": 0.9462885722585335, + "pass@64": 0.9541117889729607, + "pass@128": 0.9587020648967551 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969896788990825, + "pass@2": 0.9970370855305931, + "pass@4": 0.9971307700642925, + "pass@8": 0.9973136242144044, + "pass@16": 0.9976612728454813, + "pass@32": 0.9982843314310482, + "pass@64": 0.9992414938958318, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9976806640625, + "pass@2": 0.9994152159202757, + "pass@4": 0.9999608991258906, + "pass@8": 0.9999998130536076, + "pass@16": 0.9999999999978858, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9746279761904761, + "pass@2": 0.9800646012998375, + "pass@4": 0.9848414263693228, + "pass@8": 0.9876718146803775, + "pass@16": 0.9888463646668386, + "pass@32": 0.9897427177507346, + "pass@64": 0.9903847019122605, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9837434505988024, + "pass@2": 0.9885827508369086, + "pass@4": 0.9905320604572633, + "pass@8": 0.992091198748378, + "pass@16": 0.9934451035275379, + "pass@32": 0.993975561223535, + "pass@64": 0.9940119536626435, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9717085798816568, + "pass@2": 0.9788356706890929, + "pass@4": 0.9833554588960404, + "pass@8": 0.9872629105885962, + "pass@16": 0.990595545939145, + "pass@32": 0.9934039163490062, + "pass@64": 0.9958611615406721, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9722055288461539, + "pass@2": 0.9869693083737128, + "pass@4": 0.993037607979291, + "pass@8": 0.9952941196678669, + "pass@16": 0.996277513394104, + "pass@32": 0.9967337642373502, + "pass@64": 0.9967946729122014, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9670994718309859, + "pass@2": 0.9814191249861375, + "pass@4": 0.9867852194532019, + "pass@8": 0.9891142645184077, + "pass@16": 0.9906925954424748, + "pass@32": 0.9923033625045726, + "pass@64": 0.99428501700158, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382648, + "answer_accuracy": 0.9795011467889908, + "avg_response_len": 194.0187325933814, + "resp_tokens_sum": 75794582, + "pass_at_k": { + "pass@1": 0.9795011467889908, + "pass@2": 0.9834198128115305, + "pass@4": 0.9857896414344838, + "pass@8": 0.9876240419139255, + "pass@16": 0.9892223679888624, + "pass@32": 0.9906843232453986, + "pass@64": 0.9920135437884321, + "pass@128": 0.9931192660550459 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8994919478527608, + "pass@2": 0.9163061265156274, + "pass@4": 0.9283594290936025, + "pass@8": 0.9382051798212965, + "pass@16": 0.9469677315727241, + "pass@32": 0.9551237622514445, + "pass@64": 0.961818137346346, + "pass@128": 0.9662576687116564 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9816123188405798, + "pass@2": 0.9827905682985281, + "pass@4": 0.9834688090619108, + "pass@8": 0.9842941702692776, + "pass@16": 0.9857126174441587, + "pass@32": 0.9878730050048092, + "pass@64": 0.990240146068698, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9922770415472779, + "pass@2": 0.9936541834938971, + "pass@4": 0.9941627463114389, + "pass@8": 0.99426602927843, + "pass@16": 0.9942693391968014, + "pass@32": 0.994269340974212, + "pass@64": 0.994269340974212, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9937454841040463, + "pass@2": 0.9949379295434891, + "pass@4": 0.9956017114551432, + "pass@8": 0.9963418329269713, + "pass@16": 0.9969251127417026, + "pass@32": 0.9971024349058709, + "pass@64": 0.9971098254417226, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9880941901408451, + "pass@2": 0.9926787595652655, + "pass@4": 0.9941441718552785, + "pass@8": 0.9946804118630559, + "pass@16": 0.995070095638103, + "pass@32": 0.9957746478821857, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9570078592814372, + "pass@2": 0.9676165631335754, + "pass@4": 0.9740296437496209, + "pass@8": 0.9788848321388287, + "pass@16": 0.9824501833869013, + "pass@32": 0.9846843678105359, + "pass@64": 0.9863466253618248, + "pass@128": 0.9880239520958084 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 374220, + "answer_accuracy": 0.9787725979243388, + "avg_response_len": 207.59749539671913, + "resp_tokens_sum": 79371996, + "pass_at_k": { + "pass@1": 0.9787725979243388, + "pass@2": 0.9831177062546631, + "pass@4": 0.9860878020792264, + "pass@8": 0.9882428638856257, + "pass@16": 0.9898623209805008, + "pass@32": 0.9910769662520457, + "pass@64": 0.9919538759400394, + "pass@128": 0.9926347505858721 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9056203358208955, + "pass@2": 0.9243040457163008, + "pass@4": 0.9389661845440961, + "pass@8": 0.9507190979296448, + "pass@16": 0.9601808559838616, + "pass@32": 0.9673198046675844, + "pass@64": 0.9723514966419701, + "pass@128": 0.9761194029850746 + }, + "2": { + "pass@1": 0.9972991071428572, + "pass@2": 0.9974479752530934, + "pass@4": 0.9977247281589801, + "pass@8": 0.9982019248579823, + "pass@16": 0.9989049734161272, + "pass@32": 0.9996397297099158, + "pass@64": 0.9999812230788224, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994533527696793, + "pass@2": 0.9999397396753976, + "pass@4": 0.9999991596349289, + "pass@8": 0.9999999999107686, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9742417279411765, + "pass@2": 0.976435488073182, + "pass@4": 0.9785727789541012, + "pass@8": 0.9800637367521491, + "pass@16": 0.981034477001222, + "pass@32": 0.9818501519506909, + "pass@64": 0.982312283238623, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9912016955835962, + "pass@2": 0.9923161591196998, + "pass@4": 0.9931202484744611, + "pass@8": 0.9935946610212556, + "pass@16": 0.993688693168888, + "pass@32": 0.9936908514016762, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9939922863924051, + "pass@2": 0.9979045711651552, + "pass@4": 0.9996272230211731, + "pass@8": 0.9999848951262044, + "pass@16": 0.9999999797040798, + "pass@32": 0.999999999999996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9742164039039038, + "pass@2": 0.9776526033907924, + "pass@4": 0.9802813651671918, + "pass@8": 0.9826036723003114, + "pass@16": 0.9842749202268645, + "pass@32": 0.985456634186374, + "pass@64": 0.9864771895962706, + "pass@128": 0.987987987987988 + }, + "9": { + "pass@1": 0.972292001607717, + "pass@2": 0.9819812264222599, + "pass@4": 0.986615645311217, + "pass@8": 0.9891525246781142, + "pass@16": 0.9907615607123103, + "pass@32": 0.9917515996499101, + "pass@64": 0.9927715998185213, + "pass@128": 0.9935691318327974 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.35745859375, + "avg_response_len": 315.944015625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.35745859375, + "pass@2": 0.41038422736221175, + "pass@4": 0.45656204859392713, + "pass@8": 0.49663667668830686, + "pass@16": 0.5317553718358711, + "pass@32": 0.5628365507079407, + "pass@64": 0.5904172611029005, + "pass@128": 0.6155 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8521796875, + "pass@2": 0.8826498523622048, + "pass@4": 0.9019524208848895, + "pass@8": 0.9162788519906351, + "pass@16": 0.9278805123943094, + "pass@32": 0.9375432152594656, + "pass@64": 0.9457429549439367, + "pass@128": 0.951 + }, + "12": { + "pass@1": 0.7126328125, + "pass@2": 0.7664475885826777, + "pass@4": 0.8050542942444691, + "pass@8": 0.8339374081797831, + "pass@16": 0.8565802584389245, + "pass@32": 0.875478516227526, + "pass@64": 0.8916753179204527, + "pass@128": 0.905 + }, + "13": { + "pass@1": 0.4683203125, + "pass@2": 0.5323293553149607, + "pass@4": 0.5836485316835397, + "pass@8": 0.6262608522090582, + "pass@16": 0.6638667571257654, + "pass@32": 0.6988731675248452, + "pass@64": 0.7316326124719252, + "pass@128": 0.761 + }, + "14": { + "pass@1": 0.39525, + "pass@2": 0.45573868110236265, + "pass@4": 0.5044872852455938, + "pass@8": 0.5460352483880692, + "pass@16": 0.5834204342201049, + "pass@32": 0.6181456996660168, + "pass@64": 0.6510138582017766, + "pass@128": 0.684 + }, + "15": { + "pass@1": 0.290015625, + "pass@2": 0.35128457185039375, + "pass@4": 0.40712679471316043, + "pass@8": 0.458066279644021, + "pass@16": 0.5040513662108203, + "pass@32": 0.5438092119108932, + "pass@64": 0.5760747302445015, + "pass@128": 0.602 + }, + "16": { + "pass@1": 0.1971015625, + "pass@2": 0.24690588090551172, + "pass@4": 0.2925098887326582, + "pass@8": 0.33302631077575845, + "pass@16": 0.36878563691614125, + "pass@32": 0.4010538115703694, + "pass@64": 0.43131436565014997, + "pass@128": 0.459 + }, + "17": { + "pass@1": 0.1835234375, + "pass@2": 0.2347957677165352, + "pass@4": 0.2837368578927632, + "pass@8": 0.3292021787983135, + "pass@16": 0.37171267121835305, + "pass@32": 0.41049264568036214, + "pass@64": 0.4439672168855264, + "pass@128": 0.476 + }, + "18": { + "pass@1": 0.171921875, + "pass@2": 0.22305573326771638, + "pass@4": 0.2714720539932507, + "pass@8": 0.3150629889307357, + "pass@16": 0.3544238974627193, + "pass@32": 0.38985483100134993, + "pass@64": 0.4204924719767653, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.1479375, + "pass@2": 0.20008304625984225, + "pass@4": 0.2513866133295835, + "pass@8": 0.2975000269098746, + "pass@16": 0.33725700378345325, + "pass@32": 0.3707434569592805, + "pass@64": 0.39991453230743806, + "pass@128": 0.428 + }, + "20": { + "pass@1": 0.155703125, + "pass@2": 0.21055179625984247, + "pass@4": 0.2642457452193475, + "pass@8": 0.3109966210567916, + "pass@16": 0.34957518058801845, + "pass@32": 0.38237095127929793, + "pass@64": 0.4123445504264982, + "pass@128": 0.44 + } + }, + "per_op_accuracy": { + "11": 0.8521796875, + "12": 0.7126328125, + "13": 0.4683203125, + "14": 0.39525, + "15": 0.290015625, + "16": 0.1971015625, + "17": 0.1835234375, + "18": 0.171921875, + "19": 0.1479375, + "20": 0.155703125 + }, + "per_op_avg_response_len": { + "11": 289.2789453125, + "12": 297.2435703125, + "13": 308.2810625, + "14": 321.3185234375, + "15": 323.91484375, + "16": 322.6036796875, + "17": 326.5742265625, + "18": 324.5450234375, + "19": 321.4939296875, + "20": 324.1863515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 289.2789453125, + "12": 297.2435703125, + "13": 308.2810625, + "14": 321.3185234375, + "15": 323.91484375, + "16": 322.6036796875, + "17": 326.5742265625, + "18": 324.5450234375, + "19": 321.4939296875, + "20": 324.1863515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 155134, + "answer_accuracy": 0.36017366270430906, + "avg_response_len": 299.2867338410104, + "resp_tokens_sum": 128908782, + "pass_at_k": { + "pass@1": 0.36017366270430906, + "pass@2": 0.41512845000058607, + "pass@4": 0.4632319151413664, + "pass@8": 0.5038704634042737, + "pass@16": 0.5378097520203142, + "pass@32": 0.5667291103117928, + "pass@64": 0.5921206181496333, + "pass@128": 0.6160475482912333 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8452992209631728, + "pass@2": 0.874468838526912, + "pass@4": 0.8927729621615995, + "pass@8": 0.9068203029145412, + "pass@16": 0.9184389985389775, + "pass@32": 0.9286397978946337, + "pass@64": 0.9370345366531673, + "pass@128": 0.9405099150141643 + }, + "12": { + "pass@1": 0.718794138418079, + "pass@2": 0.7764510765603452, + "pass@4": 0.8174984948915283, + "pass@8": 0.84820170368261, + "pass@16": 0.8711515105018326, + "pass@32": 0.8890325500632591, + "pass@64": 0.9038298217779518, + "pass@128": 0.9152542372881356 + }, + "13": { + "pass@1": 0.4823912617554859, + "pass@2": 0.5498393262656431, + "pass@4": 0.6036611143630555, + "pass@8": 0.6477879485187265, + "pass@16": 0.685484891823752, + "pass@32": 0.7189389553919789, + "pass@64": 0.7495642145009295, + "pass@128": 0.780564263322884 + }, + "14": { + "pass@1": 0.38770443925233644, + "pass@2": 0.44748279858709245, + "pass@4": 0.49312651893980536, + "pass@8": 0.5296382419814589, + "pass@16": 0.5612308622050344, + "pass@32": 0.5903160848446066, + "pass@64": 0.619181232589094, + "pass@128": 0.6542056074766355 + }, + "15": { + "pass@1": 0.2938041907514451, + "pass@2": 0.35669355343407216, + "pass@4": 0.4148084477157117, + "pass@8": 0.4650869029400088, + "pass@16": 0.5067125568176679, + "pass@32": 0.5417496851194056, + "pass@64": 0.5726888869572655, + "pass@128": 0.6011560693641619 + }, + "16": { + "pass@1": 0.18674395161290322, + "pass@2": 0.23528781432562876, + "pass@4": 0.2805795692877099, + "pass@8": 0.3201898414598259, + "pass@16": 0.3533900521458434, + "pass@32": 0.38255860221232757, + "pass@64": 0.41028016987609034, + "pass@128": 0.43548387096774194 + }, + "17": { + "pass@1": 0.18230671965317918, + "pass@2": 0.23375557553138218, + "pass@4": 0.2822599248793323, + "pass@8": 0.32672688937217353, + "pass@16": 0.36842829528522153, + "pass@32": 0.4058567655828945, + "pass@64": 0.4349941917269913, + "pass@128": 0.45664739884393063 + }, + "18": { + "pass@1": 0.1603125, + "pass@2": 0.21394474128233967, + "pass@4": 0.265273910493331, + "pass@8": 0.3092573071038731, + "pass@16": 0.3474877935238888, + "pass@32": 0.3828370961500478, + "pass@64": 0.4140552777998218, + "pass@128": 0.44285714285714284 + }, + "19": { + "pass@1": 0.15153434684684686, + "pass@2": 0.21021759948925312, + "pass@4": 0.27047376904463505, + "pass@8": 0.3240379916251193, + "pass@16": 0.3656271096807543, + "pass@32": 0.39589239284381544, + "pass@64": 0.4213365552310757, + "pass@128": 0.44744744744744747 + }, + "20": { + "pass@1": 0.15498310810810811, + "pass@2": 0.21680957532335482, + "pass@4": 0.27715197115000256, + "pass@8": 0.3276990281660607, + "pass@16": 0.36812078329427916, + "pass@32": 0.4005794165660846, + "pass@64": 0.42888634723852626, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 151654, + "answer_accuracy": 0.35230356080880165, + "avg_response_len": 312.2791383251561, + "resp_tokens_sum": 134424927, + "pass_at_k": { + "pass@1": 0.35230356080880165, + "pass@2": 0.40511396162734464, + "pass@4": 0.45068913185584203, + "pass@8": 0.49027892608542734, + "pass@16": 0.5251501247292233, + "pass@32": 0.5561363670478808, + "pass@64": 0.5840671998665793, + "pass@128": 0.6095747844186739 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8519860692771084, + "pass@2": 0.8888458608054265, + "pass@4": 0.9119766091926755, + "pass@8": 0.9283114158279034, + "pass@16": 0.9401535492489448, + "pass@32": 0.9484750469432417, + "pass@64": 0.9544825870655861, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.7110599529780565, + "pass@2": 0.7596701213190828, + "pass@4": 0.7952358187789219, + "pass@8": 0.8224395278267408, + "pass@16": 0.8435730646163403, + "pass@32": 0.86120908890127, + "pass@64": 0.8775723553695687, + "pass@128": 0.8934169278996865 + }, + "13": { + "pass@1": 0.4431897759103641, + "pass@2": 0.5064896529477931, + "pass@4": 0.5575887811817638, + "pass@8": 0.6006728712300449, + "pass@16": 0.6382857482801823, + "pass@32": 0.672182533480856, + "pass@64": 0.7036470611522763, + "pass@128": 0.7310924369747899 + }, + "14": { + "pass@1": 0.39160727339181284, + "pass@2": 0.4481789180135375, + "pass@4": 0.4956121079711529, + "pass@8": 0.5395046872717323, + "pass@16": 0.580186785625428, + "pass@32": 0.6175963919547802, + "pass@64": 0.652920536861353, + "pass@128": 0.6871345029239766 + }, + "15": { + "pass@1": 0.2729403409090909, + "pass@2": 0.33778334526366016, + "pass@4": 0.39326375396257257, + "pass@8": 0.44043010034042246, + "pass@16": 0.48186062978129907, + "pass@32": 0.5183775815933022, + "pass@64": 0.5489058851771729, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.21091524216524216, + "pass@2": 0.26682271687192954, + "pass@4": 0.31628648368740236, + "pass@8": 0.35864969193198676, + "pass@16": 0.3954488342196391, + "pass@32": 0.42901714214411485, + "pass@64": 0.46241872936773004, + "pass@128": 0.49572649572649574 + }, + "17": { + "pass@1": 0.18791246118012422, + "pass@2": 0.23750657186873375, + "pass@4": 0.2855899974847864, + "pass@8": 0.33262139177019856, + "pass@16": 0.37758867649195893, + "pass@32": 0.4197480671208388, + "pass@64": 0.45981387209971636, + "pass@128": 0.5031055900621118 + }, + "18": { + "pass@1": 0.1879595588235294, + "pass@2": 0.23974315365910143, + "pass@4": 0.28761232024305783, + "pass@8": 0.329751823564591, + "pass@16": 0.3682869367529674, + "pass@32": 0.40342406625742766, + "pass@64": 0.43378958147428764, + "pass@128": 0.46176470588235297 + }, + "19": { + "pass@1": 0.14397321428571427, + "pass@2": 0.19275703330052496, + "pass@4": 0.2422015247536021, + "pass@8": 0.2878507464902696, + "pass@16": 0.3280847004008977, + "pass@32": 0.36158603274779966, + "pass@64": 0.3892965040997099, + "pass@128": 0.41369047619047616 + }, + "20": { + "pass@1": 0.13793506736526945, + "pass@2": 0.18860249481352254, + "pass@4": 0.23572179403846963, + "pass@8": 0.2758505298038162, + "pass@16": 0.3104882346741073, + "pass@32": 0.341360691539217, + "pass@64": 0.36851288153677664, + "pass@128": 0.38922155688622756 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 150759, + "answer_accuracy": 0.35996475779339854, + "avg_response_len": 336.84155094361245, + "resp_tokens_sum": 141074631, + "pass_at_k": { + "pass@1": 0.35996475779339854, + "pass@2": 0.410922000738792, + "pass@4": 0.45573885729137276, + "pass@8": 0.495731854860139, + "pass@16": 0.5323178586020396, + "pass@32": 0.5657198803478534, + "pass@64": 0.5951921570306081, + "pass@128": 0.6210268948655256 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.860094246031746, + "pass@2": 0.8852873859517562, + "pass@4": 0.9016740666345278, + "pass@8": 0.9141964920855181, + "pass@16": 0.9255256112996866, + "pass@32": 0.9359988921190586, + "pass@64": 0.9462906177764885, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.7074971330275229, + "pass@2": 0.7622297822003904, + "pass@4": 0.8011608589063977, + "pass@8": 0.8297119134538486, + "pass@16": 0.8534948504852095, + "pass@32": 0.8747256701701144, + "pass@64": 0.8922751671197711, + "pass@128": 0.9051987767584098 + }, + "13": { + "pass@1": 0.48215663580246915, + "pass@2": 0.5435611238699328, + "pass@4": 0.5926587695056635, + "pass@8": 0.633260128402649, + "pass@16": 0.6707687175863076, + "pass@32": 0.7085261614263532, + "pass@64": 0.7448136642431026, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.406134087537092, + "pass@2": 0.4712745186803429, + "pass@4": 0.5243155244503929, + "pass@8": 0.568281215623407, + "pass@16": 0.6078381209744614, + "pass@32": 0.6452114848141344, + "pass@64": 0.6794001748786187, + "pass@128": 0.7091988130563798 + }, + "15": { + "pass@1": 0.3033613040123457, + "pass@2": 0.35925956304073065, + "pass@4": 0.4130433117156653, + "pass@8": 0.46853172257542763, + "pass@16": 0.5238111534693787, + "pass@32": 0.5719111695487336, + "pass@64": 0.6073624482988279, + "pass@128": 0.6327160493827161 + }, + "16": { + "pass@1": 0.1922704646017699, + "pass@2": 0.23690821487468935, + "pass@4": 0.27880137604480854, + "pass@8": 0.318234271548925, + "pass@16": 0.35525716796412005, + "pass@32": 0.38901365189369735, + "pass@64": 0.4183437728038016, + "pass@128": 0.4424778761061947 + }, + "17": { + "pass@1": 0.18053463855421686, + "pass@2": 0.23325067000284597, + "pass@4": 0.2834787490795578, + "pass@8": 0.32846562326969203, + "pass@16": 0.369436527768843, + "pass@32": 0.40634737101135743, + "pass@64": 0.4379492763008395, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.16743951612903227, + "pass@2": 0.21504000508001017, + "pass@4": 0.26076773044498464, + "pass@8": 0.30550745623361136, + "pass@16": 0.34705035881725593, + "pass@32": 0.3828956929719597, + "pass@64": 0.4131763453082906, + "pass@128": 0.44193548387096776 + }, + "19": { + "pass@1": 0.14834308912386707, + "pass@2": 0.19732393486690303, + "pass@4": 0.2415079635060723, + "pass@8": 0.28059675189721844, + "pass@16": 0.31802633512102896, + "pass@32": 0.354738317640572, + "pass@64": 0.38914149256189534, + "pass@128": 0.4229607250755287 + }, + "20": { + "pass@1": 0.17424455705705705, + "pass@2": 0.22630923246080717, + "pass@4": 0.2799491279806238, + "pass@8": 0.32954584901807554, + "pass@16": 0.370233902011627, + "pass@32": 0.4052958996657454, + "pass@64": 0.43976604913749345, + "pass@128": 0.47147147147147145 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6510345394736842, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6510345394736842, + "pass@2": 0.6813128690944722, + "pass@4": 0.7070225767617703, + "pass@8": 0.7291047194888424, + "pass@16": 0.748383466493019, + "pass@32": 0.7654361951363342, + "pass@64": 0.78055281075161, + "pass@128": 0.7942105263157895 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.90171875, + "pass@2": 0.9185543799212602, + "pass@4": 0.9303104290401201, + "pass@8": 0.93989990916447, + "pass@16": 0.9483905366285567, + "pass@32": 0.9562143070532549, + "pass@64": 0.9627343606118026, + "pass@128": 0.967 + }, + "2": { + "pass@1": 0.9990546875, + "pass@2": 0.9991067913385827, + "pass@4": 0.999203654855643, + "pass@8": 0.9993706737002938, + "pass@16": 0.9996167406956444, + "pass@32": 0.9998739053984704, + "pass@64": 0.9999934280775878, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998828125, + "pass@2": 0.9990104576771653, + "pass@4": 0.9990614735658042, + "pass@8": 0.9991215550875039, + "pass@16": 0.9992352362204724, + "pass@32": 0.9994389763779528, + "pass@64": 0.999751968503937, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9992421875, + "pass@2": 0.9998128690944881, + "pass@4": 0.9999874877202849, + "pass@8": 0.9999999401771543, + "pass@16": 0.9999999999993234, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.97690625, + "pass@2": 0.9797711614173229, + "pass@4": 0.9822365332770904, + "pass@8": 0.9839197808629503, + "pass@16": 0.9851091800687043, + "pass@32": 0.9864141944813757, + "pass@64": 0.9875902077971946, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.9890859375, + "pass@2": 0.9915361712598425, + "pass@4": 0.9926196254218225, + "pass@8": 0.9933268121438683, + "pass@16": 0.9938099796924188, + "pass@32": 0.9939878373429921, + "pass@64": 0.9939999925233229, + "pass@128": 0.994 + }, + "7": { + "pass@1": 0.986375, + "pass@2": 0.9904328248031498, + "pass@4": 0.9927345397450318, + "pass@8": 0.994424364831558, + "pass@16": 0.9957573771225494, + "pass@32": 0.9967679662033939, + "pass@64": 0.9976010722035833, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.978515625, + "pass@2": 0.9858937007874016, + "pass@4": 0.9891826092988377, + "pass@8": 0.990850334423763, + "pass@16": 0.9918520165660328, + "pass@32": 0.9926379936242916, + "pass@64": 0.9934968420841649, + "pass@128": 0.995 + }, + "9": { + "pass@1": 0.96534375, + "pass@2": 0.9769838828740166, + "pass@4": 0.9824721196100491, + "pass@8": 0.9857095330132968, + "pass@16": 0.9879610780148321, + "pass@32": 0.9895870200289643, + "pass@64": 0.9911629214499705, + "pass@128": 0.993 + }, + "11": { + "pass@1": 0.8521796875, + "pass@2": 0.8826498523622048, + "pass@4": 0.9019524208848895, + "pass@8": 0.9162788519906351, + "pass@16": 0.9278805123943094, + "pass@32": 0.9375432152594656, + "pass@64": 0.9457429549439367, + "pass@128": 0.951 + }, + "12": { + "pass@1": 0.7126328125, + "pass@2": 0.7664475885826777, + "pass@4": 0.8050542942444691, + "pass@8": 0.8339374081797831, + "pass@16": 0.8565802584389245, + "pass@32": 0.875478516227526, + "pass@64": 0.8916753179204527, + "pass@128": 0.905 + }, + "13": { + "pass@1": 0.4683203125, + "pass@2": 0.5323293553149607, + "pass@4": 0.5836485316835397, + "pass@8": 0.6262608522090582, + "pass@16": 0.6638667571257654, + "pass@32": 0.6988731675248452, + "pass@64": 0.7316326124719252, + "pass@128": 0.761 + }, + "14": { + "pass@1": 0.39525, + "pass@2": 0.45573868110236265, + "pass@4": 0.5044872852455938, + "pass@8": 0.5460352483880692, + "pass@16": 0.5834204342201049, + "pass@32": 0.6181456996660168, + "pass@64": 0.6510138582017766, + "pass@128": 0.684 + }, + "15": { + "pass@1": 0.290015625, + "pass@2": 0.35128457185039375, + "pass@4": 0.40712679471316043, + "pass@8": 0.458066279644021, + "pass@16": 0.5040513662108203, + "pass@32": 0.5438092119108932, + "pass@64": 0.5760747302445015, + "pass@128": 0.602 + }, + "16": { + "pass@1": 0.1971015625, + "pass@2": 0.24690588090551172, + "pass@4": 0.2925098887326582, + "pass@8": 0.33302631077575845, + "pass@16": 0.36878563691614125, + "pass@32": 0.4010538115703694, + "pass@64": 0.43131436565014997, + "pass@128": 0.459 + }, + "17": { + "pass@1": 0.1835234375, + "pass@2": 0.2347957677165352, + "pass@4": 0.2837368578927632, + "pass@8": 0.3292021787983135, + "pass@16": 0.37171267121835305, + "pass@32": 0.41049264568036214, + "pass@64": 0.4439672168855264, + "pass@128": 0.476 + }, + "18": { + "pass@1": 0.171921875, + "pass@2": 0.22305573326771638, + "pass@4": 0.2714720539932507, + "pass@8": 0.3150629889307357, + "pass@16": 0.3544238974627193, + "pass@32": 0.38985483100134993, + "pass@64": 0.4204924719767653, + "pass@128": 0.449 + }, + "19": { + "pass@1": 0.1479375, + "pass@2": 0.20008304625984225, + "pass@4": 0.2513866133295835, + "pass@8": 0.2975000269098746, + "pass@16": 0.33725700378345325, + "pass@32": 0.3707434569592805, + "pass@64": 0.39991453230743806, + "pass@128": 0.428 + }, + "20": { + "pass@1": 0.155703125, + "pass@2": 0.21055179625984247, + "pass@4": 0.2642457452193475, + "pass@8": 0.3109966210567916, + "pass@16": 0.34957518058801845, + "pass@32": 0.38237095127929793, + "pass@64": 0.4123445504264982, + "pass@128": 0.44 + } + }, + "per_op_accuracy": { + "10": 0.90171875, + "2": 0.9990546875, + "3": 0.998828125, + "4": 0.9992421875, + "5": 0.97690625, + "6": 0.9890859375, + "7": 0.986375, + "8": 0.978515625, + "9": 0.96534375, + "11": 0.8521796875, + "12": 0.7126328125, + "13": 0.4683203125, + "14": 0.39525, + "15": 0.290015625, + "16": 0.1971015625, + "17": 0.1835234375, + "18": 0.171921875, + "19": 0.1479375, + "20": 0.155703125 + }, + "per_op_avg_response_len": { + "10": 273.5842265625, + "2": 114.487234375, + "3": 142.4478046875, + "4": 157.4185625, + "5": 170.3610546875, + "6": 195.172, + "7": 219.4658671875, + "8": 238.2007890625, + "9": 253.443296875, + "11": 289.2789453125, + "12": 297.2435703125, + "13": 308.2810625, + "14": 321.3185234375, + "15": 323.91484375, + "16": 322.6036796875, + "17": 326.5742265625, + "18": 324.5450234375, + "19": 321.4939296875, + "20": 324.1863515625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.5842265625, + "2": 114.487234375, + "3": 142.4478046875, + "4": 157.4185625, + "5": 170.3610546875, + "6": 195.172, + "7": 219.4658671875, + "8": 238.2007890625, + "9": 253.443296875, + "11": 289.2789453125, + "12": 297.2435703125, + "13": 308.2810625, + "14": 321.3185234375, + "15": 323.91484375, + "16": 322.6036796875, + "17": 326.5742265625, + "18": 324.5450234375, + "19": 321.4939296875, + "20": 324.1863515625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 524035, + "answer_accuracy": 0.6471741127884919, + "avg_response_len": 246.5130895806987, + "resp_tokens_sum": 199608551, + "pass_at_k": { + "pass@1": 0.6471741127884919, + "pass@2": 0.6797406917707471, + "pass@4": 0.7070043655574902, + "pass@8": 0.7296928552659782, + "pass@16": 0.7485988840793822, + "pass@32": 0.7647882080256212, + "pass@64": 0.7790428565048858, + "pass@128": 0.7922858046158711 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9000046091445427, + "pass@2": 0.9150345939888044, + "pass@4": 0.9236329891838736, + "pass@8": 0.9308381201660656, + "pass@16": 0.9381075792957397, + "pass@32": 0.9462885722585335, + "pass@64": 0.9541117889729607, + "pass@128": 0.9587020648967551 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969896788990825, + "pass@2": 0.9970370855305931, + "pass@4": 0.9971307700642925, + "pass@8": 0.9973136242144044, + "pass@16": 0.9976612728454813, + "pass@32": 0.9982843314310482, + "pass@64": 0.9992414938958318, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9976806640625, + "pass@2": 0.9994152159202757, + "pass@4": 0.9999608991258906, + "pass@8": 0.9999998130536076, + "pass@16": 0.9999999999978858, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9746279761904761, + "pass@2": 0.9800646012998375, + "pass@4": 0.9848414263693228, + "pass@8": 0.9876718146803775, + "pass@16": 0.9888463646668386, + "pass@32": 0.9897427177507346, + "pass@64": 0.9903847019122605, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9837434505988024, + "pass@2": 0.9885827508369086, + "pass@4": 0.9905320604572633, + "pass@8": 0.992091198748378, + "pass@16": 0.9934451035275379, + "pass@32": 0.993975561223535, + "pass@64": 0.9940119536626435, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9717085798816568, + "pass@2": 0.9788356706890929, + "pass@4": 0.9833554588960404, + "pass@8": 0.9872629105885962, + "pass@16": 0.990595545939145, + "pass@32": 0.9934039163490062, + "pass@64": 0.9958611615406721, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9722055288461539, + "pass@2": 0.9869693083737128, + "pass@4": 0.993037607979291, + "pass@8": 0.9952941196678669, + "pass@16": 0.996277513394104, + "pass@32": 0.9967337642373502, + "pass@64": 0.9967946729122014, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9670994718309859, + "pass@2": 0.9814191249861375, + "pass@4": 0.9867852194532019, + "pass@8": 0.9891142645184077, + "pass@16": 0.9906925954424748, + "pass@32": 0.9923033625045726, + "pass@64": 0.99428501700158, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8452992209631728, + "pass@2": 0.874468838526912, + "pass@4": 0.8927729621615995, + "pass@8": 0.9068203029145412, + "pass@16": 0.9184389985389775, + "pass@32": 0.9286397978946337, + "pass@64": 0.9370345366531673, + "pass@128": 0.9405099150141643 + }, + "12": { + "pass@1": 0.718794138418079, + "pass@2": 0.7764510765603452, + "pass@4": 0.8174984948915283, + "pass@8": 0.84820170368261, + "pass@16": 0.8711515105018326, + "pass@32": 0.8890325500632591, + "pass@64": 0.9038298217779518, + "pass@128": 0.9152542372881356 + }, + "13": { + "pass@1": 0.4823912617554859, + "pass@2": 0.5498393262656431, + "pass@4": 0.6036611143630555, + "pass@8": 0.6477879485187265, + "pass@16": 0.685484891823752, + "pass@32": 0.7189389553919789, + "pass@64": 0.7495642145009295, + "pass@128": 0.780564263322884 + }, + "14": { + "pass@1": 0.38770443925233644, + "pass@2": 0.44748279858709245, + "pass@4": 0.49312651893980536, + "pass@8": 0.5296382419814589, + "pass@16": 0.5612308622050344, + "pass@32": 0.5903160848446066, + "pass@64": 0.619181232589094, + "pass@128": 0.6542056074766355 + }, + "15": { + "pass@1": 0.2938041907514451, + "pass@2": 0.35669355343407216, + "pass@4": 0.4148084477157117, + "pass@8": 0.4650869029400088, + "pass@16": 0.5067125568176679, + "pass@32": 0.5417496851194056, + "pass@64": 0.5726888869572655, + "pass@128": 0.6011560693641619 + }, + "16": { + "pass@1": 0.18674395161290322, + "pass@2": 0.23528781432562876, + "pass@4": 0.2805795692877099, + "pass@8": 0.3201898414598259, + "pass@16": 0.3533900521458434, + "pass@32": 0.38255860221232757, + "pass@64": 0.41028016987609034, + "pass@128": 0.43548387096774194 + }, + "17": { + "pass@1": 0.18230671965317918, + "pass@2": 0.23375557553138218, + "pass@4": 0.2822599248793323, + "pass@8": 0.32672688937217353, + "pass@16": 0.36842829528522153, + "pass@32": 0.4058567655828945, + "pass@64": 0.4349941917269913, + "pass@128": 0.45664739884393063 + }, + "18": { + "pass@1": 0.1603125, + "pass@2": 0.21394474128233967, + "pass@4": 0.265273910493331, + "pass@8": 0.3092573071038731, + "pass@16": 0.3474877935238888, + "pass@32": 0.3828370961500478, + "pass@64": 0.4140552777998218, + "pass@128": 0.44285714285714284 + }, + "19": { + "pass@1": 0.15153434684684686, + "pass@2": 0.21021759948925312, + "pass@4": 0.27047376904463505, + "pass@8": 0.3240379916251193, + "pass@16": 0.3656271096807543, + "pass@32": 0.39589239284381544, + "pass@64": 0.4213365552310757, + "pass@128": 0.44744744744744747 + }, + "20": { + "pass@1": 0.15498310810810811, + "pass@2": 0.21680957532335482, + "pass@4": 0.27715197115000256, + "pass@8": 0.3276990281660607, + "pass@16": 0.36812078329427916, + "pass@32": 0.4005794165660846, + "pass@64": 0.42888634723852626, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 534302, + "answer_accuracy": 0.6506990452065472, + "avg_response_len": 256.01557506819955, + "resp_tokens_sum": 210219509, + "pass_at_k": { + "pass@1": 0.6506990452065472, + "pass@2": 0.6802487173271308, + "pass@4": 0.7052685169273947, + "pass@8": 0.7268958073806057, + "pass@16": 0.7459371062457395, + "pass@32": 0.7628768755926708, + "pass@64": 0.7781517270138143, + "pass@128": 0.792049883086516 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8994919478527608, + "pass@2": 0.9163061265156274, + "pass@4": 0.9283594290936025, + "pass@8": 0.9382051798212965, + "pass@16": 0.9469677315727241, + "pass@32": 0.9551237622514445, + "pass@64": 0.961818137346346, + "pass@128": 0.9662576687116564 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9816123188405798, + "pass@2": 0.9827905682985281, + "pass@4": 0.9834688090619108, + "pass@8": 0.9842941702692776, + "pass@16": 0.9857126174441587, + "pass@32": 0.9878730050048092, + "pass@64": 0.990240146068698, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9922770415472779, + "pass@2": 0.9936541834938971, + "pass@4": 0.9941627463114389, + "pass@8": 0.99426602927843, + "pass@16": 0.9942693391968014, + "pass@32": 0.994269340974212, + "pass@64": 0.994269340974212, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9937454841040463, + "pass@2": 0.9949379295434891, + "pass@4": 0.9956017114551432, + "pass@8": 0.9963418329269713, + "pass@16": 0.9969251127417026, + "pass@32": 0.9971024349058709, + "pass@64": 0.9971098254417226, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9880941901408451, + "pass@2": 0.9926787595652655, + "pass@4": 0.9941441718552785, + "pass@8": 0.9946804118630559, + "pass@16": 0.995070095638103, + "pass@32": 0.9957746478821857, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9570078592814372, + "pass@2": 0.9676165631335754, + "pass@4": 0.9740296437496209, + "pass@8": 0.9788848321388287, + "pass@16": 0.9824501833869013, + "pass@32": 0.9846843678105359, + "pass@64": 0.9863466253618248, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8519860692771084, + "pass@2": 0.8888458608054265, + "pass@4": 0.9119766091926755, + "pass@8": 0.9283114158279034, + "pass@16": 0.9401535492489448, + "pass@32": 0.9484750469432417, + "pass@64": 0.9544825870655861, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.7110599529780565, + "pass@2": 0.7596701213190828, + "pass@4": 0.7952358187789219, + "pass@8": 0.8224395278267408, + "pass@16": 0.8435730646163403, + "pass@32": 0.86120908890127, + "pass@64": 0.8775723553695687, + "pass@128": 0.8934169278996865 + }, + "13": { + "pass@1": 0.4431897759103641, + "pass@2": 0.5064896529477931, + "pass@4": 0.5575887811817638, + "pass@8": 0.6006728712300449, + "pass@16": 0.6382857482801823, + "pass@32": 0.672182533480856, + "pass@64": 0.7036470611522763, + "pass@128": 0.7310924369747899 + }, + "14": { + "pass@1": 0.39160727339181284, + "pass@2": 0.4481789180135375, + "pass@4": 0.4956121079711529, + "pass@8": 0.5395046872717323, + "pass@16": 0.580186785625428, + "pass@32": 0.6175963919547802, + "pass@64": 0.652920536861353, + "pass@128": 0.6871345029239766 + }, + "15": { + "pass@1": 0.2729403409090909, + "pass@2": 0.33778334526366016, + "pass@4": 0.39326375396257257, + "pass@8": 0.44043010034042246, + "pass@16": 0.48186062978129907, + "pass@32": 0.5183775815933022, + "pass@64": 0.5489058851771729, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.21091524216524216, + "pass@2": 0.26682271687192954, + "pass@4": 0.31628648368740236, + "pass@8": 0.35864969193198676, + "pass@16": 0.3954488342196391, + "pass@32": 0.42901714214411485, + "pass@64": 0.46241872936773004, + "pass@128": 0.49572649572649574 + }, + "17": { + "pass@1": 0.18791246118012422, + "pass@2": 0.23750657186873375, + "pass@4": 0.2855899974847864, + "pass@8": 0.33262139177019856, + "pass@16": 0.37758867649195893, + "pass@32": 0.4197480671208388, + "pass@64": 0.45981387209971636, + "pass@128": 0.5031055900621118 + }, + "18": { + "pass@1": 0.1879595588235294, + "pass@2": 0.23974315365910143, + "pass@4": 0.28761232024305783, + "pass@8": 0.329751823564591, + "pass@16": 0.3682869367529674, + "pass@32": 0.40342406625742766, + "pass@64": 0.43378958147428764, + "pass@128": 0.46176470588235297 + }, + "19": { + "pass@1": 0.14397321428571427, + "pass@2": 0.19275703330052496, + "pass@4": 0.2422015247536021, + "pass@8": 0.2878507464902696, + "pass@16": 0.3280847004008977, + "pass@32": 0.36158603274779966, + "pass@64": 0.3892965040997099, + "pass@128": 0.41369047619047616 + }, + "20": { + "pass@1": 0.13793506736526945, + "pass@2": 0.18860249481352254, + "pass@4": 0.23572179403846963, + "pass@8": 0.2758505298038162, + "pass@16": 0.3104882346741073, + "pass@32": 0.341360691539217, + "pass@64": 0.36851288153677664, + "pass@128": 0.38922155688622756 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 524979, + "answer_accuracy": 0.655280146588912, + "avg_response_len": 275.1620503974277, + "resp_tokens_sum": 220446627, + "pass_at_k": { + "pass@1": 0.655280146588912, + "pass@2": 0.6839925507269552, + "pass@4": 0.7088387611228634, + "pass@8": 0.7307742552370572, + "pass@16": 0.7506730765481188, + "pass@32": 0.7687142269840314, + "pass@64": 0.784539857043786, + "pass@128": 0.7983703467007509 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9056203358208955, + "pass@2": 0.9243040457163008, + "pass@4": 0.9389661845440961, + "pass@8": 0.9507190979296448, + "pass@16": 0.9601808559838616, + "pass@32": 0.9673198046675844, + "pass@64": 0.9723514966419701, + "pass@128": 0.9761194029850746 + }, + "2": { + "pass@1": 0.9972991071428572, + "pass@2": 0.9974479752530934, + "pass@4": 0.9977247281589801, + "pass@8": 0.9982019248579823, + "pass@16": 0.9989049734161272, + "pass@32": 0.9996397297099158, + "pass@64": 0.9999812230788224, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994533527696793, + "pass@2": 0.9999397396753976, + "pass@4": 0.9999991596349289, + "pass@8": 0.9999999999107686, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9742417279411765, + "pass@2": 0.976435488073182, + "pass@4": 0.9785727789541012, + "pass@8": 0.9800637367521491, + "pass@16": 0.981034477001222, + "pass@32": 0.9818501519506909, + "pass@64": 0.982312283238623, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9912016955835962, + "pass@2": 0.9923161591196998, + "pass@4": 0.9931202484744611, + "pass@8": 0.9935946610212556, + "pass@16": 0.993688693168888, + "pass@32": 0.9936908514016762, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9939922863924051, + "pass@2": 0.9979045711651552, + "pass@4": 0.9996272230211731, + "pass@8": 0.9999848951262044, + "pass@16": 0.9999999797040798, + "pass@32": 0.999999999999996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9742164039039038, + "pass@2": 0.9776526033907924, + "pass@4": 0.9802813651671918, + "pass@8": 0.9826036723003114, + "pass@16": 0.9842749202268645, + "pass@32": 0.985456634186374, + "pass@64": 0.9864771895962706, + "pass@128": 0.987987987987988 + }, + "9": { + "pass@1": 0.972292001607717, + "pass@2": 0.9819812264222599, + "pass@4": 0.986615645311217, + "pass@8": 0.9891525246781142, + "pass@16": 0.9907615607123103, + "pass@32": 0.9917515996499101, + "pass@64": 0.9927715998185213, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.860094246031746, + "pass@2": 0.8852873859517562, + "pass@4": 0.9016740666345278, + "pass@8": 0.9141964920855181, + "pass@16": 0.9255256112996866, + "pass@32": 0.9359988921190586, + "pass@64": 0.9462906177764885, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.7074971330275229, + "pass@2": 0.7622297822003904, + "pass@4": 0.8011608589063977, + "pass@8": 0.8297119134538486, + "pass@16": 0.8534948504852095, + "pass@32": 0.8747256701701144, + "pass@64": 0.8922751671197711, + "pass@128": 0.9051987767584098 + }, + "13": { + "pass@1": 0.48215663580246915, + "pass@2": 0.5435611238699328, + "pass@4": 0.5926587695056635, + "pass@8": 0.633260128402649, + "pass@16": 0.6707687175863076, + "pass@32": 0.7085261614263532, + "pass@64": 0.7448136642431026, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.406134087537092, + "pass@2": 0.4712745186803429, + "pass@4": 0.5243155244503929, + "pass@8": 0.568281215623407, + "pass@16": 0.6078381209744614, + "pass@32": 0.6452114848141344, + "pass@64": 0.6794001748786187, + "pass@128": 0.7091988130563798 + }, + "15": { + "pass@1": 0.3033613040123457, + "pass@2": 0.35925956304073065, + "pass@4": 0.4130433117156653, + "pass@8": 0.46853172257542763, + "pass@16": 0.5238111534693787, + "pass@32": 0.5719111695487336, + "pass@64": 0.6073624482988279, + "pass@128": 0.6327160493827161 + }, + "16": { + "pass@1": 0.1922704646017699, + "pass@2": 0.23690821487468935, + "pass@4": 0.27880137604480854, + "pass@8": 0.318234271548925, + "pass@16": 0.35525716796412005, + "pass@32": 0.38901365189369735, + "pass@64": 0.4183437728038016, + "pass@128": 0.4424778761061947 + }, + "17": { + "pass@1": 0.18053463855421686, + "pass@2": 0.23325067000284597, + "pass@4": 0.2834787490795578, + "pass@8": 0.32846562326969203, + "pass@16": 0.369436527768843, + "pass@32": 0.40634737101135743, + "pass@64": 0.4379492763008395, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.16743951612903227, + "pass@2": 0.21504000508001017, + "pass@4": 0.26076773044498464, + "pass@8": 0.30550745623361136, + "pass@16": 0.34705035881725593, + "pass@32": 0.3828956929719597, + "pass@64": 0.4131763453082906, + "pass@128": 0.44193548387096776 + }, + "19": { + "pass@1": 0.14834308912386707, + "pass@2": 0.19732393486690303, + "pass@4": 0.2415079635060723, + "pass@8": 0.28059675189721844, + "pass@16": 0.31802633512102896, + "pass@32": 0.354738317640572, + "pass@64": 0.38914149256189534, + "pass@128": 0.4229607250755287 + }, + "20": { + "pass@1": 0.17424455705705705, + "pass@2": 0.22630923246080717, + "pass@4": 0.2799491279806238, + "pass@8": 0.32954584901807554, + "pass@16": 0.370233902011627, + "pass@32": 0.4052958996657454, + "pass@64": 0.43976604913749345, + "pass@128": 0.47147147147147145 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..470209700c8aef574e928be3f7ce25b494720a1d --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16", "results_dir": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/checkpoint-21808_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1000360, "value_mismatch": 31766, "dependency_mismatch": 145686, "answer_mismatch": 26231, "extra_nodes": 19409, "missing_nodes": 17027, "perfect_with_extra": 601, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 327588, "dependency_mismatch": 48688, "value_mismatch": 12427, "answer_mismatch": 10107, "extra_nodes": 7062, "perfect_with_extra": 150, "missing_nodes": 6481}, "teachers_in_school": {"total": 390656, "value_mismatch": 9389, "answer_mismatch": 8008, "dependency_mismatch": 48369, "perfect": 340741, "extra_nodes": 6152, "missing_nodes": 5575, "perfect_with_extra": 234}, "movie_festival_awards": {"total": 382336, "perfect": 332031, "dependency_mismatch": 48629, "extra_nodes": 6195, "missing_nodes": 4971, "value_mismatch": 9950, "answer_mismatch": 8116, "perfect_with_extra": 217}}, "results_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/checkpoint-21808_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/checkpoint-21808_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 161658, "value_mismatch": 968006, "dependency_mismatch": 1075266, "answer_mismatch": 822454, "extra_nodes": 48344, "missing_nodes": 739860, "perfect_with_extra": 2, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 326614, "dependency_mismatch": 364116, "answer_mismatch": 275586, "extra_nodes": 18162, "missing_nodes": 258539, "perfect": 53879}, "movie_festival_awards": {"total": 418816, "perfect": 54272, "dependency_mismatch": 350719, "value_mismatch": 314558, "answer_mismatch": 268058, "missing_nodes": 236866, "extra_nodes": 15095}, "teachers_in_school": {"total": 430464, "perfect": 53507, "dependency_mismatch": 360431, "value_mismatch": 326834, "answer_mismatch": 278810, "missing_nodes": 244455, "extra_nodes": 15087, "perfect_with_extra": 2}}, "results_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/checkpoint-21808_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1162018, "value_mismatch": 999772, "dependency_mismatch": 1220952, "answer_mismatch": 848685, "extra_nodes": 67753, "missing_nodes": 756887, "perfect_with_extra": 603, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 381467, "dependency_mismatch": 412804, "value_mismatch": 339041, "answer_mismatch": 285693, "extra_nodes": 25224, "perfect_with_extra": 150, "missing_nodes": 265020}, "teachers_in_school": {"total": 821120, "value_mismatch": 336223, "answer_mismatch": 286818, "dependency_mismatch": 408800, "perfect": 394248, "extra_nodes": 21239, "missing_nodes": 250030, "perfect_with_extra": 236}, "movie_festival_awards": {"total": 801152, "perfect": 386303, "dependency_mismatch": 399348, "extra_nodes": 21290, "missing_nodes": 241837, "value_mismatch": 324508, "answer_mismatch": 276174, "perfect_with_extra": 217}}}} diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/results.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..886355817d246f24fd8701964d3300947f8d3c3e --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21808_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 0.9969418960244648, + "pass@4": 0.9969418960244648, + "pass@8": 0.9969418960244648, + "pass@16": 0.9969418960244648, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9396825396825397, + "pass@2": 0.9619047619047619, + "pass@4": 0.9809523809523809, + "pass@8": 0.9841269841269841, + "pass@16": 0.9873015873015873, + "pass@32": 0.9873015873015873, + "pass@64": 0.9873015873015873, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.9341317365269461, + "pass@2": 0.9760479041916168, + "pass@4": 0.9790419161676647, + "pass@8": 0.9850299401197605, + "pass@16": 0.9850299401197605, + "pass@32": 0.9910179640718563, + "pass@64": 0.9910179640718563, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.849112426035503, + "pass@2": 0.9142011834319527, + "pass@4": 0.9526627218934911, + "pass@8": 0.9704142011834319, + "pass@16": 0.9822485207100592, + "pass@32": 0.9822485207100592, + "pass@64": 0.9881656804733728, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.7467948717948718, + "pass@2": 0.8461538461538461, + "pass@4": 0.9134615384615384, + "pass@8": 0.9647435897435898, + "pass@16": 0.9903846153846154, + "pass@32": 0.9903846153846154, + "pass@64": 0.9935897435897436, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.6732394366197183, + "pass@2": 0.7774647887323943, + "pass@4": 0.8676056338028169, + "pass@8": 0.9464788732394366, + "pass@16": 0.971830985915493, + "pass@32": 0.9859154929577465, + "pass@64": 0.9943661971830986, + "pass@128": 0.9943661971830986 + }, + "10": { + "pass@1": 0.6371681415929203, + "pass@2": 0.6932153392330384, + "pass@4": 0.7610619469026548, + "pass@8": 0.831858407079646, + "pass@16": 0.887905604719764, + "pass@32": 0.8967551622418879, + "pass@64": 0.8997050147492626, + "pass@128": 0.9085545722713865 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 1.0, + "5": 0.9396825396825397, + "6": 0.9341317365269461, + "7": 0.849112426035503, + "8": 0.7467948717948718, + "9": 0.6732394366197183, + "10": 0.6371681415929203 + }, + "pass@2": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 1.0, + "5": 0.9619047619047619, + "6": 0.9760479041916168, + "7": 0.9142011834319527, + "8": 0.8461538461538461, + "9": 0.7774647887323943, + "10": 0.6932153392330384 + }, + "pass@4": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 1.0, + "5": 0.9809523809523809, + "6": 0.9790419161676647, + "7": 0.9526627218934911, + "8": 0.9134615384615384, + "9": 0.8676056338028169, + "10": 0.7610619469026548 + }, + "pass@8": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 1.0, + "5": 0.9841269841269841, + "6": 0.9850299401197605, + "7": 0.9704142011834319, + "8": 0.9647435897435898, + "9": 0.9464788732394366, + "10": 0.831858407079646 + }, + "pass@16": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9850299401197605, + "7": 0.9822485207100592, + "8": 0.9903846153846154, + "9": 0.971830985915493, + "10": 0.887905604719764 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9910179640718563, + "7": 0.9822485207100592, + "8": 0.9903846153846154, + "9": 0.9859154929577465, + "10": 0.8967551622418879 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9910179640718563, + "7": 0.9881656804733728, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.8997050147492626 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9910179640718563, + "7": 0.9911242603550295, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9085545722713865 + } + }, + "total_pass_at_k": { + "pass@1": 0.8611955420466059, + "pass@2": 0.9050996285038838, + "pass@4": 0.9375211077338736, + "pass@8": 0.9635258358662614, + "pass@16": 0.9773725092874029, + "pass@32": 0.9810874704491725, + "pass@64": 0.983451536643026, + "pass@128": 0.9848024316109423 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9971428571428571, + "pass@2": 0.9971428571428571, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9470588235294117, + "pass@2": 0.9558823529411765, + "pass@4": 0.9647058823529412, + "pass@8": 0.9705882352941176, + "pass@16": 0.9735294117647059, + "pass@32": 0.9764705882352941, + "pass@64": 0.9764705882352941, + "pass@128": 0.9764705882352941 + }, + "6": { + "pass@1": 0.9652996845425867, + "pass@2": 0.9747634069400631, + "pass@4": 0.9747634069400631, + "pass@8": 0.9842271293375394, + "pass@16": 0.9905362776025236, + "pass@32": 0.9905362776025236, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.8607594936708861, + "pass@2": 0.8829113924050633, + "pass@4": 0.9145569620253164, + "pass@8": 0.9620253164556962, + "pass@16": 0.990506329113924, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.7627627627627628, + "pass@2": 0.8438438438438438, + "pass@4": 0.8858858858858859, + "pass@8": 0.933933933933934, + "pass@16": 0.954954954954955, + "pass@32": 0.96996996996997, + "pass@64": 0.96996996996997, + "pass@128": 0.972972972972973 + }, + "9": { + "pass@1": 0.729903536977492, + "pass@2": 0.8231511254019293, + "pass@4": 0.8906752411575563, + "pass@8": 0.9485530546623794, + "pass@16": 0.9678456591639871, + "pass@32": 0.9903536977491961, + "pass@64": 0.9903536977491961, + "pass@128": 0.9903536977491961 + }, + "10": { + "pass@1": 0.5940298507462687, + "pass@2": 0.6626865671641791, + "pass@4": 0.746268656716418, + "pass@8": 0.7970149253731343, + "pass@16": 0.8388059701492537, + "pass@32": 0.8805970149253731, + "pass@64": 0.8985074626865671, + "pass@128": 0.9044776119402985 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.9470588235294117, + "6": 0.9652996845425867, + "7": 0.8607594936708861, + "8": 0.7627627627627628, + "9": 0.729903536977492, + "10": 0.5940298507462687 + }, + "pass@2": { + "2": 0.9971428571428571, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9558823529411765, + "6": 0.9747634069400631, + "7": 0.8829113924050633, + "8": 0.8438438438438438, + "9": 0.8231511254019293, + "10": 0.6626865671641791 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9647058823529412, + "6": 0.9747634069400631, + "7": 0.9145569620253164, + "8": 0.8858858858858859, + "9": 0.8906752411575563, + "10": 0.746268656716418 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9705882352941176, + "6": 0.9842271293375394, + "7": 0.9620253164556962, + "8": 0.933933933933934, + "9": 0.9485530546623794, + "10": 0.7970149253731343 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9735294117647059, + "6": 0.9905362776025236, + "7": 0.990506329113924, + "8": 0.954954954954955, + "9": 0.9678456591639871, + "10": 0.8388059701492537 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9764705882352941, + "6": 0.9905362776025236, + "7": 0.9936708860759493, + "8": 0.96996996996997, + "9": 0.9903536977491961, + "10": 0.8805970149253731 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9764705882352941, + "6": 0.9936908517350158, + "7": 0.9936708860759493, + "8": 0.96996996996997, + "9": 0.9903536977491961, + "10": 0.8985074626865671 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9764705882352941, + "6": 0.9936908517350158, + "7": 0.9936708860759493, + "8": 0.972972972972973, + "9": 0.9903536977491961, + "10": 0.9044776119402985 + } + }, + "total_pass_at_k": { + "pass@1": 0.8737864077669902, + "pass@2": 0.905256109809173, + "pass@4": 0.9317040508871778, + "pass@8": 0.9554737194509542, + "pass@16": 0.9685302979578172, + "pass@32": 0.9779042517576163, + "pass@64": 0.9802477402075661, + "pass@128": 0.9812520924004018 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.9940828402366864, + "pass@4": 0.9970414201183432, + "pass@8": 0.9970414201183432, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9449275362318841, + "pass@2": 0.9565217391304348, + "pass@4": 0.9739130434782609, + "pass@8": 0.9826086956521739, + "pass@16": 0.9826086956521739, + "pass@32": 0.9855072463768116, + "pass@64": 0.9855072463768116, + "pass@128": 0.9855072463768116 + }, + "6": { + "pass@1": 0.9684813753581661, + "pass@2": 0.9799426934097422, + "pass@4": 0.9856733524355301, + "pass@8": 0.994269340974212, + "pass@16": 0.994269340974212, + "pass@32": 0.994269340974212, + "pass@64": 0.994269340974212, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.9017341040462428, + "pass@4": 0.930635838150289, + "pass@8": 0.9739884393063584, + "pass@16": 0.9913294797687862, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.7605633802816901, + "pass@2": 0.8507042253521127, + "pass@4": 0.9211267605633803, + "pass@8": 0.9633802816901409, + "pass@16": 0.9887323943661972, + "pass@32": 0.9915492957746479, + "pass@64": 0.9943661971830986, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.688622754491018, + "pass@2": 0.7784431137724551, + "pass@4": 0.8592814371257484, + "pass@8": 0.9041916167664671, + "pass@16": 0.9401197604790419, + "pass@32": 0.9550898203592815, + "pass@64": 0.9640718562874252, + "pass@128": 0.9730538922155688 + }, + "10": { + "pass@1": 0.6012269938650306, + "pass@2": 0.6533742331288344, + "pass@4": 0.7239263803680982, + "pass@8": 0.7944785276073619, + "pass@16": 0.843558282208589, + "pass@32": 0.8803680981595092, + "pass@64": 0.901840490797546, + "pass@128": 0.9049079754601227 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9449275362318841, + "6": 0.9684813753581661, + "7": 0.8641618497109826, + "8": 0.7605633802816901, + "9": 0.688622754491018, + "10": 0.6012269938650306 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9565217391304348, + "6": 0.9799426934097422, + "7": 0.9017341040462428, + "8": 0.8507042253521127, + "9": 0.7784431137724551, + "10": 0.6533742331288344 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9739130434782609, + "6": 0.9856733524355301, + "7": 0.930635838150289, + "8": 0.9211267605633803, + "9": 0.8592814371257484, + "10": 0.7239263803680982 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9826086956521739, + "6": 0.994269340974212, + "7": 0.9739884393063584, + "8": 0.9633802816901409, + "9": 0.9041916167664671, + "10": 0.7944785276073619 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9826086956521739, + "6": 0.994269340974212, + "7": 0.9913294797687862, + "8": 0.9887323943661972, + "9": 0.9401197604790419, + "10": 0.843558282208589 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9855072463768116, + "6": 0.994269340974212, + "7": 0.9971098265895953, + "8": 0.9915492957746479, + "9": 0.9550898203592815, + "10": 0.8803680981595092 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9855072463768116, + "6": 0.994269340974212, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9640718562874252, + "10": 0.901840490797546 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9855072463768116, + "6": 0.994269340974212, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9730538922155688, + "10": 0.9049079754601227 + } + }, + "total_pass_at_k": { + "pass@1": 0.8682830930537353, + "pass@2": 0.9023591087811271, + "pass@4": 0.9331585845347313, + "pass@8": 0.9574049803407602, + "pass@16": 0.971821756225426, + "pass@32": 0.97870249017038, + "pass@64": 0.9823066841415465, + "pass@128": 0.9836173001310616 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99, + "pass@2": 0.996, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.944, + "pass@2": 0.958, + "pass@4": 0.973, + "pass@8": 0.979, + "pass@16": 0.981, + "pass@32": 0.983, + "pass@64": 0.983, + "pass@128": 0.983 + }, + "6": { + "pass@1": 0.956, + "pass@2": 0.977, + "pass@4": 0.98, + "pass@8": 0.988, + "pass@16": 0.99, + "pass@32": 0.992, + "pass@64": 0.993, + "pass@128": 0.993 + }, + "7": { + "pass@1": 0.858, + "pass@2": 0.9, + "pass@4": 0.933, + "pass@8": 0.969, + "pass@16": 0.988, + "pass@32": 0.991, + "pass@64": 0.993, + "pass@128": 0.994 + }, + "8": { + "pass@1": 0.757, + "pass@2": 0.847, + "pass@4": 0.907, + "pass@8": 0.954, + "pass@16": 0.978, + "pass@32": 0.984, + "pass@64": 0.986, + "pass@128": 0.987 + }, + "9": { + "pass@1": 0.696, + "pass@2": 0.792, + "pass@4": 0.872, + "pass@8": 0.933, + "pass@16": 0.96, + "pass@32": 0.977, + "pass@64": 0.983, + "pass@128": 0.986 + }, + "10": { + "pass@1": 0.611, + "pass@2": 0.67, + "pass@4": 0.744, + "pass@8": 0.808, + "pass@16": 0.857, + "pass@32": 0.886, + "pass@64": 0.9, + "pass@128": 0.906 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.999, + "4": 0.99, + "5": 0.944, + "6": 0.956, + "7": 0.858, + "8": 0.757, + "9": 0.696, + "10": 0.611 + }, + "pass@2": { + "2": 0.999, + "3": 0.999, + "4": 0.996, + "5": 0.958, + "6": 0.977, + "7": 0.9, + "8": 0.847, + "9": 0.792, + "10": 0.67 + }, + "pass@4": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.973, + "6": 0.98, + "7": 0.933, + "8": 0.907, + "9": 0.872, + "10": 0.744 + }, + "pass@8": { + "2": 1.0, + "3": 0.999, + "4": 0.999, + "5": 0.979, + "6": 0.988, + "7": 0.969, + "8": 0.954, + "9": 0.933, + "10": 0.808 + }, + "pass@16": { + "2": 1.0, + "3": 0.999, + "4": 1.0, + "5": 0.981, + "6": 0.99, + "7": 0.988, + "8": 0.978, + "9": 0.96, + "10": 0.857 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.983, + "6": 0.992, + "7": 0.991, + "8": 0.984, + "9": 0.977, + "10": 0.886 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.983, + "6": 0.993, + "7": 0.993, + "8": 0.986, + "9": 0.983, + "10": 0.9 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.983, + "6": 0.993, + "7": 0.994, + "8": 0.987, + "9": 0.986, + "10": 0.906 + } + }, + "total_pass_at_k": { + "pass@1": 0.8677777777777778, + "pass@2": 0.9042222222222223, + "pass@4": 0.9341111111111111, + "pass@8": 0.9587777777777777, + "pass@16": 0.9725555555555555, + "pass@32": 0.9792222222222222, + "pass@64": 0.982, + "pass@128": 0.9832222222222222 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21808_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.45042492917847027, + "pass@2": 0.5439093484419264, + "pass@4": 0.6572237960339944, + "pass@8": 0.7252124645892352, + "pass@16": 0.7790368271954674, + "pass@32": 0.8328611898016998, + "pass@64": 0.8583569405099151, + "pass@128": 0.8668555240793201 + }, + "12": { + "pass@1": 0.4350282485875706, + "pass@2": 0.5028248587570622, + "pass@4": 0.5677966101694916, + "pass@8": 0.6186440677966102, + "pass@16": 0.6666666666666666, + "pass@32": 0.7033898305084746, + "pass@64": 0.7542372881355932, + "pass@128": 0.7740112994350282 + }, + "13": { + "pass@1": 0.1724137931034483, + "pass@2": 0.23197492163009403, + "pass@4": 0.26959247648902823, + "pass@8": 0.3166144200626959, + "pass@16": 0.3510971786833856, + "pass@32": 0.4106583072100313, + "pass@64": 0.45141065830721006, + "pass@128": 0.4670846394984326 + }, + "14": { + "pass@1": 0.09034267912772585, + "pass@2": 0.11214953271028037, + "pass@4": 0.1277258566978193, + "pass@8": 0.1557632398753894, + "pass@16": 0.17133956386292834, + "pass@32": 0.21495327102803738, + "pass@64": 0.2585669781931464, + "pass@128": 0.29283489096573206 + }, + "15": { + "pass@1": 0.049132947976878616, + "pass@2": 0.049132947976878616, + "pass@4": 0.07225433526011561, + "pass@8": 0.0838150289017341, + "pass@16": 0.10404624277456648, + "pass@32": 0.11849710982658959, + "pass@64": 0.13872832369942195, + "pass@128": 0.15895953757225434 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0032258064516129032, + "pass@32": 0.0064516129032258064, + "pass@64": 0.00967741935483871, + "pass@128": 0.00967741935483871 + }, + "17": { + "pass@1": 0.005780346820809248, + "pass@2": 0.005780346820809248, + "pass@4": 0.005780346820809248, + "pass@8": 0.005780346820809248, + "pass@16": 0.008670520231213872, + "pass@32": 0.011560693641618497, + "pass@64": 0.014450867052023121, + "pass@128": 0.017341040462427744 + }, + "18": { + "pass@1": 0.002857142857142857, + "pass@2": 0.002857142857142857, + "pass@4": 0.002857142857142857, + "pass@8": 0.002857142857142857, + "pass@16": 0.002857142857142857, + "pass@32": 0.002857142857142857, + "pass@64": 0.002857142857142857, + "pass@128": 0.002857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.45042492917847027, + "12": 0.4350282485875706, + "13": 0.1724137931034483, + "14": 0.09034267912772585, + "15": 0.049132947976878616, + "16": 0.0, + "17": 0.005780346820809248, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5439093484419264, + "12": 0.5028248587570622, + "13": 0.23197492163009403, + "14": 0.11214953271028037, + "15": 0.049132947976878616, + "16": 0.0, + "17": 0.005780346820809248, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6572237960339944, + "12": 0.5677966101694916, + "13": 0.26959247648902823, + "14": 0.1277258566978193, + "15": 0.07225433526011561, + "16": 0.0, + "17": 0.005780346820809248, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7252124645892352, + "12": 0.6186440677966102, + "13": 0.3166144200626959, + "14": 0.1557632398753894, + "15": 0.0838150289017341, + "16": 0.0, + "17": 0.005780346820809248, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.7790368271954674, + "12": 0.6666666666666666, + "13": 0.3510971786833856, + "14": 0.17133956386292834, + "15": 0.10404624277456648, + "16": 0.0032258064516129032, + "17": 0.008670520231213872, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8328611898016998, + "12": 0.7033898305084746, + "13": 0.4106583072100313, + "14": 0.21495327102803738, + "15": 0.11849710982658959, + "16": 0.0064516129032258064, + "17": 0.011560693641618497, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8583569405099151, + "12": 0.7542372881355932, + "13": 0.45141065830721006, + "14": 0.2585669781931464, + "15": 0.13872832369942195, + "16": 0.00967741935483871, + "17": 0.014450867052023121, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8668555240793201, + "12": 0.7740112994350282, + "13": 0.4670846394984326, + "14": 0.29283489096573206, + "15": 0.15895953757225434, + "16": 0.00967741935483871, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.12392273402674592, + "pass@2": 0.1485884101040119, + "pass@4": 0.17473997028231797, + "pass@8": 0.19554234769687964, + "pass@16": 0.21367013372956908, + "pass@32": 0.2350668647845468, + "pass@64": 0.2537890044576523, + "pass@128": 0.2638930163447251 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5206349206349207, + "pass@2": 0.5873015873015873, + "pass@4": 0.6698412698412698, + "pass@8": 0.7555555555555555, + "pass@16": 0.8222222222222222, + "pass@32": 0.8571428571428571, + "pass@64": 0.8825396825396825, + "pass@128": 0.9015873015873016 + }, + "12": { + "pass@1": 0.4036697247706422, + "pass@2": 0.4709480122324159, + "pass@4": 0.5259938837920489, + "pass@8": 0.5840978593272171, + "pass@16": 0.6391437308868502, + "pass@32": 0.7033639143730887, + "pass@64": 0.7247706422018348, + "pass@128": 0.7431192660550459 + }, + "13": { + "pass@1": 0.21296296296296297, + "pass@2": 0.2654320987654321, + "pass@4": 0.2962962962962963, + "pass@8": 0.32407407407407407, + "pass@16": 0.35802469135802467, + "pass@32": 0.4074074074074074, + "pass@64": 0.44753086419753085, + "pass@128": 0.47530864197530864 + }, + "14": { + "pass@1": 0.10682492581602374, + "pass@2": 0.11869436201780416, + "pass@4": 0.1513353115727003, + "pass@8": 0.17507418397626112, + "pass@16": 0.19584569732937684, + "pass@32": 0.2344213649851632, + "pass@64": 0.27596439169139464, + "pass@128": 0.3323442136498516 + }, + "15": { + "pass@1": 0.08333333333333333, + "pass@2": 0.10185185185185185, + "pass@4": 0.12037037037037036, + "pass@8": 0.1419753086419753, + "pass@16": 0.15432098765432098, + "pass@32": 0.1728395061728395, + "pass@64": 0.2037037037037037, + "pass@128": 0.2345679012345679 + }, + "16": { + "pass@1": 0.008849557522123894, + "pass@2": 0.011799410029498525, + "pass@4": 0.02064896755162242, + "pass@8": 0.02064896755162242, + "pass@16": 0.02654867256637168, + "pass@32": 0.029498525073746312, + "pass@64": 0.032448377581120944, + "pass@128": 0.04424778761061947 + }, + "17": { + "pass@1": 0.012048192771084338, + "pass@2": 0.012048192771084338, + "pass@4": 0.024096385542168676, + "pass@8": 0.024096385542168676, + "pass@16": 0.02710843373493976, + "pass@32": 0.030120481927710843, + "pass@64": 0.03313253012048193, + "pass@128": 0.03313253012048193 + }, + "18": { + "pass@1": 0.0032258064516129032, + "pass@2": 0.0064516129032258064, + "pass@4": 0.0064516129032258064, + "pass@8": 0.0064516129032258064, + "pass@16": 0.0064516129032258064, + "pass@32": 0.012903225806451613, + "pass@64": 0.016129032258064516, + "pass@128": 0.01935483870967742 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5206349206349207, + "12": 0.4036697247706422, + "13": 0.21296296296296297, + "14": 0.10682492581602374, + "15": 0.08333333333333333, + "16": 0.008849557522123894, + "17": 0.012048192771084338, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5873015873015873, + "12": 0.4709480122324159, + "13": 0.2654320987654321, + "14": 0.11869436201780416, + "15": 0.10185185185185185, + "16": 0.011799410029498525, + "17": 0.012048192771084338, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6698412698412698, + "12": 0.5259938837920489, + "13": 0.2962962962962963, + "14": 0.1513353115727003, + "15": 0.12037037037037036, + "16": 0.02064896755162242, + "17": 0.024096385542168676, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7555555555555555, + "12": 0.5840978593272171, + "13": 0.32407407407407407, + "14": 0.17507418397626112, + "15": 0.1419753086419753, + "16": 0.02064896755162242, + "17": 0.024096385542168676, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8222222222222222, + "12": 0.6391437308868502, + "13": 0.35802469135802467, + "14": 0.19584569732937684, + "15": 0.15432098765432098, + "16": 0.02654867256637168, + "17": 0.02710843373493976, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8571428571428571, + "12": 0.7033639143730887, + "13": 0.4074074074074074, + "14": 0.2344213649851632, + "15": 0.1728395061728395, + "16": 0.029498525073746312, + "17": 0.030120481927710843, + "18": 0.012903225806451613, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8825396825396825, + "12": 0.7247706422018348, + "13": 0.44753086419753085, + "14": 0.27596439169139464, + "15": 0.2037037037037037, + "16": 0.032448377581120944, + "17": 0.03313253012048193, + "18": 0.016129032258064516, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.9015873015873016, + "12": 0.7431192660550459, + "13": 0.47530864197530864, + "14": 0.3323442136498516, + "15": 0.2345679012345679, + "16": 0.04424778761061947, + "17": 0.03313253012048193, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.1332518337408313, + "pass@2": 0.1552567237163814, + "pass@4": 0.1790953545232274, + "pass@8": 0.20048899755501223, + "pass@16": 0.2200488997555012, + "pass@32": 0.2417481662591687, + "pass@64": 0.25855745721271395, + "pass@128": 0.27536674816625917 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5090361445783133, + "pass@2": 0.6054216867469879, + "pass@4": 0.6897590361445783, + "pass@8": 0.7620481927710844, + "pass@16": 0.8162650602409639, + "pass@32": 0.8554216867469879, + "pass@64": 0.8734939759036144, + "pass@128": 0.8885542168674698 + }, + "12": { + "pass@1": 0.3824451410658307, + "pass@2": 0.4482758620689655, + "pass@4": 0.49843260188087773, + "pass@8": 0.554858934169279, + "pass@16": 0.6394984326018809, + "pass@32": 0.677115987460815, + "pass@64": 0.7115987460815048, + "pass@128": 0.7366771159874608 + }, + "13": { + "pass@1": 0.17366946778711484, + "pass@2": 0.23529411764705882, + "pass@4": 0.28291316526610644, + "pass@8": 0.3389355742296919, + "pass@16": 0.37815126050420167, + "pass@32": 0.42016806722689076, + "pass@64": 0.4369747899159664, + "pass@128": 0.46218487394957986 + }, + "14": { + "pass@1": 0.13157894736842105, + "pass@2": 0.14912280701754385, + "pass@4": 0.17251461988304093, + "pass@8": 0.19298245614035087, + "pass@16": 0.2222222222222222, + "pass@32": 0.2719298245614035, + "pass@64": 0.30701754385964913, + "pass@128": 0.3333333333333333 + }, + "15": { + "pass@1": 0.03939393939393939, + "pass@2": 0.05757575757575758, + "pass@4": 0.07575757575757576, + "pass@8": 0.08484848484848485, + "pass@16": 0.08787878787878788, + "pass@32": 0.11212121212121212, + "pass@64": 0.1303030303030303, + "pass@128": 0.1606060606060606 + }, + "16": { + "pass@1": 0.008547008547008548, + "pass@2": 0.014245014245014245, + "pass@4": 0.022792022792022793, + "pass@8": 0.02564102564102564, + "pass@16": 0.02849002849002849, + "pass@32": 0.02849002849002849, + "pass@64": 0.03418803418803419, + "pass@128": 0.037037037037037035 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.009316770186335404, + "pass@8": 0.009316770186335404, + "pass@16": 0.009316770186335404, + "pass@32": 0.012422360248447204, + "pass@64": 0.015527950310559006, + "pass@128": 0.015527950310559006 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0029411764705882353, + "pass@8": 0.0029411764705882353, + "pass@16": 0.0029411764705882353, + "pass@32": 0.0058823529411764705, + "pass@64": 0.0058823529411764705, + "pass@128": 0.008823529411764706 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0029940119760479044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5090361445783133, + "12": 0.3824451410658307, + "13": 0.17366946778711484, + "14": 0.13157894736842105, + "15": 0.03939393939393939, + "16": 0.008547008547008548, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6054216867469879, + "12": 0.4482758620689655, + "13": 0.23529411764705882, + "14": 0.14912280701754385, + "15": 0.05757575757575758, + "16": 0.014245014245014245, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6897590361445783, + "12": 0.49843260188087773, + "13": 0.28291316526610644, + "14": 0.17251461988304093, + "15": 0.07575757575757576, + "16": 0.022792022792022793, + "17": 0.009316770186335404, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7620481927710844, + "12": 0.554858934169279, + "13": 0.3389355742296919, + "14": 0.19298245614035087, + "15": 0.08484848484848485, + "16": 0.02564102564102564, + "17": 0.009316770186335404, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8162650602409639, + "12": 0.6394984326018809, + "13": 0.37815126050420167, + "14": 0.2222222222222222, + "15": 0.08787878787878788, + "16": 0.02849002849002849, + "17": 0.009316770186335404, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8554216867469879, + "12": 0.677115987460815, + "13": 0.42016806722689076, + "14": 0.2719298245614035, + "15": 0.11212121212121212, + "16": 0.02849002849002849, + "17": 0.012422360248447204, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8734939759036144, + "12": 0.7115987460815048, + "13": 0.4369747899159664, + "14": 0.30701754385964913, + "15": 0.1303030303030303, + "16": 0.03418803418803419, + "17": 0.015527950310559006, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8885542168674698, + "12": 0.7366771159874608, + "13": 0.46218487394957986, + "14": 0.3333333333333333, + "15": 0.1606060606060606, + "16": 0.037037037037037035, + "17": 0.015527950310559006, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0029940119760479044 + } + }, + "total_pass_at_k": { + "pass@1": 0.12310437109723461, + "pass@2": 0.14956883734760631, + "pass@4": 0.17395182872435325, + "pass@8": 0.19565863812072554, + "pass@16": 0.2167707404103479, + "pass@32": 0.2366934284864704, + "pass@64": 0.24977698483496877, + "pass@128": 0.26286054118346713 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.492, + "pass@2": 0.578, + "pass@4": 0.672, + "pass@8": 0.747, + "pass@16": 0.805, + "pass@32": 0.848, + "pass@64": 0.871, + "pass@128": 0.885 + }, + "12": { + "pass@1": 0.408, + "pass@2": 0.475, + "pass@4": 0.532, + "pass@8": 0.587, + "pass@16": 0.649, + "pass@32": 0.695, + "pass@64": 0.731, + "pass@128": 0.752 + }, + "13": { + "pass@1": 0.186, + "pass@2": 0.244, + "pass@4": 0.283, + "pass@8": 0.327, + "pass@16": 0.363, + "pass@32": 0.413, + "pass@64": 0.445, + "pass@128": 0.468 + }, + "14": { + "pass@1": 0.11, + "pass@2": 0.127, + "pass@4": 0.151, + "pass@8": 0.175, + "pass@16": 0.197, + "pass@32": 0.241, + "pass@64": 0.281, + "pass@128": 0.32 + }, + "15": { + "pass@1": 0.057, + "pass@2": 0.069, + "pass@4": 0.089, + "pass@8": 0.103, + "pass@16": 0.115, + "pass@32": 0.134, + "pass@64": 0.157, + "pass@128": 0.184 + }, + "16": { + "pass@1": 0.006, + "pass@2": 0.009, + "pass@4": 0.015, + "pass@8": 0.016, + "pass@16": 0.02, + "pass@32": 0.022, + "pass@64": 0.026, + "pass@128": 0.031 + }, + "17": { + "pass@1": 0.006, + "pass@2": 0.006, + "pass@4": 0.013, + "pass@8": 0.013, + "pass@16": 0.015, + "pass@32": 0.018, + "pass@64": 0.021, + "pass@128": 0.022 + }, + "18": { + "pass@1": 0.002, + "pass@2": 0.003, + "pass@4": 0.004, + "pass@8": 0.004, + "pass@16": 0.004, + "pass@32": 0.007, + "pass@64": 0.008, + "pass@128": 0.01 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.001 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.492, + "12": 0.408, + "13": 0.186, + "14": 0.11, + "15": 0.057, + "16": 0.006, + "17": 0.006, + "18": 0.002, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.578, + "12": 0.475, + "13": 0.244, + "14": 0.127, + "15": 0.069, + "16": 0.009, + "17": 0.006, + "18": 0.003, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.672, + "12": 0.532, + "13": 0.283, + "14": 0.151, + "15": 0.089, + "16": 0.015, + "17": 0.013, + "18": 0.004, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.747, + "12": 0.587, + "13": 0.327, + "14": 0.175, + "15": 0.103, + "16": 0.016, + "17": 0.013, + "18": 0.004, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.805, + "12": 0.649, + "13": 0.363, + "14": 0.197, + "15": 0.115, + "16": 0.02, + "17": 0.015, + "18": 0.004, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.848, + "12": 0.695, + "13": 0.413, + "14": 0.241, + "15": 0.134, + "16": 0.022, + "17": 0.018, + "18": 0.007, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.871, + "12": 0.731, + "13": 0.445, + "14": 0.281, + "15": 0.157, + "16": 0.026, + "17": 0.021, + "18": 0.008, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.885, + "12": 0.752, + "13": 0.468, + "14": 0.32, + "15": 0.184, + "16": 0.031, + "17": 0.022, + "18": 0.01, + "19": 0.0, + "20": 0.001 + } + }, + "total_pass_at_k": { + "pass@1": 0.1267, + "pass@2": 0.1511, + "pass@4": 0.1759, + "pass@8": 0.1972, + "pass@16": 0.2168, + "pass@32": 0.2378, + "pass@64": 0.254, + "pass@128": 0.2673 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/summary.csv b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..d036598c4b30da5b5078749903513d1c9d68ef21 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21808,0.9772300347222223,0.35745859375,0.6510345394736842,0.0,0.0,0.0,196.0645373263889,315.944015625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/summary.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f38c04a2f47296d62ed0599f13b344ed393027d3 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block16/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21808", + "id_acc": 0.9772300347222223, + "ood_acc": 0.35745859375, + "total_acc": 0.6510345394736842, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.0645373263889, + "ood_avg_resp_len": 315.944015625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 524035, + "answer_accuracy": 0.6471741127884919, + "avg_response_len": 246.5130895806987, + "resp_tokens_sum": 199608551, + "pass_at_k": { + "pass@1": 0.6471741127884919, + "pass@2": 0.6797406917707471, + "pass@4": 0.7070043655574902, + "pass@8": 0.7296928552659782, + "pass@16": 0.7485988840793822, + "pass@32": 0.7647882080256212, + "pass@64": 0.7790428565048858, + "pass@128": 0.7922858046158711 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9000046091445427, + "pass@2": 0.9150345939888044, + "pass@4": 0.9236329891838736, + "pass@8": 0.9308381201660656, + "pass@16": 0.9381075792957397, + "pass@32": 0.9462885722585335, + "pass@64": 0.9541117889729607, + "pass@128": 0.9587020648967551 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969896788990825, + "pass@2": 0.9970370855305931, + "pass@4": 0.9971307700642925, + "pass@8": 0.9973136242144044, + "pass@16": 0.9976612728454813, + "pass@32": 0.9982843314310482, + "pass@64": 0.9992414938958318, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9976806640625, + "pass@2": 0.9994152159202757, + "pass@4": 0.9999608991258906, + "pass@8": 0.9999998130536076, + "pass@16": 0.9999999999978858, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9746279761904761, + "pass@2": 0.9800646012998375, + "pass@4": 0.9848414263693228, + "pass@8": 0.9876718146803775, + "pass@16": 0.9888463646668386, + "pass@32": 0.9897427177507346, + "pass@64": 0.9903847019122605, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9837434505988024, + "pass@2": 0.9885827508369086, + "pass@4": 0.9905320604572633, + "pass@8": 0.992091198748378, + "pass@16": 0.9934451035275379, + "pass@32": 0.993975561223535, + "pass@64": 0.9940119536626435, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9717085798816568, + "pass@2": 0.9788356706890929, + "pass@4": 0.9833554588960404, + "pass@8": 0.9872629105885962, + "pass@16": 0.990595545939145, + "pass@32": 0.9934039163490062, + "pass@64": 0.9958611615406721, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9722055288461539, + "pass@2": 0.9869693083737128, + "pass@4": 0.993037607979291, + "pass@8": 0.9952941196678669, + "pass@16": 0.996277513394104, + "pass@32": 0.9967337642373502, + "pass@64": 0.9967946729122014, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9670994718309859, + "pass@2": 0.9814191249861375, + "pass@4": 0.9867852194532019, + "pass@8": 0.9891142645184077, + "pass@16": 0.9906925954424748, + "pass@32": 0.9923033625045726, + "pass@64": 0.99428501700158, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8452992209631728, + "pass@2": 0.874468838526912, + "pass@4": 0.8927729621615995, + "pass@8": 0.9068203029145412, + "pass@16": 0.9184389985389775, + "pass@32": 0.9286397978946337, + "pass@64": 0.9370345366531673, + "pass@128": 0.9405099150141643 + }, + "12": { + "pass@1": 0.718794138418079, + "pass@2": 0.7764510765603452, + "pass@4": 0.8174984948915283, + "pass@8": 0.84820170368261, + "pass@16": 0.8711515105018326, + "pass@32": 0.8890325500632591, + "pass@64": 0.9038298217779518, + "pass@128": 0.9152542372881356 + }, + "13": { + "pass@1": 0.4823912617554859, + "pass@2": 0.5498393262656431, + "pass@4": 0.6036611143630555, + "pass@8": 0.6477879485187265, + "pass@16": 0.685484891823752, + "pass@32": 0.7189389553919789, + "pass@64": 0.7495642145009295, + "pass@128": 0.780564263322884 + }, + "14": { + "pass@1": 0.38770443925233644, + "pass@2": 0.44748279858709245, + "pass@4": 0.49312651893980536, + "pass@8": 0.5296382419814589, + "pass@16": 0.5612308622050344, + "pass@32": 0.5903160848446066, + "pass@64": 0.619181232589094, + "pass@128": 0.6542056074766355 + }, + "15": { + "pass@1": 0.2938041907514451, + "pass@2": 0.35669355343407216, + "pass@4": 0.4148084477157117, + "pass@8": 0.4650869029400088, + "pass@16": 0.5067125568176679, + "pass@32": 0.5417496851194056, + "pass@64": 0.5726888869572655, + "pass@128": 0.6011560693641619 + }, + "16": { + "pass@1": 0.18674395161290322, + "pass@2": 0.23528781432562876, + "pass@4": 0.2805795692877099, + "pass@8": 0.3201898414598259, + "pass@16": 0.3533900521458434, + "pass@32": 0.38255860221232757, + "pass@64": 0.41028016987609034, + "pass@128": 0.43548387096774194 + }, + "17": { + "pass@1": 0.18230671965317918, + "pass@2": 0.23375557553138218, + "pass@4": 0.2822599248793323, + "pass@8": 0.32672688937217353, + "pass@16": 0.36842829528522153, + "pass@32": 0.4058567655828945, + "pass@64": 0.4349941917269913, + "pass@128": 0.45664739884393063 + }, + "18": { + "pass@1": 0.1603125, + "pass@2": 0.21394474128233967, + "pass@4": 0.265273910493331, + "pass@8": 0.3092573071038731, + "pass@16": 0.3474877935238888, + "pass@32": 0.3828370961500478, + "pass@64": 0.4140552777998218, + "pass@128": 0.44285714285714284 + }, + "19": { + "pass@1": 0.15153434684684686, + "pass@2": 0.21021759948925312, + "pass@4": 0.27047376904463505, + "pass@8": 0.3240379916251193, + "pass@16": 0.3656271096807543, + "pass@32": 0.39589239284381544, + "pass@64": 0.4213365552310757, + "pass@128": 0.44744744744744747 + }, + "20": { + "pass@1": 0.15498310810810811, + "pass@2": 0.21680957532335482, + "pass@4": 0.27715197115000256, + "pass@8": 0.3276990281660607, + "pass@16": 0.36812078329427916, + "pass@32": 0.4005794165660846, + "pass@64": 0.42888634723852626, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 534302, + "answer_accuracy": 0.6506990452065472, + "avg_response_len": 256.01557506819955, + "resp_tokens_sum": 210219509, + "pass_at_k": { + "pass@1": 0.6506990452065472, + "pass@2": 0.6802487173271308, + "pass@4": 0.7052685169273947, + "pass@8": 0.7268958073806057, + "pass@16": 0.7459371062457395, + "pass@32": 0.7628768755926708, + "pass@64": 0.7781517270138143, + "pass@128": 0.792049883086516 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8994919478527608, + "pass@2": 0.9163061265156274, + "pass@4": 0.9283594290936025, + "pass@8": 0.9382051798212965, + "pass@16": 0.9469677315727241, + "pass@32": 0.9551237622514445, + "pass@64": 0.961818137346346, + "pass@128": 0.9662576687116564 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9816123188405798, + "pass@2": 0.9827905682985281, + "pass@4": 0.9834688090619108, + "pass@8": 0.9842941702692776, + "pass@16": 0.9857126174441587, + "pass@32": 0.9878730050048092, + "pass@64": 0.990240146068698, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9922770415472779, + "pass@2": 0.9936541834938971, + "pass@4": 0.9941627463114389, + "pass@8": 0.99426602927843, + "pass@16": 0.9942693391968014, + "pass@32": 0.994269340974212, + "pass@64": 0.994269340974212, + "pass@128": 0.994269340974212 + }, + "7": { + "pass@1": 0.9937454841040463, + "pass@2": 0.9949379295434891, + "pass@4": 0.9956017114551432, + "pass@8": 0.9963418329269713, + "pass@16": 0.9969251127417026, + "pass@32": 0.9971024349058709, + "pass@64": 0.9971098254417226, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9880941901408451, + "pass@2": 0.9926787595652655, + "pass@4": 0.9941441718552785, + "pass@8": 0.9946804118630559, + "pass@16": 0.995070095638103, + "pass@32": 0.9957746478821857, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9570078592814372, + "pass@2": 0.9676165631335754, + "pass@4": 0.9740296437496209, + "pass@8": 0.9788848321388287, + "pass@16": 0.9824501833869013, + "pass@32": 0.9846843678105359, + "pass@64": 0.9863466253618248, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8519860692771084, + "pass@2": 0.8888458608054265, + "pass@4": 0.9119766091926755, + "pass@8": 0.9283114158279034, + "pass@16": 0.9401535492489448, + "pass@32": 0.9484750469432417, + "pass@64": 0.9544825870655861, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.7110599529780565, + "pass@2": 0.7596701213190828, + "pass@4": 0.7952358187789219, + "pass@8": 0.8224395278267408, + "pass@16": 0.8435730646163403, + "pass@32": 0.86120908890127, + "pass@64": 0.8775723553695687, + "pass@128": 0.8934169278996865 + }, + "13": { + "pass@1": 0.4431897759103641, + "pass@2": 0.5064896529477931, + "pass@4": 0.5575887811817638, + "pass@8": 0.6006728712300449, + "pass@16": 0.6382857482801823, + "pass@32": 0.672182533480856, + "pass@64": 0.7036470611522763, + "pass@128": 0.7310924369747899 + }, + "14": { + "pass@1": 0.39160727339181284, + "pass@2": 0.4481789180135375, + "pass@4": 0.4956121079711529, + "pass@8": 0.5395046872717323, + "pass@16": 0.580186785625428, + "pass@32": 0.6175963919547802, + "pass@64": 0.652920536861353, + "pass@128": 0.6871345029239766 + }, + "15": { + "pass@1": 0.2729403409090909, + "pass@2": 0.33778334526366016, + "pass@4": 0.39326375396257257, + "pass@8": 0.44043010034042246, + "pass@16": 0.48186062978129907, + "pass@32": 0.5183775815933022, + "pass@64": 0.5489058851771729, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.21091524216524216, + "pass@2": 0.26682271687192954, + "pass@4": 0.31628648368740236, + "pass@8": 0.35864969193198676, + "pass@16": 0.3954488342196391, + "pass@32": 0.42901714214411485, + "pass@64": 0.46241872936773004, + "pass@128": 0.49572649572649574 + }, + "17": { + "pass@1": 0.18791246118012422, + "pass@2": 0.23750657186873375, + "pass@4": 0.2855899974847864, + "pass@8": 0.33262139177019856, + "pass@16": 0.37758867649195893, + "pass@32": 0.4197480671208388, + "pass@64": 0.45981387209971636, + "pass@128": 0.5031055900621118 + }, + "18": { + "pass@1": 0.1879595588235294, + "pass@2": 0.23974315365910143, + "pass@4": 0.28761232024305783, + "pass@8": 0.329751823564591, + "pass@16": 0.3682869367529674, + "pass@32": 0.40342406625742766, + "pass@64": 0.43378958147428764, + "pass@128": 0.46176470588235297 + }, + "19": { + "pass@1": 0.14397321428571427, + "pass@2": 0.19275703330052496, + "pass@4": 0.2422015247536021, + "pass@8": 0.2878507464902696, + "pass@16": 0.3280847004008977, + "pass@32": 0.36158603274779966, + "pass@64": 0.3892965040997099, + "pass@128": 0.41369047619047616 + }, + "20": { + "pass@1": 0.13793506736526945, + "pass@2": 0.18860249481352254, + "pass@4": 0.23572179403846963, + "pass@8": 0.2758505298038162, + "pass@16": 0.3104882346741073, + "pass@32": 0.341360691539217, + "pass@64": 0.36851288153677664, + "pass@128": 0.38922155688622756 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 524979, + "answer_accuracy": 0.655280146588912, + "avg_response_len": 275.1620503974277, + "resp_tokens_sum": 220446627, + "pass_at_k": { + "pass@1": 0.655280146588912, + "pass@2": 0.6839925507269552, + "pass@4": 0.7088387611228634, + "pass@8": 0.7307742552370572, + "pass@16": 0.7506730765481188, + "pass@32": 0.7687142269840314, + "pass@64": 0.784539857043786, + "pass@128": 0.7983703467007509 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9056203358208955, + "pass@2": 0.9243040457163008, + "pass@4": 0.9389661845440961, + "pass@8": 0.9507190979296448, + "pass@16": 0.9601808559838616, + "pass@32": 0.9673198046675844, + "pass@64": 0.9723514966419701, + "pass@128": 0.9761194029850746 + }, + "2": { + "pass@1": 0.9972991071428572, + "pass@2": 0.9974479752530934, + "pass@4": 0.9977247281589801, + "pass@8": 0.9982019248579823, + "pass@16": 0.9989049734161272, + "pass@32": 0.9996397297099158, + "pass@64": 0.9999812230788224, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994533527696793, + "pass@2": 0.9999397396753976, + "pass@4": 0.9999991596349289, + "pass@8": 0.9999999999107686, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9742417279411765, + "pass@2": 0.976435488073182, + "pass@4": 0.9785727789541012, + "pass@8": 0.9800637367521491, + "pass@16": 0.981034477001222, + "pass@32": 0.9818501519506909, + "pass@64": 0.982312283238623, + "pass@128": 0.9823529411764705 + }, + "6": { + "pass@1": 0.9912016955835962, + "pass@2": 0.9923161591196998, + "pass@4": 0.9931202484744611, + "pass@8": 0.9935946610212556, + "pass@16": 0.993688693168888, + "pass@32": 0.9936908514016762, + "pass@64": 0.9936908517350158, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9939922863924051, + "pass@2": 0.9979045711651552, + "pass@4": 0.9996272230211731, + "pass@8": 0.9999848951262044, + "pass@16": 0.9999999797040798, + "pass@32": 0.999999999999996, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9742164039039038, + "pass@2": 0.9776526033907924, + "pass@4": 0.9802813651671918, + "pass@8": 0.9826036723003114, + "pass@16": 0.9842749202268645, + "pass@32": 0.985456634186374, + "pass@64": 0.9864771895962706, + "pass@128": 0.987987987987988 + }, + "9": { + "pass@1": 0.972292001607717, + "pass@2": 0.9819812264222599, + "pass@4": 0.986615645311217, + "pass@8": 0.9891525246781142, + "pass@16": 0.9907615607123103, + "pass@32": 0.9917515996499101, + "pass@64": 0.9927715998185213, + "pass@128": 0.9935691318327974 + }, + "11": { + "pass@1": 0.860094246031746, + "pass@2": 0.8852873859517562, + "pass@4": 0.9016740666345278, + "pass@8": 0.9141964920855181, + "pass@16": 0.9255256112996866, + "pass@32": 0.9359988921190586, + "pass@64": 0.9462906177764885, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.7074971330275229, + "pass@2": 0.7622297822003904, + "pass@4": 0.8011608589063977, + "pass@8": 0.8297119134538486, + "pass@16": 0.8534948504852095, + "pass@32": 0.8747256701701144, + "pass@64": 0.8922751671197711, + "pass@128": 0.9051987767584098 + }, + "13": { + "pass@1": 0.48215663580246915, + "pass@2": 0.5435611238699328, + "pass@4": 0.5926587695056635, + "pass@8": 0.633260128402649, + "pass@16": 0.6707687175863076, + "pass@32": 0.7085261614263532, + "pass@64": 0.7448136642431026, + "pass@128": 0.7746913580246914 + }, + "14": { + "pass@1": 0.406134087537092, + "pass@2": 0.4712745186803429, + "pass@4": 0.5243155244503929, + "pass@8": 0.568281215623407, + "pass@16": 0.6078381209744614, + "pass@32": 0.6452114848141344, + "pass@64": 0.6794001748786187, + "pass@128": 0.7091988130563798 + }, + "15": { + "pass@1": 0.3033613040123457, + "pass@2": 0.35925956304073065, + "pass@4": 0.4130433117156653, + "pass@8": 0.46853172257542763, + "pass@16": 0.5238111534693787, + "pass@32": 0.5719111695487336, + "pass@64": 0.6073624482988279, + "pass@128": 0.6327160493827161 + }, + "16": { + "pass@1": 0.1922704646017699, + "pass@2": 0.23690821487468935, + "pass@4": 0.27880137604480854, + "pass@8": 0.318234271548925, + "pass@16": 0.35525716796412005, + "pass@32": 0.38901365189369735, + "pass@64": 0.4183437728038016, + "pass@128": 0.4424778761061947 + }, + "17": { + "pass@1": 0.18053463855421686, + "pass@2": 0.23325067000284597, + "pass@4": 0.2834787490795578, + "pass@8": 0.32846562326969203, + "pass@16": 0.369436527768843, + "pass@32": 0.40634737101135743, + "pass@64": 0.4379492763008395, + "pass@128": 0.46987951807228917 + }, + "18": { + "pass@1": 0.16743951612903227, + "pass@2": 0.21504000508001017, + "pass@4": 0.26076773044498464, + "pass@8": 0.30550745623361136, + "pass@16": 0.34705035881725593, + "pass@32": 0.3828956929719597, + "pass@64": 0.4131763453082906, + "pass@128": 0.44193548387096776 + }, + "19": { + "pass@1": 0.14834308912386707, + "pass@2": 0.19732393486690303, + "pass@4": 0.2415079635060723, + "pass@8": 0.28059675189721844, + "pass@16": 0.31802633512102896, + "pass@32": 0.354738317640572, + "pass@64": 0.38914149256189534, + "pass@128": 0.4229607250755287 + }, + "20": { + "pass@1": 0.17424455705705705, + "pass@2": 0.22630923246080717, + "pass@4": 0.2799491279806238, + "pass@8": 0.32954584901807554, + "pass@16": 0.370233902011627, + "pass@32": 0.4052958996657454, + "pass@64": 0.43976604913749345, + "pass@128": 0.47147147147147145 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/checkpoint-21808_metrics.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/checkpoint-21808_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..26999b740a5d9c76304486367e5f417b691082d2 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/checkpoint-21808_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21808", + "id": { + "answer_accuracy": 0.890703125, + "avg_response_len": 200.43872916666666, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.890703125, + "pass@2": 0.9210265337926394, + "pass@4": 0.9404577235345565, + "pass@8": 0.9536115527829724, + "pass@16": 0.9630975996278396, + "pass@32": 0.9703186447941745, + "pass@64": 0.9759534203581967, + "pass@128": 0.9801111111111112 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.80096875, + "pass@2": 0.8557803887795276, + "pass@4": 0.8904167110048742, + "pass@8": 0.9129100125018699, + "pass@16": 0.9276722009841487, + "pass@32": 0.937608858739233, + "pass@64": 0.9454213165629998, + "pass@128": 0.952 + }, + "2": { + "pass@1": 0.9294921875, + "pass@2": 0.9364880659448821, + "pass@4": 0.9420603667041619, + "pass@8": 0.9474590485835576, + "pass@16": 0.9529304380551213, + "pass@32": 0.9583871718022152, + "pass@64": 0.9632877535234018, + "pass@128": 0.967 + }, + "3": { + "pass@1": 0.973203125, + "pass@2": 0.980429872047244, + "pass@4": 0.9855431728533933, + "pass@8": 0.9890036599336474, + "pass@16": 0.9911901532983721, + "pass@32": 0.9928402440869977, + "pass@64": 0.99444485216838, + "pass@128": 0.996 + }, + "4": { + "pass@1": 0.96428125, + "pass@2": 0.9740697588582677, + "pass@4": 0.9806544889388824, + "pass@8": 0.9854722781029022, + "pass@16": 0.9892539405961934, + "pass@32": 0.9919500822619794, + "pass@64": 0.9933217755386953, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.9377421875, + "pass@2": 0.9559159694881895, + "pass@4": 0.9667724132920883, + "pass@8": 0.9739256047719212, + "pass@16": 0.9791804380546189, + "pass@32": 0.9836106949889912, + "pass@64": 0.9870193916880827, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.88653125, + "pass@2": 0.9191799950787405, + "pass@4": 0.9399450903637045, + "pass@8": 0.9548495446375231, + "pass@16": 0.9663498701850942, + "pass@32": 0.9754545094030427, + "pass@64": 0.9827603868406525, + "pass@128": 0.988 + }, + "7": { + "pass@1": 0.8744140625, + "pass@2": 0.9132801427165361, + "pass@4": 0.9383020911136102, + "pass@8": 0.9556171076854798, + "pass@16": 0.9683151755830492, + "pass@32": 0.9774271188529, + "pass@64": 0.9836507404376994, + "pass@128": 0.988 + }, + "8": { + "pass@1": 0.84721875, + "pass@2": 0.893562869094489, + "pass@4": 0.9231761468878892, + "pass@8": 0.9423404399090602, + "pass@16": 0.9552222500339728, + "pass@32": 0.9643567621506189, + "pass@64": 0.9717649485611493, + "pass@128": 0.978 + }, + "9": { + "pass@1": 0.8024765625, + "pass@2": 0.8605317421259852, + "pass@4": 0.8972490306524176, + "pass@8": 0.9209262789208218, + "pass@16": 0.9377639298599652, + "pass@32": 0.9512323608615938, + "pass@64": 0.9619096179026939, + "pass@128": 0.969 + } + }, + "per_op_accuracy": { + "10": 0.80096875, + "2": 0.9294921875, + "3": 0.973203125, + "4": 0.96428125, + "5": 0.9377421875, + "6": 0.88653125, + "7": 0.8744140625, + "8": 0.84721875, + "9": 0.8024765625 + }, + "per_op_avg_response_len": { + "10": 282.7000546875, + "2": 116.5491015625, + "3": 144.1264375, + "4": 160.84509375, + "5": 177.43053125, + "6": 198.4591015625, + "7": 222.6527109375, + "8": 241.4876953125, + "9": 259.6978359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.7000546875, + "2": 116.5491015625, + "3": 144.1264375, + "4": 160.84509375, + "5": 177.43053125, + "6": 198.4591015625, + "7": 222.6527109375, + "8": 241.4876953125, + "9": 259.6978359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 333947, + "answer_accuracy": 0.8811080504896994, + "avg_response_len": 191.06698275498144, + "resp_tokens_sum": 72415915, + "pass_at_k": { + "pass@1": 0.8811080504896994, + "pass@2": 0.9155245222006851, + "pass@4": 0.9377718603626949, + "pass@8": 0.9527883652029874, + "pass@16": 0.9636765902508956, + "pass@32": 0.9715946287602495, + "pass@64": 0.9769477477480074, + "pass@128": 0.9797365754812564 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8082595870206489, + "pass@2": 0.8647357617355352, + "pass@4": 0.8976401423273417, + "pass@8": 0.9173720749762039, + "pass@16": 0.9291881918718146, + "pass@32": 0.9361090250317172, + "pass@64": 0.9411341859417028, + "pass@128": 0.943952802359882 + }, + "2": { + "pass@1": 0.9132593457943925, + "pass@2": 0.9192192827532073, + "pass@4": 0.9254616957926987, + "pass@8": 0.9326086884253552, + "pass@16": 0.9412402480297669, + "pass@32": 0.9502751094866533, + "pass@64": 0.9567691395661726, + "pass@128": 0.9595015576323987 + }, + "3": { + "pass@1": 0.9742689220183486, + "pass@2": 0.9795918213778326, + "pass@4": 0.9831441336461383, + "pass@8": 0.985580114929864, + "pass@16": 0.9875498957766747, + "pass@32": 0.9897440229097163, + "pass@64": 0.9922786486226282, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.9580322265625, + "pass@2": 0.9734970933809055, + "pass@4": 0.983606354869704, + "pass@8": 0.9907801440498372, + "pass@16": 0.9961476900698585, + "pass@32": 0.9991019833663184, + "pass@64": 0.9999517511298771, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9450892857142857, + "pass@2": 0.9643052430946132, + "pass@4": 0.9760748763547413, + "pass@8": 0.9826861265629934, + "pass@16": 0.9857626408128157, + "pass@32": 0.9875250624374391, + "pass@64": 0.9888449939948887, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.8729883982035929, + "pass@2": 0.9114836272337217, + "pass@4": 0.9363393078859157, + "pass@8": 0.9538367547277492, + "pass@16": 0.966520089086936, + "pass@32": 0.9746123891484606, + "pass@64": 0.979363177663249, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.8500600961538461, + "pass@2": 0.8949580382518755, + "pass@4": 0.9245661620330001, + "pass@8": 0.9468657824828772, + "pass@16": 0.9647895774794542, + "pass@32": 0.977670264977721, + "pass@64": 0.9850630781424923, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8358122996794872, + "pass@2": 0.8931357100494657, + "pass@4": 0.9322278360036731, + "pass@8": 0.9569910688945663, + "pass@16": 0.9718355835350385, + "pass@32": 0.9799294253747101, + "pass@64": 0.9839658835248849, + "pass@128": 0.9871794871794872 + }, + "9": { + "pass@1": 0.7866857394366197, + "pass@2": 0.8491883386935789, + "pass@4": 0.8889232061661304, + "pass@8": 0.9148338855375537, + "pass@16": 0.9351348435701646, + "pass@32": 0.9531606564413371, + "pass@64": 0.9678114627207628, + "pass@128": 0.9746478873239437 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 348776, + "answer_accuracy": 0.8927957077326344, + "avg_response_len": 198.36971657929226, + "resp_tokens_sum": 77494320, + "pass_at_k": { + "pass@1": 0.8927957077326344, + "pass@2": 0.9226083112532404, + "pass@4": 0.9418752599609538, + "pass@8": 0.9548893845015304, + "pass@16": 0.9637317301155329, + "pass@32": 0.9702815714153485, + "pass@64": 0.9757441959769306, + "pass@128": 0.9803407601572739 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.793472009202454, + "pass@2": 0.8493992590937638, + "pass@4": 0.8877754004867485, + "pass@8": 0.9141527086586699, + "pass@16": 0.9311651079164657, + "pass@32": 0.9426145462430192, + "pass@64": 0.9516821714118398, + "pass@128": 0.9601226993865031 + }, + "2": { + "pass@1": 0.9402783054711246, + "pass@2": 0.9459940346552426, + "pass@4": 0.9491347127391752, + "pass@8": 0.9519259878204085, + "pass@16": 0.9553404745234914, + "pass@32": 0.9597092653562085, + "pass@64": 0.9640150627253267, + "pass@128": 0.9665653495440729 + }, + "3": { + "pass@1": 0.9669981060606061, + "pass@2": 0.976074847888332, + "pass@4": 0.9835954573860082, + "pass@8": 0.9887292837477384, + "pass@16": 0.990873589188666, + "pass@32": 0.9916485785890059, + "pass@64": 0.9924242355459135, + "pass@128": 0.9939393939393939 + }, + "4": { + "pass@1": 0.965051775147929, + "pass@2": 0.9720598384429018, + "pass@4": 0.9769207664685405, + "pass@8": 0.9815084547831039, + "pass@16": 0.9856876197176295, + "pass@32": 0.9890133314572516, + "pass@64": 0.990732185832648, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.9434329710144927, + "pass@2": 0.9612539940659597, + "pass@4": 0.9702148456986353, + "pass@8": 0.9758259545604623, + "pass@16": 0.9803524893764493, + "pass@32": 0.9842124262842136, + "pass@64": 0.988043997616301, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.8868642550143266, + "pass@2": 0.9194193054396137, + "pass@4": 0.939174852516644, + "pass@8": 0.952673887441346, + "pass@16": 0.9630499625273448, + "pass@32": 0.9720429085728259, + "pass@64": 0.9808653443318179, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.8915281791907514, + "pass@2": 0.92678630353648, + "pass@4": 0.950118883170673, + "pass@8": 0.9654538312470585, + "pass@16": 0.9751158531352345, + "pass@32": 0.9812282655305531, + "pass@64": 0.9852157051970267, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.8579005281690141, + "pass@2": 0.8997251718975272, + "pass@4": 0.9258677762110714, + "pass@8": 0.9437238727011946, + "pass@16": 0.9556631355474996, + "pass@32": 0.9640877913679654, + "pass@64": 0.9721959531135995, + "pass@128": 0.9802816901408451 + }, + "9": { + "pass@1": 0.7888285928143712, + "pass@2": 0.851565815219954, + "pass@4": 0.8928656674277982, + "pass@8": 0.9188092984095537, + "pass@16": 0.9350753145380785, + "pass@32": 0.9466458701980125, + "pass@64": 0.9550391747311027, + "pass@128": 0.9610778443113772 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 343367, + "answer_accuracy": 0.8980765609306997, + "avg_response_len": 211.8429365793438, + "resp_tokens_sum": 80995181, + "pass_at_k": { + "pass@1": 0.8980765609306997, + "pass@2": 0.9248644552905124, + "pass@4": 0.9416718245317283, + "pass@8": 0.9531219364519798, + "pass@16": 0.961875718983944, + "pass@32": 0.9690916476159442, + "pass@64": 0.9751815252829948, + "pass@128": 0.9802477402075661 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8008861940298507, + "pass@2": 0.8529277823481023, + "pass@4": 0.8856773796932103, + "pass@8": 0.9071853613797309, + "pass@16": 0.922739041071155, + "pass@32": 0.93425539456196, + "pass@64": 0.9436669840253802, + "pass@128": 0.9522388059701492 + }, + "2": { + "pass@1": 0.9342410714285714, + "pass@2": 0.9433903965129361, + "pass@4": 0.9506338338957628, + "pass@8": 0.9568800274460122, + "pass@16": 0.9613865780552503, + "pass@32": 0.964584323870877, + "pass@64": 0.968582583102937, + "pass@128": 0.9742857142857143 + }, + "3": { + "pass@1": 0.978156887755102, + "pass@2": 0.9854187949082893, + "pass@4": 0.9897041988738288, + "pass@8": 0.9925314831336094, + "pass@16": 0.9949651689420985, + "pass@32": 0.9969385354552436, + "pass@64": 0.9984540418035831, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9693667763157895, + "pass@2": 0.9765919987797581, + "pass@4": 0.9815825623222536, + "pass@8": 0.9844233166382033, + "pass@16": 0.9863282582142694, + "pass@32": 0.9881606478134689, + "pass@64": 0.9896775917125719, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.9251608455882353, + "pass@2": 0.9427270292959704, + "pass@4": 0.9546608984538691, + "pass@8": 0.9638809428859377, + "pass@16": 0.9718929334226678, + "pass@32": 0.9793735683916001, + "pass@64": 0.9842883511825552, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9004337539432177, + "pass@2": 0.9270256340197225, + "pass@4": 0.9445922334432167, + "pass@8": 0.9583119300991347, + "pass@16": 0.9698035441893214, + "pass@32": 0.9800977802382987, + "pass@64": 0.98842612091269, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.8817246835443038, + "pass@2": 0.9180894454549986, + "pass@4": 0.9400556796500119, + "pass@8": 0.954207112641725, + "pass@16": 0.9646399152221633, + "pass@32": 0.9730050298001863, + "pass@64": 0.980426538061284, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.8465183933933934, + "pass@2": 0.8873936682745736, + "pass@4": 0.9118258303297669, + "pass@8": 0.927138893708802, + "pass@16": 0.9391865911461822, + "pass@32": 0.9500528993936376, + "pass@64": 0.9598739626007716, + "pass@128": 0.9669669669669669 + }, + "9": { + "pass@1": 0.8351587620578779, + "pass@2": 0.8831090019495152, + "pass@4": 0.9114603200725307, + "pass@8": 0.9301541604057866, + "pass@16": 0.9436524287358187, + "pass@32": 0.9539569362661837, + "pass@64": 0.9625513320792103, + "pass@128": 0.9710610932475884 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.30851953125, + "avg_response_len": 328.6832890625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.30851953125, + "pass@2": 0.3752682332677224, + "pass@4": 0.43450376761342496, + "pass@8": 0.48504148026548244, + "pass@16": 0.5275449010807912, + "pass@32": 0.5635612964335429, + "pass@64": 0.5946841594439383, + "pass@128": 0.6227 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.73865625, + "pass@2": 0.8118968996063, + "pass@4": 0.8600340128421451, + "pass@8": 0.8924111084147311, + "pass@16": 0.9156893145490389, + "pass@32": 0.9332161742632478, + "pass@64": 0.9469885923732444, + "pass@128": 0.959 + }, + "12": { + "pass@1": 0.580546875, + "pass@2": 0.6677766978346458, + "pass@4": 0.7349462101612291, + "pass@8": 0.7854534250296459, + "pass@16": 0.822971035590625, + "pass@32": 0.8516079660353347, + "pass@64": 0.8746620928637222, + "pass@128": 0.896 + }, + "13": { + "pass@1": 0.3827265625, + "pass@2": 0.4641267224409449, + "pass@4": 0.5347831144544428, + "pass@8": 0.5933991973126331, + "pass@16": 0.6415841948801748, + "pass@32": 0.683597963618523, + "pass@64": 0.7232024375652686, + "pass@128": 0.761 + }, + "14": { + "pass@1": 0.322515625, + "pass@2": 0.3969320866141736, + "pass@4": 0.4603320167791523, + "pass@8": 0.5124417839066472, + "pass@16": 0.5564519977944874, + "pass@32": 0.5959754188029391, + "pass@64": 0.6315096364458914, + "pass@128": 0.662 + }, + "15": { + "pass@1": 0.2491484375, + "pass@2": 0.31848745078740154, + "pass@4": 0.3813791015185601, + "pass@8": 0.4372522014251215, + "pass@16": 0.48708832699515886, + "pass@32": 0.530787426728577, + "pass@64": 0.5700222181353979, + "pass@128": 0.609 + }, + "16": { + "pass@1": 0.182953125, + "pass@2": 0.23924778543307085, + "pass@4": 0.29406598115860527, + "pass@8": 0.3439873332042931, + "pass@16": 0.3878827007868039, + "pass@32": 0.42520837146889945, + "pass@64": 0.45657455458503415, + "pass@128": 0.487 + }, + "17": { + "pass@1": 0.179296875, + "pass@2": 0.23684793307086602, + "pass@4": 0.29352195425571787, + "pass@8": 0.34525731254854514, + "pass@16": 0.38998667678602095, + "pass@32": 0.42788088210582303, + "pass@64": 0.4601960116995837, + "pass@128": 0.489 + }, + "18": { + "pass@1": 0.1612421875, + "pass@2": 0.218044660433071, + "pass@4": 0.27296097075365566, + "pass@8": 0.3213106546616519, + "pass@16": 0.3635569729731848, + "pass@32": 0.40157602253748653, + "pass@64": 0.4348644408693857, + "pass@128": 0.462 + }, + "19": { + "pass@1": 0.1484296875, + "pass@2": 0.20265772637795254, + "pass@4": 0.25669816685414304, + "pass@8": 0.3068975188900042, + "pass@16": 0.35125819731138486, + "pass@32": 0.3894010373998276, + "pass@64": 0.4209719100949514, + "pass@128": 0.446 + }, + "20": { + "pass@1": 0.1396796875, + "pass@2": 0.19666437007874016, + "pass@4": 0.25631614735658004, + "pass@8": 0.31200426726149993, + "pass@16": 0.35897959314091565, + "pass@32": 0.3963617013747654, + "pass@64": 0.42784969980687954, + "pass@128": 0.456 + } + }, + "per_op_accuracy": { + "11": 0.73865625, + "12": 0.580546875, + "13": 0.3827265625, + "14": 0.322515625, + "15": 0.2491484375, + "16": 0.182953125, + "17": 0.179296875, + "18": 0.1612421875, + "19": 0.1484296875, + "20": 0.1396796875 + }, + "per_op_avg_response_len": { + "11": 300.53321875, + "12": 310.8298359375, + "13": 319.9635234375, + "14": 338.1668203125, + "15": 337.2906953125, + "16": 333.285203125, + "17": 336.919984375, + "18": 342.160640625, + "19": 331.1883359375, + "20": 336.4946328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 300.53321875, + "12": 310.8298359375, + "13": 319.9635234375, + "14": 338.1668203125, + "15": 337.2906953125, + "16": 333.285203125, + "17": 336.919984375, + "18": 342.160640625, + "19": 331.1883359375, + "20": 336.4946328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 133092, + "answer_accuracy": 0.30899888558692423, + "avg_response_len": 310.9412982912333, + "resp_tokens_sum": 133928636, + "pass_at_k": { + "pass@1": 0.30899888558692423, + "pass@2": 0.3764697967731752, + "pass@4": 0.43594456866740094, + "pass@8": 0.4861778796414367, + "pass@16": 0.5284114862972554, + "pass@32": 0.5643099395881148, + "pass@64": 0.5950054743885208, + "pass@128": 0.6216939078751857 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7324938031161473, + "pass@2": 0.8083786191474651, + "pass@4": 0.8603668985427808, + "pass@8": 0.8951230630417605, + "pass@16": 0.9192438398166457, + "pass@32": 0.9382015773061856, + "pass@64": 0.9543511427116407, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.5774629237288136, + "pass@2": 0.6690998404066018, + "pass@4": 0.7407094531615751, + "pass@8": 0.7936093234416746, + "pass@16": 0.8316808174882109, + "pass@32": 0.8604117124933197, + "pass@64": 0.8853741576695201, + "pass@128": 0.9124293785310734 + }, + "13": { + "pass@1": 0.37955525078369906, + "pass@2": 0.4627638813467281, + "pass@4": 0.5334049649553994, + "pass@8": 0.5908909515982965, + "pass@16": 0.6383243449452396, + "pass@32": 0.6785025472136269, + "pass@64": 0.7141232799137383, + "pass@128": 0.7492163009404389 + }, + "14": { + "pass@1": 0.3085572429906542, + "pass@2": 0.3798913643388032, + "pass@4": 0.4410535917239317, + "pass@8": 0.4929612888049262, + "pass@16": 0.5385958716440408, + "pass@32": 0.5820172888780842, + "pass@64": 0.6215417900208923, + "pass@128": 0.6510903426791277 + }, + "15": { + "pass@1": 0.2654895231213873, + "pass@2": 0.33615796106230944, + "pass@4": 0.3985344978553981, + "pass@8": 0.4518694766045028, + "pass@16": 0.4984802935633324, + "pass@32": 0.5388247206741961, + "pass@64": 0.5739738417672182, + "pass@128": 0.6098265895953757 + }, + "16": { + "pass@1": 0.17903225806451614, + "pass@2": 0.23331375412750824, + "pass@4": 0.2866873023936522, + "pass@8": 0.3375684152141981, + "pass@16": 0.3862721919687068, + "pass@32": 0.43088065063116227, + "pass@64": 0.4678904281213043, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.16988800578034682, + "pass@2": 0.22384549519821573, + "pass@4": 0.2748313011632217, + "pass@8": 0.3193916584849494, + "pass@16": 0.358615204863562, + "pass@32": 0.39272698262401673, + "pass@64": 0.4211641810459257, + "pass@128": 0.44508670520231214 + }, + "18": { + "pass@1": 0.15439732142857143, + "pass@2": 0.21321674634420695, + "pass@4": 0.2686094804220902, + "pass@8": 0.3154271003708264, + "pass@16": 0.3563084554457416, + "pass@32": 0.3923554436588413, + "pass@64": 0.4227785903208911, + "pass@128": 0.44571428571428573 + }, + "19": { + "pass@1": 0.14902402402402404, + "pass@2": 0.20243558026530464, + "pass@4": 0.25758560106788453, + "pass@8": 0.3117411757269937, + "pass@16": 0.3584763615622298, + "pass@32": 0.3972802657507109, + "pass@64": 0.43147037921139075, + "pass@128": 0.4594594594594595 + }, + "20": { + "pass@1": 0.14071884384384384, + "pass@2": 0.20093001466032961, + "pass@4": 0.26321763270582166, + "pass@8": 0.32019753114376637, + "pass@16": 0.367272720530513, + "pass@32": 0.4034641339305123, + "pass@64": 0.43107760881473256, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 130374, + "answer_accuracy": 0.3028685325602141, + "avg_response_len": 327.1753851657746, + "resp_tokens_sum": 140837225, + "pass_at_k": { + "pass@1": 0.3028685325602141, + "pass@2": 0.37080558813957404, + "pass@4": 0.4310754379108066, + "pass@8": 0.48251005171541667, + "pass@16": 0.5256438053073414, + "pass@32": 0.5618262749127514, + "pass@64": 0.5923494032083111, + "pass@128": 0.6184953910199227 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7130788780120482, + "pass@2": 0.7939841304193148, + "pass@4": 0.8469520404452451, + "pass@8": 0.8823839807042658, + "pass@16": 0.9067006633768472, + "pass@32": 0.9217781709792446, + "pass@64": 0.9309741583437446, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.5889988244514106, + "pass@2": 0.6738543029397968, + "pass@4": 0.7364737050893712, + "pass@8": 0.7833416397619604, + "pass@16": 0.8210170887885584, + "pass@32": 0.8530612652715264, + "pass@64": 0.8769674396793894, + "pass@128": 0.8934169278996865 + }, + "13": { + "pass@1": 0.37489058123249297, + "pass@2": 0.45928484582809503, + "pass@4": 0.5333312377444418, + "pass@8": 0.5955567982367712, + "pass@16": 0.6468370691536759, + "pass@32": 0.6914674094335568, + "pass@64": 0.7330451891460972, + "pass@128": 0.7675070028011205 + }, + "14": { + "pass@1": 0.3155381944444444, + "pass@2": 0.3891212817147854, + "pass@4": 0.453302549078295, + "pass@8": 0.507271840880927, + "pass@16": 0.5527430887420266, + "pass@32": 0.5921199769806936, + "pass@64": 0.6270367990392346, + "pass@128": 0.6578947368421053 + }, + "15": { + "pass@1": 0.2109848484848485, + "pass@2": 0.27266687544738727, + "pass@4": 0.3313393129836044, + "pass@8": 0.3861669387313994, + "pass@16": 0.43703563287263564, + "pass@32": 0.4823231852380329, + "pass@64": 0.5239700082246377, + "pass@128": 0.5666666666666667 + }, + "16": { + "pass@1": 0.1877003205128205, + "pass@2": 0.2486519112995491, + "pass@4": 0.30775032955282305, + "pass@8": 0.3612737419871499, + "pass@16": 0.40587271912690925, + "pass@32": 0.43920306525713915, + "pass@64": 0.4626650339424816, + "pass@128": 0.48148148148148145 + }, + "17": { + "pass@1": 0.19596758540372672, + "pass@2": 0.25669413605907954, + "pass@4": 0.31698872724372196, + "pass@8": 0.37355683740742207, + "pass@16": 0.4234816279489652, + "pass@32": 0.46557943249256295, + "pass@64": 0.4995003793338017, + "pass@128": 0.5279503105590062 + }, + "18": { + "pass@1": 0.17325367647058823, + "pass@2": 0.23055631947660954, + "pass@4": 0.28691845293235396, + "pass@8": 0.3373942841337351, + "pass@16": 0.38137141561201654, + "pass@32": 0.4221056033676291, + "pass@64": 0.45916635273358053, + "pass@128": 0.48823529411764705 + }, + "19": { + "pass@1": 0.15171595982142858, + "pass@2": 0.20929048907949008, + "pass@4": 0.263858470258182, + "pass@8": 0.31035082337712583, + "pass@16": 0.3495046566647495, + "pass@32": 0.3829455856819846, + "pass@64": 0.41029909549921423, + "pass@128": 0.43154761904761907 + }, + "20": { + "pass@1": 0.13075411676646706, + "pass@2": 0.18799286269979726, + "pass@4": 0.24704597469852202, + "pass@8": 0.30007091187136975, + "pass@16": 0.34343731208913214, + "pass@32": 0.37867318149442, + "pass@64": 0.41008857357648265, + "pass@128": 0.44011976047904194 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 131439, + "answer_accuracy": 0.3138347150061125, + "avg_response_len": 348.4794014555318, + "resp_tokens_sum": 145948749, + "pass_at_k": { + "pass@1": 0.3138347150061125, + "pass@2": 0.37861927677935553, + "pass@4": 0.43654569219877537, + "pass@8": 0.4864746130019525, + "pass@16": 0.5286076534746139, + "pass@32": 0.5645746500885723, + "pass@64": 0.5967534016297189, + "pass@128": 0.6280562347188264 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7725198412698413, + "pass@2": 0.8347190976127983, + "pass@4": 0.873448953107052, + "pass@8": 0.8999402716418193, + "pass@16": 0.921179742389368, + "pass@32": 0.9396846498700138, + "pass@64": 0.9556165664314662, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5756402905198776, + "pass@2": 0.660415387440102, + "pass@4": 0.7272169780382953, + "pass@8": 0.7786842246092575, + "pass@16": 0.8154482412117705, + "pass@32": 0.8406595602173778, + "pass@64": 0.8608165987491957, + "pass@128": 0.8807339449541285 + }, + "13": { + "pass@1": 0.39448302469135804, + "pass@2": 0.47080356274910107, + "pass@4": 0.5377397492558798, + "pass@8": 0.59349137895756, + "pass@16": 0.6390058492431547, + "pass@32": 0.6799437836098674, + "pass@64": 0.7212962307025573, + "pass@128": 0.7654320987654321 + }, + "14": { + "pass@1": 0.3428922477744807, + "pass@2": 0.4210904460384586, + "pass@4": 0.4858289081631857, + "pass@8": 0.5362440374450701, + "pass@16": 0.5772242927210018, + "pass@32": 0.6131834924204059, + "pass@64": 0.6455434319815009, + "pass@128": 0.6765578635014837 + }, + "15": { + "pass@1": 0.27056809413580246, + "pass@2": 0.34628619556236, + "pass@4": 0.4140252838765522, + "pass@8": 0.4736736812919801, + "pass@16": 0.5259024276983834, + "pass@32": 0.5715661180455387, + "pass@64": 0.6127073029932402, + "pass@128": 0.6512345679012346 + }, + "16": { + "pass@1": 0.1816233407079646, + "pass@2": 0.23493717046431134, + "pass@4": 0.28664469540864906, + "pass@8": 0.3319588231575276, + "pass@16": 0.3707286043155161, + "pass@32": 0.40553125034803245, + "pass@64": 0.439920634081472, + "pass@128": 0.4808259587020649 + }, + "17": { + "pass@1": 0.1729339231927711, + "pass@2": 0.2311502407266863, + "pass@4": 0.290240824942063, + "pass@8": 0.34476655743241874, + "pass@16": 0.39019497501102995, + "pass@32": 0.4279541230581569, + "pass@64": 0.4627532617837625, + "pass@128": 0.49698795180722893 + }, + "18": { + "pass@1": 0.15579637096774193, + "pass@2": 0.20977306642113286, + "pass@4": 0.26256573744814155, + "pass@8": 0.310313267504495, + "pass@16": 0.3522023621260945, + "pass@32": 0.3894700390706386, + "pass@64": 0.4218560462182471, + "pass@128": 0.45161290322580644 + }, + "19": { + "pass@1": 0.14449584592145015, + "pass@2": 0.19614825927159402, + "pass@4": 0.24853690541325835, + "pass@8": 0.2985191260368612, + "pass@16": 0.34577644795107754, + "pass@32": 0.3880271665126702, + "pass@64": 0.4212440414798256, + "pass@128": 0.4471299093655589 + }, + "20": { + "pass@1": 0.1475929054054054, + "pass@2": 0.2010962734387931, + "pass@4": 0.2587126730217282, + "pass@8": 0.31578019467143653, + "pass@16": 0.3662754203197754, + "pass@32": 0.40700090737768396, + "pass@64": 0.4424362537448303, + "pass@128": 0.4774774774774775 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5842907072368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5842907072368421, + "pass@2": 0.6337853229900349, + "pass@4": 0.674166167786575, + "pass@8": 0.7069957251421944, + "pass@16": 0.733859337234662, + "pass@32": 0.7562358298675385, + "pass@64": 0.7752853882980614, + "pass@128": 0.792 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.80096875, + "pass@2": 0.8557803887795276, + "pass@4": 0.8904167110048742, + "pass@8": 0.9129100125018699, + "pass@16": 0.9276722009841487, + "pass@32": 0.937608858739233, + "pass@64": 0.9454213165629998, + "pass@128": 0.952 + }, + "2": { + "pass@1": 0.9294921875, + "pass@2": 0.9364880659448821, + "pass@4": 0.9420603667041619, + "pass@8": 0.9474590485835576, + "pass@16": 0.9529304380551213, + "pass@32": 0.9583871718022152, + "pass@64": 0.9632877535234018, + "pass@128": 0.967 + }, + "3": { + "pass@1": 0.973203125, + "pass@2": 0.980429872047244, + "pass@4": 0.9855431728533933, + "pass@8": 0.9890036599336474, + "pass@16": 0.9911901532983721, + "pass@32": 0.9928402440869977, + "pass@64": 0.99444485216838, + "pass@128": 0.996 + }, + "4": { + "pass@1": 0.96428125, + "pass@2": 0.9740697588582677, + "pass@4": 0.9806544889388824, + "pass@8": 0.9854722781029022, + "pass@16": 0.9892539405961934, + "pass@32": 0.9919500822619794, + "pass@64": 0.9933217755386953, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.9377421875, + "pass@2": 0.9559159694881895, + "pass@4": 0.9667724132920883, + "pass@8": 0.9739256047719212, + "pass@16": 0.9791804380546189, + "pass@32": 0.9836106949889912, + "pass@64": 0.9870193916880827, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.88653125, + "pass@2": 0.9191799950787405, + "pass@4": 0.9399450903637045, + "pass@8": 0.9548495446375231, + "pass@16": 0.9663498701850942, + "pass@32": 0.9754545094030427, + "pass@64": 0.9827603868406525, + "pass@128": 0.988 + }, + "7": { + "pass@1": 0.8744140625, + "pass@2": 0.9132801427165361, + "pass@4": 0.9383020911136102, + "pass@8": 0.9556171076854798, + "pass@16": 0.9683151755830492, + "pass@32": 0.9774271188529, + "pass@64": 0.9836507404376994, + "pass@128": 0.988 + }, + "8": { + "pass@1": 0.84721875, + "pass@2": 0.893562869094489, + "pass@4": 0.9231761468878892, + "pass@8": 0.9423404399090602, + "pass@16": 0.9552222500339728, + "pass@32": 0.9643567621506189, + "pass@64": 0.9717649485611493, + "pass@128": 0.978 + }, + "9": { + "pass@1": 0.8024765625, + "pass@2": 0.8605317421259852, + "pass@4": 0.8972490306524176, + "pass@8": 0.9209262789208218, + "pass@16": 0.9377639298599652, + "pass@32": 0.9512323608615938, + "pass@64": 0.9619096179026939, + "pass@128": 0.969 + }, + "11": { + "pass@1": 0.73865625, + "pass@2": 0.8118968996063, + "pass@4": 0.8600340128421451, + "pass@8": 0.8924111084147311, + "pass@16": 0.9156893145490389, + "pass@32": 0.9332161742632478, + "pass@64": 0.9469885923732444, + "pass@128": 0.959 + }, + "12": { + "pass@1": 0.580546875, + "pass@2": 0.6677766978346458, + "pass@4": 0.7349462101612291, + "pass@8": 0.7854534250296459, + "pass@16": 0.822971035590625, + "pass@32": 0.8516079660353347, + "pass@64": 0.8746620928637222, + "pass@128": 0.896 + }, + "13": { + "pass@1": 0.3827265625, + "pass@2": 0.4641267224409449, + "pass@4": 0.5347831144544428, + "pass@8": 0.5933991973126331, + "pass@16": 0.6415841948801748, + "pass@32": 0.683597963618523, + "pass@64": 0.7232024375652686, + "pass@128": 0.761 + }, + "14": { + "pass@1": 0.322515625, + "pass@2": 0.3969320866141736, + "pass@4": 0.4603320167791523, + "pass@8": 0.5124417839066472, + "pass@16": 0.5564519977944874, + "pass@32": 0.5959754188029391, + "pass@64": 0.6315096364458914, + "pass@128": 0.662 + }, + "15": { + "pass@1": 0.2491484375, + "pass@2": 0.31848745078740154, + "pass@4": 0.3813791015185601, + "pass@8": 0.4372522014251215, + "pass@16": 0.48708832699515886, + "pass@32": 0.530787426728577, + "pass@64": 0.5700222181353979, + "pass@128": 0.609 + }, + "16": { + "pass@1": 0.182953125, + "pass@2": 0.23924778543307085, + "pass@4": 0.29406598115860527, + "pass@8": 0.3439873332042931, + "pass@16": 0.3878827007868039, + "pass@32": 0.42520837146889945, + "pass@64": 0.45657455458503415, + "pass@128": 0.487 + }, + "17": { + "pass@1": 0.179296875, + "pass@2": 0.23684793307086602, + "pass@4": 0.29352195425571787, + "pass@8": 0.34525731254854514, + "pass@16": 0.38998667678602095, + "pass@32": 0.42788088210582303, + "pass@64": 0.4601960116995837, + "pass@128": 0.489 + }, + "18": { + "pass@1": 0.1612421875, + "pass@2": 0.218044660433071, + "pass@4": 0.27296097075365566, + "pass@8": 0.3213106546616519, + "pass@16": 0.3635569729731848, + "pass@32": 0.40157602253748653, + "pass@64": 0.4348644408693857, + "pass@128": 0.462 + }, + "19": { + "pass@1": 0.1484296875, + "pass@2": 0.20265772637795254, + "pass@4": 0.25669816685414304, + "pass@8": 0.3068975188900042, + "pass@16": 0.35125819731138486, + "pass@32": 0.3894010373998276, + "pass@64": 0.4209719100949514, + "pass@128": 0.446 + }, + "20": { + "pass@1": 0.1396796875, + "pass@2": 0.19666437007874016, + "pass@4": 0.25631614735658004, + "pass@8": 0.31200426726149993, + "pass@16": 0.35897959314091565, + "pass@32": 0.3963617013747654, + "pass@64": 0.42784969980687954, + "pass@128": 0.456 + } + }, + "per_op_accuracy": { + "10": 0.80096875, + "2": 0.9294921875, + "3": 0.973203125, + "4": 0.96428125, + "5": 0.9377421875, + "6": 0.88653125, + "7": 0.8744140625, + "8": 0.84721875, + "9": 0.8024765625, + "11": 0.73865625, + "12": 0.580546875, + "13": 0.3827265625, + "14": 0.322515625, + "15": 0.2491484375, + "16": 0.182953125, + "17": 0.179296875, + "18": 0.1612421875, + "19": 0.1484296875, + "20": 0.1396796875 + }, + "per_op_avg_response_len": { + "10": 282.7000546875, + "2": 116.5491015625, + "3": 144.1264375, + "4": 160.84509375, + "5": 177.43053125, + "6": 198.4591015625, + "7": 222.6527109375, + "8": 241.4876953125, + "9": 259.6978359375, + "11": 300.53321875, + "12": 310.8298359375, + "13": 319.9635234375, + "14": 338.1668203125, + "15": 337.2906953125, + "16": 333.285203125, + "17": 336.919984375, + "18": 342.160640625, + "19": 331.1883359375, + "20": 336.4946328125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 282.7000546875, + "2": 116.5491015625, + "3": 144.1264375, + "4": 160.84509375, + "5": 177.43053125, + "6": 198.4591015625, + "7": 222.6527109375, + "8": 241.4876953125, + "9": 259.6978359375, + "11": 300.53321875, + "12": 310.8298359375, + "13": 319.9635234375, + "14": 338.1668203125, + "15": 337.2906953125, + "16": 333.285203125, + "17": 336.919984375, + "18": 342.160640625, + "19": 331.1883359375, + "20": 336.4946328125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 467039, + "answer_accuracy": 0.5767850438665824, + "avg_response_len": 254.83193245139108, + "resp_tokens_sum": 206344551, + "pass_at_k": { + "pass@1": 0.5767850438665824, + "pass@2": 0.6287842201040128, + "pass@4": 0.6708340107650578, + "pass@8": 0.7045834515269525, + "pass@16": 0.7321452790267492, + "pass@32": 0.7549469874285673, + "pass@64": 0.7737805410052538, + "pass@128": 0.7892823269048371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8082595870206489, + "pass@2": 0.8647357617355352, + "pass@4": 0.8976401423273417, + "pass@8": 0.9173720749762039, + "pass@16": 0.9291881918718146, + "pass@32": 0.9361090250317172, + "pass@64": 0.9411341859417028, + "pass@128": 0.943952802359882 + }, + "2": { + "pass@1": 0.9132593457943925, + "pass@2": 0.9192192827532073, + "pass@4": 0.9254616957926987, + "pass@8": 0.9326086884253552, + "pass@16": 0.9412402480297669, + "pass@32": 0.9502751094866533, + "pass@64": 0.9567691395661726, + "pass@128": 0.9595015576323987 + }, + "3": { + "pass@1": 0.9742689220183486, + "pass@2": 0.9795918213778326, + "pass@4": 0.9831441336461383, + "pass@8": 0.985580114929864, + "pass@16": 0.9875498957766747, + "pass@32": 0.9897440229097163, + "pass@64": 0.9922786486226282, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.9580322265625, + "pass@2": 0.9734970933809055, + "pass@4": 0.983606354869704, + "pass@8": 0.9907801440498372, + "pass@16": 0.9961476900698585, + "pass@32": 0.9991019833663184, + "pass@64": 0.9999517511298771, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9450892857142857, + "pass@2": 0.9643052430946132, + "pass@4": 0.9760748763547413, + "pass@8": 0.9826861265629934, + "pass@16": 0.9857626408128157, + "pass@32": 0.9875250624374391, + "pass@64": 0.9888449939948887, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.8729883982035929, + "pass@2": 0.9114836272337217, + "pass@4": 0.9363393078859157, + "pass@8": 0.9538367547277492, + "pass@16": 0.966520089086936, + "pass@32": 0.9746123891484606, + "pass@64": 0.979363177663249, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.8500600961538461, + "pass@2": 0.8949580382518755, + "pass@4": 0.9245661620330001, + "pass@8": 0.9468657824828772, + "pass@16": 0.9647895774794542, + "pass@32": 0.977670264977721, + "pass@64": 0.9850630781424923, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8358122996794872, + "pass@2": 0.8931357100494657, + "pass@4": 0.9322278360036731, + "pass@8": 0.9569910688945663, + "pass@16": 0.9718355835350385, + "pass@32": 0.9799294253747101, + "pass@64": 0.9839658835248849, + "pass@128": 0.9871794871794872 + }, + "9": { + "pass@1": 0.7866857394366197, + "pass@2": 0.8491883386935789, + "pass@4": 0.8889232061661304, + "pass@8": 0.9148338855375537, + "pass@16": 0.9351348435701646, + "pass@32": 0.9531606564413371, + "pass@64": 0.9678114627207628, + "pass@128": 0.9746478873239437 + }, + "11": { + "pass@1": 0.7324938031161473, + "pass@2": 0.8083786191474651, + "pass@4": 0.8603668985427808, + "pass@8": 0.8951230630417605, + "pass@16": 0.9192438398166457, + "pass@32": 0.9382015773061856, + "pass@64": 0.9543511427116407, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.5774629237288136, + "pass@2": 0.6690998404066018, + "pass@4": 0.7407094531615751, + "pass@8": 0.7936093234416746, + "pass@16": 0.8316808174882109, + "pass@32": 0.8604117124933197, + "pass@64": 0.8853741576695201, + "pass@128": 0.9124293785310734 + }, + "13": { + "pass@1": 0.37955525078369906, + "pass@2": 0.4627638813467281, + "pass@4": 0.5334049649553994, + "pass@8": 0.5908909515982965, + "pass@16": 0.6383243449452396, + "pass@32": 0.6785025472136269, + "pass@64": 0.7141232799137383, + "pass@128": 0.7492163009404389 + }, + "14": { + "pass@1": 0.3085572429906542, + "pass@2": 0.3798913643388032, + "pass@4": 0.4410535917239317, + "pass@8": 0.4929612888049262, + "pass@16": 0.5385958716440408, + "pass@32": 0.5820172888780842, + "pass@64": 0.6215417900208923, + "pass@128": 0.6510903426791277 + }, + "15": { + "pass@1": 0.2654895231213873, + "pass@2": 0.33615796106230944, + "pass@4": 0.3985344978553981, + "pass@8": 0.4518694766045028, + "pass@16": 0.4984802935633324, + "pass@32": 0.5388247206741961, + "pass@64": 0.5739738417672182, + "pass@128": 0.6098265895953757 + }, + "16": { + "pass@1": 0.17903225806451614, + "pass@2": 0.23331375412750824, + "pass@4": 0.2866873023936522, + "pass@8": 0.3375684152141981, + "pass@16": 0.3862721919687068, + "pass@32": 0.43088065063116227, + "pass@64": 0.4678904281213043, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.16988800578034682, + "pass@2": 0.22384549519821573, + "pass@4": 0.2748313011632217, + "pass@8": 0.3193916584849494, + "pass@16": 0.358615204863562, + "pass@32": 0.39272698262401673, + "pass@64": 0.4211641810459257, + "pass@128": 0.44508670520231214 + }, + "18": { + "pass@1": 0.15439732142857143, + "pass@2": 0.21321674634420695, + "pass@4": 0.2686094804220902, + "pass@8": 0.3154271003708264, + "pass@16": 0.3563084554457416, + "pass@32": 0.3923554436588413, + "pass@64": 0.4227785903208911, + "pass@128": 0.44571428571428573 + }, + "19": { + "pass@1": 0.14902402402402404, + "pass@2": 0.20243558026530464, + "pass@4": 0.25758560106788453, + "pass@8": 0.3117411757269937, + "pass@16": 0.3584763615622298, + "pass@32": 0.3972802657507109, + "pass@64": 0.43147037921139075, + "pass@128": 0.4594594594594595 + }, + "20": { + "pass@1": 0.14071884384384384, + "pass@2": 0.20093001466032961, + "pass@4": 0.26321763270582166, + "pass@8": 0.32019753114376637, + "pass@16": 0.367272720530513, + "pass@32": 0.4034641339305123, + "pass@64": 0.43107760881473256, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 479150, + "answer_accuracy": 0.5835322486360094, + "avg_response_len": 265.8948083106002, + "resp_tokens_sum": 218331545, + "pass_at_k": { + "pass@1": 0.5835322486360094, + "pass@2": 0.633331217281108, + "pass@4": 0.6740935293990435, + "pass@8": 0.7072492136270658, + "pass@16": 0.7340684890976236, + "pass@32": 0.7561529413080658, + "pass@64": 0.7747532859097668, + "pass@128": 0.7906469212782541 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.793472009202454, + "pass@2": 0.8493992590937638, + "pass@4": 0.8877754004867485, + "pass@8": 0.9141527086586699, + "pass@16": 0.9311651079164657, + "pass@32": 0.9426145462430192, + "pass@64": 0.9516821714118398, + "pass@128": 0.9601226993865031 + }, + "2": { + "pass@1": 0.9402783054711246, + "pass@2": 0.9459940346552426, + "pass@4": 0.9491347127391752, + "pass@8": 0.9519259878204085, + "pass@16": 0.9553404745234914, + "pass@32": 0.9597092653562085, + "pass@64": 0.9640150627253267, + "pass@128": 0.9665653495440729 + }, + "3": { + "pass@1": 0.9669981060606061, + "pass@2": 0.976074847888332, + "pass@4": 0.9835954573860082, + "pass@8": 0.9887292837477384, + "pass@16": 0.990873589188666, + "pass@32": 0.9916485785890059, + "pass@64": 0.9924242355459135, + "pass@128": 0.9939393939393939 + }, + "4": { + "pass@1": 0.965051775147929, + "pass@2": 0.9720598384429018, + "pass@4": 0.9769207664685405, + "pass@8": 0.9815084547831039, + "pass@16": 0.9856876197176295, + "pass@32": 0.9890133314572516, + "pass@64": 0.990732185832648, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.9434329710144927, + "pass@2": 0.9612539940659597, + "pass@4": 0.9702148456986353, + "pass@8": 0.9758259545604623, + "pass@16": 0.9803524893764493, + "pass@32": 0.9842124262842136, + "pass@64": 0.988043997616301, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.8868642550143266, + "pass@2": 0.9194193054396137, + "pass@4": 0.939174852516644, + "pass@8": 0.952673887441346, + "pass@16": 0.9630499625273448, + "pass@32": 0.9720429085728259, + "pass@64": 0.9808653443318179, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.8915281791907514, + "pass@2": 0.92678630353648, + "pass@4": 0.950118883170673, + "pass@8": 0.9654538312470585, + "pass@16": 0.9751158531352345, + "pass@32": 0.9812282655305531, + "pass@64": 0.9852157051970267, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.8579005281690141, + "pass@2": 0.8997251718975272, + "pass@4": 0.9258677762110714, + "pass@8": 0.9437238727011946, + "pass@16": 0.9556631355474996, + "pass@32": 0.9640877913679654, + "pass@64": 0.9721959531135995, + "pass@128": 0.9802816901408451 + }, + "9": { + "pass@1": 0.7888285928143712, + "pass@2": 0.851565815219954, + "pass@4": 0.8928656674277982, + "pass@8": 0.9188092984095537, + "pass@16": 0.9350753145380785, + "pass@32": 0.9466458701980125, + "pass@64": 0.9550391747311027, + "pass@128": 0.9610778443113772 + }, + "11": { + "pass@1": 0.7130788780120482, + "pass@2": 0.7939841304193148, + "pass@4": 0.8469520404452451, + "pass@8": 0.8823839807042658, + "pass@16": 0.9067006633768472, + "pass@32": 0.9217781709792446, + "pass@64": 0.9309741583437446, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.5889988244514106, + "pass@2": 0.6738543029397968, + "pass@4": 0.7364737050893712, + "pass@8": 0.7833416397619604, + "pass@16": 0.8210170887885584, + "pass@32": 0.8530612652715264, + "pass@64": 0.8769674396793894, + "pass@128": 0.8934169278996865 + }, + "13": { + "pass@1": 0.37489058123249297, + "pass@2": 0.45928484582809503, + "pass@4": 0.5333312377444418, + "pass@8": 0.5955567982367712, + "pass@16": 0.6468370691536759, + "pass@32": 0.6914674094335568, + "pass@64": 0.7330451891460972, + "pass@128": 0.7675070028011205 + }, + "14": { + "pass@1": 0.3155381944444444, + "pass@2": 0.3891212817147854, + "pass@4": 0.453302549078295, + "pass@8": 0.507271840880927, + "pass@16": 0.5527430887420266, + "pass@32": 0.5921199769806936, + "pass@64": 0.6270367990392346, + "pass@128": 0.6578947368421053 + }, + "15": { + "pass@1": 0.2109848484848485, + "pass@2": 0.27266687544738727, + "pass@4": 0.3313393129836044, + "pass@8": 0.3861669387313994, + "pass@16": 0.43703563287263564, + "pass@32": 0.4823231852380329, + "pass@64": 0.5239700082246377, + "pass@128": 0.5666666666666667 + }, + "16": { + "pass@1": 0.1877003205128205, + "pass@2": 0.2486519112995491, + "pass@4": 0.30775032955282305, + "pass@8": 0.3612737419871499, + "pass@16": 0.40587271912690925, + "pass@32": 0.43920306525713915, + "pass@64": 0.4626650339424816, + "pass@128": 0.48148148148148145 + }, + "17": { + "pass@1": 0.19596758540372672, + "pass@2": 0.25669413605907954, + "pass@4": 0.31698872724372196, + "pass@8": 0.37355683740742207, + "pass@16": 0.4234816279489652, + "pass@32": 0.46557943249256295, + "pass@64": 0.4995003793338017, + "pass@128": 0.5279503105590062 + }, + "18": { + "pass@1": 0.17325367647058823, + "pass@2": 0.23055631947660954, + "pass@4": 0.28691845293235396, + "pass@8": 0.3373942841337351, + "pass@16": 0.38137141561201654, + "pass@32": 0.4221056033676291, + "pass@64": 0.45916635273358053, + "pass@128": 0.48823529411764705 + }, + "19": { + "pass@1": 0.15171595982142858, + "pass@2": 0.20929048907949008, + "pass@4": 0.263858470258182, + "pass@8": 0.31035082337712583, + "pass@16": 0.3495046566647495, + "pass@32": 0.3829455856819846, + "pass@64": 0.41029909549921423, + "pass@128": 0.43154761904761907 + }, + "20": { + "pass@1": 0.13075411676646706, + "pass@2": 0.18799286269979726, + "pass@4": 0.24704597469852202, + "pass@8": 0.30007091187136975, + "pass@16": 0.34343731208913214, + "pass@32": 0.37867318149442, + "pass@64": 0.41008857357648265, + "pass@128": 0.44011976047904194 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 474806, + "answer_accuracy": 0.5926540781274964, + "avg_response_len": 283.2720008188209, + "resp_tokens_sum": 226943930, + "pass_at_k": { + "pass@1": 0.5926540781274964, + "pass@2": 0.6393053844982961, + "pass@4": 0.6776084430021845, + "pass@8": 0.7091740146867663, + "pass@16": 0.7353773789381717, + "pass@32": 0.7576234233134128, + "pass@64": 0.7773517089235912, + "pass@128": 0.7961335676625659 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8008861940298507, + "pass@2": 0.8529277823481023, + "pass@4": 0.8856773796932103, + "pass@8": 0.9071853613797309, + "pass@16": 0.922739041071155, + "pass@32": 0.93425539456196, + "pass@64": 0.9436669840253802, + "pass@128": 0.9522388059701492 + }, + "2": { + "pass@1": 0.9342410714285714, + "pass@2": 0.9433903965129361, + "pass@4": 0.9506338338957628, + "pass@8": 0.9568800274460122, + "pass@16": 0.9613865780552503, + "pass@32": 0.964584323870877, + "pass@64": 0.968582583102937, + "pass@128": 0.9742857142857143 + }, + "3": { + "pass@1": 0.978156887755102, + "pass@2": 0.9854187949082893, + "pass@4": 0.9897041988738288, + "pass@8": 0.9925314831336094, + "pass@16": 0.9949651689420985, + "pass@32": 0.9969385354552436, + "pass@64": 0.9984540418035831, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9693667763157895, + "pass@2": 0.9765919987797581, + "pass@4": 0.9815825623222536, + "pass@8": 0.9844233166382033, + "pass@16": 0.9863282582142694, + "pass@32": 0.9881606478134689, + "pass@64": 0.9896775917125719, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.9251608455882353, + "pass@2": 0.9427270292959704, + "pass@4": 0.9546608984538691, + "pass@8": 0.9638809428859377, + "pass@16": 0.9718929334226678, + "pass@32": 0.9793735683916001, + "pass@64": 0.9842883511825552, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9004337539432177, + "pass@2": 0.9270256340197225, + "pass@4": 0.9445922334432167, + "pass@8": 0.9583119300991347, + "pass@16": 0.9698035441893214, + "pass@32": 0.9800977802382987, + "pass@64": 0.98842612091269, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.8817246835443038, + "pass@2": 0.9180894454549986, + "pass@4": 0.9400556796500119, + "pass@8": 0.954207112641725, + "pass@16": 0.9646399152221633, + "pass@32": 0.9730050298001863, + "pass@64": 0.980426538061284, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.8465183933933934, + "pass@2": 0.8873936682745736, + "pass@4": 0.9118258303297669, + "pass@8": 0.927138893708802, + "pass@16": 0.9391865911461822, + "pass@32": 0.9500528993936376, + "pass@64": 0.9598739626007716, + "pass@128": 0.9669669669669669 + }, + "9": { + "pass@1": 0.8351587620578779, + "pass@2": 0.8831090019495152, + "pass@4": 0.9114603200725307, + "pass@8": 0.9301541604057866, + "pass@16": 0.9436524287358187, + "pass@32": 0.9539569362661837, + "pass@64": 0.9625513320792103, + "pass@128": 0.9710610932475884 + }, + "11": { + "pass@1": 0.7725198412698413, + "pass@2": 0.8347190976127983, + "pass@4": 0.873448953107052, + "pass@8": 0.8999402716418193, + "pass@16": 0.921179742389368, + "pass@32": 0.9396846498700138, + "pass@64": 0.9556165664314662, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5756402905198776, + "pass@2": 0.660415387440102, + "pass@4": 0.7272169780382953, + "pass@8": 0.7786842246092575, + "pass@16": 0.8154482412117705, + "pass@32": 0.8406595602173778, + "pass@64": 0.8608165987491957, + "pass@128": 0.8807339449541285 + }, + "13": { + "pass@1": 0.39448302469135804, + "pass@2": 0.47080356274910107, + "pass@4": 0.5377397492558798, + "pass@8": 0.59349137895756, + "pass@16": 0.6390058492431547, + "pass@32": 0.6799437836098674, + "pass@64": 0.7212962307025573, + "pass@128": 0.7654320987654321 + }, + "14": { + "pass@1": 0.3428922477744807, + "pass@2": 0.4210904460384586, + "pass@4": 0.4858289081631857, + "pass@8": 0.5362440374450701, + "pass@16": 0.5772242927210018, + "pass@32": 0.6131834924204059, + "pass@64": 0.6455434319815009, + "pass@128": 0.6765578635014837 + }, + "15": { + "pass@1": 0.27056809413580246, + "pass@2": 0.34628619556236, + "pass@4": 0.4140252838765522, + "pass@8": 0.4736736812919801, + "pass@16": 0.5259024276983834, + "pass@32": 0.5715661180455387, + "pass@64": 0.6127073029932402, + "pass@128": 0.6512345679012346 + }, + "16": { + "pass@1": 0.1816233407079646, + "pass@2": 0.23493717046431134, + "pass@4": 0.28664469540864906, + "pass@8": 0.3319588231575276, + "pass@16": 0.3707286043155161, + "pass@32": 0.40553125034803245, + "pass@64": 0.439920634081472, + "pass@128": 0.4808259587020649 + }, + "17": { + "pass@1": 0.1729339231927711, + "pass@2": 0.2311502407266863, + "pass@4": 0.290240824942063, + "pass@8": 0.34476655743241874, + "pass@16": 0.39019497501102995, + "pass@32": 0.4279541230581569, + "pass@64": 0.4627532617837625, + "pass@128": 0.49698795180722893 + }, + "18": { + "pass@1": 0.15579637096774193, + "pass@2": 0.20977306642113286, + "pass@4": 0.26256573744814155, + "pass@8": 0.310313267504495, + "pass@16": 0.3522023621260945, + "pass@32": 0.3894700390706386, + "pass@64": 0.4218560462182471, + "pass@128": 0.45161290322580644 + }, + "19": { + "pass@1": 0.14449584592145015, + "pass@2": 0.19614825927159402, + "pass@4": 0.24853690541325835, + "pass@8": 0.2985191260368612, + "pass@16": 0.34577644795107754, + "pass@32": 0.3880271665126702, + "pass@64": 0.4212440414798256, + "pass@128": 0.4471299093655589 + }, + "20": { + "pass@1": 0.1475929054054054, + "pass@2": 0.2010962734387931, + "pass@4": 0.2587126730217282, + "pass@8": 0.31578019467143653, + "pass@16": 0.3662754203197754, + "pass@32": 0.40700090737768396, + "pass@64": 0.4424362537448303, + "pass@128": 0.4774774774774775 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/summary.csv b/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..7cfbf4efb377ba420f974f6283aec528dde06b67 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21808,0.890703125,0.30851953125,0.5842907072368421,0.0,0.0,0.0,200.43872916666666,328.6832890625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/summary.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..15eaf36091769d2c64f7a69604ed6b1a2a43ec8d --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block32/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21808", + "id_acc": 0.890703125, + "ood_acc": 0.30851953125, + "total_acc": 0.5842907072368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.43872916666666, + "ood_avg_resp_len": 328.6832890625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 467039, + "answer_accuracy": 0.5767850438665824, + "avg_response_len": 254.83193245139108, + "resp_tokens_sum": 206344551, + "pass_at_k": { + "pass@1": 0.5767850438665824, + "pass@2": 0.6287842201040128, + "pass@4": 0.6708340107650578, + "pass@8": 0.7045834515269525, + "pass@16": 0.7321452790267492, + "pass@32": 0.7549469874285673, + "pass@64": 0.7737805410052538, + "pass@128": 0.7892823269048371 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8082595870206489, + "pass@2": 0.8647357617355352, + "pass@4": 0.8976401423273417, + "pass@8": 0.9173720749762039, + "pass@16": 0.9291881918718146, + "pass@32": 0.9361090250317172, + "pass@64": 0.9411341859417028, + "pass@128": 0.943952802359882 + }, + "2": { + "pass@1": 0.9132593457943925, + "pass@2": 0.9192192827532073, + "pass@4": 0.9254616957926987, + "pass@8": 0.9326086884253552, + "pass@16": 0.9412402480297669, + "pass@32": 0.9502751094866533, + "pass@64": 0.9567691395661726, + "pass@128": 0.9595015576323987 + }, + "3": { + "pass@1": 0.9742689220183486, + "pass@2": 0.9795918213778326, + "pass@4": 0.9831441336461383, + "pass@8": 0.985580114929864, + "pass@16": 0.9875498957766747, + "pass@32": 0.9897440229097163, + "pass@64": 0.9922786486226282, + "pass@128": 0.9938837920489296 + }, + "4": { + "pass@1": 0.9580322265625, + "pass@2": 0.9734970933809055, + "pass@4": 0.983606354869704, + "pass@8": 0.9907801440498372, + "pass@16": 0.9961476900698585, + "pass@32": 0.9991019833663184, + "pass@64": 0.9999517511298771, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9450892857142857, + "pass@2": 0.9643052430946132, + "pass@4": 0.9760748763547413, + "pass@8": 0.9826861265629934, + "pass@16": 0.9857626408128157, + "pass@32": 0.9875250624374391, + "pass@64": 0.9888449939948887, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.8729883982035929, + "pass@2": 0.9114836272337217, + "pass@4": 0.9363393078859157, + "pass@8": 0.9538367547277492, + "pass@16": 0.966520089086936, + "pass@32": 0.9746123891484606, + "pass@64": 0.979363177663249, + "pass@128": 0.9820359281437125 + }, + "7": { + "pass@1": 0.8500600961538461, + "pass@2": 0.8949580382518755, + "pass@4": 0.9245661620330001, + "pass@8": 0.9468657824828772, + "pass@16": 0.9647895774794542, + "pass@32": 0.977670264977721, + "pass@64": 0.9850630781424923, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.8358122996794872, + "pass@2": 0.8931357100494657, + "pass@4": 0.9322278360036731, + "pass@8": 0.9569910688945663, + "pass@16": 0.9718355835350385, + "pass@32": 0.9799294253747101, + "pass@64": 0.9839658835248849, + "pass@128": 0.9871794871794872 + }, + "9": { + "pass@1": 0.7866857394366197, + "pass@2": 0.8491883386935789, + "pass@4": 0.8889232061661304, + "pass@8": 0.9148338855375537, + "pass@16": 0.9351348435701646, + "pass@32": 0.9531606564413371, + "pass@64": 0.9678114627207628, + "pass@128": 0.9746478873239437 + }, + "11": { + "pass@1": 0.7324938031161473, + "pass@2": 0.8083786191474651, + "pass@4": 0.8603668985427808, + "pass@8": 0.8951230630417605, + "pass@16": 0.9192438398166457, + "pass@32": 0.9382015773061856, + "pass@64": 0.9543511427116407, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.5774629237288136, + "pass@2": 0.6690998404066018, + "pass@4": 0.7407094531615751, + "pass@8": 0.7936093234416746, + "pass@16": 0.8316808174882109, + "pass@32": 0.8604117124933197, + "pass@64": 0.8853741576695201, + "pass@128": 0.9124293785310734 + }, + "13": { + "pass@1": 0.37955525078369906, + "pass@2": 0.4627638813467281, + "pass@4": 0.5334049649553994, + "pass@8": 0.5908909515982965, + "pass@16": 0.6383243449452396, + "pass@32": 0.6785025472136269, + "pass@64": 0.7141232799137383, + "pass@128": 0.7492163009404389 + }, + "14": { + "pass@1": 0.3085572429906542, + "pass@2": 0.3798913643388032, + "pass@4": 0.4410535917239317, + "pass@8": 0.4929612888049262, + "pass@16": 0.5385958716440408, + "pass@32": 0.5820172888780842, + "pass@64": 0.6215417900208923, + "pass@128": 0.6510903426791277 + }, + "15": { + "pass@1": 0.2654895231213873, + "pass@2": 0.33615796106230944, + "pass@4": 0.3985344978553981, + "pass@8": 0.4518694766045028, + "pass@16": 0.4984802935633324, + "pass@32": 0.5388247206741961, + "pass@64": 0.5739738417672182, + "pass@128": 0.6098265895953757 + }, + "16": { + "pass@1": 0.17903225806451614, + "pass@2": 0.23331375412750824, + "pass@4": 0.2866873023936522, + "pass@8": 0.3375684152141981, + "pass@16": 0.3862721919687068, + "pass@32": 0.43088065063116227, + "pass@64": 0.4678904281213043, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.16988800578034682, + "pass@2": 0.22384549519821573, + "pass@4": 0.2748313011632217, + "pass@8": 0.3193916584849494, + "pass@16": 0.358615204863562, + "pass@32": 0.39272698262401673, + "pass@64": 0.4211641810459257, + "pass@128": 0.44508670520231214 + }, + "18": { + "pass@1": 0.15439732142857143, + "pass@2": 0.21321674634420695, + "pass@4": 0.2686094804220902, + "pass@8": 0.3154271003708264, + "pass@16": 0.3563084554457416, + "pass@32": 0.3923554436588413, + "pass@64": 0.4227785903208911, + "pass@128": 0.44571428571428573 + }, + "19": { + "pass@1": 0.14902402402402404, + "pass@2": 0.20243558026530464, + "pass@4": 0.25758560106788453, + "pass@8": 0.3117411757269937, + "pass@16": 0.3584763615622298, + "pass@32": 0.3972802657507109, + "pass@64": 0.43147037921139075, + "pass@128": 0.4594594594594595 + }, + "20": { + "pass@1": 0.14071884384384384, + "pass@2": 0.20093001466032961, + "pass@4": 0.26321763270582166, + "pass@8": 0.32019753114376637, + "pass@16": 0.367272720530513, + "pass@32": 0.4034641339305123, + "pass@64": 0.43107760881473256, + "pass@128": 0.45045045045045046 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 479150, + "answer_accuracy": 0.5835322486360094, + "avg_response_len": 265.8948083106002, + "resp_tokens_sum": 218331545, + "pass_at_k": { + "pass@1": 0.5835322486360094, + "pass@2": 0.633331217281108, + "pass@4": 0.6740935293990435, + "pass@8": 0.7072492136270658, + "pass@16": 0.7340684890976236, + "pass@32": 0.7561529413080658, + "pass@64": 0.7747532859097668, + "pass@128": 0.7906469212782541 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.793472009202454, + "pass@2": 0.8493992590937638, + "pass@4": 0.8877754004867485, + "pass@8": 0.9141527086586699, + "pass@16": 0.9311651079164657, + "pass@32": 0.9426145462430192, + "pass@64": 0.9516821714118398, + "pass@128": 0.9601226993865031 + }, + "2": { + "pass@1": 0.9402783054711246, + "pass@2": 0.9459940346552426, + "pass@4": 0.9491347127391752, + "pass@8": 0.9519259878204085, + "pass@16": 0.9553404745234914, + "pass@32": 0.9597092653562085, + "pass@64": 0.9640150627253267, + "pass@128": 0.9665653495440729 + }, + "3": { + "pass@1": 0.9669981060606061, + "pass@2": 0.976074847888332, + "pass@4": 0.9835954573860082, + "pass@8": 0.9887292837477384, + "pass@16": 0.990873589188666, + "pass@32": 0.9916485785890059, + "pass@64": 0.9924242355459135, + "pass@128": 0.9939393939393939 + }, + "4": { + "pass@1": 0.965051775147929, + "pass@2": 0.9720598384429018, + "pass@4": 0.9769207664685405, + "pass@8": 0.9815084547831039, + "pass@16": 0.9856876197176295, + "pass@32": 0.9890133314572516, + "pass@64": 0.990732185832648, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.9434329710144927, + "pass@2": 0.9612539940659597, + "pass@4": 0.9702148456986353, + "pass@8": 0.9758259545604623, + "pass@16": 0.9803524893764493, + "pass@32": 0.9842124262842136, + "pass@64": 0.988043997616301, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.8868642550143266, + "pass@2": 0.9194193054396137, + "pass@4": 0.939174852516644, + "pass@8": 0.952673887441346, + "pass@16": 0.9630499625273448, + "pass@32": 0.9720429085728259, + "pass@64": 0.9808653443318179, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.8915281791907514, + "pass@2": 0.92678630353648, + "pass@4": 0.950118883170673, + "pass@8": 0.9654538312470585, + "pass@16": 0.9751158531352345, + "pass@32": 0.9812282655305531, + "pass@64": 0.9852157051970267, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.8579005281690141, + "pass@2": 0.8997251718975272, + "pass@4": 0.9258677762110714, + "pass@8": 0.9437238727011946, + "pass@16": 0.9556631355474996, + "pass@32": 0.9640877913679654, + "pass@64": 0.9721959531135995, + "pass@128": 0.9802816901408451 + }, + "9": { + "pass@1": 0.7888285928143712, + "pass@2": 0.851565815219954, + "pass@4": 0.8928656674277982, + "pass@8": 0.9188092984095537, + "pass@16": 0.9350753145380785, + "pass@32": 0.9466458701980125, + "pass@64": 0.9550391747311027, + "pass@128": 0.9610778443113772 + }, + "11": { + "pass@1": 0.7130788780120482, + "pass@2": 0.7939841304193148, + "pass@4": 0.8469520404452451, + "pass@8": 0.8823839807042658, + "pass@16": 0.9067006633768472, + "pass@32": 0.9217781709792446, + "pass@64": 0.9309741583437446, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.5889988244514106, + "pass@2": 0.6738543029397968, + "pass@4": 0.7364737050893712, + "pass@8": 0.7833416397619604, + "pass@16": 0.8210170887885584, + "pass@32": 0.8530612652715264, + "pass@64": 0.8769674396793894, + "pass@128": 0.8934169278996865 + }, + "13": { + "pass@1": 0.37489058123249297, + "pass@2": 0.45928484582809503, + "pass@4": 0.5333312377444418, + "pass@8": 0.5955567982367712, + "pass@16": 0.6468370691536759, + "pass@32": 0.6914674094335568, + "pass@64": 0.7330451891460972, + "pass@128": 0.7675070028011205 + }, + "14": { + "pass@1": 0.3155381944444444, + "pass@2": 0.3891212817147854, + "pass@4": 0.453302549078295, + "pass@8": 0.507271840880927, + "pass@16": 0.5527430887420266, + "pass@32": 0.5921199769806936, + "pass@64": 0.6270367990392346, + "pass@128": 0.6578947368421053 + }, + "15": { + "pass@1": 0.2109848484848485, + "pass@2": 0.27266687544738727, + "pass@4": 0.3313393129836044, + "pass@8": 0.3861669387313994, + "pass@16": 0.43703563287263564, + "pass@32": 0.4823231852380329, + "pass@64": 0.5239700082246377, + "pass@128": 0.5666666666666667 + }, + "16": { + "pass@1": 0.1877003205128205, + "pass@2": 0.2486519112995491, + "pass@4": 0.30775032955282305, + "pass@8": 0.3612737419871499, + "pass@16": 0.40587271912690925, + "pass@32": 0.43920306525713915, + "pass@64": 0.4626650339424816, + "pass@128": 0.48148148148148145 + }, + "17": { + "pass@1": 0.19596758540372672, + "pass@2": 0.25669413605907954, + "pass@4": 0.31698872724372196, + "pass@8": 0.37355683740742207, + "pass@16": 0.4234816279489652, + "pass@32": 0.46557943249256295, + "pass@64": 0.4995003793338017, + "pass@128": 0.5279503105590062 + }, + "18": { + "pass@1": 0.17325367647058823, + "pass@2": 0.23055631947660954, + "pass@4": 0.28691845293235396, + "pass@8": 0.3373942841337351, + "pass@16": 0.38137141561201654, + "pass@32": 0.4221056033676291, + "pass@64": 0.45916635273358053, + "pass@128": 0.48823529411764705 + }, + "19": { + "pass@1": 0.15171595982142858, + "pass@2": 0.20929048907949008, + "pass@4": 0.263858470258182, + "pass@8": 0.31035082337712583, + "pass@16": 0.3495046566647495, + "pass@32": 0.3829455856819846, + "pass@64": 0.41029909549921423, + "pass@128": 0.43154761904761907 + }, + "20": { + "pass@1": 0.13075411676646706, + "pass@2": 0.18799286269979726, + "pass@4": 0.24704597469852202, + "pass@8": 0.30007091187136975, + "pass@16": 0.34343731208913214, + "pass@32": 0.37867318149442, + "pass@64": 0.41008857357648265, + "pass@128": 0.44011976047904194 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 474806, + "answer_accuracy": 0.5926540781274964, + "avg_response_len": 283.2720008188209, + "resp_tokens_sum": 226943930, + "pass_at_k": { + "pass@1": 0.5926540781274964, + "pass@2": 0.6393053844982961, + "pass@4": 0.6776084430021845, + "pass@8": 0.7091740146867663, + "pass@16": 0.7353773789381717, + "pass@32": 0.7576234233134128, + "pass@64": 0.7773517089235912, + "pass@128": 0.7961335676625659 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8008861940298507, + "pass@2": 0.8529277823481023, + "pass@4": 0.8856773796932103, + "pass@8": 0.9071853613797309, + "pass@16": 0.922739041071155, + "pass@32": 0.93425539456196, + "pass@64": 0.9436669840253802, + "pass@128": 0.9522388059701492 + }, + "2": { + "pass@1": 0.9342410714285714, + "pass@2": 0.9433903965129361, + "pass@4": 0.9506338338957628, + "pass@8": 0.9568800274460122, + "pass@16": 0.9613865780552503, + "pass@32": 0.964584323870877, + "pass@64": 0.968582583102937, + "pass@128": 0.9742857142857143 + }, + "3": { + "pass@1": 0.978156887755102, + "pass@2": 0.9854187949082893, + "pass@4": 0.9897041988738288, + "pass@8": 0.9925314831336094, + "pass@16": 0.9949651689420985, + "pass@32": 0.9969385354552436, + "pass@64": 0.9984540418035831, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9693667763157895, + "pass@2": 0.9765919987797581, + "pass@4": 0.9815825623222536, + "pass@8": 0.9844233166382033, + "pass@16": 0.9863282582142694, + "pass@32": 0.9881606478134689, + "pass@64": 0.9896775917125719, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.9251608455882353, + "pass@2": 0.9427270292959704, + "pass@4": 0.9546608984538691, + "pass@8": 0.9638809428859377, + "pass@16": 0.9718929334226678, + "pass@32": 0.9793735683916001, + "pass@64": 0.9842883511825552, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9004337539432177, + "pass@2": 0.9270256340197225, + "pass@4": 0.9445922334432167, + "pass@8": 0.9583119300991347, + "pass@16": 0.9698035441893214, + "pass@32": 0.9800977802382987, + "pass@64": 0.98842612091269, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.8817246835443038, + "pass@2": 0.9180894454549986, + "pass@4": 0.9400556796500119, + "pass@8": 0.954207112641725, + "pass@16": 0.9646399152221633, + "pass@32": 0.9730050298001863, + "pass@64": 0.980426538061284, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.8465183933933934, + "pass@2": 0.8873936682745736, + "pass@4": 0.9118258303297669, + "pass@8": 0.927138893708802, + "pass@16": 0.9391865911461822, + "pass@32": 0.9500528993936376, + "pass@64": 0.9598739626007716, + "pass@128": 0.9669669669669669 + }, + "9": { + "pass@1": 0.8351587620578779, + "pass@2": 0.8831090019495152, + "pass@4": 0.9114603200725307, + "pass@8": 0.9301541604057866, + "pass@16": 0.9436524287358187, + "pass@32": 0.9539569362661837, + "pass@64": 0.9625513320792103, + "pass@128": 0.9710610932475884 + }, + "11": { + "pass@1": 0.7725198412698413, + "pass@2": 0.8347190976127983, + "pass@4": 0.873448953107052, + "pass@8": 0.8999402716418193, + "pass@16": 0.921179742389368, + "pass@32": 0.9396846498700138, + "pass@64": 0.9556165664314662, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.5756402905198776, + "pass@2": 0.660415387440102, + "pass@4": 0.7272169780382953, + "pass@8": 0.7786842246092575, + "pass@16": 0.8154482412117705, + "pass@32": 0.8406595602173778, + "pass@64": 0.8608165987491957, + "pass@128": 0.8807339449541285 + }, + "13": { + "pass@1": 0.39448302469135804, + "pass@2": 0.47080356274910107, + "pass@4": 0.5377397492558798, + "pass@8": 0.59349137895756, + "pass@16": 0.6390058492431547, + "pass@32": 0.6799437836098674, + "pass@64": 0.7212962307025573, + "pass@128": 0.7654320987654321 + }, + "14": { + "pass@1": 0.3428922477744807, + "pass@2": 0.4210904460384586, + "pass@4": 0.4858289081631857, + "pass@8": 0.5362440374450701, + "pass@16": 0.5772242927210018, + "pass@32": 0.6131834924204059, + "pass@64": 0.6455434319815009, + "pass@128": 0.6765578635014837 + }, + "15": { + "pass@1": 0.27056809413580246, + "pass@2": 0.34628619556236, + "pass@4": 0.4140252838765522, + "pass@8": 0.4736736812919801, + "pass@16": 0.5259024276983834, + "pass@32": 0.5715661180455387, + "pass@64": 0.6127073029932402, + "pass@128": 0.6512345679012346 + }, + "16": { + "pass@1": 0.1816233407079646, + "pass@2": 0.23493717046431134, + "pass@4": 0.28664469540864906, + "pass@8": 0.3319588231575276, + "pass@16": 0.3707286043155161, + "pass@32": 0.40553125034803245, + "pass@64": 0.439920634081472, + "pass@128": 0.4808259587020649 + }, + "17": { + "pass@1": 0.1729339231927711, + "pass@2": 0.2311502407266863, + "pass@4": 0.290240824942063, + "pass@8": 0.34476655743241874, + "pass@16": 0.39019497501102995, + "pass@32": 0.4279541230581569, + "pass@64": 0.4627532617837625, + "pass@128": 0.49698795180722893 + }, + "18": { + "pass@1": 0.15579637096774193, + "pass@2": 0.20977306642113286, + "pass@4": 0.26256573744814155, + "pass@8": 0.310313267504495, + "pass@16": 0.3522023621260945, + "pass@32": 0.3894700390706386, + "pass@64": 0.4218560462182471, + "pass@128": 0.45161290322580644 + }, + "19": { + "pass@1": 0.14449584592145015, + "pass@2": 0.19614825927159402, + "pass@4": 0.24853690541325835, + "pass@8": 0.2985191260368612, + "pass@16": 0.34577644795107754, + "pass@32": 0.3880271665126702, + "pass@64": 0.4212440414798256, + "pass@128": 0.4471299093655589 + }, + "20": { + "pass@1": 0.1475929054054054, + "pass@2": 0.2010962734387931, + "pass@4": 0.2587126730217282, + "pass@8": 0.31578019467143653, + "pass@16": 0.3662754203197754, + "pass@32": 0.40700090737768396, + "pass@64": 0.4424362537448303, + "pass@128": 0.4774774774774775 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/checkpoint-21808_metrics.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/checkpoint-21808_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e8520dffa5992e2bf690687472656611c6a73823 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/checkpoint-21808_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21808", + "id": { + "answer_accuracy": 0.9517916666666667, + "avg_response_len": 196.2878142361111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9517916666666667, + "pass@2": 0.966083948490801, + "pass@4": 0.9741246513248322, + "pass@8": 0.9794706282917761, + "pass@16": 0.9834737064177047, + "pass@32": 0.986842192184215, + "pass@64": 0.9896918390239997, + "pass@128": 0.9916666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8819375, + "pass@2": 0.9187347440944885, + "pass@4": 0.9413814194788149, + "pass@8": 0.9562699368944456, + "pass@16": 0.966554411117548, + "pass@32": 0.9739934405917635, + "pass@64": 0.9796453043075861, + "pass@128": 0.984 + }, + "2": { + "pass@1": 0.9615390625, + "pass@2": 0.966419414370079, + "pass@4": 0.9699190299025123, + "pass@8": 0.9725064708486456, + "pass@16": 0.9749029250306379, + "pass@32": 0.9774983735324464, + "pass@64": 0.9802985458001466, + "pass@128": 0.983 + }, + "3": { + "pass@1": 0.9939140625, + "pass@2": 0.9959056348425196, + "pass@4": 0.9970321550431197, + "pass@8": 0.9977359346939182, + "pass@16": 0.9982835963498219, + "pass@32": 0.998781158105662, + "pass@64": 0.998990332213311, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.9805546875, + "pass@2": 0.9841828248031499, + "pass@4": 0.9869512231908513, + "pass@8": 0.9893844569008565, + "pass@16": 0.9912527669160224, + "pass@32": 0.9927349388162231, + "pass@64": 0.994127141737953, + "pass@128": 0.995 + }, + "5": { + "pass@1": 0.952046875, + "pass@2": 0.9629343011811029, + "pass@4": 0.9698748169291337, + "pass@8": 0.974794103858124, + "pass@16": 0.9787874008751346, + "pass@32": 0.9824861874497502, + "pass@64": 0.9858271655075969, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.9740234375, + "pass@2": 0.9855746801181101, + "pass@4": 0.9907253421447321, + "pass@8": 0.9930694421828948, + "pass@16": 0.9941911081127991, + "pass@32": 0.9947386956493451, + "pass@64": 0.9949710417286846, + "pass@128": 0.995 + }, + "7": { + "pass@1": 0.9543125, + "pass@2": 0.9663805364173231, + "pass@4": 0.9727133587364074, + "pass@8": 0.9773424133620466, + "pass@16": 0.9813153290055051, + "pass@32": 0.9849896042653357, + "pass@64": 0.9883051829952667, + "pass@128": 0.99 + }, + "8": { + "pass@1": 0.94778125, + "pass@2": 0.967933439960631, + "pass@4": 0.9780367185039364, + "pass@8": 0.9839861528474465, + "pass@16": 0.9880879870533277, + "pass@32": 0.9914644386829476, + "pass@64": 0.9941462724320863, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.920015625, + "pass@2": 0.9466899606299218, + "pass@4": 0.9604877979940004, + "pass@8": 0.9701467430375996, + "pass@16": 0.9778878332985467, + "pass@32": 0.9848928925644655, + "pass@64": 0.9909155644933574, + "pass@128": 0.995 + } + }, + "per_op_accuracy": { + "10": 0.8819375, + "2": 0.9615390625, + "3": 0.9939140625, + "4": 0.9805546875, + "5": 0.952046875, + "6": 0.9740234375, + "7": 0.9543125, + "8": 0.94778125, + "9": 0.920015625 + }, + "per_op_avg_response_len": { + "10": 272.8321953125, + "2": 117.7809375, + "3": 143.2790859375, + "4": 157.306984375, + "5": 170.418859375, + "6": 195.5709765625, + "7": 218.4988203125, + "8": 238.285453125, + "9": 252.617015625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.8321953125, + "2": 117.7809375, + "3": 143.2790859375, + "4": 157.306984375, + "5": 170.418859375, + "6": 195.5709765625, + "7": 218.4988203125, + "8": 238.285453125, + "9": 252.617015625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 358242, + "answer_accuracy": 0.9452095997973657, + "avg_response_len": 186.16363242992233, + "resp_tokens_sum": 70557506, + "pass_at_k": { + "pass@1": 0.9452095997973657, + "pass@2": 0.9610232813318565, + "pass@4": 0.9696750693967211, + "pass@8": 0.9754612861954334, + "pass@16": 0.9800604467506913, + "pass@32": 0.9842302319534386, + "pass@64": 0.9879454057786811, + "pass@128": 0.9908814589665653 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8832964601769911, + "pass@2": 0.9172103279678535, + "pass@4": 0.937207035237852, + "pass@8": 0.9511780335899438, + "pass@16": 0.9620826801919633, + "pass@32": 0.9708490352016442, + "pass@64": 0.978042327328383, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 0.9566053348909658, + "pass@2": 0.962186787107219, + "pass@4": 0.9660673160597917, + "pass@8": 0.9685307941081832, + "pass@16": 0.9705478781887977, + "pass@32": 0.9729358037598592, + "pass@64": 0.9762551112290103, + "pass@128": 0.9813084112149533 + }, + "3": { + "pass@1": 0.9943377293577982, + "pass@2": 0.9960855667124179, + "pass@4": 0.9972139947529495, + "pass@8": 0.9981468940282231, + "pass@16": 0.9989822280288635, + "pass@32": 0.9997163691810725, + "pass@64": 0.9999905325251474, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9753662109375, + "pass@2": 0.9802957369586615, + "pass@4": 0.983529715621485, + "pass@8": 0.9857348060465551, + "pass@16": 0.9876058181840719, + "pass@32": 0.9903074532142714, + "pass@64": 0.9941683070866141, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.934672619047619, + "pass@2": 0.9468152418447692, + "pass@4": 0.9547582409341686, + "pass@8": 0.9609408470768459, + "pass@16": 0.9669133908826691, + "pass@32": 0.9727478686315325, + "pass@64": 0.9775761611445652, + "pass@128": 0.9809523809523809 + }, + "6": { + "pass@1": 0.9568909056886228, + "pass@2": 0.9743243581734176, + "pass@4": 0.982969407080103, + "pass@8": 0.9869105050103905, + "pass@16": 0.9889078989792498, + "pass@32": 0.9902473376106875, + "pass@64": 0.9909312655090486, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9426081730769231, + "pass@2": 0.9565109141312955, + "pass@4": 0.9643415889581849, + "pass@8": 0.9704004882673377, + "pass@16": 0.9755356821468868, + "pass@32": 0.9802742371656721, + "pass@64": 0.9851040508965963, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.9438351362179487, + "pass@2": 0.9649525855542097, + "pass@4": 0.9742177161749009, + "pass@8": 0.9796460229237937, + "pass@16": 0.9837141372979744, + "pass@32": 0.9876441458962127, + "pass@64": 0.990970237640019, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9236355633802817, + "pass@2": 0.9530761339691695, + "pass@4": 0.9679002519051312, + "pass@8": 0.9780128398804888, + "pass@16": 0.9860372032987277, + "pass@32": 0.9927718536917328, + "pass@64": 0.9977068384538204, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 372735, + "answer_accuracy": 0.9541258805701179, + "avg_response_len": 194.51174434796854, + "resp_tokens_sum": 75987180, + "pass_at_k": { + "pass@1": 0.9541258805701179, + "pass@2": 0.9672918429634384, + "pass@4": 0.9744399019974249, + "pass@8": 0.9792299527808267, + "pass@16": 0.9828555293464146, + "pass@32": 0.9859778112192111, + "pass@64": 0.9887684042613567, + "pass@128": 0.9908256880733946 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8846338190184049, + "pass@2": 0.9194517324042314, + "pass@4": 0.9400236116957766, + "pass@8": 0.9542828313525812, + "pass@16": 0.9646392990472547, + "pass@32": 0.9717789665917326, + "pass@64": 0.9761131191473751, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 0.9572568389057751, + "pass@2": 0.9598992114017663, + "pass@4": 0.9626694922063312, + "pass@8": 0.9658379624318189, + "pass@16": 0.9690636373795826, + "pass@32": 0.972155719959404, + "pass@64": 0.9757077858616603, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.9955965909090909, + "pass@2": 0.9970606657122405, + "pass@4": 0.9975789384281509, + "pass@8": 0.9980929433414543, + "pass@16": 0.9988386081686187, + "pass@32": 0.9996178951468804, + "pass@64": 0.9999800850835996, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837740384615384, + "pass@2": 0.9871876164795229, + "pass@4": 0.988965550626438, + "pass@8": 0.9903578338731447, + "pass@16": 0.9910320953813216, + "pass@32": 0.9911233149752038, + "pass@64": 0.9911242603502997, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.9695199275362318, + "pass@2": 0.9757246376811596, + "pass@4": 0.9795172891432047, + "pass@8": 0.9828762612538346, + "pass@16": 0.9857687021768708, + "pass@32": 0.9885129019107062, + "pass@64": 0.9915973976452171, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9840168338108882, + "pass@2": 0.995195426189563, + "pass@4": 0.9990684802472753, + "pass@8": 0.9999207029235206, + "pass@16": 0.9999989480294558, + "pass@32": 0.9999999999297677, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9557893786127167, + "pass@2": 0.9670289813845525, + "pass@4": 0.9733226897035272, + "pass@8": 0.9776014792969531, + "pass@16": 0.9806254260386871, + "pass@32": 0.9831697369077875, + "pass@64": 0.9850731496000922, + "pass@128": 0.9855491329479769 + }, + "8": { + "pass@1": 0.9510343309859155, + "pass@2": 0.9731552206942445, + "pass@4": 0.9841903806038328, + "pass@8": 0.9894380107543043, + "pass@16": 0.9924766599199953, + "pass@32": 0.9949998418278819, + "pass@64": 0.9975485092564174, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9023203592814372, + "pass@2": 0.9278947027205433, + "pass@4": 0.9418790857542502, + "pass@8": 0.95232354181696, + "pass@16": 0.9613164420633297, + "pass@32": 0.9708170510530576, + "pass@64": 0.9803618880355766, + "pass@128": 0.9880239520958084 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 365487, + "answer_accuracy": 0.9559314320388349, + "avg_response_len": 208.1385901406093, + "resp_tokens_sum": 79578876, + "pass_at_k": { + "pass@1": 0.9559314320388349, + "pass@2": 0.9698663862300954, + "pass@4": 0.9782133915445881, + "pass@8": 0.9836909843703433, + "pass@16": 0.9874888849565766, + "pass@32": 0.9903146076340048, + "pass@64": 0.9923666002342315, + "pass@128": 0.9933043187144291 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8779384328358208, + "pass@2": 0.919579636267481, + "pass@4": 0.9469269764786861, + "pass@8": 0.9633563596612335, + "pass@16": 0.9729431971434862, + "pass@32": 0.9793303717895566, + "pass@64": 0.9847047119439397, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9700892857142858, + "pass@2": 0.9764303290213722, + "pass@4": 0.9802661671755313, + "pass@8": 0.9824211322852868, + "pass@16": 0.984386055526147, + "pass@32": 0.9867049961682504, + "pass@64": 0.988322267277566, + "pass@128": 0.9885714285714285 + }, + "3": { + "pass@1": 0.9918913994169096, + "pass@2": 0.9946228421064712, + "pass@4": 0.996332737835613, + "pass@8": 0.9970006677667912, + "pass@16": 0.9970835775181903, + "pass@32": 0.9970845480611684, + "pass@64": 0.9970845481049563, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9822277046783626, + "pass@2": 0.9848502181470739, + "pass@4": 0.9881618657316958, + "pass@8": 0.9918373424468889, + "pass@16": 0.9948832071878148, + "pass@32": 0.996599044812975, + "pass@64": 0.9970563844205709, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9504136029411765, + "pass@2": 0.9648897058823532, + "pass@4": 0.9740956067072498, + "pass@8": 0.9794277261657189, + "pass@16": 0.9827043544002162, + "pass@32": 0.985393110798894, + "pass@64": 0.9876163898807621, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9810725552050473, + "pass@2": 0.9868364061452098, + "pass@4": 0.9897118630084484, + "pass@8": 0.992015798703835, + "pass@16": 0.9933635236323332, + "pass@32": 0.9936785012993264, + "pass@64": 0.9936908487339506, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9652145965189873, + "pass@2": 0.9762272781072461, + "pass@4": 0.981000794402282, + "pass@8": 0.9844839762371538, + "pass@16": 0.988252756425543, + "pass@32": 0.9920258959912787, + "pass@64": 0.9952680507296997, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9480105105105106, + "pass@2": 0.9651595493130931, + "pass@4": 0.9750546725015227, + "pass@8": 0.9822405401725065, + "pass@16": 0.9875073932275126, + "pass@32": 0.9912748387820756, + "pass@64": 0.9934950075146307, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.934887459807074, + "pass@2": 0.9595855381421378, + "pass@4": 0.9720115560313806, + "pass@8": 0.9803090736757587, + "pass@16": 0.9863824259753894, + "pass@32": 0.9910160432545946, + "pass@64": 0.9944976406378414, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.3380640625, + "avg_response_len": 313.0753828125, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.3380640625, + "pass@2": 0.41320054133858886, + "pass@4": 0.4793022547150359, + "pass@8": 0.5346731521138202, + "pass@16": 0.5801699326509802, + "pass@32": 0.6182158700022926, + "pass@64": 0.6512754042574009, + "pass@128": 0.681 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.82321875, + "pass@2": 0.8787271161417317, + "pass@4": 0.9113301397637792, + "pass@8": 0.9315777946514386, + "pass@16": 0.9457507982958181, + "pass@32": 0.9567190192205716, + "pass@64": 0.9652489621851135, + "pass@128": 0.973 + }, + "12": { + "pass@1": 0.652859375, + "pass@2": 0.7453163139763783, + "pass@4": 0.8095610776152983, + "pass@8": 0.8532979517607202, + "pass@16": 0.8827030328451128, + "pass@32": 0.9033929481150977, + "pass@64": 0.920714454766374, + "pass@128": 0.936 + }, + "13": { + "pass@1": 0.4201640625, + "pass@2": 0.5142707923228347, + "pass@4": 0.5912878345519303, + "pass@8": 0.6531636774325383, + "pass@16": 0.7039763771081896, + "pass@32": 0.7471350953865014, + "pass@64": 0.7838292368786625, + "pass@128": 0.813 + }, + "14": { + "pass@1": 0.360734375, + "pass@2": 0.446761687992126, + "pass@4": 0.5191934669103859, + "pass@8": 0.5788261284709705, + "pass@16": 0.6292851164527962, + "pass@32": 0.6751040632277482, + "pass@64": 0.7185368442123148, + "pass@128": 0.76 + }, + "15": { + "pass@1": 0.2632890625, + "pass@2": 0.3506299212598424, + "pass@4": 0.4325513665166849, + "pass@8": 0.5029998735571795, + "pass@16": 0.5618629637107032, + "pass@32": 0.6115234507892966, + "pass@64": 0.6528988035539732, + "pass@128": 0.688 + }, + "16": { + "pass@1": 0.200421875, + "pass@2": 0.269509719488189, + "pass@4": 0.33480263188976345, + "pass@8": 0.39139732602033783, + "pass@16": 0.43973064092240943, + "pass@32": 0.48171716174049706, + "pass@64": 0.5202867907135632, + "pass@128": 0.557 + }, + "17": { + "pass@1": 0.18459375, + "pass@2": 0.2548353838582678, + "pass@4": 0.32462207367829016, + "pass@8": 0.3890998520177243, + "pass@16": 0.44612510593382176, + "pass@32": 0.4951935530892446, + "pass@64": 0.5369976875500979, + "pass@128": 0.574 + }, + "18": { + "pass@1": 0.171921875, + "pass@2": 0.23883513779527538, + "pass@4": 0.30559595725534255, + "pass@8": 0.36544609818176754, + "pass@16": 0.4163473764111736, + "pass@32": 0.45949103685674647, + "pass@64": 0.49736862038028845, + "pass@128": 0.534 + }, + "19": { + "pass@1": 0.15396875, + "pass@2": 0.21837856791338575, + "pass@4": 0.2821726451068615, + "pass@8": 0.33856467466915535, + "pass@16": 0.38488238196098973, + "pass@32": 0.4221923155799813, + "pass@64": 0.45493599720825245, + "pass@128": 0.485 + }, + "20": { + "pass@1": 0.14946875, + "pass@2": 0.21474077263779529, + "pass@4": 0.2819053538620171, + "pass@8": 0.3423581443763291, + "pass@16": 0.39103553286865417, + "pass@32": 0.42969005601724, + "pass@64": 0.46193664512533655, + "pass@128": 0.49 + } + }, + "per_op_accuracy": { + "11": 0.82321875, + "12": 0.652859375, + "13": 0.4201640625, + "14": 0.360734375, + "15": 0.2632890625, + "16": 0.200421875, + "17": 0.18459375, + "18": 0.171921875, + "19": 0.15396875, + "20": 0.14946875 + }, + "per_op_avg_response_len": { + "11": 288.2266328125, + "12": 294.280890625, + "13": 304.0269453125, + "14": 316.9761328125, + "15": 318.809, + "16": 319.464953125, + "17": 323.985453125, + "18": 322.2364140625, + "19": 319.9578515625, + "20": 322.7895546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 288.2266328125, + "12": 294.280890625, + "13": 304.0269453125, + "14": 316.9761328125, + "15": 318.809, + "16": 319.464953125, + "17": 323.985453125, + "18": 322.2364140625, + "19": 319.9578515625, + "20": 322.7895546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 147744, + "answer_accuracy": 0.34301634472511144, + "avg_response_len": 296.5207838038633, + "resp_tokens_sum": 127717432, + "pass_at_k": { + "pass@1": 0.34301634472511144, + "pass@2": 0.41863574341004645, + "pass@4": 0.4839380849032075, + "pass@8": 0.5368201648121248, + "pass@16": 0.578860535559213, + "pass@32": 0.6136866155882119, + "pass@64": 0.6449572108911955, + "pass@128": 0.674591381872214 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8158640226628895, + "pass@2": 0.8690087495259979, + "pass@4": 0.9003459481693683, + "pass@8": 0.9208178113433934, + "pass@16": 0.936255057279863, + "pass@32": 0.9486361034628609, + "pass@64": 0.9589588726836659, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.6557203389830508, + "pass@2": 0.7513526513634943, + "pass@4": 0.8129071535549578, + "pass@8": 0.8537187806022632, + "pass@16": 0.8834961886746142, + "pass@32": 0.9062902420548817, + "pass@64": 0.925781100999793, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.4441859326018809, + "pass@2": 0.537450170315701, + "pass@4": 0.612993540286304, + "pass@8": 0.6709888225301639, + "pass@16": 0.7145216319389676, + "pass@32": 0.7501344384006141, + "pass@64": 0.7802929625106815, + "pass@128": 0.8025078369905956 + }, + "14": { + "pass@1": 0.34971281152647976, + "pass@2": 0.4340566512129908, + "pass@4": 0.503801084735903, + "pass@8": 0.559338023432466, + "pass@16": 0.6046153041994793, + "pass@32": 0.6462515330765554, + "pass@64": 0.6884805551577264, + "pass@128": 0.7320872274143302 + }, + "15": { + "pass@1": 0.27420520231213874, + "pass@2": 0.3648961272814163, + "pass@4": 0.4489088842218423, + "pass@8": 0.5161746732770215, + "pass@16": 0.5669481987005399, + "pass@32": 0.6095912062248127, + "pass@64": 0.6513023988475989, + "pass@128": 0.6936416184971098 + }, + "16": { + "pass@1": 0.19057459677419356, + "pass@2": 0.25924601536703096, + "pass@4": 0.3243514381266857, + "pass@8": 0.3820198056073008, + "pass@16": 0.43185101815612564, + "pass@32": 0.4727802423483779, + "pass@64": 0.507659460573341, + "pass@128": 0.5451612903225806 + }, + "17": { + "pass@1": 0.1830970014450867, + "pass@2": 0.25075561251194756, + "pass@4": 0.3177439135137008, + "pass@8": 0.3787659856147468, + "pass@16": 0.43273696124530503, + "pass@32": 0.4781735761884427, + "pass@64": 0.5152145582863207, + "pass@128": 0.546242774566474 + }, + "18": { + "pass@1": 0.17191964285714287, + "pass@2": 0.2386410292463441, + "pass@4": 0.302181122181156, + "pass@8": 0.35631130956435, + "pass@16": 0.4024186189231844, + "pass@32": 0.44305521959331307, + "pass@64": 0.4812509405361627, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.15561655405405406, + "pass@2": 0.22392286479392776, + "pass@4": 0.29112672092790204, + "pass@8": 0.3476218784231473, + "pass@16": 0.3900725810744725, + "pass@32": 0.42293491455607535, + "pass@64": 0.45500925055441316, + "pass@128": 0.4894894894894895 + }, + "20": { + "pass@1": 0.1527777777777778, + "pass@2": 0.22137873601948407, + "pass@4": 0.29151925350547414, + "pass@8": 0.3513487308384057, + "pass@16": 0.39666827573276253, + "pass@32": 0.43178798807608976, + "pass@64": 0.46000330337155665, + "pass@128": 0.4804804804804805 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 141709, + "answer_accuracy": 0.3292005835563485, + "avg_response_len": 309.35859212384776, + "resp_tokens_sum": 133167737, + "pass_at_k": { + "pass@1": 0.3292005835563485, + "pass@2": 0.4023192479647679, + "pass@4": 0.46766044983672544, + "pass@8": 0.5239849306107431, + "pass@16": 0.5718775237370038, + "pass@32": 0.6131725488567548, + "pass@64": 0.6494198603354132, + "pass@128": 0.6815343443354148 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8197242093373494, + "pass@2": 0.879619980907884, + "pass@4": 0.9152084375183525, + "pass@8": 0.9365968410373897, + "pass@16": 0.9502230602590158, + "pass@32": 0.959630325530834, + "pass@64": 0.9672099492738422, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.6610011755485894, + "pass@2": 0.7461709821538761, + "pass@4": 0.8065136687459518, + "pass@8": 0.8485894222226024, + "pass@16": 0.8752288825205958, + "pass@32": 0.8914381380619772, + "pass@64": 0.9036142400296753, + "pass@128": 0.9153605015673981 + }, + "13": { + "pass@1": 0.38804271708683474, + "pass@2": 0.4778598447252919, + "pass@4": 0.5540819498403033, + "pass@8": 0.619142374924008, + "pass@16": 0.6772598039791219, + "pass@32": 0.7320106065745692, + "pass@64": 0.781803259607603, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.3611796418128655, + "pass@2": 0.4434256573191508, + "pass@4": 0.5109200042428902, + "pass@8": 0.5672925734838943, + "pass@16": 0.6168776944628133, + "pass@32": 0.6620983039384781, + "pass@64": 0.70456770688268, + "pass@128": 0.7456140350877193 + }, + "15": { + "pass@1": 0.2217092803030303, + "pass@2": 0.3015707020997377, + "pass@4": 0.37985566946177185, + "pass@8": 0.4510174568078622, + "pass@16": 0.5158476685485044, + "pass@32": 0.5753658130979193, + "pass@64": 0.6249801268738032, + "pass@128": 0.6666666666666666 + }, + "16": { + "pass@1": 0.2050391737891738, + "pass@2": 0.2738355121475203, + "pass@4": 0.338386615668768, + "pass@8": 0.3932875901414371, + "pass@16": 0.43886406008834783, + "pass@32": 0.4774662643312779, + "pass@64": 0.5142127541987865, + "pass@128": 0.5498575498575499 + }, + "17": { + "pass@1": 0.1947059394409938, + "pass@2": 0.2682923381180613, + "pass@4": 0.3440897881747702, + "pass@8": 0.41833600483808464, + "pass@16": 0.48600653081169176, + "pass@32": 0.5444368907837626, + "pass@64": 0.5933865414431194, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.1689797794117647, + "pass@2": 0.23628271190365893, + "pass@4": 0.3063349933280399, + "pass@8": 0.37162887977886394, + "pass@16": 0.4290596832304357, + "pass@32": 0.4805349660760809, + "pass@64": 0.5257714119613119, + "pass@128": 0.5647058823529412 + }, + "19": { + "pass@1": 0.14950706845238096, + "pass@2": 0.21081337000843647, + "pass@4": 0.27120366957478526, + "pass@8": 0.32483243652106647, + "pass@16": 0.3707793523131844, + "pass@32": 0.4109668939345245, + "pass@64": 0.4485305995646864, + "pass@128": 0.48214285714285715 + }, + "20": { + "pass@1": 0.13929172904191617, + "pass@2": 0.20208254750341842, + "pass@4": 0.2660832878674596, + "pass@8": 0.32449652847649324, + "pass@16": 0.37296661066376247, + "pass@32": 0.4105719174561622, + "pass@64": 0.4410038284084724, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 143269, + "answer_accuracy": 0.3420810093215159, + "avg_response_len": 333.92067399526286, + "resp_tokens_sum": 139851321, + "pass_at_k": { + "pass@1": 0.3420810093215159, + "pass@2": 0.41879477564349493, + "pass@4": 0.48650024408623843, + "pass@8": 0.5434505944075286, + "pass@16": 0.5900395819148438, + "pass@32": 0.628057443998562, + "pass@64": 0.659680329345081, + "pass@128": 0.687041564792176 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8351438492063492, + "pass@2": 0.888676806024247, + "pass@4": 0.9195518057266644, + "pass@8": 0.9383458921358946, + "pass@16": 0.9516784351429641, + "pass@32": 0.9627085924506177, + "pass@64": 0.9702310379963924, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.6418195718654435, + "pass@2": 0.7379478045462207, + "pass@4": 0.808911574699951, + "pass@8": 0.8574357117385621, + "pass@16": 0.8891356835786846, + "pass@32": 0.9119187657061124, + "pass@64": 0.9319113224555994, + "pass@128": 0.9480122324159022 + }, + "13": { + "pass@1": 0.43190586419753085, + "pass@2": 0.5315685908185085, + "pass@4": 0.6109124663815172, + "pass@8": 0.6731000469060026, + "pass@16": 0.7230315632688672, + "pass@32": 0.7608469845357523, + "pass@64": 0.7895432659192605, + "pass@128": 0.8148148148148148 + }, + "14": { + "pass@1": 0.3707807863501484, + "pass@2": 0.4622490449543215, + "pass@4": 0.5422512678311351, + "pass@8": 0.6090936582126327, + "pass@16": 0.665375170025168, + "pass@32": 0.7157854633923258, + "pass@64": 0.7613425230053639, + "pass@128": 0.8011869436201781 + }, + "15": { + "pass@1": 0.29398148148148145, + "pass@2": 0.385362745455429, + "pass@4": 0.46875469646155304, + "pass@8": 0.5418755427677018, + "pass@16": 0.6032998652447866, + "pass@32": 0.6504141207197476, + "pass@64": 0.6830392953220654, + "pass@128": 0.7037037037037037 + }, + "16": { + "pass@1": 0.20464601769911506, + "pass@2": 0.27441648955938025, + "pass@4": 0.3406489202677099, + "pass@8": 0.3980154635469927, + "pass@16": 0.44783345192625446, + "pass@32": 0.4942909375581759, + "pass@64": 0.5381229534278857, + "pass@128": 0.5752212389380531 + }, + "17": { + "pass@1": 0.17634600903614459, + "pass@2": 0.24603556944312685, + "pass@4": 0.3129089391875711, + "pass@8": 0.37151393806373156, + "pass@16": 0.42139763379994655, + "pass@32": 0.4651711352279267, + "pass@64": 0.5050089880673075, + "pass@128": 0.5451807228915663 + }, + "18": { + "pass@1": 0.17515120967741934, + "pass@2": 0.2418537274574549, + "pass@4": 0.30864086051743567, + "pass@8": 0.36897845390139183, + "pass@16": 0.41813086287003537, + "pass@32": 0.45496716623619304, + "pass@64": 0.48441455201866307, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.1568400679758308, + "pass@2": 0.22048024656849932, + "pass@4": 0.28429913613517355, + "pass@8": 0.34339241837815315, + "pass@16": 0.3939768884772223, + "pass@32": 0.4328402195492693, + "pass@64": 0.4613644571598135, + "pass@128": 0.48338368580060426 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.22079904707857453, + "pass@4": 0.2881610339248135, + "pass@8": 0.3512828123002748, + "pass@16": 0.40352597323708034, + "pass@32": 0.4467676743469794, + "pass@64": 0.4848656649074429, + "pass@128": 0.5195195195195195 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6287771381578947, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6287771381578947, + "pass@2": 0.675092681568566, + "pass@4": 0.7136918110038668, + "pass@8": 0.7453666934612777, + "pass@16": 0.771208562329956, + "pass@32": 0.7928283384042701, + "pass@64": 0.8115779259889512, + "pass@128": 0.8281578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8819375, + "pass@2": 0.9187347440944885, + "pass@4": 0.9413814194788149, + "pass@8": 0.9562699368944456, + "pass@16": 0.966554411117548, + "pass@32": 0.9739934405917635, + "pass@64": 0.9796453043075861, + "pass@128": 0.984 + }, + "2": { + "pass@1": 0.9615390625, + "pass@2": 0.966419414370079, + "pass@4": 0.9699190299025123, + "pass@8": 0.9725064708486456, + "pass@16": 0.9749029250306379, + "pass@32": 0.9774983735324464, + "pass@64": 0.9802985458001466, + "pass@128": 0.983 + }, + "3": { + "pass@1": 0.9939140625, + "pass@2": 0.9959056348425196, + "pass@4": 0.9970321550431197, + "pass@8": 0.9977359346939182, + "pass@16": 0.9982835963498219, + "pass@32": 0.998781158105662, + "pass@64": 0.998990332213311, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.9805546875, + "pass@2": 0.9841828248031499, + "pass@4": 0.9869512231908513, + "pass@8": 0.9893844569008565, + "pass@16": 0.9912527669160224, + "pass@32": 0.9927349388162231, + "pass@64": 0.994127141737953, + "pass@128": 0.995 + }, + "5": { + "pass@1": 0.952046875, + "pass@2": 0.9629343011811029, + "pass@4": 0.9698748169291337, + "pass@8": 0.974794103858124, + "pass@16": 0.9787874008751346, + "pass@32": 0.9824861874497502, + "pass@64": 0.9858271655075969, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.9740234375, + "pass@2": 0.9855746801181101, + "pass@4": 0.9907253421447321, + "pass@8": 0.9930694421828948, + "pass@16": 0.9941911081127991, + "pass@32": 0.9947386956493451, + "pass@64": 0.9949710417286846, + "pass@128": 0.995 + }, + "7": { + "pass@1": 0.9543125, + "pass@2": 0.9663805364173231, + "pass@4": 0.9727133587364074, + "pass@8": 0.9773424133620466, + "pass@16": 0.9813153290055051, + "pass@32": 0.9849896042653357, + "pass@64": 0.9883051829952667, + "pass@128": 0.99 + }, + "8": { + "pass@1": 0.94778125, + "pass@2": 0.967933439960631, + "pass@4": 0.9780367185039364, + "pass@8": 0.9839861528474465, + "pass@16": 0.9880879870533277, + "pass@32": 0.9914644386829476, + "pass@64": 0.9941462724320863, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.920015625, + "pass@2": 0.9466899606299218, + "pass@4": 0.9604877979940004, + "pass@8": 0.9701467430375996, + "pass@16": 0.9778878332985467, + "pass@32": 0.9848928925644655, + "pass@64": 0.9909155644933574, + "pass@128": 0.995 + }, + "11": { + "pass@1": 0.82321875, + "pass@2": 0.8787271161417317, + "pass@4": 0.9113301397637792, + "pass@8": 0.9315777946514386, + "pass@16": 0.9457507982958181, + "pass@32": 0.9567190192205716, + "pass@64": 0.9652489621851135, + "pass@128": 0.973 + }, + "12": { + "pass@1": 0.652859375, + "pass@2": 0.7453163139763783, + "pass@4": 0.8095610776152983, + "pass@8": 0.8532979517607202, + "pass@16": 0.8827030328451128, + "pass@32": 0.9033929481150977, + "pass@64": 0.920714454766374, + "pass@128": 0.936 + }, + "13": { + "pass@1": 0.4201640625, + "pass@2": 0.5142707923228347, + "pass@4": 0.5912878345519303, + "pass@8": 0.6531636774325383, + "pass@16": 0.7039763771081896, + "pass@32": 0.7471350953865014, + "pass@64": 0.7838292368786625, + "pass@128": 0.813 + }, + "14": { + "pass@1": 0.360734375, + "pass@2": 0.446761687992126, + "pass@4": 0.5191934669103859, + "pass@8": 0.5788261284709705, + "pass@16": 0.6292851164527962, + "pass@32": 0.6751040632277482, + "pass@64": 0.7185368442123148, + "pass@128": 0.76 + }, + "15": { + "pass@1": 0.2632890625, + "pass@2": 0.3506299212598424, + "pass@4": 0.4325513665166849, + "pass@8": 0.5029998735571795, + "pass@16": 0.5618629637107032, + "pass@32": 0.6115234507892966, + "pass@64": 0.6528988035539732, + "pass@128": 0.688 + }, + "16": { + "pass@1": 0.200421875, + "pass@2": 0.269509719488189, + "pass@4": 0.33480263188976345, + "pass@8": 0.39139732602033783, + "pass@16": 0.43973064092240943, + "pass@32": 0.48171716174049706, + "pass@64": 0.5202867907135632, + "pass@128": 0.557 + }, + "17": { + "pass@1": 0.18459375, + "pass@2": 0.2548353838582678, + "pass@4": 0.32462207367829016, + "pass@8": 0.3890998520177243, + "pass@16": 0.44612510593382176, + "pass@32": 0.4951935530892446, + "pass@64": 0.5369976875500979, + "pass@128": 0.574 + }, + "18": { + "pass@1": 0.171921875, + "pass@2": 0.23883513779527538, + "pass@4": 0.30559595725534255, + "pass@8": 0.36544609818176754, + "pass@16": 0.4163473764111736, + "pass@32": 0.45949103685674647, + "pass@64": 0.49736862038028845, + "pass@128": 0.534 + }, + "19": { + "pass@1": 0.15396875, + "pass@2": 0.21837856791338575, + "pass@4": 0.2821726451068615, + "pass@8": 0.33856467466915535, + "pass@16": 0.38488238196098973, + "pass@32": 0.4221923155799813, + "pass@64": 0.45493599720825245, + "pass@128": 0.485 + }, + "20": { + "pass@1": 0.14946875, + "pass@2": 0.21474077263779529, + "pass@4": 0.2819053538620171, + "pass@8": 0.3423581443763291, + "pass@16": 0.39103553286865417, + "pass@32": 0.42969005601724, + "pass@64": 0.46193664512533655, + "pass@128": 0.49 + } + }, + "per_op_accuracy": { + "10": 0.8819375, + "2": 0.9615390625, + "3": 0.9939140625, + "4": 0.9805546875, + "5": 0.952046875, + "6": 0.9740234375, + "7": 0.9543125, + "8": 0.94778125, + "9": 0.920015625, + "11": 0.82321875, + "12": 0.652859375, + "13": 0.4201640625, + "14": 0.360734375, + "15": 0.2632890625, + "16": 0.200421875, + "17": 0.18459375, + "18": 0.171921875, + "19": 0.15396875, + "20": 0.14946875 + }, + "per_op_avg_response_len": { + "10": 272.8321953125, + "2": 117.7809375, + "3": 143.2790859375, + "4": 157.306984375, + "5": 170.418859375, + "6": 195.5709765625, + "7": 218.4988203125, + "8": 238.285453125, + "9": 252.617015625, + "11": 288.2266328125, + "12": 294.280890625, + "13": 304.0269453125, + "14": 316.9761328125, + "15": 318.809, + "16": 319.464953125, + "17": 323.985453125, + "18": 322.2364140625, + "19": 319.9578515625, + "20": 322.7895546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.8321953125, + "2": 117.7809375, + "3": 143.2790859375, + "4": 157.306984375, + "5": 170.418859375, + "6": 195.5709765625, + "7": 218.4988203125, + "8": 238.285453125, + "9": 252.617015625, + "11": 288.2266328125, + "12": 294.280890625, + "13": 304.0269453125, + "14": 316.9761328125, + "15": 318.809, + "16": 319.464953125, + "17": 323.985453125, + "18": 322.2364140625, + "19": 319.9578515625, + "20": 322.7895546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 505986, + "answer_accuracy": 0.6248839116345242, + "avg_response_len": 244.86610071530194, + "resp_tokens_sum": 198274938, + "pass_at_k": { + "pass@1": 0.6248839116345242, + "pass@2": 0.672510150584643, + "pass@4": 0.711296164429809, + "pass@8": 0.7421341642455704, + "pass@16": 0.766649491777681, + "pass@32": 0.7871263323219219, + "pass@64": 0.8054991086246542, + "pass@128": 0.8226367372747392 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8832964601769911, + "pass@2": 0.9172103279678535, + "pass@4": 0.937207035237852, + "pass@8": 0.9511780335899438, + "pass@16": 0.9620826801919633, + "pass@32": 0.9708490352016442, + "pass@64": 0.978042327328383, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 0.9566053348909658, + "pass@2": 0.962186787107219, + "pass@4": 0.9660673160597917, + "pass@8": 0.9685307941081832, + "pass@16": 0.9705478781887977, + "pass@32": 0.9729358037598592, + "pass@64": 0.9762551112290103, + "pass@128": 0.9813084112149533 + }, + "3": { + "pass@1": 0.9943377293577982, + "pass@2": 0.9960855667124179, + "pass@4": 0.9972139947529495, + "pass@8": 0.9981468940282231, + "pass@16": 0.9989822280288635, + "pass@32": 0.9997163691810725, + "pass@64": 0.9999905325251474, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9753662109375, + "pass@2": 0.9802957369586615, + "pass@4": 0.983529715621485, + "pass@8": 0.9857348060465551, + "pass@16": 0.9876058181840719, + "pass@32": 0.9903074532142714, + "pass@64": 0.9941683070866141, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.934672619047619, + "pass@2": 0.9468152418447692, + "pass@4": 0.9547582409341686, + "pass@8": 0.9609408470768459, + "pass@16": 0.9669133908826691, + "pass@32": 0.9727478686315325, + "pass@64": 0.9775761611445652, + "pass@128": 0.9809523809523809 + }, + "6": { + "pass@1": 0.9568909056886228, + "pass@2": 0.9743243581734176, + "pass@4": 0.982969407080103, + "pass@8": 0.9869105050103905, + "pass@16": 0.9889078989792498, + "pass@32": 0.9902473376106875, + "pass@64": 0.9909312655090486, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9426081730769231, + "pass@2": 0.9565109141312955, + "pass@4": 0.9643415889581849, + "pass@8": 0.9704004882673377, + "pass@16": 0.9755356821468868, + "pass@32": 0.9802742371656721, + "pass@64": 0.9851040508965963, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.9438351362179487, + "pass@2": 0.9649525855542097, + "pass@4": 0.9742177161749009, + "pass@8": 0.9796460229237937, + "pass@16": 0.9837141372979744, + "pass@32": 0.9876441458962127, + "pass@64": 0.990970237640019, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9236355633802817, + "pass@2": 0.9530761339691695, + "pass@4": 0.9679002519051312, + "pass@8": 0.9780128398804888, + "pass@16": 0.9860372032987277, + "pass@32": 0.9927718536917328, + "pass@64": 0.9977068384538204, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8158640226628895, + "pass@2": 0.8690087495259979, + "pass@4": 0.9003459481693683, + "pass@8": 0.9208178113433934, + "pass@16": 0.936255057279863, + "pass@32": 0.9486361034628609, + "pass@64": 0.9589588726836659, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.6557203389830508, + "pass@2": 0.7513526513634943, + "pass@4": 0.8129071535549578, + "pass@8": 0.8537187806022632, + "pass@16": 0.8834961886746142, + "pass@32": 0.9062902420548817, + "pass@64": 0.925781100999793, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.4441859326018809, + "pass@2": 0.537450170315701, + "pass@4": 0.612993540286304, + "pass@8": 0.6709888225301639, + "pass@16": 0.7145216319389676, + "pass@32": 0.7501344384006141, + "pass@64": 0.7802929625106815, + "pass@128": 0.8025078369905956 + }, + "14": { + "pass@1": 0.34971281152647976, + "pass@2": 0.4340566512129908, + "pass@4": 0.503801084735903, + "pass@8": 0.559338023432466, + "pass@16": 0.6046153041994793, + "pass@32": 0.6462515330765554, + "pass@64": 0.6884805551577264, + "pass@128": 0.7320872274143302 + }, + "15": { + "pass@1": 0.27420520231213874, + "pass@2": 0.3648961272814163, + "pass@4": 0.4489088842218423, + "pass@8": 0.5161746732770215, + "pass@16": 0.5669481987005399, + "pass@32": 0.6095912062248127, + "pass@64": 0.6513023988475989, + "pass@128": 0.6936416184971098 + }, + "16": { + "pass@1": 0.19057459677419356, + "pass@2": 0.25924601536703096, + "pass@4": 0.3243514381266857, + "pass@8": 0.3820198056073008, + "pass@16": 0.43185101815612564, + "pass@32": 0.4727802423483779, + "pass@64": 0.507659460573341, + "pass@128": 0.5451612903225806 + }, + "17": { + "pass@1": 0.1830970014450867, + "pass@2": 0.25075561251194756, + "pass@4": 0.3177439135137008, + "pass@8": 0.3787659856147468, + "pass@16": 0.43273696124530503, + "pass@32": 0.4781735761884427, + "pass@64": 0.5152145582863207, + "pass@128": 0.546242774566474 + }, + "18": { + "pass@1": 0.17191964285714287, + "pass@2": 0.2386410292463441, + "pass@4": 0.302181122181156, + "pass@8": 0.35631130956435, + "pass@16": 0.4024186189231844, + "pass@32": 0.44305521959331307, + "pass@64": 0.4812509405361627, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.15561655405405406, + "pass@2": 0.22392286479392776, + "pass@4": 0.29112672092790204, + "pass@8": 0.3476218784231473, + "pass@16": 0.3900725810744725, + "pass@32": 0.42293491455607535, + "pass@64": 0.45500925055441316, + "pass@128": 0.4894894894894895 + }, + "20": { + "pass@1": 0.1527777777777778, + "pass@2": 0.22137873601948407, + "pass@4": 0.29151925350547414, + "pass@8": 0.3513487308384057, + "pass@16": 0.39666827573276253, + "pass@32": 0.43178798807608976, + "pass@64": 0.46000330337155665, + "pass@128": 0.4804804804804805 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 514444, + "answer_accuracy": 0.6265150038971161, + "avg_response_len": 254.71906298713952, + "resp_tokens_sum": 209154917, + "pass_at_k": { + "pass@1": 0.6265150038971161, + "pass@2": 0.6711105745331151, + "pass@4": 0.7087658103970431, + "pass@8": 0.7405722739721018, + "pass@16": 0.7674043940596819, + "pass@32": 0.7905383572324718, + "pass@64": 0.8108683024339327, + "pass@128": 0.8286827747466875 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8846338190184049, + "pass@2": 0.9194517324042314, + "pass@4": 0.9400236116957766, + "pass@8": 0.9542828313525812, + "pass@16": 0.9646392990472547, + "pass@32": 0.9717789665917326, + "pass@64": 0.9761131191473751, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 0.9572568389057751, + "pass@2": 0.9598992114017663, + "pass@4": 0.9626694922063312, + "pass@8": 0.9658379624318189, + "pass@16": 0.9690636373795826, + "pass@32": 0.972155719959404, + "pass@64": 0.9757077858616603, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.9955965909090909, + "pass@2": 0.9970606657122405, + "pass@4": 0.9975789384281509, + "pass@8": 0.9980929433414543, + "pass@16": 0.9988386081686187, + "pass@32": 0.9996178951468804, + "pass@64": 0.9999800850835996, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837740384615384, + "pass@2": 0.9871876164795229, + "pass@4": 0.988965550626438, + "pass@8": 0.9903578338731447, + "pass@16": 0.9910320953813216, + "pass@32": 0.9911233149752038, + "pass@64": 0.9911242603502997, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.9695199275362318, + "pass@2": 0.9757246376811596, + "pass@4": 0.9795172891432047, + "pass@8": 0.9828762612538346, + "pass@16": 0.9857687021768708, + "pass@32": 0.9885129019107062, + "pass@64": 0.9915973976452171, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9840168338108882, + "pass@2": 0.995195426189563, + "pass@4": 0.9990684802472753, + "pass@8": 0.9999207029235206, + "pass@16": 0.9999989480294558, + "pass@32": 0.9999999999297677, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9557893786127167, + "pass@2": 0.9670289813845525, + "pass@4": 0.9733226897035272, + "pass@8": 0.9776014792969531, + "pass@16": 0.9806254260386871, + "pass@32": 0.9831697369077875, + "pass@64": 0.9850731496000922, + "pass@128": 0.9855491329479769 + }, + "8": { + "pass@1": 0.9510343309859155, + "pass@2": 0.9731552206942445, + "pass@4": 0.9841903806038328, + "pass@8": 0.9894380107543043, + "pass@16": 0.9924766599199953, + "pass@32": 0.9949998418278819, + "pass@64": 0.9975485092564174, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9023203592814372, + "pass@2": 0.9278947027205433, + "pass@4": 0.9418790857542502, + "pass@8": 0.95232354181696, + "pass@16": 0.9613164420633297, + "pass@32": 0.9708170510530576, + "pass@64": 0.9803618880355766, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8197242093373494, + "pass@2": 0.879619980907884, + "pass@4": 0.9152084375183525, + "pass@8": 0.9365968410373897, + "pass@16": 0.9502230602590158, + "pass@32": 0.959630325530834, + "pass@64": 0.9672099492738422, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.6610011755485894, + "pass@2": 0.7461709821538761, + "pass@4": 0.8065136687459518, + "pass@8": 0.8485894222226024, + "pass@16": 0.8752288825205958, + "pass@32": 0.8914381380619772, + "pass@64": 0.9036142400296753, + "pass@128": 0.9153605015673981 + }, + "13": { + "pass@1": 0.38804271708683474, + "pass@2": 0.4778598447252919, + "pass@4": 0.5540819498403033, + "pass@8": 0.619142374924008, + "pass@16": 0.6772598039791219, + "pass@32": 0.7320106065745692, + "pass@64": 0.781803259607603, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.3611796418128655, + "pass@2": 0.4434256573191508, + "pass@4": 0.5109200042428902, + "pass@8": 0.5672925734838943, + "pass@16": 0.6168776944628133, + "pass@32": 0.6620983039384781, + "pass@64": 0.70456770688268, + "pass@128": 0.7456140350877193 + }, + "15": { + "pass@1": 0.2217092803030303, + "pass@2": 0.3015707020997377, + "pass@4": 0.37985566946177185, + "pass@8": 0.4510174568078622, + "pass@16": 0.5158476685485044, + "pass@32": 0.5753658130979193, + "pass@64": 0.6249801268738032, + "pass@128": 0.6666666666666666 + }, + "16": { + "pass@1": 0.2050391737891738, + "pass@2": 0.2738355121475203, + "pass@4": 0.338386615668768, + "pass@8": 0.3932875901414371, + "pass@16": 0.43886406008834783, + "pass@32": 0.4774662643312779, + "pass@64": 0.5142127541987865, + "pass@128": 0.5498575498575499 + }, + "17": { + "pass@1": 0.1947059394409938, + "pass@2": 0.2682923381180613, + "pass@4": 0.3440897881747702, + "pass@8": 0.41833600483808464, + "pass@16": 0.48600653081169176, + "pass@32": 0.5444368907837626, + "pass@64": 0.5933865414431194, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.1689797794117647, + "pass@2": 0.23628271190365893, + "pass@4": 0.3063349933280399, + "pass@8": 0.37162887977886394, + "pass@16": 0.4290596832304357, + "pass@32": 0.4805349660760809, + "pass@64": 0.5257714119613119, + "pass@128": 0.5647058823529412 + }, + "19": { + "pass@1": 0.14950706845238096, + "pass@2": 0.21081337000843647, + "pass@4": 0.27120366957478526, + "pass@8": 0.32483243652106647, + "pass@16": 0.3707793523131844, + "pass@32": 0.4109668939345245, + "pass@64": 0.4485305995646864, + "pass@128": 0.48214285714285715 + }, + "20": { + "pass@1": 0.13929172904191617, + "pass@2": 0.20208254750341842, + "pass@4": 0.2660832878674596, + "pass@8": 0.32449652847649324, + "pass@16": 0.37296661066376247, + "pass@32": 0.4105719174561622, + "pass@64": 0.4410038284084724, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 508756, + "answer_accuracy": 0.6350305559993609, + "avg_response_len": 273.8933398406295, + "resp_tokens_sum": 219430197, + "pass_at_k": { + "pass@1": 0.6350305559993609, + "pass@2": 0.6817842149823968, + "pass@4": 0.7211618787655941, + "pass@8": 0.7535477416864765, + "pass@16": 0.7797154196182626, + "pass@32": 0.8009384390103981, + "pass@64": 0.8184491248628813, + "pass@128": 0.8332001917239176 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8779384328358208, + "pass@2": 0.919579636267481, + "pass@4": 0.9469269764786861, + "pass@8": 0.9633563596612335, + "pass@16": 0.9729431971434862, + "pass@32": 0.9793303717895566, + "pass@64": 0.9847047119439397, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9700892857142858, + "pass@2": 0.9764303290213722, + "pass@4": 0.9802661671755313, + "pass@8": 0.9824211322852868, + "pass@16": 0.984386055526147, + "pass@32": 0.9867049961682504, + "pass@64": 0.988322267277566, + "pass@128": 0.9885714285714285 + }, + "3": { + "pass@1": 0.9918913994169096, + "pass@2": 0.9946228421064712, + "pass@4": 0.996332737835613, + "pass@8": 0.9970006677667912, + "pass@16": 0.9970835775181903, + "pass@32": 0.9970845480611684, + "pass@64": 0.9970845481049563, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9822277046783626, + "pass@2": 0.9848502181470739, + "pass@4": 0.9881618657316958, + "pass@8": 0.9918373424468889, + "pass@16": 0.9948832071878148, + "pass@32": 0.996599044812975, + "pass@64": 0.9970563844205709, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9504136029411765, + "pass@2": 0.9648897058823532, + "pass@4": 0.9740956067072498, + "pass@8": 0.9794277261657189, + "pass@16": 0.9827043544002162, + "pass@32": 0.985393110798894, + "pass@64": 0.9876163898807621, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9810725552050473, + "pass@2": 0.9868364061452098, + "pass@4": 0.9897118630084484, + "pass@8": 0.992015798703835, + "pass@16": 0.9933635236323332, + "pass@32": 0.9936785012993264, + "pass@64": 0.9936908487339506, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9652145965189873, + "pass@2": 0.9762272781072461, + "pass@4": 0.981000794402282, + "pass@8": 0.9844839762371538, + "pass@16": 0.988252756425543, + "pass@32": 0.9920258959912787, + "pass@64": 0.9952680507296997, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9480105105105106, + "pass@2": 0.9651595493130931, + "pass@4": 0.9750546725015227, + "pass@8": 0.9822405401725065, + "pass@16": 0.9875073932275126, + "pass@32": 0.9912748387820756, + "pass@64": 0.9934950075146307, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.934887459807074, + "pass@2": 0.9595855381421378, + "pass@4": 0.9720115560313806, + "pass@8": 0.9803090736757587, + "pass@16": 0.9863824259753894, + "pass@32": 0.9910160432545946, + "pass@64": 0.9944976406378414, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.8351438492063492, + "pass@2": 0.888676806024247, + "pass@4": 0.9195518057266644, + "pass@8": 0.9383458921358946, + "pass@16": 0.9516784351429641, + "pass@32": 0.9627085924506177, + "pass@64": 0.9702310379963924, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.6418195718654435, + "pass@2": 0.7379478045462207, + "pass@4": 0.808911574699951, + "pass@8": 0.8574357117385621, + "pass@16": 0.8891356835786846, + "pass@32": 0.9119187657061124, + "pass@64": 0.9319113224555994, + "pass@128": 0.9480122324159022 + }, + "13": { + "pass@1": 0.43190586419753085, + "pass@2": 0.5315685908185085, + "pass@4": 0.6109124663815172, + "pass@8": 0.6731000469060026, + "pass@16": 0.7230315632688672, + "pass@32": 0.7608469845357523, + "pass@64": 0.7895432659192605, + "pass@128": 0.8148148148148148 + }, + "14": { + "pass@1": 0.3707807863501484, + "pass@2": 0.4622490449543215, + "pass@4": 0.5422512678311351, + "pass@8": 0.6090936582126327, + "pass@16": 0.665375170025168, + "pass@32": 0.7157854633923258, + "pass@64": 0.7613425230053639, + "pass@128": 0.8011869436201781 + }, + "15": { + "pass@1": 0.29398148148148145, + "pass@2": 0.385362745455429, + "pass@4": 0.46875469646155304, + "pass@8": 0.5418755427677018, + "pass@16": 0.6032998652447866, + "pass@32": 0.6504141207197476, + "pass@64": 0.6830392953220654, + "pass@128": 0.7037037037037037 + }, + "16": { + "pass@1": 0.20464601769911506, + "pass@2": 0.27441648955938025, + "pass@4": 0.3406489202677099, + "pass@8": 0.3980154635469927, + "pass@16": 0.44783345192625446, + "pass@32": 0.4942909375581759, + "pass@64": 0.5381229534278857, + "pass@128": 0.5752212389380531 + }, + "17": { + "pass@1": 0.17634600903614459, + "pass@2": 0.24603556944312685, + "pass@4": 0.3129089391875711, + "pass@8": 0.37151393806373156, + "pass@16": 0.42139763379994655, + "pass@32": 0.4651711352279267, + "pass@64": 0.5050089880673075, + "pass@128": 0.5451807228915663 + }, + "18": { + "pass@1": 0.17515120967741934, + "pass@2": 0.2418537274574549, + "pass@4": 0.30864086051743567, + "pass@8": 0.36897845390139183, + "pass@16": 0.41813086287003537, + "pass@32": 0.45496716623619304, + "pass@64": 0.48441455201866307, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.1568400679758308, + "pass@2": 0.22048024656849932, + "pass@4": 0.28429913613517355, + "pass@8": 0.34339241837815315, + "pass@16": 0.3939768884772223, + "pass@32": 0.4328402195492693, + "pass@64": 0.4613644571598135, + "pass@128": 0.48338368580060426 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.22079904707857453, + "pass@4": 0.2881610339248135, + "pass@8": 0.3512828123002748, + "pass@16": 0.40352597323708034, + "pass@32": 0.4467676743469794, + "pass@64": 0.4848656649074429, + "pass@128": 0.5195195195195195 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e3b958a67c7f0f342766cb7744a0e01a49817cc1 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4", "results_dir": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/checkpoint-21808_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 963285, "value_mismatch": 74390, "dependency_mismatch": 150777, "answer_mismatch": 55536, "extra_nodes": 43340, "missing_nodes": 55772, "perfect_with_extra": 3673, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 315039, "dependency_mismatch": 49299, "value_mismatch": 27448, "answer_mismatch": 20766, "extra_nodes": 13865, "missing_nodes": 21061, "perfect_with_extra": 783}, "teachers_in_school": {"total": 390656, "perfect": 328678, "value_mismatch": 23040, "answer_mismatch": 17921, "dependency_mismatch": 50302, "extra_nodes": 14717, "perfect_with_extra": 1710, "missing_nodes": 16958}, "movie_festival_awards": {"total": 382336, "perfect": 319568, "dependency_mismatch": 51176, "value_mismatch": 23902, "answer_mismatch": 16849, "missing_nodes": 17753, "extra_nodes": 14758, "perfect_with_extra": 1180}}, "results_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/checkpoint-21808_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/checkpoint-21808_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 142784, "value_mismatch": 1007335, "dependency_mismatch": 1079184, "answer_mismatch": 847280, "extra_nodes": 77922, "missing_nodes": 796643, "perfect_with_extra": 36, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 337352, "dependency_mismatch": 364412, "answer_mismatch": 282982, "extra_nodes": 29611, "missing_nodes": 275011, "perfect": 48068, "perfect_with_extra": 12}, "movie_festival_awards": {"total": 418816, "perfect": 47602, "value_mismatch": 328358, "answer_mismatch": 275542, "dependency_mismatch": 352360, "missing_nodes": 255373, "extra_nodes": 24140, "perfect_with_extra": 17}, "teachers_in_school": {"total": 430464, "perfect": 47114, "dependency_mismatch": 362412, "missing_nodes": 266259, "extra_nodes": 24171, "perfect_with_extra": 7, "value_mismatch": 341625, "answer_mismatch": 288756}}, "results_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/checkpoint-21808_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1106069, "value_mismatch": 1081725, "dependency_mismatch": 1229961, "answer_mismatch": 902816, "extra_nodes": 121262, "missing_nodes": 852415, "perfect_with_extra": 3709, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 363107, "dependency_mismatch": 413711, "value_mismatch": 364800, "answer_mismatch": 303748, "extra_nodes": 43476, "missing_nodes": 296072, "perfect_with_extra": 795}, "teachers_in_school": {"total": 821120, "perfect": 375792, "value_mismatch": 364665, "answer_mismatch": 306677, "dependency_mismatch": 412714, "extra_nodes": 38888, "perfect_with_extra": 1717, "missing_nodes": 283217}, "movie_festival_awards": {"total": 801152, "perfect": 367170, "dependency_mismatch": 403536, "value_mismatch": 352260, "answer_mismatch": 292391, "missing_nodes": 273126, "extra_nodes": 38898, "perfect_with_extra": 1197}}}} diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/results.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..c579369aea243cc9447c92ca54109be6144218dd --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21808_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9501557632398754, + "pass@2": 0.9595015576323987, + "pass@4": 0.9626168224299065, + "pass@8": 0.9626168224299065, + "pass@16": 0.9657320872274143, + "pass@32": 0.9657320872274143, + "pass@64": 0.9657320872274143, + "pass@128": 0.9719626168224299 + }, + "3": { + "pass@1": 0.9877675840978594, + "pass@2": 0.9969418960244648, + "pass@4": 0.9969418960244648, + "pass@8": 0.9969418960244648, + "pass@16": 0.9969418960244648, + "pass@32": 0.9969418960244648, + "pass@64": 0.9969418960244648, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.959375, + "pass@2": 0.975, + "pass@4": 0.978125, + "pass@8": 0.978125, + "pass@16": 0.978125, + "pass@32": 0.984375, + "pass@64": 0.990625, + "pass@128": 0.990625 + }, + "5": { + "pass@1": 0.8825396825396825, + "pass@2": 0.8952380952380953, + "pass@4": 0.9174603174603174, + "pass@8": 0.926984126984127, + "pass@16": 0.9333333333333333, + "pass@32": 0.9365079365079365, + "pass@64": 0.946031746031746, + "pass@128": 0.9587301587301588 + }, + "6": { + "pass@1": 0.8922155688622755, + "pass@2": 0.937125748502994, + "pass@4": 0.9520958083832335, + "pass@8": 0.9670658682634731, + "pass@16": 0.9730538922155688, + "pass@32": 0.9760479041916168, + "pass@64": 0.9790419161676647, + "pass@128": 0.9790419161676647 + }, + "7": { + "pass@1": 0.8047337278106509, + "pass@2": 0.8579881656804734, + "pass@4": 0.8994082840236687, + "pass@8": 0.9201183431952663, + "pass@16": 0.9408284023668639, + "pass@32": 0.9467455621301775, + "pass@64": 0.9497041420118343, + "pass@128": 0.9556213017751479 + }, + "8": { + "pass@1": 0.7564102564102564, + "pass@2": 0.842948717948718, + "pass@4": 0.8974358974358975, + "pass@8": 0.9391025641025641, + "pass@16": 0.9551282051282052, + "pass@32": 0.9551282051282052, + "pass@64": 0.9583333333333334, + "pass@128": 0.9615384615384616 + }, + "9": { + "pass@1": 0.676056338028169, + "pass@2": 0.7774647887323943, + "pass@4": 0.8507042253521127, + "pass@8": 0.8845070422535212, + "pass@16": 0.9352112676056338, + "pass@32": 0.9492957746478873, + "pass@64": 0.9661971830985916, + "pass@128": 0.9690140845070423 + }, + "10": { + "pass@1": 0.616519174041298, + "pass@2": 0.6784660766961652, + "pass@4": 0.7581120943952803, + "pass@8": 0.8171091445427728, + "pass@16": 0.855457227138643, + "pass@32": 0.8761061946902655, + "pass@64": 0.8849557522123894, + "pass@128": 0.887905604719764 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9501557632398754, + "3": 0.9877675840978594, + "4": 0.959375, + "5": 0.8825396825396825, + "6": 0.8922155688622755, + "7": 0.8047337278106509, + "8": 0.7564102564102564, + "9": 0.676056338028169, + "10": 0.616519174041298 + }, + "pass@2": { + "2": 0.9595015576323987, + "3": 0.9969418960244648, + "4": 0.975, + "5": 0.8952380952380953, + "6": 0.937125748502994, + "7": 0.8579881656804734, + "8": 0.842948717948718, + "9": 0.7774647887323943, + "10": 0.6784660766961652 + }, + "pass@4": { + "2": 0.9626168224299065, + "3": 0.9969418960244648, + "4": 0.978125, + "5": 0.9174603174603174, + "6": 0.9520958083832335, + "7": 0.8994082840236687, + "8": 0.8974358974358975, + "9": 0.8507042253521127, + "10": 0.7581120943952803 + }, + "pass@8": { + "2": 0.9626168224299065, + "3": 0.9969418960244648, + "4": 0.978125, + "5": 0.926984126984127, + "6": 0.9670658682634731, + "7": 0.9201183431952663, + "8": 0.9391025641025641, + "9": 0.8845070422535212, + "10": 0.8171091445427728 + }, + "pass@16": { + "2": 0.9657320872274143, + "3": 0.9969418960244648, + "4": 0.978125, + "5": 0.9333333333333333, + "6": 0.9730538922155688, + "7": 0.9408284023668639, + "8": 0.9551282051282052, + "9": 0.9352112676056338, + "10": 0.855457227138643 + }, + "pass@32": { + "2": 0.9657320872274143, + "3": 0.9969418960244648, + "4": 0.984375, + "5": 0.9365079365079365, + "6": 0.9760479041916168, + "7": 0.9467455621301775, + "8": 0.9551282051282052, + "9": 0.9492957746478873, + "10": 0.8761061946902655 + }, + "pass@64": { + "2": 0.9657320872274143, + "3": 0.9969418960244648, + "4": 0.990625, + "5": 0.946031746031746, + "6": 0.9790419161676647, + "7": 0.9497041420118343, + "8": 0.9583333333333334, + "9": 0.9661971830985916, + "10": 0.8849557522123894 + }, + "pass@128": { + "2": 0.9719626168224299, + "3": 1.0, + "4": 0.990625, + "5": 0.9587301587301588, + "6": 0.9790419161676647, + "7": 0.9556213017751479, + "8": 0.9615384615384616, + "9": 0.9690140845070423, + "10": 0.887905604719764 + } + }, + "total_pass_at_k": { + "pass@1": 0.8335021952043229, + "pass@2": 0.8780817291455589, + "pass@4": 0.911178655859507, + "pass@8": 0.9314420803782506, + "pass@16": 0.9476528199932456, + "pass@32": 0.9537318473488686, + "pass@64": 0.9594731509625126, + "pass@128": 0.9635258358662614 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9542857142857143, + "pass@2": 0.9628571428571429, + "pass@4": 0.9628571428571429, + "pass@8": 0.9657142857142857, + "pass@16": 0.9714285714285714, + "pass@32": 0.9742857142857143, + "pass@64": 0.9742857142857143, + "pass@128": 0.9742857142857143 + }, + "3": { + "pass@1": 0.9883381924198251, + "pass@2": 0.9912536443148688, + "pass@4": 0.9912536443148688, + "pass@8": 0.9912536443148688, + "pass@16": 0.9912536443148688, + "pass@32": 0.9912536443148688, + "pass@64": 0.9912536443148688, + "pass@128": 0.9912536443148688 + }, + "4": { + "pass@1": 0.9678362573099415, + "pass@2": 0.9707602339181286, + "pass@4": 0.9707602339181286, + "pass@8": 0.9824561403508771, + "pass@16": 0.9883040935672515, + "pass@32": 0.9912280701754386, + "pass@64": 0.9941520467836257, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.8764705882352941, + "pass@2": 0.9058823529411765, + "pass@4": 0.9205882352941176, + "pass@8": 0.9264705882352942, + "pass@16": 0.9294117647058824, + "pass@32": 0.9382352941176471, + "pass@64": 0.9441176470588235, + "pass@128": 0.9558823529411765 + }, + "6": { + "pass@1": 0.9495268138801262, + "pass@2": 0.9558359621451105, + "pass@4": 0.9716088328075709, + "pass@8": 0.9810725552050473, + "pass@16": 0.9842271293375394, + "pass@32": 0.9873817034700315, + "pass@64": 0.9873817034700315, + "pass@128": 0.9873817034700315 + }, + "7": { + "pass@1": 0.7943037974683544, + "pass@2": 0.8639240506329114, + "pass@4": 0.9018987341772152, + "pass@8": 0.9272151898734177, + "pass@16": 0.9430379746835443, + "pass@32": 0.9620253164556962, + "pass@64": 0.9651898734177216, + "pass@128": 0.9651898734177216 + }, + "8": { + "pass@1": 0.7177177177177178, + "pass@2": 0.7807807807807807, + "pass@4": 0.8588588588588588, + "pass@8": 0.9159159159159159, + "pass@16": 0.954954954954955, + "pass@32": 0.960960960960961, + "pass@64": 0.972972972972973, + "pass@128": 0.975975975975976 + }, + "9": { + "pass@1": 0.6945337620578779, + "pass@2": 0.7942122186495176, + "pass@4": 0.8360128617363344, + "pass@8": 0.9003215434083601, + "pass@16": 0.9453376205787781, + "pass@32": 0.9678456591639871, + "pass@64": 0.9742765273311897, + "pass@128": 0.977491961414791 + }, + "10": { + "pass@1": 0.5134328358208955, + "pass@2": 0.582089552238806, + "pass@4": 0.6686567164179105, + "pass@8": 0.7611940298507462, + "pass@16": 0.8119402985074626, + "pass@32": 0.8477611940298507, + "pass@64": 0.8776119402985074, + "pass@128": 0.8925373134328358 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9542857142857143, + "3": 0.9883381924198251, + "4": 0.9678362573099415, + "5": 0.8764705882352941, + "6": 0.9495268138801262, + "7": 0.7943037974683544, + "8": 0.7177177177177178, + "9": 0.6945337620578779, + "10": 0.5134328358208955 + }, + "pass@2": { + "2": 0.9628571428571429, + "3": 0.9912536443148688, + "4": 0.9707602339181286, + "5": 0.9058823529411765, + "6": 0.9558359621451105, + "7": 0.8639240506329114, + "8": 0.7807807807807807, + "9": 0.7942122186495176, + "10": 0.582089552238806 + }, + "pass@4": { + "2": 0.9628571428571429, + "3": 0.9912536443148688, + "4": 0.9707602339181286, + "5": 0.9205882352941176, + "6": 0.9716088328075709, + "7": 0.9018987341772152, + "8": 0.8588588588588588, + "9": 0.8360128617363344, + "10": 0.6686567164179105 + }, + "pass@8": { + "2": 0.9657142857142857, + "3": 0.9912536443148688, + "4": 0.9824561403508771, + "5": 0.9264705882352942, + "6": 0.9810725552050473, + "7": 0.9272151898734177, + "8": 0.9159159159159159, + "9": 0.9003215434083601, + "10": 0.7611940298507462 + }, + "pass@16": { + "2": 0.9714285714285714, + "3": 0.9912536443148688, + "4": 0.9883040935672515, + "5": 0.9294117647058824, + "6": 0.9842271293375394, + "7": 0.9430379746835443, + "8": 0.954954954954955, + "9": 0.9453376205787781, + "10": 0.8119402985074626 + }, + "pass@32": { + "2": 0.9742857142857143, + "3": 0.9912536443148688, + "4": 0.9912280701754386, + "5": 0.9382352941176471, + "6": 0.9873817034700315, + "7": 0.9620253164556962, + "8": 0.960960960960961, + "9": 0.9678456591639871, + "10": 0.8477611940298507 + }, + "pass@64": { + "2": 0.9742857142857143, + "3": 0.9912536443148688, + "4": 0.9941520467836257, + "5": 0.9441176470588235, + "6": 0.9873817034700315, + "7": 0.9651898734177216, + "8": 0.972972972972973, + "9": 0.9742765273311897, + "10": 0.8776119402985074 + }, + "pass@128": { + "2": 0.9742857142857143, + "3": 0.9912536443148688, + "4": 0.9941520467836257, + "5": 0.9558823529411765, + "6": 0.9873817034700315, + "7": 0.9651898734177216, + "8": 0.975975975975976, + "9": 0.977491961414791, + "10": 0.8925373134328358 + } + }, + "total_pass_at_k": { + "pass@1": 0.8305992634750586, + "pass@2": 0.8687646468028122, + "pass@4": 0.8988952125878809, + "pass@8": 0.9283562102443924, + "pass@16": 0.9467693337797121, + "pass@32": 0.957817207900904, + "pass@64": 0.9645128891864747, + "pass@128": 0.9681955138935386 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9574468085106383, + "pass@2": 0.9665653495440729, + "pass@4": 0.9665653495440729, + "pass@8": 0.9696048632218845, + "pass@16": 0.9696048632218845, + "pass@32": 0.9726443768996961, + "pass@64": 0.9756838905775076, + "pass@128": 0.9756838905775076 + }, + "3": { + "pass@1": 0.9878787878787879, + "pass@2": 0.990909090909091, + "pass@4": 0.9939393939393939, + "pass@8": 0.9939393939393939, + "pass@16": 0.9939393939393939, + "pass@32": 0.996969696969697, + "pass@64": 0.996969696969697, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9526627218934911, + "pass@2": 0.9556213017751479, + "pass@4": 0.9644970414201184, + "pass@8": 0.9644970414201184, + "pass@16": 0.9644970414201184, + "pass@32": 0.9644970414201184, + "pass@64": 0.9644970414201184, + "pass@128": 0.9644970414201184 + }, + "5": { + "pass@1": 0.9101449275362319, + "pass@2": 0.927536231884058, + "pass@4": 0.9449275362318841, + "pass@8": 0.9536231884057971, + "pass@16": 0.9652173913043478, + "pass@32": 0.9652173913043478, + "pass@64": 0.9681159420289855, + "pass@128": 0.9681159420289855 + }, + "6": { + "pass@1": 0.9369627507163324, + "pass@2": 0.9570200573065902, + "pass@4": 0.9713467048710601, + "pass@8": 0.9742120343839542, + "pass@16": 0.9856733524355301, + "pass@32": 0.9856733524355301, + "pass@64": 0.9856733524355301, + "pass@128": 0.9856733524355301 + }, + "7": { + "pass@1": 0.8352601156069365, + "pass@2": 0.8757225433526011, + "pass@4": 0.9248554913294798, + "pass@8": 0.9479768786127167, + "pass@16": 0.9624277456647399, + "pass@32": 0.976878612716763, + "pass@64": 0.9797687861271677, + "pass@128": 0.9797687861271677 + }, + "8": { + "pass@1": 0.780281690140845, + "pass@2": 0.8535211267605634, + "pass@4": 0.9267605633802817, + "pass@8": 0.9661971830985916, + "pass@16": 0.9774647887323944, + "pass@32": 0.9830985915492958, + "pass@64": 0.9887323943661972, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.625748502994012, + "pass@2": 0.7395209580838323, + "pass@4": 0.7994011976047904, + "pass@8": 0.8772455089820359, + "pass@16": 0.9161676646706587, + "pass@32": 0.9311377245508982, + "pass@64": 0.9461077844311377, + "pass@128": 0.9520958083832335 + }, + "10": { + "pass@1": 0.588957055214724, + "pass@2": 0.6840490797546013, + "pass@4": 0.7607361963190185, + "pass@8": 0.8159509202453987, + "pass@16": 0.8619631901840491, + "pass@32": 0.8803680981595092, + "pass@64": 0.8957055214723927, + "pass@128": 0.9049079754601227 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9574468085106383, + "3": 0.9878787878787879, + "4": 0.9526627218934911, + "5": 0.9101449275362319, + "6": 0.9369627507163324, + "7": 0.8352601156069365, + "8": 0.780281690140845, + "9": 0.625748502994012, + "10": 0.588957055214724 + }, + "pass@2": { + "2": 0.9665653495440729, + "3": 0.990909090909091, + "4": 0.9556213017751479, + "5": 0.927536231884058, + "6": 0.9570200573065902, + "7": 0.8757225433526011, + "8": 0.8535211267605634, + "9": 0.7395209580838323, + "10": 0.6840490797546013 + }, + "pass@4": { + "2": 0.9665653495440729, + "3": 0.9939393939393939, + "4": 0.9644970414201184, + "5": 0.9449275362318841, + "6": 0.9713467048710601, + "7": 0.9248554913294798, + "8": 0.9267605633802817, + "9": 0.7994011976047904, + "10": 0.7607361963190185 + }, + "pass@8": { + "2": 0.9696048632218845, + "3": 0.9939393939393939, + "4": 0.9644970414201184, + "5": 0.9536231884057971, + "6": 0.9742120343839542, + "7": 0.9479768786127167, + "8": 0.9661971830985916, + "9": 0.8772455089820359, + "10": 0.8159509202453987 + }, + "pass@16": { + "2": 0.9696048632218845, + "3": 0.9939393939393939, + "4": 0.9644970414201184, + "5": 0.9652173913043478, + "6": 0.9856733524355301, + "7": 0.9624277456647399, + "8": 0.9774647887323944, + "9": 0.9161676646706587, + "10": 0.8619631901840491 + }, + "pass@32": { + "2": 0.9726443768996961, + "3": 0.996969696969697, + "4": 0.9644970414201184, + "5": 0.9652173913043478, + "6": 0.9856733524355301, + "7": 0.976878612716763, + "8": 0.9830985915492958, + "9": 0.9311377245508982, + "10": 0.8803680981595092 + }, + "pass@64": { + "2": 0.9756838905775076, + "3": 0.996969696969697, + "4": 0.9644970414201184, + "5": 0.9681159420289855, + "6": 0.9856733524355301, + "7": 0.9797687861271677, + "8": 0.9887323943661972, + "9": 0.9461077844311377, + "10": 0.8957055214723927 + }, + "pass@128": { + "2": 0.9756838905775076, + "3": 1.0, + "4": 0.9644970414201184, + "5": 0.9681159420289855, + "6": 0.9856733524355301, + "7": 0.9797687861271677, + "8": 0.9943661971830986, + "9": 0.9520958083832335, + "10": 0.9049079754601227 + } + }, + "total_pass_at_k": { + "pass@1": 0.8423984272608126, + "pass@2": 0.8840104849279161, + "pass@4": 0.9177588466579292, + "pass@8": 0.9410222804718218, + "pass@16": 0.9557667103538663, + "pass@32": 0.9623197903014417, + "pass@64": 0.9672346002621232, + "pass@128": 0.9698558322411533 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.954, + "pass@2": 0.963, + "pass@4": 0.964, + "pass@8": 0.966, + "pass@16": 0.969, + "pass@32": 0.971, + "pass@64": 0.972, + "pass@128": 0.974 + }, + "3": { + "pass@1": 0.988, + "pass@2": 0.993, + "pass@4": 0.994, + "pass@8": 0.994, + "pass@16": 0.994, + "pass@32": 0.995, + "pass@64": 0.995, + "pass@128": 0.997 + }, + "4": { + "pass@1": 0.96, + "pass@2": 0.967, + "pass@4": 0.971, + "pass@8": 0.975, + "pass@16": 0.977, + "pass@32": 0.98, + "pass@64": 0.983, + "pass@128": 0.983 + }, + "5": { + "pass@1": 0.89, + "pass@2": 0.91, + "pass@4": 0.928, + "pass@8": 0.936, + "pass@16": 0.943, + "pass@32": 0.947, + "pass@64": 0.953, + "pass@128": 0.961 + }, + "6": { + "pass@1": 0.926, + "pass@2": 0.95, + "pass@4": 0.965, + "pass@8": 0.974, + "pass@16": 0.981, + "pass@32": 0.983, + "pass@64": 0.984, + "pass@128": 0.984 + }, + "7": { + "pass@1": 0.812, + "pass@2": 0.866, + "pass@4": 0.909, + "pass@8": 0.932, + "pass@16": 0.949, + "pass@32": 0.962, + "pass@64": 0.965, + "pass@128": 0.967 + }, + "8": { + "pass@1": 0.752, + "pass@2": 0.826, + "pass@4": 0.895, + "pass@8": 0.941, + "pass@16": 0.963, + "pass@32": 0.967, + "pass@64": 0.974, + "pass@128": 0.978 + }, + "9": { + "pass@1": 0.665, + "pass@2": 0.77, + "pass@4": 0.829, + "pass@8": 0.887, + "pass@16": 0.932, + "pass@32": 0.949, + "pass@64": 0.962, + "pass@128": 0.966 + }, + "10": { + "pass@1": 0.573, + "pass@2": 0.648, + "pass@4": 0.729, + "pass@8": 0.798, + "pass@16": 0.843, + "pass@32": 0.868, + "pass@64": 0.886, + "pass@128": 0.895 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.954, + "3": 0.988, + "4": 0.96, + "5": 0.89, + "6": 0.926, + "7": 0.812, + "8": 0.752, + "9": 0.665, + "10": 0.573 + }, + "pass@2": { + "2": 0.963, + "3": 0.993, + "4": 0.967, + "5": 0.91, + "6": 0.95, + "7": 0.866, + "8": 0.826, + "9": 0.77, + "10": 0.648 + }, + "pass@4": { + "2": 0.964, + "3": 0.994, + "4": 0.971, + "5": 0.928, + "6": 0.965, + "7": 0.909, + "8": 0.895, + "9": 0.829, + "10": 0.729 + }, + "pass@8": { + "2": 0.966, + "3": 0.994, + "4": 0.975, + "5": 0.936, + "6": 0.974, + "7": 0.932, + "8": 0.941, + "9": 0.887, + "10": 0.798 + }, + "pass@16": { + "2": 0.969, + "3": 0.994, + "4": 0.977, + "5": 0.943, + "6": 0.981, + "7": 0.949, + "8": 0.963, + "9": 0.932, + "10": 0.843 + }, + "pass@32": { + "2": 0.971, + "3": 0.995, + "4": 0.98, + "5": 0.947, + "6": 0.983, + "7": 0.962, + "8": 0.967, + "9": 0.949, + "10": 0.868 + }, + "pass@64": { + "2": 0.972, + "3": 0.995, + "4": 0.983, + "5": 0.953, + "6": 0.984, + "7": 0.965, + "8": 0.974, + "9": 0.962, + "10": 0.886 + }, + "pass@128": { + "2": 0.974, + "3": 0.997, + "4": 0.983, + "5": 0.961, + "6": 0.984, + "7": 0.967, + "8": 0.978, + "9": 0.966, + "10": 0.895 + } + }, + "total_pass_at_k": { + "pass@1": 0.8355555555555556, + "pass@2": 0.877, + "pass@4": 0.9093333333333333, + "pass@8": 0.9336666666666666, + "pass@16": 0.9501111111111111, + "pass@32": 0.958, + "pass@64": 0.9637777777777777, + "pass@128": 0.9672222222222222 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21808_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.42776203966005666, + "pass@2": 0.5325779036827195, + "pass@4": 0.6203966005665722, + "pass@8": 0.7025495750708215, + "pass@16": 0.7705382436260623, + "pass@32": 0.8186968838526912, + "pass@64": 0.8356940509915014, + "pass@128": 0.8526912181303116 + }, + "12": { + "pass@1": 0.3983050847457627, + "pass@2": 0.4717514124293785, + "pass@4": 0.536723163841808, + "pass@8": 0.6045197740112994, + "pass@16": 0.6751412429378532, + "pass@32": 0.7033898305084746, + "pass@64": 0.7372881355932204, + "pass@128": 0.7542372881355932 + }, + "13": { + "pass@1": 0.13793103448275862, + "pass@2": 0.20376175548589343, + "pass@4": 0.2476489028213166, + "pass@8": 0.3072100313479624, + "pass@16": 0.34796238244514105, + "pass@32": 0.38557993730407525, + "pass@64": 0.42946708463949845, + "pass@128": 0.4670846394984326 + }, + "14": { + "pass@1": 0.07476635514018691, + "pass@2": 0.1059190031152648, + "pass@4": 0.1277258566978193, + "pass@8": 0.1557632398753894, + "pass@16": 0.1806853582554517, + "pass@32": 0.23052959501557632, + "pass@64": 0.2616822429906542, + "pass@128": 0.29906542056074764 + }, + "15": { + "pass@1": 0.03757225433526012, + "pass@2": 0.049132947976878616, + "pass@4": 0.07514450867052024, + "pass@8": 0.0953757225433526, + "pass@16": 0.10404624277456648, + "pass@32": 0.13005780346820808, + "pass@64": 0.16184971098265896, + "pass@128": 0.17630057803468208 + }, + "16": { + "pass@1": 0.0032258064516129032, + "pass@2": 0.0032258064516129032, + "pass@4": 0.0064516129032258064, + "pass@8": 0.00967741935483871, + "pass@16": 0.012903225806451613, + "pass@32": 0.01935483870967742, + "pass@64": 0.01935483870967742, + "pass@128": 0.025806451612903226 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.008670520231213872, + "pass@8": 0.014450867052023121, + "pass@16": 0.017341040462427744, + "pass@32": 0.017341040462427744, + "pass@64": 0.023121387283236993, + "pass@128": 0.02601156069364162 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.002857142857142857, + "pass@64": 0.002857142857142857, + "pass@128": 0.002857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.42776203966005666, + "12": 0.3983050847457627, + "13": 0.13793103448275862, + "14": 0.07476635514018691, + "15": 0.03757225433526012, + "16": 0.0032258064516129032, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5325779036827195, + "12": 0.4717514124293785, + "13": 0.20376175548589343, + "14": 0.1059190031152648, + "15": 0.049132947976878616, + "16": 0.0032258064516129032, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6203966005665722, + "12": 0.536723163841808, + "13": 0.2476489028213166, + "14": 0.1277258566978193, + "15": 0.07514450867052024, + "16": 0.0064516129032258064, + "17": 0.008670520231213872, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7025495750708215, + "12": 0.6045197740112994, + "13": 0.3072100313479624, + "14": 0.1557632398753894, + "15": 0.0953757225433526, + "16": 0.00967741935483871, + "17": 0.014450867052023121, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.7705382436260623, + "12": 0.6751412429378532, + "13": 0.34796238244514105, + "14": 0.1806853582554517, + "15": 0.10404624277456648, + "16": 0.012903225806451613, + "17": 0.017341040462427744, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8186968838526912, + "12": 0.7033898305084746, + "13": 0.38557993730407525, + "14": 0.23052959501557632, + "15": 0.13005780346820808, + "16": 0.01935483870967742, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8356940509915014, + "12": 0.7372881355932204, + "13": 0.42946708463949845, + "14": 0.2616822429906542, + "15": 0.16184971098265896, + "16": 0.01935483870967742, + "17": 0.023121387283236993, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8526912181303116, + "12": 0.7542372881355932, + "13": 0.4670846394984326, + "14": 0.29906542056074764, + "15": 0.17630057803468208, + "16": 0.025806451612903226, + "17": 0.02601156069364162, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11114413075780089, + "pass@2": 0.14026745913818722, + "pass@4": 0.1664190193164933, + "pass@8": 0.19346210995542348, + "pass@16": 0.21575037147102527, + "pass@32": 0.23566121842496285, + "pass@64": 0.25200594353640415, + "pass@128": 0.2650817236255572 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4793650793650794, + "pass@2": 0.5619047619047619, + "pass@4": 0.6571428571428571, + "pass@8": 0.7428571428571429, + "pass@16": 0.8031746031746032, + "pass@32": 0.8380952380952381, + "pass@64": 0.8793650793650793, + "pass@128": 0.8920634920634921 + }, + "12": { + "pass@1": 0.3333333333333333, + "pass@2": 0.3944954128440367, + "pass@4": 0.4801223241590214, + "pass@8": 0.5565749235474006, + "pass@16": 0.6116207951070336, + "pass@32": 0.6788990825688074, + "pass@64": 0.709480122324159, + "pass@128": 0.7431192660550459 + }, + "13": { + "pass@1": 0.1574074074074074, + "pass@2": 0.2191358024691358, + "pass@4": 0.25308641975308643, + "pass@8": 0.30864197530864196, + "pass@16": 0.345679012345679, + "pass@32": 0.39814814814814814, + "pass@64": 0.4228395061728395, + "pass@128": 0.44135802469135804 + }, + "14": { + "pass@1": 0.10682492581602374, + "pass@2": 0.13649851632047477, + "pass@4": 0.16023738872403562, + "pass@8": 0.18991097922848665, + "pass@16": 0.2314540059347181, + "pass@32": 0.27002967359050445, + "pass@64": 0.3264094955489614, + "pass@128": 0.3620178041543027 + }, + "15": { + "pass@1": 0.06172839506172839, + "pass@2": 0.10185185185185185, + "pass@4": 0.12037037037037036, + "pass@8": 0.13580246913580246, + "pass@16": 0.15432098765432098, + "pass@32": 0.19135802469135801, + "pass@64": 0.21604938271604937, + "pass@128": 0.24074074074074073 + }, + "16": { + "pass@1": 0.008849557522123894, + "pass@2": 0.011799410029498525, + "pass@4": 0.014749262536873156, + "pass@8": 0.02064896755162242, + "pass@16": 0.029498525073746312, + "pass@32": 0.04129793510324484, + "pass@64": 0.05309734513274336, + "pass@128": 0.05604719764011799 + }, + "17": { + "pass@1": 0.009036144578313253, + "pass@2": 0.012048192771084338, + "pass@4": 0.012048192771084338, + "pass@8": 0.018072289156626505, + "pass@16": 0.02710843373493976, + "pass@32": 0.030120481927710843, + "pass@64": 0.03313253012048193, + "pass@128": 0.03313253012048193 + }, + "18": { + "pass@1": 0.0064516129032258064, + "pass@2": 0.0064516129032258064, + "pass@4": 0.00967741935483871, + "pass@8": 0.01935483870967742, + "pass@16": 0.01935483870967742, + "pass@32": 0.02258064516129032, + "pass@64": 0.025806451612903226, + "pass@128": 0.02903225806451613 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4793650793650794, + "12": 0.3333333333333333, + "13": 0.1574074074074074, + "14": 0.10682492581602374, + "15": 0.06172839506172839, + "16": 0.008849557522123894, + "17": 0.009036144578313253, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5619047619047619, + "12": 0.3944954128440367, + "13": 0.2191358024691358, + "14": 0.13649851632047477, + "15": 0.10185185185185185, + "16": 0.011799410029498525, + "17": 0.012048192771084338, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6571428571428571, + "12": 0.4801223241590214, + "13": 0.25308641975308643, + "14": 0.16023738872403562, + "15": 0.12037037037037036, + "16": 0.014749262536873156, + "17": 0.012048192771084338, + "18": 0.00967741935483871, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7428571428571429, + "12": 0.5565749235474006, + "13": 0.30864197530864196, + "14": 0.18991097922848665, + "15": 0.13580246913580246, + "16": 0.02064896755162242, + "17": 0.018072289156626505, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8031746031746032, + "12": 0.6116207951070336, + "13": 0.345679012345679, + "14": 0.2314540059347181, + "15": 0.15432098765432098, + "16": 0.029498525073746312, + "17": 0.02710843373493976, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8380952380952381, + "12": 0.6788990825688074, + "13": 0.39814814814814814, + "14": 0.27002967359050445, + "15": 0.19135802469135801, + "16": 0.04129793510324484, + "17": 0.030120481927710843, + "18": 0.02258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8793650793650793, + "12": 0.709480122324159, + "13": 0.4228395061728395, + "14": 0.3264094955489614, + "15": 0.21604938271604937, + "16": 0.05309734513274336, + "17": 0.03313253012048193, + "18": 0.025806451612903226, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8920634920634921, + "12": 0.7431192660550459, + "13": 0.44135802469135804, + "14": 0.3620178041543027, + "15": 0.24074074074074073, + "16": 0.05604719764011799, + "17": 0.03313253012048193, + "18": 0.02903225806451613, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11460880195599021, + "pass@2": 0.14242053789731052, + "pass@4": 0.16839853300733496, + "pass@8": 0.1965158924205379, + "pass@16": 0.21943765281173594, + "pass@32": 0.24419315403422984, + "pass@64": 0.26375305623471884, + "pass@128": 0.2768948655256724 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5, + "pass@2": 0.5993975903614458, + "pass@4": 0.6686746987951807, + "pass@8": 0.7530120481927711, + "pass@16": 0.8132530120481928, + "pass@32": 0.8554216867469879, + "pass@64": 0.8825301204819277, + "pass@128": 0.9006024096385542 + }, + "12": { + "pass@1": 0.3322884012539185, + "pass@2": 0.38557993730407525, + "pass@4": 0.4827586206896552, + "pass@8": 0.5454545454545454, + "pass@16": 0.6206896551724138, + "pass@32": 0.6614420062695925, + "pass@64": 0.6927899686520376, + "pass@128": 0.7366771159874608 + }, + "13": { + "pass@1": 0.13725490196078433, + "pass@2": 0.18207282913165265, + "pass@4": 0.23529411764705882, + "pass@8": 0.29411764705882354, + "pass@16": 0.35014005602240894, + "pass@32": 0.42016806722689076, + "pass@64": 0.4677871148459384, + "pass@128": 0.5070028011204482 + }, + "14": { + "pass@1": 0.1111111111111111, + "pass@2": 0.13157894736842105, + "pass@4": 0.14912280701754385, + "pass@8": 0.18421052631578946, + "pass@16": 0.20760233918128654, + "pass@32": 0.23684210526315788, + "pass@64": 0.26900584795321636, + "pass@128": 0.30701754385964913 + }, + "15": { + "pass@1": 0.024242424242424242, + "pass@2": 0.030303030303030304, + "pass@4": 0.04242424242424243, + "pass@8": 0.05454545454545454, + "pass@16": 0.0696969696969697, + "pass@32": 0.10303030303030303, + "pass@64": 0.13333333333333333, + "pass@128": 0.1606060606060606 + }, + "16": { + "pass@1": 0.011396011396011397, + "pass@2": 0.011396011396011397, + "pass@4": 0.014245014245014245, + "pass@8": 0.019943019943019943, + "pass@16": 0.02564102564102564, + "pass@32": 0.03133903133903134, + "pass@64": 0.03418803418803419, + "pass@128": 0.03418803418803419 + }, + "17": { + "pass@1": 0.003105590062111801, + "pass@2": 0.003105590062111801, + "pass@4": 0.006211180124223602, + "pass@8": 0.012422360248447204, + "pass@16": 0.012422360248447204, + "pass@32": 0.015527950310559006, + "pass@64": 0.021739130434782608, + "pass@128": 0.021739130434782608 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0029411764705882353, + "pass@16": 0.0058823529411764705, + "pass@32": 0.0058823529411764705, + "pass@64": 0.0058823529411764705, + "pass@128": 0.008823529411764706 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5, + "12": 0.3322884012539185, + "13": 0.13725490196078433, + "14": 0.1111111111111111, + "15": 0.024242424242424242, + "16": 0.011396011396011397, + "17": 0.003105590062111801, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5993975903614458, + "12": 0.38557993730407525, + "13": 0.18207282913165265, + "14": 0.13157894736842105, + "15": 0.030303030303030304, + "16": 0.011396011396011397, + "17": 0.003105590062111801, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6686746987951807, + "12": 0.4827586206896552, + "13": 0.23529411764705882, + "14": 0.14912280701754385, + "15": 0.04242424242424243, + "16": 0.014245014245014245, + "17": 0.006211180124223602, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7530120481927711, + "12": 0.5454545454545454, + "13": 0.29411764705882354, + "14": 0.18421052631578946, + "15": 0.05454545454545454, + "16": 0.019943019943019943, + "17": 0.012422360248447204, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8132530120481928, + "12": 0.6206896551724138, + "13": 0.35014005602240894, + "14": 0.20760233918128654, + "15": 0.0696969696969697, + "16": 0.02564102564102564, + "17": 0.012422360248447204, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8554216867469879, + "12": 0.6614420062695925, + "13": 0.42016806722689076, + "14": 0.23684210526315788, + "15": 0.10303030303030303, + "16": 0.03133903133903134, + "17": 0.015527950310559006, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8825301204819277, + "12": 0.6927899686520376, + "13": 0.4677871148459384, + "14": 0.26900584795321636, + "15": 0.13333333333333333, + "16": 0.03418803418803419, + "17": 0.021739130434782608, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.9006024096385542, + "12": 0.7366771159874608, + "13": 0.5070028011204482, + "14": 0.30701754385964913, + "15": 0.1606060606060606, + "16": 0.03418803418803419, + "17": 0.021739130434782608, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.11061552185548618, + "pass@2": 0.13291703835860838, + "pass@4": 0.15819209039548024, + "pass@8": 0.18495391019922688, + "pass@16": 0.2087421944692239, + "pass@32": 0.2313410645257211, + "pass@64": 0.24918227772821885, + "pass@128": 0.2661314302705917 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.468, + "pass@2": 0.564, + "pass@4": 0.648, + "pass@8": 0.732, + "pass@16": 0.795, + "pass@32": 0.837, + "pass@64": 0.865, + "pass@128": 0.881 + }, + "12": { + "pass@1": 0.356, + "pass@2": 0.419, + "pass@4": 0.501, + "pass@8": 0.57, + "pass@16": 0.637, + "pass@32": 0.682, + "pass@64": 0.714, + "pass@128": 0.745 + }, + "13": { + "pass@1": 0.144, + "pass@2": 0.201, + "pass@4": 0.245, + "pass@8": 0.303, + "pass@16": 0.348, + "pass@32": 0.402, + "pass@64": 0.441, + "pass@128": 0.473 + }, + "14": { + "pass@1": 0.098, + "pass@2": 0.125, + "pass@4": 0.146, + "pass@8": 0.177, + "pass@16": 0.207, + "pass@32": 0.246, + "pass@64": 0.286, + "pass@128": 0.323 + }, + "15": { + "pass@1": 0.041, + "pass@2": 0.06, + "pass@4": 0.079, + "pass@8": 0.095, + "pass@16": 0.109, + "pass@32": 0.141, + "pass@64": 0.17, + "pass@128": 0.192 + }, + "16": { + "pass@1": 0.008, + "pass@2": 0.009, + "pass@4": 0.012, + "pass@8": 0.017, + "pass@16": 0.023, + "pass@32": 0.031, + "pass@64": 0.036, + "pass@128": 0.039 + }, + "17": { + "pass@1": 0.004, + "pass@2": 0.005, + "pass@4": 0.009, + "pass@8": 0.015, + "pass@16": 0.019, + "pass@32": 0.021, + "pass@64": 0.026, + "pass@128": 0.027 + }, + "18": { + "pass@1": 0.002, + "pass@2": 0.002, + "pass@4": 0.003, + "pass@8": 0.007, + "pass@16": 0.008, + "pass@32": 0.01, + "pass@64": 0.011, + "pass@128": 0.013 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.468, + "12": 0.356, + "13": 0.144, + "14": 0.098, + "15": 0.041, + "16": 0.008, + "17": 0.004, + "18": 0.002, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.564, + "12": 0.419, + "13": 0.201, + "14": 0.125, + "15": 0.06, + "16": 0.009, + "17": 0.005, + "18": 0.002, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.648, + "12": 0.501, + "13": 0.245, + "14": 0.146, + "15": 0.079, + "16": 0.012, + "17": 0.009, + "18": 0.003, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.732, + "12": 0.57, + "13": 0.303, + "14": 0.177, + "15": 0.095, + "16": 0.017, + "17": 0.015, + "18": 0.007, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.795, + "12": 0.637, + "13": 0.348, + "14": 0.207, + "15": 0.109, + "16": 0.023, + "17": 0.019, + "18": 0.008, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.837, + "12": 0.682, + "13": 0.402, + "14": 0.246, + "15": 0.141, + "16": 0.031, + "17": 0.021, + "18": 0.01, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.865, + "12": 0.714, + "13": 0.441, + "14": 0.286, + "15": 0.17, + "16": 0.036, + "17": 0.026, + "18": 0.011, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.881, + "12": 0.745, + "13": 0.473, + "14": 0.323, + "15": 0.192, + "16": 0.039, + "17": 0.027, + "18": 0.013, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.1121, + "pass@2": 0.1385, + "pass@4": 0.1643, + "pass@8": 0.1916, + "pass@16": 0.2146, + "pass@32": 0.237, + "pass@64": 0.2549, + "pass@128": 0.2693 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/summary.csv b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..f7fb7971d912246dc6c9d0cbe82235be10cdb2e7 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21808,0.9517916666666667,0.3380640625,0.6287771381578947,0.0,0.0,0.0,196.2878142361111,313.0753828125,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/summary.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..77800e287b955d81a3f35a55deb514c0ffc09987 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block4/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21808", + "id_acc": 0.9517916666666667, + "ood_acc": 0.3380640625, + "total_acc": 0.6287771381578947, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.2878142361111, + "ood_avg_resp_len": 313.0753828125, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 505986, + "answer_accuracy": 0.6248839116345242, + "avg_response_len": 244.86610071530194, + "resp_tokens_sum": 198274938, + "pass_at_k": { + "pass@1": 0.6248839116345242, + "pass@2": 0.672510150584643, + "pass@4": 0.711296164429809, + "pass@8": 0.7421341642455704, + "pass@16": 0.766649491777681, + "pass@32": 0.7871263323219219, + "pass@64": 0.8054991086246542, + "pass@128": 0.8226367372747392 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8832964601769911, + "pass@2": 0.9172103279678535, + "pass@4": 0.937207035237852, + "pass@8": 0.9511780335899438, + "pass@16": 0.9620826801919633, + "pass@32": 0.9708490352016442, + "pass@64": 0.978042327328383, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 0.9566053348909658, + "pass@2": 0.962186787107219, + "pass@4": 0.9660673160597917, + "pass@8": 0.9685307941081832, + "pass@16": 0.9705478781887977, + "pass@32": 0.9729358037598592, + "pass@64": 0.9762551112290103, + "pass@128": 0.9813084112149533 + }, + "3": { + "pass@1": 0.9943377293577982, + "pass@2": 0.9960855667124179, + "pass@4": 0.9972139947529495, + "pass@8": 0.9981468940282231, + "pass@16": 0.9989822280288635, + "pass@32": 0.9997163691810725, + "pass@64": 0.9999905325251474, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9753662109375, + "pass@2": 0.9802957369586615, + "pass@4": 0.983529715621485, + "pass@8": 0.9857348060465551, + "pass@16": 0.9876058181840719, + "pass@32": 0.9903074532142714, + "pass@64": 0.9941683070866141, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.934672619047619, + "pass@2": 0.9468152418447692, + "pass@4": 0.9547582409341686, + "pass@8": 0.9609408470768459, + "pass@16": 0.9669133908826691, + "pass@32": 0.9727478686315325, + "pass@64": 0.9775761611445652, + "pass@128": 0.9809523809523809 + }, + "6": { + "pass@1": 0.9568909056886228, + "pass@2": 0.9743243581734176, + "pass@4": 0.982969407080103, + "pass@8": 0.9869105050103905, + "pass@16": 0.9889078989792498, + "pass@32": 0.9902473376106875, + "pass@64": 0.9909312655090486, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9426081730769231, + "pass@2": 0.9565109141312955, + "pass@4": 0.9643415889581849, + "pass@8": 0.9704004882673377, + "pass@16": 0.9755356821468868, + "pass@32": 0.9802742371656721, + "pass@64": 0.9851040508965963, + "pass@128": 0.9881656804733728 + }, + "8": { + "pass@1": 0.9438351362179487, + "pass@2": 0.9649525855542097, + "pass@4": 0.9742177161749009, + "pass@8": 0.9796460229237937, + "pass@16": 0.9837141372979744, + "pass@32": 0.9876441458962127, + "pass@64": 0.990970237640019, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9236355633802817, + "pass@2": 0.9530761339691695, + "pass@4": 0.9679002519051312, + "pass@8": 0.9780128398804888, + "pass@16": 0.9860372032987277, + "pass@32": 0.9927718536917328, + "pass@64": 0.9977068384538204, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8158640226628895, + "pass@2": 0.8690087495259979, + "pass@4": 0.9003459481693683, + "pass@8": 0.9208178113433934, + "pass@16": 0.936255057279863, + "pass@32": 0.9486361034628609, + "pass@64": 0.9589588726836659, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.6557203389830508, + "pass@2": 0.7513526513634943, + "pass@4": 0.8129071535549578, + "pass@8": 0.8537187806022632, + "pass@16": 0.8834961886746142, + "pass@32": 0.9062902420548817, + "pass@64": 0.925781100999793, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.4441859326018809, + "pass@2": 0.537450170315701, + "pass@4": 0.612993540286304, + "pass@8": 0.6709888225301639, + "pass@16": 0.7145216319389676, + "pass@32": 0.7501344384006141, + "pass@64": 0.7802929625106815, + "pass@128": 0.8025078369905956 + }, + "14": { + "pass@1": 0.34971281152647976, + "pass@2": 0.4340566512129908, + "pass@4": 0.503801084735903, + "pass@8": 0.559338023432466, + "pass@16": 0.6046153041994793, + "pass@32": 0.6462515330765554, + "pass@64": 0.6884805551577264, + "pass@128": 0.7320872274143302 + }, + "15": { + "pass@1": 0.27420520231213874, + "pass@2": 0.3648961272814163, + "pass@4": 0.4489088842218423, + "pass@8": 0.5161746732770215, + "pass@16": 0.5669481987005399, + "pass@32": 0.6095912062248127, + "pass@64": 0.6513023988475989, + "pass@128": 0.6936416184971098 + }, + "16": { + "pass@1": 0.19057459677419356, + "pass@2": 0.25924601536703096, + "pass@4": 0.3243514381266857, + "pass@8": 0.3820198056073008, + "pass@16": 0.43185101815612564, + "pass@32": 0.4727802423483779, + "pass@64": 0.507659460573341, + "pass@128": 0.5451612903225806 + }, + "17": { + "pass@1": 0.1830970014450867, + "pass@2": 0.25075561251194756, + "pass@4": 0.3177439135137008, + "pass@8": 0.3787659856147468, + "pass@16": 0.43273696124530503, + "pass@32": 0.4781735761884427, + "pass@64": 0.5152145582863207, + "pass@128": 0.546242774566474 + }, + "18": { + "pass@1": 0.17191964285714287, + "pass@2": 0.2386410292463441, + "pass@4": 0.302181122181156, + "pass@8": 0.35631130956435, + "pass@16": 0.4024186189231844, + "pass@32": 0.44305521959331307, + "pass@64": 0.4812509405361627, + "pass@128": 0.52 + }, + "19": { + "pass@1": 0.15561655405405406, + "pass@2": 0.22392286479392776, + "pass@4": 0.29112672092790204, + "pass@8": 0.3476218784231473, + "pass@16": 0.3900725810744725, + "pass@32": 0.42293491455607535, + "pass@64": 0.45500925055441316, + "pass@128": 0.4894894894894895 + }, + "20": { + "pass@1": 0.1527777777777778, + "pass@2": 0.22137873601948407, + "pass@4": 0.29151925350547414, + "pass@8": 0.3513487308384057, + "pass@16": 0.39666827573276253, + "pass@32": 0.43178798807608976, + "pass@64": 0.46000330337155665, + "pass@128": 0.4804804804804805 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 514444, + "answer_accuracy": 0.6265150038971161, + "avg_response_len": 254.71906298713952, + "resp_tokens_sum": 209154917, + "pass_at_k": { + "pass@1": 0.6265150038971161, + "pass@2": 0.6711105745331151, + "pass@4": 0.7087658103970431, + "pass@8": 0.7405722739721018, + "pass@16": 0.7674043940596819, + "pass@32": 0.7905383572324718, + "pass@64": 0.8108683024339327, + "pass@128": 0.8286827747466875 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8846338190184049, + "pass@2": 0.9194517324042314, + "pass@4": 0.9400236116957766, + "pass@8": 0.9542828313525812, + "pass@16": 0.9646392990472547, + "pass@32": 0.9717789665917326, + "pass@64": 0.9761131191473751, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 0.9572568389057751, + "pass@2": 0.9598992114017663, + "pass@4": 0.9626694922063312, + "pass@8": 0.9658379624318189, + "pass@16": 0.9690636373795826, + "pass@32": 0.972155719959404, + "pass@64": 0.9757077858616603, + "pass@128": 0.9787234042553191 + }, + "3": { + "pass@1": 0.9955965909090909, + "pass@2": 0.9970606657122405, + "pass@4": 0.9975789384281509, + "pass@8": 0.9980929433414543, + "pass@16": 0.9988386081686187, + "pass@32": 0.9996178951468804, + "pass@64": 0.9999800850835996, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9837740384615384, + "pass@2": 0.9871876164795229, + "pass@4": 0.988965550626438, + "pass@8": 0.9903578338731447, + "pass@16": 0.9910320953813216, + "pass@32": 0.9911233149752038, + "pass@64": 0.9911242603502997, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.9695199275362318, + "pass@2": 0.9757246376811596, + "pass@4": 0.9795172891432047, + "pass@8": 0.9828762612538346, + "pass@16": 0.9857687021768708, + "pass@32": 0.9885129019107062, + "pass@64": 0.9915973976452171, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9840168338108882, + "pass@2": 0.995195426189563, + "pass@4": 0.9990684802472753, + "pass@8": 0.9999207029235206, + "pass@16": 0.9999989480294558, + "pass@32": 0.9999999999297677, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9557893786127167, + "pass@2": 0.9670289813845525, + "pass@4": 0.9733226897035272, + "pass@8": 0.9776014792969531, + "pass@16": 0.9806254260386871, + "pass@32": 0.9831697369077875, + "pass@64": 0.9850731496000922, + "pass@128": 0.9855491329479769 + }, + "8": { + "pass@1": 0.9510343309859155, + "pass@2": 0.9731552206942445, + "pass@4": 0.9841903806038328, + "pass@8": 0.9894380107543043, + "pass@16": 0.9924766599199953, + "pass@32": 0.9949998418278819, + "pass@64": 0.9975485092564174, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9023203592814372, + "pass@2": 0.9278947027205433, + "pass@4": 0.9418790857542502, + "pass@8": 0.95232354181696, + "pass@16": 0.9613164420633297, + "pass@32": 0.9708170510530576, + "pass@64": 0.9803618880355766, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8197242093373494, + "pass@2": 0.879619980907884, + "pass@4": 0.9152084375183525, + "pass@8": 0.9365968410373897, + "pass@16": 0.9502230602590158, + "pass@32": 0.959630325530834, + "pass@64": 0.9672099492738422, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.6610011755485894, + "pass@2": 0.7461709821538761, + "pass@4": 0.8065136687459518, + "pass@8": 0.8485894222226024, + "pass@16": 0.8752288825205958, + "pass@32": 0.8914381380619772, + "pass@64": 0.9036142400296753, + "pass@128": 0.9153605015673981 + }, + "13": { + "pass@1": 0.38804271708683474, + "pass@2": 0.4778598447252919, + "pass@4": 0.5540819498403033, + "pass@8": 0.619142374924008, + "pass@16": 0.6772598039791219, + "pass@32": 0.7320106065745692, + "pass@64": 0.781803259607603, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.3611796418128655, + "pass@2": 0.4434256573191508, + "pass@4": 0.5109200042428902, + "pass@8": 0.5672925734838943, + "pass@16": 0.6168776944628133, + "pass@32": 0.6620983039384781, + "pass@64": 0.70456770688268, + "pass@128": 0.7456140350877193 + }, + "15": { + "pass@1": 0.2217092803030303, + "pass@2": 0.3015707020997377, + "pass@4": 0.37985566946177185, + "pass@8": 0.4510174568078622, + "pass@16": 0.5158476685485044, + "pass@32": 0.5753658130979193, + "pass@64": 0.6249801268738032, + "pass@128": 0.6666666666666666 + }, + "16": { + "pass@1": 0.2050391737891738, + "pass@2": 0.2738355121475203, + "pass@4": 0.338386615668768, + "pass@8": 0.3932875901414371, + "pass@16": 0.43886406008834783, + "pass@32": 0.4774662643312779, + "pass@64": 0.5142127541987865, + "pass@128": 0.5498575498575499 + }, + "17": { + "pass@1": 0.1947059394409938, + "pass@2": 0.2682923381180613, + "pass@4": 0.3440897881747702, + "pass@8": 0.41833600483808464, + "pass@16": 0.48600653081169176, + "pass@32": 0.5444368907837626, + "pass@64": 0.5933865414431194, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.1689797794117647, + "pass@2": 0.23628271190365893, + "pass@4": 0.3063349933280399, + "pass@8": 0.37162887977886394, + "pass@16": 0.4290596832304357, + "pass@32": 0.4805349660760809, + "pass@64": 0.5257714119613119, + "pass@128": 0.5647058823529412 + }, + "19": { + "pass@1": 0.14950706845238096, + "pass@2": 0.21081337000843647, + "pass@4": 0.27120366957478526, + "pass@8": 0.32483243652106647, + "pass@16": 0.3707793523131844, + "pass@32": 0.4109668939345245, + "pass@64": 0.4485305995646864, + "pass@128": 0.48214285714285715 + }, + "20": { + "pass@1": 0.13929172904191617, + "pass@2": 0.20208254750341842, + "pass@4": 0.2660832878674596, + "pass@8": 0.32449652847649324, + "pass@16": 0.37296661066376247, + "pass@32": 0.4105719174561622, + "pass@64": 0.4410038284084724, + "pass@128": 0.47005988023952094 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 508756, + "answer_accuracy": 0.6350305559993609, + "avg_response_len": 273.8933398406295, + "resp_tokens_sum": 219430197, + "pass_at_k": { + "pass@1": 0.6350305559993609, + "pass@2": 0.6817842149823968, + "pass@4": 0.7211618787655941, + "pass@8": 0.7535477416864765, + "pass@16": 0.7797154196182626, + "pass@32": 0.8009384390103981, + "pass@64": 0.8184491248628813, + "pass@128": 0.8332001917239176 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8779384328358208, + "pass@2": 0.919579636267481, + "pass@4": 0.9469269764786861, + "pass@8": 0.9633563596612335, + "pass@16": 0.9729431971434862, + "pass@32": 0.9793303717895566, + "pass@64": 0.9847047119439397, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 0.9700892857142858, + "pass@2": 0.9764303290213722, + "pass@4": 0.9802661671755313, + "pass@8": 0.9824211322852868, + "pass@16": 0.984386055526147, + "pass@32": 0.9867049961682504, + "pass@64": 0.988322267277566, + "pass@128": 0.9885714285714285 + }, + "3": { + "pass@1": 0.9918913994169096, + "pass@2": 0.9946228421064712, + "pass@4": 0.996332737835613, + "pass@8": 0.9970006677667912, + "pass@16": 0.9970835775181903, + "pass@32": 0.9970845480611684, + "pass@64": 0.9970845481049563, + "pass@128": 0.9970845481049563 + }, + "4": { + "pass@1": 0.9822277046783626, + "pass@2": 0.9848502181470739, + "pass@4": 0.9881618657316958, + "pass@8": 0.9918373424468889, + "pass@16": 0.9948832071878148, + "pass@32": 0.996599044812975, + "pass@64": 0.9970563844205709, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9504136029411765, + "pass@2": 0.9648897058823532, + "pass@4": 0.9740956067072498, + "pass@8": 0.9794277261657189, + "pass@16": 0.9827043544002162, + "pass@32": 0.985393110798894, + "pass@64": 0.9876163898807621, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9810725552050473, + "pass@2": 0.9868364061452098, + "pass@4": 0.9897118630084484, + "pass@8": 0.992015798703835, + "pass@16": 0.9933635236323332, + "pass@32": 0.9936785012993264, + "pass@64": 0.9936908487339506, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.9652145965189873, + "pass@2": 0.9762272781072461, + "pass@4": 0.981000794402282, + "pass@8": 0.9844839762371538, + "pass@16": 0.988252756425543, + "pass@32": 0.9920258959912787, + "pass@64": 0.9952680507296997, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9480105105105106, + "pass@2": 0.9651595493130931, + "pass@4": 0.9750546725015227, + "pass@8": 0.9822405401725065, + "pass@16": 0.9875073932275126, + "pass@32": 0.9912748387820756, + "pass@64": 0.9934950075146307, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.934887459807074, + "pass@2": 0.9595855381421378, + "pass@4": 0.9720115560313806, + "pass@8": 0.9803090736757587, + "pass@16": 0.9863824259753894, + "pass@32": 0.9910160432545946, + "pass@64": 0.9944976406378414, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.8351438492063492, + "pass@2": 0.888676806024247, + "pass@4": 0.9195518057266644, + "pass@8": 0.9383458921358946, + "pass@16": 0.9516784351429641, + "pass@32": 0.9627085924506177, + "pass@64": 0.9702310379963924, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.6418195718654435, + "pass@2": 0.7379478045462207, + "pass@4": 0.808911574699951, + "pass@8": 0.8574357117385621, + "pass@16": 0.8891356835786846, + "pass@32": 0.9119187657061124, + "pass@64": 0.9319113224555994, + "pass@128": 0.9480122324159022 + }, + "13": { + "pass@1": 0.43190586419753085, + "pass@2": 0.5315685908185085, + "pass@4": 0.6109124663815172, + "pass@8": 0.6731000469060026, + "pass@16": 0.7230315632688672, + "pass@32": 0.7608469845357523, + "pass@64": 0.7895432659192605, + "pass@128": 0.8148148148148148 + }, + "14": { + "pass@1": 0.3707807863501484, + "pass@2": 0.4622490449543215, + "pass@4": 0.5422512678311351, + "pass@8": 0.6090936582126327, + "pass@16": 0.665375170025168, + "pass@32": 0.7157854633923258, + "pass@64": 0.7613425230053639, + "pass@128": 0.8011869436201781 + }, + "15": { + "pass@1": 0.29398148148148145, + "pass@2": 0.385362745455429, + "pass@4": 0.46875469646155304, + "pass@8": 0.5418755427677018, + "pass@16": 0.6032998652447866, + "pass@32": 0.6504141207197476, + "pass@64": 0.6830392953220654, + "pass@128": 0.7037037037037037 + }, + "16": { + "pass@1": 0.20464601769911506, + "pass@2": 0.27441648955938025, + "pass@4": 0.3406489202677099, + "pass@8": 0.3980154635469927, + "pass@16": 0.44783345192625446, + "pass@32": 0.4942909375581759, + "pass@64": 0.5381229534278857, + "pass@128": 0.5752212389380531 + }, + "17": { + "pass@1": 0.17634600903614459, + "pass@2": 0.24603556944312685, + "pass@4": 0.3129089391875711, + "pass@8": 0.37151393806373156, + "pass@16": 0.42139763379994655, + "pass@32": 0.4651711352279267, + "pass@64": 0.5050089880673075, + "pass@128": 0.5451807228915663 + }, + "18": { + "pass@1": 0.17515120967741934, + "pass@2": 0.2418537274574549, + "pass@4": 0.30864086051743567, + "pass@8": 0.36897845390139183, + "pass@16": 0.41813086287003537, + "pass@32": 0.45496716623619304, + "pass@64": 0.48441455201866307, + "pass@128": 0.5161290322580645 + }, + "19": { + "pass@1": 0.1568400679758308, + "pass@2": 0.22048024656849932, + "pass@4": 0.28429913613517355, + "pass@8": 0.34339241837815315, + "pass@16": 0.3939768884772223, + "pass@32": 0.4328402195492693, + "pass@64": 0.4613644571598135, + "pass@128": 0.48338368580060426 + }, + "20": { + "pass@1": 0.15636730480480482, + "pass@2": 0.22079904707857453, + "pass@4": 0.2881610339248135, + "pass@8": 0.3512828123002748, + "pass@16": 0.40352597323708034, + "pass@32": 0.4467676743469794, + "pass@64": 0.4848656649074429, + "pass@128": 0.5195195195195195 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/checkpoint-21808_metrics.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/checkpoint-21808_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e659f8a28bd7a84ea8927ad6f7ddd6881aa80ce8 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/checkpoint-21808_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21808", + "id": { + "answer_accuracy": 0.9757300347222222, + "avg_response_len": 196.41278559027776, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9757300347222222, + "pass@2": 0.9824465086395398, + "pass@4": 0.9861182112235961, + "pass@8": 0.9885871035750466, + "pass@16": 0.9904074478931092, + "pass@32": 0.9918600774152049, + "pass@64": 0.9930006028480788, + "pass@128": 0.9938888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.903828125, + "pass@2": 0.9262903543307085, + "pass@4": 0.9409031223284587, + "pass@8": 0.9512946215385871, + "pass@16": 0.9590260046593875, + "pass@32": 0.9651557406992112, + "pass@64": 0.9703015072286535, + "pass@128": 0.975 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999988927165354, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994296875, + "pass@2": 0.9998980068897638, + "pass@4": 0.999994822272216, + "pass@8": 0.999999986957765, + "pass@16": 0.9999999999999764, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997421875, + "pass@2": 0.9999858513779528, + "pass@4": 0.9999999046681665, + "pass@8": 0.9999999999978996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.97234375, + "pass@2": 0.9762458169291336, + "pass@4": 0.9790829797525312, + "pass@8": 0.9813830160790908, + "pass@16": 0.9836014491610245, + "pass@32": 0.985740926280185, + "pass@64": 0.9874084610226646, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.9877890625, + "pass@2": 0.9922095226377955, + "pass@4": 0.9941044799400074, + "pass@8": 0.9949746400481112, + "pass@16": 0.9953875147349939, + "pass@32": 0.9956881719643986, + "pass@64": 0.9959404409407284, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.9828125, + "pass@2": 0.9880092273622049, + "pass@4": 0.9908369163854517, + "pass@8": 0.9931549028622938, + "pass@16": 0.9951904589008074, + "pass@32": 0.9969358460011541, + "pass@64": 0.9982484981706026, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9750859375, + "pass@2": 0.985098794291339, + "pass@4": 0.9898506652605922, + "pass@8": 0.9924436661873157, + "pass@16": 0.9939045467734713, + "pass@32": 0.9947923299637188, + "pass@64": 0.9954338069591998, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.9606015625, + "pass@2": 0.9742821112204733, + "pass@4": 0.9802910104986872, + "pass@8": 0.9840330985043604, + "pass@16": 0.9865570568083187, + "pass@32": 0.9884276818281734, + "pass@64": 0.9896727113108568, + "pass@128": 0.99 + } + }, + "per_op_accuracy": { + "10": 0.903828125, + "2": 0.9999375, + "3": 0.9994296875, + "4": 0.9997421875, + "5": 0.97234375, + "6": 0.9877890625, + "7": 0.9828125, + "8": 0.9750859375, + "9": 0.9606015625 + }, + "per_op_avg_response_len": { + "10": 274.1716484375, + "2": 114.47234375, + "3": 142.6243125, + "4": 157.5850625, + "5": 170.9491015625, + "6": 195.5380390625, + "7": 219.89640625, + "8": 238.573046875, + "9": 253.905109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.1716484375, + "2": 114.47234375, + "3": 142.6243125, + "4": 157.5850625, + "5": 170.9491015625, + "6": 195.5380390625, + "7": 219.89640625, + "8": 238.573046875, + "9": 253.905109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 368761, + "answer_accuracy": 0.9729636313745357, + "avg_response_len": 186.87297893448158, + "resp_tokens_sum": 70826354, + "pass_at_k": { + "pass@1": 0.9729636313745357, + "pass@2": 0.9805738627751331, + "pass@4": 0.9844540156581243, + "pass@8": 0.9871831067807101, + "pass@16": 0.9893726420610013, + "pass@32": 0.9912014624590227, + "pass@64": 0.9925453864607254, + "pass@128": 0.9935832489023978 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9079784292035398, + "pass@2": 0.9283034864004833, + "pass@4": 0.9397207392770592, + "pass@8": 0.9474065863832337, + "pass@16": 0.953277630712324, + "pass@32": 0.9587798739641823, + "pass@64": 0.9641137000779781, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9999026479750779, + "pass@2": 0.9999977003458681, + "pass@4": 0.9999999997079805, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992115825688074, + "pass@2": 0.9999002955765851, + "pass@4": 0.9999974601798628, + "pass@8": 0.9999999989512287, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999365234375, + "pass@2": 0.9999569389763779, + "pass@4": 0.9999997020880201, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9670138888888888, + "pass@2": 0.9722800274965628, + "pass@4": 0.9764746609054821, + "pass@8": 0.9805577074470837, + "pass@16": 0.9849640666818144, + "pass@32": 0.9887401621413379, + "pass@64": 0.9903685676767175, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9776618637724551, + "pass@2": 0.9838419863029846, + "pass@4": 0.9863429900603738, + "pass@8": 0.9880363231052547, + "pass@16": 0.9891851085789134, + "pass@32": 0.9900843472432467, + "pass@64": 0.9908396435351152, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9761002218934911, + "pass@2": 0.9821877329590457, + "pass@4": 0.9854717781209303, + "pass@8": 0.9882141392067795, + "pass@16": 0.9909368540370078, + "pass@32": 0.9934787972166277, + "pass@64": 0.995374785040164, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9728565705128205, + "pass@2": 0.9847504037956794, + "pass@4": 0.9898392754992162, + "pass@8": 0.9928595606012538, + "pass@16": 0.9949857414228591, + "pass@32": 0.9963289537684682, + "pass@64": 0.9967736090563776, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9606514084507042, + "pass@2": 0.9767279860263948, + "pass@4": 0.9843846306183557, + "pass@8": 0.9893238732854907, + "pass@16": 0.9925283410495083, + "pass@32": 0.9947305999396088, + "pass@64": 0.9964951038189134, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382007, + "answer_accuracy": 0.9778603170052425, + "avg_response_len": 194.37474402031455, + "resp_tokens_sum": 75933660, + "pass_at_k": { + "pass@1": 0.9778603170052425, + "pass@2": 0.984059399219307, + "pass@4": 0.9874203478599426, + "pass@8": 0.9896460246248755, + "pass@16": 0.9912454815153983, + "pass@32": 0.9924621077985335, + "pass@64": 0.9934031023583285, + "pass@128": 0.9941022280471822 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9004026073619632, + "pass@2": 0.9242703402009566, + "pass@4": 0.9396745401073328, + "pass@8": 0.9505904177376397, + "pass@16": 0.9586655869893964, + "pass@32": 0.96514230835694, + "pass@64": 0.9707695920724272, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9834692028985508, + "pass@2": 0.9852240956293506, + "pass@4": 0.9858998532792097, + "pass@8": 0.986317596894847, + "pass@16": 0.9869360286784715, + "pass@32": 0.9877361308097358, + "pass@64": 0.988322264064818, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.9928142908309455, + "pass@2": 0.9974357275003949, + "pass@4": 0.999494777783866, + "pass@8": 0.9999737827585785, + "pass@16": 0.9999999348964089, + "pass@32": 0.9999999999999439, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9855491329479769, + "pass@2": 0.9887166605980608, + "pass@4": 0.9913939513883019, + "pass@8": 0.9943285343504065, + "pass@16": 0.9968946491921471, + "pass@32": 0.9987241613729214, + "pass@64": 0.999808867257577, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9807658450704225, + "pass@2": 0.9904752827991573, + "pass@4": 0.9946537202215919, + "pass@8": 0.9965723900602302, + "pass@16": 0.9974438326135088, + "pass@32": 0.9978865571381552, + "pass@64": 0.9985915492939871, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9550196482035929, + "pass@2": 0.9680099692347592, + "pass@4": 0.973621248223463, + "pass@8": 0.9772931092950905, + "pass@16": 0.9797760833989643, + "pass@32": 0.9813793644561303, + "pass@64": 0.9820069826621478, + "pass@128": 0.9820359281437125 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 373273, + "answer_accuracy": 0.9762957189487781, + "avg_response_len": 207.95194540927352, + "resp_tokens_sum": 79507515, + "pass_at_k": { + "pass@1": 0.9762957189487781, + "pass@2": 0.9826548656382389, + "pass@4": 0.9864374486039932, + "pass@8": 0.988896915246943, + "pass@16": 0.9905769763342356, + "pass@32": 0.9918978283878637, + "pass@64": 0.9930405985687577, + "pass@128": 0.9939738868429863 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9029617537313432, + "pass@2": 0.9262189299565168, + "pass@4": 0.9432951989210303, + "pass@8": 0.9559143658871644, + "pass@16": 0.9651937507145258, + "pass@32": 0.9716208086596739, + "pass@64": 0.9761076892137535, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 0.9999107142857143, + "pass@2": 0.9999989454443194, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991116982507289, + "pass@2": 0.9997976974816924, + "pass@4": 0.9999873259282823, + "pass@8": 0.9999999629758403, + "pass@16": 0.9999999999999313, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999314692982456, + "pass@2": 0.9999996402587834, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9659926470588235, + "pass@2": 0.9708096919870312, + "pass@4": 0.9745823887822845, + "pass@8": 0.9771404920721685, + "pass@16": 0.9789554066534717, + "pass@32": 0.9809376766950135, + "pass@64": 0.9837387679474603, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9929268533123028, + "pass@2": 0.9952720199210117, + "pass@4": 0.9963477722816194, + "pass@8": 0.996781255357137, + "pass@16": 0.996844514797445, + "pass@32": 0.9968454258207375, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9869956487341772, + "pass@2": 0.9934614210605002, + "pass@4": 0.9959656588614716, + "pass@8": 0.9971545915353225, + "pass@16": 0.9978742203032149, + "pass@32": 0.9986754832496939, + "pass@64": 0.9996137745440047, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9711195570570571, + "pass@2": 0.9796935222624201, + "pass@4": 0.9847409628413564, + "pass@8": 0.9876525069319597, + "pass@16": 0.9891184230382688, + "pass@32": 0.9900539597715055, + "pass@64": 0.9908121349376716, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.9665393890675241, + "pass@2": 0.9782261943945113, + "pass@4": 0.9827812531271213, + "pass@8": 0.9852322346734745, + "pass@16": 0.9870234208375871, + "pass@32": 0.9888026081069605, + "pass@64": 0.9901177403407886, + "pass@128": 0.9903536977491961 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.35559765625, + "avg_response_len": 316.2850265625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.35559765625, + "pass@2": 0.41827846948819586, + "pass@4": 0.473175446278587, + "pass@8": 0.5201447088604175, + "pass@16": 0.5599174884816872, + "pass@32": 0.5933208410478391, + "pass@64": 0.6217572103229588, + "pass@128": 0.6473 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8518046875, + "pass@2": 0.8897643946850398, + "pass@4": 0.9142276405136861, + "pass@8": 0.9303760085015262, + "pass@16": 0.9415357982288092, + "pass@32": 0.9508669679427215, + "pass@64": 0.9588891537042802, + "pass@128": 0.965 + }, + "12": { + "pass@1": 0.70853125, + "pass@2": 0.77690157480315, + "pass@4": 0.823685896794151, + "pass@8": 0.8563853361924025, + "pass@16": 0.8790161878347383, + "pass@32": 0.8947983606207323, + "pass@64": 0.906980106818619, + "pass@128": 0.918 + }, + "13": { + "pass@1": 0.4635703125, + "pass@2": 0.5395426919291342, + "pass@4": 0.6009809487251592, + "pass@8": 0.6520847322363588, + "pass@16": 0.6958659697890068, + "pass@32": 0.7342348311115826, + "pass@64": 0.7677598125640247, + "pass@128": 0.796 + }, + "14": { + "pass@1": 0.388875, + "pass@2": 0.46016178641732297, + "pass@4": 0.5195261549493811, + "pass@8": 0.5691724041079831, + "pass@16": 0.6111346606619696, + "pass@32": 0.6480092126802668, + "pass@64": 0.6817971191592553, + "pass@128": 0.712 + }, + "15": { + "pass@1": 0.2777734375, + "pass@2": 0.35240452755905527, + "pass@4": 0.420563871203599, + "pass@8": 0.48226965965807667, + "pass@16": 0.538075855062058, + "pass@32": 0.5864038424932985, + "pass@64": 0.6280894095568261, + "pass@128": 0.665 + }, + "16": { + "pass@1": 0.200515625, + "pass@2": 0.25968134842519697, + "pass@4": 0.315604779246344, + "pass@8": 0.3651431944197165, + "pass@16": 0.40810283843387624, + "pass@32": 0.44496367046196783, + "pass@64": 0.478787626173846, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.190359375, + "pass@2": 0.25173658956692924, + "pass@4": 0.30874303271466064, + "pass@8": 0.35933756871553263, + "pass@16": 0.4036958904748346, + "pass@32": 0.44140474848634753, + "pass@64": 0.4733639588964715, + "pass@128": 0.504 + }, + "18": { + "pass@1": 0.17846875, + "pass@2": 0.2374504183070865, + "pass@4": 0.2935350335583049, + "pass@8": 0.34377836300060277, + "pass@16": 0.38784117204252194, + "pass@32": 0.4249526745982745, + "pass@64": 0.4549007227697489, + "pass@128": 0.482 + }, + "19": { + "pass@1": 0.1475078125, + "pass@2": 0.2060077509842518, + "pass@4": 0.2648804682227218, + "pass@8": 0.318611381141697, + "pass@16": 0.3648611668445132, + "pass@32": 0.4028502836845614, + "pass@64": 0.43388069395036394, + "pass@128": 0.46 + }, + "20": { + "pass@1": 0.1485703125, + "pass@2": 0.2091336122047243, + "pass@4": 0.2700066368578927, + "pass@8": 0.3242884406302299, + "pass@16": 0.36904534544444567, + "pass@32": 0.40472381839862426, + "pass@64": 0.4331234996361173, + "pass@128": 0.456 + } + }, + "per_op_accuracy": { + "11": 0.8518046875, + "12": 0.70853125, + "13": 0.4635703125, + "14": 0.388875, + "15": 0.2777734375, + "16": 0.200515625, + "17": 0.190359375, + "18": 0.17846875, + "19": 0.1475078125, + "20": 0.1485703125 + }, + "per_op_avg_response_len": { + "11": 290.0039921875, + "12": 297.4168359375, + "13": 307.637796875, + "14": 321.095875, + "15": 323.3938515625, + "16": 322.91203125, + "17": 327.2185234375, + "18": 325.4930859375, + "19": 322.7733671875, + "20": 324.90490625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 290.0039921875, + "12": 297.4168359375, + "13": 307.637796875, + "14": 321.095875, + "15": 323.3938515625, + "16": 322.91203125, + "17": 327.2185234375, + "18": 325.4930859375, + "19": 322.7733671875, + "20": 324.90490625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 154644, + "answer_accuracy": 0.3590360326894502, + "avg_response_len": 299.74041140416045, + "resp_tokens_sum": 129104190, + "pass_at_k": { + "pass@1": 0.3590360326894502, + "pass@2": 0.4223984962735904, + "pass@4": 0.477865819818587, + "pass@8": 0.5246184466867111, + "pass@16": 0.5636257348351528, + "pass@32": 0.5955384152855437, + "pass@64": 0.6213828000180788, + "pass@128": 0.6442793462109956 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8500354107648725, + "pass@2": 0.8847818752648838, + "pass@4": 0.9057016992918374, + "pass@8": 0.9209417546235552, + "pass@16": 0.9350945491946546, + "pass@32": 0.9489809420090249, + "pass@64": 0.9604307908065416, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.7215969279661016, + "pass@2": 0.7881981516081679, + "pass@4": 0.8320213376399981, + "pass@8": 0.8641739560917514, + "pass@16": 0.8889098921541904, + "pass@32": 0.9070642083248085, + "pass@64": 0.9203748363482039, + "pass@128": 0.9322033898305084 + }, + "13": { + "pass@1": 0.47176234326018807, + "pass@2": 0.5514414354651594, + "pass@4": 0.61641166121868, + "pass@8": 0.6679790136854478, + "pass@16": 0.7104903504508953, + "pass@32": 0.7483723201148265, + "pass@64": 0.782034213606473, + "pass@128": 0.8119122257053292 + }, + "14": { + "pass@1": 0.3773364485981308, + "pass@2": 0.44722178784310845, + "pass@4": 0.5053335929387331, + "pass@8": 0.5553041100778835, + "pass@16": 0.5983519667569652, + "pass@32": 0.6341758147700675, + "pass@64": 0.6635061144623973, + "pass@128": 0.6853582554517134 + }, + "15": { + "pass@1": 0.2905979046242775, + "pass@2": 0.3691705828137089, + "pass@4": 0.44011951603954086, + "pass@8": 0.49954595991520595, + "pass@16": 0.5489780070558901, + "pass@32": 0.589607486805191, + "pass@64": 0.622846434960636, + "pass@128": 0.6502890173410405 + }, + "16": { + "pass@1": 0.18639112903225807, + "pass@2": 0.24502595567691135, + "pass@4": 0.3011878971781754, + "pass@8": 0.3503457986058463, + "pass@16": 0.3939021510117161, + "pass@32": 0.43316290463944823, + "pass@64": 0.46712538073865945, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.18806448699421965, + "pass@2": 0.24722041270310846, + "pass@4": 0.30208944176414365, + "pass@8": 0.3499756198131455, + "pass@16": 0.39130847250177325, + "pass@32": 0.4240642793091486, + "pass@64": 0.44695993145996443, + "pass@128": 0.4653179190751445 + }, + "18": { + "pass@1": 0.1703794642857143, + "pass@2": 0.23251019403824516, + "pass@4": 0.2904351298944774, + "pass@8": 0.3390466063051701, + "pass@16": 0.3793374554178496, + "pass@32": 0.4127509001659112, + "pass@64": 0.4405737754821389, + "pass@128": 0.4685714285714286 + }, + "19": { + "pass@1": 0.14318224474474475, + "pass@2": 0.20463980220377853, + "pass@4": 0.2685763572909241, + "pass@8": 0.3284336355160867, + "pass@16": 0.3783702554316468, + "pass@32": 0.4147896381558507, + "pass@64": 0.4437101594568495, + "pass@128": 0.4744744744744745 + }, + "20": { + "pass@1": 0.15111204954954954, + "pass@2": 0.21561509836608267, + "pass@4": 0.281267438698541, + "pass@8": 0.3375878425646831, + "pass@16": 0.381100671812623, + "pass@32": 0.4144019892616114, + "pass@64": 0.4406480739566761, + "pass@128": 0.4624624624624625 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 149853, + "answer_accuracy": 0.3481197033898305, + "avg_response_len": 312.43839438373476, + "resp_tokens_sum": 134493481, + "pass_at_k": { + "pass@1": 0.3481197033898305, + "pass@2": 0.4099884673063296, + "pass@4": 0.4648495347537416, + "pass@8": 0.5129282653117243, + "pass@16": 0.5540938542061926, + "pass@32": 0.5888153511830716, + "pass@64": 0.6186960454288796, + "pass@128": 0.6461492714837942 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8364551957831325, + "pass@2": 0.8805564284697844, + "pass@4": 0.9119869102393826, + "pass@8": 0.9332457070554272, + "pass@16": 0.9464105210220005, + "pass@32": 0.9554480120240839, + "pass@64": 0.9620750241307124, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.7179907915360502, + "pass@2": 0.7807096641818678, + "pass@4": 0.8237406890557176, + "pass@8": 0.8544050832000054, + "pass@16": 0.8745406088754614, + "pass@32": 0.8870839775741599, + "pass@64": 0.8953727964648683, + "pass@128": 0.8996865203761756 + }, + "13": { + "pass@1": 0.4339110644257703, + "pass@2": 0.5064138352191269, + "pass@4": 0.5668354349613859, + "pass@8": 0.6208126022401135, + "pass@16": 0.6694676564972795, + "pass@32": 0.7126055948949221, + "pass@64": 0.7509963818526764, + "pass@128": 0.7843137254901961 + }, + "14": { + "pass@1": 0.39722679093567254, + "pass@2": 0.46368376444720705, + "pass@4": 0.5167455819667276, + "pass@8": 0.56171751122582, + "pass@16": 0.6024369898411556, + "pass@32": 0.6426240615167135, + "pass@64": 0.6832214593635492, + "pass@128": 0.7222222222222222 + }, + "15": { + "pass@1": 0.24644886363636365, + "pass@2": 0.32176874552612744, + "pass@4": 0.3923603217211486, + "pass@8": 0.4581419440373477, + "pass@16": 0.5190329686132177, + "pass@32": 0.5727951896253346, + "pass@64": 0.6198279692563144, + "pass@128": 0.6636363636363637 + }, + "16": { + "pass@1": 0.2040820868945869, + "pass@2": 0.26635372501514243, + "pass@4": 0.3238963121062858, + "pass@8": 0.37332193265107516, + "pass@16": 0.4134811645828353, + "pass@32": 0.4443577173332517, + "pass@64": 0.47302414732746867, + "pass@128": 0.50997150997151 + }, + "17": { + "pass@1": 0.19650135869565216, + "pass@2": 0.2574690052330416, + "pass@4": 0.3152138539941824, + "pass@8": 0.3709270117297045, + "pass@16": 0.42297098948583206, + "pass@32": 0.4692510903823389, + "pass@64": 0.5108174522501921, + "pass@128": 0.5527950310559007 + }, + "18": { + "pass@1": 0.18683363970588235, + "pass@2": 0.24651024779990735, + "pass@4": 0.3035248188645537, + "pass@8": 0.3556424980931623, + "pass@16": 0.4017419832141523, + "pass@32": 0.43944869520791696, + "pass@64": 0.4673817173155303, + "pass@128": 0.49117647058823527 + }, + "19": { + "pass@1": 0.14808872767857142, + "pass@2": 0.2049320250984251, + "pass@4": 0.2627956976918065, + "pass@8": 0.3139157657154089, + "pass@16": 0.3560561707464758, + "pass@32": 0.39169649693737485, + "pass@64": 0.4226012391904779, + "pass@128": 0.44642857142857145 + }, + "20": { + "pass@1": 0.13087107035928144, + "pass@2": 0.1879670776557122, + "pass@4": 0.24696301351178396, + "pass@8": 0.30177241862067616, + "pass@16": 0.34838594581032317, + "pass@32": 0.38534870948871147, + "pass@64": 0.4129125683339026, + "pass@128": 0.4341317365269461 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 150668, + "answer_accuracy": 0.359747478606357, + "avg_response_len": 337.2535027315098, + "resp_tokens_sum": 141247163, + "pass_at_k": { + "pass@1": 0.359747478606357, + "pass@2": 0.42256190078547734, + "pass@4": 0.47690922790939505, + "pass@8": 0.5229609594315465, + "pass@16": 0.5620894423597352, + "pass@32": 0.5956710320946896, + "pass@64": 0.6252885636893026, + "pass@128": 0.6515892420537898 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8699652777777778, + "pass@2": 0.9050528840144983, + "pass@4": 0.9261437665529902, + "pass@8": 0.9379237916730481, + "pass@16": 0.9436162837580702, + "pass@32": 0.9481522394334617, + "pass@64": 0.9538037350418224, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.6851586391437309, + "pass@2": 0.7609573280117508, + "pass@4": 0.8146087567609088, + "pass@8": 0.8498854256731503, + "pass@16": 0.8726716568222734, + "pass@32": 0.8890453884635856, + "pass@64": 0.9038027299053867, + "pass@128": 0.9204892966360856 + }, + "13": { + "pass@1": 0.48818479938271603, + "pass@2": 0.5643306630941965, + "pass@4": 0.6234116620839059, + "pass@8": 0.670892987256112, + "pass@16": 0.710554304400132, + "pass@32": 0.7441477272144019, + "pass@64": 0.7721765126609678, + "pass@128": 0.7932098765432098 + }, + "14": { + "pass@1": 0.39139002225519287, + "pass@2": 0.4689131901446295, + "pass@4": 0.5358667138974095, + "pass@8": 0.5899477624146936, + "pass@16": 0.6321371774697618, + "pass@32": 0.6666508816034391, + "pass@64": 0.697774235354575, + "pass@128": 0.7270029673590505 + }, + "15": { + "pass@1": 0.29598283179012347, + "pass@2": 0.3657031477836101, + "pass@4": 0.4284062113763555, + "pass@8": 0.488394802453985, + "pass@16": 0.5458289659825917, + "pass@32": 0.5968433317356244, + "pass@64": 0.6421028185366104, + "pass@128": 0.6820987654320988 + }, + "16": { + "pass@1": 0.20973912241887904, + "pass@2": 0.2661744680974612, + "pass@4": 0.32020332027080706, + "pass@8": 0.37020648522530164, + "pass@16": 0.4155200084120026, + "pass@32": 0.4563823340406111, + "pass@64": 0.4954197121915054, + "pass@128": 0.5339233038348082 + }, + "17": { + "pass@1": 0.1867940512048193, + "pass@2": 0.2508834550801631, + "pass@4": 0.3094012797534344, + "pass@8": 0.3578539351904199, + "pass@16": 0.3979111457071782, + "pass@32": 0.43246884560924453, + "pass@64": 0.46455603309265586, + "pass@128": 0.49698795180722893 + }, + "18": { + "pass@1": 0.17842741935483872, + "pass@2": 0.2330915036830072, + "pass@4": 0.286078386068677, + "pass@8": 0.3361083917487688, + "pass@16": 0.3821960914627827, + "pass@32": 0.422830010224237, + "pass@64": 0.45738747568942023, + "pass@128": 0.4870967741935484 + }, + "19": { + "pass@1": 0.1512698262839879, + "pass@2": 0.20847594083783336, + "pass@4": 0.2632785100314122, + "pass@8": 0.31349632397118765, + "pass@16": 0.3602084544258579, + "pass@32": 0.40216106105046995, + "pass@64": 0.4354416751759285, + "pass@128": 0.459214501510574 + }, + "20": { + "pass@1": 0.16378096846846846, + "pass@2": 0.22388222375919223, + "pass@4": 0.2818586584334616, + "pass@8": 0.3335726763870422, + "pass@16": 0.37771145894953373, + "pass@32": 0.41447893995579, + "pass@64": 0.44587055010516763, + "pass@128": 0.47147147147147145 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.649344572368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.649344572368421, + "pass@2": 0.6855159617177615, + "pass@4": 0.7161483349367347, + "pass@8": 0.7420384747778785, + "pass@16": 0.7638337850449982, + "pass@32": 0.7821025845902874, + "pass@64": 0.7976093436243314, + "pass@128": 0.8114736842105263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.903828125, + "pass@2": 0.9262903543307085, + "pass@4": 0.9409031223284587, + "pass@8": 0.9512946215385871, + "pass@16": 0.9590260046593875, + "pass@32": 0.9651557406992112, + "pass@64": 0.9703015072286535, + "pass@128": 0.975 + }, + "2": { + "pass@1": 0.9999375, + "pass@2": 0.9999988927165354, + "pass@4": 0.9999999999062616, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9994296875, + "pass@2": 0.9998980068897638, + "pass@4": 0.999994822272216, + "pass@8": 0.999999986957765, + "pass@16": 0.9999999999999764, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9997421875, + "pass@2": 0.9999858513779528, + "pass@4": 0.9999999046681665, + "pass@8": 0.9999999999978996, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.97234375, + "pass@2": 0.9762458169291336, + "pass@4": 0.9790829797525312, + "pass@8": 0.9813830160790908, + "pass@16": 0.9836014491610245, + "pass@32": 0.985740926280185, + "pass@64": 0.9874084610226646, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.9877890625, + "pass@2": 0.9922095226377955, + "pass@4": 0.9941044799400074, + "pass@8": 0.9949746400481112, + "pass@16": 0.9953875147349939, + "pass@32": 0.9956881719643986, + "pass@64": 0.9959404409407284, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.9828125, + "pass@2": 0.9880092273622049, + "pass@4": 0.9908369163854517, + "pass@8": 0.9931549028622938, + "pass@16": 0.9951904589008074, + "pass@32": 0.9969358460011541, + "pass@64": 0.9982484981706026, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.9750859375, + "pass@2": 0.985098794291339, + "pass@4": 0.9898506652605922, + "pass@8": 0.9924436661873157, + "pass@16": 0.9939045467734713, + "pass@32": 0.9947923299637188, + "pass@64": 0.9954338069591998, + "pass@128": 0.996 + }, + "9": { + "pass@1": 0.9606015625, + "pass@2": 0.9742821112204733, + "pass@4": 0.9802910104986872, + "pass@8": 0.9840330985043604, + "pass@16": 0.9865570568083187, + "pass@32": 0.9884276818281734, + "pass@64": 0.9896727113108568, + "pass@128": 0.99 + }, + "11": { + "pass@1": 0.8518046875, + "pass@2": 0.8897643946850398, + "pass@4": 0.9142276405136861, + "pass@8": 0.9303760085015262, + "pass@16": 0.9415357982288092, + "pass@32": 0.9508669679427215, + "pass@64": 0.9588891537042802, + "pass@128": 0.965 + }, + "12": { + "pass@1": 0.70853125, + "pass@2": 0.77690157480315, + "pass@4": 0.823685896794151, + "pass@8": 0.8563853361924025, + "pass@16": 0.8790161878347383, + "pass@32": 0.8947983606207323, + "pass@64": 0.906980106818619, + "pass@128": 0.918 + }, + "13": { + "pass@1": 0.4635703125, + "pass@2": 0.5395426919291342, + "pass@4": 0.6009809487251592, + "pass@8": 0.6520847322363588, + "pass@16": 0.6958659697890068, + "pass@32": 0.7342348311115826, + "pass@64": 0.7677598125640247, + "pass@128": 0.796 + }, + "14": { + "pass@1": 0.388875, + "pass@2": 0.46016178641732297, + "pass@4": 0.5195261549493811, + "pass@8": 0.5691724041079831, + "pass@16": 0.6111346606619696, + "pass@32": 0.6480092126802668, + "pass@64": 0.6817971191592553, + "pass@128": 0.712 + }, + "15": { + "pass@1": 0.2777734375, + "pass@2": 0.35240452755905527, + "pass@4": 0.420563871203599, + "pass@8": 0.48226965965807667, + "pass@16": 0.538075855062058, + "pass@32": 0.5864038424932985, + "pass@64": 0.6280894095568261, + "pass@128": 0.665 + }, + "16": { + "pass@1": 0.200515625, + "pass@2": 0.25968134842519697, + "pass@4": 0.315604779246344, + "pass@8": 0.3651431944197165, + "pass@16": 0.40810283843387624, + "pass@32": 0.44496367046196783, + "pass@64": 0.478787626173846, + "pass@128": 0.515 + }, + "17": { + "pass@1": 0.190359375, + "pass@2": 0.25173658956692924, + "pass@4": 0.30874303271466064, + "pass@8": 0.35933756871553263, + "pass@16": 0.4036958904748346, + "pass@32": 0.44140474848634753, + "pass@64": 0.4733639588964715, + "pass@128": 0.504 + }, + "18": { + "pass@1": 0.17846875, + "pass@2": 0.2374504183070865, + "pass@4": 0.2935350335583049, + "pass@8": 0.34377836300060277, + "pass@16": 0.38784117204252194, + "pass@32": 0.4249526745982745, + "pass@64": 0.4549007227697489, + "pass@128": 0.482 + }, + "19": { + "pass@1": 0.1475078125, + "pass@2": 0.2060077509842518, + "pass@4": 0.2648804682227218, + "pass@8": 0.318611381141697, + "pass@16": 0.3648611668445132, + "pass@32": 0.4028502836845614, + "pass@64": 0.43388069395036394, + "pass@128": 0.46 + }, + "20": { + "pass@1": 0.1485703125, + "pass@2": 0.2091336122047243, + "pass@4": 0.2700066368578927, + "pass@8": 0.3242884406302299, + "pass@16": 0.36904534544444567, + "pass@32": 0.40472381839862426, + "pass@64": 0.4331234996361173, + "pass@128": 0.456 + } + }, + "per_op_accuracy": { + "10": 0.903828125, + "2": 0.9999375, + "3": 0.9994296875, + "4": 0.9997421875, + "5": 0.97234375, + "6": 0.9877890625, + "7": 0.9828125, + "8": 0.9750859375, + "9": 0.9606015625, + "11": 0.8518046875, + "12": 0.70853125, + "13": 0.4635703125, + "14": 0.388875, + "15": 0.2777734375, + "16": 0.200515625, + "17": 0.190359375, + "18": 0.17846875, + "19": 0.1475078125, + "20": 0.1485703125 + }, + "per_op_avg_response_len": { + "10": 274.1716484375, + "2": 114.47234375, + "3": 142.6243125, + "4": 157.5850625, + "5": 170.9491015625, + "6": 195.5380390625, + "7": 219.89640625, + "8": 238.573046875, + "9": 253.905109375, + "11": 290.0039921875, + "12": 297.4168359375, + "13": 307.637796875, + "14": 321.095875, + "15": 323.3938515625, + "16": 322.91203125, + "17": 327.2185234375, + "18": 325.4930859375, + "19": 322.7733671875, + "20": 324.90490625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.1716484375, + "2": 114.47234375, + "3": 142.6243125, + "4": 157.5850625, + "5": 170.9491015625, + "6": 195.5380390625, + "7": 219.89640625, + "8": 238.573046875, + "9": 253.905109375, + "11": 290.0039921875, + "12": 297.4168359375, + "13": 307.637796875, + "14": 321.095875, + "15": 323.3938515625, + "16": 322.91203125, + "17": 327.2185234375, + "18": 325.4930859375, + "19": 322.7733671875, + "20": 324.90490625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 523405, + "answer_accuracy": 0.6463960737432817, + "avg_response_len": 246.91074533670565, + "resp_tokens_sum": 199930544, + "pass_at_k": { + "pass@1": 0.6463960737432817, + "pass@2": 0.6836626853679749, + "pass@4": 0.7149836901759768, + "pass@8": 0.7411302959656122, + "pass@16": 0.762904361502207, + "pass@32": 0.7807357410333592, + "pass@64": 0.7951122370172407, + "pass@128": 0.807777426493835 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9079784292035398, + "pass@2": 0.9283034864004833, + "pass@4": 0.9397207392770592, + "pass@8": 0.9474065863832337, + "pass@16": 0.953277630712324, + "pass@32": 0.9587798739641823, + "pass@64": 0.9641137000779781, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9999026479750779, + "pass@2": 0.9999977003458681, + "pass@4": 0.9999999997079805, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992115825688074, + "pass@2": 0.9999002955765851, + "pass@4": 0.9999974601798628, + "pass@8": 0.9999999989512287, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999365234375, + "pass@2": 0.9999569389763779, + "pass@4": 0.9999997020880201, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9670138888888888, + "pass@2": 0.9722800274965628, + "pass@4": 0.9764746609054821, + "pass@8": 0.9805577074470837, + "pass@16": 0.9849640666818144, + "pass@32": 0.9887401621413379, + "pass@64": 0.9903685676767175, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9776618637724551, + "pass@2": 0.9838419863029846, + "pass@4": 0.9863429900603738, + "pass@8": 0.9880363231052547, + "pass@16": 0.9891851085789134, + "pass@32": 0.9900843472432467, + "pass@64": 0.9908396435351152, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9761002218934911, + "pass@2": 0.9821877329590457, + "pass@4": 0.9854717781209303, + "pass@8": 0.9882141392067795, + "pass@16": 0.9909368540370078, + "pass@32": 0.9934787972166277, + "pass@64": 0.995374785040164, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9728565705128205, + "pass@2": 0.9847504037956794, + "pass@4": 0.9898392754992162, + "pass@8": 0.9928595606012538, + "pass@16": 0.9949857414228591, + "pass@32": 0.9963289537684682, + "pass@64": 0.9967736090563776, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9606514084507042, + "pass@2": 0.9767279860263948, + "pass@4": 0.9843846306183557, + "pass@8": 0.9893238732854907, + "pass@16": 0.9925283410495083, + "pass@32": 0.9947305999396088, + "pass@64": 0.9964951038189134, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8500354107648725, + "pass@2": 0.8847818752648838, + "pass@4": 0.9057016992918374, + "pass@8": 0.9209417546235552, + "pass@16": 0.9350945491946546, + "pass@32": 0.9489809420090249, + "pass@64": 0.9604307908065416, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.7215969279661016, + "pass@2": 0.7881981516081679, + "pass@4": 0.8320213376399981, + "pass@8": 0.8641739560917514, + "pass@16": 0.8889098921541904, + "pass@32": 0.9070642083248085, + "pass@64": 0.9203748363482039, + "pass@128": 0.9322033898305084 + }, + "13": { + "pass@1": 0.47176234326018807, + "pass@2": 0.5514414354651594, + "pass@4": 0.61641166121868, + "pass@8": 0.6679790136854478, + "pass@16": 0.7104903504508953, + "pass@32": 0.7483723201148265, + "pass@64": 0.782034213606473, + "pass@128": 0.8119122257053292 + }, + "14": { + "pass@1": 0.3773364485981308, + "pass@2": 0.44722178784310845, + "pass@4": 0.5053335929387331, + "pass@8": 0.5553041100778835, + "pass@16": 0.5983519667569652, + "pass@32": 0.6341758147700675, + "pass@64": 0.6635061144623973, + "pass@128": 0.6853582554517134 + }, + "15": { + "pass@1": 0.2905979046242775, + "pass@2": 0.3691705828137089, + "pass@4": 0.44011951603954086, + "pass@8": 0.49954595991520595, + "pass@16": 0.5489780070558901, + "pass@32": 0.589607486805191, + "pass@64": 0.622846434960636, + "pass@128": 0.6502890173410405 + }, + "16": { + "pass@1": 0.18639112903225807, + "pass@2": 0.24502595567691135, + "pass@4": 0.3011878971781754, + "pass@8": 0.3503457986058463, + "pass@16": 0.3939021510117161, + "pass@32": 0.43316290463944823, + "pass@64": 0.46712538073865945, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.18806448699421965, + "pass@2": 0.24722041270310846, + "pass@4": 0.30208944176414365, + "pass@8": 0.3499756198131455, + "pass@16": 0.39130847250177325, + "pass@32": 0.4240642793091486, + "pass@64": 0.44695993145996443, + "pass@128": 0.4653179190751445 + }, + "18": { + "pass@1": 0.1703794642857143, + "pass@2": 0.23251019403824516, + "pass@4": 0.2904351298944774, + "pass@8": 0.3390466063051701, + "pass@16": 0.3793374554178496, + "pass@32": 0.4127509001659112, + "pass@64": 0.4405737754821389, + "pass@128": 0.4685714285714286 + }, + "19": { + "pass@1": 0.14318224474474475, + "pass@2": 0.20463980220377853, + "pass@4": 0.2685763572909241, + "pass@8": 0.3284336355160867, + "pass@16": 0.3783702554316468, + "pass@32": 0.4147896381558507, + "pass@64": 0.4437101594568495, + "pass@128": 0.4744744744744745 + }, + "20": { + "pass@1": 0.15111204954954954, + "pass@2": 0.21561509836608267, + "pass@4": 0.281267438698541, + "pass@8": 0.3375878425646831, + "pass@16": 0.381100671812623, + "pass@32": 0.4144019892616114, + "pass@64": 0.4406480739566761, + "pass@128": 0.4624624624624625 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 531860, + "answer_accuracy": 0.647725058456742, + "avg_response_len": 256.26843944855807, + "resp_tokens_sum": 210427141, + "pass_at_k": { + "pass@1": 0.647725058456742, + "pass@2": 0.6831084180777097, + "pass@4": 0.7134677922128366, + "pass@8": 0.7397314767573606, + "pass@16": 0.7620730851567378, + "pass@32": 0.7808543069415151, + "pass@64": 0.7969666514692069, + "pass@128": 0.8116913484021824 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9004026073619632, + "pass@2": 0.9242703402009566, + "pass@4": 0.9396745401073328, + "pass@8": 0.9505904177376397, + "pass@16": 0.9586655869893964, + "pass@32": 0.96514230835694, + "pass@64": 0.9707695920724272, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9834692028985508, + "pass@2": 0.9852240956293506, + "pass@4": 0.9858998532792097, + "pass@8": 0.986317596894847, + "pass@16": 0.9869360286784715, + "pass@32": 0.9877361308097358, + "pass@64": 0.988322264064818, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.9928142908309455, + "pass@2": 0.9974357275003949, + "pass@4": 0.999494777783866, + "pass@8": 0.9999737827585785, + "pass@16": 0.9999999348964089, + "pass@32": 0.9999999999999439, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9855491329479769, + "pass@2": 0.9887166605980608, + "pass@4": 0.9913939513883019, + "pass@8": 0.9943285343504065, + "pass@16": 0.9968946491921471, + "pass@32": 0.9987241613729214, + "pass@64": 0.999808867257577, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9807658450704225, + "pass@2": 0.9904752827991573, + "pass@4": 0.9946537202215919, + "pass@8": 0.9965723900602302, + "pass@16": 0.9974438326135088, + "pass@32": 0.9978865571381552, + "pass@64": 0.9985915492939871, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9550196482035929, + "pass@2": 0.9680099692347592, + "pass@4": 0.973621248223463, + "pass@8": 0.9772931092950905, + "pass@16": 0.9797760833989643, + "pass@32": 0.9813793644561303, + "pass@64": 0.9820069826621478, + "pass@128": 0.9820359281437125 + }, + "11": { + "pass@1": 0.8364551957831325, + "pass@2": 0.8805564284697844, + "pass@4": 0.9119869102393826, + "pass@8": 0.9332457070554272, + "pass@16": 0.9464105210220005, + "pass@32": 0.9554480120240839, + "pass@64": 0.9620750241307124, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.7179907915360502, + "pass@2": 0.7807096641818678, + "pass@4": 0.8237406890557176, + "pass@8": 0.8544050832000054, + "pass@16": 0.8745406088754614, + "pass@32": 0.8870839775741599, + "pass@64": 0.8953727964648683, + "pass@128": 0.8996865203761756 + }, + "13": { + "pass@1": 0.4339110644257703, + "pass@2": 0.5064138352191269, + "pass@4": 0.5668354349613859, + "pass@8": 0.6208126022401135, + "pass@16": 0.6694676564972795, + "pass@32": 0.7126055948949221, + "pass@64": 0.7509963818526764, + "pass@128": 0.7843137254901961 + }, + "14": { + "pass@1": 0.39722679093567254, + "pass@2": 0.46368376444720705, + "pass@4": 0.5167455819667276, + "pass@8": 0.56171751122582, + "pass@16": 0.6024369898411556, + "pass@32": 0.6426240615167135, + "pass@64": 0.6832214593635492, + "pass@128": 0.7222222222222222 + }, + "15": { + "pass@1": 0.24644886363636365, + "pass@2": 0.32176874552612744, + "pass@4": 0.3923603217211486, + "pass@8": 0.4581419440373477, + "pass@16": 0.5190329686132177, + "pass@32": 0.5727951896253346, + "pass@64": 0.6198279692563144, + "pass@128": 0.6636363636363637 + }, + "16": { + "pass@1": 0.2040820868945869, + "pass@2": 0.26635372501514243, + "pass@4": 0.3238963121062858, + "pass@8": 0.37332193265107516, + "pass@16": 0.4134811645828353, + "pass@32": 0.4443577173332517, + "pass@64": 0.47302414732746867, + "pass@128": 0.50997150997151 + }, + "17": { + "pass@1": 0.19650135869565216, + "pass@2": 0.2574690052330416, + "pass@4": 0.3152138539941824, + "pass@8": 0.3709270117297045, + "pass@16": 0.42297098948583206, + "pass@32": 0.4692510903823389, + "pass@64": 0.5108174522501921, + "pass@128": 0.5527950310559007 + }, + "18": { + "pass@1": 0.18683363970588235, + "pass@2": 0.24651024779990735, + "pass@4": 0.3035248188645537, + "pass@8": 0.3556424980931623, + "pass@16": 0.4017419832141523, + "pass@32": 0.43944869520791696, + "pass@64": 0.4673817173155303, + "pass@128": 0.49117647058823527 + }, + "19": { + "pass@1": 0.14808872767857142, + "pass@2": 0.2049320250984251, + "pass@4": 0.2627956976918065, + "pass@8": 0.3139157657154089, + "pass@16": 0.3560561707464758, + "pass@32": 0.39169649693737485, + "pass@64": 0.4226012391904779, + "pass@128": 0.44642857142857145 + }, + "20": { + "pass@1": 0.13087107035928144, + "pass@2": 0.1879670776557122, + "pass@4": 0.24696301351178396, + "pass@8": 0.30177241862067616, + "pass@16": 0.34838594581032317, + "pass@32": 0.38534870948871147, + "pass@64": 0.4129125683339026, + "pass@128": 0.4341317365269461 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 523941, + "answer_accuracy": 0.6539845123022847, + "avg_response_len": 275.54656045294774, + "resp_tokens_sum": 220754678, + "pass_at_k": { + "pass@1": 0.6539845123022847, + "pass@2": 0.689856626143393, + "pass@4": 0.7200727996005184, + "pass@8": 0.7453208731590747, + "pass@16": 0.7665777414461511, + "pass@32": 0.7847634494980638, + "pass@64": 0.8007918914069807, + "pass@128": 0.8149864195558396 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9029617537313432, + "pass@2": 0.9262189299565168, + "pass@4": 0.9432951989210303, + "pass@8": 0.9559143658871644, + "pass@16": 0.9651937507145258, + "pass@32": 0.9716208086596739, + "pass@64": 0.9761076892137535, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 0.9999107142857143, + "pass@2": 0.9999989454443194, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991116982507289, + "pass@2": 0.9997976974816924, + "pass@4": 0.9999873259282823, + "pass@8": 0.9999999629758403, + "pass@16": 0.9999999999999313, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999314692982456, + "pass@2": 0.9999996402587834, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9659926470588235, + "pass@2": 0.9708096919870312, + "pass@4": 0.9745823887822845, + "pass@8": 0.9771404920721685, + "pass@16": 0.9789554066534717, + "pass@32": 0.9809376766950135, + "pass@64": 0.9837387679474603, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9929268533123028, + "pass@2": 0.9952720199210117, + "pass@4": 0.9963477722816194, + "pass@8": 0.996781255357137, + "pass@16": 0.996844514797445, + "pass@32": 0.9968454258207375, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9869956487341772, + "pass@2": 0.9934614210605002, + "pass@4": 0.9959656588614716, + "pass@8": 0.9971545915353225, + "pass@16": 0.9978742203032149, + "pass@32": 0.9986754832496939, + "pass@64": 0.9996137745440047, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9711195570570571, + "pass@2": 0.9796935222624201, + "pass@4": 0.9847409628413564, + "pass@8": 0.9876525069319597, + "pass@16": 0.9891184230382688, + "pass@32": 0.9900539597715055, + "pass@64": 0.9908121349376716, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.9665393890675241, + "pass@2": 0.9782261943945113, + "pass@4": 0.9827812531271213, + "pass@8": 0.9852322346734745, + "pass@16": 0.9870234208375871, + "pass@32": 0.9888026081069605, + "pass@64": 0.9901177403407886, + "pass@128": 0.9903536977491961 + }, + "11": { + "pass@1": 0.8699652777777778, + "pass@2": 0.9050528840144983, + "pass@4": 0.9261437665529902, + "pass@8": 0.9379237916730481, + "pass@16": 0.9436162837580702, + "pass@32": 0.9481522394334617, + "pass@64": 0.9538037350418224, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.6851586391437309, + "pass@2": 0.7609573280117508, + "pass@4": 0.8146087567609088, + "pass@8": 0.8498854256731503, + "pass@16": 0.8726716568222734, + "pass@32": 0.8890453884635856, + "pass@64": 0.9038027299053867, + "pass@128": 0.9204892966360856 + }, + "13": { + "pass@1": 0.48818479938271603, + "pass@2": 0.5643306630941965, + "pass@4": 0.6234116620839059, + "pass@8": 0.670892987256112, + "pass@16": 0.710554304400132, + "pass@32": 0.7441477272144019, + "pass@64": 0.7721765126609678, + "pass@128": 0.7932098765432098 + }, + "14": { + "pass@1": 0.39139002225519287, + "pass@2": 0.4689131901446295, + "pass@4": 0.5358667138974095, + "pass@8": 0.5899477624146936, + "pass@16": 0.6321371774697618, + "pass@32": 0.6666508816034391, + "pass@64": 0.697774235354575, + "pass@128": 0.7270029673590505 + }, + "15": { + "pass@1": 0.29598283179012347, + "pass@2": 0.3657031477836101, + "pass@4": 0.4284062113763555, + "pass@8": 0.488394802453985, + "pass@16": 0.5458289659825917, + "pass@32": 0.5968433317356244, + "pass@64": 0.6421028185366104, + "pass@128": 0.6820987654320988 + }, + "16": { + "pass@1": 0.20973912241887904, + "pass@2": 0.2661744680974612, + "pass@4": 0.32020332027080706, + "pass@8": 0.37020648522530164, + "pass@16": 0.4155200084120026, + "pass@32": 0.4563823340406111, + "pass@64": 0.4954197121915054, + "pass@128": 0.5339233038348082 + }, + "17": { + "pass@1": 0.1867940512048193, + "pass@2": 0.2508834550801631, + "pass@4": 0.3094012797534344, + "pass@8": 0.3578539351904199, + "pass@16": 0.3979111457071782, + "pass@32": 0.43246884560924453, + "pass@64": 0.46455603309265586, + "pass@128": 0.49698795180722893 + }, + "18": { + "pass@1": 0.17842741935483872, + "pass@2": 0.2330915036830072, + "pass@4": 0.286078386068677, + "pass@8": 0.3361083917487688, + "pass@16": 0.3821960914627827, + "pass@32": 0.422830010224237, + "pass@64": 0.45738747568942023, + "pass@128": 0.4870967741935484 + }, + "19": { + "pass@1": 0.1512698262839879, + "pass@2": 0.20847594083783336, + "pass@4": 0.2632785100314122, + "pass@8": 0.31349632397118765, + "pass@16": 0.3602084544258579, + "pass@32": 0.40216106105046995, + "pass@64": 0.4354416751759285, + "pass@128": 0.459214501510574 + }, + "20": { + "pass@1": 0.16378096846846846, + "pass@2": 0.22388222375919223, + "pass@4": 0.2818586584334616, + "pass@8": 0.3335726763870422, + "pass@16": 0.37771145894953373, + "pass@32": 0.41447893995579, + "pass@64": 0.44587055010516763, + "pass@128": 0.47147147147147145 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..41f7cc19fe262bfbe081f0e115644fe5bc67661e --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8", "results_dir": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/checkpoint-21808_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1000310, "value_mismatch": 32969, "dependency_mismatch": 146361, "answer_mismatch": 27959, "extra_nodes": 22072, "missing_nodes": 18132, "perfect_with_extra": 767, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 328176, "value_mismatch": 12183, "answer_mismatch": 10247, "dependency_mismatch": 48823, "extra_nodes": 8295, "missing_nodes": 6883, "perfect_with_extra": 183}, "teachers_in_school": {"total": 390656, "value_mismatch": 9952, "answer_mismatch": 8649, "perfect": 340663, "dependency_mismatch": 48401, "extra_nodes": 6709, "missing_nodes": 5536, "perfect_with_extra": 297}, "movie_festival_awards": {"total": 382336, "perfect": 331471, "dependency_mismatch": 49137, "extra_nodes": 7068, "missing_nodes": 5713, "value_mismatch": 10834, "answer_mismatch": 9063, "perfect_with_extra": 287}}, "results_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/checkpoint-21808_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/checkpoint-21808_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 158035, "value_mismatch": 972581, "dependency_mismatch": 1081387, "answer_mismatch": 824839, "extra_nodes": 54235, "missing_nodes": 743164, "perfect_with_extra": 4, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 327226, "dependency_mismatch": 365735, "answer_mismatch": 276076, "extra_nodes": 20533, "missing_nodes": 260701, "perfect": 52823, "perfect_with_extra": 2}, "movie_festival_awards": {"total": 418816, "perfect": 53124, "value_mismatch": 316268, "dependency_mismatch": 353030, "answer_mismatch": 268152, "missing_nodes": 236173, "extra_nodes": 17776}, "teachers_in_school": {"total": 430464, "perfect": 52088, "dependency_mismatch": 362622, "value_mismatch": 329087, "answer_mismatch": 280611, "extra_nodes": 15926, "missing_nodes": 246290, "perfect_with_extra": 2}}, "results_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/checkpoint-21808_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1158345, "value_mismatch": 1005550, "dependency_mismatch": 1227748, "answer_mismatch": 852798, "extra_nodes": 76307, "missing_nodes": 761296, "perfect_with_extra": 771, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 380999, "value_mismatch": 339409, "answer_mismatch": 286323, "dependency_mismatch": 414558, "extra_nodes": 28828, "missing_nodes": 267584, "perfect_with_extra": 185}, "teachers_in_school": {"total": 821120, "value_mismatch": 339039, "answer_mismatch": 289260, "perfect": 392751, "dependency_mismatch": 411023, "extra_nodes": 22635, "missing_nodes": 251826, "perfect_with_extra": 299}, "movie_festival_awards": {"total": 801152, "perfect": 384595, "dependency_mismatch": 402167, "extra_nodes": 24844, "missing_nodes": 241886, "value_mismatch": 327102, "answer_mismatch": 277215, "perfect_with_extra": 287}}}} diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/results.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..3a850361797b424bb46ee4124711c8a7f558e731 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21808_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9365079365079365, + "pass@2": 0.9492063492063492, + "pass@4": 0.9746031746031746, + "pass@8": 0.9777777777777777, + "pass@16": 0.9809523809523809, + "pass@32": 0.9841269841269841, + "pass@64": 0.9873015873015873, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.9461077844311377, + "pass@2": 0.9640718562874252, + "pass@4": 0.9760479041916168, + "pass@8": 0.9850299401197605, + "pass@16": 0.9910179640718563, + "pass@32": 0.9910179640718563, + "pass@64": 0.9910179640718563, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.863905325443787, + "pass@2": 0.8994082840236687, + "pass@4": 0.9408284023668639, + "pass@8": 0.9674556213017751, + "pass@16": 0.9792899408284024, + "pass@32": 0.9822485207100592, + "pass@64": 0.985207100591716, + "pass@128": 0.985207100591716 + }, + "8": { + "pass@1": 0.7788461538461539, + "pass@2": 0.8653846153846154, + "pass@4": 0.907051282051282, + "pass@8": 0.9615384615384616, + "pass@16": 0.9775641025641025, + "pass@32": 0.9871794871794872, + "pass@64": 0.9935897435897436, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.6957746478873239, + "pass@2": 0.8112676056338028, + "pass@4": 0.895774647887324, + "pass@8": 0.952112676056338, + "pass@16": 0.9802816901408451, + "pass@32": 0.9915492957746479, + "pass@64": 0.9943661971830986, + "pass@128": 0.9943661971830986 + }, + "10": { + "pass@1": 0.616519174041298, + "pass@2": 0.6873156342182891, + "pass@4": 0.7581120943952803, + "pass@8": 0.8436578171091446, + "pass@16": 0.8908554572271387, + "pass@32": 0.911504424778761, + "pass@64": 0.9174041297935103, + "pass@128": 0.9174041297935103 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9365079365079365, + "6": 0.9461077844311377, + "7": 0.863905325443787, + "8": 0.7788461538461539, + "9": 0.6957746478873239, + "10": 0.616519174041298 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9492063492063492, + "6": 0.9640718562874252, + "7": 0.8994082840236687, + "8": 0.8653846153846154, + "9": 0.8112676056338028, + "10": 0.6873156342182891 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9746031746031746, + "6": 0.9760479041916168, + "7": 0.9408284023668639, + "8": 0.907051282051282, + "9": 0.895774647887324, + "10": 0.7581120943952803 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9777777777777777, + "6": 0.9850299401197605, + "7": 0.9674556213017751, + "8": 0.9615384615384616, + "9": 0.952112676056338, + "10": 0.8436578171091446 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9809523809523809, + "6": 0.9910179640718563, + "7": 0.9792899408284024, + "8": 0.9775641025641025, + "9": 0.9802816901408451, + "10": 0.8908554572271387 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9841269841269841, + "6": 0.9910179640718563, + "7": 0.9822485207100592, + "8": 0.9871794871794872, + "9": 0.9915492957746479, + "10": 0.911504424778761 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9910179640718563, + "7": 0.985207100591716, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9174041297935103 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9873015873015873, + "6": 0.9910179640718563, + "7": 0.985207100591716, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9174041297935103 + } + }, + "total_pass_at_k": { + "pass@1": 0.8676122931442081, + "pass@2": 0.9064505234718001, + "pass@4": 0.9378588314758528, + "pass@8": 0.9645390070921985, + "pass@16": 0.9773725092874029, + "pass@32": 0.9827760891590679, + "pass@64": 0.9851401553529213, + "pass@128": 0.9851401553529213 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9941520467836257, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9294117647058824, + "pass@2": 0.95, + "pass@4": 0.961764705882353, + "pass@8": 0.9676470588235294, + "pass@16": 0.9676470588235294, + "pass@32": 0.9676470588235294, + "pass@64": 0.9705882352941176, + "pass@128": 0.9764705882352941 + }, + "6": { + "pass@1": 0.9652996845425867, + "pass@2": 0.9779179810725552, + "pass@4": 0.9936908517350158, + "pass@8": 0.9968454258675079, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8259493670886076, + "pass@2": 0.8607594936708861, + "pass@4": 0.9177215189873418, + "pass@8": 0.9778481012658228, + "pass@16": 0.9873417721518988, + "pass@32": 0.990506329113924, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.7507507507507507, + "pass@2": 0.8258258258258259, + "pass@4": 0.8828828828828829, + "pass@8": 0.9429429429429429, + "pass@16": 0.9669669669669669, + "pass@32": 0.972972972972973, + "pass@64": 0.978978978978979, + "pass@128": 0.9819819819819819 + }, + "9": { + "pass@1": 0.7106109324758842, + "pass@2": 0.8006430868167203, + "pass@4": 0.8938906752411575, + "pass@8": 0.9356913183279743, + "pass@16": 0.9710610932475884, + "pass@32": 0.9742765273311897, + "pass@64": 0.9807073954983923, + "pass@128": 0.9807073954983923 + }, + "10": { + "pass@1": 0.5880597014925373, + "pass@2": 0.6507462686567164, + "pass@4": 0.7283582089552239, + "pass@8": 0.7731343283582089, + "pass@16": 0.8477611940298507, + "pass@32": 0.8895522388059701, + "pass@64": 0.9164179104477612, + "pass@128": 0.9194029850746268 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9294117647058824, + "6": 0.9652996845425867, + "7": 0.8259493670886076, + "8": 0.7507507507507507, + "9": 0.7106109324758842, + "10": 0.5880597014925373 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.95, + "6": 0.9779179810725552, + "7": 0.8607594936708861, + "8": 0.8258258258258259, + "9": 0.8006430868167203, + "10": 0.6507462686567164 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.961764705882353, + "6": 0.9936908517350158, + "7": 0.9177215189873418, + "8": 0.8828828828828829, + "9": 0.8938906752411575, + "10": 0.7283582089552239 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9676470588235294, + "6": 0.9968454258675079, + "7": 0.9778481012658228, + "8": 0.9429429429429429, + "9": 0.9356913183279743, + "10": 0.7731343283582089 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9676470588235294, + "6": 0.9968454258675079, + "7": 0.9873417721518988, + "8": 0.9669669669669669, + "9": 0.9710610932475884, + "10": 0.8477611940298507 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9676470588235294, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.972972972972973, + "9": 0.9742765273311897, + "10": 0.8895522388059701 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9705882352941176, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.978978978978979, + "9": 0.9807073954983923, + "10": 0.9164179104477612 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9764705882352941, + "6": 0.9968454258675079, + "7": 0.9936708860759493, + "8": 0.9819819819819819, + "9": 0.9807073954983923, + "10": 0.9194029850746268 + } + }, + "total_pass_at_k": { + "pass@1": 0.8650820220957482, + "pass@2": 0.8972212922664882, + "pass@4": 0.9317040508871778, + "pass@8": 0.9551389353866756, + "pass@16": 0.970873786407767, + "pass@32": 0.9768998995647807, + "pass@64": 0.9819216605289588, + "pass@128": 0.983260796786073 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.985207100591716, + "pass@2": 0.9940828402366864, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9449275362318841, + "pass@2": 0.9681159420289855, + "pass@4": 0.9797101449275363, + "pass@8": 0.9884057971014493, + "pass@16": 0.9884057971014493, + "pass@32": 0.9884057971014493, + "pass@64": 0.9884057971014493, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9828080229226361, + "pass@4": 0.9885386819484241, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8757225433526011, + "pass@2": 0.9104046242774566, + "pass@4": 0.9479768786127167, + "pass@8": 0.9710982658959537, + "pass@16": 0.9855491329479769, + "pass@32": 0.9942196531791907, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7718309859154929, + "pass@2": 0.8647887323943662, + "pass@4": 0.923943661971831, + "pass@8": 0.9492957746478873, + "pass@16": 0.9746478873239437, + "pass@32": 0.9859154929577465, + "pass@64": 0.9943661971830986, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.7035928143712575, + "pass@2": 0.7574850299401198, + "pass@4": 0.8293413173652695, + "pass@8": 0.8952095808383234, + "pass@16": 0.9461077844311377, + "pass@32": 0.9640718562874252, + "pass@64": 0.9700598802395209, + "pass@128": 0.9700598802395209 + }, + "10": { + "pass@1": 0.6073619631901841, + "pass@2": 0.6932515337423313, + "pass@4": 0.754601226993865, + "pass@8": 0.8098159509202454, + "pass@16": 0.8588957055214724, + "pass@32": 0.8834355828220859, + "pass@64": 0.8987730061349694, + "pass@128": 0.9049079754601227 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.985207100591716, + "5": 0.9449275362318841, + "6": 0.9713467048710601, + "7": 0.8757225433526011, + "8": 0.7718309859154929, + "9": 0.7035928143712575, + "10": 0.6073619631901841 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9940828402366864, + "5": 0.9681159420289855, + "6": 0.9828080229226361, + "7": 0.9104046242774566, + "8": 0.8647887323943662, + "9": 0.7574850299401198, + "10": 0.6932515337423313 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9797101449275363, + "6": 0.9885386819484241, + "7": 0.9479768786127167, + "8": 0.923943661971831, + "9": 0.8293413173652695, + "10": 0.754601226993865 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9710982658959537, + "8": 0.9492957746478873, + "9": 0.8952095808383234, + "10": 0.8098159509202454 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9855491329479769, + "8": 0.9746478873239437, + "9": 0.9461077844311377, + "10": 0.8588957055214724 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9942196531791907, + "8": 0.9859154929577465, + "9": 0.9640718562874252, + "10": 0.8834355828220859 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9943661971830986, + "9": 0.9700598802395209, + "10": 0.8987730061349694 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 1.0, + "8": 0.9943661971830986, + "9": 0.9700598802395209, + "10": 0.9049079754601227 + } + }, + "total_pass_at_k": { + "pass@1": 0.8738532110091743, + "pass@2": 0.9085845347313237, + "pass@4": 0.936435124508519, + "pass@8": 0.9577326343381389, + "pass@16": 0.973132372214941, + "pass@32": 0.9800131061598951, + "pass@64": 0.9836173001310616, + "pass@128": 0.984600262123198 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.992, + "pass@2": 0.996, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.937, + "pass@2": 0.956, + "pass@4": 0.972, + "pass@8": 0.978, + "pass@16": 0.979, + "pass@32": 0.98, + "pass@64": 0.982, + "pass@128": 0.984 + }, + "6": { + "pass@1": 0.961, + "pass@2": 0.975, + "pass@4": 0.986, + "pass@8": 0.994, + "pass@16": 0.996, + "pass@32": 0.996, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "7": { + "pass@1": 0.856, + "pass@2": 0.891, + "pass@4": 0.936, + "pass@8": 0.972, + "pass@16": 0.984, + "pass@32": 0.989, + "pass@64": 0.992, + "pass@128": 0.993 + }, + "8": { + "pass@1": 0.767, + "pass@2": 0.852, + "pass@4": 0.905, + "pass@8": 0.951, + "pass@16": 0.973, + "pass@32": 0.982, + "pass@64": 0.989, + "pass@128": 0.99 + }, + "9": { + "pass@1": 0.703, + "pass@2": 0.79, + "pass@4": 0.873, + "pass@8": 0.928, + "pass@16": 0.966, + "pass@32": 0.977, + "pass@64": 0.982, + "pass@128": 0.982 + }, + "10": { + "pass@1": 0.604, + "pass@2": 0.677, + "pass@4": 0.747, + "pass@8": 0.809, + "pass@16": 0.866, + "pass@32": 0.895, + "pass@64": 0.911, + "pass@128": 0.914 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.992, + "5": 0.937, + "6": 0.961, + "7": 0.856, + "8": 0.767, + "9": 0.703, + "10": 0.604 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.956, + "6": 0.975, + "7": 0.891, + "8": 0.852, + "9": 0.79, + "10": 0.677 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.972, + "6": 0.986, + "7": 0.936, + "8": 0.905, + "9": 0.873, + "10": 0.747 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.978, + "6": 0.994, + "7": 0.972, + "8": 0.951, + "9": 0.928, + "10": 0.809 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.979, + "6": 0.996, + "7": 0.984, + "8": 0.973, + "9": 0.966, + "10": 0.866 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.98, + "6": 0.996, + "7": 0.989, + "8": 0.982, + "9": 0.977, + "10": 0.895 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.982, + "6": 0.996, + "7": 0.992, + "8": 0.989, + "9": 0.982, + "10": 0.911 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.984, + "6": 0.996, + "7": 0.993, + "8": 0.99, + "9": 0.982, + "10": 0.914 + } + }, + "total_pass_at_k": { + "pass@1": 0.8688888888888889, + "pass@2": 0.9041111111111111, + "pass@4": 0.9353333333333333, + "pass@8": 0.9591111111111111, + "pass@16": 0.9737777777777777, + "pass@32": 0.9798888888888889, + "pass@64": 0.9835555555555555, + "pass@128": 0.9843333333333333 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21808_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.45892351274787535, + "pass@2": 0.5580736543909348, + "pass@4": 0.6742209631728046, + "pass@8": 0.7365439093484419, + "pass@16": 0.8073654390934845, + "pass@32": 0.8356940509915014, + "pass@64": 0.8753541076487252, + "pass@128": 0.9008498583569405 + }, + "12": { + "pass@1": 0.4265536723163842, + "pass@2": 0.4745762711864407, + "pass@4": 0.5338983050847458, + "pass@8": 0.6016949152542372, + "pass@16": 0.6666666666666666, + "pass@32": 0.7203389830508474, + "pass@64": 0.748587570621469, + "pass@128": 0.7768361581920904 + }, + "13": { + "pass@1": 0.15673981191222572, + "pass@2": 0.2006269592476489, + "pass@4": 0.2601880877742947, + "pass@8": 0.3103448275862069, + "pass@16": 0.3667711598746082, + "pass@32": 0.41379310344827586, + "pass@64": 0.438871473354232, + "pass@128": 0.4890282131661442 + }, + "14": { + "pass@1": 0.102803738317757, + "pass@2": 0.11214953271028037, + "pass@4": 0.13395638629283488, + "pass@8": 0.16510903426791276, + "pass@16": 0.19626168224299065, + "pass@32": 0.24299065420560748, + "pass@64": 0.29595015576323985, + "pass@128": 0.32710280373831774 + }, + "15": { + "pass@1": 0.046242774566473986, + "pass@2": 0.06069364161849711, + "pass@4": 0.07514450867052024, + "pass@8": 0.08670520231213873, + "pass@16": 0.10404624277456648, + "pass@32": 0.11560693641618497, + "pass@64": 0.13583815028901733, + "pass@128": 0.16473988439306358 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0064516129032258064, + "pass@4": 0.0064516129032258064, + "pass@8": 0.00967741935483871, + "pass@16": 0.00967741935483871, + "pass@32": 0.00967741935483871, + "pass@64": 0.012903225806451613, + "pass@128": 0.01935483870967742 + }, + "17": { + "pass@1": 0.002890173410404624, + "pass@2": 0.002890173410404624, + "pass@4": 0.002890173410404624, + "pass@8": 0.005780346820809248, + "pass@16": 0.005780346820809248, + "pass@32": 0.011560693641618497, + "pass@64": 0.011560693641618497, + "pass@128": 0.023121387283236993 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.002857142857142857, + "pass@16": 0.005714285714285714, + "pass@32": 0.005714285714285714, + "pass@64": 0.005714285714285714, + "pass@128": 0.005714285714285714 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.45892351274787535, + "12": 0.4265536723163842, + "13": 0.15673981191222572, + "14": 0.102803738317757, + "15": 0.046242774566473986, + "16": 0.0, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5580736543909348, + "12": 0.4745762711864407, + "13": 0.2006269592476489, + "14": 0.11214953271028037, + "15": 0.06069364161849711, + "16": 0.0064516129032258064, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6742209631728046, + "12": 0.5338983050847458, + "13": 0.2601880877742947, + "14": 0.13395638629283488, + "15": 0.07514450867052024, + "16": 0.0064516129032258064, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7365439093484419, + "12": 0.6016949152542372, + "13": 0.3103448275862069, + "14": 0.16510903426791276, + "15": 0.08670520231213873, + "16": 0.00967741935483871, + "17": 0.005780346820809248, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8073654390934845, + "12": 0.6666666666666666, + "13": 0.3667711598746082, + "14": 0.19626168224299065, + "15": 0.10404624277456648, + "16": 0.00967741935483871, + "17": 0.005780346820809248, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8356940509915014, + "12": 0.7203389830508474, + "13": 0.41379310344827586, + "14": 0.24299065420560748, + "15": 0.11560693641618497, + "16": 0.00967741935483871, + "17": 0.011560693641618497, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8753541076487252, + "12": 0.748587570621469, + "13": 0.438871473354232, + "14": 0.29595015576323985, + "15": 0.13583815028901733, + "16": 0.012903225806451613, + "17": 0.011560693641618497, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.9008498583569405, + "12": 0.7768361581920904, + "13": 0.4890282131661442, + "14": 0.32710280373831774, + "15": 0.16473988439306358, + "16": 0.01935483870967742, + "17": 0.023121387283236993, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.12273402674591381, + "pass@2": 0.14531946508172364, + "pass@4": 0.17295690936106983, + "pass@8": 0.1964338781575037, + "pass@16": 0.22109955423476968, + "pass@32": 0.24041604754829124, + "pass@64": 0.25735512630014856, + "pass@128": 0.27548291233283806 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4888888888888889, + "pass@2": 0.5619047619047619, + "pass@4": 0.6666666666666666, + "pass@8": 0.765079365079365, + "pass@16": 0.8126984126984127, + "pass@32": 0.8666666666666667, + "pass@64": 0.8888888888888888, + "pass@128": 0.8984126984126984 + }, + "12": { + "pass@1": 0.39755351681957185, + "pass@2": 0.48623853211009177, + "pass@4": 0.5412844036697247, + "pass@8": 0.5902140672782875, + "pass@16": 0.6697247706422018, + "pass@32": 0.709480122324159, + "pass@64": 0.7431192660550459, + "pass@128": 0.7522935779816514 + }, + "13": { + "pass@1": 0.16666666666666666, + "pass@2": 0.24074074074074073, + "pass@4": 0.2962962962962963, + "pass@8": 0.3395061728395062, + "pass@16": 0.39814814814814814, + "pass@32": 0.43209876543209874, + "pass@64": 0.4722222222222222, + "pass@128": 0.5092592592592593 + }, + "14": { + "pass@1": 0.11275964391691394, + "pass@2": 0.1394658753709199, + "pass@4": 0.16913946587537093, + "pass@8": 0.19881305637982197, + "pass@16": 0.2344213649851632, + "pass@32": 0.27299703264094954, + "pass@64": 0.32344213649851633, + "pass@128": 0.33827893175074186 + }, + "15": { + "pass@1": 0.08950617283950617, + "pass@2": 0.11419753086419752, + "pass@4": 0.12962962962962962, + "pass@8": 0.14506172839506173, + "pass@16": 0.16049382716049382, + "pass@32": 0.1728395061728395, + "pass@64": 0.2037037037037037, + "pass@128": 0.21604938271604937 + }, + "16": { + "pass@1": 0.011799410029498525, + "pass@2": 0.017699115044247787, + "pass@4": 0.02359882005899705, + "pass@8": 0.029498525073746312, + "pass@16": 0.032448377581120944, + "pass@32": 0.04129793510324484, + "pass@64": 0.0471976401179941, + "pass@128": 0.061946902654867256 + }, + "17": { + "pass@1": 0.009036144578313253, + "pass@2": 0.018072289156626505, + "pass@4": 0.024096385542168676, + "pass@8": 0.02710843373493976, + "pass@16": 0.030120481927710843, + "pass@32": 0.03313253012048193, + "pass@64": 0.0391566265060241, + "pass@128": 0.0391566265060241 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0032258064516129032, + "pass@4": 0.0032258064516129032, + "pass@8": 0.0032258064516129032, + "pass@16": 0.00967741935483871, + "pass@32": 0.016129032258064516, + "pass@64": 0.01935483870967742, + "pass@128": 0.02258064516129032 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4888888888888889, + "12": 0.39755351681957185, + "13": 0.16666666666666666, + "14": 0.11275964391691394, + "15": 0.08950617283950617, + "16": 0.011799410029498525, + "17": 0.009036144578313253, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5619047619047619, + "12": 0.48623853211009177, + "13": 0.24074074074074073, + "14": 0.1394658753709199, + "15": 0.11419753086419752, + "16": 0.017699115044247787, + "17": 0.018072289156626505, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6666666666666666, + "12": 0.5412844036697247, + "13": 0.2962962962962963, + "14": 0.16913946587537093, + "15": 0.12962962962962962, + "16": 0.02359882005899705, + "17": 0.024096385542168676, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.765079365079365, + "12": 0.5902140672782875, + "13": 0.3395061728395062, + "14": 0.19881305637982197, + "15": 0.14506172839506173, + "16": 0.029498525073746312, + "17": 0.02710843373493976, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8126984126984127, + "12": 0.6697247706422018, + "13": 0.39814814814814814, + "14": 0.2344213649851632, + "15": 0.16049382716049382, + "16": 0.032448377581120944, + "17": 0.030120481927710843, + "18": 0.00967741935483871, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8666666666666667, + "12": 0.709480122324159, + "13": 0.43209876543209874, + "14": 0.27299703264094954, + "15": 0.1728395061728395, + "16": 0.04129793510324484, + "17": 0.03313253012048193, + "18": 0.016129032258064516, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8888888888888888, + "12": 0.7431192660550459, + "13": 0.4722222222222222, + "14": 0.32344213649851633, + "15": 0.2037037037037037, + "16": 0.0471976401179941, + "17": 0.0391566265060241, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.8984126984126984, + "12": 0.7522935779816514, + "13": 0.5092592592592593, + "14": 0.33827893175074186, + "15": 0.21604938271604937, + "16": 0.061946902654867256, + "17": 0.0391566265060241, + "18": 0.02258064516129032, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.12591687041564792, + "pass@2": 0.15617359413202933, + "pass@4": 0.18306845965770172, + "pass@8": 0.20721271393643031, + "pass@16": 0.2319682151589242, + "pass@32": 0.2515281173594132, + "pass@64": 0.2707823960880196, + "pass@128": 0.2808679706601467 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.49698795180722893, + "pass@2": 0.608433734939759, + "pass@4": 0.6897590361445783, + "pass@8": 0.75, + "pass@16": 0.8012048192771084, + "pass@32": 0.8614457831325302, + "pass@64": 0.8765060240963856, + "pass@128": 0.9066265060240963 + }, + "12": { + "pass@1": 0.3510971786833856, + "pass@2": 0.45768025078369906, + "pass@4": 0.5078369905956113, + "pass@8": 0.5830721003134797, + "pass@16": 0.6520376175548589, + "pass@32": 0.6927899686520376, + "pass@64": 0.7241379310344828, + "pass@128": 0.7492163009404389 + }, + "13": { + "pass@1": 0.15406162464985995, + "pass@2": 0.20168067226890757, + "pass@4": 0.27450980392156865, + "pass@8": 0.3389355742296919, + "pass@16": 0.3865546218487395, + "pass@32": 0.4341736694677871, + "pass@64": 0.47058823529411764, + "pass@128": 0.5238095238095238 + }, + "14": { + "pass@1": 0.1286549707602339, + "pass@2": 0.14912280701754385, + "pass@4": 0.16666666666666666, + "pass@8": 0.19005847953216373, + "pass@16": 0.21929824561403508, + "pass@32": 0.24561403508771928, + "pass@64": 0.28362573099415206, + "pass@128": 0.32748538011695905 + }, + "15": { + "pass@1": 0.048484848484848485, + "pass@2": 0.05454545454545454, + "pass@4": 0.07575757575757576, + "pass@8": 0.09090909090909091, + "pass@16": 0.10303030303030303, + "pass@32": 0.12424242424242424, + "pass@64": 0.15757575757575756, + "pass@128": 0.19090909090909092 + }, + "16": { + "pass@1": 0.011396011396011397, + "pass@2": 0.014245014245014245, + "pass@4": 0.014245014245014245, + "pass@8": 0.017094017094017096, + "pass@16": 0.022792022792022793, + "pass@32": 0.03418803418803419, + "pass@64": 0.037037037037037035, + "pass@128": 0.042735042735042736 + }, + "17": { + "pass@1": 0.006211180124223602, + "pass@2": 0.006211180124223602, + "pass@4": 0.012422360248447204, + "pass@8": 0.015527950310559006, + "pass@16": 0.015527950310559006, + "pass@32": 0.015527950310559006, + "pass@64": 0.015527950310559006, + "pass@128": 0.018633540372670808 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0029411764705882353, + "pass@4": 0.0058823529411764705, + "pass@8": 0.008823529411764706, + "pass@16": 0.008823529411764706, + "pass@32": 0.008823529411764706, + "pass@64": 0.008823529411764706, + "pass@128": 0.011764705882352941 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.002976190476190476, + "pass@128": 0.002976190476190476 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0029940119760479044, + "pass@128": 0.0029940119760479044 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.49698795180722893, + "12": 0.3510971786833856, + "13": 0.15406162464985995, + "14": 0.1286549707602339, + "15": 0.048484848484848485, + "16": 0.011396011396011397, + "17": 0.006211180124223602, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.608433734939759, + "12": 0.45768025078369906, + "13": 0.20168067226890757, + "14": 0.14912280701754385, + "15": 0.05454545454545454, + "16": 0.014245014245014245, + "17": 0.006211180124223602, + "18": 0.0029411764705882353, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6897590361445783, + "12": 0.5078369905956113, + "13": 0.27450980392156865, + "14": 0.16666666666666666, + "15": 0.07575757575757576, + "16": 0.014245014245014245, + "17": 0.012422360248447204, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.75, + "12": 0.5830721003134797, + "13": 0.3389355742296919, + "14": 0.19005847953216373, + "15": 0.09090909090909091, + "16": 0.017094017094017096, + "17": 0.015527950310559006, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8012048192771084, + "12": 0.6520376175548589, + "13": 0.3865546218487395, + "14": 0.21929824561403508, + "15": 0.10303030303030303, + "16": 0.022792022792022793, + "17": 0.015527950310559006, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8614457831325302, + "12": 0.6927899686520376, + "13": 0.4341736694677871, + "14": 0.24561403508771928, + "15": 0.12424242424242424, + "16": 0.03418803418803419, + "17": 0.015527950310559006, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.8765060240963856, + "12": 0.7241379310344828, + "13": 0.47058823529411764, + "14": 0.28362573099415206, + "15": 0.15757575757575756, + "16": 0.037037037037037035, + "17": 0.015527950310559006, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@128": { + "11": 0.9066265060240963, + "12": 0.7492163009404389, + "13": 0.5238095238095238, + "14": 0.32748538011695905, + "15": 0.19090909090909092, + "16": 0.042735042735042736, + "17": 0.018633540372670808, + "18": 0.011764705882352941, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + } + }, + "total_pass_at_k": { + "pass@1": 0.1183467142432352, + "pass@2": 0.14778471602735652, + "pass@4": 0.17305976806422838, + "pass@8": 0.1977401129943503, + "pass@16": 0.2191495688373476, + "pass@32": 0.23996431757359502, + "pass@64": 0.25631876300921796, + "pass@128": 0.2762414510853405 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.481, + "pass@2": 0.576, + "pass@4": 0.677, + "pass@8": 0.75, + "pass@16": 0.807, + "pass@32": 0.854, + "pass@64": 0.88, + "pass@128": 0.902 + }, + "12": { + "pass@1": 0.393, + "pass@2": 0.473, + "pass@4": 0.528, + "pass@8": 0.592, + "pass@16": 0.663, + "pass@32": 0.708, + "pass@64": 0.739, + "pass@128": 0.76 + }, + "13": { + "pass@1": 0.159, + "pass@2": 0.214, + "pass@4": 0.277, + "pass@8": 0.33, + "pass@16": 0.384, + "pass@32": 0.427, + "pass@64": 0.461, + "pass@128": 0.508 + }, + "14": { + "pass@1": 0.115, + "pass@2": 0.134, + "pass@4": 0.157, + "pass@8": 0.185, + "pass@16": 0.217, + "pass@32": 0.254, + "pass@64": 0.301, + "pass@128": 0.331 + }, + "15": { + "pass@1": 0.061, + "pass@2": 0.076, + "pass@4": 0.093, + "pass@8": 0.107, + "pass@16": 0.122, + "pass@32": 0.137, + "pass@64": 0.165, + "pass@128": 0.19 + }, + "16": { + "pass@1": 0.008, + "pass@2": 0.013, + "pass@4": 0.015, + "pass@8": 0.019, + "pass@16": 0.022, + "pass@32": 0.029, + "pass@64": 0.033, + "pass@128": 0.042 + }, + "17": { + "pass@1": 0.006, + "pass@2": 0.009, + "pass@4": 0.013, + "pass@8": 0.016, + "pass@16": 0.017, + "pass@32": 0.02, + "pass@64": 0.022, + "pass@128": 0.027 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.002, + "pass@4": 0.003, + "pass@8": 0.005, + "pass@16": 0.008, + "pass@32": 0.01, + "pass@64": 0.011, + "pass@128": 0.013 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.001, + "pass@128": 0.001 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.001, + "pass@128": 0.001 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.481, + "12": 0.393, + "13": 0.159, + "14": 0.115, + "15": 0.061, + "16": 0.008, + "17": 0.006, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.576, + "12": 0.473, + "13": 0.214, + "14": 0.134, + "15": 0.076, + "16": 0.013, + "17": 0.009, + "18": 0.002, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.677, + "12": 0.528, + "13": 0.277, + "14": 0.157, + "15": 0.093, + "16": 0.015, + "17": 0.013, + "18": 0.003, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.75, + "12": 0.592, + "13": 0.33, + "14": 0.185, + "15": 0.107, + "16": 0.019, + "17": 0.016, + "18": 0.005, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.807, + "12": 0.663, + "13": 0.384, + "14": 0.217, + "15": 0.122, + "16": 0.022, + "17": 0.017, + "18": 0.008, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.854, + "12": 0.708, + "13": 0.427, + "14": 0.254, + "15": 0.137, + "16": 0.029, + "17": 0.02, + "18": 0.01, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.88, + "12": 0.739, + "13": 0.461, + "14": 0.301, + "15": 0.165, + "16": 0.033, + "17": 0.022, + "18": 0.011, + "19": 0.001, + "20": 0.001 + }, + "pass@128": { + "11": 0.902, + "12": 0.76, + "13": 0.508, + "14": 0.331, + "15": 0.19, + "16": 0.042, + "17": 0.027, + "18": 0.013, + "19": 0.001, + "20": 0.001 + } + }, + "total_pass_at_k": { + "pass@1": 0.1223, + "pass@2": 0.1497, + "pass@4": 0.1763, + "pass@8": 0.2004, + "pass@16": 0.224, + "pass@32": 0.2439, + "pass@64": 0.2614, + "pass@128": 0.2775 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/summary.csv b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..11698aac259976ac7f49b0d9048c7e3d647f251e --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21808,0.9757300347222222,0.35559765625,0.649344572368421,0.0,0.0,0.0,196.41278559027776,316.2850265625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/summary.json b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dbea4b47eb53ac157d54c3462db7729b9bf3ec37 --- /dev/null +++ b/SDAR-100M-ep1-block16/composition/low_confidence_static/block8/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21808", + "id_acc": 0.9757300347222222, + "ood_acc": 0.35559765625, + "total_acc": 0.649344572368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.41278559027776, + "ood_avg_resp_len": 316.2850265625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 523405, + "answer_accuracy": 0.6463960737432817, + "avg_response_len": 246.91074533670565, + "resp_tokens_sum": 199930544, + "pass_at_k": { + "pass@1": 0.6463960737432817, + "pass@2": 0.6836626853679749, + "pass@4": 0.7149836901759768, + "pass@8": 0.7411302959656122, + "pass@16": 0.762904361502207, + "pass@32": 0.7807357410333592, + "pass@64": 0.7951122370172407, + "pass@128": 0.807777426493835 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9079784292035398, + "pass@2": 0.9283034864004833, + "pass@4": 0.9397207392770592, + "pass@8": 0.9474065863832337, + "pass@16": 0.953277630712324, + "pass@32": 0.9587798739641823, + "pass@64": 0.9641137000779781, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9999026479750779, + "pass@2": 0.9999977003458681, + "pass@4": 0.9999999997079805, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9992115825688074, + "pass@2": 0.9999002955765851, + "pass@4": 0.9999974601798628, + "pass@8": 0.9999999989512287, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999365234375, + "pass@2": 0.9999569389763779, + "pass@4": 0.9999997020880201, + "pass@8": 0.9999999999934361, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9670138888888888, + "pass@2": 0.9722800274965628, + "pass@4": 0.9764746609054821, + "pass@8": 0.9805577074470837, + "pass@16": 0.9849640666818144, + "pass@32": 0.9887401621413379, + "pass@64": 0.9903685676767175, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9776618637724551, + "pass@2": 0.9838419863029846, + "pass@4": 0.9863429900603738, + "pass@8": 0.9880363231052547, + "pass@16": 0.9891851085789134, + "pass@32": 0.9900843472432467, + "pass@64": 0.9908396435351152, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9761002218934911, + "pass@2": 0.9821877329590457, + "pass@4": 0.9854717781209303, + "pass@8": 0.9882141392067795, + "pass@16": 0.9909368540370078, + "pass@32": 0.9934787972166277, + "pass@64": 0.995374785040164, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.9728565705128205, + "pass@2": 0.9847504037956794, + "pass@4": 0.9898392754992162, + "pass@8": 0.9928595606012538, + "pass@16": 0.9949857414228591, + "pass@32": 0.9963289537684682, + "pass@64": 0.9967736090563776, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.9606514084507042, + "pass@2": 0.9767279860263948, + "pass@4": 0.9843846306183557, + "pass@8": 0.9893238732854907, + "pass@16": 0.9925283410495083, + "pass@32": 0.9947305999396088, + "pass@64": 0.9964951038189134, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8500354107648725, + "pass@2": 0.8847818752648838, + "pass@4": 0.9057016992918374, + "pass@8": 0.9209417546235552, + "pass@16": 0.9350945491946546, + "pass@32": 0.9489809420090249, + "pass@64": 0.9604307908065416, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.7215969279661016, + "pass@2": 0.7881981516081679, + "pass@4": 0.8320213376399981, + "pass@8": 0.8641739560917514, + "pass@16": 0.8889098921541904, + "pass@32": 0.9070642083248085, + "pass@64": 0.9203748363482039, + "pass@128": 0.9322033898305084 + }, + "13": { + "pass@1": 0.47176234326018807, + "pass@2": 0.5514414354651594, + "pass@4": 0.61641166121868, + "pass@8": 0.6679790136854478, + "pass@16": 0.7104903504508953, + "pass@32": 0.7483723201148265, + "pass@64": 0.782034213606473, + "pass@128": 0.8119122257053292 + }, + "14": { + "pass@1": 0.3773364485981308, + "pass@2": 0.44722178784310845, + "pass@4": 0.5053335929387331, + "pass@8": 0.5553041100778835, + "pass@16": 0.5983519667569652, + "pass@32": 0.6341758147700675, + "pass@64": 0.6635061144623973, + "pass@128": 0.6853582554517134 + }, + "15": { + "pass@1": 0.2905979046242775, + "pass@2": 0.3691705828137089, + "pass@4": 0.44011951603954086, + "pass@8": 0.49954595991520595, + "pass@16": 0.5489780070558901, + "pass@32": 0.589607486805191, + "pass@64": 0.622846434960636, + "pass@128": 0.6502890173410405 + }, + "16": { + "pass@1": 0.18639112903225807, + "pass@2": 0.24502595567691135, + "pass@4": 0.3011878971781754, + "pass@8": 0.3503457986058463, + "pass@16": 0.3939021510117161, + "pass@32": 0.43316290463944823, + "pass@64": 0.46712538073865945, + "pass@128": 0.5 + }, + "17": { + "pass@1": 0.18806448699421965, + "pass@2": 0.24722041270310846, + "pass@4": 0.30208944176414365, + "pass@8": 0.3499756198131455, + "pass@16": 0.39130847250177325, + "pass@32": 0.4240642793091486, + "pass@64": 0.44695993145996443, + "pass@128": 0.4653179190751445 + }, + "18": { + "pass@1": 0.1703794642857143, + "pass@2": 0.23251019403824516, + "pass@4": 0.2904351298944774, + "pass@8": 0.3390466063051701, + "pass@16": 0.3793374554178496, + "pass@32": 0.4127509001659112, + "pass@64": 0.4405737754821389, + "pass@128": 0.4685714285714286 + }, + "19": { + "pass@1": 0.14318224474474475, + "pass@2": 0.20463980220377853, + "pass@4": 0.2685763572909241, + "pass@8": 0.3284336355160867, + "pass@16": 0.3783702554316468, + "pass@32": 0.4147896381558507, + "pass@64": 0.4437101594568495, + "pass@128": 0.4744744744744745 + }, + "20": { + "pass@1": 0.15111204954954954, + "pass@2": 0.21561509836608267, + "pass@4": 0.281267438698541, + "pass@8": 0.3375878425646831, + "pass@16": 0.381100671812623, + "pass@32": 0.4144019892616114, + "pass@64": 0.4406480739566761, + "pass@128": 0.4624624624624625 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 531860, + "answer_accuracy": 0.647725058456742, + "avg_response_len": 256.26843944855807, + "resp_tokens_sum": 210427141, + "pass_at_k": { + "pass@1": 0.647725058456742, + "pass@2": 0.6831084180777097, + "pass@4": 0.7134677922128366, + "pass@8": 0.7397314767573606, + "pass@16": 0.7620730851567378, + "pass@32": 0.7808543069415151, + "pass@64": 0.7969666514692069, + "pass@128": 0.8116913484021824 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9004026073619632, + "pass@2": 0.9242703402009566, + "pass@4": 0.9396745401073328, + "pass@8": 0.9505904177376397, + "pass@16": 0.9586655869893964, + "pass@32": 0.96514230835694, + "pass@64": 0.9707695920724272, + "pass@128": 0.9754601226993865 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999075443786982, + "pass@2": 0.9999992720029819, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9834692028985508, + "pass@2": 0.9852240956293506, + "pass@4": 0.9858998532792097, + "pass@8": 0.986317596894847, + "pass@16": 0.9869360286784715, + "pass@32": 0.9877361308097358, + "pass@64": 0.988322264064818, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.9928142908309455, + "pass@2": 0.9974357275003949, + "pass@4": 0.999494777783866, + "pass@8": 0.9999737827585785, + "pass@16": 0.9999999348964089, + "pass@32": 0.9999999999999439, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9855491329479769, + "pass@2": 0.9887166605980608, + "pass@4": 0.9913939513883019, + "pass@8": 0.9943285343504065, + "pass@16": 0.9968946491921471, + "pass@32": 0.9987241613729214, + "pass@64": 0.999808867257577, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9807658450704225, + "pass@2": 0.9904752827991573, + "pass@4": 0.9946537202215919, + "pass@8": 0.9965723900602302, + "pass@16": 0.9974438326135088, + "pass@32": 0.9978865571381552, + "pass@64": 0.9985915492939871, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9550196482035929, + "pass@2": 0.9680099692347592, + "pass@4": 0.973621248223463, + "pass@8": 0.9772931092950905, + "pass@16": 0.9797760833989643, + "pass@32": 0.9813793644561303, + "pass@64": 0.9820069826621478, + "pass@128": 0.9820359281437125 + }, + "11": { + "pass@1": 0.8364551957831325, + "pass@2": 0.8805564284697844, + "pass@4": 0.9119869102393826, + "pass@8": 0.9332457070554272, + "pass@16": 0.9464105210220005, + "pass@32": 0.9554480120240839, + "pass@64": 0.9620750241307124, + "pass@128": 0.9668674698795181 + }, + "12": { + "pass@1": 0.7179907915360502, + "pass@2": 0.7807096641818678, + "pass@4": 0.8237406890557176, + "pass@8": 0.8544050832000054, + "pass@16": 0.8745406088754614, + "pass@32": 0.8870839775741599, + "pass@64": 0.8953727964648683, + "pass@128": 0.8996865203761756 + }, + "13": { + "pass@1": 0.4339110644257703, + "pass@2": 0.5064138352191269, + "pass@4": 0.5668354349613859, + "pass@8": 0.6208126022401135, + "pass@16": 0.6694676564972795, + "pass@32": 0.7126055948949221, + "pass@64": 0.7509963818526764, + "pass@128": 0.7843137254901961 + }, + "14": { + "pass@1": 0.39722679093567254, + "pass@2": 0.46368376444720705, + "pass@4": 0.5167455819667276, + "pass@8": 0.56171751122582, + "pass@16": 0.6024369898411556, + "pass@32": 0.6426240615167135, + "pass@64": 0.6832214593635492, + "pass@128": 0.7222222222222222 + }, + "15": { + "pass@1": 0.24644886363636365, + "pass@2": 0.32176874552612744, + "pass@4": 0.3923603217211486, + "pass@8": 0.4581419440373477, + "pass@16": 0.5190329686132177, + "pass@32": 0.5727951896253346, + "pass@64": 0.6198279692563144, + "pass@128": 0.6636363636363637 + }, + "16": { + "pass@1": 0.2040820868945869, + "pass@2": 0.26635372501514243, + "pass@4": 0.3238963121062858, + "pass@8": 0.37332193265107516, + "pass@16": 0.4134811645828353, + "pass@32": 0.4443577173332517, + "pass@64": 0.47302414732746867, + "pass@128": 0.50997150997151 + }, + "17": { + "pass@1": 0.19650135869565216, + "pass@2": 0.2574690052330416, + "pass@4": 0.3152138539941824, + "pass@8": 0.3709270117297045, + "pass@16": 0.42297098948583206, + "pass@32": 0.4692510903823389, + "pass@64": 0.5108174522501921, + "pass@128": 0.5527950310559007 + }, + "18": { + "pass@1": 0.18683363970588235, + "pass@2": 0.24651024779990735, + "pass@4": 0.3035248188645537, + "pass@8": 0.3556424980931623, + "pass@16": 0.4017419832141523, + "pass@32": 0.43944869520791696, + "pass@64": 0.4673817173155303, + "pass@128": 0.49117647058823527 + }, + "19": { + "pass@1": 0.14808872767857142, + "pass@2": 0.2049320250984251, + "pass@4": 0.2627956976918065, + "pass@8": 0.3139157657154089, + "pass@16": 0.3560561707464758, + "pass@32": 0.39169649693737485, + "pass@64": 0.4226012391904779, + "pass@128": 0.44642857142857145 + }, + "20": { + "pass@1": 0.13087107035928144, + "pass@2": 0.1879670776557122, + "pass@4": 0.24696301351178396, + "pass@8": 0.30177241862067616, + "pass@16": 0.34838594581032317, + "pass@32": 0.38534870948871147, + "pass@64": 0.4129125683339026, + "pass@128": 0.4341317365269461 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 523941, + "answer_accuracy": 0.6539845123022847, + "avg_response_len": 275.54656045294774, + "resp_tokens_sum": 220754678, + "pass_at_k": { + "pass@1": 0.6539845123022847, + "pass@2": 0.689856626143393, + "pass@4": 0.7200727996005184, + "pass@8": 0.7453208731590747, + "pass@16": 0.7665777414461511, + "pass@32": 0.7847634494980638, + "pass@64": 0.8007918914069807, + "pass@128": 0.8149864195558396 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9029617537313432, + "pass@2": 0.9262189299565168, + "pass@4": 0.9432951989210303, + "pass@8": 0.9559143658871644, + "pass@16": 0.9651937507145258, + "pass@32": 0.9716208086596739, + "pass@64": 0.9761076892137535, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 0.9999107142857143, + "pass@2": 0.9999989454443194, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9991116982507289, + "pass@2": 0.9997976974816924, + "pass@4": 0.9999873259282823, + "pass@8": 0.9999999629758403, + "pass@16": 0.9999999999999313, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999314692982456, + "pass@2": 0.9999996402587834, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9659926470588235, + "pass@2": 0.9708096919870312, + "pass@4": 0.9745823887822845, + "pass@8": 0.9771404920721685, + "pass@16": 0.9789554066534717, + "pass@32": 0.9809376766950135, + "pass@64": 0.9837387679474603, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9929268533123028, + "pass@2": 0.9952720199210117, + "pass@4": 0.9963477722816194, + "pass@8": 0.996781255357137, + "pass@16": 0.996844514797445, + "pass@32": 0.9968454258207375, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9869956487341772, + "pass@2": 0.9934614210605002, + "pass@4": 0.9959656588614716, + "pass@8": 0.9971545915353225, + "pass@16": 0.9978742203032149, + "pass@32": 0.9986754832496939, + "pass@64": 0.9996137745440047, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9711195570570571, + "pass@2": 0.9796935222624201, + "pass@4": 0.9847409628413564, + "pass@8": 0.9876525069319597, + "pass@16": 0.9891184230382688, + "pass@32": 0.9900539597715055, + "pass@64": 0.9908121349376716, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.9665393890675241, + "pass@2": 0.9782261943945113, + "pass@4": 0.9827812531271213, + "pass@8": 0.9852322346734745, + "pass@16": 0.9870234208375871, + "pass@32": 0.9888026081069605, + "pass@64": 0.9901177403407886, + "pass@128": 0.9903536977491961 + }, + "11": { + "pass@1": 0.8699652777777778, + "pass@2": 0.9050528840144983, + "pass@4": 0.9261437665529902, + "pass@8": 0.9379237916730481, + "pass@16": 0.9436162837580702, + "pass@32": 0.9481522394334617, + "pass@64": 0.9538037350418224, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.6851586391437309, + "pass@2": 0.7609573280117508, + "pass@4": 0.8146087567609088, + "pass@8": 0.8498854256731503, + "pass@16": 0.8726716568222734, + "pass@32": 0.8890453884635856, + "pass@64": 0.9038027299053867, + "pass@128": 0.9204892966360856 + }, + "13": { + "pass@1": 0.48818479938271603, + "pass@2": 0.5643306630941965, + "pass@4": 0.6234116620839059, + "pass@8": 0.670892987256112, + "pass@16": 0.710554304400132, + "pass@32": 0.7441477272144019, + "pass@64": 0.7721765126609678, + "pass@128": 0.7932098765432098 + }, + "14": { + "pass@1": 0.39139002225519287, + "pass@2": 0.4689131901446295, + "pass@4": 0.5358667138974095, + "pass@8": 0.5899477624146936, + "pass@16": 0.6321371774697618, + "pass@32": 0.6666508816034391, + "pass@64": 0.697774235354575, + "pass@128": 0.7270029673590505 + }, + "15": { + "pass@1": 0.29598283179012347, + "pass@2": 0.3657031477836101, + "pass@4": 0.4284062113763555, + "pass@8": 0.488394802453985, + "pass@16": 0.5458289659825917, + "pass@32": 0.5968433317356244, + "pass@64": 0.6421028185366104, + "pass@128": 0.6820987654320988 + }, + "16": { + "pass@1": 0.20973912241887904, + "pass@2": 0.2661744680974612, + "pass@4": 0.32020332027080706, + "pass@8": 0.37020648522530164, + "pass@16": 0.4155200084120026, + "pass@32": 0.4563823340406111, + "pass@64": 0.4954197121915054, + "pass@128": 0.5339233038348082 + }, + "17": { + "pass@1": 0.1867940512048193, + "pass@2": 0.2508834550801631, + "pass@4": 0.3094012797534344, + "pass@8": 0.3578539351904199, + "pass@16": 0.3979111457071782, + "pass@32": 0.43246884560924453, + "pass@64": 0.46455603309265586, + "pass@128": 0.49698795180722893 + }, + "18": { + "pass@1": 0.17842741935483872, + "pass@2": 0.2330915036830072, + "pass@4": 0.286078386068677, + "pass@8": 0.3361083917487688, + "pass@16": 0.3821960914627827, + "pass@32": 0.422830010224237, + "pass@64": 0.45738747568942023, + "pass@128": 0.4870967741935484 + }, + "19": { + "pass@1": 0.1512698262839879, + "pass@2": 0.20847594083783336, + "pass@4": 0.2632785100314122, + "pass@8": 0.31349632397118765, + "pass@16": 0.3602084544258579, + "pass@32": 0.40216106105046995, + "pass@64": 0.4354416751759285, + "pass@128": 0.459214501510574 + }, + "20": { + "pass@1": 0.16378096846846846, + "pass@2": 0.22388222375919223, + "pass@4": 0.2818586584334616, + "pass@8": 0.3335726763870422, + "pass@16": 0.37771145894953373, + "pass@32": 0.41447893995579, + "pass@64": 0.44587055010516763, + "pass@128": 0.47147147147147145 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/autoregressive/checkpoint-22350_metrics.json b/SDAR-100M-ep1-block32/composition/autoregressive/checkpoint-22350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..51dfa94e927950ee173077dbb42ee91a08610c08 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/autoregressive/checkpoint-22350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-22350", + "id": { + "answer_accuracy": 0.8321892361111111, + "avg_response_len": 197.74917100694444, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.8321892361111111, + "pass@2": 0.8881887713254415, + "pass@4": 0.923848789026372, + "pass@8": 0.9478013524644198, + "pass@16": 0.964040721619663, + "pass@32": 0.9752554708484816, + "pass@64": 0.9831959865782305, + "pass@128": 0.9887777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.678078125, + "pass@2": 0.7743121309055128, + "pass@4": 0.8371884178852642, + "pass@8": 0.8790315017338544, + "pass@16": 0.9098392032325715, + "pass@32": 0.9350052143959573, + "pass@64": 0.9560643585445312, + "pass@128": 0.974 + }, + "2": { + "pass@1": 0.9318125, + "pass@2": 0.9521502214566935, + "pass@4": 0.965556017247844, + "pass@8": 0.9755671839352523, + "pass@16": 0.9823858654164263, + "pass@32": 0.9870343999718583, + "pass@64": 0.9904259772483972, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.9588359375, + "pass@2": 0.9810745570866154, + "pass@4": 0.9894684816272957, + "pass@8": 0.9944121465234204, + "pass@16": 0.9972472859419731, + "pass@32": 0.9986524460939028, + "pass@64": 0.9994958847802009, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9255625, + "pass@2": 0.9550111958661429, + "pass@4": 0.9725522872140981, + "pass@8": 0.9839967129856895, + "pass@16": 0.9908376729123604, + "pass@32": 0.995221063782237, + "pass@64": 0.9984076897426669, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9223203125, + "pass@2": 0.9476866387795281, + "pass@4": 0.9635705636482935, + "pass@8": 0.9747443104371549, + "pass@16": 0.9822307661274378, + "pass@32": 0.9871895658524094, + "pass@64": 0.9902528606185678, + "pass@128": 0.993 + }, + "6": { + "pass@1": 0.8008984375, + "pass@2": 0.8647743602362217, + "pass@4": 0.9083693935133111, + "pass@8": 0.9381201621381264, + "pass@16": 0.9575599006895994, + "pass@32": 0.9688325339317648, + "pass@64": 0.9757075538720211, + "pass@128": 0.981 + }, + "7": { + "pass@1": 0.8202421875, + "pass@2": 0.8798455954724419, + "pass@4": 0.9180830775215603, + "pass@8": 0.944209604313889, + "pass@16": 0.9617111477217547, + "pass@32": 0.9732856240137526, + "pass@64": 0.9813414452529328, + "pass@128": 0.987 + }, + "8": { + "pass@1": 0.7606328125, + "pass@2": 0.8447492618110243, + "pass@4": 0.898190818803898, + "pass@8": 0.9336867536221927, + "pass@16": 0.9575478823734268, + "pass@32": 0.9735957552542194, + "pass@64": 0.9842146079979369, + "pass@128": 0.991 + }, + "9": { + "pass@1": 0.6913203125, + "pass@2": 0.7940949803149603, + "pass@4": 0.8616600437757783, + "pass@8": 0.9064437964902416, + "pass@16": 0.9370067701613884, + "pass@32": 0.9584826343402005, + "pass@64": 0.9728535011467876, + "pass@128": 0.98 + } + }, + "per_op_accuracy": { + "10": 0.678078125, + "2": 0.9318125, + "3": 0.9588359375, + "4": 0.9255625, + "5": 0.9223203125, + "6": 0.8008984375, + "7": 0.8202421875, + "8": 0.7606328125, + "9": 0.6913203125 + }, + "per_op_avg_response_len": { + "10": 272.82728125, + "2": 120.07925, + "3": 147.4028046875, + "4": 160.8676171875, + "5": 173.9177421875, + "6": 196.157046875, + "7": 218.3181015625, + "8": 238.53146875, + "9": 251.6412265625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.82728125, + "2": 120.07925, + "3": 147.4028046875, + "4": 160.8676171875, + "5": 173.9177421875, + "6": 196.157046875, + "7": 218.3181015625, + "8": 238.53146875, + "9": 251.6412265625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 319157, + "answer_accuracy": 0.8420851274907126, + "avg_response_len": 188.7386176545086, + "resp_tokens_sum": 71533446, + "pass_at_k": { + "pass@1": 0.8420851274907126, + "pass@2": 0.8942759731496374, + "pass@4": 0.927962050842937, + "pass@8": 0.9507840577529836, + "pass@16": 0.9664579168964791, + "pass@32": 0.9770732045957066, + "pass@64": 0.9840874601382165, + "pass@128": 0.9888551165146909 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6943906710914455, + "pass@2": 0.7924839732422831, + "pass@4": 0.860622414178315, + "pass@8": 0.9069794084790083, + "pass@16": 0.938909065525345, + "pass@32": 0.961220354064231, + "pass@64": 0.9761939203554444, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 0.9308557242990654, + "pass@2": 0.9428547446464054, + "pass@4": 0.9537001116682842, + "pass@8": 0.9648737739918477, + "pass@16": 0.9749592956005654, + "pass@32": 0.982755395867794, + "pass@64": 0.9887116454415467, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9621081804281345, + "pass@2": 0.9769709118928944, + "pass@4": 0.9848477245963518, + "pass@8": 0.9902888229327741, + "pass@16": 0.9939032445179141, + "pass@32": 0.9963332215254794, + "pass@64": 0.9984785531300948, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9477783203125, + "pass@2": 0.9676934670275597, + "pass@4": 0.9787657553547993, + "pass@8": 0.985568835042546, + "pass@16": 0.9906626007427637, + "pass@32": 0.9951571846591477, + "pass@64": 0.9986095925944035, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9187748015873016, + "pass@2": 0.9491169072615924, + "pass@4": 0.966745741603728, + "pass@8": 0.9769276192432061, + "pass@16": 0.9838399495702144, + "pass@32": 0.9891182703066478, + "pass@64": 0.9919019584676965, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.8201019835329342, + "pass@2": 0.8826146918760904, + "pass@4": 0.9236047901385079, + "pass@8": 0.9489790206766302, + "pass@16": 0.9630538085476542, + "pass@32": 0.9706400767227515, + "pass@64": 0.9751205236185113, + "pass@128": 0.9790419161676647 + }, + "7": { + "pass@1": 0.8473095414201184, + "pass@2": 0.8948160788333407, + "pass@4": 0.9233494061208317, + "pass@8": 0.9446898395536166, + "pass@16": 0.9609689351179754, + "pass@32": 0.9718427623312778, + "pass@64": 0.9791843428978619, + "pass@128": 0.985207100591716 + }, + "8": { + "pass@1": 0.7699068509615384, + "pass@2": 0.8474366072329903, + "pass@4": 0.8956249834756224, + "pass@8": 0.9289230761890581, + "pass@16": 0.952910327741177, + "pass@32": 0.9687048044336607, + "pass@64": 0.9776952048260006, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.7081205985915493, + "pass@2": 0.8081637462570703, + "pass@4": 0.873297600035911, + "pass@8": 0.9156458120013241, + "pass@16": 0.9428910229292787, + "pass@32": 0.9607310443761353, + "pass@64": 0.9728863441329083, + "pass@128": 0.9802816901408451 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 323124, + "answer_accuracy": 0.8271317988204456, + "avg_response_len": 195.94932370167103, + "resp_tokens_sum": 76548779, + "pass_at_k": { + "pass@1": 0.8271317988204456, + "pass@2": 0.8838586305223931, + "pass@4": 0.9191253907399869, + "pass@8": 0.9430054197669399, + "pass@16": 0.9596702709340951, + "pass@32": 0.9716613444688003, + "pass@64": 0.9805587584769488, + "pass@128": 0.9868938401048493 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6537576687116564, + "pass@2": 0.7460161948698127, + "pass@4": 0.8060986609802601, + "pass@8": 0.847945424633065, + "pass@16": 0.8813404119372923, + "pass@32": 0.9103958247721612, + "pass@64": 0.9355294903589877, + "pass@128": 0.9570552147239264 + }, + "2": { + "pass@1": 0.9330357142857143, + "pass@2": 0.9584501322308118, + "pass@4": 0.9728141189798079, + "pass@8": 0.9825223285201696, + "pass@16": 0.988427837610654, + "pass@32": 0.9919575137068457, + "pass@64": 0.9936774918251292, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9535274621212121, + "pass@2": 0.9845360594130279, + "pass@4": 0.9929094837577115, + "pass@8": 0.9964002893104172, + "pass@16": 0.9984903628634235, + "pass@32": 0.9996016193444951, + "pass@64": 0.9999800782817796, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9252727440828402, + "pass@2": 0.9541336398686112, + "pass@4": 0.9712754627010378, + "pass@8": 0.9821602965278723, + "pass@16": 0.9889760898065934, + "pass@32": 0.99406958704192, + "pass@64": 0.9982805345757635, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9472373188405797, + "pass@2": 0.9669444682186469, + "pass@4": 0.9776280263880057, + "pass@8": 0.9841109345296649, + "pass@16": 0.9871814520804815, + "pass@32": 0.9891453339567674, + "pass@64": 0.991220735025819, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.7716690544412608, + "pass@2": 0.8399441317149118, + "pass@4": 0.88877262772311, + "pass@8": 0.9236277234039011, + "pass@16": 0.947779342166902, + "pass@32": 0.9623149622458524, + "pass@64": 0.9719698435364026, + "pass@128": 0.9799426934097422 + }, + "7": { + "pass@1": 0.8232252528901735, + "pass@2": 0.8818197140548907, + "pass@4": 0.9189946032757454, + "pass@8": 0.9443311493694878, + "pass@16": 0.9613068615394603, + "pass@32": 0.973776344644011, + "pass@64": 0.9830204417073324, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.7471390845070423, + "pass@2": 0.8324141205500724, + "pass@4": 0.8866519479079198, + "pass@8": 0.9229622296115595, + "pass@16": 0.9482430767490266, + "pass@32": 0.9668919515308674, + "pass@64": 0.9814110244645726, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.6907980913173652, + "pass@2": 0.7911919762836525, + "pass@4": 0.8568220283280452, + "pass@8": 0.9022026761726983, + "pass@16": 0.9342429287522777, + "pass@32": 0.9557694238038187, + "pass@64": 0.968969940690975, + "pass@128": 0.9760479041916168 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 316401, + "answer_accuracy": 0.827546974389019, + "avg_response_len": 208.5203067458989, + "resp_tokens_sum": 79724820, + "pass_at_k": { + "pass@1": 0.827546974389019, + "pass@2": 0.8865789236956, + "pass@4": 0.9245975146143233, + "pass@8": 0.9497449066101576, + "pass@16": 0.9661101224491581, + "pass@32": 0.9771258973919089, + "pass@64": 0.9850068894754354, + "pass@128": 0.9906260462002009 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6852378731343284, + "pass@2": 0.7834590580561754, + "pass@4": 0.8437291223858209, + "pass@8": 0.8810008174002717, + "pass@16": 0.9081553902326077, + "pass@32": 0.932425299977489, + "pass@64": 0.9556776289164318, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 0.9315401785714286, + "pass@2": 0.954753585489314, + "pass@4": 0.9696069607370507, + "pass@8": 0.9788367325735242, + "pass@16": 0.9835176370135428, + "pass@32": 0.9863311311106975, + "pass@64": 0.9889418407176944, + "pass@128": 0.9914285714285714 + }, + "3": { + "pass@1": 0.9608236151603499, + "pass@2": 0.9816564702371388, + "pass@4": 0.9905631080007128, + "pass@8": 0.996430338052377, + "pass@16": 0.9992393738766351, + "pass@32": 0.9999502865055028, + "pass@64": 0.9999999185821354, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9050621345029239, + "pass@2": 0.9440120355021416, + "pass@4": 0.9680004067474015, + "pass@8": 0.984340659490216, + "pass@16": 0.9928412933334729, + "pass@32": 0.9964188428980713, + "pass@64": 0.9983444427656412, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9003216911764705, + "pass@2": 0.9268205042843912, + "pass@4": 0.9463646645272281, + "pass@8": 0.9632171704788541, + "pass@16": 0.9757164148383949, + "pass@32": 0.9834181485021496, + "pass@64": 0.9877429121098109, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.8128450315457413, + "pass@2": 0.8733140415807641, + "pass@4": 0.9138919448948998, + "pass@8": 0.9426343651866598, + "pass@16": 0.9625392372821305, + "pass@32": 0.9741035535727569, + "pass@64": 0.9804410838777078, + "pass@128": 0.9842271293375394 + }, + "7": { + "pass@1": 0.7880241297468354, + "pass@2": 0.8616713283663908, + "pass@4": 0.9114520427826266, + "pass@8": 0.9435628508320384, + "pass@16": 0.9629477011367917, + "pass@32": 0.9742916300599795, + "pass@64": 0.9818103307680968, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.7663288288288288, + "pass@2": 0.8553814641413071, + "pass@4": 0.9128960433999799, + "pass@8": 0.9495830700873962, + "pass@16": 0.9718125161329592, + "pass@32": 0.9853249653977751, + "pass@64": 0.9933116829048084, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.6727039790996785, + "pass@2": 0.7811534736815446, + "pass@4": 0.8535718273358913, + "pass@8": 0.9004946604118668, + "pass@16": 0.9332582598657084, + "pass@32": 0.9588299872546514, + "pass@64": 0.9769867806714441, + "pass@128": 0.9839228295819936 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.2584078125, + "avg_response_len": 304.17006484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2584078125, + "pass@2": 0.3477248892716594, + "pass@4": 0.4317873421072397, + "pass@8": 0.5051253797512297, + "pass@16": 0.566952443833158, + "pass@32": 0.618664893153937, + "pass@64": 0.6623548267258346, + "pass@128": 0.7002 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5618359375, + "pass@2": 0.6782918307086615, + "pass@4": 0.7654492692163484, + "pass@8": 0.8305453862936757, + "pass@16": 0.8790624298470032, + "pass@32": 0.9140260071161761, + "pass@64": 0.9386822610406704, + "pass@128": 0.956 + }, + "12": { + "pass@1": 0.46409375, + "pass@2": 0.5919548474409453, + "pass@4": 0.6973053483314605, + "pass@8": 0.7777032936045966, + "pass@16": 0.8362024885850111, + "pass@32": 0.8768584121841908, + "pass@64": 0.9059551505827929, + "pass@128": 0.931 + }, + "13": { + "pass@1": 0.3252265625, + "pass@2": 0.44007984744094475, + "pass@4": 0.5439967534683158, + "pass@8": 0.6302635003682064, + "pass@16": 0.7008982680987459, + "pass@32": 0.7591684187849655, + "pass@64": 0.8058926164145022, + "pass@128": 0.844 + }, + "14": { + "pass@1": 0.23825, + "pass@2": 0.33163422736220477, + "pass@4": 0.4239824708473937, + "pass@8": 0.5092990947839865, + "pass@16": 0.5861418342294777, + "pass@32": 0.6532094978929414, + "pass@64": 0.7108122471147346, + "pass@128": 0.759 + }, + "15": { + "pass@1": 0.174, + "pass@2": 0.256453001968504, + "pass@4": 0.34635185723659506, + "pass@8": 0.4333355323854744, + "pass@16": 0.5122020216484495, + "pass@32": 0.5826284225511319, + "pass@64": 0.6439820764003331, + "pass@128": 0.698 + }, + "16": { + "pass@1": 0.1874296875, + "pass@2": 0.26266769192913386, + "pass@4": 0.3367808257405326, + "pass@8": 0.40367480026754987, + "pass@16": 0.46207081755192175, + "pass@32": 0.5140916236398034, + "pass@64": 0.5627495455269993, + "pass@128": 0.607 + }, + "17": { + "pass@1": 0.1755625, + "pass@2": 0.25273708169291337, + "pass@4": 0.330940113329584, + "pass@8": 0.40066365106016755, + "pass@16": 0.45929627635289516, + "pass@32": 0.5092423424026581, + "pass@64": 0.5550927068402005, + "pass@128": 0.599 + }, + "18": { + "pass@1": 0.1615078125, + "pass@2": 0.2326395177165354, + "pass@4": 0.30590347506561655, + "pass@8": 0.3727982418756291, + "pass@16": 0.4298899429089121, + "pass@32": 0.4802052020564647, + "pass@64": 0.52596283656325, + "pass@128": 0.565 + }, + "19": { + "pass@1": 0.1524765625, + "pass@2": 0.2192069389763778, + "pass@4": 0.28596678871391046, + "pass@8": 0.3480251570933633, + "pass@16": 0.4044770759605301, + "pass@32": 0.45336777416097945, + "pass@64": 0.49190347904041687, + "pass@128": 0.522 + }, + "20": { + "pass@1": 0.1436953125, + "pass@2": 0.21158390748031486, + "pass@4": 0.2811965191226094, + "pass@8": 0.3449451397795559, + "pass@16": 0.3992832831484958, + "pass@32": 0.44385123075006017, + "pass@64": 0.4825153477344088, + "pass@128": 0.521 + } + }, + "per_op_accuracy": { + "11": 0.5618359375, + "12": 0.46409375, + "13": 0.3252265625, + "14": 0.23825, + "15": 0.174, + "16": 0.1874296875, + "17": 0.1755625, + "18": 0.1615078125, + "19": 0.1524765625, + "20": 0.1436953125 + }, + "per_op_avg_response_len": { + "11": 284.345109375, + "12": 292.093796875, + "13": 298.82365625, + "14": 304.67215625, + "15": 304.8195859375, + "16": 307.62365625, + "17": 314.1466328125, + "18": 311.1063359375, + "19": 311.447765625, + "20": 312.621953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 284.345109375, + "12": 292.093796875, + "13": 298.82365625, + "14": 304.67215625, + "15": 304.8195859375, + "16": 307.62365625, + "17": 314.1466328125, + "18": 311.1063359375, + "19": 311.447765625, + "20": 312.621953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 114722, + "answer_accuracy": 0.26634936849925706, + "avg_response_len": 291.31425520059435, + "resp_tokens_sum": 125474876, + "pass_at_k": { + "pass@1": 0.26634936849925706, + "pass@2": 0.35535437458319263, + "pass@4": 0.43819264888508686, + "pass@8": 0.5095144030113675, + "pass@16": 0.5689537497874675, + "pass@32": 0.6182117448887823, + "pass@64": 0.6590555088964505, + "pass@128": 0.6945022288261515 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5862473441926346, + "pass@2": 0.6982166358100419, + "pass@4": 0.7848791272726872, + "pass@8": 0.8518025967130893, + "pass@16": 0.8993367128402658, + "pass@32": 0.9296627844085906, + "pass@64": 0.9481568861295607, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.4626588983050847, + "pass@2": 0.5915618466123937, + "pass@4": 0.7009537414814668, + "pass@8": 0.7852130266152862, + "pass@16": 0.8448068180092854, + "pass@32": 0.8845265457694631, + "pass@64": 0.9105669496967504, + "pass@128": 0.9293785310734464 + }, + "13": { + "pass@1": 0.359203565830721, + "pass@2": 0.47705713289561363, + "pass@4": 0.5781522086384967, + "pass@8": 0.6589915053523162, + "pass@16": 0.7248504805274102, + "pass@32": 0.7792684090834485, + "pass@64": 0.8205216396970003, + "pass@128": 0.8495297805642633 + }, + "14": { + "pass@1": 0.23035922897196262, + "pass@2": 0.3267222109794686, + "pass@4": 0.42098008076093296, + "pass@8": 0.5040391320187068, + "pass@16": 0.5757914308258796, + "pass@32": 0.6382070925379484, + "pass@64": 0.6902304607748279, + "pass@128": 0.7320872274143302 + }, + "15": { + "pass@1": 0.1915643063583815, + "pass@2": 0.2748797420463337, + "pass@4": 0.35926651934042914, + "pass@8": 0.4369129733469511, + "pass@16": 0.5078082628147159, + "pass@32": 0.5739481885721509, + "pass@64": 0.6338527371652254, + "pass@128": 0.6878612716763006 + }, + "16": { + "pass@1": 0.1787046370967742, + "pass@2": 0.25333494729489453, + "pass@4": 0.3314841881659954, + "pass@8": 0.4064332975106817, + "pass@16": 0.47415692528401443, + "pass@32": 0.5324868128405753, + "pass@64": 0.5803103772512649, + "pass@128": 0.6161290322580645 + }, + "17": { + "pass@1": 0.16708815028901733, + "pass@2": 0.24038220836557264, + "pass@4": 0.3126962039788378, + "pass@8": 0.37412953912282093, + "pass@16": 0.4247942184063712, + "pass@32": 0.4686977316823931, + "pass@64": 0.5104069609249928, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.15174107142857143, + "pass@2": 0.21740122328458925, + "pass@4": 0.2878790993090149, + "pass@8": 0.3559641056655266, + "pass@16": 0.41501722330270596, + "pass@32": 0.46579089458779094, + "pass@64": 0.5139033629581451, + "pass@128": 0.5628571428571428 + }, + "19": { + "pass@1": 0.1649540165165165, + "pass@2": 0.23666493757537074, + "pass@4": 0.30362395044087953, + "pass@8": 0.36108731681792244, + "pass@16": 0.41238462464001546, + "pass@32": 0.4586458618410122, + "pass@64": 0.49736060514722336, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.14857826576576577, + "pass@2": 0.2135457307701402, + "pass@4": 0.2781666238792225, + "pass@8": 0.3375520764569661, + "pass@16": 0.38947846469059344, + "pass@32": 0.4322996299171272, + "pass@64": 0.4686906739688465, + "pass@128": 0.5045045045045045 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 108109, + "answer_accuracy": 0.25114527579542073, + "avg_response_len": 300.70828687184064, + "resp_tokens_sum": 129444092, + "pass_at_k": { + "pass@1": 0.25114527579542073, + "pass@2": 0.3405292673746968, + "pass@4": 0.4258428828477187, + "pass@8": 0.5006199635217935, + "pass@16": 0.5636703070678595, + "pass@32": 0.6166528670027084, + "pass@64": 0.6625662238245791, + "pass@128": 0.7029438001784122 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5349444653614458, + "pass@2": 0.6472616598282896, + "pass@4": 0.7313714308866506, + "pass@8": 0.7953237183465092, + "pass@16": 0.8460691697512974, + "pass@32": 0.8874826145843286, + "pass@64": 0.9217594495766532, + "pass@128": 0.9487951807228916 + }, + "12": { + "pass@1": 0.4509943181818182, + "pass@2": 0.5793468300298671, + "pass@4": 0.6863166373403949, + "pass@8": 0.7673792999388869, + "pass@16": 0.8274085956239333, + "pass@32": 0.8702289262058087, + "pass@64": 0.901284376599011, + "pass@128": 0.9310344827586207 + }, + "13": { + "pass@1": 0.29072566526610644, + "pass@2": 0.40266313218200644, + "pass@4": 0.5093617699048123, + "pass@8": 0.5996578531299764, + "pass@16": 0.6733180939052346, + "pass@32": 0.7334280975334988, + "pass@64": 0.7814138765854218, + "pass@128": 0.8235294117647058 + }, + "14": { + "pass@1": 0.2385325292397661, + "pass@2": 0.32814910266150954, + "pass@4": 0.4164776088844155, + "pass@8": 0.5006383588810326, + "pass@16": 0.5789538739078068, + "pass@32": 0.6484920703430255, + "pass@64": 0.7104199418336355, + "pass@128": 0.7631578947368421 + }, + "15": { + "pass@1": 0.1773910984848485, + "pass@2": 0.26216893342877584, + "pass@4": 0.3520031467657451, + "pass@8": 0.43594597119352674, + "pass@16": 0.510650496174987, + "pass@32": 0.5791911558699068, + "pass@64": 0.6454691653281225, + "pass@128": 0.7090909090909091 + }, + "16": { + "pass@1": 0.18540776353276353, + "pass@2": 0.26307989546178523, + "pass@4": 0.33983510180885507, + "pass@8": 0.40605426855918836, + "pass@16": 0.45938682436814693, + "pass@32": 0.5041509463348236, + "pass@64": 0.5477951643139377, + "pass@128": 0.5925925925925926 + }, + "17": { + "pass@1": 0.19257084627329193, + "pass@2": 0.2739552257054825, + "pass@4": 0.356384882402122, + "pass@8": 0.4314625004676136, + "pass@16": 0.49515481662757777, + "pass@32": 0.5487970243528865, + "pass@64": 0.5973562474326223, + "pass@128": 0.639751552795031 + }, + "18": { + "pass@1": 0.16482077205882353, + "pass@2": 0.23924559981472898, + "pass@4": 0.31721665040031777, + "pass@8": 0.3896711923876134, + "pass@16": 0.4539420417330226, + "pass@32": 0.5117977902329494, + "pass@64": 0.5613726317248905, + "pass@128": 0.5941176470588235 + }, + "19": { + "pass@1": 0.14569382440476192, + "pass@2": 0.21112570889576301, + "pass@4": 0.27935337240211033, + "pass@8": 0.34295164313173365, + "pass@16": 0.3982470420257465, + "pass@32": 0.4451936526320672, + "pass@64": 0.4832365573017363, + "pass@128": 0.5119047619047619 + }, + "20": { + "pass@1": 0.14116298652694612, + "pass@2": 0.210082541609694, + "pass@4": 0.28193506883870056, + "pass@8": 0.34853051627177145, + "pass@16": 0.4043639567430765, + "pass@32": 0.4477221435505894, + "pass@64": 0.48468313273594765, + "pass@128": 0.5239520958083832 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 107931, + "answer_accuracy": 0.25770505424816625, + "avg_response_len": 320.94933097111857, + "resp_tokens_sum": 134418715, + "pass_at_k": { + "pass@1": 0.25770505424816625, + "pass@2": 0.34727429586277253, + "pass@4": 0.43130976239522817, + "pass@8": 0.505242327033973, + "pass@16": 0.5682676734802553, + "pass@32": 0.6211989053357201, + "pass@64": 0.6655306445903684, + "pass@128": 0.703239608801956 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5628224206349206, + "pass@2": 0.6886682133483313, + "pass@4": 0.7795924515387953, + "pass@8": 0.843846333850522, + "pass@16": 0.8911162090697101, + "pass@32": 0.92447878151729, + "pass@64": 0.9459007713570995, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.47842603211009177, + "pass@2": 0.6046798622649233, + "pass@4": 0.7040755857352686, + "pass@8": 0.7796449097929059, + "pass@16": 0.8354664617911026, + "pass@32": 0.8750244266732382, + "pass@64": 0.9055190650001771, + "pass@128": 0.9327217125382263 + }, + "13": { + "pass@1": 0.32978877314814814, + "pass@2": 0.4449008761057645, + "pass@4": 0.5485310094340058, + "pass@8": 0.6357017178809099, + "pass@16": 0.7077049545874511, + "pass@32": 0.7677406341601419, + "pass@64": 0.8184612944758107, + "pass@128": 0.8611111111111112 + }, + "14": { + "pass@1": 0.24547941394658754, + "pass@2": 0.33984986506694087, + "pass@4": 0.4344585242868378, + "pass@8": 0.5230985598476801, + "pass@16": 0.6032954304388765, + "pass@32": 0.6722870419315873, + "pass@64": 0.7308149824893481, + "pass@128": 0.7804154302670623 + }, + "15": { + "pass@1": 0.1517891589506173, + "pass@2": 0.23095328144745775, + "pass@4": 0.32680433059293523, + "pass@8": 0.4268563984986593, + "pass@16": 0.5184743795580643, + "pass@32": 0.5953989749015376, + "pass@64": 0.6532845826632232, + "pass@128": 0.6975308641975309 + }, + "16": { + "pass@1": 0.19750184365781712, + "pass@2": 0.2707752653705895, + "pass@4": 0.3384619665904593, + "pass@8": 0.39868858340697155, + "pass@16": 0.4537976264326197, + "pass@32": 0.5075626238811257, + "pass@64": 0.5621747076841135, + "pass@128": 0.6135693215339233 + }, + "17": { + "pass@1": 0.16789815512048192, + "pass@2": 0.24503390036524042, + "pass@4": 0.3252749837934414, + "pass@8": 0.39844549811174756, + "pass@16": 0.4604747765970229, + "pass@32": 0.5131333897557869, + "pass@64": 0.5606722490567707, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.16890120967741937, + "pass@2": 0.24259866332232669, + "pass@4": 0.31384557797210855, + "pass@8": 0.37329870800292475, + "pass@16": 0.4203020018185082, + "pass@32": 0.46182948474688756, + "pass@64": 0.5007418217465677, + "pass@128": 0.535483870967742 + }, + "19": { + "pass@1": 0.14680891238670696, + "pass@2": 0.20984675702357458, + "pass@4": 0.27491625404830444, + "pass@8": 0.3400342250777418, + "pass@16": 0.4028458907394394, + "pass@32": 0.45635539239742556, + "pass@64": 0.4952112213686042, + "pass@128": 0.5226586102719033 + }, + "20": { + "pass@1": 0.14135228978978978, + "pass@2": 0.2111279586673286, + "pass@4": 0.2834856467828909, + "pass@8": 0.34874205971355804, + "pass@16": 0.40399217073375576, + "pass@32": 0.45152029441970093, + "pass@64": 0.49416572663356273, + "pass@128": 0.5345345345345346 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5301990131578947, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5301990131578947, + "pass@2": 0.60373409656027, + "pass@4": 0.6648690801215426, + "pass@8": 0.7148139984048492, + "pass@16": 0.7550468912057213, + "pass@32": 0.787576219430319, + "pass@64": 0.814332218234864, + "pass@128": 0.8368947368421052 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.678078125, + "pass@2": 0.7743121309055128, + "pass@4": 0.8371884178852642, + "pass@8": 0.8790315017338544, + "pass@16": 0.9098392032325715, + "pass@32": 0.9350052143959573, + "pass@64": 0.9560643585445312, + "pass@128": 0.974 + }, + "2": { + "pass@1": 0.9318125, + "pass@2": 0.9521502214566935, + "pass@4": 0.965556017247844, + "pass@8": 0.9755671839352523, + "pass@16": 0.9823858654164263, + "pass@32": 0.9870343999718583, + "pass@64": 0.9904259772483972, + "pass@128": 0.993 + }, + "3": { + "pass@1": 0.9588359375, + "pass@2": 0.9810745570866154, + "pass@4": 0.9894684816272957, + "pass@8": 0.9944121465234204, + "pass@16": 0.9972472859419731, + "pass@32": 0.9986524460939028, + "pass@64": 0.9994958847802009, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9255625, + "pass@2": 0.9550111958661429, + "pass@4": 0.9725522872140981, + "pass@8": 0.9839967129856895, + "pass@16": 0.9908376729123604, + "pass@32": 0.995221063782237, + "pass@64": 0.9984076897426669, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9223203125, + "pass@2": 0.9476866387795281, + "pass@4": 0.9635705636482935, + "pass@8": 0.9747443104371549, + "pass@16": 0.9822307661274378, + "pass@32": 0.9871895658524094, + "pass@64": 0.9902528606185678, + "pass@128": 0.993 + }, + "6": { + "pass@1": 0.8008984375, + "pass@2": 0.8647743602362217, + "pass@4": 0.9083693935133111, + "pass@8": 0.9381201621381264, + "pass@16": 0.9575599006895994, + "pass@32": 0.9688325339317648, + "pass@64": 0.9757075538720211, + "pass@128": 0.981 + }, + "7": { + "pass@1": 0.8202421875, + "pass@2": 0.8798455954724419, + "pass@4": 0.9180830775215603, + "pass@8": 0.944209604313889, + "pass@16": 0.9617111477217547, + "pass@32": 0.9732856240137526, + "pass@64": 0.9813414452529328, + "pass@128": 0.987 + }, + "8": { + "pass@1": 0.7606328125, + "pass@2": 0.8447492618110243, + "pass@4": 0.898190818803898, + "pass@8": 0.9336867536221927, + "pass@16": 0.9575478823734268, + "pass@32": 0.9735957552542194, + "pass@64": 0.9842146079979369, + "pass@128": 0.991 + }, + "9": { + "pass@1": 0.6913203125, + "pass@2": 0.7940949803149603, + "pass@4": 0.8616600437757783, + "pass@8": 0.9064437964902416, + "pass@16": 0.9370067701613884, + "pass@32": 0.9584826343402005, + "pass@64": 0.9728535011467876, + "pass@128": 0.98 + }, + "11": { + "pass@1": 0.5618359375, + "pass@2": 0.6782918307086615, + "pass@4": 0.7654492692163484, + "pass@8": 0.8305453862936757, + "pass@16": 0.8790624298470032, + "pass@32": 0.9140260071161761, + "pass@64": 0.9386822610406704, + "pass@128": 0.956 + }, + "12": { + "pass@1": 0.46409375, + "pass@2": 0.5919548474409453, + "pass@4": 0.6973053483314605, + "pass@8": 0.7777032936045966, + "pass@16": 0.8362024885850111, + "pass@32": 0.8768584121841908, + "pass@64": 0.9059551505827929, + "pass@128": 0.931 + }, + "13": { + "pass@1": 0.3252265625, + "pass@2": 0.44007984744094475, + "pass@4": 0.5439967534683158, + "pass@8": 0.6302635003682064, + "pass@16": 0.7008982680987459, + "pass@32": 0.7591684187849655, + "pass@64": 0.8058926164145022, + "pass@128": 0.844 + }, + "14": { + "pass@1": 0.23825, + "pass@2": 0.33163422736220477, + "pass@4": 0.4239824708473937, + "pass@8": 0.5092990947839865, + "pass@16": 0.5861418342294777, + "pass@32": 0.6532094978929414, + "pass@64": 0.7108122471147346, + "pass@128": 0.759 + }, + "15": { + "pass@1": 0.174, + "pass@2": 0.256453001968504, + "pass@4": 0.34635185723659506, + "pass@8": 0.4333355323854744, + "pass@16": 0.5122020216484495, + "pass@32": 0.5826284225511319, + "pass@64": 0.6439820764003331, + "pass@128": 0.698 + }, + "16": { + "pass@1": 0.1874296875, + "pass@2": 0.26266769192913386, + "pass@4": 0.3367808257405326, + "pass@8": 0.40367480026754987, + "pass@16": 0.46207081755192175, + "pass@32": 0.5140916236398034, + "pass@64": 0.5627495455269993, + "pass@128": 0.607 + }, + "17": { + "pass@1": 0.1755625, + "pass@2": 0.25273708169291337, + "pass@4": 0.330940113329584, + "pass@8": 0.40066365106016755, + "pass@16": 0.45929627635289516, + "pass@32": 0.5092423424026581, + "pass@64": 0.5550927068402005, + "pass@128": 0.599 + }, + "18": { + "pass@1": 0.1615078125, + "pass@2": 0.2326395177165354, + "pass@4": 0.30590347506561655, + "pass@8": 0.3727982418756291, + "pass@16": 0.4298899429089121, + "pass@32": 0.4802052020564647, + "pass@64": 0.52596283656325, + "pass@128": 0.565 + }, + "19": { + "pass@1": 0.1524765625, + "pass@2": 0.2192069389763778, + "pass@4": 0.28596678871391046, + "pass@8": 0.3480251570933633, + "pass@16": 0.4044770759605301, + "pass@32": 0.45336777416097945, + "pass@64": 0.49190347904041687, + "pass@128": 0.522 + }, + "20": { + "pass@1": 0.1436953125, + "pass@2": 0.21158390748031486, + "pass@4": 0.2811965191226094, + "pass@8": 0.3449451397795559, + "pass@16": 0.3992832831484958, + "pass@32": 0.44385123075006017, + "pass@64": 0.4825153477344088, + "pass@128": 0.521 + } + }, + "per_op_accuracy": { + "10": 0.678078125, + "2": 0.9318125, + "3": 0.9588359375, + "4": 0.9255625, + "5": 0.9223203125, + "6": 0.8008984375, + "7": 0.8202421875, + "8": 0.7606328125, + "9": 0.6913203125, + "11": 0.5618359375, + "12": 0.46409375, + "13": 0.3252265625, + "14": 0.23825, + "15": 0.174, + "16": 0.1874296875, + "17": 0.1755625, + "18": 0.1615078125, + "19": 0.1524765625, + "20": 0.1436953125 + }, + "per_op_avg_response_len": { + "10": 272.82728125, + "2": 120.07925, + "3": 147.4028046875, + "4": 160.8676171875, + "5": 173.9177421875, + "6": 196.157046875, + "7": 218.3181015625, + "8": 238.53146875, + "9": 251.6412265625, + "11": 284.345109375, + "12": 292.093796875, + "13": 298.82365625, + "14": 304.67215625, + "15": 304.8195859375, + "16": 307.62365625, + "17": 314.1466328125, + "18": 311.1063359375, + "19": 311.447765625, + "20": 312.621953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.82728125, + "2": 120.07925, + "3": 147.4028046875, + "4": 160.8676171875, + "5": 173.9177421875, + "6": 196.157046875, + "7": 218.3181015625, + "8": 238.53146875, + "9": 251.6412265625, + "11": 284.345109375, + "12": 292.093796875, + "13": 298.82365625, + "14": 304.67215625, + "15": 304.8195859375, + "16": 307.62365625, + "17": 314.1466328125, + "18": 311.1063359375, + "19": 311.447765625, + "20": 312.621953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 433879, + "answer_accuracy": 0.5358330204710717, + "avg_response_len": 243.30185197992412, + "resp_tokens_sum": 197008322, + "pass_at_k": { + "pass@1": 0.5358330204710717, + "pass@2": 0.6076064854518745, + "pass@4": 0.6674381751571705, + "pass@8": 0.7160587355579924, + "pass@16": 0.7550128453944618, + "pass@32": 0.78618341453662, + "pass@64": 0.8111926583790442, + "pass@128": 0.8322794815049004 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6943906710914455, + "pass@2": 0.7924839732422831, + "pass@4": 0.860622414178315, + "pass@8": 0.9069794084790083, + "pass@16": 0.938909065525345, + "pass@32": 0.961220354064231, + "pass@64": 0.9761939203554444, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 0.9308557242990654, + "pass@2": 0.9428547446464054, + "pass@4": 0.9537001116682842, + "pass@8": 0.9648737739918477, + "pass@16": 0.9749592956005654, + "pass@32": 0.982755395867794, + "pass@64": 0.9887116454415467, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9621081804281345, + "pass@2": 0.9769709118928944, + "pass@4": 0.9848477245963518, + "pass@8": 0.9902888229327741, + "pass@16": 0.9939032445179141, + "pass@32": 0.9963332215254794, + "pass@64": 0.9984785531300948, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9477783203125, + "pass@2": 0.9676934670275597, + "pass@4": 0.9787657553547993, + "pass@8": 0.985568835042546, + "pass@16": 0.9906626007427637, + "pass@32": 0.9951571846591477, + "pass@64": 0.9986095925944035, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9187748015873016, + "pass@2": 0.9491169072615924, + "pass@4": 0.966745741603728, + "pass@8": 0.9769276192432061, + "pass@16": 0.9838399495702144, + "pass@32": 0.9891182703066478, + "pass@64": 0.9919019584676965, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.8201019835329342, + "pass@2": 0.8826146918760904, + "pass@4": 0.9236047901385079, + "pass@8": 0.9489790206766302, + "pass@16": 0.9630538085476542, + "pass@32": 0.9706400767227515, + "pass@64": 0.9751205236185113, + "pass@128": 0.9790419161676647 + }, + "7": { + "pass@1": 0.8473095414201184, + "pass@2": 0.8948160788333407, + "pass@4": 0.9233494061208317, + "pass@8": 0.9446898395536166, + "pass@16": 0.9609689351179754, + "pass@32": 0.9718427623312778, + "pass@64": 0.9791843428978619, + "pass@128": 0.985207100591716 + }, + "8": { + "pass@1": 0.7699068509615384, + "pass@2": 0.8474366072329903, + "pass@4": 0.8956249834756224, + "pass@8": 0.9289230761890581, + "pass@16": 0.952910327741177, + "pass@32": 0.9687048044336607, + "pass@64": 0.9776952048260006, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.7081205985915493, + "pass@2": 0.8081637462570703, + "pass@4": 0.873297600035911, + "pass@8": 0.9156458120013241, + "pass@16": 0.9428910229292787, + "pass@32": 0.9607310443761353, + "pass@64": 0.9728863441329083, + "pass@128": 0.9802816901408451 + }, + "11": { + "pass@1": 0.5862473441926346, + "pass@2": 0.6982166358100419, + "pass@4": 0.7848791272726872, + "pass@8": 0.8518025967130893, + "pass@16": 0.8993367128402658, + "pass@32": 0.9296627844085906, + "pass@64": 0.9481568861295607, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.4626588983050847, + "pass@2": 0.5915618466123937, + "pass@4": 0.7009537414814668, + "pass@8": 0.7852130266152862, + "pass@16": 0.8448068180092854, + "pass@32": 0.8845265457694631, + "pass@64": 0.9105669496967504, + "pass@128": 0.9293785310734464 + }, + "13": { + "pass@1": 0.359203565830721, + "pass@2": 0.47705713289561363, + "pass@4": 0.5781522086384967, + "pass@8": 0.6589915053523162, + "pass@16": 0.7248504805274102, + "pass@32": 0.7792684090834485, + "pass@64": 0.8205216396970003, + "pass@128": 0.8495297805642633 + }, + "14": { + "pass@1": 0.23035922897196262, + "pass@2": 0.3267222109794686, + "pass@4": 0.42098008076093296, + "pass@8": 0.5040391320187068, + "pass@16": 0.5757914308258796, + "pass@32": 0.6382070925379484, + "pass@64": 0.6902304607748279, + "pass@128": 0.7320872274143302 + }, + "15": { + "pass@1": 0.1915643063583815, + "pass@2": 0.2748797420463337, + "pass@4": 0.35926651934042914, + "pass@8": 0.4369129733469511, + "pass@16": 0.5078082628147159, + "pass@32": 0.5739481885721509, + "pass@64": 0.6338527371652254, + "pass@128": 0.6878612716763006 + }, + "16": { + "pass@1": 0.1787046370967742, + "pass@2": 0.25333494729489453, + "pass@4": 0.3314841881659954, + "pass@8": 0.4064332975106817, + "pass@16": 0.47415692528401443, + "pass@32": 0.5324868128405753, + "pass@64": 0.5803103772512649, + "pass@128": 0.6161290322580645 + }, + "17": { + "pass@1": 0.16708815028901733, + "pass@2": 0.24038220836557264, + "pass@4": 0.3126962039788378, + "pass@8": 0.37412953912282093, + "pass@16": 0.4247942184063712, + "pass@32": 0.4686977316823931, + "pass@64": 0.5104069609249928, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.15174107142857143, + "pass@2": 0.21740122328458925, + "pass@4": 0.2878790993090149, + "pass@8": 0.3559641056655266, + "pass@16": 0.41501722330270596, + "pass@32": 0.46579089458779094, + "pass@64": 0.5139033629581451, + "pass@128": 0.5628571428571428 + }, + "19": { + "pass@1": 0.1649540165165165, + "pass@2": 0.23666493757537074, + "pass@4": 0.30362395044087953, + "pass@8": 0.36108731681792244, + "pass@16": 0.41238462464001546, + "pass@32": 0.4586458618410122, + "pass@64": 0.49736060514722336, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.14857826576576577, + "pass@2": 0.2135457307701402, + "pass@4": 0.2781666238792225, + "pass@8": 0.3375520764569661, + "pass@16": 0.38947846469059344, + "pass@32": 0.4322996299171272, + "pass@64": 0.4686906739688465, + "pass@128": 0.5045045045045045 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 431233, + "answer_accuracy": 0.5251765880748246, + "avg_response_len": 250.86816908612627, + "resp_tokens_sum": 205992871, + "pass_at_k": { + "pass@1": 0.5251765880748246, + "pass@2": 0.5990236113071689, + "pass@4": 0.6605269380444797, + "pass@8": 0.7110892406005488, + "pass@16": 0.7520712251847446, + "pass@32": 0.7855516781058292, + "pass@64": 0.8138543322827327, + "pass@128": 0.8380358534684333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6537576687116564, + "pass@2": 0.7460161948698127, + "pass@4": 0.8060986609802601, + "pass@8": 0.847945424633065, + "pass@16": 0.8813404119372923, + "pass@32": 0.9103958247721612, + "pass@64": 0.9355294903589877, + "pass@128": 0.9570552147239264 + }, + "2": { + "pass@1": 0.9330357142857143, + "pass@2": 0.9584501322308118, + "pass@4": 0.9728141189798079, + "pass@8": 0.9825223285201696, + "pass@16": 0.988427837610654, + "pass@32": 0.9919575137068457, + "pass@64": 0.9936774918251292, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9535274621212121, + "pass@2": 0.9845360594130279, + "pass@4": 0.9929094837577115, + "pass@8": 0.9964002893104172, + "pass@16": 0.9984903628634235, + "pass@32": 0.9996016193444951, + "pass@64": 0.9999800782817796, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9252727440828402, + "pass@2": 0.9541336398686112, + "pass@4": 0.9712754627010378, + "pass@8": 0.9821602965278723, + "pass@16": 0.9889760898065934, + "pass@32": 0.99406958704192, + "pass@64": 0.9982805345757635, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9472373188405797, + "pass@2": 0.9669444682186469, + "pass@4": 0.9776280263880057, + "pass@8": 0.9841109345296649, + "pass@16": 0.9871814520804815, + "pass@32": 0.9891453339567674, + "pass@64": 0.991220735025819, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.7716690544412608, + "pass@2": 0.8399441317149118, + "pass@4": 0.88877262772311, + "pass@8": 0.9236277234039011, + "pass@16": 0.947779342166902, + "pass@32": 0.9623149622458524, + "pass@64": 0.9719698435364026, + "pass@128": 0.9799426934097422 + }, + "7": { + "pass@1": 0.8232252528901735, + "pass@2": 0.8818197140548907, + "pass@4": 0.9189946032757454, + "pass@8": 0.9443311493694878, + "pass@16": 0.9613068615394603, + "pass@32": 0.973776344644011, + "pass@64": 0.9830204417073324, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.7471390845070423, + "pass@2": 0.8324141205500724, + "pass@4": 0.8866519479079198, + "pass@8": 0.9229622296115595, + "pass@16": 0.9482430767490266, + "pass@32": 0.9668919515308674, + "pass@64": 0.9814110244645726, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.6907980913173652, + "pass@2": 0.7911919762836525, + "pass@4": 0.8568220283280452, + "pass@8": 0.9022026761726983, + "pass@16": 0.9342429287522777, + "pass@32": 0.9557694238038187, + "pass@64": 0.968969940690975, + "pass@128": 0.9760479041916168 + }, + "11": { + "pass@1": 0.5349444653614458, + "pass@2": 0.6472616598282896, + "pass@4": 0.7313714308866506, + "pass@8": 0.7953237183465092, + "pass@16": 0.8460691697512974, + "pass@32": 0.8874826145843286, + "pass@64": 0.9217594495766532, + "pass@128": 0.9487951807228916 + }, + "12": { + "pass@1": 0.4509943181818182, + "pass@2": 0.5793468300298671, + "pass@4": 0.6863166373403949, + "pass@8": 0.7673792999388869, + "pass@16": 0.8274085956239333, + "pass@32": 0.8702289262058087, + "pass@64": 0.901284376599011, + "pass@128": 0.9310344827586207 + }, + "13": { + "pass@1": 0.29072566526610644, + "pass@2": 0.40266313218200644, + "pass@4": 0.5093617699048123, + "pass@8": 0.5996578531299764, + "pass@16": 0.6733180939052346, + "pass@32": 0.7334280975334988, + "pass@64": 0.7814138765854218, + "pass@128": 0.8235294117647058 + }, + "14": { + "pass@1": 0.2385325292397661, + "pass@2": 0.32814910266150954, + "pass@4": 0.4164776088844155, + "pass@8": 0.5006383588810326, + "pass@16": 0.5789538739078068, + "pass@32": 0.6484920703430255, + "pass@64": 0.7104199418336355, + "pass@128": 0.7631578947368421 + }, + "15": { + "pass@1": 0.1773910984848485, + "pass@2": 0.26216893342877584, + "pass@4": 0.3520031467657451, + "pass@8": 0.43594597119352674, + "pass@16": 0.510650496174987, + "pass@32": 0.5791911558699068, + "pass@64": 0.6454691653281225, + "pass@128": 0.7090909090909091 + }, + "16": { + "pass@1": 0.18540776353276353, + "pass@2": 0.26307989546178523, + "pass@4": 0.33983510180885507, + "pass@8": 0.40605426855918836, + "pass@16": 0.45938682436814693, + "pass@32": 0.5041509463348236, + "pass@64": 0.5477951643139377, + "pass@128": 0.5925925925925926 + }, + "17": { + "pass@1": 0.19257084627329193, + "pass@2": 0.2739552257054825, + "pass@4": 0.356384882402122, + "pass@8": 0.4314625004676136, + "pass@16": 0.49515481662757777, + "pass@32": 0.5487970243528865, + "pass@64": 0.5973562474326223, + "pass@128": 0.639751552795031 + }, + "18": { + "pass@1": 0.16482077205882353, + "pass@2": 0.23924559981472898, + "pass@4": 0.31721665040031777, + "pass@8": 0.3896711923876134, + "pass@16": 0.4539420417330226, + "pass@32": 0.5117977902329494, + "pass@64": 0.5613726317248905, + "pass@128": 0.5941176470588235 + }, + "19": { + "pass@1": 0.14569382440476192, + "pass@2": 0.21112570889576301, + "pass@4": 0.27935337240211033, + "pass@8": 0.34295164313173365, + "pass@16": 0.3982470420257465, + "pass@32": 0.4451936526320672, + "pass@64": 0.4832365573017363, + "pass@128": 0.5119047619047619 + }, + "20": { + "pass@1": 0.14116298652694612, + "pass@2": 0.210082541609694, + "pass@4": 0.28193506883870056, + "pass@8": 0.34853051627177145, + "pass@16": 0.4043639567430765, + "pass@32": 0.4477221435505894, + "pass@64": 0.48468313273594765, + "pass@128": 0.5239520958083832 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 424332, + "answer_accuracy": 0.5296523006870107, + "avg_response_len": 267.2945146489056, + "resp_tokens_sum": 214143535, + "pass_at_k": { + "pass@1": 0.5296523006870107, + "pass@2": 0.6046481452535212, + "pass@4": 0.6667228500895, + "pass@8": 0.7173735309314182, + "pass@16": 0.758131133309332, + "pass@32": 0.7910589349365911, + "pass@64": 0.8179951826110925, + "pass@128": 0.8403898386323694 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6852378731343284, + "pass@2": 0.7834590580561754, + "pass@4": 0.8437291223858209, + "pass@8": 0.8810008174002717, + "pass@16": 0.9081553902326077, + "pass@32": 0.932425299977489, + "pass@64": 0.9556776289164318, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 0.9315401785714286, + "pass@2": 0.954753585489314, + "pass@4": 0.9696069607370507, + "pass@8": 0.9788367325735242, + "pass@16": 0.9835176370135428, + "pass@32": 0.9863311311106975, + "pass@64": 0.9889418407176944, + "pass@128": 0.9914285714285714 + }, + "3": { + "pass@1": 0.9608236151603499, + "pass@2": 0.9816564702371388, + "pass@4": 0.9905631080007128, + "pass@8": 0.996430338052377, + "pass@16": 0.9992393738766351, + "pass@32": 0.9999502865055028, + "pass@64": 0.9999999185821354, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9050621345029239, + "pass@2": 0.9440120355021416, + "pass@4": 0.9680004067474015, + "pass@8": 0.984340659490216, + "pass@16": 0.9928412933334729, + "pass@32": 0.9964188428980713, + "pass@64": 0.9983444427656412, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9003216911764705, + "pass@2": 0.9268205042843912, + "pass@4": 0.9463646645272281, + "pass@8": 0.9632171704788541, + "pass@16": 0.9757164148383949, + "pass@32": 0.9834181485021496, + "pass@64": 0.9877429121098109, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.8128450315457413, + "pass@2": 0.8733140415807641, + "pass@4": 0.9138919448948998, + "pass@8": 0.9426343651866598, + "pass@16": 0.9625392372821305, + "pass@32": 0.9741035535727569, + "pass@64": 0.9804410838777078, + "pass@128": 0.9842271293375394 + }, + "7": { + "pass@1": 0.7880241297468354, + "pass@2": 0.8616713283663908, + "pass@4": 0.9114520427826266, + "pass@8": 0.9435628508320384, + "pass@16": 0.9629477011367917, + "pass@32": 0.9742916300599795, + "pass@64": 0.9818103307680968, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.7663288288288288, + "pass@2": 0.8553814641413071, + "pass@4": 0.9128960433999799, + "pass@8": 0.9495830700873962, + "pass@16": 0.9718125161329592, + "pass@32": 0.9853249653977751, + "pass@64": 0.9933116829048084, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.6727039790996785, + "pass@2": 0.7811534736815446, + "pass@4": 0.8535718273358913, + "pass@8": 0.9004946604118668, + "pass@16": 0.9332582598657084, + "pass@32": 0.9588299872546514, + "pass@64": 0.9769867806714441, + "pass@128": 0.9839228295819936 + }, + "11": { + "pass@1": 0.5628224206349206, + "pass@2": 0.6886682133483313, + "pass@4": 0.7795924515387953, + "pass@8": 0.843846333850522, + "pass@16": 0.8911162090697101, + "pass@32": 0.92447878151729, + "pass@64": 0.9459007713570995, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.47842603211009177, + "pass@2": 0.6046798622649233, + "pass@4": 0.7040755857352686, + "pass@8": 0.7796449097929059, + "pass@16": 0.8354664617911026, + "pass@32": 0.8750244266732382, + "pass@64": 0.9055190650001771, + "pass@128": 0.9327217125382263 + }, + "13": { + "pass@1": 0.32978877314814814, + "pass@2": 0.4449008761057645, + "pass@4": 0.5485310094340058, + "pass@8": 0.6357017178809099, + "pass@16": 0.7077049545874511, + "pass@32": 0.7677406341601419, + "pass@64": 0.8184612944758107, + "pass@128": 0.8611111111111112 + }, + "14": { + "pass@1": 0.24547941394658754, + "pass@2": 0.33984986506694087, + "pass@4": 0.4344585242868378, + "pass@8": 0.5230985598476801, + "pass@16": 0.6032954304388765, + "pass@32": 0.6722870419315873, + "pass@64": 0.7308149824893481, + "pass@128": 0.7804154302670623 + }, + "15": { + "pass@1": 0.1517891589506173, + "pass@2": 0.23095328144745775, + "pass@4": 0.32680433059293523, + "pass@8": 0.4268563984986593, + "pass@16": 0.5184743795580643, + "pass@32": 0.5953989749015376, + "pass@64": 0.6532845826632232, + "pass@128": 0.6975308641975309 + }, + "16": { + "pass@1": 0.19750184365781712, + "pass@2": 0.2707752653705895, + "pass@4": 0.3384619665904593, + "pass@8": 0.39868858340697155, + "pass@16": 0.4537976264326197, + "pass@32": 0.5075626238811257, + "pass@64": 0.5621747076841135, + "pass@128": 0.6135693215339233 + }, + "17": { + "pass@1": 0.16789815512048192, + "pass@2": 0.24503390036524042, + "pass@4": 0.3252749837934414, + "pass@8": 0.39844549811174756, + "pass@16": 0.4604747765970229, + "pass@32": 0.5131333897557869, + "pass@64": 0.5606722490567707, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.16890120967741937, + "pass@2": 0.24259866332232669, + "pass@4": 0.31384557797210855, + "pass@8": 0.37329870800292475, + "pass@16": 0.4203020018185082, + "pass@32": 0.46182948474688756, + "pass@64": 0.5007418217465677, + "pass@128": 0.535483870967742 + }, + "19": { + "pass@1": 0.14680891238670696, + "pass@2": 0.20984675702357458, + "pass@4": 0.27491625404830444, + "pass@8": 0.3400342250777418, + "pass@16": 0.4028458907394394, + "pass@32": 0.45635539239742556, + "pass@64": 0.4952112213686042, + "pass@128": 0.5226586102719033 + }, + "20": { + "pass@1": 0.14135228978978978, + "pass@2": 0.2111279586673286, + "pass@4": 0.2834856467828909, + "pass@8": 0.34874205971355804, + "pass@16": 0.40399217073375576, + "pass@32": 0.45152029441970093, + "pass@64": 0.49416572663356273, + "pass@128": 0.5345345345345346 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep1-block32/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a735f3a430ef2f336de5c6b6e034ce675aa032fc --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block32/composition/autoregressive", "results_dir": "results/SDAR-100M-ep1-block32/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block32/composition/autoregressive/checkpoint-22350_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 777182, "value_mismatch": 208496, "dependency_mismatch": 297633, "answer_mismatch": 193241, "extra_nodes": 128523, "missing_nodes": 215158, "perfect_with_extra": 5993, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 260920, "value_mismatch": 67024, "dependency_mismatch": 94398, "answer_mismatch": 59818, "missing_nodes": 63720, "extra_nodes": 40811, "perfect_with_extra": 1821}, "teachers_in_school": {"total": 390656, "perfect": 254978, "value_mismatch": 69090, "dependency_mismatch": 101615, "answer_mismatch": 67508, "missing_nodes": 81866, "extra_nodes": 51955, "perfect_with_extra": 2125}, "movie_festival_awards": {"total": 382336, "value_mismatch": 72382, "answer_mismatch": 65915, "dependency_mismatch": 101620, "perfect": 261284, "missing_nodes": 69572, "extra_nodes": 35757, "perfect_with_extra": 2047}}, "results_path": "results/SDAR-100M-ep1-block32/composition/autoregressive/graphs/checkpoint-22350_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block32/composition/autoregressive/checkpoint-22350_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 60305, "value_mismatch": 1100531, "dependency_mismatch": 1156484, "answer_mismatch": 948917, "extra_nodes": 230324, "missing_nodes": 1056593, "perfect_with_extra": 8, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 368942, "dependency_mismatch": 390561, "answer_mismatch": 315901, "missing_nodes": 350438, "extra_nodes": 75113, "perfect": 22445, "perfect_with_extra": 1}, "movie_festival_awards": {"total": 418816, "perfect": 19199, "value_mismatch": 361604, "dependency_mismatch": 379867, "answer_mismatch": 310775, "extra_nodes": 72773, "missing_nodes": 348425, "perfect_with_extra": 2}, "teachers_in_school": {"total": 430464, "perfect": 18661, "value_mismatch": 369985, "dependency_mismatch": 386056, "answer_mismatch": 322241, "extra_nodes": 82438, "missing_nodes": 357730, "perfect_with_extra": 5}}, "results_path": "results/SDAR-100M-ep1-block32/composition/autoregressive/graphs/checkpoint-22350_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 837487, "value_mismatch": 1309027, "dependency_mismatch": 1454117, "answer_mismatch": 1142158, "extra_nodes": 358847, "missing_nodes": 1271751, "perfect_with_extra": 6001, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block32/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 283365, "value_mismatch": 435966, "dependency_mismatch": 484959, "answer_mismatch": 375719, "missing_nodes": 414158, "extra_nodes": 115924, "perfect_with_extra": 1822}, "teachers_in_school": {"total": 821120, "perfect": 273639, "value_mismatch": 439075, "dependency_mismatch": 487671, "answer_mismatch": 389749, "missing_nodes": 439596, "extra_nodes": 134393, "perfect_with_extra": 2130}, "movie_festival_awards": {"total": 801152, "value_mismatch": 433986, "answer_mismatch": 376690, "dependency_mismatch": 481487, "perfect": 280483, "missing_nodes": 417997, "extra_nodes": 108530, "perfect_with_extra": 2049}}}} diff --git a/SDAR-100M-ep1-block32/composition/autoregressive/graphs/results.json b/SDAR-100M-ep1-block32/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..97939dea451760193b2775aa3d674ae87f762164 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-22350_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.897196261682243, + "pass@2": 0.9283489096573209, + "pass@4": 0.9470404984423676, + "pass@8": 0.9626168224299065, + "pass@16": 0.9657320872274143, + "pass@32": 0.9719626168224299, + "pass@64": 0.9781931464174455, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.9296636085626911, + "pass@2": 0.9541284403669725, + "pass@4": 0.9663608562691132, + "pass@8": 0.9724770642201835, + "pass@16": 0.981651376146789, + "pass@32": 0.9847094801223242, + "pass@64": 0.9908256880733946, + "pass@128": 0.9908256880733946 + }, + "4": { + "pass@1": 0.915625, + "pass@2": 0.953125, + "pass@4": 0.96875, + "pass@8": 0.975, + "pass@16": 0.98125, + "pass@32": 0.984375, + "pass@64": 0.9875, + "pass@128": 0.9875 + }, + "5": { + "pass@1": 0.7428571428571429, + "pass@2": 0.8412698412698413, + "pass@4": 0.8857142857142857, + "pass@8": 0.9142857142857143, + "pass@16": 0.9301587301587302, + "pass@32": 0.9428571428571428, + "pass@64": 0.9587301587301588, + "pass@128": 0.9650793650793651 + }, + "6": { + "pass@1": 0.7125748502994012, + "pass@2": 0.8053892215568862, + "pass@4": 0.8592814371257484, + "pass@8": 0.9161676646706587, + "pass@16": 0.9341317365269461, + "pass@32": 0.9431137724550899, + "pass@64": 0.9520958083832335, + "pass@128": 0.9580838323353293 + }, + "7": { + "pass@1": 0.7011834319526628, + "pass@2": 0.772189349112426, + "pass@4": 0.8372781065088757, + "pass@8": 0.8757396449704142, + "pass@16": 0.8964497041420119, + "pass@32": 0.9142011834319527, + "pass@64": 0.9201183431952663, + "pass@128": 0.9319526627218935 + }, + "8": { + "pass@1": 0.48717948717948717, + "pass@2": 0.6474358974358975, + "pass@4": 0.7692307692307693, + "pass@8": 0.8237179487179487, + "pass@16": 0.8685897435897436, + "pass@32": 0.9038461538461539, + "pass@64": 0.9230769230769231, + "pass@128": 0.9358974358974359 + }, + "9": { + "pass@1": 0.4563380281690141, + "pass@2": 0.5746478873239437, + "pass@4": 0.6788732394366197, + "pass@8": 0.7690140845070422, + "pass@16": 0.856338028169014, + "pass@32": 0.8929577464788733, + "pass@64": 0.9183098591549296, + "pass@128": 0.9408450704225352 + }, + "10": { + "pass@1": 0.3834808259587021, + "pass@2": 0.471976401179941, + "pass@4": 0.5811209439528023, + "pass@8": 0.6696165191740413, + "pass@16": 0.7197640117994101, + "pass@32": 0.7669616519174042, + "pass@64": 0.799410029498525, + "pass@128": 0.8407079646017699 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.897196261682243, + "3": 0.9296636085626911, + "4": 0.915625, + "5": 0.7428571428571429, + "6": 0.7125748502994012, + "7": 0.7011834319526628, + "8": 0.48717948717948717, + "9": 0.4563380281690141, + "10": 0.3834808259587021 + }, + "pass@2": { + "2": 0.9283489096573209, + "3": 0.9541284403669725, + "4": 0.953125, + "5": 0.8412698412698413, + "6": 0.8053892215568862, + "7": 0.772189349112426, + "8": 0.6474358974358975, + "9": 0.5746478873239437, + "10": 0.471976401179941 + }, + "pass@4": { + "2": 0.9470404984423676, + "3": 0.9663608562691132, + "4": 0.96875, + "5": 0.8857142857142857, + "6": 0.8592814371257484, + "7": 0.8372781065088757, + "8": 0.7692307692307693, + "9": 0.6788732394366197, + "10": 0.5811209439528023 + }, + "pass@8": { + "2": 0.9626168224299065, + "3": 0.9724770642201835, + "4": 0.975, + "5": 0.9142857142857143, + "6": 0.9161676646706587, + "7": 0.8757396449704142, + "8": 0.8237179487179487, + "9": 0.7690140845070422, + "10": 0.6696165191740413 + }, + "pass@16": { + "2": 0.9657320872274143, + "3": 0.981651376146789, + "4": 0.98125, + "5": 0.9301587301587302, + "6": 0.9341317365269461, + "7": 0.8964497041420119, + "8": 0.8685897435897436, + "9": 0.856338028169014, + "10": 0.7197640117994101 + }, + "pass@32": { + "2": 0.9719626168224299, + "3": 0.9847094801223242, + "4": 0.984375, + "5": 0.9428571428571428, + "6": 0.9431137724550899, + "7": 0.9142011834319527, + "8": 0.9038461538461539, + "9": 0.8929577464788733, + "10": 0.7669616519174042 + }, + "pass@64": { + "2": 0.9781931464174455, + "3": 0.9908256880733946, + "4": 0.9875, + "5": 0.9587301587301588, + "6": 0.9520958083832335, + "7": 0.9201183431952663, + "8": 0.9230769230769231, + "9": 0.9183098591549296, + "10": 0.799410029498525 + }, + "pass@128": { + "2": 0.9875389408099688, + "3": 0.9908256880733946, + "4": 0.9875, + "5": 0.9650793650793651, + "6": 0.9580838323353293, + "7": 0.9319526627218935, + "8": 0.9358974358974359, + "9": 0.9408450704225352, + "10": 0.8407079646017699 + } + }, + "total_pass_at_k": { + "pass@1": 0.6882809861533266, + "pass@2": 0.7686592367443431, + "pass@4": 0.8297872340425532, + "pass@8": 0.8733535967578521, + "pass@16": 0.9023978385680513, + "pass@32": 0.9216480918608578, + "pass@64": 0.9354947652819994, + "pass@128": 0.9479905437352246 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9228571428571428, + "pass@2": 0.9428571428571428, + "pass@4": 0.96, + "pass@8": 0.9685714285714285, + "pass@16": 0.9828571428571429, + "pass@32": 0.9857142857142858, + "pass@64": 0.9885714285714285, + "pass@128": 0.9885714285714285 + }, + "3": { + "pass@1": 0.9329446064139941, + "pass@2": 0.9650145772594753, + "pass@4": 0.9737609329446064, + "pass@8": 0.9766763848396501, + "pass@16": 0.9825072886297376, + "pass@32": 0.9912536443148688, + "pass@64": 0.9912536443148688, + "pass@128": 0.9912536443148688 + }, + "4": { + "pass@1": 0.8771929824561403, + "pass@2": 0.9298245614035088, + "pass@4": 0.9532163742690059, + "pass@8": 0.9707602339181286, + "pass@16": 0.97953216374269, + "pass@32": 0.9883040935672515, + "pass@64": 0.9912280701754386, + "pass@128": 0.9912280701754386 + }, + "5": { + "pass@1": 0.8264705882352941, + "pass@2": 0.8676470588235294, + "pass@4": 0.8970588235294118, + "pass@8": 0.9147058823529411, + "pass@16": 0.9264705882352942, + "pass@32": 0.9470588235294117, + "pass@64": 0.9588235294117647, + "pass@128": 0.961764705882353 + }, + "6": { + "pass@1": 0.7287066246056783, + "pass@2": 0.8107255520504731, + "pass@4": 0.8675078864353313, + "pass@8": 0.8958990536277602, + "pass@16": 0.9305993690851735, + "pass@32": 0.9463722397476341, + "pass@64": 0.9652996845425867, + "pass@128": 0.9684542586750788 + }, + "7": { + "pass@1": 0.6044303797468354, + "pass@2": 0.6993670886075949, + "pass@4": 0.7689873417721519, + "pass@8": 0.8386075949367089, + "pass@16": 0.8607594936708861, + "pass@32": 0.8955696202531646, + "pass@64": 0.9177215189873418, + "pass@128": 0.9272151898734177 + }, + "8": { + "pass@1": 0.5315315315315315, + "pass@2": 0.6726726726726727, + "pass@4": 0.7597597597597597, + "pass@8": 0.8348348348348348, + "pass@16": 0.8858858858858859, + "pass@32": 0.918918918918919, + "pass@64": 0.948948948948949, + "pass@128": 0.963963963963964 + }, + "9": { + "pass@1": 0.3729903536977492, + "pass@2": 0.5048231511254019, + "pass@4": 0.6270096463022508, + "pass@8": 0.7170418006430869, + "pass@16": 0.797427652733119, + "pass@32": 0.8520900321543409, + "pass@64": 0.8842443729903537, + "pass@128": 0.909967845659164 + }, + "10": { + "pass@1": 0.3641791044776119, + "pass@2": 0.46865671641791046, + "pass@4": 0.5462686567164179, + "pass@8": 0.6208955223880597, + "pass@16": 0.7134328358208956, + "pass@32": 0.7671641791044777, + "pass@64": 0.7940298507462686, + "pass@128": 0.8059701492537313 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9228571428571428, + "3": 0.9329446064139941, + "4": 0.8771929824561403, + "5": 0.8264705882352941, + "6": 0.7287066246056783, + "7": 0.6044303797468354, + "8": 0.5315315315315315, + "9": 0.3729903536977492, + "10": 0.3641791044776119 + }, + "pass@2": { + "2": 0.9428571428571428, + "3": 0.9650145772594753, + "4": 0.9298245614035088, + "5": 0.8676470588235294, + "6": 0.8107255520504731, + "7": 0.6993670886075949, + "8": 0.6726726726726727, + "9": 0.5048231511254019, + "10": 0.46865671641791046 + }, + "pass@4": { + "2": 0.96, + "3": 0.9737609329446064, + "4": 0.9532163742690059, + "5": 0.8970588235294118, + "6": 0.8675078864353313, + "7": 0.7689873417721519, + "8": 0.7597597597597597, + "9": 0.6270096463022508, + "10": 0.5462686567164179 + }, + "pass@8": { + "2": 0.9685714285714285, + "3": 0.9766763848396501, + "4": 0.9707602339181286, + "5": 0.9147058823529411, + "6": 0.8958990536277602, + "7": 0.8386075949367089, + "8": 0.8348348348348348, + "9": 0.7170418006430869, + "10": 0.6208955223880597 + }, + "pass@16": { + "2": 0.9828571428571429, + "3": 0.9825072886297376, + "4": 0.97953216374269, + "5": 0.9264705882352942, + "6": 0.9305993690851735, + "7": 0.8607594936708861, + "8": 0.8858858858858859, + "9": 0.797427652733119, + "10": 0.7134328358208956 + }, + "pass@32": { + "2": 0.9857142857142858, + "3": 0.9912536443148688, + "4": 0.9883040935672515, + "5": 0.9470588235294117, + "6": 0.9463722397476341, + "7": 0.8955696202531646, + "8": 0.918918918918919, + "9": 0.8520900321543409, + "10": 0.7671641791044777 + }, + "pass@64": { + "2": 0.9885714285714285, + "3": 0.9912536443148688, + "4": 0.9912280701754386, + "5": 0.9588235294117647, + "6": 0.9652996845425867, + "7": 0.9177215189873418, + "8": 0.948948948948949, + "9": 0.8842443729903537, + "10": 0.7940298507462686 + }, + "pass@128": { + "2": 0.9885714285714285, + "3": 0.9912536443148688, + "4": 0.9912280701754386, + "5": 0.961764705882353, + "6": 0.9684542586750788, + "7": 0.9272151898734177, + "8": 0.963963963963964, + "9": 0.909967845659164, + "10": 0.8059701492537313 + } + }, + "total_pass_at_k": { + "pass@1": 0.6899899564780716, + "pass@2": 0.7666555071978574, + "pass@4": 0.8202209574824239, + "pass@8": 0.8620689655172413, + "pass@16": 0.8972212922664882, + "pass@32": 0.9226648811516572, + "pass@64": 0.9387345162370271, + "pass@128": 0.9460997656511551 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.8358662613981763, + "pass@2": 0.9148936170212766, + "pass@4": 0.9544072948328267, + "pass@8": 0.9696048632218845, + "pass@16": 0.9817629179331308, + "pass@32": 0.9848024316109423, + "pass@64": 0.9878419452887538, + "pass@128": 0.9908814589665653 + }, + "3": { + "pass@1": 0.8333333333333334, + "pass@2": 0.9272727272727272, + "pass@4": 0.9575757575757575, + "pass@8": 0.9666666666666667, + "pass@16": 0.9787878787878788, + "pass@32": 0.9848484848484849, + "pass@64": 0.9878787878787879, + "pass@128": 0.990909090909091 + }, + "4": { + "pass@1": 0.8579881656804734, + "pass@2": 0.9053254437869822, + "pass@4": 0.9289940828402367, + "pass@8": 0.9437869822485208, + "pass@16": 0.9526627218934911, + "pass@32": 0.9585798816568047, + "pass@64": 0.9585798816568047, + "pass@128": 0.9644970414201184 + }, + "5": { + "pass@1": 0.8608695652173913, + "pass@2": 0.9159420289855073, + "pass@4": 0.9478260869565217, + "pass@8": 0.9565217391304348, + "pass@16": 0.9652173913043478, + "pass@32": 0.9710144927536232, + "pass@64": 0.9710144927536232, + "pass@128": 0.9768115942028985 + }, + "6": { + "pass@1": 0.66189111747851, + "pass@2": 0.7478510028653295, + "pass@4": 0.8223495702005731, + "pass@8": 0.8624641833810889, + "pass@16": 0.8968481375358166, + "pass@32": 0.9140401146131805, + "pass@64": 0.9426934097421203, + "pass@128": 0.9570200573065902 + }, + "7": { + "pass@1": 0.6040462427745664, + "pass@2": 0.6994219653179191, + "pass@4": 0.7803468208092486, + "pass@8": 0.8497109826589595, + "pass@16": 0.8930635838150289, + "pass@32": 0.9248554913294798, + "pass@64": 0.9479768786127167, + "pass@128": 0.9508670520231214 + }, + "8": { + "pass@1": 0.5380281690140845, + "pass@2": 0.6450704225352113, + "pass@4": 0.7464788732394366, + "pass@8": 0.8225352112676056, + "pass@16": 0.8676056338028169, + "pass@32": 0.8845070422535212, + "pass@64": 0.9014084507042254, + "pass@128": 0.923943661971831 + }, + "9": { + "pass@1": 0.3652694610778443, + "pass@2": 0.5239520958083832, + "pass@4": 0.6437125748502994, + "pass@8": 0.7455089820359282, + "pass@16": 0.8233532934131736, + "pass@32": 0.8622754491017964, + "pass@64": 0.8832335329341318, + "pass@128": 0.907185628742515 + }, + "10": { + "pass@1": 0.38650306748466257, + "pass@2": 0.48466257668711654, + "pass@4": 0.5552147239263804, + "pass@8": 0.6411042944785276, + "pass@16": 0.7055214723926381, + "pass@32": 0.745398773006135, + "pass@64": 0.7914110429447853, + "pass@128": 0.8251533742331288 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.8358662613981763, + "3": 0.8333333333333334, + "4": 0.8579881656804734, + "5": 0.8608695652173913, + "6": 0.66189111747851, + "7": 0.6040462427745664, + "8": 0.5380281690140845, + "9": 0.3652694610778443, + "10": 0.38650306748466257 + }, + "pass@2": { + "2": 0.9148936170212766, + "3": 0.9272727272727272, + "4": 0.9053254437869822, + "5": 0.9159420289855073, + "6": 0.7478510028653295, + "7": 0.6994219653179191, + "8": 0.6450704225352113, + "9": 0.5239520958083832, + "10": 0.48466257668711654 + }, + "pass@4": { + "2": 0.9544072948328267, + "3": 0.9575757575757575, + "4": 0.9289940828402367, + "5": 0.9478260869565217, + "6": 0.8223495702005731, + "7": 0.7803468208092486, + "8": 0.7464788732394366, + "9": 0.6437125748502994, + "10": 0.5552147239263804 + }, + "pass@8": { + "2": 0.9696048632218845, + "3": 0.9666666666666667, + "4": 0.9437869822485208, + "5": 0.9565217391304348, + "6": 0.8624641833810889, + "7": 0.8497109826589595, + "8": 0.8225352112676056, + "9": 0.7455089820359282, + "10": 0.6411042944785276 + }, + "pass@16": { + "2": 0.9817629179331308, + "3": 0.9787878787878788, + "4": 0.9526627218934911, + "5": 0.9652173913043478, + "6": 0.8968481375358166, + "7": 0.8930635838150289, + "8": 0.8676056338028169, + "9": 0.8233532934131736, + "10": 0.7055214723926381 + }, + "pass@32": { + "2": 0.9848024316109423, + "3": 0.9848484848484849, + "4": 0.9585798816568047, + "5": 0.9710144927536232, + "6": 0.9140401146131805, + "7": 0.9248554913294798, + "8": 0.8845070422535212, + "9": 0.8622754491017964, + "10": 0.745398773006135 + }, + "pass@64": { + "2": 0.9878419452887538, + "3": 0.9878787878787879, + "4": 0.9585798816568047, + "5": 0.9710144927536232, + "6": 0.9426934097421203, + "7": 0.9479768786127167, + "8": 0.9014084507042254, + "9": 0.8832335329341318, + "10": 0.7914110429447853 + }, + "pass@128": { + "2": 0.9908814589665653, + "3": 0.990909090909091, + "4": 0.9644970414201184, + "5": 0.9768115942028985, + "6": 0.9570200573065902, + "7": 0.9508670520231214, + "8": 0.923943661971831, + "9": 0.907185628742515, + "10": 0.8251533742331288 + } + }, + "total_pass_at_k": { + "pass@1": 0.6605504587155964, + "pass@2": 0.7516382699868939, + "pass@4": 0.8155307994757536, + "pass@8": 0.8623853211009175, + "pass@16": 0.8964613368283093, + "pass@32": 0.9148099606815203, + "pass@64": 0.9305373525557011, + "pass@128": 0.9433158584534731 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.886, + "pass@2": 0.929, + "pass@4": 0.954, + "pass@8": 0.967, + "pass@16": 0.977, + "pass@32": 0.981, + "pass@64": 0.985, + "pass@128": 0.989 + }, + "3": { + "pass@1": 0.899, + "pass@2": 0.949, + "pass@4": 0.966, + "pass@8": 0.972, + "pass@16": 0.981, + "pass@32": 0.987, + "pass@64": 0.99, + "pass@128": 0.991 + }, + "4": { + "pass@1": 0.883, + "pass@2": 0.929, + "pass@4": 0.95, + "pass@8": 0.963, + "pass@16": 0.971, + "pass@32": 0.977, + "pass@64": 0.979, + "pass@128": 0.981 + }, + "5": { + "pass@1": 0.812, + "pass@2": 0.876, + "pass@4": 0.911, + "pass@8": 0.929, + "pass@16": 0.941, + "pass@32": 0.954, + "pass@64": 0.963, + "pass@128": 0.968 + }, + "6": { + "pass@1": 0.7, + "pass@2": 0.787, + "pass@4": 0.849, + "pass@8": 0.891, + "pass@16": 0.92, + "pass@32": 0.934, + "pass@64": 0.953, + "pass@128": 0.961 + }, + "7": { + "pass@1": 0.637, + "pass@2": 0.724, + "pass@4": 0.796, + "pass@8": 0.855, + "pass@16": 0.884, + "pass@32": 0.912, + "pass@64": 0.929, + "pass@128": 0.937 + }, + "8": { + "pass@1": 0.52, + "pass@2": 0.655, + "pass@4": 0.758, + "pass@8": 0.827, + "pass@16": 0.874, + "pass@32": 0.902, + "pass@64": 0.924, + "pass@128": 0.941 + }, + "9": { + "pass@1": 0.4, + "pass@2": 0.536, + "pass@4": 0.651, + "pass@8": 0.745, + "pass@16": 0.827, + "pass@32": 0.87, + "pass@64": 0.896, + "pass@128": 0.92 + }, + "10": { + "pass@1": 0.378, + "pass@2": 0.475, + "pass@4": 0.561, + "pass@8": 0.644, + "pass@16": 0.713, + "pass@32": 0.76, + "pass@64": 0.795, + "pass@128": 0.824 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.886, + "3": 0.899, + "4": 0.883, + "5": 0.812, + "6": 0.7, + "7": 0.637, + "8": 0.52, + "9": 0.4, + "10": 0.378 + }, + "pass@2": { + "2": 0.929, + "3": 0.949, + "4": 0.929, + "5": 0.876, + "6": 0.787, + "7": 0.724, + "8": 0.655, + "9": 0.536, + "10": 0.475 + }, + "pass@4": { + "2": 0.954, + "3": 0.966, + "4": 0.95, + "5": 0.911, + "6": 0.849, + "7": 0.796, + "8": 0.758, + "9": 0.651, + "10": 0.561 + }, + "pass@8": { + "2": 0.967, + "3": 0.972, + "4": 0.963, + "5": 0.929, + "6": 0.891, + "7": 0.855, + "8": 0.827, + "9": 0.745, + "10": 0.644 + }, + "pass@16": { + "2": 0.977, + "3": 0.981, + "4": 0.971, + "5": 0.941, + "6": 0.92, + "7": 0.884, + "8": 0.874, + "9": 0.827, + "10": 0.713 + }, + "pass@32": { + "2": 0.981, + "3": 0.987, + "4": 0.977, + "5": 0.954, + "6": 0.934, + "7": 0.912, + "8": 0.902, + "9": 0.87, + "10": 0.76 + }, + "pass@64": { + "2": 0.985, + "3": 0.99, + "4": 0.979, + "5": 0.963, + "6": 0.953, + "7": 0.929, + "8": 0.924, + "9": 0.896, + "10": 0.795 + }, + "pass@128": { + "2": 0.989, + "3": 0.991, + "4": 0.981, + "5": 0.968, + "6": 0.961, + "7": 0.937, + "8": 0.941, + "9": 0.92, + "10": 0.824 + } + }, + "total_pass_at_k": { + "pass@1": 0.6794444444444444, + "pass@2": 0.7622222222222222, + "pass@4": 0.8217777777777778, + "pass@8": 0.8658888888888889, + "pass@16": 0.8986666666666666, + "pass@32": 0.9196666666666666, + "pass@64": 0.9348888888888889, + "pass@128": 0.9457777777777778 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-22350_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.23796033994334279, + "pass@2": 0.32577903682719545, + "pass@4": 0.43059490084985835, + "pass@8": 0.5524079320113314, + "pass@16": 0.6402266288951841, + "pass@32": 0.6940509915014165, + "pass@64": 0.7280453257790368, + "pass@128": 0.773371104815864 + }, + "12": { + "pass@1": 0.17796610169491525, + "pass@2": 0.3050847457627119, + "pass@4": 0.403954802259887, + "pass@8": 0.4604519774011299, + "pass@16": 0.5141242937853108, + "pass@32": 0.5819209039548022, + "pass@64": 0.632768361581921, + "pass@128": 0.6864406779661016 + }, + "13": { + "pass@1": 0.0877742946708464, + "pass@2": 0.109717868338558, + "pass@4": 0.15047021943573669, + "pass@8": 0.1974921630094044, + "pass@16": 0.2413793103448276, + "pass@32": 0.29780564263322884, + "pass@64": 0.3542319749216301, + "pass@128": 0.4012539184952978 + }, + "14": { + "pass@1": 0.006230529595015576, + "pass@2": 0.012461059190031152, + "pass@4": 0.021806853582554516, + "pass@8": 0.028037383177570093, + "pass@16": 0.04672897196261682, + "pass@32": 0.059190031152647975, + "pass@64": 0.102803738317757, + "pass@128": 0.1277258566978193 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.014450867052023121, + "pass@16": 0.017341040462427744, + "pass@32": 0.017341040462427744, + "pass@64": 0.028901734104046242, + "pass@128": 0.03468208092485549 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0032258064516129032, + "pass@8": 0.0032258064516129032, + "pass@16": 0.0032258064516129032, + "pass@32": 0.0032258064516129032, + "pass@64": 0.0064516129032258064, + "pass@128": 0.0064516129032258064 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.23796033994334279, + "12": 0.17796610169491525, + "13": 0.0877742946708464, + "14": 0.006230529595015576, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.32577903682719545, + "12": 0.3050847457627119, + "13": 0.109717868338558, + "14": 0.012461059190031152, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.43059490084985835, + "12": 0.403954802259887, + "13": 0.15047021943573669, + "14": 0.021806853582554516, + "15": 0.0, + "16": 0.0032258064516129032, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.5524079320113314, + "12": 0.4604519774011299, + "13": 0.1974921630094044, + "14": 0.028037383177570093, + "15": 0.014450867052023121, + "16": 0.0032258064516129032, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.6402266288951841, + "12": 0.5141242937853108, + "13": 0.2413793103448276, + "14": 0.04672897196261682, + "15": 0.017341040462427744, + "16": 0.0032258064516129032, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.6940509915014165, + "12": 0.5819209039548022, + "13": 0.29780564263322884, + "14": 0.059190031152647975, + "15": 0.017341040462427744, + "16": 0.0032258064516129032, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.7280453257790368, + "12": 0.632768361581921, + "13": 0.3542319749216301, + "14": 0.102803738317757, + "15": 0.028901734104046242, + "16": 0.0064516129032258064, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.773371104815864, + "12": 0.6864406779661016, + "13": 0.4012539184952978, + "14": 0.1277258566978193, + "15": 0.03468208092485549, + "16": 0.0064516129032258064, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.05260029717682021, + "pass@2": 0.07786032689450223, + "pass@4": 0.10430906389301635, + "pass@8": 0.12956909361069838, + "pass@16": 0.15066864784546805, + "pass@32": 0.1699851411589896, + "pass@64": 0.18989598811292718, + "pass@128": 0.20772659732540863 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.19682539682539682, + "pass@2": 0.3111111111111111, + "pass@4": 0.41904761904761906, + "pass@8": 0.5238095238095238, + "pass@16": 0.6158730158730159, + "pass@32": 0.7015873015873015, + "pass@64": 0.7523809523809524, + "pass@128": 0.7873015873015873 + }, + "12": { + "pass@1": 0.1620795107033639, + "pass@2": 0.24464831804281345, + "pass@4": 0.3149847094801223, + "pass@8": 0.41590214067278286, + "pass@16": 0.48623853211009177, + "pass@32": 0.5657492354740061, + "pass@64": 0.6116207951070336, + "pass@128": 0.6605504587155964 + }, + "13": { + "pass@1": 0.05864197530864197, + "pass@2": 0.08950617283950617, + "pass@4": 0.11728395061728394, + "pass@8": 0.16358024691358025, + "pass@16": 0.20679012345679013, + "pass@32": 0.26851851851851855, + "pass@64": 0.31790123456790126, + "pass@128": 0.3487654320987654 + }, + "14": { + "pass@1": 0.008902077151335312, + "pass@2": 0.011869436201780416, + "pass@4": 0.017804154302670624, + "pass@8": 0.020771513353115726, + "pass@16": 0.02967359050445104, + "pass@32": 0.04747774480712166, + "pass@64": 0.08902077151335312, + "pass@128": 0.11572700296735905 + }, + "15": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0030864197530864196, + "pass@128": 0.006172839506172839 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0058997050147492625, + "pass@128": 0.0058997050147492625 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.19682539682539682, + "12": 0.1620795107033639, + "13": 0.05864197530864197, + "14": 0.008902077151335312, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.3111111111111111, + "12": 0.24464831804281345, + "13": 0.08950617283950617, + "14": 0.011869436201780416, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.41904761904761906, + "12": 0.3149847094801223, + "13": 0.11728395061728394, + "14": 0.017804154302670624, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.5238095238095238, + "12": 0.41590214067278286, + "13": 0.16358024691358025, + "14": 0.020771513353115726, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.6158730158730159, + "12": 0.48623853211009177, + "13": 0.20679012345679013, + "14": 0.02967359050445104, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.7015873015873015, + "12": 0.5657492354740061, + "13": 0.26851851851851855, + "14": 0.04747774480712166, + "15": 0.0, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.7523809523809524, + "12": 0.6116207951070336, + "13": 0.31790123456790126, + "14": 0.08902077151335312, + "15": 0.0030864197530864196, + "16": 0.0058997050147492625, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.7873015873015873, + "12": 0.6605504587155964, + "13": 0.3487654320987654, + "14": 0.11572700296735905, + "15": 0.006172839506172839, + "16": 0.0058997050147492625, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.04187041564792176, + "pass@2": 0.06448655256723716, + "pass@4": 0.08526894865525672, + "pass@8": 0.11033007334963325, + "pass@16": 0.13141809290953546, + "pass@32": 0.15556234718826406, + "pass@64": 0.17512224938875307, + "pass@128": 0.18948655256723718 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.19879518072289157, + "pass@2": 0.32831325301204817, + "pass@4": 0.4186746987951807, + "pass@8": 0.5240963855421686, + "pass@16": 0.6234939759036144, + "pass@32": 0.6987951807228916, + "pass@64": 0.7379518072289156, + "pass@128": 0.786144578313253 + }, + "12": { + "pass@1": 0.12225705329153605, + "pass@2": 0.21003134796238246, + "pass@4": 0.27586206896551724, + "pass@8": 0.3573667711598746, + "pass@16": 0.4106583072100313, + "pass@32": 0.4702194357366771, + "pass@64": 0.5391849529780565, + "pass@128": 0.5893416927899686 + }, + "13": { + "pass@1": 0.047619047619047616, + "pass@2": 0.0896358543417367, + "pass@4": 0.12324929971988796, + "pass@8": 0.17647058823529413, + "pass@16": 0.21288515406162464, + "pass@32": 0.2969187675070028, + "pass@64": 0.3389355742296919, + "pass@128": 0.39215686274509803 + }, + "14": { + "pass@1": 0.005847953216374269, + "pass@2": 0.011695906432748537, + "pass@4": 0.03508771929824561, + "pass@8": 0.049707602339181284, + "pass@16": 0.06140350877192982, + "pass@32": 0.07602339181286549, + "pass@64": 0.11988304093567251, + "pass@128": 0.16374269005847952 + }, + "15": { + "pass@1": 0.006060606060606061, + "pass@2": 0.006060606060606061, + "pass@4": 0.006060606060606061, + "pass@8": 0.006060606060606061, + "pass@16": 0.006060606060606061, + "pass@32": 0.00909090909090909, + "pass@64": 0.00909090909090909, + "pass@128": 0.00909090909090909 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.005698005698005698, + "pass@32": 0.008547008547008548, + "pass@64": 0.008547008547008548, + "pass@128": 0.011396011396011397 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.19879518072289157, + "12": 0.12225705329153605, + "13": 0.047619047619047616, + "14": 0.005847953216374269, + "15": 0.006060606060606061, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.32831325301204817, + "12": 0.21003134796238246, + "13": 0.0896358543417367, + "14": 0.011695906432748537, + "15": 0.006060606060606061, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.4186746987951807, + "12": 0.27586206896551724, + "13": 0.12324929971988796, + "14": 0.03508771929824561, + "15": 0.006060606060606061, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.5240963855421686, + "12": 0.3573667711598746, + "13": 0.17647058823529413, + "14": 0.049707602339181284, + "15": 0.006060606060606061, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.6234939759036144, + "12": 0.4106583072100313, + "13": 0.21288515406162464, + "14": 0.06140350877192982, + "15": 0.006060606060606061, + "16": 0.005698005698005698, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.6987951807228916, + "12": 0.4702194357366771, + "13": 0.2969187675070028, + "14": 0.07602339181286549, + "15": 0.00909090909090909, + "16": 0.008547008547008548, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.7379518072289156, + "12": 0.5391849529780565, + "13": 0.3389355742296919, + "14": 0.11988304093567251, + "15": 0.00909090909090909, + "16": 0.008547008547008548, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.786144578313253, + "12": 0.5893416927899686, + "13": 0.39215686274509803, + "14": 0.16374269005847952, + "15": 0.00909090909090909, + "16": 0.011396011396011397, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.03746654772524532, + "pass@2": 0.06363366042224204, + "pass@4": 0.0847457627118644, + "pass@8": 0.11002081474873625, + "pass@16": 0.1305382099316087, + "pass@32": 0.15462384775498067, + "pass@64": 0.17395182872435325, + "pass@128": 0.19387451680047577 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.212, + "pass@2": 0.322, + "pass@4": 0.423, + "pass@8": 0.534, + "pass@16": 0.627, + "pass@32": 0.698, + "pass@64": 0.739, + "pass@128": 0.782 + }, + "12": { + "pass@1": 0.155, + "pass@2": 0.255, + "pass@4": 0.334, + "pass@8": 0.413, + "pass@16": 0.472, + "pass@32": 0.541, + "pass@64": 0.596, + "pass@128": 0.647 + }, + "13": { + "pass@1": 0.064, + "pass@2": 0.096, + "pass@4": 0.13, + "pass@8": 0.179, + "pass@16": 0.22, + "pass@32": 0.288, + "pass@64": 0.337, + "pass@128": 0.381 + }, + "14": { + "pass@1": 0.007, + "pass@2": 0.012, + "pass@4": 0.025, + "pass@8": 0.033, + "pass@16": 0.046, + "pass@32": 0.061, + "pass@64": 0.104, + "pass@128": 0.136 + }, + "15": { + "pass@1": 0.002, + "pass@2": 0.002, + "pass@4": 0.002, + "pass@8": 0.007, + "pass@16": 0.008, + "pass@32": 0.009, + "pass@64": 0.014, + "pass@128": 0.017 + }, + "16": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.003, + "pass@32": 0.004, + "pass@64": 0.007, + "pass@128": 0.008 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.212, + "12": 0.155, + "13": 0.064, + "14": 0.007, + "15": 0.002, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.322, + "12": 0.255, + "13": 0.096, + "14": 0.012, + "15": 0.002, + "16": 0.0, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.423, + "12": 0.334, + "13": 0.13, + "14": 0.025, + "15": 0.002, + "16": 0.001, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.534, + "12": 0.413, + "13": 0.179, + "14": 0.033, + "15": 0.007, + "16": 0.001, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.627, + "12": 0.472, + "13": 0.22, + "14": 0.046, + "15": 0.008, + "16": 0.003, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.698, + "12": 0.541, + "13": 0.288, + "14": 0.061, + "15": 0.009, + "16": 0.004, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.739, + "12": 0.596, + "13": 0.337, + "14": 0.104, + "15": 0.014, + "16": 0.007, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.782, + "12": 0.647, + "13": 0.381, + "14": 0.136, + "15": 0.017, + "16": 0.008, + "17": 0.0, + "18": 0.0, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.044, + "pass@2": 0.0687, + "pass@4": 0.0915, + "pass@8": 0.1167, + "pass@16": 0.1376, + "pass@32": 0.1601, + "pass@64": 0.1797, + "pass@128": 0.1971 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block32/composition/autoregressive/summary.csv b/SDAR-100M-ep1-block32/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..fdc2b20467faac34d1617044a097822b76d4a6e1 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-22350,0.8321892361111111,0.2584078125,0.5301990131578947,0.0,0.0,0.0,197.74917100694444,304.17006484375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block32/composition/autoregressive/summary.json b/SDAR-100M-ep1-block32/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0e8d2d672185af3fbde6a776df96648082761256 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-22350", + "id_acc": 0.8321892361111111, + "ood_acc": 0.2584078125, + "total_acc": 0.5301990131578947, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.74917100694444, + "ood_avg_resp_len": 304.17006484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 433879, + "answer_accuracy": 0.5358330204710717, + "avg_response_len": 243.30185197992412, + "resp_tokens_sum": 197008322, + "pass_at_k": { + "pass@1": 0.5358330204710717, + "pass@2": 0.6076064854518745, + "pass@4": 0.6674381751571705, + "pass@8": 0.7160587355579924, + "pass@16": 0.7550128453944618, + "pass@32": 0.78618341453662, + "pass@64": 0.8111926583790442, + "pass@128": 0.8322794815049004 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6943906710914455, + "pass@2": 0.7924839732422831, + "pass@4": 0.860622414178315, + "pass@8": 0.9069794084790083, + "pass@16": 0.938909065525345, + "pass@32": 0.961220354064231, + "pass@64": 0.9761939203554444, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 0.9308557242990654, + "pass@2": 0.9428547446464054, + "pass@4": 0.9537001116682842, + "pass@8": 0.9648737739918477, + "pass@16": 0.9749592956005654, + "pass@32": 0.982755395867794, + "pass@64": 0.9887116454415467, + "pass@128": 0.9937694704049844 + }, + "3": { + "pass@1": 0.9621081804281345, + "pass@2": 0.9769709118928944, + "pass@4": 0.9848477245963518, + "pass@8": 0.9902888229327741, + "pass@16": 0.9939032445179141, + "pass@32": 0.9963332215254794, + "pass@64": 0.9984785531300948, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9477783203125, + "pass@2": 0.9676934670275597, + "pass@4": 0.9787657553547993, + "pass@8": 0.985568835042546, + "pass@16": 0.9906626007427637, + "pass@32": 0.9951571846591477, + "pass@64": 0.9986095925944035, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9187748015873016, + "pass@2": 0.9491169072615924, + "pass@4": 0.966745741603728, + "pass@8": 0.9769276192432061, + "pass@16": 0.9838399495702144, + "pass@32": 0.9891182703066478, + "pass@64": 0.9919019584676965, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.8201019835329342, + "pass@2": 0.8826146918760904, + "pass@4": 0.9236047901385079, + "pass@8": 0.9489790206766302, + "pass@16": 0.9630538085476542, + "pass@32": 0.9706400767227515, + "pass@64": 0.9751205236185113, + "pass@128": 0.9790419161676647 + }, + "7": { + "pass@1": 0.8473095414201184, + "pass@2": 0.8948160788333407, + "pass@4": 0.9233494061208317, + "pass@8": 0.9446898395536166, + "pass@16": 0.9609689351179754, + "pass@32": 0.9718427623312778, + "pass@64": 0.9791843428978619, + "pass@128": 0.985207100591716 + }, + "8": { + "pass@1": 0.7699068509615384, + "pass@2": 0.8474366072329903, + "pass@4": 0.8956249834756224, + "pass@8": 0.9289230761890581, + "pass@16": 0.952910327741177, + "pass@32": 0.9687048044336607, + "pass@64": 0.9776952048260006, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.7081205985915493, + "pass@2": 0.8081637462570703, + "pass@4": 0.873297600035911, + "pass@8": 0.9156458120013241, + "pass@16": 0.9428910229292787, + "pass@32": 0.9607310443761353, + "pass@64": 0.9728863441329083, + "pass@128": 0.9802816901408451 + }, + "11": { + "pass@1": 0.5862473441926346, + "pass@2": 0.6982166358100419, + "pass@4": 0.7848791272726872, + "pass@8": 0.8518025967130893, + "pass@16": 0.8993367128402658, + "pass@32": 0.9296627844085906, + "pass@64": 0.9481568861295607, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.4626588983050847, + "pass@2": 0.5915618466123937, + "pass@4": 0.7009537414814668, + "pass@8": 0.7852130266152862, + "pass@16": 0.8448068180092854, + "pass@32": 0.8845265457694631, + "pass@64": 0.9105669496967504, + "pass@128": 0.9293785310734464 + }, + "13": { + "pass@1": 0.359203565830721, + "pass@2": 0.47705713289561363, + "pass@4": 0.5781522086384967, + "pass@8": 0.6589915053523162, + "pass@16": 0.7248504805274102, + "pass@32": 0.7792684090834485, + "pass@64": 0.8205216396970003, + "pass@128": 0.8495297805642633 + }, + "14": { + "pass@1": 0.23035922897196262, + "pass@2": 0.3267222109794686, + "pass@4": 0.42098008076093296, + "pass@8": 0.5040391320187068, + "pass@16": 0.5757914308258796, + "pass@32": 0.6382070925379484, + "pass@64": 0.6902304607748279, + "pass@128": 0.7320872274143302 + }, + "15": { + "pass@1": 0.1915643063583815, + "pass@2": 0.2748797420463337, + "pass@4": 0.35926651934042914, + "pass@8": 0.4369129733469511, + "pass@16": 0.5078082628147159, + "pass@32": 0.5739481885721509, + "pass@64": 0.6338527371652254, + "pass@128": 0.6878612716763006 + }, + "16": { + "pass@1": 0.1787046370967742, + "pass@2": 0.25333494729489453, + "pass@4": 0.3314841881659954, + "pass@8": 0.4064332975106817, + "pass@16": 0.47415692528401443, + "pass@32": 0.5324868128405753, + "pass@64": 0.5803103772512649, + "pass@128": 0.6161290322580645 + }, + "17": { + "pass@1": 0.16708815028901733, + "pass@2": 0.24038220836557264, + "pass@4": 0.3126962039788378, + "pass@8": 0.37412953912282093, + "pass@16": 0.4247942184063712, + "pass@32": 0.4686977316823931, + "pass@64": 0.5104069609249928, + "pass@128": 0.5549132947976878 + }, + "18": { + "pass@1": 0.15174107142857143, + "pass@2": 0.21740122328458925, + "pass@4": 0.2878790993090149, + "pass@8": 0.3559641056655266, + "pass@16": 0.41501722330270596, + "pass@32": 0.46579089458779094, + "pass@64": 0.5139033629581451, + "pass@128": 0.5628571428571428 + }, + "19": { + "pass@1": 0.1649540165165165, + "pass@2": 0.23666493757537074, + "pass@4": 0.30362395044087953, + "pass@8": 0.36108731681792244, + "pass@16": 0.41238462464001546, + "pass@32": 0.4586458618410122, + "pass@64": 0.49736060514722336, + "pass@128": 0.5315315315315315 + }, + "20": { + "pass@1": 0.14857826576576577, + "pass@2": 0.2135457307701402, + "pass@4": 0.2781666238792225, + "pass@8": 0.3375520764569661, + "pass@16": 0.38947846469059344, + "pass@32": 0.4322996299171272, + "pass@64": 0.4686906739688465, + "pass@128": 0.5045045045045045 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 431233, + "answer_accuracy": 0.5251765880748246, + "avg_response_len": 250.86816908612627, + "resp_tokens_sum": 205992871, + "pass_at_k": { + "pass@1": 0.5251765880748246, + "pass@2": 0.5990236113071689, + "pass@4": 0.6605269380444797, + "pass@8": 0.7110892406005488, + "pass@16": 0.7520712251847446, + "pass@32": 0.7855516781058292, + "pass@64": 0.8138543322827327, + "pass@128": 0.8380358534684333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6537576687116564, + "pass@2": 0.7460161948698127, + "pass@4": 0.8060986609802601, + "pass@8": 0.847945424633065, + "pass@16": 0.8813404119372923, + "pass@32": 0.9103958247721612, + "pass@64": 0.9355294903589877, + "pass@128": 0.9570552147239264 + }, + "2": { + "pass@1": 0.9330357142857143, + "pass@2": 0.9584501322308118, + "pass@4": 0.9728141189798079, + "pass@8": 0.9825223285201696, + "pass@16": 0.988427837610654, + "pass@32": 0.9919575137068457, + "pass@64": 0.9936774918251292, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9535274621212121, + "pass@2": 0.9845360594130279, + "pass@4": 0.9929094837577115, + "pass@8": 0.9964002893104172, + "pass@16": 0.9984903628634235, + "pass@32": 0.9996016193444951, + "pass@64": 0.9999800782817796, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9252727440828402, + "pass@2": 0.9541336398686112, + "pass@4": 0.9712754627010378, + "pass@8": 0.9821602965278723, + "pass@16": 0.9889760898065934, + "pass@32": 0.99406958704192, + "pass@64": 0.9982805345757635, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9472373188405797, + "pass@2": 0.9669444682186469, + "pass@4": 0.9776280263880057, + "pass@8": 0.9841109345296649, + "pass@16": 0.9871814520804815, + "pass@32": 0.9891453339567674, + "pass@64": 0.991220735025819, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.7716690544412608, + "pass@2": 0.8399441317149118, + "pass@4": 0.88877262772311, + "pass@8": 0.9236277234039011, + "pass@16": 0.947779342166902, + "pass@32": 0.9623149622458524, + "pass@64": 0.9719698435364026, + "pass@128": 0.9799426934097422 + }, + "7": { + "pass@1": 0.8232252528901735, + "pass@2": 0.8818197140548907, + "pass@4": 0.9189946032757454, + "pass@8": 0.9443311493694878, + "pass@16": 0.9613068615394603, + "pass@32": 0.973776344644011, + "pass@64": 0.9830204417073324, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.7471390845070423, + "pass@2": 0.8324141205500724, + "pass@4": 0.8866519479079198, + "pass@8": 0.9229622296115595, + "pass@16": 0.9482430767490266, + "pass@32": 0.9668919515308674, + "pass@64": 0.9814110244645726, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.6907980913173652, + "pass@2": 0.7911919762836525, + "pass@4": 0.8568220283280452, + "pass@8": 0.9022026761726983, + "pass@16": 0.9342429287522777, + "pass@32": 0.9557694238038187, + "pass@64": 0.968969940690975, + "pass@128": 0.9760479041916168 + }, + "11": { + "pass@1": 0.5349444653614458, + "pass@2": 0.6472616598282896, + "pass@4": 0.7313714308866506, + "pass@8": 0.7953237183465092, + "pass@16": 0.8460691697512974, + "pass@32": 0.8874826145843286, + "pass@64": 0.9217594495766532, + "pass@128": 0.9487951807228916 + }, + "12": { + "pass@1": 0.4509943181818182, + "pass@2": 0.5793468300298671, + "pass@4": 0.6863166373403949, + "pass@8": 0.7673792999388869, + "pass@16": 0.8274085956239333, + "pass@32": 0.8702289262058087, + "pass@64": 0.901284376599011, + "pass@128": 0.9310344827586207 + }, + "13": { + "pass@1": 0.29072566526610644, + "pass@2": 0.40266313218200644, + "pass@4": 0.5093617699048123, + "pass@8": 0.5996578531299764, + "pass@16": 0.6733180939052346, + "pass@32": 0.7334280975334988, + "pass@64": 0.7814138765854218, + "pass@128": 0.8235294117647058 + }, + "14": { + "pass@1": 0.2385325292397661, + "pass@2": 0.32814910266150954, + "pass@4": 0.4164776088844155, + "pass@8": 0.5006383588810326, + "pass@16": 0.5789538739078068, + "pass@32": 0.6484920703430255, + "pass@64": 0.7104199418336355, + "pass@128": 0.7631578947368421 + }, + "15": { + "pass@1": 0.1773910984848485, + "pass@2": 0.26216893342877584, + "pass@4": 0.3520031467657451, + "pass@8": 0.43594597119352674, + "pass@16": 0.510650496174987, + "pass@32": 0.5791911558699068, + "pass@64": 0.6454691653281225, + "pass@128": 0.7090909090909091 + }, + "16": { + "pass@1": 0.18540776353276353, + "pass@2": 0.26307989546178523, + "pass@4": 0.33983510180885507, + "pass@8": 0.40605426855918836, + "pass@16": 0.45938682436814693, + "pass@32": 0.5041509463348236, + "pass@64": 0.5477951643139377, + "pass@128": 0.5925925925925926 + }, + "17": { + "pass@1": 0.19257084627329193, + "pass@2": 0.2739552257054825, + "pass@4": 0.356384882402122, + "pass@8": 0.4314625004676136, + "pass@16": 0.49515481662757777, + "pass@32": 0.5487970243528865, + "pass@64": 0.5973562474326223, + "pass@128": 0.639751552795031 + }, + "18": { + "pass@1": 0.16482077205882353, + "pass@2": 0.23924559981472898, + "pass@4": 0.31721665040031777, + "pass@8": 0.3896711923876134, + "pass@16": 0.4539420417330226, + "pass@32": 0.5117977902329494, + "pass@64": 0.5613726317248905, + "pass@128": 0.5941176470588235 + }, + "19": { + "pass@1": 0.14569382440476192, + "pass@2": 0.21112570889576301, + "pass@4": 0.27935337240211033, + "pass@8": 0.34295164313173365, + "pass@16": 0.3982470420257465, + "pass@32": 0.4451936526320672, + "pass@64": 0.4832365573017363, + "pass@128": 0.5119047619047619 + }, + "20": { + "pass@1": 0.14116298652694612, + "pass@2": 0.210082541609694, + "pass@4": 0.28193506883870056, + "pass@8": 0.34853051627177145, + "pass@16": 0.4043639567430765, + "pass@32": 0.4477221435505894, + "pass@64": 0.48468313273594765, + "pass@128": 0.5239520958083832 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 424332, + "answer_accuracy": 0.5296523006870107, + "avg_response_len": 267.2945146489056, + "resp_tokens_sum": 214143535, + "pass_at_k": { + "pass@1": 0.5296523006870107, + "pass@2": 0.6046481452535212, + "pass@4": 0.6667228500895, + "pass@8": 0.7173735309314182, + "pass@16": 0.758131133309332, + "pass@32": 0.7910589349365911, + "pass@64": 0.8179951826110925, + "pass@128": 0.8403898386323694 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.6852378731343284, + "pass@2": 0.7834590580561754, + "pass@4": 0.8437291223858209, + "pass@8": 0.8810008174002717, + "pass@16": 0.9081553902326077, + "pass@32": 0.932425299977489, + "pass@64": 0.9556776289164318, + "pass@128": 0.9791044776119403 + }, + "2": { + "pass@1": 0.9315401785714286, + "pass@2": 0.954753585489314, + "pass@4": 0.9696069607370507, + "pass@8": 0.9788367325735242, + "pass@16": 0.9835176370135428, + "pass@32": 0.9863311311106975, + "pass@64": 0.9889418407176944, + "pass@128": 0.9914285714285714 + }, + "3": { + "pass@1": 0.9608236151603499, + "pass@2": 0.9816564702371388, + "pass@4": 0.9905631080007128, + "pass@8": 0.996430338052377, + "pass@16": 0.9992393738766351, + "pass@32": 0.9999502865055028, + "pass@64": 0.9999999185821354, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9050621345029239, + "pass@2": 0.9440120355021416, + "pass@4": 0.9680004067474015, + "pass@8": 0.984340659490216, + "pass@16": 0.9928412933334729, + "pass@32": 0.9964188428980713, + "pass@64": 0.9983444427656412, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9003216911764705, + "pass@2": 0.9268205042843912, + "pass@4": 0.9463646645272281, + "pass@8": 0.9632171704788541, + "pass@16": 0.9757164148383949, + "pass@32": 0.9834181485021496, + "pass@64": 0.9877429121098109, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.8128450315457413, + "pass@2": 0.8733140415807641, + "pass@4": 0.9138919448948998, + "pass@8": 0.9426343651866598, + "pass@16": 0.9625392372821305, + "pass@32": 0.9741035535727569, + "pass@64": 0.9804410838777078, + "pass@128": 0.9842271293375394 + }, + "7": { + "pass@1": 0.7880241297468354, + "pass@2": 0.8616713283663908, + "pass@4": 0.9114520427826266, + "pass@8": 0.9435628508320384, + "pass@16": 0.9629477011367917, + "pass@32": 0.9742916300599795, + "pass@64": 0.9818103307680968, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.7663288288288288, + "pass@2": 0.8553814641413071, + "pass@4": 0.9128960433999799, + "pass@8": 0.9495830700873962, + "pass@16": 0.9718125161329592, + "pass@32": 0.9853249653977751, + "pass@64": 0.9933116829048084, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.6727039790996785, + "pass@2": 0.7811534736815446, + "pass@4": 0.8535718273358913, + "pass@8": 0.9004946604118668, + "pass@16": 0.9332582598657084, + "pass@32": 0.9588299872546514, + "pass@64": 0.9769867806714441, + "pass@128": 0.9839228295819936 + }, + "11": { + "pass@1": 0.5628224206349206, + "pass@2": 0.6886682133483313, + "pass@4": 0.7795924515387953, + "pass@8": 0.843846333850522, + "pass@16": 0.8911162090697101, + "pass@32": 0.92447878151729, + "pass@64": 0.9459007713570995, + "pass@128": 0.9587301587301588 + }, + "12": { + "pass@1": 0.47842603211009177, + "pass@2": 0.6046798622649233, + "pass@4": 0.7040755857352686, + "pass@8": 0.7796449097929059, + "pass@16": 0.8354664617911026, + "pass@32": 0.8750244266732382, + "pass@64": 0.9055190650001771, + "pass@128": 0.9327217125382263 + }, + "13": { + "pass@1": 0.32978877314814814, + "pass@2": 0.4449008761057645, + "pass@4": 0.5485310094340058, + "pass@8": 0.6357017178809099, + "pass@16": 0.7077049545874511, + "pass@32": 0.7677406341601419, + "pass@64": 0.8184612944758107, + "pass@128": 0.8611111111111112 + }, + "14": { + "pass@1": 0.24547941394658754, + "pass@2": 0.33984986506694087, + "pass@4": 0.4344585242868378, + "pass@8": 0.5230985598476801, + "pass@16": 0.6032954304388765, + "pass@32": 0.6722870419315873, + "pass@64": 0.7308149824893481, + "pass@128": 0.7804154302670623 + }, + "15": { + "pass@1": 0.1517891589506173, + "pass@2": 0.23095328144745775, + "pass@4": 0.32680433059293523, + "pass@8": 0.4268563984986593, + "pass@16": 0.5184743795580643, + "pass@32": 0.5953989749015376, + "pass@64": 0.6532845826632232, + "pass@128": 0.6975308641975309 + }, + "16": { + "pass@1": 0.19750184365781712, + "pass@2": 0.2707752653705895, + "pass@4": 0.3384619665904593, + "pass@8": 0.39868858340697155, + "pass@16": 0.4537976264326197, + "pass@32": 0.5075626238811257, + "pass@64": 0.5621747076841135, + "pass@128": 0.6135693215339233 + }, + "17": { + "pass@1": 0.16789815512048192, + "pass@2": 0.24503390036524042, + "pass@4": 0.3252749837934414, + "pass@8": 0.39844549811174756, + "pass@16": 0.4604747765970229, + "pass@32": 0.5131333897557869, + "pass@64": 0.5606722490567707, + "pass@128": 0.6054216867469879 + }, + "18": { + "pass@1": 0.16890120967741937, + "pass@2": 0.24259866332232669, + "pass@4": 0.31384557797210855, + "pass@8": 0.37329870800292475, + "pass@16": 0.4203020018185082, + "pass@32": 0.46182948474688756, + "pass@64": 0.5007418217465677, + "pass@128": 0.535483870967742 + }, + "19": { + "pass@1": 0.14680891238670696, + "pass@2": 0.20984675702357458, + "pass@4": 0.27491625404830444, + "pass@8": 0.3400342250777418, + "pass@16": 0.4028458907394394, + "pass@32": 0.45635539239742556, + "pass@64": 0.4952112213686042, + "pass@128": 0.5226586102719033 + }, + "20": { + "pass@1": 0.14135228978978978, + "pass@2": 0.2111279586673286, + "pass@4": 0.2834856467828909, + "pass@8": 0.34874205971355804, + "pass@16": 0.40399217073375576, + "pass@32": 0.45152029441970093, + "pass@64": 0.49416572663356273, + "pass@128": 0.5345345345345346 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/checkpoint-22350_metrics.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/checkpoint-22350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..fe03222304b763d771d37e74c350ac9460405b69 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/checkpoint-22350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-22350", + "id": { + "answer_accuracy": 0.9711449652777778, + "avg_response_len": 196.12382552083332, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9711449652777778, + "pass@2": 0.9772521872265921, + "pass@4": 0.9807891801962249, + "pass@8": 0.9832802236569779, + "pass@16": 0.9852935447531236, + "pass@32": 0.9869718664085532, + "pass@64": 0.9883399028888511, + "pass@128": 0.9895555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.893609375, + "pass@2": 0.9112267470472445, + "pass@4": 0.9226303084926886, + "pass@8": 0.9307501273420538, + "pass@16": 0.9379550421861264, + "pass@32": 0.9454594928804226, + "pass@64": 0.9529819802272322, + "pass@128": 0.96 + }, + "2": { + "pass@1": 0.999796875, + "pass@2": 0.9999814222440945, + "pass@4": 0.9999997754968128, + "pass@8": 0.9999999999829965, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995546875, + "pass@2": 0.9998916092519686, + "pass@4": 0.9999895073115861, + "pass@8": 0.9999999174442428, + "pass@16": 0.9999999999982161, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9980078125, + "pass@2": 0.998831938976378, + "pass@4": 0.9989930251218597, + "pass@8": 0.9989999850429369, + "pass@16": 0.998999999999976, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9709375, + "pass@2": 0.9759972933070866, + "pass@4": 0.9792994161979752, + "pass@8": 0.9816848512692736, + "pass@16": 0.9839470682252031, + "pass@32": 0.9859399222232034, + "pass@64": 0.987406075384883, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.9853984375, + "pass@2": 0.9884852362204726, + "pass@4": 0.9899697619985, + "pass@8": 0.9908947205863909, + "pass@16": 0.991577305674489, + "pass@32": 0.9919372248725359, + "pass@64": 0.9919995560032147, + "pass@128": 0.992 + }, + "7": { + "pass@1": 0.9759296875, + "pass@2": 0.9813612204724409, + "pass@4": 0.9843149371016123, + "pass@8": 0.986289730181675, + "pass@16": 0.9875135423464853, + "pass@32": 0.9882564902121656, + "pass@64": 0.9889711688992653, + "pass@128": 0.99 + }, + "8": { + "pass@1": 0.96628125, + "pass@2": 0.9747928149606305, + "pass@4": 0.9798431611361081, + "pass@8": 0.9841081004997401, + "pass@16": 0.9874667032833114, + "pass@32": 0.9893074023160056, + "pass@64": 0.9899354839743831, + "pass@128": 0.99 + }, + "9": { + "pass@1": 0.9507890625, + "pass@2": 0.9647014025590553, + "pass@4": 0.9720627289088867, + "pass@8": 0.9767945805634863, + "pass@16": 0.980182241064308, + "pass@32": 0.9828462651726455, + "pass@64": 0.9847648615106731, + "pass@128": 0.986 + } + }, + "per_op_accuracy": { + "10": 0.893609375, + "2": 0.999796875, + "3": 0.9995546875, + "4": 0.9980078125, + "5": 0.9709375, + "6": 0.9853984375, + "7": 0.9759296875, + "8": 0.96628125, + "9": 0.9507890625 + }, + "per_op_avg_response_len": { + "10": 273.24859375, + "2": 114.352421875, + "3": 142.668546875, + "4": 157.6628125, + "5": 171.03015625, + "6": 195.4423125, + "7": 219.505125, + "8": 238.1589375, + "9": 253.0455234375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.24859375, + "2": 114.352421875, + "3": 142.668546875, + "4": 157.6628125, + "5": 171.03015625, + "6": 195.4423125, + "7": 219.505125, + "8": 238.1589375, + "9": 253.0455234375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 368498, + "answer_accuracy": 0.972269714623438, + "avg_response_len": 186.58243889311044, + "resp_tokens_sum": 70716237, + "pass_at_k": { + "pass@1": 0.972269714623438, + "pass@2": 0.9792173584684908, + "pass@4": 0.9827895848125371, + "pass@8": 0.9851507218358346, + "pass@16": 0.9869154105069612, + "pass@32": 0.9882053773375488, + "pass@64": 0.9893108618124246, + "pass@128": 0.9905437352245863 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9188560103244838, + "pass@2": 0.9359165592409355, + "pass@4": 0.9455644327644885, + "pass@8": 0.9517458870956133, + "pass@16": 0.956694467156006, + "pass@32": 0.9617356981168745, + "pass@64": 0.9673014491619945, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9943115234375, + "pass@2": 0.9964159387303149, + "pass@4": 0.9968543389107613, + "pass@8": 0.9968749534243821, + "pass@16": 0.996874999999925, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9741071428571428, + "pass@2": 0.979054961879765, + "pass@4": 0.9813411186696901, + "pass@8": 0.9823550160738614, + "pass@16": 0.983211768811447, + "pass@32": 0.9839110645580986, + "pass@64": 0.9841223976613889, + "pass@128": 0.9841269841269841 + }, + "6": { + "pass@1": 0.9783869760479041, + "pass@2": 0.9840478982978925, + "pass@4": 0.986558948189111, + "pass@8": 0.9875558074943762, + "pass@16": 0.987963102168246, + "pass@32": 0.9880233341018126, + "pass@64": 0.9880239520939974, + "pass@128": 0.9880239520958084 + }, + "7": { + "pass@1": 0.9787120931952663, + "pass@2": 0.9838508421469508, + "pass@4": 0.9854995232659527, + "pass@8": 0.9862073748551169, + "pass@16": 0.9870352684360165, + "pass@32": 0.9882217903235329, + "pass@64": 0.9895597074034385, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.9613882211538461, + "pass@2": 0.9697350406319402, + "pass@4": 0.9748580922576983, + "pass@8": 0.9800074606967099, + "pass@16": 0.9842336623573533, + "pass@32": 0.9863401319498917, + "pass@64": 0.987086901456941, + "pass@128": 0.9871794871794872 + }, + "9": { + "pass@1": 0.9488556338028169, + "pass@2": 0.966649453809471, + "pass@4": 0.9763733091462155, + "pass@8": 0.9831086093800043, + "pass@16": 0.9874560548129, + "pass@32": 0.9897432546798159, + "pass@64": 0.9915307632141225, + "pass@128": 0.9943661971830986 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 380308, + "answer_accuracy": 0.9735112221494102, + "avg_response_len": 194.05903915465268, + "resp_tokens_sum": 75810328, + "pass_at_k": { + "pass@1": 0.9735112221494102, + "pass@2": 0.9787569513601512, + "pass@4": 0.9820316175971947, + "pass@8": 0.984491002664138, + "pass@16": 0.9865770081045345, + "pass@32": 0.9884411361014508, + "pass@64": 0.9900010810421327, + "pass@128": 0.9911533420707732 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8771568251533742, + "pass@2": 0.8951967900101444, + "pass@4": 0.9078227647732684, + "pass@8": 0.9182064081148184, + "pass@16": 0.9283328420291027, + "pass@32": 0.9387767921163173, + "pass@64": 0.9482679411296987, + "pass@128": 0.9539877300613497 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9987926136363636, + "pass@2": 0.999673034478645, + "pass@4": 0.9999682039745033, + "pass@8": 0.9999997498310389, + "pass@16": 0.9999999999945943, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9761548913043478, + "pass@2": 0.9806904741526873, + "pass@4": 0.9844550450215461, + "pass@8": 0.9874040842618942, + "pass@16": 0.9900450599720992, + "pass@32": 0.9923200953194031, + "pass@64": 0.9941919743959912, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9873522564469914, + "pass@2": 0.9888020187036076, + "pass@4": 0.9894069616656083, + "pass@8": 0.990037327806008, + "pass@16": 0.990786092204451, + "pass@32": 0.9913001711780407, + "pass@64": 0.9914031358915707, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9812590317919075, + "pass@2": 0.9833455890719586, + "pass@4": 0.9854745302790909, + "pass@8": 0.9875048076156542, + "pass@16": 0.9886360584882343, + "pass@32": 0.9891581060193411, + "pass@64": 0.9898843928705129, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.9748459507042253, + "pass@2": 0.9835442774758789, + "pass@4": 0.9879733210285333, + "pass@8": 0.9911431418738322, + "pass@16": 0.99338839349148, + "pass@32": 0.9942874400881745, + "pass@64": 0.9943660901808677, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.9630660553892215, + "pass@2": 0.9748923658588334, + "pass@4": 0.9807665619267653, + "pass@8": 0.9838964621748524, + "pass@16": 0.9860050132910293, + "pass@32": 0.9884616006328849, + "pass@64": 0.9905750388985808, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 369953, + "answer_accuracy": 0.9676122572815534, + "avg_response_len": 207.69187834784066, + "resp_tokens_sum": 79408082, + "pass_at_k": { + "pass@1": 0.9676122572815534, + "pass@2": 0.9737666123411428, + "pass@4": 0.9775367138364485, + "pass@8": 0.9801888802899018, + "pass@16": 0.9823744035928921, + "pass@32": 0.9842478500163607, + "pass@64": 0.985680068574648, + "pass@128": 0.986943421493137 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.884071828358209, + "pass@2": 0.9018414326007759, + "pass@4": 0.9138320730431082, + "pass@8": 0.921710395734955, + "pass@16": 0.9283555561753828, + "pass@32": 0.9354921104145758, + "pass@64": 0.9430789258299536, + "pass@128": 0.9522388059701492 + }, + "2": { + "pass@1": 0.9994419642857143, + "pass@2": 0.9999469206974128, + "pass@4": 0.9999993585623226, + "pass@8": 0.9999999999514184, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999985652303667, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9995202850877193, + "pass@2": 0.9999381245107519, + "pass@4": 0.9999989376327959, + "pass@8": 0.9999999998454224, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9627068014705882, + "pass@2": 0.9684022550949515, + "pass@4": 0.972176392013498, + "pass@8": 0.9752606239283936, + "pass@16": 0.9784406334094804, + "pass@32": 0.9813456000064947, + "pass@64": 0.9835626145733192, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9906348580441641, + "pass@2": 0.9928117781117265, + "pass@4": 0.9941831030979172, + "pass@8": 0.9953566355803547, + "pass@16": 0.9962564144193743, + "pass@32": 0.9967623707930423, + "pass@64": 0.996844989203979, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9671182753164557, + "pass@2": 0.9765255126831456, + "pass@4": 0.9817781986112496, + "pass@8": 0.9850473862203449, + "pass@16": 0.9867960296777932, + "pass@32": 0.987306393671264, + "pass@64": 0.9873417337459038, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.9617351726726727, + "pass@2": 0.9702019933319148, + "pass@4": 0.9758465687284583, + "pass@8": 0.9804503225139821, + "pass@16": 0.9841829457607862, + "pass@32": 0.9867784982472595, + "pass@64": 0.9878811072240533, + "pass@128": 0.987987987987988 + }, + "9": { + "pass@1": 0.9398110932475884, + "pass@2": 0.9515331069448313, + "pass@4": 0.9577947668117046, + "pass@8": 0.9619601474829078, + "pass@16": 0.9656259394422011, + "pass@32": 0.9689428782955865, + "pass@64": 0.9708018571624879, + "pass@128": 0.9710610932475884 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.35872109375, + "avg_response_len": 308.98884765625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.35872109375, + "pass@2": 0.41100600393701425, + "pass@4": 0.4551079980596173, + "pass@8": 0.49294967546783774, + "pass@16": 0.5268697916522348, + "pass@32": 0.5577538666702029, + "pass@64": 0.5862138770542571, + "pass@128": 0.6133 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8412578125, + "pass@2": 0.8727789124015755, + "pass@4": 0.8934548995125605, + "pass@8": 0.9075329905007414, + "pass@16": 0.9182517278001402, + "pass@32": 0.9274958720939278, + "pass@64": 0.935855727747411, + "pass@128": 0.943 + }, + "12": { + "pass@1": 0.73640625, + "pass@2": 0.7898687253937015, + "pass@4": 0.8290561062992129, + "pass@8": 0.85906084829708, + "pass@16": 0.8824044988142241, + "pass@32": 0.9000195055978156, + "pass@64": 0.9129328391884791, + "pass@128": 0.923 + }, + "13": { + "pass@1": 0.515921875, + "pass@2": 0.5871433316929133, + "pass@4": 0.6421769704724403, + "pass@8": 0.6854092934107159, + "pass@16": 0.7227572317193646, + "pass@32": 0.7569873779406892, + "pass@64": 0.7891560872092849, + "pass@128": 0.821 + }, + "14": { + "pass@1": 0.3721171875, + "pass@2": 0.43426537893700806, + "pass@4": 0.48625735648668855, + "pass@8": 0.5317790693685567, + "pass@16": 0.5730995366637053, + "pass@32": 0.6105820811127548, + "pass@64": 0.644858778110634, + "pass@128": 0.678 + }, + "15": { + "pass@1": 0.24996875, + "pass@2": 0.3086731053149607, + "pass@4": 0.36075944197600296, + "pass@8": 0.4084037456271807, + "pass@16": 0.45326894538704726, + "pass@32": 0.49515244148618925, + "pass@64": 0.5344616927593079, + "pass@128": 0.572 + }, + "16": { + "pass@1": 0.211671875, + "pass@2": 0.26013779527559056, + "pass@4": 0.3009350399325082, + "pass@8": 0.3366217553121025, + "pass@16": 0.3708985859048203, + "pass@32": 0.404999043991755, + "pass@64": 0.43899124743731543, + "pass@128": 0.475 + }, + "17": { + "pass@1": 0.180671875, + "pass@2": 0.23036884842519675, + "pass@4": 0.2757752613423318, + "pass@8": 0.31739147878842566, + "pass@16": 0.35620930247494764, + "pass@32": 0.39195929393152495, + "pass@64": 0.4252221583031679, + "pass@128": 0.456 + }, + "18": { + "pass@1": 0.1679921875, + "pass@2": 0.21613029035433068, + "pass@4": 0.26094241919760003, + "pass@8": 0.3013311650532052, + "pass@16": 0.3382275798525124, + "pass@32": 0.3717808442919629, + "pass@64": 0.4022645565483507, + "pass@128": 0.43 + }, + "19": { + "pass@1": 0.158765625, + "pass@2": 0.21092691929133855, + "pass@4": 0.25805894235095594, + "pass@8": 0.2979729863463644, + "pass@16": 0.33230309169158023, + "pass@32": 0.3635323991056966, + "pass@64": 0.3926283939468405, + "pass@128": 0.421 + }, + "20": { + "pass@1": 0.1524375, + "pass@2": 0.19976673228346445, + "pass@4": 0.2436635430258716, + "pass@8": 0.2839934219739605, + "pass@16": 0.3212774162139301, + "pass@32": 0.35502980714971066, + "pass@64": 0.3857672892917629, + "pass@128": 0.414 + } + }, + "per_op_accuracy": { + "11": 0.8412578125, + "12": 0.73640625, + "13": 0.515921875, + "14": 0.3721171875, + "15": 0.24996875, + "16": 0.211671875, + "17": 0.180671875, + "18": 0.1679921875, + "19": 0.158765625, + "20": 0.1524375 + }, + "per_op_avg_response_len": { + "11": 288.5913671875, + "12": 297.7015234375, + "13": 307.1338671875, + "14": 315.290421875, + "15": 311.8331640625, + "16": 310.4463359375, + "17": 315.9180625, + "18": 314.4411640625, + "19": 312.2506640625, + "20": 316.28190625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 288.5913671875, + "12": 297.7015234375, + "13": 307.1338671875, + "14": 315.290421875, + "15": 311.8331640625, + "16": 310.4463359375, + "17": 315.9180625, + "18": 314.4411640625, + "19": 312.2506640625, + "20": 316.28190625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 156483, + "answer_accuracy": 0.3633056277860327, + "avg_response_len": 293.30141158989596, + "resp_tokens_sum": 126330784, + "pass_at_k": { + "pass@1": 0.3633056277860327, + "pass@2": 0.41730934322752805, + "pass@4": 0.4622783424453071, + "pass@8": 0.4998161733608168, + "pass@16": 0.5321484198312386, + "pass@32": 0.5607132784926532, + "pass@64": 0.5864180840706916, + "pass@128": 0.611292719167905 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8217289305949008, + "pass@2": 0.8557530642858737, + "pass@4": 0.8781868379235878, + "pass@8": 0.8928124041359038, + "pass@16": 0.90415402974055, + "pass@32": 0.9134579174534119, + "pass@64": 0.9219840469693056, + "pass@128": 0.9320113314447592 + }, + "12": { + "pass@1": 0.7351694915254238, + "pass@2": 0.788351767204946, + "pass@4": 0.8296121509705352, + "pass@8": 0.8631458462120339, + "pass@16": 0.8882166657918442, + "pass@32": 0.9056702384232721, + "pass@64": 0.9167729138632694, + "pass@128": 0.923728813559322 + }, + "13": { + "pass@1": 0.5215762147335423, + "pass@2": 0.602112670624244, + "pass@4": 0.6653502091627258, + "pass@8": 0.7142055909099876, + "pass@16": 0.7537658644644787, + "pass@32": 0.7883226602707918, + "pass@64": 0.8193713978709286, + "pass@128": 0.8463949843260188 + }, + "14": { + "pass@1": 0.3340147975077882, + "pass@2": 0.397272533544288, + "pass@4": 0.4527668489452836, + "pass@8": 0.5028074674389132, + "pass@16": 0.5481987968856181, + "pass@32": 0.5876974515021655, + "pass@64": 0.6210981694258276, + "pass@128": 0.6510903426791277 + }, + "15": { + "pass@1": 0.27140534682080925, + "pass@2": 0.32894390617177177, + "pass@4": 0.3785269061381776, + "pass@8": 0.42405549456100855, + "pass@16": 0.46767789842350194, + "pass@32": 0.5084233147816724, + "pass@64": 0.5450481432462191, + "pass@128": 0.5809248554913294 + }, + "16": { + "pass@1": 0.2229586693548387, + "pass@2": 0.27569334201168394, + "pass@4": 0.31796193711673143, + "pass@8": 0.35152289464494024, + "pass@16": 0.380881906006343, + "pass@32": 0.40832885380195727, + "pass@64": 0.4346231078734797, + "pass@128": 0.4645161290322581 + }, + "17": { + "pass@1": 0.1877032153179191, + "pass@2": 0.2378149037367438, + "pass@4": 0.28208731558482936, + "pass@8": 0.3191592222713673, + "pass@16": 0.3518104209106715, + "pass@32": 0.3820936581550958, + "pass@64": 0.41107801511864955, + "pass@128": 0.4421965317919075 + }, + "18": { + "pass@1": 0.16616071428571427, + "pass@2": 0.21133858267716538, + "pass@4": 0.2523077339439713, + "pass@8": 0.28922715703966473, + "pass@16": 0.3217381726043357, + "pass@32": 0.35019897098246955, + "pass@64": 0.37583858090720185, + "pass@128": 0.4 + }, + "19": { + "pass@1": 0.17372841591591592, + "pass@2": 0.2298645840722612, + "pass@4": 0.27871888086061297, + "pass@8": 0.31920558010112116, + "pass@16": 0.35332419941478427, + "pass@32": 0.38412272386745866, + "pass@64": 0.413135659750011, + "pass@128": 0.43843843843843844 + }, + "20": { + "pass@1": 0.16403903903903905, + "pass@2": 0.21445424262845508, + "pass@4": 0.25837019950208934, + "pass@8": 0.29514379006139285, + "pass@16": 0.32666536880092817, + "pass@32": 0.3556550693689438, + "pass@64": 0.3840131899528545, + "pass@128": 0.4144144144144144 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 154623, + "answer_accuracy": 0.3592007694023194, + "avg_response_len": 305.7103660236396, + "resp_tokens_sum": 131597307, + "pass_at_k": { + "pass@1": 0.3592007694023194, + "pass@2": 0.4109362085900062, + "pass@4": 0.4542472720292225, + "pass@8": 0.4916467539822679, + "pass@16": 0.5255968806707716, + "pass@32": 0.5569423202420237, + "pass@64": 0.5866195188846247, + "pass@128": 0.6149271483794231 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8489975527108434, + "pass@2": 0.882086540057869, + "pass@4": 0.9039521694765564, + "pass@8": 0.9186704477726292, + "pass@16": 0.9294459191329357, + "pass@32": 0.9399804052926921, + "pass@64": 0.9501763525977892, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.740130289968652, + "pass@2": 0.7924894478315606, + "pass@4": 0.8298025234110624, + "pass@8": 0.8583415348389902, + "pass@16": 0.8811457315129261, + "pass@32": 0.898488218747374, + "pass@64": 0.9119921067515657, + "pass@128": 0.9216300940438872 + }, + "13": { + "pass@1": 0.4885329131652661, + "pass@2": 0.5524010785416529, + "pass@4": 0.6016827056281829, + "pass@8": 0.6424268441509672, + "pass@16": 0.6814935198674684, + "pass@32": 0.719642728564925, + "pass@64": 0.7559772746835947, + "pass@128": 0.7927170868347339 + }, + "14": { + "pass@1": 0.4045138888888889, + "pass@2": 0.4613771037666344, + "pass@4": 0.505874457852856, + "pass@8": 0.5456155359914777, + "pass@16": 0.5829306088795296, + "pass@32": 0.6183434242052145, + "pass@64": 0.6531282479677968, + "pass@128": 0.6871345029239766 + }, + "15": { + "pass@1": 0.26129261363636364, + "pass@2": 0.3235124373657837, + "pass@4": 0.37589588829805376, + "pass@8": 0.42043231436330086, + "pass@16": 0.45993852789377043, + "pass@32": 0.49402710124836785, + "pass@64": 0.5220525337703099, + "pass@128": 0.5454545454545454 + }, + "16": { + "pass@1": 0.21287393162393162, + "pass@2": 0.26000866478228674, + "pass@4": 0.300428627244244, + "pass@8": 0.33815033021506824, + "pass@16": 0.3761938301457166, + "pass@32": 0.41416824184837775, + "pass@64": 0.4506074301431214, + "pass@128": 0.4843304843304843 + }, + "17": { + "pass@1": 0.1943662655279503, + "pass@2": 0.24511312784271538, + "pass@4": 0.2901540472231345, + "pass@8": 0.3315383250511585, + "pass@16": 0.3698795046319779, + "pass@32": 0.40524016010241276, + "pass@64": 0.43942643349381627, + "pass@128": 0.4720496894409938 + }, + "18": { + "pass@1": 0.17589613970588236, + "pass@2": 0.22981378821213524, + "pass@4": 0.2799638864994816, + "pass@8": 0.32366264093031627, + "pass@16": 0.3623485931753083, + "pass@32": 0.39840148766647915, + "pass@64": 0.43414839380764614, + "pass@128": 0.4676470588235294 + }, + "19": { + "pass@1": 0.14046223958333334, + "pass@2": 0.19077498125234343, + "pass@4": 0.2388992863057297, + "pass@8": 0.2804475024195655, + "pass@16": 0.315365405487468, + "pass@32": 0.34737330408706285, + "pass@64": 0.38110748946513323, + "pass@128": 0.41964285714285715 + }, + "20": { + "pass@1": 0.13994666916167664, + "pass@2": 0.18604130322033105, + "pass@4": 0.2294044430823391, + "pass@8": 0.2700214660362194, + "pass@16": 0.3089193843131219, + "pass@32": 0.34388208870332215, + "pass@64": 0.37594382323777076, + "pass@128": 0.40718562874251496 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 148057, + "answer_accuracy": 0.35351323731662593, + "avg_response_len": 328.4918293474939, + "resp_tokens_sum": 137577634, + "pass_at_k": { + "pass@1": 0.35351323731662593, + "pass@2": 0.4045952414184789, + "pass@4": 0.44861851541364584, + "pass@8": 0.4872271692166349, + "pass@16": 0.5227494419603741, + "pass@32": 0.5555444564793105, + "pass@64": 0.5855869424314432, + "pass@128": 0.6136919315403423 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8549851190476191, + "pass@2": 0.8820487282839649, + "pass@4": 0.8995010332041826, + "pass@8": 0.9122908672389048, + "pass@16": 0.9222517782526704, + "pass@32": 0.9300689926212681, + "pass@64": 0.9363073336659694, + "pass@128": 0.9396825396825397 + }, + "12": { + "pass@1": 0.7341121941896025, + "pass@2": 0.7889543301066725, + "pass@4": 0.8277259935397981, + "pass@8": 0.855340272551626, + "pass@16": 0.8773403999121976, + "pass@32": 0.8953960226775695, + "pass@64": 0.9096934117037676, + "pass@128": 0.9235474006116208 + }, + "13": { + "pass@1": 0.5405333719135802, + "pass@2": 0.6106858170506467, + "pass@4": 0.6639800241983641, + "pass@8": 0.7044176745325131, + "pass@16": 0.7376936245756492, + "pass@32": 0.7672839358537915, + "pass@64": 0.7959651982296765, + "pass@128": 0.8271604938271605 + }, + "14": { + "pass@1": 0.37553319732937684, + "pass@2": 0.44198791151662414, + "pass@4": 0.49824965397500376, + "pass@8": 0.5453334095299108, + "pass@16": 0.5868411116517013, + "pass@32": 0.6245037035678842, + "pass@64": 0.6590991243915612, + "pass@128": 0.6943620178041543 + }, + "15": { + "pass@1": 0.21554301697530864, + "pass@2": 0.271911757558083, + "pass@4": 0.32636879417850534, + "pass@8": 0.3794379036085877, + "pass@16": 0.4310885133567016, + "pass@32": 0.4821266393820033, + "pass@64": 0.5357953674441768, + "pass@128": 0.5895061728395061 + }, + "16": { + "pass@1": 0.20010601032448377, + "pass@2": 0.24604666051146248, + "pass@4": 0.285889059774608, + "pass@8": 0.3214126609636647, + "pass@16": 0.35628660962155617, + "pass@32": 0.3924603139361884, + "pass@64": 0.4309583363312713, + "pass@128": 0.4749262536873156 + }, + "17": { + "pass@1": 0.16006212349397592, + "pass@2": 0.2083086282136419, + "pass@4": 0.255251346217265, + "pass@8": 0.30182845547006043, + "pass@16": 0.34753522996493463, + "pass@32": 0.3893601104123036, + "pass@64": 0.4261863056840549, + "pass@128": 0.45481927710843373 + }, + "18": { + "pass@1": 0.16139112903225805, + "pass@2": 0.2065325755651512, + "pass@4": 0.24982900292705337, + "pass@8": 0.2905043941065011, + "pass@16": 0.33038934761738825, + "pass@32": 0.36695064077901834, + "pass@64": 0.39713096560074196, + "pass@128": 0.42258064516129035 + }, + "19": { + "pass@1": 0.16229229607250756, + "pass@2": 0.21233117551204875, + "pass@4": 0.25672324720739215, + "pass@8": 0.29440231830730196, + "pass@16": 0.3283485107029244, + "pass@32": 0.35922085161504386, + "pass@64": 0.3836921534435109, + "pass@128": 0.40483383685800606 + }, + "20": { + "pass@1": 0.1533643018018018, + "pass@2": 0.19884586850630145, + "pass@4": 0.2432588066131373, + "pass@8": 0.2868569676499088, + "pass@16": 0.32828460673465015, + "pass@32": 0.3655857400088371, + "pass@64": 0.3973743545827233, + "pass@128": 0.42042042042042044 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6488166118421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6488166118421053, + "pass@2": 0.6792278802320614, + "pass@4": 0.7041148738085226, + "pass@8": 0.7252115140837497, + "pass@16": 0.7440178852263484, + "pass@32": 0.7610676560199607, + "pass@64": 0.7766946261338011, + "pass@128": 0.7915263157894736 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.893609375, + "pass@2": 0.9112267470472445, + "pass@4": 0.9226303084926886, + "pass@8": 0.9307501273420538, + "pass@16": 0.9379550421861264, + "pass@32": 0.9454594928804226, + "pass@64": 0.9529819802272322, + "pass@128": 0.96 + }, + "2": { + "pass@1": 0.999796875, + "pass@2": 0.9999814222440945, + "pass@4": 0.9999997754968128, + "pass@8": 0.9999999999829965, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9995546875, + "pass@2": 0.9998916092519686, + "pass@4": 0.9999895073115861, + "pass@8": 0.9999999174442428, + "pass@16": 0.9999999999982161, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9980078125, + "pass@2": 0.998831938976378, + "pass@4": 0.9989930251218597, + "pass@8": 0.9989999850429369, + "pass@16": 0.998999999999976, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9709375, + "pass@2": 0.9759972933070866, + "pass@4": 0.9792994161979752, + "pass@8": 0.9816848512692736, + "pass@16": 0.9839470682252031, + "pass@32": 0.9859399222232034, + "pass@64": 0.987406075384883, + "pass@128": 0.989 + }, + "6": { + "pass@1": 0.9853984375, + "pass@2": 0.9884852362204726, + "pass@4": 0.9899697619985, + "pass@8": 0.9908947205863909, + "pass@16": 0.991577305674489, + "pass@32": 0.9919372248725359, + "pass@64": 0.9919995560032147, + "pass@128": 0.992 + }, + "7": { + "pass@1": 0.9759296875, + "pass@2": 0.9813612204724409, + "pass@4": 0.9843149371016123, + "pass@8": 0.986289730181675, + "pass@16": 0.9875135423464853, + "pass@32": 0.9882564902121656, + "pass@64": 0.9889711688992653, + "pass@128": 0.99 + }, + "8": { + "pass@1": 0.96628125, + "pass@2": 0.9747928149606305, + "pass@4": 0.9798431611361081, + "pass@8": 0.9841081004997401, + "pass@16": 0.9874667032833114, + "pass@32": 0.9893074023160056, + "pass@64": 0.9899354839743831, + "pass@128": 0.99 + }, + "9": { + "pass@1": 0.9507890625, + "pass@2": 0.9647014025590553, + "pass@4": 0.9720627289088867, + "pass@8": 0.9767945805634863, + "pass@16": 0.980182241064308, + "pass@32": 0.9828462651726455, + "pass@64": 0.9847648615106731, + "pass@128": 0.986 + }, + "11": { + "pass@1": 0.8412578125, + "pass@2": 0.8727789124015755, + "pass@4": 0.8934548995125605, + "pass@8": 0.9075329905007414, + "pass@16": 0.9182517278001402, + "pass@32": 0.9274958720939278, + "pass@64": 0.935855727747411, + "pass@128": 0.943 + }, + "12": { + "pass@1": 0.73640625, + "pass@2": 0.7898687253937015, + "pass@4": 0.8290561062992129, + "pass@8": 0.85906084829708, + "pass@16": 0.8824044988142241, + "pass@32": 0.9000195055978156, + "pass@64": 0.9129328391884791, + "pass@128": 0.923 + }, + "13": { + "pass@1": 0.515921875, + "pass@2": 0.5871433316929133, + "pass@4": 0.6421769704724403, + "pass@8": 0.6854092934107159, + "pass@16": 0.7227572317193646, + "pass@32": 0.7569873779406892, + "pass@64": 0.7891560872092849, + "pass@128": 0.821 + }, + "14": { + "pass@1": 0.3721171875, + "pass@2": 0.43426537893700806, + "pass@4": 0.48625735648668855, + "pass@8": 0.5317790693685567, + "pass@16": 0.5730995366637053, + "pass@32": 0.6105820811127548, + "pass@64": 0.644858778110634, + "pass@128": 0.678 + }, + "15": { + "pass@1": 0.24996875, + "pass@2": 0.3086731053149607, + "pass@4": 0.36075944197600296, + "pass@8": 0.4084037456271807, + "pass@16": 0.45326894538704726, + "pass@32": 0.49515244148618925, + "pass@64": 0.5344616927593079, + "pass@128": 0.572 + }, + "16": { + "pass@1": 0.211671875, + "pass@2": 0.26013779527559056, + "pass@4": 0.3009350399325082, + "pass@8": 0.3366217553121025, + "pass@16": 0.3708985859048203, + "pass@32": 0.404999043991755, + "pass@64": 0.43899124743731543, + "pass@128": 0.475 + }, + "17": { + "pass@1": 0.180671875, + "pass@2": 0.23036884842519675, + "pass@4": 0.2757752613423318, + "pass@8": 0.31739147878842566, + "pass@16": 0.35620930247494764, + "pass@32": 0.39195929393152495, + "pass@64": 0.4252221583031679, + "pass@128": 0.456 + }, + "18": { + "pass@1": 0.1679921875, + "pass@2": 0.21613029035433068, + "pass@4": 0.26094241919760003, + "pass@8": 0.3013311650532052, + "pass@16": 0.3382275798525124, + "pass@32": 0.3717808442919629, + "pass@64": 0.4022645565483507, + "pass@128": 0.43 + }, + "19": { + "pass@1": 0.158765625, + "pass@2": 0.21092691929133855, + "pass@4": 0.25805894235095594, + "pass@8": 0.2979729863463644, + "pass@16": 0.33230309169158023, + "pass@32": 0.3635323991056966, + "pass@64": 0.3926283939468405, + "pass@128": 0.421 + }, + "20": { + "pass@1": 0.1524375, + "pass@2": 0.19976673228346445, + "pass@4": 0.2436635430258716, + "pass@8": 0.2839934219739605, + "pass@16": 0.3212774162139301, + "pass@32": 0.35502980714971066, + "pass@64": 0.3857672892917629, + "pass@128": 0.414 + } + }, + "per_op_accuracy": { + "10": 0.893609375, + "2": 0.999796875, + "3": 0.9995546875, + "4": 0.9980078125, + "5": 0.9709375, + "6": 0.9853984375, + "7": 0.9759296875, + "8": 0.96628125, + "9": 0.9507890625, + "11": 0.8412578125, + "12": 0.73640625, + "13": 0.515921875, + "14": 0.3721171875, + "15": 0.24996875, + "16": 0.211671875, + "17": 0.180671875, + "18": 0.1679921875, + "19": 0.158765625, + "20": 0.1524375 + }, + "per_op_avg_response_len": { + "10": 273.24859375, + "2": 114.352421875, + "3": 142.668546875, + "4": 157.6628125, + "5": 171.03015625, + "6": 195.4423125, + "7": 219.505125, + "8": 238.1589375, + "9": 253.0455234375, + "11": 288.5913671875, + "12": 297.7015234375, + "13": 307.1338671875, + "14": 315.290421875, + "15": 311.8331640625, + "16": 310.4463359375, + "17": 315.9180625, + "18": 314.4411640625, + "19": 312.2506640625, + "20": 316.28190625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.24859375, + "2": 114.352421875, + "3": 142.668546875, + "4": 157.6628125, + "5": 171.03015625, + "6": 195.4423125, + "7": 219.505125, + "8": 238.1589375, + "9": 253.0455234375, + "11": 288.5913671875, + "12": 297.7015234375, + "13": 307.1338671875, + "14": 315.290421875, + "15": 311.8331640625, + "16": 310.4463359375, + "17": 315.9180625, + "18": 314.4411640625, + "19": 312.2506640625, + "20": 316.28190625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 524981, + "answer_accuracy": 0.6483424063389187, + "avg_response_len": 243.34964457200442, + "resp_tokens_sum": 197047021, + "pass_at_k": { + "pass@1": 0.6483424063389187, + "pass@2": 0.6803206668330436, + "pass@4": 0.7059131493769168, + "pass@8": 0.7269858853485713, + "pass@16": 0.7450104273226786, + "pass@32": 0.7608087740158496, + "pass@64": 0.774999417439848, + "pass@128": 0.7888080935820424 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9188560103244838, + "pass@2": 0.9359165592409355, + "pass@4": 0.9455644327644885, + "pass@8": 0.9517458870956133, + "pass@16": 0.956694467156006, + "pass@32": 0.9617356981168745, + "pass@64": 0.9673014491619945, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9943115234375, + "pass@2": 0.9964159387303149, + "pass@4": 0.9968543389107613, + "pass@8": 0.9968749534243821, + "pass@16": 0.996874999999925, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9741071428571428, + "pass@2": 0.979054961879765, + "pass@4": 0.9813411186696901, + "pass@8": 0.9823550160738614, + "pass@16": 0.983211768811447, + "pass@32": 0.9839110645580986, + "pass@64": 0.9841223976613889, + "pass@128": 0.9841269841269841 + }, + "6": { + "pass@1": 0.9783869760479041, + "pass@2": 0.9840478982978925, + "pass@4": 0.986558948189111, + "pass@8": 0.9875558074943762, + "pass@16": 0.987963102168246, + "pass@32": 0.9880233341018126, + "pass@64": 0.9880239520939974, + "pass@128": 0.9880239520958084 + }, + "7": { + "pass@1": 0.9787120931952663, + "pass@2": 0.9838508421469508, + "pass@4": 0.9854995232659527, + "pass@8": 0.9862073748551169, + "pass@16": 0.9870352684360165, + "pass@32": 0.9882217903235329, + "pass@64": 0.9895597074034385, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.9613882211538461, + "pass@2": 0.9697350406319402, + "pass@4": 0.9748580922576983, + "pass@8": 0.9800074606967099, + "pass@16": 0.9842336623573533, + "pass@32": 0.9863401319498917, + "pass@64": 0.987086901456941, + "pass@128": 0.9871794871794872 + }, + "9": { + "pass@1": 0.9488556338028169, + "pass@2": 0.966649453809471, + "pass@4": 0.9763733091462155, + "pass@8": 0.9831086093800043, + "pass@16": 0.9874560548129, + "pass@32": 0.9897432546798159, + "pass@64": 0.9915307632141225, + "pass@128": 0.9943661971830986 + }, + "11": { + "pass@1": 0.8217289305949008, + "pass@2": 0.8557530642858737, + "pass@4": 0.8781868379235878, + "pass@8": 0.8928124041359038, + "pass@16": 0.90415402974055, + "pass@32": 0.9134579174534119, + "pass@64": 0.9219840469693056, + "pass@128": 0.9320113314447592 + }, + "12": { + "pass@1": 0.7351694915254238, + "pass@2": 0.788351767204946, + "pass@4": 0.8296121509705352, + "pass@8": 0.8631458462120339, + "pass@16": 0.8882166657918442, + "pass@32": 0.9056702384232721, + "pass@64": 0.9167729138632694, + "pass@128": 0.923728813559322 + }, + "13": { + "pass@1": 0.5215762147335423, + "pass@2": 0.602112670624244, + "pass@4": 0.6653502091627258, + "pass@8": 0.7142055909099876, + "pass@16": 0.7537658644644787, + "pass@32": 0.7883226602707918, + "pass@64": 0.8193713978709286, + "pass@128": 0.8463949843260188 + }, + "14": { + "pass@1": 0.3340147975077882, + "pass@2": 0.397272533544288, + "pass@4": 0.4527668489452836, + "pass@8": 0.5028074674389132, + "pass@16": 0.5481987968856181, + "pass@32": 0.5876974515021655, + "pass@64": 0.6210981694258276, + "pass@128": 0.6510903426791277 + }, + "15": { + "pass@1": 0.27140534682080925, + "pass@2": 0.32894390617177177, + "pass@4": 0.3785269061381776, + "pass@8": 0.42405549456100855, + "pass@16": 0.46767789842350194, + "pass@32": 0.5084233147816724, + "pass@64": 0.5450481432462191, + "pass@128": 0.5809248554913294 + }, + "16": { + "pass@1": 0.2229586693548387, + "pass@2": 0.27569334201168394, + "pass@4": 0.31796193711673143, + "pass@8": 0.35152289464494024, + "pass@16": 0.380881906006343, + "pass@32": 0.40832885380195727, + "pass@64": 0.4346231078734797, + "pass@128": 0.4645161290322581 + }, + "17": { + "pass@1": 0.1877032153179191, + "pass@2": 0.2378149037367438, + "pass@4": 0.28208731558482936, + "pass@8": 0.3191592222713673, + "pass@16": 0.3518104209106715, + "pass@32": 0.3820936581550958, + "pass@64": 0.41107801511864955, + "pass@128": 0.4421965317919075 + }, + "18": { + "pass@1": 0.16616071428571427, + "pass@2": 0.21133858267716538, + "pass@4": 0.2523077339439713, + "pass@8": 0.28922715703966473, + "pass@16": 0.3217381726043357, + "pass@32": 0.35019897098246955, + "pass@64": 0.37583858090720185, + "pass@128": 0.4 + }, + "19": { + "pass@1": 0.17372841591591592, + "pass@2": 0.2298645840722612, + "pass@4": 0.27871888086061297, + "pass@8": 0.31920558010112116, + "pass@16": 0.35332419941478427, + "pass@32": 0.38412272386745866, + "pass@64": 0.413135659750011, + "pass@128": 0.43843843843843844 + }, + "20": { + "pass@1": 0.16403903903903905, + "pass@2": 0.21445424262845508, + "pass@4": 0.25837019950208934, + "pass@8": 0.29514379006139285, + "pass@16": 0.32666536880092817, + "pass@32": 0.3556550693689438, + "pass@64": 0.3840131899528545, + "pass@128": 0.4144144144144144 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 534931, + "answer_accuracy": 0.6514650720966485, + "avg_response_len": 252.5911377143414, + "resp_tokens_sum": 207407635, + "pass_at_k": { + "pass@1": 0.6514650720966485, + "pass@2": 0.6810825697645152, + "pass@4": 0.7053459193672471, + "pass@8": 0.7261223030044127, + "pass@16": 0.7449127573547755, + "pass@32": 0.7622321699696901, + "pass@64": 0.778532305744286, + "pass@128": 0.7939204988308651 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8771568251533742, + "pass@2": 0.8951967900101444, + "pass@4": 0.9078227647732684, + "pass@8": 0.9182064081148184, + "pass@16": 0.9283328420291027, + "pass@32": 0.9387767921163173, + "pass@64": 0.9482679411296987, + "pass@128": 0.9539877300613497 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9987926136363636, + "pass@2": 0.999673034478645, + "pass@4": 0.9999682039745033, + "pass@8": 0.9999997498310389, + "pass@16": 0.9999999999945943, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9761548913043478, + "pass@2": 0.9806904741526873, + "pass@4": 0.9844550450215461, + "pass@8": 0.9874040842618942, + "pass@16": 0.9900450599720992, + "pass@32": 0.9923200953194031, + "pass@64": 0.9941919743959912, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9873522564469914, + "pass@2": 0.9888020187036076, + "pass@4": 0.9894069616656083, + "pass@8": 0.990037327806008, + "pass@16": 0.990786092204451, + "pass@32": 0.9913001711780407, + "pass@64": 0.9914031358915707, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9812590317919075, + "pass@2": 0.9833455890719586, + "pass@4": 0.9854745302790909, + "pass@8": 0.9875048076156542, + "pass@16": 0.9886360584882343, + "pass@32": 0.9891581060193411, + "pass@64": 0.9898843928705129, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.9748459507042253, + "pass@2": 0.9835442774758789, + "pass@4": 0.9879733210285333, + "pass@8": 0.9911431418738322, + "pass@16": 0.99338839349148, + "pass@32": 0.9942874400881745, + "pass@64": 0.9943660901808677, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.9630660553892215, + "pass@2": 0.9748923658588334, + "pass@4": 0.9807665619267653, + "pass@8": 0.9838964621748524, + "pass@16": 0.9860050132910293, + "pass@32": 0.9884616006328849, + "pass@64": 0.9905750388985808, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.8489975527108434, + "pass@2": 0.882086540057869, + "pass@4": 0.9039521694765564, + "pass@8": 0.9186704477726292, + "pass@16": 0.9294459191329357, + "pass@32": 0.9399804052926921, + "pass@64": 0.9501763525977892, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.740130289968652, + "pass@2": 0.7924894478315606, + "pass@4": 0.8298025234110624, + "pass@8": 0.8583415348389902, + "pass@16": 0.8811457315129261, + "pass@32": 0.898488218747374, + "pass@64": 0.9119921067515657, + "pass@128": 0.9216300940438872 + }, + "13": { + "pass@1": 0.4885329131652661, + "pass@2": 0.5524010785416529, + "pass@4": 0.6016827056281829, + "pass@8": 0.6424268441509672, + "pass@16": 0.6814935198674684, + "pass@32": 0.719642728564925, + "pass@64": 0.7559772746835947, + "pass@128": 0.7927170868347339 + }, + "14": { + "pass@1": 0.4045138888888889, + "pass@2": 0.4613771037666344, + "pass@4": 0.505874457852856, + "pass@8": 0.5456155359914777, + "pass@16": 0.5829306088795296, + "pass@32": 0.6183434242052145, + "pass@64": 0.6531282479677968, + "pass@128": 0.6871345029239766 + }, + "15": { + "pass@1": 0.26129261363636364, + "pass@2": 0.3235124373657837, + "pass@4": 0.37589588829805376, + "pass@8": 0.42043231436330086, + "pass@16": 0.45993852789377043, + "pass@32": 0.49402710124836785, + "pass@64": 0.5220525337703099, + "pass@128": 0.5454545454545454 + }, + "16": { + "pass@1": 0.21287393162393162, + "pass@2": 0.26000866478228674, + "pass@4": 0.300428627244244, + "pass@8": 0.33815033021506824, + "pass@16": 0.3761938301457166, + "pass@32": 0.41416824184837775, + "pass@64": 0.4506074301431214, + "pass@128": 0.4843304843304843 + }, + "17": { + "pass@1": 0.1943662655279503, + "pass@2": 0.24511312784271538, + "pass@4": 0.2901540472231345, + "pass@8": 0.3315383250511585, + "pass@16": 0.3698795046319779, + "pass@32": 0.40524016010241276, + "pass@64": 0.43942643349381627, + "pass@128": 0.4720496894409938 + }, + "18": { + "pass@1": 0.17589613970588236, + "pass@2": 0.22981378821213524, + "pass@4": 0.2799638864994816, + "pass@8": 0.32366264093031627, + "pass@16": 0.3623485931753083, + "pass@32": 0.39840148766647915, + "pass@64": 0.43414839380764614, + "pass@128": 0.4676470588235294 + }, + "19": { + "pass@1": 0.14046223958333334, + "pass@2": 0.19077498125234343, + "pass@4": 0.2388992863057297, + "pass@8": 0.2804475024195655, + "pass@16": 0.315365405487468, + "pass@32": 0.34737330408706285, + "pass@64": 0.38110748946513323, + "pass@128": 0.41964285714285715 + }, + "20": { + "pass@1": 0.13994666916167664, + "pass@2": 0.18604130322033105, + "pass@4": 0.2294044430823391, + "pass@8": 0.2700214660362194, + "pass@16": 0.3089193843131219, + "pass@32": 0.34388208870332215, + "pass@64": 0.37594382323777076, + "pass@128": 0.40718562874251496 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 518010, + "answer_accuracy": 0.6465814227512382, + "avg_response_len": 270.8421323294456, + "resp_tokens_sum": 216985716, + "pass_at_k": { + "pass@1": 0.6465814227512382, + "pass@2": 0.6762224797865887, + "pass@4": 0.7010356201730161, + "pass@8": 0.7224846593869281, + "pass@16": 0.7420975423592175, + "pass@32": 0.7601357708258796, + "pass@64": 0.776524499196065, + "pass@128": 0.7918197795174948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.884071828358209, + "pass@2": 0.9018414326007759, + "pass@4": 0.9138320730431082, + "pass@8": 0.921710395734955, + "pass@16": 0.9283555561753828, + "pass@32": 0.9354921104145758, + "pass@64": 0.9430789258299536, + "pass@128": 0.9522388059701492 + }, + "2": { + "pass@1": 0.9994419642857143, + "pass@2": 0.9999469206974128, + "pass@4": 0.9999993585623226, + "pass@8": 0.9999999999514184, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999985652303667, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9995202850877193, + "pass@2": 0.9999381245107519, + "pass@4": 0.9999989376327959, + "pass@8": 0.9999999998454224, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9627068014705882, + "pass@2": 0.9684022550949515, + "pass@4": 0.972176392013498, + "pass@8": 0.9752606239283936, + "pass@16": 0.9784406334094804, + "pass@32": 0.9813456000064947, + "pass@64": 0.9835626145733192, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9906348580441641, + "pass@2": 0.9928117781117265, + "pass@4": 0.9941831030979172, + "pass@8": 0.9953566355803547, + "pass@16": 0.9962564144193743, + "pass@32": 0.9967623707930423, + "pass@64": 0.996844989203979, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9671182753164557, + "pass@2": 0.9765255126831456, + "pass@4": 0.9817781986112496, + "pass@8": 0.9850473862203449, + "pass@16": 0.9867960296777932, + "pass@32": 0.987306393671264, + "pass@64": 0.9873417337459038, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.9617351726726727, + "pass@2": 0.9702019933319148, + "pass@4": 0.9758465687284583, + "pass@8": 0.9804503225139821, + "pass@16": 0.9841829457607862, + "pass@32": 0.9867784982472595, + "pass@64": 0.9878811072240533, + "pass@128": 0.987987987987988 + }, + "9": { + "pass@1": 0.9398110932475884, + "pass@2": 0.9515331069448313, + "pass@4": 0.9577947668117046, + "pass@8": 0.9619601474829078, + "pass@16": 0.9656259394422011, + "pass@32": 0.9689428782955865, + "pass@64": 0.9708018571624879, + "pass@128": 0.9710610932475884 + }, + "11": { + "pass@1": 0.8549851190476191, + "pass@2": 0.8820487282839649, + "pass@4": 0.8995010332041826, + "pass@8": 0.9122908672389048, + "pass@16": 0.9222517782526704, + "pass@32": 0.9300689926212681, + "pass@64": 0.9363073336659694, + "pass@128": 0.9396825396825397 + }, + "12": { + "pass@1": 0.7341121941896025, + "pass@2": 0.7889543301066725, + "pass@4": 0.8277259935397981, + "pass@8": 0.855340272551626, + "pass@16": 0.8773403999121976, + "pass@32": 0.8953960226775695, + "pass@64": 0.9096934117037676, + "pass@128": 0.9235474006116208 + }, + "13": { + "pass@1": 0.5405333719135802, + "pass@2": 0.6106858170506467, + "pass@4": 0.6639800241983641, + "pass@8": 0.7044176745325131, + "pass@16": 0.7376936245756492, + "pass@32": 0.7672839358537915, + "pass@64": 0.7959651982296765, + "pass@128": 0.8271604938271605 + }, + "14": { + "pass@1": 0.37553319732937684, + "pass@2": 0.44198791151662414, + "pass@4": 0.49824965397500376, + "pass@8": 0.5453334095299108, + "pass@16": 0.5868411116517013, + "pass@32": 0.6245037035678842, + "pass@64": 0.6590991243915612, + "pass@128": 0.6943620178041543 + }, + "15": { + "pass@1": 0.21554301697530864, + "pass@2": 0.271911757558083, + "pass@4": 0.32636879417850534, + "pass@8": 0.3794379036085877, + "pass@16": 0.4310885133567016, + "pass@32": 0.4821266393820033, + "pass@64": 0.5357953674441768, + "pass@128": 0.5895061728395061 + }, + "16": { + "pass@1": 0.20010601032448377, + "pass@2": 0.24604666051146248, + "pass@4": 0.285889059774608, + "pass@8": 0.3214126609636647, + "pass@16": 0.35628660962155617, + "pass@32": 0.3924603139361884, + "pass@64": 0.4309583363312713, + "pass@128": 0.4749262536873156 + }, + "17": { + "pass@1": 0.16006212349397592, + "pass@2": 0.2083086282136419, + "pass@4": 0.255251346217265, + "pass@8": 0.30182845547006043, + "pass@16": 0.34753522996493463, + "pass@32": 0.3893601104123036, + "pass@64": 0.4261863056840549, + "pass@128": 0.45481927710843373 + }, + "18": { + "pass@1": 0.16139112903225805, + "pass@2": 0.2065325755651512, + "pass@4": 0.24982900292705337, + "pass@8": 0.2905043941065011, + "pass@16": 0.33038934761738825, + "pass@32": 0.36695064077901834, + "pass@64": 0.39713096560074196, + "pass@128": 0.42258064516129035 + }, + "19": { + "pass@1": 0.16229229607250756, + "pass@2": 0.21233117551204875, + "pass@4": 0.25672324720739215, + "pass@8": 0.29440231830730196, + "pass@16": 0.3283485107029244, + "pass@32": 0.35922085161504386, + "pass@64": 0.3836921534435109, + "pass@128": 0.40483383685800606 + }, + "20": { + "pass@1": 0.1533643018018018, + "pass@2": 0.19884586850630145, + "pass@4": 0.2432588066131373, + "pass@8": 0.2868569676499088, + "pass@16": 0.32828460673465015, + "pass@32": 0.3655857400088371, + "pass@64": 0.3973743545827233, + "pass@128": 0.42042042042042044 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/summary.csv b/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..008855e18e742eda293c49a00ea409d3e1130656 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-22350,0.9711449652777778,0.35872109375,0.6488166118421053,0.0,0.0,0.0,196.12382552083332,308.98884765625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/summary.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..134bed9b207424ab6645b36ad653ae6f049bfd9c --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block16/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-22350", + "id_acc": 0.9711449652777778, + "ood_acc": 0.35872109375, + "total_acc": 0.6488166118421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.12382552083332, + "ood_avg_resp_len": 308.98884765625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 524981, + "answer_accuracy": 0.6483424063389187, + "avg_response_len": 243.34964457200442, + "resp_tokens_sum": 197047021, + "pass_at_k": { + "pass@1": 0.6483424063389187, + "pass@2": 0.6803206668330436, + "pass@4": 0.7059131493769168, + "pass@8": 0.7269858853485713, + "pass@16": 0.7450104273226786, + "pass@32": 0.7608087740158496, + "pass@64": 0.774999417439848, + "pass@128": 0.7888080935820424 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9188560103244838, + "pass@2": 0.9359165592409355, + "pass@4": 0.9455644327644885, + "pass@8": 0.9517458870956133, + "pass@16": 0.956694467156006, + "pass@32": 0.9617356981168745, + "pass@64": 0.9673014491619945, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9943115234375, + "pass@2": 0.9964159387303149, + "pass@4": 0.9968543389107613, + "pass@8": 0.9968749534243821, + "pass@16": 0.996874999999925, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9741071428571428, + "pass@2": 0.979054961879765, + "pass@4": 0.9813411186696901, + "pass@8": 0.9823550160738614, + "pass@16": 0.983211768811447, + "pass@32": 0.9839110645580986, + "pass@64": 0.9841223976613889, + "pass@128": 0.9841269841269841 + }, + "6": { + "pass@1": 0.9783869760479041, + "pass@2": 0.9840478982978925, + "pass@4": 0.986558948189111, + "pass@8": 0.9875558074943762, + "pass@16": 0.987963102168246, + "pass@32": 0.9880233341018126, + "pass@64": 0.9880239520939974, + "pass@128": 0.9880239520958084 + }, + "7": { + "pass@1": 0.9787120931952663, + "pass@2": 0.9838508421469508, + "pass@4": 0.9854995232659527, + "pass@8": 0.9862073748551169, + "pass@16": 0.9870352684360165, + "pass@32": 0.9882217903235329, + "pass@64": 0.9895597074034385, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.9613882211538461, + "pass@2": 0.9697350406319402, + "pass@4": 0.9748580922576983, + "pass@8": 0.9800074606967099, + "pass@16": 0.9842336623573533, + "pass@32": 0.9863401319498917, + "pass@64": 0.987086901456941, + "pass@128": 0.9871794871794872 + }, + "9": { + "pass@1": 0.9488556338028169, + "pass@2": 0.966649453809471, + "pass@4": 0.9763733091462155, + "pass@8": 0.9831086093800043, + "pass@16": 0.9874560548129, + "pass@32": 0.9897432546798159, + "pass@64": 0.9915307632141225, + "pass@128": 0.9943661971830986 + }, + "11": { + "pass@1": 0.8217289305949008, + "pass@2": 0.8557530642858737, + "pass@4": 0.8781868379235878, + "pass@8": 0.8928124041359038, + "pass@16": 0.90415402974055, + "pass@32": 0.9134579174534119, + "pass@64": 0.9219840469693056, + "pass@128": 0.9320113314447592 + }, + "12": { + "pass@1": 0.7351694915254238, + "pass@2": 0.788351767204946, + "pass@4": 0.8296121509705352, + "pass@8": 0.8631458462120339, + "pass@16": 0.8882166657918442, + "pass@32": 0.9056702384232721, + "pass@64": 0.9167729138632694, + "pass@128": 0.923728813559322 + }, + "13": { + "pass@1": 0.5215762147335423, + "pass@2": 0.602112670624244, + "pass@4": 0.6653502091627258, + "pass@8": 0.7142055909099876, + "pass@16": 0.7537658644644787, + "pass@32": 0.7883226602707918, + "pass@64": 0.8193713978709286, + "pass@128": 0.8463949843260188 + }, + "14": { + "pass@1": 0.3340147975077882, + "pass@2": 0.397272533544288, + "pass@4": 0.4527668489452836, + "pass@8": 0.5028074674389132, + "pass@16": 0.5481987968856181, + "pass@32": 0.5876974515021655, + "pass@64": 0.6210981694258276, + "pass@128": 0.6510903426791277 + }, + "15": { + "pass@1": 0.27140534682080925, + "pass@2": 0.32894390617177177, + "pass@4": 0.3785269061381776, + "pass@8": 0.42405549456100855, + "pass@16": 0.46767789842350194, + "pass@32": 0.5084233147816724, + "pass@64": 0.5450481432462191, + "pass@128": 0.5809248554913294 + }, + "16": { + "pass@1": 0.2229586693548387, + "pass@2": 0.27569334201168394, + "pass@4": 0.31796193711673143, + "pass@8": 0.35152289464494024, + "pass@16": 0.380881906006343, + "pass@32": 0.40832885380195727, + "pass@64": 0.4346231078734797, + "pass@128": 0.4645161290322581 + }, + "17": { + "pass@1": 0.1877032153179191, + "pass@2": 0.2378149037367438, + "pass@4": 0.28208731558482936, + "pass@8": 0.3191592222713673, + "pass@16": 0.3518104209106715, + "pass@32": 0.3820936581550958, + "pass@64": 0.41107801511864955, + "pass@128": 0.4421965317919075 + }, + "18": { + "pass@1": 0.16616071428571427, + "pass@2": 0.21133858267716538, + "pass@4": 0.2523077339439713, + "pass@8": 0.28922715703966473, + "pass@16": 0.3217381726043357, + "pass@32": 0.35019897098246955, + "pass@64": 0.37583858090720185, + "pass@128": 0.4 + }, + "19": { + "pass@1": 0.17372841591591592, + "pass@2": 0.2298645840722612, + "pass@4": 0.27871888086061297, + "pass@8": 0.31920558010112116, + "pass@16": 0.35332419941478427, + "pass@32": 0.38412272386745866, + "pass@64": 0.413135659750011, + "pass@128": 0.43843843843843844 + }, + "20": { + "pass@1": 0.16403903903903905, + "pass@2": 0.21445424262845508, + "pass@4": 0.25837019950208934, + "pass@8": 0.29514379006139285, + "pass@16": 0.32666536880092817, + "pass@32": 0.3556550693689438, + "pass@64": 0.3840131899528545, + "pass@128": 0.4144144144144144 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 534931, + "answer_accuracy": 0.6514650720966485, + "avg_response_len": 252.5911377143414, + "resp_tokens_sum": 207407635, + "pass_at_k": { + "pass@1": 0.6514650720966485, + "pass@2": 0.6810825697645152, + "pass@4": 0.7053459193672471, + "pass@8": 0.7261223030044127, + "pass@16": 0.7449127573547755, + "pass@32": 0.7622321699696901, + "pass@64": 0.778532305744286, + "pass@128": 0.7939204988308651 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8771568251533742, + "pass@2": 0.8951967900101444, + "pass@4": 0.9078227647732684, + "pass@8": 0.9182064081148184, + "pass@16": 0.9283328420291027, + "pass@32": 0.9387767921163173, + "pass@64": 0.9482679411296987, + "pass@128": 0.9539877300613497 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9987926136363636, + "pass@2": 0.999673034478645, + "pass@4": 0.9999682039745033, + "pass@8": 0.9999997498310389, + "pass@16": 0.9999999999945943, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9761548913043478, + "pass@2": 0.9806904741526873, + "pass@4": 0.9844550450215461, + "pass@8": 0.9874040842618942, + "pass@16": 0.9900450599720992, + "pass@32": 0.9923200953194031, + "pass@64": 0.9941919743959912, + "pass@128": 0.9971014492753624 + }, + "6": { + "pass@1": 0.9873522564469914, + "pass@2": 0.9888020187036076, + "pass@4": 0.9894069616656083, + "pass@8": 0.990037327806008, + "pass@16": 0.990786092204451, + "pass@32": 0.9913001711780407, + "pass@64": 0.9914031358915707, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.9812590317919075, + "pass@2": 0.9833455890719586, + "pass@4": 0.9854745302790909, + "pass@8": 0.9875048076156542, + "pass@16": 0.9886360584882343, + "pass@32": 0.9891581060193411, + "pass@64": 0.9898843928705129, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.9748459507042253, + "pass@2": 0.9835442774758789, + "pass@4": 0.9879733210285333, + "pass@8": 0.9911431418738322, + "pass@16": 0.99338839349148, + "pass@32": 0.9942874400881745, + "pass@64": 0.9943660901808677, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.9630660553892215, + "pass@2": 0.9748923658588334, + "pass@4": 0.9807665619267653, + "pass@8": 0.9838964621748524, + "pass@16": 0.9860050132910293, + "pass@32": 0.9884616006328849, + "pass@64": 0.9905750388985808, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.8489975527108434, + "pass@2": 0.882086540057869, + "pass@4": 0.9039521694765564, + "pass@8": 0.9186704477726292, + "pass@16": 0.9294459191329357, + "pass@32": 0.9399804052926921, + "pass@64": 0.9501763525977892, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.740130289968652, + "pass@2": 0.7924894478315606, + "pass@4": 0.8298025234110624, + "pass@8": 0.8583415348389902, + "pass@16": 0.8811457315129261, + "pass@32": 0.898488218747374, + "pass@64": 0.9119921067515657, + "pass@128": 0.9216300940438872 + }, + "13": { + "pass@1": 0.4885329131652661, + "pass@2": 0.5524010785416529, + "pass@4": 0.6016827056281829, + "pass@8": 0.6424268441509672, + "pass@16": 0.6814935198674684, + "pass@32": 0.719642728564925, + "pass@64": 0.7559772746835947, + "pass@128": 0.7927170868347339 + }, + "14": { + "pass@1": 0.4045138888888889, + "pass@2": 0.4613771037666344, + "pass@4": 0.505874457852856, + "pass@8": 0.5456155359914777, + "pass@16": 0.5829306088795296, + "pass@32": 0.6183434242052145, + "pass@64": 0.6531282479677968, + "pass@128": 0.6871345029239766 + }, + "15": { + "pass@1": 0.26129261363636364, + "pass@2": 0.3235124373657837, + "pass@4": 0.37589588829805376, + "pass@8": 0.42043231436330086, + "pass@16": 0.45993852789377043, + "pass@32": 0.49402710124836785, + "pass@64": 0.5220525337703099, + "pass@128": 0.5454545454545454 + }, + "16": { + "pass@1": 0.21287393162393162, + "pass@2": 0.26000866478228674, + "pass@4": 0.300428627244244, + "pass@8": 0.33815033021506824, + "pass@16": 0.3761938301457166, + "pass@32": 0.41416824184837775, + "pass@64": 0.4506074301431214, + "pass@128": 0.4843304843304843 + }, + "17": { + "pass@1": 0.1943662655279503, + "pass@2": 0.24511312784271538, + "pass@4": 0.2901540472231345, + "pass@8": 0.3315383250511585, + "pass@16": 0.3698795046319779, + "pass@32": 0.40524016010241276, + "pass@64": 0.43942643349381627, + "pass@128": 0.4720496894409938 + }, + "18": { + "pass@1": 0.17589613970588236, + "pass@2": 0.22981378821213524, + "pass@4": 0.2799638864994816, + "pass@8": 0.32366264093031627, + "pass@16": 0.3623485931753083, + "pass@32": 0.39840148766647915, + "pass@64": 0.43414839380764614, + "pass@128": 0.4676470588235294 + }, + "19": { + "pass@1": 0.14046223958333334, + "pass@2": 0.19077498125234343, + "pass@4": 0.2388992863057297, + "pass@8": 0.2804475024195655, + "pass@16": 0.315365405487468, + "pass@32": 0.34737330408706285, + "pass@64": 0.38110748946513323, + "pass@128": 0.41964285714285715 + }, + "20": { + "pass@1": 0.13994666916167664, + "pass@2": 0.18604130322033105, + "pass@4": 0.2294044430823391, + "pass@8": 0.2700214660362194, + "pass@16": 0.3089193843131219, + "pass@32": 0.34388208870332215, + "pass@64": 0.37594382323777076, + "pass@128": 0.40718562874251496 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 518010, + "answer_accuracy": 0.6465814227512382, + "avg_response_len": 270.8421323294456, + "resp_tokens_sum": 216985716, + "pass_at_k": { + "pass@1": 0.6465814227512382, + "pass@2": 0.6762224797865887, + "pass@4": 0.7010356201730161, + "pass@8": 0.7224846593869281, + "pass@16": 0.7420975423592175, + "pass@32": 0.7601357708258796, + "pass@64": 0.776524499196065, + "pass@128": 0.7918197795174948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.884071828358209, + "pass@2": 0.9018414326007759, + "pass@4": 0.9138320730431082, + "pass@8": 0.921710395734955, + "pass@16": 0.9283555561753828, + "pass@32": 0.9354921104145758, + "pass@64": 0.9430789258299536, + "pass@128": 0.9522388059701492 + }, + "2": { + "pass@1": 0.9994419642857143, + "pass@2": 0.9999469206974128, + "pass@4": 0.9999993585623226, + "pass@8": 0.9999999999514184, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9998633381924198, + "pass@2": 0.9999985652303667, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9995202850877193, + "pass@2": 0.9999381245107519, + "pass@4": 0.9999989376327959, + "pass@8": 0.9999999998454224, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9627068014705882, + "pass@2": 0.9684022550949515, + "pass@4": 0.972176392013498, + "pass@8": 0.9752606239283936, + "pass@16": 0.9784406334094804, + "pass@32": 0.9813456000064947, + "pass@64": 0.9835626145733192, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9906348580441641, + "pass@2": 0.9928117781117265, + "pass@4": 0.9941831030979172, + "pass@8": 0.9953566355803547, + "pass@16": 0.9962564144193743, + "pass@32": 0.9967623707930423, + "pass@64": 0.996844989203979, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9671182753164557, + "pass@2": 0.9765255126831456, + "pass@4": 0.9817781986112496, + "pass@8": 0.9850473862203449, + "pass@16": 0.9867960296777932, + "pass@32": 0.987306393671264, + "pass@64": 0.9873417337459038, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.9617351726726727, + "pass@2": 0.9702019933319148, + "pass@4": 0.9758465687284583, + "pass@8": 0.9804503225139821, + "pass@16": 0.9841829457607862, + "pass@32": 0.9867784982472595, + "pass@64": 0.9878811072240533, + "pass@128": 0.987987987987988 + }, + "9": { + "pass@1": 0.9398110932475884, + "pass@2": 0.9515331069448313, + "pass@4": 0.9577947668117046, + "pass@8": 0.9619601474829078, + "pass@16": 0.9656259394422011, + "pass@32": 0.9689428782955865, + "pass@64": 0.9708018571624879, + "pass@128": 0.9710610932475884 + }, + "11": { + "pass@1": 0.8549851190476191, + "pass@2": 0.8820487282839649, + "pass@4": 0.8995010332041826, + "pass@8": 0.9122908672389048, + "pass@16": 0.9222517782526704, + "pass@32": 0.9300689926212681, + "pass@64": 0.9363073336659694, + "pass@128": 0.9396825396825397 + }, + "12": { + "pass@1": 0.7341121941896025, + "pass@2": 0.7889543301066725, + "pass@4": 0.8277259935397981, + "pass@8": 0.855340272551626, + "pass@16": 0.8773403999121976, + "pass@32": 0.8953960226775695, + "pass@64": 0.9096934117037676, + "pass@128": 0.9235474006116208 + }, + "13": { + "pass@1": 0.5405333719135802, + "pass@2": 0.6106858170506467, + "pass@4": 0.6639800241983641, + "pass@8": 0.7044176745325131, + "pass@16": 0.7376936245756492, + "pass@32": 0.7672839358537915, + "pass@64": 0.7959651982296765, + "pass@128": 0.8271604938271605 + }, + "14": { + "pass@1": 0.37553319732937684, + "pass@2": 0.44198791151662414, + "pass@4": 0.49824965397500376, + "pass@8": 0.5453334095299108, + "pass@16": 0.5868411116517013, + "pass@32": 0.6245037035678842, + "pass@64": 0.6590991243915612, + "pass@128": 0.6943620178041543 + }, + "15": { + "pass@1": 0.21554301697530864, + "pass@2": 0.271911757558083, + "pass@4": 0.32636879417850534, + "pass@8": 0.3794379036085877, + "pass@16": 0.4310885133567016, + "pass@32": 0.4821266393820033, + "pass@64": 0.5357953674441768, + "pass@128": 0.5895061728395061 + }, + "16": { + "pass@1": 0.20010601032448377, + "pass@2": 0.24604666051146248, + "pass@4": 0.285889059774608, + "pass@8": 0.3214126609636647, + "pass@16": 0.35628660962155617, + "pass@32": 0.3924603139361884, + "pass@64": 0.4309583363312713, + "pass@128": 0.4749262536873156 + }, + "17": { + "pass@1": 0.16006212349397592, + "pass@2": 0.2083086282136419, + "pass@4": 0.255251346217265, + "pass@8": 0.30182845547006043, + "pass@16": 0.34753522996493463, + "pass@32": 0.3893601104123036, + "pass@64": 0.4261863056840549, + "pass@128": 0.45481927710843373 + }, + "18": { + "pass@1": 0.16139112903225805, + "pass@2": 0.2065325755651512, + "pass@4": 0.24982900292705337, + "pass@8": 0.2905043941065011, + "pass@16": 0.33038934761738825, + "pass@32": 0.36695064077901834, + "pass@64": 0.39713096560074196, + "pass@128": 0.42258064516129035 + }, + "19": { + "pass@1": 0.16229229607250756, + "pass@2": 0.21233117551204875, + "pass@4": 0.25672324720739215, + "pass@8": 0.29440231830730196, + "pass@16": 0.3283485107029244, + "pass@32": 0.35922085161504386, + "pass@64": 0.3836921534435109, + "pass@128": 0.40483383685800606 + }, + "20": { + "pass@1": 0.1533643018018018, + "pass@2": 0.19884586850630145, + "pass@4": 0.2432588066131373, + "pass@8": 0.2868569676499088, + "pass@16": 0.32828460673465015, + "pass@32": 0.3655857400088371, + "pass@64": 0.3973743545827233, + "pass@128": 0.42042042042042044 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/checkpoint-22350_metrics.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/checkpoint-22350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..87207f5cb1918b4bb35e9cf2bd7f49a5b9ba2d73 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/checkpoint-22350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-22350", + "id": { + "answer_accuracy": 0.9704357638888889, + "avg_response_len": 196.0621032986111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9704357638888889, + "pass@2": 0.9767754811898485, + "pass@4": 0.98051369874599, + "pass@8": 0.9830728620214397, + "pass@16": 0.9851039830409487, + "pass@32": 0.9867661849562662, + "pass@64": 0.9879488743177828, + "pass@128": 0.9886666666666667 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.89928125, + "pass@2": 0.916050565944882, + "pass@4": 0.9260351500749907, + "pass@8": 0.9330535750028963, + "pass@16": 0.9383338135951381, + "pass@32": 0.94246185430624, + "pass@64": 0.945903048004897, + "pass@128": 0.948 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999904035433071, + "pass@4": 0.9999999329771279, + "pass@8": 0.9999999999990998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9981171875, + "pass@2": 0.9982573818897639, + "pass@4": 0.9984843227409074, + "pass@8": 0.9988590133679512, + "pass@16": 0.9993672588247003, + "pass@32": 0.9998241803921225, + "pass@64": 0.9999927603421557, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9933359375, + "pass@2": 0.9954790846456693, + "pass@4": 0.9968326808211472, + "pass@8": 0.997768162863856, + "pass@16": 0.9984728041659322, + "pass@32": 0.998887565153522, + "pass@64": 0.9989968420736931, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9701796875, + "pass@2": 0.973689345472441, + "pass@4": 0.9764589475065616, + "pass@8": 0.979163521366934, + "pass@16": 0.9821057314308723, + "pass@32": 0.9846806191317323, + "pass@64": 0.9863707931066418, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.98528125, + "pass@2": 0.9902500000000002, + "pass@4": 0.9929865528683917, + "pass@8": 0.9945421401299686, + "pass@16": 0.995530902448261, + "pass@32": 0.9963896873403176, + "pass@64": 0.9972217048693288, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9786171875, + "pass@2": 0.983411909448819, + "pass@4": 0.9861397134420696, + "pass@8": 0.9878044142052393, + "pass@16": 0.9890889706751056, + "pass@32": 0.9902669840948403, + "pass@64": 0.9912530039689931, + "pass@128": 0.992 + }, + "8": { + "pass@1": 0.95978125, + "pass@2": 0.9705783710629925, + "pass@4": 0.9771872181289839, + "pass@8": 0.9815502427887551, + "pass@16": 0.9847671920564206, + "pass@32": 0.9873725342613687, + "pass@64": 0.9888198336231544, + "pass@128": 0.989 + }, + "9": { + "pass@1": 0.9494296875, + "pass@2": 0.9632722687007877, + "pass@4": 0.9704987701537305, + "pass@8": 0.974914688468251, + "pass@16": 0.9782691741721024, + "pass@32": 0.981012239926249, + "pass@64": 0.9829818828711779, + "pass@128": 0.984 + } + }, + "per_op_accuracy": { + "10": 0.89928125, + "2": 0.9998984375, + "3": 0.9981171875, + "4": 0.9933359375, + "5": 0.9701796875, + "6": 0.98528125, + "7": 0.9786171875, + "8": 0.95978125, + "9": 0.9494296875 + }, + "per_op_avg_response_len": { + "10": 273.267484375, + "2": 114.288421875, + "3": 142.6334140625, + "4": 157.7698125, + "5": 171.13790625, + "6": 195.2028984375, + "7": 219.23784375, + "8": 237.87965625, + "9": 253.1414921875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.267484375, + "2": 114.288421875, + "3": 142.6334140625, + "4": 157.7698125, + "5": 171.13790625, + "6": 195.2028984375, + "7": 219.23784375, + "8": 237.87965625, + "9": 253.1414921875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 366743, + "answer_accuracy": 0.967639205504897, + "avg_response_len": 186.43606731256332, + "resp_tokens_sum": 70660761, + "pass_at_k": { + "pass@1": 0.967639205504897, + "pass@2": 0.9765381305395345, + "pass@4": 0.98116641688786, + "pass@8": 0.9837187188521721, + "pass@16": 0.9854161336599176, + "pass@32": 0.9868387639028573, + "pass@64": 0.9880772059737805, + "pass@128": 0.9888551165146909 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9232577433628318, + "pass@2": 0.9413916422781228, + "pass@4": 0.9499321857444808, + "pass@8": 0.9535468343450594, + "pass@16": 0.9556178797377718, + "pass@32": 0.9578424969991439, + "pass@64": 0.9603845492642211, + "pass@128": 0.9616519174041298 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9871337890625, + "pass@2": 0.9921521284448819, + "pass@4": 0.9948190980033745, + "pass@8": 0.9961782344257655, + "pass@16": 0.9967684498096553, + "pass@32": 0.9968730556584248, + "pass@64": 0.9968749999667598, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9740575396825397, + "pass@2": 0.9775235126859141, + "pass@4": 0.979264063123062, + "pass@8": 0.980565740435064, + "pass@16": 0.9821661054021831, + "pass@32": 0.9838715548767617, + "pass@64": 0.9855250123602817, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.9662939745508982, + "pass@2": 0.9776823076288367, + "pass@4": 0.9837688846154713, + "pass@8": 0.987005164857485, + "pass@16": 0.9892880596338539, + "pass@32": 0.991436482518866, + "pass@64": 0.993178757094046, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9721015162721893, + "pass@2": 0.9787191911661931, + "pass@4": 0.9822101936777133, + "pass@8": 0.9845253393977582, + "pass@16": 0.986677006424018, + "pass@32": 0.98918608445364, + "pass@64": 0.9918771028850546, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.9414563301282052, + "pass@2": 0.9573518416868564, + "pass@4": 0.9671929527438877, + "pass@8": 0.9740068690727082, + "pass@16": 0.978930348505969, + "pass@32": 0.9823055554325953, + "pass@64": 0.9838388835534627, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.9477112676056338, + "pass@2": 0.966034642896751, + "pass@4": 0.974834940878869, + "pass@8": 0.979053728156307, + "pass@16": 0.9807582382316763, + "pass@32": 0.9815156079864068, + "pass@64": 0.9823999112442319, + "pass@128": 0.9830985915492958 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 380272, + "answer_accuracy": 0.9734190694626474, + "avg_response_len": 194.1332911820118, + "resp_tokens_sum": 75839335, + "pass_at_k": { + "pass@1": 0.9734190694626474, + "pass@2": 0.9783102164193357, + "pass@4": 0.9812914630039696, + "pass@8": 0.9835696312785593, + "pass@16": 0.9855613939498288, + "pass@32": 0.9872534388268681, + "pass@64": 0.9884731541106981, + "pass@128": 0.9891874180865007 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8846098542944786, + "pass@2": 0.9002780662769918, + "pass@4": 0.9112371256507044, + "pass@8": 0.920703275994343, + "pass@16": 0.928180327350798, + "pass@32": 0.9340597884127482, + "pass@64": 0.9389806489566199, + "pass@128": 0.941717791411043 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9972064393939394, + "pass@2": 0.9974264047959914, + "pass@4": 0.9978202057129222, + "pass@8": 0.9984495141444165, + "pass@16": 0.9992439943304724, + "pass@32": 0.9998493181625818, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979428624260355, + "pass@2": 0.9990397719330942, + "pass@4": 0.9997296856938741, + "pass@8": 0.999977983651016, + "pass@16": 0.999999898523723, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9704483695652174, + "pass@2": 0.9743309939518432, + "pass@4": 0.9780603405552569, + "pass@8": 0.9814549315371892, + "pass@16": 0.9847236492950605, + "pass@32": 0.9876855114215582, + "pass@64": 0.9896688751350827, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9924785100286533, + "pass@2": 0.9948129357218599, + "pass@4": 0.9960934836583821, + "pass@8": 0.9969392940630192, + "pass@16": 0.9974515227801705, + "pass@32": 0.9978507255774547, + "pass@64": 0.9985673352433165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9881909320809249, + "pass@2": 0.9905351087797553, + "pass@4": 0.9912523347876314, + "pass@8": 0.9913285922353743, + "pass@16": 0.9913294796978018, + "pass@32": 0.9913294797687862, + "pass@64": 0.9913294797687862, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.968794014084507, + "pass@2": 0.9768676527669956, + "pass@4": 0.9810858080239967, + "pass@8": 0.9833780732591677, + "pass@16": 0.9854529291572443, + "pass@32": 0.9874980186150435, + "pass@64": 0.9886114136046574, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9583645209580839, + "pass@2": 0.9689787501768117, + "pass@4": 0.9740201422239886, + "pass@8": 0.9778193850824559, + "pass@16": 0.9817966272607928, + "pass@32": 0.9852710419406909, + "pass@64": 0.9874857148809907, + "pass@128": 0.9880239520958084 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 370927, + "answer_accuracy": 0.970159754770673, + "avg_response_len": 207.57513548292602, + "resp_tokens_sum": 79363447, + "pass_at_k": { + "pass@1": 0.970159754770673, + "pass@2": 0.9754426333007337, + "pass@4": 0.9790719729564274, + "pass@8": 0.9819250475425196, + "pass@16": 0.9843271848900711, + "pass@32": 0.9861963807801907, + "pass@64": 0.9872859710833027, + "pass@128": 0.9879477736859725 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8892957089552239, + "pass@2": 0.9057556704665646, + "pass@4": 0.9162532422253183, + "pass@8": 0.9243341199575077, + "pass@16": 0.9307240765602192, + "pass@32": 0.9350739008984293, + "pass@64": 0.9379850574461723, + "pass@128": 0.9402985074626866 + }, + "2": { + "pass@1": 0.9997098214285715, + "pass@2": 0.999972581552306, + "pass@4": 0.9999998085060796, + "pass@8": 0.9999999999974281, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9971984329446064, + "pass@2": 0.9973955344229929, + "pass@4": 0.9976782940397756, + "pass@8": 0.9981652294469208, + "pass@16": 0.9988826259348237, + "pass@32": 0.9996323772550161, + "pass@64": 0.9999808398763494, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9945860745614035, + "pass@2": 0.9950729842980154, + "pass@4": 0.9958536131448481, + "pass@8": 0.9970718402735894, + "pass@16": 0.9985582880872056, + "pass@32": 0.9996730624062373, + "pass@64": 0.9999907663284503, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9663143382352941, + "pass@2": 0.9694860178323298, + "pass@4": 0.9722351474448048, + "pass@8": 0.9755392992634085, + "pass@16": 0.9793933800658496, + "pass@32": 0.9823811114856312, + "pass@64": 0.9838078008574993, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9973629731861199, + "pass@2": 0.9984681220348246, + "pass@4": 0.9992779798424248, + "pass@8": 0.9998441685791019, + "pass@16": 0.9999940349535459, + "pass@32": 0.9999999966955364, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9751038370253164, + "pass@2": 0.9806318835343366, + "pass@4": 0.9847448105774753, + "pass@8": 0.9874530271372075, + "pass@16": 0.9892156409124943, + "pass@32": 0.9902597707262977, + "pass@64": 0.9905017189678624, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.9673423423423423, + "pass@2": 0.9762660051784069, + "pass@4": 0.9823950601332488, + "pass@8": 0.9866693202134098, + "pass@16": 0.9895049053205298, + "pass@32": 0.9919861992734489, + "pass@64": 0.9937088592336957, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9417956189710611, + "pass@2": 0.9539905720687647, + "pass@4": 0.9617673268132481, + "pass@8": 0.9670705477659864, + "pass@16": 0.9716396337451848, + "pass@32": 0.9758638941571824, + "pass@64": 0.9788092784862537, + "pass@128": 0.9807073954983923 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.353303125, + "avg_response_len": 308.236484375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.353303125, + "pass@2": 0.39791903297244535, + "pass@4": 0.43620034877203023, + "pass@8": 0.4701827701295203, + "pass@16": 0.5009912046366604, + "pass@32": 0.5290502619178181, + "pass@64": 0.5549034521568353, + "pass@128": 0.5784 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8391015625, + "pass@2": 0.8645938730314966, + "pass@4": 0.8826258601424821, + "pass@8": 0.8977953889910106, + "pass@16": 0.911162419977714, + "pass@32": 0.922565457278282, + "pass@64": 0.931881624184718, + "pass@128": 0.94 + }, + "12": { + "pass@1": 0.7364140625, + "pass@2": 0.7842556594488193, + "pass@4": 0.8196690350581183, + "pass@8": 0.8477787303726111, + "pass@16": 0.8712383209271597, + "pass@32": 0.8908579516177915, + "pass@64": 0.9070877279688662, + "pass@128": 0.92 + }, + "13": { + "pass@1": 0.49853125, + "pass@2": 0.5598706938976378, + "pass@4": 0.6103975771466054, + "pass@8": 0.6532738076367062, + "pass@16": 0.6902455496313759, + "pass@32": 0.721566325921807, + "pass@64": 0.7486950938125937, + "pass@128": 0.773 + }, + "14": { + "pass@1": 0.3530859375, + "pass@2": 0.4050979330708665, + "pass@4": 0.45004108942632143, + "pass@8": 0.48990172291383877, + "pass@16": 0.5260877540170674, + "pass@32": 0.5591971927785748, + "pass@64": 0.5905743368061539, + "pass@128": 0.621 + }, + "15": { + "pass@1": 0.2445078125, + "pass@2": 0.29235617618110243, + "pass@4": 0.33575429396325446, + "pass@8": 0.37687534175533055, + "pass@16": 0.4155688891509993, + "pass@32": 0.4491113618001927, + "pass@64": 0.47753770845147786, + "pass@128": 0.503 + }, + "16": { + "pass@1": 0.2133203125, + "pass@2": 0.2559495570866139, + "pass@4": 0.29070916432320926, + "pass@8": 0.32089867725181254, + "pass@16": 0.34890487844481394, + "pass@32": 0.3763310644511031, + "pass@64": 0.40407359451227987, + "pass@128": 0.432 + }, + "17": { + "pass@1": 0.1782734375, + "pass@2": 0.2215953494094489, + "pass@4": 0.26197571006749126, + "pass@8": 0.29855956944584344, + "pass@16": 0.3316988459268027, + "pass@32": 0.3625499449066916, + "pass@64": 0.3928001504568992, + "pass@128": 0.421 + }, + "18": { + "pass@1": 0.162375, + "pass@2": 0.2024637057086614, + "pass@4": 0.2393894426321706, + "pass@8": 0.2732510647392577, + "pass@16": 0.3047129285521866, + "pass@32": 0.33544284753184744, + "pass@64": 0.3652063873930492, + "pass@128": 0.392 + }, + "19": { + "pass@1": 0.153375, + "pass@2": 0.1973847194881888, + "pass@4": 0.23735144422572174, + "pass@8": 0.2726833003540313, + "pass@16": 0.30426648375269755, + "pass@32": 0.3336304035031561, + "pass@64": 0.3604725830723611, + "pass@128": 0.381 + }, + "20": { + "pass@1": 0.154046875, + "pass@2": 0.19562266240157478, + "pass@4": 0.2340898707349079, + "pass@8": 0.27081009783472154, + "pass@16": 0.306025975985714, + "pass@32": 0.33925006938873714, + "pass@64": 0.37070531490994785, + "pass@128": 0.401 + } + }, + "per_op_accuracy": { + "11": 0.8391015625, + "12": 0.7364140625, + "13": 0.49853125, + "14": 0.3530859375, + "15": 0.2445078125, + "16": 0.2133203125, + "17": 0.1782734375, + "18": 0.162375, + "19": 0.153375, + "20": 0.154046875 + }, + "per_op_avg_response_len": { + "11": 288.8068515625, + "12": 297.392421875, + "13": 307.364375, + "14": 313.2124140625, + "15": 311.81865625, + "16": 307.8609140625, + "17": 315.13540625, + "18": 313.5347421875, + "19": 311.662953125, + "20": 315.576109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 288.8068515625, + "12": 297.392421875, + "13": 307.364375, + "14": 313.2124140625, + "15": 311.81865625, + "16": 307.8609140625, + "17": 315.13540625, + "18": 313.5347421875, + "19": 311.662953125, + "20": 315.576109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 152535, + "answer_accuracy": 0.35413958023774145, + "avg_response_len": 292.40680256315005, + "resp_tokens_sum": 125945458, + "pass_at_k": { + "pass@1": 0.35413958023774145, + "pass@2": 0.39982464812626606, + "pass@4": 0.4387393648416536, + "pass@8": 0.4728202552422867, + "pass@16": 0.5030330218565757, + "pass@32": 0.5303371194995841, + "pass@64": 0.5555134095212355, + "pass@128": 0.5777117384843982 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.80875973796034, + "pass@2": 0.8375635720818182, + "pass@4": 0.8604180507004612, + "pass@8": 0.8809243943571061, + "pass@16": 0.8994562696166066, + "pass@32": 0.9151986207448092, + "pass@64": 0.9273850927591833, + "pass@128": 0.9376770538243626 + }, + "12": { + "pass@1": 0.7334701624293786, + "pass@2": 0.7817570704880107, + "pass@4": 0.8171735532528774, + "pass@8": 0.8454429859144871, + "pass@16": 0.8691713238808132, + "pass@32": 0.8891811697633311, + "pass@64": 0.906188340693502, + "pass@128": 0.9180790960451978 + }, + "13": { + "pass@1": 0.5085961990595611, + "pass@2": 0.5776671994174711, + "pass@4": 0.6343787514413363, + "pass@8": 0.6803702374386639, + "pass@16": 0.7154196073579733, + "pass@32": 0.7422161611800479, + "pass@64": 0.76511848727167, + "pass@128": 0.786833855799373 + }, + "14": { + "pass@1": 0.30926304517133957, + "pass@2": 0.3654867141315279, + "pass@4": 0.4139567650422203, + "pass@8": 0.45759488846050517, + "pass@16": 0.49837696805182397, + "pass@32": 0.5345293388750569, + "pass@64": 0.5677636542177898, + "pass@128": 0.6012461059190031 + }, + "15": { + "pass@1": 0.2457550578034682, + "pass@2": 0.29157006679259007, + "pass@4": 0.33352885432312285, + "pass@8": 0.37290426380904296, + "pass@16": 0.4105600444996007, + "pass@32": 0.4447149669473933, + "pass@64": 0.4736651101410065, + "pass@128": 0.49710982658959535 + }, + "16": { + "pass@1": 0.22184979838709679, + "pass@2": 0.26663068326136635, + "pass@4": 0.30263250511024836, + "pass@8": 0.3322350800902826, + "pass@16": 0.3567520519571173, + "pass@32": 0.3789453955002686, + "pass@64": 0.4006622686591528, + "pass@128": 0.42258064516129035 + }, + "17": { + "pass@1": 0.1810196893063584, + "pass@2": 0.22069610224841837, + "pass@4": 0.2561653288642387, + "pass@8": 0.2871249636415423, + "pass@16": 0.31473971022818265, + "pass@32": 0.3398529280723456, + "pass@64": 0.3637549328856826, + "pass@128": 0.38439306358381503 + }, + "18": { + "pass@1": 0.1610044642857143, + "pass@2": 0.19778894825646798, + "pass@4": 0.23138643687396207, + "pass@8": 0.26264321026468734, + "pass@16": 0.29186077506817065, + "pass@32": 0.32167058957108174, + "pass@64": 0.3512736526766329, + "pass@128": 0.3742857142857143 + }, + "19": { + "pass@1": 0.16532939189189189, + "pass@2": 0.2147416855832209, + "pass@4": 0.2589971599833805, + "pass@8": 0.296842774153626, + "pass@16": 0.32952997822341357, + "pass@32": 0.3608921725080611, + "pass@64": 0.3914425486882824, + "pass@128": 0.4144144144144144 + }, + "20": { + "pass@1": 0.17171077327327328, + "pass@2": 0.2125411582842685, + "pass@4": 0.24931663215521463, + "pass@8": 0.28445708954372556, + "pass@16": 0.31796378367059186, + "pass@32": 0.3504592271011099, + "pass@64": 0.3830969877606695, + "pass@128": 0.4174174174174174 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 151686, + "answer_accuracy": 0.3523778991971454, + "avg_response_len": 304.6506072517098, + "resp_tokens_sum": 131141119, + "pass_at_k": { + "pass@1": 0.3523778991971454, + "pass@2": 0.3970092481052502, + "pass@4": 0.4350496245117003, + "pass@8": 0.4686835007331392, + "pass@16": 0.49963910654999955, + "pass@32": 0.528194672586522, + "pass@64": 0.5543945358977235, + "pass@128": 0.5792447219744276 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8485739834337349, + "pass@2": 0.8717270657432884, + "pass@4": 0.8875434908927042, + "pass@8": 0.900419181851555, + "pass@16": 0.911604703248638, + "pass@32": 0.9220797702645877, + "pass@64": 0.9313253131337487, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.7480162617554859, + "pass@2": 0.7935103392738131, + "pass@4": 0.8251117530763199, + "pass@8": 0.8481994805867784, + "pass@16": 0.8673263961259938, + "pass@32": 0.8842887166756281, + "pass@64": 0.8996039865796801, + "pass@128": 0.9153605015673981 + }, + "13": { + "pass@1": 0.4614189425770308, + "pass@2": 0.5181631707801232, + "pass@4": 0.5654716180435426, + "pass@8": 0.6084657762411573, + "pass@16": 0.650034763496025, + "pass@32": 0.6886026576493296, + "pass@64": 0.7222463189761011, + "pass@128": 0.7535014005602241 + }, + "14": { + "pass@1": 0.38966557017543857, + "pass@2": 0.43866052516461757, + "pass@4": 0.4775347327746312, + "pass@8": 0.5106014603664322, + "pass@16": 0.5421969559925761, + "pass@32": 0.5743889726779975, + "pass@64": 0.6071513767259988, + "pass@128": 0.6432748538011696 + }, + "15": { + "pass@1": 0.25414299242424243, + "pass@2": 0.3020221904080171, + "pass@4": 0.34498625342286765, + "pass@8": 0.38554057450292495, + "pass@16": 0.42102806619022154, + "pass@32": 0.4483743656957863, + "pass@64": 0.46978329116358775, + "pass@128": 0.4909090909090909 + }, + "16": { + "pass@1": 0.19816150284900286, + "pass@2": 0.2385198645041165, + "pass@4": 0.2712453993464492, + "pass@8": 0.3002115489394276, + "pass@16": 0.33021755688763793, + "pass@32": 0.3629484109481779, + "pass@64": 0.39703810753951246, + "pass@128": 0.4301994301994302 + }, + "17": { + "pass@1": 0.19232822204968944, + "pass@2": 0.24024803455274613, + "pass@4": 0.28360422241474476, + "pass@8": 0.32167776261409275, + "pass@16": 0.3578691530927828, + "pass@32": 0.3935301291607172, + "pass@64": 0.4269889693410022, + "pass@128": 0.45652173913043476 + }, + "18": { + "pass@1": 0.1627297794117647, + "pass@2": 0.21125484888837412, + "pass@4": 0.2557750033084101, + "pass@8": 0.294215639407568, + "pass@16": 0.32724684342872096, + "pass@32": 0.3559548424763023, + "pass@64": 0.382033785221312, + "pass@128": 0.4088235294117647 + }, + "19": { + "pass@1": 0.14957682291666666, + "pass@2": 0.19273652805118105, + "pass@4": 0.23311154353473673, + "pass@8": 0.27019836298649635, + "pass@16": 0.3039869333375479, + "pass@32": 0.3345311995737657, + "pass@64": 0.36212989556933894, + "pass@128": 0.38392857142857145 + }, + "20": { + "pass@1": 0.13704622005988024, + "pass@2": 0.18041611167428911, + "pass@4": 0.22258806767791758, + "pass@8": 0.26292508542741727, + "pass@16": 0.299340227526878, + "pass@32": 0.330316341657262, + "pass@64": 0.35724850294988825, + "pass@128": 0.38023952095808383 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 148007, + "answer_accuracy": 0.35339385314792177, + "avg_response_len": 328.20169955302566, + "resp_tokens_sum": 137456123, + "pass_at_k": { + "pass@1": 0.35339385314792177, + "pass@2": 0.3968943421153967, + "pass@4": 0.43477189419171175, + "pass@8": 0.4690112865034621, + "pass@16": 0.5002810551624386, + "pass@32": 0.5286062127608766, + "pass@64": 0.5547992280884292, + "pass@128": 0.578239608801956 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8631200396825397, + "pass@2": 0.8873668135233092, + "pass@4": 0.9023296802185442, + "pass@8": 0.9139362203436046, + "pass@16": 0.9238145883317608, + "pass@32": 0.9313328902460989, + "pass@64": 0.9375069285089583, + "pass@128": 0.9428571428571428 + }, + "12": { + "pass@1": 0.7282826834862385, + "pass@2": 0.7779322882804786, + "pass@4": 0.8170610029824253, + "pass@8": 0.8498968778339464, + "pass@16": 0.8772922076732741, + "pass@32": 0.8990817030643646, + "pass@64": 0.9153620294937252, + "pass@128": 0.926605504587156 + }, + "13": { + "pass@1": 0.5295138888888888, + "pass@2": 0.5883043373918536, + "pass@4": 0.6362882339360357, + "pass@8": 0.6759673449866654, + "pass@16": 0.7097663096176011, + "pass@32": 0.7375562398906225, + "pass@64": 0.7616677793162406, + "pass@128": 0.7808641975308642 + }, + "14": { + "pass@1": 0.3577058605341246, + "pass@2": 0.40876794726512283, + "pass@4": 0.4565106505603714, + "pass@8": 0.4996678465065184, + "pass@16": 0.5361346834509522, + "pass@32": 0.5672766953822151, + "pass@64": 0.5954790295606865, + "pass@128": 0.6172106824925816 + }, + "15": { + "pass@1": 0.23336226851851852, + "pass@2": 0.28335064887722383, + "pass@4": 0.3287279220305795, + "pass@8": 0.37229036077603983, + "pass@16": 0.4153575676276682, + "pass@32": 0.4545569214715594, + "pass@64": 0.48957127857625266, + "pass@128": 0.5216049382716049 + }, + "16": { + "pass@1": 0.22121589233038347, + "pass@2": 0.26422882842078366, + "pass@4": 0.29995856214875805, + "pass@8": 0.33195147122739216, + "pass@16": 0.3610778167272766, + "pass@32": 0.3877967539917674, + "pass@64": 0.4144776267893019, + "pass@128": 0.4424778761061947 + }, + "17": { + "pass@1": 0.16177993222891565, + "pass@2": 0.20444165935395123, + "pass@4": 0.24705405621360577, + "pass@8": 0.288054495373893, + "pass@16": 0.3239910809999265, + "pass@32": 0.3561570186864724, + "pass@64": 0.38991113117665654, + "pass@128": 0.4246987951807229 + }, + "18": { + "pass@1": 0.16353326612903227, + "pass@2": 0.19809975869951738, + "pass@4": 0.2304538341981446, + "pass@8": 0.26223427015498135, + "pass@16": 0.2945088081050385, + "pass@32": 0.32849514432266463, + "pass@64": 0.3624810386483276, + "pass@128": 0.3935483870967742 + }, + "19": { + "pass@1": 0.14520392749244712, + "pass@2": 0.18464128327425824, + "pass@4": 0.21987888617397153, + "pass@8": 0.2509003220465607, + "pass@16": 0.2791341129996524, + "pass@32": 0.30528950755645506, + "pass@64": 0.3276331403258758, + "pass@128": 0.34441087613293053 + }, + "20": { + "pass@1": 0.15343468468468469, + "pass@2": 0.193956382563666, + "pass@4": 0.23039945232071216, + "pass@8": 0.26507179724896024, + "pass@16": 0.3007939940823713, + "pass@32": 0.3370014674190456, + "pass@64": 0.3718108648660131, + "pass@128": 0.40540540540540543 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6456291118421053, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6456291118421053, + "pass@2": 0.672114192654361, + "pass@4": 0.6940329882333646, + "pass@8": 0.7131307083941159, + "pass@16": 0.7303077838808043, + "pass@32": 0.7458630675676212, + "pass@64": 0.7600302310751801, + "pass@128": 0.7727368421052632 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.89928125, + "pass@2": 0.916050565944882, + "pass@4": 0.9260351500749907, + "pass@8": 0.9330535750028963, + "pass@16": 0.9383338135951381, + "pass@32": 0.94246185430624, + "pass@64": 0.945903048004897, + "pass@128": 0.948 + }, + "2": { + "pass@1": 0.9998984375, + "pass@2": 0.9999904035433071, + "pass@4": 0.9999999329771279, + "pass@8": 0.9999999999990998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9981171875, + "pass@2": 0.9982573818897639, + "pass@4": 0.9984843227409074, + "pass@8": 0.9988590133679512, + "pass@16": 0.9993672588247003, + "pass@32": 0.9998241803921225, + "pass@64": 0.9999927603421557, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9933359375, + "pass@2": 0.9954790846456693, + "pass@4": 0.9968326808211472, + "pass@8": 0.997768162863856, + "pass@16": 0.9984728041659322, + "pass@32": 0.998887565153522, + "pass@64": 0.9989968420736931, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9701796875, + "pass@2": 0.973689345472441, + "pass@4": 0.9764589475065616, + "pass@8": 0.979163521366934, + "pass@16": 0.9821057314308723, + "pass@32": 0.9846806191317323, + "pass@64": 0.9863707931066418, + "pass@128": 0.988 + }, + "6": { + "pass@1": 0.98528125, + "pass@2": 0.9902500000000002, + "pass@4": 0.9929865528683917, + "pass@8": 0.9945421401299686, + "pass@16": 0.995530902448261, + "pass@32": 0.9963896873403176, + "pass@64": 0.9972217048693288, + "pass@128": 0.998 + }, + "7": { + "pass@1": 0.9786171875, + "pass@2": 0.983411909448819, + "pass@4": 0.9861397134420696, + "pass@8": 0.9878044142052393, + "pass@16": 0.9890889706751056, + "pass@32": 0.9902669840948403, + "pass@64": 0.9912530039689931, + "pass@128": 0.992 + }, + "8": { + "pass@1": 0.95978125, + "pass@2": 0.9705783710629925, + "pass@4": 0.9771872181289839, + "pass@8": 0.9815502427887551, + "pass@16": 0.9847671920564206, + "pass@32": 0.9873725342613687, + "pass@64": 0.9888198336231544, + "pass@128": 0.989 + }, + "9": { + "pass@1": 0.9494296875, + "pass@2": 0.9632722687007877, + "pass@4": 0.9704987701537305, + "pass@8": 0.974914688468251, + "pass@16": 0.9782691741721024, + "pass@32": 0.981012239926249, + "pass@64": 0.9829818828711779, + "pass@128": 0.984 + }, + "11": { + "pass@1": 0.8391015625, + "pass@2": 0.8645938730314966, + "pass@4": 0.8826258601424821, + "pass@8": 0.8977953889910106, + "pass@16": 0.911162419977714, + "pass@32": 0.922565457278282, + "pass@64": 0.931881624184718, + "pass@128": 0.94 + }, + "12": { + "pass@1": 0.7364140625, + "pass@2": 0.7842556594488193, + "pass@4": 0.8196690350581183, + "pass@8": 0.8477787303726111, + "pass@16": 0.8712383209271597, + "pass@32": 0.8908579516177915, + "pass@64": 0.9070877279688662, + "pass@128": 0.92 + }, + "13": { + "pass@1": 0.49853125, + "pass@2": 0.5598706938976378, + "pass@4": 0.6103975771466054, + "pass@8": 0.6532738076367062, + "pass@16": 0.6902455496313759, + "pass@32": 0.721566325921807, + "pass@64": 0.7486950938125937, + "pass@128": 0.773 + }, + "14": { + "pass@1": 0.3530859375, + "pass@2": 0.4050979330708665, + "pass@4": 0.45004108942632143, + "pass@8": 0.48990172291383877, + "pass@16": 0.5260877540170674, + "pass@32": 0.5591971927785748, + "pass@64": 0.5905743368061539, + "pass@128": 0.621 + }, + "15": { + "pass@1": 0.2445078125, + "pass@2": 0.29235617618110243, + "pass@4": 0.33575429396325446, + "pass@8": 0.37687534175533055, + "pass@16": 0.4155688891509993, + "pass@32": 0.4491113618001927, + "pass@64": 0.47753770845147786, + "pass@128": 0.503 + }, + "16": { + "pass@1": 0.2133203125, + "pass@2": 0.2559495570866139, + "pass@4": 0.29070916432320926, + "pass@8": 0.32089867725181254, + "pass@16": 0.34890487844481394, + "pass@32": 0.3763310644511031, + "pass@64": 0.40407359451227987, + "pass@128": 0.432 + }, + "17": { + "pass@1": 0.1782734375, + "pass@2": 0.2215953494094489, + "pass@4": 0.26197571006749126, + "pass@8": 0.29855956944584344, + "pass@16": 0.3316988459268027, + "pass@32": 0.3625499449066916, + "pass@64": 0.3928001504568992, + "pass@128": 0.421 + }, + "18": { + "pass@1": 0.162375, + "pass@2": 0.2024637057086614, + "pass@4": 0.2393894426321706, + "pass@8": 0.2732510647392577, + "pass@16": 0.3047129285521866, + "pass@32": 0.33544284753184744, + "pass@64": 0.3652063873930492, + "pass@128": 0.392 + }, + "19": { + "pass@1": 0.153375, + "pass@2": 0.1973847194881888, + "pass@4": 0.23735144422572174, + "pass@8": 0.2726833003540313, + "pass@16": 0.30426648375269755, + "pass@32": 0.3336304035031561, + "pass@64": 0.3604725830723611, + "pass@128": 0.381 + }, + "20": { + "pass@1": 0.154046875, + "pass@2": 0.19562266240157478, + "pass@4": 0.2340898707349079, + "pass@8": 0.27081009783472154, + "pass@16": 0.306025975985714, + "pass@32": 0.33925006938873714, + "pass@64": 0.37070531490994785, + "pass@128": 0.401 + } + }, + "per_op_accuracy": { + "10": 0.89928125, + "2": 0.9998984375, + "3": 0.9981171875, + "4": 0.9933359375, + "5": 0.9701796875, + "6": 0.98528125, + "7": 0.9786171875, + "8": 0.95978125, + "9": 0.9494296875, + "11": 0.8391015625, + "12": 0.7364140625, + "13": 0.49853125, + "14": 0.3530859375, + "15": 0.2445078125, + "16": 0.2133203125, + "17": 0.1782734375, + "18": 0.162375, + "19": 0.153375, + "20": 0.154046875 + }, + "per_op_avg_response_len": { + "10": 273.267484375, + "2": 114.288421875, + "3": 142.6334140625, + "4": 157.7698125, + "5": 171.13790625, + "6": 195.2028984375, + "7": 219.23784375, + "8": 237.87965625, + "9": 253.1414921875, + "11": 288.8068515625, + "12": 297.392421875, + "13": 307.364375, + "14": 313.2124140625, + "15": 311.81865625, + "16": 307.8609140625, + "17": 315.13540625, + "18": 313.5347421875, + "19": 311.662953125, + "20": 315.576109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.267484375, + "2": 114.288421875, + "3": 142.6334140625, + "4": 157.7698125, + "5": 171.13790625, + "6": 195.2028984375, + "7": 219.23784375, + "8": 237.87965625, + "9": 253.1414921875, + "11": 288.8068515625, + "12": 297.392421875, + "13": 307.364375, + "14": 313.2124140625, + "15": 311.81865625, + "16": 307.8609140625, + "17": 315.13540625, + "18": 313.5347421875, + "19": 311.662953125, + "20": 315.576109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 519278, + "answer_accuracy": 0.6412993005058488, + "avg_response_len": 242.80526176691433, + "resp_tokens_sum": 196606219, + "pass_at_k": { + "pass@1": 0.6412993005058488, + "pass@2": 0.669765941427828, + "pass@4": 0.6926322673248675, + "pass@8": 0.7119556252626574, + "pass@16": 0.7288212599295653, + "pass@32": 0.7440110633943208, + "pass@64": 0.7579828058690048, + "pass@128": 0.7701549162187796 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9232577433628318, + "pass@2": 0.9413916422781228, + "pass@4": 0.9499321857444808, + "pass@8": 0.9535468343450594, + "pass@16": 0.9556178797377718, + "pass@32": 0.9578424969991439, + "pass@64": 0.9603845492642211, + "pass@128": 0.9616519174041298 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9871337890625, + "pass@2": 0.9921521284448819, + "pass@4": 0.9948190980033745, + "pass@8": 0.9961782344257655, + "pass@16": 0.9967684498096553, + "pass@32": 0.9968730556584248, + "pass@64": 0.9968749999667598, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9740575396825397, + "pass@2": 0.9775235126859141, + "pass@4": 0.979264063123062, + "pass@8": 0.980565740435064, + "pass@16": 0.9821661054021831, + "pass@32": 0.9838715548767617, + "pass@64": 0.9855250123602817, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.9662939745508982, + "pass@2": 0.9776823076288367, + "pass@4": 0.9837688846154713, + "pass@8": 0.987005164857485, + "pass@16": 0.9892880596338539, + "pass@32": 0.991436482518866, + "pass@64": 0.993178757094046, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9721015162721893, + "pass@2": 0.9787191911661931, + "pass@4": 0.9822101936777133, + "pass@8": 0.9845253393977582, + "pass@16": 0.986677006424018, + "pass@32": 0.98918608445364, + "pass@64": 0.9918771028850546, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.9414563301282052, + "pass@2": 0.9573518416868564, + "pass@4": 0.9671929527438877, + "pass@8": 0.9740068690727082, + "pass@16": 0.978930348505969, + "pass@32": 0.9823055554325953, + "pass@64": 0.9838388835534627, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.9477112676056338, + "pass@2": 0.966034642896751, + "pass@4": 0.974834940878869, + "pass@8": 0.979053728156307, + "pass@16": 0.9807582382316763, + "pass@32": 0.9815156079864068, + "pass@64": 0.9823999112442319, + "pass@128": 0.9830985915492958 + }, + "11": { + "pass@1": 0.80875973796034, + "pass@2": 0.8375635720818182, + "pass@4": 0.8604180507004612, + "pass@8": 0.8809243943571061, + "pass@16": 0.8994562696166066, + "pass@32": 0.9151986207448092, + "pass@64": 0.9273850927591833, + "pass@128": 0.9376770538243626 + }, + "12": { + "pass@1": 0.7334701624293786, + "pass@2": 0.7817570704880107, + "pass@4": 0.8171735532528774, + "pass@8": 0.8454429859144871, + "pass@16": 0.8691713238808132, + "pass@32": 0.8891811697633311, + "pass@64": 0.906188340693502, + "pass@128": 0.9180790960451978 + }, + "13": { + "pass@1": 0.5085961990595611, + "pass@2": 0.5776671994174711, + "pass@4": 0.6343787514413363, + "pass@8": 0.6803702374386639, + "pass@16": 0.7154196073579733, + "pass@32": 0.7422161611800479, + "pass@64": 0.76511848727167, + "pass@128": 0.786833855799373 + }, + "14": { + "pass@1": 0.30926304517133957, + "pass@2": 0.3654867141315279, + "pass@4": 0.4139567650422203, + "pass@8": 0.45759488846050517, + "pass@16": 0.49837696805182397, + "pass@32": 0.5345293388750569, + "pass@64": 0.5677636542177898, + "pass@128": 0.6012461059190031 + }, + "15": { + "pass@1": 0.2457550578034682, + "pass@2": 0.29157006679259007, + "pass@4": 0.33352885432312285, + "pass@8": 0.37290426380904296, + "pass@16": 0.4105600444996007, + "pass@32": 0.4447149669473933, + "pass@64": 0.4736651101410065, + "pass@128": 0.49710982658959535 + }, + "16": { + "pass@1": 0.22184979838709679, + "pass@2": 0.26663068326136635, + "pass@4": 0.30263250511024836, + "pass@8": 0.3322350800902826, + "pass@16": 0.3567520519571173, + "pass@32": 0.3789453955002686, + "pass@64": 0.4006622686591528, + "pass@128": 0.42258064516129035 + }, + "17": { + "pass@1": 0.1810196893063584, + "pass@2": 0.22069610224841837, + "pass@4": 0.2561653288642387, + "pass@8": 0.2871249636415423, + "pass@16": 0.31473971022818265, + "pass@32": 0.3398529280723456, + "pass@64": 0.3637549328856826, + "pass@128": 0.38439306358381503 + }, + "18": { + "pass@1": 0.1610044642857143, + "pass@2": 0.19778894825646798, + "pass@4": 0.23138643687396207, + "pass@8": 0.26264321026468734, + "pass@16": 0.29186077506817065, + "pass@32": 0.32167058957108174, + "pass@64": 0.3512736526766329, + "pass@128": 0.3742857142857143 + }, + "19": { + "pass@1": 0.16532939189189189, + "pass@2": 0.2147416855832209, + "pass@4": 0.2589971599833805, + "pass@8": 0.296842774153626, + "pass@16": 0.32952997822341357, + "pass@32": 0.3608921725080611, + "pass@64": 0.3914425486882824, + "pass@128": 0.4144144144144144 + }, + "20": { + "pass@1": 0.17171077327327328, + "pass@2": 0.2125411582842685, + "pass@4": 0.24931663215521463, + "pass@8": 0.28445708954372556, + "pass@16": 0.31796378367059186, + "pass@32": 0.3504592271011099, + "pass@64": 0.3830969877606695, + "pass@128": 0.4174174174174174 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 531958, + "answer_accuracy": 0.6478444076383476, + "avg_response_len": 252.07089584957131, + "resp_tokens_sum": 206980454, + "pass_at_k": { + "pass@1": 0.6478444076383476, + "pass@2": 0.6735689605439996, + "pass@4": 0.6949296075324933, + "pass@8": 0.7136456940962927, + "pass@16": 0.7308214637042161, + "pass@32": 0.7465964425889449, + "pass@64": 0.7609117522322542, + "pass@128": 0.7742790335151988 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8846098542944786, + "pass@2": 0.9002780662769918, + "pass@4": 0.9112371256507044, + "pass@8": 0.920703275994343, + "pass@16": 0.928180327350798, + "pass@32": 0.9340597884127482, + "pass@64": 0.9389806489566199, + "pass@128": 0.941717791411043 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9972064393939394, + "pass@2": 0.9974264047959914, + "pass@4": 0.9978202057129222, + "pass@8": 0.9984495141444165, + "pass@16": 0.9992439943304724, + "pass@32": 0.9998493181625818, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979428624260355, + "pass@2": 0.9990397719330942, + "pass@4": 0.9997296856938741, + "pass@8": 0.999977983651016, + "pass@16": 0.999999898523723, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9704483695652174, + "pass@2": 0.9743309939518432, + "pass@4": 0.9780603405552569, + "pass@8": 0.9814549315371892, + "pass@16": 0.9847236492950605, + "pass@32": 0.9876855114215582, + "pass@64": 0.9896688751350827, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9924785100286533, + "pass@2": 0.9948129357218599, + "pass@4": 0.9960934836583821, + "pass@8": 0.9969392940630192, + "pass@16": 0.9974515227801705, + "pass@32": 0.9978507255774547, + "pass@64": 0.9985673352433165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9881909320809249, + "pass@2": 0.9905351087797553, + "pass@4": 0.9912523347876314, + "pass@8": 0.9913285922353743, + "pass@16": 0.9913294796978018, + "pass@32": 0.9913294797687862, + "pass@64": 0.9913294797687862, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.968794014084507, + "pass@2": 0.9768676527669956, + "pass@4": 0.9810858080239967, + "pass@8": 0.9833780732591677, + "pass@16": 0.9854529291572443, + "pass@32": 0.9874980186150435, + "pass@64": 0.9886114136046574, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9583645209580839, + "pass@2": 0.9689787501768117, + "pass@4": 0.9740201422239886, + "pass@8": 0.9778193850824559, + "pass@16": 0.9817966272607928, + "pass@32": 0.9852710419406909, + "pass@64": 0.9874857148809907, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8485739834337349, + "pass@2": 0.8717270657432884, + "pass@4": 0.8875434908927042, + "pass@8": 0.900419181851555, + "pass@16": 0.911604703248638, + "pass@32": 0.9220797702645877, + "pass@64": 0.9313253131337487, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.7480162617554859, + "pass@2": 0.7935103392738131, + "pass@4": 0.8251117530763199, + "pass@8": 0.8481994805867784, + "pass@16": 0.8673263961259938, + "pass@32": 0.8842887166756281, + "pass@64": 0.8996039865796801, + "pass@128": 0.9153605015673981 + }, + "13": { + "pass@1": 0.4614189425770308, + "pass@2": 0.5181631707801232, + "pass@4": 0.5654716180435426, + "pass@8": 0.6084657762411573, + "pass@16": 0.650034763496025, + "pass@32": 0.6886026576493296, + "pass@64": 0.7222463189761011, + "pass@128": 0.7535014005602241 + }, + "14": { + "pass@1": 0.38966557017543857, + "pass@2": 0.43866052516461757, + "pass@4": 0.4775347327746312, + "pass@8": 0.5106014603664322, + "pass@16": 0.5421969559925761, + "pass@32": 0.5743889726779975, + "pass@64": 0.6071513767259988, + "pass@128": 0.6432748538011696 + }, + "15": { + "pass@1": 0.25414299242424243, + "pass@2": 0.3020221904080171, + "pass@4": 0.34498625342286765, + "pass@8": 0.38554057450292495, + "pass@16": 0.42102806619022154, + "pass@32": 0.4483743656957863, + "pass@64": 0.46978329116358775, + "pass@128": 0.4909090909090909 + }, + "16": { + "pass@1": 0.19816150284900286, + "pass@2": 0.2385198645041165, + "pass@4": 0.2712453993464492, + "pass@8": 0.3002115489394276, + "pass@16": 0.33021755688763793, + "pass@32": 0.3629484109481779, + "pass@64": 0.39703810753951246, + "pass@128": 0.4301994301994302 + }, + "17": { + "pass@1": 0.19232822204968944, + "pass@2": 0.24024803455274613, + "pass@4": 0.28360422241474476, + "pass@8": 0.32167776261409275, + "pass@16": 0.3578691530927828, + "pass@32": 0.3935301291607172, + "pass@64": 0.4269889693410022, + "pass@128": 0.45652173913043476 + }, + "18": { + "pass@1": 0.1627297794117647, + "pass@2": 0.21125484888837412, + "pass@4": 0.2557750033084101, + "pass@8": 0.294215639407568, + "pass@16": 0.32724684342872096, + "pass@32": 0.3559548424763023, + "pass@64": 0.382033785221312, + "pass@128": 0.4088235294117647 + }, + "19": { + "pass@1": 0.14957682291666666, + "pass@2": 0.19273652805118105, + "pass@4": 0.23311154353473673, + "pass@8": 0.27019836298649635, + "pass@16": 0.3039869333375479, + "pass@32": 0.3345311995737657, + "pass@64": 0.36212989556933894, + "pass@128": 0.38392857142857145 + }, + "20": { + "pass@1": 0.13704622005988024, + "pass@2": 0.18041611167428911, + "pass@4": 0.22258806767791758, + "pass@8": 0.26292508542741727, + "pass@16": 0.299340227526878, + "pass@32": 0.330316341657262, + "pass@64": 0.35724850294988825, + "pass@128": 0.38023952095808383 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 518934, + "answer_accuracy": 0.6477347619428023, + "avg_response_len": 270.6347484622144, + "resp_tokens_sum": 216819570, + "pass_at_k": { + "pass@1": 0.6477347619428023, + "pass@2": 0.6729965542532144, + "pass@4": 0.694529736541958, + "pass@8": 0.7137905490411974, + "pass@16": 0.7312837376191367, + "pass@32": 0.7469832429372135, + "pass@64": 0.7611960808325885, + "pass@128": 0.7737657772807157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8892957089552239, + "pass@2": 0.9057556704665646, + "pass@4": 0.9162532422253183, + "pass@8": 0.9243341199575077, + "pass@16": 0.9307240765602192, + "pass@32": 0.9350739008984293, + "pass@64": 0.9379850574461723, + "pass@128": 0.9402985074626866 + }, + "2": { + "pass@1": 0.9997098214285715, + "pass@2": 0.999972581552306, + "pass@4": 0.9999998085060796, + "pass@8": 0.9999999999974281, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9971984329446064, + "pass@2": 0.9973955344229929, + "pass@4": 0.9976782940397756, + "pass@8": 0.9981652294469208, + "pass@16": 0.9988826259348237, + "pass@32": 0.9996323772550161, + "pass@64": 0.9999808398763494, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9945860745614035, + "pass@2": 0.9950729842980154, + "pass@4": 0.9958536131448481, + "pass@8": 0.9970718402735894, + "pass@16": 0.9985582880872056, + "pass@32": 0.9996730624062373, + "pass@64": 0.9999907663284503, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9663143382352941, + "pass@2": 0.9694860178323298, + "pass@4": 0.9722351474448048, + "pass@8": 0.9755392992634085, + "pass@16": 0.9793933800658496, + "pass@32": 0.9823811114856312, + "pass@64": 0.9838078008574993, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9973629731861199, + "pass@2": 0.9984681220348246, + "pass@4": 0.9992779798424248, + "pass@8": 0.9998441685791019, + "pass@16": 0.9999940349535459, + "pass@32": 0.9999999966955364, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9751038370253164, + "pass@2": 0.9806318835343366, + "pass@4": 0.9847448105774753, + "pass@8": 0.9874530271372075, + "pass@16": 0.9892156409124943, + "pass@32": 0.9902597707262977, + "pass@64": 0.9905017189678624, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.9673423423423423, + "pass@2": 0.9762660051784069, + "pass@4": 0.9823950601332488, + "pass@8": 0.9866693202134098, + "pass@16": 0.9895049053205298, + "pass@32": 0.9919861992734489, + "pass@64": 0.9937088592336957, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9417956189710611, + "pass@2": 0.9539905720687647, + "pass@4": 0.9617673268132481, + "pass@8": 0.9670705477659864, + "pass@16": 0.9716396337451848, + "pass@32": 0.9758638941571824, + "pass@64": 0.9788092784862537, + "pass@128": 0.9807073954983923 + }, + "11": { + "pass@1": 0.8631200396825397, + "pass@2": 0.8873668135233092, + "pass@4": 0.9023296802185442, + "pass@8": 0.9139362203436046, + "pass@16": 0.9238145883317608, + "pass@32": 0.9313328902460989, + "pass@64": 0.9375069285089583, + "pass@128": 0.9428571428571428 + }, + "12": { + "pass@1": 0.7282826834862385, + "pass@2": 0.7779322882804786, + "pass@4": 0.8170610029824253, + "pass@8": 0.8498968778339464, + "pass@16": 0.8772922076732741, + "pass@32": 0.8990817030643646, + "pass@64": 0.9153620294937252, + "pass@128": 0.926605504587156 + }, + "13": { + "pass@1": 0.5295138888888888, + "pass@2": 0.5883043373918536, + "pass@4": 0.6362882339360357, + "pass@8": 0.6759673449866654, + "pass@16": 0.7097663096176011, + "pass@32": 0.7375562398906225, + "pass@64": 0.7616677793162406, + "pass@128": 0.7808641975308642 + }, + "14": { + "pass@1": 0.3577058605341246, + "pass@2": 0.40876794726512283, + "pass@4": 0.4565106505603714, + "pass@8": 0.4996678465065184, + "pass@16": 0.5361346834509522, + "pass@32": 0.5672766953822151, + "pass@64": 0.5954790295606865, + "pass@128": 0.6172106824925816 + }, + "15": { + "pass@1": 0.23336226851851852, + "pass@2": 0.28335064887722383, + "pass@4": 0.3287279220305795, + "pass@8": 0.37229036077603983, + "pass@16": 0.4153575676276682, + "pass@32": 0.4545569214715594, + "pass@64": 0.48957127857625266, + "pass@128": 0.5216049382716049 + }, + "16": { + "pass@1": 0.22121589233038347, + "pass@2": 0.26422882842078366, + "pass@4": 0.29995856214875805, + "pass@8": 0.33195147122739216, + "pass@16": 0.3610778167272766, + "pass@32": 0.3877967539917674, + "pass@64": 0.4144776267893019, + "pass@128": 0.4424778761061947 + }, + "17": { + "pass@1": 0.16177993222891565, + "pass@2": 0.20444165935395123, + "pass@4": 0.24705405621360577, + "pass@8": 0.288054495373893, + "pass@16": 0.3239910809999265, + "pass@32": 0.3561570186864724, + "pass@64": 0.38991113117665654, + "pass@128": 0.4246987951807229 + }, + "18": { + "pass@1": 0.16353326612903227, + "pass@2": 0.19809975869951738, + "pass@4": 0.2304538341981446, + "pass@8": 0.26223427015498135, + "pass@16": 0.2945088081050385, + "pass@32": 0.32849514432266463, + "pass@64": 0.3624810386483276, + "pass@128": 0.3935483870967742 + }, + "19": { + "pass@1": 0.14520392749244712, + "pass@2": 0.18464128327425824, + "pass@4": 0.21987888617397153, + "pass@8": 0.2509003220465607, + "pass@16": 0.2791341129996524, + "pass@32": 0.30528950755645506, + "pass@64": 0.3276331403258758, + "pass@128": 0.34441087613293053 + }, + "20": { + "pass@1": 0.15343468468468469, + "pass@2": 0.193956382563666, + "pass@4": 0.23039945232071216, + "pass@8": 0.26507179724896024, + "pass@16": 0.3007939940823713, + "pass@32": 0.3370014674190456, + "pass@64": 0.3718108648660131, + "pass@128": 0.40540540540540543 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/summary.csv b/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..63d9e8a3cc68a82cfe359fb0f1dd786f72d559a6 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-22350,0.9704357638888889,0.353303125,0.6456291118421053,0.0,0.0,0.0,196.0621032986111,308.236484375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/summary.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e81d73d258e821c20bf96181c652e1fb85a89156 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block32/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-22350", + "id_acc": 0.9704357638888889, + "ood_acc": 0.353303125, + "total_acc": 0.6456291118421053, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.0621032986111, + "ood_avg_resp_len": 308.236484375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 519278, + "answer_accuracy": 0.6412993005058488, + "avg_response_len": 242.80526176691433, + "resp_tokens_sum": 196606219, + "pass_at_k": { + "pass@1": 0.6412993005058488, + "pass@2": 0.669765941427828, + "pass@4": 0.6926322673248675, + "pass@8": 0.7119556252626574, + "pass@16": 0.7288212599295653, + "pass@32": 0.7440110633943208, + "pass@64": 0.7579828058690048, + "pass@128": 0.7701549162187796 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9232577433628318, + "pass@2": 0.9413916422781228, + "pass@4": 0.9499321857444808, + "pass@8": 0.9535468343450594, + "pass@16": 0.9556178797377718, + "pass@32": 0.9578424969991439, + "pass@64": 0.9603845492642211, + "pass@128": 0.9616519174041298 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9871337890625, + "pass@2": 0.9921521284448819, + "pass@4": 0.9948190980033745, + "pass@8": 0.9961782344257655, + "pass@16": 0.9967684498096553, + "pass@32": 0.9968730556584248, + "pass@64": 0.9968749999667598, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9740575396825397, + "pass@2": 0.9775235126859141, + "pass@4": 0.979264063123062, + "pass@8": 0.980565740435064, + "pass@16": 0.9821661054021831, + "pass@32": 0.9838715548767617, + "pass@64": 0.9855250123602817, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.9662939745508982, + "pass@2": 0.9776823076288367, + "pass@4": 0.9837688846154713, + "pass@8": 0.987005164857485, + "pass@16": 0.9892880596338539, + "pass@32": 0.991436482518866, + "pass@64": 0.993178757094046, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9721015162721893, + "pass@2": 0.9787191911661931, + "pass@4": 0.9822101936777133, + "pass@8": 0.9845253393977582, + "pass@16": 0.986677006424018, + "pass@32": 0.98918608445364, + "pass@64": 0.9918771028850546, + "pass@128": 0.9940828402366864 + }, + "8": { + "pass@1": 0.9414563301282052, + "pass@2": 0.9573518416868564, + "pass@4": 0.9671929527438877, + "pass@8": 0.9740068690727082, + "pass@16": 0.978930348505969, + "pass@32": 0.9823055554325953, + "pass@64": 0.9838388835534627, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.9477112676056338, + "pass@2": 0.966034642896751, + "pass@4": 0.974834940878869, + "pass@8": 0.979053728156307, + "pass@16": 0.9807582382316763, + "pass@32": 0.9815156079864068, + "pass@64": 0.9823999112442319, + "pass@128": 0.9830985915492958 + }, + "11": { + "pass@1": 0.80875973796034, + "pass@2": 0.8375635720818182, + "pass@4": 0.8604180507004612, + "pass@8": 0.8809243943571061, + "pass@16": 0.8994562696166066, + "pass@32": 0.9151986207448092, + "pass@64": 0.9273850927591833, + "pass@128": 0.9376770538243626 + }, + "12": { + "pass@1": 0.7334701624293786, + "pass@2": 0.7817570704880107, + "pass@4": 0.8171735532528774, + "pass@8": 0.8454429859144871, + "pass@16": 0.8691713238808132, + "pass@32": 0.8891811697633311, + "pass@64": 0.906188340693502, + "pass@128": 0.9180790960451978 + }, + "13": { + "pass@1": 0.5085961990595611, + "pass@2": 0.5776671994174711, + "pass@4": 0.6343787514413363, + "pass@8": 0.6803702374386639, + "pass@16": 0.7154196073579733, + "pass@32": 0.7422161611800479, + "pass@64": 0.76511848727167, + "pass@128": 0.786833855799373 + }, + "14": { + "pass@1": 0.30926304517133957, + "pass@2": 0.3654867141315279, + "pass@4": 0.4139567650422203, + "pass@8": 0.45759488846050517, + "pass@16": 0.49837696805182397, + "pass@32": 0.5345293388750569, + "pass@64": 0.5677636542177898, + "pass@128": 0.6012461059190031 + }, + "15": { + "pass@1": 0.2457550578034682, + "pass@2": 0.29157006679259007, + "pass@4": 0.33352885432312285, + "pass@8": 0.37290426380904296, + "pass@16": 0.4105600444996007, + "pass@32": 0.4447149669473933, + "pass@64": 0.4736651101410065, + "pass@128": 0.49710982658959535 + }, + "16": { + "pass@1": 0.22184979838709679, + "pass@2": 0.26663068326136635, + "pass@4": 0.30263250511024836, + "pass@8": 0.3322350800902826, + "pass@16": 0.3567520519571173, + "pass@32": 0.3789453955002686, + "pass@64": 0.4006622686591528, + "pass@128": 0.42258064516129035 + }, + "17": { + "pass@1": 0.1810196893063584, + "pass@2": 0.22069610224841837, + "pass@4": 0.2561653288642387, + "pass@8": 0.2871249636415423, + "pass@16": 0.31473971022818265, + "pass@32": 0.3398529280723456, + "pass@64": 0.3637549328856826, + "pass@128": 0.38439306358381503 + }, + "18": { + "pass@1": 0.1610044642857143, + "pass@2": 0.19778894825646798, + "pass@4": 0.23138643687396207, + "pass@8": 0.26264321026468734, + "pass@16": 0.29186077506817065, + "pass@32": 0.32167058957108174, + "pass@64": 0.3512736526766329, + "pass@128": 0.3742857142857143 + }, + "19": { + "pass@1": 0.16532939189189189, + "pass@2": 0.2147416855832209, + "pass@4": 0.2589971599833805, + "pass@8": 0.296842774153626, + "pass@16": 0.32952997822341357, + "pass@32": 0.3608921725080611, + "pass@64": 0.3914425486882824, + "pass@128": 0.4144144144144144 + }, + "20": { + "pass@1": 0.17171077327327328, + "pass@2": 0.2125411582842685, + "pass@4": 0.24931663215521463, + "pass@8": 0.28445708954372556, + "pass@16": 0.31796378367059186, + "pass@32": 0.3504592271011099, + "pass@64": 0.3830969877606695, + "pass@128": 0.4174174174174174 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 531958, + "answer_accuracy": 0.6478444076383476, + "avg_response_len": 252.07089584957131, + "resp_tokens_sum": 206980454, + "pass_at_k": { + "pass@1": 0.6478444076383476, + "pass@2": 0.6735689605439996, + "pass@4": 0.6949296075324933, + "pass@8": 0.7136456940962927, + "pass@16": 0.7308214637042161, + "pass@32": 0.7465964425889449, + "pass@64": 0.7609117522322542, + "pass@128": 0.7742790335151988 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8846098542944786, + "pass@2": 0.9002780662769918, + "pass@4": 0.9112371256507044, + "pass@8": 0.920703275994343, + "pass@16": 0.928180327350798, + "pass@32": 0.9340597884127482, + "pass@64": 0.9389806489566199, + "pass@128": 0.941717791411043 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9972064393939394, + "pass@2": 0.9974264047959914, + "pass@4": 0.9978202057129222, + "pass@8": 0.9984495141444165, + "pass@16": 0.9992439943304724, + "pass@32": 0.9998493181625818, + "pass@64": 0.9999979765592963, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9979428624260355, + "pass@2": 0.9990397719330942, + "pass@4": 0.9997296856938741, + "pass@8": 0.999977983651016, + "pass@16": 0.999999898523723, + "pass@32": 0.9999999999996828, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9704483695652174, + "pass@2": 0.9743309939518432, + "pass@4": 0.9780603405552569, + "pass@8": 0.9814549315371892, + "pass@16": 0.9847236492950605, + "pass@32": 0.9876855114215582, + "pass@64": 0.9896688751350827, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9924785100286533, + "pass@2": 0.9948129357218599, + "pass@4": 0.9960934836583821, + "pass@8": 0.9969392940630192, + "pass@16": 0.9974515227801705, + "pass@32": 0.9978507255774547, + "pass@64": 0.9985673352433165, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9881909320809249, + "pass@2": 0.9905351087797553, + "pass@4": 0.9912523347876314, + "pass@8": 0.9913285922353743, + "pass@16": 0.9913294796978018, + "pass@32": 0.9913294797687862, + "pass@64": 0.9913294797687862, + "pass@128": 0.9913294797687862 + }, + "8": { + "pass@1": 0.968794014084507, + "pass@2": 0.9768676527669956, + "pass@4": 0.9810858080239967, + "pass@8": 0.9833780732591677, + "pass@16": 0.9854529291572443, + "pass@32": 0.9874980186150435, + "pass@64": 0.9886114136046574, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9583645209580839, + "pass@2": 0.9689787501768117, + "pass@4": 0.9740201422239886, + "pass@8": 0.9778193850824559, + "pass@16": 0.9817966272607928, + "pass@32": 0.9852710419406909, + "pass@64": 0.9874857148809907, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8485739834337349, + "pass@2": 0.8717270657432884, + "pass@4": 0.8875434908927042, + "pass@8": 0.900419181851555, + "pass@16": 0.911604703248638, + "pass@32": 0.9220797702645877, + "pass@64": 0.9313253131337487, + "pass@128": 0.9397590361445783 + }, + "12": { + "pass@1": 0.7480162617554859, + "pass@2": 0.7935103392738131, + "pass@4": 0.8251117530763199, + "pass@8": 0.8481994805867784, + "pass@16": 0.8673263961259938, + "pass@32": 0.8842887166756281, + "pass@64": 0.8996039865796801, + "pass@128": 0.9153605015673981 + }, + "13": { + "pass@1": 0.4614189425770308, + "pass@2": 0.5181631707801232, + "pass@4": 0.5654716180435426, + "pass@8": 0.6084657762411573, + "pass@16": 0.650034763496025, + "pass@32": 0.6886026576493296, + "pass@64": 0.7222463189761011, + "pass@128": 0.7535014005602241 + }, + "14": { + "pass@1": 0.38966557017543857, + "pass@2": 0.43866052516461757, + "pass@4": 0.4775347327746312, + "pass@8": 0.5106014603664322, + "pass@16": 0.5421969559925761, + "pass@32": 0.5743889726779975, + "pass@64": 0.6071513767259988, + "pass@128": 0.6432748538011696 + }, + "15": { + "pass@1": 0.25414299242424243, + "pass@2": 0.3020221904080171, + "pass@4": 0.34498625342286765, + "pass@8": 0.38554057450292495, + "pass@16": 0.42102806619022154, + "pass@32": 0.4483743656957863, + "pass@64": 0.46978329116358775, + "pass@128": 0.4909090909090909 + }, + "16": { + "pass@1": 0.19816150284900286, + "pass@2": 0.2385198645041165, + "pass@4": 0.2712453993464492, + "pass@8": 0.3002115489394276, + "pass@16": 0.33021755688763793, + "pass@32": 0.3629484109481779, + "pass@64": 0.39703810753951246, + "pass@128": 0.4301994301994302 + }, + "17": { + "pass@1": 0.19232822204968944, + "pass@2": 0.24024803455274613, + "pass@4": 0.28360422241474476, + "pass@8": 0.32167776261409275, + "pass@16": 0.3578691530927828, + "pass@32": 0.3935301291607172, + "pass@64": 0.4269889693410022, + "pass@128": 0.45652173913043476 + }, + "18": { + "pass@1": 0.1627297794117647, + "pass@2": 0.21125484888837412, + "pass@4": 0.2557750033084101, + "pass@8": 0.294215639407568, + "pass@16": 0.32724684342872096, + "pass@32": 0.3559548424763023, + "pass@64": 0.382033785221312, + "pass@128": 0.4088235294117647 + }, + "19": { + "pass@1": 0.14957682291666666, + "pass@2": 0.19273652805118105, + "pass@4": 0.23311154353473673, + "pass@8": 0.27019836298649635, + "pass@16": 0.3039869333375479, + "pass@32": 0.3345311995737657, + "pass@64": 0.36212989556933894, + "pass@128": 0.38392857142857145 + }, + "20": { + "pass@1": 0.13704622005988024, + "pass@2": 0.18041611167428911, + "pass@4": 0.22258806767791758, + "pass@8": 0.26292508542741727, + "pass@16": 0.299340227526878, + "pass@32": 0.330316341657262, + "pass@64": 0.35724850294988825, + "pass@128": 0.38023952095808383 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 518934, + "answer_accuracy": 0.6477347619428023, + "avg_response_len": 270.6347484622144, + "resp_tokens_sum": 216819570, + "pass_at_k": { + "pass@1": 0.6477347619428023, + "pass@2": 0.6729965542532144, + "pass@4": 0.694529736541958, + "pass@8": 0.7137905490411974, + "pass@16": 0.7312837376191367, + "pass@32": 0.7469832429372135, + "pass@64": 0.7611960808325885, + "pass@128": 0.7737657772807157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8892957089552239, + "pass@2": 0.9057556704665646, + "pass@4": 0.9162532422253183, + "pass@8": 0.9243341199575077, + "pass@16": 0.9307240765602192, + "pass@32": 0.9350739008984293, + "pass@64": 0.9379850574461723, + "pass@128": 0.9402985074626866 + }, + "2": { + "pass@1": 0.9997098214285715, + "pass@2": 0.999972581552306, + "pass@4": 0.9999998085060796, + "pass@8": 0.9999999999974281, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9971984329446064, + "pass@2": 0.9973955344229929, + "pass@4": 0.9976782940397756, + "pass@8": 0.9981652294469208, + "pass@16": 0.9988826259348237, + "pass@32": 0.9996323772550161, + "pass@64": 0.9999808398763494, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9945860745614035, + "pass@2": 0.9950729842980154, + "pass@4": 0.9958536131448481, + "pass@8": 0.9970718402735894, + "pass@16": 0.9985582880872056, + "pass@32": 0.9996730624062373, + "pass@64": 0.9999907663284503, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9663143382352941, + "pass@2": 0.9694860178323298, + "pass@4": 0.9722351474448048, + "pass@8": 0.9755392992634085, + "pass@16": 0.9793933800658496, + "pass@32": 0.9823811114856312, + "pass@64": 0.9838078008574993, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9973629731861199, + "pass@2": 0.9984681220348246, + "pass@4": 0.9992779798424248, + "pass@8": 0.9998441685791019, + "pass@16": 0.9999940349535459, + "pass@32": 0.9999999966955364, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9751038370253164, + "pass@2": 0.9806318835343366, + "pass@4": 0.9847448105774753, + "pass@8": 0.9874530271372075, + "pass@16": 0.9892156409124943, + "pass@32": 0.9902597707262977, + "pass@64": 0.9905017189678624, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.9673423423423423, + "pass@2": 0.9762660051784069, + "pass@4": 0.9823950601332488, + "pass@8": 0.9866693202134098, + "pass@16": 0.9895049053205298, + "pass@32": 0.9919861992734489, + "pass@64": 0.9937088592336957, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9417956189710611, + "pass@2": 0.9539905720687647, + "pass@4": 0.9617673268132481, + "pass@8": 0.9670705477659864, + "pass@16": 0.9716396337451848, + "pass@32": 0.9758638941571824, + "pass@64": 0.9788092784862537, + "pass@128": 0.9807073954983923 + }, + "11": { + "pass@1": 0.8631200396825397, + "pass@2": 0.8873668135233092, + "pass@4": 0.9023296802185442, + "pass@8": 0.9139362203436046, + "pass@16": 0.9238145883317608, + "pass@32": 0.9313328902460989, + "pass@64": 0.9375069285089583, + "pass@128": 0.9428571428571428 + }, + "12": { + "pass@1": 0.7282826834862385, + "pass@2": 0.7779322882804786, + "pass@4": 0.8170610029824253, + "pass@8": 0.8498968778339464, + "pass@16": 0.8772922076732741, + "pass@32": 0.8990817030643646, + "pass@64": 0.9153620294937252, + "pass@128": 0.926605504587156 + }, + "13": { + "pass@1": 0.5295138888888888, + "pass@2": 0.5883043373918536, + "pass@4": 0.6362882339360357, + "pass@8": 0.6759673449866654, + "pass@16": 0.7097663096176011, + "pass@32": 0.7375562398906225, + "pass@64": 0.7616677793162406, + "pass@128": 0.7808641975308642 + }, + "14": { + "pass@1": 0.3577058605341246, + "pass@2": 0.40876794726512283, + "pass@4": 0.4565106505603714, + "pass@8": 0.4996678465065184, + "pass@16": 0.5361346834509522, + "pass@32": 0.5672766953822151, + "pass@64": 0.5954790295606865, + "pass@128": 0.6172106824925816 + }, + "15": { + "pass@1": 0.23336226851851852, + "pass@2": 0.28335064887722383, + "pass@4": 0.3287279220305795, + "pass@8": 0.37229036077603983, + "pass@16": 0.4153575676276682, + "pass@32": 0.4545569214715594, + "pass@64": 0.48957127857625266, + "pass@128": 0.5216049382716049 + }, + "16": { + "pass@1": 0.22121589233038347, + "pass@2": 0.26422882842078366, + "pass@4": 0.29995856214875805, + "pass@8": 0.33195147122739216, + "pass@16": 0.3610778167272766, + "pass@32": 0.3877967539917674, + "pass@64": 0.4144776267893019, + "pass@128": 0.4424778761061947 + }, + "17": { + "pass@1": 0.16177993222891565, + "pass@2": 0.20444165935395123, + "pass@4": 0.24705405621360577, + "pass@8": 0.288054495373893, + "pass@16": 0.3239910809999265, + "pass@32": 0.3561570186864724, + "pass@64": 0.38991113117665654, + "pass@128": 0.4246987951807229 + }, + "18": { + "pass@1": 0.16353326612903227, + "pass@2": 0.19809975869951738, + "pass@4": 0.2304538341981446, + "pass@8": 0.26223427015498135, + "pass@16": 0.2945088081050385, + "pass@32": 0.32849514432266463, + "pass@64": 0.3624810386483276, + "pass@128": 0.3935483870967742 + }, + "19": { + "pass@1": 0.14520392749244712, + "pass@2": 0.18464128327425824, + "pass@4": 0.21987888617397153, + "pass@8": 0.2509003220465607, + "pass@16": 0.2791341129996524, + "pass@32": 0.30528950755645506, + "pass@64": 0.3276331403258758, + "pass@128": 0.34441087613293053 + }, + "20": { + "pass@1": 0.15343468468468469, + "pass@2": 0.193956382563666, + "pass@4": 0.23039945232071216, + "pass@8": 0.26507179724896024, + "pass@16": 0.3007939940823713, + "pass@32": 0.3370014674190456, + "pass@64": 0.3718108648660131, + "pass@128": 0.40540540540540543 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/checkpoint-22350_metrics.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/checkpoint-22350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2d6ff2f038788a9ae9ff51e0a25db93bb15373 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/checkpoint-22350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-22350", + "id": { + "answer_accuracy": 0.9301788194444445, + "avg_response_len": 196.3179861111111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9301788194444445, + "pass@2": 0.9509085739282445, + "pass@4": 0.9640604516935356, + "pass@8": 0.9732268796355469, + "pass@16": 0.9794884848692531, + "pass@32": 0.9838704871906203, + "pass@64": 0.9873198892516832, + "pass@128": 0.9902222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.842375, + "pass@2": 0.8851686761811031, + "pass@4": 0.910745237720285, + "pass@8": 0.9282465725653735, + "pass@16": 0.9404889074173671, + "pass@32": 0.949493652362537, + "pass@64": 0.9567908707914207, + "pass@128": 0.963 + }, + "2": { + "pass@1": 0.9682265625, + "pass@2": 0.973982283464567, + "pass@4": 0.9788314984064492, + "pass@8": 0.9834760495500084, + "pass@16": 0.9876226291653293, + "pass@32": 0.9907697851838029, + "pass@64": 0.993023081142325, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.99221875, + "pass@2": 0.9951913139763781, + "pass@4": 0.9967813782339707, + "pass@8": 0.9978019018273887, + "pass@16": 0.9982985859310679, + "pass@32": 0.998581266552144, + "pass@64": 0.9988779527556754, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.9907265625, + "pass@2": 0.9943736466535433, + "pass@4": 0.9962627990251218, + "pass@8": 0.9974505179380317, + "pass@16": 0.9980318766060151, + "pass@32": 0.9982480594537915, + "pass@64": 0.9984999999225515, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.96440625, + "pass@2": 0.9729023129921262, + "pass@4": 0.9791136643232095, + "pass@8": 0.9842856809776525, + "pass@16": 0.9881543397420988, + "pass@32": 0.9909226792166675, + "pass@64": 0.9931605575768252, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.93046875, + "pass@2": 0.9518184055118113, + "pass@4": 0.9655051450131226, + "pass@8": 0.9747187754836814, + "pass@16": 0.9810129355110199, + "pass@32": 0.9858519076302539, + "pass@64": 0.9899221984968051, + "pass@128": 0.992 + }, + "7": { + "pass@1": 0.917796875, + "pass@2": 0.9403061023622051, + "pass@4": 0.9557353846081736, + "pass@8": 0.9669838256112889, + "pass@16": 0.9746646172909683, + "pass@32": 0.9800978019508552, + "pass@64": 0.9848486304305892, + "pass@128": 0.989 + }, + "8": { + "pass@1": 0.897046875, + "pass@2": 0.9290511811023631, + "pass@4": 0.9491970415260598, + "pass@8": 0.9627610929940758, + "pass@16": 0.9722511641735038, + "pass@32": 0.9792204884202885, + "pass@64": 0.9844638888225717, + "pass@128": 0.989 + }, + "9": { + "pass@1": 0.86834375, + "pass@2": 0.9153832431102378, + "pass@4": 0.9443719163854521, + "pass@8": 0.9633174997724309, + "pass@16": 0.9748713079858962, + "pass@32": 0.9816487439452415, + "pass@64": 0.9862918233263783, + "pass@128": 0.991 + } + }, + "per_op_accuracy": { + "10": 0.842375, + "2": 0.9682265625, + "3": 0.99221875, + "4": 0.9907265625, + "5": 0.96440625, + "6": 0.93046875, + "7": 0.917796875, + "8": 0.897046875, + "9": 0.86834375 + }, + "per_op_avg_response_len": { + "10": 272.7595703125, + "2": 116.34684375, + "3": 143.7793828125, + "4": 157.8691015625, + "5": 172.0627734375, + "6": 195.70890625, + "7": 218.2772265625, + "8": 237.9661171875, + "9": 252.091953125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.7595703125, + "2": 116.34684375, + "3": 143.7793828125, + "4": 157.8691015625, + "5": 172.0627734375, + "6": 195.70890625, + "7": 218.2772265625, + "8": 237.9661171875, + "9": 252.091953125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 355674, + "answer_accuracy": 0.9384340172239108, + "avg_response_len": 186.9961214539007, + "resp_tokens_sum": 70873026, + "pass_at_k": { + "pass@1": 0.9384340172239108, + "pass@2": 0.958179221945663, + "pass@4": 0.9704006980130531, + "pass@8": 0.9785403585472229, + "pass@16": 0.983421701641228, + "pass@32": 0.9862881069457309, + "pass@64": 0.9883645914608837, + "pass@128": 0.9902060114826072 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8670953171091446, + "pass@2": 0.9082642324576685, + "pass@4": 0.9344302874751275, + "pass@8": 0.9526031337738796, + "pass@16": 0.9629272671248992, + "pass@32": 0.9682270358052656, + "pass@64": 0.9723204617269156, + "pass@128": 0.976401179941003 + }, + "2": { + "pass@1": 0.9599639797507789, + "pass@2": 0.963836022395565, + "pass@4": 0.967983610109484, + "pass@8": 0.97280747375255, + "pass@16": 0.9773608156099567, + "pass@32": 0.9804489575520082, + "pass@64": 0.9824826081902464, + "pass@128": 0.9844236760124611 + }, + "3": { + "pass@1": 0.9889860474006116, + "pass@2": 0.9924104240410316, + "pass@4": 0.9947170838163579, + "pass@8": 0.9962247167758309, + "pass@16": 0.9968678062677515, + "pass@32": 0.9969414013686603, + "pass@64": 0.9969418960237719, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.9893798828125, + "pass@2": 0.99249969242126, + "pass@4": 0.99361593257874, + "pass@8": 0.993938088917673, + "pass@16": 0.9941406211230228, + "pass@32": 0.99453125, + "pass@64": 0.9953125, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9679563492063492, + "pass@2": 0.9771223909511312, + "pass@4": 0.9827705200540406, + "pass@8": 0.986222767171394, + "pass@16": 0.9884563271556579, + "pass@32": 0.9898710602540834, + "pass@64": 0.9904321086955653, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9334534056886228, + "pass@2": 0.9554382868122028, + "pass@4": 0.9689164457923299, + "pass@8": 0.977704755737177, + "pass@16": 0.9830688043745892, + "pass@32": 0.986982241257549, + "pass@64": 0.9900670186881447, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9388406065088757, + "pass@2": 0.9596263773703585, + "pass@4": 0.9736909371165883, + "pass@8": 0.9825845522685667, + "pass@16": 0.9870793750133965, + "pass@32": 0.9887747654939287, + "pass@64": 0.9896445965776214, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.9217998798076923, + "pass@2": 0.9474363706339591, + "pass@4": 0.9611889871939082, + "pass@8": 0.9712367571433583, + "pass@16": 0.9788797736051013, + "pass@32": 0.9848084145955653, + "pass@64": 0.989404376252009, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.8873239436619719, + "pass@2": 0.9320952783630917, + "pass@4": 0.958994276683724, + "pass@8": 0.9748610209136754, + "pass@16": 0.9827990984511219, + "pass@32": 0.9866721612838238, + "pass@64": 0.9892717297087438, + "pass@128": 0.9915492957746479 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 361088, + "answer_accuracy": 0.9243119266055045, + "avg_response_len": 193.81848736484272, + "resp_tokens_sum": 75716355, + "pass_at_k": { + "pass@1": 0.9243119266055045, + "pass@2": 0.9454117072450259, + "pass@4": 0.9588298313239195, + "pass@8": 0.968254630894631, + "pass@16": 0.975258408470879, + "pass@32": 0.9807515653079653, + "pass@64": 0.9851800662305187, + "pass@128": 0.9885321100917431 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8187787576687117, + "pass@2": 0.8631999208975412, + "pass@4": 0.8889652101577791, + "pass@8": 0.9047884008029274, + "pass@16": 0.9158893168854205, + "pass@32": 0.92519370344476, + "pass@64": 0.9330087088515969, + "pass@128": 0.9386503067484663 + }, + "2": { + "pass@1": 0.9752089665653495, + "pass@2": 0.9798478896919802, + "pass@4": 0.9832519405818955, + "pass@8": 0.9870105278631578, + "pass@16": 0.9908648345867141, + "pass@32": 0.9942662077657641, + "pass@64": 0.996862131434271, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942945075757575, + "pass@2": 0.9963180028632787, + "pass@4": 0.9970773806683255, + "pass@8": 0.9974988500184285, + "pass@16": 0.9979768035033618, + "pass@32": 0.9987316926293286, + "pass@64": 0.9996301598663804, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994221523668639, + "pass@2": 0.9966635896659369, + "pass@4": 0.9981680000798717, + "pass@8": 0.9992769656375657, + "pass@16": 0.9998620255616644, + "pass@32": 0.9999962385209578, + "pass@64": 0.999999999802363, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9785326086956522, + "pass@2": 0.9834570780554607, + "pass@4": 0.9865280187802611, + "pass@8": 0.9891545568020785, + "pass@16": 0.9914041524744066, + "pass@32": 0.9936531060212447, + "pass@64": 0.996363467201046, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.921181053008596, + "pass@2": 0.9443954041919544, + "pass@4": 0.9595953014827298, + "pass@8": 0.970122744120205, + "pass@16": 0.9776500126894409, + "pass@32": 0.9831185714417165, + "pass@64": 0.9870883967891798, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.9094337066473989, + "pass@2": 0.9308299861180648, + "pass@4": 0.9450992366019274, + "pass@8": 0.9571766432271784, + "pass@16": 0.9673169836926407, + "pass@32": 0.9756833949170814, + "pass@64": 0.9829578324080214, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.8751760563380282, + "pass@2": 0.912961974603527, + "pass@4": 0.9388333774299338, + "pass@8": 0.9557506365071035, + "pass@16": 0.967302846071004, + "pass@32": 0.9765007185796407, + "pass@64": 0.9839139050837533, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.8521940494011976, + "pass@2": 0.9009305454052524, + "pass@4": 0.931433134855149, + "pass@8": 0.9526636585593052, + "pass@16": 0.9679941054740805, + "pass@32": 0.9783950530139177, + "pass@64": 0.9854222638839946, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 354804, + "answer_accuracy": 0.9279900401740877, + "avg_response_len": 208.11259991211918, + "resp_tokens_sum": 79568939, + "pass_at_k": { + "pass@1": 0.9279900401740877, + "pass@2": 0.9493176962375031, + "pass@4": 0.9631198370353543, + "pass@8": 0.973040100961226, + "pass@16": 0.9799116312723414, + "pass@32": 0.9846607039602808, + "pass@64": 0.988470668166694, + "pass@128": 0.991965182457315 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8403218283582089, + "pass@2": 0.8831759019861319, + "pass@4": 0.9079722738948675, + "pass@8": 0.926427138967982, + "pass@16": 0.9417213330071029, + "pass@32": 0.9541837011986865, + "pass@64": 0.9642190900906727, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9692410714285714, + "pass@2": 0.9777741844769405, + "pass@4": 0.9846253460281751, + "pass@8": 0.9899382480241745, + "pass@16": 0.9939865050728705, + "pass@32": 0.9969488212990623, + "pass@64": 0.9990814933468017, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9933035714285714, + "pass@2": 0.996758496705769, + "pass@4": 0.9984645953512371, + "pass@8": 0.999597081444929, + "pass@16": 0.9999722102781452, + "pass@32": 0.9999999117694275, + "pass@64": 0.9999999999999893, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9885325292397661, + "pass@2": 0.9938638940691626, + "pass@4": 0.996856481207393, + "pass@8": 0.9989319154937989, + "pass@16": 0.9998640737041085, + "pass@32": 0.9999980433734141, + "pass@64": 0.9999999999688675, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9467830882352941, + "pass@2": 0.9582825526864288, + "pass@4": 0.9682023059617548, + "pass@8": 0.9775505506527833, + "pass@16": 0.9845769531894009, + "pass@32": 0.9891263931155932, + "pass@64": 0.9924383739804744, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9375492902208202, + "pass@2": 0.9561767244591276, + "pass@4": 0.968417324608872, + "pass@8": 0.9766326478533517, + "pass@16": 0.9825492126854667, + "pass@32": 0.987670213303071, + "pass@64": 0.9928894756326212, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9044452136075949, + "pass@2": 0.9300164924499152, + "pass@4": 0.9481756708813297, + "pass@8": 0.9610352164174365, + "pass@16": 0.9694307347429958, + "pass@32": 0.9756502423183432, + "pass@64": 0.9817890720701818, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.8971706081081081, + "pass@2": 0.928977575015961, + "pass@4": 0.9490096952970964, + "pass@8": 0.9622935399559339, + "pass@16": 0.9713158091696881, + "pass@32": 0.9768844083204202, + "pass@64": 0.9804212826643012, + "pass@128": 0.984984984984985 + }, + "9": { + "pass@1": 0.864022307073955, + "pass@2": 0.9118283508874097, + "pass@4": 0.9415764666273624, + "pass@8": 0.9615825575217636, + "pass@16": 0.9732077067762223, + "pass@32": 0.9794089356361269, + "pass@64": 0.9838241901688739, + "pass@128": 0.9903536977491961 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.32165859375, + "avg_response_len": 305.1192109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.32165859375, + "pass@2": 0.39340520423229014, + "pass@4": 0.45623275829583937, + "pass@8": 0.5101233036129434, + "pass@16": 0.5564067968545583, + "pass@32": 0.5962946042852723, + "pass@64": 0.631478754254601, + "pass@128": 0.6633 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7714921875, + "pass@2": 0.8339756397637798, + "pass@4": 0.8748189698162727, + "pass@8": 0.9036720365863427, + "pass@16": 0.9257487272967078, + "pass@32": 0.9433549183704942, + "pass@64": 0.9577135339252582, + "pass@128": 0.968 + }, + "12": { + "pass@1": 0.6492265625, + "pass@2": 0.738015378937008, + "pass@4": 0.7991055609298835, + "pass@8": 0.8428072476124577, + "pass@16": 0.8753151803199343, + "pass@32": 0.9001017609139902, + "pass@64": 0.9203730703066859, + "pass@128": 0.935 + }, + "13": { + "pass@1": 0.4325390625, + "pass@2": 0.5346312746062992, + "pass@4": 0.6184741975065617, + "pass@8": 0.6859102514737139, + "pass@16": 0.7412768270826603, + "pass@32": 0.7877307736119611, + "pass@64": 0.8271118052806022, + "pass@128": 0.86 + }, + "14": { + "pass@1": 0.3015234375, + "pass@2": 0.3842117372047243, + "pass@4": 0.4582512019122605, + "pass@8": 0.5227826980417837, + "pass@16": 0.5803850954009546, + "pass@32": 0.6335778715202134, + "pass@64": 0.6848542922039572, + "pass@128": 0.734 + }, + "15": { + "pass@1": 0.2051953125, + "pass@2": 0.27879281496062985, + "pass@4": 0.35127285339332537, + "pass@8": 0.4183948297476776, + "pass@16": 0.4791631947684879, + "pass@32": 0.5314374456317077, + "pass@64": 0.576463195483568, + "pass@128": 0.62 + }, + "16": { + "pass@1": 0.2015, + "pass@2": 0.2660114419291342, + "pass@4": 0.3259961936632919, + "pass@8": 0.379400400754768, + "pass@16": 0.4267204838940661, + "pass@32": 0.470111663420348, + "pass@64": 0.510720644829728, + "pass@128": 0.55 + }, + "17": { + "pass@1": 0.18478125, + "pass@2": 0.24815108267716524, + "pass@4": 0.308099278496438, + "pass@8": 0.3627436118672181, + "pass@16": 0.41280616180438556, + "pass@32": 0.4578132921923998, + "pass@64": 0.4981033892990208, + "pass@128": 0.537 + }, + "18": { + "pass@1": 0.1721953125, + "pass@2": 0.23473548228346444, + "pass@4": 0.29553349006374185, + "pass@8": 0.3495150086940284, + "pass@16": 0.3951179909040983, + "pass@32": 0.4341696571812153, + "pass@64": 0.46857661135579487, + "pass@128": 0.499 + }, + "19": { + "pass@1": 0.1551875, + "pass@2": 0.21523179133858275, + "pass@4": 0.27394932714660647, + "pass@8": 0.3269668371516335, + "pass@16": 0.3717734691703706, + "pass@32": 0.40766288096324815, + "pass@64": 0.4368859774688005, + "pass@128": 0.462 + }, + "20": { + "pass@1": 0.1429453125, + "pass@2": 0.20029539862204715, + "pass@4": 0.25682651002999607, + "pass@8": 0.3090401141997698, + "pass@16": 0.3557608379038028, + "pass@32": 0.3969857790471327, + "pass@64": 0.43398502239256936, + "pass@128": 0.468 + } + }, + "per_op_accuracy": { + "11": 0.7714921875, + "12": 0.6492265625, + "13": 0.4325390625, + "14": 0.3015234375, + "15": 0.2051953125, + "16": 0.2015, + "17": 0.18478125, + "18": 0.1721953125, + "19": 0.1551875, + "20": 0.1429453125 + }, + "per_op_avg_response_len": { + "11": 286.6291328125, + "12": 294.07084375, + "13": 301.6905859375, + "14": 307.832859375, + "15": 306.8066328125, + "16": 307.8000625, + "17": 313.14009375, + "18": 311.6820625, + "19": 309.7851640625, + "20": 311.754671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 286.6291328125, + "12": 294.07084375, + "13": 301.6905859375, + "14": 307.832859375, + "15": 306.8066328125, + "16": 307.8000625, + "17": 313.14009375, + "18": 311.6820625, + "19": 309.7851640625, + "20": 311.754671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 145408, + "answer_accuracy": 0.33759286775631503, + "avg_response_len": 292.0508590267459, + "resp_tokens_sum": 125792146, + "pass_at_k": { + "pass@1": 0.33759286775631503, + "pass@2": 0.4119178946660279, + "pass@4": 0.475692191057842, + "pass@8": 0.5285756495166668, + "pass@16": 0.5724484194434191, + "pass@32": 0.6091434406202278, + "pass@64": 0.641526600648974, + "pass@128": 0.6719167904903418 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7816041076487252, + "pass@2": 0.8401684938993113, + "pass@4": 0.8784991090879926, + "pass@8": 0.9062103229646153, + "pass@16": 0.928634342538452, + "pass@32": 0.9474247894334389, + "pass@64": 0.9635688141184316, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.6643714689265536, + "pass@2": 0.7570847374215929, + "pass@4": 0.8195590050714002, + "pass@8": 0.8638611475587994, + "pass@16": 0.8949019817729846, + "pass@32": 0.9149040498901152, + "pass@64": 0.9297771688529582, + "pass@128": 0.940677966101695 + }, + "13": { + "pass@1": 0.4798932210031348, + "pass@2": 0.5873326925925011, + "pass@4": 0.6713887188474476, + "pass@8": 0.7363040893811634, + "pass@16": 0.7878785286983827, + "pass@32": 0.8272839467742679, + "pass@64": 0.8553831050059669, + "pass@128": 0.8746081504702194 + }, + "14": { + "pass@1": 0.31096670560747663, + "pass@2": 0.40393041553217085, + "pass@4": 0.48734351284360494, + "pass@8": 0.5564510286229742, + "pass@16": 0.613397458431016, + "pass@32": 0.6620214973063218, + "pass@64": 0.7095034575721406, + "pass@128": 0.7601246105919003 + }, + "15": { + "pass@1": 0.22798500722543352, + "pass@2": 0.3055256076191342, + "pass@4": 0.3797083284567752, + "pass@8": 0.4463900949283278, + "pass@16": 0.506202646502336, + "pass@32": 0.5582980599444395, + "pass@64": 0.6044086870879166, + "pass@128": 0.6473988439306358 + }, + "16": { + "pass@1": 0.2020665322580645, + "pass@2": 0.272865601981204, + "pass@4": 0.3412203653978737, + "pass@8": 0.40107650120527977, + "pass@16": 0.45179670722165527, + "pass@32": 0.494565665067616, + "pass@64": 0.5308362340060577, + "pass@128": 0.5645161290322581 + }, + "17": { + "pass@1": 0.19440932080924855, + "pass@2": 0.2594631133084519, + "pass@4": 0.3184659128591581, + "pass@8": 0.36832649582409616, + "pass@16": 0.4102210447790027, + "pass@32": 0.44760686822032736, + "pass@64": 0.48478608490693575, + "pass@128": 0.5260115606936416 + }, + "18": { + "pass@1": 0.1690625, + "pass@2": 0.22736255624296942, + "pass@4": 0.28266300096416525, + "pass@8": 0.3321534924751763, + "pass@16": 0.37531477121326395, + "pass@32": 0.4134994424323006, + "pass@64": 0.4477984271129805, + "pass@128": 0.48 + }, + "19": { + "pass@1": 0.1649540165165165, + "pass@2": 0.22900225520796397, + "pass@4": 0.28934828247820366, + "pass@8": 0.3409590956948496, + "pass@16": 0.38376503808474804, + "pass@32": 0.41959872780770363, + "pass@64": 0.44923215559385915, + "pass@128": 0.4744744744744745 + }, + "20": { + "pass@1": 0.1474756006006006, + "pass@2": 0.20596506940010878, + "pass@4": 0.26194670120063823, + "pass@8": 0.3106229089309864, + "pass@16": 0.3518269313013121, + "pass@32": 0.3880737873045299, + "pass@64": 0.42370540748975977, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 133267, + "answer_accuracy": 0.3095891874814154, + "avg_response_len": 300.28181683021114, + "resp_tokens_sum": 129260512, + "pass_at_k": { + "pass@1": 0.3095891874814154, + "pass@2": 0.3798773226356308, + "pass@4": 0.4414370225911792, + "pass@8": 0.4952648669725449, + "pass@16": 0.5427859106206214, + "pass@32": 0.5845796498627995, + "pass@64": 0.6217558330606549, + "pass@128": 0.655367231638418 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.760706890060241, + "pass@2": 0.827785181078645, + "pass@4": 0.8710749813088118, + "pass@8": 0.9003279944961914, + "pass@16": 0.9204542771350541, + "pass@32": 0.9356246993576314, + "pass@64": 0.9476432584619248, + "pass@128": 0.9548192771084337 + }, + "12": { + "pass@1": 0.6126077586206896, + "pass@2": 0.7043927257917213, + "pass@4": 0.7676139801686229, + "pass@8": 0.8119565346138998, + "pass@16": 0.84612392537352, + "pass@32": 0.875273652747933, + "pass@64": 0.9018302739049897, + "pass@128": 0.9216300940438872 + }, + "13": { + "pass@1": 0.3867953431372549, + "pass@2": 0.48342107512296223, + "pass@4": 0.5674021487860234, + "pass@8": 0.6372697246312006, + "pass@16": 0.6963878858222297, + "pass@32": 0.7491304071442696, + "pass@64": 0.7981109820911362, + "pass@128": 0.8431372549019608 + }, + "14": { + "pass@1": 0.29877101608187134, + "pass@2": 0.3703949526868352, + "pass@4": 0.43101541802888665, + "pass@8": 0.4865946482695511, + "pass@16": 0.5415986897105027, + "pass@32": 0.5971263312909335, + "pass@64": 0.6528434464388628, + "pass@128": 0.7046783625730995 + }, + "15": { + "pass@1": 0.20783617424242423, + "pass@2": 0.2801408524218563, + "pass@4": 0.34825574416834243, + "pass@8": 0.41106899030231925, + "pass@16": 0.4680849755795498, + "pass@32": 0.5172851517081245, + "pass@64": 0.5596152305263337, + "pass@128": 0.6 + }, + "16": { + "pass@1": 0.19382122507122507, + "pass@2": 0.25543441685173957, + "pass@4": 0.3137022444843969, + "pass@8": 0.36664821250482243, + "pass@16": 0.4134268702032012, + "pass@32": 0.45505377915145834, + "pass@64": 0.49311278811442033, + "pass@128": 0.5327635327635327 + }, + "17": { + "pass@1": 0.19494856366459629, + "pass@2": 0.25942795703526167, + "pass@4": 0.3195814134452136, + "pass@8": 0.375845879351016, + "pass@16": 0.4307573783826269, + "pass@32": 0.48238930194144775, + "pass@64": 0.5288904166601099, + "pass@128": 0.577639751552795 + }, + "18": { + "pass@1": 0.16665900735294117, + "pass@2": 0.22696524722093553, + "pass@4": 0.2879940161891529, + "pass@8": 0.3464190547517433, + "pass@16": 0.3993496945302608, + "pass@32": 0.4459605878543203, + "pass@64": 0.4871371319920805, + "pass@128": 0.5235294117647059 + }, + "19": { + "pass@1": 0.14764694940476192, + "pass@2": 0.20653436386857893, + "pass@4": 0.2659328434950095, + "pass@8": 0.3216636232453182, + "pass@16": 0.36939694692013475, + "pass@32": 0.4057203341614509, + "pass@64": 0.4357478660293507, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.1414436751497006, + "pass@2": 0.1992263013343391, + "pass@4": 0.25540902390943654, + "pass@8": 0.3068920327741638, + "pass@16": 0.3528934687646693, + "pass@32": 0.39143395959153, + "pass@64": 0.42026686041833455, + "pass@128": 0.44011976047904194 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 133048, + "answer_accuracy": 0.3176764975550122, + "avg_response_len": 323.53093482579465, + "resp_tokens_sum": 135499932, + "pass_at_k": { + "pass@1": 0.3176764975550122, + "pass@2": 0.38827044338265077, + "pass@4": 0.4514274612086248, + "pass@8": 0.5064181625541392, + "pass@16": 0.5539089302265313, + "pass@32": 0.5951213333364987, + "pass@64": 0.6311386506048924, + "pass@128": 0.6625916870415648 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7715277777777778, + "pass@2": 0.8335602580927388, + "pass@4": 0.8746409222656686, + "pass@8": 0.9043520584352333, + "pass@16": 0.9280951884818978, + "pass@32": 0.9469414841706554, + "pass@64": 0.9617656531177552, + "pass@128": 0.9714285714285714 + }, + "12": { + "pass@1": 0.6685540902140673, + "pass@2": 0.750171566856895, + "pass@4": 0.8076844448343031, + "pass@8": 0.8501109077517056, + "pass@16": 0.8825882158353087, + "pass@32": 0.9082979572669679, + "pass@64": 0.9282816365658905, + "pass@128": 0.9418960244648318 + }, + "13": { + "pass@1": 0.4363184799382716, + "pass@2": 0.5391692032419559, + "pass@4": 0.6226500897457258, + "pass@8": 0.6898887508264963, + "pass@16": 0.7448552505843882, + "pass@32": 0.7913198741989679, + "pass@64": 0.831231401781366, + "pass@128": 0.8641975308641975 + }, + "14": { + "pass@1": 0.29532177299703266, + "pass@2": 0.3794510385756675, + "pass@4": 0.4581800039275504, + "pass@8": 0.5274378283252875, + "pass@16": 0.5883019565685669, + "pass@32": 0.6434770492088578, + "pass@64": 0.6938611977483916, + "pass@128": 0.7388724035608308 + }, + "15": { + "pass@1": 0.1781684027777778, + "pass@2": 0.2488718315592497, + "pass@4": 0.323979556085582, + "pass@8": 0.3959601546379987, + "pass@16": 0.4615711022760134, + "pass@32": 0.5171673358865758, + "pass@64": 0.5637801348069706, + "pass@128": 0.6111111111111112 + }, + "16": { + "pass@1": 0.2089325221238938, + "pass@2": 0.27069505899705004, + "pass@4": 0.32480351792309137, + "pass@8": 0.3727821911266633, + "pass@16": 0.4175536083009712, + "pass@32": 0.46334050373812846, + "pass@64": 0.5105570019459841, + "pass@128": 0.5545722713864307 + }, + "17": { + "pass@1": 0.16488610692771086, + "pass@2": 0.22542482923821255, + "pass@4": 0.2861592093909947, + "pass@8": 0.3442176842200421, + "pass@16": 0.39808977250495636, + "pass@32": 0.44461433904524217, + "pass@64": 0.4821225595080291, + "pass@128": 0.5090361445783133 + }, + "18": { + "pass@1": 0.18180443548387096, + "pass@2": 0.251581946913894, + "pass@4": 0.3183337878128136, + "pass@8": 0.37251228294233507, + "pass@16": 0.4128352414166689, + "pass@32": 0.44457500793368165, + "pass@64": 0.47167915157723894, + "pass@128": 0.4935483870967742 + }, + "19": { + "pass@1": 0.15301642749244712, + "pass@2": 0.21020693079905794, + "pass@4": 0.26659490533849434, + "pass@8": 0.31827335611728563, + "pass@16": 0.36212186502412136, + "pass@32": 0.3976268046073573, + "pass@64": 0.4256205035352982, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.13992117117117117, + "pass@2": 0.1956980356340592, + "pass@4": 0.2531280616949908, + "pass@8": 0.3096118515891317, + "pass@16": 0.36257072436356236, + "pass@32": 0.411466262375835, + "pass@64": 0.45802399495121837, + "pass@128": 0.5045045045045045 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6099050164473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6099050164473684, + "pass@2": 0.6574857477724582, + "pass@4": 0.6967827183263103, + "pass@8": 0.7294881554131272, + "pass@16": 0.7568139122299491, + "pass@32": 0.7798831803983461, + "pass@64": 0.8000350813584826, + "pass@128": 0.8181578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.842375, + "pass@2": 0.8851686761811031, + "pass@4": 0.910745237720285, + "pass@8": 0.9282465725653735, + "pass@16": 0.9404889074173671, + "pass@32": 0.949493652362537, + "pass@64": 0.9567908707914207, + "pass@128": 0.963 + }, + "2": { + "pass@1": 0.9682265625, + "pass@2": 0.973982283464567, + "pass@4": 0.9788314984064492, + "pass@8": 0.9834760495500084, + "pass@16": 0.9876226291653293, + "pass@32": 0.9907697851838029, + "pass@64": 0.993023081142325, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.99221875, + "pass@2": 0.9951913139763781, + "pass@4": 0.9967813782339707, + "pass@8": 0.9978019018273887, + "pass@16": 0.9982985859310679, + "pass@32": 0.998581266552144, + "pass@64": 0.9988779527556754, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.9907265625, + "pass@2": 0.9943736466535433, + "pass@4": 0.9962627990251218, + "pass@8": 0.9974505179380317, + "pass@16": 0.9980318766060151, + "pass@32": 0.9982480594537915, + "pass@64": 0.9984999999225515, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.96440625, + "pass@2": 0.9729023129921262, + "pass@4": 0.9791136643232095, + "pass@8": 0.9842856809776525, + "pass@16": 0.9881543397420988, + "pass@32": 0.9909226792166675, + "pass@64": 0.9931605575768252, + "pass@128": 0.995 + }, + "6": { + "pass@1": 0.93046875, + "pass@2": 0.9518184055118113, + "pass@4": 0.9655051450131226, + "pass@8": 0.9747187754836814, + "pass@16": 0.9810129355110199, + "pass@32": 0.9858519076302539, + "pass@64": 0.9899221984968051, + "pass@128": 0.992 + }, + "7": { + "pass@1": 0.917796875, + "pass@2": 0.9403061023622051, + "pass@4": 0.9557353846081736, + "pass@8": 0.9669838256112889, + "pass@16": 0.9746646172909683, + "pass@32": 0.9800978019508552, + "pass@64": 0.9848486304305892, + "pass@128": 0.989 + }, + "8": { + "pass@1": 0.897046875, + "pass@2": 0.9290511811023631, + "pass@4": 0.9491970415260598, + "pass@8": 0.9627610929940758, + "pass@16": 0.9722511641735038, + "pass@32": 0.9792204884202885, + "pass@64": 0.9844638888225717, + "pass@128": 0.989 + }, + "9": { + "pass@1": 0.86834375, + "pass@2": 0.9153832431102378, + "pass@4": 0.9443719163854521, + "pass@8": 0.9633174997724309, + "pass@16": 0.9748713079858962, + "pass@32": 0.9816487439452415, + "pass@64": 0.9862918233263783, + "pass@128": 0.991 + }, + "11": { + "pass@1": 0.7714921875, + "pass@2": 0.8339756397637798, + "pass@4": 0.8748189698162727, + "pass@8": 0.9036720365863427, + "pass@16": 0.9257487272967078, + "pass@32": 0.9433549183704942, + "pass@64": 0.9577135339252582, + "pass@128": 0.968 + }, + "12": { + "pass@1": 0.6492265625, + "pass@2": 0.738015378937008, + "pass@4": 0.7991055609298835, + "pass@8": 0.8428072476124577, + "pass@16": 0.8753151803199343, + "pass@32": 0.9001017609139902, + "pass@64": 0.9203730703066859, + "pass@128": 0.935 + }, + "13": { + "pass@1": 0.4325390625, + "pass@2": 0.5346312746062992, + "pass@4": 0.6184741975065617, + "pass@8": 0.6859102514737139, + "pass@16": 0.7412768270826603, + "pass@32": 0.7877307736119611, + "pass@64": 0.8271118052806022, + "pass@128": 0.86 + }, + "14": { + "pass@1": 0.3015234375, + "pass@2": 0.3842117372047243, + "pass@4": 0.4582512019122605, + "pass@8": 0.5227826980417837, + "pass@16": 0.5803850954009546, + "pass@32": 0.6335778715202134, + "pass@64": 0.6848542922039572, + "pass@128": 0.734 + }, + "15": { + "pass@1": 0.2051953125, + "pass@2": 0.27879281496062985, + "pass@4": 0.35127285339332537, + "pass@8": 0.4183948297476776, + "pass@16": 0.4791631947684879, + "pass@32": 0.5314374456317077, + "pass@64": 0.576463195483568, + "pass@128": 0.62 + }, + "16": { + "pass@1": 0.2015, + "pass@2": 0.2660114419291342, + "pass@4": 0.3259961936632919, + "pass@8": 0.379400400754768, + "pass@16": 0.4267204838940661, + "pass@32": 0.470111663420348, + "pass@64": 0.510720644829728, + "pass@128": 0.55 + }, + "17": { + "pass@1": 0.18478125, + "pass@2": 0.24815108267716524, + "pass@4": 0.308099278496438, + "pass@8": 0.3627436118672181, + "pass@16": 0.41280616180438556, + "pass@32": 0.4578132921923998, + "pass@64": 0.4981033892990208, + "pass@128": 0.537 + }, + "18": { + "pass@1": 0.1721953125, + "pass@2": 0.23473548228346444, + "pass@4": 0.29553349006374185, + "pass@8": 0.3495150086940284, + "pass@16": 0.3951179909040983, + "pass@32": 0.4341696571812153, + "pass@64": 0.46857661135579487, + "pass@128": 0.499 + }, + "19": { + "pass@1": 0.1551875, + "pass@2": 0.21523179133858275, + "pass@4": 0.27394932714660647, + "pass@8": 0.3269668371516335, + "pass@16": 0.3717734691703706, + "pass@32": 0.40766288096324815, + "pass@64": 0.4368859774688005, + "pass@128": 0.462 + }, + "20": { + "pass@1": 0.1429453125, + "pass@2": 0.20029539862204715, + "pass@4": 0.25682651002999607, + "pass@8": 0.3090401141997698, + "pass@16": 0.3557608379038028, + "pass@32": 0.3969857790471327, + "pass@64": 0.43398502239256936, + "pass@128": 0.468 + } + }, + "per_op_accuracy": { + "10": 0.842375, + "2": 0.9682265625, + "3": 0.99221875, + "4": 0.9907265625, + "5": 0.96440625, + "6": 0.93046875, + "7": 0.917796875, + "8": 0.897046875, + "9": 0.86834375, + "11": 0.7714921875, + "12": 0.6492265625, + "13": 0.4325390625, + "14": 0.3015234375, + "15": 0.2051953125, + "16": 0.2015, + "17": 0.18478125, + "18": 0.1721953125, + "19": 0.1551875, + "20": 0.1429453125 + }, + "per_op_avg_response_len": { + "10": 272.7595703125, + "2": 116.34684375, + "3": 143.7793828125, + "4": 157.8691015625, + "5": 172.0627734375, + "6": 195.70890625, + "7": 218.2772265625, + "8": 237.9661171875, + "9": 252.091953125, + "11": 286.6291328125, + "12": 294.07084375, + "13": 301.6905859375, + "14": 307.832859375, + "15": 306.8066328125, + "16": 307.8000625, + "17": 313.14009375, + "18": 311.6820625, + "19": 309.7851640625, + "20": 311.754671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.7595703125, + "2": 116.34684375, + "3": 143.7793828125, + "4": 157.8691015625, + "5": 172.0627734375, + "6": 195.70890625, + "7": 218.2772265625, + "8": 237.9661171875, + "9": 252.091953125, + "11": 286.6291328125, + "12": 294.07084375, + "13": 301.6905859375, + "14": 307.832859375, + "15": 306.8066328125, + "16": 307.8000625, + "17": 313.14009375, + "18": 311.6820625, + "19": 309.7851640625, + "20": 311.754671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 501082, + "answer_accuracy": 0.6188275569079987, + "avg_response_len": 242.87806769680682, + "resp_tokens_sum": 196665172, + "pass_at_k": { + "pass@1": 0.6188275569079987, + "pass@2": 0.6676054997996059, + "pass@4": 0.7072495557581842, + "pass@8": 0.7391898612522771, + "pass@16": 0.7648119807124286, + "pass@32": 0.7856728995183987, + "pass@64": 0.8038704657760815, + "pass@128": 0.8208978817578249 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8670953171091446, + "pass@2": 0.9082642324576685, + "pass@4": 0.9344302874751275, + "pass@8": 0.9526031337738796, + "pass@16": 0.9629272671248992, + "pass@32": 0.9682270358052656, + "pass@64": 0.9723204617269156, + "pass@128": 0.976401179941003 + }, + "2": { + "pass@1": 0.9599639797507789, + "pass@2": 0.963836022395565, + "pass@4": 0.967983610109484, + "pass@8": 0.97280747375255, + "pass@16": 0.9773608156099567, + "pass@32": 0.9804489575520082, + "pass@64": 0.9824826081902464, + "pass@128": 0.9844236760124611 + }, + "3": { + "pass@1": 0.9889860474006116, + "pass@2": 0.9924104240410316, + "pass@4": 0.9947170838163579, + "pass@8": 0.9962247167758309, + "pass@16": 0.9968678062677515, + "pass@32": 0.9969414013686603, + "pass@64": 0.9969418960237719, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.9893798828125, + "pass@2": 0.99249969242126, + "pass@4": 0.99361593257874, + "pass@8": 0.993938088917673, + "pass@16": 0.9941406211230228, + "pass@32": 0.99453125, + "pass@64": 0.9953125, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9679563492063492, + "pass@2": 0.9771223909511312, + "pass@4": 0.9827705200540406, + "pass@8": 0.986222767171394, + "pass@16": 0.9884563271556579, + "pass@32": 0.9898710602540834, + "pass@64": 0.9904321086955653, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9334534056886228, + "pass@2": 0.9554382868122028, + "pass@4": 0.9689164457923299, + "pass@8": 0.977704755737177, + "pass@16": 0.9830688043745892, + "pass@32": 0.986982241257549, + "pass@64": 0.9900670186881447, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9388406065088757, + "pass@2": 0.9596263773703585, + "pass@4": 0.9736909371165883, + "pass@8": 0.9825845522685667, + "pass@16": 0.9870793750133965, + "pass@32": 0.9887747654939287, + "pass@64": 0.9896445965776214, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.9217998798076923, + "pass@2": 0.9474363706339591, + "pass@4": 0.9611889871939082, + "pass@8": 0.9712367571433583, + "pass@16": 0.9788797736051013, + "pass@32": 0.9848084145955653, + "pass@64": 0.989404376252009, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.8873239436619719, + "pass@2": 0.9320952783630917, + "pass@4": 0.958994276683724, + "pass@8": 0.9748610209136754, + "pass@16": 0.9827990984511219, + "pass@32": 0.9866721612838238, + "pass@64": 0.9892717297087438, + "pass@128": 0.9915492957746479 + }, + "11": { + "pass@1": 0.7816041076487252, + "pass@2": 0.8401684938993113, + "pass@4": 0.8784991090879926, + "pass@8": 0.9062103229646153, + "pass@16": 0.928634342538452, + "pass@32": 0.9474247894334389, + "pass@64": 0.9635688141184316, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.6643714689265536, + "pass@2": 0.7570847374215929, + "pass@4": 0.8195590050714002, + "pass@8": 0.8638611475587994, + "pass@16": 0.8949019817729846, + "pass@32": 0.9149040498901152, + "pass@64": 0.9297771688529582, + "pass@128": 0.940677966101695 + }, + "13": { + "pass@1": 0.4798932210031348, + "pass@2": 0.5873326925925011, + "pass@4": 0.6713887188474476, + "pass@8": 0.7363040893811634, + "pass@16": 0.7878785286983827, + "pass@32": 0.8272839467742679, + "pass@64": 0.8553831050059669, + "pass@128": 0.8746081504702194 + }, + "14": { + "pass@1": 0.31096670560747663, + "pass@2": 0.40393041553217085, + "pass@4": 0.48734351284360494, + "pass@8": 0.5564510286229742, + "pass@16": 0.613397458431016, + "pass@32": 0.6620214973063218, + "pass@64": 0.7095034575721406, + "pass@128": 0.7601246105919003 + }, + "15": { + "pass@1": 0.22798500722543352, + "pass@2": 0.3055256076191342, + "pass@4": 0.3797083284567752, + "pass@8": 0.4463900949283278, + "pass@16": 0.506202646502336, + "pass@32": 0.5582980599444395, + "pass@64": 0.6044086870879166, + "pass@128": 0.6473988439306358 + }, + "16": { + "pass@1": 0.2020665322580645, + "pass@2": 0.272865601981204, + "pass@4": 0.3412203653978737, + "pass@8": 0.40107650120527977, + "pass@16": 0.45179670722165527, + "pass@32": 0.494565665067616, + "pass@64": 0.5308362340060577, + "pass@128": 0.5645161290322581 + }, + "17": { + "pass@1": 0.19440932080924855, + "pass@2": 0.2594631133084519, + "pass@4": 0.3184659128591581, + "pass@8": 0.36832649582409616, + "pass@16": 0.4102210447790027, + "pass@32": 0.44760686822032736, + "pass@64": 0.48478608490693575, + "pass@128": 0.5260115606936416 + }, + "18": { + "pass@1": 0.1690625, + "pass@2": 0.22736255624296942, + "pass@4": 0.28266300096416525, + "pass@8": 0.3321534924751763, + "pass@16": 0.37531477121326395, + "pass@32": 0.4134994424323006, + "pass@64": 0.4477984271129805, + "pass@128": 0.48 + }, + "19": { + "pass@1": 0.1649540165165165, + "pass@2": 0.22900225520796397, + "pass@4": 0.28934828247820366, + "pass@8": 0.3409590956948496, + "pass@16": 0.38376503808474804, + "pass@32": 0.41959872780770363, + "pass@64": 0.44923215559385915, + "pass@128": 0.4744744744744745 + }, + "20": { + "pass@1": 0.1474756006006006, + "pass@2": 0.20596506940010878, + "pass@4": 0.26194670120063823, + "pass@8": 0.3106229089309864, + "pass@16": 0.3518269313013121, + "pass@32": 0.3880737873045299, + "pass@64": 0.42370540748975977, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 494355, + "answer_accuracy": 0.6020496395167576, + "avg_response_len": 249.6308298421668, + "resp_tokens_sum": 204976867, + "pass_at_k": { + "pass@1": 0.6020496395167576, + "pass@2": 0.6489359261941491, + "pass@4": 0.6875917930124271, + "pass@8": 0.720294447563385, + "pass@16": 0.7485389992315395, + "pass@32": 0.7730623756521464, + "pass@64": 0.7946585235726484, + "pass@128": 0.8138737334372564 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8187787576687117, + "pass@2": 0.8631999208975412, + "pass@4": 0.8889652101577791, + "pass@8": 0.9047884008029274, + "pass@16": 0.9158893168854205, + "pass@32": 0.92519370344476, + "pass@64": 0.9330087088515969, + "pass@128": 0.9386503067484663 + }, + "2": { + "pass@1": 0.9752089665653495, + "pass@2": 0.9798478896919802, + "pass@4": 0.9832519405818955, + "pass@8": 0.9870105278631578, + "pass@16": 0.9908648345867141, + "pass@32": 0.9942662077657641, + "pass@64": 0.996862131434271, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942945075757575, + "pass@2": 0.9963180028632787, + "pass@4": 0.9970773806683255, + "pass@8": 0.9974988500184285, + "pass@16": 0.9979768035033618, + "pass@32": 0.9987316926293286, + "pass@64": 0.9996301598663804, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994221523668639, + "pass@2": 0.9966635896659369, + "pass@4": 0.9981680000798717, + "pass@8": 0.9992769656375657, + "pass@16": 0.9998620255616644, + "pass@32": 0.9999962385209578, + "pass@64": 0.999999999802363, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9785326086956522, + "pass@2": 0.9834570780554607, + "pass@4": 0.9865280187802611, + "pass@8": 0.9891545568020785, + "pass@16": 0.9914041524744066, + "pass@32": 0.9936531060212447, + "pass@64": 0.996363467201046, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.921181053008596, + "pass@2": 0.9443954041919544, + "pass@4": 0.9595953014827298, + "pass@8": 0.970122744120205, + "pass@16": 0.9776500126894409, + "pass@32": 0.9831185714417165, + "pass@64": 0.9870883967891798, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.9094337066473989, + "pass@2": 0.9308299861180648, + "pass@4": 0.9450992366019274, + "pass@8": 0.9571766432271784, + "pass@16": 0.9673169836926407, + "pass@32": 0.9756833949170814, + "pass@64": 0.9829578324080214, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.8751760563380282, + "pass@2": 0.912961974603527, + "pass@4": 0.9388333774299338, + "pass@8": 0.9557506365071035, + "pass@16": 0.967302846071004, + "pass@32": 0.9765007185796407, + "pass@64": 0.9839139050837533, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.8521940494011976, + "pass@2": 0.9009305454052524, + "pass@4": 0.931433134855149, + "pass@8": 0.9526636585593052, + "pass@16": 0.9679941054740805, + "pass@32": 0.9783950530139177, + "pass@64": 0.9854222638839946, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.760706890060241, + "pass@2": 0.827785181078645, + "pass@4": 0.8710749813088118, + "pass@8": 0.9003279944961914, + "pass@16": 0.9204542771350541, + "pass@32": 0.9356246993576314, + "pass@64": 0.9476432584619248, + "pass@128": 0.9548192771084337 + }, + "12": { + "pass@1": 0.6126077586206896, + "pass@2": 0.7043927257917213, + "pass@4": 0.7676139801686229, + "pass@8": 0.8119565346138998, + "pass@16": 0.84612392537352, + "pass@32": 0.875273652747933, + "pass@64": 0.9018302739049897, + "pass@128": 0.9216300940438872 + }, + "13": { + "pass@1": 0.3867953431372549, + "pass@2": 0.48342107512296223, + "pass@4": 0.5674021487860234, + "pass@8": 0.6372697246312006, + "pass@16": 0.6963878858222297, + "pass@32": 0.7491304071442696, + "pass@64": 0.7981109820911362, + "pass@128": 0.8431372549019608 + }, + "14": { + "pass@1": 0.29877101608187134, + "pass@2": 0.3703949526868352, + "pass@4": 0.43101541802888665, + "pass@8": 0.4865946482695511, + "pass@16": 0.5415986897105027, + "pass@32": 0.5971263312909335, + "pass@64": 0.6528434464388628, + "pass@128": 0.7046783625730995 + }, + "15": { + "pass@1": 0.20783617424242423, + "pass@2": 0.2801408524218563, + "pass@4": 0.34825574416834243, + "pass@8": 0.41106899030231925, + "pass@16": 0.4680849755795498, + "pass@32": 0.5172851517081245, + "pass@64": 0.5596152305263337, + "pass@128": 0.6 + }, + "16": { + "pass@1": 0.19382122507122507, + "pass@2": 0.25543441685173957, + "pass@4": 0.3137022444843969, + "pass@8": 0.36664821250482243, + "pass@16": 0.4134268702032012, + "pass@32": 0.45505377915145834, + "pass@64": 0.49311278811442033, + "pass@128": 0.5327635327635327 + }, + "17": { + "pass@1": 0.19494856366459629, + "pass@2": 0.25942795703526167, + "pass@4": 0.3195814134452136, + "pass@8": 0.375845879351016, + "pass@16": 0.4307573783826269, + "pass@32": 0.48238930194144775, + "pass@64": 0.5288904166601099, + "pass@128": 0.577639751552795 + }, + "18": { + "pass@1": 0.16665900735294117, + "pass@2": 0.22696524722093553, + "pass@4": 0.2879940161891529, + "pass@8": 0.3464190547517433, + "pass@16": 0.3993496945302608, + "pass@32": 0.4459605878543203, + "pass@64": 0.4871371319920805, + "pass@128": 0.5235294117647059 + }, + "19": { + "pass@1": 0.14764694940476192, + "pass@2": 0.20653436386857893, + "pass@4": 0.2659328434950095, + "pass@8": 0.3216636232453182, + "pass@16": 0.36939694692013475, + "pass@32": 0.4057203341614509, + "pass@64": 0.4357478660293507, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.1414436751497006, + "pass@2": 0.1992263013343391, + "pass@4": 0.25540902390943654, + "pass@8": 0.3068920327741638, + "pass@16": 0.3528934687646693, + "pass@32": 0.39143395959153, + "pass@64": 0.42026686041833455, + "pass@128": 0.44011976047904194 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 487852, + "answer_accuracy": 0.6089381290941045, + "avg_response_len": 268.44952143912764, + "resp_tokens_sum": 215068871, + "pass_at_k": { + "pass@1": 0.6089381290941045, + "pass@2": 0.6560205862612997, + "pass@4": 0.695623838680178, + "pass@8": 0.7291054496642166, + "pass@16": 0.7572113855746511, + "pass@32": 0.7810222919645945, + "pass@64": 0.8016692044405078, + "pass@128": 0.8197795174948075 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8403218283582089, + "pass@2": 0.8831759019861319, + "pass@4": 0.9079722738948675, + "pass@8": 0.926427138967982, + "pass@16": 0.9417213330071029, + "pass@32": 0.9541837011986865, + "pass@64": 0.9642190900906727, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9692410714285714, + "pass@2": 0.9777741844769405, + "pass@4": 0.9846253460281751, + "pass@8": 0.9899382480241745, + "pass@16": 0.9939865050728705, + "pass@32": 0.9969488212990623, + "pass@64": 0.9990814933468017, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9933035714285714, + "pass@2": 0.996758496705769, + "pass@4": 0.9984645953512371, + "pass@8": 0.999597081444929, + "pass@16": 0.9999722102781452, + "pass@32": 0.9999999117694275, + "pass@64": 0.9999999999999893, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9885325292397661, + "pass@2": 0.9938638940691626, + "pass@4": 0.996856481207393, + "pass@8": 0.9989319154937989, + "pass@16": 0.9998640737041085, + "pass@32": 0.9999980433734141, + "pass@64": 0.9999999999688675, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9467830882352941, + "pass@2": 0.9582825526864288, + "pass@4": 0.9682023059617548, + "pass@8": 0.9775505506527833, + "pass@16": 0.9845769531894009, + "pass@32": 0.9891263931155932, + "pass@64": 0.9924383739804744, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9375492902208202, + "pass@2": 0.9561767244591276, + "pass@4": 0.968417324608872, + "pass@8": 0.9766326478533517, + "pass@16": 0.9825492126854667, + "pass@32": 0.987670213303071, + "pass@64": 0.9928894756326212, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9044452136075949, + "pass@2": 0.9300164924499152, + "pass@4": 0.9481756708813297, + "pass@8": 0.9610352164174365, + "pass@16": 0.9694307347429958, + "pass@32": 0.9756502423183432, + "pass@64": 0.9817890720701818, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.8971706081081081, + "pass@2": 0.928977575015961, + "pass@4": 0.9490096952970964, + "pass@8": 0.9622935399559339, + "pass@16": 0.9713158091696881, + "pass@32": 0.9768844083204202, + "pass@64": 0.9804212826643012, + "pass@128": 0.984984984984985 + }, + "9": { + "pass@1": 0.864022307073955, + "pass@2": 0.9118283508874097, + "pass@4": 0.9415764666273624, + "pass@8": 0.9615825575217636, + "pass@16": 0.9732077067762223, + "pass@32": 0.9794089356361269, + "pass@64": 0.9838241901688739, + "pass@128": 0.9903536977491961 + }, + "11": { + "pass@1": 0.7715277777777778, + "pass@2": 0.8335602580927388, + "pass@4": 0.8746409222656686, + "pass@8": 0.9043520584352333, + "pass@16": 0.9280951884818978, + "pass@32": 0.9469414841706554, + "pass@64": 0.9617656531177552, + "pass@128": 0.9714285714285714 + }, + "12": { + "pass@1": 0.6685540902140673, + "pass@2": 0.750171566856895, + "pass@4": 0.8076844448343031, + "pass@8": 0.8501109077517056, + "pass@16": 0.8825882158353087, + "pass@32": 0.9082979572669679, + "pass@64": 0.9282816365658905, + "pass@128": 0.9418960244648318 + }, + "13": { + "pass@1": 0.4363184799382716, + "pass@2": 0.5391692032419559, + "pass@4": 0.6226500897457258, + "pass@8": 0.6898887508264963, + "pass@16": 0.7448552505843882, + "pass@32": 0.7913198741989679, + "pass@64": 0.831231401781366, + "pass@128": 0.8641975308641975 + }, + "14": { + "pass@1": 0.29532177299703266, + "pass@2": 0.3794510385756675, + "pass@4": 0.4581800039275504, + "pass@8": 0.5274378283252875, + "pass@16": 0.5883019565685669, + "pass@32": 0.6434770492088578, + "pass@64": 0.6938611977483916, + "pass@128": 0.7388724035608308 + }, + "15": { + "pass@1": 0.1781684027777778, + "pass@2": 0.2488718315592497, + "pass@4": 0.323979556085582, + "pass@8": 0.3959601546379987, + "pass@16": 0.4615711022760134, + "pass@32": 0.5171673358865758, + "pass@64": 0.5637801348069706, + "pass@128": 0.6111111111111112 + }, + "16": { + "pass@1": 0.2089325221238938, + "pass@2": 0.27069505899705004, + "pass@4": 0.32480351792309137, + "pass@8": 0.3727821911266633, + "pass@16": 0.4175536083009712, + "pass@32": 0.46334050373812846, + "pass@64": 0.5105570019459841, + "pass@128": 0.5545722713864307 + }, + "17": { + "pass@1": 0.16488610692771086, + "pass@2": 0.22542482923821255, + "pass@4": 0.2861592093909947, + "pass@8": 0.3442176842200421, + "pass@16": 0.39808977250495636, + "pass@32": 0.44461433904524217, + "pass@64": 0.4821225595080291, + "pass@128": 0.5090361445783133 + }, + "18": { + "pass@1": 0.18180443548387096, + "pass@2": 0.251581946913894, + "pass@4": 0.3183337878128136, + "pass@8": 0.37251228294233507, + "pass@16": 0.4128352414166689, + "pass@32": 0.44457500793368165, + "pass@64": 0.47167915157723894, + "pass@128": 0.4935483870967742 + }, + "19": { + "pass@1": 0.15301642749244712, + "pass@2": 0.21020693079905794, + "pass@4": 0.26659490533849434, + "pass@8": 0.31827335611728563, + "pass@16": 0.36212186502412136, + "pass@32": 0.3976268046073573, + "pass@64": 0.4256205035352982, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.13992117117117117, + "pass@2": 0.1956980356340592, + "pass@4": 0.2531280616949908, + "pass@8": 0.3096118515891317, + "pass@16": 0.36257072436356236, + "pass@32": 0.411466262375835, + "pass@64": 0.45802399495121837, + "pass@128": 0.5045045045045045 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/summary.csv b/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..74245e09e73ce32a44c83b32956d8f840615a52b --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-22350,0.9301788194444445,0.32165859375,0.6099050164473684,0.0,0.0,0.0,196.3179861111111,305.1192109375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/summary.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4e9555e6a11f4c2b5878fc4cf87ffc51dbb7787a --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block4/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-22350", + "id_acc": 0.9301788194444445, + "ood_acc": 0.32165859375, + "total_acc": 0.6099050164473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.3179861111111, + "ood_avg_resp_len": 305.1192109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 501082, + "answer_accuracy": 0.6188275569079987, + "avg_response_len": 242.87806769680682, + "resp_tokens_sum": 196665172, + "pass_at_k": { + "pass@1": 0.6188275569079987, + "pass@2": 0.6676054997996059, + "pass@4": 0.7072495557581842, + "pass@8": 0.7391898612522771, + "pass@16": 0.7648119807124286, + "pass@32": 0.7856728995183987, + "pass@64": 0.8038704657760815, + "pass@128": 0.8208978817578249 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8670953171091446, + "pass@2": 0.9082642324576685, + "pass@4": 0.9344302874751275, + "pass@8": 0.9526031337738796, + "pass@16": 0.9629272671248992, + "pass@32": 0.9682270358052656, + "pass@64": 0.9723204617269156, + "pass@128": 0.976401179941003 + }, + "2": { + "pass@1": 0.9599639797507789, + "pass@2": 0.963836022395565, + "pass@4": 0.967983610109484, + "pass@8": 0.97280747375255, + "pass@16": 0.9773608156099567, + "pass@32": 0.9804489575520082, + "pass@64": 0.9824826081902464, + "pass@128": 0.9844236760124611 + }, + "3": { + "pass@1": 0.9889860474006116, + "pass@2": 0.9924104240410316, + "pass@4": 0.9947170838163579, + "pass@8": 0.9962247167758309, + "pass@16": 0.9968678062677515, + "pass@32": 0.9969414013686603, + "pass@64": 0.9969418960237719, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.9893798828125, + "pass@2": 0.99249969242126, + "pass@4": 0.99361593257874, + "pass@8": 0.993938088917673, + "pass@16": 0.9941406211230228, + "pass@32": 0.99453125, + "pass@64": 0.9953125, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9679563492063492, + "pass@2": 0.9771223909511312, + "pass@4": 0.9827705200540406, + "pass@8": 0.986222767171394, + "pass@16": 0.9884563271556579, + "pass@32": 0.9898710602540834, + "pass@64": 0.9904321086955653, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9334534056886228, + "pass@2": 0.9554382868122028, + "pass@4": 0.9689164457923299, + "pass@8": 0.977704755737177, + "pass@16": 0.9830688043745892, + "pass@32": 0.986982241257549, + "pass@64": 0.9900670186881447, + "pass@128": 0.9910179640718563 + }, + "7": { + "pass@1": 0.9388406065088757, + "pass@2": 0.9596263773703585, + "pass@4": 0.9736909371165883, + "pass@8": 0.9825845522685667, + "pass@16": 0.9870793750133965, + "pass@32": 0.9887747654939287, + "pass@64": 0.9896445965776214, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.9217998798076923, + "pass@2": 0.9474363706339591, + "pass@4": 0.9611889871939082, + "pass@8": 0.9712367571433583, + "pass@16": 0.9788797736051013, + "pass@32": 0.9848084145955653, + "pass@64": 0.989404376252009, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.8873239436619719, + "pass@2": 0.9320952783630917, + "pass@4": 0.958994276683724, + "pass@8": 0.9748610209136754, + "pass@16": 0.9827990984511219, + "pass@32": 0.9866721612838238, + "pass@64": 0.9892717297087438, + "pass@128": 0.9915492957746479 + }, + "11": { + "pass@1": 0.7816041076487252, + "pass@2": 0.8401684938993113, + "pass@4": 0.8784991090879926, + "pass@8": 0.9062103229646153, + "pass@16": 0.928634342538452, + "pass@32": 0.9474247894334389, + "pass@64": 0.9635688141184316, + "pass@128": 0.9773371104815864 + }, + "12": { + "pass@1": 0.6643714689265536, + "pass@2": 0.7570847374215929, + "pass@4": 0.8195590050714002, + "pass@8": 0.8638611475587994, + "pass@16": 0.8949019817729846, + "pass@32": 0.9149040498901152, + "pass@64": 0.9297771688529582, + "pass@128": 0.940677966101695 + }, + "13": { + "pass@1": 0.4798932210031348, + "pass@2": 0.5873326925925011, + "pass@4": 0.6713887188474476, + "pass@8": 0.7363040893811634, + "pass@16": 0.7878785286983827, + "pass@32": 0.8272839467742679, + "pass@64": 0.8553831050059669, + "pass@128": 0.8746081504702194 + }, + "14": { + "pass@1": 0.31096670560747663, + "pass@2": 0.40393041553217085, + "pass@4": 0.48734351284360494, + "pass@8": 0.5564510286229742, + "pass@16": 0.613397458431016, + "pass@32": 0.6620214973063218, + "pass@64": 0.7095034575721406, + "pass@128": 0.7601246105919003 + }, + "15": { + "pass@1": 0.22798500722543352, + "pass@2": 0.3055256076191342, + "pass@4": 0.3797083284567752, + "pass@8": 0.4463900949283278, + "pass@16": 0.506202646502336, + "pass@32": 0.5582980599444395, + "pass@64": 0.6044086870879166, + "pass@128": 0.6473988439306358 + }, + "16": { + "pass@1": 0.2020665322580645, + "pass@2": 0.272865601981204, + "pass@4": 0.3412203653978737, + "pass@8": 0.40107650120527977, + "pass@16": 0.45179670722165527, + "pass@32": 0.494565665067616, + "pass@64": 0.5308362340060577, + "pass@128": 0.5645161290322581 + }, + "17": { + "pass@1": 0.19440932080924855, + "pass@2": 0.2594631133084519, + "pass@4": 0.3184659128591581, + "pass@8": 0.36832649582409616, + "pass@16": 0.4102210447790027, + "pass@32": 0.44760686822032736, + "pass@64": 0.48478608490693575, + "pass@128": 0.5260115606936416 + }, + "18": { + "pass@1": 0.1690625, + "pass@2": 0.22736255624296942, + "pass@4": 0.28266300096416525, + "pass@8": 0.3321534924751763, + "pass@16": 0.37531477121326395, + "pass@32": 0.4134994424323006, + "pass@64": 0.4477984271129805, + "pass@128": 0.48 + }, + "19": { + "pass@1": 0.1649540165165165, + "pass@2": 0.22900225520796397, + "pass@4": 0.28934828247820366, + "pass@8": 0.3409590956948496, + "pass@16": 0.38376503808474804, + "pass@32": 0.41959872780770363, + "pass@64": 0.44923215559385915, + "pass@128": 0.4744744744744745 + }, + "20": { + "pass@1": 0.1474756006006006, + "pass@2": 0.20596506940010878, + "pass@4": 0.26194670120063823, + "pass@8": 0.3106229089309864, + "pass@16": 0.3518269313013121, + "pass@32": 0.3880737873045299, + "pass@64": 0.42370540748975977, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 494355, + "answer_accuracy": 0.6020496395167576, + "avg_response_len": 249.6308298421668, + "resp_tokens_sum": 204976867, + "pass_at_k": { + "pass@1": 0.6020496395167576, + "pass@2": 0.6489359261941491, + "pass@4": 0.6875917930124271, + "pass@8": 0.720294447563385, + "pass@16": 0.7485389992315395, + "pass@32": 0.7730623756521464, + "pass@64": 0.7946585235726484, + "pass@128": 0.8138737334372564 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8187787576687117, + "pass@2": 0.8631999208975412, + "pass@4": 0.8889652101577791, + "pass@8": 0.9047884008029274, + "pass@16": 0.9158893168854205, + "pass@32": 0.92519370344476, + "pass@64": 0.9330087088515969, + "pass@128": 0.9386503067484663 + }, + "2": { + "pass@1": 0.9752089665653495, + "pass@2": 0.9798478896919802, + "pass@4": 0.9832519405818955, + "pass@8": 0.9870105278631578, + "pass@16": 0.9908648345867141, + "pass@32": 0.9942662077657641, + "pass@64": 0.996862131434271, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9942945075757575, + "pass@2": 0.9963180028632787, + "pass@4": 0.9970773806683255, + "pass@8": 0.9974988500184285, + "pass@16": 0.9979768035033618, + "pass@32": 0.9987316926293286, + "pass@64": 0.9996301598663804, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.994221523668639, + "pass@2": 0.9966635896659369, + "pass@4": 0.9981680000798717, + "pass@8": 0.9992769656375657, + "pass@16": 0.9998620255616644, + "pass@32": 0.9999962385209578, + "pass@64": 0.999999999802363, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9785326086956522, + "pass@2": 0.9834570780554607, + "pass@4": 0.9865280187802611, + "pass@8": 0.9891545568020785, + "pass@16": 0.9914041524744066, + "pass@32": 0.9936531060212447, + "pass@64": 0.996363467201046, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.921181053008596, + "pass@2": 0.9443954041919544, + "pass@4": 0.9595953014827298, + "pass@8": 0.970122744120205, + "pass@16": 0.9776500126894409, + "pass@32": 0.9831185714417165, + "pass@64": 0.9870883967891798, + "pass@128": 0.9885386819484241 + }, + "7": { + "pass@1": 0.9094337066473989, + "pass@2": 0.9308299861180648, + "pass@4": 0.9450992366019274, + "pass@8": 0.9571766432271784, + "pass@16": 0.9673169836926407, + "pass@32": 0.9756833949170814, + "pass@64": 0.9829578324080214, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.8751760563380282, + "pass@2": 0.912961974603527, + "pass@4": 0.9388333774299338, + "pass@8": 0.9557506365071035, + "pass@16": 0.967302846071004, + "pass@32": 0.9765007185796407, + "pass@64": 0.9839139050837533, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.8521940494011976, + "pass@2": 0.9009305454052524, + "pass@4": 0.931433134855149, + "pass@8": 0.9526636585593052, + "pass@16": 0.9679941054740805, + "pass@32": 0.9783950530139177, + "pass@64": 0.9854222638839946, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.760706890060241, + "pass@2": 0.827785181078645, + "pass@4": 0.8710749813088118, + "pass@8": 0.9003279944961914, + "pass@16": 0.9204542771350541, + "pass@32": 0.9356246993576314, + "pass@64": 0.9476432584619248, + "pass@128": 0.9548192771084337 + }, + "12": { + "pass@1": 0.6126077586206896, + "pass@2": 0.7043927257917213, + "pass@4": 0.7676139801686229, + "pass@8": 0.8119565346138998, + "pass@16": 0.84612392537352, + "pass@32": 0.875273652747933, + "pass@64": 0.9018302739049897, + "pass@128": 0.9216300940438872 + }, + "13": { + "pass@1": 0.3867953431372549, + "pass@2": 0.48342107512296223, + "pass@4": 0.5674021487860234, + "pass@8": 0.6372697246312006, + "pass@16": 0.6963878858222297, + "pass@32": 0.7491304071442696, + "pass@64": 0.7981109820911362, + "pass@128": 0.8431372549019608 + }, + "14": { + "pass@1": 0.29877101608187134, + "pass@2": 0.3703949526868352, + "pass@4": 0.43101541802888665, + "pass@8": 0.4865946482695511, + "pass@16": 0.5415986897105027, + "pass@32": 0.5971263312909335, + "pass@64": 0.6528434464388628, + "pass@128": 0.7046783625730995 + }, + "15": { + "pass@1": 0.20783617424242423, + "pass@2": 0.2801408524218563, + "pass@4": 0.34825574416834243, + "pass@8": 0.41106899030231925, + "pass@16": 0.4680849755795498, + "pass@32": 0.5172851517081245, + "pass@64": 0.5596152305263337, + "pass@128": 0.6 + }, + "16": { + "pass@1": 0.19382122507122507, + "pass@2": 0.25543441685173957, + "pass@4": 0.3137022444843969, + "pass@8": 0.36664821250482243, + "pass@16": 0.4134268702032012, + "pass@32": 0.45505377915145834, + "pass@64": 0.49311278811442033, + "pass@128": 0.5327635327635327 + }, + "17": { + "pass@1": 0.19494856366459629, + "pass@2": 0.25942795703526167, + "pass@4": 0.3195814134452136, + "pass@8": 0.375845879351016, + "pass@16": 0.4307573783826269, + "pass@32": 0.48238930194144775, + "pass@64": 0.5288904166601099, + "pass@128": 0.577639751552795 + }, + "18": { + "pass@1": 0.16665900735294117, + "pass@2": 0.22696524722093553, + "pass@4": 0.2879940161891529, + "pass@8": 0.3464190547517433, + "pass@16": 0.3993496945302608, + "pass@32": 0.4459605878543203, + "pass@64": 0.4871371319920805, + "pass@128": 0.5235294117647059 + }, + "19": { + "pass@1": 0.14764694940476192, + "pass@2": 0.20653436386857893, + "pass@4": 0.2659328434950095, + "pass@8": 0.3216636232453182, + "pass@16": 0.36939694692013475, + "pass@32": 0.4057203341614509, + "pass@64": 0.4357478660293507, + "pass@128": 0.46130952380952384 + }, + "20": { + "pass@1": 0.1414436751497006, + "pass@2": 0.1992263013343391, + "pass@4": 0.25540902390943654, + "pass@8": 0.3068920327741638, + "pass@16": 0.3528934687646693, + "pass@32": 0.39143395959153, + "pass@64": 0.42026686041833455, + "pass@128": 0.44011976047904194 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 487852, + "answer_accuracy": 0.6089381290941045, + "avg_response_len": 268.44952143912764, + "resp_tokens_sum": 215068871, + "pass_at_k": { + "pass@1": 0.6089381290941045, + "pass@2": 0.6560205862612997, + "pass@4": 0.695623838680178, + "pass@8": 0.7291054496642166, + "pass@16": 0.7572113855746511, + "pass@32": 0.7810222919645945, + "pass@64": 0.8016692044405078, + "pass@128": 0.8197795174948075 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8403218283582089, + "pass@2": 0.8831759019861319, + "pass@4": 0.9079722738948675, + "pass@8": 0.926427138967982, + "pass@16": 0.9417213330071029, + "pass@32": 0.9541837011986865, + "pass@64": 0.9642190900906727, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9692410714285714, + "pass@2": 0.9777741844769405, + "pass@4": 0.9846253460281751, + "pass@8": 0.9899382480241745, + "pass@16": 0.9939865050728705, + "pass@32": 0.9969488212990623, + "pass@64": 0.9990814933468017, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9933035714285714, + "pass@2": 0.996758496705769, + "pass@4": 0.9984645953512371, + "pass@8": 0.999597081444929, + "pass@16": 0.9999722102781452, + "pass@32": 0.9999999117694275, + "pass@64": 0.9999999999999893, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9885325292397661, + "pass@2": 0.9938638940691626, + "pass@4": 0.996856481207393, + "pass@8": 0.9989319154937989, + "pass@16": 0.9998640737041085, + "pass@32": 0.9999980433734141, + "pass@64": 0.9999999999688675, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9467830882352941, + "pass@2": 0.9582825526864288, + "pass@4": 0.9682023059617548, + "pass@8": 0.9775505506527833, + "pass@16": 0.9845769531894009, + "pass@32": 0.9891263931155932, + "pass@64": 0.9924383739804744, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9375492902208202, + "pass@2": 0.9561767244591276, + "pass@4": 0.968417324608872, + "pass@8": 0.9766326478533517, + "pass@16": 0.9825492126854667, + "pass@32": 0.987670213303071, + "pass@64": 0.9928894756326212, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9044452136075949, + "pass@2": 0.9300164924499152, + "pass@4": 0.9481756708813297, + "pass@8": 0.9610352164174365, + "pass@16": 0.9694307347429958, + "pass@32": 0.9756502423183432, + "pass@64": 0.9817890720701818, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.8971706081081081, + "pass@2": 0.928977575015961, + "pass@4": 0.9490096952970964, + "pass@8": 0.9622935399559339, + "pass@16": 0.9713158091696881, + "pass@32": 0.9768844083204202, + "pass@64": 0.9804212826643012, + "pass@128": 0.984984984984985 + }, + "9": { + "pass@1": 0.864022307073955, + "pass@2": 0.9118283508874097, + "pass@4": 0.9415764666273624, + "pass@8": 0.9615825575217636, + "pass@16": 0.9732077067762223, + "pass@32": 0.9794089356361269, + "pass@64": 0.9838241901688739, + "pass@128": 0.9903536977491961 + }, + "11": { + "pass@1": 0.7715277777777778, + "pass@2": 0.8335602580927388, + "pass@4": 0.8746409222656686, + "pass@8": 0.9043520584352333, + "pass@16": 0.9280951884818978, + "pass@32": 0.9469414841706554, + "pass@64": 0.9617656531177552, + "pass@128": 0.9714285714285714 + }, + "12": { + "pass@1": 0.6685540902140673, + "pass@2": 0.750171566856895, + "pass@4": 0.8076844448343031, + "pass@8": 0.8501109077517056, + "pass@16": 0.8825882158353087, + "pass@32": 0.9082979572669679, + "pass@64": 0.9282816365658905, + "pass@128": 0.9418960244648318 + }, + "13": { + "pass@1": 0.4363184799382716, + "pass@2": 0.5391692032419559, + "pass@4": 0.6226500897457258, + "pass@8": 0.6898887508264963, + "pass@16": 0.7448552505843882, + "pass@32": 0.7913198741989679, + "pass@64": 0.831231401781366, + "pass@128": 0.8641975308641975 + }, + "14": { + "pass@1": 0.29532177299703266, + "pass@2": 0.3794510385756675, + "pass@4": 0.4581800039275504, + "pass@8": 0.5274378283252875, + "pass@16": 0.5883019565685669, + "pass@32": 0.6434770492088578, + "pass@64": 0.6938611977483916, + "pass@128": 0.7388724035608308 + }, + "15": { + "pass@1": 0.1781684027777778, + "pass@2": 0.2488718315592497, + "pass@4": 0.323979556085582, + "pass@8": 0.3959601546379987, + "pass@16": 0.4615711022760134, + "pass@32": 0.5171673358865758, + "pass@64": 0.5637801348069706, + "pass@128": 0.6111111111111112 + }, + "16": { + "pass@1": 0.2089325221238938, + "pass@2": 0.27069505899705004, + "pass@4": 0.32480351792309137, + "pass@8": 0.3727821911266633, + "pass@16": 0.4175536083009712, + "pass@32": 0.46334050373812846, + "pass@64": 0.5105570019459841, + "pass@128": 0.5545722713864307 + }, + "17": { + "pass@1": 0.16488610692771086, + "pass@2": 0.22542482923821255, + "pass@4": 0.2861592093909947, + "pass@8": 0.3442176842200421, + "pass@16": 0.39808977250495636, + "pass@32": 0.44461433904524217, + "pass@64": 0.4821225595080291, + "pass@128": 0.5090361445783133 + }, + "18": { + "pass@1": 0.18180443548387096, + "pass@2": 0.251581946913894, + "pass@4": 0.3183337878128136, + "pass@8": 0.37251228294233507, + "pass@16": 0.4128352414166689, + "pass@32": 0.44457500793368165, + "pass@64": 0.47167915157723894, + "pass@128": 0.4935483870967742 + }, + "19": { + "pass@1": 0.15301642749244712, + "pass@2": 0.21020693079905794, + "pass@4": 0.26659490533849434, + "pass@8": 0.31827335611728563, + "pass@16": 0.36212186502412136, + "pass@32": 0.3976268046073573, + "pass@64": 0.4256205035352982, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.13992117117117117, + "pass@2": 0.1956980356340592, + "pass@4": 0.2531280616949908, + "pass@8": 0.3096118515891317, + "pass@16": 0.36257072436356236, + "pass@32": 0.411466262375835, + "pass@64": 0.45802399495121837, + "pass@128": 0.5045045045045045 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/checkpoint-22350_metrics.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/checkpoint-22350_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..47da2ef59b0c5d9a74fa7f5856d5d575a5bef8f3 --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/checkpoint-22350_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-22350", + "id": { + "answer_accuracy": 0.9594730902777778, + "avg_response_len": 196.4464236111111, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9594730902777778, + "pass@2": 0.9696449174321878, + "pass@4": 0.9756305321938904, + "pass@8": 0.9798652956829302, + "pass@16": 0.9831418877038463, + "pass@32": 0.9858580959018016, + "pass@64": 0.9882768607295374, + "pass@128": 0.9903333333333333 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8758984375, + "pass@2": 0.9029356545275596, + "pass@4": 0.9205091601049858, + "pass@8": 0.9336229453171274, + "pass@16": 0.943493966561507, + "pass@32": 0.951005378590098, + "pass@64": 0.9571638138706535, + "pass@128": 0.962 + }, + "2": { + "pass@1": 0.9993828125, + "pass@2": 0.9999259350393701, + "pass@4": 0.9999978398012749, + "pass@8": 0.9999999978161459, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.994234375, + "pass@2": 0.9956544045275592, + "pass@4": 0.9966550927071617, + "pass@8": 0.9974267305248449, + "pass@16": 0.9979805710795902, + "pass@32": 0.9985379257191752, + "pass@64": 0.9989423622047243, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.99565625, + "pass@2": 0.9963619586614175, + "pass@4": 0.9966482469066367, + "pass@8": 0.9969211450968901, + "pass@16": 0.9971078675022533, + "pass@32": 0.9972498583460825, + "pass@64": 0.9974999999997747, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.96821875, + "pass@2": 0.9733453494094488, + "pass@4": 0.9769739426321712, + "pass@8": 0.9799816813729223, + "pass@16": 0.9826650295347344, + "pass@32": 0.9853613180116391, + "pass@64": 0.9882905385698499, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.967828125, + "pass@2": 0.9744089566929135, + "pass@4": 0.9783110363704537, + "pass@8": 0.9811095183238062, + "pass@16": 0.9832091588734683, + "pass@32": 0.9851414571924823, + "pass@64": 0.9873614320200583, + "pass@128": 0.99 + }, + "7": { + "pass@1": 0.9619765625, + "pass@2": 0.9719731791338586, + "pass@4": 0.9779432251593553, + "pass@8": 0.982517209325641, + "pass@16": 0.9864979778259129, + "pass@32": 0.9898456073057555, + "pass@64": 0.9925945720220554, + "pass@128": 0.994 + }, + "8": { + "pass@1": 0.94234375, + "pass@2": 0.9593125000000006, + "pass@4": 0.9693958648293964, + "pass@8": 0.9756817987942217, + "pass@16": 0.9799939958787885, + "pass@32": 0.9833089714374633, + "pass@64": 0.9860169295370107, + "pass@128": 0.988 + }, + "9": { + "pass@1": 0.92971875, + "pass@2": 0.9528863188976383, + "pass@4": 0.9642403812335956, + "pass@8": 0.9715266345747782, + "pass@16": 0.977328422078353, + "pass@32": 0.9822723465135123, + "pass@64": 0.9866220983417069, + "pass@128": 0.991 + } + }, + "per_op_accuracy": { + "10": 0.8758984375, + "2": 0.9993828125, + "3": 0.994234375, + "4": 0.99565625, + "5": 0.96821875, + "6": 0.967828125, + "7": 0.9619765625, + "8": 0.94234375, + "9": 0.92971875 + }, + "per_op_avg_response_len": { + "10": 273.4673203125, + "2": 114.53275, + "3": 143.1052265625, + "4": 157.8867265625, + "5": 171.5562421875, + "6": 195.9602734375, + "7": 219.664953125, + "8": 238.6634453125, + "9": 253.180875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.4673203125, + "2": 114.53275, + "3": 143.1052265625, + "4": 157.8867265625, + "5": 171.5562421875, + "6": 195.9602734375, + "7": 219.664953125, + "8": 238.6634453125, + "9": 253.180875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 367054, + "answer_accuracy": 0.9684597686592368, + "avg_response_len": 187.11052537149612, + "resp_tokens_sum": 70916386, + "pass_at_k": { + "pass@1": 0.9684597686592368, + "pass@2": 0.9777915061149282, + "pass@4": 0.9823845372925141, + "pass@8": 0.9852640107569967, + "pass@16": 0.9874225147591853, + "pass@32": 0.9893086612028916, + "pass@64": 0.9911180926775749, + "pass@128": 0.9925700776764607 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.903530604719764, + "pass@2": 0.9280135095115319, + "pass@4": 0.9426811864224937, + "pass@8": 0.9534636795009439, + "pass@16": 0.9602740600181923, + "pass@32": 0.9643415604348307, + "pass@64": 0.9677587233741899, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9996592679127726, + "pass@2": 0.9999701044962838, + "pass@4": 0.9999997912060058, + "pass@8": 0.9999999999971957, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9957234327217125, + "pass@2": 0.9966740109321197, + "pass@4": 0.9969229032724121, + "pass@8": 0.9969418134426237, + "pass@16": 0.9969418960240432, + "pass@32": 0.9969418960244648, + "pass@64": 0.9969418960244648, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.9931884765625, + "pass@2": 0.9937711460383859, + "pass@4": 0.9938475112485939, + "pass@8": 0.993945312498918, + "pass@16": 0.994140625, + "pass@32": 0.99453125, + "pass@64": 0.9953125, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9752232142857142, + "pass@2": 0.9805809430071241, + "pass@4": 0.9836755715059426, + "pass@8": 0.9860623829306817, + "pass@16": 0.9877131700577393, + "pass@32": 0.988865914595378, + "pass@64": 0.9904761738606371, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9744807260479041, + "pass@2": 0.9818922686123818, + "pass@4": 0.9849555498115132, + "pass@8": 0.9867536020108717, + "pass@16": 0.9885545833866755, + "pass@32": 0.9907015576214719, + "pass@64": 0.9928842794533806, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9752218934911243, + "pass@2": 0.9823286003820528, + "pass@4": 0.9855232493127706, + "pass@8": 0.9876027174492761, + "pass@16": 0.9900139637057965, + "pass@32": 0.9935202105812683, + "pass@64": 0.9977249219587196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9639172676282052, + "pass@2": 0.9756736763072885, + "pass@4": 0.9796417202657361, + "pass@8": 0.9811646330398488, + "pass@16": 0.9820810896845035, + "pass@32": 0.9829762025036928, + "pass@64": 0.9837834645669173, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.9407350352112676, + "pass@2": 0.9646823361428418, + "pass@4": 0.97650347359749, + "pass@8": 0.9829905290283689, + "pass@16": 0.9880542555562098, + "pass@32": 0.9923821609397722, + "pass@64": 0.9953117085182914, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 373913, + "answer_accuracy": 0.9571413212647444, + "avg_response_len": 194.10338763515728, + "resp_tokens_sum": 75827653, + "pass_at_k": { + "pass@1": 0.9571413212647444, + "pass@2": 0.9674451889041404, + "pass@4": 0.9737081396833102, + "pass@8": 0.9782248564292898, + "pass@16": 0.9816633587917186, + "pass@32": 0.9844800736899023, + "pass@64": 0.9866525302173448, + "pass@128": 0.9882044560943644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8560678680981595, + "pass@2": 0.8841039533114343, + "pass@4": 0.9030514761191657, + "pass@8": 0.9173781283934727, + "pass@16": 0.9282431728633065, + "pass@32": 0.9367494154153667, + "pass@64": 0.9431173930127746, + "pass@128": 0.9478527607361963 + }, + "2": { + "pass@1": 0.9994300911854104, + "pass@2": 0.9999536294665294, + "pass@4": 0.9999994814477978, + "pass@8": 0.9999999999726388, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.994720643939394, + "pass@2": 0.9957684621808638, + "pass@4": 0.9967587758916499, + "pass@8": 0.9976809114058548, + "pass@16": 0.998458723990178, + "pass@32": 0.9992998922984194, + "pass@64": 0.9999126700071582, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996070636094675, + "pass@2": 0.9999854400596374, + "pass@4": 0.9999999800320818, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9763360507246377, + "pass@2": 0.9799897295446762, + "pass@4": 0.9829189930063094, + "pass@8": 0.9854568297076476, + "pass@16": 0.9878492664588717, + "pass@32": 0.9902406260662067, + "pass@64": 0.992681422004542, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9599973137535817, + "pass@2": 0.9650911349637887, + "pass@4": 0.9684557643940639, + "pass@8": 0.9713198049675233, + "pass@16": 0.9734459817059731, + "pass@32": 0.9751500499585644, + "pass@64": 0.9770377542411987, + "pass@128": 0.9799426934097422 + }, + "7": { + "pass@1": 0.9596278901734104, + "pass@2": 0.9690860253288427, + "pass@4": 0.9756533710020351, + "pass@8": 0.9813592130007283, + "pass@16": 0.9863418091645095, + "pass@32": 0.9903767960067987, + "pass@64": 0.9930983479380481, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9377860915492958, + "pass@2": 0.958912678828879, + "pass@4": 0.9724244942093503, + "pass@8": 0.9806020235436738, + "pass@16": 0.9851324251816836, + "pass@32": 0.9876871801465659, + "pass@64": 0.9886708658809289, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9292196856287425, + "pass@2": 0.9527257001744545, + "pass@4": 0.9627166653756599, + "pass@8": 0.9688774460237405, + "pass@16": 0.9742544799641086, + "pass@32": 0.9797131952662533, + "pass@64": 0.9843850850382375, + "pass@128": 0.9880239520958084 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 364346, + "answer_accuracy": 0.9529471459658521, + "avg_response_len": 208.0950812897556, + "resp_tokens_sum": 79562241, + "pass_at_k": { + "pass@1": 0.9529471459658521, + "pass@2": 0.9638168365410227, + "pass@4": 0.9708995421856171, + "pass@8": 0.9761897099004756, + "pass@16": 0.9804092240041317, + "pass@32": 0.9838455749557632, + "pass@64": 0.9871200374704368, + "pass@128": 0.9902912621359223 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8672341417910447, + "pass@2": 0.8958841373839462, + "pass@4": 0.9150610647549656, + "pass@8": 0.9293536958508508, + "pass@16": 0.9413545846623945, + "pass@32": 0.9513829259023318, + "pass@64": 0.9601114523123534, + "pass@128": 0.9671641791044776 + }, + "2": { + "pass@1": 0.9990848214285715, + "pass@2": 0.999859392575928, + "pass@4": 0.9999945069366328, + "pass@8": 0.9999999937887084, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9923469387755102, + "pass@2": 0.9945726251693029, + "pass@4": 0.9963000212619196, + "pass@8": 0.9976444803649415, + "pass@16": 0.9985107643235256, + "pass@32": 0.9993264176696698, + "pass@64": 0.9999159798902688, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9940606725146199, + "pass@2": 0.9952050093244924, + "pass@4": 0.9959562867141608, + "pass@8": 0.9966627049626813, + "pass@16": 0.9970259283691618, + "pass@32": 0.9970756091990718, + "pass@64": 0.997076023391154, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9534926470588235, + "pass@2": 0.9598996931449746, + "pass@4": 0.9647326030018306, + "pass@8": 0.9687924250018215, + "pass@16": 0.9727276001124571, + "pass@32": 0.9771633497683939, + "pass@64": 0.9818101565064178, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9694400630914827, + "pass@2": 0.976782753545791, + "pass@4": 0.9821603184854258, + "pass@8": 0.9859407044747921, + "pass@16": 0.988325805700107, + "pass@32": 0.9902831845784584, + "pass@64": 0.9929082222474788, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9503807357594937, + "pass@2": 0.9640580615219775, + "pass@4": 0.9723427231801719, + "pass@8": 0.9783455795238412, + "pass@16": 0.9829082029190926, + "pass@32": 0.9853335592118175, + "pass@64": 0.986555443080517, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.9269894894894894, + "pass@2": 0.9444093453689912, + "pass@4": 0.9565674254119138, + "pass@8": 0.9652994442275824, + "pass@16": 0.972560615488667, + "pass@32": 0.9789532952080486, + "pass@64": 0.9852802738870954, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.9176798633440515, + "pass@2": 0.9395939090310658, + "pass@4": 0.9518787198424955, + "pass@8": 0.961285947902821, + "pass@16": 0.968386382790631, + "pass@32": 0.9734806178809153, + "pass@64": 0.9791055415271452, + "pass@128": 0.9871382636655949 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.34515703125, + "avg_response_len": 307.698259375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.34515703125, + "pass@2": 0.40644681348425915, + "pass@4": 0.4592774762092244, + "pass@8": 0.5046793996427845, + "pass@16": 0.5445875157924662, + "pass@32": 0.5802760670277693, + "pass@64": 0.6119854380648104, + "pass@128": 0.64 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8140234375, + "pass@2": 0.8588109005905517, + "pass@4": 0.8887584455380577, + "pass@8": 0.9097936555920381, + "pass@16": 0.9248400415246867, + "pass@32": 0.9360444039813549, + "pass@64": 0.9458932703788226, + "pass@128": 0.956 + }, + "12": { + "pass@1": 0.714375, + "pass@2": 0.7795043061023637, + "pass@4": 0.8227757677165364, + "pass@8": 0.8550922374688296, + "pass@16": 0.8813360138608634, + "pass@32": 0.9026568521768252, + "pass@64": 0.9193196211034207, + "pass@128": 0.931 + }, + "13": { + "pass@1": 0.476984375, + "pass@2": 0.5629181840551186, + "pass@4": 0.6314860646794139, + "pass@8": 0.6868325978065871, + "pass@16": 0.7348996152149023, + "pass@32": 0.777958903163057, + "pass@64": 0.8138307124115921, + "pass@128": 0.841 + }, + "14": { + "pass@1": 0.3407890625, + "pass@2": 0.4136942667322834, + "pass@4": 0.47691905446194205, + "pass@8": 0.5313429294341345, + "pass@16": 0.580416669664554, + "pass@32": 0.628523766424757, + "pass@64": 0.6769620921517351, + "pass@128": 0.723 + }, + "15": { + "pass@1": 0.2291171875, + "pass@2": 0.2934741633858266, + "pass@4": 0.35368036154855625, + "pass@8": 0.4087269808191484, + "pass@16": 0.4590751347471268, + "pass@32": 0.5043894620327677, + "pass@64": 0.5447927760704238, + "pass@128": 0.583 + }, + "16": { + "pass@1": 0.2074609375, + "pass@2": 0.26580967027559066, + "pass@4": 0.3179536690101236, + "pass@8": 0.3633628118263016, + "pass@16": 0.4042266391373354, + "pass@32": 0.4416219500061143, + "pass@64": 0.476138517525895, + "pass@128": 0.51 + }, + "17": { + "pass@1": 0.1913671875, + "pass@2": 0.24750873523622044, + "pass@4": 0.2996941448256466, + "pass@8": 0.3473043174921732, + "pass@16": 0.39080969893918205, + "pass@32": 0.4300398519553629, + "pass@64": 0.46458386098575105, + "pass@128": 0.494 + }, + "18": { + "pass@1": 0.1696875, + "pass@2": 0.2261791338582675, + "pass@4": 0.280628838582677, + "pass@8": 0.3297475065578793, + "pass@16": 0.37338743223144355, + "pass@32": 0.4132763199734128, + "pass@64": 0.4490263179563619, + "pass@128": 0.48 + }, + "19": { + "pass@1": 0.1585703125, + "pass@2": 0.21429613681102355, + "pass@4": 0.26714785686164205, + "pass@8": 0.3140845213579394, + "pass@16": 0.35491944726490754, + "pass@32": 0.3889688163342846, + "pass@64": 0.4168338509607699, + "pass@128": 0.441 + }, + "20": { + "pass@1": 0.1491953125, + "pass@2": 0.20227263779527538, + "pass@4": 0.25373055886764123, + "pass@8": 0.30050643807278393, + "pass@16": 0.34196446533954206, + "pass@32": 0.37928034422974605, + "pass@64": 0.4124733611033092, + "pass@128": 0.441 + } + }, + "per_op_accuracy": { + "11": 0.8140234375, + "12": 0.714375, + "13": 0.476984375, + "14": 0.3407890625, + "15": 0.2291171875, + "16": 0.2074609375, + "17": 0.1913671875, + "18": 0.1696875, + "19": 0.1585703125, + "20": 0.1491953125 + }, + "per_op_avg_response_len": { + "11": 287.598140625, + "12": 296.5236484375, + "13": 305.18253125, + "14": 311.5532265625, + "15": 309.950296875, + "16": 309.6932734375, + "17": 315.4138671875, + "18": 313.953265625, + "19": 311.659640625, + "20": 315.454703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 287.598140625, + "12": 296.5236484375, + "13": 305.18253125, + "14": 311.5532265625, + "15": 309.950296875, + "16": 309.6932734375, + "17": 315.4138671875, + "18": 313.953265625, + "19": 311.659640625, + "20": 315.454703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 152460, + "answer_accuracy": 0.35396545319465084, + "avg_response_len": 293.341607076523, + "resp_tokens_sum": 126348097, + "pass_at_k": { + "pass@1": 0.35396545319465084, + "pass@2": 0.4171479215172418, + "pass@4": 0.4711401781500948, + "pass@8": 0.5168576815296596, + "pass@16": 0.5564031448279884, + "pass@32": 0.5910112304245334, + "pass@64": 0.6208255133136782, + "pass@128": 0.6457652303120357 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8119909702549575, + "pass@2": 0.8574068445941428, + "pass@4": 0.8905700936639295, + "pass@8": 0.9141548025681611, + "pass@16": 0.9310147321010921, + "pass@32": 0.9431402746849071, + "pass@64": 0.9527867386839005, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.72314177259887, + "pass@2": 0.7903432092174907, + "pass@4": 0.8314442794756585, + "pass@8": 0.8607643836517445, + "pass@16": 0.8863590398904893, + "pass@32": 0.9076966113232872, + "pass@64": 0.924174427048371, + "pass@128": 0.9350282485875706 + }, + "13": { + "pass@1": 0.5077390282131662, + "pass@2": 0.6026861748080863, + "pass@4": 0.6746213919576663, + "pass@8": 0.7280871199402102, + "pass@16": 0.7710803144173798, + "pass@32": 0.8074469976041617, + "pass@64": 0.8368862455010191, + "pass@128": 0.8589341692789969 + }, + "14": { + "pass@1": 0.32676207165109034, + "pass@2": 0.4022562673240613, + "pass@4": 0.47134501353218683, + "pass@8": 0.5316685145841253, + "pass@16": 0.583733926077304, + "pass@32": 0.6325814592753726, + "pass@64": 0.6818888012957016, + "pass@128": 0.7289719626168224 + }, + "15": { + "pass@1": 0.2350298049132948, + "pass@2": 0.2988132794365298, + "pass@4": 0.3616546562460039, + "pass@8": 0.4242653266749827, + "pass@16": 0.4833672306164851, + "pass@32": 0.5327217007147764, + "pass@64": 0.5712522977311489, + "pass@128": 0.6040462427745664 + }, + "16": { + "pass@1": 0.20788810483870968, + "pass@2": 0.27152257429514853, + "pass@4": 0.327680491853841, + "pass@8": 0.3755843283869693, + "pass@16": 0.41732714469475174, + "pass@32": 0.4543796479160046, + "pass@64": 0.4873940801671596, + "pass@128": 0.5161290322580645 + }, + "17": { + "pass@1": 0.19716401734104047, + "pass@2": 0.25503077920895717, + "pass@4": 0.30815697911316003, + "pass@8": 0.35562136170719266, + "pass@16": 0.3982127691151806, + "pass@32": 0.4370893005671065, + "pass@64": 0.47201922867722684, + "pass@128": 0.5028901734104047 + }, + "18": { + "pass@1": 0.16732142857142857, + "pass@2": 0.22041725253093356, + "pass@4": 0.2704885762493973, + "pass@8": 0.3153377965478373, + "pass@16": 0.3545991358083943, + "pass@32": 0.38914450742366186, + "pass@64": 0.4186827347023023, + "pass@128": 0.44285714285714284 + }, + "19": { + "pass@1": 0.17396302552552553, + "pass@2": 0.23597921248019668, + "pass@4": 0.2925832649297215, + "pass@8": 0.33894785591823995, + "pass@16": 0.3781170566234184, + "pass@32": 0.41324923316576234, + "pass@64": 0.44271916868136607, + "pass@128": 0.46546546546546547 + }, + "20": { + "pass@1": 0.15355198948948948, + "pass@2": 0.20524424522948134, + "pass@4": 0.25447808072414363, + "pass@8": 0.29844947745611566, + "pass@16": 0.3366465437604806, + "pass@32": 0.3712929813911508, + "pass@64": 0.4015159625628319, + "pass@128": 0.4264264264264264 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 147116, + "answer_accuracy": 0.3417614481118049, + "avg_response_len": 303.8969507322331, + "resp_tokens_sum": 130816697, + "pass_at_k": { + "pass@1": 0.3417614481118049, + "pass@2": 0.4025617623231984, + "pass@4": 0.4549984818746463, + "pass@8": 0.4999763688473229, + "pass@16": 0.5399769334339257, + "pass@32": 0.576309597033437, + "pass@64": 0.6086037792110182, + "pass@128": 0.6366339577757955 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8160062123493976, + "pass@2": 0.8610585303813678, + "pass@4": 0.8899617457456372, + "pass@8": 0.9105718643838208, + "pass@16": 0.9254026184703923, + "pass@32": 0.9363826773485632, + "pass@64": 0.9448460895593244, + "pass@128": 0.9518072289156626 + }, + "12": { + "pass@1": 0.7139743338557993, + "pass@2": 0.7795156030163154, + "pass@4": 0.8230373401600654, + "pass@8": 0.8540952671660289, + "pass@16": 0.8784286480198956, + "pass@32": 0.8988179779778064, + "pass@64": 0.914751375375745, + "pass@128": 0.9247648902821317 + }, + "13": { + "pass@1": 0.4431022408963585, + "pass@2": 0.5229334844173891, + "pass@4": 0.5902965916025199, + "pass@8": 0.647391393076343, + "pass@16": 0.699989527769502, + "pass@32": 0.7489458173941425, + "pass@64": 0.7897583683021384, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.35249908625730997, + "pass@2": 0.4206065524704149, + "pass@4": 0.47809296233804116, + "pass@8": 0.5299356203940037, + "pass@16": 0.5811376255515341, + "pass@32": 0.6335831039468219, + "pass@64": 0.6852598413531686, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.23425662878787878, + "pass@2": 0.2991070895967547, + "pass@4": 0.3562961430957494, + "pass@8": 0.40513356161577346, + "pass@16": 0.44987841221925085, + "pass@32": 0.49389229180815036, + "pass@64": 0.5351252376246326, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.2116942663817664, + "pass@2": 0.2658307535276038, + "pass@4": 0.3131332528412581, + "pass@8": 0.35405246824329895, + "pass@16": 0.3925773455929453, + "pass@32": 0.4296501256109476, + "pass@64": 0.4638134199632514, + "pass@128": 0.4985754985754986 + }, + "17": { + "pass@1": 0.19443905279503104, + "pass@2": 0.2521755942192007, + "pass@4": 0.3052168306213276, + "pass@8": 0.3531225638086416, + "pass@16": 0.3969801368618287, + "pass@32": 0.4357863106880458, + "pass@64": 0.46768276618636456, + "pass@128": 0.4906832298136646 + }, + "18": { + "pass@1": 0.17817095588235293, + "pass@2": 0.2388269308707734, + "pass@4": 0.2978523059617549, + "pass@8": 0.3508819219268661, + "pass@16": 0.39719267118461893, + "pass@32": 0.4392130178110388, + "pass@64": 0.4782813253719725, + "pass@128": 0.5147058823529411 + }, + "19": { + "pass@1": 0.150390625, + "pass@2": 0.20729415658980124, + "pass@4": 0.2629280089988752, + "pass@8": 0.3126190421848842, + "pass@16": 0.35452784964061956, + "pass@32": 0.38692673717859644, + "pass@64": 0.41329885722988, + "pass@128": 0.4375 + }, + "20": { + "pass@1": 0.1395256362275449, + "pass@2": 0.19365010137205899, + "pass@4": 0.24706315462438455, + "pass@8": 0.29426874511051465, + "pass@16": 0.334118849246813, + "pass@32": 0.3683037737540345, + "pass@64": 0.39960945985179536, + "pass@128": 0.4281437125748503 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 142225, + "answer_accuracy": 0.3395882678789731, + "avg_response_len": 326.3700001910147, + "resp_tokens_sum": 136688978, + "pass_at_k": { + "pass@1": 0.3395882678789731, + "pass@2": 0.39943464921885613, + "pass@4": 0.45147560148922516, + "pass@8": 0.49698880490432945, + "pass@16": 0.5371748619926855, + "pass@32": 0.5733125626759336, + "pass@64": 0.6063698101653117, + "pass@128": 0.6375305623471883 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8142113095238095, + "pass@2": 0.8580154043244594, + "pass@4": 0.8854600091655204, + "pass@8": 0.9040861946350753, + "pass@16": 0.917327529399465, + "pass@32": 0.9277359940884433, + "pass@64": 0.9392719044117785, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.7052752293577982, + "pass@2": 0.7677594271472944, + "pass@4": 0.8131363341623575, + "pass@8": 0.8499243285325645, + "pass@16": 0.8787344801874143, + "pass@32": 0.9009459229157835, + "pass@64": 0.9185204439860378, + "pass@128": 0.9327217125382263 + }, + "13": { + "pass@1": 0.4840374228395062, + "pass@2": 0.5678211122047245, + "pass@4": 0.6344011032185792, + "pass@8": 0.6896730840043367, + "pass@16": 0.7377430046359499, + "pass@32": 0.7808939941901869, + "pass@64": 0.817655131521307, + "pass@128": 0.845679012345679 + }, + "14": { + "pass@1": 0.34226632047477745, + "pass@2": 0.41757437381714485, + "pass@4": 0.4810371275919886, + "pass@8": 0.5324609913290231, + "pass@16": 0.5765252565433672, + "pass@32": 0.6195243218028179, + "pass@64": 0.6638484308398551, + "pass@128": 0.7091988130563798 + }, + "15": { + "pass@1": 0.2175684799382716, + "pass@2": 0.2820352751045008, + "pass@4": 0.3425003804501287, + "pass@8": 0.3957935260999967, + "pass@16": 0.44250060778231465, + "pass@32": 0.4848249916319935, + "pass@64": 0.5263831871583865, + "pass@128": 0.5709876543209876 + }, + "16": { + "pass@1": 0.20268713126843657, + "pass@2": 0.26056365119736136, + "pass@4": 0.3140499846258156, + "pass@8": 0.361826706999834, + "pass@16": 0.40430848371338823, + "pass@32": 0.44235122437377733, + "pass@64": 0.4786072043273569, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.1823465737951807, + "pass@2": 0.2351432056968028, + "pass@4": 0.285518104194054, + "pass@8": 0.33299355661175367, + "pass@16": 0.3771098697464484, + "pass@32": 0.41711976481202867, + "pass@64": 0.4538293890103033, + "pass@128": 0.4879518072289157 + }, + "18": { + "pass@1": 0.16305443548387097, + "pass@2": 0.21881270637541272, + "pass@4": 0.27318726731739185, + "pass@8": 0.3228368526161366, + "pass@16": 0.3684910532120487, + "pass@32": 0.4120752139334768, + "pass@64": 0.451199065109953, + "pass@128": 0.4838709677419355 + }, + "19": { + "pass@1": 0.15138783987915408, + "pass@2": 0.19958979589409337, + "pass@4": 0.2458423523154772, + "pass@8": 0.2905585714895618, + "pass@16": 0.33197918407873356, + "pass@32": 0.36661461630839115, + "pass@64": 0.3943805189143051, + "pass@128": 0.4199395770392749 + }, + "20": { + "pass@1": 0.15453734984984985, + "pass@2": 0.20794946028705857, + "pass@4": 0.2596704634893611, + "pass@8": 0.3088198234624205, + "pass@16": 0.3551515634200191, + "pass@32": 0.39827724021815375, + "pass@64": 0.4363332912293886, + "pass@128": 0.46846846846846846 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6361488486842105, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6361488486842105, + "pass@2": 0.6732248627227313, + "pass@4": 0.7038657658861541, + "pass@8": 0.7297674556618075, + "pass@16": 0.7523237972241775, + "pass@32": 0.7723938701786406, + "pass@64": 0.7902287435375767, + "pass@128": 0.8059473684210526 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8758984375, + "pass@2": 0.9029356545275596, + "pass@4": 0.9205091601049858, + "pass@8": 0.9336229453171274, + "pass@16": 0.943493966561507, + "pass@32": 0.951005378590098, + "pass@64": 0.9571638138706535, + "pass@128": 0.962 + }, + "2": { + "pass@1": 0.9993828125, + "pass@2": 0.9999259350393701, + "pass@4": 0.9999978398012749, + "pass@8": 0.9999999978161459, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.994234375, + "pass@2": 0.9956544045275592, + "pass@4": 0.9966550927071617, + "pass@8": 0.9974267305248449, + "pass@16": 0.9979805710795902, + "pass@32": 0.9985379257191752, + "pass@64": 0.9989423622047243, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.99565625, + "pass@2": 0.9963619586614175, + "pass@4": 0.9966482469066367, + "pass@8": 0.9969211450968901, + "pass@16": 0.9971078675022533, + "pass@32": 0.9972498583460825, + "pass@64": 0.9974999999997747, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.96821875, + "pass@2": 0.9733453494094488, + "pass@4": 0.9769739426321712, + "pass@8": 0.9799816813729223, + "pass@16": 0.9826650295347344, + "pass@32": 0.9853613180116391, + "pass@64": 0.9882905385698499, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.967828125, + "pass@2": 0.9744089566929135, + "pass@4": 0.9783110363704537, + "pass@8": 0.9811095183238062, + "pass@16": 0.9832091588734683, + "pass@32": 0.9851414571924823, + "pass@64": 0.9873614320200583, + "pass@128": 0.99 + }, + "7": { + "pass@1": 0.9619765625, + "pass@2": 0.9719731791338586, + "pass@4": 0.9779432251593553, + "pass@8": 0.982517209325641, + "pass@16": 0.9864979778259129, + "pass@32": 0.9898456073057555, + "pass@64": 0.9925945720220554, + "pass@128": 0.994 + }, + "8": { + "pass@1": 0.94234375, + "pass@2": 0.9593125000000006, + "pass@4": 0.9693958648293964, + "pass@8": 0.9756817987942217, + "pass@16": 0.9799939958787885, + "pass@32": 0.9833089714374633, + "pass@64": 0.9860169295370107, + "pass@128": 0.988 + }, + "9": { + "pass@1": 0.92971875, + "pass@2": 0.9528863188976383, + "pass@4": 0.9642403812335956, + "pass@8": 0.9715266345747782, + "pass@16": 0.977328422078353, + "pass@32": 0.9822723465135123, + "pass@64": 0.9866220983417069, + "pass@128": 0.991 + }, + "11": { + "pass@1": 0.8140234375, + "pass@2": 0.8588109005905517, + "pass@4": 0.8887584455380577, + "pass@8": 0.9097936555920381, + "pass@16": 0.9248400415246867, + "pass@32": 0.9360444039813549, + "pass@64": 0.9458932703788226, + "pass@128": 0.956 + }, + "12": { + "pass@1": 0.714375, + "pass@2": 0.7795043061023637, + "pass@4": 0.8227757677165364, + "pass@8": 0.8550922374688296, + "pass@16": 0.8813360138608634, + "pass@32": 0.9026568521768252, + "pass@64": 0.9193196211034207, + "pass@128": 0.931 + }, + "13": { + "pass@1": 0.476984375, + "pass@2": 0.5629181840551186, + "pass@4": 0.6314860646794139, + "pass@8": 0.6868325978065871, + "pass@16": 0.7348996152149023, + "pass@32": 0.777958903163057, + "pass@64": 0.8138307124115921, + "pass@128": 0.841 + }, + "14": { + "pass@1": 0.3407890625, + "pass@2": 0.4136942667322834, + "pass@4": 0.47691905446194205, + "pass@8": 0.5313429294341345, + "pass@16": 0.580416669664554, + "pass@32": 0.628523766424757, + "pass@64": 0.6769620921517351, + "pass@128": 0.723 + }, + "15": { + "pass@1": 0.2291171875, + "pass@2": 0.2934741633858266, + "pass@4": 0.35368036154855625, + "pass@8": 0.4087269808191484, + "pass@16": 0.4590751347471268, + "pass@32": 0.5043894620327677, + "pass@64": 0.5447927760704238, + "pass@128": 0.583 + }, + "16": { + "pass@1": 0.2074609375, + "pass@2": 0.26580967027559066, + "pass@4": 0.3179536690101236, + "pass@8": 0.3633628118263016, + "pass@16": 0.4042266391373354, + "pass@32": 0.4416219500061143, + "pass@64": 0.476138517525895, + "pass@128": 0.51 + }, + "17": { + "pass@1": 0.1913671875, + "pass@2": 0.24750873523622044, + "pass@4": 0.2996941448256466, + "pass@8": 0.3473043174921732, + "pass@16": 0.39080969893918205, + "pass@32": 0.4300398519553629, + "pass@64": 0.46458386098575105, + "pass@128": 0.494 + }, + "18": { + "pass@1": 0.1696875, + "pass@2": 0.2261791338582675, + "pass@4": 0.280628838582677, + "pass@8": 0.3297475065578793, + "pass@16": 0.37338743223144355, + "pass@32": 0.4132763199734128, + "pass@64": 0.4490263179563619, + "pass@128": 0.48 + }, + "19": { + "pass@1": 0.1585703125, + "pass@2": 0.21429613681102355, + "pass@4": 0.26714785686164205, + "pass@8": 0.3140845213579394, + "pass@16": 0.35491944726490754, + "pass@32": 0.3889688163342846, + "pass@64": 0.4168338509607699, + "pass@128": 0.441 + }, + "20": { + "pass@1": 0.1491953125, + "pass@2": 0.20227263779527538, + "pass@4": 0.25373055886764123, + "pass@8": 0.30050643807278393, + "pass@16": 0.34196446533954206, + "pass@32": 0.37928034422974605, + "pass@64": 0.4124733611033092, + "pass@128": 0.441 + } + }, + "per_op_accuracy": { + "10": 0.8758984375, + "2": 0.9993828125, + "3": 0.994234375, + "4": 0.99565625, + "5": 0.96821875, + "6": 0.967828125, + "7": 0.9619765625, + "8": 0.94234375, + "9": 0.92971875, + "11": 0.8140234375, + "12": 0.714375, + "13": 0.476984375, + "14": 0.3407890625, + "15": 0.2291171875, + "16": 0.2074609375, + "17": 0.1913671875, + "18": 0.1696875, + "19": 0.1585703125, + "20": 0.1491953125 + }, + "per_op_avg_response_len": { + "10": 273.4673203125, + "2": 114.53275, + "3": 143.1052265625, + "4": 157.8867265625, + "5": 171.5562421875, + "6": 195.9602734375, + "7": 219.664953125, + "8": 238.6634453125, + "9": 253.180875, + "11": 287.598140625, + "12": 296.5236484375, + "13": 305.18253125, + "14": 311.5532265625, + "15": 309.950296875, + "16": 309.6932734375, + "17": 315.4138671875, + "18": 313.953265625, + "19": 311.659640625, + "20": 315.454703125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.4673203125, + "2": 114.53275, + "3": 143.1052265625, + "4": 157.8867265625, + "5": 171.5562421875, + "6": 195.9602734375, + "7": 219.664953125, + "8": 238.6634453125, + "9": 253.180875, + "11": 287.598140625, + "12": 296.5236484375, + "13": 305.18253125, + "14": 311.5532265625, + "15": 309.950296875, + "16": 309.6932734375, + "17": 315.4138671875, + "18": 313.953265625, + "19": 311.659640625, + "20": 315.454703125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 519514, + "answer_accuracy": 0.6415907564021499, + "avg_response_len": 243.61820636065445, + "resp_tokens_sum": 197264483, + "pass_at_k": { + "pass@1": 0.6415907564021499, + "pass@2": 0.6795674052342421, + "pass@4": 0.7104374509007589, + "pass@8": 0.7361038308882039, + "pass@16": 0.7581496440955067, + "pass@32": 0.7774416276004311, + "pass@64": 0.7941477275875496, + "pass@128": 0.8080935820423648 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.903530604719764, + "pass@2": 0.9280135095115319, + "pass@4": 0.9426811864224937, + "pass@8": 0.9534636795009439, + "pass@16": 0.9602740600181923, + "pass@32": 0.9643415604348307, + "pass@64": 0.9677587233741899, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9996592679127726, + "pass@2": 0.9999701044962838, + "pass@4": 0.9999997912060058, + "pass@8": 0.9999999999971957, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9957234327217125, + "pass@2": 0.9966740109321197, + "pass@4": 0.9969229032724121, + "pass@8": 0.9969418134426237, + "pass@16": 0.9969418960240432, + "pass@32": 0.9969418960244648, + "pass@64": 0.9969418960244648, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.9931884765625, + "pass@2": 0.9937711460383859, + "pass@4": 0.9938475112485939, + "pass@8": 0.993945312498918, + "pass@16": 0.994140625, + "pass@32": 0.99453125, + "pass@64": 0.9953125, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9752232142857142, + "pass@2": 0.9805809430071241, + "pass@4": 0.9836755715059426, + "pass@8": 0.9860623829306817, + "pass@16": 0.9877131700577393, + "pass@32": 0.988865914595378, + "pass@64": 0.9904761738606371, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9744807260479041, + "pass@2": 0.9818922686123818, + "pass@4": 0.9849555498115132, + "pass@8": 0.9867536020108717, + "pass@16": 0.9885545833866755, + "pass@32": 0.9907015576214719, + "pass@64": 0.9928842794533806, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9752218934911243, + "pass@2": 0.9823286003820528, + "pass@4": 0.9855232493127706, + "pass@8": 0.9876027174492761, + "pass@16": 0.9900139637057965, + "pass@32": 0.9935202105812683, + "pass@64": 0.9977249219587196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9639172676282052, + "pass@2": 0.9756736763072885, + "pass@4": 0.9796417202657361, + "pass@8": 0.9811646330398488, + "pass@16": 0.9820810896845035, + "pass@32": 0.9829762025036928, + "pass@64": 0.9837834645669173, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.9407350352112676, + "pass@2": 0.9646823361428418, + "pass@4": 0.97650347359749, + "pass@8": 0.9829905290283689, + "pass@16": 0.9880542555562098, + "pass@32": 0.9923821609397722, + "pass@64": 0.9953117085182914, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8119909702549575, + "pass@2": 0.8574068445941428, + "pass@4": 0.8905700936639295, + "pass@8": 0.9141548025681611, + "pass@16": 0.9310147321010921, + "pass@32": 0.9431402746849071, + "pass@64": 0.9527867386839005, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.72314177259887, + "pass@2": 0.7903432092174907, + "pass@4": 0.8314442794756585, + "pass@8": 0.8607643836517445, + "pass@16": 0.8863590398904893, + "pass@32": 0.9076966113232872, + "pass@64": 0.924174427048371, + "pass@128": 0.9350282485875706 + }, + "13": { + "pass@1": 0.5077390282131662, + "pass@2": 0.6026861748080863, + "pass@4": 0.6746213919576663, + "pass@8": 0.7280871199402102, + "pass@16": 0.7710803144173798, + "pass@32": 0.8074469976041617, + "pass@64": 0.8368862455010191, + "pass@128": 0.8589341692789969 + }, + "14": { + "pass@1": 0.32676207165109034, + "pass@2": 0.4022562673240613, + "pass@4": 0.47134501353218683, + "pass@8": 0.5316685145841253, + "pass@16": 0.583733926077304, + "pass@32": 0.6325814592753726, + "pass@64": 0.6818888012957016, + "pass@128": 0.7289719626168224 + }, + "15": { + "pass@1": 0.2350298049132948, + "pass@2": 0.2988132794365298, + "pass@4": 0.3616546562460039, + "pass@8": 0.4242653266749827, + "pass@16": 0.4833672306164851, + "pass@32": 0.5327217007147764, + "pass@64": 0.5712522977311489, + "pass@128": 0.6040462427745664 + }, + "16": { + "pass@1": 0.20788810483870968, + "pass@2": 0.27152257429514853, + "pass@4": 0.327680491853841, + "pass@8": 0.3755843283869693, + "pass@16": 0.41732714469475174, + "pass@32": 0.4543796479160046, + "pass@64": 0.4873940801671596, + "pass@128": 0.5161290322580645 + }, + "17": { + "pass@1": 0.19716401734104047, + "pass@2": 0.25503077920895717, + "pass@4": 0.30815697911316003, + "pass@8": 0.35562136170719266, + "pass@16": 0.3982127691151806, + "pass@32": 0.4370893005671065, + "pass@64": 0.47201922867722684, + "pass@128": 0.5028901734104047 + }, + "18": { + "pass@1": 0.16732142857142857, + "pass@2": 0.22041725253093356, + "pass@4": 0.2704885762493973, + "pass@8": 0.3153377965478373, + "pass@16": 0.3545991358083943, + "pass@32": 0.38914450742366186, + "pass@64": 0.4186827347023023, + "pass@128": 0.44285714285714284 + }, + "19": { + "pass@1": 0.17396302552552553, + "pass@2": 0.23597921248019668, + "pass@4": 0.2925832649297215, + "pass@8": 0.33894785591823995, + "pass@16": 0.3781170566234184, + "pass@32": 0.41324923316576234, + "pass@64": 0.44271916868136607, + "pass@128": 0.46546546546546547 + }, + "20": { + "pass@1": 0.15355198948948948, + "pass@2": 0.20524424522948134, + "pass@4": 0.25447808072414363, + "pass@8": 0.29844947745611566, + "pass@16": 0.3366465437604806, + "pass@32": 0.3712929813911508, + "pass@64": 0.4015159625628319, + "pass@128": 0.4264264264264264 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 521029, + "answer_accuracy": 0.6345345381917381, + "avg_response_len": 251.66157199922057, + "resp_tokens_sum": 206644350, + "pass_at_k": { + "pass@1": 0.6345345381917381, + "pass@2": 0.6713106661306874, + "pass@4": 0.7017797563301446, + "pass@8": 0.7275078394786831, + "pass@16": 0.7501136396213048, + "pass@32": 0.7705007575565151, + "pass@64": 0.7884642294169927, + "pass@128": 0.8038971161340608 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8560678680981595, + "pass@2": 0.8841039533114343, + "pass@4": 0.9030514761191657, + "pass@8": 0.9173781283934727, + "pass@16": 0.9282431728633065, + "pass@32": 0.9367494154153667, + "pass@64": 0.9431173930127746, + "pass@128": 0.9478527607361963 + }, + "2": { + "pass@1": 0.9994300911854104, + "pass@2": 0.9999536294665294, + "pass@4": 0.9999994814477978, + "pass@8": 0.9999999999726388, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.994720643939394, + "pass@2": 0.9957684621808638, + "pass@4": 0.9967587758916499, + "pass@8": 0.9976809114058548, + "pass@16": 0.998458723990178, + "pass@32": 0.9992998922984194, + "pass@64": 0.9999126700071582, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996070636094675, + "pass@2": 0.9999854400596374, + "pass@4": 0.9999999800320818, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9763360507246377, + "pass@2": 0.9799897295446762, + "pass@4": 0.9829189930063094, + "pass@8": 0.9854568297076476, + "pass@16": 0.9878492664588717, + "pass@32": 0.9902406260662067, + "pass@64": 0.992681422004542, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9599973137535817, + "pass@2": 0.9650911349637887, + "pass@4": 0.9684557643940639, + "pass@8": 0.9713198049675233, + "pass@16": 0.9734459817059731, + "pass@32": 0.9751500499585644, + "pass@64": 0.9770377542411987, + "pass@128": 0.9799426934097422 + }, + "7": { + "pass@1": 0.9596278901734104, + "pass@2": 0.9690860253288427, + "pass@4": 0.9756533710020351, + "pass@8": 0.9813592130007283, + "pass@16": 0.9863418091645095, + "pass@32": 0.9903767960067987, + "pass@64": 0.9930983479380481, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9377860915492958, + "pass@2": 0.958912678828879, + "pass@4": 0.9724244942093503, + "pass@8": 0.9806020235436738, + "pass@16": 0.9851324251816836, + "pass@32": 0.9876871801465659, + "pass@64": 0.9886708658809289, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9292196856287425, + "pass@2": 0.9527257001744545, + "pass@4": 0.9627166653756599, + "pass@8": 0.9688774460237405, + "pass@16": 0.9742544799641086, + "pass@32": 0.9797131952662533, + "pass@64": 0.9843850850382375, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8160062123493976, + "pass@2": 0.8610585303813678, + "pass@4": 0.8899617457456372, + "pass@8": 0.9105718643838208, + "pass@16": 0.9254026184703923, + "pass@32": 0.9363826773485632, + "pass@64": 0.9448460895593244, + "pass@128": 0.9518072289156626 + }, + "12": { + "pass@1": 0.7139743338557993, + "pass@2": 0.7795156030163154, + "pass@4": 0.8230373401600654, + "pass@8": 0.8540952671660289, + "pass@16": 0.8784286480198956, + "pass@32": 0.8988179779778064, + "pass@64": 0.914751375375745, + "pass@128": 0.9247648902821317 + }, + "13": { + "pass@1": 0.4431022408963585, + "pass@2": 0.5229334844173891, + "pass@4": 0.5902965916025199, + "pass@8": 0.647391393076343, + "pass@16": 0.699989527769502, + "pass@32": 0.7489458173941425, + "pass@64": 0.7897583683021384, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.35249908625730997, + "pass@2": 0.4206065524704149, + "pass@4": 0.47809296233804116, + "pass@8": 0.5299356203940037, + "pass@16": 0.5811376255515341, + "pass@32": 0.6335831039468219, + "pass@64": 0.6852598413531686, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.23425662878787878, + "pass@2": 0.2991070895967547, + "pass@4": 0.3562961430957494, + "pass@8": 0.40513356161577346, + "pass@16": 0.44987841221925085, + "pass@32": 0.49389229180815036, + "pass@64": 0.5351252376246326, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.2116942663817664, + "pass@2": 0.2658307535276038, + "pass@4": 0.3131332528412581, + "pass@8": 0.35405246824329895, + "pass@16": 0.3925773455929453, + "pass@32": 0.4296501256109476, + "pass@64": 0.4638134199632514, + "pass@128": 0.4985754985754986 + }, + "17": { + "pass@1": 0.19443905279503104, + "pass@2": 0.2521755942192007, + "pass@4": 0.3052168306213276, + "pass@8": 0.3531225638086416, + "pass@16": 0.3969801368618287, + "pass@32": 0.4357863106880458, + "pass@64": 0.46768276618636456, + "pass@128": 0.4906832298136646 + }, + "18": { + "pass@1": 0.17817095588235293, + "pass@2": 0.2388269308707734, + "pass@4": 0.2978523059617549, + "pass@8": 0.3508819219268661, + "pass@16": 0.39719267118461893, + "pass@32": 0.4392130178110388, + "pass@64": 0.4782813253719725, + "pass@128": 0.5147058823529411 + }, + "19": { + "pass@1": 0.150390625, + "pass@2": 0.20729415658980124, + "pass@4": 0.2629280089988752, + "pass@8": 0.3126190421848842, + "pass@16": 0.35452784964061956, + "pass@32": 0.38692673717859644, + "pass@64": 0.41329885722988, + "pass@128": 0.4375 + }, + "20": { + "pass@1": 0.1395256362275449, + "pass@2": 0.19365010137205899, + "pass@4": 0.24706315462438455, + "pass@8": 0.29426874511051465, + "pass@16": 0.334118849246813, + "pass@32": 0.3683037737540345, + "pass@64": 0.39960945985179536, + "pass@128": 0.4281437125748503 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 506571, + "answer_accuracy": 0.6323032333439846, + "avg_response_len": 269.9253312729669, + "resp_tokens_sum": 216251219, + "pass_at_k": { + "pass@1": 0.6323032333439846, + "pass@2": 0.668776332160433, + "pass@4": 0.6993617351943084, + "pass@8": 0.725679187269482, + "pass@16": 0.748700830889992, + "pass@32": 0.7692323753744258, + "pass@64": 0.7880763014515265, + "pass@128": 0.8058795334718006 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8672341417910447, + "pass@2": 0.8958841373839462, + "pass@4": 0.9150610647549656, + "pass@8": 0.9293536958508508, + "pass@16": 0.9413545846623945, + "pass@32": 0.9513829259023318, + "pass@64": 0.9601114523123534, + "pass@128": 0.9671641791044776 + }, + "2": { + "pass@1": 0.9990848214285715, + "pass@2": 0.999859392575928, + "pass@4": 0.9999945069366328, + "pass@8": 0.9999999937887084, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9923469387755102, + "pass@2": 0.9945726251693029, + "pass@4": 0.9963000212619196, + "pass@8": 0.9976444803649415, + "pass@16": 0.9985107643235256, + "pass@32": 0.9993264176696698, + "pass@64": 0.9999159798902688, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9940606725146199, + "pass@2": 0.9952050093244924, + "pass@4": 0.9959562867141608, + "pass@8": 0.9966627049626813, + "pass@16": 0.9970259283691618, + "pass@32": 0.9970756091990718, + "pass@64": 0.997076023391154, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9534926470588235, + "pass@2": 0.9598996931449746, + "pass@4": 0.9647326030018306, + "pass@8": 0.9687924250018215, + "pass@16": 0.9727276001124571, + "pass@32": 0.9771633497683939, + "pass@64": 0.9818101565064178, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9694400630914827, + "pass@2": 0.976782753545791, + "pass@4": 0.9821603184854258, + "pass@8": 0.9859407044747921, + "pass@16": 0.988325805700107, + "pass@32": 0.9902831845784584, + "pass@64": 0.9929082222474788, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9503807357594937, + "pass@2": 0.9640580615219775, + "pass@4": 0.9723427231801719, + "pass@8": 0.9783455795238412, + "pass@16": 0.9829082029190926, + "pass@32": 0.9853335592118175, + "pass@64": 0.986555443080517, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.9269894894894894, + "pass@2": 0.9444093453689912, + "pass@4": 0.9565674254119138, + "pass@8": 0.9652994442275824, + "pass@16": 0.972560615488667, + "pass@32": 0.9789532952080486, + "pass@64": 0.9852802738870954, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.9176798633440515, + "pass@2": 0.9395939090310658, + "pass@4": 0.9518787198424955, + "pass@8": 0.961285947902821, + "pass@16": 0.968386382790631, + "pass@32": 0.9734806178809153, + "pass@64": 0.9791055415271452, + "pass@128": 0.9871382636655949 + }, + "11": { + "pass@1": 0.8142113095238095, + "pass@2": 0.8580154043244594, + "pass@4": 0.8854600091655204, + "pass@8": 0.9040861946350753, + "pass@16": 0.917327529399465, + "pass@32": 0.9277359940884433, + "pass@64": 0.9392719044117785, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.7052752293577982, + "pass@2": 0.7677594271472944, + "pass@4": 0.8131363341623575, + "pass@8": 0.8499243285325645, + "pass@16": 0.8787344801874143, + "pass@32": 0.9009459229157835, + "pass@64": 0.9185204439860378, + "pass@128": 0.9327217125382263 + }, + "13": { + "pass@1": 0.4840374228395062, + "pass@2": 0.5678211122047245, + "pass@4": 0.6344011032185792, + "pass@8": 0.6896730840043367, + "pass@16": 0.7377430046359499, + "pass@32": 0.7808939941901869, + "pass@64": 0.817655131521307, + "pass@128": 0.845679012345679 + }, + "14": { + "pass@1": 0.34226632047477745, + "pass@2": 0.41757437381714485, + "pass@4": 0.4810371275919886, + "pass@8": 0.5324609913290231, + "pass@16": 0.5765252565433672, + "pass@32": 0.6195243218028179, + "pass@64": 0.6638484308398551, + "pass@128": 0.7091988130563798 + }, + "15": { + "pass@1": 0.2175684799382716, + "pass@2": 0.2820352751045008, + "pass@4": 0.3425003804501287, + "pass@8": 0.3957935260999967, + "pass@16": 0.44250060778231465, + "pass@32": 0.4848249916319935, + "pass@64": 0.5263831871583865, + "pass@128": 0.5709876543209876 + }, + "16": { + "pass@1": 0.20268713126843657, + "pass@2": 0.26056365119736136, + "pass@4": 0.3140499846258156, + "pass@8": 0.361826706999834, + "pass@16": 0.40430848371338823, + "pass@32": 0.44235122437377733, + "pass@64": 0.4786072043273569, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.1823465737951807, + "pass@2": 0.2351432056968028, + "pass@4": 0.285518104194054, + "pass@8": 0.33299355661175367, + "pass@16": 0.3771098697464484, + "pass@32": 0.41711976481202867, + "pass@64": 0.4538293890103033, + "pass@128": 0.4879518072289157 + }, + "18": { + "pass@1": 0.16305443548387097, + "pass@2": 0.21881270637541272, + "pass@4": 0.27318726731739185, + "pass@8": 0.3228368526161366, + "pass@16": 0.3684910532120487, + "pass@32": 0.4120752139334768, + "pass@64": 0.451199065109953, + "pass@128": 0.4838709677419355 + }, + "19": { + "pass@1": 0.15138783987915408, + "pass@2": 0.19958979589409337, + "pass@4": 0.2458423523154772, + "pass@8": 0.2905585714895618, + "pass@16": 0.33197918407873356, + "pass@32": 0.36661461630839115, + "pass@64": 0.3943805189143051, + "pass@128": 0.4199395770392749 + }, + "20": { + "pass@1": 0.15453734984984985, + "pass@2": 0.20794946028705857, + "pass@4": 0.2596704634893611, + "pass@8": 0.3088198234624205, + "pass@16": 0.3551515634200191, + "pass@32": 0.39827724021815375, + "pass@64": 0.4363332912293886, + "pass@128": 0.46846846846846846 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/summary.csv b/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..4b94ec2b260459451ff33498d506f7f6ef868f0b --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-22350,0.9594730902777778,0.34515703125,0.6361488486842105,0.0,0.0,0.0,196.4464236111111,307.698259375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/summary.json b/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cdde1e182d6ebc8405e6e8f6cba9bb960e5c3ddf --- /dev/null +++ b/SDAR-100M-ep1-block32/composition/low_confidence_static/block8/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-22350", + "id_acc": 0.9594730902777778, + "ood_acc": 0.34515703125, + "total_acc": 0.6361488486842105, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.4464236111111, + "ood_avg_resp_len": 307.698259375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 519514, + "answer_accuracy": 0.6415907564021499, + "avg_response_len": 243.61820636065445, + "resp_tokens_sum": 197264483, + "pass_at_k": { + "pass@1": 0.6415907564021499, + "pass@2": 0.6795674052342421, + "pass@4": 0.7104374509007589, + "pass@8": 0.7361038308882039, + "pass@16": 0.7581496440955067, + "pass@32": 0.7774416276004311, + "pass@64": 0.7941477275875496, + "pass@128": 0.8080935820423648 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.903530604719764, + "pass@2": 0.9280135095115319, + "pass@4": 0.9426811864224937, + "pass@8": 0.9534636795009439, + "pass@16": 0.9602740600181923, + "pass@32": 0.9643415604348307, + "pass@64": 0.9677587233741899, + "pass@128": 0.9705014749262537 + }, + "2": { + "pass@1": 0.9996592679127726, + "pass@2": 0.9999701044962838, + "pass@4": 0.9999997912060058, + "pass@8": 0.9999999999971957, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9957234327217125, + "pass@2": 0.9966740109321197, + "pass@4": 0.9969229032724121, + "pass@8": 0.9969418134426237, + "pass@16": 0.9969418960240432, + "pass@32": 0.9969418960244648, + "pass@64": 0.9969418960244648, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.9931884765625, + "pass@2": 0.9937711460383859, + "pass@4": 0.9938475112485939, + "pass@8": 0.993945312498918, + "pass@16": 0.994140625, + "pass@32": 0.99453125, + "pass@64": 0.9953125, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9752232142857142, + "pass@2": 0.9805809430071241, + "pass@4": 0.9836755715059426, + "pass@8": 0.9860623829306817, + "pass@16": 0.9877131700577393, + "pass@32": 0.988865914595378, + "pass@64": 0.9904761738606371, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9744807260479041, + "pass@2": 0.9818922686123818, + "pass@4": 0.9849555498115132, + "pass@8": 0.9867536020108717, + "pass@16": 0.9885545833866755, + "pass@32": 0.9907015576214719, + "pass@64": 0.9928842794533806, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.9752218934911243, + "pass@2": 0.9823286003820528, + "pass@4": 0.9855232493127706, + "pass@8": 0.9876027174492761, + "pass@16": 0.9900139637057965, + "pass@32": 0.9935202105812683, + "pass@64": 0.9977249219587196, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9639172676282052, + "pass@2": 0.9756736763072885, + "pass@4": 0.9796417202657361, + "pass@8": 0.9811646330398488, + "pass@16": 0.9820810896845035, + "pass@32": 0.9829762025036928, + "pass@64": 0.9837834645669173, + "pass@128": 0.9839743589743589 + }, + "9": { + "pass@1": 0.9407350352112676, + "pass@2": 0.9646823361428418, + "pass@4": 0.97650347359749, + "pass@8": 0.9829905290283689, + "pass@16": 0.9880542555562098, + "pass@32": 0.9923821609397722, + "pass@64": 0.9953117085182914, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.8119909702549575, + "pass@2": 0.8574068445941428, + "pass@4": 0.8905700936639295, + "pass@8": 0.9141548025681611, + "pass@16": 0.9310147321010921, + "pass@32": 0.9431402746849071, + "pass@64": 0.9527867386839005, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.72314177259887, + "pass@2": 0.7903432092174907, + "pass@4": 0.8314442794756585, + "pass@8": 0.8607643836517445, + "pass@16": 0.8863590398904893, + "pass@32": 0.9076966113232872, + "pass@64": 0.924174427048371, + "pass@128": 0.9350282485875706 + }, + "13": { + "pass@1": 0.5077390282131662, + "pass@2": 0.6026861748080863, + "pass@4": 0.6746213919576663, + "pass@8": 0.7280871199402102, + "pass@16": 0.7710803144173798, + "pass@32": 0.8074469976041617, + "pass@64": 0.8368862455010191, + "pass@128": 0.8589341692789969 + }, + "14": { + "pass@1": 0.32676207165109034, + "pass@2": 0.4022562673240613, + "pass@4": 0.47134501353218683, + "pass@8": 0.5316685145841253, + "pass@16": 0.583733926077304, + "pass@32": 0.6325814592753726, + "pass@64": 0.6818888012957016, + "pass@128": 0.7289719626168224 + }, + "15": { + "pass@1": 0.2350298049132948, + "pass@2": 0.2988132794365298, + "pass@4": 0.3616546562460039, + "pass@8": 0.4242653266749827, + "pass@16": 0.4833672306164851, + "pass@32": 0.5327217007147764, + "pass@64": 0.5712522977311489, + "pass@128": 0.6040462427745664 + }, + "16": { + "pass@1": 0.20788810483870968, + "pass@2": 0.27152257429514853, + "pass@4": 0.327680491853841, + "pass@8": 0.3755843283869693, + "pass@16": 0.41732714469475174, + "pass@32": 0.4543796479160046, + "pass@64": 0.4873940801671596, + "pass@128": 0.5161290322580645 + }, + "17": { + "pass@1": 0.19716401734104047, + "pass@2": 0.25503077920895717, + "pass@4": 0.30815697911316003, + "pass@8": 0.35562136170719266, + "pass@16": 0.3982127691151806, + "pass@32": 0.4370893005671065, + "pass@64": 0.47201922867722684, + "pass@128": 0.5028901734104047 + }, + "18": { + "pass@1": 0.16732142857142857, + "pass@2": 0.22041725253093356, + "pass@4": 0.2704885762493973, + "pass@8": 0.3153377965478373, + "pass@16": 0.3545991358083943, + "pass@32": 0.38914450742366186, + "pass@64": 0.4186827347023023, + "pass@128": 0.44285714285714284 + }, + "19": { + "pass@1": 0.17396302552552553, + "pass@2": 0.23597921248019668, + "pass@4": 0.2925832649297215, + "pass@8": 0.33894785591823995, + "pass@16": 0.3781170566234184, + "pass@32": 0.41324923316576234, + "pass@64": 0.44271916868136607, + "pass@128": 0.46546546546546547 + }, + "20": { + "pass@1": 0.15355198948948948, + "pass@2": 0.20524424522948134, + "pass@4": 0.25447808072414363, + "pass@8": 0.29844947745611566, + "pass@16": 0.3366465437604806, + "pass@32": 0.3712929813911508, + "pass@64": 0.4015159625628319, + "pass@128": 0.4264264264264264 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 521029, + "answer_accuracy": 0.6345345381917381, + "avg_response_len": 251.66157199922057, + "resp_tokens_sum": 206644350, + "pass_at_k": { + "pass@1": 0.6345345381917381, + "pass@2": 0.6713106661306874, + "pass@4": 0.7017797563301446, + "pass@8": 0.7275078394786831, + "pass@16": 0.7501136396213048, + "pass@32": 0.7705007575565151, + "pass@64": 0.7884642294169927, + "pass@128": 0.8038971161340608 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8560678680981595, + "pass@2": 0.8841039533114343, + "pass@4": 0.9030514761191657, + "pass@8": 0.9173781283934727, + "pass@16": 0.9282431728633065, + "pass@32": 0.9367494154153667, + "pass@64": 0.9431173930127746, + "pass@128": 0.9478527607361963 + }, + "2": { + "pass@1": 0.9994300911854104, + "pass@2": 0.9999536294665294, + "pass@4": 0.9999994814477978, + "pass@8": 0.9999999999726388, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.994720643939394, + "pass@2": 0.9957684621808638, + "pass@4": 0.9967587758916499, + "pass@8": 0.9976809114058548, + "pass@16": 0.998458723990178, + "pass@32": 0.9992998922984194, + "pass@64": 0.9999126700071582, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9996070636094675, + "pass@2": 0.9999854400596374, + "pass@4": 0.9999999800320818, + "pass@8": 0.999999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9763360507246377, + "pass@2": 0.9799897295446762, + "pass@4": 0.9829189930063094, + "pass@8": 0.9854568297076476, + "pass@16": 0.9878492664588717, + "pass@32": 0.9902406260662067, + "pass@64": 0.992681422004542, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9599973137535817, + "pass@2": 0.9650911349637887, + "pass@4": 0.9684557643940639, + "pass@8": 0.9713198049675233, + "pass@16": 0.9734459817059731, + "pass@32": 0.9751500499585644, + "pass@64": 0.9770377542411987, + "pass@128": 0.9799426934097422 + }, + "7": { + "pass@1": 0.9596278901734104, + "pass@2": 0.9690860253288427, + "pass@4": 0.9756533710020351, + "pass@8": 0.9813592130007283, + "pass@16": 0.9863418091645095, + "pass@32": 0.9903767960067987, + "pass@64": 0.9930983479380481, + "pass@128": 0.9942196531791907 + }, + "8": { + "pass@1": 0.9377860915492958, + "pass@2": 0.958912678828879, + "pass@4": 0.9724244942093503, + "pass@8": 0.9806020235436738, + "pass@16": 0.9851324251816836, + "pass@32": 0.9876871801465659, + "pass@64": 0.9886708658809289, + "pass@128": 0.9887323943661972 + }, + "9": { + "pass@1": 0.9292196856287425, + "pass@2": 0.9527257001744545, + "pass@4": 0.9627166653756599, + "pass@8": 0.9688774460237405, + "pass@16": 0.9742544799641086, + "pass@32": 0.9797131952662533, + "pass@64": 0.9843850850382375, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.8160062123493976, + "pass@2": 0.8610585303813678, + "pass@4": 0.8899617457456372, + "pass@8": 0.9105718643838208, + "pass@16": 0.9254026184703923, + "pass@32": 0.9363826773485632, + "pass@64": 0.9448460895593244, + "pass@128": 0.9518072289156626 + }, + "12": { + "pass@1": 0.7139743338557993, + "pass@2": 0.7795156030163154, + "pass@4": 0.8230373401600654, + "pass@8": 0.8540952671660289, + "pass@16": 0.8784286480198956, + "pass@32": 0.8988179779778064, + "pass@64": 0.914751375375745, + "pass@128": 0.9247648902821317 + }, + "13": { + "pass@1": 0.4431022408963585, + "pass@2": 0.5229334844173891, + "pass@4": 0.5902965916025199, + "pass@8": 0.647391393076343, + "pass@16": 0.699989527769502, + "pass@32": 0.7489458173941425, + "pass@64": 0.7897583683021384, + "pass@128": 0.8207282913165266 + }, + "14": { + "pass@1": 0.35249908625730997, + "pass@2": 0.4206065524704149, + "pass@4": 0.47809296233804116, + "pass@8": 0.5299356203940037, + "pass@16": 0.5811376255515341, + "pass@32": 0.6335831039468219, + "pass@64": 0.6852598413531686, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.23425662878787878, + "pass@2": 0.2991070895967547, + "pass@4": 0.3562961430957494, + "pass@8": 0.40513356161577346, + "pass@16": 0.44987841221925085, + "pass@32": 0.49389229180815036, + "pass@64": 0.5351252376246326, + "pass@128": 0.5727272727272728 + }, + "16": { + "pass@1": 0.2116942663817664, + "pass@2": 0.2658307535276038, + "pass@4": 0.3131332528412581, + "pass@8": 0.35405246824329895, + "pass@16": 0.3925773455929453, + "pass@32": 0.4296501256109476, + "pass@64": 0.4638134199632514, + "pass@128": 0.4985754985754986 + }, + "17": { + "pass@1": 0.19443905279503104, + "pass@2": 0.2521755942192007, + "pass@4": 0.3052168306213276, + "pass@8": 0.3531225638086416, + "pass@16": 0.3969801368618287, + "pass@32": 0.4357863106880458, + "pass@64": 0.46768276618636456, + "pass@128": 0.4906832298136646 + }, + "18": { + "pass@1": 0.17817095588235293, + "pass@2": 0.2388269308707734, + "pass@4": 0.2978523059617549, + "pass@8": 0.3508819219268661, + "pass@16": 0.39719267118461893, + "pass@32": 0.4392130178110388, + "pass@64": 0.4782813253719725, + "pass@128": 0.5147058823529411 + }, + "19": { + "pass@1": 0.150390625, + "pass@2": 0.20729415658980124, + "pass@4": 0.2629280089988752, + "pass@8": 0.3126190421848842, + "pass@16": 0.35452784964061956, + "pass@32": 0.38692673717859644, + "pass@64": 0.41329885722988, + "pass@128": 0.4375 + }, + "20": { + "pass@1": 0.1395256362275449, + "pass@2": 0.19365010137205899, + "pass@4": 0.24706315462438455, + "pass@8": 0.29426874511051465, + "pass@16": 0.334118849246813, + "pass@32": 0.3683037737540345, + "pass@64": 0.39960945985179536, + "pass@128": 0.4281437125748503 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 506571, + "answer_accuracy": 0.6323032333439846, + "avg_response_len": 269.9253312729669, + "resp_tokens_sum": 216251219, + "pass_at_k": { + "pass@1": 0.6323032333439846, + "pass@2": 0.668776332160433, + "pass@4": 0.6993617351943084, + "pass@8": 0.725679187269482, + "pass@16": 0.748700830889992, + "pass@32": 0.7692323753744258, + "pass@64": 0.7880763014515265, + "pass@128": 0.8058795334718006 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8672341417910447, + "pass@2": 0.8958841373839462, + "pass@4": 0.9150610647549656, + "pass@8": 0.9293536958508508, + "pass@16": 0.9413545846623945, + "pass@32": 0.9513829259023318, + "pass@64": 0.9601114523123534, + "pass@128": 0.9671641791044776 + }, + "2": { + "pass@1": 0.9990848214285715, + "pass@2": 0.999859392575928, + "pass@4": 0.9999945069366328, + "pass@8": 0.9999999937887084, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9923469387755102, + "pass@2": 0.9945726251693029, + "pass@4": 0.9963000212619196, + "pass@8": 0.9976444803649415, + "pass@16": 0.9985107643235256, + "pass@32": 0.9993264176696698, + "pass@64": 0.9999159798902688, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9940606725146199, + "pass@2": 0.9952050093244924, + "pass@4": 0.9959562867141608, + "pass@8": 0.9966627049626813, + "pass@16": 0.9970259283691618, + "pass@32": 0.9970756091990718, + "pass@64": 0.997076023391154, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9534926470588235, + "pass@2": 0.9598996931449746, + "pass@4": 0.9647326030018306, + "pass@8": 0.9687924250018215, + "pass@16": 0.9727276001124571, + "pass@32": 0.9771633497683939, + "pass@64": 0.9818101565064178, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9694400630914827, + "pass@2": 0.976782753545791, + "pass@4": 0.9821603184854258, + "pass@8": 0.9859407044747921, + "pass@16": 0.988325805700107, + "pass@32": 0.9902831845784584, + "pass@64": 0.9929082222474788, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9503807357594937, + "pass@2": 0.9640580615219775, + "pass@4": 0.9723427231801719, + "pass@8": 0.9783455795238412, + "pass@16": 0.9829082029190926, + "pass@32": 0.9853335592118175, + "pass@64": 0.986555443080517, + "pass@128": 0.9873417721518988 + }, + "8": { + "pass@1": 0.9269894894894894, + "pass@2": 0.9444093453689912, + "pass@4": 0.9565674254119138, + "pass@8": 0.9652994442275824, + "pass@16": 0.972560615488667, + "pass@32": 0.9789532952080486, + "pass@64": 0.9852802738870954, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.9176798633440515, + "pass@2": 0.9395939090310658, + "pass@4": 0.9518787198424955, + "pass@8": 0.961285947902821, + "pass@16": 0.968386382790631, + "pass@32": 0.9734806178809153, + "pass@64": 0.9791055415271452, + "pass@128": 0.9871382636655949 + }, + "11": { + "pass@1": 0.8142113095238095, + "pass@2": 0.8580154043244594, + "pass@4": 0.8854600091655204, + "pass@8": 0.9040861946350753, + "pass@16": 0.917327529399465, + "pass@32": 0.9277359940884433, + "pass@64": 0.9392719044117785, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.7052752293577982, + "pass@2": 0.7677594271472944, + "pass@4": 0.8131363341623575, + "pass@8": 0.8499243285325645, + "pass@16": 0.8787344801874143, + "pass@32": 0.9009459229157835, + "pass@64": 0.9185204439860378, + "pass@128": 0.9327217125382263 + }, + "13": { + "pass@1": 0.4840374228395062, + "pass@2": 0.5678211122047245, + "pass@4": 0.6344011032185792, + "pass@8": 0.6896730840043367, + "pass@16": 0.7377430046359499, + "pass@32": 0.7808939941901869, + "pass@64": 0.817655131521307, + "pass@128": 0.845679012345679 + }, + "14": { + "pass@1": 0.34226632047477745, + "pass@2": 0.41757437381714485, + "pass@4": 0.4810371275919886, + "pass@8": 0.5324609913290231, + "pass@16": 0.5765252565433672, + "pass@32": 0.6195243218028179, + "pass@64": 0.6638484308398551, + "pass@128": 0.7091988130563798 + }, + "15": { + "pass@1": 0.2175684799382716, + "pass@2": 0.2820352751045008, + "pass@4": 0.3425003804501287, + "pass@8": 0.3957935260999967, + "pass@16": 0.44250060778231465, + "pass@32": 0.4848249916319935, + "pass@64": 0.5263831871583865, + "pass@128": 0.5709876543209876 + }, + "16": { + "pass@1": 0.20268713126843657, + "pass@2": 0.26056365119736136, + "pass@4": 0.3140499846258156, + "pass@8": 0.361826706999834, + "pass@16": 0.40430848371338823, + "pass@32": 0.44235122437377733, + "pass@64": 0.4786072043273569, + "pass@128": 0.5162241887905604 + }, + "17": { + "pass@1": 0.1823465737951807, + "pass@2": 0.2351432056968028, + "pass@4": 0.285518104194054, + "pass@8": 0.33299355661175367, + "pass@16": 0.3771098697464484, + "pass@32": 0.41711976481202867, + "pass@64": 0.4538293890103033, + "pass@128": 0.4879518072289157 + }, + "18": { + "pass@1": 0.16305443548387097, + "pass@2": 0.21881270637541272, + "pass@4": 0.27318726731739185, + "pass@8": 0.3228368526161366, + "pass@16": 0.3684910532120487, + "pass@32": 0.4120752139334768, + "pass@64": 0.451199065109953, + "pass@128": 0.4838709677419355 + }, + "19": { + "pass@1": 0.15138783987915408, + "pass@2": 0.19958979589409337, + "pass@4": 0.2458423523154772, + "pass@8": 0.2905585714895618, + "pass@16": 0.33197918407873356, + "pass@32": 0.36661461630839115, + "pass@64": 0.3943805189143051, + "pass@128": 0.4199395770392749 + }, + "20": { + "pass@1": 0.15453734984984985, + "pass@2": 0.20794946028705857, + "pass@4": 0.2596704634893611, + "pass@8": 0.3088198234624205, + "pass@16": 0.3551515634200191, + "pass@32": 0.39827724021815375, + "pass@64": 0.4363332912293886, + "pass@128": 0.46846846846846846 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/autoregressive/checkpoint-21546_metrics.json b/SDAR-100M-ep1-block8/composition/autoregressive/checkpoint-21546_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..df936d546b7e2bd60e3cd8227a0f89c61f70475b --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/autoregressive/checkpoint-21546_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21546", + "id": { + "answer_accuracy": 0.9743967013888889, + "avg_response_len": 200.49886805555556, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9743967013888889, + "pass@2": 0.9864769384295486, + "pass@4": 0.9910679221972267, + "pass@8": 0.9936657904035767, + "pass@16": 0.9954608405910889, + "pass@32": 0.9968089043247439, + "pass@64": 0.9978034336435224, + "pass@128": 0.9984444444444445 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9031875, + "pass@2": 0.941331446850394, + "pass@4": 0.9604867477502804, + "pass@8": 0.9718874688192747, + "pass@16": 0.9795211090723244, + "pass@32": 0.9853481677577, + "pass@64": 0.9897948033393734, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.998984375, + "pass@2": 0.9989998769685039, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.9967421875, + "pass@2": 0.9984013287401574, + "pass@4": 0.9994809550056243, + "pass@8": 0.9999265037605697, + "pass@16": 0.9999979377046996, + "pass@32": 0.999999998949342, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99365625, + "pass@2": 0.9952604576771655, + "pass@4": 0.99574416319835, + "pass@8": 0.9962986058824905, + "pass@16": 0.9970440246751775, + "pass@32": 0.9978894518600503, + "pass@64": 0.9986635853175072, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9736484375, + "pass@2": 0.9834765009842525, + "pass@4": 0.9883558332395947, + "pass@8": 0.9917795126232883, + "pass@16": 0.9943345640274536, + "pass@32": 0.9959711027429112, + "pass@64": 0.9969557329122339, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.99153125, + "pass@2": 0.9966617864173237, + "pass@4": 0.9979242754030744, + "pass@8": 0.9985776657381249, + "pass@16": 0.9989941048487985, + "pass@32": 0.9992400364799199, + "pass@64": 0.9994999879304772, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.98465625, + "pass@2": 0.9936507135826778, + "pass@4": 0.9957305050618672, + "pass@8": 0.9968353787827728, + "pass@16": 0.9978504011995109, + "pass@32": 0.998650949929657, + "pass@64": 0.998969403273414, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.970953125, + "pass@2": 0.9874678887795286, + "pass@4": 0.9930164595987997, + "pass@8": 0.9957940320963058, + "pass@16": 0.9971187403386395, + "pass@32": 0.9978872841808387, + "pass@64": 0.9987513007268689, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9562109375, + "pass@2": 0.9830424458661438, + "pass@4": 0.9898723605174348, + "pass@8": 0.9928929459293645, + "pass@16": 0.9952866834531895, + "pass@32": 0.9972931470222736, + "pass@64": 0.9985960892918249, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9031875, + "2": 0.998984375, + "3": 0.9967421875, + "4": 0.99365625, + "5": 0.9736484375, + "6": 0.99153125, + "7": 0.98465625, + "8": 0.970953125, + "9": 0.9562109375 + }, + "per_op_avg_response_len": { + "10": 280.954671875, + "2": 115.8335, + "3": 145.085, + "4": 160.184578125, + "5": 174.1452421875, + "6": 199.4692734375, + "7": 224.743640625, + "8": 244.058078125, + "9": 260.015828125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.954671875, + "2": 115.8335, + "3": 145.085, + "4": 160.184578125, + "5": 174.1452421875, + "6": 199.4692734375, + "7": 224.743640625, + "8": 244.058078125, + "9": 260.015828125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 368862, + "answer_accuracy": 0.9732301165146909, + "avg_response_len": 191.24051735055724, + "resp_tokens_sum": 72481686, + "pass_at_k": { + "pass@1": 0.9732301165146909, + "pass@2": 0.9862903190957539, + "pass@4": 0.9910474259993093, + "pass@8": 0.9936329623708255, + "pass@16": 0.9954282803015323, + "pass@32": 0.9967457117389424, + "pass@64": 0.9977712919476621, + "pass@128": 0.9986491050320837 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9078401548672567, + "pass@2": 0.9451660598564567, + "pass@4": 0.9627415558121603, + "pass@8": 0.9724228068912785, + "pass@16": 0.9786434853777529, + "pass@32": 0.9833303999293327, + "pass@64": 0.9874800169721869, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9974913990825688, + "pass@2": 0.998993925931277, + "pass@4": 0.999781667199857, + "pass@8": 0.9999876849218283, + "pass@16": 0.9999999719746717, + "pass@32": 0.9999999999999853, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9863525390625, + "pass@2": 0.9885619156003939, + "pass@4": 0.9898306295697413, + "pass@8": 0.9915581459135234, + "pass@16": 0.9938875771099299, + "pass@32": 0.996529537062657, + "pass@64": 0.9989487041172099, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9679315476190476, + "pass@2": 0.9789893450818652, + "pass@4": 0.9852945253867076, + "pass@8": 0.9903654763094896, + "pass@16": 0.9944361907939756, + "pass@32": 0.9969451642580184, + "pass@64": 0.9983683333808681, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9894975673652695, + "pass@2": 0.996165027229007, + "pass@4": 0.998533353147473, + "pass@8": 0.999620189417031, + "pass@16": 0.9999636861019171, + "pass@32": 0.9999998029761207, + "pass@64": 0.999999999999911, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860392011834319, + "pass@2": 0.9959920124167174, + "pass@4": 0.9977027054310518, + "pass@8": 0.9983763394978522, + "pass@16": 0.9991472297394309, + "pass@32": 0.9997987731838602, + "pass@64": 0.999995725631176, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9701522435897436, + "pass@2": 0.9889413612961848, + "pass@4": 0.9943698822502955, + "pass@8": 0.9962621361835249, + "pass@16": 0.9970686948727937, + "pass@32": 0.9975932975941004, + "pass@64": 0.998397435811393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9600132042253521, + "pass@2": 0.9878528058112463, + "pass@4": 0.9946557779749362, + "pass@8": 0.9974029094078771, + "pass@16": 0.9989479245309896, + "pass@32": 0.9997375771890578, + "pass@64": 0.9999912792521506, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 380951, + "answer_accuracy": 0.9751571715268676, + "avg_response_len": 198.48051738614024, + "resp_tokens_sum": 77537605, + "pass_at_k": { + "pass@1": 0.9751571715268676, + "pass@2": 0.9865409912565422, + "pass@4": 0.9909553005681806, + "pass@8": 0.9937105260964556, + "pass@16": 0.9957349188644483, + "pass@32": 0.9972766366991683, + "pass@64": 0.9984251370487833, + "pass@128": 0.9990170380078637 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8947948619631901, + "pass@2": 0.9342487228877832, + "pass@4": 0.9553914430174755, + "pass@8": 0.9685340029247087, + "pass@16": 0.9778976377826292, + "pass@32": 0.9854515349169342, + "pass@64": 0.991566746760942, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9943181818181818, + "pass@2": 0.9965987383679314, + "pass@4": 0.9986907074115734, + "pass@8": 0.9997900403945096, + "pass@16": 0.999993778423042, + "pass@32": 0.9999999968162022, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986825073964497, + "pass@2": 0.9998864324651727, + "pass@4": 0.9999964917477031, + "pass@8": 0.999999997761824, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9817028985507247, + "pass@2": 0.9890858581536004, + "pass@4": 0.9919462368834332, + "pass@8": 0.9941619355764031, + "pass@16": 0.9961781821357537, + "pass@32": 0.9975886476682482, + "pass@64": 0.9985464572309225, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9912249283667621, + "pass@2": 0.9971071734765248, + "pass@4": 0.998236967692792, + "pass@8": 0.9989742809016088, + "pass@16": 0.9996596953889924, + "pass@32": 0.9999716397876668, + "pass@64": 0.9999999654169256, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866781069364162, + "pass@2": 0.9940098595876383, + "pass@4": 0.9954577830950322, + "pass@8": 0.9965385997356417, + "pass@16": 0.997911202425725, + "pass@32": 0.9992275285274425, + "pass@64": 0.9999158252269373, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9713248239436619, + "pass@2": 0.9873073084174341, + "pass@4": 0.9928848371066289, + "pass@8": 0.9962645987397817, + "pass@16": 0.9977159520362265, + "pass@32": 0.9984178461141198, + "pass@64": 0.9993013196994364, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9549728667664671, + "pass@2": 0.9786595607996605, + "pass@4": 0.9846184301812573, + "pass@8": 0.9881064318849376, + "pass@16": 0.9915007486161633, + "pass@32": 0.9943169158716935, + "pass@64": 0.996171853688097, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 372692, + "answer_accuracy": 0.9747761131570137, + "avg_response_len": 211.73890243136927, + "resp_tokens_sum": 80955405, + "pass_at_k": { + "pass@1": 0.9747761131570137, + "pass@2": 0.9865964866784955, + "pass@4": 0.9912033123726127, + "pass@8": 0.9936526235037797, + "pass@16": 0.9952130749824784, + "pass@32": 0.9963936361760367, + "pass@64": 0.9972000632948728, + "pass@128": 0.9976565115500502 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9066464552238805, + "pass@2": 0.9443434892466809, + "pass@4": 0.9631634325560046, + "pass@8": 0.9746091114318679, + "pass@16": 0.981989067498895, + "pass@32": 0.9872894382051791, + "pass@64": 0.9904128899753277, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983600583090378, + "pass@2": 0.9995706451872088, + "pass@4": 0.9999545667214339, + "pass@8": 0.9999994678161623, + "pass@16": 0.9999999999690319, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9955226608187134, + "pass@2": 0.996956229566699, + "pass@4": 0.9970747003664014, + "pass@8": 0.9970760232358671, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9707720588235295, + "pass@2": 0.9819418712366834, + "pass@4": 0.9875488412294049, + "pass@8": 0.9906721170939422, + "pass@16": 0.992369679677989, + "pass@32": 0.9934273310473238, + "pass@64": 0.9940328828017424, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.994011238170347, + "pass@2": 0.9966948384212229, + "pass@4": 0.9969382767414325, + "pass@8": 0.9970425818238966, + "pass@16": 0.9972397476340691, + "pass@32": 0.9976340694006309, + "pass@64": 0.998422712933754, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9809632120253164, + "pass@2": 0.9907531707864047, + "pass@4": 0.9939196128964891, + "pass@8": 0.9955120902657174, + "pass@16": 0.9963967136338048, + "pass@32": 0.996791897857649, + "pass@64": 0.9968353559542914, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9713072447447447, + "pass@2": 0.9862585272280152, + "pass@4": 0.9918887061977614, + "pass@8": 0.9948537929562266, + "pass@16": 0.9965289626590625, + "pass@32": 0.9975971169999002, + "pass@64": 0.9984964932745174, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9532003617363344, + "pass@2": 0.9822585417879847, + "pass@4": 0.9900546741343824, + "pass@8": 0.9928854174919578, + "pass@16": 0.9951733768710284, + "pass@32": 0.9976992193537383, + "pass@64": 0.9996070290208584, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.39801640625, + "avg_response_len": 321.86122421875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.39801640625, + "pass@2": 0.483714000984252, + "pass@4": 0.5557793787214041, + "pass@8": 0.6147551850177316, + "pass@16": 0.6623456607483118, + "pass@32": 0.7008492513637058, + "pass@64": 0.732434425375126, + "pass@128": 0.7577 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8663046875, + "pass@2": 0.9220928887795296, + "pass@4": 0.9506338244281964, + "pass@8": 0.9675263429637743, + "pass@16": 0.9781564796497898, + "pass@32": 0.9851142696937852, + "pass@64": 0.9903520557897221, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.776015625, + "pass@2": 0.8638099163385842, + "pass@4": 0.9110736210161228, + "pass@8": 0.9377246430833253, + "pass@16": 0.9543967079760017, + "pass@32": 0.9664445886852736, + "pass@64": 0.9755977359015238, + "pass@128": 0.982 + }, + "13": { + "pass@1": 0.570796875, + "pass@2": 0.6869044045275595, + "pass@4": 0.7703089637232837, + "pass@8": 0.8289863404139657, + "pass@16": 0.8706306155014618, + "pass@32": 0.9017770640160171, + "pass@64": 0.9262811180583888, + "pass@128": 0.946 + }, + "14": { + "pass@1": 0.465671875, + "pass@2": 0.5816224163385827, + "pass@4": 0.6775207412823394, + "pass@8": 0.7499487856814693, + "pass@16": 0.802954341064699, + "pass@32": 0.8430218421609508, + "pass@64": 0.8751080533100261, + "pass@128": 0.901 + }, + "15": { + "pass@1": 0.36846875, + "pass@2": 0.47901058070866176, + "pass@4": 0.5728318718597668, + "pass@8": 0.6493805015717913, + "pass@16": 0.7133682179646545, + "pass@32": 0.7662561068084073, + "pass@64": 0.8078740144258999, + "pass@128": 0.837 + }, + "16": { + "pass@1": 0.2422890625, + "pass@2": 0.32881323818897656, + "pass@4": 0.41162225215598025, + "pass@8": 0.48517711191976115, + "pass@16": 0.5477937709328996, + "pass@32": 0.6006260501592771, + "pass@64": 0.6451632488133172, + "pass@128": 0.681 + }, + "17": { + "pass@1": 0.205359375, + "pass@2": 0.28227411417322856, + "pass@4": 0.35604794225721725, + "pass@8": 0.42237966102226454, + "pass@16": 0.48057563857933505, + "pass@32": 0.530299008710456, + "pass@64": 0.5723552718529699, + "pass@128": 0.607 + }, + "18": { + "pass@1": 0.186046875, + "pass@2": 0.26339591535433055, + "pass@4": 0.34232630324334395, + "pass@8": 0.4130072150998003, + "pass@16": 0.4705819052420476, + "pass@32": 0.516768489074943, + "pass@64": 0.5562738374202079, + "pass@128": 0.591 + }, + "19": { + "pass@1": 0.15340625, + "pass@2": 0.2179523868110237, + "pass@4": 0.2843356102362201, + "pass@8": 0.3457563272501176, + "pass@16": 0.4002832993363438, + "pass@32": 0.44730833071019604, + "pass@64": 0.48591384451975916, + "pass@128": 0.516 + }, + "20": { + "pass@1": 0.1458046875, + "pass@2": 0.2112641486220472, + "pass@4": 0.2810926570116231, + "pass@8": 0.34766492117099124, + "pass@16": 0.4047156312357237, + "pass@32": 0.4508767636177382, + "pass@64": 0.48942507365937193, + "pass@128": 0.521 + } + }, + "per_op_accuracy": { + "11": 0.8663046875, + "12": 0.776015625, + "13": 0.570796875, + "14": 0.465671875, + "15": 0.36846875, + "16": 0.2422890625, + "17": 0.205359375, + "18": 0.186046875, + "19": 0.15340625, + "20": 0.1458046875 + }, + "per_op_avg_response_len": { + "11": 297.5433046875, + "12": 306.14871875, + "13": 317.1943125, + "14": 328.4833515625, + "15": 332.1365, + "16": 322.0154765625, + "17": 329.8279375, + "18": 333.6520390625, + "19": 323.0419375, + "20": 328.5686640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.5433046875, + "12": 306.14871875, + "13": 317.1943125, + "14": 328.4833515625, + "15": 332.1365, + "16": 322.0154765625, + "17": 329.8279375, + "18": 333.6520390625, + "19": 323.0419375, + "20": 328.5686640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 171481, + "answer_accuracy": 0.3981263930163447, + "avg_response_len": 305.5984676820208, + "resp_tokens_sum": 131627372, + "pass_at_k": { + "pass@1": 0.3981263930163447, + "pass@2": 0.48285525938622587, + "pass@4": 0.5534416722519682, + "pass@8": 0.610445945899272, + "pass@16": 0.6561711137327331, + "pass@32": 0.6939297865406735, + "pass@64": 0.7267991664141054, + "pass@128": 0.7560178306092125 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8549043909348442, + "pass@2": 0.9105383272735389, + "pass@4": 0.9394676640101288, + "pass@8": 0.9567641672474922, + "pass@16": 0.9689281622131964, + "pass@32": 0.9792995108322007, + "pass@64": 0.989377221593619, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.7604387358757062, + "pass@2": 0.8512497080608568, + "pass@4": 0.9022677093859018, + "pass@8": 0.9306185243521176, + "pass@16": 0.947629076857396, + "pass@32": 0.9613728628093047, + "pass@64": 0.9722807480889516, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.6151792711598746, + "pass@2": 0.7272553717325303, + "pass@4": 0.8012909736204601, + "pass@8": 0.8513849982189186, + "pass@16": 0.8858777402360082, + "pass@32": 0.9113844472956131, + "pass@64": 0.9327076612903673, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.4572381230529595, + "pass@2": 0.5804936437559792, + "pass@4": 0.6836558835519382, + "pass@8": 0.7584661690159947, + "pass@16": 0.8090695107690499, + "pass@32": 0.8452244630136615, + "pass@64": 0.8744698913107548, + "pass@128": 0.9034267912772586 + }, + "15": { + "pass@1": 0.3591943641618497, + "pass@2": 0.46582035694779467, + "pass@4": 0.557234378759013, + "pass@8": 0.6333810374573632, + "pass@16": 0.6991402754904769, + "pass@32": 0.7537907497738942, + "pass@64": 0.7989936415700499, + "pass@128": 0.838150289017341 + }, + "16": { + "pass@1": 0.2480594758064516, + "pass@2": 0.3369277051054099, + "pass@4": 0.41882028587152403, + "pass@8": 0.48785088132261173, + "pass@16": 0.5464977216622237, + "pass@32": 0.5997006252636178, + "pass@64": 0.6464878664953922, + "pass@128": 0.6838709677419355 + }, + "17": { + "pass@1": 0.19413836705202311, + "pass@2": 0.2639743866915478, + "pass@4": 0.3301886702384742, + "pass@8": 0.3913913054415934, + "pass@16": 0.44674849852516724, + "pass@32": 0.49537678199969953, + "pass@64": 0.5397606358748235, + "pass@128": 0.5809248554913294 + }, + "18": { + "pass@1": 0.16417410714285716, + "pass@2": 0.23900625703037112, + "pass@4": 0.31737553055868006, + "pass@8": 0.3877756897146819, + "pass@16": 0.44484807064643955, + "pass@32": 0.4911672177360723, + "pass@64": 0.5336450417048393, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.1477805930930931, + "pass@2": 0.21373120167411505, + "pass@4": 0.2819873895267596, + "pass@8": 0.3444790109299255, + "pass@16": 0.398624559518731, + "pass@32": 0.4448169543649647, + "pass@64": 0.48421600118684854, + "pass@128": 0.5165165165165165 + }, + "20": { + "pass@1": 0.15219125375375375, + "pass@2": 0.21601005533092138, + "pass@4": 0.2833787189889551, + "pass@8": 0.34737669283247447, + "pass@16": 0.40206167741449067, + "pass@32": 0.4470107741738278, + "pass@64": 0.4875770578788106, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 169440, + "answer_accuracy": 0.39362176628010703, + "avg_response_len": 316.94848349687777, + "resp_tokens_sum": 136434912, + "pass_at_k": { + "pass@1": 0.39362176628010703, + "pass@2": 0.47886799609460307, + "pass@4": 0.5513894517366877, + "pass@8": 0.6120301912117956, + "pass@16": 0.661500303177071, + "pass@32": 0.7007946708589267, + "pass@64": 0.7309419358097482, + "pass@128": 0.751709782931906 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8858951430722891, + "pass@2": 0.9403234246039275, + "pass@4": 0.9654911386264966, + "pass@8": 0.9802103258206285, + "pass@16": 0.9892633104282124, + "pass@32": 0.9943230187012654, + "pass@64": 0.9966323831791278, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.770865987460815, + "pass@2": 0.8529438081603437, + "pass@4": 0.8969163801625106, + "pass@8": 0.9239001333151642, + "pass@16": 0.944273209688222, + "pass@32": 0.9607568063656658, + "pass@64": 0.9735690051847953, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.55249912464986, + "pass@2": 0.670897295926244, + "pass@4": 0.7573695231373386, + "pass@8": 0.8195883022487538, + "pass@16": 0.8639400641786898, + "pass@32": 0.8959443632193429, + "pass@64": 0.9194230288568553, + "pass@128": 0.938375350140056 + }, + "14": { + "pass@1": 0.4458607456140351, + "pass@2": 0.5551821441727679, + "pass@4": 0.6490961617736377, + "pass@8": 0.7264317974040324, + "pass@16": 0.7878208627637026, + "pass@32": 0.8337530255493398, + "pass@64": 0.8650871772849, + "pass@128": 0.8830409356725146 + }, + "15": { + "pass@1": 0.3692708333333333, + "pass@2": 0.4844406167979, + "pass@4": 0.5807708542114053, + "pass@8": 0.6566659618047733, + "pass@16": 0.71822453037185, + "pass@32": 0.7690809020128775, + "pass@64": 0.8074737398378564, + "pass@128": 0.8272727272727273 + }, + "16": { + "pass@1": 0.2302795584045584, + "pass@2": 0.3132508075913587, + "pass@4": 0.3953974561940442, + "pass@8": 0.4699005180824398, + "pass@16": 0.5316027123774437, + "pass@32": 0.5810105949043947, + "pass@64": 0.6208832644017411, + "pass@128": 0.6495726495726496 + }, + "17": { + "pass@1": 0.21753687888198758, + "pass@2": 0.29977732063383367, + "pass@4": 0.3802775075281738, + "pass@8": 0.4530973687186417, + "pass@16": 0.5146178409330402, + "pass@32": 0.5636207169721692, + "pass@64": 0.6026909514942224, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.19386488970588237, + "pass@2": 0.27067797591477505, + "pass@4": 0.34721876282008873, + "pass@8": 0.4179357468470566, + "pass@16": 0.4785538356301472, + "pass@32": 0.5267746944091646, + "pass@64": 0.5637993560978994, + "pass@128": 0.5882352941176471 + }, + "19": { + "pass@1": 0.14381045386904762, + "pass@2": 0.20448420509936244, + "pass@4": 0.26923672822147243, + "pass@8": 0.3304539572282448, + "pass@16": 0.3857649133616538, + "pass@32": 0.4348844940956561, + "pass@64": 0.4749150124076846, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.140812125748503, + "pass@2": 0.2086588388184262, + "pass@4": 0.2810939201462094, + "pass@8": 0.3485268808493757, + "pass@16": 0.4055889275550898, + "pass@32": 0.45131218202379575, + "pass@64": 0.4876867028458811, + "pass@128": 0.5149700598802395 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 168540, + "answer_accuracy": 0.40242015586797064, + "avg_response_len": 343.6355893757641, + "resp_tokens_sum": 143920083, + "pass_at_k": { + "pass@1": 0.40242015586797064, + "pass@2": 0.48957793127803706, + "pass@4": 0.562695548256653, + "pass@8": 0.621987686179996, + "pass@16": 0.6695645752407798, + "pass@32": 0.708021486500331, + "pass@64": 0.7397638534961841, + "pass@128": 0.7655867970660146 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8584325396825396, + "pass@2": 0.9158269278840144, + "pass@4": 0.947487876217854, + "pass@8": 0.9662182976284491, + "pass@16": 0.976791807385288, + "pass@32": 0.9819247624165014, + "pass@64": 0.9848252233387427, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.7979023318042814, + "pass@2": 0.8880074767030265, + "pass@4": 0.934417512604502, + "pass@8": 0.9589038010248868, + "pass@16": 0.9715989629294857, + "pass@32": 0.9774837125997929, + "pass@64": 0.9811677016027062, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5472608024691358, + "pass@2": 0.6648135996889273, + "pass@4": 0.7540624178343447, + "pass@8": 0.817288586664587, + "pass@16": 0.8629907822666152, + "pass@32": 0.8987447149981824, + "pass@64": 0.927510348163161, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.4938102744807122, + "pass@2": 0.6095301584149166, + "pass@4": 0.7005231909067747, + "pass@8": 0.7657016935167862, + "pass@16": 0.8124875164470288, + "pass@32": 0.8503301329249001, + "pass@64": 0.8858854705277092, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.37755594135802467, + "pass@2": 0.48756584463400393, + "pass@4": 0.5814024534548923, + "pass@8": 0.6590459728887901, + "pass@16": 0.7236160111797545, + "pass@32": 0.7766907707481191, + "pass@64": 0.8177650626424972, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.24944690265486727, + "pass@2": 0.33750624230599463, + "pass@4": 0.4218391044592877, + "pass@8": 0.49854943027379045, + "pass@16": 0.5657431421036233, + "pass@32": 0.6217821165667055, + "pass@64": 0.6690913993944967, + "pass@128": 0.7109144542772862 + }, + "17": { + "pass@1": 0.20524284638554216, + "pass@2": 0.2843695154634285, + "pass@4": 0.3594979064175711, + "pass@8": 0.4248822789520208, + "pass@16": 0.48281238948550975, + "pass@32": 0.5343758170889205, + "pass@64": 0.5769024260817495, + "pass@128": 0.608433734939759 + }, + "18": { + "pass@1": 0.20216733870967743, + "pass@2": 0.28294585026670055, + "pass@4": 0.36513060706121425, + "pass@8": 0.4360889347472986, + "pass@16": 0.49089282710175347, + "pass@32": 0.5346986024780715, + "pass@64": 0.5735686830652535, + "pass@128": 0.6096774193548387 + }, + "19": { + "pass@1": 0.16880664652567975, + "pass@2": 0.23587070616361772, + "pass@4": 0.30202498139394224, + "pass@8": 0.3625748851412769, + "pass@16": 0.416689758692117, + "pass@32": 0.4624262685514275, + "pass@64": 0.49878692433702804, + "pass@128": 0.5287009063444109 + }, + "20": { + "pass@1": 0.14442567567567569, + "pass@2": 0.20913137546995805, + "pass@4": 0.27880532810650926, + "pass@8": 0.34708860136362146, + "pass@16": 0.40649366622612243, + "pass@32": 0.4543060270928111, + "pass@64": 0.493016680586197, + "pass@128": 0.5255255255255256 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6710386513157894, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6710386513157894, + "pass@2": 0.7218648660898945, + "pass@4": 0.7619686887888791, + "pass@8": 0.7942391559899847, + "pass@16": 0.8201370617264716, + "pass@32": 0.8410406659242107, + "pass@64": 0.8581355345548908, + "pass@128": 0.8717368421052631 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9031875, + "pass@2": 0.941331446850394, + "pass@4": 0.9604867477502804, + "pass@8": 0.9718874688192747, + "pass@16": 0.9795211090723244, + "pass@32": 0.9853481677577, + "pass@64": 0.9897948033393734, + "pass@128": 0.992 + }, + "2": { + "pass@1": 0.998984375, + "pass@2": 0.9989998769685039, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.9967421875, + "pass@2": 0.9984013287401574, + "pass@4": 0.9994809550056243, + "pass@8": 0.9999265037605697, + "pass@16": 0.9999979377046996, + "pass@32": 0.999999998949342, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99365625, + "pass@2": 0.9952604576771655, + "pass@4": 0.99574416319835, + "pass@8": 0.9962986058824905, + "pass@16": 0.9970440246751775, + "pass@32": 0.9978894518600503, + "pass@64": 0.9986635853175072, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9736484375, + "pass@2": 0.9834765009842525, + "pass@4": 0.9883558332395947, + "pass@8": 0.9917795126232883, + "pass@16": 0.9943345640274536, + "pass@32": 0.9959711027429112, + "pass@64": 0.9969557329122339, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.99153125, + "pass@2": 0.9966617864173237, + "pass@4": 0.9979242754030744, + "pass@8": 0.9985776657381249, + "pass@16": 0.9989941048487985, + "pass@32": 0.9992400364799199, + "pass@64": 0.9994999879304772, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.98465625, + "pass@2": 0.9936507135826778, + "pass@4": 0.9957305050618672, + "pass@8": 0.9968353787827728, + "pass@16": 0.9978504011995109, + "pass@32": 0.998650949929657, + "pass@64": 0.998969403273414, + "pass@128": 0.999 + }, + "8": { + "pass@1": 0.970953125, + "pass@2": 0.9874678887795286, + "pass@4": 0.9930164595987997, + "pass@8": 0.9957940320963058, + "pass@16": 0.9971187403386395, + "pass@32": 0.9978872841808387, + "pass@64": 0.9987513007268689, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9562109375, + "pass@2": 0.9830424458661438, + "pass@4": 0.9898723605174348, + "pass@8": 0.9928929459293645, + "pass@16": 0.9952866834531895, + "pass@32": 0.9972931470222736, + "pass@64": 0.9985960892918249, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.8663046875, + "pass@2": 0.9220928887795296, + "pass@4": 0.9506338244281964, + "pass@8": 0.9675263429637743, + "pass@16": 0.9781564796497898, + "pass@32": 0.9851142696937852, + "pass@64": 0.9903520557897221, + "pass@128": 0.995 + }, + "12": { + "pass@1": 0.776015625, + "pass@2": 0.8638099163385842, + "pass@4": 0.9110736210161228, + "pass@8": 0.9377246430833253, + "pass@16": 0.9543967079760017, + "pass@32": 0.9664445886852736, + "pass@64": 0.9755977359015238, + "pass@128": 0.982 + }, + "13": { + "pass@1": 0.570796875, + "pass@2": 0.6869044045275595, + "pass@4": 0.7703089637232837, + "pass@8": 0.8289863404139657, + "pass@16": 0.8706306155014618, + "pass@32": 0.9017770640160171, + "pass@64": 0.9262811180583888, + "pass@128": 0.946 + }, + "14": { + "pass@1": 0.465671875, + "pass@2": 0.5816224163385827, + "pass@4": 0.6775207412823394, + "pass@8": 0.7499487856814693, + "pass@16": 0.802954341064699, + "pass@32": 0.8430218421609508, + "pass@64": 0.8751080533100261, + "pass@128": 0.901 + }, + "15": { + "pass@1": 0.36846875, + "pass@2": 0.47901058070866176, + "pass@4": 0.5728318718597668, + "pass@8": 0.6493805015717913, + "pass@16": 0.7133682179646545, + "pass@32": 0.7662561068084073, + "pass@64": 0.8078740144258999, + "pass@128": 0.837 + }, + "16": { + "pass@1": 0.2422890625, + "pass@2": 0.32881323818897656, + "pass@4": 0.41162225215598025, + "pass@8": 0.48517711191976115, + "pass@16": 0.5477937709328996, + "pass@32": 0.6006260501592771, + "pass@64": 0.6451632488133172, + "pass@128": 0.681 + }, + "17": { + "pass@1": 0.205359375, + "pass@2": 0.28227411417322856, + "pass@4": 0.35604794225721725, + "pass@8": 0.42237966102226454, + "pass@16": 0.48057563857933505, + "pass@32": 0.530299008710456, + "pass@64": 0.5723552718529699, + "pass@128": 0.607 + }, + "18": { + "pass@1": 0.186046875, + "pass@2": 0.26339591535433055, + "pass@4": 0.34232630324334395, + "pass@8": 0.4130072150998003, + "pass@16": 0.4705819052420476, + "pass@32": 0.516768489074943, + "pass@64": 0.5562738374202079, + "pass@128": 0.591 + }, + "19": { + "pass@1": 0.15340625, + "pass@2": 0.2179523868110237, + "pass@4": 0.2843356102362201, + "pass@8": 0.3457563272501176, + "pass@16": 0.4002832993363438, + "pass@32": 0.44730833071019604, + "pass@64": 0.48591384451975916, + "pass@128": 0.516 + }, + "20": { + "pass@1": 0.1458046875, + "pass@2": 0.2112641486220472, + "pass@4": 0.2810926570116231, + "pass@8": 0.34766492117099124, + "pass@16": 0.4047156312357237, + "pass@32": 0.4508767636177382, + "pass@64": 0.48942507365937193, + "pass@128": 0.521 + } + }, + "per_op_accuracy": { + "10": 0.9031875, + "2": 0.998984375, + "3": 0.9967421875, + "4": 0.99365625, + "5": 0.9736484375, + "6": 0.99153125, + "7": 0.98465625, + "8": 0.970953125, + "9": 0.9562109375, + "11": 0.8663046875, + "12": 0.776015625, + "13": 0.570796875, + "14": 0.465671875, + "15": 0.36846875, + "16": 0.2422890625, + "17": 0.205359375, + "18": 0.186046875, + "19": 0.15340625, + "20": 0.1458046875 + }, + "per_op_avg_response_len": { + "10": 280.954671875, + "2": 115.8335, + "3": 145.085, + "4": 160.184578125, + "5": 174.1452421875, + "6": 199.4692734375, + "7": 224.743640625, + "8": 244.058078125, + "9": 260.015828125, + "11": 297.5433046875, + "12": 306.14871875, + "13": 317.1943125, + "14": 328.4833515625, + "15": 332.1365, + "16": 322.0154765625, + "17": 329.8279375, + "18": 333.6520390625, + "19": 323.0419375, + "20": 328.5686640625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.954671875, + "2": 115.8335, + "3": 145.085, + "4": 160.184578125, + "5": 174.1452421875, + "6": 199.4692734375, + "7": 224.743640625, + "8": 244.058078125, + "9": 260.015828125, + "11": 297.5433046875, + "12": 306.14871875, + "13": 317.1943125, + "14": 328.4833515625, + "15": 332.1365, + "16": 322.0154765625, + "17": 329.8279375, + "18": 333.6520390625, + "19": 323.0419375, + "20": 328.5686640625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 540343, + "answer_accuracy": 0.6673142092159342, + "avg_response_len": 252.07113746838445, + "resp_tokens_sum": 204109058, + "pass_at_k": { + "pass@1": 0.6673142092159342, + "pass@2": 0.7184972467083713, + "pass@4": 0.7582710489269354, + "pass@8": 0.7898036372954599, + "pass@16": 0.8149666354226288, + "pass@32": 0.8356683187114132, + "pass@64": 0.8536326257414687, + "pass@128": 0.8695858362314258 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9078401548672567, + "pass@2": 0.9451660598564567, + "pass@4": 0.9627415558121603, + "pass@8": 0.9724228068912785, + "pass@16": 0.9786434853777529, + "pass@32": 0.9833303999293327, + "pass@64": 0.9874800169721869, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9974913990825688, + "pass@2": 0.998993925931277, + "pass@4": 0.999781667199857, + "pass@8": 0.9999876849218283, + "pass@16": 0.9999999719746717, + "pass@32": 0.9999999999999853, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9863525390625, + "pass@2": 0.9885619156003939, + "pass@4": 0.9898306295697413, + "pass@8": 0.9915581459135234, + "pass@16": 0.9938875771099299, + "pass@32": 0.996529537062657, + "pass@64": 0.9989487041172099, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9679315476190476, + "pass@2": 0.9789893450818652, + "pass@4": 0.9852945253867076, + "pass@8": 0.9903654763094896, + "pass@16": 0.9944361907939756, + "pass@32": 0.9969451642580184, + "pass@64": 0.9983683333808681, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9894975673652695, + "pass@2": 0.996165027229007, + "pass@4": 0.998533353147473, + "pass@8": 0.999620189417031, + "pass@16": 0.9999636861019171, + "pass@32": 0.9999998029761207, + "pass@64": 0.999999999999911, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860392011834319, + "pass@2": 0.9959920124167174, + "pass@4": 0.9977027054310518, + "pass@8": 0.9983763394978522, + "pass@16": 0.9991472297394309, + "pass@32": 0.9997987731838602, + "pass@64": 0.999995725631176, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9701522435897436, + "pass@2": 0.9889413612961848, + "pass@4": 0.9943698822502955, + "pass@8": 0.9962621361835249, + "pass@16": 0.9970686948727937, + "pass@32": 0.9975932975941004, + "pass@64": 0.998397435811393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9600132042253521, + "pass@2": 0.9878528058112463, + "pass@4": 0.9946557779749362, + "pass@8": 0.9974029094078771, + "pass@16": 0.9989479245309896, + "pass@32": 0.9997375771890578, + "pass@64": 0.9999912792521506, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8549043909348442, + "pass@2": 0.9105383272735389, + "pass@4": 0.9394676640101288, + "pass@8": 0.9567641672474922, + "pass@16": 0.9689281622131964, + "pass@32": 0.9792995108322007, + "pass@64": 0.989377221593619, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.7604387358757062, + "pass@2": 0.8512497080608568, + "pass@4": 0.9022677093859018, + "pass@8": 0.9306185243521176, + "pass@16": 0.947629076857396, + "pass@32": 0.9613728628093047, + "pass@64": 0.9722807480889516, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.6151792711598746, + "pass@2": 0.7272553717325303, + "pass@4": 0.8012909736204601, + "pass@8": 0.8513849982189186, + "pass@16": 0.8858777402360082, + "pass@32": 0.9113844472956131, + "pass@64": 0.9327076612903673, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.4572381230529595, + "pass@2": 0.5804936437559792, + "pass@4": 0.6836558835519382, + "pass@8": 0.7584661690159947, + "pass@16": 0.8090695107690499, + "pass@32": 0.8452244630136615, + "pass@64": 0.8744698913107548, + "pass@128": 0.9034267912772586 + }, + "15": { + "pass@1": 0.3591943641618497, + "pass@2": 0.46582035694779467, + "pass@4": 0.557234378759013, + "pass@8": 0.6333810374573632, + "pass@16": 0.6991402754904769, + "pass@32": 0.7537907497738942, + "pass@64": 0.7989936415700499, + "pass@128": 0.838150289017341 + }, + "16": { + "pass@1": 0.2480594758064516, + "pass@2": 0.3369277051054099, + "pass@4": 0.41882028587152403, + "pass@8": 0.48785088132261173, + "pass@16": 0.5464977216622237, + "pass@32": 0.5997006252636178, + "pass@64": 0.6464878664953922, + "pass@128": 0.6838709677419355 + }, + "17": { + "pass@1": 0.19413836705202311, + "pass@2": 0.2639743866915478, + "pass@4": 0.3301886702384742, + "pass@8": 0.3913913054415934, + "pass@16": 0.44674849852516724, + "pass@32": 0.49537678199969953, + "pass@64": 0.5397606358748235, + "pass@128": 0.5809248554913294 + }, + "18": { + "pass@1": 0.16417410714285716, + "pass@2": 0.23900625703037112, + "pass@4": 0.31737553055868006, + "pass@8": 0.3877756897146819, + "pass@16": 0.44484807064643955, + "pass@32": 0.4911672177360723, + "pass@64": 0.5336450417048393, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.1477805930930931, + "pass@2": 0.21373120167411505, + "pass@4": 0.2819873895267596, + "pass@8": 0.3444790109299255, + "pass@16": 0.398624559518731, + "pass@32": 0.4448169543649647, + "pass@64": 0.48421600118684854, + "pass@128": 0.5165165165165165 + }, + "20": { + "pass@1": 0.15219125375375375, + "pass@2": 0.21601005533092138, + "pass@4": 0.2833787189889551, + "pass@8": 0.34737669283247447, + "pass@16": 0.40206167741449067, + "pass@32": 0.4470107741738278, + "pass@64": 0.4875770578788106, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 550391, + "answer_accuracy": 0.6702930144193296, + "avg_response_len": 260.58617132696804, + "resp_tokens_sum": 213972517, + "pass_at_k": { + "pass@1": 0.6702930144193296, + "pass@2": 0.7203984686174745, + "pass@4": 0.7605172725681272, + "pass@8": 0.7936184035372785, + "pass@16": 0.8205157430956861, + "pass@32": 0.8418489124402874, + "pass@64": 0.8581997269526279, + "pass@128": 0.8693686671862821 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8947948619631901, + "pass@2": 0.9342487228877832, + "pass@4": 0.9553914430174755, + "pass@8": 0.9685340029247087, + "pass@16": 0.9778976377826292, + "pass@32": 0.9854515349169342, + "pass@64": 0.991566746760942, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9943181818181818, + "pass@2": 0.9965987383679314, + "pass@4": 0.9986907074115734, + "pass@8": 0.9997900403945096, + "pass@16": 0.999993778423042, + "pass@32": 0.9999999968162022, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986825073964497, + "pass@2": 0.9998864324651727, + "pass@4": 0.9999964917477031, + "pass@8": 0.999999997761824, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9817028985507247, + "pass@2": 0.9890858581536004, + "pass@4": 0.9919462368834332, + "pass@8": 0.9941619355764031, + "pass@16": 0.9961781821357537, + "pass@32": 0.9975886476682482, + "pass@64": 0.9985464572309225, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9912249283667621, + "pass@2": 0.9971071734765248, + "pass@4": 0.998236967692792, + "pass@8": 0.9989742809016088, + "pass@16": 0.9996596953889924, + "pass@32": 0.9999716397876668, + "pass@64": 0.9999999654169256, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866781069364162, + "pass@2": 0.9940098595876383, + "pass@4": 0.9954577830950322, + "pass@8": 0.9965385997356417, + "pass@16": 0.997911202425725, + "pass@32": 0.9992275285274425, + "pass@64": 0.9999158252269373, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9713248239436619, + "pass@2": 0.9873073084174341, + "pass@4": 0.9928848371066289, + "pass@8": 0.9962645987397817, + "pass@16": 0.9977159520362265, + "pass@32": 0.9984178461141198, + "pass@64": 0.9993013196994364, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9549728667664671, + "pass@2": 0.9786595607996605, + "pass@4": 0.9846184301812573, + "pass@8": 0.9881064318849376, + "pass@16": 0.9915007486161633, + "pass@32": 0.9943169158716935, + "pass@64": 0.996171853688097, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.8858951430722891, + "pass@2": 0.9403234246039275, + "pass@4": 0.9654911386264966, + "pass@8": 0.9802103258206285, + "pass@16": 0.9892633104282124, + "pass@32": 0.9943230187012654, + "pass@64": 0.9966323831791278, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.770865987460815, + "pass@2": 0.8529438081603437, + "pass@4": 0.8969163801625106, + "pass@8": 0.9239001333151642, + "pass@16": 0.944273209688222, + "pass@32": 0.9607568063656658, + "pass@64": 0.9735690051847953, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.55249912464986, + "pass@2": 0.670897295926244, + "pass@4": 0.7573695231373386, + "pass@8": 0.8195883022487538, + "pass@16": 0.8639400641786898, + "pass@32": 0.8959443632193429, + "pass@64": 0.9194230288568553, + "pass@128": 0.938375350140056 + }, + "14": { + "pass@1": 0.4458607456140351, + "pass@2": 0.5551821441727679, + "pass@4": 0.6490961617736377, + "pass@8": 0.7264317974040324, + "pass@16": 0.7878208627637026, + "pass@32": 0.8337530255493398, + "pass@64": 0.8650871772849, + "pass@128": 0.8830409356725146 + }, + "15": { + "pass@1": 0.3692708333333333, + "pass@2": 0.4844406167979, + "pass@4": 0.5807708542114053, + "pass@8": 0.6566659618047733, + "pass@16": 0.71822453037185, + "pass@32": 0.7690809020128775, + "pass@64": 0.8074737398378564, + "pass@128": 0.8272727272727273 + }, + "16": { + "pass@1": 0.2302795584045584, + "pass@2": 0.3132508075913587, + "pass@4": 0.3953974561940442, + "pass@8": 0.4699005180824398, + "pass@16": 0.5316027123774437, + "pass@32": 0.5810105949043947, + "pass@64": 0.6208832644017411, + "pass@128": 0.6495726495726496 + }, + "17": { + "pass@1": 0.21753687888198758, + "pass@2": 0.29977732063383367, + "pass@4": 0.3802775075281738, + "pass@8": 0.4530973687186417, + "pass@16": 0.5146178409330402, + "pass@32": 0.5636207169721692, + "pass@64": 0.6026909514942224, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.19386488970588237, + "pass@2": 0.27067797591477505, + "pass@4": 0.34721876282008873, + "pass@8": 0.4179357468470566, + "pass@16": 0.4785538356301472, + "pass@32": 0.5267746944091646, + "pass@64": 0.5637993560978994, + "pass@128": 0.5882352941176471 + }, + "19": { + "pass@1": 0.14381045386904762, + "pass@2": 0.20448420509936244, + "pass@4": 0.26923672822147243, + "pass@8": 0.3304539572282448, + "pass@16": 0.3857649133616538, + "pass@32": 0.4348844940956561, + "pass@64": 0.4749150124076846, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.140812125748503, + "pass@2": 0.2086588388184262, + "pass@4": 0.2810939201462094, + "pass@8": 0.3485268808493757, + "pass@16": 0.4055889275550898, + "pass@32": 0.45131218202379575, + "pass@64": 0.4876867028458811, + "pass@128": 0.5149700598802395 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 541232, + "answer_accuracy": 0.6755671832561112, + "avg_response_len": 280.69016616072855, + "resp_tokens_sum": 224875488, + "pass_at_k": { + "pass@1": 0.6755671832561112, + "pass@2": 0.7267714805640489, + "pass@4": 0.7671935018298011, + "pass@8": 0.7993583792277895, + "pass@16": 0.8249747156351722, + "pass@32": 0.8456421305459204, + "pass@64": 0.8626208528041777, + "pass@128": 0.8763380731746285 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9066464552238805, + "pass@2": 0.9443434892466809, + "pass@4": 0.9631634325560046, + "pass@8": 0.9746091114318679, + "pass@16": 0.981989067498895, + "pass@32": 0.9872894382051791, + "pass@64": 0.9904128899753277, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983600583090378, + "pass@2": 0.9995706451872088, + "pass@4": 0.9999545667214339, + "pass@8": 0.9999994678161623, + "pass@16": 0.9999999999690319, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9955226608187134, + "pass@2": 0.996956229566699, + "pass@4": 0.9970747003664014, + "pass@8": 0.9970760232358671, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9707720588235295, + "pass@2": 0.9819418712366834, + "pass@4": 0.9875488412294049, + "pass@8": 0.9906721170939422, + "pass@16": 0.992369679677989, + "pass@32": 0.9934273310473238, + "pass@64": 0.9940328828017424, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.994011238170347, + "pass@2": 0.9966948384212229, + "pass@4": 0.9969382767414325, + "pass@8": 0.9970425818238966, + "pass@16": 0.9972397476340691, + "pass@32": 0.9976340694006309, + "pass@64": 0.998422712933754, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9809632120253164, + "pass@2": 0.9907531707864047, + "pass@4": 0.9939196128964891, + "pass@8": 0.9955120902657174, + "pass@16": 0.9963967136338048, + "pass@32": 0.996791897857649, + "pass@64": 0.9968353559542914, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9713072447447447, + "pass@2": 0.9862585272280152, + "pass@4": 0.9918887061977614, + "pass@8": 0.9948537929562266, + "pass@16": 0.9965289626590625, + "pass@32": 0.9975971169999002, + "pass@64": 0.9984964932745174, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9532003617363344, + "pass@2": 0.9822585417879847, + "pass@4": 0.9900546741343824, + "pass@8": 0.9928854174919578, + "pass@16": 0.9951733768710284, + "pass@32": 0.9976992193537383, + "pass@64": 0.9996070290208584, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8584325396825396, + "pass@2": 0.9158269278840144, + "pass@4": 0.947487876217854, + "pass@8": 0.9662182976284491, + "pass@16": 0.976791807385288, + "pass@32": 0.9819247624165014, + "pass@64": 0.9848252233387427, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.7979023318042814, + "pass@2": 0.8880074767030265, + "pass@4": 0.934417512604502, + "pass@8": 0.9589038010248868, + "pass@16": 0.9715989629294857, + "pass@32": 0.9774837125997929, + "pass@64": 0.9811677016027062, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5472608024691358, + "pass@2": 0.6648135996889273, + "pass@4": 0.7540624178343447, + "pass@8": 0.817288586664587, + "pass@16": 0.8629907822666152, + "pass@32": 0.8987447149981824, + "pass@64": 0.927510348163161, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.4938102744807122, + "pass@2": 0.6095301584149166, + "pass@4": 0.7005231909067747, + "pass@8": 0.7657016935167862, + "pass@16": 0.8124875164470288, + "pass@32": 0.8503301329249001, + "pass@64": 0.8858854705277092, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.37755594135802467, + "pass@2": 0.48756584463400393, + "pass@4": 0.5814024534548923, + "pass@8": 0.6590459728887901, + "pass@16": 0.7236160111797545, + "pass@32": 0.7766907707481191, + "pass@64": 0.8177650626424972, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.24944690265486727, + "pass@2": 0.33750624230599463, + "pass@4": 0.4218391044592877, + "pass@8": 0.49854943027379045, + "pass@16": 0.5657431421036233, + "pass@32": 0.6217821165667055, + "pass@64": 0.6690913993944967, + "pass@128": 0.7109144542772862 + }, + "17": { + "pass@1": 0.20524284638554216, + "pass@2": 0.2843695154634285, + "pass@4": 0.3594979064175711, + "pass@8": 0.4248822789520208, + "pass@16": 0.48281238948550975, + "pass@32": 0.5343758170889205, + "pass@64": 0.5769024260817495, + "pass@128": 0.608433734939759 + }, + "18": { + "pass@1": 0.20216733870967743, + "pass@2": 0.28294585026670055, + "pass@4": 0.36513060706121425, + "pass@8": 0.4360889347472986, + "pass@16": 0.49089282710175347, + "pass@32": 0.5346986024780715, + "pass@64": 0.5735686830652535, + "pass@128": 0.6096774193548387 + }, + "19": { + "pass@1": 0.16880664652567975, + "pass@2": 0.23587070616361772, + "pass@4": 0.30202498139394224, + "pass@8": 0.3625748851412769, + "pass@16": 0.416689758692117, + "pass@32": 0.4624262685514275, + "pass@64": 0.49878692433702804, + "pass@128": 0.5287009063444109 + }, + "20": { + "pass@1": 0.14442567567567569, + "pass@2": 0.20913137546995805, + "pass@4": 0.27880532810650926, + "pass@8": 0.34708860136362146, + "pass@16": 0.40649366622612243, + "pass@32": 0.4543060270928111, + "pass@64": 0.493016680586197, + "pass@128": 0.5255255255255256 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep1-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1f0f86763be975f8ac060973199a1547a7527b92 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block8/composition/autoregressive", "results_dir": "results/SDAR-100M-ep1-block8/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block8/composition/autoregressive/checkpoint-21546_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 996291, "value_mismatch": 33225, "dependency_mismatch": 150212, "answer_mismatch": 29492, "extra_nodes": 23474, "missing_nodes": 19445, "perfect_with_extra": 1603, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 327351, "dependency_mismatch": 49818, "value_mismatch": 11432, "answer_mismatch": 10143, "extra_nodes": 8546, "perfect_with_extra": 660, "missing_nodes": 6557}, "teachers_in_school": {"total": 390656, "perfect": 338347, "value_mismatch": 10569, "answer_mismatch": 9705, "dependency_mismatch": 50603, "extra_nodes": 7257, "perfect_with_extra": 394, "missing_nodes": 6133}, "movie_festival_awards": {"total": 382336, "perfect": 330593, "dependency_mismatch": 49791, "value_mismatch": 11224, "answer_mismatch": 9644, "extra_nodes": 7671, "missing_nodes": 6755, "perfect_with_extra": 549}}, "results_path": "results/SDAR-100M-ep1-block8/composition/autoregressive/graphs/checkpoint-21546_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block8/composition/autoregressive/checkpoint-21546_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 199073, "value_mismatch": 844664, "dependency_mismatch": 966335, "answer_mismatch": 770536, "extra_nodes": 88262, "missing_nodes": 816971, "perfect_with_extra": 198, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 284675, "dependency_mismatch": 325194, "answer_mismatch": 259235, "extra_nodes": 32237, "missing_nodes": 281041, "perfect": 67553, "perfect_with_extra": 35}, "movie_festival_awards": {"total": 418816, "perfect": 66722, "dependency_mismatch": 318657, "value_mismatch": 275946, "answer_mismatch": 250282, "extra_nodes": 28262, "missing_nodes": 259754, "perfect_with_extra": 30}, "teachers_in_school": {"total": 430464, "perfect": 64798, "value_mismatch": 284043, "dependency_mismatch": 322484, "answer_mismatch": 261019, "extra_nodes": 27763, "missing_nodes": 276176, "perfect_with_extra": 133}}, "results_path": "results/SDAR-100M-ep1-block8/composition/autoregressive/graphs/checkpoint-21546_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1195364, "value_mismatch": 877889, "dependency_mismatch": 1116547, "answer_mismatch": 800028, "extra_nodes": 111736, "missing_nodes": 836416, "perfect_with_extra": 1801, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block8/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 394904, "dependency_mismatch": 375012, "value_mismatch": 296107, "answer_mismatch": 269378, "extra_nodes": 40783, "perfect_with_extra": 695, "missing_nodes": 287598}, "teachers_in_school": {"total": 821120, "perfect": 403145, "value_mismatch": 294612, "answer_mismatch": 270724, "dependency_mismatch": 373087, "extra_nodes": 35020, "perfect_with_extra": 527, "missing_nodes": 282309}, "movie_festival_awards": {"total": 801152, "perfect": 397315, "dependency_mismatch": 368448, "value_mismatch": 287170, "answer_mismatch": 259926, "extra_nodes": 35933, "missing_nodes": 266509, "perfect_with_extra": 579}}}} diff --git a/SDAR-100M-ep1-block8/composition/autoregressive/graphs/results.json b/SDAR-100M-ep1-block8/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..42f25541e57ce1fca1993d86e4645a618f7d9049 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21546_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.975, + "pass@2": 0.984375, + "pass@4": 0.9875, + "pass@8": 0.9875, + "pass@16": 0.990625, + "pass@32": 0.996875, + "pass@64": 0.996875, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9238095238095239, + "pass@2": 0.9523809523809523, + "pass@4": 0.9777777777777777, + "pass@8": 0.9841269841269841, + "pass@16": 0.9904761904761905, + "pass@32": 0.9936507936507937, + "pass@64": 0.9968253968253968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9461077844311377, + "pass@2": 0.9790419161676647, + "pass@4": 0.9880239520958084, + "pass@8": 0.9910179640718563, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8698224852071006, + "pass@2": 0.908284023668639, + "pass@4": 0.9437869822485208, + "pass@8": 0.9644970414201184, + "pass@16": 0.9940828402366864, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.7756410256410257, + "pass@2": 0.8974358974358975, + "pass@4": 0.9487179487179487, + "pass@8": 0.9743589743589743, + "pass@16": 0.9903846153846154, + "pass@32": 0.9967948717948718, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7380281690140845, + "pass@2": 0.8169014084507042, + "pass@4": 0.8873239436619719, + "pass@8": 0.9408450704225352, + "pass@16": 0.9943661971830986, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6106194690265486, + "pass@2": 0.6607669616519174, + "pass@4": 0.7522123893805309, + "pass@8": 0.8495575221238938, + "pass@16": 0.887905604719764, + "pass@32": 0.9203539823008849, + "pass@64": 0.9262536873156342, + "pass@128": 0.9410029498525073 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.975, + "5": 0.9238095238095239, + "6": 0.9461077844311377, + "7": 0.8698224852071006, + "8": 0.7756410256410257, + "9": 0.7380281690140845, + "10": 0.6106194690265486 + }, + "pass@2": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.984375, + "5": 0.9523809523809523, + "6": 0.9790419161676647, + "7": 0.908284023668639, + "8": 0.8974358974358975, + "9": 0.8169014084507042, + "10": 0.6607669616519174 + }, + "pass@4": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.9875, + "5": 0.9777777777777777, + "6": 0.9880239520958084, + "7": 0.9437869822485208, + "8": 0.9487179487179487, + "9": 0.8873239436619719, + "10": 0.7522123893805309 + }, + "pass@8": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.9875, + "5": 0.9841269841269841, + "6": 0.9910179640718563, + "7": 0.9644970414201184, + "8": 0.9743589743589743, + "9": 0.9408450704225352, + "10": 0.8495575221238938 + }, + "pass@16": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.990625, + "5": 0.9904761904761905, + "6": 0.9970059880239521, + "7": 0.9940828402366864, + "8": 0.9903846153846154, + "9": 0.9943661971830986, + "10": 0.887905604719764 + }, + "pass@32": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9203539823008849 + }, + "pass@64": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.996875, + "5": 0.9968253968253968, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 1.0, + "10": 0.9262536873156342 + }, + "pass@128": { + "2": 0.9968847352024922, + "3": 1.0, + "4": 0.996875, + "5": 1.0, + "6": 1.0, + "7": 0.9970414201183432, + "8": 1.0, + "9": 1.0, + "10": 0.9410029498525073 + } + }, + "total_pass_at_k": { + "pass@1": 0.8682877406281662, + "pass@2": 0.9084768659236744, + "pass@4": 0.9408983451536643, + "pass@8": 0.9645390070921985, + "pass@16": 0.9821006416751098, + "pass@32": 0.9888551165146909, + "pass@64": 0.9902060114826072, + "pass@128": 0.9922323539344816 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9824561403508771, + "pass@2": 0.9912280701754386, + "pass@4": 0.9970760233918129, + "pass@8": 0.9970760233918129, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9205882352941176, + "pass@2": 0.9588235294117647, + "pass@4": 0.9764705882352941, + "pass@8": 0.9911764705882353, + "pass@16": 0.9941176470588236, + "pass@32": 0.9941176470588236, + "pass@64": 0.9941176470588236, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.9589905362776026, + "pass@2": 0.9716088328075709, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 0.9968454258675079, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8512658227848101, + "pass@2": 0.8955696202531646, + "pass@4": 0.9588607594936709, + "pass@8": 0.9746835443037974, + "pass@16": 0.9873417721518988, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.7507507507507507, + "pass@2": 0.8618618618618619, + "pass@4": 0.9279279279279279, + "pass@8": 0.978978978978979, + "pass@16": 0.993993993993994, + "pass@32": 0.996996996996997, + "pass@64": 0.996996996996997, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.729903536977492, + "pass@2": 0.8102893890675241, + "pass@4": 0.8778135048231511, + "pass@8": 0.9356913183279743, + "pass@16": 0.9646302250803859, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5552238805970149, + "pass@2": 0.6358208955223881, + "pass@4": 0.7343283582089553, + "pass@8": 0.835820895522388, + "pass@16": 0.8626865671641791, + "pass@32": 0.9223880597014925, + "pass@64": 0.9313432835820895, + "pass@128": 0.9522388059701492 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9824561403508771, + "5": 0.9205882352941176, + "6": 0.9589905362776026, + "7": 0.8512658227848101, + "8": 0.7507507507507507, + "9": 0.729903536977492, + "10": 0.5552238805970149 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9588235294117647, + "6": 0.9716088328075709, + "7": 0.8955696202531646, + "8": 0.8618618618618619, + "9": 0.8102893890675241, + "10": 0.6358208955223881 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9764705882352941, + "6": 0.9842271293375394, + "7": 0.9588607594936709, + "8": 0.9279279279279279, + "9": 0.8778135048231511, + "10": 0.7343283582089553 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9936908517350158, + "7": 0.9746835443037974, + "8": 0.978978978978979, + "9": 0.9356913183279743, + "10": 0.835820895522388 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 0.9968454258675079, + "7": 0.9873417721518988, + "8": 0.993993993993994, + "9": 0.9646302250803859, + "10": 0.8626865671641791 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9223880597014925 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9936708860759493, + "8": 0.996996996996997, + "9": 1.0, + "10": 0.9313432835820895 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9941176470588236, + "6": 1.0, + "7": 0.9936708860759493, + "8": 1.0, + "9": 1.0, + "10": 0.9522388059701492 + } + }, + "total_pass_at_k": { + "pass@1": 0.863073317710077, + "pass@2": 0.9042517576163375, + "pass@4": 0.9404084365584198, + "pass@8": 0.9678607298292601, + "pass@16": 0.9775694676933377, + "pass@32": 0.9889521258788082, + "pass@64": 0.9902912621359223, + "pass@128": 0.9929695346501507 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9733727810650887, + "pass@2": 0.9911242603550295, + "pass@4": 0.9970414201183432, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9391304347826087, + "pass@2": 0.9797101449275363, + "pass@4": 0.9884057971014493, + "pass@8": 0.9971014492753624, + "pass@16": 0.9971014492753624, + "pass@32": 0.9971014492753624, + "pass@64": 0.9971014492753624, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9684813753581661, + "pass@2": 0.9828080229226361, + "pass@4": 0.9914040114613181, + "pass@8": 0.994269340974212, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.930635838150289, + "pass@4": 0.9624277456647399, + "pass@8": 0.9942196531791907, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7915492957746478, + "pass@2": 0.8788732394366198, + "pass@4": 0.9323943661971831, + "pass@8": 0.9774647887323944, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7005988023952096, + "pass@2": 0.7904191616766467, + "pass@4": 0.874251497005988, + "pass@8": 0.9431137724550899, + "pass@16": 0.9700598802395209, + "pass@32": 0.9880239520958084, + "pass@64": 0.9940119760479041, + "pass@128": 0.9970059880239521 + }, + "10": { + "pass@1": 0.5858895705521472, + "pass@2": 0.6993865030674846, + "pass@4": 0.7699386503067485, + "pass@8": 0.8374233128834356, + "pass@16": 0.8773006134969326, + "pass@32": 0.911042944785276, + "pass@64": 0.941717791411043, + "pass@128": 0.9570552147239264 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9733727810650887, + "5": 0.9391304347826087, + "6": 0.9684813753581661, + "7": 0.8641618497109826, + "8": 0.7915492957746478, + "9": 0.7005988023952096, + "10": 0.5858895705521472 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9911242603550295, + "5": 0.9797101449275363, + "6": 0.9828080229226361, + "7": 0.930635838150289, + "8": 0.8788732394366198, + "9": 0.7904191616766467, + "10": 0.6993865030674846 + }, + "pass@4": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9970414201183432, + "5": 0.9884057971014493, + "6": 0.9914040114613181, + "7": 0.9624277456647399, + "8": 0.9323943661971831, + "9": 0.874251497005988, + "10": 0.7699386503067485 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.994269340974212, + "7": 0.9942196531791907, + "8": 0.9774647887323944, + "9": 0.9431137724550899, + "10": 0.8374233128834356 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9700598802395209, + "10": 0.8773006134969326 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.911042944785276 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9940119760479041, + "10": 0.941717791411043 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9970059880239521, + "10": 0.9570552147239264 + } + }, + "total_pass_at_k": { + "pass@1": 0.8695937090432503, + "pass@2": 0.9174311926605505, + "pass@4": 0.9465923984272608, + "pass@8": 0.9721494102228048, + "pass@16": 0.9829619921363041, + "pass@32": 0.9885321100917431, + "pass@64": 0.9927916120576671, + "pass@128": 0.9950851900393185 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "3": { + "pass@1": 0.999, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.977, + "pass@2": 0.989, + "pass@4": 0.994, + "pass@8": 0.995, + "pass@16": 0.996, + "pass@32": 0.998, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.928, + "pass@2": 0.964, + "pass@4": 0.981, + "pass@8": 0.991, + "pass@16": 0.994, + "pass@32": 0.995, + "pass@64": 0.996, + "pass@128": 0.998 + }, + "6": { + "pass@1": 0.958, + "pass@2": 0.978, + "pass@4": 0.988, + "pass@8": 0.993, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.862, + "pass@2": 0.912, + "pass@4": 0.955, + "pass@8": 0.978, + "pass@16": 0.994, + "pass@32": 0.997, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.773, + "pass@2": 0.879, + "pass@4": 0.936, + "pass@8": 0.977, + "pass@16": 0.994, + "pass@32": 0.997, + "pass@64": 0.999, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.723, + "pass@2": 0.806, + "pass@4": 0.88, + "pass@8": 0.94, + "pass@16": 0.977, + "pass@32": 0.995, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.584, + "pass@2": 0.665, + "pass@4": 0.752, + "pass@8": 0.841, + "pass@16": 0.876, + "pass@32": 0.918, + "pass@64": 0.933, + "pass@128": 0.95 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.999, + "3": 0.999, + "4": 0.977, + "5": 0.928, + "6": 0.958, + "7": 0.862, + "8": 0.773, + "9": 0.723, + "10": 0.584 + }, + "pass@2": { + "2": 0.999, + "3": 0.999, + "4": 0.989, + "5": 0.964, + "6": 0.978, + "7": 0.912, + "8": 0.879, + "9": 0.806, + "10": 0.665 + }, + "pass@4": { + "2": 0.999, + "3": 0.999, + "4": 0.994, + "5": 0.981, + "6": 0.988, + "7": 0.955, + "8": 0.936, + "9": 0.88, + "10": 0.752 + }, + "pass@8": { + "2": 0.999, + "3": 1.0, + "4": 0.995, + "5": 0.991, + "6": 0.993, + "7": 0.978, + "8": 0.977, + "9": 0.94, + "10": 0.841 + }, + "pass@16": { + "2": 0.999, + "3": 1.0, + "4": 0.996, + "5": 0.994, + "6": 0.998, + "7": 0.994, + "8": 0.994, + "9": 0.977, + "10": 0.876 + }, + "pass@32": { + "2": 0.999, + "3": 1.0, + "4": 0.998, + "5": 0.995, + "6": 1.0, + "7": 0.997, + "8": 0.997, + "9": 0.995, + "10": 0.918 + }, + "pass@64": { + "2": 0.999, + "3": 1.0, + "4": 0.998, + "5": 0.996, + "6": 1.0, + "7": 0.997, + "8": 0.999, + "9": 0.998, + "10": 0.933 + }, + "pass@128": { + "2": 0.999, + "3": 1.0, + "4": 0.998, + "5": 0.998, + "6": 1.0, + "7": 0.997, + "8": 1.0, + "9": 0.999, + "10": 0.95 + } + }, + "total_pass_at_k": { + "pass@1": 0.867, + "pass@2": 0.9101111111111111, + "pass@4": 0.9426666666666667, + "pass@8": 0.9682222222222222, + "pass@16": 0.9808888888888889, + "pass@32": 0.9887777777777778, + "pass@64": 0.9911111111111112, + "pass@128": 0.9934444444444445 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21546_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4730878186968839, + "pass@2": 0.5920679886685553, + "pass@4": 0.7025495750708215, + "pass@8": 0.7677053824362606, + "pass@16": 0.8356940509915014, + "pass@32": 0.8725212464589235, + "pass@64": 0.9121813031161473, + "pass@128": 0.9490084985835694 + }, + "12": { + "pass@1": 0.4915254237288136, + "pass@2": 0.5649717514124294, + "pass@4": 0.6610169491525424, + "pass@8": 0.7372881355932204, + "pass@16": 0.7768361581920904, + "pass@32": 0.8220338983050848, + "pass@64": 0.8418079096045198, + "pass@128": 0.8615819209039548 + }, + "13": { + "pass@1": 0.2727272727272727, + "pass@2": 0.3322884012539185, + "pass@4": 0.4200626959247649, + "pass@8": 0.4890282131661442, + "pass@16": 0.5611285266457681, + "pass@32": 0.6300940438871473, + "pass@64": 0.670846394984326, + "pass@128": 0.6990595611285266 + }, + "14": { + "pass@1": 0.13707165109034267, + "pass@2": 0.21806853582554517, + "pass@4": 0.27414330218068533, + "pass@8": 0.3302180685358255, + "pass@16": 0.3894080996884735, + "pass@32": 0.43613707165109034, + "pass@64": 0.4797507788161994, + "pass@128": 0.5482866043613707 + }, + "15": { + "pass@1": 0.0953757225433526, + "pass@2": 0.13583815028901733, + "pass@4": 0.18497109826589594, + "pass@8": 0.20520231213872833, + "pass@16": 0.2398843930635838, + "pass@32": 0.28034682080924855, + "pass@64": 0.3352601156069364, + "pass@128": 0.3872832369942196 + }, + "16": { + "pass@1": 0.02258064516129032, + "pass@2": 0.02903225806451613, + "pass@4": 0.03225806451612903, + "pass@8": 0.041935483870967745, + "pass@16": 0.05161290322580645, + "pass@32": 0.07096774193548387, + "pass@64": 0.08387096774193549, + "pass@128": 0.0967741935483871 + }, + "17": { + "pass@1": 0.014450867052023121, + "pass@2": 0.017341040462427744, + "pass@4": 0.02601156069364162, + "pass@8": 0.03468208092485549, + "pass@16": 0.04046242774566474, + "pass@32": 0.049132947976878616, + "pass@64": 0.057803468208092484, + "pass@128": 0.07225433526011561 + }, + "18": { + "pass@1": 0.002857142857142857, + "pass@2": 0.002857142857142857, + "pass@4": 0.011428571428571429, + "pass@8": 0.011428571428571429, + "pass@16": 0.014285714285714285, + "pass@32": 0.022857142857142857, + "pass@64": 0.025714285714285714, + "pass@128": 0.02857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.003003003003003003, + "pass@128": 0.003003003003003003 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.003003003003003003, + "pass@64": 0.006006006006006006, + "pass@128": 0.009009009009009009 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4730878186968839, + "12": 0.4915254237288136, + "13": 0.2727272727272727, + "14": 0.13707165109034267, + "15": 0.0953757225433526, + "16": 0.02258064516129032, + "17": 0.014450867052023121, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5920679886685553, + "12": 0.5649717514124294, + "13": 0.3322884012539185, + "14": 0.21806853582554517, + "15": 0.13583815028901733, + "16": 0.02903225806451613, + "17": 0.017341040462427744, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.7025495750708215, + "12": 0.6610169491525424, + "13": 0.4200626959247649, + "14": 0.27414330218068533, + "15": 0.18497109826589594, + "16": 0.03225806451612903, + "17": 0.02601156069364162, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7677053824362606, + "12": 0.7372881355932204, + "13": 0.4890282131661442, + "14": 0.3302180685358255, + "15": 0.20520231213872833, + "16": 0.041935483870967745, + "17": 0.03468208092485549, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8356940509915014, + "12": 0.7768361581920904, + "13": 0.5611285266457681, + "14": 0.3894080996884735, + "15": 0.2398843930635838, + "16": 0.05161290322580645, + "17": 0.04046242774566474, + "18": 0.014285714285714285, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.8725212464589235, + "12": 0.8220338983050848, + "13": 0.6300940438871473, + "14": 0.43613707165109034, + "15": 0.28034682080924855, + "16": 0.07096774193548387, + "17": 0.049132947976878616, + "18": 0.022857142857142857, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@64": { + "11": 0.9121813031161473, + "12": 0.8418079096045198, + "13": 0.670846394984326, + "14": 0.4797507788161994, + "15": 0.3352601156069364, + "16": 0.08387096774193549, + "17": 0.057803468208092484, + "18": 0.025714285714285714, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9490084985835694, + "12": 0.8615819209039548, + "13": 0.6990595611285266, + "14": 0.5482866043613707, + "15": 0.3872832369942196, + "16": 0.0967741935483871, + "17": 0.07225433526011561, + "18": 0.02857142857142857, + "19": 0.003003003003003003, + "20": 0.009009009009009009 + } + }, + "total_pass_at_k": { + "pass@1": 0.15393759286775632, + "pass@2": 0.19257057949479942, + "pass@4": 0.2350668647845468, + "pass@8": 0.26567607726597325, + "pass@16": 0.2947994056463596, + "pass@32": 0.3224368499257058, + "pass@64": 0.3453194650817236, + "pass@128": 0.36909361069836555 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4888888888888889, + "pass@2": 0.5841269841269842, + "pass@4": 0.6825396825396826, + "pass@8": 0.780952380952381, + "pass@16": 0.8698412698412699, + "pass@32": 0.926984126984127, + "pass@64": 0.946031746031746, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.5045871559633027, + "pass@2": 0.5749235474006116, + "pass@4": 0.654434250764526, + "pass@8": 0.7339449541284404, + "pass@16": 0.8134556574923547, + "pass@32": 0.8593272171253823, + "pass@64": 0.8837920489296636, + "pass@128": 0.8960244648318043 + }, + "13": { + "pass@1": 0.2808641975308642, + "pass@2": 0.38271604938271603, + "pass@4": 0.4382716049382716, + "pass@8": 0.5123456790123457, + "pass@16": 0.5617283950617284, + "pass@32": 0.6203703703703703, + "pass@64": 0.6728395061728395, + "pass@128": 0.7160493827160493 + }, + "14": { + "pass@1": 0.18397626112759644, + "pass@2": 0.2522255192878338, + "pass@4": 0.2997032640949555, + "pass@8": 0.3827893175074184, + "pass@16": 0.44510385756676557, + "pass@32": 0.49554896142433236, + "pass@64": 0.5578635014836796, + "pass@128": 0.6112759643916914 + }, + "15": { + "pass@1": 0.13580246913580246, + "pass@2": 0.16666666666666666, + "pass@4": 0.19135802469135801, + "pass@8": 0.23765432098765432, + "pass@16": 0.2654320987654321, + "pass@32": 0.31790123456790126, + "pass@64": 0.36728395061728397, + "pass@128": 0.44753086419753085 + }, + "16": { + "pass@1": 0.014749262536873156, + "pass@2": 0.038348082595870206, + "pass@4": 0.058997050147492625, + "pass@8": 0.07669616519174041, + "pass@16": 0.08849557522123894, + "pass@32": 0.11209439528023599, + "pass@64": 0.15339233038348082, + "pass@128": 0.19174041297935104 + }, + "17": { + "pass@1": 0.012048192771084338, + "pass@2": 0.02108433734939759, + "pass@4": 0.03313253012048193, + "pass@8": 0.03614457831325301, + "pass@16": 0.05120481927710843, + "pass@32": 0.06927710843373494, + "pass@64": 0.07530120481927711, + "pass@128": 0.09036144578313253 + }, + "18": { + "pass@1": 0.016129032258064516, + "pass@2": 0.016129032258064516, + "pass@4": 0.03225806451612903, + "pass@8": 0.04516129032258064, + "pass@16": 0.06451612903225806, + "pass@32": 0.07741935483870968, + "pass@64": 0.08064516129032258, + "pass@128": 0.08709677419354839 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0030211480362537764, + "pass@4": 0.0030211480362537764, + "pass@8": 0.006042296072507553, + "pass@16": 0.00906344410876133, + "pass@32": 0.00906344410876133, + "pass@64": 0.015105740181268883, + "pass@128": 0.015105740181268883 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.006006006006006006 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.4888888888888889, + "12": 0.5045871559633027, + "13": 0.2808641975308642, + "14": 0.18397626112759644, + "15": 0.13580246913580246, + "16": 0.014749262536873156, + "17": 0.012048192771084338, + "18": 0.016129032258064516, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5841269841269842, + "12": 0.5749235474006116, + "13": 0.38271604938271603, + "14": 0.2522255192878338, + "15": 0.16666666666666666, + "16": 0.038348082595870206, + "17": 0.02108433734939759, + "18": 0.016129032258064516, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@4": { + "11": 0.6825396825396826, + "12": 0.654434250764526, + "13": 0.4382716049382716, + "14": 0.2997032640949555, + "15": 0.19135802469135801, + "16": 0.058997050147492625, + "17": 0.03313253012048193, + "18": 0.03225806451612903, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@8": { + "11": 0.780952380952381, + "12": 0.7339449541284404, + "13": 0.5123456790123457, + "14": 0.3827893175074184, + "15": 0.23765432098765432, + "16": 0.07669616519174041, + "17": 0.03614457831325301, + "18": 0.04516129032258064, + "19": 0.006042296072507553, + "20": 0.0 + }, + "pass@16": { + "11": 0.8698412698412699, + "12": 0.8134556574923547, + "13": 0.5617283950617284, + "14": 0.44510385756676557, + "15": 0.2654320987654321, + "16": 0.08849557522123894, + "17": 0.05120481927710843, + "18": 0.06451612903225806, + "19": 0.00906344410876133, + "20": 0.0 + }, + "pass@32": { + "11": 0.926984126984127, + "12": 0.8593272171253823, + "13": 0.6203703703703703, + "14": 0.49554896142433236, + "15": 0.31790123456790126, + "16": 0.11209439528023599, + "17": 0.06927710843373494, + "18": 0.07741935483870968, + "19": 0.00906344410876133, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.946031746031746, + "12": 0.8837920489296636, + "13": 0.6728395061728395, + "14": 0.5578635014836796, + "15": 0.36728395061728397, + "16": 0.15339233038348082, + "17": 0.07530120481927711, + "18": 0.08064516129032258, + "19": 0.015105740181268883, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9555555555555556, + "12": 0.8960244648318043, + "13": 0.7160493827160493, + "14": 0.6112759643916914, + "15": 0.44753086419753085, + "16": 0.19174041297935104, + "17": 0.09036144578313253, + "18": 0.08709677419354839, + "19": 0.015105740181268883, + "20": 0.006006006006006006 + } + }, + "total_pass_at_k": { + "pass@1": 0.16198044009779952, + "pass@2": 0.20201711491442542, + "pass@4": 0.2371638141809291, + "pass@8": 0.2787286063569682, + "pass@16": 0.3141809290953545, + "pass@32": 0.3465770171149144, + "pass@64": 0.37316625916870416, + "pass@128": 0.3991442542787286 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5391566265060241, + "pass@2": 0.6536144578313253, + "pass@4": 0.7439759036144579, + "pass@8": 0.8072289156626506, + "pass@16": 0.8644578313253012, + "pass@32": 0.9246987951807228, + "pass@64": 0.963855421686747, + "pass@128": 0.9698795180722891 + }, + "12": { + "pass@1": 0.43260188087774293, + "pass@2": 0.5015673981191222, + "pass@4": 0.5987460815047022, + "pass@8": 0.6865203761755486, + "pass@16": 0.786833855799373, + "pass@32": 0.8213166144200627, + "pass@64": 0.8620689655172413, + "pass@128": 0.877742946708464 + }, + "13": { + "pass@1": 0.24929971988795518, + "pass@2": 0.3137254901960784, + "pass@4": 0.39775910364145656, + "pass@8": 0.49299719887955185, + "pass@16": 0.5798319327731093, + "pass@32": 0.6386554621848739, + "pass@64": 0.680672268907563, + "pass@128": 0.7310924369747899 + }, + "14": { + "pass@1": 0.14327485380116958, + "pass@2": 0.1871345029239766, + "pass@4": 0.24561403508771928, + "pass@8": 0.30409356725146197, + "pass@16": 0.3654970760233918, + "pass@32": 0.43859649122807015, + "pass@64": 0.5, + "pass@128": 0.5467836257309941 + }, + "15": { + "pass@1": 0.09090909090909091, + "pass@2": 0.13636363636363635, + "pass@4": 0.1787878787878788, + "pass@8": 0.22727272727272727, + "pass@16": 0.26666666666666666, + "pass@32": 0.3303030303030303, + "pass@64": 0.3696969696969697, + "pass@128": 0.42424242424242425 + }, + "16": { + "pass@1": 0.019943019943019943, + "pass@2": 0.02849002849002849, + "pass@4": 0.045584045584045586, + "pass@8": 0.06267806267806268, + "pass@16": 0.08262108262108261, + "pass@32": 0.09116809116809117, + "pass@64": 0.10826210826210826, + "pass@128": 0.1282051282051282 + }, + "17": { + "pass@1": 0.021739130434782608, + "pass@2": 0.031055900621118012, + "pass@4": 0.031055900621118012, + "pass@8": 0.037267080745341616, + "pass@16": 0.037267080745341616, + "pass@32": 0.052795031055900624, + "pass@64": 0.06832298136645963, + "pass@128": 0.08695652173913043 + }, + "18": { + "pass@1": 0.0029411764705882353, + "pass@2": 0.0058823529411764705, + "pass@4": 0.0058823529411764705, + "pass@8": 0.01764705882352941, + "pass@16": 0.020588235294117647, + "pass@32": 0.03529411764705882, + "pass@64": 0.03529411764705882, + "pass@128": 0.047058823529411764 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.002976190476190476, + "pass@4": 0.002976190476190476, + "pass@8": 0.002976190476190476, + "pass@16": 0.008928571428571428, + "pass@32": 0.008928571428571428, + "pass@64": 0.011904761904761904, + "pass@128": 0.01488095238095238 + }, + "20": { + "pass@1": 0.0029940119760479044, + "pass@2": 0.0029940119760479044, + "pass@4": 0.005988023952095809, + "pass@8": 0.005988023952095809, + "pass@16": 0.005988023952095809, + "pass@32": 0.005988023952095809, + "pass@64": 0.008982035928143712, + "pass@128": 0.011976047904191617 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5391566265060241, + "12": 0.43260188087774293, + "13": 0.24929971988795518, + "14": 0.14327485380116958, + "15": 0.09090909090909091, + "16": 0.019943019943019943, + "17": 0.021739130434782608, + "18": 0.0029411764705882353, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@2": { + "11": 0.6536144578313253, + "12": 0.5015673981191222, + "13": 0.3137254901960784, + "14": 0.1871345029239766, + "15": 0.13636363636363635, + "16": 0.02849002849002849, + "17": 0.031055900621118012, + "18": 0.0058823529411764705, + "19": 0.002976190476190476, + "20": 0.0029940119760479044 + }, + "pass@4": { + "11": 0.7439759036144579, + "12": 0.5987460815047022, + "13": 0.39775910364145656, + "14": 0.24561403508771928, + "15": 0.1787878787878788, + "16": 0.045584045584045586, + "17": 0.031055900621118012, + "18": 0.0058823529411764705, + "19": 0.002976190476190476, + "20": 0.005988023952095809 + }, + "pass@8": { + "11": 0.8072289156626506, + "12": 0.6865203761755486, + "13": 0.49299719887955185, + "14": 0.30409356725146197, + "15": 0.22727272727272727, + "16": 0.06267806267806268, + "17": 0.037267080745341616, + "18": 0.01764705882352941, + "19": 0.002976190476190476, + "20": 0.005988023952095809 + }, + "pass@16": { + "11": 0.8644578313253012, + "12": 0.786833855799373, + "13": 0.5798319327731093, + "14": 0.3654970760233918, + "15": 0.26666666666666666, + "16": 0.08262108262108261, + "17": 0.037267080745341616, + "18": 0.020588235294117647, + "19": 0.008928571428571428, + "20": 0.005988023952095809 + }, + "pass@32": { + "11": 0.9246987951807228, + "12": 0.8213166144200627, + "13": 0.6386554621848739, + "14": 0.43859649122807015, + "15": 0.3303030303030303, + "16": 0.09116809116809117, + "17": 0.052795031055900624, + "18": 0.03529411764705882, + "19": 0.008928571428571428, + "20": 0.005988023952095809 + }, + "pass@64": { + "11": 0.963855421686747, + "12": 0.8620689655172413, + "13": 0.680672268907563, + "14": 0.5, + "15": 0.3696969696969697, + "16": 0.10826210826210826, + "17": 0.06832298136645963, + "18": 0.03529411764705882, + "19": 0.011904761904761904, + "20": 0.008982035928143712 + }, + "pass@128": { + "11": 0.9698795180722891, + "12": 0.877742946708464, + "13": 0.7310924369747899, + "14": 0.5467836257309941, + "15": 0.42424242424242425, + "16": 0.1282051282051282, + "17": 0.08695652173913043, + "18": 0.047058823529411764, + "19": 0.01488095238095238, + "20": 0.011976047904191617 + } + }, + "total_pass_at_k": { + "pass@1": 0.14927148379423133, + "pass@2": 0.18495391019922688, + "pass@4": 0.22420457924472198, + "pass@8": 0.2631578947368421, + "pass@16": 0.3006244424620874, + "pass@32": 0.33363068688670827, + "pass@64": 0.359797799583705, + "pass@128": 0.3829913767469521 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5, + "pass@2": 0.61, + "pass@4": 0.71, + "pass@8": 0.785, + "pass@16": 0.856, + "pass@32": 0.907, + "pass@64": 0.94, + "pass@128": 0.958 + }, + "12": { + "pass@1": 0.477, + "pass@2": 0.548, + "pass@4": 0.639, + "pass@8": 0.72, + "pass@16": 0.792, + "pass@32": 0.834, + "pass@64": 0.862, + "pass@128": 0.878 + }, + "13": { + "pass@1": 0.267, + "pass@2": 0.342, + "pass@4": 0.418, + "pass@8": 0.498, + "pass@16": 0.568, + "pass@32": 0.63, + "pass@64": 0.675, + "pass@128": 0.716 + }, + "14": { + "pass@1": 0.155, + "pass@2": 0.219, + "pass@4": 0.273, + "pass@8": 0.339, + "pass@16": 0.4, + "pass@32": 0.457, + "pass@64": 0.513, + "pass@128": 0.569 + }, + "15": { + "pass@1": 0.107, + "pass@2": 0.146, + "pass@4": 0.185, + "pass@8": 0.223, + "pass@16": 0.257, + "pass@32": 0.309, + "pass@64": 0.357, + "pass@128": 0.419 + }, + "16": { + "pass@1": 0.019, + "pass@2": 0.032, + "pass@4": 0.046, + "pass@8": 0.061, + "pass@16": 0.075, + "pass@32": 0.092, + "pass@64": 0.116, + "pass@128": 0.14 + }, + "17": { + "pass@1": 0.016, + "pass@2": 0.023, + "pass@4": 0.03, + "pass@8": 0.036, + "pass@16": 0.043, + "pass@32": 0.057, + "pass@64": 0.067, + "pass@128": 0.083 + }, + "18": { + "pass@1": 0.007, + "pass@2": 0.008, + "pass@4": 0.016, + "pass@8": 0.024, + "pass@16": 0.032, + "pass@32": 0.044, + "pass@64": 0.046, + "pass@128": 0.053 + }, + "19": { + "pass@1": 0.001, + "pass@2": 0.002, + "pass@4": 0.002, + "pass@8": 0.003, + "pass@16": 0.006, + "pass@32": 0.006, + "pass@64": 0.01, + "pass@128": 0.011 + }, + "20": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.002, + "pass@8": 0.002, + "pass@16": 0.002, + "pass@32": 0.005, + "pass@64": 0.007, + "pass@128": 0.009 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5, + "12": 0.477, + "13": 0.267, + "14": 0.155, + "15": 0.107, + "16": 0.019, + "17": 0.016, + "18": 0.007, + "19": 0.001, + "20": 0.001 + }, + "pass@2": { + "11": 0.61, + "12": 0.548, + "13": 0.342, + "14": 0.219, + "15": 0.146, + "16": 0.032, + "17": 0.023, + "18": 0.008, + "19": 0.002, + "20": 0.001 + }, + "pass@4": { + "11": 0.71, + "12": 0.639, + "13": 0.418, + "14": 0.273, + "15": 0.185, + "16": 0.046, + "17": 0.03, + "18": 0.016, + "19": 0.002, + "20": 0.002 + }, + "pass@8": { + "11": 0.785, + "12": 0.72, + "13": 0.498, + "14": 0.339, + "15": 0.223, + "16": 0.061, + "17": 0.036, + "18": 0.024, + "19": 0.003, + "20": 0.002 + }, + "pass@16": { + "11": 0.856, + "12": 0.792, + "13": 0.568, + "14": 0.4, + "15": 0.257, + "16": 0.075, + "17": 0.043, + "18": 0.032, + "19": 0.006, + "20": 0.002 + }, + "pass@32": { + "11": 0.907, + "12": 0.834, + "13": 0.63, + "14": 0.457, + "15": 0.309, + "16": 0.092, + "17": 0.057, + "18": 0.044, + "19": 0.006, + "20": 0.005 + }, + "pass@64": { + "11": 0.94, + "12": 0.862, + "13": 0.675, + "14": 0.513, + "15": 0.357, + "16": 0.116, + "17": 0.067, + "18": 0.046, + "19": 0.01, + "20": 0.007 + }, + "pass@128": { + "11": 0.958, + "12": 0.878, + "13": 0.716, + "14": 0.569, + "15": 0.419, + "16": 0.14, + "17": 0.083, + "18": 0.053, + "19": 0.011, + "20": 0.009 + } + }, + "total_pass_at_k": { + "pass@1": 0.155, + "pass@2": 0.1931, + "pass@4": 0.2321, + "pass@8": 0.2691, + "pass@16": 0.3031, + "pass@32": 0.3341, + "pass@64": 0.3593, + "pass@128": 0.3836 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block8/composition/autoregressive/summary.csv b/SDAR-100M-ep1-block8/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..8691e7f00b7aac21f74583cbf24a939d6e7a36cb --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21546,0.9743967013888889,0.39801640625,0.6710386513157894,0.0,0.0,0.0,200.49886805555556,321.86122421875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block8/composition/autoregressive/summary.json b/SDAR-100M-ep1-block8/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d952199261dd01db332624b549afd02441a908 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21546", + "id_acc": 0.9743967013888889, + "ood_acc": 0.39801640625, + "total_acc": 0.6710386513157894, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 200.49886805555556, + "ood_avg_resp_len": 321.86122421875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 540343, + "answer_accuracy": 0.6673142092159342, + "avg_response_len": 252.07113746838445, + "resp_tokens_sum": 204109058, + "pass_at_k": { + "pass@1": 0.6673142092159342, + "pass@2": 0.7184972467083713, + "pass@4": 0.7582710489269354, + "pass@8": 0.7898036372954599, + "pass@16": 0.8149666354226288, + "pass@32": 0.8356683187114132, + "pass@64": 0.8536326257414687, + "pass@128": 0.8695858362314258 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9078401548672567, + "pass@2": 0.9451660598564567, + "pass@4": 0.9627415558121603, + "pass@8": 0.9724228068912785, + "pass@16": 0.9786434853777529, + "pass@32": 0.9833303999293327, + "pass@64": 0.9874800169721869, + "pass@128": 0.9911504424778761 + }, + "2": { + "pass@1": 0.9968847352024922, + "pass@2": 0.9968847352024922, + "pass@4": 0.9968847352024922, + "pass@8": 0.9968847352024922, + "pass@16": 0.9968847352024922, + "pass@32": 0.9968847352024922, + "pass@64": 0.9968847352024922, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9974913990825688, + "pass@2": 0.998993925931277, + "pass@4": 0.999781667199857, + "pass@8": 0.9999876849218283, + "pass@16": 0.9999999719746717, + "pass@32": 0.9999999999999853, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9863525390625, + "pass@2": 0.9885619156003939, + "pass@4": 0.9898306295697413, + "pass@8": 0.9915581459135234, + "pass@16": 0.9938875771099299, + "pass@32": 0.996529537062657, + "pass@64": 0.9989487041172099, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9679315476190476, + "pass@2": 0.9789893450818652, + "pass@4": 0.9852945253867076, + "pass@8": 0.9903654763094896, + "pass@16": 0.9944361907939756, + "pass@32": 0.9969451642580184, + "pass@64": 0.9983683333808681, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9894975673652695, + "pass@2": 0.996165027229007, + "pass@4": 0.998533353147473, + "pass@8": 0.999620189417031, + "pass@16": 0.9999636861019171, + "pass@32": 0.9999998029761207, + "pass@64": 0.999999999999911, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9860392011834319, + "pass@2": 0.9959920124167174, + "pass@4": 0.9977027054310518, + "pass@8": 0.9983763394978522, + "pass@16": 0.9991472297394309, + "pass@32": 0.9997987731838602, + "pass@64": 0.999995725631176, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9701522435897436, + "pass@2": 0.9889413612961848, + "pass@4": 0.9943698822502955, + "pass@8": 0.9962621361835249, + "pass@16": 0.9970686948727937, + "pass@32": 0.9975932975941004, + "pass@64": 0.998397435811393, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9600132042253521, + "pass@2": 0.9878528058112463, + "pass@4": 0.9946557779749362, + "pass@8": 0.9974029094078771, + "pass@16": 0.9989479245309896, + "pass@32": 0.9997375771890578, + "pass@64": 0.9999912792521506, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8549043909348442, + "pass@2": 0.9105383272735389, + "pass@4": 0.9394676640101288, + "pass@8": 0.9567641672474922, + "pass@16": 0.9689281622131964, + "pass@32": 0.9792995108322007, + "pass@64": 0.989377221593619, + "pass@128": 1.0 + }, + "12": { + "pass@1": 0.7604387358757062, + "pass@2": 0.8512497080608568, + "pass@4": 0.9022677093859018, + "pass@8": 0.9306185243521176, + "pass@16": 0.947629076857396, + "pass@32": 0.9613728628093047, + "pass@64": 0.9722807480889516, + "pass@128": 0.9774011299435028 + }, + "13": { + "pass@1": 0.6151792711598746, + "pass@2": 0.7272553717325303, + "pass@4": 0.8012909736204601, + "pass@8": 0.8513849982189186, + "pass@16": 0.8858777402360082, + "pass@32": 0.9113844472956131, + "pass@64": 0.9327076612903673, + "pass@128": 0.9529780564263323 + }, + "14": { + "pass@1": 0.4572381230529595, + "pass@2": 0.5804936437559792, + "pass@4": 0.6836558835519382, + "pass@8": 0.7584661690159947, + "pass@16": 0.8090695107690499, + "pass@32": 0.8452244630136615, + "pass@64": 0.8744698913107548, + "pass@128": 0.9034267912772586 + }, + "15": { + "pass@1": 0.3591943641618497, + "pass@2": 0.46582035694779467, + "pass@4": 0.557234378759013, + "pass@8": 0.6333810374573632, + "pass@16": 0.6991402754904769, + "pass@32": 0.7537907497738942, + "pass@64": 0.7989936415700499, + "pass@128": 0.838150289017341 + }, + "16": { + "pass@1": 0.2480594758064516, + "pass@2": 0.3369277051054099, + "pass@4": 0.41882028587152403, + "pass@8": 0.48785088132261173, + "pass@16": 0.5464977216622237, + "pass@32": 0.5997006252636178, + "pass@64": 0.6464878664953922, + "pass@128": 0.6838709677419355 + }, + "17": { + "pass@1": 0.19413836705202311, + "pass@2": 0.2639743866915478, + "pass@4": 0.3301886702384742, + "pass@8": 0.3913913054415934, + "pass@16": 0.44674849852516724, + "pass@32": 0.49537678199969953, + "pass@64": 0.5397606358748235, + "pass@128": 0.5809248554913294 + }, + "18": { + "pass@1": 0.16417410714285716, + "pass@2": 0.23900625703037112, + "pass@4": 0.31737553055868006, + "pass@8": 0.3877756897146819, + "pass@16": 0.44484807064643955, + "pass@32": 0.4911672177360723, + "pass@64": 0.5336450417048393, + "pass@128": 0.5771428571428572 + }, + "19": { + "pass@1": 0.1477805930930931, + "pass@2": 0.21373120167411505, + "pass@4": 0.2819873895267596, + "pass@8": 0.3444790109299255, + "pass@16": 0.398624559518731, + "pass@32": 0.4448169543649647, + "pass@64": 0.48421600118684854, + "pass@128": 0.5165165165165165 + }, + "20": { + "pass@1": 0.15219125375375375, + "pass@2": 0.21601005533092138, + "pass@4": 0.2833787189889551, + "pass@8": 0.34737669283247447, + "pass@16": 0.40206167741449067, + "pass@32": 0.4470107741738278, + "pass@64": 0.4875770578788106, + "pass@128": 0.5225225225225225 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 550391, + "answer_accuracy": 0.6702930144193296, + "avg_response_len": 260.58617132696804, + "resp_tokens_sum": 213972517, + "pass_at_k": { + "pass@1": 0.6702930144193296, + "pass@2": 0.7203984686174745, + "pass@4": 0.7605172725681272, + "pass@8": 0.7936184035372785, + "pass@16": 0.8205157430956861, + "pass@32": 0.8418489124402874, + "pass@64": 0.8581997269526279, + "pass@128": 0.8693686671862821 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.8947948619631901, + "pass@2": 0.9342487228877832, + "pass@4": 0.9553914430174755, + "pass@8": 0.9685340029247087, + "pass@16": 0.9778976377826292, + "pass@32": 0.9854515349169342, + "pass@64": 0.991566746760942, + "pass@128": 0.9938650306748467 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9943181818181818, + "pass@2": 0.9965987383679314, + "pass@4": 0.9986907074115734, + "pass@8": 0.9997900403945096, + "pass@16": 0.999993778423042, + "pass@32": 0.9999999968162022, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9986825073964497, + "pass@2": 0.9998864324651727, + "pass@4": 0.9999964917477031, + "pass@8": 0.999999997761824, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9817028985507247, + "pass@2": 0.9890858581536004, + "pass@4": 0.9919462368834332, + "pass@8": 0.9941619355764031, + "pass@16": 0.9961781821357537, + "pass@32": 0.9975886476682482, + "pass@64": 0.9985464572309225, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9912249283667621, + "pass@2": 0.9971071734765248, + "pass@4": 0.998236967692792, + "pass@8": 0.9989742809016088, + "pass@16": 0.9996596953889924, + "pass@32": 0.9999716397876668, + "pass@64": 0.9999999654169256, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9866781069364162, + "pass@2": 0.9940098595876383, + "pass@4": 0.9954577830950322, + "pass@8": 0.9965385997356417, + "pass@16": 0.997911202425725, + "pass@32": 0.9992275285274425, + "pass@64": 0.9999158252269373, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9713248239436619, + "pass@2": 0.9873073084174341, + "pass@4": 0.9928848371066289, + "pass@8": 0.9962645987397817, + "pass@16": 0.9977159520362265, + "pass@32": 0.9984178461141198, + "pass@64": 0.9993013196994364, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9549728667664671, + "pass@2": 0.9786595607996605, + "pass@4": 0.9846184301812573, + "pass@8": 0.9881064318849376, + "pass@16": 0.9915007486161633, + "pass@32": 0.9943169158716935, + "pass@64": 0.996171853688097, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.8858951430722891, + "pass@2": 0.9403234246039275, + "pass@4": 0.9654911386264966, + "pass@8": 0.9802103258206285, + "pass@16": 0.9892633104282124, + "pass@32": 0.9943230187012654, + "pass@64": 0.9966323831791278, + "pass@128": 0.9969879518072289 + }, + "12": { + "pass@1": 0.770865987460815, + "pass@2": 0.8529438081603437, + "pass@4": 0.8969163801625106, + "pass@8": 0.9239001333151642, + "pass@16": 0.944273209688222, + "pass@32": 0.9607568063656658, + "pass@64": 0.9735690051847953, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.55249912464986, + "pass@2": 0.670897295926244, + "pass@4": 0.7573695231373386, + "pass@8": 0.8195883022487538, + "pass@16": 0.8639400641786898, + "pass@32": 0.8959443632193429, + "pass@64": 0.9194230288568553, + "pass@128": 0.938375350140056 + }, + "14": { + "pass@1": 0.4458607456140351, + "pass@2": 0.5551821441727679, + "pass@4": 0.6490961617736377, + "pass@8": 0.7264317974040324, + "pass@16": 0.7878208627637026, + "pass@32": 0.8337530255493398, + "pass@64": 0.8650871772849, + "pass@128": 0.8830409356725146 + }, + "15": { + "pass@1": 0.3692708333333333, + "pass@2": 0.4844406167979, + "pass@4": 0.5807708542114053, + "pass@8": 0.6566659618047733, + "pass@16": 0.71822453037185, + "pass@32": 0.7690809020128775, + "pass@64": 0.8074737398378564, + "pass@128": 0.8272727272727273 + }, + "16": { + "pass@1": 0.2302795584045584, + "pass@2": 0.3132508075913587, + "pass@4": 0.3953974561940442, + "pass@8": 0.4699005180824398, + "pass@16": 0.5316027123774437, + "pass@32": 0.5810105949043947, + "pass@64": 0.6208832644017411, + "pass@128": 0.6495726495726496 + }, + "17": { + "pass@1": 0.21753687888198758, + "pass@2": 0.29977732063383367, + "pass@4": 0.3802775075281738, + "pass@8": 0.4530973687186417, + "pass@16": 0.5146178409330402, + "pass@32": 0.5636207169721692, + "pass@64": 0.6026909514942224, + "pass@128": 0.6335403726708074 + }, + "18": { + "pass@1": 0.19386488970588237, + "pass@2": 0.27067797591477505, + "pass@4": 0.34721876282008873, + "pass@8": 0.4179357468470566, + "pass@16": 0.4785538356301472, + "pass@32": 0.5267746944091646, + "pass@64": 0.5637993560978994, + "pass@128": 0.5882352941176471 + }, + "19": { + "pass@1": 0.14381045386904762, + "pass@2": 0.20448420509936244, + "pass@4": 0.26923672822147243, + "pass@8": 0.3304539572282448, + "pass@16": 0.3857649133616538, + "pass@32": 0.4348844940956561, + "pass@64": 0.4749150124076846, + "pass@128": 0.5029761904761905 + }, + "20": { + "pass@1": 0.140812125748503, + "pass@2": 0.2086588388184262, + "pass@4": 0.2810939201462094, + "pass@8": 0.3485268808493757, + "pass@16": 0.4055889275550898, + "pass@32": 0.45131218202379575, + "pass@64": 0.4876867028458811, + "pass@128": 0.5149700598802395 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 541232, + "answer_accuracy": 0.6755671832561112, + "avg_response_len": 280.69016616072855, + "resp_tokens_sum": 224875488, + "pass_at_k": { + "pass@1": 0.6755671832561112, + "pass@2": 0.7267714805640489, + "pass@4": 0.7671935018298011, + "pass@8": 0.7993583792277895, + "pass@16": 0.8249747156351722, + "pass@32": 0.8456421305459204, + "pass@64": 0.8626208528041777, + "pass@128": 0.8763380731746285 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9066464552238805, + "pass@2": 0.9443434892466809, + "pass@4": 0.9631634325560046, + "pass@8": 0.9746091114318679, + "pass@16": 0.981989067498895, + "pass@32": 0.9872894382051791, + "pass@64": 0.9904128899753277, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 0.9999553571428571, + "pass@2": 0.9999996484814399, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9983600583090378, + "pass@2": 0.9995706451872088, + "pass@4": 0.9999545667214339, + "pass@8": 0.9999994678161623, + "pass@16": 0.9999999999690319, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9955226608187134, + "pass@2": 0.996956229566699, + "pass@4": 0.9970747003664014, + "pass@8": 0.9970760232358671, + "pass@16": 0.9970760233918129, + "pass@32": 0.9970760233918129, + "pass@64": 0.9970760233918129, + "pass@128": 0.9970760233918129 + }, + "5": { + "pass@1": 0.9707720588235295, + "pass@2": 0.9819418712366834, + "pass@4": 0.9875488412294049, + "pass@8": 0.9906721170939422, + "pass@16": 0.992369679677989, + "pass@32": 0.9934273310473238, + "pass@64": 0.9940328828017424, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.994011238170347, + "pass@2": 0.9966948384212229, + "pass@4": 0.9969382767414325, + "pass@8": 0.9970425818238966, + "pass@16": 0.9972397476340691, + "pass@32": 0.9976340694006309, + "pass@64": 0.998422712933754, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9809632120253164, + "pass@2": 0.9907531707864047, + "pass@4": 0.9939196128964891, + "pass@8": 0.9955120902657174, + "pass@16": 0.9963967136338048, + "pass@32": 0.996791897857649, + "pass@64": 0.9968353559542914, + "pass@128": 0.9968354430379747 + }, + "8": { + "pass@1": 0.9713072447447447, + "pass@2": 0.9862585272280152, + "pass@4": 0.9918887061977614, + "pass@8": 0.9948537929562266, + "pass@16": 0.9965289626590625, + "pass@32": 0.9975971169999002, + "pass@64": 0.9984964932745174, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9532003617363344, + "pass@2": 0.9822585417879847, + "pass@4": 0.9900546741343824, + "pass@8": 0.9928854174919578, + "pass@16": 0.9951733768710284, + "pass@32": 0.9976992193537383, + "pass@64": 0.9996070290208584, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8584325396825396, + "pass@2": 0.9158269278840144, + "pass@4": 0.947487876217854, + "pass@8": 0.9662182976284491, + "pass@16": 0.976791807385288, + "pass@32": 0.9819247624165014, + "pass@64": 0.9848252233387427, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.7979023318042814, + "pass@2": 0.8880074767030265, + "pass@4": 0.934417512604502, + "pass@8": 0.9589038010248868, + "pass@16": 0.9715989629294857, + "pass@32": 0.9774837125997929, + "pass@64": 0.9811677016027062, + "pass@128": 0.9847094801223242 + }, + "13": { + "pass@1": 0.5472608024691358, + "pass@2": 0.6648135996889273, + "pass@4": 0.7540624178343447, + "pass@8": 0.817288586664587, + "pass@16": 0.8629907822666152, + "pass@32": 0.8987447149981824, + "pass@64": 0.927510348163161, + "pass@128": 0.9475308641975309 + }, + "14": { + "pass@1": 0.4938102744807122, + "pass@2": 0.6095301584149166, + "pass@4": 0.7005231909067747, + "pass@8": 0.7657016935167862, + "pass@16": 0.8124875164470288, + "pass@32": 0.8503301329249001, + "pass@64": 0.8858854705277092, + "pass@128": 0.9169139465875371 + }, + "15": { + "pass@1": 0.37755594135802467, + "pass@2": 0.48756584463400393, + "pass@4": 0.5814024534548923, + "pass@8": 0.6590459728887901, + "pass@16": 0.7236160111797545, + "pass@32": 0.7766907707481191, + "pass@64": 0.8177650626424972, + "pass@128": 0.845679012345679 + }, + "16": { + "pass@1": 0.24944690265486727, + "pass@2": 0.33750624230599463, + "pass@4": 0.4218391044592877, + "pass@8": 0.49854943027379045, + "pass@16": 0.5657431421036233, + "pass@32": 0.6217821165667055, + "pass@64": 0.6690913993944967, + "pass@128": 0.7109144542772862 + }, + "17": { + "pass@1": 0.20524284638554216, + "pass@2": 0.2843695154634285, + "pass@4": 0.3594979064175711, + "pass@8": 0.4248822789520208, + "pass@16": 0.48281238948550975, + "pass@32": 0.5343758170889205, + "pass@64": 0.5769024260817495, + "pass@128": 0.608433734939759 + }, + "18": { + "pass@1": 0.20216733870967743, + "pass@2": 0.28294585026670055, + "pass@4": 0.36513060706121425, + "pass@8": 0.4360889347472986, + "pass@16": 0.49089282710175347, + "pass@32": 0.5346986024780715, + "pass@64": 0.5735686830652535, + "pass@128": 0.6096774193548387 + }, + "19": { + "pass@1": 0.16880664652567975, + "pass@2": 0.23587070616361772, + "pass@4": 0.30202498139394224, + "pass@8": 0.3625748851412769, + "pass@16": 0.416689758692117, + "pass@32": 0.4624262685514275, + "pass@64": 0.49878692433702804, + "pass@128": 0.5287009063444109 + }, + "20": { + "pass@1": 0.14442567567567569, + "pass@2": 0.20913137546995805, + "pass@4": 0.27880532810650926, + "pass@8": 0.34708860136362146, + "pass@16": 0.40649366622612243, + "pass@32": 0.4543060270928111, + "pass@64": 0.493016680586197, + "pass@128": 0.5255255255255256 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/checkpoint-21546_metrics.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/checkpoint-21546_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..d82e47f4cf40d09d64bd27c56eed98c29e444ec0 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/checkpoint-21546_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21546", + "id": { + "answer_accuracy": 0.8766857638888889, + "avg_response_len": 199.3285625, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.8766857638888889, + "pass@2": 0.9269500218722498, + "pass@4": 0.9548419176977839, + "pass@8": 0.969719400744483, + "pass@16": 0.9783104081158266, + "pass@32": 0.983883652581512, + "pass@64": 0.9877247653158312, + "pass@128": 0.9905555555555555 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.742609375, + "pass@2": 0.8420033218503942, + "pass@4": 0.9007638144919381, + "pass@8": 0.9314799273056936, + "pass@16": 0.9482071956036686, + "pass@32": 0.9590574677640448, + "pass@64": 0.9667893709367122, + "pass@128": 0.972 + }, + "2": { + "pass@1": 0.9700859375, + "pass@2": 0.9776165108267717, + "pass@4": 0.9835557621859767, + "pass@8": 0.9877506181583803, + "pass@16": 0.9904904799979233, + "pass@32": 0.9922714851547506, + "pass@64": 0.9936245623896852, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.98003125, + "pass@2": 0.9902121062992125, + "pass@4": 0.9938104119797523, + "pass@8": 0.9953216741446105, + "pass@16": 0.9962975091437134, + "pass@32": 0.9973458976443671, + "pass@64": 0.9986160966112356, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9623828125, + "pass@2": 0.9767733759842523, + "pass@4": 0.9844144256655416, + "pass@8": 0.988053791245531, + "pass@16": 0.9898672400273606, + "pass@32": 0.9912520377201507, + "pass@64": 0.9926321551350659, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.8991953125, + "pass@2": 0.926057086614174, + "pass@4": 0.9433322265654293, + "pass@8": 0.9550924227478708, + "pass@16": 0.9633991311457364, + "pass@32": 0.9697422400328563, + "pass@64": 0.9744431275938922, + "pass@128": 0.977 + }, + "6": { + "pass@1": 0.868984375, + "pass@2": 0.9273538385826782, + "pass@4": 0.9600408220847391, + "pass@8": 0.977614400273202, + "pass@16": 0.9874573317988508, + "pass@32": 0.9932872465112752, + "pass@64": 0.9966020228329742, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.8641640625, + "pass@2": 0.9231895915354337, + "pass@4": 0.9538450460254959, + "pass@8": 0.9702194025693823, + "pass@16": 0.9804944578875513, + "pass@32": 0.9874538270747679, + "pass@64": 0.9922845131764251, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.8278203125, + "pass@2": 0.9064554625984261, + "pass@4": 0.9468264335395588, + "pass@8": 0.9668031353266834, + "pass@16": 0.977404550975094, + "pass@32": 0.984057116904717, + "pass@64": 0.9888479828753439, + "pass@128": 0.992 + }, + "9": { + "pass@1": 0.7748984375, + "pass@2": 0.8728889025590546, + "pass@4": 0.9269883167416576, + "pass@8": 0.9551392349290369, + "pass@16": 0.9711757764625201, + "pass@32": 0.9804855544266728, + "pass@64": 0.985683056291135, + "pass@128": 0.989 + } + }, + "per_op_accuracy": { + "10": 0.742609375, + "2": 0.9700859375, + "3": 0.98003125, + "4": 0.9623828125, + "5": 0.8991953125, + "6": 0.868984375, + "7": 0.8641640625, + "8": 0.8278203125, + "9": 0.7748984375 + }, + "per_op_avg_response_len": { + "10": 277.3975078125, + "2": 116.822421875, + "3": 143.9019609375, + "4": 159.7976015625, + "5": 173.9955078125, + "6": 198.53321875, + "7": 223.23425, + "8": 242.378515625, + "9": 257.896078125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.3975078125, + "2": 116.822421875, + "3": 143.9019609375, + "4": 159.7976015625, + "5": 173.9955078125, + "6": 198.53321875, + "7": 223.23425, + "8": 242.378515625, + "9": 257.896078125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 330972, + "answer_accuracy": 0.8732586119554204, + "avg_response_len": 189.62009772880785, + "resp_tokens_sum": 71867534, + "pass_at_k": { + "pass@1": 0.8732586119554204, + "pass@2": 0.9279041250162939, + "pass@4": 0.957443514849399, + "pass@8": 0.9719675992314021, + "pass@16": 0.9797481070178121, + "pass@32": 0.9847516079214117, + "pass@64": 0.9884079617600531, + "pass@128": 0.9912191827085444 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7548626474926253, + "pass@2": 0.8580390739321302, + "pass@4": 0.9158941481872283, + "pass@8": 0.9422260710100495, + "pass@16": 0.9541179241145453, + "pass@32": 0.9627354979168701, + "pass@64": 0.9700056559069217, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9521271417445483, + "pass@2": 0.9600596070351018, + "pass@4": 0.967638862899147, + "pass@8": 0.9736349720045984, + "pass@16": 0.9777725738486086, + "pass@32": 0.9807478572199139, + "pass@64": 0.983644411229528, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.9738627675840978, + "pass@2": 0.9860556779599798, + "pass@4": 0.9911855877533655, + "pass@8": 0.9935549584465483, + "pass@16": 0.9950485984443537, + "pass@32": 0.9964154468880052, + "pass@64": 0.9980977119454602, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9468017578125, + "pass@2": 0.9668841504675199, + "pass@4": 0.9788314503655793, + "pass@8": 0.98423139689816, + "pass@16": 0.9869775056679988, + "pass@32": 0.988862756689439, + "pass@64": 0.9906228273186573, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8940972222222222, + "pass@2": 0.925370656792901, + "pass@4": 0.9444052689842337, + "pass@8": 0.956068245645845, + "pass@16": 0.9641325616691484, + "pass@32": 0.9702121366978933, + "pass@64": 0.9736402664360028, + "pass@128": 0.9746031746031746 + }, + "6": { + "pass@1": 0.874625748502994, + "pass@2": 0.9345000942995885, + "pass@4": 0.9660633426622565, + "pass@8": 0.9817471947893537, + "pass@16": 0.9901813682097385, + "pass@32": 0.9945595172809533, + "pass@64": 0.996547366723047, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.8708394970414202, + "pass@2": 0.9296885919955274, + "pass@4": 0.9574697000707745, + "pass@8": 0.9709659057101204, + "pass@16": 0.9797032271920411, + "pass@32": 0.9856532059055091, + "pass@64": 0.9903706220860208, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.8371394230769231, + "pass@2": 0.921493681228548, + "pass@4": 0.9620283975921279, + "pass@8": 0.9787379257212695, + "pass@16": 0.9860862836307394, + "pass@32": 0.9901621390004783, + "pass@64": 0.993815134792471, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7703125, + "pass@2": 0.8768198264389484, + "pass@4": 0.9369461128978587, + "pass@8": 0.9678318575372462, + "pass@16": 0.9839113625433821, + "pass@32": 0.993005940915507, + "pass@64": 0.9981911215016727, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 341727, + "answer_accuracy": 0.8747516997051114, + "avg_response_len": 197.293109538827, + "resp_tokens_sum": 77073737, + "pass_at_k": { + "pass@1": 0.8747516997051114, + "pass@2": 0.9247403600066086, + "pass@4": 0.9534785004245888, + "pass@8": 0.969580216485743, + "pass@16": 0.9790434021820338, + "pass@32": 0.9851216451915528, + "pass@64": 0.9892037390438363, + "pass@128": 0.9921363040629095 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7257716641104295, + "pass@2": 0.829145482102314, + "pass@4": 0.8909685864612009, + "pass@8": 0.923828926172029, + "pass@16": 0.9424341083861851, + "pass@32": 0.9547993211002885, + "pass@64": 0.9635684888575998, + "pass@128": 0.9693251533742331 + }, + "2": { + "pass@1": 0.969961056231003, + "pass@2": 0.9800300062226267, + "pass@4": 0.9871420935604933, + "pass@8": 0.9922565877731916, + "pass@16": 0.9959260151333992, + "pass@32": 0.9983344837584649, + "pass@64": 0.9996191683758167, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9798295454545455, + "pass@2": 0.9895311381531855, + "pass@4": 0.9931646833918487, + "pass@8": 0.9950567296773937, + "pass@16": 0.9963382395784798, + "pass@32": 0.9977819591272412, + "pass@64": 0.9992064994093033, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9674093934911243, + "pass@2": 0.9813876642361278, + "pass@4": 0.9896786493589189, + "pass@8": 0.9942140522761386, + "pass@16": 0.9964544182239704, + "pass@32": 0.9981567480120942, + "pass@64": 0.9996475795537232, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8929347826086956, + "pass@2": 0.9245599395184297, + "pass@4": 0.9455650755612065, + "pass@8": 0.9585984031211746, + "pass@16": 0.9661113498054109, + "pass@32": 0.9713447438413154, + "pass@64": 0.9761022999756817, + "pass@128": 0.9797101449275363 + }, + "6": { + "pass@1": 0.8609643624641834, + "pass@2": 0.9185087313584368, + "pass@4": 0.9530109303135099, + "pass@8": 0.9730129786135173, + "pass@16": 0.9845427515918137, + "pass@32": 0.9913868693492569, + "pass@64": 0.9955681335039379, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8571396315028902, + "pass@2": 0.9151441815347509, + "pass@4": 0.9492684707764129, + "pass@8": 0.9707522712002761, + "pass@16": 0.9846804474521327, + "pass@32": 0.9932585974949043, + "pass@64": 0.9976034786090924, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8400528169014084, + "pass@2": 0.9146213402462021, + "pass@4": 0.9519286311042101, + "pass@8": 0.9705870784036298, + "pass@16": 0.9806850356855231, + "pass@32": 0.9870184734714211, + "pass@64": 0.9913704266572482, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.7795424775449101, + "pass@2": 0.8696370791880805, + "pass@4": 0.9201581606191439, + "pass@8": 0.9472707355109283, + "pass@16": 0.9634659109310358, + "pass@32": 0.9732413865436497, + "pass@64": 0.979391838196961, + "pass@128": 0.9850299401197605 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 337243, + "answer_accuracy": 0.8820592358553733, + "avg_response_len": 211.03226742969534, + "resp_tokens_sum": 80685233, + "pass_at_k": { + "pass@1": 0.8820592358553733, + "pass@2": 0.9282619698483507, + "pass@4": 0.9536560523987928, + "pass@8": 0.9676329844866791, + "pass@16": 0.976136278775741, + "pass@32": 0.9817583197367524, + "pass@64": 0.9855363580546227, + "pass@128": 0.9882825577502511 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7465951492537314, + "pass@2": 0.8382885033493949, + "pass@4": 0.8949848927466156, + "pass@8": 0.9280509232871795, + "pass@16": 0.9478438805819164, + "pass@32": 0.9594792695269607, + "pass@64": 0.9666690334826524, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9866741071428572, + "pass@2": 0.9914500140607426, + "pass@4": 0.9947826811827094, + "pass@8": 0.9964610707643552, + "pass@16": 0.9970452137532337, + "pass@32": 0.9971410795160666, + "pass@64": 0.9971428571124661, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9861060495626822, + "pass@2": 0.9948298076260877, + "pass@4": 0.9969340503355447, + "pass@8": 0.9972608802887731, + "pass@16": 0.9974489749023661, + "pass@32": 0.997813411078717, + "pass@64": 0.9985422740524781, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9719937865497076, + "pass@2": 0.981466132522908, + "pass@4": 0.9844356083778999, + "pass@8": 0.9855420893824135, + "pass@16": 0.9860609498652024, + "pass@32": 0.9866636688638668, + "pass@64": 0.9875788552746696, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.9102711397058824, + "pass@2": 0.9282122076192687, + "pass@4": 0.940072399314056, + "pass@8": 0.9506308126253655, + "pass@16": 0.9599675251090849, + "pass@32": 0.9676808245522525, + "pass@64": 0.9735033828674721, + "pass@128": 0.9764705882352941 + }, + "6": { + "pass@1": 0.871870070977918, + "pass@2": 0.9295623338880746, + "pass@4": 0.9614348610603484, + "pass@8": 0.9783258917269404, + "pass@16": 0.9877960079219414, + "pass@32": 0.9940389600521968, + "pass@64": 0.99779786673376, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8647151898734177, + "pass@2": 0.9250473437655732, + "pass@4": 0.9549791028890373, + "pass@8": 0.9688374702660314, + "pass@16": 0.9767573806272281, + "pass@32": 0.9830239517261676, + "pass@64": 0.9885077193436845, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.8060482357357357, + "pass@2": 0.883660235038188, + "pass@4": 0.9271438722186747, + "pass@8": 0.9515870560611387, + "pass@16": 0.9657731015433726, + "pass@32": 0.975180094457099, + "pass@64": 0.9815049830533622, + "pass@128": 0.984984984984985 + }, + "9": { + "pass@1": 0.7751456993569131, + "pass@2": 0.8718941470238243, + "pass@4": 0.9229569807592373, + "pass@8": 0.9491012856677312, + "pass@16": 0.9649184196419777, + "pass@32": 0.9739737051321179, + "pass@64": 0.9781618463030722, + "pass@128": 0.9807073954983923 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.329509375, + "avg_response_len": 317.06312109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.329509375, + "pass@2": 0.41763699557087164, + "pass@4": 0.49376160724596774, + "pass@8": 0.5560516866609775, + "pass@16": 0.6061437186667719, + "pass@32": 0.6467827413738537, + "pass@64": 0.6807164882999561, + "pass@128": 0.71 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6998515625, + "pass@2": 0.8083144685039375, + "pass@4": 0.8770636429508802, + "pass@8": 0.9181008360941916, + "pass@16": 0.943863718342336, + "pass@32": 0.960634794271071, + "pass@64": 0.9709948538828274, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.613703125, + "pass@2": 0.7349860974409431, + "pass@4": 0.8186638415823015, + "pass@8": 0.8712536742701642, + "pass@16": 0.9053730755637114, + "pass@32": 0.9288760881260807, + "pass@64": 0.9450337096196038, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.4653828125, + "pass@2": 0.5840756643700786, + "pass@4": 0.676996273247094, + "pass@8": 0.7453395469249404, + "pass@16": 0.7961783778372612, + "pass@32": 0.8359978716006594, + "pass@64": 0.8684670915496531, + "pass@128": 0.893 + }, + "14": { + "pass@1": 0.3721796875, + "pass@2": 0.4779044045275593, + "pass@4": 0.5697991311398568, + "pass@8": 0.6449930551382927, + "pass@16": 0.7034564240199858, + "pass@32": 0.7480998447079009, + "pass@64": 0.7847394786790167, + "pass@128": 0.818 + }, + "15": { + "pass@1": 0.282671875, + "pass@2": 0.3776302903543311, + "pass@4": 0.46403711426696603, + "pass@8": 0.5361525115190808, + "pass@16": 0.5960660457857662, + "pass@32": 0.6488017024787309, + "pass@64": 0.6972755900163513, + "pass@128": 0.74 + }, + "16": { + "pass@1": 0.2092421875, + "pass@2": 0.2815675442913384, + "pass@4": 0.35104423256467915, + "pass@8": 0.4147365013606016, + "pass@16": 0.473807645456793, + "pass@32": 0.5276581635478398, + "pass@64": 0.5752541026701634, + "pass@128": 0.618 + }, + "17": { + "pass@1": 0.181265625, + "pass@2": 0.2510859990157479, + "pass@4": 0.32178556730408703, + "pass@8": 0.38730375998214156, + "pass@16": 0.4440593399814598, + "pass@32": 0.4919418160037072, + "pass@64": 0.5322604937723371, + "pass@128": 0.569 + }, + "18": { + "pass@1": 0.18325, + "pass@2": 0.2523038877952756, + "pass@4": 0.32132108848893887, + "pass@8": 0.38317692535437, + "pass@16": 0.43339469992733715, + "pass@32": 0.47394288781054794, + "pass@64": 0.5101970847647627, + "pass@128": 0.543 + }, + "19": { + "pass@1": 0.15084375, + "pass@2": 0.21060839074803153, + "pass@4": 0.27216627474690647, + "pass@8": 0.3298577004903936, + "pass@16": 0.3790432843348919, + "pass@32": 0.4189484785223196, + "pass@64": 0.4522162312278507, + "pass@128": 0.479 + }, + "20": { + "pass@1": 0.136703125, + "pass@2": 0.19789320866141724, + "pass@4": 0.2647389061679785, + "pass@8": 0.32960235547558203, + "pass@16": 0.38619457541803326, + "pass@32": 0.43292576666967225, + "pass@64": 0.47072624681695024, + "pass@128": 0.502 + } + }, + "per_op_accuracy": { + "11": 0.6998515625, + "12": 0.613703125, + "13": 0.4653828125, + "14": 0.3721796875, + "15": 0.282671875, + "16": 0.2092421875, + "17": 0.181265625, + "18": 0.18325, + "19": 0.15084375, + "20": 0.136703125 + }, + "per_op_avg_response_len": { + "11": 293.3047734375, + "12": 301.717640625, + "13": 312.774609375, + "14": 322.1216484375, + "15": 324.38078125, + "16": 317.1935703125, + "17": 325.424015625, + "18": 328.242046875, + "19": 320.539703125, + "20": 324.932421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 293.3047734375, + "12": 301.717640625, + "13": 312.774609375, + "14": 322.1216484375, + "15": 324.38078125, + "16": 317.1935703125, + "17": 325.424015625, + "18": 328.242046875, + "19": 320.539703125, + "20": 324.932421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 145035, + "answer_accuracy": 0.33672687592867756, + "avg_response_len": 301.7581305720654, + "resp_tokens_sum": 129973262, + "pass_at_k": { + "pass@1": 0.33672687592867756, + "pass@2": 0.42635137210281937, + "pass@4": 0.5027781829648718, + "pass@8": 0.5637722736865154, + "pass@16": 0.6114019206850503, + "pass@32": 0.6496656037448822, + "pass@64": 0.6823712670647403, + "pass@128": 0.7114413075780089 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7148326841359773, + "pass@2": 0.8247275183466801, + "pass@4": 0.8932718025154789, + "pass@8": 0.9331762977429914, + "pass@16": 0.9563250136105693, + "pass@32": 0.9703408496523926, + "pass@64": 0.9791441852491506, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.610787429378531, + "pass@2": 0.7386105504025978, + "pass@4": 0.8302062342842734, + "pass@8": 0.8869758223344203, + "pass@16": 0.9190229298739069, + "pass@32": 0.9382509994759017, + "pass@64": 0.9504507398120785, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.4938528605015674, + "pass@2": 0.6135426437192993, + "pass@4": 0.7050806219755448, + "pass@8": 0.7709119554640657, + "pass@16": 0.8185553713018926, + "pass@32": 0.8529551509357786, + "pass@64": 0.8789136115726013, + "pass@128": 0.896551724137931 + }, + "14": { + "pass@1": 0.3722984813084112, + "pass@2": 0.4832002600142271, + "pass@4": 0.5817811873983043, + "pass@8": 0.6600182956969374, + "pass@16": 0.7170059764733474, + "pass@32": 0.759723872506481, + "pass@64": 0.7973315503500162, + "pass@128": 0.8348909657320872 + }, + "15": { + "pass@1": 0.2915236632947977, + "pass@2": 0.38716340573938335, + "pass@4": 0.4724536182435286, + "pass@8": 0.5428825309662252, + "pass@16": 0.6009330341758621, + "pass@32": 0.6513234112092107, + "pass@64": 0.6978355864283469, + "pass@128": 0.7398843930635838 + }, + "16": { + "pass@1": 0.2159022177419355, + "pass@2": 0.294323485521971, + "pass@4": 0.3682993205486411, + "pass@8": 0.4345881141350131, + "pass@16": 0.49488662642953934, + "pass@32": 0.5483336279194447, + "pass@64": 0.5945119316894635, + "pass@128": 0.635483870967742 + }, + "17": { + "pass@1": 0.17767792630057805, + "pass@2": 0.24014361260297645, + "pass@4": 0.3024947796445963, + "pass@8": 0.36105936344277334, + "pass@16": 0.41425311433468764, + "pass@32": 0.4629499781277512, + "pass@64": 0.5069431688928129, + "pass@128": 0.5491329479768786 + }, + "18": { + "pass@1": 0.17955357142857142, + "pass@2": 0.24916057367829025, + "pass@4": 0.3168455503776314, + "pass@8": 0.37584058796751063, + "pass@16": 0.4238660306777394, + "pass@32": 0.4629081102911771, + "pass@64": 0.4982544518716383, + "pass@128": 0.5285714285714286 + }, + "19": { + "pass@1": 0.15315315315315314, + "pass@2": 0.21435005379395142, + "pass@4": 0.27603142118496454, + "pass@8": 0.3325890592148852, + "pass@16": 0.38104382491172395, + "pass@32": 0.4215814504369368, + "pass@64": 0.456708682959238, + "pass@128": 0.4864864864864865 + }, + "20": { + "pass@1": 0.13323479729729729, + "pass@2": 0.19331757939041397, + "pass@4": 0.2577742100931077, + "pass@8": 0.3186896897271893, + "pass@16": 0.37015786252682287, + "pass@32": 0.4131022452662304, + "pass@64": 0.4508702308920766, + "pass@128": 0.48348348348348347 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 140988, + "answer_accuracy": 0.3275256467439786, + "avg_response_len": 312.5581628196551, + "resp_tokens_sum": 134545037, + "pass_at_k": { + "pass@1": 0.3275256467439786, + "pass@2": 0.4141252595990182, + "pass@4": 0.4896123724944743, + "pass@8": 0.5520520892984585, + "pass@16": 0.6028639246470211, + "pass@32": 0.6447117705212217, + "pass@64": 0.6803548434824219, + "pass@128": 0.7124591138864109 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.7141613328313253, + "pass@2": 0.8234709260269424, + "pass@4": 0.8924713595665003, + "pass@8": 0.9303646055635199, + "pass@16": 0.9515900662979302, + "pass@32": 0.9648603754748566, + "pass@64": 0.9738234865676904, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.625342868338558, + "pass@2": 0.7412462512033179, + "pass@4": 0.8177234021295925, + "pass@8": 0.864841454508076, + "pass@16": 0.8973079096792422, + "pass@32": 0.9206851353500622, + "pass@64": 0.9360387543338561, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.45526960784313725, + "pass@2": 0.573507700324224, + "pass@4": 0.6657258832666921, + "pass@8": 0.7338637568424262, + "pass@16": 0.7852004994961729, + "pass@32": 0.8273113155682195, + "pass@64": 0.8624061133822667, + "pass@128": 0.8907563025210085 + }, + "14": { + "pass@1": 0.35798154239766083, + "pass@2": 0.45727353570935186, + "pass@4": 0.545599068756756, + "pass@8": 0.6210483819213921, + "pass@16": 0.68125171142204, + "pass@32": 0.7270073779808284, + "pass@64": 0.7631432072915221, + "pass@128": 0.7953216374269005 + }, + "15": { + "pass@1": 0.2765151515151515, + "pass@2": 0.36803939990455725, + "pass@4": 0.45397499403483615, + "pass@8": 0.5263473308035848, + "pass@16": 0.5859555392776711, + "pass@32": 0.6386314053079184, + "pass@64": 0.688474647802565, + "pass@128": 0.7333333333333333 + }, + "16": { + "pass@1": 0.19366542022792022, + "pass@2": 0.26023509881777596, + "pass@4": 0.3251300206063985, + "pass@8": 0.38605801417611135, + "pass@16": 0.44536576095291924, + "pass@32": 0.502109158651507, + "pass@64": 0.5541207044234497, + "pass@128": 0.603988603988604 + }, + "17": { + "pass@1": 0.20055318322981366, + "pass@2": 0.2769511572602337, + "pass@4": 0.3524708197383709, + "pass@8": 0.4216463758523163, + "pass@16": 0.4810756708240572, + "pass@32": 0.5287041717254924, + "pass@64": 0.5680075699156929, + "pass@128": 0.6055900621118012 + }, + "18": { + "pass@1": 0.19087775735294119, + "pass@2": 0.26312348019916604, + "pass@4": 0.3357639033172321, + "pass@8": 0.400534103180857, + "pass@16": 0.45257040732607284, + "pass@32": 0.49584444773542125, + "pass@64": 0.5358095049537671, + "pass@128": 0.5735294117647058 + }, + "19": { + "pass@1": 0.14274088541666666, + "pass@2": 0.20004298779058863, + "pass@4": 0.26311496163425996, + "pass@8": 0.3261681997932631, + "pass@16": 0.3816211319972022, + "pass@32": 0.4273529590398094, + "pass@64": 0.466436005280597, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.12951440868263472, + "pass@2": 0.18884229572351371, + "pass@4": 0.25496642738145764, + "pass@8": 0.3193190721203961, + "pass@16": 0.3750415414954998, + "pass@32": 0.4213703302393265, + "pass@64": 0.46037260804896324, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 135749, + "answer_accuracy": 0.32412563034841074, + "avg_response_len": 337.4333740831296, + "resp_tokens_sum": 141322496, + "pass_at_k": { + "pass@1": 0.32412563034841074, + "pass@2": 0.4122843339092855, + "pass@4": 0.4887533857224919, + "pass@8": 0.5522224906307847, + "pass@16": 0.6041070736474479, + "pass@32": 0.6459465076021101, + "pass@64": 0.6793863785743481, + "pass@128": 0.7059902200488998 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6679811507936508, + "pass@2": 0.7739470066241724, + "pass@4": 0.8426609691645686, + "pass@8": 0.8882811236312015, + "pass@16": 0.9217558302441041, + "pass@32": 0.9453042212257897, + "pass@64": 0.958881139521916, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.6055045871559633, + "pass@2": 0.7249553775674832, + "pass@4": 0.8070858390407616, + "pass@8": 0.8604887130755426, + "pass@16": 0.8984640220204004, + "pass@32": 0.926717664021228, + "pass@64": 0.9479442969224088, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.44849537037037035, + "pass@2": 0.5667077531107221, + "pass@4": 0.6617636250329818, + "pass@8": 0.7328063948739421, + "pass@16": 0.7862427039253898, + "pass@32": 0.828873576525592, + "pass@64": 0.8648600832732204, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.3864753338278932, + "pass@2": 0.4937969491109606, + "pass@4": 0.582945069644484, + "pass@8": 0.6549811145473605, + "pass@16": 0.7130843331326536, + "pass@32": 0.7584331108720384, + "pass@64": 0.7946619410176868, + "pass@128": 0.8249258160237388 + }, + "15": { + "pass@1": 0.2794897762345679, + "pass@2": 0.37721836419753096, + "pass@4": 0.4652975750716343, + "pass@8": 0.5389522735789639, + "pass@16": 0.6011663209854563, + "pass@32": 0.65646740261954, + "pass@64": 0.705641491781776, + "pass@128": 0.7469135802469136 + }, + "16": { + "pass@1": 0.21928005162241887, + "pass@2": 0.2919903955589622, + "pass@4": 0.36209677274853913, + "pass@8": 0.42627676402045145, + "pass@16": 0.4839805580211237, + "pass@32": 0.5352047911685343, + "pass@64": 0.5795251816926208, + "pass@128": 0.616519174041298 + }, + "17": { + "pass@1": 0.16629800451807228, + "pass@2": 0.23740372414856273, + "pass@4": 0.312128824070184, + "pass@8": 0.3813466482123387, + "pass@16": 0.4392210735431096, + "pass@32": 0.48650114540932765, + "pass@64": 0.5239750597065395, + "pass@128": 0.5542168674698795 + }, + "18": { + "pass@1": 0.17905745967741934, + "pass@2": 0.24398614109728226, + "pass@4": 0.31053360880293185, + "pass@8": 0.3724229822072563, + "pass@16": 0.42312145386859323, + "pass@32": 0.46238044186642907, + "pass@64": 0.495589661049704, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.15674565709969787, + "pass@2": 0.2175691206556129, + "pass@4": 0.277465813846532, + "pass@8": 0.33085507142386955, + "pass@16": 0.37441386794026765, + "pass@32": 0.40776816099529734, + "pass@64": 0.43326206050798804, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.14738175675675674, + "pass@2": 0.21154693078905676, + "pass@4": 0.2815054278125145, + "pass@8": 0.35082918530995677, + "pass@16": 0.41341781482613965, + "pass@32": 0.464339425513764, + "pass@64": 0.5009669935181226, + "pass@128": 0.5255255255255256 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5886981907894737, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5886981907894737, + "pass@2": 0.6588905343451912, + "pass@4": 0.7121680700915433, + "pass@8": 0.751999551226857, + "pass@16": 0.782433203142648, + "pass@32": 0.8064621203669733, + "pass@64": 0.8261414616232666, + "pass@128": 0.8428947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.742609375, + "pass@2": 0.8420033218503942, + "pass@4": 0.9007638144919381, + "pass@8": 0.9314799273056936, + "pass@16": 0.9482071956036686, + "pass@32": 0.9590574677640448, + "pass@64": 0.9667893709367122, + "pass@128": 0.972 + }, + "2": { + "pass@1": 0.9700859375, + "pass@2": 0.9776165108267717, + "pass@4": 0.9835557621859767, + "pass@8": 0.9877506181583803, + "pass@16": 0.9904904799979233, + "pass@32": 0.9922714851547506, + "pass@64": 0.9936245623896852, + "pass@128": 0.995 + }, + "3": { + "pass@1": 0.98003125, + "pass@2": 0.9902121062992125, + "pass@4": 0.9938104119797523, + "pass@8": 0.9953216741446105, + "pass@16": 0.9962975091437134, + "pass@32": 0.9973458976443671, + "pass@64": 0.9986160966112356, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9623828125, + "pass@2": 0.9767733759842523, + "pass@4": 0.9844144256655416, + "pass@8": 0.988053791245531, + "pass@16": 0.9898672400273606, + "pass@32": 0.9912520377201507, + "pass@64": 0.9926321551350659, + "pass@128": 0.994 + }, + "5": { + "pass@1": 0.8991953125, + "pass@2": 0.926057086614174, + "pass@4": 0.9433322265654293, + "pass@8": 0.9550924227478708, + "pass@16": 0.9633991311457364, + "pass@32": 0.9697422400328563, + "pass@64": 0.9744431275938922, + "pass@128": 0.977 + }, + "6": { + "pass@1": 0.868984375, + "pass@2": 0.9273538385826782, + "pass@4": 0.9600408220847391, + "pass@8": 0.977614400273202, + "pass@16": 0.9874573317988508, + "pass@32": 0.9932872465112752, + "pass@64": 0.9966020228329742, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.8641640625, + "pass@2": 0.9231895915354337, + "pass@4": 0.9538450460254959, + "pass@8": 0.9702194025693823, + "pass@16": 0.9804944578875513, + "pass@32": 0.9874538270747679, + "pass@64": 0.9922845131764251, + "pass@128": 0.997 + }, + "8": { + "pass@1": 0.8278203125, + "pass@2": 0.9064554625984261, + "pass@4": 0.9468264335395588, + "pass@8": 0.9668031353266834, + "pass@16": 0.977404550975094, + "pass@32": 0.984057116904717, + "pass@64": 0.9888479828753439, + "pass@128": 0.992 + }, + "9": { + "pass@1": 0.7748984375, + "pass@2": 0.8728889025590546, + "pass@4": 0.9269883167416576, + "pass@8": 0.9551392349290369, + "pass@16": 0.9711757764625201, + "pass@32": 0.9804855544266728, + "pass@64": 0.985683056291135, + "pass@128": 0.989 + }, + "11": { + "pass@1": 0.6998515625, + "pass@2": 0.8083144685039375, + "pass@4": 0.8770636429508802, + "pass@8": 0.9181008360941916, + "pass@16": 0.943863718342336, + "pass@32": 0.960634794271071, + "pass@64": 0.9709948538828274, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.613703125, + "pass@2": 0.7349860974409431, + "pass@4": 0.8186638415823015, + "pass@8": 0.8712536742701642, + "pass@16": 0.9053730755637114, + "pass@32": 0.9288760881260807, + "pass@64": 0.9450337096196038, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.4653828125, + "pass@2": 0.5840756643700786, + "pass@4": 0.676996273247094, + "pass@8": 0.7453395469249404, + "pass@16": 0.7961783778372612, + "pass@32": 0.8359978716006594, + "pass@64": 0.8684670915496531, + "pass@128": 0.893 + }, + "14": { + "pass@1": 0.3721796875, + "pass@2": 0.4779044045275593, + "pass@4": 0.5697991311398568, + "pass@8": 0.6449930551382927, + "pass@16": 0.7034564240199858, + "pass@32": 0.7480998447079009, + "pass@64": 0.7847394786790167, + "pass@128": 0.818 + }, + "15": { + "pass@1": 0.282671875, + "pass@2": 0.3776302903543311, + "pass@4": 0.46403711426696603, + "pass@8": 0.5361525115190808, + "pass@16": 0.5960660457857662, + "pass@32": 0.6488017024787309, + "pass@64": 0.6972755900163513, + "pass@128": 0.74 + }, + "16": { + "pass@1": 0.2092421875, + "pass@2": 0.2815675442913384, + "pass@4": 0.35104423256467915, + "pass@8": 0.4147365013606016, + "pass@16": 0.473807645456793, + "pass@32": 0.5276581635478398, + "pass@64": 0.5752541026701634, + "pass@128": 0.618 + }, + "17": { + "pass@1": 0.181265625, + "pass@2": 0.2510859990157479, + "pass@4": 0.32178556730408703, + "pass@8": 0.38730375998214156, + "pass@16": 0.4440593399814598, + "pass@32": 0.4919418160037072, + "pass@64": 0.5322604937723371, + "pass@128": 0.569 + }, + "18": { + "pass@1": 0.18325, + "pass@2": 0.2523038877952756, + "pass@4": 0.32132108848893887, + "pass@8": 0.38317692535437, + "pass@16": 0.43339469992733715, + "pass@32": 0.47394288781054794, + "pass@64": 0.5101970847647627, + "pass@128": 0.543 + }, + "19": { + "pass@1": 0.15084375, + "pass@2": 0.21060839074803153, + "pass@4": 0.27216627474690647, + "pass@8": 0.3298577004903936, + "pass@16": 0.3790432843348919, + "pass@32": 0.4189484785223196, + "pass@64": 0.4522162312278507, + "pass@128": 0.479 + }, + "20": { + "pass@1": 0.136703125, + "pass@2": 0.19789320866141724, + "pass@4": 0.2647389061679785, + "pass@8": 0.32960235547558203, + "pass@16": 0.38619457541803326, + "pass@32": 0.43292576666967225, + "pass@64": 0.47072624681695024, + "pass@128": 0.502 + } + }, + "per_op_accuracy": { + "10": 0.742609375, + "2": 0.9700859375, + "3": 0.98003125, + "4": 0.9623828125, + "5": 0.8991953125, + "6": 0.868984375, + "7": 0.8641640625, + "8": 0.8278203125, + "9": 0.7748984375, + "11": 0.6998515625, + "12": 0.613703125, + "13": 0.4653828125, + "14": 0.3721796875, + "15": 0.282671875, + "16": 0.2092421875, + "17": 0.181265625, + "18": 0.18325, + "19": 0.15084375, + "20": 0.136703125 + }, + "per_op_avg_response_len": { + "10": 277.3975078125, + "2": 116.822421875, + "3": 143.9019609375, + "4": 159.7976015625, + "5": 173.9955078125, + "6": 198.53321875, + "7": 223.23425, + "8": 242.378515625, + "9": 257.896078125, + "11": 293.3047734375, + "12": 301.717640625, + "13": 312.774609375, + "14": 322.1216484375, + "15": 324.38078125, + "16": 317.1935703125, + "17": 325.424015625, + "18": 328.242046875, + "19": 320.539703125, + "20": 324.932421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.3975078125, + "2": 116.822421875, + "3": 143.9019609375, + "4": 159.7976015625, + "5": 173.9955078125, + "6": 198.53321875, + "7": 223.23425, + "8": 242.378515625, + "9": 257.896078125, + "11": 293.3047734375, + "12": 301.717640625, + "13": 312.774609375, + "14": 322.1216484375, + "15": 324.38078125, + "16": 317.1935703125, + "17": 325.424015625, + "18": 328.242046875, + "19": 320.539703125, + "20": 324.932421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 476007, + "answer_accuracy": 0.5878603679260196, + "avg_response_len": 249.26987334018338, + "resp_tokens_sum": 201840796, + "pass_at_k": { + "pass@1": 0.5878603679260196, + "pass@2": 0.6611123113024427, + "pass@4": 0.715592607199787, + "pass@8": 0.7548355615364056, + "pass@16": 0.7838130901019593, + "pass@32": 0.8065087365881829, + "pass@64": 0.8256173393051519, + "pass@128": 0.8423964590578564 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7548626474926253, + "pass@2": 0.8580390739321302, + "pass@4": 0.9158941481872283, + "pass@8": 0.9422260710100495, + "pass@16": 0.9541179241145453, + "pass@32": 0.9627354979168701, + "pass@64": 0.9700056559069217, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9521271417445483, + "pass@2": 0.9600596070351018, + "pass@4": 0.967638862899147, + "pass@8": 0.9736349720045984, + "pass@16": 0.9777725738486086, + "pass@32": 0.9807478572199139, + "pass@64": 0.983644411229528, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.9738627675840978, + "pass@2": 0.9860556779599798, + "pass@4": 0.9911855877533655, + "pass@8": 0.9935549584465483, + "pass@16": 0.9950485984443537, + "pass@32": 0.9964154468880052, + "pass@64": 0.9980977119454602, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9468017578125, + "pass@2": 0.9668841504675199, + "pass@4": 0.9788314503655793, + "pass@8": 0.98423139689816, + "pass@16": 0.9869775056679988, + "pass@32": 0.988862756689439, + "pass@64": 0.9906228273186573, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8940972222222222, + "pass@2": 0.925370656792901, + "pass@4": 0.9444052689842337, + "pass@8": 0.956068245645845, + "pass@16": 0.9641325616691484, + "pass@32": 0.9702121366978933, + "pass@64": 0.9736402664360028, + "pass@128": 0.9746031746031746 + }, + "6": { + "pass@1": 0.874625748502994, + "pass@2": 0.9345000942995885, + "pass@4": 0.9660633426622565, + "pass@8": 0.9817471947893537, + "pass@16": 0.9901813682097385, + "pass@32": 0.9945595172809533, + "pass@64": 0.996547366723047, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.8708394970414202, + "pass@2": 0.9296885919955274, + "pass@4": 0.9574697000707745, + "pass@8": 0.9709659057101204, + "pass@16": 0.9797032271920411, + "pass@32": 0.9856532059055091, + "pass@64": 0.9903706220860208, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.8371394230769231, + "pass@2": 0.921493681228548, + "pass@4": 0.9620283975921279, + "pass@8": 0.9787379257212695, + "pass@16": 0.9860862836307394, + "pass@32": 0.9901621390004783, + "pass@64": 0.993815134792471, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7703125, + "pass@2": 0.8768198264389484, + "pass@4": 0.9369461128978587, + "pass@8": 0.9678318575372462, + "pass@16": 0.9839113625433821, + "pass@32": 0.993005940915507, + "pass@64": 0.9981911215016727, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.7148326841359773, + "pass@2": 0.8247275183466801, + "pass@4": 0.8932718025154789, + "pass@8": 0.9331762977429914, + "pass@16": 0.9563250136105693, + "pass@32": 0.9703408496523926, + "pass@64": 0.9791441852491506, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.610787429378531, + "pass@2": 0.7386105504025978, + "pass@4": 0.8302062342842734, + "pass@8": 0.8869758223344203, + "pass@16": 0.9190229298739069, + "pass@32": 0.9382509994759017, + "pass@64": 0.9504507398120785, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.4938528605015674, + "pass@2": 0.6135426437192993, + "pass@4": 0.7050806219755448, + "pass@8": 0.7709119554640657, + "pass@16": 0.8185553713018926, + "pass@32": 0.8529551509357786, + "pass@64": 0.8789136115726013, + "pass@128": 0.896551724137931 + }, + "14": { + "pass@1": 0.3722984813084112, + "pass@2": 0.4832002600142271, + "pass@4": 0.5817811873983043, + "pass@8": 0.6600182956969374, + "pass@16": 0.7170059764733474, + "pass@32": 0.759723872506481, + "pass@64": 0.7973315503500162, + "pass@128": 0.8348909657320872 + }, + "15": { + "pass@1": 0.2915236632947977, + "pass@2": 0.38716340573938335, + "pass@4": 0.4724536182435286, + "pass@8": 0.5428825309662252, + "pass@16": 0.6009330341758621, + "pass@32": 0.6513234112092107, + "pass@64": 0.6978355864283469, + "pass@128": 0.7398843930635838 + }, + "16": { + "pass@1": 0.2159022177419355, + "pass@2": 0.294323485521971, + "pass@4": 0.3682993205486411, + "pass@8": 0.4345881141350131, + "pass@16": 0.49488662642953934, + "pass@32": 0.5483336279194447, + "pass@64": 0.5945119316894635, + "pass@128": 0.635483870967742 + }, + "17": { + "pass@1": 0.17767792630057805, + "pass@2": 0.24014361260297645, + "pass@4": 0.3024947796445963, + "pass@8": 0.36105936344277334, + "pass@16": 0.41425311433468764, + "pass@32": 0.4629499781277512, + "pass@64": 0.5069431688928129, + "pass@128": 0.5491329479768786 + }, + "18": { + "pass@1": 0.17955357142857142, + "pass@2": 0.24916057367829025, + "pass@4": 0.3168455503776314, + "pass@8": 0.37584058796751063, + "pass@16": 0.4238660306777394, + "pass@32": 0.4629081102911771, + "pass@64": 0.4982544518716383, + "pass@128": 0.5285714285714286 + }, + "19": { + "pass@1": 0.15315315315315314, + "pass@2": 0.21435005379395142, + "pass@4": 0.27603142118496454, + "pass@8": 0.3325890592148852, + "pass@16": 0.38104382491172395, + "pass@32": 0.4215814504369368, + "pass@64": 0.456708682959238, + "pass@128": 0.4864864864864865 + }, + "20": { + "pass@1": 0.13323479729729729, + "pass@2": 0.19331757939041397, + "pass@4": 0.2577742100931077, + "pass@8": 0.3186896897271893, + "pass@16": 0.37015786252682287, + "pass@32": 0.4131022452662304, + "pass@64": 0.4508702308920766, + "pass@128": 0.48348348348348347 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 482715, + "answer_accuracy": 0.5878738795791114, + "avg_response_len": 257.7196682579891, + "resp_tokens_sum": 211618774, + "pass_at_k": { + "pass@1": 0.5878738795791114, + "pass@2": 0.6570554679301156, + "pass@4": 0.710301292594661, + "pass@8": 0.750695245054591, + "pass@16": 0.7818350494228476, + "pass@32": 0.8066651512685107, + "pass@64": 0.8272927747768041, + "pass@128": 0.8455183164458301 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7257716641104295, + "pass@2": 0.829145482102314, + "pass@4": 0.8909685864612009, + "pass@8": 0.923828926172029, + "pass@16": 0.9424341083861851, + "pass@32": 0.9547993211002885, + "pass@64": 0.9635684888575998, + "pass@128": 0.9693251533742331 + }, + "2": { + "pass@1": 0.969961056231003, + "pass@2": 0.9800300062226267, + "pass@4": 0.9871420935604933, + "pass@8": 0.9922565877731916, + "pass@16": 0.9959260151333992, + "pass@32": 0.9983344837584649, + "pass@64": 0.9996191683758167, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9798295454545455, + "pass@2": 0.9895311381531855, + "pass@4": 0.9931646833918487, + "pass@8": 0.9950567296773937, + "pass@16": 0.9963382395784798, + "pass@32": 0.9977819591272412, + "pass@64": 0.9992064994093033, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9674093934911243, + "pass@2": 0.9813876642361278, + "pass@4": 0.9896786493589189, + "pass@8": 0.9942140522761386, + "pass@16": 0.9964544182239704, + "pass@32": 0.9981567480120942, + "pass@64": 0.9996475795537232, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8929347826086956, + "pass@2": 0.9245599395184297, + "pass@4": 0.9455650755612065, + "pass@8": 0.9585984031211746, + "pass@16": 0.9661113498054109, + "pass@32": 0.9713447438413154, + "pass@64": 0.9761022999756817, + "pass@128": 0.9797101449275363 + }, + "6": { + "pass@1": 0.8609643624641834, + "pass@2": 0.9185087313584368, + "pass@4": 0.9530109303135099, + "pass@8": 0.9730129786135173, + "pass@16": 0.9845427515918137, + "pass@32": 0.9913868693492569, + "pass@64": 0.9955681335039379, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8571396315028902, + "pass@2": 0.9151441815347509, + "pass@4": 0.9492684707764129, + "pass@8": 0.9707522712002761, + "pass@16": 0.9846804474521327, + "pass@32": 0.9932585974949043, + "pass@64": 0.9976034786090924, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8400528169014084, + "pass@2": 0.9146213402462021, + "pass@4": 0.9519286311042101, + "pass@8": 0.9705870784036298, + "pass@16": 0.9806850356855231, + "pass@32": 0.9870184734714211, + "pass@64": 0.9913704266572482, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.7795424775449101, + "pass@2": 0.8696370791880805, + "pass@4": 0.9201581606191439, + "pass@8": 0.9472707355109283, + "pass@16": 0.9634659109310358, + "pass@32": 0.9732413865436497, + "pass@64": 0.979391838196961, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.7141613328313253, + "pass@2": 0.8234709260269424, + "pass@4": 0.8924713595665003, + "pass@8": 0.9303646055635199, + "pass@16": 0.9515900662979302, + "pass@32": 0.9648603754748566, + "pass@64": 0.9738234865676904, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.625342868338558, + "pass@2": 0.7412462512033179, + "pass@4": 0.8177234021295925, + "pass@8": 0.864841454508076, + "pass@16": 0.8973079096792422, + "pass@32": 0.9206851353500622, + "pass@64": 0.9360387543338561, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.45526960784313725, + "pass@2": 0.573507700324224, + "pass@4": 0.6657258832666921, + "pass@8": 0.7338637568424262, + "pass@16": 0.7852004994961729, + "pass@32": 0.8273113155682195, + "pass@64": 0.8624061133822667, + "pass@128": 0.8907563025210085 + }, + "14": { + "pass@1": 0.35798154239766083, + "pass@2": 0.45727353570935186, + "pass@4": 0.545599068756756, + "pass@8": 0.6210483819213921, + "pass@16": 0.68125171142204, + "pass@32": 0.7270073779808284, + "pass@64": 0.7631432072915221, + "pass@128": 0.7953216374269005 + }, + "15": { + "pass@1": 0.2765151515151515, + "pass@2": 0.36803939990455725, + "pass@4": 0.45397499403483615, + "pass@8": 0.5263473308035848, + "pass@16": 0.5859555392776711, + "pass@32": 0.6386314053079184, + "pass@64": 0.688474647802565, + "pass@128": 0.7333333333333333 + }, + "16": { + "pass@1": 0.19366542022792022, + "pass@2": 0.26023509881777596, + "pass@4": 0.3251300206063985, + "pass@8": 0.38605801417611135, + "pass@16": 0.44536576095291924, + "pass@32": 0.502109158651507, + "pass@64": 0.5541207044234497, + "pass@128": 0.603988603988604 + }, + "17": { + "pass@1": 0.20055318322981366, + "pass@2": 0.2769511572602337, + "pass@4": 0.3524708197383709, + "pass@8": 0.4216463758523163, + "pass@16": 0.4810756708240572, + "pass@32": 0.5287041717254924, + "pass@64": 0.5680075699156929, + "pass@128": 0.6055900621118012 + }, + "18": { + "pass@1": 0.19087775735294119, + "pass@2": 0.26312348019916604, + "pass@4": 0.3357639033172321, + "pass@8": 0.400534103180857, + "pass@16": 0.45257040732607284, + "pass@32": 0.49584444773542125, + "pass@64": 0.5358095049537671, + "pass@128": 0.5735294117647058 + }, + "19": { + "pass@1": 0.14274088541666666, + "pass@2": 0.20004298779058863, + "pass@4": 0.26311496163425996, + "pass@8": 0.3261681997932631, + "pass@16": 0.3816211319972022, + "pass@32": 0.4273529590398094, + "pass@64": 0.466436005280597, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.12951440868263472, + "pass@2": 0.18884229572351371, + "pass@4": 0.25496642738145764, + "pass@8": 0.3193190721203961, + "pass@16": 0.3750415414954998, + "pass@32": 0.4213703302393265, + "pass@64": 0.46037260804896324, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 472992, + "answer_accuracy": 0.5903898386323694, + "avg_response_len": 277.1106219543857, + "resp_tokens_sum": 222007729, + "pass_at_k": { + "pass@1": 0.5903898386323694, + "pass@2": 0.6585257779977994, + "pass@4": 0.7106201799966714, + "pass@8": 0.750469997444581, + "pass@16": 0.7816516072340053, + "pass@32": 0.8062069138724718, + "pass@64": 0.8254911858450941, + "pass@128": 0.8407093784949673 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7465951492537314, + "pass@2": 0.8382885033493949, + "pass@4": 0.8949848927466156, + "pass@8": 0.9280509232871795, + "pass@16": 0.9478438805819164, + "pass@32": 0.9594792695269607, + "pass@64": 0.9666690334826524, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9866741071428572, + "pass@2": 0.9914500140607426, + "pass@4": 0.9947826811827094, + "pass@8": 0.9964610707643552, + "pass@16": 0.9970452137532337, + "pass@32": 0.9971410795160666, + "pass@64": 0.9971428571124661, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9861060495626822, + "pass@2": 0.9948298076260877, + "pass@4": 0.9969340503355447, + "pass@8": 0.9972608802887731, + "pass@16": 0.9974489749023661, + "pass@32": 0.997813411078717, + "pass@64": 0.9985422740524781, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9719937865497076, + "pass@2": 0.981466132522908, + "pass@4": 0.9844356083778999, + "pass@8": 0.9855420893824135, + "pass@16": 0.9860609498652024, + "pass@32": 0.9866636688638668, + "pass@64": 0.9875788552746696, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.9102711397058824, + "pass@2": 0.9282122076192687, + "pass@4": 0.940072399314056, + "pass@8": 0.9506308126253655, + "pass@16": 0.9599675251090849, + "pass@32": 0.9676808245522525, + "pass@64": 0.9735033828674721, + "pass@128": 0.9764705882352941 + }, + "6": { + "pass@1": 0.871870070977918, + "pass@2": 0.9295623338880746, + "pass@4": 0.9614348610603484, + "pass@8": 0.9783258917269404, + "pass@16": 0.9877960079219414, + "pass@32": 0.9940389600521968, + "pass@64": 0.99779786673376, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8647151898734177, + "pass@2": 0.9250473437655732, + "pass@4": 0.9549791028890373, + "pass@8": 0.9688374702660314, + "pass@16": 0.9767573806272281, + "pass@32": 0.9830239517261676, + "pass@64": 0.9885077193436845, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.8060482357357357, + "pass@2": 0.883660235038188, + "pass@4": 0.9271438722186747, + "pass@8": 0.9515870560611387, + "pass@16": 0.9657731015433726, + "pass@32": 0.975180094457099, + "pass@64": 0.9815049830533622, + "pass@128": 0.984984984984985 + }, + "9": { + "pass@1": 0.7751456993569131, + "pass@2": 0.8718941470238243, + "pass@4": 0.9229569807592373, + "pass@8": 0.9491012856677312, + "pass@16": 0.9649184196419777, + "pass@32": 0.9739737051321179, + "pass@64": 0.9781618463030722, + "pass@128": 0.9807073954983923 + }, + "11": { + "pass@1": 0.6679811507936508, + "pass@2": 0.7739470066241724, + "pass@4": 0.8426609691645686, + "pass@8": 0.8882811236312015, + "pass@16": 0.9217558302441041, + "pass@32": 0.9453042212257897, + "pass@64": 0.958881139521916, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.6055045871559633, + "pass@2": 0.7249553775674832, + "pass@4": 0.8070858390407616, + "pass@8": 0.8604887130755426, + "pass@16": 0.8984640220204004, + "pass@32": 0.926717664021228, + "pass@64": 0.9479442969224088, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.44849537037037035, + "pass@2": 0.5667077531107221, + "pass@4": 0.6617636250329818, + "pass@8": 0.7328063948739421, + "pass@16": 0.7862427039253898, + "pass@32": 0.828873576525592, + "pass@64": 0.8648600832732204, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.3864753338278932, + "pass@2": 0.4937969491109606, + "pass@4": 0.582945069644484, + "pass@8": 0.6549811145473605, + "pass@16": 0.7130843331326536, + "pass@32": 0.7584331108720384, + "pass@64": 0.7946619410176868, + "pass@128": 0.8249258160237388 + }, + "15": { + "pass@1": 0.2794897762345679, + "pass@2": 0.37721836419753096, + "pass@4": 0.4652975750716343, + "pass@8": 0.5389522735789639, + "pass@16": 0.6011663209854563, + "pass@32": 0.65646740261954, + "pass@64": 0.705641491781776, + "pass@128": 0.7469135802469136 + }, + "16": { + "pass@1": 0.21928005162241887, + "pass@2": 0.2919903955589622, + "pass@4": 0.36209677274853913, + "pass@8": 0.42627676402045145, + "pass@16": 0.4839805580211237, + "pass@32": 0.5352047911685343, + "pass@64": 0.5795251816926208, + "pass@128": 0.616519174041298 + }, + "17": { + "pass@1": 0.16629800451807228, + "pass@2": 0.23740372414856273, + "pass@4": 0.312128824070184, + "pass@8": 0.3813466482123387, + "pass@16": 0.4392210735431096, + "pass@32": 0.48650114540932765, + "pass@64": 0.5239750597065395, + "pass@128": 0.5542168674698795 + }, + "18": { + "pass@1": 0.17905745967741934, + "pass@2": 0.24398614109728226, + "pass@4": 0.31053360880293185, + "pass@8": 0.3724229822072563, + "pass@16": 0.42312145386859323, + "pass@32": 0.46238044186642907, + "pass@64": 0.495589661049704, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.15674565709969787, + "pass@2": 0.2175691206556129, + "pass@4": 0.277465813846532, + "pass@8": 0.33085507142386955, + "pass@16": 0.37441386794026765, + "pass@32": 0.40776816099529734, + "pass@64": 0.43326206050798804, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.14738175675675674, + "pass@2": 0.21154693078905676, + "pass@4": 0.2815054278125145, + "pass@8": 0.35082918530995677, + "pass@16": 0.41341781482613965, + "pass@32": 0.464339425513764, + "pass@64": 0.5009669935181226, + "pass@128": 0.5255255255255256 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..682e0d2ecd594d94b51c60b861ebdde13b76c3b9 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16", "results_dir": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/checkpoint-21546_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 664863, "value_mismatch": 160010, "dependency_mismatch": 422962, "answer_mismatch": 141566, "extra_nodes": 122500, "missing_nodes": 191203, "perfect_with_extra": 4744, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "dependency_mismatch": 130390, "missing_nodes": 47874, "value_mismatch": 57044, "answer_mismatch": 47899, "perfect": 224111, "extra_nodes": 23121, "perfect_with_extra": 1253}, "teachers_in_school": {"total": 390656, "perfect": 225323, "dependency_mismatch": 143404, "value_mismatch": 53320, "answer_mismatch": 48721, "missing_nodes": 55728, "extra_nodes": 28712, "perfect_with_extra": 1265}, "movie_festival_awards": {"total": 382336, "perfect": 215429, "value_mismatch": 49646, "dependency_mismatch": 149168, "answer_mismatch": 44946, "missing_nodes": 87601, "extra_nodes": 70667, "perfect_with_extra": 2226}}, "results_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/checkpoint-21546_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/checkpoint-21546_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 75414, "value_mismatch": 952071, "dependency_mismatch": 1135057, "answer_mismatch": 858153, "extra_nodes": 250125, "missing_nodes": 930031, "perfect_with_extra": 423, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 320370, "dependency_mismatch": 379019, "answer_mismatch": 285649, "extra_nodes": 43111, "missing_nodes": 305568, "perfect": 27563, "perfect_with_extra": 174}, "movie_festival_awards": {"total": 418816, "dependency_mismatch": 374936, "extra_nodes": 144331, "missing_nodes": 316286, "value_mismatch": 311900, "answer_mismatch": 283060, "perfect": 22445, "perfect_with_extra": 213}, "teachers_in_school": {"total": 430464, "perfect": 25406, "dependency_mismatch": 381102, "value_mismatch": 319801, "answer_mismatch": 289444, "missing_nodes": 308177, "extra_nodes": 62683, "perfect_with_extra": 36}}, "results_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/checkpoint-21546_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 740277, "value_mismatch": 1112081, "dependency_mismatch": 1558019, "answer_mismatch": 999719, "extra_nodes": 372625, "missing_nodes": 1121234, "perfect_with_extra": 5167, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/block16_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "dependency_mismatch": 509409, "missing_nodes": 353442, "value_mismatch": 377414, "answer_mismatch": 333548, "perfect": 251674, "extra_nodes": 66232, "perfect_with_extra": 1427}, "teachers_in_school": {"total": 821120, "perfect": 250729, "dependency_mismatch": 524506, "value_mismatch": 373121, "answer_mismatch": 338165, "missing_nodes": 363905, "extra_nodes": 91395, "perfect_with_extra": 1301}, "movie_festival_awards": {"total": 801152, "perfect": 237874, "value_mismatch": 361546, "dependency_mismatch": 524104, "answer_mismatch": 328006, "missing_nodes": 403887, "extra_nodes": 214998, "perfect_with_extra": 2439}}}} diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/results.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..e171edc5869ff1d1062a8e5491c3da68ae77e1b9 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21546_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.8847352024922118, + "pass@2": 0.8878504672897196, + "pass@4": 0.9003115264797508, + "pass@8": 0.9096573208722741, + "pass@16": 0.9127725856697819, + "pass@32": 0.9158878504672897, + "pass@64": 0.9252336448598131, + "pass@128": 0.9345794392523364 + }, + "3": { + "pass@1": 0.8807339449541285, + "pass@2": 0.8929663608562691, + "pass@4": 0.8990825688073395, + "pass@8": 0.908256880733945, + "pass@16": 0.9113149847094801, + "pass@32": 0.9143730886850153, + "pass@64": 0.9296636085626911, + "pass@128": 0.9357798165137615 + }, + "4": { + "pass@1": 0.775, + "pass@2": 0.79375, + "pass@4": 0.81875, + "pass@8": 0.84375, + "pass@16": 0.865625, + "pass@32": 0.878125, + "pass@64": 0.884375, + "pass@128": 0.890625 + }, + "5": { + "pass@1": 0.6, + "pass@2": 0.6793650793650794, + "pass@4": 0.7555555555555555, + "pass@8": 0.8253968253968254, + "pass@16": 0.8761904761904762, + "pass@32": 0.9015873015873016, + "pass@64": 0.9238095238095239, + "pass@128": 0.9333333333333333 + }, + "6": { + "pass@1": 0.6676646706586826, + "pass@2": 0.7574850299401198, + "pass@4": 0.8023952095808383, + "pass@8": 0.8473053892215568, + "pass@16": 0.8712574850299402, + "pass@32": 0.8952095808383234, + "pass@64": 0.9161676646706587, + "pass@128": 0.9251497005988024 + }, + "7": { + "pass@1": 0.4970414201183432, + "pass@2": 0.621301775147929, + "pass@4": 0.7396449704142012, + "pass@8": 0.7899408284023669, + "pass@16": 0.8402366863905325, + "pass@32": 0.8816568047337278, + "pass@64": 0.9023668639053254, + "pass@128": 0.9201183431952663 + }, + "8": { + "pass@1": 0.4583333333333333, + "pass@2": 0.5865384615384616, + "pass@4": 0.7051282051282052, + "pass@8": 0.7724358974358975, + "pass@16": 0.8173076923076923, + "pass@32": 0.8685897435897436, + "pass@64": 0.9006410256410257, + "pass@128": 0.9038461538461539 + }, + "9": { + "pass@1": 0.27605633802816903, + "pass@2": 0.4056338028169014, + "pass@4": 0.5211267605633803, + "pass@8": 0.6366197183098592, + "pass@16": 0.7323943661971831, + "pass@32": 0.7859154929577464, + "pass@64": 0.8338028169014085, + "pass@128": 0.8450704225352113 + }, + "10": { + "pass@1": 0.3215339233038348, + "pass@2": 0.4365781710914454, + "pass@4": 0.528023598820059, + "pass@8": 0.6194690265486725, + "pass@16": 0.6755162241887905, + "pass@32": 0.7256637168141593, + "pass@64": 0.7640117994100295, + "pass@128": 0.7876106194690266 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.8847352024922118, + "3": 0.8807339449541285, + "4": 0.775, + "5": 0.6, + "6": 0.6676646706586826, + "7": 0.4970414201183432, + "8": 0.4583333333333333, + "9": 0.27605633802816903, + "10": 0.3215339233038348 + }, + "pass@2": { + "2": 0.8878504672897196, + "3": 0.8929663608562691, + "4": 0.79375, + "5": 0.6793650793650794, + "6": 0.7574850299401198, + "7": 0.621301775147929, + "8": 0.5865384615384616, + "9": 0.4056338028169014, + "10": 0.4365781710914454 + }, + "pass@4": { + "2": 0.9003115264797508, + "3": 0.8990825688073395, + "4": 0.81875, + "5": 0.7555555555555555, + "6": 0.8023952095808383, + "7": 0.7396449704142012, + "8": 0.7051282051282052, + "9": 0.5211267605633803, + "10": 0.528023598820059 + }, + "pass@8": { + "2": 0.9096573208722741, + "3": 0.908256880733945, + "4": 0.84375, + "5": 0.8253968253968254, + "6": 0.8473053892215568, + "7": 0.7899408284023669, + "8": 0.7724358974358975, + "9": 0.6366197183098592, + "10": 0.6194690265486725 + }, + "pass@16": { + "2": 0.9127725856697819, + "3": 0.9113149847094801, + "4": 0.865625, + "5": 0.8761904761904762, + "6": 0.8712574850299402, + "7": 0.8402366863905325, + "8": 0.8173076923076923, + "9": 0.7323943661971831, + "10": 0.6755162241887905 + }, + "pass@32": { + "2": 0.9158878504672897, + "3": 0.9143730886850153, + "4": 0.878125, + "5": 0.9015873015873016, + "6": 0.8952095808383234, + "7": 0.8816568047337278, + "8": 0.8685897435897436, + "9": 0.7859154929577464, + "10": 0.7256637168141593 + }, + "pass@64": { + "2": 0.9252336448598131, + "3": 0.9296636085626911, + "4": 0.884375, + "5": 0.9238095238095239, + "6": 0.9161676646706587, + "7": 0.9023668639053254, + "8": 0.9006410256410257, + "9": 0.8338028169014085, + "10": 0.7640117994100295 + }, + "pass@128": { + "2": 0.9345794392523364, + "3": 0.9357798165137615, + "4": 0.890625, + "5": 0.9333333333333333, + "6": 0.9251497005988024, + "7": 0.9201183431952663, + "8": 0.9038461538461539, + "9": 0.8450704225352113, + "10": 0.7876106194690266 + } + }, + "total_pass_at_k": { + "pass@1": 0.5910165484633569, + "pass@2": 0.6697061803444783, + "pass@4": 0.7379263762242486, + "pass@8": 0.7922998986828774, + "pass@16": 0.8318135764944276, + "pass@32": 0.8615332657885849, + "pass@64": 0.8855116514690983, + "pass@128": 0.8963188112124282 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.9342857142857143, + "pass@2": 0.9371428571428572, + "pass@4": 0.9428571428571428, + "pass@8": 0.9485714285714286, + "pass@16": 0.9485714285714286, + "pass@32": 0.9514285714285714, + "pass@64": 0.9514285714285714, + "pass@128": 0.9514285714285714 + }, + "3": { + "pass@1": 0.8862973760932945, + "pass@2": 0.8979591836734694, + "pass@4": 0.9037900874635568, + "pass@8": 0.9067055393586005, + "pass@16": 0.9067055393586005, + "pass@32": 0.9096209912536443, + "pass@64": 0.9154518950437318, + "pass@128": 0.9212827988338192 + }, + "4": { + "pass@1": 0.7865497076023392, + "pass@2": 0.8070175438596491, + "pass@4": 0.8304093567251462, + "pass@8": 0.8508771929824561, + "pass@16": 0.8567251461988304, + "pass@32": 0.8654970760233918, + "pass@64": 0.8830409356725146, + "pass@128": 0.8888888888888888 + }, + "5": { + "pass@1": 0.6176470588235294, + "pass@2": 0.6676470588235294, + "pass@4": 0.7235294117647059, + "pass@8": 0.7617647058823529, + "pass@16": 0.7941176470588235, + "pass@32": 0.8264705882352941, + "pass@64": 0.8441176470588235, + "pass@128": 0.8588235294117647 + }, + "6": { + "pass@1": 0.5362776025236593, + "pass@2": 0.5930599369085173, + "pass@4": 0.6498422712933754, + "pass@8": 0.6908517350157729, + "pass@16": 0.7160883280757098, + "pass@32": 0.7444794952681388, + "pass@64": 0.7823343848580442, + "pass@128": 0.807570977917981 + }, + "7": { + "pass@1": 0.4525316455696203, + "pass@2": 0.5348101265822784, + "pass@4": 0.5981012658227848, + "pass@8": 0.6740506329113924, + "pass@16": 0.740506329113924, + "pass@32": 0.7784810126582279, + "pass@64": 0.8069620253164557, + "pass@128": 0.8481012658227848 + }, + "8": { + "pass@1": 0.33933933933933935, + "pass@2": 0.45645645645645644, + "pass@4": 0.5495495495495496, + "pass@8": 0.6156156156156156, + "pass@16": 0.6666666666666666, + "pass@32": 0.7027027027027027, + "pass@64": 0.7507507507507507, + "pass@128": 0.7927927927927928 + }, + "9": { + "pass@1": 0.2508038585209003, + "pass@2": 0.3086816720257235, + "pass@4": 0.4212218649517685, + "pass@8": 0.4919614147909968, + "pass@16": 0.572347266881029, + "pass@32": 0.6591639871382636, + "pass@64": 0.729903536977492, + "pass@128": 0.7556270096463023 + }, + "10": { + "pass@1": 0.2537313432835821, + "pass@2": 0.33432835820895523, + "pass@4": 0.417910447761194, + "pass@8": 0.4955223880597015, + "pass@16": 0.5671641791044776, + "pass@32": 0.6298507462686567, + "pass@64": 0.6597014925373135, + "pass@128": 0.6716417910447762 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.9342857142857143, + "3": 0.8862973760932945, + "4": 0.7865497076023392, + "5": 0.6176470588235294, + "6": 0.5362776025236593, + "7": 0.4525316455696203, + "8": 0.33933933933933935, + "9": 0.2508038585209003, + "10": 0.2537313432835821 + }, + "pass@2": { + "2": 0.9371428571428572, + "3": 0.8979591836734694, + "4": 0.8070175438596491, + "5": 0.6676470588235294, + "6": 0.5930599369085173, + "7": 0.5348101265822784, + "8": 0.45645645645645644, + "9": 0.3086816720257235, + "10": 0.33432835820895523 + }, + "pass@4": { + "2": 0.9428571428571428, + "3": 0.9037900874635568, + "4": 0.8304093567251462, + "5": 0.7235294117647059, + "6": 0.6498422712933754, + "7": 0.5981012658227848, + "8": 0.5495495495495496, + "9": 0.4212218649517685, + "10": 0.417910447761194 + }, + "pass@8": { + "2": 0.9485714285714286, + "3": 0.9067055393586005, + "4": 0.8508771929824561, + "5": 0.7617647058823529, + "6": 0.6908517350157729, + "7": 0.6740506329113924, + "8": 0.6156156156156156, + "9": 0.4919614147909968, + "10": 0.4955223880597015 + }, + "pass@16": { + "2": 0.9485714285714286, + "3": 0.9067055393586005, + "4": 0.8567251461988304, + "5": 0.7941176470588235, + "6": 0.7160883280757098, + "7": 0.740506329113924, + "8": 0.6666666666666666, + "9": 0.572347266881029, + "10": 0.5671641791044776 + }, + "pass@32": { + "2": 0.9514285714285714, + "3": 0.9096209912536443, + "4": 0.8654970760233918, + "5": 0.8264705882352941, + "6": 0.7444794952681388, + "7": 0.7784810126582279, + "8": 0.7027027027027027, + "9": 0.6591639871382636, + "10": 0.6298507462686567 + }, + "pass@64": { + "2": 0.9514285714285714, + "3": 0.9154518950437318, + "4": 0.8830409356725146, + "5": 0.8441176470588235, + "6": 0.7823343848580442, + "7": 0.8069620253164557, + "8": 0.7507507507507507, + "9": 0.729903536977492, + "10": 0.6597014925373135 + }, + "pass@128": { + "2": 0.9514285714285714, + "3": 0.9212827988338192, + "4": 0.8888888888888888, + "5": 0.8588235294117647, + "6": 0.807570977917981, + "7": 0.8481012658227848, + "8": 0.7927927927927928, + "9": 0.7556270096463023, + "10": 0.6716417910447762 + } + }, + "total_pass_at_k": { + "pass@1": 0.56879812520924, + "pass@2": 0.6213592233009708, + "pass@4": 0.675929025778373, + "pass@8": 0.7194509541345832, + "pass@16": 0.7556076330766656, + "pass@32": 0.7880816873116839, + "pass@64": 0.8158687646468028, + "pass@128": 0.8346166722464011 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.8966565349544073, + "pass@2": 0.9148936170212766, + "pass@4": 0.9209726443768997, + "pass@8": 0.9331306990881459, + "pass@16": 0.9361702127659575, + "pass@32": 0.9483282674772037, + "pass@64": 0.9544072948328267, + "pass@128": 0.9574468085106383 + }, + "3": { + "pass@1": 0.8696969696969697, + "pass@2": 0.8787878787878788, + "pass@4": 0.8878787878787879, + "pass@8": 0.8909090909090909, + "pass@16": 0.8939393939393939, + "pass@32": 0.9030303030303031, + "pass@64": 0.906060606060606, + "pass@128": 0.9090909090909091 + }, + "4": { + "pass@1": 0.7899408284023669, + "pass@2": 0.8224852071005917, + "pass@4": 0.8461538461538461, + "pass@8": 0.8846153846153846, + "pass@16": 0.9142011834319527, + "pass@32": 0.9319526627218935, + "pass@64": 0.9378698224852071, + "pass@128": 0.9467455621301775 + }, + "5": { + "pass@1": 0.5594202898550724, + "pass@2": 0.6289855072463768, + "pass@4": 0.7246376811594203, + "pass@8": 0.7971014492753623, + "pass@16": 0.8376811594202899, + "pass@32": 0.8579710144927536, + "pass@64": 0.8869565217391304, + "pass@128": 0.9101449275362319 + }, + "6": { + "pass@1": 0.6246418338108882, + "pass@2": 0.6848137535816619, + "pass@4": 0.7507163323782235, + "pass@8": 0.8223495702005731, + "pass@16": 0.8710601719197708, + "pass@32": 0.8825214899713467, + "pass@64": 0.8997134670487106, + "pass@128": 0.9255014326647565 + }, + "7": { + "pass@1": 0.4653179190751445, + "pass@2": 0.5838150289017341, + "pass@4": 0.6647398843930635, + "pass@8": 0.7658959537572254, + "pass@16": 0.8121387283236994, + "pass@32": 0.846820809248555, + "pass@64": 0.8930635838150289, + "pass@128": 0.9219653179190751 + }, + "8": { + "pass@1": 0.38028169014084506, + "pass@2": 0.5126760563380282, + "pass@4": 0.6197183098591549, + "pass@8": 0.7070422535211267, + "pass@16": 0.7774647887323943, + "pass@32": 0.8140845070422535, + "pass@64": 0.847887323943662, + "pass@128": 0.8676056338028169 + }, + "9": { + "pass@1": 0.2964071856287425, + "pass@2": 0.4281437125748503, + "pass@4": 0.5179640718562875, + "pass@8": 0.6317365269461078, + "pass@16": 0.7155688622754491, + "pass@32": 0.7784431137724551, + "pass@64": 0.8143712574850299, + "pass@128": 0.8532934131736527 + }, + "10": { + "pass@1": 0.27300613496932513, + "pass@2": 0.3619631901840491, + "pass@4": 0.4539877300613497, + "pass@8": 0.5613496932515337, + "pass@16": 0.6503067484662577, + "pass@32": 0.7147239263803681, + "pass@64": 0.7638036809815951, + "pass@128": 0.7883435582822086 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.8966565349544073, + "3": 0.8696969696969697, + "4": 0.7899408284023669, + "5": 0.5594202898550724, + "6": 0.6246418338108882, + "7": 0.4653179190751445, + "8": 0.38028169014084506, + "9": 0.2964071856287425, + "10": 0.27300613496932513 + }, + "pass@2": { + "2": 0.9148936170212766, + "3": 0.8787878787878788, + "4": 0.8224852071005917, + "5": 0.6289855072463768, + "6": 0.6848137535816619, + "7": 0.5838150289017341, + "8": 0.5126760563380282, + "9": 0.4281437125748503, + "10": 0.3619631901840491 + }, + "pass@4": { + "2": 0.9209726443768997, + "3": 0.8878787878787879, + "4": 0.8461538461538461, + "5": 0.7246376811594203, + "6": 0.7507163323782235, + "7": 0.6647398843930635, + "8": 0.6197183098591549, + "9": 0.5179640718562875, + "10": 0.4539877300613497 + }, + "pass@8": { + "2": 0.9331306990881459, + "3": 0.8909090909090909, + "4": 0.8846153846153846, + "5": 0.7971014492753623, + "6": 0.8223495702005731, + "7": 0.7658959537572254, + "8": 0.7070422535211267, + "9": 0.6317365269461078, + "10": 0.5613496932515337 + }, + "pass@16": { + "2": 0.9361702127659575, + "3": 0.8939393939393939, + "4": 0.9142011834319527, + "5": 0.8376811594202899, + "6": 0.8710601719197708, + "7": 0.8121387283236994, + "8": 0.7774647887323943, + "9": 0.7155688622754491, + "10": 0.6503067484662577 + }, + "pass@32": { + "2": 0.9483282674772037, + "3": 0.9030303030303031, + "4": 0.9319526627218935, + "5": 0.8579710144927536, + "6": 0.8825214899713467, + "7": 0.846820809248555, + "8": 0.8140845070422535, + "9": 0.7784431137724551, + "10": 0.7147239263803681 + }, + "pass@64": { + "2": 0.9544072948328267, + "3": 0.906060606060606, + "4": 0.9378698224852071, + "5": 0.8869565217391304, + "6": 0.8997134670487106, + "7": 0.8930635838150289, + "8": 0.847887323943662, + "9": 0.8143712574850299, + "10": 0.7638036809815951 + }, + "pass@128": { + "2": 0.9574468085106383, + "3": 0.9090909090909091, + "4": 0.9467455621301775, + "5": 0.9101449275362319, + "6": 0.9255014326647565, + "7": 0.9219653179190751, + "8": 0.8676056338028169, + "9": 0.8532934131736527, + "10": 0.7883435582822086 + } + }, + "total_pass_at_k": { + "pass@1": 0.5714285714285714, + "pass@2": 0.645478374836173, + "pass@4": 0.7093709043250328, + "pass@8": 0.7771952817824378, + "pass@16": 0.823394495412844, + "pass@32": 0.8532110091743119, + "pass@64": 0.8784403669724771, + "pass@128": 0.8980996068152032 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 0.906, + "pass@2": 0.914, + "pass@4": 0.922, + "pass@8": 0.931, + "pass@16": 0.933, + "pass@32": 0.939, + "pass@64": 0.944, + "pass@128": 0.948 + }, + "3": { + "pass@1": 0.879, + "pass@2": 0.89, + "pass@4": 0.897, + "pass@8": 0.902, + "pass@16": 0.904, + "pass@32": 0.909, + "pass@64": 0.917, + "pass@128": 0.922 + }, + "4": { + "pass@1": 0.784, + "pass@2": 0.808, + "pass@4": 0.832, + "pass@8": 0.86, + "pass@16": 0.879, + "pass@32": 0.892, + "pass@64": 0.902, + "pass@128": 0.909 + }, + "5": { + "pass@1": 0.592, + "pass@2": 0.658, + "pass@4": 0.734, + "pass@8": 0.794, + "pass@16": 0.835, + "pass@32": 0.861, + "pass@64": 0.884, + "pass@128": 0.9 + }, + "6": { + "pass@1": 0.611, + "pass@2": 0.68, + "pass@4": 0.736, + "pass@8": 0.789, + "pass@16": 0.822, + "pass@32": 0.843, + "pass@64": 0.868, + "pass@128": 0.888 + }, + "7": { + "pass@1": 0.472, + "pass@2": 0.581, + "pass@4": 0.669, + "pass@8": 0.745, + "pass@16": 0.799, + "pass@32": 0.837, + "pass@64": 0.869, + "pass@128": 0.898 + }, + "8": { + "pass@1": 0.391, + "pass@2": 0.517, + "pass@4": 0.623, + "pass@8": 0.697, + "pass@16": 0.753, + "pass@32": 0.794, + "pass@64": 0.832, + "pass@128": 0.854 + }, + "9": { + "pass@1": 0.275, + "pass@2": 0.383, + "pass@4": 0.489, + "pass@8": 0.59, + "pass@16": 0.677, + "pass@32": 0.744, + "pass@64": 0.795, + "pass@128": 0.82 + }, + "10": { + "pass@1": 0.283, + "pass@2": 0.378, + "pass@4": 0.467, + "pass@8": 0.559, + "pass@16": 0.631, + "pass@32": 0.69, + "pass@64": 0.729, + "pass@128": 0.749 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 0.906, + "3": 0.879, + "4": 0.784, + "5": 0.592, + "6": 0.611, + "7": 0.472, + "8": 0.391, + "9": 0.275, + "10": 0.283 + }, + "pass@2": { + "2": 0.914, + "3": 0.89, + "4": 0.808, + "5": 0.658, + "6": 0.68, + "7": 0.581, + "8": 0.517, + "9": 0.383, + "10": 0.378 + }, + "pass@4": { + "2": 0.922, + "3": 0.897, + "4": 0.832, + "5": 0.734, + "6": 0.736, + "7": 0.669, + "8": 0.623, + "9": 0.489, + "10": 0.467 + }, + "pass@8": { + "2": 0.931, + "3": 0.902, + "4": 0.86, + "5": 0.794, + "6": 0.789, + "7": 0.745, + "8": 0.697, + "9": 0.59, + "10": 0.559 + }, + "pass@16": { + "2": 0.933, + "3": 0.904, + "4": 0.879, + "5": 0.835, + "6": 0.822, + "7": 0.799, + "8": 0.753, + "9": 0.677, + "10": 0.631 + }, + "pass@32": { + "2": 0.939, + "3": 0.909, + "4": 0.892, + "5": 0.861, + "6": 0.843, + "7": 0.837, + "8": 0.794, + "9": 0.744, + "10": 0.69 + }, + "pass@64": { + "2": 0.944, + "3": 0.917, + "4": 0.902, + "5": 0.884, + "6": 0.868, + "7": 0.869, + "8": 0.832, + "9": 0.795, + "10": 0.729 + }, + "pass@128": { + "2": 0.948, + "3": 0.922, + "4": 0.909, + "5": 0.9, + "6": 0.888, + "7": 0.898, + "8": 0.854, + "9": 0.82, + "10": 0.749 + } + }, + "total_pass_at_k": { + "pass@1": 0.577, + "pass@2": 0.6454444444444445, + "pass@4": 0.7076666666666667, + "pass@8": 0.763, + "pass@16": 0.8036666666666666, + "pass@32": 0.8343333333333334, + "pass@64": 0.86, + "pass@128": 0.8764444444444445 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21546_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.20396600566572237, + "pass@2": 0.31444759206798867, + "pass@4": 0.4192634560906516, + "pass@8": 0.48725212464589235, + "pass@16": 0.5637393767705382, + "pass@32": 0.6345609065155807, + "pass@64": 0.6997167138810199, + "pass@128": 0.7365439093484419 + }, + "12": { + "pass@1": 0.24293785310734464, + "pass@2": 0.3192090395480226, + "pass@4": 0.4067796610169492, + "pass@8": 0.4858757062146893, + "pass@16": 0.5508474576271186, + "pass@32": 0.6129943502824858, + "pass@64": 0.6694915254237288, + "pass@128": 0.6892655367231638 + }, + "13": { + "pass@1": 0.11912225705329153, + "pass@2": 0.16927899686520376, + "pass@4": 0.2225705329153605, + "pass@8": 0.26959247648902823, + "pass@16": 0.36363636363636365, + "pass@32": 0.4263322884012539, + "pass@64": 0.4827586206896552, + "pass@128": 0.5297805642633229 + }, + "14": { + "pass@1": 0.03426791277258567, + "pass@2": 0.06230529595015576, + "pass@4": 0.09968847352024922, + "pass@8": 0.12461059190031153, + "pass@16": 0.1838006230529595, + "pass@32": 0.22118380062305296, + "pass@64": 0.26791277258566976, + "pass@128": 0.30218068535825543 + }, + "15": { + "pass@1": 0.017341040462427744, + "pass@2": 0.031791907514450865, + "pass@4": 0.06069364161849711, + "pass@8": 0.08092485549132948, + "pass@16": 0.10404624277456648, + "pass@32": 0.12138728323699421, + "pass@64": 0.16184971098265896, + "pass@128": 0.20520231213872833 + }, + "16": { + "pass@1": 0.016129032258064516, + "pass@2": 0.02903225806451613, + "pass@4": 0.02903225806451613, + "pass@8": 0.03225806451612903, + "pass@16": 0.03225806451612903, + "pass@32": 0.04516129032258064, + "pass@64": 0.04838709677419355, + "pass@128": 0.054838709677419356 + }, + "17": { + "pass@1": 0.002890173410404624, + "pass@2": 0.002890173410404624, + "pass@4": 0.002890173410404624, + "pass@8": 0.005780346820809248, + "pass@16": 0.005780346820809248, + "pass@32": 0.005780346820809248, + "pass@64": 0.008670520231213872, + "pass@128": 0.014450867052023121 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.002857142857142857, + "pass@4": 0.005714285714285714, + "pass@8": 0.005714285714285714, + "pass@16": 0.005714285714285714, + "pass@32": 0.008571428571428572, + "pass@64": 0.011428571428571429, + "pass@128": 0.014285714285714285 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.20396600566572237, + "12": 0.24293785310734464, + "13": 0.11912225705329153, + "14": 0.03426791277258567, + "15": 0.017341040462427744, + "16": 0.016129032258064516, + "17": 0.002890173410404624, + "18": 0.0, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.31444759206798867, + "12": 0.3192090395480226, + "13": 0.16927899686520376, + "14": 0.06230529595015576, + "15": 0.031791907514450865, + "16": 0.02903225806451613, + "17": 0.002890173410404624, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.4192634560906516, + "12": 0.4067796610169492, + "13": 0.2225705329153605, + "14": 0.09968847352024922, + "15": 0.06069364161849711, + "16": 0.02903225806451613, + "17": 0.002890173410404624, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.48725212464589235, + "12": 0.4858757062146893, + "13": 0.26959247648902823, + "14": 0.12461059190031153, + "15": 0.08092485549132948, + "16": 0.03225806451612903, + "17": 0.005780346820809248, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.5637393767705382, + "12": 0.5508474576271186, + "13": 0.36363636363636365, + "14": 0.1838006230529595, + "15": 0.10404624277456648, + "16": 0.03225806451612903, + "17": 0.005780346820809248, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@32": { + "11": 0.6345609065155807, + "12": 0.6129943502824858, + "13": 0.4263322884012539, + "14": 0.22118380062305296, + "15": 0.12138728323699421, + "16": 0.04516129032258064, + "17": 0.005780346820809248, + "18": 0.008571428571428572, + "19": 0.0, + "20": 0.0 + }, + "pass@64": { + "11": 0.6997167138810199, + "12": 0.6694915254237288, + "13": 0.4827586206896552, + "14": 0.26791277258566976, + "15": 0.16184971098265896, + "16": 0.04838709677419355, + "17": 0.008670520231213872, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.0 + }, + "pass@128": { + "11": 0.7365439093484419, + "12": 0.6892655367231638, + "13": 0.5297805642633229, + "14": 0.30218068535825543, + "15": 0.20520231213872833, + "16": 0.054838709677419356, + "17": 0.014450867052023121, + "18": 0.014285714285714285, + "19": 0.0, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.06508172362555721, + "pass@2": 0.0950965824665676, + "pass@4": 0.12719167904903417, + "pass@8": 0.15215453194650816, + "pass@16": 0.18395245170876673, + "pass@32": 0.21069836552748886, + "pass@64": 0.23833580980683508, + "pass@128": 0.25794947994056466 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.19047619047619047, + "pass@2": 0.26666666666666666, + "pass@4": 0.34285714285714286, + "pass@8": 0.4253968253968254, + "pass@16": 0.5238095238095238, + "pass@32": 0.5873015873015873, + "pass@64": 0.6317460317460317, + "pass@128": 0.6698412698412698 + }, + "12": { + "pass@1": 0.1926605504587156, + "pass@2": 0.25688073394495414, + "pass@4": 0.3425076452599388, + "pass@8": 0.41590214067278286, + "pass@16": 0.4740061162079511, + "pass@32": 0.5596330275229358, + "pass@64": 0.6024464831804281, + "pass@128": 0.6391437308868502 + }, + "13": { + "pass@1": 0.06481481481481481, + "pass@2": 0.08950617283950617, + "pass@4": 0.14814814814814814, + "pass@8": 0.20987654320987653, + "pass@16": 0.2716049382716049, + "pass@32": 0.30864197530864196, + "pass@64": 0.39814814814814814, + "pass@128": 0.4444444444444444 + }, + "14": { + "pass@1": 0.06528189910979229, + "pass@2": 0.08902077151335312, + "pass@4": 0.12462908011869436, + "pass@8": 0.1543026706231454, + "pass@16": 0.19287833827893175, + "pass@32": 0.22255192878338279, + "pass@64": 0.26409495548961426, + "pass@128": 0.2878338278931751 + }, + "15": { + "pass@1": 0.021604938271604937, + "pass@2": 0.033950617283950615, + "pass@4": 0.06172839506172839, + "pass@8": 0.07098765432098765, + "pass@16": 0.10493827160493827, + "pass@32": 0.14814814814814814, + "pass@64": 0.1697530864197531, + "pass@128": 0.19444444444444445 + }, + "16": { + "pass@1": 0.008849557522123894, + "pass@2": 0.011799410029498525, + "pass@4": 0.02359882005899705, + "pass@8": 0.02359882005899705, + "pass@16": 0.02654867256637168, + "pass@32": 0.04424778761061947, + "pass@64": 0.05604719764011799, + "pass@128": 0.06489675516224189 + }, + "17": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0030120481927710845, + "pass@8": 0.012048192771084338, + "pass@16": 0.012048192771084338, + "pass@32": 0.012048192771084338, + "pass@64": 0.012048192771084338, + "pass@128": 0.015060240963855422 + }, + "18": { + "pass@1": 0.0032258064516129032, + "pass@2": 0.0064516129032258064, + "pass@4": 0.012903225806451613, + "pass@8": 0.01935483870967742, + "pass@16": 0.01935483870967742, + "pass@32": 0.02258064516129032, + "pass@64": 0.025806451612903226, + "pass@128": 0.03225806451612903 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0030211480362537764, + "pass@32": 0.0030211480362537764, + "pass@64": 0.0030211480362537764, + "pass@128": 0.006042296072507553 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.19047619047619047, + "12": 0.1926605504587156, + "13": 0.06481481481481481, + "14": 0.06528189910979229, + "15": 0.021604938271604937, + "16": 0.008849557522123894, + "17": 0.0, + "18": 0.0032258064516129032, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.26666666666666666, + "12": 0.25688073394495414, + "13": 0.08950617283950617, + "14": 0.08902077151335312, + "15": 0.033950617283950615, + "16": 0.011799410029498525, + "17": 0.0, + "18": 0.0064516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.34285714285714286, + "12": 0.3425076452599388, + "13": 0.14814814814814814, + "14": 0.12462908011869436, + "15": 0.06172839506172839, + "16": 0.02359882005899705, + "17": 0.0030120481927710845, + "18": 0.012903225806451613, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.4253968253968254, + "12": 0.41590214067278286, + "13": 0.20987654320987653, + "14": 0.1543026706231454, + "15": 0.07098765432098765, + "16": 0.02359882005899705, + "17": 0.012048192771084338, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.5238095238095238, + "12": 0.4740061162079511, + "13": 0.2716049382716049, + "14": 0.19287833827893175, + "15": 0.10493827160493827, + "16": 0.02654867256637168, + "17": 0.012048192771084338, + "18": 0.01935483870967742, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@32": { + "11": 0.5873015873015873, + "12": 0.5596330275229358, + "13": 0.30864197530864196, + "14": 0.22255192878338279, + "15": 0.14814814814814814, + "16": 0.04424778761061947, + "17": 0.012048192771084338, + "18": 0.02258064516129032, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@64": { + "11": 0.6317460317460317, + "12": 0.6024464831804281, + "13": 0.39814814814814814, + "14": 0.26409495548961426, + "15": 0.1697530864197531, + "16": 0.05604719764011799, + "17": 0.012048192771084338, + "18": 0.025806451612903226, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@128": { + "11": 0.6698412698412698, + "12": 0.6391437308868502, + "13": 0.4444444444444444, + "14": 0.2878338278931751, + "15": 0.19444444444444445, + "16": 0.06489675516224189, + "17": 0.015060240963855422, + "18": 0.03225806451612903, + "19": 0.006042296072507553, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.05409535452322738, + "pass@2": 0.0745721271393643, + "pass@4": 0.10482885085574573, + "pass@8": 0.13172371638141808, + "pass@16": 0.1610635696821516, + "pass@32": 0.18887530562347188, + "pass@64": 0.21424205378973105, + "pass@128": 0.23319070904645478 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.22289156626506024, + "pass@2": 0.30120481927710846, + "pass@4": 0.4126506024096386, + "pass@8": 0.5271084337349398, + "pass@16": 0.6355421686746988, + "pass@32": 0.6927710843373494, + "pass@64": 0.7469879518072289, + "pass@128": 0.7891566265060241 + }, + "12": { + "pass@1": 0.18495297805642633, + "pass@2": 0.26959247648902823, + "pass@4": 0.36363636363636365, + "pass@8": 0.4827586206896552, + "pass@16": 0.5987460815047022, + "pass@32": 0.6426332288401254, + "pass@64": 0.6865203761755486, + "pass@128": 0.7084639498432602 + }, + "13": { + "pass@1": 0.0896358543417367, + "pass@2": 0.12324929971988796, + "pass@4": 0.1876750700280112, + "pass@8": 0.25210084033613445, + "pass@16": 0.32492997198879553, + "pass@32": 0.3949579831932773, + "pass@64": 0.4481792717086835, + "pass@128": 0.48179271708683474 + }, + "14": { + "pass@1": 0.04093567251461988, + "pass@2": 0.07602339181286549, + "pass@4": 0.10526315789473684, + "pass@8": 0.14035087719298245, + "pass@16": 0.1871345029239766, + "pass@32": 0.23391812865497075, + "pass@64": 0.27485380116959063, + "pass@128": 0.31871345029239767 + }, + "15": { + "pass@1": 0.024242424242424242, + "pass@2": 0.045454545454545456, + "pass@4": 0.051515151515151514, + "pass@8": 0.07272727272727272, + "pass@16": 0.10303030303030303, + "pass@32": 0.13333333333333333, + "pass@64": 0.16363636363636364, + "pass@128": 0.18484848484848485 + }, + "16": { + "pass@1": 0.011396011396011397, + "pass@2": 0.014245014245014245, + "pass@4": 0.019943019943019943, + "pass@8": 0.03133903133903134, + "pass@16": 0.037037037037037035, + "pass@32": 0.039886039886039885, + "pass@64": 0.045584045584045586, + "pass@128": 0.05413105413105413 + }, + "17": { + "pass@1": 0.009316770186335404, + "pass@2": 0.012422360248447204, + "pass@4": 0.012422360248447204, + "pass@8": 0.015527950310559006, + "pass@16": 0.015527950310559006, + "pass@32": 0.018633540372670808, + "pass@64": 0.027950310559006212, + "pass@128": 0.027950310559006212 + }, + "18": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0029411764705882353, + "pass@16": 0.0029411764705882353, + "pass@32": 0.008823529411764706, + "pass@64": 0.008823529411764706, + "pass@128": 0.008823529411764706 + }, + "19": { + "pass@1": 0.002976190476190476, + "pass@2": 0.002976190476190476, + "pass@4": 0.002976190476190476, + "pass@8": 0.002976190476190476, + "pass@16": 0.002976190476190476, + "pass@32": 0.002976190476190476, + "pass@64": 0.002976190476190476, + "pass@128": 0.005952380952380952 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.22289156626506024, + "12": 0.18495297805642633, + "13": 0.0896358543417367, + "14": 0.04093567251461988, + "15": 0.024242424242424242, + "16": 0.011396011396011397, + "17": 0.009316770186335404, + "18": 0.0, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@2": { + "11": 0.30120481927710846, + "12": 0.26959247648902823, + "13": 0.12324929971988796, + "14": 0.07602339181286549, + "15": 0.045454545454545456, + "16": 0.014245014245014245, + "17": 0.012422360248447204, + "18": 0.0, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@4": { + "11": 0.4126506024096386, + "12": 0.36363636363636365, + "13": 0.1876750700280112, + "14": 0.10526315789473684, + "15": 0.051515151515151514, + "16": 0.019943019943019943, + "17": 0.012422360248447204, + "18": 0.0, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@8": { + "11": 0.5271084337349398, + "12": 0.4827586206896552, + "13": 0.25210084033613445, + "14": 0.14035087719298245, + "15": 0.07272727272727272, + "16": 0.03133903133903134, + "17": 0.015527950310559006, + "18": 0.0029411764705882353, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@16": { + "11": 0.6355421686746988, + "12": 0.5987460815047022, + "13": 0.32492997198879553, + "14": 0.1871345029239766, + "15": 0.10303030303030303, + "16": 0.037037037037037035, + "17": 0.015527950310559006, + "18": 0.0029411764705882353, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@32": { + "11": 0.6927710843373494, + "12": 0.6426332288401254, + "13": 0.3949579831932773, + "14": 0.23391812865497075, + "15": 0.13333333333333333, + "16": 0.039886039886039885, + "17": 0.018633540372670808, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@64": { + "11": 0.7469879518072289, + "12": 0.6865203761755486, + "13": 0.4481792717086835, + "14": 0.27485380116959063, + "15": 0.16363636363636364, + "16": 0.045584045584045586, + "17": 0.027950310559006212, + "18": 0.008823529411764706, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@128": { + "11": 0.7891566265060241, + "12": 0.7084639498432602, + "13": 0.48179271708683474, + "14": 0.31871345029239767, + "15": 0.18484848484848485, + "16": 0.05413105413105413, + "17": 0.027950310559006212, + "18": 0.008823529411764706, + "19": 0.005952380952380952, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.05798394290811775, + "pass@2": 0.08355634849836456, + "pass@4": 0.11448111804936069, + "pass@8": 0.15135295866785609, + "pass@16": 0.18911685994647637, + "pass@32": 0.2152839726434731, + "pass@64": 0.23907225691347012, + "pass@128": 0.2566161165625929 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.206, + "pass@2": 0.295, + "pass@4": 0.393, + "pass@8": 0.481, + "pass@16": 0.575, + "pass@32": 0.639, + "pass@64": 0.694, + "pass@128": 0.733 + }, + "12": { + "pass@1": 0.208, + "pass@2": 0.283, + "pass@4": 0.372, + "pass@8": 0.462, + "pass@16": 0.541, + "pass@32": 0.605, + "pass@64": 0.653, + "pass@128": 0.679 + }, + "13": { + "pass@1": 0.091, + "pass@2": 0.127, + "pass@4": 0.186, + "pass@8": 0.244, + "pass@16": 0.32, + "pass@32": 0.377, + "pass@64": 0.443, + "pass@128": 0.485 + }, + "14": { + "pass@1": 0.047, + "pass@2": 0.076, + "pass@4": 0.11, + "pass@8": 0.14, + "pass@16": 0.188, + "pass@32": 0.226, + "pass@64": 0.269, + "pass@128": 0.303 + }, + "15": { + "pass@1": 0.021, + "pass@2": 0.037, + "pass@4": 0.058, + "pass@8": 0.075, + "pass@16": 0.104, + "pass@32": 0.134, + "pass@64": 0.165, + "pass@128": 0.195 + }, + "16": { + "pass@1": 0.012, + "pass@2": 0.018, + "pass@4": 0.024, + "pass@8": 0.029, + "pass@16": 0.032, + "pass@32": 0.043, + "pass@64": 0.05, + "pass@128": 0.058 + }, + "17": { + "pass@1": 0.004, + "pass@2": 0.005, + "pass@4": 0.006, + "pass@8": 0.011, + "pass@16": 0.011, + "pass@32": 0.012, + "pass@64": 0.016, + "pass@128": 0.019 + }, + "18": { + "pass@1": 0.001, + "pass@2": 0.003, + "pass@4": 0.006, + "pass@8": 0.009, + "pass@16": 0.009, + "pass@32": 0.013, + "pass@64": 0.015, + "pass@128": 0.018 + }, + "19": { + "pass@1": 0.001, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.001, + "pass@16": 0.002, + "pass@32": 0.002, + "pass@64": 0.002, + "pass@128": 0.004 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.206, + "12": 0.208, + "13": 0.091, + "14": 0.047, + "15": 0.021, + "16": 0.012, + "17": 0.004, + "18": 0.001, + "19": 0.001, + "20": 0.0 + }, + "pass@2": { + "11": 0.295, + "12": 0.283, + "13": 0.127, + "14": 0.076, + "15": 0.037, + "16": 0.018, + "17": 0.005, + "18": 0.003, + "19": 0.001, + "20": 0.0 + }, + "pass@4": { + "11": 0.393, + "12": 0.372, + "13": 0.186, + "14": 0.11, + "15": 0.058, + "16": 0.024, + "17": 0.006, + "18": 0.006, + "19": 0.001, + "20": 0.0 + }, + "pass@8": { + "11": 0.481, + "12": 0.462, + "13": 0.244, + "14": 0.14, + "15": 0.075, + "16": 0.029, + "17": 0.011, + "18": 0.009, + "19": 0.001, + "20": 0.0 + }, + "pass@16": { + "11": 0.575, + "12": 0.541, + "13": 0.32, + "14": 0.188, + "15": 0.104, + "16": 0.032, + "17": 0.011, + "18": 0.009, + "19": 0.002, + "20": 0.0 + }, + "pass@32": { + "11": 0.639, + "12": 0.605, + "13": 0.377, + "14": 0.226, + "15": 0.134, + "16": 0.043, + "17": 0.012, + "18": 0.013, + "19": 0.002, + "20": 0.0 + }, + "pass@64": { + "11": 0.694, + "12": 0.653, + "13": 0.443, + "14": 0.269, + "15": 0.165, + "16": 0.05, + "17": 0.016, + "18": 0.015, + "19": 0.002, + "20": 0.0 + }, + "pass@128": { + "11": 0.733, + "12": 0.679, + "13": 0.485, + "14": 0.303, + "15": 0.195, + "16": 0.058, + "17": 0.019, + "18": 0.018, + "19": 0.004, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.0591, + "pass@2": 0.0845, + "pass@4": 0.1156, + "pass@8": 0.1452, + "pass@16": 0.1782, + "pass@32": 0.2051, + "pass@64": 0.2307, + "pass@128": 0.2494 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/summary.csv b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..725e7cf22b8445b26020a6a364f092061cee44d5 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21546,0.8766857638888889,0.329509375,0.5886981907894737,0.0,0.0,0.0,199.3285625,317.06312109375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/summary.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..887c63e2bc340991da7896376b176cf9146828e7 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block16/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21546", + "id_acc": 0.8766857638888889, + "ood_acc": 0.329509375, + "total_acc": 0.5886981907894737, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.3285625, + "ood_avg_resp_len": 317.06312109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 476007, + "answer_accuracy": 0.5878603679260196, + "avg_response_len": 249.26987334018338, + "resp_tokens_sum": 201840796, + "pass_at_k": { + "pass@1": 0.5878603679260196, + "pass@2": 0.6611123113024427, + "pass@4": 0.715592607199787, + "pass@8": 0.7548355615364056, + "pass@16": 0.7838130901019593, + "pass@32": 0.8065087365881829, + "pass@64": 0.8256173393051519, + "pass@128": 0.8423964590578564 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7548626474926253, + "pass@2": 0.8580390739321302, + "pass@4": 0.9158941481872283, + "pass@8": 0.9422260710100495, + "pass@16": 0.9541179241145453, + "pass@32": 0.9627354979168701, + "pass@64": 0.9700056559069217, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9521271417445483, + "pass@2": 0.9600596070351018, + "pass@4": 0.967638862899147, + "pass@8": 0.9736349720045984, + "pass@16": 0.9777725738486086, + "pass@32": 0.9807478572199139, + "pass@64": 0.983644411229528, + "pass@128": 0.9875389408099688 + }, + "3": { + "pass@1": 0.9738627675840978, + "pass@2": 0.9860556779599798, + "pass@4": 0.9911855877533655, + "pass@8": 0.9935549584465483, + "pass@16": 0.9950485984443537, + "pass@32": 0.9964154468880052, + "pass@64": 0.9980977119454602, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9468017578125, + "pass@2": 0.9668841504675199, + "pass@4": 0.9788314503655793, + "pass@8": 0.98423139689816, + "pass@16": 0.9869775056679988, + "pass@32": 0.988862756689439, + "pass@64": 0.9906228273186573, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.8940972222222222, + "pass@2": 0.925370656792901, + "pass@4": 0.9444052689842337, + "pass@8": 0.956068245645845, + "pass@16": 0.9641325616691484, + "pass@32": 0.9702121366978933, + "pass@64": 0.9736402664360028, + "pass@128": 0.9746031746031746 + }, + "6": { + "pass@1": 0.874625748502994, + "pass@2": 0.9345000942995885, + "pass@4": 0.9660633426622565, + "pass@8": 0.9817471947893537, + "pass@16": 0.9901813682097385, + "pass@32": 0.9945595172809533, + "pass@64": 0.996547366723047, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.8708394970414202, + "pass@2": 0.9296885919955274, + "pass@4": 0.9574697000707745, + "pass@8": 0.9709659057101204, + "pass@16": 0.9797032271920411, + "pass@32": 0.9856532059055091, + "pass@64": 0.9903706220860208, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.8371394230769231, + "pass@2": 0.921493681228548, + "pass@4": 0.9620283975921279, + "pass@8": 0.9787379257212695, + "pass@16": 0.9860862836307394, + "pass@32": 0.9901621390004783, + "pass@64": 0.993815134792471, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7703125, + "pass@2": 0.8768198264389484, + "pass@4": 0.9369461128978587, + "pass@8": 0.9678318575372462, + "pass@16": 0.9839113625433821, + "pass@32": 0.993005940915507, + "pass@64": 0.9981911215016727, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.7148326841359773, + "pass@2": 0.8247275183466801, + "pass@4": 0.8932718025154789, + "pass@8": 0.9331762977429914, + "pass@16": 0.9563250136105693, + "pass@32": 0.9703408496523926, + "pass@64": 0.9791441852491506, + "pass@128": 0.9886685552407932 + }, + "12": { + "pass@1": 0.610787429378531, + "pass@2": 0.7386105504025978, + "pass@4": 0.8302062342842734, + "pass@8": 0.8869758223344203, + "pass@16": 0.9190229298739069, + "pass@32": 0.9382509994759017, + "pass@64": 0.9504507398120785, + "pass@128": 0.96045197740113 + }, + "13": { + "pass@1": 0.4938528605015674, + "pass@2": 0.6135426437192993, + "pass@4": 0.7050806219755448, + "pass@8": 0.7709119554640657, + "pass@16": 0.8185553713018926, + "pass@32": 0.8529551509357786, + "pass@64": 0.8789136115726013, + "pass@128": 0.896551724137931 + }, + "14": { + "pass@1": 0.3722984813084112, + "pass@2": 0.4832002600142271, + "pass@4": 0.5817811873983043, + "pass@8": 0.6600182956969374, + "pass@16": 0.7170059764733474, + "pass@32": 0.759723872506481, + "pass@64": 0.7973315503500162, + "pass@128": 0.8348909657320872 + }, + "15": { + "pass@1": 0.2915236632947977, + "pass@2": 0.38716340573938335, + "pass@4": 0.4724536182435286, + "pass@8": 0.5428825309662252, + "pass@16": 0.6009330341758621, + "pass@32": 0.6513234112092107, + "pass@64": 0.6978355864283469, + "pass@128": 0.7398843930635838 + }, + "16": { + "pass@1": 0.2159022177419355, + "pass@2": 0.294323485521971, + "pass@4": 0.3682993205486411, + "pass@8": 0.4345881141350131, + "pass@16": 0.49488662642953934, + "pass@32": 0.5483336279194447, + "pass@64": 0.5945119316894635, + "pass@128": 0.635483870967742 + }, + "17": { + "pass@1": 0.17767792630057805, + "pass@2": 0.24014361260297645, + "pass@4": 0.3024947796445963, + "pass@8": 0.36105936344277334, + "pass@16": 0.41425311433468764, + "pass@32": 0.4629499781277512, + "pass@64": 0.5069431688928129, + "pass@128": 0.5491329479768786 + }, + "18": { + "pass@1": 0.17955357142857142, + "pass@2": 0.24916057367829025, + "pass@4": 0.3168455503776314, + "pass@8": 0.37584058796751063, + "pass@16": 0.4238660306777394, + "pass@32": 0.4629081102911771, + "pass@64": 0.4982544518716383, + "pass@128": 0.5285714285714286 + }, + "19": { + "pass@1": 0.15315315315315314, + "pass@2": 0.21435005379395142, + "pass@4": 0.27603142118496454, + "pass@8": 0.3325890592148852, + "pass@16": 0.38104382491172395, + "pass@32": 0.4215814504369368, + "pass@64": 0.456708682959238, + "pass@128": 0.4864864864864865 + }, + "20": { + "pass@1": 0.13323479729729729, + "pass@2": 0.19331757939041397, + "pass@4": 0.2577742100931077, + "pass@8": 0.3186896897271893, + "pass@16": 0.37015786252682287, + "pass@32": 0.4131022452662304, + "pass@64": 0.4508702308920766, + "pass@128": 0.48348348348348347 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 482715, + "answer_accuracy": 0.5878738795791114, + "avg_response_len": 257.7196682579891, + "resp_tokens_sum": 211618774, + "pass_at_k": { + "pass@1": 0.5878738795791114, + "pass@2": 0.6570554679301156, + "pass@4": 0.710301292594661, + "pass@8": 0.750695245054591, + "pass@16": 0.7818350494228476, + "pass@32": 0.8066651512685107, + "pass@64": 0.8272927747768041, + "pass@128": 0.8455183164458301 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7257716641104295, + "pass@2": 0.829145482102314, + "pass@4": 0.8909685864612009, + "pass@8": 0.923828926172029, + "pass@16": 0.9424341083861851, + "pass@32": 0.9547993211002885, + "pass@64": 0.9635684888575998, + "pass@128": 0.9693251533742331 + }, + "2": { + "pass@1": 0.969961056231003, + "pass@2": 0.9800300062226267, + "pass@4": 0.9871420935604933, + "pass@8": 0.9922565877731916, + "pass@16": 0.9959260151333992, + "pass@32": 0.9983344837584649, + "pass@64": 0.9996191683758167, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9798295454545455, + "pass@2": 0.9895311381531855, + "pass@4": 0.9931646833918487, + "pass@8": 0.9950567296773937, + "pass@16": 0.9963382395784798, + "pass@32": 0.9977819591272412, + "pass@64": 0.9992064994093033, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9674093934911243, + "pass@2": 0.9813876642361278, + "pass@4": 0.9896786493589189, + "pass@8": 0.9942140522761386, + "pass@16": 0.9964544182239704, + "pass@32": 0.9981567480120942, + "pass@64": 0.9996475795537232, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.8929347826086956, + "pass@2": 0.9245599395184297, + "pass@4": 0.9455650755612065, + "pass@8": 0.9585984031211746, + "pass@16": 0.9661113498054109, + "pass@32": 0.9713447438413154, + "pass@64": 0.9761022999756817, + "pass@128": 0.9797101449275363 + }, + "6": { + "pass@1": 0.8609643624641834, + "pass@2": 0.9185087313584368, + "pass@4": 0.9530109303135099, + "pass@8": 0.9730129786135173, + "pass@16": 0.9845427515918137, + "pass@32": 0.9913868693492569, + "pass@64": 0.9955681335039379, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8571396315028902, + "pass@2": 0.9151441815347509, + "pass@4": 0.9492684707764129, + "pass@8": 0.9707522712002761, + "pass@16": 0.9846804474521327, + "pass@32": 0.9932585974949043, + "pass@64": 0.9976034786090924, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8400528169014084, + "pass@2": 0.9146213402462021, + "pass@4": 0.9519286311042101, + "pass@8": 0.9705870784036298, + "pass@16": 0.9806850356855231, + "pass@32": 0.9870184734714211, + "pass@64": 0.9913704266572482, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.7795424775449101, + "pass@2": 0.8696370791880805, + "pass@4": 0.9201581606191439, + "pass@8": 0.9472707355109283, + "pass@16": 0.9634659109310358, + "pass@32": 0.9732413865436497, + "pass@64": 0.979391838196961, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.7141613328313253, + "pass@2": 0.8234709260269424, + "pass@4": 0.8924713595665003, + "pass@8": 0.9303646055635199, + "pass@16": 0.9515900662979302, + "pass@32": 0.9648603754748566, + "pass@64": 0.9738234865676904, + "pass@128": 0.9819277108433735 + }, + "12": { + "pass@1": 0.625342868338558, + "pass@2": 0.7412462512033179, + "pass@4": 0.8177234021295925, + "pass@8": 0.864841454508076, + "pass@16": 0.8973079096792422, + "pass@32": 0.9206851353500622, + "pass@64": 0.9360387543338561, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.45526960784313725, + "pass@2": 0.573507700324224, + "pass@4": 0.6657258832666921, + "pass@8": 0.7338637568424262, + "pass@16": 0.7852004994961729, + "pass@32": 0.8273113155682195, + "pass@64": 0.8624061133822667, + "pass@128": 0.8907563025210085 + }, + "14": { + "pass@1": 0.35798154239766083, + "pass@2": 0.45727353570935186, + "pass@4": 0.545599068756756, + "pass@8": 0.6210483819213921, + "pass@16": 0.68125171142204, + "pass@32": 0.7270073779808284, + "pass@64": 0.7631432072915221, + "pass@128": 0.7953216374269005 + }, + "15": { + "pass@1": 0.2765151515151515, + "pass@2": 0.36803939990455725, + "pass@4": 0.45397499403483615, + "pass@8": 0.5263473308035848, + "pass@16": 0.5859555392776711, + "pass@32": 0.6386314053079184, + "pass@64": 0.688474647802565, + "pass@128": 0.7333333333333333 + }, + "16": { + "pass@1": 0.19366542022792022, + "pass@2": 0.26023509881777596, + "pass@4": 0.3251300206063985, + "pass@8": 0.38605801417611135, + "pass@16": 0.44536576095291924, + "pass@32": 0.502109158651507, + "pass@64": 0.5541207044234497, + "pass@128": 0.603988603988604 + }, + "17": { + "pass@1": 0.20055318322981366, + "pass@2": 0.2769511572602337, + "pass@4": 0.3524708197383709, + "pass@8": 0.4216463758523163, + "pass@16": 0.4810756708240572, + "pass@32": 0.5287041717254924, + "pass@64": 0.5680075699156929, + "pass@128": 0.6055900621118012 + }, + "18": { + "pass@1": 0.19087775735294119, + "pass@2": 0.26312348019916604, + "pass@4": 0.3357639033172321, + "pass@8": 0.400534103180857, + "pass@16": 0.45257040732607284, + "pass@32": 0.49584444773542125, + "pass@64": 0.5358095049537671, + "pass@128": 0.5735294117647058 + }, + "19": { + "pass@1": 0.14274088541666666, + "pass@2": 0.20004298779058863, + "pass@4": 0.26311496163425996, + "pass@8": 0.3261681997932631, + "pass@16": 0.3816211319972022, + "pass@32": 0.4273529590398094, + "pass@64": 0.466436005280597, + "pass@128": 0.5 + }, + "20": { + "pass@1": 0.12951440868263472, + "pass@2": 0.18884229572351371, + "pass@4": 0.25496642738145764, + "pass@8": 0.3193190721203961, + "pass@16": 0.3750415414954998, + "pass@32": 0.4213703302393265, + "pass@64": 0.46037260804896324, + "pass@128": 0.49700598802395207 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 472992, + "answer_accuracy": 0.5903898386323694, + "avg_response_len": 277.1106219543857, + "resp_tokens_sum": 222007729, + "pass_at_k": { + "pass@1": 0.5903898386323694, + "pass@2": 0.6585257779977994, + "pass@4": 0.7106201799966714, + "pass@8": 0.750469997444581, + "pass@16": 0.7816516072340053, + "pass@32": 0.8062069138724718, + "pass@64": 0.8254911858450941, + "pass@128": 0.8407093784949673 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.7465951492537314, + "pass@2": 0.8382885033493949, + "pass@4": 0.8949848927466156, + "pass@8": 0.9280509232871795, + "pass@16": 0.9478438805819164, + "pass@32": 0.9594792695269607, + "pass@64": 0.9666690334826524, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9866741071428572, + "pass@2": 0.9914500140607426, + "pass@4": 0.9947826811827094, + "pass@8": 0.9964610707643552, + "pass@16": 0.9970452137532337, + "pass@32": 0.9971410795160666, + "pass@64": 0.9971428571124661, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9861060495626822, + "pass@2": 0.9948298076260877, + "pass@4": 0.9969340503355447, + "pass@8": 0.9972608802887731, + "pass@16": 0.9974489749023661, + "pass@32": 0.997813411078717, + "pass@64": 0.9985422740524781, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9719937865497076, + "pass@2": 0.981466132522908, + "pass@4": 0.9844356083778999, + "pass@8": 0.9855420893824135, + "pass@16": 0.9860609498652024, + "pass@32": 0.9866636688638668, + "pass@64": 0.9875788552746696, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.9102711397058824, + "pass@2": 0.9282122076192687, + "pass@4": 0.940072399314056, + "pass@8": 0.9506308126253655, + "pass@16": 0.9599675251090849, + "pass@32": 0.9676808245522525, + "pass@64": 0.9735033828674721, + "pass@128": 0.9764705882352941 + }, + "6": { + "pass@1": 0.871870070977918, + "pass@2": 0.9295623338880746, + "pass@4": 0.9614348610603484, + "pass@8": 0.9783258917269404, + "pass@16": 0.9877960079219414, + "pass@32": 0.9940389600521968, + "pass@64": 0.99779786673376, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8647151898734177, + "pass@2": 0.9250473437655732, + "pass@4": 0.9549791028890373, + "pass@8": 0.9688374702660314, + "pass@16": 0.9767573806272281, + "pass@32": 0.9830239517261676, + "pass@64": 0.9885077193436845, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.8060482357357357, + "pass@2": 0.883660235038188, + "pass@4": 0.9271438722186747, + "pass@8": 0.9515870560611387, + "pass@16": 0.9657731015433726, + "pass@32": 0.975180094457099, + "pass@64": 0.9815049830533622, + "pass@128": 0.984984984984985 + }, + "9": { + "pass@1": 0.7751456993569131, + "pass@2": 0.8718941470238243, + "pass@4": 0.9229569807592373, + "pass@8": 0.9491012856677312, + "pass@16": 0.9649184196419777, + "pass@32": 0.9739737051321179, + "pass@64": 0.9781618463030722, + "pass@128": 0.9807073954983923 + }, + "11": { + "pass@1": 0.6679811507936508, + "pass@2": 0.7739470066241724, + "pass@4": 0.8426609691645686, + "pass@8": 0.8882811236312015, + "pass@16": 0.9217558302441041, + "pass@32": 0.9453042212257897, + "pass@64": 0.958881139521916, + "pass@128": 0.9682539682539683 + }, + "12": { + "pass@1": 0.6055045871559633, + "pass@2": 0.7249553775674832, + "pass@4": 0.8070858390407616, + "pass@8": 0.8604887130755426, + "pass@16": 0.8984640220204004, + "pass@32": 0.926717664021228, + "pass@64": 0.9479442969224088, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.44849537037037035, + "pass@2": 0.5667077531107221, + "pass@4": 0.6617636250329818, + "pass@8": 0.7328063948739421, + "pass@16": 0.7862427039253898, + "pass@32": 0.828873576525592, + "pass@64": 0.8648600832732204, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.3864753338278932, + "pass@2": 0.4937969491109606, + "pass@4": 0.582945069644484, + "pass@8": 0.6549811145473605, + "pass@16": 0.7130843331326536, + "pass@32": 0.7584331108720384, + "pass@64": 0.7946619410176868, + "pass@128": 0.8249258160237388 + }, + "15": { + "pass@1": 0.2794897762345679, + "pass@2": 0.37721836419753096, + "pass@4": 0.4652975750716343, + "pass@8": 0.5389522735789639, + "pass@16": 0.6011663209854563, + "pass@32": 0.65646740261954, + "pass@64": 0.705641491781776, + "pass@128": 0.7469135802469136 + }, + "16": { + "pass@1": 0.21928005162241887, + "pass@2": 0.2919903955589622, + "pass@4": 0.36209677274853913, + "pass@8": 0.42627676402045145, + "pass@16": 0.4839805580211237, + "pass@32": 0.5352047911685343, + "pass@64": 0.5795251816926208, + "pass@128": 0.616519174041298 + }, + "17": { + "pass@1": 0.16629800451807228, + "pass@2": 0.23740372414856273, + "pass@4": 0.312128824070184, + "pass@8": 0.3813466482123387, + "pass@16": 0.4392210735431096, + "pass@32": 0.48650114540932765, + "pass@64": 0.5239750597065395, + "pass@128": 0.5542168674698795 + }, + "18": { + "pass@1": 0.17905745967741934, + "pass@2": 0.24398614109728226, + "pass@4": 0.31053360880293185, + "pass@8": 0.3724229822072563, + "pass@16": 0.42312145386859323, + "pass@32": 0.46238044186642907, + "pass@64": 0.495589661049704, + "pass@128": 0.5258064516129032 + }, + "19": { + "pass@1": 0.15674565709969787, + "pass@2": 0.2175691206556129, + "pass@4": 0.277465813846532, + "pass@8": 0.33085507142386955, + "pass@16": 0.37441386794026765, + "pass@32": 0.40776816099529734, + "pass@64": 0.43326206050798804, + "pass@128": 0.4501510574018127 + }, + "20": { + "pass@1": 0.14738175675675674, + "pass@2": 0.21154693078905676, + "pass@4": 0.2815054278125145, + "pass@8": 0.35082918530995677, + "pass@16": 0.41341781482613965, + "pass@32": 0.464339425513764, + "pass@64": 0.5009669935181226, + "pass@128": 0.5255255255255256 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/checkpoint-21546_metrics.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/checkpoint-21546_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..360a2a10d277cedd1d7fc17914cfcdeab2ce9ea7 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/checkpoint-21546_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21546", + "id": { + "answer_accuracy": 0.6596380208333333, + "avg_response_len": 207.44738020833333, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.6596380208333333, + "pass@2": 0.7660776738845149, + "pass@4": 0.8495281832270917, + "pass@8": 0.9074004116124759, + "pass@16": 0.9438217583554562, + "pass@32": 0.9655632281139891, + "pass@64": 0.9779978046376833, + "pass@128": 0.985 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.4250078125, + "pass@2": 0.5768004429133857, + "pass@4": 0.7134305258717659, + "pass@8": 0.8199063979574568, + "pass@16": 0.8935452112743179, + "pass@32": 0.9398754942428027, + "pass@64": 0.9654533308462334, + "pass@128": 0.979 + }, + "2": { + "pass@1": 0.9381015625, + "pass@2": 0.9528553149606297, + "pass@4": 0.9630730704911884, + "pass@8": 0.9702266488878839, + "pass@16": 0.975452197911979, + "pass@32": 0.9793132877954412, + "pass@64": 0.9817224170335146, + "pass@128": 0.983 + }, + "3": { + "pass@1": 0.8848359375, + "pass@2": 0.9430001230314963, + "pass@4": 0.9707062199100114, + "pass@8": 0.9812595682703927, + "pass@16": 0.9852764841848526, + "pass@32": 0.987617675650772, + "pass@64": 0.9896682272274708, + "pass@128": 0.991 + }, + "4": { + "pass@1": 0.773953125, + "pass@2": 0.858044414370079, + "pass@4": 0.9149909086051744, + "pass@8": 0.9485047792560358, + "pass@16": 0.9672467913518666, + "pass@32": 0.9788297174812043, + "pass@64": 0.98618043104808, + "pass@128": 0.99 + }, + "5": { + "pass@1": 0.6465078125, + "pass@2": 0.7576323818897631, + "pass@4": 0.8412265929883768, + "pass@8": 0.8968452107771091, + "pass@16": 0.931368042735231, + "pass@32": 0.9520664882811875, + "pass@64": 0.9645204472984087, + "pass@128": 0.973 + }, + "6": { + "pass@1": 0.6936328125, + "pass@2": 0.803946727362204, + "pass@4": 0.8832361657292838, + "pass@8": 0.9314466219085178, + "pass@16": 0.9575838221416829, + "pass@32": 0.9724694137582386, + "pass@64": 0.9823185183926962, + "pass@128": 0.99 + }, + "7": { + "pass@1": 0.598171875, + "pass@2": 0.7292191190944884, + "pass@4": 0.8330857781214833, + "pass@8": 0.9052266116175698, + "pass@16": 0.9468352343911222, + "pass@32": 0.9687958802123822, + "pass@64": 0.9813171526673037, + "pass@128": 0.988 + }, + "8": { + "pass@1": 0.53165625, + "pass@2": 0.6782041092519686, + "pass@4": 0.7972615198725161, + "pass@8": 0.8804777104729525, + "pass@16": 0.9322194970921365, + "pass@32": 0.9615292324789348, + "pass@64": 0.9769648857667702, + "pass@128": 0.986 + }, + "9": { + "pass@1": 0.444875, + "pass@2": 0.5949964320866137, + "pass@4": 0.7287428674540681, + "pass@8": 0.8327101553644152, + "pass@16": 0.9048685441158348, + "pass@32": 0.9495718631248805, + "pass@64": 0.9738348314586076, + "pass@128": 0.985 + } + }, + "per_op_accuracy": { + "10": 0.4250078125, + "2": 0.9381015625, + "3": 0.8848359375, + "4": 0.773953125, + "5": 0.6465078125, + "6": 0.6936328125, + "7": 0.598171875, + "8": 0.53165625, + "9": 0.444875 + }, + "per_op_avg_response_len": { + "10": 285.6915, + "2": 120.4114609375, + "3": 151.4393359375, + "4": 169.4210625, + "5": 181.6033359375, + "6": 205.7773359375, + "7": 232.946625, + "8": 252.271203125, + "9": 267.4645625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 285.6915, + "2": 120.4114609375, + "3": 151.4393359375, + "4": 169.4210625, + "5": 181.6033359375, + "6": 205.7773359375, + "7": 232.946625, + "8": 252.271203125, + "9": 267.4645625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 239443, + "answer_accuracy": 0.6317623902397839, + "avg_response_len": 197.64661431948667, + "resp_tokens_sum": 74909648, + "pass_at_k": { + "pass@1": 0.6317623902397839, + "pass@2": 0.7417875541488187, + "pass@4": 0.8307686099774532, + "pass@8": 0.8937960029314846, + "pass@16": 0.9341744735458538, + "pass@32": 0.9589139467414423, + "pass@64": 0.9735013156369298, + "pass@128": 0.9817629179331308 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.371427912979351, + "pass@2": 0.523322997235965, + "pass@4": 0.6718812374669977, + "pass@8": 0.7933920591197899, + "pass@16": 0.8779724027500238, + "pass@32": 0.9314886148824195, + "pass@64": 0.9611391857168347, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9139164719626168, + "pass@2": 0.9309390867613512, + "pass@4": 0.9439649050877985, + "pass@8": 0.9533337561204487, + "pass@16": 0.9605609138316018, + "pass@32": 0.966119078288411, + "pass@64": 0.9697424698773338, + "pass@128": 0.9719626168224299 + }, + "3": { + "pass@1": 0.9014000382262997, + "pass@2": 0.9560262105998218, + "pass@4": 0.9798727122985766, + "pass@8": 0.9879021076001016, + "pass@16": 0.9912037087331826, + "pass@32": 0.9936467679984826, + "pass@64": 0.9958006824234752, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.7223876953125, + "pass@2": 0.8201191098671256, + "pass@4": 0.8897095618907006, + "pass@8": 0.9320338826425963, + "pass@16": 0.956386741149801, + "pass@32": 0.9713648996011031, + "pass@64": 0.9804295567954489, + "pass@128": 0.984375 + }, + "5": { + "pass@1": 0.629811507936508, + "pass@2": 0.7491188601424822, + "pass@4": 0.8407026499663729, + "pass@8": 0.9022890763307583, + "pass@16": 0.9401223531283432, + "pass@32": 0.96133074752117, + "pass@64": 0.9711185300201212, + "pass@128": 0.9777777777777777 + }, + "6": { + "pass@1": 0.6804126122754491, + "pass@2": 0.7908088842001034, + "pass@4": 0.8711181910644397, + "pass@8": 0.9226813205784111, + "pass@16": 0.9523163473121824, + "pass@32": 0.9702453653333176, + "pass@64": 0.9832286283027216, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.5659208579881657, + "pass@2": 0.703538502306295, + "pass@4": 0.8161317657962868, + "pass@8": 0.8944902079834598, + "pass@16": 0.9397179109827034, + "pass@32": 0.9645216715274736, + "pass@64": 0.9810000758817367, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.5125951522435898, + "pass@2": 0.6551375902230964, + "pass@4": 0.7706036381870534, + "pass@8": 0.8512998182377252, + "pass@16": 0.9029006956766283, + "pass@32": 0.9340055360413846, + "pass@64": 0.9518592357512387, + "pass@128": 0.9647435897435898 + }, + "9": { + "pass@1": 0.4185519366197183, + "pass@2": 0.5713655456360205, + "pass@4": 0.7097021433060301, + "pass@8": 0.8166626354564073, + "pass@16": 0.8912920455560849, + "pass@32": 0.9391179746651805, + "pass@64": 0.9667627928829464, + "pass@128": 0.9802816901408451 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 257791, + "answer_accuracy": 0.6598925909239842, + "avg_response_len": 205.77035038499344, + "resp_tokens_sum": 80385422, + "pass_at_k": { + "pass@1": 0.6598925909239842, + "pass@2": 0.770424236140495, + "pass@4": 0.8549053976481159, + "pass@8": 0.9115704794913504, + "pass@16": 0.9465323709033038, + "pass@32": 0.9671355260264496, + "pass@64": 0.9787999133794949, + "pass@128": 0.9855832241153342 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.42939992331288346, + "pass@2": 0.5845611323124487, + "pass@4": 0.7219675550525508, + "pass@8": 0.8269844795437781, + "pass@16": 0.8993004726366725, + "pass@32": 0.9453174771206605, + "pass@64": 0.9721708965924318, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9301624240121581, + "pass@2": 0.9487538293085703, + "pass@4": 0.9608639527012007, + "pass@8": 0.9687376166611839, + "pass@16": 0.9746019959801766, + "pass@32": 0.9795867935302768, + "pass@64": 0.9830901743667175, + "pass@128": 0.9848024316109423 + }, + "3": { + "pass@1": 0.860653409090909, + "pass@2": 0.9269099707706995, + "pass@4": 0.9605988600856707, + "pass@8": 0.9739233460421274, + "pass@16": 0.9793009784562008, + "pass@32": 0.9826759601394346, + "pass@64": 0.9857259818108475, + "pass@128": 0.9878787878787879 + }, + "4": { + "pass@1": 0.7838618713017751, + "pass@2": 0.8724043266318782, + "pass@4": 0.9301489839657617, + "pass@8": 0.960023013690949, + "pass@16": 0.97335069655027, + "pass@32": 0.9804791565050158, + "pass@64": 0.9861651795826174, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.6375226449275362, + "pass@2": 0.7545318384115028, + "pass@4": 0.8361816756601076, + "pass@8": 0.8851649784226489, + "pass@16": 0.9156569142518054, + "pass@32": 0.9365426588707715, + "pass@64": 0.9514844580155023, + "pass@128": 0.9623188405797102 + }, + "6": { + "pass@1": 0.6906787249283668, + "pass@2": 0.8065539477246575, + "pass@4": 0.8880656017567995, + "pass@8": 0.9351576339738463, + "pass@16": 0.9588356055013254, + "pass@32": 0.9712646483259476, + "pass@64": 0.9791367802266179, + "pass@128": 0.9856733524355301 + }, + "7": { + "pass@1": 0.6023753612716763, + "pass@2": 0.7327194085385282, + "pass@4": 0.8357873419724274, + "pass@8": 0.9079161103488637, + "pass@16": 0.9501706105430924, + "pass@32": 0.9716194979336098, + "pass@64": 0.982082756700385, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.5243617957746479, + "pass@2": 0.673133733503383, + "pass@4": 0.7976204885480864, + "pass@8": 0.885373322021841, + "pass@16": 0.9396246435828248, + "pass@32": 0.9694345592489272, + "pass@64": 0.9842985934546734, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.48940400449101795, + "pass@2": 0.6394930365646656, + "pass@4": 0.7650968950737445, + "pass@8": 0.8614388295275515, + "pass@16": 0.9280775013732009, + "pass@32": 0.9675055589240936, + "pass@64": 0.9853653966250359, + "pass@128": 0.9910179640718563 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 262669, + "answer_accuracy": 0.6870109014060931, + "avg_response_len": 218.87636006026113, + "resp_tokens_sum": 83684312, + "pass_at_k": { + "pass@1": 0.6870109014060931, + "pass@2": 0.7857152154085045, + "pass@4": 0.8626302381247399, + "pass@8": 0.9166255896968976, + "pass@16": 0.9506154713869845, + "pass@32": 0.9705481189460016, + "pass@64": 0.9816355910625663, + "pass@128": 0.987612989621694 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.47495335820895523, + "pass@2": 0.623364231989658, + "pass@4": 0.7471682490248422, + "pass@8": 0.8398494017450072, + "pass@16": 0.9037033213806163, + "pass@32": 0.9430667350935134, + "pass@64": 0.963281894325653, + "pass@128": 0.9761194029850746 + }, + "2": { + "pass@1": 0.9677455357142857, + "pass@2": 0.9768110236220473, + "pass@4": 0.9826745586266004, + "pass@8": 0.9871195351191154, + "pass@16": 0.9899088225558785, + "pass@32": 0.9911571674097157, + "pass@64": 0.9914240481035442, + "pass@128": 0.9914285714285714 + }, + "3": { + "pass@1": 0.8923104956268222, + "pass@2": 0.9460619877872416, + "pass@4": 0.9716916010498681, + "pass@8": 0.9819850579920034, + "pass@16": 0.9853747770803383, + "pass@32": 0.9866242439336862, + "pass@64": 0.9876146649487308, + "pass@128": 0.9883381924198251 + }, + "4": { + "pass@1": 0.8124086257309941, + "pass@2": 0.8793381193304781, + "pass@4": 0.9236651819290124, + "pass@8": 0.952532626265685, + "pass@16": 0.9713757273390042, + "pass@32": 0.9841841950589343, + "pass@64": 0.9915764391070508, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.67109375, + "pass@2": 0.7686660490968045, + "pass@4": 0.846831118077152, + "pass@8": 0.9036536299326656, + "pass@16": 0.9391996061262696, + "pass@32": 0.9592355455929494, + "pass@64": 0.9716352127256529, + "pass@128": 0.9794117647058823 + }, + "6": { + "pass@1": 0.7108142744479495, + "pass@2": 0.8149187138279639, + "pass@4": 0.890687050159739, + "pass@8": 0.9365963614462343, + "pass@16": 0.9617556334998504, + "pass@32": 0.9761391150509602, + "pass@64": 0.9848625244179728, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.628065664556962, + "pass@2": 0.7528550315209811, + "pass@4": 0.8482620916451892, + "pass@8": 0.9137656555014311, + "pass@16": 0.950796041870444, + "pass@32": 0.9702759776932505, + "pass@64": 0.9808180164586792, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.5572916666666666, + "pass@2": 0.7052213384644487, + "pass@4": 0.8218555895603137, + "pass@8": 0.902596510105195, + "pass@16": 0.9517949896970779, + "pass@32": 0.978889599701662, + "pass@64": 0.9926690796876108, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.42710008038585207, + "pass@2": 0.5741832449299945, + "pass@4": 0.7114348669639761, + "pass@8": 0.8201747611420893, + "pass@16": 0.8954404581504106, + "pass@32": 0.9422447763282753, + "pass@64": 0.9695241077569068, + "pass@128": 0.9839228295819936 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.2007515625, + "avg_response_len": 315.76594609375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2007515625, + "pass@2": 0.2946044414370124, + "pass@4": 0.3959062469722573, + "pass@8": 0.4913513469485593, + "pass@16": 0.5715126672075507, + "pass@32": 0.6334514774533725, + "pass@64": 0.6804898489824805, + "pass@128": 0.7188 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3785546875, + "pass@2": 0.5274135088582681, + "pass@4": 0.6649970094675667, + "pass@8": 0.7742118241198892, + "pass@16": 0.8527273139381635, + "pass@32": 0.9043119669349217, + "pass@64": 0.935317522955068, + "pass@128": 0.956 + }, + "12": { + "pass@1": 0.315734375, + "pass@2": 0.44720546259842486, + "pass@4": 0.5770580259655036, + "pass@8": 0.6903081858777339, + "pass@16": 0.780733671495773, + "pass@32": 0.8487832332702647, + "pass@64": 0.8976167035212879, + "pass@128": 0.932 + }, + "13": { + "pass@1": 0.2449609375, + "pass@2": 0.35291830708661415, + "pass@4": 0.4684043848893887, + "pass@8": 0.5766009104446622, + "pass@16": 0.668861788051177, + "pass@32": 0.7432899891795103, + "pass@64": 0.803186790851953, + "pass@128": 0.855 + }, + "14": { + "pass@1": 0.2227265625, + "pass@2": 0.3258378444881892, + "pass@4": 0.4331095793025868, + "pass@8": 0.5307248037101999, + "pass@16": 0.6139301662190384, + "pass@32": 0.6838601209647185, + "pass@64": 0.7439559725492523, + "pass@128": 0.795 + }, + "15": { + "pass@1": 0.173203125, + "pass@2": 0.26356569881889763, + "pass@4": 0.3660749837832765, + "pass@8": 0.4664740438327597, + "pass@16": 0.5550727709276908, + "pass@32": 0.6283213058634126, + "pass@64": 0.6859979395274989, + "pass@128": 0.732 + }, + "16": { + "pass@1": 0.15965625, + "pass@2": 0.23523462106299184, + "pass@4": 0.320119775309336, + "pass@8": 0.4041228952662305, + "pass@16": 0.47892231066690466, + "pass@32": 0.5433424540250937, + "pass@64": 0.59955952289602, + "pass@128": 0.647 + }, + "17": { + "pass@1": 0.1449140625, + "pass@2": 0.22285482283464564, + "pass@4": 0.31279313873265785, + "pass@8": 0.4010318023396403, + "pass@16": 0.474828282235777, + "pass@32": 0.5299228587969564, + "pass@64": 0.5725745778361933, + "pass@128": 0.611 + }, + "18": { + "pass@1": 0.1400546875, + "pass@2": 0.2118804133858266, + "pass@4": 0.2962147456880388, + "pass@8": 0.38244051847904775, + "pass@16": 0.45735655353352805, + "pass@32": 0.5135397102666166, + "pass@64": 0.5539213413626312, + "pass@128": 0.588 + }, + "19": { + "pass@1": 0.1136875, + "pass@2": 0.17863988681102355, + "pass@4": 0.2585911073303335, + "pass@8": 0.3412827702769446, + "pass@16": 0.41318235526965275, + "pass@32": 0.4677034691616054, + "pass@64": 0.5075086796015467, + "pass@128": 0.54 + }, + "20": { + "pass@1": 0.1140234375, + "pass@2": 0.18049384842519672, + "pass@4": 0.261699719253843, + "pass@8": 0.34631571513835946, + "pass@16": 0.41951145973762105, + "pass@32": 0.4714396660706176, + "pass@64": 0.5052594387232707, + "pass@128": 0.532 + } + }, + "per_op_accuracy": { + "11": 0.3785546875, + "12": 0.315734375, + "13": 0.2449609375, + "14": 0.2227265625, + "15": 0.173203125, + "16": 0.15965625, + "17": 0.1449140625, + "18": 0.1400546875, + "19": 0.1136875, + "20": 0.1140234375 + }, + "per_op_avg_response_len": { + "11": 299.1008828125, + "12": 301.861171875, + "13": 311.158953125, + "14": 317.5734609375, + "15": 320.3238359375, + "16": 315.5424453125, + "17": 324.372078125, + "18": 324.3397421875, + "19": 320.361328125, + "20": 323.0255625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 299.1008828125, + "12": 301.861171875, + "13": 311.158953125, + "14": 317.5734609375, + "15": 320.3238359375, + "16": 315.5424453125, + "17": 324.372078125, + "18": 324.3397421875, + "19": 320.361328125, + "20": 323.0255625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 86787, + "answer_accuracy": 0.20149284918276375, + "avg_response_len": 300.62174498514116, + "resp_tokens_sum": 129483798, + "pass_at_k": { + "pass@1": 0.20149284918276375, + "pass@2": 0.29490613044190395, + "pass@4": 0.39521196005774306, + "pass@8": 0.4895170668304611, + "pass@16": 0.5691828698218978, + "pass@32": 0.6316325760197526, + "pass@64": 0.6794396869878608, + "pass@128": 0.7179791976225854 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3921963526912181, + "pass@2": 0.5415787206397362, + "pass@4": 0.6760388732392862, + "pass@8": 0.7790355829347181, + "pass@16": 0.8543600624135301, + "pass@32": 0.9077318451268638, + "pass@64": 0.9404013355393905, + "pass@128": 0.9631728045325779 + }, + "12": { + "pass@1": 0.2990819209039548, + "pass@2": 0.42569940277592416, + "pass@4": 0.5545588146820628, + "pass@8": 0.6733555030767344, + "pass@16": 0.7733669214501476, + "pass@32": 0.8514951163900696, + "pass@64": 0.905878813478266, + "pass@128": 0.9350282485875706 + }, + "13": { + "pass@1": 0.26119220219435735, + "pass@2": 0.37001201774738973, + "pass@4": 0.48377948671384713, + "pass@8": 0.5913443740517459, + "pass@16": 0.6861284752670558, + "pass@32": 0.764477112035867, + "pass@64": 0.8285231373938065, + "pass@128": 0.8840125391849529 + }, + "14": { + "pass@1": 0.21434482087227413, + "pass@2": 0.3150246369612677, + "pass@4": 0.4199875996808808, + "pass@8": 0.5154268383070109, + "pass@16": 0.597963503716804, + "pass@32": 0.670862546098087, + "pass@64": 0.736826979824222, + "pass@128": 0.7912772585669782 + }, + "15": { + "pass@1": 0.17119761560693642, + "pass@2": 0.2615891402303036, + "pass@4": 0.3643629110125685, + "pass@8": 0.4638810821555635, + "pass@16": 0.5488619660921231, + "pass@32": 0.6166011084092652, + "pass@64": 0.6693182181911753, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.16743951612903227, + "pass@2": 0.2455216535433071, + "pass@4": 0.332099522539521, + "pass@8": 0.4180017910081934, + "pass@16": 0.49562935124361973, + "pass@32": 0.5626821505928203, + "pass@64": 0.6208124764624271, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.13091582369942195, + "pass@2": 0.2026965232579309, + "pass@4": 0.28583829468932, + "pass@8": 0.36784673635324805, + "pass@16": 0.4371569990362648, + "pass@32": 0.49075281833375817, + "pass@64": 0.5351253719194886, + "pass@128": 0.5809248554913294 + }, + "18": { + "pass@1": 0.140625, + "pass@2": 0.21281390607424075, + "pass@4": 0.2967270649204563, + "pass@8": 0.38152916752215504, + "pass@16": 0.45486813332834614, + "pass@32": 0.5073866663449251, + "pass@64": 0.5420570454433494, + "pass@128": 0.5742857142857143 + }, + "19": { + "pass@1": 0.11249530780780781, + "pass@2": 0.17809714241800853, + "pass@4": 0.25848685102425245, + "pass@8": 0.3399542597368296, + "pass@16": 0.40889877405287706, + "pass@32": 0.46292300786984686, + "pass@64": 0.5045700654986855, + "pass@128": 0.5345345345345346 + }, + "20": { + "pass@1": 0.11549831081081081, + "pass@2": 0.18252775413208475, + "pass@4": 0.26408397570799147, + "pass@8": 0.3473522314072748, + "pass@16": 0.4172471386714921, + "pass@32": 0.465631560238557, + "pass@64": 0.4982591181602525, + "pass@128": 0.5255255255255256 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 86131, + "answer_accuracy": 0.20008874145108535, + "avg_response_len": 312.1805447145406, + "resp_tokens_sum": 134382486, + "pass_at_k": { + "pass@1": 0.20008874145108535, + "pass@2": 0.2934413859368162, + "pass@4": 0.3948427501702806, + "pass@8": 0.4910969097668026, + "pass@16": 0.5722420666703005, + "pass@32": 0.6348362570771938, + "pass@64": 0.6826214913662856, + "pass@128": 0.7237585489146595 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3600809487951807, + "pass@2": 0.5060478133004463, + "pass@4": 0.647385364574157, + "pass@8": 0.765477598410561, + "pass@16": 0.8514365450245749, + "pass@32": 0.9050546071868998, + "pass@64": 0.9358720866999578, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.3449990203761755, + "pass@2": 0.47940900143163934, + "pass@4": 0.6083281063221331, + "pass@8": 0.716982685718736, + "pass@16": 0.7997644746548964, + "pass@32": 0.8590672882733515, + "pass@64": 0.9032935756972884, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.23816088935574228, + "pass@2": 0.34847689075630256, + "pass@4": 0.4657425271946047, + "pass@8": 0.5719295256799126, + "pass@16": 0.6597921500111992, + "pass@32": 0.7322101517746895, + "pass@64": 0.7924964201460486, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.22480354532163743, + "pass@2": 0.32782389660174055, + "pass@4": 0.4342883085338015, + "pass@8": 0.5318416120559322, + "pass@16": 0.614830521987863, + "pass@32": 0.6842228584791915, + "pass@64": 0.7460521925082643, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.1651278409090909, + "pass@2": 0.24669828203292754, + "pass@4": 0.3395767744940974, + "pass@8": 0.4333264190796036, + "pass@16": 0.5224560313801565, + "pass@32": 0.6018586127709377, + "pass@64": 0.6662757719754229, + "pass@128": 0.7181818181818181 + }, + "16": { + "pass@1": 0.15731837606837606, + "pass@2": 0.2312620577876484, + "pass@4": 0.31380667774434196, + "pass@8": 0.3952425891605544, + "pass@16": 0.4682784764472947, + "pass@32": 0.5316408674982099, + "pass@64": 0.587040470177276, + "pass@128": 0.6381766381766382 + }, + "17": { + "pass@1": 0.1607628105590062, + "pass@2": 0.2452281355455568, + "pass@4": 0.3430799177082678, + "pass@8": 0.4389751464459959, + "pass@16": 0.5188112906635529, + "pass@32": 0.5781936339374035, + "pass@64": 0.6212796797280741, + "pass@128": 0.65527950310559 + }, + "18": { + "pass@1": 0.1396829044117647, + "pass@2": 0.2133181594488189, + "pass@4": 0.3000968705198614, + "pass@8": 0.39007173591004785, + "pass@16": 0.4709636864254259, + "pass@32": 0.5343133483461368, + "pass@64": 0.5793604777810965, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.10928199404761904, + "pass@2": 0.17229316062054745, + "pass@4": 0.2538720503129072, + "pass@8": 0.34257004009238523, + "pass@16": 0.420559560046734, + "pass@32": 0.4759493512788768, + "pass@64": 0.5138949420123206, + "pass@128": 0.5505952380952381 + }, + "20": { + "pass@1": 0.10689558383233533, + "pass@2": 0.1719055736951294, + "pass@4": 0.25167281522444424, + "pass@8": 0.3348155934449863, + "pass@16": 0.4058721468049287, + "pass@32": 0.45523155365388795, + "pass@64": 0.48832122929571226, + "pass@128": 0.5149700598802395 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 84044, + "answer_accuracy": 0.20067046149144255, + "avg_response_len": 335.0257081868888, + "resp_tokens_sum": 140314127, + "pass_at_k": { + "pass@1": 0.20067046149144255, + "pass@2": 0.295489579623338, + "pass@4": 0.3977133420860615, + "pass@8": 0.49349927630048923, + "pass@16": 0.5731589990563593, + "pass@32": 0.6338987847422449, + "pass@64": 0.6793789326545677, + "pass@128": 0.7145476772616137 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.3827380952380952, + "pass@2": 0.5340582427196598, + "pass@4": 0.6711852259538985, + "pass@8": 0.7780117481638958, + "pass@16": 0.8522580284381895, + "pass@32": 0.8996968127590125, + "pass@64": 0.9290359324611956, + "pass@128": 0.946031746031746 + }, + "12": { + "pass@1": 0.3052131116207951, + "pass@2": 0.43907156745888415, + "pass@4": 0.5709099071905003, + "pass@8": 0.6826387187898862, + "pass@16": 0.7701434675459031, + "pass@32": 0.8358149758684412, + "pass@64": 0.8831344125460127, + "pass@128": 0.9174311926605505 + }, + "13": { + "pass@1": 0.23647280092592593, + "pass@2": 0.3409821710654225, + "pass@4": 0.45619952598517804, + "pass@8": 0.5672321433778583, + "pass@16": 0.6618549595277416, + "pass@32": 0.7346381674583468, + "pass@64": 0.7900206420715697, + "pass@128": 0.8333333333333334 + }, + "14": { + "pass@1": 0.22860255934718102, + "pass@2": 0.3341221465454799, + "pass@4": 0.44441233912897393, + "pass@8": 0.5441630780134137, + "pass@16": 0.6282250534305504, + "pass@32": 0.695872480912015, + "pass@64": 0.7486191756316015, + "pass@128": 0.7893175074183977 + }, + "15": { + "pass@1": 0.18356963734567902, + "pass@2": 0.28285624453193353, + "pass@4": 0.3948922252773957, + "pass@8": 0.5030045404650186, + "pass@16": 0.5949260503221181, + "pass@32": 0.6677900621586327, + "pass@64": 0.723897627411919, + "pass@128": 0.7654320987654321 + }, + "16": { + "pass@1": 0.1549594395280236, + "pass@2": 0.22994078519499225, + "pass@4": 0.31570141425905845, + "pass@8": 0.40062593291544546, + "pass@16": 0.47466509306307514, + "pass@32": 0.537772987756483, + "pass@64": 0.5930868736296269, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.1441312123493976, + "pass@2": 0.22216375343895262, + "pass@4": 0.311510106229191, + "pass@8": 0.3988157662825498, + "pass@16": 0.47142989450471834, + "pass@32": 0.5239278118241953, + "pass@64": 0.564364886384367, + "pass@128": 0.5993975903614458 + }, + "18": { + "pass@1": 0.13981854838709679, + "pass@2": 0.20924958724917453, + "pass@4": 0.2913785064165365, + "pass@8": 0.37509974076412117, + "pass@16": 0.4452421080127842, + "pass@32": 0.49770270518776283, + "pass@64": 0.539415525844793, + "pass@128": 0.5774193548387097 + }, + "19": { + "pass@1": 0.11935895015105741, + "pass@2": 0.1856285088136642, + "pass@4": 0.263486335450516, + "pass@8": 0.3413125930922642, + "pass@16": 0.4100031762366848, + "pass@32": 0.46414237314575196, + "pass@64": 0.5039823180493795, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.11969782282282282, + "pass@2": 0.1870740080631812, + "pass@4": 0.26937247765200556, + "pass@8": 0.35681385546279937, + "pass@16": 0.43545605263414, + "pass@32": 0.49350455732966975, + "pass@64": 0.5292488342076239, + "pass@128": 0.5555555555555556 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.4181188322368421, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.4181188322368421, + "pass@2": 0.5179338673331793, + "pass@4": 0.6107797957245282, + "pass@8": 0.6884272196841036, + "pass@16": 0.7478696051197281, + "pass@32": 0.7907675698715844, + "pass@64": 0.8214146700823132, + "pass@128": 0.8448947368421053 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.4250078125, + "pass@2": 0.5768004429133857, + "pass@4": 0.7134305258717659, + "pass@8": 0.8199063979574568, + "pass@16": 0.8935452112743179, + "pass@32": 0.9398754942428027, + "pass@64": 0.9654533308462334, + "pass@128": 0.979 + }, + "2": { + "pass@1": 0.9381015625, + "pass@2": 0.9528553149606297, + "pass@4": 0.9630730704911884, + "pass@8": 0.9702266488878839, + "pass@16": 0.975452197911979, + "pass@32": 0.9793132877954412, + "pass@64": 0.9817224170335146, + "pass@128": 0.983 + }, + "3": { + "pass@1": 0.8848359375, + "pass@2": 0.9430001230314963, + "pass@4": 0.9707062199100114, + "pass@8": 0.9812595682703927, + "pass@16": 0.9852764841848526, + "pass@32": 0.987617675650772, + "pass@64": 0.9896682272274708, + "pass@128": 0.991 + }, + "4": { + "pass@1": 0.773953125, + "pass@2": 0.858044414370079, + "pass@4": 0.9149909086051744, + "pass@8": 0.9485047792560358, + "pass@16": 0.9672467913518666, + "pass@32": 0.9788297174812043, + "pass@64": 0.98618043104808, + "pass@128": 0.99 + }, + "5": { + "pass@1": 0.6465078125, + "pass@2": 0.7576323818897631, + "pass@4": 0.8412265929883768, + "pass@8": 0.8968452107771091, + "pass@16": 0.931368042735231, + "pass@32": 0.9520664882811875, + "pass@64": 0.9645204472984087, + "pass@128": 0.973 + }, + "6": { + "pass@1": 0.6936328125, + "pass@2": 0.803946727362204, + "pass@4": 0.8832361657292838, + "pass@8": 0.9314466219085178, + "pass@16": 0.9575838221416829, + "pass@32": 0.9724694137582386, + "pass@64": 0.9823185183926962, + "pass@128": 0.99 + }, + "7": { + "pass@1": 0.598171875, + "pass@2": 0.7292191190944884, + "pass@4": 0.8330857781214833, + "pass@8": 0.9052266116175698, + "pass@16": 0.9468352343911222, + "pass@32": 0.9687958802123822, + "pass@64": 0.9813171526673037, + "pass@128": 0.988 + }, + "8": { + "pass@1": 0.53165625, + "pass@2": 0.6782041092519686, + "pass@4": 0.7972615198725161, + "pass@8": 0.8804777104729525, + "pass@16": 0.9322194970921365, + "pass@32": 0.9615292324789348, + "pass@64": 0.9769648857667702, + "pass@128": 0.986 + }, + "9": { + "pass@1": 0.444875, + "pass@2": 0.5949964320866137, + "pass@4": 0.7287428674540681, + "pass@8": 0.8327101553644152, + "pass@16": 0.9048685441158348, + "pass@32": 0.9495718631248805, + "pass@64": 0.9738348314586076, + "pass@128": 0.985 + }, + "11": { + "pass@1": 0.3785546875, + "pass@2": 0.5274135088582681, + "pass@4": 0.6649970094675667, + "pass@8": 0.7742118241198892, + "pass@16": 0.8527273139381635, + "pass@32": 0.9043119669349217, + "pass@64": 0.935317522955068, + "pass@128": 0.956 + }, + "12": { + "pass@1": 0.315734375, + "pass@2": 0.44720546259842486, + "pass@4": 0.5770580259655036, + "pass@8": 0.6903081858777339, + "pass@16": 0.780733671495773, + "pass@32": 0.8487832332702647, + "pass@64": 0.8976167035212879, + "pass@128": 0.932 + }, + "13": { + "pass@1": 0.2449609375, + "pass@2": 0.35291830708661415, + "pass@4": 0.4684043848893887, + "pass@8": 0.5766009104446622, + "pass@16": 0.668861788051177, + "pass@32": 0.7432899891795103, + "pass@64": 0.803186790851953, + "pass@128": 0.855 + }, + "14": { + "pass@1": 0.2227265625, + "pass@2": 0.3258378444881892, + "pass@4": 0.4331095793025868, + "pass@8": 0.5307248037101999, + "pass@16": 0.6139301662190384, + "pass@32": 0.6838601209647185, + "pass@64": 0.7439559725492523, + "pass@128": 0.795 + }, + "15": { + "pass@1": 0.173203125, + "pass@2": 0.26356569881889763, + "pass@4": 0.3660749837832765, + "pass@8": 0.4664740438327597, + "pass@16": 0.5550727709276908, + "pass@32": 0.6283213058634126, + "pass@64": 0.6859979395274989, + "pass@128": 0.732 + }, + "16": { + "pass@1": 0.15965625, + "pass@2": 0.23523462106299184, + "pass@4": 0.320119775309336, + "pass@8": 0.4041228952662305, + "pass@16": 0.47892231066690466, + "pass@32": 0.5433424540250937, + "pass@64": 0.59955952289602, + "pass@128": 0.647 + }, + "17": { + "pass@1": 0.1449140625, + "pass@2": 0.22285482283464564, + "pass@4": 0.31279313873265785, + "pass@8": 0.4010318023396403, + "pass@16": 0.474828282235777, + "pass@32": 0.5299228587969564, + "pass@64": 0.5725745778361933, + "pass@128": 0.611 + }, + "18": { + "pass@1": 0.1400546875, + "pass@2": 0.2118804133858266, + "pass@4": 0.2962147456880388, + "pass@8": 0.38244051847904775, + "pass@16": 0.45735655353352805, + "pass@32": 0.5135397102666166, + "pass@64": 0.5539213413626312, + "pass@128": 0.588 + }, + "19": { + "pass@1": 0.1136875, + "pass@2": 0.17863988681102355, + "pass@4": 0.2585911073303335, + "pass@8": 0.3412827702769446, + "pass@16": 0.41318235526965275, + "pass@32": 0.4677034691616054, + "pass@64": 0.5075086796015467, + "pass@128": 0.54 + }, + "20": { + "pass@1": 0.1140234375, + "pass@2": 0.18049384842519672, + "pass@4": 0.261699719253843, + "pass@8": 0.34631571513835946, + "pass@16": 0.41951145973762105, + "pass@32": 0.4714396660706176, + "pass@64": 0.5052594387232707, + "pass@128": 0.532 + } + }, + "per_op_accuracy": { + "10": 0.4250078125, + "2": 0.9381015625, + "3": 0.8848359375, + "4": 0.773953125, + "5": 0.6465078125, + "6": 0.6936328125, + "7": 0.598171875, + "8": 0.53165625, + "9": 0.444875, + "11": 0.3785546875, + "12": 0.315734375, + "13": 0.2449609375, + "14": 0.2227265625, + "15": 0.173203125, + "16": 0.15965625, + "17": 0.1449140625, + "18": 0.1400546875, + "19": 0.1136875, + "20": 0.1140234375 + }, + "per_op_avg_response_len": { + "10": 285.6915, + "2": 120.4114609375, + "3": 151.4393359375, + "4": 169.4210625, + "5": 181.6033359375, + "6": 205.7773359375, + "7": 232.946625, + "8": 252.271203125, + "9": 267.4645625, + "11": 299.1008828125, + "12": 301.861171875, + "13": 311.158953125, + "14": 317.5734609375, + "15": 320.3238359375, + "16": 315.5424453125, + "17": 324.372078125, + "18": 324.3397421875, + "19": 320.361328125, + "20": 323.0255625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 285.6915, + "2": 120.4114609375, + "3": 151.4393359375, + "4": 169.4210625, + "5": 181.6033359375, + "6": 205.7773359375, + "7": 232.946625, + "8": 252.271203125, + "9": 267.4645625, + "11": 299.1008828125, + "12": 301.861171875, + "13": 311.158953125, + "14": 317.5734609375, + "15": 320.3238359375, + "16": 315.5424453125, + "17": 324.372078125, + "18": 324.3397421875, + "19": 320.361328125, + "20": 323.0255625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 326230, + "answer_accuracy": 0.4028883773316472, + "avg_response_len": 252.42235170328803, + "resp_tokens_sum": 204393446, + "pass_at_k": { + "pass@1": 0.4028883773316472, + "pass@2": 0.5040771540897414, + "pass@4": 0.599082216208909, + "pass@8": 0.678747216971971, + "pass@16": 0.7400238654947899, + "pass@32": 0.7848226074309053, + "pass@64": 0.8170806105461791, + "pass@128": 0.8414479924122669 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.371427912979351, + "pass@2": 0.523322997235965, + "pass@4": 0.6718812374669977, + "pass@8": 0.7933920591197899, + "pass@16": 0.8779724027500238, + "pass@32": 0.9314886148824195, + "pass@64": 0.9611391857168347, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9139164719626168, + "pass@2": 0.9309390867613512, + "pass@4": 0.9439649050877985, + "pass@8": 0.9533337561204487, + "pass@16": 0.9605609138316018, + "pass@32": 0.966119078288411, + "pass@64": 0.9697424698773338, + "pass@128": 0.9719626168224299 + }, + "3": { + "pass@1": 0.9014000382262997, + "pass@2": 0.9560262105998218, + "pass@4": 0.9798727122985766, + "pass@8": 0.9879021076001016, + "pass@16": 0.9912037087331826, + "pass@32": 0.9936467679984826, + "pass@64": 0.9958006824234752, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.7223876953125, + "pass@2": 0.8201191098671256, + "pass@4": 0.8897095618907006, + "pass@8": 0.9320338826425963, + "pass@16": 0.956386741149801, + "pass@32": 0.9713648996011031, + "pass@64": 0.9804295567954489, + "pass@128": 0.984375 + }, + "5": { + "pass@1": 0.629811507936508, + "pass@2": 0.7491188601424822, + "pass@4": 0.8407026499663729, + "pass@8": 0.9022890763307583, + "pass@16": 0.9401223531283432, + "pass@32": 0.96133074752117, + "pass@64": 0.9711185300201212, + "pass@128": 0.9777777777777777 + }, + "6": { + "pass@1": 0.6804126122754491, + "pass@2": 0.7908088842001034, + "pass@4": 0.8711181910644397, + "pass@8": 0.9226813205784111, + "pass@16": 0.9523163473121824, + "pass@32": 0.9702453653333176, + "pass@64": 0.9832286283027216, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.5659208579881657, + "pass@2": 0.703538502306295, + "pass@4": 0.8161317657962868, + "pass@8": 0.8944902079834598, + "pass@16": 0.9397179109827034, + "pass@32": 0.9645216715274736, + "pass@64": 0.9810000758817367, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.5125951522435898, + "pass@2": 0.6551375902230964, + "pass@4": 0.7706036381870534, + "pass@8": 0.8512998182377252, + "pass@16": 0.9029006956766283, + "pass@32": 0.9340055360413846, + "pass@64": 0.9518592357512387, + "pass@128": 0.9647435897435898 + }, + "9": { + "pass@1": 0.4185519366197183, + "pass@2": 0.5713655456360205, + "pass@4": 0.7097021433060301, + "pass@8": 0.8166626354564073, + "pass@16": 0.8912920455560849, + "pass@32": 0.9391179746651805, + "pass@64": 0.9667627928829464, + "pass@128": 0.9802816901408451 + }, + "11": { + "pass@1": 0.3921963526912181, + "pass@2": 0.5415787206397362, + "pass@4": 0.6760388732392862, + "pass@8": 0.7790355829347181, + "pass@16": 0.8543600624135301, + "pass@32": 0.9077318451268638, + "pass@64": 0.9404013355393905, + "pass@128": 0.9631728045325779 + }, + "12": { + "pass@1": 0.2990819209039548, + "pass@2": 0.42569940277592416, + "pass@4": 0.5545588146820628, + "pass@8": 0.6733555030767344, + "pass@16": 0.7733669214501476, + "pass@32": 0.8514951163900696, + "pass@64": 0.905878813478266, + "pass@128": 0.9350282485875706 + }, + "13": { + "pass@1": 0.26119220219435735, + "pass@2": 0.37001201774738973, + "pass@4": 0.48377948671384713, + "pass@8": 0.5913443740517459, + "pass@16": 0.6861284752670558, + "pass@32": 0.764477112035867, + "pass@64": 0.8285231373938065, + "pass@128": 0.8840125391849529 + }, + "14": { + "pass@1": 0.21434482087227413, + "pass@2": 0.3150246369612677, + "pass@4": 0.4199875996808808, + "pass@8": 0.5154268383070109, + "pass@16": 0.597963503716804, + "pass@32": 0.670862546098087, + "pass@64": 0.736826979824222, + "pass@128": 0.7912772585669782 + }, + "15": { + "pass@1": 0.17119761560693642, + "pass@2": 0.2615891402303036, + "pass@4": 0.3643629110125685, + "pass@8": 0.4638810821555635, + "pass@16": 0.5488619660921231, + "pass@32": 0.6166011084092652, + "pass@64": 0.6693182181911753, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.16743951612903227, + "pass@2": 0.2455216535433071, + "pass@4": 0.332099522539521, + "pass@8": 0.4180017910081934, + "pass@16": 0.49562935124361973, + "pass@32": 0.5626821505928203, + "pass@64": 0.6208124764624271, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.13091582369942195, + "pass@2": 0.2026965232579309, + "pass@4": 0.28583829468932, + "pass@8": 0.36784673635324805, + "pass@16": 0.4371569990362648, + "pass@32": 0.49075281833375817, + "pass@64": 0.5351253719194886, + "pass@128": 0.5809248554913294 + }, + "18": { + "pass@1": 0.140625, + "pass@2": 0.21281390607424075, + "pass@4": 0.2967270649204563, + "pass@8": 0.38152916752215504, + "pass@16": 0.45486813332834614, + "pass@32": 0.5073866663449251, + "pass@64": 0.5420570454433494, + "pass@128": 0.5742857142857143 + }, + "19": { + "pass@1": 0.11249530780780781, + "pass@2": 0.17809714241800853, + "pass@4": 0.25848685102425245, + "pass@8": 0.3399542597368296, + "pass@16": 0.40889877405287706, + "pass@32": 0.46292300786984686, + "pass@64": 0.5045700654986855, + "pass@128": 0.5345345345345346 + }, + "20": { + "pass@1": 0.11549831081081081, + "pass@2": 0.18252775413208475, + "pass@4": 0.26408397570799147, + "pass@8": 0.3473522314072748, + "pass@16": 0.4172471386714921, + "pass@32": 0.465631560238557, + "pass@64": 0.4982591181602525, + "pass@128": 0.5255255255255256 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 343922, + "answer_accuracy": 0.4188449922057677, + "avg_response_len": 261.5548372954014, + "resp_tokens_sum": 214767908, + "pass_at_k": { + "pass@1": 0.4188449922057677, + "pass@2": 0.5203707170080039, + "pass@4": 0.6137221266476559, + "pass@8": 0.6911413890808137, + "pass@16": 0.7503143984737648, + "pass@32": 0.7929309365523514, + "pass@64": 0.8235313189554284, + "pass@128": 0.8483242400623539 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.42939992331288346, + "pass@2": 0.5845611323124487, + "pass@4": 0.7219675550525508, + "pass@8": 0.8269844795437781, + "pass@16": 0.8993004726366725, + "pass@32": 0.9453174771206605, + "pass@64": 0.9721708965924318, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9301624240121581, + "pass@2": 0.9487538293085703, + "pass@4": 0.9608639527012007, + "pass@8": 0.9687376166611839, + "pass@16": 0.9746019959801766, + "pass@32": 0.9795867935302768, + "pass@64": 0.9830901743667175, + "pass@128": 0.9848024316109423 + }, + "3": { + "pass@1": 0.860653409090909, + "pass@2": 0.9269099707706995, + "pass@4": 0.9605988600856707, + "pass@8": 0.9739233460421274, + "pass@16": 0.9793009784562008, + "pass@32": 0.9826759601394346, + "pass@64": 0.9857259818108475, + "pass@128": 0.9878787878787879 + }, + "4": { + "pass@1": 0.7838618713017751, + "pass@2": 0.8724043266318782, + "pass@4": 0.9301489839657617, + "pass@8": 0.960023013690949, + "pass@16": 0.97335069655027, + "pass@32": 0.9804791565050158, + "pass@64": 0.9861651795826174, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.6375226449275362, + "pass@2": 0.7545318384115028, + "pass@4": 0.8361816756601076, + "pass@8": 0.8851649784226489, + "pass@16": 0.9156569142518054, + "pass@32": 0.9365426588707715, + "pass@64": 0.9514844580155023, + "pass@128": 0.9623188405797102 + }, + "6": { + "pass@1": 0.6906787249283668, + "pass@2": 0.8065539477246575, + "pass@4": 0.8880656017567995, + "pass@8": 0.9351576339738463, + "pass@16": 0.9588356055013254, + "pass@32": 0.9712646483259476, + "pass@64": 0.9791367802266179, + "pass@128": 0.9856733524355301 + }, + "7": { + "pass@1": 0.6023753612716763, + "pass@2": 0.7327194085385282, + "pass@4": 0.8357873419724274, + "pass@8": 0.9079161103488637, + "pass@16": 0.9501706105430924, + "pass@32": 0.9716194979336098, + "pass@64": 0.982082756700385, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.5243617957746479, + "pass@2": 0.673133733503383, + "pass@4": 0.7976204885480864, + "pass@8": 0.885373322021841, + "pass@16": 0.9396246435828248, + "pass@32": 0.9694345592489272, + "pass@64": 0.9842985934546734, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.48940400449101795, + "pass@2": 0.6394930365646656, + "pass@4": 0.7650968950737445, + "pass@8": 0.8614388295275515, + "pass@16": 0.9280775013732009, + "pass@32": 0.9675055589240936, + "pass@64": 0.9853653966250359, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.3600809487951807, + "pass@2": 0.5060478133004463, + "pass@4": 0.647385364574157, + "pass@8": 0.765477598410561, + "pass@16": 0.8514365450245749, + "pass@32": 0.9050546071868998, + "pass@64": 0.9358720866999578, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.3449990203761755, + "pass@2": 0.47940900143163934, + "pass@4": 0.6083281063221331, + "pass@8": 0.716982685718736, + "pass@16": 0.7997644746548964, + "pass@32": 0.8590672882733515, + "pass@64": 0.9032935756972884, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.23816088935574228, + "pass@2": 0.34847689075630256, + "pass@4": 0.4657425271946047, + "pass@8": 0.5719295256799126, + "pass@16": 0.6597921500111992, + "pass@32": 0.7322101517746895, + "pass@64": 0.7924964201460486, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.22480354532163743, + "pass@2": 0.32782389660174055, + "pass@4": 0.4342883085338015, + "pass@8": 0.5318416120559322, + "pass@16": 0.614830521987863, + "pass@32": 0.6842228584791915, + "pass@64": 0.7460521925082643, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.1651278409090909, + "pass@2": 0.24669828203292754, + "pass@4": 0.3395767744940974, + "pass@8": 0.4333264190796036, + "pass@16": 0.5224560313801565, + "pass@32": 0.6018586127709377, + "pass@64": 0.6662757719754229, + "pass@128": 0.7181818181818181 + }, + "16": { + "pass@1": 0.15731837606837606, + "pass@2": 0.2312620577876484, + "pass@4": 0.31380667774434196, + "pass@8": 0.3952425891605544, + "pass@16": 0.4682784764472947, + "pass@32": 0.5316408674982099, + "pass@64": 0.587040470177276, + "pass@128": 0.6381766381766382 + }, + "17": { + "pass@1": 0.1607628105590062, + "pass@2": 0.2452281355455568, + "pass@4": 0.3430799177082678, + "pass@8": 0.4389751464459959, + "pass@16": 0.5188112906635529, + "pass@32": 0.5781936339374035, + "pass@64": 0.6212796797280741, + "pass@128": 0.65527950310559 + }, + "18": { + "pass@1": 0.1396829044117647, + "pass@2": 0.2133181594488189, + "pass@4": 0.3000968705198614, + "pass@8": 0.39007173591004785, + "pass@16": 0.4709636864254259, + "pass@32": 0.5343133483461368, + "pass@64": 0.5793604777810965, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.10928199404761904, + "pass@2": 0.17229316062054745, + "pass@4": 0.2538720503129072, + "pass@8": 0.34257004009238523, + "pass@16": 0.420559560046734, + "pass@32": 0.4759493512788768, + "pass@64": 0.5138949420123206, + "pass@128": 0.5505952380952381 + }, + "20": { + "pass@1": 0.10689558383233533, + "pass@2": 0.1719055736951294, + "pass@4": 0.25167281522444424, + "pass@8": 0.3348155934449863, + "pass@16": 0.4058721468049287, + "pass@32": 0.45523155365388795, + "pass@64": 0.48832122929571226, + "pass@128": 0.5149700598802395 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 346713, + "answer_accuracy": 0.4327680639878575, + "avg_response_len": 279.5954313288864, + "resp_tokens_sum": 223998439, + "pass_at_k": { + "pass@1": 0.4327680639878575, + "pass@2": 0.5294413249644969, + "pass@4": 0.619586927078482, + "pass@8": 0.6954290251605538, + "pass@16": 0.7532936024836905, + "pass@32": 0.7945588840019715, + "pass@64": 0.8236257194679109, + "pass@128": 0.8448633967087394 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.47495335820895523, + "pass@2": 0.623364231989658, + "pass@4": 0.7471682490248422, + "pass@8": 0.8398494017450072, + "pass@16": 0.9037033213806163, + "pass@32": 0.9430667350935134, + "pass@64": 0.963281894325653, + "pass@128": 0.9761194029850746 + }, + "2": { + "pass@1": 0.9677455357142857, + "pass@2": 0.9768110236220473, + "pass@4": 0.9826745586266004, + "pass@8": 0.9871195351191154, + "pass@16": 0.9899088225558785, + "pass@32": 0.9911571674097157, + "pass@64": 0.9914240481035442, + "pass@128": 0.9914285714285714 + }, + "3": { + "pass@1": 0.8923104956268222, + "pass@2": 0.9460619877872416, + "pass@4": 0.9716916010498681, + "pass@8": 0.9819850579920034, + "pass@16": 0.9853747770803383, + "pass@32": 0.9866242439336862, + "pass@64": 0.9876146649487308, + "pass@128": 0.9883381924198251 + }, + "4": { + "pass@1": 0.8124086257309941, + "pass@2": 0.8793381193304781, + "pass@4": 0.9236651819290124, + "pass@8": 0.952532626265685, + "pass@16": 0.9713757273390042, + "pass@32": 0.9841841950589343, + "pass@64": 0.9915764391070508, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.67109375, + "pass@2": 0.7686660490968045, + "pass@4": 0.846831118077152, + "pass@8": 0.9036536299326656, + "pass@16": 0.9391996061262696, + "pass@32": 0.9592355455929494, + "pass@64": 0.9716352127256529, + "pass@128": 0.9794117647058823 + }, + "6": { + "pass@1": 0.7108142744479495, + "pass@2": 0.8149187138279639, + "pass@4": 0.890687050159739, + "pass@8": 0.9365963614462343, + "pass@16": 0.9617556334998504, + "pass@32": 0.9761391150509602, + "pass@64": 0.9848625244179728, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.628065664556962, + "pass@2": 0.7528550315209811, + "pass@4": 0.8482620916451892, + "pass@8": 0.9137656555014311, + "pass@16": 0.950796041870444, + "pass@32": 0.9702759776932505, + "pass@64": 0.9808180164586792, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.5572916666666666, + "pass@2": 0.7052213384644487, + "pass@4": 0.8218555895603137, + "pass@8": 0.902596510105195, + "pass@16": 0.9517949896970779, + "pass@32": 0.978889599701662, + "pass@64": 0.9926690796876108, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.42710008038585207, + "pass@2": 0.5741832449299945, + "pass@4": 0.7114348669639761, + "pass@8": 0.8201747611420893, + "pass@16": 0.8954404581504106, + "pass@32": 0.9422447763282753, + "pass@64": 0.9695241077569068, + "pass@128": 0.9839228295819936 + }, + "11": { + "pass@1": 0.3827380952380952, + "pass@2": 0.5340582427196598, + "pass@4": 0.6711852259538985, + "pass@8": 0.7780117481638958, + "pass@16": 0.8522580284381895, + "pass@32": 0.8996968127590125, + "pass@64": 0.9290359324611956, + "pass@128": 0.946031746031746 + }, + "12": { + "pass@1": 0.3052131116207951, + "pass@2": 0.43907156745888415, + "pass@4": 0.5709099071905003, + "pass@8": 0.6826387187898862, + "pass@16": 0.7701434675459031, + "pass@32": 0.8358149758684412, + "pass@64": 0.8831344125460127, + "pass@128": 0.9174311926605505 + }, + "13": { + "pass@1": 0.23647280092592593, + "pass@2": 0.3409821710654225, + "pass@4": 0.45619952598517804, + "pass@8": 0.5672321433778583, + "pass@16": 0.6618549595277416, + "pass@32": 0.7346381674583468, + "pass@64": 0.7900206420715697, + "pass@128": 0.8333333333333334 + }, + "14": { + "pass@1": 0.22860255934718102, + "pass@2": 0.3341221465454799, + "pass@4": 0.44441233912897393, + "pass@8": 0.5441630780134137, + "pass@16": 0.6282250534305504, + "pass@32": 0.695872480912015, + "pass@64": 0.7486191756316015, + "pass@128": 0.7893175074183977 + }, + "15": { + "pass@1": 0.18356963734567902, + "pass@2": 0.28285624453193353, + "pass@4": 0.3948922252773957, + "pass@8": 0.5030045404650186, + "pass@16": 0.5949260503221181, + "pass@32": 0.6677900621586327, + "pass@64": 0.723897627411919, + "pass@128": 0.7654320987654321 + }, + "16": { + "pass@1": 0.1549594395280236, + "pass@2": 0.22994078519499225, + "pass@4": 0.31570141425905845, + "pass@8": 0.40062593291544546, + "pass@16": 0.47466509306307514, + "pass@32": 0.537772987756483, + "pass@64": 0.5930868736296269, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.1441312123493976, + "pass@2": 0.22216375343895262, + "pass@4": 0.311510106229191, + "pass@8": 0.3988157662825498, + "pass@16": 0.47142989450471834, + "pass@32": 0.5239278118241953, + "pass@64": 0.564364886384367, + "pass@128": 0.5993975903614458 + }, + "18": { + "pass@1": 0.13981854838709679, + "pass@2": 0.20924958724917453, + "pass@4": 0.2913785064165365, + "pass@8": 0.37509974076412117, + "pass@16": 0.4452421080127842, + "pass@32": 0.49770270518776283, + "pass@64": 0.539415525844793, + "pass@128": 0.5774193548387097 + }, + "19": { + "pass@1": 0.11935895015105741, + "pass@2": 0.1856285088136642, + "pass@4": 0.263486335450516, + "pass@8": 0.3413125930922642, + "pass@16": 0.4100031762366848, + "pass@32": 0.46414237314575196, + "pass@64": 0.5039823180493795, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.11969782282282282, + "pass@2": 0.1870740080631812, + "pass@4": 0.26937247765200556, + "pass@8": 0.35681385546279937, + "pass@16": 0.43545605263414, + "pass@32": 0.49350455732966975, + "pass@64": 0.5292488342076239, + "pass@128": 0.5555555555555556 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/summary.csv b/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..56d464f84d26cb9740e5bd80d0611a0d6f6d6ae0 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21546,0.6596380208333333,0.2007515625,0.4181188322368421,0.0,0.0,0.0,207.44738020833333,315.76594609375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/summary.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e6ed6a61bd6704a5f31d12832fa2c93715038222 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block32/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21546", + "id_acc": 0.6596380208333333, + "ood_acc": 0.2007515625, + "total_acc": 0.4181188322368421, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 207.44738020833333, + "ood_avg_resp_len": 315.76594609375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 326230, + "answer_accuracy": 0.4028883773316472, + "avg_response_len": 252.42235170328803, + "resp_tokens_sum": 204393446, + "pass_at_k": { + "pass@1": 0.4028883773316472, + "pass@2": 0.5040771540897414, + "pass@4": 0.599082216208909, + "pass@8": 0.678747216971971, + "pass@16": 0.7400238654947899, + "pass@32": 0.7848226074309053, + "pass@64": 0.8170806105461791, + "pass@128": 0.8414479924122669 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.371427912979351, + "pass@2": 0.523322997235965, + "pass@4": 0.6718812374669977, + "pass@8": 0.7933920591197899, + "pass@16": 0.8779724027500238, + "pass@32": 0.9314886148824195, + "pass@64": 0.9611391857168347, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 0.9139164719626168, + "pass@2": 0.9309390867613512, + "pass@4": 0.9439649050877985, + "pass@8": 0.9533337561204487, + "pass@16": 0.9605609138316018, + "pass@32": 0.966119078288411, + "pass@64": 0.9697424698773338, + "pass@128": 0.9719626168224299 + }, + "3": { + "pass@1": 0.9014000382262997, + "pass@2": 0.9560262105998218, + "pass@4": 0.9798727122985766, + "pass@8": 0.9879021076001016, + "pass@16": 0.9912037087331826, + "pass@32": 0.9936467679984826, + "pass@64": 0.9958006824234752, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.7223876953125, + "pass@2": 0.8201191098671256, + "pass@4": 0.8897095618907006, + "pass@8": 0.9320338826425963, + "pass@16": 0.956386741149801, + "pass@32": 0.9713648996011031, + "pass@64": 0.9804295567954489, + "pass@128": 0.984375 + }, + "5": { + "pass@1": 0.629811507936508, + "pass@2": 0.7491188601424822, + "pass@4": 0.8407026499663729, + "pass@8": 0.9022890763307583, + "pass@16": 0.9401223531283432, + "pass@32": 0.96133074752117, + "pass@64": 0.9711185300201212, + "pass@128": 0.9777777777777777 + }, + "6": { + "pass@1": 0.6804126122754491, + "pass@2": 0.7908088842001034, + "pass@4": 0.8711181910644397, + "pass@8": 0.9226813205784111, + "pass@16": 0.9523163473121824, + "pass@32": 0.9702453653333176, + "pass@64": 0.9832286283027216, + "pass@128": 0.9940119760479041 + }, + "7": { + "pass@1": 0.5659208579881657, + "pass@2": 0.703538502306295, + "pass@4": 0.8161317657962868, + "pass@8": 0.8944902079834598, + "pass@16": 0.9397179109827034, + "pass@32": 0.9645216715274736, + "pass@64": 0.9810000758817367, + "pass@128": 0.9911242603550295 + }, + "8": { + "pass@1": 0.5125951522435898, + "pass@2": 0.6551375902230964, + "pass@4": 0.7706036381870534, + "pass@8": 0.8512998182377252, + "pass@16": 0.9029006956766283, + "pass@32": 0.9340055360413846, + "pass@64": 0.9518592357512387, + "pass@128": 0.9647435897435898 + }, + "9": { + "pass@1": 0.4185519366197183, + "pass@2": 0.5713655456360205, + "pass@4": 0.7097021433060301, + "pass@8": 0.8166626354564073, + "pass@16": 0.8912920455560849, + "pass@32": 0.9391179746651805, + "pass@64": 0.9667627928829464, + "pass@128": 0.9802816901408451 + }, + "11": { + "pass@1": 0.3921963526912181, + "pass@2": 0.5415787206397362, + "pass@4": 0.6760388732392862, + "pass@8": 0.7790355829347181, + "pass@16": 0.8543600624135301, + "pass@32": 0.9077318451268638, + "pass@64": 0.9404013355393905, + "pass@128": 0.9631728045325779 + }, + "12": { + "pass@1": 0.2990819209039548, + "pass@2": 0.42569940277592416, + "pass@4": 0.5545588146820628, + "pass@8": 0.6733555030767344, + "pass@16": 0.7733669214501476, + "pass@32": 0.8514951163900696, + "pass@64": 0.905878813478266, + "pass@128": 0.9350282485875706 + }, + "13": { + "pass@1": 0.26119220219435735, + "pass@2": 0.37001201774738973, + "pass@4": 0.48377948671384713, + "pass@8": 0.5913443740517459, + "pass@16": 0.6861284752670558, + "pass@32": 0.764477112035867, + "pass@64": 0.8285231373938065, + "pass@128": 0.8840125391849529 + }, + "14": { + "pass@1": 0.21434482087227413, + "pass@2": 0.3150246369612677, + "pass@4": 0.4199875996808808, + "pass@8": 0.5154268383070109, + "pass@16": 0.597963503716804, + "pass@32": 0.670862546098087, + "pass@64": 0.736826979824222, + "pass@128": 0.7912772585669782 + }, + "15": { + "pass@1": 0.17119761560693642, + "pass@2": 0.2615891402303036, + "pass@4": 0.3643629110125685, + "pass@8": 0.4638810821555635, + "pass@16": 0.5488619660921231, + "pass@32": 0.6166011084092652, + "pass@64": 0.6693182181911753, + "pass@128": 0.7138728323699421 + }, + "16": { + "pass@1": 0.16743951612903227, + "pass@2": 0.2455216535433071, + "pass@4": 0.332099522539521, + "pass@8": 0.4180017910081934, + "pass@16": 0.49562935124361973, + "pass@32": 0.5626821505928203, + "pass@64": 0.6208124764624271, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.13091582369942195, + "pass@2": 0.2026965232579309, + "pass@4": 0.28583829468932, + "pass@8": 0.36784673635324805, + "pass@16": 0.4371569990362648, + "pass@32": 0.49075281833375817, + "pass@64": 0.5351253719194886, + "pass@128": 0.5809248554913294 + }, + "18": { + "pass@1": 0.140625, + "pass@2": 0.21281390607424075, + "pass@4": 0.2967270649204563, + "pass@8": 0.38152916752215504, + "pass@16": 0.45486813332834614, + "pass@32": 0.5073866663449251, + "pass@64": 0.5420570454433494, + "pass@128": 0.5742857142857143 + }, + "19": { + "pass@1": 0.11249530780780781, + "pass@2": 0.17809714241800853, + "pass@4": 0.25848685102425245, + "pass@8": 0.3399542597368296, + "pass@16": 0.40889877405287706, + "pass@32": 0.46292300786984686, + "pass@64": 0.5045700654986855, + "pass@128": 0.5345345345345346 + }, + "20": { + "pass@1": 0.11549831081081081, + "pass@2": 0.18252775413208475, + "pass@4": 0.26408397570799147, + "pass@8": 0.3473522314072748, + "pass@16": 0.4172471386714921, + "pass@32": 0.465631560238557, + "pass@64": 0.4982591181602525, + "pass@128": 0.5255255255255256 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 343922, + "answer_accuracy": 0.4188449922057677, + "avg_response_len": 261.5548372954014, + "resp_tokens_sum": 214767908, + "pass_at_k": { + "pass@1": 0.4188449922057677, + "pass@2": 0.5203707170080039, + "pass@4": 0.6137221266476559, + "pass@8": 0.6911413890808137, + "pass@16": 0.7503143984737648, + "pass@32": 0.7929309365523514, + "pass@64": 0.8235313189554284, + "pass@128": 0.8483242400623539 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.42939992331288346, + "pass@2": 0.5845611323124487, + "pass@4": 0.7219675550525508, + "pass@8": 0.8269844795437781, + "pass@16": 0.8993004726366725, + "pass@32": 0.9453174771206605, + "pass@64": 0.9721708965924318, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 0.9301624240121581, + "pass@2": 0.9487538293085703, + "pass@4": 0.9608639527012007, + "pass@8": 0.9687376166611839, + "pass@16": 0.9746019959801766, + "pass@32": 0.9795867935302768, + "pass@64": 0.9830901743667175, + "pass@128": 0.9848024316109423 + }, + "3": { + "pass@1": 0.860653409090909, + "pass@2": 0.9269099707706995, + "pass@4": 0.9605988600856707, + "pass@8": 0.9739233460421274, + "pass@16": 0.9793009784562008, + "pass@32": 0.9826759601394346, + "pass@64": 0.9857259818108475, + "pass@128": 0.9878787878787879 + }, + "4": { + "pass@1": 0.7838618713017751, + "pass@2": 0.8724043266318782, + "pass@4": 0.9301489839657617, + "pass@8": 0.960023013690949, + "pass@16": 0.97335069655027, + "pass@32": 0.9804791565050158, + "pass@64": 0.9861651795826174, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.6375226449275362, + "pass@2": 0.7545318384115028, + "pass@4": 0.8361816756601076, + "pass@8": 0.8851649784226489, + "pass@16": 0.9156569142518054, + "pass@32": 0.9365426588707715, + "pass@64": 0.9514844580155023, + "pass@128": 0.9623188405797102 + }, + "6": { + "pass@1": 0.6906787249283668, + "pass@2": 0.8065539477246575, + "pass@4": 0.8880656017567995, + "pass@8": 0.9351576339738463, + "pass@16": 0.9588356055013254, + "pass@32": 0.9712646483259476, + "pass@64": 0.9791367802266179, + "pass@128": 0.9856733524355301 + }, + "7": { + "pass@1": 0.6023753612716763, + "pass@2": 0.7327194085385282, + "pass@4": 0.8357873419724274, + "pass@8": 0.9079161103488637, + "pass@16": 0.9501706105430924, + "pass@32": 0.9716194979336098, + "pass@64": 0.982082756700385, + "pass@128": 0.9884393063583815 + }, + "8": { + "pass@1": 0.5243617957746479, + "pass@2": 0.673133733503383, + "pass@4": 0.7976204885480864, + "pass@8": 0.885373322021841, + "pass@16": 0.9396246435828248, + "pass@32": 0.9694345592489272, + "pass@64": 0.9842985934546734, + "pass@128": 0.9915492957746479 + }, + "9": { + "pass@1": 0.48940400449101795, + "pass@2": 0.6394930365646656, + "pass@4": 0.7650968950737445, + "pass@8": 0.8614388295275515, + "pass@16": 0.9280775013732009, + "pass@32": 0.9675055589240936, + "pass@64": 0.9853653966250359, + "pass@128": 0.9910179640718563 + }, + "11": { + "pass@1": 0.3600809487951807, + "pass@2": 0.5060478133004463, + "pass@4": 0.647385364574157, + "pass@8": 0.765477598410561, + "pass@16": 0.8514365450245749, + "pass@32": 0.9050546071868998, + "pass@64": 0.9358720866999578, + "pass@128": 0.9578313253012049 + }, + "12": { + "pass@1": 0.3449990203761755, + "pass@2": 0.47940900143163934, + "pass@4": 0.6083281063221331, + "pass@8": 0.716982685718736, + "pass@16": 0.7997644746548964, + "pass@32": 0.8590672882733515, + "pass@64": 0.9032935756972884, + "pass@128": 0.9435736677115988 + }, + "13": { + "pass@1": 0.23816088935574228, + "pass@2": 0.34847689075630256, + "pass@4": 0.4657425271946047, + "pass@8": 0.5719295256799126, + "pass@16": 0.6597921500111992, + "pass@32": 0.7322101517746895, + "pass@64": 0.7924964201460486, + "pass@128": 0.8487394957983193 + }, + "14": { + "pass@1": 0.22480354532163743, + "pass@2": 0.32782389660174055, + "pass@4": 0.4342883085338015, + "pass@8": 0.5318416120559322, + "pass@16": 0.614830521987863, + "pass@32": 0.6842228584791915, + "pass@64": 0.7460521925082643, + "pass@128": 0.804093567251462 + }, + "15": { + "pass@1": 0.1651278409090909, + "pass@2": 0.24669828203292754, + "pass@4": 0.3395767744940974, + "pass@8": 0.4333264190796036, + "pass@16": 0.5224560313801565, + "pass@32": 0.6018586127709377, + "pass@64": 0.6662757719754229, + "pass@128": 0.7181818181818181 + }, + "16": { + "pass@1": 0.15731837606837606, + "pass@2": 0.2312620577876484, + "pass@4": 0.31380667774434196, + "pass@8": 0.3952425891605544, + "pass@16": 0.4682784764472947, + "pass@32": 0.5316408674982099, + "pass@64": 0.587040470177276, + "pass@128": 0.6381766381766382 + }, + "17": { + "pass@1": 0.1607628105590062, + "pass@2": 0.2452281355455568, + "pass@4": 0.3430799177082678, + "pass@8": 0.4389751464459959, + "pass@16": 0.5188112906635529, + "pass@32": 0.5781936339374035, + "pass@64": 0.6212796797280741, + "pass@128": 0.65527950310559 + }, + "18": { + "pass@1": 0.1396829044117647, + "pass@2": 0.2133181594488189, + "pass@4": 0.3000968705198614, + "pass@8": 0.39007173591004785, + "pass@16": 0.4709636864254259, + "pass@32": 0.5343133483461368, + "pass@64": 0.5793604777810965, + "pass@128": 0.611764705882353 + }, + "19": { + "pass@1": 0.10928199404761904, + "pass@2": 0.17229316062054745, + "pass@4": 0.2538720503129072, + "pass@8": 0.34257004009238523, + "pass@16": 0.420559560046734, + "pass@32": 0.4759493512788768, + "pass@64": 0.5138949420123206, + "pass@128": 0.5505952380952381 + }, + "20": { + "pass@1": 0.10689558383233533, + "pass@2": 0.1719055736951294, + "pass@4": 0.25167281522444424, + "pass@8": 0.3348155934449863, + "pass@16": 0.4058721468049287, + "pass@32": 0.45523155365388795, + "pass@64": 0.48832122929571226, + "pass@128": 0.5149700598802395 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 346713, + "answer_accuracy": 0.4327680639878575, + "avg_response_len": 279.5954313288864, + "resp_tokens_sum": 223998439, + "pass_at_k": { + "pass@1": 0.4327680639878575, + "pass@2": 0.5294413249644969, + "pass@4": 0.619586927078482, + "pass@8": 0.6954290251605538, + "pass@16": 0.7532936024836905, + "pass@32": 0.7945588840019715, + "pass@64": 0.8236257194679109, + "pass@128": 0.8448633967087394 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.47495335820895523, + "pass@2": 0.623364231989658, + "pass@4": 0.7471682490248422, + "pass@8": 0.8398494017450072, + "pass@16": 0.9037033213806163, + "pass@32": 0.9430667350935134, + "pass@64": 0.963281894325653, + "pass@128": 0.9761194029850746 + }, + "2": { + "pass@1": 0.9677455357142857, + "pass@2": 0.9768110236220473, + "pass@4": 0.9826745586266004, + "pass@8": 0.9871195351191154, + "pass@16": 0.9899088225558785, + "pass@32": 0.9911571674097157, + "pass@64": 0.9914240481035442, + "pass@128": 0.9914285714285714 + }, + "3": { + "pass@1": 0.8923104956268222, + "pass@2": 0.9460619877872416, + "pass@4": 0.9716916010498681, + "pass@8": 0.9819850579920034, + "pass@16": 0.9853747770803383, + "pass@32": 0.9866242439336862, + "pass@64": 0.9876146649487308, + "pass@128": 0.9883381924198251 + }, + "4": { + "pass@1": 0.8124086257309941, + "pass@2": 0.8793381193304781, + "pass@4": 0.9236651819290124, + "pass@8": 0.952532626265685, + "pass@16": 0.9713757273390042, + "pass@32": 0.9841841950589343, + "pass@64": 0.9915764391070508, + "pass@128": 0.9941520467836257 + }, + "5": { + "pass@1": 0.67109375, + "pass@2": 0.7686660490968045, + "pass@4": 0.846831118077152, + "pass@8": 0.9036536299326656, + "pass@16": 0.9391996061262696, + "pass@32": 0.9592355455929494, + "pass@64": 0.9716352127256529, + "pass@128": 0.9794117647058823 + }, + "6": { + "pass@1": 0.7108142744479495, + "pass@2": 0.8149187138279639, + "pass@4": 0.890687050159739, + "pass@8": 0.9365963614462343, + "pass@16": 0.9617556334998504, + "pass@32": 0.9761391150509602, + "pass@64": 0.9848625244179728, + "pass@128": 0.9905362776025236 + }, + "7": { + "pass@1": 0.628065664556962, + "pass@2": 0.7528550315209811, + "pass@4": 0.8482620916451892, + "pass@8": 0.9137656555014311, + "pass@16": 0.950796041870444, + "pass@32": 0.9702759776932505, + "pass@64": 0.9808180164586792, + "pass@128": 0.9841772151898734 + }, + "8": { + "pass@1": 0.5572916666666666, + "pass@2": 0.7052213384644487, + "pass@4": 0.8218555895603137, + "pass@8": 0.902596510105195, + "pass@16": 0.9517949896970779, + "pass@32": 0.978889599701662, + "pass@64": 0.9926690796876108, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.42710008038585207, + "pass@2": 0.5741832449299945, + "pass@4": 0.7114348669639761, + "pass@8": 0.8201747611420893, + "pass@16": 0.8954404581504106, + "pass@32": 0.9422447763282753, + "pass@64": 0.9695241077569068, + "pass@128": 0.9839228295819936 + }, + "11": { + "pass@1": 0.3827380952380952, + "pass@2": 0.5340582427196598, + "pass@4": 0.6711852259538985, + "pass@8": 0.7780117481638958, + "pass@16": 0.8522580284381895, + "pass@32": 0.8996968127590125, + "pass@64": 0.9290359324611956, + "pass@128": 0.946031746031746 + }, + "12": { + "pass@1": 0.3052131116207951, + "pass@2": 0.43907156745888415, + "pass@4": 0.5709099071905003, + "pass@8": 0.6826387187898862, + "pass@16": 0.7701434675459031, + "pass@32": 0.8358149758684412, + "pass@64": 0.8831344125460127, + "pass@128": 0.9174311926605505 + }, + "13": { + "pass@1": 0.23647280092592593, + "pass@2": 0.3409821710654225, + "pass@4": 0.45619952598517804, + "pass@8": 0.5672321433778583, + "pass@16": 0.6618549595277416, + "pass@32": 0.7346381674583468, + "pass@64": 0.7900206420715697, + "pass@128": 0.8333333333333334 + }, + "14": { + "pass@1": 0.22860255934718102, + "pass@2": 0.3341221465454799, + "pass@4": 0.44441233912897393, + "pass@8": 0.5441630780134137, + "pass@16": 0.6282250534305504, + "pass@32": 0.695872480912015, + "pass@64": 0.7486191756316015, + "pass@128": 0.7893175074183977 + }, + "15": { + "pass@1": 0.18356963734567902, + "pass@2": 0.28285624453193353, + "pass@4": 0.3948922252773957, + "pass@8": 0.5030045404650186, + "pass@16": 0.5949260503221181, + "pass@32": 0.6677900621586327, + "pass@64": 0.723897627411919, + "pass@128": 0.7654320987654321 + }, + "16": { + "pass@1": 0.1549594395280236, + "pass@2": 0.22994078519499225, + "pass@4": 0.31570141425905845, + "pass@8": 0.40062593291544546, + "pass@16": 0.47466509306307514, + "pass@32": 0.537772987756483, + "pass@64": 0.5930868736296269, + "pass@128": 0.6371681415929203 + }, + "17": { + "pass@1": 0.1441312123493976, + "pass@2": 0.22216375343895262, + "pass@4": 0.311510106229191, + "pass@8": 0.3988157662825498, + "pass@16": 0.47142989450471834, + "pass@32": 0.5239278118241953, + "pass@64": 0.564364886384367, + "pass@128": 0.5993975903614458 + }, + "18": { + "pass@1": 0.13981854838709679, + "pass@2": 0.20924958724917453, + "pass@4": 0.2913785064165365, + "pass@8": 0.37509974076412117, + "pass@16": 0.4452421080127842, + "pass@32": 0.49770270518776283, + "pass@64": 0.539415525844793, + "pass@128": 0.5774193548387097 + }, + "19": { + "pass@1": 0.11935895015105741, + "pass@2": 0.1856285088136642, + "pass@4": 0.263486335450516, + "pass@8": 0.3413125930922642, + "pass@16": 0.4100031762366848, + "pass@32": 0.46414237314575196, + "pass@64": 0.5039823180493795, + "pass@128": 0.5347432024169184 + }, + "20": { + "pass@1": 0.11969782282282282, + "pass@2": 0.1870740080631812, + "pass@4": 0.26937247765200556, + "pass@8": 0.35681385546279937, + "pass@16": 0.43545605263414, + "pass@32": 0.49350455732966975, + "pass@64": 0.5292488342076239, + "pass@128": 0.5555555555555556 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/checkpoint-21546_metrics.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/checkpoint-21546_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..21157ca2b8dc6abb53360e5be81f7de537e09736 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/checkpoint-21546_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21546", + "id": { + "answer_accuracy": 0.9822352430555555, + "avg_response_len": 197.35695486111112, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9822352430555555, + "pass@2": 0.9879565972222177, + "pass@4": 0.9909751535745521, + "pass@8": 0.9928967721890887, + "pass@16": 0.9942459390861659, + "pass@32": 0.9953664906888682, + "pass@64": 0.9963702035746539, + "pass@128": 0.9971111111111111 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9219140625, + "pass@2": 0.9456259842519695, + "pass@4": 0.9600568006186735, + "pass@8": 0.9686591967389316, + "pass@16": 0.9746726986292997, + "pass@32": 0.9802680649056352, + "pass@64": 0.9855262359076933, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977890625, + "pass@2": 0.9989580462598426, + "pass@4": 0.9996748009936257, + "pass@8": 0.9999555402203036, + "pass@16": 0.9999989376601343, + "pass@32": 0.9999999997300467, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996859375, + "pass@2": 0.9970784940944883, + "pass@4": 0.997182251312336, + "pass@8": 0.9973515354285698, + "pass@16": 0.9976530446194225, + "pass@32": 0.9981275815523059, + "pass@64": 0.9986924094488189, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9785, + "pass@2": 0.9830538877952757, + "pass@4": 0.9861205081552304, + "pass@8": 0.9885940186235782, + "pass@16": 0.9905107873626512, + "pass@32": 0.9919421813668161, + "pass@64": 0.9931093862513866, + "pass@128": 0.994 + }, + "6": { + "pass@1": 0.9963515625, + "pass@2": 0.9976577263779527, + "pass@4": 0.9984900437757778, + "pass@8": 0.9989215901193106, + "pass@16": 0.9989984729965745, + "pass@32": 0.9989999998140913, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.991859375, + "pass@2": 0.9952082923228346, + "pass@4": 0.9963165047806523, + "pass@8": 0.9968138304751346, + "pass@16": 0.9970783819357514, + "pass@32": 0.9972487286222231, + "pass@64": 0.9974999999331987, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.982046875, + "pass@2": 0.9887955216535437, + "pass@4": 0.992182715785527, + "pass@8": 0.9948031279074375, + "pass@16": 0.9965073225879685, + "pass@32": 0.9973598741837058, + "pass@64": 0.9978808542313305, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.9748046875, + "pass@2": 0.985231422244095, + "pass@4": 0.9887527567491561, + "pass@8": 0.9909721101885259, + "pass@16": 0.9927938059836875, + "pass@32": 0.9943519860249862, + "pass@64": 0.9956229463994606, + "pass@128": 0.996 + } + }, + "per_op_accuracy": { + "10": 0.9219140625, + "2": 0.9999921875, + "3": 0.9977890625, + "4": 0.996859375, + "5": 0.9785, + "6": 0.9963515625, + "7": 0.991859375, + "8": 0.982046875, + "9": 0.9748046875 + }, + "per_op_avg_response_len": { + "10": 276.033921875, + "2": 114.5776484375, + "3": 143.12903125, + "4": 158.215875, + "5": 171.626671875, + "6": 196.33803125, + "7": 220.828546875, + "8": 240.013625, + "9": 255.4492421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.033921875, + "2": 114.5776484375, + "3": 143.12903125, + "4": 158.215875, + "5": 171.626671875, + "6": 196.33803125, + "7": 220.828546875, + "8": 240.013625, + "9": 255.4492421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371920, + "answer_accuracy": 0.9812985477879095, + "avg_response_len": 187.99772564167512, + "resp_tokens_sum": 71252642, + "pass_at_k": { + "pass@1": 0.9812985477879095, + "pass@2": 0.9879377195536738, + "pass@4": 0.9914442672882716, + "pass@8": 0.9935793573255105, + "pass@16": 0.9948305686016683, + "pass@32": 0.9956933072803694, + "pass@64": 0.9965955023722005, + "pass@128": 0.9976359338061466 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9263228244837758, + "pass@2": 0.9494500238078647, + "pass@4": 0.9622405067176335, + "pass@8": 0.9693425057647705, + "pass@16": 0.9736827240563464, + "pass@32": 0.9773741161261673, + "pass@64": 0.98185566893626, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9965596330275229, + "pass@2": 0.9981052397120085, + "pass@4": 0.999307317089951, + "pass@8": 0.999884609180392, + "pass@16": 0.9999968325205879, + "pass@32": 0.9999999991746348, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9903076171875, + "pass@2": 0.9908726008858268, + "pass@4": 0.9911945356439821, + "pass@8": 0.9917235482142811, + "pass@16": 0.9926657644356954, + "pass@32": 0.9941486923509562, + "pass@64": 0.9959137795275591, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9702628968253968, + "pass@2": 0.9771036432945881, + "pass@4": 0.9829424443968311, + "pass@8": 0.9876832076495944, + "pass@16": 0.990954409287743, + "pass@32": 0.9930190209714992, + "pass@64": 0.9948502012050503, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9975463688764207, + "pass@4": 0.9990146021230881, + "pass@8": 0.9998480009664775, + "pass@16": 0.9999970866060833, + "pass@32": 0.9999999996392369, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9940366124260355, + "pass@2": 0.9978666047383871, + "pass@4": 0.9993812003159368, + "pass@8": 0.9999391517555988, + "pass@16": 0.9999995060547365, + "pass@32": 0.9999999999906712, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9850260416666666, + "pass@2": 0.991938282354129, + "pass@4": 0.9944415109169044, + "pass@8": 0.9959676378945411, + "pass@16": 0.996687988729117, + "pass@32": 0.9967934570195686, + "pass@64": 0.9967948717815591, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.975594190140845, + "pass@2": 0.9893763169568592, + "pass@4": 0.9948080432727598, + "pass@8": 0.9979627772031147, + "pass@16": 0.9995681970104956, + "pass@32": 0.9999716229169351, + "pass@64": 0.9999999660012311, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 383817, + "answer_accuracy": 0.9824935493119266, + "avg_response_len": 195.34848562418085, + "resp_tokens_sum": 76314058, + "pass_at_k": { + "pass@1": 0.9824935493119266, + "pass@2": 0.9876375114807903, + "pass@4": 0.990440648740173, + "pass@8": 0.9922396044415076, + "pass@16": 0.9937546394179761, + "pass@32": 0.9953122006321661, + "pass@64": 0.9967446326239127, + "pass@128": 0.9977064220183486 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9111148389570553, + "pass@2": 0.9357741624800733, + "pass@4": 0.9522138641795541, + "pass@8": 0.9627448771437234, + "pass@16": 0.9721080231114817, + "pass@32": 0.9821285716890078, + "pass@64": 0.9909827842779778, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9971117424242424, + "pass@2": 0.9987439602720115, + "pass@4": 0.9997009899330764, + "pass@8": 0.9999796152070378, + "pass@16": 0.9999999194724303, + "pass@32": 0.9999999999998216, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226677, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863677536231884, + "pass@2": 0.9885042222982998, + "pass@4": 0.9891683455328956, + "pass@8": 0.9898325450069416, + "pass@16": 0.9908752954090346, + "pass@32": 0.9922112515258646, + "pass@64": 0.9934649176276082, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9970003581661891, + "pass@2": 0.9985179816348173, + "pass@4": 0.9994818757433256, + "pass@8": 0.9999207959785322, + "pass@16": 0.9999984128084317, + "pass@32": 0.9999999998125675, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9931809971098265, + "pass@2": 0.9950314477037914, + "pass@4": 0.9958010177376672, + "pass@8": 0.9966315484192274, + "pass@16": 0.9973368465089668, + "pass@32": 0.997828695449064, + "pass@64": 0.9985549131017303, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9819762323943662, + "pass@2": 0.9893354219807033, + "pass@4": 0.9935911558590389, + "pass@8": 0.9965179072847223, + "pass@16": 0.9981742152631411, + "pass@32": 0.999119008113794, + "pass@64": 0.9998322278254919, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9731942365269461, + "pass@2": 0.9808708125088408, + "pass@4": 0.9824776866402178, + "pass@8": 0.9832461410936331, + "pass@16": 0.9842818929540148, + "pass@32": 0.9858385141976975, + "pass@64": 0.9874802206610402, + "pass@128": 0.9880239520958084 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375798, + "answer_accuracy": 0.9828998577167727, + "avg_response_len": 208.6868932038835, + "resp_tokens_sum": 79788512, + "pass_at_k": { + "pass@1": 0.9828998577167727, + "pass@2": 0.9883013399270856, + "pass@4": 0.9910562593824592, + "pass@8": 0.992891596888341, + "pass@16": 0.9941683892341109, + "pass@32": 0.9950979902957031, + "pass@64": 0.9957642888783468, + "pass@128": 0.9959825912286575 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9279617537313433, + "pass@2": 0.9513434304853687, + "pass@4": 0.9654792511010749, + "pass@8": 0.9737231562263299, + "pass@16": 0.97817026758166, + "pass@32": 0.981386045278353, + "pass@64": 0.9839306760109867, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996127915451894, + "pass@2": 0.9999770436858658, + "pass@4": 0.9999999464352669, + "pass@8": 0.9999999999999796, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9781479779411765, + "pass@2": 0.9830361278369619, + "pass@4": 0.9859722322393524, + "pass@8": 0.9881811181957687, + "pass@16": 0.9897299162379272, + "pass@32": 0.9906714940716781, + "pass@64": 0.9911358126184439, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.996352523659306, + "pass@2": 0.9968279608286346, + "pass@4": 0.9968454007326369, + "pass@8": 0.9968454258675057, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9880834651898734, + "pass@2": 0.9925585411890763, + "pass@4": 0.9936028700526358, + "pass@8": 0.9936705108502834, + "pass@16": 0.9936708860669233, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9793308933933934, + "pass@2": 0.9852753836513678, + "pass@4": 0.988564877025507, + "pass@8": 0.9918839873821754, + "pass@16": 0.9945610320902971, + "pass@32": 0.9960152183579682, + "pass@64": 0.9968180581304334, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.975633038585209, + "pass@2": 0.985183210117224, + "pass@4": 0.9885799165578577, + "pass@8": 0.9912897529136566, + "pass@16": 0.9942023916344718, + "pass@32": 0.9970803734644477, + "pass@64": 0.9993715908946497, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.42139921875, + "avg_response_len": 319.1119796875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.42139921875, + "pass@2": 0.48749451279527367, + "pass@4": 0.5435850034870588, + "pass@8": 0.5904528279076726, + "pass@16": 0.6293411435709784, + "pass@32": 0.662450740286997, + "pass@64": 0.6913326959769053, + "pass@128": 0.7164 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.89640625, + "pass@2": 0.925331692913386, + "pass@4": 0.9431302344394448, + "pass@8": 0.9552568652263491, + "pass@16": 0.9638813417090932, + "pass@32": 0.9705131055432534, + "pass@64": 0.9761280577411631, + "pass@128": 0.981 + }, + "12": { + "pass@1": 0.820921875, + "pass@2": 0.874889886811025, + "pass@4": 0.9070598801087356, + "pass@8": 0.9269613699012635, + "pass@16": 0.9399997328861264, + "pass@32": 0.9500446728410242, + "pass@64": 0.9586033946386087, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.6281796875, + "pass@2": 0.7100658218503938, + "pass@4": 0.7667898410198721, + "pass@8": 0.8094543607592102, + "pass@16": 0.8437353266471783, + "pass@32": 0.8719723610710303, + "pass@64": 0.8963146468910601, + "pass@128": 0.918 + }, + "14": { + "pass@1": 0.4969375, + "pass@2": 0.5895782480314973, + "pass@4": 0.6646629949381327, + "pass@8": 0.723076427932374, + "pass@16": 0.7681673812774344, + "pass@32": 0.8030320381123471, + "pass@64": 0.8308881099479088, + "pass@128": 0.855 + }, + "15": { + "pass@1": 0.3896015625, + "pass@2": 0.4741213090551181, + "pass@4": 0.5445334146044238, + "pass@8": 0.6034925539343172, + "pass@16": 0.6542334370777839, + "pass@32": 0.6998373192002232, + "pass@64": 0.7402673283471181, + "pass@128": 0.775 + }, + "16": { + "pass@1": 0.257046875, + "pass@2": 0.3274960629921262, + "pass@4": 0.3930217805586801, + "pass@8": 0.45165701119567986, + "pass@16": 0.502763039125505, + "pass@32": 0.5485593190178388, + "pass@64": 0.5892055941829706, + "pass@128": 0.621 + }, + "17": { + "pass@1": 0.21765625, + "pass@2": 0.28583046259842515, + "pass@4": 0.3484196591676038, + "pass@8": 0.4016824467198299, + "pass@16": 0.4471924710378919, + "pass@32": 0.48900647190022234, + "pass@64": 0.5271945765396434, + "pass@128": 0.56 + }, + "18": { + "pass@1": 0.1981953125, + "pass@2": 0.26509670275590547, + "pass@4": 0.3312123272403448, + "pass@8": 0.390450283347242, + "pass@16": 0.4390495972184917, + "pass@32": 0.4777098247538077, + "pass@64": 0.5115464964164609, + "pass@128": 0.545 + }, + "19": { + "pass@1": 0.1574140625, + "pass@2": 0.21163508858267716, + "pass@4": 0.26538212701537295, + "pass@8": 0.31474668871579253, + "pass@16": 0.35921081485879913, + "pass@32": 0.40040054147524284, + "pass@64": 0.4373684826848517, + "pass@128": 0.468 + }, + "20": { + "pass@1": 0.1516328125, + "pass@2": 0.2108998523622048, + "pass@4": 0.2716377757780275, + "pass@8": 0.327750271344641, + "pass@16": 0.37517829387134566, + "pass@32": 0.41343174895495494, + "pass@64": 0.4458102723792253, + "pass@128": 0.475 + } + }, + "per_op_accuracy": { + "11": 0.89640625, + "12": 0.820921875, + "13": 0.6281796875, + "14": 0.4969375, + "15": 0.3896015625, + "16": 0.257046875, + "17": 0.21765625, + "18": 0.1981953125, + "19": 0.1574140625, + "20": 0.1516328125 + }, + "per_op_avg_response_len": { + "11": 292.72021875, + "12": 301.7166484375, + "13": 314.3682578125, + "14": 325.4328828125, + "15": 329.5058515625, + "16": 319.9487890625, + "17": 327.4527578125, + "18": 331.063671875, + "19": 321.687125, + "20": 327.22359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 292.72021875, + "12": 301.7166484375, + "13": 314.3682578125, + "14": 325.4328828125, + "15": 329.5058515625, + "16": 319.9487890625, + "17": 327.4527578125, + "18": 331.063671875, + "19": 321.687125, + "20": 327.22359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 180966, + "answer_accuracy": 0.42014765973254087, + "avg_response_len": 302.93900910104014, + "resp_tokens_sum": 130481890, + "pass_at_k": { + "pass@1": 0.42014765973254087, + "pass@2": 0.48744841086450624, + "pass@4": 0.5444472647642686, + "pass@8": 0.5915130598082231, + "pass@16": 0.6291897584149792, + "pass@32": 0.6599489512056094, + "pass@64": 0.686270415578776, + "pass@128": 0.7096582466567608 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8872388456090652, + "pass@2": 0.9170830452142494, + "pass@4": 0.9344475201046042, + "pass@8": 0.9461166366837217, + "pass@16": 0.9548923315039325, + "pass@32": 0.9624344015568419, + "pass@64": 0.9692996308589977, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.7989495056497176, + "pass@2": 0.8576045976244497, + "pass@4": 0.894183136271101, + "pass@8": 0.9153254162241761, + "pass@16": 0.9267540179385538, + "pass@32": 0.9355352501223162, + "pass@64": 0.9451007362794269, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.661662421630094, + "pass@2": 0.7403545621158639, + "pass@4": 0.7929696285025494, + "pass@8": 0.8331626588813359, + "pass@16": 0.8660458043380581, + "pass@32": 0.8937908906213489, + "pass@64": 0.9188840917684389, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.48459404205607476, + "pass@2": 0.586037726592587, + "pass@4": 0.6693883112741745, + "pass@8": 0.7337120781768929, + "pass@16": 0.7796279839990717, + "pass@32": 0.8097918673125195, + "pass@64": 0.8326759446166977, + "pass@128": 0.8566978193146417 + }, + "15": { + "pass@1": 0.3742097182080925, + "pass@2": 0.46095314562377676, + "pass@4": 0.5352501313961474, + "pass@8": 0.5976363331573419, + "pass@16": 0.6497626597389893, + "pass@32": 0.6938640362470392, + "pass@64": 0.7308787729367424, + "pass@128": 0.7630057803468208 + }, + "16": { + "pass@1": 0.2728074596774194, + "pass@2": 0.35024368173736364, + "pass@4": 0.41988924942849887, + "pass@8": 0.47919964105067914, + "pass@16": 0.527871351126739, + "pass@32": 0.5696261229797946, + "pass@64": 0.6054567059936721, + "pass@128": 0.632258064516129 + }, + "17": { + "pass@1": 0.21050848988439305, + "pass@2": 0.2739904305675663, + "pass@4": 0.3313912844528826, + "pass@8": 0.38125538448297414, + "pass@16": 0.4236245875549352, + "pass@32": 0.4619779463348103, + "pass@64": 0.49696813014542734, + "pass@128": 0.5260115606936416 + }, + "18": { + "pass@1": 0.1708705357142857, + "pass@2": 0.2367473987626546, + "pass@4": 0.30414279554341417, + "pass@8": 0.3648555156644124, + "pass@16": 0.41361773639831223, + "pass@32": 0.4516806081441688, + "pass@64": 0.48544624589394647, + "pass@128": 0.5228571428571429 + }, + "19": { + "pass@1": 0.15662537537537538, + "pass@2": 0.21153806365420533, + "pass@4": 0.266338078235716, + "pass@8": 0.3166763163826258, + "pass@16": 0.360523558204304, + "pass@32": 0.39946955168091575, + "pass@64": 0.4333781248476876, + "pass@128": 0.4594594594594595 + }, + "20": { + "pass@1": 0.15707113363363365, + "pass@2": 0.21765934241327928, + "pass@4": 0.27828566361637236, + "pass@8": 0.3325391712216513, + "pass@16": 0.3774587634032869, + "pass@32": 0.41180155357726206, + "pass@64": 0.43666908597212345, + "pass@128": 0.45645645645645644 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 180142, + "answer_accuracy": 0.41848331103181685, + "avg_response_len": 314.57921916815343, + "resp_tokens_sum": 135415029, + "pass_at_k": { + "pass@1": 0.41848331103181685, + "pass@2": 0.4836045806495437, + "pass@4": 0.5395686343734254, + "pass@8": 0.5870149662069744, + "pass@16": 0.6273912746139623, + "pass@32": 0.6623282945159469, + "pass@64": 0.6922196744122944, + "pass@128": 0.7175141242937854 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9079207454819277, + "pass@2": 0.93692449423679, + "pass@4": 0.9551024624369245, + "pass@8": 0.9671201826101439, + "pass@16": 0.9746815567314097, + "pass@32": 0.9788712127467571, + "pass@64": 0.9815593127389013, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8269004702194357, + "pass@2": 0.8749591180608697, + "pass@4": 0.9040032188374567, + "pass@8": 0.9231087891699578, + "pass@16": 0.9369859413563386, + "pass@32": 0.9479069421599462, + "pass@64": 0.9560498758422292, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6125262605042017, + "pass@2": 0.6965695234786827, + "pass@4": 0.7562029827679098, + "pass@8": 0.8020303886222311, + "pass@16": 0.838161985699861, + "pass@32": 0.8655366886816216, + "pass@64": 0.8870740770303306, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.4824561403508772, + "pass@2": 0.5683321246028454, + "pass@4": 0.6404780016533019, + "pass@8": 0.6993768785639923, + "pass@16": 0.7467342166602401, + "pass@32": 0.7841871616007284, + "pass@64": 0.8128076930956771, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.39372632575757577, + "pass@2": 0.4847448401336197, + "pass@4": 0.557197479292361, + "pass@8": 0.613319566937348, + "pass@16": 0.6601720515546455, + "pass@32": 0.702804164645992, + "pass@64": 0.7402769189342715, + "pass@128": 0.7727272727272727 + }, + "16": { + "pass@1": 0.24263265669515668, + "pass@2": 0.30694675785719094, + "pass@4": 0.36897373752426255, + "pass@8": 0.4271218724609568, + "pass@16": 0.48113967991388973, + "pass@32": 0.5317545339132025, + "pass@64": 0.5758856604621956, + "pass@128": 0.6096866096866097 + }, + "17": { + "pass@1": 0.22648971273291926, + "pass@2": 0.2974423968063773, + "pass@4": 0.3655080216098764, + "pass@8": 0.4248164502069451, + "pass@16": 0.4753100911077305, + "pass@32": 0.5195282979937258, + "pass@64": 0.5565277964456121, + "pass@128": 0.5838509316770186 + }, + "18": { + "pass@1": 0.21300551470588236, + "pass@2": 0.2771649924733672, + "pass@4": 0.3376023183682923, + "pass@8": 0.39235717588844027, + "pass@16": 0.44123591873712553, + "pass@32": 0.48463721239549934, + "pass@64": 0.5242288971716302, + "pass@128": 0.5617647058823529 + }, + "19": { + "pass@1": 0.14706566220238096, + "pass@2": 0.20007777348143976, + "pass@4": 0.25505773050690084, + "pass@8": 0.306085461392637, + "pass@16": 0.3516417990127139, + "pass@32": 0.39376010269129014, + "pass@64": 0.43223091722228596, + "pass@128": 0.46726190476190477 + }, + "20": { + "pass@1": 0.14556044161676646, + "pass@2": 0.20420244648498276, + "pass@4": 0.26505273339709784, + "pass@8": 0.3224901187290411, + "pass@16": 0.37382821128207, + "pass@32": 0.4189057357484239, + "pass@64": 0.45902656447888046, + "pass@128": 0.4940119760479042 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 178283, + "answer_accuracy": 0.4256833549816626, + "avg_response_len": 340.4034587981357, + "resp_tokens_sum": 142566415, + "pass_at_k": { + "pass@1": 0.4256833549816626, + "pass@2": 0.491540042991839, + "pass@4": 0.5468263054832088, + "pass@8": 0.5928959356564724, + "pass@16": 0.6315009297422057, + "pass@32": 0.6651494888771128, + "pass@64": 0.6956272146387121, + "pass@128": 0.7221882640586798 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8945436507936508, + "pass@2": 0.9223569710036246, + "pass@4": 0.9402420390903516, + "pass@8": 0.9529961646045297, + "pass@16": 0.96257165667739, + "pass@32": 0.9707572036881429, + "pass@64": 0.9780558610114967, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8388761467889908, + "pass@2": 0.8935348641912881, + "pass@4": 0.9239817218948547, + "pass@8": 0.9433164183874273, + "pass@16": 0.9572791903461986, + "pass@32": 0.9678375527483201, + "pass@64": 0.975711998782937, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6124614197530864, + "pass@2": 0.6951154217701951, + "pass@4": 0.752679211973503, + "pass@8": 0.7942920488824928, + "pass@16": 0.8279101424953335, + "pass@32": 0.8575816947638316, + "pass@64": 0.884275234929319, + "pass@128": 0.9074074074074074 + }, + "14": { + "pass@1": 0.5233911350148368, + "pass@2": 0.614512021308909, + "pass@4": 0.6847058470435999, + "pass@8": 0.7369969091059508, + "pass@16": 0.7790020662193811, + "pass@32": 0.8157176244438528, + "pass@64": 0.8475338299324224, + "pass@128": 0.8724035608308606 + }, + "15": { + "pass@1": 0.40183738425925924, + "pass@2": 0.4773633439049284, + "pass@4": 0.5415484597990062, + "pass@8": 0.5997374246066435, + "pass@16": 0.652959196898339, + "pass@32": 0.7031944084122549, + "pass@64": 0.750283610070048, + "pass@128": 0.7901234567901234 + }, + "16": { + "pass@1": 0.2575589970501475, + "pass@2": 0.32797111989872935, + "pass@4": 0.39335200992575037, + "pass@8": 0.4518741747379751, + "pass@16": 0.5021914236768155, + "pass@32": 0.5466943347804367, + "pass@64": 0.5881361312764039, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.21653802710843373, + "pass@2": 0.2869075959349209, + "pass@4": 0.34959244514616383, + "pass@8": 0.4005336949460392, + "pass@16": 0.44448347713010716, + "pass@32": 0.48757226058553715, + "pass@64": 0.5302459429332484, + "pass@128": 0.572289156626506 + }, + "18": { + "pass@1": 0.2128024193548387, + "pass@2": 0.28386779273558566, + "pass@4": 0.35476632437074396, + "pass@8": 0.41725617116976693, + "pass@16": 0.46536508744664384, + "pass@32": 0.49949986996412565, + "pass@64": 0.5271047913394351, + "pass@128": 0.5516129032258065 + }, + "19": { + "pass@1": 0.16871223564954682, + "pass@2": 0.22346459666484297, + "pass@4": 0.27490075381438345, + "pass@8": 0.32159746324003746, + "pass@16": 0.3655734909622184, + "pass@32": 0.40807790411245964, + "pass@64": 0.4465981236371106, + "pass@128": 0.4773413897280967 + }, + "20": { + "pass@1": 0.1522850975975976, + "pass@2": 0.21085788051831356, + "pass@4": 0.2715947052226581, + "pass@8": 0.3282373203373321, + "pass@16": 0.37425196123074994, + "pass@32": 0.4095715191404, + "pass@64": 0.4416954781218096, + "pass@128": 0.4744744744744745 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6870583881578948, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6870583881578948, + "pass@2": 0.7245555001553888, + "pass@4": 0.7555066535284859, + "pass@8": 0.7810841699357192, + "pass@16": 0.8021907835518599, + "pass@32": 0.8201476746878913, + "pass@64": 0.8358241469442598, + "pass@128": 0.8493684210526316 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9219140625, + "pass@2": 0.9456259842519695, + "pass@4": 0.9600568006186735, + "pass@8": 0.9686591967389316, + "pass@16": 0.9746726986292997, + "pass@32": 0.9802680649056352, + "pass@64": 0.9855262359076933, + "pass@128": 0.99 + }, + "2": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9977890625, + "pass@2": 0.9989580462598426, + "pass@4": 0.9996748009936257, + "pass@8": 0.9999555402203036, + "pass@16": 0.9999989376601343, + "pass@32": 0.9999999997300467, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.996859375, + "pass@2": 0.9970784940944883, + "pass@4": 0.997182251312336, + "pass@8": 0.9973515354285698, + "pass@16": 0.9976530446194225, + "pass@32": 0.9981275815523059, + "pass@64": 0.9986924094488189, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.9785, + "pass@2": 0.9830538877952757, + "pass@4": 0.9861205081552304, + "pass@8": 0.9885940186235782, + "pass@16": 0.9905107873626512, + "pass@32": 0.9919421813668161, + "pass@64": 0.9931093862513866, + "pass@128": 0.994 + }, + "6": { + "pass@1": 0.9963515625, + "pass@2": 0.9976577263779527, + "pass@4": 0.9984900437757778, + "pass@8": 0.9989215901193106, + "pass@16": 0.9989984729965745, + "pass@32": 0.9989999998140913, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.991859375, + "pass@2": 0.9952082923228346, + "pass@4": 0.9963165047806523, + "pass@8": 0.9968138304751346, + "pass@16": 0.9970783819357514, + "pass@32": 0.9972487286222231, + "pass@64": 0.9974999999331987, + "pass@128": 0.998 + }, + "8": { + "pass@1": 0.982046875, + "pass@2": 0.9887955216535437, + "pass@4": 0.992182715785527, + "pass@8": 0.9948031279074375, + "pass@16": 0.9965073225879685, + "pass@32": 0.9973598741837058, + "pass@64": 0.9978808542313305, + "pass@128": 0.998 + }, + "9": { + "pass@1": 0.9748046875, + "pass@2": 0.985231422244095, + "pass@4": 0.9887527567491561, + "pass@8": 0.9909721101885259, + "pass@16": 0.9927938059836875, + "pass@32": 0.9943519860249862, + "pass@64": 0.9956229463994606, + "pass@128": 0.996 + }, + "11": { + "pass@1": 0.89640625, + "pass@2": 0.925331692913386, + "pass@4": 0.9431302344394448, + "pass@8": 0.9552568652263491, + "pass@16": 0.9638813417090932, + "pass@32": 0.9705131055432534, + "pass@64": 0.9761280577411631, + "pass@128": 0.981 + }, + "12": { + "pass@1": 0.820921875, + "pass@2": 0.874889886811025, + "pass@4": 0.9070598801087356, + "pass@8": 0.9269613699012635, + "pass@16": 0.9399997328861264, + "pass@32": 0.9500446728410242, + "pass@64": 0.9586033946386087, + "pass@128": 0.966 + }, + "13": { + "pass@1": 0.6281796875, + "pass@2": 0.7100658218503938, + "pass@4": 0.7667898410198721, + "pass@8": 0.8094543607592102, + "pass@16": 0.8437353266471783, + "pass@32": 0.8719723610710303, + "pass@64": 0.8963146468910601, + "pass@128": 0.918 + }, + "14": { + "pass@1": 0.4969375, + "pass@2": 0.5895782480314973, + "pass@4": 0.6646629949381327, + "pass@8": 0.723076427932374, + "pass@16": 0.7681673812774344, + "pass@32": 0.8030320381123471, + "pass@64": 0.8308881099479088, + "pass@128": 0.855 + }, + "15": { + "pass@1": 0.3896015625, + "pass@2": 0.4741213090551181, + "pass@4": 0.5445334146044238, + "pass@8": 0.6034925539343172, + "pass@16": 0.6542334370777839, + "pass@32": 0.6998373192002232, + "pass@64": 0.7402673283471181, + "pass@128": 0.775 + }, + "16": { + "pass@1": 0.257046875, + "pass@2": 0.3274960629921262, + "pass@4": 0.3930217805586801, + "pass@8": 0.45165701119567986, + "pass@16": 0.502763039125505, + "pass@32": 0.5485593190178388, + "pass@64": 0.5892055941829706, + "pass@128": 0.621 + }, + "17": { + "pass@1": 0.21765625, + "pass@2": 0.28583046259842515, + "pass@4": 0.3484196591676038, + "pass@8": 0.4016824467198299, + "pass@16": 0.4471924710378919, + "pass@32": 0.48900647190022234, + "pass@64": 0.5271945765396434, + "pass@128": 0.56 + }, + "18": { + "pass@1": 0.1981953125, + "pass@2": 0.26509670275590547, + "pass@4": 0.3312123272403448, + "pass@8": 0.390450283347242, + "pass@16": 0.4390495972184917, + "pass@32": 0.4777098247538077, + "pass@64": 0.5115464964164609, + "pass@128": 0.545 + }, + "19": { + "pass@1": 0.1574140625, + "pass@2": 0.21163508858267716, + "pass@4": 0.26538212701537295, + "pass@8": 0.31474668871579253, + "pass@16": 0.35921081485879913, + "pass@32": 0.40040054147524284, + "pass@64": 0.4373684826848517, + "pass@128": 0.468 + }, + "20": { + "pass@1": 0.1516328125, + "pass@2": 0.2108998523622048, + "pass@4": 0.2716377757780275, + "pass@8": 0.327750271344641, + "pass@16": 0.37517829387134566, + "pass@32": 0.41343174895495494, + "pass@64": 0.4458102723792253, + "pass@128": 0.475 + } + }, + "per_op_accuracy": { + "10": 0.9219140625, + "2": 0.9999921875, + "3": 0.9977890625, + "4": 0.996859375, + "5": 0.9785, + "6": 0.9963515625, + "7": 0.991859375, + "8": 0.982046875, + "9": 0.9748046875, + "11": 0.89640625, + "12": 0.820921875, + "13": 0.6281796875, + "14": 0.4969375, + "15": 0.3896015625, + "16": 0.257046875, + "17": 0.21765625, + "18": 0.1981953125, + "19": 0.1574140625, + "20": 0.1516328125 + }, + "per_op_avg_response_len": { + "10": 276.033921875, + "2": 114.5776484375, + "3": 143.12903125, + "4": 158.215875, + "5": 171.626671875, + "6": 196.33803125, + "7": 220.828546875, + "8": 240.013625, + "9": 255.4492421875, + "11": 292.72021875, + "12": 301.7166484375, + "13": 314.3682578125, + "14": 325.4328828125, + "15": 329.5058515625, + "16": 319.9487890625, + "17": 327.4527578125, + "18": 331.063671875, + "19": 321.687125, + "20": 327.22359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.033921875, + "2": 114.5776484375, + "3": 143.12903125, + "4": 158.215875, + "5": 171.626671875, + "6": 196.33803125, + "7": 220.828546875, + "8": 240.013625, + "9": 255.4492421875, + "11": 292.72021875, + "12": 301.7166484375, + "13": 314.3682578125, + "14": 325.4328828125, + "15": 329.5058515625, + "16": 319.9487890625, + "17": 327.4527578125, + "18": 331.063671875, + "19": 321.687125, + "20": 327.22359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 552886, + "answer_accuracy": 0.6828045961112867, + "avg_response_len": 249.1386391479608, + "resp_tokens_sum": 201734532, + "pass_at_k": { + "pass@1": 0.6828045961112867, + "pass@2": 0.7217115855449684, + "pass@4": 0.7536723871913236, + "pass@8": 0.7797075439923347, + "pass@16": 0.800334627046473, + "pass@32": 0.817100237695868, + "pass@64": 0.8315237481736786, + "pass@128": 0.8444514701233007 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9263228244837758, + "pass@2": 0.9494500238078647, + "pass@4": 0.9622405067176335, + "pass@8": 0.9693425057647705, + "pass@16": 0.9736827240563464, + "pass@32": 0.9773741161261673, + "pass@64": 0.98185566893626, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9965596330275229, + "pass@2": 0.9981052397120085, + "pass@4": 0.999307317089951, + "pass@8": 0.999884609180392, + "pass@16": 0.9999968325205879, + "pass@32": 0.9999999991746348, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9903076171875, + "pass@2": 0.9908726008858268, + "pass@4": 0.9911945356439821, + "pass@8": 0.9917235482142811, + "pass@16": 0.9926657644356954, + "pass@32": 0.9941486923509562, + "pass@64": 0.9959137795275591, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9702628968253968, + "pass@2": 0.9771036432945881, + "pass@4": 0.9829424443968311, + "pass@8": 0.9876832076495944, + "pass@16": 0.990954409287743, + "pass@32": 0.9930190209714992, + "pass@64": 0.9948502012050503, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9975463688764207, + "pass@4": 0.9990146021230881, + "pass@8": 0.9998480009664775, + "pass@16": 0.9999970866060833, + "pass@32": 0.9999999996392369, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9940366124260355, + "pass@2": 0.9978666047383871, + "pass@4": 0.9993812003159368, + "pass@8": 0.9999391517555988, + "pass@16": 0.9999995060547365, + "pass@32": 0.9999999999906712, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9850260416666666, + "pass@2": 0.991938282354129, + "pass@4": 0.9944415109169044, + "pass@8": 0.9959676378945411, + "pass@16": 0.996687988729117, + "pass@32": 0.9967934570195686, + "pass@64": 0.9967948717815591, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.975594190140845, + "pass@2": 0.9893763169568592, + "pass@4": 0.9948080432727598, + "pass@8": 0.9979627772031147, + "pass@16": 0.9995681970104956, + "pass@32": 0.9999716229169351, + "pass@64": 0.9999999660012311, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8872388456090652, + "pass@2": 0.9170830452142494, + "pass@4": 0.9344475201046042, + "pass@8": 0.9461166366837217, + "pass@16": 0.9548923315039325, + "pass@32": 0.9624344015568419, + "pass@64": 0.9692996308589977, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.7989495056497176, + "pass@2": 0.8576045976244497, + "pass@4": 0.894183136271101, + "pass@8": 0.9153254162241761, + "pass@16": 0.9267540179385538, + "pass@32": 0.9355352501223162, + "pass@64": 0.9451007362794269, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.661662421630094, + "pass@2": 0.7403545621158639, + "pass@4": 0.7929696285025494, + "pass@8": 0.8331626588813359, + "pass@16": 0.8660458043380581, + "pass@32": 0.8937908906213489, + "pass@64": 0.9188840917684389, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.48459404205607476, + "pass@2": 0.586037726592587, + "pass@4": 0.6693883112741745, + "pass@8": 0.7337120781768929, + "pass@16": 0.7796279839990717, + "pass@32": 0.8097918673125195, + "pass@64": 0.8326759446166977, + "pass@128": 0.8566978193146417 + }, + "15": { + "pass@1": 0.3742097182080925, + "pass@2": 0.46095314562377676, + "pass@4": 0.5352501313961474, + "pass@8": 0.5976363331573419, + "pass@16": 0.6497626597389893, + "pass@32": 0.6938640362470392, + "pass@64": 0.7308787729367424, + "pass@128": 0.7630057803468208 + }, + "16": { + "pass@1": 0.2728074596774194, + "pass@2": 0.35024368173736364, + "pass@4": 0.41988924942849887, + "pass@8": 0.47919964105067914, + "pass@16": 0.527871351126739, + "pass@32": 0.5696261229797946, + "pass@64": 0.6054567059936721, + "pass@128": 0.632258064516129 + }, + "17": { + "pass@1": 0.21050848988439305, + "pass@2": 0.2739904305675663, + "pass@4": 0.3313912844528826, + "pass@8": 0.38125538448297414, + "pass@16": 0.4236245875549352, + "pass@32": 0.4619779463348103, + "pass@64": 0.49696813014542734, + "pass@128": 0.5260115606936416 + }, + "18": { + "pass@1": 0.1708705357142857, + "pass@2": 0.2367473987626546, + "pass@4": 0.30414279554341417, + "pass@8": 0.3648555156644124, + "pass@16": 0.41361773639831223, + "pass@32": 0.4516806081441688, + "pass@64": 0.48544624589394647, + "pass@128": 0.5228571428571429 + }, + "19": { + "pass@1": 0.15662537537537538, + "pass@2": 0.21153806365420533, + "pass@4": 0.266338078235716, + "pass@8": 0.3166763163826258, + "pass@16": 0.360523558204304, + "pass@32": 0.39946955168091575, + "pass@64": 0.4333781248476876, + "pass@128": 0.4594594594594595 + }, + "20": { + "pass@1": 0.15707113363363365, + "pass@2": 0.21765934241327928, + "pass@4": 0.27828566361637236, + "pass@8": 0.3325391712216513, + "pass@16": 0.3774587634032869, + "pass@32": 0.41180155357726206, + "pass@64": 0.43666908597212345, + "pass@128": 0.45645645645645644 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 563959, + "answer_accuracy": 0.6868167868277475, + "avg_response_len": 257.8540128117693, + "resp_tokens_sum": 211729087, + "pass_at_k": { + "pass@1": 0.6868167868277475, + "pass@2": 0.7234032563934145, + "pass@4": 0.7540754758149335, + "pass@8": 0.7798046148261166, + "pass@16": 0.8016922862089578, + "pass@32": 0.8207486969269729, + "pass@64": 0.8371004495427515, + "pass@128": 0.8508183943881528 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9111148389570553, + "pass@2": 0.9357741624800733, + "pass@4": 0.9522138641795541, + "pass@8": 0.9627448771437234, + "pass@16": 0.9721080231114817, + "pass@32": 0.9821285716890078, + "pass@64": 0.9909827842779778, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9971117424242424, + "pass@2": 0.9987439602720115, + "pass@4": 0.9997009899330764, + "pass@8": 0.9999796152070378, + "pass@16": 0.9999999194724303, + "pass@32": 0.9999999999998216, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226677, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863677536231884, + "pass@2": 0.9885042222982998, + "pass@4": 0.9891683455328956, + "pass@8": 0.9898325450069416, + "pass@16": 0.9908752954090346, + "pass@32": 0.9922112515258646, + "pass@64": 0.9934649176276082, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9970003581661891, + "pass@2": 0.9985179816348173, + "pass@4": 0.9994818757433256, + "pass@8": 0.9999207959785322, + "pass@16": 0.9999984128084317, + "pass@32": 0.9999999998125675, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9931809971098265, + "pass@2": 0.9950314477037914, + "pass@4": 0.9958010177376672, + "pass@8": 0.9966315484192274, + "pass@16": 0.9973368465089668, + "pass@32": 0.997828695449064, + "pass@64": 0.9985549131017303, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9819762323943662, + "pass@2": 0.9893354219807033, + "pass@4": 0.9935911558590389, + "pass@8": 0.9965179072847223, + "pass@16": 0.9981742152631411, + "pass@32": 0.999119008113794, + "pass@64": 0.9998322278254919, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9731942365269461, + "pass@2": 0.9808708125088408, + "pass@4": 0.9824776866402178, + "pass@8": 0.9832461410936331, + "pass@16": 0.9842818929540148, + "pass@32": 0.9858385141976975, + "pass@64": 0.9874802206610402, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.9079207454819277, + "pass@2": 0.93692449423679, + "pass@4": 0.9551024624369245, + "pass@8": 0.9671201826101439, + "pass@16": 0.9746815567314097, + "pass@32": 0.9788712127467571, + "pass@64": 0.9815593127389013, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8269004702194357, + "pass@2": 0.8749591180608697, + "pass@4": 0.9040032188374567, + "pass@8": 0.9231087891699578, + "pass@16": 0.9369859413563386, + "pass@32": 0.9479069421599462, + "pass@64": 0.9560498758422292, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6125262605042017, + "pass@2": 0.6965695234786827, + "pass@4": 0.7562029827679098, + "pass@8": 0.8020303886222311, + "pass@16": 0.838161985699861, + "pass@32": 0.8655366886816216, + "pass@64": 0.8870740770303306, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.4824561403508772, + "pass@2": 0.5683321246028454, + "pass@4": 0.6404780016533019, + "pass@8": 0.6993768785639923, + "pass@16": 0.7467342166602401, + "pass@32": 0.7841871616007284, + "pass@64": 0.8128076930956771, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.39372632575757577, + "pass@2": 0.4847448401336197, + "pass@4": 0.557197479292361, + "pass@8": 0.613319566937348, + "pass@16": 0.6601720515546455, + "pass@32": 0.702804164645992, + "pass@64": 0.7402769189342715, + "pass@128": 0.7727272727272727 + }, + "16": { + "pass@1": 0.24263265669515668, + "pass@2": 0.30694675785719094, + "pass@4": 0.36897373752426255, + "pass@8": 0.4271218724609568, + "pass@16": 0.48113967991388973, + "pass@32": 0.5317545339132025, + "pass@64": 0.5758856604621956, + "pass@128": 0.6096866096866097 + }, + "17": { + "pass@1": 0.22648971273291926, + "pass@2": 0.2974423968063773, + "pass@4": 0.3655080216098764, + "pass@8": 0.4248164502069451, + "pass@16": 0.4753100911077305, + "pass@32": 0.5195282979937258, + "pass@64": 0.5565277964456121, + "pass@128": 0.5838509316770186 + }, + "18": { + "pass@1": 0.21300551470588236, + "pass@2": 0.2771649924733672, + "pass@4": 0.3376023183682923, + "pass@8": 0.39235717588844027, + "pass@16": 0.44123591873712553, + "pass@32": 0.48463721239549934, + "pass@64": 0.5242288971716302, + "pass@128": 0.5617647058823529 + }, + "19": { + "pass@1": 0.14706566220238096, + "pass@2": 0.20007777348143976, + "pass@4": 0.25505773050690084, + "pass@8": 0.306085461392637, + "pass@16": 0.3516417990127139, + "pass@32": 0.39376010269129014, + "pass@64": 0.43223091722228596, + "pass@128": 0.46726190476190477 + }, + "20": { + "pass@1": 0.14556044161676646, + "pass@2": 0.20420244648498276, + "pass@4": 0.26505273339709784, + "pass@8": 0.3224901187290411, + "pass@16": 0.37382821128207, + "pass@32": 0.4189057357484239, + "pass@64": 0.45902656447888046, + "pass@128": 0.4940119760479042 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 554081, + "answer_accuracy": 0.6916053383128296, + "avg_response_len": 277.5439953966289, + "resp_tokens_sum": 222354927, + "pass_at_k": { + "pass@1": 0.6916053383128296, + "pass@2": 0.7286108201040862, + "pass@4": 0.7588274034696325, + "pass@8": 0.7837869789700335, + "pass@16": 0.8045777313882145, + "pass@32": 0.8226117310463649, + "pass@64": 0.8388624663967905, + "pass@128": 0.8528518932736859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9279617537313433, + "pass@2": 0.9513434304853687, + "pass@4": 0.9654792511010749, + "pass@8": 0.9737231562263299, + "pass@16": 0.97817026758166, + "pass@32": 0.981386045278353, + "pass@64": 0.9839306760109867, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996127915451894, + "pass@2": 0.9999770436858658, + "pass@4": 0.9999999464352669, + "pass@8": 0.9999999999999796, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9781479779411765, + "pass@2": 0.9830361278369619, + "pass@4": 0.9859722322393524, + "pass@8": 0.9881811181957687, + "pass@16": 0.9897299162379272, + "pass@32": 0.9906714940716781, + "pass@64": 0.9911358126184439, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.996352523659306, + "pass@2": 0.9968279608286346, + "pass@4": 0.9968454007326369, + "pass@8": 0.9968454258675057, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9880834651898734, + "pass@2": 0.9925585411890763, + "pass@4": 0.9936028700526358, + "pass@8": 0.9936705108502834, + "pass@16": 0.9936708860669233, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9793308933933934, + "pass@2": 0.9852753836513678, + "pass@4": 0.988564877025507, + "pass@8": 0.9918839873821754, + "pass@16": 0.9945610320902971, + "pass@32": 0.9960152183579682, + "pass@64": 0.9968180581304334, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.975633038585209, + "pass@2": 0.985183210117224, + "pass@4": 0.9885799165578577, + "pass@8": 0.9912897529136566, + "pass@16": 0.9942023916344718, + "pass@32": 0.9970803734644477, + "pass@64": 0.9993715908946497, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8945436507936508, + "pass@2": 0.9223569710036246, + "pass@4": 0.9402420390903516, + "pass@8": 0.9529961646045297, + "pass@16": 0.96257165667739, + "pass@32": 0.9707572036881429, + "pass@64": 0.9780558610114967, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8388761467889908, + "pass@2": 0.8935348641912881, + "pass@4": 0.9239817218948547, + "pass@8": 0.9433164183874273, + "pass@16": 0.9572791903461986, + "pass@32": 0.9678375527483201, + "pass@64": 0.975711998782937, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6124614197530864, + "pass@2": 0.6951154217701951, + "pass@4": 0.752679211973503, + "pass@8": 0.7942920488824928, + "pass@16": 0.8279101424953335, + "pass@32": 0.8575816947638316, + "pass@64": 0.884275234929319, + "pass@128": 0.9074074074074074 + }, + "14": { + "pass@1": 0.5233911350148368, + "pass@2": 0.614512021308909, + "pass@4": 0.6847058470435999, + "pass@8": 0.7369969091059508, + "pass@16": 0.7790020662193811, + "pass@32": 0.8157176244438528, + "pass@64": 0.8475338299324224, + "pass@128": 0.8724035608308606 + }, + "15": { + "pass@1": 0.40183738425925924, + "pass@2": 0.4773633439049284, + "pass@4": 0.5415484597990062, + "pass@8": 0.5997374246066435, + "pass@16": 0.652959196898339, + "pass@32": 0.7031944084122549, + "pass@64": 0.750283610070048, + "pass@128": 0.7901234567901234 + }, + "16": { + "pass@1": 0.2575589970501475, + "pass@2": 0.32797111989872935, + "pass@4": 0.39335200992575037, + "pass@8": 0.4518741747379751, + "pass@16": 0.5021914236768155, + "pass@32": 0.5466943347804367, + "pass@64": 0.5881361312764039, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.21653802710843373, + "pass@2": 0.2869075959349209, + "pass@4": 0.34959244514616383, + "pass@8": 0.4005336949460392, + "pass@16": 0.44448347713010716, + "pass@32": 0.48757226058553715, + "pass@64": 0.5302459429332484, + "pass@128": 0.572289156626506 + }, + "18": { + "pass@1": 0.2128024193548387, + "pass@2": 0.28386779273558566, + "pass@4": 0.35476632437074396, + "pass@8": 0.41725617116976693, + "pass@16": 0.46536508744664384, + "pass@32": 0.49949986996412565, + "pass@64": 0.5271047913394351, + "pass@128": 0.5516129032258065 + }, + "19": { + "pass@1": 0.16871223564954682, + "pass@2": 0.22346459666484297, + "pass@4": 0.27490075381438345, + "pass@8": 0.32159746324003746, + "pass@16": 0.3655734909622184, + "pass@32": 0.40807790411245964, + "pass@64": 0.4465981236371106, + "pass@128": 0.4773413897280967 + }, + "20": { + "pass@1": 0.1522850975975976, + "pass@2": 0.21085788051831356, + "pass@4": 0.2715947052226581, + "pass@8": 0.3282373203373321, + "pass@16": 0.37425196123074994, + "pass@32": 0.4095715191404, + "pass@64": 0.4416954781218096, + "pass@128": 0.4744744744744745 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..65be86e861e7057d008bf91feebfba796be37e61 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4", "results_dir": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/checkpoint-21546_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1012497, "value_mismatch": 23626, "dependency_mismatch": 136143, "answer_mismatch": 20463, "extra_nodes": 19057, "missing_nodes": 16085, "perfect_with_extra": 1376, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 333223, "value_mismatch": 8063, "answer_mismatch": 7086, "dependency_mismatch": 44696, "extra_nodes": 6823, "perfect_with_extra": 581, "missing_nodes": 5332}, "teachers_in_school": {"total": 390656, "perfect": 343813, "dependency_mismatch": 45743, "value_mismatch": 7649, "answer_mismatch": 6839, "extra_nodes": 6081, "missing_nodes": 5181, "perfect_with_extra": 305}, "movie_festival_awards": {"total": 382336, "perfect": 335461, "value_mismatch": 7914, "dependency_mismatch": 45704, "answer_mismatch": 6538, "missing_nodes": 5572, "extra_nodes": 6153, "perfect_with_extra": 490}}, "results_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/checkpoint-21546_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/checkpoint-21546_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 222324, "value_mismatch": 815309, "dependency_mismatch": 963475, "answer_mismatch": 740598, "extra_nodes": 70467, "missing_nodes": 791654, "perfect_with_extra": 143, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 275351, "dependency_mismatch": 324672, "answer_mismatch": 249751, "extra_nodes": 24754, "missing_nodes": 273860, "perfect": 74587, "perfect_with_extra": 2}, "movie_festival_awards": {"total": 418816, "perfect": 74098, "value_mismatch": 266366, "dependency_mismatch": 317108, "answer_mismatch": 240533, "extra_nodes": 22761, "missing_nodes": 251464, "perfect_with_extra": 4}, "teachers_in_school": {"total": 430464, "perfect": 73639, "value_mismatch": 273592, "dependency_mismatch": 321695, "answer_mismatch": 250314, "extra_nodes": 22952, "perfect_with_extra": 137, "missing_nodes": 266330}}, "results_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/checkpoint-21546_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1234821, "value_mismatch": 838935, "dependency_mismatch": 1099618, "answer_mismatch": 761061, "extra_nodes": 89524, "missing_nodes": 807739, "perfect_with_extra": 1519, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/block4_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 407810, "value_mismatch": 283414, "answer_mismatch": 256837, "dependency_mismatch": 369368, "extra_nodes": 31577, "perfect_with_extra": 583, "missing_nodes": 279192}, "teachers_in_school": {"total": 821120, "perfect": 417452, "dependency_mismatch": 367438, "value_mismatch": 281241, "answer_mismatch": 257153, "extra_nodes": 29033, "missing_nodes": 271511, "perfect_with_extra": 442}, "movie_festival_awards": {"total": 801152, "perfect": 409559, "value_mismatch": 274280, "dependency_mismatch": 362812, "answer_mismatch": 247071, "missing_nodes": 257036, "extra_nodes": 28914, "perfect_with_extra": 494}}}} diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/results.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..18fa07e891a15d342de583c60cae3e4afef1c5d4 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21546_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875, + "pass@2": 0.9875, + "pass@4": 0.990625, + "pass@8": 0.990625, + "pass@16": 0.990625, + "pass@32": 0.990625, + "pass@64": 0.99375, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9333333333333333, + "pass@2": 0.9523809523809523, + "pass@4": 0.9777777777777777, + "pass@8": 0.9873015873015873, + "pass@16": 0.9873015873015873, + "pass@32": 0.9904761904761905, + "pass@64": 0.9904761904761905, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.9640718562874252, + "pass@2": 0.9670658682634731, + "pass@4": 0.9760479041916168, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8964497041420119, + "pass@2": 0.9378698224852071, + "pass@4": 0.9674556213017751, + "pass@8": 0.9822485207100592, + "pass@16": 0.9822485207100592, + "pass@32": 0.9970414201183432, + "pass@64": 0.9970414201183432, + "pass@128": 0.9970414201183432 + }, + "8": { + "pass@1": 0.8108974358974359, + "pass@2": 0.8846153846153846, + "pass@4": 0.9391025641025641, + "pass@8": 0.9839743589743589, + "pass@16": 0.9871794871794872, + "pass@32": 0.9935897435897436, + "pass@64": 0.9967948717948718, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.7408450704225352, + "pass@2": 0.819718309859155, + "pass@4": 0.8929577464788733, + "pass@8": 0.9408450704225352, + "pass@16": 0.9802816901408451, + "pass@32": 0.9943661971830986, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6430678466076696, + "pass@2": 0.7050147492625368, + "pass@4": 0.7728613569321534, + "pass@8": 0.8348082595870207, + "pass@16": 0.8849557522123894, + "pass@32": 0.9203539823008849, + "pass@64": 0.9292035398230089, + "pass@128": 0.9380530973451328 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9333333333333333, + "6": 0.9640718562874252, + "7": 0.8964497041420119, + "8": 0.8108974358974359, + "9": 0.7408450704225352, + "10": 0.6430678466076696 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9875, + "5": 0.9523809523809523, + "6": 0.9670658682634731, + "7": 0.9378698224852071, + "8": 0.8846153846153846, + "9": 0.819718309859155, + "10": 0.7050147492625368 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9777777777777777, + "6": 0.9760479041916168, + "7": 0.9674556213017751, + "8": 0.9391025641025641, + "9": 0.8929577464788733, + "10": 0.7728613569321534 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9873015873015873, + "6": 0.9970059880239521, + "7": 0.9822485207100592, + "8": 0.9839743589743589, + "9": 0.9408450704225352, + "10": 0.8348082595870207 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9873015873015873, + "6": 1.0, + "7": 0.9822485207100592, + "8": 0.9871794871794872, + "9": 0.9802816901408451, + "10": 0.8849557522123894 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9904761904761905, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9203539823008849 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9904761904761905, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9292035398230089 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.996875, + "5": 0.9936507936507937, + "6": 1.0, + "7": 0.9970414201183432, + "8": 0.9967948717948718, + "9": 1.0, + "10": 0.9380530973451328 + } + }, + "total_pass_at_k": { + "pass@1": 0.8838230327592029, + "pass@2": 0.9152313407632556, + "pass@4": 0.944951030057413, + "pass@8": 0.9675785207700102, + "pass@16": 0.9787234042553191, + "pass@32": 0.9871664978047957, + "pass@64": 0.9895305639986491, + "pass@128": 0.9912191827085444 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9941520467836257, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9441176470588235, + "pass@2": 0.9676470588235294, + "pass@4": 0.9823529411764705, + "pass@8": 0.9852941176470589, + "pass@16": 0.9882352941176471, + "pass@32": 0.9882352941176471, + "pass@64": 0.9882352941176471, + "pass@128": 0.9882352941176471 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9747634069400631, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8670886075949367, + "pass@2": 0.9145569620253164, + "pass@4": 0.9493670886075949, + "pass@8": 0.9715189873417721, + "pass@16": 0.990506329113924, + "pass@32": 0.990506329113924, + "pass@64": 0.990506329113924, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.8018018018018018, + "pass@2": 0.8588588588588588, + "pass@4": 0.9129129129129129, + "pass@8": 0.9519519519519519, + "pass@16": 0.9819819819819819, + "pass@32": 0.993993993993994, + "pass@64": 0.996996996996997, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.7459807073954984, + "pass@2": 0.8102893890675241, + "pass@4": 0.9003215434083601, + "pass@8": 0.932475884244373, + "pass@16": 0.9807073954983923, + "pass@32": 0.9935691318327974, + "pass@64": 0.9967845659163987, + "pass@128": 0.9967845659163987 + }, + "10": { + "pass@1": 0.591044776119403, + "pass@2": 0.6597014925373135, + "pass@4": 0.7492537313432835, + "pass@8": 0.8238805970149253, + "pass@16": 0.8686567164179104, + "pass@32": 0.8955223880597015, + "pass@64": 0.9194029850746268, + "pass@128": 0.9402985074626866 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9441176470588235, + "6": 0.9621451104100947, + "7": 0.8670886075949367, + "8": 0.8018018018018018, + "9": 0.7459807073954984, + "10": 0.591044776119403 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9676470588235294, + "6": 0.9747634069400631, + "7": 0.9145569620253164, + "8": 0.8588588588588588, + "9": 0.8102893890675241, + "10": 0.6597014925373135 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9823529411764705, + "6": 0.9842271293375394, + "7": 0.9493670886075949, + "8": 0.9129129129129129, + "9": 0.9003215434083601, + "10": 0.7492537313432835 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9936908517350158, + "7": 0.9715189873417721, + "8": 0.9519519519519519, + "9": 0.932475884244373, + "10": 0.8238805970149253 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.9819819819819819, + "9": 0.9807073954983923, + "10": 0.8686567164179104 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.993993993993994, + "9": 0.9935691318327974, + "10": 0.8955223880597015 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9194029850746268 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9882352941176471, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.996996996996997, + "9": 0.9967845659163987, + "10": 0.9402985074626866 + } + }, + "total_pass_at_k": { + "pass@1": 0.8801473049882825, + "pass@2": 0.9102778707733512, + "pass@4": 0.942417140944091, + "pass@8": 0.9625041848008035, + "pass@16": 0.9785738198861734, + "pass@32": 0.9842651489789086, + "pass@64": 0.987612989621694, + "pass@128": 0.9899564780716438 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.990909090909091, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9970414201183432, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9536231884057971, + "pass@2": 0.9710144927536232, + "pass@4": 0.9826086956521739, + "pass@8": 0.9884057971014493, + "pass@16": 0.9884057971014493, + "pass@32": 0.9884057971014493, + "pass@64": 0.991304347826087, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9742120343839542, + "pass@2": 0.9799426934097422, + "pass@4": 0.9914040114613181, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8641618497109826, + "pass@2": 0.9075144508670521, + "pass@4": 0.953757225433526, + "pass@8": 0.9797687861271677, + "pass@16": 0.9884393063583815, + "pass@32": 0.9884393063583815, + "pass@64": 0.9971098265895953, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.7971830985915493, + "pass@2": 0.8591549295774648, + "pass@4": 0.9183098591549296, + "pass@8": 0.9633802816901409, + "pass@16": 0.9887323943661972, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.7395209580838323, + "pass@2": 0.8053892215568862, + "pass@4": 0.8712574850299402, + "pass@8": 0.9221556886227545, + "pass@16": 0.9670658682634731, + "pass@32": 0.9850299401197605, + "pass@64": 0.9880239520958084, + "pass@128": 0.9880239520958084 + }, + "10": { + "pass@1": 0.6319018404907976, + "pass@2": 0.7024539877300614, + "pass@4": 0.7760736196319018, + "pass@8": 0.8312883435582822, + "pass@16": 0.8742331288343558, + "pass@32": 0.9171779141104295, + "pass@64": 0.9294478527607362, + "pass@128": 0.9447852760736196 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.990909090909091, + "4": 0.9792899408284024, + "5": 0.9536231884057971, + "6": 0.9742120343839542, + "7": 0.8641618497109826, + "8": 0.7971830985915493, + "9": 0.7395209580838323, + "10": 0.6319018404907976 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9710144927536232, + "6": 0.9799426934097422, + "7": 0.9075144508670521, + "8": 0.8591549295774648, + "9": 0.8053892215568862, + "10": 0.7024539877300614 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9826086956521739, + "6": 0.9914040114613181, + "7": 0.953757225433526, + "8": 0.9183098591549296, + "9": 0.8712574850299402, + "10": 0.7760736196319018 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9797687861271677, + "8": 0.9633802816901409, + "9": 0.9221556886227545, + "10": 0.8312883435582822 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9884393063583815, + "8": 0.9887323943661972, + "9": 0.9670658682634731, + "10": 0.8742331288343558 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9884393063583815, + "8": 0.9943661971830986, + "9": 0.9850299401197605, + "10": 0.9171779141104295 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.991304347826087, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.9294478527607362 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.991304347826087, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9880239520958084, + "10": 0.9447852760736196 + } + }, + "total_pass_at_k": { + "pass@1": 0.8817169069462647, + "pass@2": 0.9141546526867628, + "pass@4": 0.9442988204456094, + "pass@8": 0.9655963302752294, + "pass@16": 0.9790301441677588, + "pass@32": 0.9862385321100917, + "pass@64": 0.9895150720838795, + "pass@128": 0.991480996068152 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.997, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.986, + "pass@2": 0.993, + "pass@4": 0.996, + "pass@8": 0.997, + "pass@16": 0.997, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.999 + }, + "5": { + "pass@1": 0.944, + "pass@2": 0.964, + "pass@4": 0.981, + "pass@8": 0.987, + "pass@16": 0.988, + "pass@32": 0.989, + "pass@64": 0.99, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.967, + "pass@2": 0.974, + "pass@4": 0.984, + "pass@8": 0.997, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.876, + "pass@2": 0.92, + "pass@4": 0.957, + "pass@8": 0.978, + "pass@16": 0.987, + "pass@32": 0.992, + "pass@64": 0.995, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.803, + "pass@2": 0.867, + "pass@4": 0.923, + "pass@8": 0.966, + "pass@16": 0.986, + "pass@32": 0.994, + "pass@64": 0.997, + "pass@128": 0.997 + }, + "9": { + "pass@1": 0.742, + "pass@2": 0.812, + "pass@4": 0.888, + "pass@8": 0.932, + "pass@16": 0.976, + "pass@32": 0.991, + "pass@64": 0.995, + "pass@128": 0.995 + }, + "10": { + "pass@1": 0.622, + "pass@2": 0.689, + "pass@4": 0.766, + "pass@8": 0.83, + "pass@16": 0.876, + "pass@32": 0.911, + "pass@64": 0.926, + "pass@128": 0.941 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.997, + "4": 0.986, + "5": 0.944, + "6": 0.967, + "7": 0.876, + "8": 0.803, + "9": 0.742, + "10": 0.622 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.993, + "5": 0.964, + "6": 0.974, + "7": 0.92, + "8": 0.867, + "9": 0.812, + "10": 0.689 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.981, + "6": 0.984, + "7": 0.957, + "8": 0.923, + "9": 0.888, + "10": 0.766 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.987, + "6": 0.997, + "7": 0.978, + "8": 0.966, + "9": 0.932, + "10": 0.83 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.988, + "6": 0.999, + "7": 0.987, + "8": 0.986, + "9": 0.976, + "10": 0.876 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.997, + "5": 0.989, + "6": 0.999, + "7": 0.992, + "8": 0.994, + "9": 0.991, + "10": 0.911 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.998, + "5": 0.99, + "6": 0.999, + "7": 0.995, + "8": 0.997, + "9": 0.995, + "10": 0.926 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.991, + "6": 0.999, + "7": 0.996, + "8": 0.997, + "9": 0.995, + "10": 0.941 + } + }, + "total_pass_at_k": { + "pass@1": 0.8818888888888889, + "pass@2": 0.9132222222222223, + "pass@4": 0.9438888888888889, + "pass@8": 0.9652222222222222, + "pass@16": 0.9787777777777777, + "pass@32": 0.9858888888888889, + "pass@64": 0.9888888888888889, + "pass@128": 0.9908888888888889 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21546_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.509915014164306, + "pass@2": 0.6118980169971672, + "pass@4": 0.7195467422096318, + "pass@8": 0.7960339943342776, + "pass@16": 0.8413597733711048, + "pass@32": 0.8781869688385269, + "pass@64": 0.9036827195467422, + "pass@128": 0.9178470254957507 + }, + "12": { + "pass@1": 0.5169491525423728, + "pass@2": 0.576271186440678, + "pass@4": 0.6694915254237288, + "pass@8": 0.711864406779661, + "pass@16": 0.7796610169491526, + "pass@32": 0.8107344632768362, + "pass@64": 0.8220338983050848, + "pass@128": 0.8502824858757062 + }, + "13": { + "pass@1": 0.3322884012539185, + "pass@2": 0.41379310344827586, + "pass@4": 0.4702194357366771, + "pass@8": 0.542319749216301, + "pass@16": 0.6018808777429467, + "pass@32": 0.6363636363636364, + "pass@64": 0.6739811912225705, + "pass@128": 0.7115987460815048 + }, + "14": { + "pass@1": 0.1806853582554517, + "pass@2": 0.2523364485981308, + "pass@4": 0.308411214953271, + "pass@8": 0.34890965732087226, + "pass@16": 0.3862928348909657, + "pass@32": 0.4423676012461059, + "pass@64": 0.48909657320872274, + "pass@128": 0.5295950155763239 + }, + "15": { + "pass@1": 0.10115606936416185, + "pass@2": 0.14450867052023122, + "pass@4": 0.18208092485549132, + "pass@8": 0.21098265895953758, + "pass@16": 0.24277456647398843, + "pass@32": 0.2861271676300578, + "pass@64": 0.32947976878612717, + "pass@128": 0.35260115606936415 + }, + "16": { + "pass@1": 0.02258064516129032, + "pass@2": 0.03225806451612903, + "pass@4": 0.04516129032258064, + "pass@8": 0.054838709677419356, + "pass@16": 0.06451612903225806, + "pass@32": 0.07419354838709677, + "pass@64": 0.08387096774193549, + "pass@128": 0.0967741935483871 + }, + "17": { + "pass@1": 0.011560693641618497, + "pass@2": 0.02023121387283237, + "pass@4": 0.023121387283236993, + "pass@8": 0.02601156069364162, + "pass@16": 0.03468208092485549, + "pass@32": 0.04046242774566474, + "pass@64": 0.046242774566473986, + "pass@128": 0.06069364161849711 + }, + "18": { + "pass@1": 0.002857142857142857, + "pass@2": 0.002857142857142857, + "pass@4": 0.005714285714285714, + "pass@8": 0.005714285714285714, + "pass@16": 0.011428571428571429, + "pass@32": 0.014285714285714285, + "pass@64": 0.014285714285714285, + "pass@128": 0.022857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.003003003003003003, + "pass@128": 0.003003003003003003 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.003003003003003003, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.006006006006006006 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.509915014164306, + "12": 0.5169491525423728, + "13": 0.3322884012539185, + "14": 0.1806853582554517, + "15": 0.10115606936416185, + "16": 0.02258064516129032, + "17": 0.011560693641618497, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6118980169971672, + "12": 0.576271186440678, + "13": 0.41379310344827586, + "14": 0.2523364485981308, + "15": 0.14450867052023122, + "16": 0.03225806451612903, + "17": 0.02023121387283237, + "18": 0.002857142857142857, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.7195467422096318, + "12": 0.6694915254237288, + "13": 0.4702194357366771, + "14": 0.308411214953271, + "15": 0.18208092485549132, + "16": 0.04516129032258064, + "17": 0.023121387283236993, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7960339943342776, + "12": 0.711864406779661, + "13": 0.542319749216301, + "14": 0.34890965732087226, + "15": 0.21098265895953758, + "16": 0.054838709677419356, + "17": 0.02601156069364162, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8413597733711048, + "12": 0.7796610169491526, + "13": 0.6018808777429467, + "14": 0.3862928348909657, + "15": 0.24277456647398843, + "16": 0.06451612903225806, + "17": 0.03468208092485549, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.8781869688385269, + "12": 0.8107344632768362, + "13": 0.6363636363636364, + "14": 0.4423676012461059, + "15": 0.2861271676300578, + "16": 0.07419354838709677, + "17": 0.04046242774566474, + "18": 0.014285714285714285, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.9036827195467422, + "12": 0.8220338983050848, + "13": 0.6739811912225705, + "14": 0.48909657320872274, + "15": 0.32947976878612717, + "16": 0.08387096774193549, + "17": 0.046242774566473986, + "18": 0.014285714285714285, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9178470254957507, + "12": 0.8502824858757062, + "13": 0.7115987460815048, + "14": 0.5295950155763239, + "15": 0.35260115606936415, + "16": 0.0967741935483871, + "17": 0.06069364161849711, + "18": 0.022857142857142857, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + } + }, + "total_pass_at_k": { + "pass@1": 0.17057949479940565, + "pass@2": 0.20832095096582467, + "pass@4": 0.24576523031203565, + "pass@8": 0.2731054977711738, + "pass@16": 0.300148588410104, + "pass@32": 0.3224368499257058, + "pass@64": 0.34056463595839526, + "pass@128": 0.3583952451708767 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5365079365079365, + "pass@2": 0.5904761904761905, + "pass@4": 0.6952380952380952, + "pass@8": 0.8063492063492064, + "pass@16": 0.8825396825396825, + "pass@32": 0.9174603174603174, + "pass@64": 0.946031746031746, + "pass@128": 0.9523809523809523 + }, + "12": { + "pass@1": 0.5015290519877675, + "pass@2": 0.5871559633027523, + "pass@4": 0.6788990825688074, + "pass@8": 0.7614678899082569, + "pass@16": 0.8042813455657493, + "pass@32": 0.8409785932721713, + "pass@64": 0.8593272171253823, + "pass@128": 0.8868501529051988 + }, + "13": { + "pass@1": 0.3395061728395062, + "pass@2": 0.41358024691358025, + "pass@4": 0.46296296296296297, + "pass@8": 0.5246913580246914, + "pass@16": 0.5617283950617284, + "pass@32": 0.6111111111111112, + "pass@64": 0.654320987654321, + "pass@128": 0.6975308641975309 + }, + "14": { + "pass@1": 0.20771513353115728, + "pass@2": 0.2433234421364985, + "pass@4": 0.2997032640949555, + "pass@8": 0.3560830860534125, + "pass@16": 0.42433234421364985, + "pass@32": 0.47774480712166173, + "pass@64": 0.516320474777448, + "pass@128": 0.5786350148367952 + }, + "15": { + "pass@1": 0.12962962962962962, + "pass@2": 0.16049382716049382, + "pass@4": 0.2006172839506173, + "pass@8": 0.2191358024691358, + "pass@16": 0.2654320987654321, + "pass@32": 0.3148148148148148, + "pass@64": 0.37037037037037035, + "pass@128": 0.4104938271604938 + }, + "16": { + "pass@1": 0.035398230088495575, + "pass@2": 0.0471976401179941, + "pass@4": 0.058997050147492625, + "pass@8": 0.07964601769911504, + "pass@16": 0.0943952802359882, + "pass@32": 0.10619469026548672, + "pass@64": 0.13864306784660768, + "pass@128": 0.15634218289085547 + }, + "17": { + "pass@1": 0.02108433734939759, + "pass@2": 0.030120481927710843, + "pass@4": 0.03313253012048193, + "pass@8": 0.04819277108433735, + "pass@16": 0.060240963855421686, + "pass@32": 0.07530120481927711, + "pass@64": 0.08132530120481928, + "pass@128": 0.09939759036144578 + }, + "18": { + "pass@1": 0.01935483870967742, + "pass@2": 0.025806451612903226, + "pass@4": 0.04516129032258064, + "pass@8": 0.04838709677419355, + "pass@16": 0.05806451612903226, + "pass@32": 0.07419354838709677, + "pass@64": 0.07741935483870968, + "pass@128": 0.09032258064516129 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.006042296072507553, + "pass@32": 0.012084592145015106, + "pass@64": 0.015105740181268883, + "pass@128": 0.01812688821752266 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5365079365079365, + "12": 0.5015290519877675, + "13": 0.3395061728395062, + "14": 0.20771513353115728, + "15": 0.12962962962962962, + "16": 0.035398230088495575, + "17": 0.02108433734939759, + "18": 0.01935483870967742, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5904761904761905, + "12": 0.5871559633027523, + "13": 0.41358024691358025, + "14": 0.2433234421364985, + "15": 0.16049382716049382, + "16": 0.0471976401179941, + "17": 0.030120481927710843, + "18": 0.025806451612903226, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6952380952380952, + "12": 0.6788990825688074, + "13": 0.46296296296296297, + "14": 0.2997032640949555, + "15": 0.2006172839506173, + "16": 0.058997050147492625, + "17": 0.03313253012048193, + "18": 0.04516129032258064, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.8063492063492064, + "12": 0.7614678899082569, + "13": 0.5246913580246914, + "14": 0.3560830860534125, + "15": 0.2191358024691358, + "16": 0.07964601769911504, + "17": 0.04819277108433735, + "18": 0.04838709677419355, + "19": 0.0, + "20": 0.0 + }, + "pass@16": { + "11": 0.8825396825396825, + "12": 0.8042813455657493, + "13": 0.5617283950617284, + "14": 0.42433234421364985, + "15": 0.2654320987654321, + "16": 0.0943952802359882, + "17": 0.060240963855421686, + "18": 0.05806451612903226, + "19": 0.006042296072507553, + "20": 0.0 + }, + "pass@32": { + "11": 0.9174603174603174, + "12": 0.8409785932721713, + "13": 0.6111111111111112, + "14": 0.47774480712166173, + "15": 0.3148148148148148, + "16": 0.10619469026548672, + "17": 0.07530120481927711, + "18": 0.07419354838709677, + "19": 0.012084592145015106, + "20": 0.0 + }, + "pass@64": { + "11": 0.946031746031746, + "12": 0.8593272171253823, + "13": 0.654320987654321, + "14": 0.516320474777448, + "15": 0.37037037037037035, + "16": 0.13864306784660768, + "17": 0.08132530120481928, + "18": 0.07741935483870968, + "19": 0.015105740181268883, + "20": 0.0 + }, + "pass@128": { + "11": 0.9523809523809523, + "12": 0.8868501529051988, + "13": 0.6975308641975309, + "14": 0.5786350148367952, + "15": 0.4104938271604938, + "16": 0.15634218289085547, + "17": 0.09939759036144578, + "18": 0.09032258064516129, + "19": 0.01812688821752266, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.17726161369193155, + "pass@2": 0.2078239608801956, + "pass@4": 0.24511002444987776, + "pass@8": 0.2817848410757946, + "pass@16": 0.31295843520782396, + "pass@32": 0.34015892420537897, + "pass@64": 0.363080684596577, + "pass@128": 0.3863080684596577 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5993975903614458, + "pass@2": 0.6746987951807228, + "pass@4": 0.7469879518072289, + "pass@8": 0.8313253012048193, + "pass@16": 0.8855421686746988, + "pass@32": 0.9246987951807228, + "pass@64": 0.9518072289156626, + "pass@128": 0.963855421686747 + }, + "12": { + "pass@1": 0.46394984326018807, + "pass@2": 0.554858934169279, + "pass@4": 0.658307210031348, + "pass@8": 0.7241379310344828, + "pass@16": 0.7836990595611285, + "pass@32": 0.8181818181818182, + "pass@64": 0.8526645768025078, + "pass@128": 0.8652037617554859 + }, + "13": { + "pass@1": 0.3333333333333333, + "pass@2": 0.4005602240896359, + "pass@4": 0.45938375350140054, + "pass@8": 0.5350140056022409, + "pass@16": 0.6078431372549019, + "pass@32": 0.6610644257703081, + "pass@64": 0.7002801120448179, + "pass@128": 0.7254901960784313 + }, + "14": { + "pass@1": 0.16666666666666666, + "pass@2": 0.20760233918128654, + "pass@4": 0.2719298245614035, + "pass@8": 0.31871345029239767, + "pass@16": 0.3654970760233918, + "pass@32": 0.41812865497076024, + "pass@64": 0.47368421052631576, + "pass@128": 0.5292397660818714 + }, + "15": { + "pass@1": 0.13333333333333333, + "pass@2": 0.16666666666666666, + "pass@4": 0.19090909090909092, + "pass@8": 0.22121212121212122, + "pass@16": 0.2606060606060606, + "pass@32": 0.296969696969697, + "pass@64": 0.3484848484848485, + "pass@128": 0.38181818181818183 + }, + "16": { + "pass@1": 0.03418803418803419, + "pass@2": 0.042735042735042736, + "pass@4": 0.045584045584045586, + "pass@8": 0.05982905982905983, + "pass@16": 0.07407407407407407, + "pass@32": 0.09116809116809117, + "pass@64": 0.09971509971509972, + "pass@128": 0.11396011396011396 + }, + "17": { + "pass@1": 0.018633540372670808, + "pass@2": 0.024844720496894408, + "pass@4": 0.027950310559006212, + "pass@8": 0.040372670807453416, + "pass@16": 0.043478260869565216, + "pass@32": 0.049689440993788817, + "pass@64": 0.059006211180124224, + "pass@128": 0.07142857142857142 + }, + "18": { + "pass@1": 0.0058823529411764705, + "pass@2": 0.011764705882352941, + "pass@4": 0.011764705882352941, + "pass@8": 0.011764705882352941, + "pass@16": 0.023529411764705882, + "pass@32": 0.023529411764705882, + "pass@64": 0.029411764705882353, + "pass@128": 0.029411764705882353 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.005952380952380952, + "pass@128": 0.008928571428571428 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.005988023952095809, + "pass@16": 0.005988023952095809, + "pass@32": 0.005988023952095809, + "pass@64": 0.005988023952095809, + "pass@128": 0.008982035928143712 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5993975903614458, + "12": 0.46394984326018807, + "13": 0.3333333333333333, + "14": 0.16666666666666666, + "15": 0.13333333333333333, + "16": 0.03418803418803419, + "17": 0.018633540372670808, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6746987951807228, + "12": 0.554858934169279, + "13": 0.4005602240896359, + "14": 0.20760233918128654, + "15": 0.16666666666666666, + "16": 0.042735042735042736, + "17": 0.024844720496894408, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.7469879518072289, + "12": 0.658307210031348, + "13": 0.45938375350140054, + "14": 0.2719298245614035, + "15": 0.19090909090909092, + "16": 0.045584045584045586, + "17": 0.027950310559006212, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.8313253012048193, + "12": 0.7241379310344828, + "13": 0.5350140056022409, + "14": 0.31871345029239767, + "15": 0.22121212121212122, + "16": 0.05982905982905983, + "17": 0.040372670807453416, + "18": 0.011764705882352941, + "19": 0.0, + "20": 0.005988023952095809 + }, + "pass@16": { + "11": 0.8855421686746988, + "12": 0.7836990595611285, + "13": 0.6078431372549019, + "14": 0.3654970760233918, + "15": 0.2606060606060606, + "16": 0.07407407407407407, + "17": 0.043478260869565216, + "18": 0.023529411764705882, + "19": 0.0, + "20": 0.005988023952095809 + }, + "pass@32": { + "11": 0.9246987951807228, + "12": 0.8181818181818182, + "13": 0.6610644257703081, + "14": 0.41812865497076024, + "15": 0.296969696969697, + "16": 0.09116809116809117, + "17": 0.049689440993788817, + "18": 0.023529411764705882, + "19": 0.0, + "20": 0.005988023952095809 + }, + "pass@64": { + "11": 0.9518072289156626, + "12": 0.8526645768025078, + "13": 0.7002801120448179, + "14": 0.47368421052631576, + "15": 0.3484848484848485, + "16": 0.09971509971509972, + "17": 0.059006211180124224, + "18": 0.029411764705882353, + "19": 0.005952380952380952, + "20": 0.005988023952095809 + }, + "pass@128": { + "11": 0.963855421686747, + "12": 0.8652037617554859, + "13": 0.7254901960784313, + "14": 0.5292397660818714, + "15": 0.38181818181818183, + "16": 0.11396011396011396, + "17": 0.07142857142857142, + "18": 0.029411764705882353, + "19": 0.008928571428571428, + "20": 0.008982035928143712 + } + }, + "total_pass_at_k": { + "pass@1": 0.1745465358311032, + "pass@2": 0.2072554267023491, + "pass@4": 0.23996431757359502, + "pass@8": 0.2735652691049658, + "pass@16": 0.303895331549212, + "pass@32": 0.327980969372584, + "pass@64": 0.35176925364258105, + "pass@128": 0.3690157597383289 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.548, + "pass@2": 0.626, + "pass@4": 0.721, + "pass@8": 0.811, + "pass@16": 0.869, + "pass@32": 0.906, + "pass@64": 0.933, + "pass@128": 0.944 + }, + "12": { + "pass@1": 0.495, + "pass@2": 0.573, + "pass@4": 0.669, + "pass@8": 0.732, + "pass@16": 0.789, + "pass@32": 0.823, + "pass@64": 0.844, + "pass@128": 0.867 + }, + "13": { + "pass@1": 0.335, + "pass@2": 0.409, + "pass@4": 0.464, + "pass@8": 0.534, + "pass@16": 0.591, + "pass@32": 0.637, + "pass@64": 0.677, + "pass@128": 0.712 + }, + "14": { + "pass@1": 0.185, + "pass@2": 0.234, + "pass@4": 0.293, + "pass@8": 0.341, + "pass@16": 0.392, + "pass@32": 0.446, + "pass@64": 0.493, + "pass@128": 0.546 + }, + "15": { + "pass@1": 0.121, + "pass@2": 0.157, + "pass@4": 0.191, + "pass@8": 0.217, + "pass@16": 0.256, + "pass@32": 0.299, + "pass@64": 0.349, + "pass@128": 0.381 + }, + "16": { + "pass@1": 0.031, + "pass@2": 0.041, + "pass@4": 0.05, + "pass@8": 0.065, + "pass@16": 0.078, + "pass@32": 0.091, + "pass@64": 0.108, + "pass@128": 0.123 + }, + "17": { + "pass@1": 0.017, + "pass@2": 0.025, + "pass@4": 0.028, + "pass@8": 0.038, + "pass@16": 0.046, + "pass@32": 0.055, + "pass@64": 0.062, + "pass@128": 0.077 + }, + "18": { + "pass@1": 0.009, + "pass@2": 0.013, + "pass@4": 0.02, + "pass@8": 0.021, + "pass@16": 0.03, + "pass@32": 0.036, + "pass@64": 0.039, + "pass@128": 0.046 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.002, + "pass@32": 0.004, + "pass@64": 0.008, + "pass@128": 0.01 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.002, + "pass@16": 0.003, + "pass@32": 0.004, + "pass@64": 0.004, + "pass@128": 0.005 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.548, + "12": 0.495, + "13": 0.335, + "14": 0.185, + "15": 0.121, + "16": 0.031, + "17": 0.017, + "18": 0.009, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.626, + "12": 0.573, + "13": 0.409, + "14": 0.234, + "15": 0.157, + "16": 0.041, + "17": 0.025, + "18": 0.013, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.721, + "12": 0.669, + "13": 0.464, + "14": 0.293, + "15": 0.191, + "16": 0.05, + "17": 0.028, + "18": 0.02, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.811, + "12": 0.732, + "13": 0.534, + "14": 0.341, + "15": 0.217, + "16": 0.065, + "17": 0.038, + "18": 0.021, + "19": 0.0, + "20": 0.002 + }, + "pass@16": { + "11": 0.869, + "12": 0.789, + "13": 0.591, + "14": 0.392, + "15": 0.256, + "16": 0.078, + "17": 0.046, + "18": 0.03, + "19": 0.002, + "20": 0.003 + }, + "pass@32": { + "11": 0.906, + "12": 0.823, + "13": 0.637, + "14": 0.446, + "15": 0.299, + "16": 0.091, + "17": 0.055, + "18": 0.036, + "19": 0.004, + "20": 0.004 + }, + "pass@64": { + "11": 0.933, + "12": 0.844, + "13": 0.677, + "14": 0.493, + "15": 0.349, + "16": 0.108, + "17": 0.062, + "18": 0.039, + "19": 0.008, + "20": 0.004 + }, + "pass@128": { + "11": 0.944, + "12": 0.867, + "13": 0.712, + "14": 0.546, + "15": 0.381, + "16": 0.123, + "17": 0.077, + "18": 0.046, + "19": 0.01, + "20": 0.005 + } + }, + "total_pass_at_k": { + "pass@1": 0.1741, + "pass@2": 0.2078, + "pass@4": 0.2436, + "pass@8": 0.2761, + "pass@16": 0.3056, + "pass@32": 0.3301, + "pass@64": 0.3517, + "pass@128": 0.3711 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/summary.csv b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..090734c126796cdb97c3acd433633f2d3dff0ccf --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21546,0.9822352430555555,0.42139921875,0.6870583881578948,0.0,0.0,0.0,197.35695486111112,319.1119796875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/summary.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4804119755d722580288c9c99280891d629e8e2d --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block4/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21546", + "id_acc": 0.9822352430555555, + "ood_acc": 0.42139921875, + "total_acc": 0.6870583881578948, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.35695486111112, + "ood_avg_resp_len": 319.1119796875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 552886, + "answer_accuracy": 0.6828045961112867, + "avg_response_len": 249.1386391479608, + "resp_tokens_sum": 201734532, + "pass_at_k": { + "pass@1": 0.6828045961112867, + "pass@2": 0.7217115855449684, + "pass@4": 0.7536723871913236, + "pass@8": 0.7797075439923347, + "pass@16": 0.800334627046473, + "pass@32": 0.817100237695868, + "pass@64": 0.8315237481736786, + "pass@128": 0.8444514701233007 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9263228244837758, + "pass@2": 0.9494500238078647, + "pass@4": 0.9622405067176335, + "pass@8": 0.9693425057647705, + "pass@16": 0.9736827240563464, + "pass@32": 0.9773741161261673, + "pass@64": 0.98185566893626, + "pass@128": 0.9882005899705014 + }, + "2": { + "pass@1": 0.9999756619937694, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9965596330275229, + "pass@2": 0.9981052397120085, + "pass@4": 0.999307317089951, + "pass@8": 0.999884609180392, + "pass@16": 0.9999968325205879, + "pass@32": 0.9999999991746348, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9903076171875, + "pass@2": 0.9908726008858268, + "pass@4": 0.9911945356439821, + "pass@8": 0.9917235482142811, + "pass@16": 0.9926657644356954, + "pass@32": 0.9941486923509562, + "pass@64": 0.9959137795275591, + "pass@128": 0.996875 + }, + "5": { + "pass@1": 0.9702628968253968, + "pass@2": 0.9771036432945881, + "pass@4": 0.9829424443968311, + "pass@8": 0.9876832076495944, + "pass@16": 0.990954409287743, + "pass@32": 0.9930190209714992, + "pass@64": 0.9948502012050503, + "pass@128": 0.9968253968253968 + }, + "6": { + "pass@1": 0.9956727170658682, + "pass@2": 0.9975463688764207, + "pass@4": 0.9990146021230881, + "pass@8": 0.9998480009664775, + "pass@16": 0.9999970866060833, + "pass@32": 0.9999999996392369, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9940366124260355, + "pass@2": 0.9978666047383871, + "pass@4": 0.9993812003159368, + "pass@8": 0.9999391517555988, + "pass@16": 0.9999995060547365, + "pass@32": 0.9999999999906712, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9850260416666666, + "pass@2": 0.991938282354129, + "pass@4": 0.9944415109169044, + "pass@8": 0.9959676378945411, + "pass@16": 0.996687988729117, + "pass@32": 0.9967934570195686, + "pass@64": 0.9967948717815591, + "pass@128": 0.9967948717948718 + }, + "9": { + "pass@1": 0.975594190140845, + "pass@2": 0.9893763169568592, + "pass@4": 0.9948080432727598, + "pass@8": 0.9979627772031147, + "pass@16": 0.9995681970104956, + "pass@32": 0.9999716229169351, + "pass@64": 0.9999999660012311, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8872388456090652, + "pass@2": 0.9170830452142494, + "pass@4": 0.9344475201046042, + "pass@8": 0.9461166366837217, + "pass@16": 0.9548923315039325, + "pass@32": 0.9624344015568419, + "pass@64": 0.9692996308589977, + "pass@128": 0.9745042492917847 + }, + "12": { + "pass@1": 0.7989495056497176, + "pass@2": 0.8576045976244497, + "pass@4": 0.894183136271101, + "pass@8": 0.9153254162241761, + "pass@16": 0.9267540179385538, + "pass@32": 0.9355352501223162, + "pass@64": 0.9451007362794269, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.661662421630094, + "pass@2": 0.7403545621158639, + "pass@4": 0.7929696285025494, + "pass@8": 0.8331626588813359, + "pass@16": 0.8660458043380581, + "pass@32": 0.8937908906213489, + "pass@64": 0.9188840917684389, + "pass@128": 0.9435736677115988 + }, + "14": { + "pass@1": 0.48459404205607476, + "pass@2": 0.586037726592587, + "pass@4": 0.6693883112741745, + "pass@8": 0.7337120781768929, + "pass@16": 0.7796279839990717, + "pass@32": 0.8097918673125195, + "pass@64": 0.8326759446166977, + "pass@128": 0.8566978193146417 + }, + "15": { + "pass@1": 0.3742097182080925, + "pass@2": 0.46095314562377676, + "pass@4": 0.5352501313961474, + "pass@8": 0.5976363331573419, + "pass@16": 0.6497626597389893, + "pass@32": 0.6938640362470392, + "pass@64": 0.7308787729367424, + "pass@128": 0.7630057803468208 + }, + "16": { + "pass@1": 0.2728074596774194, + "pass@2": 0.35024368173736364, + "pass@4": 0.41988924942849887, + "pass@8": 0.47919964105067914, + "pass@16": 0.527871351126739, + "pass@32": 0.5696261229797946, + "pass@64": 0.6054567059936721, + "pass@128": 0.632258064516129 + }, + "17": { + "pass@1": 0.21050848988439305, + "pass@2": 0.2739904305675663, + "pass@4": 0.3313912844528826, + "pass@8": 0.38125538448297414, + "pass@16": 0.4236245875549352, + "pass@32": 0.4619779463348103, + "pass@64": 0.49696813014542734, + "pass@128": 0.5260115606936416 + }, + "18": { + "pass@1": 0.1708705357142857, + "pass@2": 0.2367473987626546, + "pass@4": 0.30414279554341417, + "pass@8": 0.3648555156644124, + "pass@16": 0.41361773639831223, + "pass@32": 0.4516806081441688, + "pass@64": 0.48544624589394647, + "pass@128": 0.5228571428571429 + }, + "19": { + "pass@1": 0.15662537537537538, + "pass@2": 0.21153806365420533, + "pass@4": 0.266338078235716, + "pass@8": 0.3166763163826258, + "pass@16": 0.360523558204304, + "pass@32": 0.39946955168091575, + "pass@64": 0.4333781248476876, + "pass@128": 0.4594594594594595 + }, + "20": { + "pass@1": 0.15707113363363365, + "pass@2": 0.21765934241327928, + "pass@4": 0.27828566361637236, + "pass@8": 0.3325391712216513, + "pass@16": 0.3774587634032869, + "pass@32": 0.41180155357726206, + "pass@64": 0.43666908597212345, + "pass@128": 0.45645645645645644 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 563959, + "answer_accuracy": 0.6868167868277475, + "avg_response_len": 257.8540128117693, + "resp_tokens_sum": 211729087, + "pass_at_k": { + "pass@1": 0.6868167868277475, + "pass@2": 0.7234032563934145, + "pass@4": 0.7540754758149335, + "pass@8": 0.7798046148261166, + "pass@16": 0.8016922862089578, + "pass@32": 0.8207486969269729, + "pass@64": 0.8371004495427515, + "pass@128": 0.8508183943881528 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9111148389570553, + "pass@2": 0.9357741624800733, + "pass@4": 0.9522138641795541, + "pass@8": 0.9627448771437234, + "pass@16": 0.9721080231114817, + "pass@32": 0.9821285716890078, + "pass@64": 0.9909827842779778, + "pass@128": 0.9969325153374233 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9971117424242424, + "pass@2": 0.9987439602720115, + "pass@4": 0.9997009899330764, + "pass@8": 0.9999796152070378, + "pass@16": 0.9999999194724303, + "pass@32": 0.9999999999998216, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9998844304733728, + "pass@2": 0.9999978160089457, + "pass@4": 0.9999999997226677, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9863677536231884, + "pass@2": 0.9885042222982998, + "pass@4": 0.9891683455328956, + "pass@8": 0.9898325450069416, + "pass@16": 0.9908752954090346, + "pass@32": 0.9922112515258646, + "pass@64": 0.9934649176276082, + "pass@128": 0.9942028985507246 + }, + "6": { + "pass@1": 0.9970003581661891, + "pass@2": 0.9985179816348173, + "pass@4": 0.9994818757433256, + "pass@8": 0.9999207959785322, + "pass@16": 0.9999984128084317, + "pass@32": 0.9999999998125675, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9931809971098265, + "pass@2": 0.9950314477037914, + "pass@4": 0.9958010177376672, + "pass@8": 0.9966315484192274, + "pass@16": 0.9973368465089668, + "pass@32": 0.997828695449064, + "pass@64": 0.9985549131017303, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9819762323943662, + "pass@2": 0.9893354219807033, + "pass@4": 0.9935911558590389, + "pass@8": 0.9965179072847223, + "pass@16": 0.9981742152631411, + "pass@32": 0.999119008113794, + "pass@64": 0.9998322278254919, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9731942365269461, + "pass@2": 0.9808708125088408, + "pass@4": 0.9824776866402178, + "pass@8": 0.9832461410936331, + "pass@16": 0.9842818929540148, + "pass@32": 0.9858385141976975, + "pass@64": 0.9874802206610402, + "pass@128": 0.9880239520958084 + }, + "11": { + "pass@1": 0.9079207454819277, + "pass@2": 0.93692449423679, + "pass@4": 0.9551024624369245, + "pass@8": 0.9671201826101439, + "pass@16": 0.9746815567314097, + "pass@32": 0.9788712127467571, + "pass@64": 0.9815593127389013, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8269004702194357, + "pass@2": 0.8749591180608697, + "pass@4": 0.9040032188374567, + "pass@8": 0.9231087891699578, + "pass@16": 0.9369859413563386, + "pass@32": 0.9479069421599462, + "pass@64": 0.9560498758422292, + "pass@128": 0.9623824451410659 + }, + "13": { + "pass@1": 0.6125262605042017, + "pass@2": 0.6965695234786827, + "pass@4": 0.7562029827679098, + "pass@8": 0.8020303886222311, + "pass@16": 0.838161985699861, + "pass@32": 0.8655366886816216, + "pass@64": 0.8870740770303306, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.4824561403508772, + "pass@2": 0.5683321246028454, + "pass@4": 0.6404780016533019, + "pass@8": 0.6993768785639923, + "pass@16": 0.7467342166602401, + "pass@32": 0.7841871616007284, + "pass@64": 0.8128076930956771, + "pass@128": 0.8362573099415205 + }, + "15": { + "pass@1": 0.39372632575757577, + "pass@2": 0.4847448401336197, + "pass@4": 0.557197479292361, + "pass@8": 0.613319566937348, + "pass@16": 0.6601720515546455, + "pass@32": 0.702804164645992, + "pass@64": 0.7402769189342715, + "pass@128": 0.7727272727272727 + }, + "16": { + "pass@1": 0.24263265669515668, + "pass@2": 0.30694675785719094, + "pass@4": 0.36897373752426255, + "pass@8": 0.4271218724609568, + "pass@16": 0.48113967991388973, + "pass@32": 0.5317545339132025, + "pass@64": 0.5758856604621956, + "pass@128": 0.6096866096866097 + }, + "17": { + "pass@1": 0.22648971273291926, + "pass@2": 0.2974423968063773, + "pass@4": 0.3655080216098764, + "pass@8": 0.4248164502069451, + "pass@16": 0.4753100911077305, + "pass@32": 0.5195282979937258, + "pass@64": 0.5565277964456121, + "pass@128": 0.5838509316770186 + }, + "18": { + "pass@1": 0.21300551470588236, + "pass@2": 0.2771649924733672, + "pass@4": 0.3376023183682923, + "pass@8": 0.39235717588844027, + "pass@16": 0.44123591873712553, + "pass@32": 0.48463721239549934, + "pass@64": 0.5242288971716302, + "pass@128": 0.5617647058823529 + }, + "19": { + "pass@1": 0.14706566220238096, + "pass@2": 0.20007777348143976, + "pass@4": 0.25505773050690084, + "pass@8": 0.306085461392637, + "pass@16": 0.3516417990127139, + "pass@32": 0.39376010269129014, + "pass@64": 0.43223091722228596, + "pass@128": 0.46726190476190477 + }, + "20": { + "pass@1": 0.14556044161676646, + "pass@2": 0.20420244648498276, + "pass@4": 0.26505273339709784, + "pass@8": 0.3224901187290411, + "pass@16": 0.37382821128207, + "pass@32": 0.4189057357484239, + "pass@64": 0.45902656447888046, + "pass@128": 0.4940119760479042 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 554081, + "answer_accuracy": 0.6916053383128296, + "avg_response_len": 277.5439953966289, + "resp_tokens_sum": 222354927, + "pass_at_k": { + "pass@1": 0.6916053383128296, + "pass@2": 0.7286108201040862, + "pass@4": 0.7588274034696325, + "pass@8": 0.7837869789700335, + "pass@16": 0.8045777313882145, + "pass@32": 0.8226117310463649, + "pass@64": 0.8388624663967905, + "pass@128": 0.8528518932736859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9279617537313433, + "pass@2": 0.9513434304853687, + "pass@4": 0.9654792511010749, + "pass@8": 0.9737231562263299, + "pass@16": 0.97817026758166, + "pass@32": 0.981386045278353, + "pass@64": 0.9839306760109867, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9996127915451894, + "pass@2": 0.9999770436858658, + "pass@4": 0.9999999464352669, + "pass@8": 0.9999999999999796, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9781479779411765, + "pass@2": 0.9830361278369619, + "pass@4": 0.9859722322393524, + "pass@8": 0.9881811181957687, + "pass@16": 0.9897299162379272, + "pass@32": 0.9906714940716781, + "pass@64": 0.9911358126184439, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.996352523659306, + "pass@2": 0.9968279608286346, + "pass@4": 0.9968454007326369, + "pass@8": 0.9968454258675057, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9880834651898734, + "pass@2": 0.9925585411890763, + "pass@4": 0.9936028700526358, + "pass@8": 0.9936705108502834, + "pass@16": 0.9936708860669233, + "pass@32": 0.9936708860759493, + "pass@64": 0.9936708860759493, + "pass@128": 0.9936708860759493 + }, + "8": { + "pass@1": 0.9793308933933934, + "pass@2": 0.9852753836513678, + "pass@4": 0.988564877025507, + "pass@8": 0.9918839873821754, + "pass@16": 0.9945610320902971, + "pass@32": 0.9960152183579682, + "pass@64": 0.9968180581304334, + "pass@128": 0.996996996996997 + }, + "9": { + "pass@1": 0.975633038585209, + "pass@2": 0.985183210117224, + "pass@4": 0.9885799165578577, + "pass@8": 0.9912897529136566, + "pass@16": 0.9942023916344718, + "pass@32": 0.9970803734644477, + "pass@64": 0.9993715908946497, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8945436507936508, + "pass@2": 0.9223569710036246, + "pass@4": 0.9402420390903516, + "pass@8": 0.9529961646045297, + "pass@16": 0.96257165667739, + "pass@32": 0.9707572036881429, + "pass@64": 0.9780558610114967, + "pass@128": 0.9841269841269841 + }, + "12": { + "pass@1": 0.8388761467889908, + "pass@2": 0.8935348641912881, + "pass@4": 0.9239817218948547, + "pass@8": 0.9433164183874273, + "pass@16": 0.9572791903461986, + "pass@32": 0.9678375527483201, + "pass@64": 0.975711998782937, + "pass@128": 0.981651376146789 + }, + "13": { + "pass@1": 0.6124614197530864, + "pass@2": 0.6951154217701951, + "pass@4": 0.752679211973503, + "pass@8": 0.7942920488824928, + "pass@16": 0.8279101424953335, + "pass@32": 0.8575816947638316, + "pass@64": 0.884275234929319, + "pass@128": 0.9074074074074074 + }, + "14": { + "pass@1": 0.5233911350148368, + "pass@2": 0.614512021308909, + "pass@4": 0.6847058470435999, + "pass@8": 0.7369969091059508, + "pass@16": 0.7790020662193811, + "pass@32": 0.8157176244438528, + "pass@64": 0.8475338299324224, + "pass@128": 0.8724035608308606 + }, + "15": { + "pass@1": 0.40183738425925924, + "pass@2": 0.4773633439049284, + "pass@4": 0.5415484597990062, + "pass@8": 0.5997374246066435, + "pass@16": 0.652959196898339, + "pass@32": 0.7031944084122549, + "pass@64": 0.750283610070048, + "pass@128": 0.7901234567901234 + }, + "16": { + "pass@1": 0.2575589970501475, + "pass@2": 0.32797111989872935, + "pass@4": 0.39335200992575037, + "pass@8": 0.4518741747379751, + "pass@16": 0.5021914236768155, + "pass@32": 0.5466943347804367, + "pass@64": 0.5881361312764039, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.21653802710843373, + "pass@2": 0.2869075959349209, + "pass@4": 0.34959244514616383, + "pass@8": 0.4005336949460392, + "pass@16": 0.44448347713010716, + "pass@32": 0.48757226058553715, + "pass@64": 0.5302459429332484, + "pass@128": 0.572289156626506 + }, + "18": { + "pass@1": 0.2128024193548387, + "pass@2": 0.28386779273558566, + "pass@4": 0.35476632437074396, + "pass@8": 0.41725617116976693, + "pass@16": 0.46536508744664384, + "pass@32": 0.49949986996412565, + "pass@64": 0.5271047913394351, + "pass@128": 0.5516129032258065 + }, + "19": { + "pass@1": 0.16871223564954682, + "pass@2": 0.22346459666484297, + "pass@4": 0.27490075381438345, + "pass@8": 0.32159746324003746, + "pass@16": 0.3655734909622184, + "pass@32": 0.40807790411245964, + "pass@64": 0.4465981236371106, + "pass@128": 0.4773413897280967 + }, + "20": { + "pass@1": 0.1522850975975976, + "pass@2": 0.21085788051831356, + "pass@4": 0.2715947052226581, + "pass@8": 0.3282373203373321, + "pass@16": 0.37425196123074994, + "pass@32": 0.4095715191404, + "pass@64": 0.4416954781218096, + "pass@128": 0.4744744744744745 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/checkpoint-21546_metrics.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/checkpoint-21546_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..9b51d7a906e091e838a02e089c1bb73ec9031393 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/checkpoint-21546_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-21546", + "id": { + "answer_accuracy": 0.9797482638888889, + "avg_response_len": 196.59117881944445, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9797482638888889, + "pass@2": 0.9861229358048901, + "pass@4": 0.9888449128233961, + "pass@8": 0.990602009745777, + "pass@16": 0.9918611680460637, + "pass@32": 0.9927922537316319, + "pass@64": 0.9935465364724203, + "pass@128": 0.9942222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.916296875, + "pass@2": 0.9394058809055122, + "pass@4": 0.9528618595800525, + "pass@8": 0.9615409457400279, + "pass@16": 0.9674628728972483, + "pass@32": 0.9717329785336175, + "pass@64": 0.9754449907658412, + "pass@128": 0.979 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9981171875, + "pass@2": 0.9985055364173229, + "pass@4": 0.9987710755530559, + "pass@8": 0.998950601439429, + "pass@16": 0.9989981090802741, + "pass@32": 0.998999998952485, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.99703125, + "pass@2": 0.9971211860236221, + "pass@4": 0.9972299803149606, + "pass@8": 0.9974122820506877, + "pass@16": 0.997667419611923, + "pass@32": 0.9979072527311064, + "pass@64": 0.9979969041357232, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9782734375, + "pass@2": 0.9810522883858267, + "pass@4": 0.9831453160854895, + "pass@8": 0.984949122876631, + "pass@16": 0.9863992255520356, + "pass@32": 0.9876978908604928, + "pass@64": 0.9891785804477966, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.9946640625, + "pass@2": 0.9978164370078745, + "pass@4": 0.9983948684851892, + "pass@8": 0.9987443749724515, + "pass@16": 0.9989454139612846, + "pass@32": 0.9989981933094014, + "pass@64": 0.9989999998360333, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9909453125, + "pass@2": 0.9939223671259843, + "pass@4": 0.9948598255530557, + "pass@8": 0.9954570343475408, + "pass@16": 0.9958527932799116, + "pass@32": 0.9959896697616626, + "pass@64": 0.9959999879263536, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.9757578125, + "pass@2": 0.9845514271653546, + "pass@4": 0.9871017754030745, + "pass@8": 0.9888908328480559, + "pass@16": 0.9905008788416947, + "pass@32": 0.9920584406618639, + "pass@64": 0.9933081767467343, + "pass@128": 0.994 + }, + "9": { + "pass@1": 0.9666484375, + "pass@2": 0.9827312992126, + "pass@4": 0.9872395144356954, + "pass@8": 0.9894728934371575, + "pass@16": 0.9909237991901977, + "pass@32": 0.9917458587740582, + "pass@64": 0.991990188393302, + "pass@128": 0.992 + } + }, + "per_op_accuracy": { + "10": 0.916296875, + "2": 1.0, + "3": 0.9981171875, + "4": 0.99703125, + "5": 0.9782734375, + "6": 0.9946640625, + "7": 0.9909453125, + "8": 0.9757578125, + "9": 0.9666484375 + }, + "per_op_avg_response_len": { + "10": 274.7838515625, + "2": 114.360515625, + "3": 142.7705703125, + "4": 157.8014921875, + "5": 170.854, + "6": 195.624046875, + "7": 219.9284765625, + "8": 238.8914453125, + "9": 254.3062109375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.7838515625, + "2": 114.360515625, + "3": 142.7705703125, + "4": 157.8014921875, + "5": 170.854, + "6": 195.624046875, + "7": 219.9284765625, + "8": 238.8914453125, + "9": 254.3062109375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 371002, + "answer_accuracy": 0.9788764353259034, + "avg_response_len": 187.21866029213103, + "resp_tokens_sum": 70957370, + "pass_at_k": { + "pass@1": 0.9788764353259034, + "pass@2": 0.9860764163123242, + "pass@4": 0.9891164809439359, + "pass@8": 0.9910691954398814, + "pass@16": 0.9923680420455808, + "pass@32": 0.9932405304294623, + "pass@64": 0.993936925038791, + "pass@128": 0.994596420128335 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9221515486725663, + "pass@2": 0.9428836259958658, + "pass@4": 0.953557931641288, + "pass@8": 0.9603148730351716, + "pass@16": 0.9648846877139904, + "pass@32": 0.9680322261071029, + "pass@64": 0.9708755602536454, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9978736620795107, + "pass@2": 0.9985266319921019, + "pass@4": 0.9993000800358671, + "pass@8": 0.9998489340659815, + "pass@16": 0.9999942173708687, + "pass@32": 0.9999999967965904, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9907470703125, + "pass@2": 0.9910037063238188, + "pass@4": 0.991343688484252, + "pass@8": 0.9919133814083988, + "pass@16": 0.9927106862872591, + "pass@32": 0.9934601647847077, + "pass@64": 0.9937403254241349, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9669642857142857, + "pass@2": 0.9718843738282714, + "pass@4": 0.9763600398164514, + "pass@8": 0.9802523092172719, + "pass@16": 0.9830724326754721, + "pass@32": 0.9851478752003268, + "pass@64": 0.987257685923108, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9989277105238344, + "pass@4": 0.9996942885881777, + "pass@8": 0.9999718501697081, + "pass@16": 0.9999998309063756, + "pass@32": 0.999999999998971, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9926960059171598, + "pass@2": 0.9966541257047011, + "pass@4": 0.9984301580460726, + "pass@8": 0.9994525764888945, + "pass@16": 0.9999158567707788, + "pass@32": 0.9999987203425054, + "pass@64": 0.9999999999877117, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9768880208333334, + "pass@2": 0.9874886432465175, + "pass@4": 0.9895625994467037, + "pass@8": 0.9907045168965894, + "pass@16": 0.9916775319238652, + "pass@32": 0.9925913730797592, + "pass@64": 0.9933988491820491, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9691681338028169, + "pass@2": 0.9880122269047353, + "pass@4": 0.9939504940403571, + "pass@8": 0.9970743852484422, + "pass@16": 0.9988665501562034, + "pass@32": 0.9997334583192324, + "pass@64": 0.9999912787935963, + "pass@128": 1.0 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 382535, + "answer_accuracy": 0.9792118897444299, + "avg_response_len": 194.5653976900393, + "resp_tokens_sum": 76008140, + "pass_at_k": { + "pass@1": 0.9792118897444299, + "pass@2": 0.9855319878664834, + "pass@4": 0.9883347573751037, + "pass@8": 0.9901135290360076, + "pass@16": 0.9914019930292206, + "pass@32": 0.9923524181343902, + "pass@64": 0.993097748699178, + "pass@128": 0.9937745740498034 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.902032208588957, + "pass@2": 0.928237539853147, + "pass@4": 0.9445890421672748, + "pass@8": 0.9554783664156684, + "pass@16": 0.9637377662720875, + "pass@32": 0.9702591907243098, + "pass@64": 0.975278890857173, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966619318181819, + "pass@2": 0.9969518014793606, + "pass@4": 0.9969696373180625, + "pass@8": 0.9969696969696016, + "pass@16": 0.996969696969697, + "pass@32": 0.996969696969697, + "pass@64": 0.996969696969697, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875226449275363, + "pass@2": 0.9883676395070182, + "pass@4": 0.9884954541008462, + "pass@8": 0.9885869564329156, + "pass@16": 0.988768115942029, + "pass@32": 0.9891304347826086, + "pass@64": 0.9898550724637681, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9941350286532952, + "pass@2": 0.9977561734314014, + "pass@4": 0.9985594472610693, + "pass@8": 0.9992944900423253, + "pass@16": 0.9998437548382668, + "pass@32": 0.9999948232370918, + "pass@64": 0.9999999995301811, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928648843930635, + "pass@2": 0.994768672340813, + "pass@4": 0.9953643609758319, + "pass@8": 0.9960753760142124, + "pass@16": 0.9967665713625197, + "pass@32": 0.9970812204794673, + "pass@64": 0.9971097917066677, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9756382042253521, + "pass@2": 0.9838094016857049, + "pass@4": 0.9872804353681142, + "pass@8": 0.9894908076168281, + "pass@16": 0.9907958224370905, + "pass@32": 0.991898265474723, + "pass@64": 0.9929392340213707, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.9612649700598802, + "pass@2": 0.9778384913244379, + "pass@4": 0.9821942192667527, + "pass@8": 0.983871105890422, + "pass@16": 0.9847190114530727, + "pass@32": 0.985008695002592, + "pass@64": 0.985029924450078, + "pass@128": 0.9850299401197605 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 375133, + "answer_accuracy": 0.9811605498828255, + "avg_response_len": 207.95197941078004, + "resp_tokens_sum": 79507528, + "pass_at_k": { + "pass@1": 0.9811605498828255, + "pass@2": 0.9867728579092088, + "pass@4": 0.9890969654592476, + "pass@8": 0.9906380011371257, + "pass@16": 0.9918278731812604, + "pass@32": 0.9927972858509846, + "pass@64": 0.9936180998265927, + "pass@128": 0.9943086709072648 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9242537313432836, + "pass@2": 0.9467549065695148, + "pass@4": 0.960208038827236, + "pass@8": 0.9686813621778994, + "pass@16": 0.9736968715149406, + "pass@32": 0.9769121125587591, + "pass@64": 0.9802306191654232, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997494533527697, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9793658088235294, + "pass@2": 0.982123161764706, + "pass@4": 0.9840028584662212, + "pass@8": 0.9856092514818663, + "pass@16": 0.9870776742919471, + "pass@32": 0.9886067946246756, + "pass@64": 0.9902717923000518, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9942576892744479, + "pass@2": 0.9967119153481209, + "pass@4": 0.9968445709861974, + "pass@8": 0.9968454258391087, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9869709256329114, + "pass@2": 0.9900737721768167, + "pass@4": 0.9904884912526759, + "pass@8": 0.9905062765613193, + "pass@16": 0.9905063291137864, + "pass@32": 0.990506329113924, + "pass@64": 0.990506329113924, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.9748263888888888, + "pass@2": 0.9825904891111588, + "pass@4": 0.9846056751352027, + "pass@8": 0.9865519125295662, + "pass@16": 0.9890839997485931, + "pass@32": 0.9917298739262829, + "pass@64": 0.9936165397127583, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9695538585209004, + "pass@2": 0.9819578860419779, + "pass@4": 0.9849974913706525, + "pass@8": 0.9868120170628933, + "pass@16": 0.9885209776830189, + "pass@32": 0.9898635914465111, + "pass@64": 0.9903321210779717, + "pass@128": 0.9903536977491961 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.41910390625, + "avg_response_len": 317.97294609375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.41910390625, + "pass@2": 0.48035402312992054, + "pass@4": 0.5309406500562398, + "pass@8": 0.5724556171009065, + "pass@16": 0.6070143353709206, + "pass@32": 0.6366226510598554, + "pass@64": 0.6627574545524926, + "pass@128": 0.6862 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8883046875, + "pass@2": 0.9191022391732293, + "pass@4": 0.9375445604611923, + "pass@8": 0.9501022353260171, + "pass@16": 0.9590303020747554, + "pass@32": 0.9660979044974181, + "pass@64": 0.9720907279383584, + "pass@128": 0.977 + }, + "12": { + "pass@1": 0.81803125, + "pass@2": 0.8704116633858281, + "pass@4": 0.9011900994563182, + "pass@8": 0.9199582373358968, + "pass@16": 0.9323515597146504, + "pass@32": 0.9418769310531213, + "pass@64": 0.9506652345944766, + "pass@128": 0.959 + }, + "13": { + "pass@1": 0.6158359375, + "pass@2": 0.6948138533464573, + "pass@4": 0.74916785639295, + "pass@8": 0.7890487229339681, + "pass@16": 0.820990559237083, + "pass@32": 0.8486991307097956, + "pass@64": 0.8733152240819593, + "pass@128": 0.893 + }, + "14": { + "pass@1": 0.5010546875, + "pass@2": 0.5843780757874018, + "pass@4": 0.6505600973003363, + "pass@8": 0.7016133180700607, + "pass@16": 0.7422435474989071, + "pass@32": 0.7771215371291468, + "pass@64": 0.8078247615201297, + "pass@128": 0.835 + }, + "15": { + "pass@1": 0.38159375, + "pass@2": 0.4551263533464563, + "pass@4": 0.5163803885451813, + "pass@8": 0.5676965353330589, + "pass@16": 0.611672517651196, + "pass@32": 0.650257501041337, + "pass@64": 0.6861249290677806, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.2539375, + "pass@2": 0.31808612204724435, + "pass@4": 0.37620671831646035, + "pass@8": 0.4274761518103465, + "pass@16": 0.4726381425836207, + "pass@32": 0.5117128471984673, + "pass@64": 0.5449662833006126, + "pass@128": 0.573 + }, + "17": { + "pass@1": 0.2166328125, + "pass@2": 0.27816412401574825, + "pass@4": 0.33224575131233564, + "pass@8": 0.3770731582144976, + "pass@16": 0.41496914260503864, + "pass@32": 0.4475605408923135, + "pass@64": 0.47527931585822586, + "pass@128": 0.501 + }, + "18": { + "pass@1": 0.1997109375, + "pass@2": 0.26007074311023637, + "pass@4": 0.3188129970941129, + "pass@8": 0.37140755571370326, + "pass@16": 0.41483116407611115, + "pass@32": 0.44973332592553017, + "pass@64": 0.47895408756267954, + "pass@128": 0.504 + }, + "19": { + "pass@1": 0.16115625, + "pass@2": 0.212988065944882, + "pass@4": 0.26191477990251183, + "pass@8": 0.3044093655498218, + "pass@16": 0.34145186161861824, + "pass@32": 0.375304737715687, + "pass@64": 0.40757055863951047, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.15478125, + "pass@2": 0.2103989911417322, + "pass@4": 0.26538325178102734, + "pass@8": 0.3157708907216943, + "pass@16": 0.3599645566490957, + "pass@32": 0.39786205443572714, + "pass@64": 0.4307834229611375, + "pass@128": 0.46 + } + }, + "per_op_accuracy": { + "11": 0.8883046875, + "12": 0.81803125, + "13": 0.6158359375, + "14": 0.5010546875, + "15": 0.38159375, + "16": 0.2539375, + "17": 0.2166328125, + "18": 0.1997109375, + "19": 0.16115625, + "20": 0.15478125 + }, + "per_op_avg_response_len": { + "11": 291.286734375, + "12": 300.4761953125, + "13": 312.851765625, + "14": 324.2943515625, + "15": 328.16921875, + "16": 318.110484375, + "17": 326.76978125, + "18": 330.2153671875, + "19": 320.9465078125, + "20": 326.6090546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 291.286734375, + "12": 300.4761953125, + "13": 312.851765625, + "14": 324.2943515625, + "15": 328.16921875, + "16": 318.110484375, + "17": 326.76978125, + "18": 330.2153671875, + "19": 320.9465078125, + "20": 326.6090546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 180349, + "answer_accuracy": 0.41871517459138186, + "avg_response_len": 301.9295226597325, + "resp_tokens_sum": 130047084, + "pass_at_k": { + "pass@1": 0.41871517459138186, + "pass@2": 0.48204456774812554, + "pass@4": 0.5339043831909582, + "pass@8": 0.5756415085447332, + "pass@16": 0.6096503811949622, + "pass@32": 0.6382359297554154, + "pass@64": 0.663527388043035, + "pass@128": 0.687667161961367 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8784525495750708, + "pass@2": 0.9102096623987864, + "pass@4": 0.9281930339444113, + "pass@8": 0.9398208465107002, + "pass@16": 0.9490450774095365, + "pass@32": 0.9577149931620413, + "pass@64": 0.9651973316007761, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.798927436440678, + "pass@2": 0.8551790699096934, + "pass@4": 0.8885044120544257, + "pass@8": 0.9085335882562506, + "pass@16": 0.9217108196453194, + "pass@32": 0.9324307195802108, + "pass@64": 0.9433761539311727, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.6550989420062696, + "pass@2": 0.7359161719694916, + "pass@4": 0.7893547024294372, + "pass@8": 0.8251472079307166, + "pass@16": 0.8499261844339199, + "pass@32": 0.8690833324313215, + "pass@64": 0.887678042849582, + "pass@128": 0.9090909090909091 + }, + "14": { + "pass@1": 0.4941588785046729, + "pass@2": 0.5814315193661542, + "pass@4": 0.6522121046201, + "pass@8": 0.7054398474703508, + "pass@16": 0.7459692840506352, + "pass@32": 0.7797983464083315, + "pass@64": 0.8106658828186043, + "pass@128": 0.8411214953271028 + }, + "15": { + "pass@1": 0.36416184971098264, + "pass@2": 0.44411809885758474, + "pass@4": 0.5126349126879371, + "pass@8": 0.5702821524993034, + "pass@16": 0.6204606367913936, + "pass@32": 0.6645106758255857, + "pass@64": 0.7048887406631363, + "pass@128": 0.7456647398843931 + }, + "16": { + "pass@1": 0.26522177419354837, + "pass@2": 0.3349060198120396, + "pass@4": 0.39511467790316557, + "pass@8": 0.44689684761816845, + "pass@16": 0.4919076005163578, + "pass@32": 0.5289581705941678, + "pass@64": 0.557172877415548, + "pass@128": 0.5806451612903226 + }, + "17": { + "pass@1": 0.20822796242774566, + "pass@2": 0.2647282212917027, + "pass@4": 0.3119168785260225, + "pass@8": 0.353016882625172, + "pass@16": 0.39074588899191076, + "pass@32": 0.42409412095318333, + "pass@64": 0.450657781803306, + "pass@128": 0.47398843930635837 + }, + "18": { + "pass@1": 0.17683035714285714, + "pass@2": 0.23971140326209206, + "pass@4": 0.3019964647276231, + "pass@8": 0.35749287383033507, + "pass@16": 0.40207686154338784, + "pass@32": 0.437154461831571, + "pass@64": 0.467470346580905, + "pass@128": 0.4942857142857143 + }, + "19": { + "pass@1": 0.15927646396396397, + "pass@2": 0.21241775732425341, + "pass@4": 0.2633034674156721, + "pass@8": 0.3066705995032504, + "pass@16": 0.34242761359959684, + "pass@32": 0.3739619165348702, + "pass@64": 0.40473258265774553, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.15995683183183182, + "pass@2": 0.21900899423045087, + "pass@4": 0.2765225937410976, + "pass@8": 0.32656027758034484, + "pass@16": 0.3677726314607193, + "pass@32": 0.4017703052549921, + "pass@64": 0.4317523725638508, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 178989, + "answer_accuracy": 0.4158048059768064, + "avg_response_len": 313.66422279214987, + "resp_tokens_sum": 135021156, + "pass_at_k": { + "pass@1": 0.4158048059768064, + "pass@2": 0.47560848458561567, + "pass@4": 0.5256885308772188, + "pass@8": 0.5680850342220793, + "pass@16": 0.6042945537921133, + "pass@32": 0.635731773785242, + "pass@64": 0.6633729853941945, + "pass@128": 0.6862920011894142 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8986257530120482, + "pass@2": 0.9296826825016609, + "pass@4": 0.9489027942478575, + "pass@8": 0.9620662130010917, + "pass@16": 0.9704384000316348, + "pass@32": 0.9762796497655822, + "pass@64": 0.9813886069516573, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8237166927899686, + "pass@2": 0.8677596543084939, + "pass@4": 0.8935126335109363, + "pass@8": 0.9109319966161393, + "pass@16": 0.9238355493757084, + "pass@32": 0.9343567941155181, + "pass@64": 0.9447863035001147, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.5984768907563025, + "pass@2": 0.6788860721453939, + "pass@4": 0.7344253578071649, + "pass@8": 0.7775856673290666, + "pass@16": 0.8146780213187624, + "pass@32": 0.8475796206853868, + "pass@64": 0.8756093944707887, + "pass@128": 0.8935574229691877 + }, + "14": { + "pass@1": 0.48645376461988304, + "pass@2": 0.5669478404015286, + "pass@4": 0.6318697369517405, + "pass@8": 0.68492196122115, + "pass@16": 0.7292486257783408, + "pass@32": 0.7674189239076338, + "pass@64": 0.7990489164697606, + "pass@128": 0.8216374269005848 + }, + "15": { + "pass@1": 0.38359375, + "pass@2": 0.46019185456931494, + "pass@4": 0.5203419740146114, + "pass@8": 0.5682835301967295, + "pass@16": 0.6074261748472496, + "pass@32": 0.6400943942606238, + "pass@64": 0.6706869139430693, + "pass@128": 0.7 + }, + "16": { + "pass@1": 0.24243233618233617, + "pass@2": 0.3018344662045447, + "pass@4": 0.3575915131335078, + "pass@8": 0.4085050939332824, + "pass@16": 0.4546885384517169, + "pass@32": 0.49601365095000066, + "pass@64": 0.5296128337074691, + "pass@128": 0.5527065527065527 + }, + "17": { + "pass@1": 0.2296195652173913, + "pass@2": 0.29389167726316806, + "pass@4": 0.3527888585937627, + "pass@8": 0.4015310017732799, + "pass@16": 0.4411821281483737, + "pass@32": 0.47576190293681564, + "pass@64": 0.5078436970671439, + "pass@128": 0.5372670807453416 + }, + "18": { + "pass@1": 0.21167279411764706, + "pass@2": 0.26834254863362644, + "pass@4": 0.3236940603012858, + "pass@8": 0.3749530206752505, + "pass@16": 0.4188367625939018, + "pass@32": 0.45438906059280315, + "pass@64": 0.4836671899265589, + "pass@128": 0.5088235294117647 + }, + "19": { + "pass@1": 0.14625186011904762, + "pass@2": 0.1951546828130859, + "pass@4": 0.24635013452559498, + "pass@8": 0.29440603480328803, + "pass@16": 0.3372258333217718, + "pass@32": 0.37594895700766007, + "pass@64": 0.411830564552925, + "pass@128": 0.44642857142857145 + }, + "20": { + "pass@1": 0.1511742140718563, + "pass@2": 0.20507213918619455, + "pass@4": 0.25712532359353274, + "pass@8": 0.3054460965504392, + "pass@16": 0.351170875678362, + "pass@32": 0.39341225491743276, + "pass@64": 0.4319755908517848, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 177115, + "answer_accuracy": 0.4228945408007335, + "avg_response_len": 338.9009278537592, + "resp_tokens_sum": 141937131, + "pass_at_k": { + "pass@1": 0.4228945408007335, + "pass@2": 0.4834929484001711, + "pass@4": 0.533290868516131, + "pass@8": 0.5736713094948613, + "pass@16": 0.6070987886873918, + "pass@32": 0.6358791722896623, + "pass@64": 0.6613329874325706, + "pass@128": 0.684596577017115 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.8884672619047619, + "pass@2": 0.9179161198600174, + "pass@4": 0.9360529898048454, + "pass@8": 0.9490142660043712, + "pass@16": 0.958196320440275, + "pass@32": 0.9647608513937915, + "pass@64": 0.9700160392867752, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8331660932721713, + "pass@2": 0.8894891220592839, + "pass@4": 0.9224128669696099, + "pass@8": 0.9411316213230453, + "pass@16": 0.9521785605790735, + "pass@32": 0.9594392629935055, + "pass@64": 0.9642912699886997, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.5963059413580247, + "pass@2": 0.6718958849761836, + "pass@4": 0.7258452271012411, + "pass@8": 0.7661379023691128, + "pass@16": 0.7994569530613119, + "pass@32": 0.8298630341960512, + "pass@64": 0.8566462178607112, + "pass@128": 0.8765432098765432 + }, + "14": { + "pass@1": 0.5224406528189911, + "pass@2": 0.6048735805743123, + "pass@4": 0.667954188960801, + "pass@8": 0.714907466749094, + "pass@16": 0.7518824252298536, + "pass@32": 0.7844183856251077, + "pass@64": 0.814024580720177, + "pass@128": 0.8427299703264095 + }, + "15": { + "pass@1": 0.39817226080246915, + "pass@2": 0.46172277510450077, + "pass@4": 0.516345238673869, + "pass@8": 0.5643374864301807, + "pass@16": 0.6066126531536483, + "pass@32": 0.6453878305545641, + "pass@64": 0.6818109357935861, + "pass@128": 0.7160493827160493 + }, + "16": { + "pass@1": 0.2555309734513274, + "pass@2": 0.31953203028824917, + "pass@4": 0.378190404296808, + "pass@8": 0.4293594132098313, + "pass@16": 0.47360209270500697, + "pass@32": 0.5121977074655605, + "pass@64": 0.5497008456356067, + "pass@128": 0.5870206489675516 + }, + "17": { + "pass@1": 0.21279649849397592, + "pass@2": 0.2769127691869842, + "pass@4": 0.3335075267323512, + "pass@8": 0.3784226935397351, + "pass@16": 0.41479042093392043, + "pass@32": 0.44466458523149915, + "pass@64": 0.4693555810200645, + "pass@128": 0.4939759036144578 + }, + "18": { + "pass@1": 0.21242439516129033, + "pass@2": 0.2739847917195834, + "pass@4": 0.33244598044196094, + "pass@8": 0.38322910594677906, + "pass@16": 0.4248379459806415, + "pass@32": 0.458828979622347, + "pass@64": 0.48675039253010505, + "pass@128": 0.5096774193548387 + }, + "19": { + "pass@1": 0.17817692598187312, + "pass@2": 0.23166459012298704, + "pass@4": 0.27631746239424, + "pass@8": 0.31228890701309736, + "pass@16": 0.34476007943757514, + "pass@32": 0.3760017219184328, + "pass@64": 0.40610132605649063, + "pass@128": 0.4350453172205438 + }, + "20": { + "pass@1": 0.15322353603603603, + "pass@2": 0.2071318365609704, + "pass@4": 0.2625266365915972, + "pass@8": 0.3153373034222012, + "pass@16": 0.36097657025856844, + "pass@32": 0.39841696589607256, + "pass@64": 0.42861872538402224, + "pass@128": 0.45645645645645644 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.6846722861842105, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.6846722861842105, + "pass@2": 0.7199287712391035, + "pass@4": 0.7478426692617197, + "pass@8": 0.7705249609853274, + "pass@16": 0.7893102034802012, + "pass@32": 0.8053345681149181, + "pass@64": 0.8194470196724575, + "pass@128": 0.8321052631578948 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.916296875, + "pass@2": 0.9394058809055122, + "pass@4": 0.9528618595800525, + "pass@8": 0.9615409457400279, + "pass@16": 0.9674628728972483, + "pass@32": 0.9717329785336175, + "pass@64": 0.9754449907658412, + "pass@128": 0.979 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9981171875, + "pass@2": 0.9985055364173229, + "pass@4": 0.9987710755530559, + "pass@8": 0.998950601439429, + "pass@16": 0.9989981090802741, + "pass@32": 0.998999998952485, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.99703125, + "pass@2": 0.9971211860236221, + "pass@4": 0.9972299803149606, + "pass@8": 0.9974122820506877, + "pass@16": 0.997667419611923, + "pass@32": 0.9979072527311064, + "pass@64": 0.9979969041357232, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.9782734375, + "pass@2": 0.9810522883858267, + "pass@4": 0.9831453160854895, + "pass@8": 0.984949122876631, + "pass@16": 0.9863992255520356, + "pass@32": 0.9876978908604928, + "pass@64": 0.9891785804477966, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.9946640625, + "pass@2": 0.9978164370078745, + "pass@4": 0.9983948684851892, + "pass@8": 0.9987443749724515, + "pass@16": 0.9989454139612846, + "pass@32": 0.9989981933094014, + "pass@64": 0.9989999998360333, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.9909453125, + "pass@2": 0.9939223671259843, + "pass@4": 0.9948598255530557, + "pass@8": 0.9954570343475408, + "pass@16": 0.9958527932799116, + "pass@32": 0.9959896697616626, + "pass@64": 0.9959999879263536, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.9757578125, + "pass@2": 0.9845514271653546, + "pass@4": 0.9871017754030745, + "pass@8": 0.9888908328480559, + "pass@16": 0.9905008788416947, + "pass@32": 0.9920584406618639, + "pass@64": 0.9933081767467343, + "pass@128": 0.994 + }, + "9": { + "pass@1": 0.9666484375, + "pass@2": 0.9827312992126, + "pass@4": 0.9872395144356954, + "pass@8": 0.9894728934371575, + "pass@16": 0.9909237991901977, + "pass@32": 0.9917458587740582, + "pass@64": 0.991990188393302, + "pass@128": 0.992 + }, + "11": { + "pass@1": 0.8883046875, + "pass@2": 0.9191022391732293, + "pass@4": 0.9375445604611923, + "pass@8": 0.9501022353260171, + "pass@16": 0.9590303020747554, + "pass@32": 0.9660979044974181, + "pass@64": 0.9720907279383584, + "pass@128": 0.977 + }, + "12": { + "pass@1": 0.81803125, + "pass@2": 0.8704116633858281, + "pass@4": 0.9011900994563182, + "pass@8": 0.9199582373358968, + "pass@16": 0.9323515597146504, + "pass@32": 0.9418769310531213, + "pass@64": 0.9506652345944766, + "pass@128": 0.959 + }, + "13": { + "pass@1": 0.6158359375, + "pass@2": 0.6948138533464573, + "pass@4": 0.74916785639295, + "pass@8": 0.7890487229339681, + "pass@16": 0.820990559237083, + "pass@32": 0.8486991307097956, + "pass@64": 0.8733152240819593, + "pass@128": 0.893 + }, + "14": { + "pass@1": 0.5010546875, + "pass@2": 0.5843780757874018, + "pass@4": 0.6505600973003363, + "pass@8": 0.7016133180700607, + "pass@16": 0.7422435474989071, + "pass@32": 0.7771215371291468, + "pass@64": 0.8078247615201297, + "pass@128": 0.835 + }, + "15": { + "pass@1": 0.38159375, + "pass@2": 0.4551263533464563, + "pass@4": 0.5163803885451813, + "pass@8": 0.5676965353330589, + "pass@16": 0.611672517651196, + "pass@32": 0.650257501041337, + "pass@64": 0.6861249290677806, + "pass@128": 0.721 + }, + "16": { + "pass@1": 0.2539375, + "pass@2": 0.31808612204724435, + "pass@4": 0.37620671831646035, + "pass@8": 0.4274761518103465, + "pass@16": 0.4726381425836207, + "pass@32": 0.5117128471984673, + "pass@64": 0.5449662833006126, + "pass@128": 0.573 + }, + "17": { + "pass@1": 0.2166328125, + "pass@2": 0.27816412401574825, + "pass@4": 0.33224575131233564, + "pass@8": 0.3770731582144976, + "pass@16": 0.41496914260503864, + "pass@32": 0.4475605408923135, + "pass@64": 0.47527931585822586, + "pass@128": 0.501 + }, + "18": { + "pass@1": 0.1997109375, + "pass@2": 0.26007074311023637, + "pass@4": 0.3188129970941129, + "pass@8": 0.37140755571370326, + "pass@16": 0.41483116407611115, + "pass@32": 0.44973332592553017, + "pass@64": 0.47895408756267954, + "pass@128": 0.504 + }, + "19": { + "pass@1": 0.16115625, + "pass@2": 0.212988065944882, + "pass@4": 0.26191477990251183, + "pass@8": 0.3044093655498218, + "pass@16": 0.34145186161861824, + "pass@32": 0.375304737715687, + "pass@64": 0.40757055863951047, + "pass@128": 0.439 + }, + "20": { + "pass@1": 0.15478125, + "pass@2": 0.2103989911417322, + "pass@4": 0.26538325178102734, + "pass@8": 0.3157708907216943, + "pass@16": 0.3599645566490957, + "pass@32": 0.39786205443572714, + "pass@64": 0.4307834229611375, + "pass@128": 0.46 + } + }, + "per_op_accuracy": { + "10": 0.916296875, + "2": 1.0, + "3": 0.9981171875, + "4": 0.99703125, + "5": 0.9782734375, + "6": 0.9946640625, + "7": 0.9909453125, + "8": 0.9757578125, + "9": 0.9666484375, + "11": 0.8883046875, + "12": 0.81803125, + "13": 0.6158359375, + "14": 0.5010546875, + "15": 0.38159375, + "16": 0.2539375, + "17": 0.2166328125, + "18": 0.1997109375, + "19": 0.16115625, + "20": 0.15478125 + }, + "per_op_avg_response_len": { + "10": 274.7838515625, + "2": 114.360515625, + "3": 142.7705703125, + "4": 157.8014921875, + "5": 170.854, + "6": 195.624046875, + "7": 219.9284765625, + "8": 238.8914453125, + "9": 254.3062109375, + "11": 291.286734375, + "12": 300.4761953125, + "13": 312.851765625, + "14": 324.2943515625, + "15": 328.16921875, + "16": 318.110484375, + "17": 326.76978125, + "18": 330.2153671875, + "19": 320.9465078125, + "20": 326.6090546875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 274.7838515625, + "2": 114.360515625, + "3": 142.7705703125, + "4": 157.8014921875, + "5": 170.854, + "6": 195.624046875, + "7": 219.9284765625, + "8": 238.8914453125, + "9": 254.3062109375, + "11": 291.286734375, + "12": 300.4761953125, + "13": 312.851765625, + "14": 324.2943515625, + "15": 328.16921875, + "16": 318.110484375, + "17": 326.76978125, + "18": 330.2153671875, + "19": 320.9465078125, + "20": 326.6090546875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 551351, + "answer_accuracy": 0.6809088978027189, + "avg_response_len": 248.23700551296238, + "resp_tokens_sum": 201004454, + "pass_at_k": { + "pass@1": 0.6809088978027189, + "pass@2": 0.7179658930087301, + "pass@4": 0.7469747311907298, + "pass@8": 0.7700900354015988, + "pass@16": 0.7887883821084508, + "pass@32": 0.8044023259925125, + "pass@64": 0.8181816148916696, + "pass@128": 0.8313310148593108 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9221515486725663, + "pass@2": 0.9428836259958658, + "pass@4": 0.953557931641288, + "pass@8": 0.9603148730351716, + "pass@16": 0.9648846877139904, + "pass@32": 0.9680322261071029, + "pass@64": 0.9708755602536454, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9978736620795107, + "pass@2": 0.9985266319921019, + "pass@4": 0.9993000800358671, + "pass@8": 0.9998489340659815, + "pass@16": 0.9999942173708687, + "pass@32": 0.9999999967965904, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9907470703125, + "pass@2": 0.9910037063238188, + "pass@4": 0.991343688484252, + "pass@8": 0.9919133814083988, + "pass@16": 0.9927106862872591, + "pass@32": 0.9934601647847077, + "pass@64": 0.9937403254241349, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9669642857142857, + "pass@2": 0.9718843738282714, + "pass@4": 0.9763600398164514, + "pass@8": 0.9802523092172719, + "pass@16": 0.9830724326754721, + "pass@32": 0.9851478752003268, + "pass@64": 0.987257685923108, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9989277105238344, + "pass@4": 0.9996942885881777, + "pass@8": 0.9999718501697081, + "pass@16": 0.9999998309063756, + "pass@32": 0.999999999998971, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9926960059171598, + "pass@2": 0.9966541257047011, + "pass@4": 0.9984301580460726, + "pass@8": 0.9994525764888945, + "pass@16": 0.9999158567707788, + "pass@32": 0.9999987203425054, + "pass@64": 0.9999999999877117, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9768880208333334, + "pass@2": 0.9874886432465175, + "pass@4": 0.9895625994467037, + "pass@8": 0.9907045168965894, + "pass@16": 0.9916775319238652, + "pass@32": 0.9925913730797592, + "pass@64": 0.9933988491820491, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9691681338028169, + "pass@2": 0.9880122269047353, + "pass@4": 0.9939504940403571, + "pass@8": 0.9970743852484422, + "pass@16": 0.9988665501562034, + "pass@32": 0.9997334583192324, + "pass@64": 0.9999912787935963, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8784525495750708, + "pass@2": 0.9102096623987864, + "pass@4": 0.9281930339444113, + "pass@8": 0.9398208465107002, + "pass@16": 0.9490450774095365, + "pass@32": 0.9577149931620413, + "pass@64": 0.9651973316007761, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.798927436440678, + "pass@2": 0.8551790699096934, + "pass@4": 0.8885044120544257, + "pass@8": 0.9085335882562506, + "pass@16": 0.9217108196453194, + "pass@32": 0.9324307195802108, + "pass@64": 0.9433761539311727, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.6550989420062696, + "pass@2": 0.7359161719694916, + "pass@4": 0.7893547024294372, + "pass@8": 0.8251472079307166, + "pass@16": 0.8499261844339199, + "pass@32": 0.8690833324313215, + "pass@64": 0.887678042849582, + "pass@128": 0.9090909090909091 + }, + "14": { + "pass@1": 0.4941588785046729, + "pass@2": 0.5814315193661542, + "pass@4": 0.6522121046201, + "pass@8": 0.7054398474703508, + "pass@16": 0.7459692840506352, + "pass@32": 0.7797983464083315, + "pass@64": 0.8106658828186043, + "pass@128": 0.8411214953271028 + }, + "15": { + "pass@1": 0.36416184971098264, + "pass@2": 0.44411809885758474, + "pass@4": 0.5126349126879371, + "pass@8": 0.5702821524993034, + "pass@16": 0.6204606367913936, + "pass@32": 0.6645106758255857, + "pass@64": 0.7048887406631363, + "pass@128": 0.7456647398843931 + }, + "16": { + "pass@1": 0.26522177419354837, + "pass@2": 0.3349060198120396, + "pass@4": 0.39511467790316557, + "pass@8": 0.44689684761816845, + "pass@16": 0.4919076005163578, + "pass@32": 0.5289581705941678, + "pass@64": 0.557172877415548, + "pass@128": 0.5806451612903226 + }, + "17": { + "pass@1": 0.20822796242774566, + "pass@2": 0.2647282212917027, + "pass@4": 0.3119168785260225, + "pass@8": 0.353016882625172, + "pass@16": 0.39074588899191076, + "pass@32": 0.42409412095318333, + "pass@64": 0.450657781803306, + "pass@128": 0.47398843930635837 + }, + "18": { + "pass@1": 0.17683035714285714, + "pass@2": 0.23971140326209206, + "pass@4": 0.3019964647276231, + "pass@8": 0.35749287383033507, + "pass@16": 0.40207686154338784, + "pass@32": 0.437154461831571, + "pass@64": 0.467470346580905, + "pass@128": 0.4942857142857143 + }, + "19": { + "pass@1": 0.15927646396396397, + "pass@2": 0.21241775732425341, + "pass@4": 0.2633034674156721, + "pass@8": 0.3066705995032504, + "pass@16": 0.34242761359959684, + "pass@32": 0.3739619165348702, + "pass@64": 0.40473258265774553, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.15995683183183182, + "pass@2": 0.21900899423045087, + "pass@4": 0.2765225937410976, + "pass@8": 0.32656027758034484, + "pass@16": 0.3677726314607193, + "pass@32": 0.4017703052549921, + "pass@64": 0.4317523725638508, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 561524, + "answer_accuracy": 0.6838513250194855, + "avg_response_len": 257.001773187841, + "resp_tokens_sum": 211029296, + "pass_at_k": { + "pass@1": 0.6838513250194855, + "pass@2": 0.7182096587108185, + "pass@4": 0.7457970707480713, + "pass@8": 0.7688692845996465, + "pass@16": 0.7884647649459229, + "pass@32": 0.8053975893041211, + "pass@64": 0.8202428182245672, + "pass@128": 0.8325798908807482 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.902032208588957, + "pass@2": 0.928237539853147, + "pass@4": 0.9445890421672748, + "pass@8": 0.9554783664156684, + "pass@16": 0.9637377662720875, + "pass@32": 0.9702591907243098, + "pass@64": 0.975278890857173, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966619318181819, + "pass@2": 0.9969518014793606, + "pass@4": 0.9969696373180625, + "pass@8": 0.9969696969696016, + "pass@16": 0.996969696969697, + "pass@32": 0.996969696969697, + "pass@64": 0.996969696969697, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875226449275363, + "pass@2": 0.9883676395070182, + "pass@4": 0.9884954541008462, + "pass@8": 0.9885869564329156, + "pass@16": 0.988768115942029, + "pass@32": 0.9891304347826086, + "pass@64": 0.9898550724637681, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9941350286532952, + "pass@2": 0.9977561734314014, + "pass@4": 0.9985594472610693, + "pass@8": 0.9992944900423253, + "pass@16": 0.9998437548382668, + "pass@32": 0.9999948232370918, + "pass@64": 0.9999999995301811, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928648843930635, + "pass@2": 0.994768672340813, + "pass@4": 0.9953643609758319, + "pass@8": 0.9960753760142124, + "pass@16": 0.9967665713625197, + "pass@32": 0.9970812204794673, + "pass@64": 0.9971097917066677, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9756382042253521, + "pass@2": 0.9838094016857049, + "pass@4": 0.9872804353681142, + "pass@8": 0.9894908076168281, + "pass@16": 0.9907958224370905, + "pass@32": 0.991898265474723, + "pass@64": 0.9929392340213707, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.9612649700598802, + "pass@2": 0.9778384913244379, + "pass@4": 0.9821942192667527, + "pass@8": 0.983871105890422, + "pass@16": 0.9847190114530727, + "pass@32": 0.985008695002592, + "pass@64": 0.985029924450078, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.8986257530120482, + "pass@2": 0.9296826825016609, + "pass@4": 0.9489027942478575, + "pass@8": 0.9620662130010917, + "pass@16": 0.9704384000316348, + "pass@32": 0.9762796497655822, + "pass@64": 0.9813886069516573, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8237166927899686, + "pass@2": 0.8677596543084939, + "pass@4": 0.8935126335109363, + "pass@8": 0.9109319966161393, + "pass@16": 0.9238355493757084, + "pass@32": 0.9343567941155181, + "pass@64": 0.9447863035001147, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.5984768907563025, + "pass@2": 0.6788860721453939, + "pass@4": 0.7344253578071649, + "pass@8": 0.7775856673290666, + "pass@16": 0.8146780213187624, + "pass@32": 0.8475796206853868, + "pass@64": 0.8756093944707887, + "pass@128": 0.8935574229691877 + }, + "14": { + "pass@1": 0.48645376461988304, + "pass@2": 0.5669478404015286, + "pass@4": 0.6318697369517405, + "pass@8": 0.68492196122115, + "pass@16": 0.7292486257783408, + "pass@32": 0.7674189239076338, + "pass@64": 0.7990489164697606, + "pass@128": 0.8216374269005848 + }, + "15": { + "pass@1": 0.38359375, + "pass@2": 0.46019185456931494, + "pass@4": 0.5203419740146114, + "pass@8": 0.5682835301967295, + "pass@16": 0.6074261748472496, + "pass@32": 0.6400943942606238, + "pass@64": 0.6706869139430693, + "pass@128": 0.7 + }, + "16": { + "pass@1": 0.24243233618233617, + "pass@2": 0.3018344662045447, + "pass@4": 0.3575915131335078, + "pass@8": 0.4085050939332824, + "pass@16": 0.4546885384517169, + "pass@32": 0.49601365095000066, + "pass@64": 0.5296128337074691, + "pass@128": 0.5527065527065527 + }, + "17": { + "pass@1": 0.2296195652173913, + "pass@2": 0.29389167726316806, + "pass@4": 0.3527888585937627, + "pass@8": 0.4015310017732799, + "pass@16": 0.4411821281483737, + "pass@32": 0.47576190293681564, + "pass@64": 0.5078436970671439, + "pass@128": 0.5372670807453416 + }, + "18": { + "pass@1": 0.21167279411764706, + "pass@2": 0.26834254863362644, + "pass@4": 0.3236940603012858, + "pass@8": 0.3749530206752505, + "pass@16": 0.4188367625939018, + "pass@32": 0.45438906059280315, + "pass@64": 0.4836671899265589, + "pass@128": 0.5088235294117647 + }, + "19": { + "pass@1": 0.14625186011904762, + "pass@2": 0.1951546828130859, + "pass@4": 0.24635013452559498, + "pass@8": 0.29440603480328803, + "pass@16": 0.3372258333217718, + "pass@32": 0.37594895700766007, + "pass@64": 0.411830564552925, + "pass@128": 0.44642857142857145 + }, + "20": { + "pass@1": 0.1511742140718563, + "pass@2": 0.20507213918619455, + "pass@4": 0.25712532359353274, + "pass@8": 0.3054460965504392, + "pass@16": 0.351170875678362, + "pass@32": 0.39341225491743276, + "pass@64": 0.4319755908517848, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 552248, + "answer_accuracy": 0.6893173829685253, + "avg_response_len": 276.4077965230069, + "resp_tokens_sum": 221444659, + "pass_at_k": { + "pass@1": 0.6893173829685253, + "pass@2": 0.7236746211439762, + "pass@4": 0.7508164814845082, + "pass@8": 0.7726614849119308, + "pass@16": 0.790704121070079, + "pass@32": 0.8062121975664928, + "pass@64": 0.8199103369645999, + "pass@128": 0.8324013420674229 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9242537313432836, + "pass@2": 0.9467549065695148, + "pass@4": 0.960208038827236, + "pass@8": 0.9686813621778994, + "pass@16": 0.9736968715149406, + "pass@32": 0.9769121125587591, + "pass@64": 0.9802306191654232, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997494533527697, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9793658088235294, + "pass@2": 0.982123161764706, + "pass@4": 0.9840028584662212, + "pass@8": 0.9856092514818663, + "pass@16": 0.9870776742919471, + "pass@32": 0.9886067946246756, + "pass@64": 0.9902717923000518, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9942576892744479, + "pass@2": 0.9967119153481209, + "pass@4": 0.9968445709861974, + "pass@8": 0.9968454258391087, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9869709256329114, + "pass@2": 0.9900737721768167, + "pass@4": 0.9904884912526759, + "pass@8": 0.9905062765613193, + "pass@16": 0.9905063291137864, + "pass@32": 0.990506329113924, + "pass@64": 0.990506329113924, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.9748263888888888, + "pass@2": 0.9825904891111588, + "pass@4": 0.9846056751352027, + "pass@8": 0.9865519125295662, + "pass@16": 0.9890839997485931, + "pass@32": 0.9917298739262829, + "pass@64": 0.9936165397127583, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9695538585209004, + "pass@2": 0.9819578860419779, + "pass@4": 0.9849974913706525, + "pass@8": 0.9868120170628933, + "pass@16": 0.9885209776830189, + "pass@32": 0.9898635914465111, + "pass@64": 0.9903321210779717, + "pass@128": 0.9903536977491961 + }, + "11": { + "pass@1": 0.8884672619047619, + "pass@2": 0.9179161198600174, + "pass@4": 0.9360529898048454, + "pass@8": 0.9490142660043712, + "pass@16": 0.958196320440275, + "pass@32": 0.9647608513937915, + "pass@64": 0.9700160392867752, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8331660932721713, + "pass@2": 0.8894891220592839, + "pass@4": 0.9224128669696099, + "pass@8": 0.9411316213230453, + "pass@16": 0.9521785605790735, + "pass@32": 0.9594392629935055, + "pass@64": 0.9642912699886997, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.5963059413580247, + "pass@2": 0.6718958849761836, + "pass@4": 0.7258452271012411, + "pass@8": 0.7661379023691128, + "pass@16": 0.7994569530613119, + "pass@32": 0.8298630341960512, + "pass@64": 0.8566462178607112, + "pass@128": 0.8765432098765432 + }, + "14": { + "pass@1": 0.5224406528189911, + "pass@2": 0.6048735805743123, + "pass@4": 0.667954188960801, + "pass@8": 0.714907466749094, + "pass@16": 0.7518824252298536, + "pass@32": 0.7844183856251077, + "pass@64": 0.814024580720177, + "pass@128": 0.8427299703264095 + }, + "15": { + "pass@1": 0.39817226080246915, + "pass@2": 0.46172277510450077, + "pass@4": 0.516345238673869, + "pass@8": 0.5643374864301807, + "pass@16": 0.6066126531536483, + "pass@32": 0.6453878305545641, + "pass@64": 0.6818109357935861, + "pass@128": 0.7160493827160493 + }, + "16": { + "pass@1": 0.2555309734513274, + "pass@2": 0.31953203028824917, + "pass@4": 0.378190404296808, + "pass@8": 0.4293594132098313, + "pass@16": 0.47360209270500697, + "pass@32": 0.5121977074655605, + "pass@64": 0.5497008456356067, + "pass@128": 0.5870206489675516 + }, + "17": { + "pass@1": 0.21279649849397592, + "pass@2": 0.2769127691869842, + "pass@4": 0.3335075267323512, + "pass@8": 0.3784226935397351, + "pass@16": 0.41479042093392043, + "pass@32": 0.44466458523149915, + "pass@64": 0.4693555810200645, + "pass@128": 0.4939759036144578 + }, + "18": { + "pass@1": 0.21242439516129033, + "pass@2": 0.2739847917195834, + "pass@4": 0.33244598044196094, + "pass@8": 0.38322910594677906, + "pass@16": 0.4248379459806415, + "pass@32": 0.458828979622347, + "pass@64": 0.48675039253010505, + "pass@128": 0.5096774193548387 + }, + "19": { + "pass@1": 0.17817692598187312, + "pass@2": 0.23166459012298704, + "pass@4": 0.27631746239424, + "pass@8": 0.31228890701309736, + "pass@16": 0.34476007943757514, + "pass@32": 0.3760017219184328, + "pass@64": 0.40610132605649063, + "pass@128": 0.4350453172205438 + }, + "20": { + "pass@1": 0.15322353603603603, + "pass@2": 0.2071318365609704, + "pass@4": 0.2625266365915972, + "pass@8": 0.3153373034222012, + "pass@16": 0.36097657025856844, + "pass@32": 0.39841696589607256, + "pass@64": 0.42861872538402224, + "pass@128": 0.45645645645645644 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..39d0628d52a51d9a155d92b8e975137ea2643d2e --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8", "results_dir": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs", "per_file": [{"file": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/checkpoint-21546_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1011011, "value_mismatch": 25823, "dependency_mismatch": 134717, "answer_mismatch": 23330, "extra_nodes": 18355, "missing_nodes": 16084, "perfect_with_extra": 1056, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 332677, "dependency_mismatch": 44197, "value_mismatch": 8793, "answer_mismatch": 8006, "extra_nodes": 6460, "missing_nodes": 5606, "perfect_with_extra": 379}, "teachers_in_school": {"total": 390656, "perfect": 342922, "value_mismatch": 8768, "answer_mismatch": 8121, "dependency_mismatch": 45559, "extra_nodes": 6044, "missing_nodes": 5259, "perfect_with_extra": 209}, "movie_festival_awards": {"total": 382336, "perfect": 335412, "dependency_mismatch": 44961, "value_mismatch": 8262, "answer_mismatch": 7203, "extra_nodes": 5851, "missing_nodes": 5219, "perfect_with_extra": 468}}, "results_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/checkpoint-21546_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/checkpoint-21546_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 217622, "value_mismatch": 819433, "dependency_mismatch": 967607, "answer_mismatch": 743546, "extra_nodes": 63420, "missing_nodes": 791203, "perfect_with_extra": 101, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 276950, "dependency_mismatch": 325782, "answer_mismatch": 250370, "perfect": 72990, "extra_nodes": 22005, "missing_nodes": 274239}, "movie_festival_awards": {"total": 418816, "perfect": 72366, "value_mismatch": 267866, "answer_mismatch": 241703, "dependency_mismatch": 318714, "missing_nodes": 251621, "extra_nodes": 20554, "perfect_with_extra": 2}, "teachers_in_school": {"total": 430464, "perfect": 72266, "value_mismatch": 274617, "dependency_mismatch": 323111, "answer_mismatch": 251473, "extra_nodes": 20861, "missing_nodes": 265343, "perfect_with_extra": 99}}, "results_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/checkpoint-21546_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1228633, "value_mismatch": 845256, "dependency_mismatch": 1102324, "answer_mismatch": 766876, "extra_nodes": 81775, "missing_nodes": 807287, "perfect_with_extra": 1157, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/block8_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 405667, "dependency_mismatch": 369979, "value_mismatch": 285743, "answer_mismatch": 258376, "extra_nodes": 28465, "missing_nodes": 279845, "perfect_with_extra": 379}, "teachers_in_school": {"total": 821120, "perfect": 415188, "value_mismatch": 283385, "answer_mismatch": 259594, "dependency_mismatch": 368670, "extra_nodes": 26905, "missing_nodes": 270602, "perfect_with_extra": 308}, "movie_festival_awards": {"total": 801152, "perfect": 407778, "dependency_mismatch": 363675, "value_mismatch": 276128, "answer_mismatch": 248906, "extra_nodes": 26405, "missing_nodes": 256840, "perfect_with_extra": 470}}}} diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/results.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..bfb35a1dbe2c005278d90c7caaad670c434c92c7 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-21546_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9969418960244648, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9875, + "pass@2": 0.990625, + "pass@4": 0.990625, + "pass@8": 0.990625, + "pass@16": 0.990625, + "pass@32": 0.990625, + "pass@64": 0.99375, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9238095238095239, + "pass@2": 0.9587301587301588, + "pass@4": 0.9650793650793651, + "pass@8": 0.9746031746031746, + "pass@16": 0.9841269841269841, + "pass@32": 0.9841269841269841, + "pass@64": 0.9841269841269841, + "pass@128": 0.9873015873015873 + }, + "6": { + "pass@1": 0.9491017964071856, + "pass@2": 0.9730538922155688, + "pass@4": 0.9850299401197605, + "pass@8": 0.9880239520958084, + "pass@16": 0.9970059880239521, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8816568047337278, + "pass@2": 0.9319526627218935, + "pass@4": 0.9674556213017751, + "pass@8": 0.9881656804733728, + "pass@16": 0.9970414201183432, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8044871794871795, + "pass@2": 0.8910256410256411, + "pass@4": 0.9455128205128205, + "pass@8": 0.9807692307692307, + "pass@16": 0.9903846153846154, + "pass@32": 0.9935897435897436, + "pass@64": 0.9935897435897436, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.7549295774647887, + "pass@2": 0.8366197183098592, + "pass@4": 0.9323943661971831, + "pass@8": 0.9577464788732394, + "pass@16": 0.9802816901408451, + "pass@32": 0.9943661971830986, + "pass@64": 0.9971830985915493, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6076696165191741, + "pass@2": 0.7286135693215339, + "pass@4": 0.7846607669616519, + "pass@8": 0.8495575221238938, + "pass@16": 0.8908554572271387, + "pass@32": 0.9144542772861357, + "pass@64": 0.9203539823008849, + "pass@128": 0.9321533923303835 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.9969418960244648, + "4": 0.9875, + "5": 0.9238095238095239, + "6": 0.9491017964071856, + "7": 0.8816568047337278, + "8": 0.8044871794871795, + "9": 0.7549295774647887, + "10": 0.6076696165191741 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9587301587301588, + "6": 0.9730538922155688, + "7": 0.9319526627218935, + "8": 0.8910256410256411, + "9": 0.8366197183098592, + "10": 0.7286135693215339 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9650793650793651, + "6": 0.9850299401197605, + "7": 0.9674556213017751, + "8": 0.9455128205128205, + "9": 0.9323943661971831, + "10": 0.7846607669616519 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9746031746031746, + "6": 0.9880239520958084, + "7": 0.9881656804733728, + "8": 0.9807692307692307, + "9": 0.9577464788732394, + "10": 0.8495575221238938 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9841269841269841, + "6": 0.9970059880239521, + "7": 0.9970414201183432, + "8": 0.9903846153846154, + "9": 0.9802816901408451, + "10": 0.8908554572271387 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 0.990625, + "5": 0.9841269841269841, + "6": 1.0, + "7": 1.0, + "8": 0.9935897435897436, + "9": 0.9943661971830986, + "10": 0.9144542772861357 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9841269841269841, + "6": 1.0, + "7": 1.0, + "8": 0.9935897435897436, + "9": 0.9971830985915493, + "10": 0.9203539823008849 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 1.0, + "7": 1.0, + "8": 0.9935897435897436, + "9": 1.0, + "10": 0.9321533923303835 + } + }, + "total_pass_at_k": { + "pass@1": 0.8760553866936845, + "pass@2": 0.9216480918608578, + "pass@4": 0.9513677811550152, + "pass@8": 0.9692671394799054, + "pass@16": 0.9807497467071935, + "pass@32": 0.9861533265788585, + "pass@64": 0.9875042215467748, + "pass@128": 0.9895305639986491 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9912280701754386, + "pass@2": 0.9941520467836257, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9294117647058824, + "pass@2": 0.9558823529411765, + "pass@4": 0.9794117647058823, + "pass@8": 0.9823529411764705, + "pass@16": 0.9823529411764705, + "pass@32": 0.9823529411764705, + "pass@64": 0.9852941176470589, + "pass@128": 0.9852941176470589 + }, + "6": { + "pass@1": 0.9621451104100947, + "pass@2": 0.9779179810725552, + "pass@4": 0.9842271293375394, + "pass@8": 0.9936908517350158, + "pass@16": 0.9936908517350158, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.8670886075949367, + "pass@2": 0.9240506329113924, + "pass@4": 0.9430379746835443, + "pass@8": 0.9620253164556962, + "pass@16": 0.9810126582278481, + "pass@32": 0.990506329113924, + "pass@64": 0.990506329113924, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.7867867867867868, + "pass@2": 0.8558558558558559, + "pass@4": 0.9069069069069069, + "pass@8": 0.963963963963964, + "pass@16": 0.984984984984985, + "pass@32": 0.990990990990991, + "pass@64": 0.990990990990991, + "pass@128": 0.990990990990991 + }, + "9": { + "pass@1": 0.7556270096463023, + "pass@2": 0.797427652733119, + "pass@4": 0.864951768488746, + "pass@8": 0.9260450160771704, + "pass@16": 0.954983922829582, + "pass@32": 0.977491961414791, + "pass@64": 0.9871382636655949, + "pass@128": 0.9871382636655949 + }, + "10": { + "pass@1": 0.6238805970149254, + "pass@2": 0.7014925373134329, + "pass@4": 0.7611940298507462, + "pass@8": 0.8238805970149253, + "pass@16": 0.8776119402985074, + "pass@32": 0.8985074626865671, + "pass@64": 0.9164179104477612, + "pass@128": 0.9253731343283582 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9912280701754386, + "5": 0.9294117647058824, + "6": 0.9621451104100947, + "7": 0.8670886075949367, + "8": 0.7867867867867868, + "9": 0.7556270096463023, + "10": 0.6238805970149254 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9941520467836257, + "5": 0.9558823529411765, + "6": 0.9779179810725552, + "7": 0.9240506329113924, + "8": 0.8558558558558559, + "9": 0.797427652733119, + "10": 0.7014925373134329 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9794117647058823, + "6": 0.9842271293375394, + "7": 0.9430379746835443, + "8": 0.9069069069069069, + "9": 0.864951768488746, + "10": 0.7611940298507462 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9823529411764705, + "6": 0.9936908517350158, + "7": 0.9620253164556962, + "8": 0.963963963963964, + "9": 0.9260450160771704, + "10": 0.8238805970149253 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9823529411764705, + "6": 0.9936908517350158, + "7": 0.9810126582278481, + "8": 0.984984984984985, + "9": 0.954983922829582, + "10": 0.8776119402985074 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9823529411764705, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.990990990990991, + "9": 0.977491961414791, + "10": 0.8985074626865671 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.990990990990991, + "9": 0.9871382636655949, + "10": 0.9164179104477612 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9852941176470589, + "6": 0.9968454258675079, + "7": 0.990506329113924, + "8": 0.990990990990991, + "9": 0.9871382636655949, + "10": 0.9253731343283582 + } + }, + "total_pass_at_k": { + "pass@1": 0.8814864412453968, + "pass@2": 0.913290927351858, + "pass@4": 0.9387345162370271, + "pass@8": 0.9618346166722465, + "pass@16": 0.975225979243388, + "pass@32": 0.9819216605289588, + "pass@64": 0.9852695011717443, + "pass@128": 0.9862738533645798 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.996969696969697, + "pass@2": 0.996969696969697, + "pass@4": 0.996969696969697, + "pass@8": 0.996969696969697, + "pass@16": 0.996969696969697, + "pass@32": 0.996969696969697, + "pass@64": 0.996969696969697, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 0.9792899408284024, + "pass@2": 0.9911242603550295, + "pass@4": 0.9940828402366864, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9507246376811594, + "pass@2": 0.9681159420289855, + "pass@4": 0.9797101449275363, + "pass@8": 0.9855072463768116, + "pass@16": 0.9884057971014493, + "pass@32": 0.9884057971014493, + "pass@64": 0.9884057971014493, + "pass@128": 0.9884057971014493 + }, + "6": { + "pass@1": 0.9627507163323782, + "pass@2": 0.9770773638968482, + "pass@4": 0.9885386819484241, + "pass@8": 0.997134670487106, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8670520231213873, + "pass@2": 0.9219653179190751, + "pass@4": 0.9624277456647399, + "pass@8": 0.9855491329479769, + "pass@16": 0.9913294797687862, + "pass@32": 0.9971098265895953, + "pass@64": 0.9971098265895953, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.7718309859154929, + "pass@2": 0.8450704225352113, + "pass@4": 0.9014084507042254, + "pass@8": 0.9577464788732394, + "pass@16": 0.9746478873239437, + "pass@32": 0.9830985915492958, + "pass@64": 0.9859154929577465, + "pass@128": 0.9859154929577465 + }, + "9": { + "pass@1": 0.7305389221556886, + "pass@2": 0.8053892215568862, + "pass@4": 0.8532934131736527, + "pass@8": 0.9221556886227545, + "pass@16": 0.9640718562874252, + "pass@32": 0.9790419161676647, + "pass@64": 0.9850299401197605, + "pass@128": 0.9850299401197605 + }, + "10": { + "pass@1": 0.6288343558282209, + "pass@2": 0.6809815950920245, + "pass@4": 0.7515337423312883, + "pass@8": 0.7914110429447853, + "pass@16": 0.8466257668711656, + "pass@32": 0.8711656441717791, + "pass@64": 0.911042944785276, + "pass@128": 0.9263803680981595 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9792899408284024, + "5": 0.9507246376811594, + "6": 0.9627507163323782, + "7": 0.8670520231213873, + "8": 0.7718309859154929, + "9": 0.7305389221556886, + "10": 0.6288343558282209 + }, + "pass@2": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9911242603550295, + "5": 0.9681159420289855, + "6": 0.9770773638968482, + "7": 0.9219653179190751, + "8": 0.8450704225352113, + "9": 0.8053892215568862, + "10": 0.6809815950920245 + }, + "pass@4": { + "2": 1.0, + "3": 0.996969696969697, + "4": 0.9940828402366864, + "5": 0.9797101449275363, + "6": 0.9885386819484241, + "7": 0.9624277456647399, + "8": 0.9014084507042254, + "9": 0.8532934131736527, + "10": 0.7515337423312883 + }, + "pass@8": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9855072463768116, + "6": 0.997134670487106, + "7": 0.9855491329479769, + "8": 0.9577464788732394, + "9": 0.9221556886227545, + "10": 0.7914110429447853 + }, + "pass@16": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9913294797687862, + "8": 0.9746478873239437, + "9": 0.9640718562874252, + "10": 0.8466257668711656 + }, + "pass@32": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9830985915492958, + "9": 0.9790419161676647, + "10": 0.8711656441717791 + }, + "pass@64": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9859154929577465, + "9": 0.9850299401197605, + "10": 0.911042944785276 + }, + "pass@128": { + "2": 1.0, + "3": 0.996969696969697, + "4": 1.0, + "5": 0.9884057971014493, + "6": 1.0, + "7": 0.9971098265895953, + "8": 0.9859154929577465, + "9": 0.9850299401197605, + "10": 0.9263803680981595 + } + }, + "total_pass_at_k": { + "pass@1": 0.8768020969855832, + "pass@2": 0.9102228047182176, + "pass@4": 0.9370904325032765, + "pass@8": 0.9603538663171691, + "pass@16": 0.9741153342070773, + "pass@32": 0.9800131061598951, + "pass@64": 0.9852555701179554, + "pass@128": 0.9868938401048493 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.998, + "pass@2": 0.999, + "pass@4": 0.999, + "pass@8": 0.999, + "pass@16": 0.999, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.986, + "pass@2": 0.992, + "pass@4": 0.995, + "pass@8": 0.997, + "pass@16": 0.997, + "pass@32": 0.997, + "pass@64": 0.998, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.935, + "pass@2": 0.961, + "pass@4": 0.975, + "pass@8": 0.981, + "pass@16": 0.985, + "pass@32": 0.985, + "pass@64": 0.986, + "pass@128": 0.987 + }, + "6": { + "pass@1": 0.958, + "pass@2": 0.976, + "pass@4": 0.986, + "pass@8": 0.993, + "pass@16": 0.997, + "pass@32": 0.999, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "7": { + "pass@1": 0.872, + "pass@2": 0.926, + "pass@4": 0.958, + "pass@8": 0.979, + "pass@16": 0.99, + "pass@32": 0.996, + "pass@64": 0.996, + "pass@128": 0.996 + }, + "8": { + "pass@1": 0.787, + "pass@2": 0.863, + "pass@4": 0.917, + "pass@8": 0.967, + "pass@16": 0.983, + "pass@32": 0.989, + "pass@64": 0.99, + "pass@128": 0.99 + }, + "9": { + "pass@1": 0.747, + "pass@2": 0.814, + "pass@4": 0.885, + "pass@8": 0.936, + "pass@16": 0.967, + "pass@32": 0.984, + "pass@64": 0.99, + "pass@128": 0.991 + }, + "10": { + "pass@1": 0.62, + "pass@2": 0.704, + "pass@4": 0.766, + "pass@8": 0.822, + "pass@16": 0.872, + "pass@32": 0.895, + "pass@64": 0.916, + "pass@128": 0.928 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 0.998, + "4": 0.986, + "5": 0.935, + "6": 0.958, + "7": 0.872, + "8": 0.787, + "9": 0.747, + "10": 0.62 + }, + "pass@2": { + "2": 1.0, + "3": 0.999, + "4": 0.992, + "5": 0.961, + "6": 0.976, + "7": 0.926, + "8": 0.863, + "9": 0.814, + "10": 0.704 + }, + "pass@4": { + "2": 1.0, + "3": 0.999, + "4": 0.995, + "5": 0.975, + "6": 0.986, + "7": 0.958, + "8": 0.917, + "9": 0.885, + "10": 0.766 + }, + "pass@8": { + "2": 1.0, + "3": 0.999, + "4": 0.997, + "5": 0.981, + "6": 0.993, + "7": 0.979, + "8": 0.967, + "9": 0.936, + "10": 0.822 + }, + "pass@16": { + "2": 1.0, + "3": 0.999, + "4": 0.997, + "5": 0.985, + "6": 0.997, + "7": 0.99, + "8": 0.983, + "9": 0.967, + "10": 0.872 + }, + "pass@32": { + "2": 1.0, + "3": 0.999, + "4": 0.997, + "5": 0.985, + "6": 0.999, + "7": 0.996, + "8": 0.989, + "9": 0.984, + "10": 0.895 + }, + "pass@64": { + "2": 1.0, + "3": 0.999, + "4": 0.998, + "5": 0.986, + "6": 0.999, + "7": 0.996, + "8": 0.99, + "9": 0.99, + "10": 0.916 + }, + "pass@128": { + "2": 1.0, + "3": 0.999, + "4": 0.998, + "5": 0.987, + "6": 0.999, + "7": 0.996, + "8": 0.99, + "9": 0.991, + "10": 0.928 + } + }, + "total_pass_at_k": { + "pass@1": 0.8781111111111111, + "pass@2": 0.915, + "pass@4": 0.9423333333333334, + "pass@8": 0.9637777777777777, + "pass@16": 0.9766666666666667, + "pass@32": 0.9826666666666667, + "pass@64": 0.986, + "pass@128": 0.9875555555555555 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-21546_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5070821529745042, + "pass@2": 0.5920679886685553, + "pass@4": 0.6798866855524079, + "pass@8": 0.7677053824362606, + "pass@16": 0.8328611898016998, + "pass@32": 0.8725212464589235, + "pass@64": 0.8951841359773371, + "pass@128": 0.9093484419263456 + }, + "12": { + "pass@1": 0.53954802259887, + "pass@2": 0.576271186440678, + "pass@4": 0.635593220338983, + "pass@8": 0.692090395480226, + "pass@16": 0.7457627118644068, + "pass@32": 0.7994350282485876, + "pass@64": 0.8220338983050848, + "pass@128": 0.8305084745762712 + }, + "13": { + "pass@1": 0.28213166144200624, + "pass@2": 0.3824451410658307, + "pass@4": 0.438871473354232, + "pass@8": 0.5172413793103449, + "pass@16": 0.5799373040752351, + "pass@32": 0.6238244514106583, + "pass@64": 0.658307210031348, + "pass@128": 0.6802507836990596 + }, + "14": { + "pass@1": 0.23052959501557632, + "pass@2": 0.2616822429906542, + "pass@4": 0.2897196261682243, + "pass@8": 0.32087227414330216, + "pass@16": 0.37383177570093457, + "pass@32": 0.40809968847352024, + "pass@64": 0.4672897196261682, + "pass@128": 0.5077881619937694 + }, + "15": { + "pass@1": 0.11560693641618497, + "pass@2": 0.14739884393063585, + "pass@4": 0.17630057803468208, + "pass@8": 0.1907514450867052, + "pass@16": 0.2254335260115607, + "pass@32": 0.2630057803468208, + "pass@64": 0.29190751445086704, + "pass@128": 0.3236994219653179 + }, + "16": { + "pass@1": 0.016129032258064516, + "pass@2": 0.02258064516129032, + "pass@4": 0.03225806451612903, + "pass@8": 0.03870967741935484, + "pass@16": 0.04516129032258064, + "pass@32": 0.04838709677419355, + "pass@64": 0.06774193548387097, + "pass@128": 0.08387096774193549 + }, + "17": { + "pass@1": 0.005780346820809248, + "pass@2": 0.008670520231213872, + "pass@4": 0.014450867052023121, + "pass@8": 0.014450867052023121, + "pass@16": 0.017341040462427744, + "pass@32": 0.02601156069364162, + "pass@64": 0.028901734104046242, + "pass@128": 0.04335260115606936 + }, + "18": { + "pass@1": 0.005714285714285714, + "pass@2": 0.005714285714285714, + "pass@4": 0.008571428571428572, + "pass@8": 0.008571428571428572, + "pass@16": 0.011428571428571429, + "pass@32": 0.011428571428571429, + "pass@64": 0.017142857142857144, + "pass@128": 0.02857142857142857 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.003003003003003003 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.003003003003003003, + "pass@16": 0.003003003003003003, + "pass@32": 0.006006006006006006, + "pass@64": 0.006006006006006006, + "pass@128": 0.006006006006006006 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5070821529745042, + "12": 0.53954802259887, + "13": 0.28213166144200624, + "14": 0.23052959501557632, + "15": 0.11560693641618497, + "16": 0.016129032258064516, + "17": 0.005780346820809248, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.5920679886685553, + "12": 0.576271186440678, + "13": 0.3824451410658307, + "14": 0.2616822429906542, + "15": 0.14739884393063585, + "16": 0.02258064516129032, + "17": 0.008670520231213872, + "18": 0.005714285714285714, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.6798866855524079, + "12": 0.635593220338983, + "13": 0.438871473354232, + "14": 0.2897196261682243, + "15": 0.17630057803468208, + "16": 0.03225806451612903, + "17": 0.014450867052023121, + "18": 0.008571428571428572, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7677053824362606, + "12": 0.692090395480226, + "13": 0.5172413793103449, + "14": 0.32087227414330216, + "15": 0.1907514450867052, + "16": 0.03870967741935484, + "17": 0.014450867052023121, + "18": 0.008571428571428572, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@16": { + "11": 0.8328611898016998, + "12": 0.7457627118644068, + "13": 0.5799373040752351, + "14": 0.37383177570093457, + "15": 0.2254335260115607, + "16": 0.04516129032258064, + "17": 0.017341040462427744, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.003003003003003003 + }, + "pass@32": { + "11": 0.8725212464589235, + "12": 0.7994350282485876, + "13": 0.6238244514106583, + "14": 0.40809968847352024, + "15": 0.2630057803468208, + "16": 0.04838709677419355, + "17": 0.02601156069364162, + "18": 0.011428571428571429, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@64": { + "11": 0.8951841359773371, + "12": 0.8220338983050848, + "13": 0.658307210031348, + "14": 0.4672897196261682, + "15": 0.29190751445086704, + "16": 0.06774193548387097, + "17": 0.028901734104046242, + "18": 0.017142857142857144, + "19": 0.0, + "20": 0.006006006006006006 + }, + "pass@128": { + "11": 0.9093484419263456, + "12": 0.8305084745762712, + "13": 0.6802507836990596, + "14": 0.5077881619937694, + "15": 0.3236994219653179, + "16": 0.08387096774193549, + "17": 0.04335260115606936, + "18": 0.02857142857142857, + "19": 0.003003003003003003, + "20": 0.006006006006006006 + } + }, + "total_pass_at_k": { + "pass@1": 0.17325408618127786, + "pass@2": 0.20267459138187222, + "pass@4": 0.23090638930163448, + "pass@8": 0.25884101040118873, + "pass@16": 0.28707280832095095, + "pass@32": 0.3096582466567608, + "pass@64": 0.3289747399702823, + "pass@128": 0.3450222882615156 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5015873015873016, + "pass@2": 0.6126984126984127, + "pass@4": 0.6698412698412698, + "pass@8": 0.780952380952381, + "pass@16": 0.8444444444444444, + "pass@32": 0.8952380952380953, + "pass@64": 0.9142857142857143, + "pass@128": 0.926984126984127 + }, + "12": { + "pass@1": 0.5168195718654435, + "pass@2": 0.599388379204893, + "pass@4": 0.6605504587155964, + "pass@8": 0.7308868501529052, + "pass@16": 0.7951070336391437, + "pass@32": 0.8165137614678899, + "pass@64": 0.8470948012232415, + "pass@128": 0.8685015290519877 + }, + "13": { + "pass@1": 0.3333333333333333, + "pass@2": 0.4104938271604938, + "pass@4": 0.4506172839506173, + "pass@8": 0.5092592592592593, + "pass@16": 0.5617283950617284, + "pass@32": 0.6049382716049383, + "pass@64": 0.6388888888888888, + "pass@128": 0.6820987654320988 + }, + "14": { + "pass@1": 0.19584569732937684, + "pass@2": 0.23738872403560832, + "pass@4": 0.27596439169139464, + "pass@8": 0.32047477744807124, + "pass@16": 0.37091988130563797, + "pass@32": 0.41543026706231456, + "pass@64": 0.47774480712166173, + "pass@128": 0.543026706231454 + }, + "15": { + "pass@1": 0.1388888888888889, + "pass@2": 0.16049382716049382, + "pass@4": 0.18518518518518517, + "pass@8": 0.22530864197530864, + "pass@16": 0.23765432098765432, + "pass@32": 0.25617283950617287, + "pass@64": 0.3117283950617284, + "pass@128": 0.35802469135802467 + }, + "16": { + "pass@1": 0.02654867256637168, + "pass@2": 0.038348082595870206, + "pass@4": 0.0471976401179941, + "pass@8": 0.07374631268436578, + "pass@16": 0.0855457227138643, + "pass@32": 0.10324483775811209, + "pass@64": 0.12094395280235988, + "pass@128": 0.14454277286135694 + }, + "17": { + "pass@1": 0.015060240963855422, + "pass@2": 0.018072289156626505, + "pass@4": 0.024096385542168676, + "pass@8": 0.02710843373493976, + "pass@16": 0.03614457831325301, + "pass@32": 0.045180722891566265, + "pass@64": 0.05120481927710843, + "pass@128": 0.06325301204819277 + }, + "18": { + "pass@1": 0.025806451612903226, + "pass@2": 0.025806451612903226, + "pass@4": 0.03870967741935484, + "pass@8": 0.04516129032258064, + "pass@16": 0.054838709677419356, + "pass@32": 0.06774193548387097, + "pass@64": 0.07419354838709677, + "pass@128": 0.07741935483870968 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0030211480362537764, + "pass@4": 0.0030211480362537764, + "pass@8": 0.0030211480362537764, + "pass@16": 0.00906344410876133, + "pass@32": 0.012084592145015106, + "pass@64": 0.012084592145015106, + "pass@128": 0.012084592145015106 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0, + "pass@32": 0.0, + "pass@64": 0.0, + "pass@128": 0.0 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5015873015873016, + "12": 0.5168195718654435, + "13": 0.3333333333333333, + "14": 0.19584569732937684, + "15": 0.1388888888888889, + "16": 0.02654867256637168, + "17": 0.015060240963855422, + "18": 0.025806451612903226, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6126984126984127, + "12": 0.599388379204893, + "13": 0.4104938271604938, + "14": 0.23738872403560832, + "15": 0.16049382716049382, + "16": 0.038348082595870206, + "17": 0.018072289156626505, + "18": 0.025806451612903226, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@4": { + "11": 0.6698412698412698, + "12": 0.6605504587155964, + "13": 0.4506172839506173, + "14": 0.27596439169139464, + "15": 0.18518518518518517, + "16": 0.0471976401179941, + "17": 0.024096385542168676, + "18": 0.03870967741935484, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@8": { + "11": 0.780952380952381, + "12": 0.7308868501529052, + "13": 0.5092592592592593, + "14": 0.32047477744807124, + "15": 0.22530864197530864, + "16": 0.07374631268436578, + "17": 0.02710843373493976, + "18": 0.04516129032258064, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@16": { + "11": 0.8444444444444444, + "12": 0.7951070336391437, + "13": 0.5617283950617284, + "14": 0.37091988130563797, + "15": 0.23765432098765432, + "16": 0.0855457227138643, + "17": 0.03614457831325301, + "18": 0.054838709677419356, + "19": 0.00906344410876133, + "20": 0.0 + }, + "pass@32": { + "11": 0.8952380952380953, + "12": 0.8165137614678899, + "13": 0.6049382716049383, + "14": 0.41543026706231456, + "15": 0.25617283950617287, + "16": 0.10324483775811209, + "17": 0.045180722891566265, + "18": 0.06774193548387097, + "19": 0.012084592145015106, + "20": 0.0 + }, + "pass@64": { + "11": 0.9142857142857143, + "12": 0.8470948012232415, + "13": 0.6388888888888888, + "14": 0.47774480712166173, + "15": 0.3117283950617284, + "16": 0.12094395280235988, + "17": 0.05120481927710843, + "18": 0.07419354838709677, + "19": 0.012084592145015106, + "20": 0.0 + }, + "pass@128": { + "11": 0.926984126984127, + "12": 0.8685015290519877, + "13": 0.6820987654320988, + "14": 0.543026706231454, + "15": 0.35802469135802467, + "16": 0.14454277286135694, + "17": 0.06325301204819277, + "18": 0.07741935483870968, + "19": 0.012084592145015106, + "20": 0.0 + } + }, + "total_pass_at_k": { + "pass@1": 0.17359413202933985, + "pass@2": 0.20843520782396088, + "pass@4": 0.23319070904645478, + "pass@8": 0.26894865525672373, + "pass@16": 0.296760391198044, + "pass@32": 0.31876528117359415, + "pass@64": 0.3419926650366748, + "pass@128": 0.36491442542787283 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5933734939759037, + "pass@2": 0.6536144578313253, + "pass@4": 0.7319277108433735, + "pass@8": 0.7951807228915663, + "pass@16": 0.8644578313253012, + "pass@32": 0.9307228915662651, + "pass@64": 0.9487951807228916, + "pass@128": 0.9518072289156626 + }, + "12": { + "pass@1": 0.46394984326018807, + "pass@2": 0.554858934169279, + "pass@4": 0.6269592476489029, + "pass@8": 0.7084639498432602, + "pass@16": 0.7774294670846394, + "pass@32": 0.8056426332288401, + "pass@64": 0.8181818181818182, + "pass@128": 0.8338557993730408 + }, + "13": { + "pass@1": 0.3081232492997199, + "pass@2": 0.38375350140056025, + "pass@4": 0.45098039215686275, + "pass@8": 0.5098039215686274, + "pass@16": 0.5658263305322129, + "pass@32": 0.6162464985994398, + "pass@64": 0.6498599439775911, + "pass@128": 0.6918767507002801 + }, + "14": { + "pass@1": 0.15789473684210525, + "pass@2": 0.19005847953216373, + "pass@4": 0.23976608187134502, + "pass@8": 0.29239766081871343, + "pass@16": 0.36257309941520466, + "pass@32": 0.40058479532163743, + "pass@64": 0.4473684210526316, + "pass@128": 0.5 + }, + "15": { + "pass@1": 0.12727272727272726, + "pass@2": 0.15757575757575756, + "pass@4": 0.18181818181818182, + "pass@8": 0.2, + "pass@16": 0.23939393939393938, + "pass@32": 0.2696969696969697, + "pass@64": 0.3212121212121212, + "pass@128": 0.35454545454545455 + }, + "16": { + "pass@1": 0.022792022792022793, + "pass@2": 0.02564102564102564, + "pass@4": 0.03133903133903134, + "pass@8": 0.04843304843304843, + "pass@16": 0.06837606837606838, + "pass@32": 0.07977207977207977, + "pass@64": 0.08831908831908832, + "pass@128": 0.10541310541310542 + }, + "17": { + "pass@1": 0.018633540372670808, + "pass@2": 0.021739130434782608, + "pass@4": 0.024844720496894408, + "pass@8": 0.037267080745341616, + "pass@16": 0.043478260869565216, + "pass@32": 0.055900621118012424, + "pass@64": 0.059006211180124224, + "pass@128": 0.07453416149068323 + }, + "18": { + "pass@1": 0.0058823529411764705, + "pass@2": 0.008823529411764706, + "pass@4": 0.008823529411764706, + "pass@8": 0.01764705882352941, + "pass@16": 0.01764705882352941, + "pass@32": 0.01764705882352941, + "pass@64": 0.023529411764705882, + "pass@128": 0.029411764705882353 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.002976190476190476, + "pass@16": 0.005952380952380952, + "pass@32": 0.005952380952380952, + "pass@64": 0.008928571428571428, + "pass@128": 0.008928571428571428 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.0, + "pass@16": 0.0029940119760479044, + "pass@32": 0.0029940119760479044, + "pass@64": 0.0029940119760479044, + "pass@128": 0.005988023952095809 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5933734939759037, + "12": 0.46394984326018807, + "13": 0.3081232492997199, + "14": 0.15789473684210525, + "15": 0.12727272727272726, + "16": 0.022792022792022793, + "17": 0.018633540372670808, + "18": 0.0058823529411764705, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6536144578313253, + "12": 0.554858934169279, + "13": 0.38375350140056025, + "14": 0.19005847953216373, + "15": 0.15757575757575756, + "16": 0.02564102564102564, + "17": 0.021739130434782608, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0 + }, + "pass@4": { + "11": 0.7319277108433735, + "12": 0.6269592476489029, + "13": 0.45098039215686275, + "14": 0.23976608187134502, + "15": 0.18181818181818182, + "16": 0.03133903133903134, + "17": 0.024844720496894408, + "18": 0.008823529411764706, + "19": 0.0, + "20": 0.0 + }, + "pass@8": { + "11": 0.7951807228915663, + "12": 0.7084639498432602, + "13": 0.5098039215686274, + "14": 0.29239766081871343, + "15": 0.2, + "16": 0.04843304843304843, + "17": 0.037267080745341616, + "18": 0.01764705882352941, + "19": 0.002976190476190476, + "20": 0.0 + }, + "pass@16": { + "11": 0.8644578313253012, + "12": 0.7774294670846394, + "13": 0.5658263305322129, + "14": 0.36257309941520466, + "15": 0.23939393939393938, + "16": 0.06837606837606838, + "17": 0.043478260869565216, + "18": 0.01764705882352941, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@32": { + "11": 0.9307228915662651, + "12": 0.8056426332288401, + "13": 0.6162464985994398, + "14": 0.40058479532163743, + "15": 0.2696969696969697, + "16": 0.07977207977207977, + "17": 0.055900621118012424, + "18": 0.01764705882352941, + "19": 0.005952380952380952, + "20": 0.0029940119760479044 + }, + "pass@64": { + "11": 0.9487951807228916, + "12": 0.8181818181818182, + "13": 0.6498599439775911, + "14": 0.4473684210526316, + "15": 0.3212121212121212, + "16": 0.08831908831908832, + "17": 0.059006211180124224, + "18": 0.023529411764705882, + "19": 0.008928571428571428, + "20": 0.0029940119760479044 + }, + "pass@128": { + "11": 0.9518072289156626, + "12": 0.8338557993730408, + "13": 0.6918767507002801, + "14": 0.5, + "15": 0.35454545454545455, + "16": 0.10541310541310542, + "17": 0.07453416149068323, + "18": 0.029411764705882353, + "19": 0.008928571428571428, + "20": 0.005988023952095809 + } + }, + "total_pass_at_k": { + "pass@1": 0.16859946476360393, + "pass@2": 0.1983348201011002, + "pass@4": 0.22836752899197146, + "pass@8": 0.2598870056497175, + "pass@16": 0.29348795718108833, + "pass@32": 0.31727624145108535, + "pass@64": 0.335712161760333, + "pass@128": 0.3547427891763307 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.534, + "pass@2": 0.619, + "pass@4": 0.694, + "pass@8": 0.781, + "pass@16": 0.847, + "pass@32": 0.899, + "pass@64": 0.919, + "pass@128": 0.929 + }, + "12": { + "pass@1": 0.508, + "pass@2": 0.577, + "pass@4": 0.641, + "pass@8": 0.71, + "pass@16": 0.772, + "pass@32": 0.807, + "pass@64": 0.829, + "pass@128": 0.844 + }, + "13": { + "pass@1": 0.308, + "pass@2": 0.392, + "pass@4": 0.447, + "pass@8": 0.512, + "pass@16": 0.569, + "pass@32": 0.615, + "pass@64": 0.649, + "pass@128": 0.685 + }, + "14": { + "pass@1": 0.194, + "pass@2": 0.229, + "pass@4": 0.268, + "pass@8": 0.311, + "pass@16": 0.369, + "pass@32": 0.408, + "pass@64": 0.464, + "pass@128": 0.517 + }, + "15": { + "pass@1": 0.127, + "pass@2": 0.155, + "pass@4": 0.181, + "pass@8": 0.205, + "pass@16": 0.234, + "pass@32": 0.263, + "pass@64": 0.308, + "pass@128": 0.345 + }, + "16": { + "pass@1": 0.022, + "pass@2": 0.029, + "pass@4": 0.037, + "pass@8": 0.054, + "pass@16": 0.067, + "pass@32": 0.078, + "pass@64": 0.093, + "pass@128": 0.112 + }, + "17": { + "pass@1": 0.013, + "pass@2": 0.016, + "pass@4": 0.021, + "pass@8": 0.026, + "pass@16": 0.032, + "pass@32": 0.042, + "pass@64": 0.046, + "pass@128": 0.06 + }, + "18": { + "pass@1": 0.012, + "pass@2": 0.013, + "pass@4": 0.018, + "pass@8": 0.023, + "pass@16": 0.027, + "pass@32": 0.031, + "pass@64": 0.037, + "pass@128": 0.044 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.001, + "pass@4": 0.001, + "pass@8": 0.002, + "pass@16": 0.005, + "pass@32": 0.006, + "pass@64": 0.007, + "pass@128": 0.008 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0, + "pass@8": 0.001, + "pass@16": 0.002, + "pass@32": 0.003, + "pass@64": 0.003, + "pass@128": 0.004 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.534, + "12": 0.508, + "13": 0.308, + "14": 0.194, + "15": 0.127, + "16": 0.022, + "17": 0.013, + "18": 0.012, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.619, + "12": 0.577, + "13": 0.392, + "14": 0.229, + "15": 0.155, + "16": 0.029, + "17": 0.016, + "18": 0.013, + "19": 0.001, + "20": 0.0 + }, + "pass@4": { + "11": 0.694, + "12": 0.641, + "13": 0.447, + "14": 0.268, + "15": 0.181, + "16": 0.037, + "17": 0.021, + "18": 0.018, + "19": 0.001, + "20": 0.0 + }, + "pass@8": { + "11": 0.781, + "12": 0.71, + "13": 0.512, + "14": 0.311, + "15": 0.205, + "16": 0.054, + "17": 0.026, + "18": 0.023, + "19": 0.002, + "20": 0.001 + }, + "pass@16": { + "11": 0.847, + "12": 0.772, + "13": 0.569, + "14": 0.369, + "15": 0.234, + "16": 0.067, + "17": 0.032, + "18": 0.027, + "19": 0.005, + "20": 0.002 + }, + "pass@32": { + "11": 0.899, + "12": 0.807, + "13": 0.615, + "14": 0.408, + "15": 0.263, + "16": 0.078, + "17": 0.042, + "18": 0.031, + "19": 0.006, + "20": 0.003 + }, + "pass@64": { + "11": 0.919, + "12": 0.829, + "13": 0.649, + "14": 0.464, + "15": 0.308, + "16": 0.093, + "17": 0.046, + "18": 0.037, + "19": 0.007, + "20": 0.003 + }, + "pass@128": { + "11": 0.929, + "12": 0.844, + "13": 0.685, + "14": 0.517, + "15": 0.345, + "16": 0.112, + "17": 0.06, + "18": 0.044, + "19": 0.008, + "20": 0.004 + } + }, + "total_pass_at_k": { + "pass@1": 0.1718, + "pass@2": 0.2031, + "pass@4": 0.2308, + "pass@8": 0.2625, + "pass@16": 0.2924, + "pass@32": 0.3152, + "pass@64": 0.3355, + "pass@128": 0.3548 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/summary.csv b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..0dc368297fbf357b1e4a47baa614ada08cffb88f --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-21546,0.9797482638888889,0.41910390625,0.6846722861842105,0.0,0.0,0.0,196.59117881944445,317.97294609375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/summary.json b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..04003d29dd4becf2a09056a3f278a4f8a90a7018 --- /dev/null +++ b/SDAR-100M-ep1-block8/composition/low_confidence_static/block8/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-21546", + "id_acc": 0.9797482638888889, + "ood_acc": 0.41910390625, + "total_acc": 0.6846722861842105, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.59117881944445, + "ood_avg_resp_len": 317.97294609375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 551351, + "answer_accuracy": 0.6809088978027189, + "avg_response_len": 248.23700551296238, + "resp_tokens_sum": 201004454, + "pass_at_k": { + "pass@1": 0.6809088978027189, + "pass@2": 0.7179658930087301, + "pass@4": 0.7469747311907298, + "pass@8": 0.7700900354015988, + "pass@16": 0.7887883821084508, + "pass@32": 0.8044023259925125, + "pass@64": 0.8181816148916696, + "pass@128": 0.8313310148593108 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9221515486725663, + "pass@2": 0.9428836259958658, + "pass@4": 0.953557931641288, + "pass@8": 0.9603148730351716, + "pass@16": 0.9648846877139904, + "pass@32": 0.9680322261071029, + "pass@64": 0.9708755602536454, + "pass@128": 0.9734513274336283 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9978736620795107, + "pass@2": 0.9985266319921019, + "pass@4": 0.9993000800358671, + "pass@8": 0.9998489340659815, + "pass@16": 0.9999942173708687, + "pass@32": 0.9999999967965904, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9907470703125, + "pass@2": 0.9910037063238188, + "pass@4": 0.991343688484252, + "pass@8": 0.9919133814083988, + "pass@16": 0.9927106862872591, + "pass@32": 0.9934601647847077, + "pass@64": 0.9937403254241349, + "pass@128": 0.99375 + }, + "5": { + "pass@1": 0.9669642857142857, + "pass@2": 0.9718843738282714, + "pass@4": 0.9763600398164514, + "pass@8": 0.9802523092172719, + "pass@16": 0.9830724326754721, + "pass@32": 0.9851478752003268, + "pass@64": 0.987257685923108, + "pass@128": 0.9904761904761905 + }, + "6": { + "pass@1": 0.9956025449101796, + "pass@2": 0.9989277105238344, + "pass@4": 0.9996942885881777, + "pass@8": 0.9999718501697081, + "pass@16": 0.9999998309063756, + "pass@32": 0.999999999998971, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9926960059171598, + "pass@2": 0.9966541257047011, + "pass@4": 0.9984301580460726, + "pass@8": 0.9994525764888945, + "pass@16": 0.9999158567707788, + "pass@32": 0.9999987203425054, + "pass@64": 0.9999999999877117, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9768880208333334, + "pass@2": 0.9874886432465175, + "pass@4": 0.9895625994467037, + "pass@8": 0.9907045168965894, + "pass@16": 0.9916775319238652, + "pass@32": 0.9925913730797592, + "pass@64": 0.9933988491820491, + "pass@128": 0.9935897435897436 + }, + "9": { + "pass@1": 0.9691681338028169, + "pass@2": 0.9880122269047353, + "pass@4": 0.9939504940403571, + "pass@8": 0.9970743852484422, + "pass@16": 0.9988665501562034, + "pass@32": 0.9997334583192324, + "pass@64": 0.9999912787935963, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.8784525495750708, + "pass@2": 0.9102096623987864, + "pass@4": 0.9281930339444113, + "pass@8": 0.9398208465107002, + "pass@16": 0.9490450774095365, + "pass@32": 0.9577149931620413, + "pass@64": 0.9651973316007761, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.798927436440678, + "pass@2": 0.8551790699096934, + "pass@4": 0.8885044120544257, + "pass@8": 0.9085335882562506, + "pass@16": 0.9217108196453194, + "pass@32": 0.9324307195802108, + "pass@64": 0.9433761539311727, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.6550989420062696, + "pass@2": 0.7359161719694916, + "pass@4": 0.7893547024294372, + "pass@8": 0.8251472079307166, + "pass@16": 0.8499261844339199, + "pass@32": 0.8690833324313215, + "pass@64": 0.887678042849582, + "pass@128": 0.9090909090909091 + }, + "14": { + "pass@1": 0.4941588785046729, + "pass@2": 0.5814315193661542, + "pass@4": 0.6522121046201, + "pass@8": 0.7054398474703508, + "pass@16": 0.7459692840506352, + "pass@32": 0.7797983464083315, + "pass@64": 0.8106658828186043, + "pass@128": 0.8411214953271028 + }, + "15": { + "pass@1": 0.36416184971098264, + "pass@2": 0.44411809885758474, + "pass@4": 0.5126349126879371, + "pass@8": 0.5702821524993034, + "pass@16": 0.6204606367913936, + "pass@32": 0.6645106758255857, + "pass@64": 0.7048887406631363, + "pass@128": 0.7456647398843931 + }, + "16": { + "pass@1": 0.26522177419354837, + "pass@2": 0.3349060198120396, + "pass@4": 0.39511467790316557, + "pass@8": 0.44689684761816845, + "pass@16": 0.4919076005163578, + "pass@32": 0.5289581705941678, + "pass@64": 0.557172877415548, + "pass@128": 0.5806451612903226 + }, + "17": { + "pass@1": 0.20822796242774566, + "pass@2": 0.2647282212917027, + "pass@4": 0.3119168785260225, + "pass@8": 0.353016882625172, + "pass@16": 0.39074588899191076, + "pass@32": 0.42409412095318333, + "pass@64": 0.450657781803306, + "pass@128": 0.47398843930635837 + }, + "18": { + "pass@1": 0.17683035714285714, + "pass@2": 0.23971140326209206, + "pass@4": 0.3019964647276231, + "pass@8": 0.35749287383033507, + "pass@16": 0.40207686154338784, + "pass@32": 0.437154461831571, + "pass@64": 0.467470346580905, + "pass@128": 0.4942857142857143 + }, + "19": { + "pass@1": 0.15927646396396397, + "pass@2": 0.21241775732425341, + "pass@4": 0.2633034674156721, + "pass@8": 0.3066705995032504, + "pass@16": 0.34242761359959684, + "pass@32": 0.3739619165348702, + "pass@64": 0.40473258265774553, + "pass@128": 0.43543543543543545 + }, + "20": { + "pass@1": 0.15995683183183182, + "pass@2": 0.21900899423045087, + "pass@4": 0.2765225937410976, + "pass@8": 0.32656027758034484, + "pass@16": 0.3677726314607193, + "pass@32": 0.4017703052549921, + "pass@64": 0.4317523725638508, + "pass@128": 0.4594594594594595 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 561524, + "answer_accuracy": 0.6838513250194855, + "avg_response_len": 257.001773187841, + "resp_tokens_sum": 211029296, + "pass_at_k": { + "pass@1": 0.6838513250194855, + "pass@2": 0.7182096587108185, + "pass@4": 0.7457970707480713, + "pass@8": 0.7688692845996465, + "pass@16": 0.7884647649459229, + "pass@32": 0.8053975893041211, + "pass@64": 0.8202428182245672, + "pass@128": 0.8325798908807482 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.902032208588957, + "pass@2": 0.928237539853147, + "pass@4": 0.9445890421672748, + "pass@8": 0.9554783664156684, + "pass@16": 0.9637377662720875, + "pass@32": 0.9702591907243098, + "pass@64": 0.975278890857173, + "pass@128": 0.9785276073619632 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9966619318181819, + "pass@2": 0.9969518014793606, + "pass@4": 0.9969696373180625, + "pass@8": 0.9969696969696016, + "pass@16": 0.996969696969697, + "pass@32": 0.996969696969697, + "pass@64": 0.996969696969697, + "pass@128": 0.996969696969697 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9875226449275363, + "pass@2": 0.9883676395070182, + "pass@4": 0.9884954541008462, + "pass@8": 0.9885869564329156, + "pass@16": 0.988768115942029, + "pass@32": 0.9891304347826086, + "pass@64": 0.9898550724637681, + "pass@128": 0.991304347826087 + }, + "6": { + "pass@1": 0.9941350286532952, + "pass@2": 0.9977561734314014, + "pass@4": 0.9985594472610693, + "pass@8": 0.9992944900423253, + "pass@16": 0.9998437548382668, + "pass@32": 0.9999948232370918, + "pass@64": 0.9999999995301811, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9928648843930635, + "pass@2": 0.994768672340813, + "pass@4": 0.9953643609758319, + "pass@8": 0.9960753760142124, + "pass@16": 0.9967665713625197, + "pass@32": 0.9970812204794673, + "pass@64": 0.9971097917066677, + "pass@128": 0.9971098265895953 + }, + "8": { + "pass@1": 0.9756382042253521, + "pass@2": 0.9838094016857049, + "pass@4": 0.9872804353681142, + "pass@8": 0.9894908076168281, + "pass@16": 0.9907958224370905, + "pass@32": 0.991898265474723, + "pass@64": 0.9929392340213707, + "pass@128": 0.9943661971830986 + }, + "9": { + "pass@1": 0.9612649700598802, + "pass@2": 0.9778384913244379, + "pass@4": 0.9821942192667527, + "pass@8": 0.983871105890422, + "pass@16": 0.9847190114530727, + "pass@32": 0.985008695002592, + "pass@64": 0.985029924450078, + "pass@128": 0.9850299401197605 + }, + "11": { + "pass@1": 0.8986257530120482, + "pass@2": 0.9296826825016609, + "pass@4": 0.9489027942478575, + "pass@8": 0.9620662130010917, + "pass@16": 0.9704384000316348, + "pass@32": 0.9762796497655822, + "pass@64": 0.9813886069516573, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8237166927899686, + "pass@2": 0.8677596543084939, + "pass@4": 0.8935126335109363, + "pass@8": 0.9109319966161393, + "pass@16": 0.9238355493757084, + "pass@32": 0.9343567941155181, + "pass@64": 0.9447863035001147, + "pass@128": 0.9561128526645768 + }, + "13": { + "pass@1": 0.5984768907563025, + "pass@2": 0.6788860721453939, + "pass@4": 0.7344253578071649, + "pass@8": 0.7775856673290666, + "pass@16": 0.8146780213187624, + "pass@32": 0.8475796206853868, + "pass@64": 0.8756093944707887, + "pass@128": 0.8935574229691877 + }, + "14": { + "pass@1": 0.48645376461988304, + "pass@2": 0.5669478404015286, + "pass@4": 0.6318697369517405, + "pass@8": 0.68492196122115, + "pass@16": 0.7292486257783408, + "pass@32": 0.7674189239076338, + "pass@64": 0.7990489164697606, + "pass@128": 0.8216374269005848 + }, + "15": { + "pass@1": 0.38359375, + "pass@2": 0.46019185456931494, + "pass@4": 0.5203419740146114, + "pass@8": 0.5682835301967295, + "pass@16": 0.6074261748472496, + "pass@32": 0.6400943942606238, + "pass@64": 0.6706869139430693, + "pass@128": 0.7 + }, + "16": { + "pass@1": 0.24243233618233617, + "pass@2": 0.3018344662045447, + "pass@4": 0.3575915131335078, + "pass@8": 0.4085050939332824, + "pass@16": 0.4546885384517169, + "pass@32": 0.49601365095000066, + "pass@64": 0.5296128337074691, + "pass@128": 0.5527065527065527 + }, + "17": { + "pass@1": 0.2296195652173913, + "pass@2": 0.29389167726316806, + "pass@4": 0.3527888585937627, + "pass@8": 0.4015310017732799, + "pass@16": 0.4411821281483737, + "pass@32": 0.47576190293681564, + "pass@64": 0.5078436970671439, + "pass@128": 0.5372670807453416 + }, + "18": { + "pass@1": 0.21167279411764706, + "pass@2": 0.26834254863362644, + "pass@4": 0.3236940603012858, + "pass@8": 0.3749530206752505, + "pass@16": 0.4188367625939018, + "pass@32": 0.45438906059280315, + "pass@64": 0.4836671899265589, + "pass@128": 0.5088235294117647 + }, + "19": { + "pass@1": 0.14625186011904762, + "pass@2": 0.1951546828130859, + "pass@4": 0.24635013452559498, + "pass@8": 0.29440603480328803, + "pass@16": 0.3372258333217718, + "pass@32": 0.37594895700766007, + "pass@64": 0.411830564552925, + "pass@128": 0.44642857142857145 + }, + "20": { + "pass@1": 0.1511742140718563, + "pass@2": 0.20507213918619455, + "pass@4": 0.25712532359353274, + "pass@8": 0.3054460965504392, + "pass@16": 0.351170875678362, + "pass@32": 0.39341225491743276, + "pass@64": 0.4319755908517848, + "pass@128": 0.46407185628742514 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 552248, + "answer_accuracy": 0.6893173829685253, + "avg_response_len": 276.4077965230069, + "resp_tokens_sum": 221444659, + "pass_at_k": { + "pass@1": 0.6893173829685253, + "pass@2": 0.7236746211439762, + "pass@4": 0.7508164814845082, + "pass@8": 0.7726614849119308, + "pass@16": 0.790704121070079, + "pass@32": 0.8062121975664928, + "pass@64": 0.8199103369645999, + "pass@128": 0.8324013420674229 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9242537313432836, + "pass@2": 0.9467549065695148, + "pass@4": 0.960208038827236, + "pass@8": 0.9686813621778994, + "pass@16": 0.9736968715149406, + "pass@32": 0.9769121125587591, + "pass@64": 0.9802306191654232, + "pass@128": 0.9850746268656716 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9997494533527697, + "pass@2": 0.9999802719175408, + "pass@4": 0.9999999098144802, + "pass@8": 0.9999999999996636, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999771564327485, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9793658088235294, + "pass@2": 0.982123161764706, + "pass@4": 0.9840028584662212, + "pass@8": 0.9856092514818663, + "pass@16": 0.9870776742919471, + "pass@32": 0.9886067946246756, + "pass@64": 0.9902717923000518, + "pass@128": 0.9911764705882353 + }, + "6": { + "pass@1": 0.9942576892744479, + "pass@2": 0.9967119153481209, + "pass@4": 0.9968445709861974, + "pass@8": 0.9968454258391087, + "pass@16": 0.9968454258675079, + "pass@32": 0.9968454258675079, + "pass@64": 0.9968454258675079, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.9869709256329114, + "pass@2": 0.9900737721768167, + "pass@4": 0.9904884912526759, + "pass@8": 0.9905062765613193, + "pass@16": 0.9905063291137864, + "pass@32": 0.990506329113924, + "pass@64": 0.990506329113924, + "pass@128": 0.990506329113924 + }, + "8": { + "pass@1": 0.9748263888888888, + "pass@2": 0.9825904891111588, + "pass@4": 0.9846056751352027, + "pass@8": 0.9865519125295662, + "pass@16": 0.9890839997485931, + "pass@32": 0.9917298739262829, + "pass@64": 0.9936165397127583, + "pass@128": 0.993993993993994 + }, + "9": { + "pass@1": 0.9695538585209004, + "pass@2": 0.9819578860419779, + "pass@4": 0.9849974913706525, + "pass@8": 0.9868120170628933, + "pass@16": 0.9885209776830189, + "pass@32": 0.9898635914465111, + "pass@64": 0.9903321210779717, + "pass@128": 0.9903536977491961 + }, + "11": { + "pass@1": 0.8884672619047619, + "pass@2": 0.9179161198600174, + "pass@4": 0.9360529898048454, + "pass@8": 0.9490142660043712, + "pass@16": 0.958196320440275, + "pass@32": 0.9647608513937915, + "pass@64": 0.9700160392867752, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.8331660932721713, + "pass@2": 0.8894891220592839, + "pass@4": 0.9224128669696099, + "pass@8": 0.9411316213230453, + "pass@16": 0.9521785605790735, + "pass@32": 0.9594392629935055, + "pass@64": 0.9642912699886997, + "pass@128": 0.9663608562691132 + }, + "13": { + "pass@1": 0.5963059413580247, + "pass@2": 0.6718958849761836, + "pass@4": 0.7258452271012411, + "pass@8": 0.7661379023691128, + "pass@16": 0.7994569530613119, + "pass@32": 0.8298630341960512, + "pass@64": 0.8566462178607112, + "pass@128": 0.8765432098765432 + }, + "14": { + "pass@1": 0.5224406528189911, + "pass@2": 0.6048735805743123, + "pass@4": 0.667954188960801, + "pass@8": 0.714907466749094, + "pass@16": 0.7518824252298536, + "pass@32": 0.7844183856251077, + "pass@64": 0.814024580720177, + "pass@128": 0.8427299703264095 + }, + "15": { + "pass@1": 0.39817226080246915, + "pass@2": 0.46172277510450077, + "pass@4": 0.516345238673869, + "pass@8": 0.5643374864301807, + "pass@16": 0.6066126531536483, + "pass@32": 0.6453878305545641, + "pass@64": 0.6818109357935861, + "pass@128": 0.7160493827160493 + }, + "16": { + "pass@1": 0.2555309734513274, + "pass@2": 0.31953203028824917, + "pass@4": 0.378190404296808, + "pass@8": 0.4293594132098313, + "pass@16": 0.47360209270500697, + "pass@32": 0.5121977074655605, + "pass@64": 0.5497008456356067, + "pass@128": 0.5870206489675516 + }, + "17": { + "pass@1": 0.21279649849397592, + "pass@2": 0.2769127691869842, + "pass@4": 0.3335075267323512, + "pass@8": 0.3784226935397351, + "pass@16": 0.41479042093392043, + "pass@32": 0.44466458523149915, + "pass@64": 0.4693555810200645, + "pass@128": 0.4939759036144578 + }, + "18": { + "pass@1": 0.21242439516129033, + "pass@2": 0.2739847917195834, + "pass@4": 0.33244598044196094, + "pass@8": 0.38322910594677906, + "pass@16": 0.4248379459806415, + "pass@32": 0.458828979622347, + "pass@64": 0.48675039253010505, + "pass@128": 0.5096774193548387 + }, + "19": { + "pass@1": 0.17817692598187312, + "pass@2": 0.23166459012298704, + "pass@4": 0.27631746239424, + "pass@8": 0.31228890701309736, + "pass@16": 0.34476007943757514, + "pass@32": 0.3760017219184328, + "pass@64": 0.40610132605649063, + "pass@128": 0.4350453172205438 + }, + "20": { + "pass@1": 0.15322353603603603, + "pass@2": 0.2071318365609704, + "pass@4": 0.2625266365915972, + "pass@8": 0.3153373034222012, + "pass@16": 0.36097657025856844, + "pass@32": 0.39841696589607256, + "pass@64": 0.42861872538402224, + "pass@128": 0.45645645645645644 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/autoregressive/checkpoint-85656_metrics.json b/SDAR-100M-ep4-block4/composition/autoregressive/checkpoint-85656_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..8be75d16ce77cf6d8b31e65ec9e297b7dea91f68 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/autoregressive/checkpoint-85656_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85656", + "id": { + "answer_accuracy": 0.9929774305555555, + "avg_response_len": 199.76994010416666, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9929774305555555, + "pass@2": 0.9950068350831136, + "pass@4": 0.9962146814668997, + "pass@8": 0.9970507342214603, + "pass@16": 0.9977145566634437, + "pass@32": 0.9983026964922151, + "pass@64": 0.9988362880858953, + "pass@128": 0.9992222222222222 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9498515625, + "pass@2": 0.9612437253937011, + "pass@4": 0.9692006616047992, + "pass@8": 0.9753473828502525, + "pass@16": 0.9806921510790775, + "pass@32": 0.9857609556456873, + "pass@64": 0.9905268983487232, + "pass@128": 0.994 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997703125, + "pass@2": 0.9991603100393702, + "pass@4": 0.999694971784777, + "pass@8": 0.9999281438834521, + "pass@16": 0.9999959201199151, + "pass@32": 0.9999999941392415, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993984375, + "pass@2": 0.9998156988188976, + "pass@4": 0.999972421728534, + "pass@8": 0.9999993800605017, + "pass@16": 0.9999999998409674, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990703125, + "pass@2": 0.9998489173228348, + "pass@4": 0.9999933370828648, + "pass@8": 0.99999998953737, + "pass@16": 0.9999999999999933, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958671875, + "pass@2": 0.9982325295275593, + "pass@4": 0.9993691691976004, + "pass@8": 0.9998216035878049, + "pass@16": 0.9999758853724429, + "pass@32": 0.9999997005598059, + "pass@64": 0.9999999999984025, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949296875, + "pass@2": 0.9967603346456695, + "pass@4": 0.9977015718035245, + "pass@8": 0.9983601080737634, + "pass@16": 0.9987670535585976, + "pass@32": 0.9989636180852017, + "pass@64": 0.9989996944259328, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9498515625, + "2": 1.0, + "3": 0.9999921875, + "4": 0.999984375, + "5": 0.997703125, + "6": 0.9993984375, + "7": 0.9990703125, + "8": 0.9958671875, + "9": 0.9949296875 + }, + "per_op_avg_response_len": { + "10": 280.4086875, + "2": 115.5650078125, + "3": 144.6046875, + "4": 159.7549921875, + "5": 172.636140625, + "6": 198.8291953125, + "7": 223.7235625, + "8": 243.297953125, + "9": 259.109234375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.4086875, + "2": 115.5650078125, + "3": 144.6046875, + "4": 159.7549921875, + "5": 172.636140625, + "6": 198.8291953125, + "7": 223.7235625, + "8": 243.297953125, + "9": 259.109234375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376729, + "answer_accuracy": 0.9939869343127322, + "avg_response_len": 190.30633918017563, + "resp_tokens_sum": 72127625, + "pass_at_k": { + "pass@1": 0.9939869343127322, + "pass@2": 0.9957977742808748, + "pass@4": 0.9967838066910838, + "pass@8": 0.997507734715208, + "pass@16": 0.9981395525497816, + "pass@32": 0.9987116780030536, + "pass@64": 0.9992272260300609, + "pass@128": 0.999662276258021 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9570658185840708, + "pass@2": 0.9683732840917013, + "pass@4": 0.9756881081236524, + "pass@8": 0.9813912893648465, + "pass@16": 0.9867117841326309, + "pass@32": 0.9916970042118012, + "pass@64": 0.9962000480088795, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999755859375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967757936507936, + "pass@2": 0.9981674165729285, + "pass@4": 0.9991240716934194, + "pass@8": 0.999773855222213, + "pass@16": 0.9999870485045957, + "pass@32": 0.9999999813944173, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982664571005917, + "pass@2": 0.9996865972836976, + "pass@4": 0.9999851585674571, + "pass@8": 0.9999999736396237, + "pass@16": 0.9999999999999807, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986979166666666, + "pass@2": 0.9998899814506359, + "pass@4": 0.9999984557218807, + "pass@8": 0.9999999997174955, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965669014084507, + "pass@2": 0.9971723549961184, + "pass@4": 0.9971830946307767, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387468, + "answer_accuracy": 0.991839367627785, + "avg_response_len": 197.7715688482962, + "resp_tokens_sum": 77260650, + "pass_at_k": { + "pass@1": 0.991839367627785, + "pass@2": 0.9938764821828464, + "pass@4": 0.9952322765539793, + "pass@8": 0.9961826036601426, + "pass@16": 0.9969388964253476, + "pass@32": 0.9976471979981268, + "pass@64": 0.9982991620167976, + "pass@128": 0.9986893840104849 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9357026457055214, + "pass@2": 0.9485471716342208, + "pass@4": 0.9583134079099005, + "pass@8": 0.965796860977208, + "pass@16": 0.9720041059256965, + "pass@32": 0.9780843207095543, + "pass@64": 0.9840777547518659, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993206521739131, + "pass@2": 0.9999322435239073, + "pass@4": 0.9999991343473369, + "pass@8": 0.9999999999112675, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995299068767909, + "pass@2": 0.9999576969067977, + "pass@4": 0.9999996293443263, + "pass@8": 0.9999999999871032, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993000361271677, + "pass@2": 0.9998751905921442, + "pass@4": 0.9999952453558912, + "pass@8": 0.9999999955120726, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959727112676057, + "pass@2": 0.9982917683264945, + "pass@4": 0.9995983774035287, + "pass@8": 0.9999709649128528, + "pass@16": 0.9999998400094289, + "pass@32": 0.9999999999990319, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9944330089820359, + "pass@2": 0.9963245261445614, + "pass@4": 0.9975550485081578, + "pass@8": 0.9985324513055662, + "pass@16": 0.9993539825001094, + "pass@32": 0.9998914962254856, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379713, + "answer_accuracy": 0.9931395421827921, + "avg_response_len": 211.1930239370606, + "resp_tokens_sum": 80746696, + "pass_at_k": { + "pass@1": 0.9931395421827921, + "pass@2": 0.9953777309944145, + "pass@4": 0.9966542931192021, + "pass@8": 0.997484733555627, + "pass@16": 0.9980857994579618, + "pass@32": 0.9985670376875161, + "pass@64": 0.9989975674666158, + "pass@128": 0.9993304318714429 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9663844899518158, + "pass@4": 0.9732305133500102, + "pass@8": 0.9785252509761181, + "pass@16": 0.9830552767950438, + "pass@32": 0.9872244258703623, + "pass@64": 0.9910618926107597, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996920955882353, + "pass@2": 0.999296911185734, + "pass@4": 0.999915258386819, + "pass@8": 0.9999981750560808, + "pass@16": 0.9999999995322573, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986691640378549, + "pass@2": 0.9994651816985022, + "pass@4": 0.9999134103702337, + "pass@8": 0.9999980443690937, + "pass@16": 0.9999999994983201, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996785996835443, + "pass@2": 0.9999937705571613, + "pass@4": 0.9999999955503979, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931024774774775, + "pass@2": 0.9966164491262917, + "pass@4": 0.998535216318681, + "pass@8": 0.9994952286242749, + "pass@16": 0.9999277542615488, + "pass@32": 0.9999991007812299, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935942524115756, + "pass@2": 0.9967580607387904, + "pass@4": 0.9984507620831962, + "pass@8": 0.9995185509250936, + "pass@16": 0.9999447697863701, + "pass@32": 0.9999995445205452, + "pass@64": 0.9999999999992755, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4781125, + "avg_response_len": 339.323959375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4781125, + "pass@2": 0.5529424950787339, + "pass@4": 0.6176803829396221, + "pass@8": 0.6719861775834345, + "pass@16": 0.7161999270259896, + "pass@32": 0.7523642026721988, + "pass@64": 0.7828406985303541, + "pass@128": 0.8089 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9316796875, + "pass@2": 0.9470157480314969, + "pass@4": 0.956975347394076, + "pass@8": 0.9640530297112921, + "pass@16": 0.9695196965365632, + "pass@32": 0.9741427538312455, + "pass@64": 0.9777855256541594, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.8436328125, + "pass@2": 0.8805028297244099, + "pass@4": 0.9018184634420694, + "pass@8": 0.91728746008682, + "pass@16": 0.9299946292916604, + "pass@32": 0.9416037330773744, + "pass@64": 0.9533957528471143, + "pass@128": 0.964 + }, + "13": { + "pass@1": 0.690921875, + "pass@2": 0.7605370324803152, + "pass@4": 0.8059136189538799, + "pass@8": 0.8380230859721491, + "pass@16": 0.8624349007355268, + "pass@32": 0.8811619623917599, + "pass@64": 0.8960270574818192, + "pass@128": 0.909 + }, + "14": { + "pass@1": 0.6462734375, + "pass@2": 0.7379026820866139, + "pass@4": 0.8006842908698911, + "pass@8": 0.841684698811715, + "pass@16": 0.8690849272798142, + "pass@32": 0.889385735955718, + "pass@64": 0.9058336603243624, + "pass@128": 0.92 + }, + "15": { + "pass@1": 0.5211953125, + "pass@2": 0.633149729330709, + "pass@4": 0.7199158272403456, + "pass@8": 0.7849333216439665, + "pass@16": 0.8312279471595976, + "pass@32": 0.8627367338301414, + "pass@64": 0.8823570069503487, + "pass@128": 0.894 + }, + "16": { + "pass@1": 0.3261328125, + "pass@2": 0.4267666092519691, + "pass@4": 0.5178725984251967, + "pass@8": 0.5934596861485518, + "pass@16": 0.6541046667364472, + "pass@32": 0.7062095340546287, + "pass@64": 0.7543217872577387, + "pass@128": 0.799 + }, + "17": { + "pass@1": 0.2642734375, + "pass@2": 0.3609981545275593, + "pass@4": 0.4540516601049867, + "pass@8": 0.5339925275657648, + "pass@16": 0.6000821631488521, + "pass@32": 0.6569883152303568, + "pass@64": 0.7087518472835821, + "pass@128": 0.754 + }, + "18": { + "pass@1": 0.226265625, + "pass@2": 0.3107898622047244, + "pass@4": 0.39696340373078376, + "pass@8": 0.47788413159213766, + "pass@16": 0.5471182562347857, + "pass@32": 0.6027876625656634, + "pass@64": 0.6488177025454803, + "pass@128": 0.689 + }, + "19": { + "pass@1": 0.173015625, + "pass@2": 0.24631409940944873, + "pass@4": 0.32354224221972244, + "pass@8": 0.3961275565048202, + "pass@16": 0.4602082281492824, + "pass@32": 0.5160631421848889, + "pass@64": 0.5648830568956981, + "pass@128": 0.607 + }, + "20": { + "pass@1": 0.157734375, + "pass@2": 0.22544820374015714, + "pass@4": 0.2990663770153726, + "pass@8": 0.37241627779712855, + "pass@16": 0.43822385498722355, + "pass@32": 0.49256245360018824, + "pass@64": 0.5362335880631475, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "11": 0.9316796875, + "12": 0.8436328125, + "13": 0.690921875, + "14": 0.6462734375, + "15": 0.5211953125, + "16": 0.3261328125, + "17": 0.2642734375, + "18": 0.226265625, + "19": 0.173015625, + "20": 0.157734375 + }, + "per_op_avg_response_len": { + "11": 297.4695390625, + "12": 306.54790625, + "13": 321.9113125, + "14": 343.8243046875, + "15": 351.8442890625, + "16": 352.2420703125, + "17": 359.2610234375, + "18": 354.801859375, + "19": 353.430875, + "20": 351.9064140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 297.4695390625, + "12": 306.54790625, + "13": 321.9113125, + "14": 343.8243046875, + "15": 351.8442890625, + "16": 352.2420703125, + "17": 359.2610234375, + "18": 354.801859375, + "19": 353.430875, + "20": 351.9064140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 205545, + "answer_accuracy": 0.47721257429420505, + "avg_response_len": 322.7680859955423, + "resp_tokens_sum": 139022670, + "pass_at_k": { + "pass@1": 0.47721257429420505, + "pass@2": 0.5505735132384112, + "pass@4": 0.6141749104263174, + "pass@8": 0.6681048464613938, + "pass@16": 0.7128412924487995, + "pass@32": 0.7499614160859973, + "pass@64": 0.7808582526897707, + "pass@128": 0.8056463595839525 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9157445113314447, + "pass@2": 0.9303819483170127, + "pass@4": 0.9399434625381455, + "pass@8": 0.9476715847990214, + "pass@16": 0.954937662674825, + "pass@32": 0.9618599495743576, + "pass@64": 0.9670685637911212, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.8374382062146892, + "pass@2": 0.8719144812936519, + "pass@4": 0.8911755683611579, + "pass@8": 0.9053322956848698, + "pass@16": 0.9172509395306903, + "pass@32": 0.9285946036232826, + "pass@64": 0.9412245539437281, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.7017535266457681, + "pass@2": 0.7656527688643157, + "pass@4": 0.8069338072552844, + "pass@8": 0.836520332654227, + "pass@16": 0.8591068301655219, + "pass@32": 0.8770989104903102, + "pass@64": 0.8922208279506292, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.6434968847352025, + "pass@2": 0.7339530134667749, + "pass@4": 0.7953253649952631, + "pass@8": 0.8356009050741141, + "pass@16": 0.8650482090662411, + "pass@32": 0.8884090804371552, + "pass@64": 0.9065243627350636, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.5236407153179191, + "pass@2": 0.6345384967684675, + "pass@4": 0.720673474883558, + "pass@8": 0.7874274695235931, + "pass@16": 0.836765843937725, + "pass@32": 0.870818671155785, + "pass@64": 0.8910456399594974, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.31060987903225806, + "pass@2": 0.41360133032766044, + "pass@4": 0.5086247351137556, + "pass@8": 0.5870659401872468, + "pass@16": 0.6479544799527384, + "pass@32": 0.6986068415947772, + "pass@64": 0.7455048157861739, + "pass@128": 0.7903225806451613 + }, + "17": { + "pass@1": 0.2576770231213873, + "pass@2": 0.3527515674070362, + "pass@4": 0.4467490030147967, + "pass@8": 0.5316486877319969, + "pass@16": 0.6051723920177607, + "pass@32": 0.6689063240851189, + "pass@64": 0.7234904728932069, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.2110267857142857, + "pass@2": 0.29613364735658004, + "pass@4": 0.384165431463924, + "pass@8": 0.4662889678969746, + "pass@16": 0.5357500897786317, + "pass@32": 0.5902205339958437, + "pass@64": 0.6345563661728516, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.173681493993994, + "pass@2": 0.24716990612659911, + "pass@4": 0.32139107752296747, + "pass@8": 0.3897048009291002, + "pass@16": 0.4521456987554777, + "pass@32": 0.5096928722985181, + "pass@64": 0.5592708746194527, + "pass@128": 0.5975975975975976 + }, + "20": { + "pass@1": 0.17227383633633633, + "pass@2": 0.24018629850322742, + "pass@4": 0.31119543052613924, + "pass@8": 0.3805273280744088, + "pass@16": 0.44240840872374637, + "pass@32": 0.4948028929687008, + "pass@64": 0.538554484333375, + "pass@128": 0.5735735735735735 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 208656, + "answer_accuracy": 0.4847234611953613, + "avg_response_len": 335.9651167112697, + "resp_tokens_sum": 144620888, + "pass_at_k": { + "pass@1": 0.4847234611953613, + "pass@2": 0.5609376865776503, + "pass@4": 0.626486374974763, + "pass@8": 0.6807188703564199, + "pass@16": 0.7239437820581853, + "pass@32": 0.7585821692071032, + "pass@64": 0.7883349119708191, + "pass@128": 0.8153434433541481 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9468655873493976, + "pass@2": 0.9604431505549762, + "pass@4": 0.9680644524103162, + "pass@8": 0.9726580226305667, + "pass@16": 0.9756575108892106, + "pass@32": 0.9783564878872142, + "pass@64": 0.9817601740575576, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8422070924764891, + "pass@2": 0.8776021739935335, + "pass@4": 0.8994707533619423, + "pass@8": 0.9159269312012803, + "pass@16": 0.9295581121779324, + "pass@32": 0.9422592174940747, + "pass@64": 0.9548234063200668, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7061887254901961, + "pass@2": 0.7791781082511747, + "pass@4": 0.8260199885728562, + "pass@8": 0.8570431299381749, + "pass@16": 0.8790524526209159, + "pass@32": 0.8947079171348, + "pass@64": 0.9077045369983032, + "pass@128": 0.9215686274509803 + }, + "14": { + "pass@1": 0.6597222222222222, + "pass@2": 0.7535168301330757, + "pass@4": 0.8180587859631577, + "pass@8": 0.8601767977854613, + "pass@16": 0.8856046539907487, + "pass@32": 0.9014211318157292, + "pass@64": 0.9136951455705985, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.5109611742424243, + "pass@2": 0.6282312544738723, + "pass@4": 0.7217386031291542, + "pass@8": 0.7923839668825581, + "pass@16": 0.8425049221743166, + "pass@32": 0.8757183743773951, + "pass@64": 0.8958328669936695, + "pass@128": 0.9090909090909091 + }, + "16": { + "pass@1": 0.3457532051282051, + "pass@2": 0.4452947988873186, + "pass@4": 0.532903049779034, + "pass@8": 0.6048692395044325, + "pass@16": 0.6625327790912872, + "pass@32": 0.7115800723039244, + "pass@64": 0.757156872133029, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.2784355590062112, + "pass@2": 0.3800656881205065, + "pass@4": 0.477290026246719, + "pass@8": 0.5595626443779531, + "pass@16": 0.625599140698911, + "pass@32": 0.6790160770546658, + "pass@64": 0.7283353115816313, + "pass@128": 0.7763975155279503 + }, + "18": { + "pass@1": 0.23963694852941175, + "pass@2": 0.3307752431681334, + "pass@4": 0.42159714759919714, + "pass@8": 0.5055019564772302, + "pass@16": 0.5780629837271026, + "pass@32": 0.6386943668162116, + "pass@64": 0.6911164787819537, + "pass@128": 0.7352941176470589 + }, + "19": { + "pass@1": 0.17643229166666666, + "pass@2": 0.24522813554555678, + "pass@4": 0.3185077649334011, + "pass@8": 0.3868667149725815, + "pass@16": 0.4454668435699914, + "pass@32": 0.4963013158912973, + "pass@64": 0.5420733465747838, + "pass@128": 0.5833333333333334 + }, + "20": { + "pass@1": 0.1466364146706587, + "pass@2": 0.2122326459286152, + "pass@4": 0.2832578821547007, + "pass@8": 0.35382239266907534, + "pass@16": 0.41688701623004404, + "pass@32": 0.46919719938448934, + "pass@64": 0.5122032313480497, + "pass@128": 0.5508982035928144 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 197783, + "answer_accuracy": 0.47224318077628363, + "avg_response_len": 359.80265796913204, + "resp_tokens_sum": 150691110, + "pass_at_k": { + "pass@1": 0.47224318077628363, + "pass@2": 0.5471612587957022, + "pass@4": 0.6122345894778913, + "pass@8": 0.6670022635950903, + "pass@16": 0.7116947989327357, + "pass@32": 0.7484443846421354, + "pass@64": 0.7792324743412167, + "pass@128": 0.8056234718826406 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.933531746031746, + "pass@2": 0.9515041088613923, + "pass@4": 0.9643743076758263, + "pass@8": 0.9733412595679051, + "pass@16": 0.9793917714194672, + "pass@32": 0.9834661512791815, + "pass@64": 0.9856061741231261, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8517297400611621, + "pass@2": 0.892629999518409, + "pass@4": 0.9156304034243424, + "pass@8": 0.9315569919301774, + "pass@16": 0.9442163881132603, + "pass@32": 0.9550475627343186, + "pass@64": 0.9651791869569815, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.6634355709876543, + "pass@2": 0.7349605387868178, + "pass@4": 0.7827549337004169, + "pass@8": 0.8185453779861791, + "pass@16": 0.8474015318736398, + "pass@32": 0.8702366775562574, + "pass@64": 0.8869076964727599, + "pass@128": 0.8981481481481481 + }, + "14": { + "pass@1": 0.6352698442136498, + "pass@2": 0.7258190173835839, + "pass@4": 0.7881565101098262, + "pass@8": 0.828713185282778, + "pass@16": 0.8561651647024184, + "pass@32": 0.8781020595086385, + "pass@64": 0.8971976264132405, + "pass@128": 0.913946587537092 + }, + "15": { + "pass@1": 0.5290075231481481, + "pass@2": 0.6366762205939533, + "pass@4": 0.7172502033889275, + "pass@8": 0.7746811978937012, + "pass@16": 0.8138282124679664, + "pass@32": 0.8408839816842546, + "pass@64": 0.8593529920260841, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.3200129056047198, + "pass@2": 0.419621600120781, + "pass@4": 0.510766843856907, + "pass@8": 0.587493043139971, + "pass@16": 0.6510022787317294, + "pass@32": 0.7076012028954857, + "pass@64": 0.7594490626705939, + "pass@128": 0.8053097345132744 + }, + "17": { + "pass@1": 0.2574124623493976, + "pass@2": 0.35109927900578686, + "pass@4": 0.4391238452121193, + "pass@8": 0.5116352714481722, + "pass@16": 0.5700288921857198, + "pass@32": 0.6232034647147692, + "pass@64": 0.6743981124495337, + "pass@128": 0.7198795180722891 + }, + "18": { + "pass@1": 0.22880544354838708, + "pass@2": 0.3054177514605029, + "pass@4": 0.3843950726925263, + "pass@8": 0.46068492782560816, + "pass@16": 0.5260139043388721, + "pass@32": 0.5775948095148585, + "pass@64": 0.6185270051584428, + "pass@128": 0.6548387096774193 + }, + "19": { + "pass@1": 0.16887745468277945, + "pass@2": 0.2465554898065991, + "pass@4": 0.33081693168263315, + "pass@8": 0.4119898536696145, + "pass@16": 0.4832835378374356, + "pass@32": 0.542532246435064, + "pass@64": 0.5936834175175014, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.1543262012012012, + "pass@2": 0.22396535314842403, + "pass@4": 0.30279329132281885, + "pass@8": 0.38295495014077957, + "pass@16": 0.4554402145987438, + "pass@32": 0.513757434375952, + "pass@64": 0.5580152117413981, + "pass@128": 0.5945945945945946 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7219958881578947, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7219958881578947, + "pass@2": 0.7623413929755306, + "pass@4": 0.7969861032946366, + "pass@8": 0.8259641254646152, + "pass@16": 0.8495489621174196, + "pass@32": 0.8688613839553769, + "pass@64": 0.8851543988461342, + "pass@128": 0.8990526315789473 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9498515625, + "pass@2": 0.9612437253937011, + "pass@4": 0.9692006616047992, + "pass@8": 0.9753473828502525, + "pass@16": 0.9806921510790775, + "pass@32": 0.9857609556456873, + "pass@64": 0.9905268983487232, + "pass@128": 0.994 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999921875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997703125, + "pass@2": 0.9991603100393702, + "pass@4": 0.999694971784777, + "pass@8": 0.9999281438834521, + "pass@16": 0.9999959201199151, + "pass@32": 0.9999999941392415, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9993984375, + "pass@2": 0.9998156988188976, + "pass@4": 0.999972421728534, + "pass@8": 0.9999993800605017, + "pass@16": 0.9999999998409674, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9990703125, + "pass@2": 0.9998489173228348, + "pass@4": 0.9999933370828648, + "pass@8": 0.99999998953737, + "pass@16": 0.9999999999999933, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9958671875, + "pass@2": 0.9982325295275593, + "pass@4": 0.9993691691976004, + "pass@8": 0.9998216035878049, + "pass@16": 0.9999758853724429, + "pass@32": 0.9999997005598059, + "pass@64": 0.9999999999984025, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9949296875, + "pass@2": 0.9967603346456695, + "pass@4": 0.9977015718035245, + "pass@8": 0.9983601080737634, + "pass@16": 0.9987670535585976, + "pass@32": 0.9989636180852017, + "pass@64": 0.9989996944259328, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9316796875, + "pass@2": 0.9470157480314969, + "pass@4": 0.956975347394076, + "pass@8": 0.9640530297112921, + "pass@16": 0.9695196965365632, + "pass@32": 0.9741427538312455, + "pass@64": 0.9777855256541594, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.8436328125, + "pass@2": 0.8805028297244099, + "pass@4": 0.9018184634420694, + "pass@8": 0.91728746008682, + "pass@16": 0.9299946292916604, + "pass@32": 0.9416037330773744, + "pass@64": 0.9533957528471143, + "pass@128": 0.964 + }, + "13": { + "pass@1": 0.690921875, + "pass@2": 0.7605370324803152, + "pass@4": 0.8059136189538799, + "pass@8": 0.8380230859721491, + "pass@16": 0.8624349007355268, + "pass@32": 0.8811619623917599, + "pass@64": 0.8960270574818192, + "pass@128": 0.909 + }, + "14": { + "pass@1": 0.6462734375, + "pass@2": 0.7379026820866139, + "pass@4": 0.8006842908698911, + "pass@8": 0.841684698811715, + "pass@16": 0.8690849272798142, + "pass@32": 0.889385735955718, + "pass@64": 0.9058336603243624, + "pass@128": 0.92 + }, + "15": { + "pass@1": 0.5211953125, + "pass@2": 0.633149729330709, + "pass@4": 0.7199158272403456, + "pass@8": 0.7849333216439665, + "pass@16": 0.8312279471595976, + "pass@32": 0.8627367338301414, + "pass@64": 0.8823570069503487, + "pass@128": 0.894 + }, + "16": { + "pass@1": 0.3261328125, + "pass@2": 0.4267666092519691, + "pass@4": 0.5178725984251967, + "pass@8": 0.5934596861485518, + "pass@16": 0.6541046667364472, + "pass@32": 0.7062095340546287, + "pass@64": 0.7543217872577387, + "pass@128": 0.799 + }, + "17": { + "pass@1": 0.2642734375, + "pass@2": 0.3609981545275593, + "pass@4": 0.4540516601049867, + "pass@8": 0.5339925275657648, + "pass@16": 0.6000821631488521, + "pass@32": 0.6569883152303568, + "pass@64": 0.7087518472835821, + "pass@128": 0.754 + }, + "18": { + "pass@1": 0.226265625, + "pass@2": 0.3107898622047244, + "pass@4": 0.39696340373078376, + "pass@8": 0.47788413159213766, + "pass@16": 0.5471182562347857, + "pass@32": 0.6027876625656634, + "pass@64": 0.6488177025454803, + "pass@128": 0.689 + }, + "19": { + "pass@1": 0.173015625, + "pass@2": 0.24631409940944873, + "pass@4": 0.32354224221972244, + "pass@8": 0.3961275565048202, + "pass@16": 0.4602082281492824, + "pass@32": 0.5160631421848889, + "pass@64": 0.5648830568956981, + "pass@128": 0.607 + }, + "20": { + "pass@1": 0.157734375, + "pass@2": 0.22544820374015714, + "pass@4": 0.2990663770153726, + "pass@8": 0.37241627779712855, + "pass@16": 0.43822385498722355, + "pass@32": 0.49256245360018824, + "pass@64": 0.5362335880631475, + "pass@128": 0.573 + } + }, + "per_op_accuracy": { + "10": 0.9498515625, + "2": 1.0, + "3": 0.9999921875, + "4": 0.999984375, + "5": 0.997703125, + "6": 0.9993984375, + "7": 0.9990703125, + "8": 0.9958671875, + "9": 0.9949296875, + "11": 0.9316796875, + "12": 0.8436328125, + "13": 0.690921875, + "14": 0.6462734375, + "15": 0.5211953125, + "16": 0.3261328125, + "17": 0.2642734375, + "18": 0.226265625, + "19": 0.173015625, + "20": 0.157734375 + }, + "per_op_avg_response_len": { + "10": 280.4086875, + "2": 115.5650078125, + "3": 144.6046875, + "4": 159.7549921875, + "5": 172.636140625, + "6": 198.8291953125, + "7": 223.7235625, + "8": 243.297953125, + "9": 259.109234375, + "11": 297.4695390625, + "12": 306.54790625, + "13": 321.9113125, + "14": 343.8243046875, + "15": 351.8442890625, + "16": 352.2420703125, + "17": 359.2610234375, + "18": 354.801859375, + "19": 353.430875, + "20": 351.9064140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 280.4086875, + "2": 115.5650078125, + "3": 144.6046875, + "4": 159.7549921875, + "5": 172.636140625, + "6": 198.8291953125, + "7": 223.7235625, + "8": 243.297953125, + "9": 259.109234375, + "11": 297.4695390625, + "12": 306.54790625, + "13": 321.9113125, + "14": 343.8243046875, + "15": 351.8442890625, + "16": 352.2420703125, + "17": 359.2610234375, + "18": 354.801859375, + "19": 353.430875, + "20": 351.9064140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 582274, + "answer_accuracy": 0.7190982650964275, + "avg_response_len": 260.7669427264464, + "resp_tokens_sum": 211150295, + "pass_at_k": { + "pass@1": 0.7190982650964275, + "pass@2": 0.7589688715922986, + "pass@4": 0.7932620020861227, + "pass@8": 0.8222878929551545, + "pass@16": 0.8463803610796929, + "pass@32": 0.8663935257186914, + "pass@64": 0.8830698445425451, + "pass@128": 0.8964590578564654 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9570658185840708, + "pass@2": 0.9683732840917013, + "pass@4": 0.9756881081236524, + "pass@8": 0.9813912893648465, + "pass@16": 0.9867117841326309, + "pass@32": 0.9916970042118012, + "pass@64": 0.9962000480088795, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999755859375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967757936507936, + "pass@2": 0.9981674165729285, + "pass@4": 0.9991240716934194, + "pass@8": 0.999773855222213, + "pass@16": 0.9999870485045957, + "pass@32": 0.9999999813944173, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982664571005917, + "pass@2": 0.9996865972836976, + "pass@4": 0.9999851585674571, + "pass@8": 0.9999999736396237, + "pass@16": 0.9999999999999807, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986979166666666, + "pass@2": 0.9998899814506359, + "pass@4": 0.9999984557218807, + "pass@8": 0.9999999997174955, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965669014084507, + "pass@2": 0.9971723549961184, + "pass@4": 0.9971830946307767, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9157445113314447, + "pass@2": 0.9303819483170127, + "pass@4": 0.9399434625381455, + "pass@8": 0.9476715847990214, + "pass@16": 0.954937662674825, + "pass@32": 0.9618599495743576, + "pass@64": 0.9670685637911212, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.8374382062146892, + "pass@2": 0.8719144812936519, + "pass@4": 0.8911755683611579, + "pass@8": 0.9053322956848698, + "pass@16": 0.9172509395306903, + "pass@32": 0.9285946036232826, + "pass@64": 0.9412245539437281, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.7017535266457681, + "pass@2": 0.7656527688643157, + "pass@4": 0.8069338072552844, + "pass@8": 0.836520332654227, + "pass@16": 0.8591068301655219, + "pass@32": 0.8770989104903102, + "pass@64": 0.8922208279506292, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.6434968847352025, + "pass@2": 0.7339530134667749, + "pass@4": 0.7953253649952631, + "pass@8": 0.8356009050741141, + "pass@16": 0.8650482090662411, + "pass@32": 0.8884090804371552, + "pass@64": 0.9065243627350636, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.5236407153179191, + "pass@2": 0.6345384967684675, + "pass@4": 0.720673474883558, + "pass@8": 0.7874274695235931, + "pass@16": 0.836765843937725, + "pass@32": 0.870818671155785, + "pass@64": 0.8910456399594974, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.31060987903225806, + "pass@2": 0.41360133032766044, + "pass@4": 0.5086247351137556, + "pass@8": 0.5870659401872468, + "pass@16": 0.6479544799527384, + "pass@32": 0.6986068415947772, + "pass@64": 0.7455048157861739, + "pass@128": 0.7903225806451613 + }, + "17": { + "pass@1": 0.2576770231213873, + "pass@2": 0.3527515674070362, + "pass@4": 0.4467490030147967, + "pass@8": 0.5316486877319969, + "pass@16": 0.6051723920177607, + "pass@32": 0.6689063240851189, + "pass@64": 0.7234904728932069, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.2110267857142857, + "pass@2": 0.29613364735658004, + "pass@4": 0.384165431463924, + "pass@8": 0.4662889678969746, + "pass@16": 0.5357500897786317, + "pass@32": 0.5902205339958437, + "pass@64": 0.6345563661728516, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.173681493993994, + "pass@2": 0.24716990612659911, + "pass@4": 0.32139107752296747, + "pass@8": 0.3897048009291002, + "pass@16": 0.4521456987554777, + "pass@32": 0.5096928722985181, + "pass@64": 0.5592708746194527, + "pass@128": 0.5975975975975976 + }, + "20": { + "pass@1": 0.17227383633633633, + "pass@2": 0.24018629850322742, + "pass@4": 0.31119543052613924, + "pass@8": 0.3805273280744088, + "pass@16": 0.44240840872374637, + "pass@32": 0.4948028929687008, + "pass@64": 0.538554484333375, + "pass@128": 0.5735735735735735 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 596124, + "answer_accuracy": 0.7259888932190179, + "avg_response_len": 270.2181629968823, + "resp_tokens_sum": 221881538, + "pass_at_k": { + "pass@1": 0.7259888932190179, + "pass@2": 0.766912620979365, + "pass@4": 0.8019209021173521, + "pass@8": 0.8308038764426154, + "pass@16": 0.8538239206472131, + "pass@32": 0.8723197323980967, + "pass@64": 0.8882274904806203, + "pass@128": 0.9025720966484801 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9357026457055214, + "pass@2": 0.9485471716342208, + "pass@4": 0.9583134079099005, + "pass@8": 0.965796860977208, + "pass@16": 0.9720041059256965, + "pass@32": 0.9780843207095543, + "pass@64": 0.9840777547518659, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993206521739131, + "pass@2": 0.9999322435239073, + "pass@4": 0.9999991343473369, + "pass@8": 0.9999999999112675, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995299068767909, + "pass@2": 0.9999576969067977, + "pass@4": 0.9999996293443263, + "pass@8": 0.9999999999871032, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993000361271677, + "pass@2": 0.9998751905921442, + "pass@4": 0.9999952453558912, + "pass@8": 0.9999999955120726, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959727112676057, + "pass@2": 0.9982917683264945, + "pass@4": 0.9995983774035287, + "pass@8": 0.9999709649128528, + "pass@16": 0.9999998400094289, + "pass@32": 0.9999999999990319, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9944330089820359, + "pass@2": 0.9963245261445614, + "pass@4": 0.9975550485081578, + "pass@8": 0.9985324513055662, + "pass@16": 0.9993539825001094, + "pass@32": 0.9998914962254856, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9468655873493976, + "pass@2": 0.9604431505549762, + "pass@4": 0.9680644524103162, + "pass@8": 0.9726580226305667, + "pass@16": 0.9756575108892106, + "pass@32": 0.9783564878872142, + "pass@64": 0.9817601740575576, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8422070924764891, + "pass@2": 0.8776021739935335, + "pass@4": 0.8994707533619423, + "pass@8": 0.9159269312012803, + "pass@16": 0.9295581121779324, + "pass@32": 0.9422592174940747, + "pass@64": 0.9548234063200668, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7061887254901961, + "pass@2": 0.7791781082511747, + "pass@4": 0.8260199885728562, + "pass@8": 0.8570431299381749, + "pass@16": 0.8790524526209159, + "pass@32": 0.8947079171348, + "pass@64": 0.9077045369983032, + "pass@128": 0.9215686274509803 + }, + "14": { + "pass@1": 0.6597222222222222, + "pass@2": 0.7535168301330757, + "pass@4": 0.8180587859631577, + "pass@8": 0.8601767977854613, + "pass@16": 0.8856046539907487, + "pass@32": 0.9014211318157292, + "pass@64": 0.9136951455705985, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.5109611742424243, + "pass@2": 0.6282312544738723, + "pass@4": 0.7217386031291542, + "pass@8": 0.7923839668825581, + "pass@16": 0.8425049221743166, + "pass@32": 0.8757183743773951, + "pass@64": 0.8958328669936695, + "pass@128": 0.9090909090909091 + }, + "16": { + "pass@1": 0.3457532051282051, + "pass@2": 0.4452947988873186, + "pass@4": 0.532903049779034, + "pass@8": 0.6048692395044325, + "pass@16": 0.6625327790912872, + "pass@32": 0.7115800723039244, + "pass@64": 0.757156872133029, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.2784355590062112, + "pass@2": 0.3800656881205065, + "pass@4": 0.477290026246719, + "pass@8": 0.5595626443779531, + "pass@16": 0.625599140698911, + "pass@32": 0.6790160770546658, + "pass@64": 0.7283353115816313, + "pass@128": 0.7763975155279503 + }, + "18": { + "pass@1": 0.23963694852941175, + "pass@2": 0.3307752431681334, + "pass@4": 0.42159714759919714, + "pass@8": 0.5055019564772302, + "pass@16": 0.5780629837271026, + "pass@32": 0.6386943668162116, + "pass@64": 0.6911164787819537, + "pass@128": 0.7352941176470589 + }, + "19": { + "pass@1": 0.17643229166666666, + "pass@2": 0.24522813554555678, + "pass@4": 0.3185077649334011, + "pass@8": 0.3868667149725815, + "pass@16": 0.4454668435699914, + "pass@32": 0.4963013158912973, + "pass@64": 0.5420733465747838, + "pass@128": 0.5833333333333334 + }, + "20": { + "pass@1": 0.1466364146706587, + "pass@2": 0.2122326459286152, + "pass@4": 0.2832578821547007, + "pass@8": 0.35382239266907534, + "pass@16": 0.41688701623004404, + "pass@32": 0.46919719938448934, + "pass@64": 0.5122032313480497, + "pass@128": 0.5508982035928144 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 577496, + "answer_accuracy": 0.7208320019172392, + "avg_response_len": 288.8812684733983, + "resp_tokens_sum": 231437806, + "pass_at_k": { + "pass@1": 0.7208320019172392, + "pass@2": 0.7610648540117928, + "pass@4": 0.7956922751747378, + "pass@8": 0.824719333058601, + "pass@16": 0.8483699736521619, + "pass@32": 0.8678111148940243, + "pass@64": 0.8841115817330713, + "pass@128": 0.8980667838312829 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9663844899518158, + "pass@4": 0.9732305133500102, + "pass@8": 0.9785252509761181, + "pass@16": 0.9830552767950438, + "pass@32": 0.9872244258703623, + "pass@64": 0.9910618926107597, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996920955882353, + "pass@2": 0.999296911185734, + "pass@4": 0.999915258386819, + "pass@8": 0.9999981750560808, + "pass@16": 0.9999999995322573, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986691640378549, + "pass@2": 0.9994651816985022, + "pass@4": 0.9999134103702337, + "pass@8": 0.9999980443690937, + "pass@16": 0.9999999994983201, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996785996835443, + "pass@2": 0.9999937705571613, + "pass@4": 0.9999999955503979, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931024774774775, + "pass@2": 0.9966164491262917, + "pass@4": 0.998535216318681, + "pass@8": 0.9994952286242749, + "pass@16": 0.9999277542615488, + "pass@32": 0.9999991007812299, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935942524115756, + "pass@2": 0.9967580607387904, + "pass@4": 0.9984507620831962, + "pass@8": 0.9995185509250936, + "pass@16": 0.9999447697863701, + "pass@32": 0.9999995445205452, + "pass@64": 0.9999999999992755, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.933531746031746, + "pass@2": 0.9515041088613923, + "pass@4": 0.9643743076758263, + "pass@8": 0.9733412595679051, + "pass@16": 0.9793917714194672, + "pass@32": 0.9834661512791815, + "pass@64": 0.9856061741231261, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8517297400611621, + "pass@2": 0.892629999518409, + "pass@4": 0.9156304034243424, + "pass@8": 0.9315569919301774, + "pass@16": 0.9442163881132603, + "pass@32": 0.9550475627343186, + "pass@64": 0.9651791869569815, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.6634355709876543, + "pass@2": 0.7349605387868178, + "pass@4": 0.7827549337004169, + "pass@8": 0.8185453779861791, + "pass@16": 0.8474015318736398, + "pass@32": 0.8702366775562574, + "pass@64": 0.8869076964727599, + "pass@128": 0.8981481481481481 + }, + "14": { + "pass@1": 0.6352698442136498, + "pass@2": 0.7258190173835839, + "pass@4": 0.7881565101098262, + "pass@8": 0.828713185282778, + "pass@16": 0.8561651647024184, + "pass@32": 0.8781020595086385, + "pass@64": 0.8971976264132405, + "pass@128": 0.913946587537092 + }, + "15": { + "pass@1": 0.5290075231481481, + "pass@2": 0.6366762205939533, + "pass@4": 0.7172502033889275, + "pass@8": 0.7746811978937012, + "pass@16": 0.8138282124679664, + "pass@32": 0.8408839816842546, + "pass@64": 0.8593529920260841, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.3200129056047198, + "pass@2": 0.419621600120781, + "pass@4": 0.510766843856907, + "pass@8": 0.587493043139971, + "pass@16": 0.6510022787317294, + "pass@32": 0.7076012028954857, + "pass@64": 0.7594490626705939, + "pass@128": 0.8053097345132744 + }, + "17": { + "pass@1": 0.2574124623493976, + "pass@2": 0.35109927900578686, + "pass@4": 0.4391238452121193, + "pass@8": 0.5116352714481722, + "pass@16": 0.5700288921857198, + "pass@32": 0.6232034647147692, + "pass@64": 0.6743981124495337, + "pass@128": 0.7198795180722891 + }, + "18": { + "pass@1": 0.22880544354838708, + "pass@2": 0.3054177514605029, + "pass@4": 0.3843950726925263, + "pass@8": 0.46068492782560816, + "pass@16": 0.5260139043388721, + "pass@32": 0.5775948095148585, + "pass@64": 0.6185270051584428, + "pass@128": 0.6548387096774193 + }, + "19": { + "pass@1": 0.16887745468277945, + "pass@2": 0.2465554898065991, + "pass@4": 0.33081693168263315, + "pass@8": 0.4119898536696145, + "pass@16": 0.4832835378374356, + "pass@32": 0.542532246435064, + "pass@64": 0.5936834175175014, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.1543262012012012, + "pass@2": 0.22396535314842403, + "pass@4": 0.30279329132281885, + "pass@8": 0.38295495014077957, + "pass@16": 0.4554402145987438, + "pass@32": 0.513757434375952, + "pass@64": 0.5580152117413981, + "pass@128": 0.5945945945945946 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl b/SDAR-100M-ep4-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3f76fbf3159b3664e69213ddbf52e89cf3118bea --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl @@ -0,0 +1 @@ +{"gold_dir": "results_decrypted/composition/op-difficulty-10B/graphs", "generated_path": "results/SDAR-100M-ep4-block4/composition/autoregressive", "results_dir": "results/SDAR-100M-ep4-block4/composition/autoregressive/graphs", "per_file": [{"file": "results/SDAR-100M-ep4-block4/composition/autoregressive/checkpoint-85656_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1024297, "value_mismatch": 9155, "dependency_mismatch": 125927, "answer_mismatch": 8090, "extra_nodes": 9195, "missing_nodes": 9144, "perfect_with_extra": 245, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 337884, "value_mismatch": 2699, "answer_mismatch": 2279, "dependency_mismatch": 40588, "extra_nodes": 2936, "missing_nodes": 2796, "perfect_with_extra": 41}, "teachers_in_school": {"total": 390656, "perfect": 346816, "value_mismatch": 3368, "dependency_mismatch": 43244, "answer_mismatch": 3188, "extra_nodes": 3434, "perfect_with_extra": 158, "missing_nodes": 3477}, "movie_festival_awards": {"total": 382336, "perfect": 339597, "dependency_mismatch": 42095, "missing_nodes": 2871, "value_mismatch": 3088, "answer_mismatch": 2623, "extra_nodes": 2825, "perfect_with_extra": 46}}, "results_path": "results/SDAR-100M-ep4-block4/composition/autoregressive/graphs/checkpoint-85656_id_generations.jsonl.graph_results.jsonl"}, {"file": "results/SDAR-100M-ep4-block4/composition/autoregressive/checkpoint-85656_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 284221, "value_mismatch": 793692, "dependency_mismatch": 961034, "answer_mismatch": 668015, "extra_nodes": 62311, "missing_nodes": 600302, "perfect_with_extra": 60, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 268415, "dependency_mismatch": 324348, "answer_mismatch": 225174, "extra_nodes": 22247, "missing_nodes": 206424, "perfect": 95236, "perfect_with_extra": 30}, "movie_festival_awards": {"total": 418816, "perfect": 92294, "value_mismatch": 260812, "dependency_mismatch": 315791, "answer_mismatch": 221033, "missing_nodes": 196831, "extra_nodes": 20877, "perfect_with_extra": 26}, "teachers_in_school": {"total": 430464, "perfect": 96691, "value_mismatch": 264465, "dependency_mismatch": 320895, "answer_mismatch": 221808, "extra_nodes": 19187, "missing_nodes": 197047, "perfect_with_extra": 4}}, "results_path": "results/SDAR-100M-ep4-block4/composition/autoregressive/graphs/checkpoint-85656_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1308518, "value_mismatch": 802847, "dependency_mismatch": 1086961, "answer_mismatch": 676105, "extra_nodes": 71506, "missing_nodes": 609446, "perfect_with_extra": 305, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "results/SDAR-100M-ep4-block4/composition/autoregressive/graphs/autoregressive_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 433120, "value_mismatch": 271114, "answer_mismatch": 227453, "dependency_mismatch": 364936, "extra_nodes": 25183, "missing_nodes": 209220, "perfect_with_extra": 71}, "teachers_in_school": {"total": 821120, "perfect": 443507, "value_mismatch": 267833, "dependency_mismatch": 364139, "answer_mismatch": 224996, "extra_nodes": 22621, "perfect_with_extra": 162, "missing_nodes": 200524}, "movie_festival_awards": {"total": 801152, "perfect": 431891, "dependency_mismatch": 357886, "missing_nodes": 199702, "value_mismatch": 263900, "answer_mismatch": 223656, "extra_nodes": 23702, "perfect_with_extra": 72}}}} diff --git a/SDAR-100M-ep4-block4/composition/autoregressive/graphs/results.json b/SDAR-100M-ep4-block4/composition/autoregressive/graphs/results.json new file mode 100644 index 0000000000000000000000000000000000000000..e6b258b7241ced50a781d20ccae8c7bc7ec24bad --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/autoregressive/graphs/results.json @@ -0,0 +1,1780 @@ +{ + "files": { + "checkpoint-85656_id_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.99375, + "pass@2": 0.99375, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9555555555555556, + "pass@2": 0.9873015873015873, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9640718562874252, + "pass@2": 0.9880239520958084, + "pass@4": 0.9910179640718563, + "pass@8": 0.9970059880239521, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8905325443786982, + "pass@2": 0.9260355029585798, + "pass@4": 0.9674556213017751, + "pass@8": 0.9881656804733728, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8365384615384616, + "pass@2": 0.9006410256410257, + "pass@4": 0.9519230769230769, + "pass@8": 0.9839743589743589, + "pass@16": 0.9967948717948718, + "pass@32": 0.9967948717948718, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.752112676056338, + "pass@2": 0.8309859154929577, + "pass@4": 0.9014084507042254, + "pass@8": 0.9661971830985916, + "pass@16": 0.9887323943661972, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "10": { + "pass@1": 0.6342182890855457, + "pass@2": 0.7197640117994101, + "pass@4": 0.8053097345132744, + "pass@8": 0.8790560471976401, + "pass@16": 0.9144542772861357, + "pass@32": 0.9321533923303835, + "pass@64": 0.9528023598820059, + "pass@128": 0.9616519174041298 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9555555555555556, + "6": 0.9640718562874252, + "7": 0.8905325443786982, + "8": 0.8365384615384616, + "9": 0.752112676056338, + "10": 0.6342182890855457 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.99375, + "5": 0.9873015873015873, + "6": 0.9880239520958084, + "7": 0.9260355029585798, + "8": 0.9006410256410257, + "9": 0.8309859154929577, + "10": 0.7197640117994101 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9910179640718563, + "7": 0.9674556213017751, + "8": 0.9519230769230769, + "9": 0.9014084507042254, + "10": 0.8053097345132744 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9970059880239521, + "7": 0.9881656804733728, + "8": 0.9839743589743589, + "9": 0.9661971830985916, + "10": 0.8790560471976401 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9887323943661972, + "10": 0.9144542772861357 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9967948717948718, + "9": 0.9971830985915493, + "10": 0.9321533923303835 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9528023598820059 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9971830985915493, + "10": 0.9616519174041298 + } + }, + "total_pass_at_k": { + "pass@1": 0.889226612630868, + "pass@2": 0.9253630530226274, + "pass@4": 0.956095913542722, + "pass@8": 0.9787234042553191, + "pass@16": 0.9885173927727119, + "pass@32": 0.9915569064505235, + "pass@64": 0.994258696386356, + "pass@128": 0.9952718676122931 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9883040935672515, + "pass@2": 0.9970760233918129, + "pass@4": 0.9970760233918129, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.961764705882353, + "pass@2": 0.9764705882352941, + "pass@4": 0.9911764705882353, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9747634069400631, + "pass@2": 0.9779179810725552, + "pass@4": 0.9905362776025236, + "pass@8": 0.9968454258675079, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8670886075949367, + "pass@2": 0.9050632911392406, + "pass@4": 0.9335443037974683, + "pass@8": 0.9746835443037974, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8048048048048048, + "pass@2": 0.8738738738738738, + "pass@4": 0.9459459459459459, + "pass@8": 0.987987987987988, + "pass@16": 0.996996996996997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7813504823151125, + "pass@2": 0.8488745980707395, + "pass@4": 0.9067524115755627, + "pass@8": 0.9453376205787781, + "pass@16": 0.9839228295819936, + "pass@32": 0.9967845659163987, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.5970149253731343, + "pass@2": 0.6417910447761194, + "pass@4": 0.755223880597015, + "pass@8": 0.817910447761194, + "pass@16": 0.8925373134328358, + "pass@32": 0.9283582089552239, + "pass@64": 0.9552238805970149, + "pass@128": 0.9611940298507463 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9883040935672515, + "5": 0.961764705882353, + "6": 0.9747634069400631, + "7": 0.8670886075949367, + "8": 0.8048048048048048, + "9": 0.7813504823151125, + "10": 0.5970149253731343 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9764705882352941, + "6": 0.9779179810725552, + "7": 0.9050632911392406, + "8": 0.8738738738738738, + "9": 0.8488745980707395, + "10": 0.6417910447761194 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.9970760233918129, + "5": 0.9911764705882353, + "6": 0.9905362776025236, + "7": 0.9335443037974683, + "8": 0.9459459459459459, + "9": 0.9067524115755627, + "10": 0.755223880597015 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9968454258675079, + "7": 0.9746835443037974, + "8": 0.987987987987988, + "9": 0.9453376205787781, + "10": 0.817910447761194 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.996996996996997, + "9": 0.9839228295819936, + "10": 0.8925373134328358 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.9967845659163987, + "10": 0.9283582089552239 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9552238805970149 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9611940298507463 + } + }, + "total_pass_at_k": { + "pass@1": 0.887847338466689, + "pass@2": 0.9146300636089723, + "pass@4": 0.9474389019082692, + "pass@8": 0.9695346501506529, + "pass@16": 0.9859390693003013, + "pass@32": 0.9916303983930365, + "pass@64": 0.9949782390358219, + "pass@128": 0.995647807164379 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9822485207100592, + "pass@2": 0.9970414201183432, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9507246376811594, + "pass@2": 0.9739130434782609, + "pass@4": 0.9971014492753624, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9713467048710601, + "pass@2": 0.9770773638968482, + "pass@4": 0.9885386819484241, + "pass@8": 0.9914040114613181, + "pass@16": 0.997134670487106, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.8786127167630058, + "pass@2": 0.9190751445086706, + "pass@4": 0.9508670520231214, + "pass@8": 0.976878612716763, + "pass@16": 0.9942196531791907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.8140845070422535, + "pass@2": 0.8788732394366198, + "pass@4": 0.9295774647887324, + "pass@8": 0.9802816901408451, + "pass@16": 0.9915492957746479, + "pass@32": 0.9971830985915493, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.7455089820359282, + "pass@2": 0.8173652694610778, + "pass@4": 0.8832335329341318, + "pass@8": 0.9341317365269461, + "pass@16": 0.9790419161676647, + "pass@32": 0.9910179640718563, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "10": { + "pass@1": 0.6503067484662577, + "pass@2": 0.7239263803680982, + "pass@4": 0.7914110429447853, + "pass@8": 0.8374233128834356, + "pass@16": 0.901840490797546, + "pass@32": 0.941717791411043, + "pass@64": 0.9570552147239264, + "pass@128": 0.9601226993865031 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.9822485207100592, + "5": 0.9507246376811594, + "6": 0.9713467048710601, + "7": 0.8786127167630058, + "8": 0.8140845070422535, + "9": 0.7455089820359282, + "10": 0.6503067484662577 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.9970414201183432, + "5": 0.9739130434782609, + "6": 0.9770773638968482, + "7": 0.9190751445086706, + "8": 0.8788732394366198, + "9": 0.8173652694610778, + "10": 0.7239263803680982 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.9971014492753624, + "6": 0.9885386819484241, + "7": 0.9508670520231214, + "8": 0.9295774647887324, + "9": 0.8832335329341318, + "10": 0.7914110429447853 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.9914040114613181, + "7": 0.976878612716763, + "8": 0.9802816901408451, + "9": 0.9341317365269461, + "10": 0.8374233128834356 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.997134670487106, + "7": 0.9942196531791907, + "8": 0.9915492957746479, + "9": 0.9790419161676647, + "10": 0.901840490797546 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.9971830985915493, + "9": 0.9910179640718563, + "10": 0.941717791411043 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9570552147239264 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 1.0, + "10": 0.9601226993865031 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885976408912188, + "pass@2": 0.9213630406290957, + "pass@4": 0.9495412844036697, + "pass@8": 0.9695281782437746, + "pass@16": 0.9852555701179554, + "pass@32": 0.9924639580602883, + "pass@64": 0.9954128440366973, + "pass@128": 0.995740498034076 + } + } + }, + "total": { + "per_op_pass_at_k": { + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.988, + "pass@2": 0.996, + "pass@4": 0.999, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.956, + "pass@2": 0.979, + "pass@4": 0.996, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.97, + "pass@2": 0.981, + "pass@4": 0.99, + "pass@8": 0.995, + "pass@16": 0.999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.879, + "pass@2": 0.917, + "pass@4": 0.951, + "pass@8": 0.98, + "pass@16": 0.998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.818, + "pass@2": 0.884, + "pass@4": 0.942, + "pass@8": 0.984, + "pass@16": 0.995, + "pass@32": 0.998, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.759, + "pass@2": 0.832, + "pass@4": 0.897, + "pass@8": 0.949, + "pass@16": 0.984, + "pass@32": 0.995, + "pass@64": 0.999, + "pass@128": 0.999 + }, + "10": { + "pass@1": 0.627, + "pass@2": 0.695, + "pass@4": 0.784, + "pass@8": 0.845, + "pass@16": 0.903, + "pass@32": 0.934, + "pass@64": 0.955, + "pass@128": 0.961 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "2": 1.0, + "3": 1.0, + "4": 0.988, + "5": 0.956, + "6": 0.97, + "7": 0.879, + "8": 0.818, + "9": 0.759, + "10": 0.627 + }, + "pass@2": { + "2": 1.0, + "3": 1.0, + "4": 0.996, + "5": 0.979, + "6": 0.981, + "7": 0.917, + "8": 0.884, + "9": 0.832, + "10": 0.695 + }, + "pass@4": { + "2": 1.0, + "3": 1.0, + "4": 0.999, + "5": 0.996, + "6": 0.99, + "7": 0.951, + "8": 0.942, + "9": 0.897, + "10": 0.784 + }, + "pass@8": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.995, + "7": 0.98, + "8": 0.984, + "9": 0.949, + "10": 0.845 + }, + "pass@16": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 0.999, + "7": 0.998, + "8": 0.995, + "9": 0.984, + "10": 0.903 + }, + "pass@32": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 0.998, + "9": 0.995, + "10": 0.934 + }, + "pass@64": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.999, + "10": 0.955 + }, + "pass@128": { + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 1.0, + "6": 1.0, + "7": 1.0, + "8": 1.0, + "9": 0.999, + "10": 0.961 + } + }, + "total_pass_at_k": { + "pass@1": 0.8885555555555555, + "pass@2": 0.9204444444444444, + "pass@4": 0.951, + "pass@8": 0.9725555555555555, + "pass@16": 0.9865555555555555, + "pass@32": 0.9918888888888889, + "pass@64": 0.9948888888888889, + "pass@128": 0.9955555555555555 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "2": 321, + "3": 327, + "4": 320, + "5": 315, + "6": 334, + "7": 338, + "8": 312, + "9": 355, + "10": 339 + }, + "total": 2961 + }, + "movie_festival_awards": { + "per_op": { + "2": 350, + "3": 343, + "4": 342, + "5": 340, + "6": 317, + "7": 316, + "8": 333, + "9": 311, + "10": 335 + }, + "total": 2987 + }, + "teachers_in_school": { + "per_op": { + "2": 329, + "3": 330, + "4": 338, + "5": 345, + "6": 349, + "7": 346, + "8": 355, + "9": 334, + "10": 326 + }, + "total": 3052 + } + }, + "total": { + "per_op": { + "2": 1000, + "3": 1000, + "4": 1000, + "5": 1000, + "6": 1000, + "7": 1000, + "8": 1000, + "9": 1000, + "10": 1000 + }, + "total": 9000 + } + } + }, + "checkpoint-85656_ood_generations.jsonl.graph_results.jsonl": { + "per_template": { + "crazy_zootopia": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5637393767705382, + "pass@2": 0.6118980169971672, + "pass@4": 0.6968838526912181, + "pass@8": 0.8073654390934845, + "pass@16": 0.8640226628895185, + "pass@32": 0.8980169971671388, + "pass@64": 0.9150141643059491, + "pass@128": 0.9206798866855525 + }, + "12": { + "pass@1": 0.5451977401129944, + "pass@2": 0.6045197740112994, + "pass@4": 0.6751412429378532, + "pass@8": 0.7288135593220338, + "pass@16": 0.7853107344632768, + "pass@32": 0.8192090395480226, + "pass@64": 0.8361581920903954, + "pass@128": 0.847457627118644 + }, + "13": { + "pass@1": 0.3793103448275862, + "pass@2": 0.45454545454545453, + "pass@4": 0.5078369905956113, + "pass@8": 0.5517241379310345, + "pass@16": 0.6112852664576802, + "pass@32": 0.6802507836990596, + "pass@64": 0.7084639498432602, + "pass@128": 0.7335423197492164 + }, + "14": { + "pass@1": 0.308411214953271, + "pass@2": 0.367601246105919, + "pass@4": 0.4236760124610592, + "pass@8": 0.4797507788161994, + "pass@16": 0.5451713395638629, + "pass@32": 0.5981308411214953, + "pass@64": 0.6417445482866043, + "pass@128": 0.6978193146417445 + }, + "15": { + "pass@1": 0.21965317919075145, + "pass@2": 0.2832369942196532, + "pass@4": 0.3236994219653179, + "pass@8": 0.3670520231213873, + "pass@16": 0.4393063583815029, + "pass@32": 0.523121387283237, + "pass@64": 0.5809248554913294, + "pass@128": 0.615606936416185 + }, + "16": { + "pass@1": 0.09032258064516129, + "pass@2": 0.10967741935483871, + "pass@4": 0.13548387096774195, + "pass@8": 0.17419354838709677, + "pass@16": 0.2064516129032258, + "pass@32": 0.24516129032258063, + "pass@64": 0.3, + "pass@128": 0.3258064516129032 + }, + "17": { + "pass@1": 0.05202312138728324, + "pass@2": 0.08092485549132948, + "pass@4": 0.12138728323699421, + "pass@8": 0.16184971098265896, + "pass@16": 0.19653179190751446, + "pass@32": 0.2254335260115607, + "pass@64": 0.26878612716763006, + "pass@128": 0.2976878612716763 + }, + "18": { + "pass@1": 0.008571428571428572, + "pass@2": 0.02, + "pass@4": 0.03142857142857143, + "pass@8": 0.06, + "pass@16": 0.07142857142857142, + "pass@32": 0.09428571428571429, + "pass@64": 0.1, + "pass@128": 0.11428571428571428 + }, + "19": { + "pass@1": 0.006006006006006006, + "pass@2": 0.009009009009009009, + "pass@4": 0.009009009009009009, + "pass@8": 0.015015015015015015, + "pass@16": 0.015015015015015015, + "pass@32": 0.018018018018018018, + "pass@64": 0.02702702702702703, + "pass@128": 0.042042042042042045 + }, + "20": { + "pass@1": 0.006006006006006006, + "pass@2": 0.006006006006006006, + "pass@4": 0.015015015015015015, + "pass@8": 0.018018018018018018, + "pass@16": 0.02702702702702703, + "pass@32": 0.03303303303303303, + "pass@64": 0.03903903903903904, + "pass@128": 0.04504504504504504 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5637393767705382, + "12": 0.5451977401129944, + "13": 0.3793103448275862, + "14": 0.308411214953271, + "15": 0.21965317919075145, + "16": 0.09032258064516129, + "17": 0.05202312138728324, + "18": 0.008571428571428572, + "19": 0.006006006006006006, + "20": 0.006006006006006006 + }, + "pass@2": { + "11": 0.6118980169971672, + "12": 0.6045197740112994, + "13": 0.45454545454545453, + "14": 0.367601246105919, + "15": 0.2832369942196532, + "16": 0.10967741935483871, + "17": 0.08092485549132948, + "18": 0.02, + "19": 0.009009009009009009, + "20": 0.006006006006006006 + }, + "pass@4": { + "11": 0.6968838526912181, + "12": 0.6751412429378532, + "13": 0.5078369905956113, + "14": 0.4236760124610592, + "15": 0.3236994219653179, + "16": 0.13548387096774195, + "17": 0.12138728323699421, + "18": 0.03142857142857143, + "19": 0.009009009009009009, + "20": 0.015015015015015015 + }, + "pass@8": { + "11": 0.8073654390934845, + "12": 0.7288135593220338, + "13": 0.5517241379310345, + "14": 0.4797507788161994, + "15": 0.3670520231213873, + "16": 0.17419354838709677, + "17": 0.16184971098265896, + "18": 0.06, + "19": 0.015015015015015015, + "20": 0.018018018018018018 + }, + "pass@16": { + "11": 0.8640226628895185, + "12": 0.7853107344632768, + "13": 0.6112852664576802, + "14": 0.5451713395638629, + "15": 0.4393063583815029, + "16": 0.2064516129032258, + "17": 0.19653179190751446, + "18": 0.07142857142857142, + "19": 0.015015015015015015, + "20": 0.02702702702702703 + }, + "pass@32": { + "11": 0.8980169971671388, + "12": 0.8192090395480226, + "13": 0.6802507836990596, + "14": 0.5981308411214953, + "15": 0.523121387283237, + "16": 0.24516129032258063, + "17": 0.2254335260115607, + "18": 0.09428571428571429, + "19": 0.018018018018018018, + "20": 0.03303303303303303 + }, + "pass@64": { + "11": 0.9150141643059491, + "12": 0.8361581920903954, + "13": 0.7084639498432602, + "14": 0.6417445482866043, + "15": 0.5809248554913294, + "16": 0.3, + "17": 0.26878612716763006, + "18": 0.1, + "19": 0.02702702702702703, + "20": 0.03903903903903904 + }, + "pass@128": { + "11": 0.9206798866855525, + "12": 0.847457627118644, + "13": 0.7335423197492164, + "14": 0.6978193146417445, + "15": 0.615606936416185, + "16": 0.3258064516129032, + "17": 0.2976878612716763, + "18": 0.11428571428571428, + "19": 0.042042042042042045, + "20": 0.04504504504504504 + } + }, + "total_pass_at_k": { + "pass@1": 0.2202080237741456, + "pass@2": 0.25705794947994054, + "pass@4": 0.2965824665676077, + "pass@8": 0.33937592867756317, + "pass@16": 0.3791976225854383, + "pass@32": 0.4163447251114413, + "pass@64": 0.4442793462109955, + "pass@128": 0.4662704309063893 + } + }, + "movie_festival_awards": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5428571428571428, + "pass@2": 0.6349206349206349, + "pass@4": 0.7142857142857143, + "pass@8": 0.819047619047619, + "pass@16": 0.9047619047619048, + "pass@32": 0.9428571428571428, + "pass@64": 0.9714285714285714, + "pass@128": 0.9746031746031746 + }, + "12": { + "pass@1": 0.5351681957186545, + "pass@2": 0.6055045871559633, + "pass@4": 0.6758409785932722, + "pass@8": 0.7339449541284404, + "pass@16": 0.8073394495412844, + "pass@32": 0.8501529051987767, + "pass@64": 0.8776758409785933, + "pass@128": 0.8929663608562691 + }, + "13": { + "pass@1": 0.3950617283950617, + "pass@2": 0.4567901234567901, + "pass@4": 0.5185185185185185, + "pass@8": 0.5740740740740741, + "pass@16": 0.6450617283950617, + "pass@32": 0.691358024691358, + "pass@64": 0.7160493827160493, + "pass@128": 0.7283950617283951 + }, + "14": { + "pass@1": 0.3026706231454006, + "pass@2": 0.3649851632047478, + "pass@4": 0.41839762611275966, + "pass@8": 0.4807121661721068, + "pass@16": 0.5281899109792285, + "pass@32": 0.6023738872403561, + "pass@64": 0.6824925816023739, + "pass@128": 0.7299703264094956 + }, + "15": { + "pass@1": 0.2345679012345679, + "pass@2": 0.2962962962962963, + "pass@4": 0.32407407407407407, + "pass@8": 0.36728395061728397, + "pass@16": 0.43209876543209874, + "pass@32": 0.5030864197530864, + "pass@64": 0.5524691358024691, + "pass@128": 0.6234567901234568 + }, + "16": { + "pass@1": 0.07964601769911504, + "pass@2": 0.11209439528023599, + "pass@4": 0.14749262536873156, + "pass@8": 0.18584070796460178, + "pass@16": 0.22123893805309736, + "pass@32": 0.2536873156342183, + "pass@64": 0.30678466076696165, + "pass@128": 0.35398230088495575 + }, + "17": { + "pass@1": 0.04819277108433735, + "pass@2": 0.06927710843373494, + "pass@4": 0.09337349397590361, + "pass@8": 0.12951807228915663, + "pass@16": 0.15963855421686746, + "pass@32": 0.1927710843373494, + "pass@64": 0.21084337349397592, + "pass@128": 0.25 + }, + "18": { + "pass@1": 0.02903225806451613, + "pass@2": 0.05161290322580645, + "pass@4": 0.07096774193548387, + "pass@8": 0.1032258064516129, + "pass@16": 0.10967741935483871, + "pass@32": 0.12580645161290321, + "pass@64": 0.15161290322580645, + "pass@128": 0.17096774193548386 + }, + "19": { + "pass@1": 0.0, + "pass@2": 0.0030211480362537764, + "pass@4": 0.006042296072507553, + "pass@8": 0.01812688821752266, + "pass@16": 0.021148036253776436, + "pass@32": 0.04229607250755287, + "pass@64": 0.054380664652567974, + "pass@128": 0.05740181268882175 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.003003003003003003, + "pass@8": 0.006006006006006006, + "pass@16": 0.009009009009009009, + "pass@32": 0.012012012012012012, + "pass@64": 0.021021021021021023, + "pass@128": 0.02702702702702703 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.5428571428571428, + "12": 0.5351681957186545, + "13": 0.3950617283950617, + "14": 0.3026706231454006, + "15": 0.2345679012345679, + "16": 0.07964601769911504, + "17": 0.04819277108433735, + "18": 0.02903225806451613, + "19": 0.0, + "20": 0.0 + }, + "pass@2": { + "11": 0.6349206349206349, + "12": 0.6055045871559633, + "13": 0.4567901234567901, + "14": 0.3649851632047478, + "15": 0.2962962962962963, + "16": 0.11209439528023599, + "17": 0.06927710843373494, + "18": 0.05161290322580645, + "19": 0.0030211480362537764, + "20": 0.0 + }, + "pass@4": { + "11": 0.7142857142857143, + "12": 0.6758409785932722, + "13": 0.5185185185185185, + "14": 0.41839762611275966, + "15": 0.32407407407407407, + "16": 0.14749262536873156, + "17": 0.09337349397590361, + "18": 0.07096774193548387, + "19": 0.006042296072507553, + "20": 0.003003003003003003 + }, + "pass@8": { + "11": 0.819047619047619, + "12": 0.7339449541284404, + "13": 0.5740740740740741, + "14": 0.4807121661721068, + "15": 0.36728395061728397, + "16": 0.18584070796460178, + "17": 0.12951807228915663, + "18": 0.1032258064516129, + "19": 0.01812688821752266, + "20": 0.006006006006006006 + }, + "pass@16": { + "11": 0.9047619047619048, + "12": 0.8073394495412844, + "13": 0.6450617283950617, + "14": 0.5281899109792285, + "15": 0.43209876543209874, + "16": 0.22123893805309736, + "17": 0.15963855421686746, + "18": 0.10967741935483871, + "19": 0.021148036253776436, + "20": 0.009009009009009009 + }, + "pass@32": { + "11": 0.9428571428571428, + "12": 0.8501529051987767, + "13": 0.691358024691358, + "14": 0.6023738872403561, + "15": 0.5030864197530864, + "16": 0.2536873156342183, + "17": 0.1927710843373494, + "18": 0.12580645161290321, + "19": 0.04229607250755287, + "20": 0.012012012012012012 + }, + "pass@64": { + "11": 0.9714285714285714, + "12": 0.8776758409785933, + "13": 0.7160493827160493, + "14": 0.6824925816023739, + "15": 0.5524691358024691, + "16": 0.30678466076696165, + "17": 0.21084337349397592, + "18": 0.15161290322580645, + "19": 0.054380664652567974, + "20": 0.021021021021021023 + }, + "pass@128": { + "11": 0.9746031746031746, + "12": 0.8929663608562691, + "13": 0.7283950617283951, + "14": 0.7299703264094956, + "15": 0.6234567901234568, + "16": 0.35398230088495575, + "17": 0.25, + "18": 0.17096774193548386, + "19": 0.05740181268882175, + "20": 0.02702702702702703 + } + }, + "total_pass_at_k": { + "pass@1": 0.21515892420537897, + "pass@2": 0.25764058679706603, + "pass@4": 0.2952322738386308, + "pass@8": 0.33954767726161367, + "pass@16": 0.38141809290953543, + "pass@32": 0.4193154034229829, + "pass@64": 0.45232273838630804, + "pass@128": 0.4789119804400978 + } + }, + "teachers_in_school": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.6024096385542169, + "pass@2": 0.6626506024096386, + "pass@4": 0.7740963855421686, + "pass@8": 0.8373493975903614, + "pass@16": 0.8945783132530121, + "pass@32": 0.9397590361445783, + "pass@64": 0.9518072289156626, + "pass@128": 0.963855421686747 + }, + "12": { + "pass@1": 0.5015673981191222, + "pass@2": 0.5705329153605015, + "pass@4": 0.6394984326018809, + "pass@8": 0.7053291536050157, + "pass@16": 0.7711598746081505, + "pass@32": 0.8181818181818182, + "pass@64": 0.8432601880877743, + "pass@128": 0.8526645768025078 + }, + "13": { + "pass@1": 0.40336134453781514, + "pass@2": 0.47058823529411764, + "pass@4": 0.5546218487394958, + "pass@8": 0.6498599439775911, + "pass@16": 0.7142857142857143, + "pass@32": 0.7591036414565826, + "pass@64": 0.7815126050420168, + "pass@128": 0.7871148459383753 + }, + "14": { + "pass@1": 0.3128654970760234, + "pass@2": 0.3830409356725146, + "pass@4": 0.43859649122807015, + "pass@8": 0.4853801169590643, + "pass@16": 0.5350877192982456, + "pass@32": 0.6228070175438597, + "pass@64": 0.6900584795321637, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.23030303030303031, + "pass@2": 0.29393939393939394, + "pass@4": 0.3393939393939394, + "pass@8": 0.4212121212121212, + "pass@16": 0.4666666666666667, + "pass@32": 0.509090909090909, + "pass@64": 0.5848484848484848, + "pass@128": 0.6484848484848484 + }, + "16": { + "pass@1": 0.1111111111111111, + "pass@2": 0.1339031339031339, + "pass@4": 0.16524216524216523, + "pass@8": 0.21082621082621084, + "pass@16": 0.24786324786324787, + "pass@32": 0.29914529914529914, + "pass@64": 0.33903133903133903, + "pass@128": 0.3789173789173789 + }, + "17": { + "pass@1": 0.040372670807453416, + "pass@2": 0.08385093167701864, + "pass@4": 0.10248447204968944, + "pass@8": 0.11801242236024845, + "pass@16": 0.15838509316770186, + "pass@32": 0.17701863354037267, + "pass@64": 0.2204968944099379, + "pass@128": 0.2701863354037267 + }, + "18": { + "pass@1": 0.03235294117647059, + "pass@2": 0.03823529411764706, + "pass@4": 0.058823529411764705, + "pass@8": 0.07941176470588235, + "pass@16": 0.09705882352941177, + "pass@32": 0.10588235294117647, + "pass@64": 0.12941176470588237, + "pass@128": 0.16470588235294117 + }, + "19": { + "pass@1": 0.005952380952380952, + "pass@2": 0.011904761904761904, + "pass@4": 0.01488095238095238, + "pass@8": 0.01488095238095238, + "pass@16": 0.017857142857142856, + "pass@32": 0.02976190476190476, + "pass@64": 0.03571428571428571, + "pass@128": 0.041666666666666664 + }, + "20": { + "pass@1": 0.0, + "pass@2": 0.0, + "pass@4": 0.0029940119760479044, + "pass@8": 0.011976047904191617, + "pass@16": 0.014970059880239521, + "pass@32": 0.023952095808383235, + "pass@64": 0.02694610778443114, + "pass@128": 0.03592814371257485 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.6024096385542169, + "12": 0.5015673981191222, + "13": 0.40336134453781514, + "14": 0.3128654970760234, + "15": 0.23030303030303031, + "16": 0.1111111111111111, + "17": 0.040372670807453416, + "18": 0.03235294117647059, + "19": 0.005952380952380952, + "20": 0.0 + }, + "pass@2": { + "11": 0.6626506024096386, + "12": 0.5705329153605015, + "13": 0.47058823529411764, + "14": 0.3830409356725146, + "15": 0.29393939393939394, + "16": 0.1339031339031339, + "17": 0.08385093167701864, + "18": 0.03823529411764706, + "19": 0.011904761904761904, + "20": 0.0 + }, + "pass@4": { + "11": 0.7740963855421686, + "12": 0.6394984326018809, + "13": 0.5546218487394958, + "14": 0.43859649122807015, + "15": 0.3393939393939394, + "16": 0.16524216524216523, + "17": 0.10248447204968944, + "18": 0.058823529411764705, + "19": 0.01488095238095238, + "20": 0.0029940119760479044 + }, + "pass@8": { + "11": 0.8373493975903614, + "12": 0.7053291536050157, + "13": 0.6498599439775911, + "14": 0.4853801169590643, + "15": 0.4212121212121212, + "16": 0.21082621082621084, + "17": 0.11801242236024845, + "18": 0.07941176470588235, + "19": 0.01488095238095238, + "20": 0.011976047904191617 + }, + "pass@16": { + "11": 0.8945783132530121, + "12": 0.7711598746081505, + "13": 0.7142857142857143, + "14": 0.5350877192982456, + "15": 0.4666666666666667, + "16": 0.24786324786324787, + "17": 0.15838509316770186, + "18": 0.09705882352941177, + "19": 0.017857142857142856, + "20": 0.014970059880239521 + }, + "pass@32": { + "11": 0.9397590361445783, + "12": 0.8181818181818182, + "13": 0.7591036414565826, + "14": 0.6228070175438597, + "15": 0.509090909090909, + "16": 0.29914529914529914, + "17": 0.17701863354037267, + "18": 0.10588235294117647, + "19": 0.02976190476190476, + "20": 0.023952095808383235 + }, + "pass@64": { + "11": 0.9518072289156626, + "12": 0.8432601880877743, + "13": 0.7815126050420168, + "14": 0.6900584795321637, + "15": 0.5848484848484848, + "16": 0.33903133903133903, + "17": 0.2204968944099379, + "18": 0.12941176470588237, + "19": 0.03571428571428571, + "20": 0.02694610778443114 + }, + "pass@128": { + "11": 0.963855421686747, + "12": 0.8526645768025078, + "13": 0.7871148459383753, + "14": 0.7309941520467836, + "15": 0.6484848484848484, + "16": 0.3789173789173789, + "17": 0.2701863354037267, + "18": 0.16470588235294117, + "19": 0.041666666666666664, + "20": 0.03592814371257485 + } + }, + "total_pass_at_k": { + "pass@1": 0.22360987213797204, + "pass@2": 0.264347308950342, + "pass@4": 0.3086529884032114, + "pass@8": 0.35325602140945583, + "pass@16": 0.39161462979482603, + "pass@32": 0.42848647041332144, + "pass@64": 0.46030330062444247, + "pass@128": 0.48736247398156407 + } + } + }, + "total": { + "per_op_pass_at_k": { + "11": { + "pass@1": 0.57, + "pass@2": 0.636, + "pass@4": 0.728, + "pass@8": 0.821, + "pass@16": 0.887, + "pass@32": 0.926, + "pass@64": 0.945, + "pass@128": 0.952 + }, + "12": { + "pass@1": 0.528, + "pass@2": 0.594, + "pass@4": 0.664, + "pass@8": 0.723, + "pass@16": 0.788, + "pass@32": 0.829, + "pass@64": 0.852, + "pass@128": 0.864 + }, + "13": { + "pass@1": 0.393, + "pass@2": 0.461, + "pass@4": 0.528, + "pass@8": 0.594, + "pass@16": 0.659, + "pass@32": 0.712, + "pass@64": 0.737, + "pass@128": 0.751 + }, + "14": { + "pass@1": 0.308, + "pass@2": 0.372, + "pass@4": 0.427, + "pass@8": 0.482, + "pass@16": 0.536, + "pass@32": 0.608, + "pass@64": 0.672, + "pass@128": 0.72 + }, + "15": { + "pass@1": 0.228, + "pass@2": 0.291, + "pass@4": 0.329, + "pass@8": 0.385, + "pass@16": 0.446, + "pass@32": 0.512, + "pass@64": 0.573, + "pass@128": 0.629 + }, + "16": { + "pass@1": 0.094, + "pass@2": 0.119, + "pass@4": 0.15, + "pass@8": 0.191, + "pass@16": 0.226, + "pass@32": 0.267, + "pass@64": 0.316, + "pass@128": 0.354 + }, + "17": { + "pass@1": 0.047, + "pass@2": 0.078, + "pass@4": 0.106, + "pass@8": 0.137, + "pass@16": 0.172, + "pass@32": 0.199, + "pass@64": 0.234, + "pass@128": 0.273 + }, + "18": { + "pass@1": 0.023, + "pass@2": 0.036, + "pass@4": 0.053, + "pass@8": 0.08, + "pass@16": 0.092, + "pass@32": 0.108, + "pass@64": 0.126, + "pass@128": 0.149 + }, + "19": { + "pass@1": 0.004, + "pass@2": 0.008, + "pass@4": 0.01, + "pass@8": 0.016, + "pass@16": 0.018, + "pass@32": 0.03, + "pass@64": 0.039, + "pass@128": 0.047 + }, + "20": { + "pass@1": 0.002, + "pass@2": 0.002, + "pass@4": 0.007, + "pass@8": 0.012, + "pass@16": 0.017, + "pass@32": 0.023, + "pass@64": 0.029, + "pass@128": 0.036 + } + }, + "per_op_pass_at_columns": { + "pass@1": { + "11": 0.57, + "12": 0.528, + "13": 0.393, + "14": 0.308, + "15": 0.228, + "16": 0.094, + "17": 0.047, + "18": 0.023, + "19": 0.004, + "20": 0.002 + }, + "pass@2": { + "11": 0.636, + "12": 0.594, + "13": 0.461, + "14": 0.372, + "15": 0.291, + "16": 0.119, + "17": 0.078, + "18": 0.036, + "19": 0.008, + "20": 0.002 + }, + "pass@4": { + "11": 0.728, + "12": 0.664, + "13": 0.528, + "14": 0.427, + "15": 0.329, + "16": 0.15, + "17": 0.106, + "18": 0.053, + "19": 0.01, + "20": 0.007 + }, + "pass@8": { + "11": 0.821, + "12": 0.723, + "13": 0.594, + "14": 0.482, + "15": 0.385, + "16": 0.191, + "17": 0.137, + "18": 0.08, + "19": 0.016, + "20": 0.012 + }, + "pass@16": { + "11": 0.887, + "12": 0.788, + "13": 0.659, + "14": 0.536, + "15": 0.446, + "16": 0.226, + "17": 0.172, + "18": 0.092, + "19": 0.018, + "20": 0.017 + }, + "pass@32": { + "11": 0.926, + "12": 0.829, + "13": 0.712, + "14": 0.608, + "15": 0.512, + "16": 0.267, + "17": 0.199, + "18": 0.108, + "19": 0.03, + "20": 0.023 + }, + "pass@64": { + "11": 0.945, + "12": 0.852, + "13": 0.737, + "14": 0.672, + "15": 0.573, + "16": 0.316, + "17": 0.234, + "18": 0.126, + "19": 0.039, + "20": 0.029 + }, + "pass@128": { + "11": 0.952, + "12": 0.864, + "13": 0.751, + "14": 0.72, + "15": 0.629, + "16": 0.354, + "17": 0.273, + "18": 0.149, + "19": 0.047, + "20": 0.036 + } + }, + "total_pass_at_k": { + "pass@1": 0.2197, + "pass@2": 0.2597, + "pass@4": 0.3002, + "pass@8": 0.3441, + "pass@16": 0.3841, + "pass@32": 0.4214, + "pass@64": 0.4523, + "pass@128": 0.4775 + } + }, + "ks": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128 + ], + "prompt_counts": { + "per_template": { + "crazy_zootopia": { + "per_op": { + "11": 353, + "12": 354, + "13": 319, + "14": 321, + "15": 346, + "16": 310, + "17": 346, + "18": 350, + "19": 333, + "20": 333 + }, + "total": 3365 + }, + "movie_festival_awards": { + "per_op": { + "11": 315, + "12": 327, + "13": 324, + "14": 337, + "15": 324, + "16": 339, + "17": 332, + "18": 310, + "19": 331, + "20": 333 + }, + "total": 3272 + }, + "teachers_in_school": { + "per_op": { + "11": 332, + "12": 319, + "13": 357, + "14": 342, + "15": 330, + "16": 351, + "17": 322, + "18": 340, + "19": 336, + "20": 334 + }, + "total": 3363 + } + }, + "total": { + "per_op": { + "11": 1000, + "12": 1000, + "13": 1000, + "14": 1000, + "15": 1000, + "16": 1000, + "17": 1000, + "18": 1000, + "19": 1000, + "20": 1000 + }, + "total": 10000 + } + } + } + } +} diff --git a/SDAR-100M-ep4-block4/composition/autoregressive/summary.csv b/SDAR-100M-ep4-block4/composition/autoregressive/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..6601986597bc7be49faac43e23d3ac8081837435 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/autoregressive/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85656,0.9929774305555555,0.4781125,0.7219958881578947,0.0,0.0,0.0,199.76994010416666,339.323959375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4/composition/autoregressive/summary.json b/SDAR-100M-ep4-block4/composition/autoregressive/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8e6d3b0e71126501ef102f9075118cb3b3f2f5b2 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/autoregressive/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85656", + "id_acc": 0.9929774305555555, + "ood_acc": 0.4781125, + "total_acc": 0.7219958881578947, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.76994010416666, + "ood_avg_resp_len": 339.323959375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 582274, + "answer_accuracy": 0.7190982650964275, + "avg_response_len": 260.7669427264464, + "resp_tokens_sum": 211150295, + "pass_at_k": { + "pass@1": 0.7190982650964275, + "pass@2": 0.7589688715922986, + "pass@4": 0.7932620020861227, + "pass@8": 0.8222878929551545, + "pass@16": 0.8463803610796929, + "pass@32": 0.8663935257186914, + "pass@64": 0.8830698445425451, + "pass@128": 0.8964590578564654 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9570658185840708, + "pass@2": 0.9683732840917013, + "pass@4": 0.9756881081236524, + "pass@8": 0.9813912893648465, + "pass@16": 0.9867117841326309, + "pass@32": 0.9916970042118012, + "pass@64": 0.9962000480088795, + "pass@128": 1.0 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999755859375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9967757936507936, + "pass@2": 0.9981674165729285, + "pass@4": 0.9991240716934194, + "pass@8": 0.999773855222213, + "pass@16": 0.9999870485045957, + "pass@32": 0.9999999813944173, + "pass@64": 0.9999999999999998, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9982664571005917, + "pass@2": 0.9996865972836976, + "pass@4": 0.9999851585674571, + "pass@8": 0.9999999736396237, + "pass@16": 0.9999999999999807, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9986979166666666, + "pass@2": 0.9998899814506359, + "pass@4": 0.9999984557218807, + "pass@8": 0.9999999997174955, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9965669014084507, + "pass@2": 0.9971723549961184, + "pass@4": 0.9971830946307767, + "pass@8": 0.9971830985915493, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9157445113314447, + "pass@2": 0.9303819483170127, + "pass@4": 0.9399434625381455, + "pass@8": 0.9476715847990214, + "pass@16": 0.954937662674825, + "pass@32": 0.9618599495743576, + "pass@64": 0.9670685637911212, + "pass@128": 0.9688385269121813 + }, + "12": { + "pass@1": 0.8374382062146892, + "pass@2": 0.8719144812936519, + "pass@4": 0.8911755683611579, + "pass@8": 0.9053322956848698, + "pass@16": 0.9172509395306903, + "pass@32": 0.9285946036232826, + "pass@64": 0.9412245539437281, + "pass@128": 0.9548022598870056 + }, + "13": { + "pass@1": 0.7017535266457681, + "pass@2": 0.7656527688643157, + "pass@4": 0.8069338072552844, + "pass@8": 0.836520332654227, + "pass@16": 0.8591068301655219, + "pass@32": 0.8770989104903102, + "pass@64": 0.8922208279506292, + "pass@128": 0.9059561128526645 + }, + "14": { + "pass@1": 0.6434968847352025, + "pass@2": 0.7339530134667749, + "pass@4": 0.7953253649952631, + "pass@8": 0.8356009050741141, + "pass@16": 0.8650482090662411, + "pass@32": 0.8884090804371552, + "pass@64": 0.9065243627350636, + "pass@128": 0.9190031152647975 + }, + "15": { + "pass@1": 0.5236407153179191, + "pass@2": 0.6345384967684675, + "pass@4": 0.720673474883558, + "pass@8": 0.7874274695235931, + "pass@16": 0.836765843937725, + "pass@32": 0.870818671155785, + "pass@64": 0.8910456399594974, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.31060987903225806, + "pass@2": 0.41360133032766044, + "pass@4": 0.5086247351137556, + "pass@8": 0.5870659401872468, + "pass@16": 0.6479544799527384, + "pass@32": 0.6986068415947772, + "pass@64": 0.7455048157861739, + "pass@128": 0.7903225806451613 + }, + "17": { + "pass@1": 0.2576770231213873, + "pass@2": 0.3527515674070362, + "pass@4": 0.4467490030147967, + "pass@8": 0.5316486877319969, + "pass@16": 0.6051723920177607, + "pass@32": 0.6689063240851189, + "pass@64": 0.7234904728932069, + "pass@128": 0.7658959537572254 + }, + "18": { + "pass@1": 0.2110267857142857, + "pass@2": 0.29613364735658004, + "pass@4": 0.384165431463924, + "pass@8": 0.4662889678969746, + "pass@16": 0.5357500897786317, + "pass@32": 0.5902205339958437, + "pass@64": 0.6345563661728516, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.173681493993994, + "pass@2": 0.24716990612659911, + "pass@4": 0.32139107752296747, + "pass@8": 0.3897048009291002, + "pass@16": 0.4521456987554777, + "pass@32": 0.5096928722985181, + "pass@64": 0.5592708746194527, + "pass@128": 0.5975975975975976 + }, + "20": { + "pass@1": 0.17227383633633633, + "pass@2": 0.24018629850322742, + "pass@4": 0.31119543052613924, + "pass@8": 0.3805273280744088, + "pass@16": 0.44240840872374637, + "pass@32": 0.4948028929687008, + "pass@64": 0.538554484333375, + "pass@128": 0.5735735735735735 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 596124, + "answer_accuracy": 0.7259888932190179, + "avg_response_len": 270.2181629968823, + "resp_tokens_sum": 221881538, + "pass_at_k": { + "pass@1": 0.7259888932190179, + "pass@2": 0.766912620979365, + "pass@4": 0.8019209021173521, + "pass@8": 0.8308038764426154, + "pass@16": 0.8538239206472131, + "pass@32": 0.8723197323980967, + "pass@64": 0.8882274904806203, + "pass@128": 0.9025720966484801 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9357026457055214, + "pass@2": 0.9485471716342208, + "pass@4": 0.9583134079099005, + "pass@8": 0.965796860977208, + "pass@16": 0.9720041059256965, + "pass@32": 0.9780843207095543, + "pass@64": 0.9840777547518659, + "pass@128": 0.9877300613496932 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999763257575758, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9999768860946746, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9993206521739131, + "pass@2": 0.9999322435239073, + "pass@4": 0.9999991343473369, + "pass@8": 0.9999999999112675, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995299068767909, + "pass@2": 0.9999576969067977, + "pass@4": 0.9999996293443263, + "pass@8": 0.9999999999871032, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993000361271677, + "pass@2": 0.9998751905921442, + "pass@4": 0.9999952453558912, + "pass@8": 0.9999999955120726, + "pass@16": 0.9999999999999997, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9959727112676057, + "pass@2": 0.9982917683264945, + "pass@4": 0.9995983774035287, + "pass@8": 0.9999709649128528, + "pass@16": 0.9999998400094289, + "pass@32": 0.9999999999990319, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9944330089820359, + "pass@2": 0.9963245261445614, + "pass@4": 0.9975550485081578, + "pass@8": 0.9985324513055662, + "pass@16": 0.9993539825001094, + "pass@32": 0.9998914962254856, + "pass@64": 0.999999085108258, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9468655873493976, + "pass@2": 0.9604431505549762, + "pass@4": 0.9680644524103162, + "pass@8": 0.9726580226305667, + "pass@16": 0.9756575108892106, + "pass@32": 0.9783564878872142, + "pass@64": 0.9817601740575576, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.8422070924764891, + "pass@2": 0.8776021739935335, + "pass@4": 0.8994707533619423, + "pass@8": 0.9159269312012803, + "pass@16": 0.9295581121779324, + "pass@32": 0.9422592174940747, + "pass@64": 0.9548234063200668, + "pass@128": 0.9655172413793104 + }, + "13": { + "pass@1": 0.7061887254901961, + "pass@2": 0.7791781082511747, + "pass@4": 0.8260199885728562, + "pass@8": 0.8570431299381749, + "pass@16": 0.8790524526209159, + "pass@32": 0.8947079171348, + "pass@64": 0.9077045369983032, + "pass@128": 0.9215686274509803 + }, + "14": { + "pass@1": 0.6597222222222222, + "pass@2": 0.7535168301330757, + "pass@4": 0.8180587859631577, + "pass@8": 0.8601767977854613, + "pass@16": 0.8856046539907487, + "pass@32": 0.9014211318157292, + "pass@64": 0.9136951455705985, + "pass@128": 0.9269005847953217 + }, + "15": { + "pass@1": 0.5109611742424243, + "pass@2": 0.6282312544738723, + "pass@4": 0.7217386031291542, + "pass@8": 0.7923839668825581, + "pass@16": 0.8425049221743166, + "pass@32": 0.8757183743773951, + "pass@64": 0.8958328669936695, + "pass@128": 0.9090909090909091 + }, + "16": { + "pass@1": 0.3457532051282051, + "pass@2": 0.4452947988873186, + "pass@4": 0.532903049779034, + "pass@8": 0.6048692395044325, + "pass@16": 0.6625327790912872, + "pass@32": 0.7115800723039244, + "pass@64": 0.757156872133029, + "pass@128": 0.8005698005698005 + }, + "17": { + "pass@1": 0.2784355590062112, + "pass@2": 0.3800656881205065, + "pass@4": 0.477290026246719, + "pass@8": 0.5595626443779531, + "pass@16": 0.625599140698911, + "pass@32": 0.6790160770546658, + "pass@64": 0.7283353115816313, + "pass@128": 0.7763975155279503 + }, + "18": { + "pass@1": 0.23963694852941175, + "pass@2": 0.3307752431681334, + "pass@4": 0.42159714759919714, + "pass@8": 0.5055019564772302, + "pass@16": 0.5780629837271026, + "pass@32": 0.6386943668162116, + "pass@64": 0.6911164787819537, + "pass@128": 0.7352941176470589 + }, + "19": { + "pass@1": 0.17643229166666666, + "pass@2": 0.24522813554555678, + "pass@4": 0.3185077649334011, + "pass@8": 0.3868667149725815, + "pass@16": 0.4454668435699914, + "pass@32": 0.4963013158912973, + "pass@64": 0.5420733465747838, + "pass@128": 0.5833333333333334 + }, + "20": { + "pass@1": 0.1466364146706587, + "pass@2": 0.2122326459286152, + "pass@4": 0.2832578821547007, + "pass@8": 0.35382239266907534, + "pass@16": 0.41688701623004404, + "pass@32": 0.46919719938448934, + "pass@64": 0.5122032313480497, + "pass@128": 0.5508982035928144 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 577496, + "answer_accuracy": 0.7208320019172392, + "avg_response_len": 288.8812684733983, + "resp_tokens_sum": 231437806, + "pass_at_k": { + "pass@1": 0.7208320019172392, + "pass@2": 0.7610648540117928, + "pass@4": 0.7956922751747378, + "pass@8": 0.824719333058601, + "pass@16": 0.8483699736521619, + "pass@32": 0.8678111148940243, + "pass@64": 0.8841115817330713, + "pass@128": 0.8980667838312829 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9563199626865672, + "pass@2": 0.9663844899518158, + "pass@4": 0.9732305133500102, + "pass@8": 0.9785252509761181, + "pass@16": 0.9830552767950438, + "pass@32": 0.9872244258703623, + "pass@64": 0.9910618926107597, + "pass@128": 0.9940298507462687 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.996920955882353, + "pass@2": 0.999296911185734, + "pass@4": 0.999915258386819, + "pass@8": 0.9999981750560808, + "pass@16": 0.9999999995322573, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9986691640378549, + "pass@2": 0.9994651816985022, + "pass@4": 0.9999134103702337, + "pass@8": 0.9999980443690937, + "pass@16": 0.9999999994983201, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996785996835443, + "pass@2": 0.9999937705571613, + "pass@4": 0.9999999955503979, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9931024774774775, + "pass@2": 0.9966164491262917, + "pass@4": 0.998535216318681, + "pass@8": 0.9994952286242749, + "pass@16": 0.9999277542615488, + "pass@32": 0.9999991007812299, + "pass@64": 0.9999999999952028, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9935942524115756, + "pass@2": 0.9967580607387904, + "pass@4": 0.9984507620831962, + "pass@8": 0.9995185509250936, + "pass@16": 0.9999447697863701, + "pass@32": 0.9999995445205452, + "pass@64": 0.9999999999992755, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.933531746031746, + "pass@2": 0.9515041088613923, + "pass@4": 0.9643743076758263, + "pass@8": 0.9733412595679051, + "pass@16": 0.9793917714194672, + "pass@32": 0.9834661512791815, + "pass@64": 0.9856061741231261, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8517297400611621, + "pass@2": 0.892629999518409, + "pass@4": 0.9156304034243424, + "pass@8": 0.9315569919301774, + "pass@16": 0.9442163881132603, + "pass@32": 0.9550475627343186, + "pass@64": 0.9651791869569815, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.6634355709876543, + "pass@2": 0.7349605387868178, + "pass@4": 0.7827549337004169, + "pass@8": 0.8185453779861791, + "pass@16": 0.8474015318736398, + "pass@32": 0.8702366775562574, + "pass@64": 0.8869076964727599, + "pass@128": 0.8981481481481481 + }, + "14": { + "pass@1": 0.6352698442136498, + "pass@2": 0.7258190173835839, + "pass@4": 0.7881565101098262, + "pass@8": 0.828713185282778, + "pass@16": 0.8561651647024184, + "pass@32": 0.8781020595086385, + "pass@64": 0.8971976264132405, + "pass@128": 0.913946587537092 + }, + "15": { + "pass@1": 0.5290075231481481, + "pass@2": 0.6366762205939533, + "pass@4": 0.7172502033889275, + "pass@8": 0.7746811978937012, + "pass@16": 0.8138282124679664, + "pass@32": 0.8408839816842546, + "pass@64": 0.8593529920260841, + "pass@128": 0.8734567901234568 + }, + "16": { + "pass@1": 0.3200129056047198, + "pass@2": 0.419621600120781, + "pass@4": 0.510766843856907, + "pass@8": 0.587493043139971, + "pass@16": 0.6510022787317294, + "pass@32": 0.7076012028954857, + "pass@64": 0.7594490626705939, + "pass@128": 0.8053097345132744 + }, + "17": { + "pass@1": 0.2574124623493976, + "pass@2": 0.35109927900578686, + "pass@4": 0.4391238452121193, + "pass@8": 0.5116352714481722, + "pass@16": 0.5700288921857198, + "pass@32": 0.6232034647147692, + "pass@64": 0.6743981124495337, + "pass@128": 0.7198795180722891 + }, + "18": { + "pass@1": 0.22880544354838708, + "pass@2": 0.3054177514605029, + "pass@4": 0.3843950726925263, + "pass@8": 0.46068492782560816, + "pass@16": 0.5260139043388721, + "pass@32": 0.5775948095148585, + "pass@64": 0.6185270051584428, + "pass@128": 0.6548387096774193 + }, + "19": { + "pass@1": 0.16887745468277945, + "pass@2": 0.2465554898065991, + "pass@4": 0.33081693168263315, + "pass@8": 0.4119898536696145, + "pass@16": 0.4832835378374356, + "pass@32": 0.542532246435064, + "pass@64": 0.5936834175175014, + "pass@128": 0.6404833836858006 + }, + "20": { + "pass@1": 0.1543262012012012, + "pass@2": 0.22396535314842403, + "pass@4": 0.30279329132281885, + "pass@8": 0.38295495014077957, + "pass@16": 0.4554402145987438, + "pass@32": 0.513757434375952, + "pass@64": 0.5580152117413981, + "pass@128": 0.5945945945945946 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/checkpoint-85656_metrics.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/checkpoint-85656_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..6f3b616b5763fbb2dc0541b6c7ddac159d49836f --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/checkpoint-85656_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85656", + "id": { + "answer_accuracy": 0.5962326388888889, + "avg_response_len": 196.43200520833332, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.5962326388888889, + "pass@2": 0.7000273813429638, + "pass@4": 0.7923565615443822, + "pass@8": 0.8639872273979543, + "pass@16": 0.9128541063527003, + "pass@32": 0.9430173529001582, + "pass@64": 0.9604671529603527, + "pass@128": 0.97 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.294984375, + "pass@2": 0.44116104822834673, + "pass@4": 0.5971287656542937, + "pass@8": 0.7389807252973897, + "pass@16": 0.84768807963811, + "pass@32": 0.9169143421175255, + "pass@64": 0.9549806583238332, + "pass@128": 0.974 + }, + "2": { + "pass@1": 0.9836640625, + "pass@2": 0.9895119340551181, + "pass@4": 0.9916571213910761, + "pass@8": 0.9926734396375929, + "pass@16": 0.9936280770317396, + "pass@32": 0.9946112162573953, + "pass@64": 0.9954085252332271, + "pass@128": 0.996 + }, + "3": { + "pass@1": 0.9901796875, + "pass@2": 0.9978218503937013, + "pass@4": 0.9992752950881137, + "pass@8": 0.9996609731267692, + "pass@16": 0.9998980295834996, + "pass@32": 0.9999929051783535, + "pass@64": 0.9999999947663261, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.81384375, + "pass@2": 0.8893309547244093, + "pass@4": 0.9378742193475821, + "pass@8": 0.9656068663773961, + "pass@16": 0.9801139588085326, + "pass@32": 0.987525844495081, + "pass@64": 0.9913506819842142, + "pass@128": 0.993 + }, + "5": { + "pass@1": 0.5399140625, + "pass@2": 0.6332266240157477, + "pass@4": 0.7043046991001116, + "pass@8": 0.7557135471072218, + "pass@16": 0.7935086678160447, + "pass@32": 0.8229397434474545, + "pass@64": 0.8457480381228596, + "pass@128": 0.862 + }, + "6": { + "pass@1": 0.5930078125, + "pass@2": 0.7389413139763772, + "pass@4": 0.8507973174915651, + "pass@8": 0.9205187483944762, + "pass@16": 0.9567578696086759, + "pass@32": 0.9749960639929603, + "pass@64": 0.9853870906007994, + "pass@128": 0.991 + }, + "7": { + "pass@1": 0.4573359375, + "pass@2": 0.6167848179133851, + "pass@4": 0.7557966017060365, + "pass@8": 0.8538009052018047, + "pass@16": 0.9125834476186985, + "pass@32": 0.9449341004863567, + "pass@64": 0.9625863934398923, + "pass@128": 0.973 + }, + "8": { + "pass@1": 0.395015625, + "pass@2": 0.5535820620078733, + "pass@4": 0.7038110569928769, + "pass@8": 0.8219207348844457, + "pass@16": 0.8994292729064922, + "pass@32": 0.9432539236445819, + "pass@64": 0.9661139429859158, + "pass@128": 0.977 + }, + "9": { + "pass@1": 0.2981484375, + "pass@2": 0.4398858267716532, + "pass@4": 0.5905639771278585, + "pass@8": 0.727009106554544, + "pass@16": 0.8320795541623611, + "pass@32": 0.9019880364816364, + "pass@64": 0.9426290511859993, + "pass@128": 0.964 + } + }, + "per_op_accuracy": { + "10": 0.294984375, + "2": 0.9836640625, + "3": 0.9901796875, + "4": 0.81384375, + "5": 0.5399140625, + "6": 0.5930078125, + "7": 0.4573359375, + "8": 0.395015625, + "9": 0.2981484375 + }, + "per_op_avg_response_len": { + "10": 273.9497890625, + "2": 114.8402265625, + "3": 144.3747265625, + "4": 159.5347578125, + "5": 169.4598828125, + "6": 194.099921875, + "7": 221.0559296875, + "8": 238.8773671875, + "9": 251.6954453125 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.9497890625, + "2": 114.8402265625, + "3": 144.3747265625, + "4": 159.5347578125, + "5": 169.4598828125, + "6": 194.099921875, + "7": 221.0559296875, + "8": 238.8773671875, + "9": 251.6954453125 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 222373, + "answer_accuracy": 0.586723763086795, + "avg_response_len": 186.14557740206013, + "resp_tokens_sum": 70550663, + "pass_at_k": { + "pass@1": 0.586723763086795, + "pass@2": 0.6891751978476113, + "pass@4": 0.7818710631029303, + "pass@8": 0.8555671751136579, + "pass@16": 0.9072101078915619, + "pass@32": 0.9392771485998755, + "pass@64": 0.9578337516070504, + "pass@128": 0.9689294157379263 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2789684734513274, + "pass@2": 0.4280777471953171, + "pass@4": 0.591814418385755, + "pass@8": 0.7408074226575495, + "pass@16": 0.852499483578908, + "pass@32": 0.9212267885243984, + "pass@64": 0.9583472694281799, + "pass@128": 0.976401179941003 + }, + "2": { + "pass@1": 0.9758080218068536, + "pass@2": 0.9829028380798195, + "pass@4": 0.98677001268533, + "pass@8": 0.9896416905457607, + "pass@16": 0.992610833071678, + "pass@32": 0.9956735708953123, + "pass@64": 0.9981573994804587, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9917335626911316, + "pass@2": 0.996875300994486, + "pass@4": 0.9981438487046914, + "pass@8": 0.9989815709816434, + "pass@16": 0.9996882265536386, + "pass@32": 0.9999783032978147, + "pass@64": 0.9999999839948809, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.8259765625, + "pass@2": 0.8985351562499997, + "pass@4": 0.9441926254335391, + "pass@8": 0.9721901401524639, + "pass@16": 0.9876806739981561, + "pass@32": 0.9947236639399566, + "pass@64": 0.9979643470112908, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.5100446428571429, + "pass@2": 0.5957895888013999, + "pass@4": 0.6638044452776733, + "pass@8": 0.7162705451375567, + "pass@16": 0.7588625747565263, + "pass@32": 0.7968042579192405, + "pass@64": 0.8300021626967635, + "pass@128": 0.8539682539682539 + }, + "6": { + "pass@1": 0.6049775449101796, + "pass@2": 0.7494286770946295, + "pass@4": 0.8553464221725272, + "pass@8": 0.9199166321160519, + "pass@16": 0.9540052822565772, + "pass@32": 0.9713208827373582, + "pass@64": 0.981635383428142, + "pass@128": 0.9880239520958084 + }, + "7": { + "pass@1": 0.45532082100591714, + "pass@2": 0.6107571023389085, + "pass@4": 0.7487294283295948, + "pass@8": 0.8499193199413594, + "pass@16": 0.9137974968766152, + "pass@32": 0.9478084366667775, + "pass@64": 0.9629303699693006, + "pass@128": 0.9733727810650887 + }, + "8": { + "pass@1": 0.384364983974359, + "pass@2": 0.537295893902685, + "pass@4": 0.6853231852628991, + "pass@8": 0.8045472369839184, + "pass@16": 0.8847355896474814, + "pass@32": 0.9309828246393891, + "pass@64": 0.9555114813719695, + "pass@128": 0.967948717948718 + }, + "9": { + "pass@1": 0.2938820422535211, + "pass@2": 0.4350788787845184, + "pass@4": 0.5845965955840028, + "pass@8": 0.7199691354405304, + "pass@16": 0.8255981073771215, + "pass@32": 0.8950638108118301, + "pass@64": 0.9340900185268076, + "pass@128": 0.9577464788732394 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 234481, + "answer_accuracy": 0.6002237262450852, + "avg_response_len": 194.40882003604193, + "resp_tokens_sum": 75946972, + "pass_at_k": { + "pass@1": 0.6002237262450852, + "pass@2": 0.706425082042497, + "pass@4": 0.7995963684523874, + "pass@8": 0.8706780920406209, + "pass@16": 0.9178523682280787, + "pass@32": 0.9460563837069758, + "pass@64": 0.9616752278909281, + "pass@128": 0.9695281782437746 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.28875095858895705, + "pass@2": 0.43106990604318646, + "pass@4": 0.5864057556455747, + "pass@8": 0.732640888827075, + "pass@16": 0.8480012279076753, + "pass@32": 0.9214811867500342, + "pass@64": 0.9574177953270581, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 0.9847074468085106, + "pass@2": 0.9920482014216307, + "pass@4": 0.9937663534839298, + "pass@8": 0.9939191283233895, + "pass@16": 0.9939209724505289, + "pass@32": 0.993920972644377, + "pass@64": 0.993920972644377, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9926846590909091, + "pass@2": 0.9993184800763542, + "pass@4": 0.9999869709468137, + "pass@8": 0.9999999918335785, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7997873520710059, + "pass@2": 0.8798932319573219, + "pass@4": 0.9325595224228621, + "pass@8": 0.9613584139789305, + "pass@16": 0.975826780702137, + "pass@32": 0.984061812308243, + "pass@64": 0.9891169464353992, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.543659420289855, + "pass@2": 0.636248644870478, + "pass@4": 0.7052871915467087, + "pass@8": 0.7567229407325373, + "pass@16": 0.7959486963018207, + "pass@32": 0.8241830240851653, + "pass@64": 0.8421962984097515, + "pass@128": 0.855072463768116 + }, + "6": { + "pass@1": 0.605860494269341, + "pass@2": 0.7515733225413447, + "pass@4": 0.8630037455024421, + "pass@8": 0.9309214882963924, + "pass@16": 0.9633535490981642, + "pass@32": 0.9774744246618872, + "pass@64": 0.9856872646797787, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.46477601156069365, + "pass@2": 0.6259927859451095, + "pass@4": 0.7673764361463481, + "pass@8": 0.8647695309449625, + "pass@16": 0.9189930954067715, + "pass@32": 0.9480705098855492, + "pass@64": 0.9656972328682472, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.4154049295774648, + "pass@2": 0.5812035599423316, + "pass@4": 0.7328646960503166, + "pass@8": 0.8477899594817437, + "pass@16": 0.9197686125029038, + "pass@32": 0.9579332312942341, + "pass@64": 0.9763086603935157, + "pass@128": 0.9830985915492958 + }, + "9": { + "pass@1": 0.32508420658682635, + "pass@2": 0.4706348867226176, + "pass@4": 0.6193361934735703, + "pass@8": 0.7496807347629907, + "pass@16": 0.8464234084190709, + "pass@32": 0.9094847682238388, + "pass@64": 0.9470349431575563, + "pass@128": 0.9670658682634731 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 230006, + "answer_accuracy": 0.6015808085035153, + "avg_response_len": 208.69610761215267, + "resp_tokens_sum": 79792035, + "pass_at_k": { + "pass@1": 0.6015808085035153, + "pass@2": 0.7042481824125018, + "pass@4": 0.7953534380766873, + "pass@8": 0.8654975239913543, + "pass@16": 0.913341948401476, + "pass@32": 0.9436198379723506, + "pass@64": 0.9618432683667312, + "pass@128": 0.9715433545363241 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3172574626865672, + "pass@2": 0.46422060465389586, + "pass@4": 0.6129414969584025, + "pass@8": 0.7433017291309109, + "pass@16": 0.8425144907670432, + "pass@32": 0.9081062505290839, + "pass@64": 0.9492021871971893, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9898883928571428, + "pass@2": 0.9931893278965128, + "pass@4": 0.9941566200653488, + "pass@8": 0.9942830392971678, + "pass@16": 0.9942857133700204, + "pass@32": 0.9942857142857143, + "pass@64": 0.9942857142857143, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9862882653061225, + "pass@2": 0.9972843397764055, + "pass@4": 0.9996692599102955, + "pass@8": 0.9999825134422476, + "pass@16": 0.9999999402345775, + "pass@32": 0.9999999999999069, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.8163834064327485, + "pass@2": 0.890046176382557, + "pass@4": 0.9372147971635119, + "pass@8": 0.9636458409465761, + "pass@16": 0.9772710270520469, + "pass@32": 0.9842145598658137, + "pass@64": 0.9873700673843149, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.5637867647058824, + "pass@2": 0.6648444737146825, + "pass@4": 0.7408300522177375, + "pass@8": 0.7912320906946062, + "pass@16": 0.823131342775329, + "pass@32": 0.8458919379220898, + "pass@64": 0.8639400997706301, + "pass@128": 0.8764705882352941 + }, + "6": { + "pass@1": 0.5662460567823344, + "pass@2": 0.7139843730594401, + "pass@4": 0.8325656318788228, + "pass@8": 0.9097002960639579, + "pass@16": 0.9523965826489672, + "pass@32": 0.9761397948002681, + "pass@64": 0.9890095178629523, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.4513449367088608, + "pass@2": 0.6131500423602112, + "pass@4": 0.7506766076708764, + "pass@8": 0.8459427764388251, + "pass@16": 0.9042667172900669, + "pass@32": 0.9384254825081815, + "pass@64": 0.9588122968919452, + "pass@128": 0.9715189873417721 + }, + "8": { + "pass@1": 0.38325825825825827, + "pass@2": 0.5393948328249514, + "pass@4": 0.6901599282071716, + "pass@8": 0.8106203042926252, + "pass@16": 0.8915132477415891, + "pass@32": 0.9391020578007191, + "pass@64": 0.9651795386130988, + "pass@128": 0.978978978978979 + }, + "9": { + "pass@1": 0.2740906350482315, + "pass@2": 0.4123497512469298, + "pass@4": 0.5664755532969952, + "pass@8": 0.7106967783354227, + "pass@16": 0.824073336435741, + "pass@32": 0.9018407429475074, + "pass@64": 0.9476444488564583, + "pass@128": 0.9678456591639871 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.16850078125, + "avg_response_len": 317.42343671875, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.16850078125, + "pass@2": 0.2639801550196887, + "pass@4": 0.37491377890889155, + "pass@8": 0.4842486487235785, + "pass@16": 0.5795502706707474, + "pass@32": 0.6588488877373366, + "pass@64": 0.7246587170491089, + "pass@128": 0.7793 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.2485859375, + "pass@2": 0.37810875984252, + "pass@4": 0.520798607049118, + "pass@8": 0.6561497426876107, + "pass@16": 0.7696150043421611, + "pass@32": 0.8554422184015045, + "pass@64": 0.9139319281938864, + "pass@128": 0.95 + }, + "12": { + "pass@1": 0.2306796875, + "pass@2": 0.3520704970472437, + "pass@4": 0.48756367425946673, + "pass@8": 0.6168539258274401, + "pass@16": 0.7289486020309567, + "pass@32": 0.8221630933625493, + "pass@64": 0.8923056471761678, + "pass@128": 0.94 + }, + "13": { + "pass@1": 0.2017421875, + "pass@2": 0.3116034694881889, + "pass@4": 0.43469092538432663, + "pass@8": 0.5533254273383503, + "pass@16": 0.6587606900127015, + "pass@32": 0.751402670674864, + "pass@64": 0.8280399221692232, + "pass@128": 0.886 + }, + "14": { + "pass@1": 0.1823203125, + "pass@2": 0.2840174704724409, + "pass@4": 0.40260358670791113, + "pass@8": 0.5209449489752673, + "pass@16": 0.6274527677177079, + "pass@32": 0.7212299705906391, + "pass@64": 0.8005728072905972, + "pass@128": 0.865 + }, + "15": { + "pass@1": 0.1433203125, + "pass@2": 0.23249852362204737, + "pass@4": 0.34314680736782827, + "pass@8": 0.45891608152280405, + "pass@16": 0.5651978335549982, + "pass@32": 0.6594439925983914, + "pass@64": 0.7455794986603643, + "pass@128": 0.821 + }, + "16": { + "pass@1": 0.1641796875, + "pass@2": 0.25789062500000004, + "pass@4": 0.36468310639295015, + "pass@8": 0.46519237448797657, + "pass@16": 0.5504731356685236, + "pass@32": 0.626458234090645, + "pass@64": 0.6999773208675142, + "pass@128": 0.77 + }, + "17": { + "pass@1": 0.1411953125, + "pass@2": 0.22621788877952767, + "pass@4": 0.32764529593175784, + "pass@8": 0.42876161935287177, + "pass@16": 0.5162824882246035, + "pass@32": 0.5883479803882948, + "pass@64": 0.650545993754578, + "pass@128": 0.707 + }, + "18": { + "pass@1": 0.1320078125, + "pass@2": 0.2094924950787402, + "pass@4": 0.30187055352455916, + "pass@8": 0.39599135008192393, + "pass@16": 0.4785160508794164, + "pass@32": 0.5446384215366399, + "pass@64": 0.5997152957495735, + "pass@128": 0.649 + }, + "19": { + "pass@1": 0.12775, + "pass@2": 0.2039974163385826, + "pass@4": 0.2945484766591674, + "pass@8": 0.38373792069473334, + "pass@16": 0.45854154870837044, + "pass@32": 0.5170329730512411, + "pass@64": 0.566199179328516, + "pass@128": 0.612 + }, + "20": { + "pass@1": 0.1132265625, + "pass@2": 0.183904404527559, + "pass@4": 0.2715867558117733, + "pass@8": 0.36261309626668203, + "pass@16": 0.4417145855678602, + "pass@32": 0.5023293226786212, + "pass@64": 0.5497195773006157, + "pass@128": 0.593 + } + }, + "per_op_accuracy": { + "11": 0.2485859375, + "12": 0.2306796875, + "13": 0.2017421875, + "14": 0.1823203125, + "15": 0.1433203125, + "16": 0.1641796875, + "17": 0.1411953125, + "18": 0.1320078125, + "19": 0.12775, + "20": 0.1132265625 + }, + "per_op_avg_response_len": { + "11": 286.6140859375, + "12": 297.71921875, + "13": 310.05965625, + "14": 321.387109375, + "15": 319.5639921875, + "16": 327.8243203125, + "17": 331.366328125, + "18": 326.1514140625, + "19": 326.956828125, + "20": 326.5914140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 286.6140859375, + "12": 297.71921875, + "13": 310.05965625, + "14": 321.387109375, + "15": 319.5639921875, + "16": 327.8243203125, + "17": 331.366328125, + "18": 326.1514140625, + "19": 326.956828125, + "20": 326.5914140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 73749, + "answer_accuracy": 0.17122260401188708, + "avg_response_len": 301.5612068164933, + "resp_tokens_sum": 129888443, + "pass_at_k": { + "pass@1": 0.17122260401188708, + "pass@2": 0.2669104871827875, + "pass@4": 0.3767544757592551, + "pass@8": 0.4834035503088786, + "pass@16": 0.5750512726058271, + "pass@32": 0.651884120830565, + "pass@64": 0.717570455765211, + "pass@128": 0.7726597325408618 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.2533197592067989, + "pass@2": 0.38688282103901334, + "pass@4": 0.5323678361911565, + "pass@8": 0.6645512690820914, + "pass@16": 0.7685657965631995, + "pass@32": 0.8465825707903929, + "pass@64": 0.9054490438521088, + "pass@128": 0.9518413597733711 + }, + "12": { + "pass@1": 0.2280411370056497, + "pass@2": 0.34637050135682185, + "pass@4": 0.4763980580287633, + "pass@8": 0.6009860957867592, + "pass@16": 0.7120890452970247, + "pass@32": 0.8080114985944731, + "pass@64": 0.8822205445332022, + "pass@128": 0.9265536723163842 + }, + "13": { + "pass@1": 0.20368828369905956, + "pass@2": 0.31595992335793444, + "pass@4": 0.4411990043172506, + "pass@8": 0.5599757590099455, + "pass@16": 0.664349775924061, + "pass@32": 0.7572927869826414, + "pass@64": 0.8338433992759554, + "pass@128": 0.8871473354231975 + }, + "14": { + "pass@1": 0.17399240654205608, + "pass@2": 0.2759075201756323, + "pass@4": 0.396560774529352, + "pass@8": 0.5163368704240593, + "pass@16": 0.6228329513919489, + "pass@32": 0.7182782309133995, + "pass@64": 0.8011897358792905, + "pass@128": 0.8660436137071651 + }, + "15": { + "pass@1": 0.15435332369942195, + "pass@2": 0.24282648149833852, + "pass@4": 0.34869666183345543, + "pass@8": 0.45676819695321474, + "pass@16": 0.5541090817165775, + "pass@32": 0.6384982135537257, + "pass@64": 0.7144541649154501, + "pass@128": 0.7774566473988439 + }, + "16": { + "pass@1": 0.16794354838709677, + "pass@2": 0.2621713868427737, + "pass@4": 0.37036601170821387, + "pass@8": 0.47215387892998295, + "pass@16": 0.5571265858208673, + "pass@32": 0.6332602700575862, + "pass@64": 0.7098035889377893, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.1376896676300578, + "pass@2": 0.21637470984479543, + "pass@4": 0.3087660631113307, + "pass@8": 0.40222533552236417, + "pass@16": 0.4867482784741931, + "pass@32": 0.5601146970644826, + "pass@64": 0.6261372264623557, + "pass@128": 0.6878612716763006 + }, + "18": { + "pass@1": 0.13252232142857143, + "pass@2": 0.21067913385826756, + "pass@4": 0.30457685414323193, + "pass@8": 0.4001484284475485, + "pass@16": 0.48298183754403196, + "pass@32": 0.5496689805711834, + "pass@64": 0.6102941385353717, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.12868337087087087, + "pass@2": 0.2052394421981036, + "pass@4": 0.29570594019216084, + "pass@8": 0.38331477986605983, + "pass@16": 0.45245381726726425, + "pass@32": 0.5028735361578468, + "pass@64": 0.5445303267550561, + "pass@128": 0.5795795795795796 + }, + "20": { + "pass@1": 0.12865990990990991, + "pass@2": 0.2023191991203802, + "pass@4": 0.28769374689650284, + "pass@8": 0.3713682642874231, + "pass@16": 0.4421942211987564, + "pass@32": 0.4975821498514105, + "pass@64": 0.5426479505551459, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 72798, + "answer_accuracy": 0.16911518733273861, + "avg_response_len": 315.7855523342254, + "resp_tokens_sum": 135934312, + "pass_at_k": { + "pass@1": 0.16911518733273861, + "pass@2": 0.265243210914983, + "pass@4": 0.37740085159961956, + "pass@8": 0.48879596912903306, + "pass@16": 0.5873576684001752, + "pass@32": 0.6697681048234034, + "pass@64": 0.7368174770578717, + "pass@128": 0.7915551590841511 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.22632718373493976, + "pass@2": 0.34282800493311844, + "pass@4": 0.47730566331693486, + "pass@8": 0.6155398332967894, + "pass@16": 0.741496303293349, + "pass@32": 0.8402857635506765, + "pass@64": 0.9063125577386248, + "pass@128": 0.9427710843373494 + }, + "12": { + "pass@1": 0.24872648902821318, + "pass@2": 0.3795660497864882, + "pass@4": 0.5243167942565169, + "pass@8": 0.6566606510122126, + "pass@16": 0.7632659524563736, + "pass@32": 0.8451385844187532, + "pass@64": 0.9036814870973651, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.19793855042016806, + "pass@2": 0.3077669059749886, + "pass@4": 0.4309012725825237, + "pass@8": 0.5507201025037028, + "pass@16": 0.659313944560305, + "pass@32": 0.7545849536297392, + "pass@64": 0.8333705544433158, + "pass@128": 0.896358543417367 + }, + "14": { + "pass@1": 0.2007721125730994, + "pass@2": 0.30511487256527137, + "pass@4": 0.41963403944024513, + "pass@8": 0.5284681700916576, + "pass@16": 0.6283094308749538, + "pass@32": 0.7210936702586249, + "pass@64": 0.799903130756725, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.14105113636363636, + "pass@2": 0.23116723335719394, + "pass@4": 0.343928793559896, + "pass@8": 0.46371082103721484, + "pass@16": 0.5760989853454654, + "pass@32": 0.6761599186144563, + "pass@64": 0.7661271643204762, + "pass@128": 0.8484848484848485 + }, + "16": { + "pass@1": 0.15892094017094016, + "pass@2": 0.250165093545999, + "pass@4": 0.3544416253523868, + "pass@8": 0.45494536835752714, + "pass@16": 0.5440344713666627, + "pass@32": 0.6268711139804762, + "pass@64": 0.7067472076557446, + "pass@128": 0.7749287749287749 + }, + "17": { + "pass@1": 0.14596273291925466, + "pass@2": 0.23891837739032604, + "pass@4": 0.3522555392920606, + "pass@8": 0.4648475282331197, + "pass@16": 0.560597983365616, + "pass@32": 0.6383266290994818, + "pass@64": 0.7019445712769129, + "pass@128": 0.7546583850931677 + }, + "18": { + "pass@1": 0.14418658088235295, + "pass@2": 0.2277142919175543, + "pass@4": 0.326042368049582, + "pass@8": 0.42390522986713636, + "pass@16": 0.5069815929983243, + "pass@32": 0.5700109209299284, + "pass@64": 0.6170846108556122, + "pass@128": 0.6529411764705882 + }, + "19": { + "pass@1": 0.12446521577380952, + "pass@2": 0.1989730531730408, + "pass@4": 0.28920478829655233, + "pass@8": 0.3814292207708836, + "pass@16": 0.46396975192531037, + "pass@32": 0.5318783006080356, + "pass@64": 0.5880874256563429, + "pass@128": 0.6398809523809523 + }, + "20": { + "pass@1": 0.10404191616766467, + "pass@2": 0.17241832771464963, + "pass@4": 0.25979317517944994, + "pass@8": 0.3531867827175506, + "pass@16": 0.43568691680178195, + "pass@32": 0.4989061678982635, + "pass@64": 0.5489351028795153, + "pass@128": 0.5958083832335329 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 69134, + "answer_accuracy": 0.16507010238386308, + "avg_response_len": 335.41995530256725, + "resp_tokens_sum": 140479244, + "pass_at_k": { + "pass@1": 0.16507010238386308, + "pass@2": 0.25966835040332653, + "pass@4": 0.3704645214637776, + "pass@8": 0.48044397807620187, + "pass@16": 0.5761526086671405, + "pass@32": 0.6547887146263415, + "pass@64": 0.7194515316306361, + "pass@128": 0.7735330073349633 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.26674107142857145, + "pass@2": 0.40546103612048484, + "pass@4": 0.5536739068330748, + "pass@8": 0.6895362541177696, + "pass@16": 0.8004270014665438, + "pass@32": 0.8813450711545094, + "pass@64": 0.9314687191900564, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.21593081039755352, + "pass@2": 0.3314182559175517, + "pass@4": 0.46379726100751173, + "pass@8": 0.5951990833211992, + "pass@16": 0.7137224500373927, + "pass@32": 0.8150697689006822, + "pass@64": 0.8921259328053642, + "pass@128": 0.9480122324159022 + }, + "13": { + "pass@1": 0.20401716820987653, + "pass@2": 0.31154156945659583, + "pass@4": 0.43245891572581213, + "pass@8": 0.5496484062356665, + "pass@16": 0.652648250879313, + "pass@32": 0.7420970468567396, + "pass@64": 0.8164524378516309, + "pass@128": 0.8734567901234568 + }, + "14": { + "pass@1": 0.17152726261127596, + "pass@2": 0.2703319586906236, + "pass@4": 0.39107636971936394, + "pass@8": 0.5176994047412352, + "pass@16": 0.6309838722897884, + "pass@32": 0.724179890916876, + "pass@64": 0.8006647814674875, + "pass@128": 0.8635014836795252 + }, + "15": { + "pass@1": 0.13384934413580246, + "pass@2": 0.22282522844366667, + "pass@4": 0.33642364351446796, + "pass@8": 0.45632627911948054, + "pass@16": 0.5659365003613015, + "pass@32": 0.6647865356976304, + "pass@64": 0.7578901030057378, + "pass@128": 0.8395061728395061 + }, + "16": { + "pass@1": 0.1661827064896755, + "pass@2": 0.2619750656167981, + "pass@4": 0.3700903606628819, + "pass@8": 0.46943612898581016, + "pass@16": 0.5510554413402865, + "pass@32": 0.61981058809925, + "pass@64": 0.6839821192024573, + "pass@128": 0.7492625368731564 + }, + "17": { + "pass@1": 0.1402249623493976, + "pass@2": 0.22415819775163626, + "pass@4": 0.3234516700096222, + "pass@8": 0.4214179191899702, + "pass@16": 0.5040814253879703, + "pass@32": 0.5692985561263602, + "pass@64": 0.6261336188175767, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.1180695564516129, + "pass@2": 0.18816754508509018, + "pass@4": 0.27230403044377516, + "pass@8": 0.3606826515176, + "pass@16": 0.4422537616760494, + "pass@32": 0.5111308555501609, + "pass@64": 0.5687212244235005, + "pass@128": 0.6161290322580645 + }, + "19": { + "pass@1": 0.13014539274924472, + "pass@2": 0.20784814568118562, + "pass@4": 0.2988084281194458, + "pass@8": 0.38650719281063034, + "pass@16": 0.45915586378086676, + "pass@32": 0.5162083882367925, + "pass@64": 0.565780077034896, + "pass@128": 0.6163141993957704 + }, + "20": { + "pass@1": 0.10700544294294294, + "pass@2": 0.17701017947080935, + "pass@4": 0.26730876151742283, + "pass@8": 0.36331254904296867, + "pass@16": 0.44728071981044754, + "pass@32": 0.5105099300302764, + "pass@64": 0.5575780342462267, + "pass@128": 0.5975975975975976 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.37111060855263156, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.37111060855263156, + "pass@2": 0.4705288411728007, + "pass@4": 0.5726498338414601, + "pass@8": 0.6641248175693354, + "pass@16": 0.7374310349411671, + "pass@32": 0.7934550028144993, + "pass@64": 0.8363574498491765, + "pass@128": 0.8696315789473684 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.294984375, + "pass@2": 0.44116104822834673, + "pass@4": 0.5971287656542937, + "pass@8": 0.7389807252973897, + "pass@16": 0.84768807963811, + "pass@32": 0.9169143421175255, + "pass@64": 0.9549806583238332, + "pass@128": 0.974 + }, + "2": { + "pass@1": 0.9836640625, + "pass@2": 0.9895119340551181, + "pass@4": 0.9916571213910761, + "pass@8": 0.9926734396375929, + "pass@16": 0.9936280770317396, + "pass@32": 0.9946112162573953, + "pass@64": 0.9954085252332271, + "pass@128": 0.996 + }, + "3": { + "pass@1": 0.9901796875, + "pass@2": 0.9978218503937013, + "pass@4": 0.9992752950881137, + "pass@8": 0.9996609731267692, + "pass@16": 0.9998980295834996, + "pass@32": 0.9999929051783535, + "pass@64": 0.9999999947663261, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.81384375, + "pass@2": 0.8893309547244093, + "pass@4": 0.9378742193475821, + "pass@8": 0.9656068663773961, + "pass@16": 0.9801139588085326, + "pass@32": 0.987525844495081, + "pass@64": 0.9913506819842142, + "pass@128": 0.993 + }, + "5": { + "pass@1": 0.5399140625, + "pass@2": 0.6332266240157477, + "pass@4": 0.7043046991001116, + "pass@8": 0.7557135471072218, + "pass@16": 0.7935086678160447, + "pass@32": 0.8229397434474545, + "pass@64": 0.8457480381228596, + "pass@128": 0.862 + }, + "6": { + "pass@1": 0.5930078125, + "pass@2": 0.7389413139763772, + "pass@4": 0.8507973174915651, + "pass@8": 0.9205187483944762, + "pass@16": 0.9567578696086759, + "pass@32": 0.9749960639929603, + "pass@64": 0.9853870906007994, + "pass@128": 0.991 + }, + "7": { + "pass@1": 0.4573359375, + "pass@2": 0.6167848179133851, + "pass@4": 0.7557966017060365, + "pass@8": 0.8538009052018047, + "pass@16": 0.9125834476186985, + "pass@32": 0.9449341004863567, + "pass@64": 0.9625863934398923, + "pass@128": 0.973 + }, + "8": { + "pass@1": 0.395015625, + "pass@2": 0.5535820620078733, + "pass@4": 0.7038110569928769, + "pass@8": 0.8219207348844457, + "pass@16": 0.8994292729064922, + "pass@32": 0.9432539236445819, + "pass@64": 0.9661139429859158, + "pass@128": 0.977 + }, + "9": { + "pass@1": 0.2981484375, + "pass@2": 0.4398858267716532, + "pass@4": 0.5905639771278585, + "pass@8": 0.727009106554544, + "pass@16": 0.8320795541623611, + "pass@32": 0.9019880364816364, + "pass@64": 0.9426290511859993, + "pass@128": 0.964 + }, + "11": { + "pass@1": 0.2485859375, + "pass@2": 0.37810875984252, + "pass@4": 0.520798607049118, + "pass@8": 0.6561497426876107, + "pass@16": 0.7696150043421611, + "pass@32": 0.8554422184015045, + "pass@64": 0.9139319281938864, + "pass@128": 0.95 + }, + "12": { + "pass@1": 0.2306796875, + "pass@2": 0.3520704970472437, + "pass@4": 0.48756367425946673, + "pass@8": 0.6168539258274401, + "pass@16": 0.7289486020309567, + "pass@32": 0.8221630933625493, + "pass@64": 0.8923056471761678, + "pass@128": 0.94 + }, + "13": { + "pass@1": 0.2017421875, + "pass@2": 0.3116034694881889, + "pass@4": 0.43469092538432663, + "pass@8": 0.5533254273383503, + "pass@16": 0.6587606900127015, + "pass@32": 0.751402670674864, + "pass@64": 0.8280399221692232, + "pass@128": 0.886 + }, + "14": { + "pass@1": 0.1823203125, + "pass@2": 0.2840174704724409, + "pass@4": 0.40260358670791113, + "pass@8": 0.5209449489752673, + "pass@16": 0.6274527677177079, + "pass@32": 0.7212299705906391, + "pass@64": 0.8005728072905972, + "pass@128": 0.865 + }, + "15": { + "pass@1": 0.1433203125, + "pass@2": 0.23249852362204737, + "pass@4": 0.34314680736782827, + "pass@8": 0.45891608152280405, + "pass@16": 0.5651978335549982, + "pass@32": 0.6594439925983914, + "pass@64": 0.7455794986603643, + "pass@128": 0.821 + }, + "16": { + "pass@1": 0.1641796875, + "pass@2": 0.25789062500000004, + "pass@4": 0.36468310639295015, + "pass@8": 0.46519237448797657, + "pass@16": 0.5504731356685236, + "pass@32": 0.626458234090645, + "pass@64": 0.6999773208675142, + "pass@128": 0.77 + }, + "17": { + "pass@1": 0.1411953125, + "pass@2": 0.22621788877952767, + "pass@4": 0.32764529593175784, + "pass@8": 0.42876161935287177, + "pass@16": 0.5162824882246035, + "pass@32": 0.5883479803882948, + "pass@64": 0.650545993754578, + "pass@128": 0.707 + }, + "18": { + "pass@1": 0.1320078125, + "pass@2": 0.2094924950787402, + "pass@4": 0.30187055352455916, + "pass@8": 0.39599135008192393, + "pass@16": 0.4785160508794164, + "pass@32": 0.5446384215366399, + "pass@64": 0.5997152957495735, + "pass@128": 0.649 + }, + "19": { + "pass@1": 0.12775, + "pass@2": 0.2039974163385826, + "pass@4": 0.2945484766591674, + "pass@8": 0.38373792069473334, + "pass@16": 0.45854154870837044, + "pass@32": 0.5170329730512411, + "pass@64": 0.566199179328516, + "pass@128": 0.612 + }, + "20": { + "pass@1": 0.1132265625, + "pass@2": 0.183904404527559, + "pass@4": 0.2715867558117733, + "pass@8": 0.36261309626668203, + "pass@16": 0.4417145855678602, + "pass@32": 0.5023293226786212, + "pass@64": 0.5497195773006157, + "pass@128": 0.593 + } + }, + "per_op_accuracy": { + "10": 0.294984375, + "2": 0.9836640625, + "3": 0.9901796875, + "4": 0.81384375, + "5": 0.5399140625, + "6": 0.5930078125, + "7": 0.4573359375, + "8": 0.395015625, + "9": 0.2981484375, + "11": 0.2485859375, + "12": 0.2306796875, + "13": 0.2017421875, + "14": 0.1823203125, + "15": 0.1433203125, + "16": 0.1641796875, + "17": 0.1411953125, + "18": 0.1320078125, + "19": 0.12775, + "20": 0.1132265625 + }, + "per_op_avg_response_len": { + "10": 273.9497890625, + "2": 114.8402265625, + "3": 144.3747265625, + "4": 159.5347578125, + "5": 169.4598828125, + "6": 194.099921875, + "7": 221.0559296875, + "8": 238.8773671875, + "9": 251.6954453125, + "11": 286.6140859375, + "12": 297.71921875, + "13": 310.05965625, + "14": 321.387109375, + "15": 319.5639921875, + "16": 327.8243203125, + "17": 331.366328125, + "18": 326.1514140625, + "19": 326.956828125, + "20": 326.5914140625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 273.9497890625, + "2": 114.8402265625, + "3": 144.3747265625, + "4": 159.5347578125, + "5": 169.4598828125, + "6": 194.099921875, + "7": 221.0559296875, + "8": 238.8773671875, + "9": 251.6954453125, + "11": 286.6140859375, + "12": 297.71921875, + "13": 310.05965625, + "14": 321.387109375, + "15": 319.5639921875, + "16": 327.8243203125, + "17": 331.366328125, + "18": 326.1514140625, + "19": 326.956828125, + "20": 326.5914140625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 296122, + "answer_accuracy": 0.3657055208662662, + "avg_response_len": 247.53881056354726, + "resp_tokens_sum": 200439106, + "pass_at_k": { + "pass@1": 0.3657055208662662, + "pass@2": 0.4645592080614758, + "pass@4": 0.5663767038851872, + "pass@8": 0.6576015416220393, + "pass@16": 0.7305242905130579, + "pass@32": 0.7864036837810774, + "pass@64": 0.8300300857031987, + "pass@128": 0.8645273474549479 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2789684734513274, + "pass@2": 0.4280777471953171, + "pass@4": 0.591814418385755, + "pass@8": 0.7408074226575495, + "pass@16": 0.852499483578908, + "pass@32": 0.9212267885243984, + "pass@64": 0.9583472694281799, + "pass@128": 0.976401179941003 + }, + "2": { + "pass@1": 0.9758080218068536, + "pass@2": 0.9829028380798195, + "pass@4": 0.98677001268533, + "pass@8": 0.9896416905457607, + "pass@16": 0.992610833071678, + "pass@32": 0.9956735708953123, + "pass@64": 0.9981573994804587, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9917335626911316, + "pass@2": 0.996875300994486, + "pass@4": 0.9981438487046914, + "pass@8": 0.9989815709816434, + "pass@16": 0.9996882265536386, + "pass@32": 0.9999783032978147, + "pass@64": 0.9999999839948809, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.8259765625, + "pass@2": 0.8985351562499997, + "pass@4": 0.9441926254335391, + "pass@8": 0.9721901401524639, + "pass@16": 0.9876806739981561, + "pass@32": 0.9947236639399566, + "pass@64": 0.9979643470112908, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.5100446428571429, + "pass@2": 0.5957895888013999, + "pass@4": 0.6638044452776733, + "pass@8": 0.7162705451375567, + "pass@16": 0.7588625747565263, + "pass@32": 0.7968042579192405, + "pass@64": 0.8300021626967635, + "pass@128": 0.8539682539682539 + }, + "6": { + "pass@1": 0.6049775449101796, + "pass@2": 0.7494286770946295, + "pass@4": 0.8553464221725272, + "pass@8": 0.9199166321160519, + "pass@16": 0.9540052822565772, + "pass@32": 0.9713208827373582, + "pass@64": 0.981635383428142, + "pass@128": 0.9880239520958084 + }, + "7": { + "pass@1": 0.45532082100591714, + "pass@2": 0.6107571023389085, + "pass@4": 0.7487294283295948, + "pass@8": 0.8499193199413594, + "pass@16": 0.9137974968766152, + "pass@32": 0.9478084366667775, + "pass@64": 0.9629303699693006, + "pass@128": 0.9733727810650887 + }, + "8": { + "pass@1": 0.384364983974359, + "pass@2": 0.537295893902685, + "pass@4": 0.6853231852628991, + "pass@8": 0.8045472369839184, + "pass@16": 0.8847355896474814, + "pass@32": 0.9309828246393891, + "pass@64": 0.9555114813719695, + "pass@128": 0.967948717948718 + }, + "9": { + "pass@1": 0.2938820422535211, + "pass@2": 0.4350788787845184, + "pass@4": 0.5845965955840028, + "pass@8": 0.7199691354405304, + "pass@16": 0.8255981073771215, + "pass@32": 0.8950638108118301, + "pass@64": 0.9340900185268076, + "pass@128": 0.9577464788732394 + }, + "11": { + "pass@1": 0.2533197592067989, + "pass@2": 0.38688282103901334, + "pass@4": 0.5323678361911565, + "pass@8": 0.6645512690820914, + "pass@16": 0.7685657965631995, + "pass@32": 0.8465825707903929, + "pass@64": 0.9054490438521088, + "pass@128": 0.9518413597733711 + }, + "12": { + "pass@1": 0.2280411370056497, + "pass@2": 0.34637050135682185, + "pass@4": 0.4763980580287633, + "pass@8": 0.6009860957867592, + "pass@16": 0.7120890452970247, + "pass@32": 0.8080114985944731, + "pass@64": 0.8822205445332022, + "pass@128": 0.9265536723163842 + }, + "13": { + "pass@1": 0.20368828369905956, + "pass@2": 0.31595992335793444, + "pass@4": 0.4411990043172506, + "pass@8": 0.5599757590099455, + "pass@16": 0.664349775924061, + "pass@32": 0.7572927869826414, + "pass@64": 0.8338433992759554, + "pass@128": 0.8871473354231975 + }, + "14": { + "pass@1": 0.17399240654205608, + "pass@2": 0.2759075201756323, + "pass@4": 0.396560774529352, + "pass@8": 0.5163368704240593, + "pass@16": 0.6228329513919489, + "pass@32": 0.7182782309133995, + "pass@64": 0.8011897358792905, + "pass@128": 0.8660436137071651 + }, + "15": { + "pass@1": 0.15435332369942195, + "pass@2": 0.24282648149833852, + "pass@4": 0.34869666183345543, + "pass@8": 0.45676819695321474, + "pass@16": 0.5541090817165775, + "pass@32": 0.6384982135537257, + "pass@64": 0.7144541649154501, + "pass@128": 0.7774566473988439 + }, + "16": { + "pass@1": 0.16794354838709677, + "pass@2": 0.2621713868427737, + "pass@4": 0.37036601170821387, + "pass@8": 0.47215387892998295, + "pass@16": 0.5571265858208673, + "pass@32": 0.6332602700575862, + "pass@64": 0.7098035889377893, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.1376896676300578, + "pass@2": 0.21637470984479543, + "pass@4": 0.3087660631113307, + "pass@8": 0.40222533552236417, + "pass@16": 0.4867482784741931, + "pass@32": 0.5601146970644826, + "pass@64": 0.6261372264623557, + "pass@128": 0.6878612716763006 + }, + "18": { + "pass@1": 0.13252232142857143, + "pass@2": 0.21067913385826756, + "pass@4": 0.30457685414323193, + "pass@8": 0.4001484284475485, + "pass@16": 0.48298183754403196, + "pass@32": 0.5496689805711834, + "pass@64": 0.6102941385353717, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.12868337087087087, + "pass@2": 0.2052394421981036, + "pass@4": 0.29570594019216084, + "pass@8": 0.38331477986605983, + "pass@16": 0.45245381726726425, + "pass@32": 0.5028735361578468, + "pass@64": 0.5445303267550561, + "pass@128": 0.5795795795795796 + }, + "20": { + "pass@1": 0.12865990990990991, + "pass@2": 0.2023191991203802, + "pass@4": 0.28769374689650284, + "pass@8": 0.3713682642874231, + "pass@16": 0.4421942211987564, + "pass@32": 0.4975821498514105, + "pass@64": 0.5426479505551459, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 307279, + "answer_accuracy": 0.37421935892439595, + "avg_response_len": 258.0393657443492, + "resp_tokens_sum": 211881284, + "pass_at_k": { + "pass@1": 0.37421935892439595, + "pass@2": 0.4751398704132225, + "pass@4": 0.5782645643719765, + "pass@8": 0.6704801841136393, + "pass@16": 0.7445938061826964, + "pass@32": 0.8012149991574135, + "pass@64": 0.8437957865734628, + "pass@128": 0.8762275915822292 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.28875095858895705, + "pass@2": 0.43106990604318646, + "pass@4": 0.5864057556455747, + "pass@8": 0.732640888827075, + "pass@16": 0.8480012279076753, + "pass@32": 0.9214811867500342, + "pass@64": 0.9574177953270581, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 0.9847074468085106, + "pass@2": 0.9920482014216307, + "pass@4": 0.9937663534839298, + "pass@8": 0.9939191283233895, + "pass@16": 0.9939209724505289, + "pass@32": 0.993920972644377, + "pass@64": 0.993920972644377, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9926846590909091, + "pass@2": 0.9993184800763542, + "pass@4": 0.9999869709468137, + "pass@8": 0.9999999918335785, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7997873520710059, + "pass@2": 0.8798932319573219, + "pass@4": 0.9325595224228621, + "pass@8": 0.9613584139789305, + "pass@16": 0.975826780702137, + "pass@32": 0.984061812308243, + "pass@64": 0.9891169464353992, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.543659420289855, + "pass@2": 0.636248644870478, + "pass@4": 0.7052871915467087, + "pass@8": 0.7567229407325373, + "pass@16": 0.7959486963018207, + "pass@32": 0.8241830240851653, + "pass@64": 0.8421962984097515, + "pass@128": 0.855072463768116 + }, + "6": { + "pass@1": 0.605860494269341, + "pass@2": 0.7515733225413447, + "pass@4": 0.8630037455024421, + "pass@8": 0.9309214882963924, + "pass@16": 0.9633535490981642, + "pass@32": 0.9774744246618872, + "pass@64": 0.9856872646797787, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.46477601156069365, + "pass@2": 0.6259927859451095, + "pass@4": 0.7673764361463481, + "pass@8": 0.8647695309449625, + "pass@16": 0.9189930954067715, + "pass@32": 0.9480705098855492, + "pass@64": 0.9656972328682472, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.4154049295774648, + "pass@2": 0.5812035599423316, + "pass@4": 0.7328646960503166, + "pass@8": 0.8477899594817437, + "pass@16": 0.9197686125029038, + "pass@32": 0.9579332312942341, + "pass@64": 0.9763086603935157, + "pass@128": 0.9830985915492958 + }, + "9": { + "pass@1": 0.32508420658682635, + "pass@2": 0.4706348867226176, + "pass@4": 0.6193361934735703, + "pass@8": 0.7496807347629907, + "pass@16": 0.8464234084190709, + "pass@32": 0.9094847682238388, + "pass@64": 0.9470349431575563, + "pass@128": 0.9670658682634731 + }, + "11": { + "pass@1": 0.22632718373493976, + "pass@2": 0.34282800493311844, + "pass@4": 0.47730566331693486, + "pass@8": 0.6155398332967894, + "pass@16": 0.741496303293349, + "pass@32": 0.8402857635506765, + "pass@64": 0.9063125577386248, + "pass@128": 0.9427710843373494 + }, + "12": { + "pass@1": 0.24872648902821318, + "pass@2": 0.3795660497864882, + "pass@4": 0.5243167942565169, + "pass@8": 0.6566606510122126, + "pass@16": 0.7632659524563736, + "pass@32": 0.8451385844187532, + "pass@64": 0.9036814870973651, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.19793855042016806, + "pass@2": 0.3077669059749886, + "pass@4": 0.4309012725825237, + "pass@8": 0.5507201025037028, + "pass@16": 0.659313944560305, + "pass@32": 0.7545849536297392, + "pass@64": 0.8333705544433158, + "pass@128": 0.896358543417367 + }, + "14": { + "pass@1": 0.2007721125730994, + "pass@2": 0.30511487256527137, + "pass@4": 0.41963403944024513, + "pass@8": 0.5284681700916576, + "pass@16": 0.6283094308749538, + "pass@32": 0.7210936702586249, + "pass@64": 0.799903130756725, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.14105113636363636, + "pass@2": 0.23116723335719394, + "pass@4": 0.343928793559896, + "pass@8": 0.46371082103721484, + "pass@16": 0.5760989853454654, + "pass@32": 0.6761599186144563, + "pass@64": 0.7661271643204762, + "pass@128": 0.8484848484848485 + }, + "16": { + "pass@1": 0.15892094017094016, + "pass@2": 0.250165093545999, + "pass@4": 0.3544416253523868, + "pass@8": 0.45494536835752714, + "pass@16": 0.5440344713666627, + "pass@32": 0.6268711139804762, + "pass@64": 0.7067472076557446, + "pass@128": 0.7749287749287749 + }, + "17": { + "pass@1": 0.14596273291925466, + "pass@2": 0.23891837739032604, + "pass@4": 0.3522555392920606, + "pass@8": 0.4648475282331197, + "pass@16": 0.560597983365616, + "pass@32": 0.6383266290994818, + "pass@64": 0.7019445712769129, + "pass@128": 0.7546583850931677 + }, + "18": { + "pass@1": 0.14418658088235295, + "pass@2": 0.2277142919175543, + "pass@4": 0.326042368049582, + "pass@8": 0.42390522986713636, + "pass@16": 0.5069815929983243, + "pass@32": 0.5700109209299284, + "pass@64": 0.6170846108556122, + "pass@128": 0.6529411764705882 + }, + "19": { + "pass@1": 0.12446521577380952, + "pass@2": 0.1989730531730408, + "pass@4": 0.28920478829655233, + "pass@8": 0.3814292207708836, + "pass@16": 0.46396975192531037, + "pass@32": 0.5318783006080356, + "pass@64": 0.5880874256563429, + "pass@128": 0.6398809523809523 + }, + "20": { + "pass@1": 0.10404191616766467, + "pass@2": 0.17241832771464963, + "pass@4": 0.25979317517944994, + "pass@8": 0.3531867827175506, + "pass@16": 0.43568691680178195, + "pass@32": 0.4989061678982635, + "pass@64": 0.5489351028795153, + "pass@128": 0.5958083832335329 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 299140, + "answer_accuracy": 0.37338732225595145, + "avg_response_len": 274.94318056997923, + "resp_tokens_sum": 220271279, + "pass_at_k": { + "pass@1": 0.37338732225595145, + "pass@2": 0.47183642169449863, + "pass@4": 0.5732354423653239, + "pass@8": 0.6642041540865311, + "pass@16": 0.7370704162700414, + "pass@32": 0.7926283959547556, + "pass@64": 0.8351288151632671, + "pass@128": 0.8680300367470842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3172574626865672, + "pass@2": 0.46422060465389586, + "pass@4": 0.6129414969584025, + "pass@8": 0.7433017291309109, + "pass@16": 0.8425144907670432, + "pass@32": 0.9081062505290839, + "pass@64": 0.9492021871971893, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9898883928571428, + "pass@2": 0.9931893278965128, + "pass@4": 0.9941566200653488, + "pass@8": 0.9942830392971678, + "pass@16": 0.9942857133700204, + "pass@32": 0.9942857142857143, + "pass@64": 0.9942857142857143, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9862882653061225, + "pass@2": 0.9972843397764055, + "pass@4": 0.9996692599102955, + "pass@8": 0.9999825134422476, + "pass@16": 0.9999999402345775, + "pass@32": 0.9999999999999069, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.8163834064327485, + "pass@2": 0.890046176382557, + "pass@4": 0.9372147971635119, + "pass@8": 0.9636458409465761, + "pass@16": 0.9772710270520469, + "pass@32": 0.9842145598658137, + "pass@64": 0.9873700673843149, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.5637867647058824, + "pass@2": 0.6648444737146825, + "pass@4": 0.7408300522177375, + "pass@8": 0.7912320906946062, + "pass@16": 0.823131342775329, + "pass@32": 0.8458919379220898, + "pass@64": 0.8639400997706301, + "pass@128": 0.8764705882352941 + }, + "6": { + "pass@1": 0.5662460567823344, + "pass@2": 0.7139843730594401, + "pass@4": 0.8325656318788228, + "pass@8": 0.9097002960639579, + "pass@16": 0.9523965826489672, + "pass@32": 0.9761397948002681, + "pass@64": 0.9890095178629523, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.4513449367088608, + "pass@2": 0.6131500423602112, + "pass@4": 0.7506766076708764, + "pass@8": 0.8459427764388251, + "pass@16": 0.9042667172900669, + "pass@32": 0.9384254825081815, + "pass@64": 0.9588122968919452, + "pass@128": 0.9715189873417721 + }, + "8": { + "pass@1": 0.38325825825825827, + "pass@2": 0.5393948328249514, + "pass@4": 0.6901599282071716, + "pass@8": 0.8106203042926252, + "pass@16": 0.8915132477415891, + "pass@32": 0.9391020578007191, + "pass@64": 0.9651795386130988, + "pass@128": 0.978978978978979 + }, + "9": { + "pass@1": 0.2740906350482315, + "pass@2": 0.4123497512469298, + "pass@4": 0.5664755532969952, + "pass@8": 0.7106967783354227, + "pass@16": 0.824073336435741, + "pass@32": 0.9018407429475074, + "pass@64": 0.9476444488564583, + "pass@128": 0.9678456591639871 + }, + "11": { + "pass@1": 0.26674107142857145, + "pass@2": 0.40546103612048484, + "pass@4": 0.5536739068330748, + "pass@8": 0.6895362541177696, + "pass@16": 0.8004270014665438, + "pass@32": 0.8813450711545094, + "pass@64": 0.9314687191900564, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.21593081039755352, + "pass@2": 0.3314182559175517, + "pass@4": 0.46379726100751173, + "pass@8": 0.5951990833211992, + "pass@16": 0.7137224500373927, + "pass@32": 0.8150697689006822, + "pass@64": 0.8921259328053642, + "pass@128": 0.9480122324159022 + }, + "13": { + "pass@1": 0.20401716820987653, + "pass@2": 0.31154156945659583, + "pass@4": 0.43245891572581213, + "pass@8": 0.5496484062356665, + "pass@16": 0.652648250879313, + "pass@32": 0.7420970468567396, + "pass@64": 0.8164524378516309, + "pass@128": 0.8734567901234568 + }, + "14": { + "pass@1": 0.17152726261127596, + "pass@2": 0.2703319586906236, + "pass@4": 0.39107636971936394, + "pass@8": 0.5176994047412352, + "pass@16": 0.6309838722897884, + "pass@32": 0.724179890916876, + "pass@64": 0.8006647814674875, + "pass@128": 0.8635014836795252 + }, + "15": { + "pass@1": 0.13384934413580246, + "pass@2": 0.22282522844366667, + "pass@4": 0.33642364351446796, + "pass@8": 0.45632627911948054, + "pass@16": 0.5659365003613015, + "pass@32": 0.6647865356976304, + "pass@64": 0.7578901030057378, + "pass@128": 0.8395061728395061 + }, + "16": { + "pass@1": 0.1661827064896755, + "pass@2": 0.2619750656167981, + "pass@4": 0.3700903606628819, + "pass@8": 0.46943612898581016, + "pass@16": 0.5510554413402865, + "pass@32": 0.61981058809925, + "pass@64": 0.6839821192024573, + "pass@128": 0.7492625368731564 + }, + "17": { + "pass@1": 0.1402249623493976, + "pass@2": 0.22415819775163626, + "pass@4": 0.3234516700096222, + "pass@8": 0.4214179191899702, + "pass@16": 0.5040814253879703, + "pass@32": 0.5692985561263602, + "pass@64": 0.6261336188175767, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.1180695564516129, + "pass@2": 0.18816754508509018, + "pass@4": 0.27230403044377516, + "pass@8": 0.3606826515176, + "pass@16": 0.4422537616760494, + "pass@32": 0.5111308555501609, + "pass@64": 0.5687212244235005, + "pass@128": 0.6161290322580645 + }, + "19": { + "pass@1": 0.13014539274924472, + "pass@2": 0.20784814568118562, + "pass@4": 0.2988084281194458, + "pass@8": 0.38650719281063034, + "pass@16": 0.45915586378086676, + "pass@32": 0.5162083882367925, + "pass@64": 0.565780077034896, + "pass@128": 0.6163141993957704 + }, + "20": { + "pass@1": 0.10700544294294294, + "pass@2": 0.17701017947080935, + "pass@4": 0.26730876151742283, + "pass@8": 0.36331254904296867, + "pass@16": 0.44728071981044754, + "pass@32": 0.5105099300302764, + "pass@64": 0.5575780342462267, + "pass@128": 0.5975975975975976 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/summary.csv b/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..df37b8d863bb150606a75db160a3804a22cd84b4 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85656,0.5962326388888889,0.16850078125,0.37111060855263156,0.0,0.0,0.0,196.43200520833332,317.42343671875,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/summary.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8d82d5c554c559ab77c70867bc00f3667c7be5a3 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block16/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85656", + "id_acc": 0.5962326388888889, + "ood_acc": 0.16850078125, + "total_acc": 0.37111060855263156, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 196.43200520833332, + "ood_avg_resp_len": 317.42343671875, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 296122, + "answer_accuracy": 0.3657055208662662, + "avg_response_len": 247.53881056354726, + "resp_tokens_sum": 200439106, + "pass_at_k": { + "pass@1": 0.3657055208662662, + "pass@2": 0.4645592080614758, + "pass@4": 0.5663767038851872, + "pass@8": 0.6576015416220393, + "pass@16": 0.7305242905130579, + "pass@32": 0.7864036837810774, + "pass@64": 0.8300300857031987, + "pass@128": 0.8645273474549479 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.2789684734513274, + "pass@2": 0.4280777471953171, + "pass@4": 0.591814418385755, + "pass@8": 0.7408074226575495, + "pass@16": 0.852499483578908, + "pass@32": 0.9212267885243984, + "pass@64": 0.9583472694281799, + "pass@128": 0.976401179941003 + }, + "2": { + "pass@1": 0.9758080218068536, + "pass@2": 0.9829028380798195, + "pass@4": 0.98677001268533, + "pass@8": 0.9896416905457607, + "pass@16": 0.992610833071678, + "pass@32": 0.9956735708953123, + "pass@64": 0.9981573994804587, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9917335626911316, + "pass@2": 0.996875300994486, + "pass@4": 0.9981438487046914, + "pass@8": 0.9989815709816434, + "pass@16": 0.9996882265536386, + "pass@32": 0.9999783032978147, + "pass@64": 0.9999999839948809, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.8259765625, + "pass@2": 0.8985351562499997, + "pass@4": 0.9441926254335391, + "pass@8": 0.9721901401524639, + "pass@16": 0.9876806739981561, + "pass@32": 0.9947236639399566, + "pass@64": 0.9979643470112908, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.5100446428571429, + "pass@2": 0.5957895888013999, + "pass@4": 0.6638044452776733, + "pass@8": 0.7162705451375567, + "pass@16": 0.7588625747565263, + "pass@32": 0.7968042579192405, + "pass@64": 0.8300021626967635, + "pass@128": 0.8539682539682539 + }, + "6": { + "pass@1": 0.6049775449101796, + "pass@2": 0.7494286770946295, + "pass@4": 0.8553464221725272, + "pass@8": 0.9199166321160519, + "pass@16": 0.9540052822565772, + "pass@32": 0.9713208827373582, + "pass@64": 0.981635383428142, + "pass@128": 0.9880239520958084 + }, + "7": { + "pass@1": 0.45532082100591714, + "pass@2": 0.6107571023389085, + "pass@4": 0.7487294283295948, + "pass@8": 0.8499193199413594, + "pass@16": 0.9137974968766152, + "pass@32": 0.9478084366667775, + "pass@64": 0.9629303699693006, + "pass@128": 0.9733727810650887 + }, + "8": { + "pass@1": 0.384364983974359, + "pass@2": 0.537295893902685, + "pass@4": 0.6853231852628991, + "pass@8": 0.8045472369839184, + "pass@16": 0.8847355896474814, + "pass@32": 0.9309828246393891, + "pass@64": 0.9555114813719695, + "pass@128": 0.967948717948718 + }, + "9": { + "pass@1": 0.2938820422535211, + "pass@2": 0.4350788787845184, + "pass@4": 0.5845965955840028, + "pass@8": 0.7199691354405304, + "pass@16": 0.8255981073771215, + "pass@32": 0.8950638108118301, + "pass@64": 0.9340900185268076, + "pass@128": 0.9577464788732394 + }, + "11": { + "pass@1": 0.2533197592067989, + "pass@2": 0.38688282103901334, + "pass@4": 0.5323678361911565, + "pass@8": 0.6645512690820914, + "pass@16": 0.7685657965631995, + "pass@32": 0.8465825707903929, + "pass@64": 0.9054490438521088, + "pass@128": 0.9518413597733711 + }, + "12": { + "pass@1": 0.2280411370056497, + "pass@2": 0.34637050135682185, + "pass@4": 0.4763980580287633, + "pass@8": 0.6009860957867592, + "pass@16": 0.7120890452970247, + "pass@32": 0.8080114985944731, + "pass@64": 0.8822205445332022, + "pass@128": 0.9265536723163842 + }, + "13": { + "pass@1": 0.20368828369905956, + "pass@2": 0.31595992335793444, + "pass@4": 0.4411990043172506, + "pass@8": 0.5599757590099455, + "pass@16": 0.664349775924061, + "pass@32": 0.7572927869826414, + "pass@64": 0.8338433992759554, + "pass@128": 0.8871473354231975 + }, + "14": { + "pass@1": 0.17399240654205608, + "pass@2": 0.2759075201756323, + "pass@4": 0.396560774529352, + "pass@8": 0.5163368704240593, + "pass@16": 0.6228329513919489, + "pass@32": 0.7182782309133995, + "pass@64": 0.8011897358792905, + "pass@128": 0.8660436137071651 + }, + "15": { + "pass@1": 0.15435332369942195, + "pass@2": 0.24282648149833852, + "pass@4": 0.34869666183345543, + "pass@8": 0.45676819695321474, + "pass@16": 0.5541090817165775, + "pass@32": 0.6384982135537257, + "pass@64": 0.7144541649154501, + "pass@128": 0.7774566473988439 + }, + "16": { + "pass@1": 0.16794354838709677, + "pass@2": 0.2621713868427737, + "pass@4": 0.37036601170821387, + "pass@8": 0.47215387892998295, + "pass@16": 0.5571265858208673, + "pass@32": 0.6332602700575862, + "pass@64": 0.7098035889377893, + "pass@128": 0.7870967741935484 + }, + "17": { + "pass@1": 0.1376896676300578, + "pass@2": 0.21637470984479543, + "pass@4": 0.3087660631113307, + "pass@8": 0.40222533552236417, + "pass@16": 0.4867482784741931, + "pass@32": 0.5601146970644826, + "pass@64": 0.6261372264623557, + "pass@128": 0.6878612716763006 + }, + "18": { + "pass@1": 0.13252232142857143, + "pass@2": 0.21067913385826756, + "pass@4": 0.30457685414323193, + "pass@8": 0.4001484284475485, + "pass@16": 0.48298183754403196, + "pass@32": 0.5496689805711834, + "pass@64": 0.6102941385353717, + "pass@128": 0.6742857142857143 + }, + "19": { + "pass@1": 0.12868337087087087, + "pass@2": 0.2052394421981036, + "pass@4": 0.29570594019216084, + "pass@8": 0.38331477986605983, + "pass@16": 0.45245381726726425, + "pass@32": 0.5028735361578468, + "pass@64": 0.5445303267550561, + "pass@128": 0.5795795795795796 + }, + "20": { + "pass@1": 0.12865990990990991, + "pass@2": 0.2023191991203802, + "pass@4": 0.28769374689650284, + "pass@8": 0.3713682642874231, + "pass@16": 0.4421942211987564, + "pass@32": 0.4975821498514105, + "pass@64": 0.5426479505551459, + "pass@128": 0.5855855855855856 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 307279, + "answer_accuracy": 0.37421935892439595, + "avg_response_len": 258.0393657443492, + "resp_tokens_sum": 211881284, + "pass_at_k": { + "pass@1": 0.37421935892439595, + "pass@2": 0.4751398704132225, + "pass@4": 0.5782645643719765, + "pass@8": 0.6704801841136393, + "pass@16": 0.7445938061826964, + "pass@32": 0.8012149991574135, + "pass@64": 0.8437957865734628, + "pass@128": 0.8762275915822292 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.28875095858895705, + "pass@2": 0.43106990604318646, + "pass@4": 0.5864057556455747, + "pass@8": 0.732640888827075, + "pass@16": 0.8480012279076753, + "pass@32": 0.9214811867500342, + "pass@64": 0.9574177953270581, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 0.9847074468085106, + "pass@2": 0.9920482014216307, + "pass@4": 0.9937663534839298, + "pass@8": 0.9939191283233895, + "pass@16": 0.9939209724505289, + "pass@32": 0.993920972644377, + "pass@64": 0.993920972644377, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9926846590909091, + "pass@2": 0.9993184800763542, + "pass@4": 0.9999869709468137, + "pass@8": 0.9999999918335785, + "pass@16": 0.999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7997873520710059, + "pass@2": 0.8798932319573219, + "pass@4": 0.9325595224228621, + "pass@8": 0.9613584139789305, + "pass@16": 0.975826780702137, + "pass@32": 0.984061812308243, + "pass@64": 0.9891169464353992, + "pass@128": 0.9911242603550295 + }, + "5": { + "pass@1": 0.543659420289855, + "pass@2": 0.636248644870478, + "pass@4": 0.7052871915467087, + "pass@8": 0.7567229407325373, + "pass@16": 0.7959486963018207, + "pass@32": 0.8241830240851653, + "pass@64": 0.8421962984097515, + "pass@128": 0.855072463768116 + }, + "6": { + "pass@1": 0.605860494269341, + "pass@2": 0.7515733225413447, + "pass@4": 0.8630037455024421, + "pass@8": 0.9309214882963924, + "pass@16": 0.9633535490981642, + "pass@32": 0.9774744246618872, + "pass@64": 0.9856872646797787, + "pass@128": 0.9914040114613181 + }, + "7": { + "pass@1": 0.46477601156069365, + "pass@2": 0.6259927859451095, + "pass@4": 0.7673764361463481, + "pass@8": 0.8647695309449625, + "pass@16": 0.9189930954067715, + "pass@32": 0.9480705098855492, + "pass@64": 0.9656972328682472, + "pass@128": 0.9739884393063584 + }, + "8": { + "pass@1": 0.4154049295774648, + "pass@2": 0.5812035599423316, + "pass@4": 0.7328646960503166, + "pass@8": 0.8477899594817437, + "pass@16": 0.9197686125029038, + "pass@32": 0.9579332312942341, + "pass@64": 0.9763086603935157, + "pass@128": 0.9830985915492958 + }, + "9": { + "pass@1": 0.32508420658682635, + "pass@2": 0.4706348867226176, + "pass@4": 0.6193361934735703, + "pass@8": 0.7496807347629907, + "pass@16": 0.8464234084190709, + "pass@32": 0.9094847682238388, + "pass@64": 0.9470349431575563, + "pass@128": 0.9670658682634731 + }, + "11": { + "pass@1": 0.22632718373493976, + "pass@2": 0.34282800493311844, + "pass@4": 0.47730566331693486, + "pass@8": 0.6155398332967894, + "pass@16": 0.741496303293349, + "pass@32": 0.8402857635506765, + "pass@64": 0.9063125577386248, + "pass@128": 0.9427710843373494 + }, + "12": { + "pass@1": 0.24872648902821318, + "pass@2": 0.3795660497864882, + "pass@4": 0.5243167942565169, + "pass@8": 0.6566606510122126, + "pass@16": 0.7632659524563736, + "pass@32": 0.8451385844187532, + "pass@64": 0.9036814870973651, + "pass@128": 0.9467084639498433 + }, + "13": { + "pass@1": 0.19793855042016806, + "pass@2": 0.3077669059749886, + "pass@4": 0.4309012725825237, + "pass@8": 0.5507201025037028, + "pass@16": 0.659313944560305, + "pass@32": 0.7545849536297392, + "pass@64": 0.8333705544433158, + "pass@128": 0.896358543417367 + }, + "14": { + "pass@1": 0.2007721125730994, + "pass@2": 0.30511487256527137, + "pass@4": 0.41963403944024513, + "pass@8": 0.5284681700916576, + "pass@16": 0.6283094308749538, + "pass@32": 0.7210936702586249, + "pass@64": 0.799903130756725, + "pass@128": 0.8654970760233918 + }, + "15": { + "pass@1": 0.14105113636363636, + "pass@2": 0.23116723335719394, + "pass@4": 0.343928793559896, + "pass@8": 0.46371082103721484, + "pass@16": 0.5760989853454654, + "pass@32": 0.6761599186144563, + "pass@64": 0.7661271643204762, + "pass@128": 0.8484848484848485 + }, + "16": { + "pass@1": 0.15892094017094016, + "pass@2": 0.250165093545999, + "pass@4": 0.3544416253523868, + "pass@8": 0.45494536835752714, + "pass@16": 0.5440344713666627, + "pass@32": 0.6268711139804762, + "pass@64": 0.7067472076557446, + "pass@128": 0.7749287749287749 + }, + "17": { + "pass@1": 0.14596273291925466, + "pass@2": 0.23891837739032604, + "pass@4": 0.3522555392920606, + "pass@8": 0.4648475282331197, + "pass@16": 0.560597983365616, + "pass@32": 0.6383266290994818, + "pass@64": 0.7019445712769129, + "pass@128": 0.7546583850931677 + }, + "18": { + "pass@1": 0.14418658088235295, + "pass@2": 0.2277142919175543, + "pass@4": 0.326042368049582, + "pass@8": 0.42390522986713636, + "pass@16": 0.5069815929983243, + "pass@32": 0.5700109209299284, + "pass@64": 0.6170846108556122, + "pass@128": 0.6529411764705882 + }, + "19": { + "pass@1": 0.12446521577380952, + "pass@2": 0.1989730531730408, + "pass@4": 0.28920478829655233, + "pass@8": 0.3814292207708836, + "pass@16": 0.46396975192531037, + "pass@32": 0.5318783006080356, + "pass@64": 0.5880874256563429, + "pass@128": 0.6398809523809523 + }, + "20": { + "pass@1": 0.10404191616766467, + "pass@2": 0.17241832771464963, + "pass@4": 0.25979317517944994, + "pass@8": 0.3531867827175506, + "pass@16": 0.43568691680178195, + "pass@32": 0.4989061678982635, + "pass@64": 0.5489351028795153, + "pass@128": 0.5958083832335329 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 299140, + "answer_accuracy": 0.37338732225595145, + "avg_response_len": 274.94318056997923, + "resp_tokens_sum": 220271279, + "pass_at_k": { + "pass@1": 0.37338732225595145, + "pass@2": 0.47183642169449863, + "pass@4": 0.5732354423653239, + "pass@8": 0.6642041540865311, + "pass@16": 0.7370704162700414, + "pass@32": 0.7926283959547556, + "pass@64": 0.8351288151632671, + "pass@128": 0.8680300367470842 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.3172574626865672, + "pass@2": 0.46422060465389586, + "pass@4": 0.6129414969584025, + "pass@8": 0.7433017291309109, + "pass@16": 0.8425144907670432, + "pass@32": 0.9081062505290839, + "pass@64": 0.9492021871971893, + "pass@128": 0.9731343283582089 + }, + "2": { + "pass@1": 0.9898883928571428, + "pass@2": 0.9931893278965128, + "pass@4": 0.9941566200653488, + "pass@8": 0.9942830392971678, + "pass@16": 0.9942857133700204, + "pass@32": 0.9942857142857143, + "pass@64": 0.9942857142857143, + "pass@128": 0.9942857142857143 + }, + "3": { + "pass@1": 0.9862882653061225, + "pass@2": 0.9972843397764055, + "pass@4": 0.9996692599102955, + "pass@8": 0.9999825134422476, + "pass@16": 0.9999999402345775, + "pass@32": 0.9999999999999069, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.8163834064327485, + "pass@2": 0.890046176382557, + "pass@4": 0.9372147971635119, + "pass@8": 0.9636458409465761, + "pass@16": 0.9772710270520469, + "pass@32": 0.9842145598658137, + "pass@64": 0.9873700673843149, + "pass@128": 0.9883040935672515 + }, + "5": { + "pass@1": 0.5637867647058824, + "pass@2": 0.6648444737146825, + "pass@4": 0.7408300522177375, + "pass@8": 0.7912320906946062, + "pass@16": 0.823131342775329, + "pass@32": 0.8458919379220898, + "pass@64": 0.8639400997706301, + "pass@128": 0.8764705882352941 + }, + "6": { + "pass@1": 0.5662460567823344, + "pass@2": 0.7139843730594401, + "pass@4": 0.8325656318788228, + "pass@8": 0.9097002960639579, + "pass@16": 0.9523965826489672, + "pass@32": 0.9761397948002681, + "pass@64": 0.9890095178629523, + "pass@128": 0.9936908517350158 + }, + "7": { + "pass@1": 0.4513449367088608, + "pass@2": 0.6131500423602112, + "pass@4": 0.7506766076708764, + "pass@8": 0.8459427764388251, + "pass@16": 0.9042667172900669, + "pass@32": 0.9384254825081815, + "pass@64": 0.9588122968919452, + "pass@128": 0.9715189873417721 + }, + "8": { + "pass@1": 0.38325825825825827, + "pass@2": 0.5393948328249514, + "pass@4": 0.6901599282071716, + "pass@8": 0.8106203042926252, + "pass@16": 0.8915132477415891, + "pass@32": 0.9391020578007191, + "pass@64": 0.9651795386130988, + "pass@128": 0.978978978978979 + }, + "9": { + "pass@1": 0.2740906350482315, + "pass@2": 0.4123497512469298, + "pass@4": 0.5664755532969952, + "pass@8": 0.7106967783354227, + "pass@16": 0.824073336435741, + "pass@32": 0.9018407429475074, + "pass@64": 0.9476444488564583, + "pass@128": 0.9678456591639871 + }, + "11": { + "pass@1": 0.26674107142857145, + "pass@2": 0.40546103612048484, + "pass@4": 0.5536739068330748, + "pass@8": 0.6895362541177696, + "pass@16": 0.8004270014665438, + "pass@32": 0.8813450711545094, + "pass@64": 0.9314687191900564, + "pass@128": 0.9555555555555556 + }, + "12": { + "pass@1": 0.21593081039755352, + "pass@2": 0.3314182559175517, + "pass@4": 0.46379726100751173, + "pass@8": 0.5951990833211992, + "pass@16": 0.7137224500373927, + "pass@32": 0.8150697689006822, + "pass@64": 0.8921259328053642, + "pass@128": 0.9480122324159022 + }, + "13": { + "pass@1": 0.20401716820987653, + "pass@2": 0.31154156945659583, + "pass@4": 0.43245891572581213, + "pass@8": 0.5496484062356665, + "pass@16": 0.652648250879313, + "pass@32": 0.7420970468567396, + "pass@64": 0.8164524378516309, + "pass@128": 0.8734567901234568 + }, + "14": { + "pass@1": 0.17152726261127596, + "pass@2": 0.2703319586906236, + "pass@4": 0.39107636971936394, + "pass@8": 0.5176994047412352, + "pass@16": 0.6309838722897884, + "pass@32": 0.724179890916876, + "pass@64": 0.8006647814674875, + "pass@128": 0.8635014836795252 + }, + "15": { + "pass@1": 0.13384934413580246, + "pass@2": 0.22282522844366667, + "pass@4": 0.33642364351446796, + "pass@8": 0.45632627911948054, + "pass@16": 0.5659365003613015, + "pass@32": 0.6647865356976304, + "pass@64": 0.7578901030057378, + "pass@128": 0.8395061728395061 + }, + "16": { + "pass@1": 0.1661827064896755, + "pass@2": 0.2619750656167981, + "pass@4": 0.3700903606628819, + "pass@8": 0.46943612898581016, + "pass@16": 0.5510554413402865, + "pass@32": 0.61981058809925, + "pass@64": 0.6839821192024573, + "pass@128": 0.7492625368731564 + }, + "17": { + "pass@1": 0.1402249623493976, + "pass@2": 0.22415819775163626, + "pass@4": 0.3234516700096222, + "pass@8": 0.4214179191899702, + "pass@16": 0.5040814253879703, + "pass@32": 0.5692985561263602, + "pass@64": 0.6261336188175767, + "pass@128": 0.6807228915662651 + }, + "18": { + "pass@1": 0.1180695564516129, + "pass@2": 0.18816754508509018, + "pass@4": 0.27230403044377516, + "pass@8": 0.3606826515176, + "pass@16": 0.4422537616760494, + "pass@32": 0.5111308555501609, + "pass@64": 0.5687212244235005, + "pass@128": 0.6161290322580645 + }, + "19": { + "pass@1": 0.13014539274924472, + "pass@2": 0.20784814568118562, + "pass@4": 0.2988084281194458, + "pass@8": 0.38650719281063034, + "pass@16": 0.45915586378086676, + "pass@32": 0.5162083882367925, + "pass@64": 0.565780077034896, + "pass@128": 0.6163141993957704 + }, + "20": { + "pass@1": 0.10700544294294294, + "pass@2": 0.17701017947080935, + "pass@4": 0.26730876151742283, + "pass@8": 0.36331254904296867, + "pass@16": 0.44728071981044754, + "pass@32": 0.5105099300302764, + "pass@64": 0.5575780342462267, + "pass@128": 0.5975975975975976 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/checkpoint-85656_metrics.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/checkpoint-85656_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..bec61c41eefa83a9c2a1c1ed9257a4ee29f16649 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/checkpoint-85656_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85656", + "id": { + "answer_accuracy": 0.99334375, + "avg_response_len": 197.87361805555557, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.99334375, + "pass@2": 0.9950787948381443, + "pass@4": 0.9961389760029997, + "pass@8": 0.9968228839822204, + "pass@16": 0.9973031747765807, + "pass@32": 0.9977420715597295, + "pass@64": 0.99825459463593, + "pass@128": 0.9987777777777778 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9520703125, + "pass@2": 0.9622504921259845, + "pass@4": 0.9693263488001502, + "pass@8": 0.9740394352292095, + "pass@16": 0.977639988144664, + "pass@32": 0.9812472061219915, + "pass@64": 0.9855393884549314, + "pass@128": 0.99 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9973984375, + "pass@2": 0.9982915846456695, + "pass@4": 0.9986447747469067, + "pass@8": 0.9989552510738148, + "pass@16": 0.9992111792809737, + "pass@32": 0.9994386769381023, + "pass@64": 0.9997519685023395, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996640625, + "pass@2": 0.9999740403543307, + "pass@4": 0.9999997768091488, + "pass@8": 0.9999999999933986, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999171875, + "pass@2": 0.9998625738188976, + "pass@4": 0.9999931821334833, + "pass@8": 0.999999978289487, + "pass@16": 0.9999999999999217, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961015625, + "pass@2": 0.9980688976377954, + "pass@4": 0.9991727544056992, + "pass@8": 0.9997782511963624, + "pass@16": 0.9999797882169323, + "pass@32": 0.9999998558241998, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.99571875, + "pass@2": 0.997261687992126, + "pass@4": 0.9981139471316086, + "pass@8": 0.9986330400577111, + "pass@16": 0.9988976173467354, + "pass@32": 0.9989929051532713, + "pass@64": 0.9989999947663261, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.9520703125, + "2": 1.0, + "3": 0.999984375, + "4": 0.999984375, + "5": 0.9973984375, + "6": 0.9996640625, + "7": 0.999171875, + "8": 0.9961015625, + "9": 0.99571875 + }, + "per_op_avg_response_len": { + "10": 277.5632734375, + "2": 114.91821875, + "3": 143.4999609375, + "4": 158.580390625, + "5": 171.0569453125, + "6": 196.9027421875, + "7": 221.39425, + "8": 240.706765625, + "9": 256.240015625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.5632734375, + "2": 114.91821875, + "3": 143.4999609375, + "4": 158.580390625, + "5": 171.0569453125, + "6": 196.9027421875, + "7": 221.39425, + "8": 240.706765625, + "9": 256.240015625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376910, + "answer_accuracy": 0.9944644967916244, + "avg_response_len": 188.41718380614657, + "resp_tokens_sum": 71411620, + "pass_at_k": { + "pass@1": 0.9944644967916244, + "pass@2": 0.9958677455876526, + "pass@4": 0.9966693331266722, + "pass@8": 0.997218035897361, + "pass@16": 0.9976537136156286, + "pass@32": 0.9980687057108327, + "pass@64": 0.998567997202145, + "pass@128": 0.9989868287740629 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9606148598820059, + "pass@2": 0.970110604952036, + "pass@4": 0.9766451804143947, + "pass@8": 0.9812420969145403, + "pass@16": 0.984712123290507, + "pass@32": 0.9877358738401864, + "pass@64": 0.9911736613911921, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963293650793651, + "pass@2": 0.9968914041994751, + "pass@4": 0.9970213660792401, + "pass@8": 0.997211286089119, + "pass@16": 0.9975721784776902, + "pass@32": 0.9982189726284215, + "pass@64": 0.9992125984251968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9996622093835906, + "pass@4": 0.999980712377669, + "pass@8": 0.9999999358582831, + "pass@16": 0.9999999999997683, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9995993589743589, + "pass@2": 0.9999917190339189, + "pass@4": 0.9999999969955679, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967209507042254, + "pass@2": 0.9971629976710658, + "pass@4": 0.9971830114545541, + "pass@8": 0.9971830985912242, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387763, + "answer_accuracy": 0.9925945076998689, + "avg_response_len": 195.85308045953474, + "resp_tokens_sum": 76511181, + "pass_at_k": { + "pass@1": 0.9925945076998689, + "pass@2": 0.9943995675999217, + "pass@4": 0.9955660793158555, + "pass@8": 0.9962802922646274, + "pass@16": 0.9967742326179531, + "pass@32": 0.9972096989004664, + "pass@64": 0.9976823307959984, + "pass@128": 0.9983617300131061 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9403278374233128, + "pass@2": 0.952113575431139, + "pass@4": 0.9607398866169331, + "pass@8": 0.9662308127069366, + "pass@16": 0.9701134353995957, + "pass@32": 0.9738990670749782, + "pass@64": 0.9783020822793281, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9997735507246377, + "pass@2": 0.9999942941914869, + "pass@4": 0.9999999959244225, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994851361031518, + "pass@2": 0.9999626322676716, + "pass@4": 0.999999727111475, + "pass@8": 0.9999999999939815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995032514450867, + "pass@2": 0.9999455958991397, + "pass@4": 0.9999991709851297, + "pass@8": 0.9999999999115423, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9990167877342796, + "pass@4": 0.9998786036428545, + "pass@8": 0.9999981102949981, + "pass@16": 0.9999999997381142, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952984655688623, + "pass@2": 0.9967109334480646, + "pass@4": 0.9979339827094966, + "pass@8": 0.9989726884046817, + "pass@16": 0.999694545217648, + "pass@32": 0.9999787580173054, + "pass@64": 0.9999999843303174, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379659, + "answer_accuracy": 0.9929983051556746, + "avg_response_len": 209.3122462964513, + "resp_tokens_sum": 80027607, + "pass_at_k": { + "pass@1": 0.9929983051556746, + "pass@2": 0.9949907192980607, + "pass@4": 0.9961985987837058, + "pass@8": 0.9969855706582709, + "pass@16": 0.9974961396127747, + "pass@32": 0.9979622381598815, + "pass@64": 0.998528636899375, + "pass@128": 0.9989956478071644 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9548507462686567, + "pass@2": 0.9641611088259492, + "pass@4": 0.9702759092613421, + "pass@8": 0.9743496102469225, + "pass@16": 0.9778077564445188, + "pass@32": 0.9818317881305241, + "pass@64": 0.9868807117022572, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9959788602941176, + "pass@2": 0.9978610612552107, + "pass@4": 0.9987736642294712, + "pass@8": 0.9995108704580657, + "pass@16": 0.9999292442955922, + "pass@32": 0.9999991192945575, + "pass@64": 0.9999999999953015, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.999959248242629, + "pass@4": 0.999999596363546, + "pass@8": 0.9999999999858014, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997033227848101, + "pass@2": 0.9999859837536131, + "pass@4": 0.999999962623343, + "pass@8": 0.99999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920701951951952, + "pass@2": 0.9952568217824125, + "pass@4": 0.9976451983479543, + "pass@8": 0.999336102227142, + "pass@16": 0.9999393042339394, + "pass@32": 0.9999995670396389, + "pass@64": 0.9999999999993233, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950261254019293, + "pass@2": 0.9979658265184698, + "pass@4": 0.999369864438167, + "pass@8": 0.9999233509024527, + "pass@16": 0.9999988400129937, + "pass@32": 0.9999999999211941, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.4865421875, + "avg_response_len": 336.17079375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.4865421875, + "pass@2": 0.5550204970472387, + "pass@4": 0.614388754405692, + "pass@8": 0.6643861191735821, + "pass@16": 0.7050709922660653, + "pass@32": 0.7386174171445529, + "pass@64": 0.7673693823426535, + "pass@128": 0.7924 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.936265625, + "pass@2": 0.9495132874015753, + "pass@4": 0.958506849550056, + "pass@8": 0.965062425401987, + "pass@16": 0.9703263571802695, + "pass@32": 0.9747704580686662, + "pass@64": 0.9780232404547895, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.849375, + "pass@2": 0.8816822096456706, + "pass@4": 0.9012263803899512, + "pass@8": 0.915389220673123, + "pass@16": 0.9268891903487071, + "pass@32": 0.9374594637955742, + "pass@64": 0.9478378870624736, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.7028828125, + "pass@2": 0.7634910187007871, + "pass@4": 0.80349823856393, + "pass@8": 0.8328035833884266, + "pass@16": 0.8556732101613651, + "pass@32": 0.8749960895614166, + "pass@64": 0.8919409402783807, + "pass@128": 0.906 + }, + "14": { + "pass@1": 0.663578125, + "pass@2": 0.7455086122047233, + "pass@4": 0.8028635613048376, + "pass@8": 0.8408190447612812, + "pass@16": 0.8657194023573667, + "pass@32": 0.8846751496105657, + "pass@64": 0.9007706768086038, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.53509375, + "pass@2": 0.634340920275591, + "pass@4": 0.7128604744094488, + "pass@8": 0.7738329492369901, + "pass@16": 0.8191749591839835, + "pass@32": 0.8522332964352862, + "pass@64": 0.8774544898600107, + "pass@128": 0.896 + }, + "16": { + "pass@1": 0.338671875, + "pass@2": 0.43235433070866197, + "pass@4": 0.5150519089801271, + "pass@8": 0.5826601206991043, + "pass@16": 0.6371272371185606, + "pass@32": 0.6838205096197499, + "pass@64": 0.7247935340279482, + "pass@128": 0.76 + }, + "17": { + "pass@1": 0.2698671875, + "pass@2": 0.35958489173228403, + "pass@4": 0.443685483783277, + "pass@8": 0.5150248219266278, + "pass@16": 0.5743802663139518, + "pass@32": 0.6261713784938127, + "pass@64": 0.6728045483676934, + "pass@128": 0.716 + }, + "18": { + "pass@1": 0.2368671875, + "pass@2": 0.31734301181102353, + "pass@4": 0.3965205211848512, + "pass@8": 0.4693614104730791, + "pass@16": 0.5308608509282342, + "pass@32": 0.5811864399109724, + "pass@64": 0.6247096074512953, + "pass@128": 0.664 + }, + "19": { + "pass@1": 0.1771015625, + "pass@2": 0.24749889271653538, + "pass@4": 0.3212013932320957, + "pass@8": 0.3898091878663418, + "pass@16": 0.44785443289121296, + "pass@32": 0.4956818275816109, + "pass@64": 0.5372515790393153, + "pass@128": 0.576 + }, + "20": { + "pass@1": 0.15571875, + "pass@2": 0.21888779527559046, + "pass@4": 0.2884727326584174, + "pass@8": 0.3590984273088792, + "pass@16": 0.4227040161768628, + "pass@32": 0.4751795583678563, + "pass@64": 0.5181073200759404, + "pass@128": 0.553 + } + }, + "per_op_accuracy": { + "11": 0.936265625, + "12": 0.849375, + "13": 0.7028828125, + "14": 0.663578125, + "15": 0.53509375, + "16": 0.338671875, + "17": 0.2698671875, + "18": 0.2368671875, + "19": 0.1771015625, + "20": 0.15571875 + }, + "per_op_avg_response_len": { + "11": 294.3470546875, + "12": 303.4772734375, + "13": 318.9317890625, + "14": 340.3666171875, + "15": 348.7077109375, + "16": 348.68609375, + "17": 355.6078203125, + "18": 351.8683515625, + "19": 350.7575546875, + "20": 348.957671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 294.3470546875, + "12": 303.4772734375, + "13": 318.9317890625, + "14": 340.3666171875, + "15": 348.7077109375, + "16": 348.68609375, + "17": 355.6078203125, + "18": 351.8683515625, + "19": 350.7575546875, + "20": 348.957671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 208964, + "answer_accuracy": 0.4851504457652303, + "avg_response_len": 319.48618592124814, + "resp_tokens_sum": 137609090, + "pass_at_k": { + "pass@1": 0.4851504457652303, + "pass@2": 0.5526870956230776, + "pass@4": 0.6109670868871723, + "pass@8": 0.6604481002812127, + "pass@16": 0.7017868095093525, + "pass@32": 0.7370719098332393, + "pass@64": 0.7681035990180648, + "pass@128": 0.7961367013372956 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9175593130311614, + "pass@2": 0.9301508721643506, + "pass@4": 0.9383426105022994, + "pass@8": 0.94506009266078, + "pass@16": 0.9523354989438444, + "pass@32": 0.9605310994724667, + "pass@64": 0.9676012340007842, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.8390271892655368, + "pass@2": 0.8704061290537837, + "pass@4": 0.8879430150680314, + "pass@8": 0.8998900279443047, + "pass@16": 0.9101521093680011, + "pass@32": 0.9205997134915045, + "pass@64": 0.9318575609376857, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.7098354231974922, + "pass@2": 0.7657214196677613, + "pass@4": 0.8028330453599253, + "pass@8": 0.8308064695801197, + "pass@16": 0.8536985149124111, + "pass@32": 0.8754225625963826, + "pass@64": 0.8965705154324992, + "pass@128": 0.9153605015673981 + }, + "14": { + "pass@1": 0.6602414330218068, + "pass@2": 0.742746507591925, + "pass@4": 0.7996196558958166, + "pass@8": 0.8367759768205015, + "pass@16": 0.8619997000724972, + "pass@32": 0.8821281369592076, + "pass@64": 0.8977516145766892, + "pass@128": 0.9096573208722741 + }, + "15": { + "pass@1": 0.5442557803468208, + "pass@2": 0.6445744532565655, + "pass@4": 0.7226312514765129, + "pass@8": 0.7833987025767928, + "pass@16": 0.8303731247677375, + "pass@32": 0.8669906303856226, + "pass@64": 0.896652764809943, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.32200100806451615, + "pass@2": 0.41802014541529064, + "pass@4": 0.5057130259927185, + "pass@8": 0.579146486298523, + "pass@16": 0.6372956308256016, + "pass@32": 0.6853502169400502, + "pass@64": 0.7263549298081786, + "pass@128": 0.7612903225806451 + }, + "17": { + "pass@1": 0.266573338150289, + "pass@2": 0.3552971815119928, + "pass@4": 0.44027677365835044, + "pass@8": 0.5145766287777432, + "pass@16": 0.5764773175385587, + "pass@32": 0.6297498942971839, + "pass@64": 0.6792948234940933, + "pass@128": 0.7283236994219653 + }, + "18": { + "pass@1": 0.22410714285714287, + "pass@2": 0.3040772637795276, + "pass@4": 0.3812210715624831, + "pass@8": 0.45209828595059937, + "pass@16": 0.5148515801438031, + "pass@32": 0.5696556136715276, + "pass@64": 0.6196317222927293, + "pass@128": 0.6714285714285714 + }, + "19": { + "pass@1": 0.17450262762762764, + "pass@2": 0.24425742179187057, + "pass@4": 0.3161038088144387, + "pass@8": 0.38331538160776063, + "pass@16": 0.44096227160841417, + "pass@32": 0.48847477835581016, + "pass@64": 0.5293757971334146, + "pass@128": 0.5675675675675675 + }, + "20": { + "pass@1": 0.16915352852852852, + "pass@2": 0.2322273060462035, + "pass@4": 0.29948549364887955, + "pass@8": 0.36665142621617874, + "pass@16": 0.42865530956555087, + "pass@32": 0.4819577047980443, + "pass@64": 0.526854050384028, + "pass@128": 0.5645645645645646 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 212515, + "answer_accuracy": 0.4936882062146893, + "avg_response_len": 333.1074793710972, + "resp_tokens_sum": 143390778, + "pass_at_k": { + "pass@1": 0.4936882062146893, + "pass@2": 0.562544998138617, + "pass@4": 0.62185408450282, + "pass@8": 0.6710257625823851, + "pass@16": 0.7103437147363697, + "pass@32": 0.7425658536381914, + "pass@64": 0.7704331070929666, + "pass@128": 0.7948260481712757 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9503247364457831, + "pass@2": 0.9617861208613987, + "pass@4": 0.9686446664046513, + "pass@8": 0.9728691216772047, + "pass@16": 0.9758402586096212, + "pass@32": 0.9778266280336569, + "pass@64": 0.9788090621047877, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8506073667711599, + "pass@2": 0.8813039178782122, + "pass@4": 0.9007288483532033, + "pass@8": 0.9156088842870286, + "pass@16": 0.92726192452425, + "pass@32": 0.9371659829216429, + "pass@64": 0.9472018257825825, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7220325630252101, + "pass@2": 0.7833849583140341, + "pass@4": 0.8213891459470926, + "pass@8": 0.8472609166913442, + "pass@16": 0.8663873407991339, + "pass@32": 0.8816572307661266, + "pass@64": 0.8938391501339014, + "pass@128": 0.9019607843137255 + }, + "14": { + "pass@1": 0.6827713815789473, + "pass@2": 0.7667430757010633, + "pass@4": 0.8250682096755444, + "pass@8": 0.8622464010462382, + "pass@16": 0.8840158404678026, + "pass@32": 0.8985840968140026, + "pass@64": 0.911342591360829, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5276751893939394, + "pass@2": 0.6309297452875214, + "pass@4": 0.7133767940939197, + "pass@8": 0.7782515948523318, + "pass@16": 0.8285203684912462, + "pass@32": 0.8654085228745337, + "pass@64": 0.8911681455877384, + "pass@128": 0.906060606060606 + }, + "16": { + "pass@1": 0.3476451210826211, + "pass@2": 0.4371421220584603, + "pass@4": 0.5146949160735249, + "pass@8": 0.5768380160414723, + "pass@16": 0.6278066916101048, + "pass@32": 0.6740581101475652, + "pass@64": 0.7166717203668512, + "pass@128": 0.7549857549857549 + }, + "17": { + "pass@1": 0.28217197204968947, + "pass@2": 0.37584135203208296, + "pass@4": 0.4652980295164967, + "pass@8": 0.5403919264533021, + "pass@16": 0.6006332682410064, + "pass@32": 0.6515994259098172, + "pass@64": 0.6960613563926029, + "pass@128": 0.7360248447204969 + }, + "18": { + "pass@1": 0.2483685661764706, + "pass@2": 0.33491054886521515, + "pass@4": 0.41869472667019536, + "pass@8": 0.4939067588373051, + "pass@16": 0.5575300786689851, + "pass@32": 0.6101799014188762, + "pass@64": 0.6536175854405368, + "pass@128": 0.6882352941176471 + }, + "19": { + "pass@1": 0.18145461309523808, + "pass@2": 0.2484313484251969, + "pass@4": 0.32077272958067743, + "pass@8": 0.3879525679017902, + "pass@16": 0.4426682347327378, + "pass@32": 0.4877902642399231, + "pass@64": 0.5304925094750792, + "pass@128": 0.5744047619047619 + }, + "20": { + "pass@1": 0.14885853293413173, + "pass@2": 0.20804846998915552, + "pass@4": 0.2722438149572623, + "pass@8": 0.3372685915989853, + "pass@16": 0.39522203120650934, + "pass@32": 0.44381516350834477, + "pass@64": 0.48748600114513696, + "pass@128": 0.5269461077844312 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 201295, + "answer_accuracy": 0.48062872478606355, + "avg_response_len": 356.4781383710269, + "resp_tokens_sum": 149298748, + "pass_at_k": { + "pass@1": 0.48062872478606355, + "pass@2": 0.5496864501713434, + "pass@4": 0.6102347220350524, + "pass@8": 0.6616117648915107, + "pass@16": 0.7030291552576929, + "pass@32": 0.7361486030169303, + "pass@64": 0.7634653647851404, + "pass@128": 0.7860635696821516 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9424107142857143, + "pass@2": 0.9582763092113488, + "pass@4": 0.9704186628457155, + "pass@8": 0.979249727923491, + "pass@16": 0.9846760641101544, + "pass@32": 0.9875064744371793, + "pass@64": 0.9888742737578513, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.859375, + "pass@2": 0.8942583796864841, + "pass@4": 0.916091897343107, + "pass@8": 0.9319538736797466, + "pass@16": 0.9446446168477055, + "pass@32": 0.9559976044880033, + "pass@64": 0.9657581898039411, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.6749373070987654, + "pass@2": 0.7393748632983379, + "pass@4": 0.7844400370092625, + "pass@8": 0.8188400380665399, + "pass@16": 0.8458120470339987, + "pass@32": 0.8672366071285914, + "pass@64": 0.885291263140771, + "pass@128": 0.9012345679012346 + }, + "14": { + "pass@1": 0.6472783753709199, + "pass@2": 0.7265900634360616, + "pass@4": 0.7834193592084369, + "pass@8": 0.8229248873711785, + "pass@16": 0.850694602949872, + "pass@32": 0.8729859244399738, + "pass@64": 0.8929176328904539, + "pass@128": 0.9109792284866469 + }, + "15": { + "pass@1": 0.5328655478395061, + "pass@2": 0.6268868626664722, + "pass@4": 0.7019003683567334, + "pass@8": 0.7591171970498471, + "pass@16": 0.7976979518896169, + "pass@32": 0.8230547091767454, + "pass@64": 0.8429850160241235, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.34462573746312686, + "pass@2": 0.4405050170719808, + "pass@4": 0.5239615202966885, + "pass@8": 0.591901375563439, + "pass@16": 0.6466237250958037, + "pass@32": 0.6925296333526205, + "pass@64": 0.7317750204679877, + "pass@128": 0.7640117994100295 + }, + "17": { + "pass@1": 0.26136577560240964, + "pass@2": 0.3482866011289251, + "pass@4": 0.42627636925655377, + "pass@8": 0.49088887967339034, + "pass@16": 0.5467325363614678, + "pass@32": 0.5977798190482726, + "pass@64": 0.6434842851816812, + "pass@128": 0.6837349397590361 + }, + "18": { + "pass@1": 0.23865927419354838, + "pass@2": 0.31305284798069594, + "pass@4": 0.3894739970003749, + "pass@8": 0.4619313302764042, + "pass@16": 0.5196857133240281, + "pass@32": 0.5624058343339355, + "pass@64": 0.5987371793517974, + "pass@128": 0.6290322580645161 + }, + "19": { + "pass@1": 0.17529739425981872, + "pass@2": 0.2498134084259105, + "pass@4": 0.3267649176972214, + "pass@8": 0.39822689720832727, + "pass@16": 0.46005277817344775, + "pass@32": 0.5109431951797943, + "pass@64": 0.5520361190642347, + "pass@128": 0.5861027190332326 + }, + "20": { + "pass@1": 0.14916478978978978, + "pass@2": 0.216420160317798, + "pass@4": 0.29373762485770377, + "pass@8": 0.3734408191736654, + "pass@16": 0.44431723624192304, + "pass@32": 0.499859994169132, + "pass@64": 0.5400738646113595, + "pass@128": 0.5675675675675675 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7266060855263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7266060855263158, + "pass@2": 0.7634691644218625, + "pass@4": 0.7952178067412438, + "pass@8": 0.8218561656618946, + "pass@16": 0.8434967629289458, + "pass@32": 0.8613606744991223, + "pass@64": 0.8767360618499952, + "pass@128": 0.8901578947368421 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9520703125, + "pass@2": 0.9622504921259845, + "pass@4": 0.9693263488001502, + "pass@8": 0.9740394352292095, + "pass@16": 0.977639988144664, + "pass@32": 0.9812472061219915, + "pass@64": 0.9855393884549314, + "pass@128": 0.99 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.999984375, + "pass@2": 0.9999998769685039, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999984375, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9973984375, + "pass@2": 0.9982915846456695, + "pass@4": 0.9986447747469067, + "pass@8": 0.9989552510738148, + "pass@16": 0.9992111792809737, + "pass@32": 0.9994386769381023, + "pass@64": 0.9997519685023395, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996640625, + "pass@2": 0.9999740403543307, + "pass@4": 0.9999997768091488, + "pass@8": 0.9999999999933986, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.999171875, + "pass@2": 0.9998625738188976, + "pass@4": 0.9999931821334833, + "pass@8": 0.999999978289487, + "pass@16": 0.9999999999999217, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9961015625, + "pass@2": 0.9980688976377954, + "pass@4": 0.9991727544056992, + "pass@8": 0.9997782511963624, + "pass@16": 0.9999797882169323, + "pass@32": 0.9999998558241998, + "pass@64": 0.9999999999997746, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.99571875, + "pass@2": 0.997261687992126, + "pass@4": 0.9981139471316086, + "pass@8": 0.9986330400577111, + "pass@16": 0.9988976173467354, + "pass@32": 0.9989929051532713, + "pass@64": 0.9989999947663261, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.936265625, + "pass@2": 0.9495132874015753, + "pass@4": 0.958506849550056, + "pass@8": 0.965062425401987, + "pass@16": 0.9703263571802695, + "pass@32": 0.9747704580686662, + "pass@64": 0.9780232404547895, + "pass@128": 0.98 + }, + "12": { + "pass@1": 0.849375, + "pass@2": 0.8816822096456706, + "pass@4": 0.9012263803899512, + "pass@8": 0.915389220673123, + "pass@16": 0.9268891903487071, + "pass@32": 0.9374594637955742, + "pass@64": 0.9478378870624736, + "pass@128": 0.958 + }, + "13": { + "pass@1": 0.7028828125, + "pass@2": 0.7634910187007871, + "pass@4": 0.80349823856393, + "pass@8": 0.8328035833884266, + "pass@16": 0.8556732101613651, + "pass@32": 0.8749960895614166, + "pass@64": 0.8919409402783807, + "pass@128": 0.906 + }, + "14": { + "pass@1": 0.663578125, + "pass@2": 0.7455086122047233, + "pass@4": 0.8028635613048376, + "pass@8": 0.8408190447612812, + "pass@16": 0.8657194023573667, + "pass@32": 0.8846751496105657, + "pass@64": 0.9007706768086038, + "pass@128": 0.915 + }, + "15": { + "pass@1": 0.53509375, + "pass@2": 0.634340920275591, + "pass@4": 0.7128604744094488, + "pass@8": 0.7738329492369901, + "pass@16": 0.8191749591839835, + "pass@32": 0.8522332964352862, + "pass@64": 0.8774544898600107, + "pass@128": 0.896 + }, + "16": { + "pass@1": 0.338671875, + "pass@2": 0.43235433070866197, + "pass@4": 0.5150519089801271, + "pass@8": 0.5826601206991043, + "pass@16": 0.6371272371185606, + "pass@32": 0.6838205096197499, + "pass@64": 0.7247935340279482, + "pass@128": 0.76 + }, + "17": { + "pass@1": 0.2698671875, + "pass@2": 0.35958489173228403, + "pass@4": 0.443685483783277, + "pass@8": 0.5150248219266278, + "pass@16": 0.5743802663139518, + "pass@32": 0.6261713784938127, + "pass@64": 0.6728045483676934, + "pass@128": 0.716 + }, + "18": { + "pass@1": 0.2368671875, + "pass@2": 0.31734301181102353, + "pass@4": 0.3965205211848512, + "pass@8": 0.4693614104730791, + "pass@16": 0.5308608509282342, + "pass@32": 0.5811864399109724, + "pass@64": 0.6247096074512953, + "pass@128": 0.664 + }, + "19": { + "pass@1": 0.1771015625, + "pass@2": 0.24749889271653538, + "pass@4": 0.3212013932320957, + "pass@8": 0.3898091878663418, + "pass@16": 0.44785443289121296, + "pass@32": 0.4956818275816109, + "pass@64": 0.5372515790393153, + "pass@128": 0.576 + }, + "20": { + "pass@1": 0.15571875, + "pass@2": 0.21888779527559046, + "pass@4": 0.2884727326584174, + "pass@8": 0.3590984273088792, + "pass@16": 0.4227040161768628, + "pass@32": 0.4751795583678563, + "pass@64": 0.5181073200759404, + "pass@128": 0.553 + } + }, + "per_op_accuracy": { + "10": 0.9520703125, + "2": 1.0, + "3": 0.999984375, + "4": 0.999984375, + "5": 0.9973984375, + "6": 0.9996640625, + "7": 0.999171875, + "8": 0.9961015625, + "9": 0.99571875, + "11": 0.936265625, + "12": 0.849375, + "13": 0.7028828125, + "14": 0.663578125, + "15": 0.53509375, + "16": 0.338671875, + "17": 0.2698671875, + "18": 0.2368671875, + "19": 0.1771015625, + "20": 0.15571875 + }, + "per_op_avg_response_len": { + "10": 277.5632734375, + "2": 114.91821875, + "3": 143.4999609375, + "4": 158.580390625, + "5": 171.0569453125, + "6": 196.9027421875, + "7": 221.39425, + "8": 240.706765625, + "9": 256.240015625, + "11": 294.3470546875, + "12": 303.4772734375, + "13": 318.9317890625, + "14": 340.3666171875, + "15": 348.7077109375, + "16": 348.68609375, + "17": 355.6078203125, + "18": 351.8683515625, + "19": 350.7575546875, + "20": 348.957671875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 277.5632734375, + "2": 114.91821875, + "3": 143.4999609375, + "4": 158.580390625, + "5": 171.0569453125, + "6": 196.9027421875, + "7": 221.39425, + "8": 240.706765625, + "9": 256.240015625, + "11": 294.3470546875, + "12": 303.4772734375, + "13": 318.9317890625, + "14": 340.3666171875, + "15": 348.7077109375, + "16": 348.68609375, + "17": 355.6078203125, + "18": 351.8683515625, + "19": 350.7575546875, + "20": 348.957671875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 585874, + "answer_accuracy": 0.7235442024976289, + "avg_response_len": 258.13694228185267, + "resp_tokens_sum": 209020710, + "pass_at_k": { + "pass@1": 0.7235442024976289, + "pass@2": 0.7601259044351346, + "pass@4": 0.7915020775787805, + "pass@8": 0.8180794280332532, + "pass@16": 0.8402727252631793, + "pass@32": 0.8592362336703517, + "pass@64": 0.8759766756894397, + "pass@128": 0.8910844135314575 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9606148598820059, + "pass@2": 0.970110604952036, + "pass@4": 0.9766451804143947, + "pass@8": 0.9812420969145403, + "pass@16": 0.984712123290507, + "pass@32": 0.9877358738401864, + "pass@64": 0.9911736613911921, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963293650793651, + "pass@2": 0.9968914041994751, + "pass@4": 0.9970213660792401, + "pass@8": 0.997211286089119, + "pass@16": 0.9975721784776902, + "pass@32": 0.9982189726284215, + "pass@64": 0.9992125984251968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9996622093835906, + "pass@4": 0.999980712377669, + "pass@8": 0.9999999358582831, + "pass@16": 0.9999999999997683, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9995993589743589, + "pass@2": 0.9999917190339189, + "pass@4": 0.9999999969955679, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967209507042254, + "pass@2": 0.9971629976710658, + "pass@4": 0.9971830114545541, + "pass@8": 0.9971830985912242, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9175593130311614, + "pass@2": 0.9301508721643506, + "pass@4": 0.9383426105022994, + "pass@8": 0.94506009266078, + "pass@16": 0.9523354989438444, + "pass@32": 0.9605310994724667, + "pass@64": 0.9676012340007842, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.8390271892655368, + "pass@2": 0.8704061290537837, + "pass@4": 0.8879430150680314, + "pass@8": 0.8998900279443047, + "pass@16": 0.9101521093680011, + "pass@32": 0.9205997134915045, + "pass@64": 0.9318575609376857, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.7098354231974922, + "pass@2": 0.7657214196677613, + "pass@4": 0.8028330453599253, + "pass@8": 0.8308064695801197, + "pass@16": 0.8536985149124111, + "pass@32": 0.8754225625963826, + "pass@64": 0.8965705154324992, + "pass@128": 0.9153605015673981 + }, + "14": { + "pass@1": 0.6602414330218068, + "pass@2": 0.742746507591925, + "pass@4": 0.7996196558958166, + "pass@8": 0.8367759768205015, + "pass@16": 0.8619997000724972, + "pass@32": 0.8821281369592076, + "pass@64": 0.8977516145766892, + "pass@128": 0.9096573208722741 + }, + "15": { + "pass@1": 0.5442557803468208, + "pass@2": 0.6445744532565655, + "pass@4": 0.7226312514765129, + "pass@8": 0.7833987025767928, + "pass@16": 0.8303731247677375, + "pass@32": 0.8669906303856226, + "pass@64": 0.896652764809943, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.32200100806451615, + "pass@2": 0.41802014541529064, + "pass@4": 0.5057130259927185, + "pass@8": 0.579146486298523, + "pass@16": 0.6372956308256016, + "pass@32": 0.6853502169400502, + "pass@64": 0.7263549298081786, + "pass@128": 0.7612903225806451 + }, + "17": { + "pass@1": 0.266573338150289, + "pass@2": 0.3552971815119928, + "pass@4": 0.44027677365835044, + "pass@8": 0.5145766287777432, + "pass@16": 0.5764773175385587, + "pass@32": 0.6297498942971839, + "pass@64": 0.6792948234940933, + "pass@128": 0.7283236994219653 + }, + "18": { + "pass@1": 0.22410714285714287, + "pass@2": 0.3040772637795276, + "pass@4": 0.3812210715624831, + "pass@8": 0.45209828595059937, + "pass@16": 0.5148515801438031, + "pass@32": 0.5696556136715276, + "pass@64": 0.6196317222927293, + "pass@128": 0.6714285714285714 + }, + "19": { + "pass@1": 0.17450262762762764, + "pass@2": 0.24425742179187057, + "pass@4": 0.3161038088144387, + "pass@8": 0.38331538160776063, + "pass@16": 0.44096227160841417, + "pass@32": 0.48847477835581016, + "pass@64": 0.5293757971334146, + "pass@128": 0.5675675675675675 + }, + "20": { + "pass@1": 0.16915352852852852, + "pass@2": 0.2322273060462035, + "pass@4": 0.29948549364887955, + "pass@8": 0.36665142621617874, + "pass@16": 0.42865530956555087, + "pass@32": 0.4819577047980443, + "pass@64": 0.526854050384028, + "pass@128": 0.5645645645645646 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 600278, + "answer_accuracy": 0.7310478371005456, + "avg_response_len": 267.8073351032736, + "resp_tokens_sum": 219901959, + "pass_at_k": { + "pass@1": 0.7310478371005456, + "pass@2": 0.7680041011777181, + "pass@4": 0.7996512798526805, + "pass@8": 0.8257688373431301, + "pass@16": 0.846615880063671, + "pass@32": 0.8637151935821454, + "pass@64": 0.8785491835920611, + "pass@128": 0.8916601714731099 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9403278374233128, + "pass@2": 0.952113575431139, + "pass@4": 0.9607398866169331, + "pass@8": 0.9662308127069366, + "pass@16": 0.9701134353995957, + "pass@32": 0.9738990670749782, + "pass@64": 0.9783020822793281, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9997735507246377, + "pass@2": 0.9999942941914869, + "pass@4": 0.9999999959244225, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994851361031518, + "pass@2": 0.9999626322676716, + "pass@4": 0.999999727111475, + "pass@8": 0.9999999999939815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995032514450867, + "pass@2": 0.9999455958991397, + "pass@4": 0.9999991709851297, + "pass@8": 0.9999999999115423, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9990167877342796, + "pass@4": 0.9998786036428545, + "pass@8": 0.9999981102949981, + "pass@16": 0.9999999997381142, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952984655688623, + "pass@2": 0.9967109334480646, + "pass@4": 0.9979339827094966, + "pass@8": 0.9989726884046817, + "pass@16": 0.999694545217648, + "pass@32": 0.9999787580173054, + "pass@64": 0.9999999843303174, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9503247364457831, + "pass@2": 0.9617861208613987, + "pass@4": 0.9686446664046513, + "pass@8": 0.9728691216772047, + "pass@16": 0.9758402586096212, + "pass@32": 0.9778266280336569, + "pass@64": 0.9788090621047877, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8506073667711599, + "pass@2": 0.8813039178782122, + "pass@4": 0.9007288483532033, + "pass@8": 0.9156088842870286, + "pass@16": 0.92726192452425, + "pass@32": 0.9371659829216429, + "pass@64": 0.9472018257825825, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7220325630252101, + "pass@2": 0.7833849583140341, + "pass@4": 0.8213891459470926, + "pass@8": 0.8472609166913442, + "pass@16": 0.8663873407991339, + "pass@32": 0.8816572307661266, + "pass@64": 0.8938391501339014, + "pass@128": 0.9019607843137255 + }, + "14": { + "pass@1": 0.6827713815789473, + "pass@2": 0.7667430757010633, + "pass@4": 0.8250682096755444, + "pass@8": 0.8622464010462382, + "pass@16": 0.8840158404678026, + "pass@32": 0.8985840968140026, + "pass@64": 0.911342591360829, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5276751893939394, + "pass@2": 0.6309297452875214, + "pass@4": 0.7133767940939197, + "pass@8": 0.7782515948523318, + "pass@16": 0.8285203684912462, + "pass@32": 0.8654085228745337, + "pass@64": 0.8911681455877384, + "pass@128": 0.906060606060606 + }, + "16": { + "pass@1": 0.3476451210826211, + "pass@2": 0.4371421220584603, + "pass@4": 0.5146949160735249, + "pass@8": 0.5768380160414723, + "pass@16": 0.6278066916101048, + "pass@32": 0.6740581101475652, + "pass@64": 0.7166717203668512, + "pass@128": 0.7549857549857549 + }, + "17": { + "pass@1": 0.28217197204968947, + "pass@2": 0.37584135203208296, + "pass@4": 0.4652980295164967, + "pass@8": 0.5403919264533021, + "pass@16": 0.6006332682410064, + "pass@32": 0.6515994259098172, + "pass@64": 0.6960613563926029, + "pass@128": 0.7360248447204969 + }, + "18": { + "pass@1": 0.2483685661764706, + "pass@2": 0.33491054886521515, + "pass@4": 0.41869472667019536, + "pass@8": 0.4939067588373051, + "pass@16": 0.5575300786689851, + "pass@32": 0.6101799014188762, + "pass@64": 0.6536175854405368, + "pass@128": 0.6882352941176471 + }, + "19": { + "pass@1": 0.18145461309523808, + "pass@2": 0.2484313484251969, + "pass@4": 0.32077272958067743, + "pass@8": 0.3879525679017902, + "pass@16": 0.4426682347327378, + "pass@32": 0.4877902642399231, + "pass@64": 0.5304925094750792, + "pass@128": 0.5744047619047619 + }, + "20": { + "pass@1": 0.14885853293413173, + "pass@2": 0.20804846998915552, + "pass@4": 0.2722438149572623, + "pass@8": 0.3372685915989853, + "pass@16": 0.39522203120650934, + "pass@32": 0.44381516350834477, + "pass@64": 0.48748600114513696, + "pass@128": 0.5269461077844312 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 580954, + "answer_accuracy": 0.7251482864674869, + "avg_response_len": 286.24574987018696, + "resp_tokens_sum": 229326355, + "pass_at_k": { + "pass@1": 0.7251482864674869, + "pass@2": 0.7622002466055128, + "pass@4": 0.7944293377641134, + "pass@8": 0.8216631401631663, + "pass@16": 0.8435584542301595, + "pass@32": 0.8610946532121718, + "pass@64": 0.8756452647380498, + "pass@128": 0.8876817382968525 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9548507462686567, + "pass@2": 0.9641611088259492, + "pass@4": 0.9702759092613421, + "pass@8": 0.9743496102469225, + "pass@16": 0.9778077564445188, + "pass@32": 0.9818317881305241, + "pass@64": 0.9868807117022572, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9959788602941176, + "pass@2": 0.9978610612552107, + "pass@4": 0.9987736642294712, + "pass@8": 0.9995108704580657, + "pass@16": 0.9999292442955922, + "pass@32": 0.9999991192945575, + "pass@64": 0.9999999999953015, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.999959248242629, + "pass@4": 0.999999596363546, + "pass@8": 0.9999999999858014, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997033227848101, + "pass@2": 0.9999859837536131, + "pass@4": 0.999999962623343, + "pass@8": 0.99999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920701951951952, + "pass@2": 0.9952568217824125, + "pass@4": 0.9976451983479543, + "pass@8": 0.999336102227142, + "pass@16": 0.9999393042339394, + "pass@32": 0.9999995670396389, + "pass@64": 0.9999999999993233, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950261254019293, + "pass@2": 0.9979658265184698, + "pass@4": 0.999369864438167, + "pass@8": 0.9999233509024527, + "pass@16": 0.9999988400129937, + "pass@32": 0.9999999999211941, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9424107142857143, + "pass@2": 0.9582763092113488, + "pass@4": 0.9704186628457155, + "pass@8": 0.979249727923491, + "pass@16": 0.9846760641101544, + "pass@32": 0.9875064744371793, + "pass@64": 0.9888742737578513, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.859375, + "pass@2": 0.8942583796864841, + "pass@4": 0.916091897343107, + "pass@8": 0.9319538736797466, + "pass@16": 0.9446446168477055, + "pass@32": 0.9559976044880033, + "pass@64": 0.9657581898039411, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.6749373070987654, + "pass@2": 0.7393748632983379, + "pass@4": 0.7844400370092625, + "pass@8": 0.8188400380665399, + "pass@16": 0.8458120470339987, + "pass@32": 0.8672366071285914, + "pass@64": 0.885291263140771, + "pass@128": 0.9012345679012346 + }, + "14": { + "pass@1": 0.6472783753709199, + "pass@2": 0.7265900634360616, + "pass@4": 0.7834193592084369, + "pass@8": 0.8229248873711785, + "pass@16": 0.850694602949872, + "pass@32": 0.8729859244399738, + "pass@64": 0.8929176328904539, + "pass@128": 0.9109792284866469 + }, + "15": { + "pass@1": 0.5328655478395061, + "pass@2": 0.6268868626664722, + "pass@4": 0.7019003683567334, + "pass@8": 0.7591171970498471, + "pass@16": 0.7976979518896169, + "pass@32": 0.8230547091767454, + "pass@64": 0.8429850160241235, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.34462573746312686, + "pass@2": 0.4405050170719808, + "pass@4": 0.5239615202966885, + "pass@8": 0.591901375563439, + "pass@16": 0.6466237250958037, + "pass@32": 0.6925296333526205, + "pass@64": 0.7317750204679877, + "pass@128": 0.7640117994100295 + }, + "17": { + "pass@1": 0.26136577560240964, + "pass@2": 0.3482866011289251, + "pass@4": 0.42627636925655377, + "pass@8": 0.49088887967339034, + "pass@16": 0.5467325363614678, + "pass@32": 0.5977798190482726, + "pass@64": 0.6434842851816812, + "pass@128": 0.6837349397590361 + }, + "18": { + "pass@1": 0.23865927419354838, + "pass@2": 0.31305284798069594, + "pass@4": 0.3894739970003749, + "pass@8": 0.4619313302764042, + "pass@16": 0.5196857133240281, + "pass@32": 0.5624058343339355, + "pass@64": 0.5987371793517974, + "pass@128": 0.6290322580645161 + }, + "19": { + "pass@1": 0.17529739425981872, + "pass@2": 0.2498134084259105, + "pass@4": 0.3267649176972214, + "pass@8": 0.39822689720832727, + "pass@16": 0.46005277817344775, + "pass@32": 0.5109431951797943, + "pass@64": 0.5520361190642347, + "pass@128": 0.5861027190332326 + }, + "20": { + "pass@1": 0.14916478978978978, + "pass@2": 0.216420160317798, + "pass@4": 0.29373762485770377, + "pass@8": 0.3734408191736654, + "pass@16": 0.44431723624192304, + "pass@32": 0.499859994169132, + "pass@64": 0.5400738646113595, + "pass@128": 0.5675675675675675 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/summary.csv b/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..516f21cbab5129ee08677d1dacfc2f2ed15b0ed6 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85656,0.99334375,0.4865421875,0.7266060855263158,0.0,0.0,0.0,197.87361805555557,336.17079375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/summary.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fbafd6bc7af2eb6812f83430b5fcc5b4a7f6c2aa --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block2/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85656", + "id_acc": 0.99334375, + "ood_acc": 0.4865421875, + "total_acc": 0.7266060855263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.87361805555557, + "ood_avg_resp_len": 336.17079375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 585874, + "answer_accuracy": 0.7235442024976289, + "avg_response_len": 258.13694228185267, + "resp_tokens_sum": 209020710, + "pass_at_k": { + "pass@1": 0.7235442024976289, + "pass@2": 0.7601259044351346, + "pass@4": 0.7915020775787805, + "pass@8": 0.8180794280332532, + "pass@16": 0.8402727252631793, + "pass@32": 0.8592362336703517, + "pass@64": 0.8759766756894397, + "pass@128": 0.8910844135314575 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9606148598820059, + "pass@2": 0.970110604952036, + "pass@4": 0.9766451804143947, + "pass@8": 0.9812420969145403, + "pass@16": 0.984712123290507, + "pass@32": 0.9877358738401864, + "pass@64": 0.9911736613911921, + "pass@128": 0.9941002949852508 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.999951171875, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9963293650793651, + "pass@2": 0.9968914041994751, + "pass@4": 0.9970213660792401, + "pass@8": 0.997211286089119, + "pass@16": 0.9975721784776902, + "pass@32": 0.9982189726284215, + "pass@64": 0.9992125984251968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999532185628742, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9983357988165681, + "pass@2": 0.9996622093835906, + "pass@4": 0.999980712377669, + "pass@8": 0.9999999358582831, + "pass@16": 0.9999999999997683, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9995993589743589, + "pass@2": 0.9999917190339189, + "pass@4": 0.9999999969955679, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9967209507042254, + "pass@2": 0.9971629976710658, + "pass@4": 0.9971830114545541, + "pass@8": 0.9971830985912242, + "pass@16": 0.9971830985915493, + "pass@32": 0.9971830985915493, + "pass@64": 0.9971830985915493, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9175593130311614, + "pass@2": 0.9301508721643506, + "pass@4": 0.9383426105022994, + "pass@8": 0.94506009266078, + "pass@16": 0.9523354989438444, + "pass@32": 0.9605310994724667, + "pass@64": 0.9676012340007842, + "pass@128": 0.9716713881019831 + }, + "12": { + "pass@1": 0.8390271892655368, + "pass@2": 0.8704061290537837, + "pass@4": 0.8879430150680314, + "pass@8": 0.8998900279443047, + "pass@16": 0.9101521093680011, + "pass@32": 0.9205997134915045, + "pass@64": 0.9318575609376857, + "pass@128": 0.943502824858757 + }, + "13": { + "pass@1": 0.7098354231974922, + "pass@2": 0.7657214196677613, + "pass@4": 0.8028330453599253, + "pass@8": 0.8308064695801197, + "pass@16": 0.8536985149124111, + "pass@32": 0.8754225625963826, + "pass@64": 0.8965705154324992, + "pass@128": 0.9153605015673981 + }, + "14": { + "pass@1": 0.6602414330218068, + "pass@2": 0.742746507591925, + "pass@4": 0.7996196558958166, + "pass@8": 0.8367759768205015, + "pass@16": 0.8619997000724972, + "pass@32": 0.8821281369592076, + "pass@64": 0.8977516145766892, + "pass@128": 0.9096573208722741 + }, + "15": { + "pass@1": 0.5442557803468208, + "pass@2": 0.6445744532565655, + "pass@4": 0.7226312514765129, + "pass@8": 0.7833987025767928, + "pass@16": 0.8303731247677375, + "pass@32": 0.8669906303856226, + "pass@64": 0.896652764809943, + "pass@128": 0.9190751445086706 + }, + "16": { + "pass@1": 0.32200100806451615, + "pass@2": 0.41802014541529064, + "pass@4": 0.5057130259927185, + "pass@8": 0.579146486298523, + "pass@16": 0.6372956308256016, + "pass@32": 0.6853502169400502, + "pass@64": 0.7263549298081786, + "pass@128": 0.7612903225806451 + }, + "17": { + "pass@1": 0.266573338150289, + "pass@2": 0.3552971815119928, + "pass@4": 0.44027677365835044, + "pass@8": 0.5145766287777432, + "pass@16": 0.5764773175385587, + "pass@32": 0.6297498942971839, + "pass@64": 0.6792948234940933, + "pass@128": 0.7283236994219653 + }, + "18": { + "pass@1": 0.22410714285714287, + "pass@2": 0.3040772637795276, + "pass@4": 0.3812210715624831, + "pass@8": 0.45209828595059937, + "pass@16": 0.5148515801438031, + "pass@32": 0.5696556136715276, + "pass@64": 0.6196317222927293, + "pass@128": 0.6714285714285714 + }, + "19": { + "pass@1": 0.17450262762762764, + "pass@2": 0.24425742179187057, + "pass@4": 0.3161038088144387, + "pass@8": 0.38331538160776063, + "pass@16": 0.44096227160841417, + "pass@32": 0.48847477835581016, + "pass@64": 0.5293757971334146, + "pass@128": 0.5675675675675675 + }, + "20": { + "pass@1": 0.16915352852852852, + "pass@2": 0.2322273060462035, + "pass@4": 0.29948549364887955, + "pass@8": 0.36665142621617874, + "pass@16": 0.42865530956555087, + "pass@32": 0.4819577047980443, + "pass@64": 0.526854050384028, + "pass@128": 0.5645645645645646 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 600278, + "answer_accuracy": 0.7310478371005456, + "avg_response_len": 267.8073351032736, + "resp_tokens_sum": 219901959, + "pass_at_k": { + "pass@1": 0.7310478371005456, + "pass@2": 0.7680041011777181, + "pass@4": 0.7996512798526805, + "pass@8": 0.8257688373431301, + "pass@16": 0.846615880063671, + "pass@32": 0.8637151935821454, + "pass@64": 0.8785491835920611, + "pass@128": 0.8916601714731099 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9403278374233128, + "pass@2": 0.952113575431139, + "pass@4": 0.9607398866169331, + "pass@8": 0.9662308127069366, + "pass@16": 0.9701134353995957, + "pass@32": 0.9738990670749782, + "pass@64": 0.9783020822793281, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 0.9999526515151516, + "pass@2": 0.9999996271772846, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9997735507246377, + "pass@2": 0.9999942941914869, + "pass@4": 0.9999999959244225, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9994851361031518, + "pass@2": 0.9999626322676716, + "pass@4": 0.999999727111475, + "pass@8": 0.9999999999939815, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995032514450867, + "pass@2": 0.9999455958991397, + "pass@4": 0.9999991709851297, + "pass@8": 0.9999999999115423, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9968089788732394, + "pass@2": 0.9990167877342796, + "pass@4": 0.9998786036428545, + "pass@8": 0.9999981102949981, + "pass@16": 0.9999999997381142, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9952984655688623, + "pass@2": 0.9967109334480646, + "pass@4": 0.9979339827094966, + "pass@8": 0.9989726884046817, + "pass@16": 0.999694545217648, + "pass@32": 0.9999787580173054, + "pass@64": 0.9999999843303174, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9503247364457831, + "pass@2": 0.9617861208613987, + "pass@4": 0.9686446664046513, + "pass@8": 0.9728691216772047, + "pass@16": 0.9758402586096212, + "pass@32": 0.9778266280336569, + "pass@64": 0.9788090621047877, + "pass@128": 0.9789156626506024 + }, + "12": { + "pass@1": 0.8506073667711599, + "pass@2": 0.8813039178782122, + "pass@4": 0.9007288483532033, + "pass@8": 0.9156088842870286, + "pass@16": 0.92726192452425, + "pass@32": 0.9371659829216429, + "pass@64": 0.9472018257825825, + "pass@128": 0.9592476489028213 + }, + "13": { + "pass@1": 0.7220325630252101, + "pass@2": 0.7833849583140341, + "pass@4": 0.8213891459470926, + "pass@8": 0.8472609166913442, + "pass@16": 0.8663873407991339, + "pass@32": 0.8816572307661266, + "pass@64": 0.8938391501339014, + "pass@128": 0.9019607843137255 + }, + "14": { + "pass@1": 0.6827713815789473, + "pass@2": 0.7667430757010633, + "pass@4": 0.8250682096755444, + "pass@8": 0.8622464010462382, + "pass@16": 0.8840158404678026, + "pass@32": 0.8985840968140026, + "pass@64": 0.911342591360829, + "pass@128": 0.9239766081871345 + }, + "15": { + "pass@1": 0.5276751893939394, + "pass@2": 0.6309297452875214, + "pass@4": 0.7133767940939197, + "pass@8": 0.7782515948523318, + "pass@16": 0.8285203684912462, + "pass@32": 0.8654085228745337, + "pass@64": 0.8911681455877384, + "pass@128": 0.906060606060606 + }, + "16": { + "pass@1": 0.3476451210826211, + "pass@2": 0.4371421220584603, + "pass@4": 0.5146949160735249, + "pass@8": 0.5768380160414723, + "pass@16": 0.6278066916101048, + "pass@32": 0.6740581101475652, + "pass@64": 0.7166717203668512, + "pass@128": 0.7549857549857549 + }, + "17": { + "pass@1": 0.28217197204968947, + "pass@2": 0.37584135203208296, + "pass@4": 0.4652980295164967, + "pass@8": 0.5403919264533021, + "pass@16": 0.6006332682410064, + "pass@32": 0.6515994259098172, + "pass@64": 0.6960613563926029, + "pass@128": 0.7360248447204969 + }, + "18": { + "pass@1": 0.2483685661764706, + "pass@2": 0.33491054886521515, + "pass@4": 0.41869472667019536, + "pass@8": 0.4939067588373051, + "pass@16": 0.5575300786689851, + "pass@32": 0.6101799014188762, + "pass@64": 0.6536175854405368, + "pass@128": 0.6882352941176471 + }, + "19": { + "pass@1": 0.18145461309523808, + "pass@2": 0.2484313484251969, + "pass@4": 0.32077272958067743, + "pass@8": 0.3879525679017902, + "pass@16": 0.4426682347327378, + "pass@32": 0.4877902642399231, + "pass@64": 0.5304925094750792, + "pass@128": 0.5744047619047619 + }, + "20": { + "pass@1": 0.14885853293413173, + "pass@2": 0.20804846998915552, + "pass@4": 0.2722438149572623, + "pass@8": 0.3372685915989853, + "pass@16": 0.39522203120650934, + "pass@32": 0.44381516350834477, + "pass@64": 0.48748600114513696, + "pass@128": 0.5269461077844312 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 580954, + "answer_accuracy": 0.7251482864674869, + "avg_response_len": 286.24574987018696, + "resp_tokens_sum": 229326355, + "pass_at_k": { + "pass@1": 0.7251482864674869, + "pass@2": 0.7622002466055128, + "pass@4": 0.7944293377641134, + "pass@8": 0.8216631401631663, + "pass@16": 0.8435584542301595, + "pass@32": 0.8610946532121718, + "pass@64": 0.8756452647380498, + "pass@128": 0.8876817382968525 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9548507462686567, + "pass@2": 0.9641611088259492, + "pass@4": 0.9702759092613421, + "pass@8": 0.9743496102469225, + "pass@16": 0.9778077564445188, + "pass@32": 0.9818317881305241, + "pass@64": 0.9868807117022572, + "pass@128": 0.991044776119403 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9959788602941176, + "pass@2": 0.9978610612552107, + "pass@4": 0.9987736642294712, + "pass@8": 0.9995108704580657, + "pass@16": 0.9999292442955922, + "pass@32": 0.9999991192945575, + "pass@64": 0.9999999999953015, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9995563880126183, + "pass@2": 0.999959248242629, + "pass@4": 0.999999596363546, + "pass@8": 0.9999999999858014, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9997033227848101, + "pass@2": 0.9999859837536131, + "pass@4": 0.999999962623343, + "pass@8": 0.99999999999998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9920701951951952, + "pass@2": 0.9952568217824125, + "pass@4": 0.9976451983479543, + "pass@8": 0.999336102227142, + "pass@16": 0.9999393042339394, + "pass@32": 0.9999995670396389, + "pass@64": 0.9999999999993233, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950261254019293, + "pass@2": 0.9979658265184698, + "pass@4": 0.999369864438167, + "pass@8": 0.9999233509024527, + "pass@16": 0.9999988400129937, + "pass@32": 0.9999999999211941, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9424107142857143, + "pass@2": 0.9582763092113488, + "pass@4": 0.9704186628457155, + "pass@8": 0.979249727923491, + "pass@16": 0.9846760641101544, + "pass@32": 0.9875064744371793, + "pass@64": 0.9888742737578513, + "pass@128": 0.9904761904761905 + }, + "12": { + "pass@1": 0.859375, + "pass@2": 0.8942583796864841, + "pass@4": 0.916091897343107, + "pass@8": 0.9319538736797466, + "pass@16": 0.9446446168477055, + "pass@32": 0.9559976044880033, + "pass@64": 0.9657581898039411, + "pass@128": 0.9724770642201835 + }, + "13": { + "pass@1": 0.6749373070987654, + "pass@2": 0.7393748632983379, + "pass@4": 0.7844400370092625, + "pass@8": 0.8188400380665399, + "pass@16": 0.8458120470339987, + "pass@32": 0.8672366071285914, + "pass@64": 0.885291263140771, + "pass@128": 0.9012345679012346 + }, + "14": { + "pass@1": 0.6472783753709199, + "pass@2": 0.7265900634360616, + "pass@4": 0.7834193592084369, + "pass@8": 0.8229248873711785, + "pass@16": 0.850694602949872, + "pass@32": 0.8729859244399738, + "pass@64": 0.8929176328904539, + "pass@128": 0.9109792284866469 + }, + "15": { + "pass@1": 0.5328655478395061, + "pass@2": 0.6268868626664722, + "pass@4": 0.7019003683567334, + "pass@8": 0.7591171970498471, + "pass@16": 0.7976979518896169, + "pass@32": 0.8230547091767454, + "pass@64": 0.8429850160241235, + "pass@128": 0.8611111111111112 + }, + "16": { + "pass@1": 0.34462573746312686, + "pass@2": 0.4405050170719808, + "pass@4": 0.5239615202966885, + "pass@8": 0.591901375563439, + "pass@16": 0.6466237250958037, + "pass@32": 0.6925296333526205, + "pass@64": 0.7317750204679877, + "pass@128": 0.7640117994100295 + }, + "17": { + "pass@1": 0.26136577560240964, + "pass@2": 0.3482866011289251, + "pass@4": 0.42627636925655377, + "pass@8": 0.49088887967339034, + "pass@16": 0.5467325363614678, + "pass@32": 0.5977798190482726, + "pass@64": 0.6434842851816812, + "pass@128": 0.6837349397590361 + }, + "18": { + "pass@1": 0.23865927419354838, + "pass@2": 0.31305284798069594, + "pass@4": 0.3894739970003749, + "pass@8": 0.4619313302764042, + "pass@16": 0.5196857133240281, + "pass@32": 0.5624058343339355, + "pass@64": 0.5987371793517974, + "pass@128": 0.6290322580645161 + }, + "19": { + "pass@1": 0.17529739425981872, + "pass@2": 0.2498134084259105, + "pass@4": 0.3267649176972214, + "pass@8": 0.39822689720832727, + "pass@16": 0.46005277817344775, + "pass@32": 0.5109431951797943, + "pass@64": 0.5520361190642347, + "pass@128": 0.5861027190332326 + }, + "20": { + "pass@1": 0.14916478978978978, + "pass@2": 0.216420160317798, + "pass@4": 0.29373762485770377, + "pass@8": 0.3734408191736654, + "pass@16": 0.44431723624192304, + "pass@32": 0.499859994169132, + "pass@64": 0.5400738646113595, + "pass@128": 0.5675675675675675 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/checkpoint-85656_metrics.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/checkpoint-85656_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..b452dd43ae60151ea2d5adafe637229366cb3ad8 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/checkpoint-85656_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85656", + "id": { + "answer_accuracy": 0.49327430555555557, + "avg_response_len": 199.13701649305557, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.49327430555555557, + "pass@2": 0.587658327865265, + "pass@4": 0.6823666375140448, + "pass@8": 0.7663682914554482, + "pass@16": 0.8323122048385166, + "pass@32": 0.8811013697007457, + "pass@64": 0.9161224577305216, + "pass@128": 0.9395555555555556 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.190796875, + "pass@2": 0.3003977608267716, + "pass@4": 0.43215709983127026, + "pass@8": 0.5686598935179051, + "pass@16": 0.6915305344763011, + "pass@32": 0.7914435747314094, + "pass@64": 0.8667204616843248, + "pass@128": 0.915 + }, + "2": { + "pass@1": 0.9806953125, + "pass@2": 0.9874446358267717, + "pass@4": 0.9907921147356582, + "pass@8": 0.992629310685749, + "pass@16": 0.9938021523891528, + "pass@32": 0.9946754172048058, + "pass@64": 0.9953712414614146, + "pass@128": 0.996 + }, + "3": { + "pass@1": 0.971625, + "pass@2": 0.9875855068897645, + "pass@4": 0.9931698494563178, + "pass@8": 0.9955605027834186, + "pass@16": 0.9969572346659715, + "pass@32": 0.9980001237695623, + "pass@64": 0.9987513647462997, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.706859375, + "pass@2": 0.8033623277559058, + "pass@4": 0.8713294938132747, + "pass@8": 0.9168010997977786, + "pass@16": 0.9457339750167953, + "pass@32": 0.9627812219276177, + "pass@64": 0.9726259406408313, + "pass@128": 0.978 + }, + "5": { + "pass@1": 0.3789453125, + "pass@2": 0.4657646407480316, + "pass@4": 0.5468323475815521, + "pass@8": 0.6180357705552324, + "pass@16": 0.6779580589440685, + "pass@32": 0.7276554152082413, + "pass@64": 0.768848988855058, + "pass@128": 0.803 + }, + "6": { + "pass@1": 0.42653125, + "pass@2": 0.5740638533464576, + "pass@4": 0.7139497283464574, + "pass@8": 0.8232841091018387, + "pass@16": 0.8924874328530586, + "pass@32": 0.9325368862764091, + "pass@64": 0.9569446412663042, + "pass@128": 0.972 + }, + "7": { + "pass@1": 0.320046875, + "pass@2": 0.4576422244094486, + "pass@4": 0.5986938634233218, + "pass@8": 0.7207198041731779, + "pass@16": 0.8103197939740765, + "pass@32": 0.8708360674901743, + "pass@64": 0.9107117986457854, + "pass@128": 0.937 + }, + "8": { + "pass@1": 0.2600234375, + "pass@2": 0.39415624999999993, + "pass@4": 0.5410903218972629, + "pass@8": 0.6742588911332974, + "pass@16": 0.777802389503416, + "pass@32": 0.8536262533516904, + "pass@64": 0.9061390567254755, + "pass@128": 0.94 + }, + "9": { + "pass@1": 0.2039453125, + "pass@2": 0.3185077509842525, + "pass@4": 0.45328491854143194, + "pass@8": 0.5873652413506206, + "pass@16": 0.7042182717236272, + "pass@32": 0.7983573673467786, + "pass@64": 0.8689886255490779, + "pass@128": 0.916 + } + }, + "per_op_accuracy": { + "10": 0.190796875, + "2": 0.9806953125, + "3": 0.971625, + "4": 0.706859375, + "5": 0.3789453125, + "6": 0.42653125, + "7": 0.320046875, + "8": 0.2600234375, + "9": 0.2039453125 + }, + "per_op_avg_response_len": { + "10": 272.0839765625, + "2": 115.115125, + "3": 145.4905390625, + "4": 162.5349921875, + "5": 170.5468828125, + "6": 203.4081484375, + "7": 225.40946875, + "8": 243.94565625, + "9": 253.698359375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.0839765625, + "2": 115.115125, + "3": 145.4905390625, + "4": 162.5349921875, + "5": 170.5468828125, + "6": 203.4081484375, + "7": 225.40946875, + "8": 243.94565625, + "9": 253.698359375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 183868, + "answer_accuracy": 0.48512960148598444, + "avg_response_len": 189.95709061550153, + "resp_tokens_sum": 71995257, + "pass_at_k": { + "pass@1": 0.48512960148598444, + "pass@2": 0.5774719898709482, + "pass@4": 0.670778007802219, + "pass@8": 0.7539905822474905, + "pass@16": 0.8195878198360237, + "pass@32": 0.8686400870494339, + "pass@64": 0.9047041895476484, + "pass@128": 0.9297534616683553 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.19035766961651918, + "pass@2": 0.30144277111931805, + "pass@4": 0.43816523764175513, + "pass@8": 0.5809281738806453, + "pass@16": 0.7052121639952813, + "pass@32": 0.8019947591187581, + "pass@64": 0.8754555701737227, + "pass@128": 0.9233038348082596 + }, + "2": { + "pass@1": 0.9800915109034268, + "pass@2": 0.9870491144798491, + "pass@4": 0.990911137860104, + "pass@8": 0.9931530593847345, + "pass@16": 0.9945585526843174, + "pass@32": 0.9955696135172313, + "pass@64": 0.9965045227606399, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9717842125382263, + "pass@2": 0.9858160111006766, + "pass@4": 0.991448598168348, + "pass@8": 0.994291079132131, + "pass@16": 0.995887473442639, + "pass@32": 0.9967283718547971, + "pass@64": 0.9969374773681062, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.6986328125, + "pass@2": 0.7886968657726381, + "pass@4": 0.8542420943475817, + "pass@8": 0.9004456167198726, + "pass@16": 0.9312778575005979, + "pass@32": 0.9517112440915444, + "pass@64": 0.9667046571410877, + "pass@128": 0.978125 + }, + "5": { + "pass@1": 0.33509424603174603, + "pass@2": 0.41271325459317576, + "pass@4": 0.48771068527148376, + "pass@8": 0.5572081248784712, + "pass@16": 0.6188679542444174, + "pass@32": 0.6719682822822658, + "pass@64": 0.7166542430331456, + "pass@128": 0.7523809523809524 + }, + "6": { + "pass@1": 0.46206025449101795, + "pass@2": 0.6031718551086802, + "pass@4": 0.7298749273107324, + "pass@8": 0.8273623466954299, + "pass@16": 0.8904114372774126, + "pass@32": 0.9291838638462578, + "pass@64": 0.9545298379466033, + "pass@128": 0.9730538922155688 + }, + "7": { + "pass@1": 0.30755362426035504, + "pass@2": 0.44404614918697294, + "pass@4": 0.5866653091255597, + "pass@8": 0.7100621776567952, + "pass@16": 0.8009828251974904, + "pass@32": 0.8641776511072881, + "pass@64": 0.9077531903073257, + "pass@128": 0.9408284023668639 + }, + "8": { + "pass@1": 0.25122696314102566, + "pass@2": 0.38249427430345234, + "pass@4": 0.5251050217280533, + "pass@8": 0.6536996226393975, + "pass@16": 0.7536627127551332, + "pass@32": 0.826661161406808, + "pass@64": 0.8762717660769928, + "pass@128": 0.9038461538461539 + }, + "9": { + "pass@1": 0.2078125, + "pass@2": 0.3245858517245202, + "pass@4": 0.4576318187339257, + "pass@8": 0.5851793502978136, + "pass@16": 0.6946462264181785, + "pass@32": 0.7833343799733238, + "pass@64": 0.8508485859197652, + "pass@128": 0.8985915492957747 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 192930, + "answer_accuracy": 0.49386160714285715, + "avg_response_len": 196.86668577981652, + "resp_tokens_sum": 76907152, + "pass_at_k": { + "pass@1": 0.49386160714285715, + "pass@2": 0.59013863053529, + "pass@4": 0.686525142867957, + "pass@8": 0.7719897156750463, + "pass@16": 0.838220505712873, + "pass@32": 0.8861594424074006, + "pass@64": 0.919862873560046, + "pass@128": 0.9423328964613368 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1796875, + "pass@2": 0.28581424206560085, + "pass@4": 0.41622926026117907, + "pass@8": 0.5547185574304757, + "pass@16": 0.6839554801528199, + "pass@32": 0.791310676081314, + "pass@64": 0.8702003501063394, + "pass@128": 0.9171779141104295 + }, + "2": { + "pass@1": 0.9739504179331308, + "pass@2": 0.9832606113730465, + "pass@4": 0.9877440161355209, + "pass@8": 0.9903351488073417, + "pass@16": 0.9921767708249385, + "pass@32": 0.9934576683623969, + "pass@64": 0.9939005764597241, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9731060606060606, + "pass@2": 0.9891795663326179, + "pass@4": 0.9937926068900476, + "pass@8": 0.9955231703667864, + "pass@16": 0.9969587575218833, + "pass@32": 0.9984506286504, + "pass@64": 0.9996207784594809, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7112610946745562, + "pass@2": 0.8105136601360479, + "pass@4": 0.8784895620480868, + "pass@8": 0.9216631817833484, + "pass@16": 0.9464260283198788, + "pass@32": 0.9590859028287348, + "pass@64": 0.9663197612118946, + "pass@128": 0.9704142011834319 + }, + "5": { + "pass@1": 0.3811141304347826, + "pass@2": 0.4672704125299555, + "pass@4": 0.547470422990604, + "pass@8": 0.6177559162397293, + "pass@16": 0.6765391597013457, + "pass@32": 0.7272516871961043, + "pass@64": 0.7712716097104394, + "pass@128": 0.8115942028985508 + }, + "6": { + "pass@1": 0.41883058739255014, + "pass@2": 0.5677878866502717, + "pass@4": 0.7127793116118358, + "pass@8": 0.8278492564390045, + "pass@16": 0.8988014599895945, + "pass@32": 0.9373003380525965, + "pass@64": 0.9604309989332116, + "pass@128": 0.9742120343839542 + }, + "7": { + "pass@1": 0.33487626445086704, + "pass@2": 0.46564541042738106, + "pass@4": 0.5980823447466463, + "pass@8": 0.714922667139698, + "pass@16": 0.8033677613399027, + "pass@32": 0.8641866922325745, + "pass@64": 0.9019377879132182, + "pass@128": 0.9219653179190751 + }, + "8": { + "pass@1": 0.28347271126760565, + "pass@2": 0.4233832621714537, + "pass@4": 0.5744736294406855, + "pass@8": 0.7088275207620468, + "pass@16": 0.8087186748834445, + "pass@32": 0.8784898814053248, + "pass@64": 0.9269338685585622, + "pass@128": 0.9605633802816902 + }, + "9": { + "pass@1": 0.21727638473053892, + "pass@2": 0.33913595053986484, + "pass@4": 0.48270442579408074, + "pass@8": 0.6238536005858858, + "pass@16": 0.7419666380525494, + "pass@32": 0.8299157749956388, + "pass@64": 0.891739006586996, + "pass@128": 0.9341317365269461 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 191454, + "answer_accuracy": 0.5007480331436224, + "avg_response_len": 210.55677205390023, + "resp_tokens_sum": 80503434, + "pass_at_k": { + "pass@1": 0.5007480331436224, + "pass@2": 0.5952217235975331, + "pass@4": 0.689605396883562, + "pass@8": 0.7728945084780705, + "pass@16": 0.8388889606884414, + "pass@32": 0.8882860432962518, + "pass@64": 0.9236195260189742, + "pass@128": 0.9464345497154335 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.20205223880597015, + "pass@2": 0.3135319955341403, + "pass@4": 0.4415771505240951, + "pass@8": 0.569811918955316, + "pass@16": 0.6850570876181581, + "pass@32": 0.7808957342914687, + "pass@64": 0.8544946545097453, + "pass@128": 0.9044776119402985 + }, + "2": { + "pass@1": 0.9875892857142857, + "pass@2": 0.9917403683914513, + "pass@4": 0.9935481662113664, + "pass@8": 0.9943054704732391, + "pass@16": 0.9946362839316639, + "pass@32": 0.9949999953558449, + "pass@64": 0.9957142857142857, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9700482871720116, + "pass@2": 0.9877388174054772, + "pass@4": 0.994211654756712, + "pass@8": 0.9968066288226595, + "pass@16": 0.9979756293527906, + "pass@32": 0.9987791216280217, + "pass@64": 0.9996441771291553, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7102065058479532, + "pass@2": 0.8100167063820963, + "pass@4": 0.8802413790929641, + "pass@8": 0.9272992602475084, + "pass@16": 0.9585762077324128, + "pass@32": 0.9767911949187461, + "pass@64": 0.9843987458071987, + "pass@128": 0.9853801169590644 + }, + "5": { + "pass@1": 0.41737132352941175, + "pass@2": 0.513387274201019, + "pass@4": 0.6009593699684594, + "pass@8": 0.6746747650464919, + "pass@16": 0.7341430684121006, + "pass@32": 0.7796575711960942, + "pass@64": 0.814747638086809, + "pass@128": 0.8411764705882353 + }, + "6": { + "pass@1": 0.3975749211356467, + "pass@2": 0.5503043574107656, + "pass@4": 0.6984590626881418, + "pass@8": 0.8139611823607342, + "pass@16": 0.887723354151534, + "pass@32": 0.9308254188372326, + "pass@64": 0.9556506522537759, + "pass@128": 0.9684542586750788 + }, + "7": { + "pass@1": 0.3171726661392405, + "pass@2": 0.46342187967208204, + "pass@4": 0.6122294229637435, + "pass@8": 0.7384669154900192, + "pass@16": 0.8279188406130379, + "pass@32": 0.8852386895678486, + "pass@64": 0.9234833724808077, + "pass@128": 0.9493670886075949 + }, + "8": { + "pass@1": 0.2432667042042042, + "pass@2": 0.3739248598992691, + "pass@4": 0.5204787287287282, + "pass@8": 0.6566691861840236, + "pass@16": 0.7674609415621451, + "pass@32": 0.8523845738554838, + "pass@64": 0.9119541812947833, + "pass@128": 0.9519519519519519 + }, + "9": { + "pass@1": 0.18521402733118972, + "pass@2": 0.28941596830138966, + "pass@4": 0.41672779638477636, + "pass@8": 0.5506735350456974, + "pass@16": 0.6746045152270984, + "pass@32": 0.7816134842691489, + "pass@64": 0.8652622165514577, + "pass@128": 0.9163987138263665 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.12814140625, + "avg_response_len": 306.88617109375, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.12814140625, + "pass@2": 0.20927265009842677, + "pass@4": 0.3111864025121896, + "pass@8": 0.41765540559337877, + "pass@16": 0.509781611871109, + "pass@32": 0.5812336685755211, + "pass@64": 0.6386194765971214, + "pass@128": 0.6901 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.171453125, + "pass@2": 0.27239197834645684, + "pass@4": 0.39325133211473556, + "pass@8": 0.5174260619010626, + "pass@16": 0.631814840300865, + "pass@32": 0.7319037429686506, + "pass@64": 0.8148390571373263, + "pass@128": 0.88 + }, + "12": { + "pass@1": 0.167859375, + "pass@2": 0.2672691929133858, + "pass@4": 0.3847895885826772, + "pass@8": 0.5019834476236162, + "pass@16": 0.6056284077005843, + "pass@32": 0.694149067950932, + "pass@64": 0.7716060354346793, + "pass@128": 0.841 + }, + "13": { + "pass@1": 0.147890625, + "pass@2": 0.23848548228346456, + "pass@4": 0.34836997919010115, + "pass@8": 0.4591470270639935, + "pass@16": 0.5553441471442907, + "pass@32": 0.636624422981234, + "pass@64": 0.7092317327628829, + "pass@128": 0.777 + }, + "14": { + "pass@1": 0.1368671875, + "pass@2": 0.22146862696850358, + "pass@4": 0.3267951149231343, + "pass@8": 0.4351780676273756, + "pass@16": 0.5280781674583069, + "pass@32": 0.6026351462793775, + "pass@64": 0.6678938901052593, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1123515625, + "pass@2": 0.18843528543307092, + "pass@4": 0.28883970463067066, + "pass@8": 0.3976124523685819, + "pass@16": 0.4939233993402138, + "pass@32": 0.5725014214834312, + "pass@64": 0.6423549838394249, + "pass@128": 0.708 + }, + "16": { + "pass@1": 0.12721875, + "pass@2": 0.20742445866141734, + "pass@4": 0.3063593675478061, + "pass@8": 0.4061218939267599, + "pass@16": 0.4879354435232922, + "pass@32": 0.5483041968695981, + "pass@64": 0.5979801919405243, + "pass@128": 0.648 + }, + "17": { + "pass@1": 0.11103125, + "pass@2": 0.1865286663385827, + "pass@4": 0.28527401659167617, + "pass@8": 0.3900816215096604, + "pass@16": 0.47768656098507156, + "pass@32": 0.5406021652419373, + "pass@64": 0.5883553332453951, + "pass@128": 0.632 + }, + "18": { + "pass@1": 0.1110078125, + "pass@2": 0.18122563976377937, + "pass@4": 0.2712755418072743, + "pass@8": 0.368057092070443, + "pass@16": 0.4507021167752477, + "pass@32": 0.5089708953402846, + "pass@64": 0.5504486046107775, + "pass@128": 0.589 + }, + "19": { + "pass@1": 0.100734375, + "pass@2": 0.1696690452755905, + "pass@4": 0.2611097652793399, + "pass@8": 0.3598293124601181, + "pass@16": 0.44099038348320846, + "pass@32": 0.4914653739624523, + "pass@64": 0.5203726748494747, + "pass@128": 0.544 + }, + "20": { + "pass@1": 0.095, + "pass@2": 0.15982812499999996, + "pass@4": 0.24579961445444298, + "pass@8": 0.34111707938207964, + "pass@16": 0.425712651999865, + "pass@32": 0.4851802526772986, + "pass@64": 0.5231122620453987, + "pass@128": 0.552 + } + }, + "per_op_accuracy": { + "11": 0.171453125, + "12": 0.167859375, + "13": 0.147890625, + "14": 0.1368671875, + "15": 0.1123515625, + "16": 0.12721875, + "17": 0.11103125, + "18": 0.1110078125, + "19": 0.100734375, + "20": 0.095 + }, + "per_op_avg_response_len": { + "11": 283.170015625, + "12": 292.4186640625, + "13": 300.93834375, + "14": 308.9383125, + "15": 305.68665625, + "16": 315.7982890625, + "17": 317.9190078125, + "18": 313.0223125, + "19": 316.4489296875, + "20": 314.5211796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 283.170015625, + "12": 292.4186640625, + "13": 300.93834375, + "14": 308.9383125, + "15": 305.68665625, + "16": 315.7982890625, + "17": 317.9190078125, + "18": 313.0223125, + "19": 316.4489296875, + "20": 314.5211796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 55817, + "answer_accuracy": 0.12958998885586925, + "avg_response_len": 294.9892435921248, + "resp_tokens_sum": 127057767, + "pass_at_k": { + "pass@1": 0.12958998885586925, + "pass@2": 0.2111846050122271, + "pass@4": 0.3136799228755404, + "pass@8": 0.42079987204734587, + "pass@16": 0.5135734732930936, + "pass@32": 0.5855753311528196, + "pass@64": 0.6426218308024394, + "pass@128": 0.6933135215453194 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.18229904390934845, + "pass@2": 0.2881442946844819, + "pass@4": 0.41277183172570875, + "pass@8": 0.5372213699972903, + "pass@16": 0.6464648060956772, + "pass@32": 0.7364996012096983, + "pass@64": 0.8090871748200775, + "pass@128": 0.8725212464589235 + }, + "12": { + "pass@1": 0.167041843220339, + "pass@2": 0.267159418234797, + "pass@4": 0.38463744918537746, + "pass@8": 0.5005959279433829, + "pass@16": 0.6027360469209624, + "pass@32": 0.6906821588303728, + "pass@64": 0.7683561968655983, + "pass@128": 0.8389830508474576 + }, + "13": { + "pass@1": 0.14863342476489028, + "pass@2": 0.24290428381013504, + "pass@4": 0.3593948659043012, + "pass@8": 0.4767867703415537, + "pass@16": 0.5776974942645328, + "pass@32": 0.6625444544056447, + "pass@64": 0.7355174465136388, + "pass@128": 0.7962382445141066 + }, + "14": { + "pass@1": 0.13086545950155765, + "pass@2": 0.21606400397380243, + "pass@4": 0.3265159209070825, + "pass@8": 0.44258768544823535, + "pass@16": 0.5412763209024235, + "pass@32": 0.6182166903432107, + "pass@64": 0.6805139060593539, + "pass@128": 0.735202492211838 + }, + "15": { + "pass@1": 0.11870032514450866, + "pass@2": 0.19539072811888392, + "pass@4": 0.2952038571948737, + "pass@8": 0.4034823819223066, + "pass@16": 0.4996538273439035, + "pass@32": 0.577105706738467, + "pass@64": 0.6430681278184355, + "pass@128": 0.7052023121387283 + }, + "16": { + "pass@1": 0.12497479838709677, + "pass@2": 0.20588566802133618, + "pass@4": 0.30794752833315187, + "pass@8": 0.41409910598717037, + "pass@16": 0.503439116374473, + "pass@32": 0.5680757189322889, + "pass@64": 0.6185146367153228, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.1048139450867052, + "pass@2": 0.1729335686814438, + "pass@4": 0.26166974377480257, + "pass@8": 0.3569233003143246, + "pass@16": 0.43881371763626015, + "pass@32": 0.5017114929543303, + "pass@64": 0.5540504002245086, + "pass@128": 0.6011560693641619 + }, + "18": { + "pass@1": 0.11404017857142858, + "pass@2": 0.1839486079865017, + "pass@4": 0.27364456023354217, + "pass@8": 0.3720505437784955, + "pass@16": 0.45863649753082086, + "pass@32": 0.5215180865558517, + "pass@64": 0.5696621746276215, + "pass@128": 0.62 + }, + "19": { + "pass@1": 0.10076482732732733, + "pass@2": 0.1690570984370197, + "pass@4": 0.2604623833619896, + "pass@8": 0.3599502461168844, + "pass@16": 0.44260688970777834, + "pass@32": 0.4944624925735973, + "pass@64": 0.521548379452954, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.10095251501501501, + "pass@2": 0.16684992078692867, + "pass@4": 0.2509089626747106, + "pass@8": 0.3406114996432136, + "pass@16": 0.42006647369881606, + "pass@32": 0.47932235146072466, + "pass@64": 0.5187436798897297, + "pass@128": 0.5465465465465466 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 55692, + "answer_accuracy": 0.12937667261373773, + "avg_response_len": 306.0040119498959, + "resp_tokens_sum": 131723711, + "pass_at_k": { + "pass@1": 0.12937667261373773, + "pass@2": 0.21112182774097948, + "pass@4": 0.3140564988754203, + "pass@8": 0.4217546405011227, + "pass@16": 0.5151074911033252, + "pass@32": 0.5886237408356023, + "pass@64": 0.6499092652529367, + "pass@128": 0.7068093963722867 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.1560617469879518, + "pass@2": 0.2487022400626127, + "pass@4": 0.363962768961109, + "pass@8": 0.4871624174058488, + "pass@16": 0.603968486823371, + "pass@32": 0.7099225827109938, + "pass@64": 0.7991877595200533, + "pass@128": 0.8644578313253012 + }, + "12": { + "pass@1": 0.18140184169278997, + "pass@2": 0.28670233937254724, + "pass@4": 0.41065066504226133, + "pass@8": 0.5328363958358436, + "pass@16": 0.6391938939640938, + "pass@32": 0.7286125927091673, + "pass@64": 0.8047119385141658, + "pass@128": 0.8714733542319749 + }, + "13": { + "pass@1": 0.14758403361344538, + "pass@2": 0.2368842221928142, + "pass@4": 0.34356088472134266, + "pass@8": 0.44992350437981904, + "pass@16": 0.5427658276100494, + "pass@32": 0.6231971069674507, + "pass@64": 0.6981882477135789, + "pass@128": 0.773109243697479 + }, + "14": { + "pass@1": 0.1457419590643275, + "pass@2": 0.2322633190588018, + "pass@4": 0.33790162002775964, + "pass@8": 0.44425079336957135, + "pass@16": 0.5325720105945908, + "pass@32": 0.6005776223976965, + "pass@64": 0.662597142298673, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.11176609848484849, + "pass@2": 0.1877408434741111, + "pass@4": 0.28652309569258383, + "pass@8": 0.39240725982235936, + "pass@16": 0.4878854499058535, + "pass@32": 0.5707635144848856, + "pass@64": 0.6501004572511563, + "pass@128": 0.7272727272727273 + }, + "16": { + "pass@1": 0.1274928774928775, + "pass@2": 0.20607284866635253, + "pass@4": 0.302366788019019, + "pass@8": 0.39909323245321887, + "pass@16": 0.479867770840156, + "pass@32": 0.5436816319849745, + "pass@64": 0.5996621177657357, + "pass@128": 0.6552706552706553 + }, + "17": { + "pass@1": 0.1155861801242236, + "pass@2": 0.19869586614173215, + "pass@4": 0.31046421002498914, + "pass@8": 0.43139028333994395, + "pass@16": 0.5326025992065495, + "pass@32": 0.6042294785582262, + "pass@64": 0.6602624716039003, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.11557904411764706, + "pass@2": 0.19096840261695236, + "pass@4": 0.28819167300778564, + "pass@8": 0.39090138579917816, + "pass@16": 0.4763199553646875, + "pass@32": 0.5355086068492452, + "pass@64": 0.5739343318595364, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.10209728422619048, + "pass@2": 0.1712254229940008, + "pass@4": 0.26240663471753534, + "pass@8": 0.3616555082558499, + "pass@16": 0.44495512112719054, + "pass@32": 0.5003037881586762, + "pass@64": 0.5392674835648679, + "pass@128": 0.5773809523809523 + }, + "20": { + "pass@1": 0.09110684880239521, + "pass@2": 0.15343316811259372, + "pass@4": 0.23748076793544526, + "pass@8": 0.33288966241110335, + "pass@16": 0.4179387610759886, + "pass@32": 0.4779039161554723, + "pass@64": 0.5203573952212737, + "pass@128": 0.5568862275449101 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 52512, + "answer_accuracy": 0.1253820293398533, + "avg_response_len": 320.02793828316015, + "resp_tokens_sum": 134032821, + "pass_at_k": { + "pass@1": 0.1253820293398533, + "pass@2": 0.20540574523997496, + "pass@4": 0.30567209013680813, + "pass@8": 0.41020832227661524, + "pass@16": 0.5004079732577005, + "pass@32": 0.5691729999987193, + "pass@64": 0.6228995862699644, + "pass@128": 0.6696210268948656 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.17552083333333332, + "pass@2": 0.2797076146731657, + "pass@4": 0.40224519435070627, + "pass@8": 0.5271396689310396, + "pass@16": 0.644746749599154, + "pass@32": 0.7499209088304015, + "pass@64": 0.8377807881434293, + "pass@128": 0.9047619047619048 + }, + "12": { + "pass@1": 0.15553325688073394, + "pass@2": 0.24843031375665203, + "pass@4": 0.3597259003748386, + "pass@8": 0.47338739712545613, + "pass@16": 0.576015274972535, + "pass@32": 0.664281855201088, + "pass@64": 0.7428282365695322, + "pass@128": 0.8134556574923547 + }, + "13": { + "pass@1": 0.14749710648148148, + "pass@2": 0.2358992235345583, + "pass@4": 0.3428141392626847, + "pass@8": 0.4519424575970455, + "pass@16": 0.5471952037565342, + "pass@32": 0.6258993050569552, + "pass@64": 0.6955199471953205, + "pass@128": 0.7623456790123457 + }, + "14": { + "pass@1": 0.13357752225519287, + "pass@2": 0.21566179992523188, + "pass@4": 0.3157897633901105, + "pass@8": 0.41891290583411983, + "pass@16": 0.510946115208545, + "pass@32": 0.5898814297305477, + "pass@64": 0.6612483786173892, + "pass@128": 0.7240356083086054 + }, + "15": { + "pass@1": 0.10616801697530864, + "pass@2": 0.18171486159716133, + "pass@4": 0.2844029273539879, + "pass@8": 0.3966455323521176, + "pass@16": 0.4939536315132476, + "pass@32": 0.5693545900367545, + "pass@64": 0.6337045083992685, + "pass@128": 0.691358024691358 + }, + "16": { + "pass@1": 0.12898691002949852, + "pass@2": 0.21023106694074747, + "pass@4": 0.3090409769022235, + "pass@8": 0.40610456188689503, + "pass@16": 0.4821112975879393, + "pass@32": 0.5350102394509227, + "pass@64": 0.57746091806195, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.1130929969879518, + "pass@2": 0.18889633336495587, + "pass@4": 0.28544219764321627, + "pass@8": 0.38457376013687594, + "pass@16": 0.46493667993499693, + "pass@32": 0.5194219776626258, + "pass@64": 0.5543655991302966, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.10257056451612903, + "pass@2": 0.16746570993141993, + "pass@4": 0.250047667428668, + "pass@8": 0.33849332443951263, + "pass@16": 0.41364696069505363, + "pass@32": 0.4656988345709459, + "pass@64": 0.5029973247060224, + "pass@128": 0.5387096774193548 + }, + "19": { + "pass@1": 0.0993202416918429, + "pass@2": 0.16870480172229227, + "pass@4": 0.2604445992589294, + "pass@8": 0.3578538662514504, + "pass@16": 0.4353394819087087, + "pass@32": 0.47947822088860853, + "pass@64": 0.5000096375040654, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.09295232732732733, + "pass@2": 0.15922049017521456, + "pass@4": 0.24903409427425166, + "pass@8": 0.3498747830497944, + "pass@16": 0.43915606624257963, + "pass@32": 0.4983363412761255, + "pass@64": 0.5302439838985393, + "pass@128": 0.5525525525525525 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.30109909539473684, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.30109909539473684, + "pass@2": 0.38850797114586283, + "pass@4": 0.4870086190919986, + "pass@8": 0.5828351936333129, + "pass@16": 0.6625592611714813, + "pass@32": 0.7232762638453931, + "pass@64": 0.7700682571339985, + "pass@128": 0.8082631578947368 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.190796875, + "pass@2": 0.3003977608267716, + "pass@4": 0.43215709983127026, + "pass@8": 0.5686598935179051, + "pass@16": 0.6915305344763011, + "pass@32": 0.7914435747314094, + "pass@64": 0.8667204616843248, + "pass@128": 0.915 + }, + "2": { + "pass@1": 0.9806953125, + "pass@2": 0.9874446358267717, + "pass@4": 0.9907921147356582, + "pass@8": 0.992629310685749, + "pass@16": 0.9938021523891528, + "pass@32": 0.9946754172048058, + "pass@64": 0.9953712414614146, + "pass@128": 0.996 + }, + "3": { + "pass@1": 0.971625, + "pass@2": 0.9875855068897645, + "pass@4": 0.9931698494563178, + "pass@8": 0.9955605027834186, + "pass@16": 0.9969572346659715, + "pass@32": 0.9980001237695623, + "pass@64": 0.9987513647462997, + "pass@128": 0.999 + }, + "4": { + "pass@1": 0.706859375, + "pass@2": 0.8033623277559058, + "pass@4": 0.8713294938132747, + "pass@8": 0.9168010997977786, + "pass@16": 0.9457339750167953, + "pass@32": 0.9627812219276177, + "pass@64": 0.9726259406408313, + "pass@128": 0.978 + }, + "5": { + "pass@1": 0.3789453125, + "pass@2": 0.4657646407480316, + "pass@4": 0.5468323475815521, + "pass@8": 0.6180357705552324, + "pass@16": 0.6779580589440685, + "pass@32": 0.7276554152082413, + "pass@64": 0.768848988855058, + "pass@128": 0.803 + }, + "6": { + "pass@1": 0.42653125, + "pass@2": 0.5740638533464576, + "pass@4": 0.7139497283464574, + "pass@8": 0.8232841091018387, + "pass@16": 0.8924874328530586, + "pass@32": 0.9325368862764091, + "pass@64": 0.9569446412663042, + "pass@128": 0.972 + }, + "7": { + "pass@1": 0.320046875, + "pass@2": 0.4576422244094486, + "pass@4": 0.5986938634233218, + "pass@8": 0.7207198041731779, + "pass@16": 0.8103197939740765, + "pass@32": 0.8708360674901743, + "pass@64": 0.9107117986457854, + "pass@128": 0.937 + }, + "8": { + "pass@1": 0.2600234375, + "pass@2": 0.39415624999999993, + "pass@4": 0.5410903218972629, + "pass@8": 0.6742588911332974, + "pass@16": 0.777802389503416, + "pass@32": 0.8536262533516904, + "pass@64": 0.9061390567254755, + "pass@128": 0.94 + }, + "9": { + "pass@1": 0.2039453125, + "pass@2": 0.3185077509842525, + "pass@4": 0.45328491854143194, + "pass@8": 0.5873652413506206, + "pass@16": 0.7042182717236272, + "pass@32": 0.7983573673467786, + "pass@64": 0.8689886255490779, + "pass@128": 0.916 + }, + "11": { + "pass@1": 0.171453125, + "pass@2": 0.27239197834645684, + "pass@4": 0.39325133211473556, + "pass@8": 0.5174260619010626, + "pass@16": 0.631814840300865, + "pass@32": 0.7319037429686506, + "pass@64": 0.8148390571373263, + "pass@128": 0.88 + }, + "12": { + "pass@1": 0.167859375, + "pass@2": 0.2672691929133858, + "pass@4": 0.3847895885826772, + "pass@8": 0.5019834476236162, + "pass@16": 0.6056284077005843, + "pass@32": 0.694149067950932, + "pass@64": 0.7716060354346793, + "pass@128": 0.841 + }, + "13": { + "pass@1": 0.147890625, + "pass@2": 0.23848548228346456, + "pass@4": 0.34836997919010115, + "pass@8": 0.4591470270639935, + "pass@16": 0.5553441471442907, + "pass@32": 0.636624422981234, + "pass@64": 0.7092317327628829, + "pass@128": 0.777 + }, + "14": { + "pass@1": 0.1368671875, + "pass@2": 0.22146862696850358, + "pass@4": 0.3267951149231343, + "pass@8": 0.4351780676273756, + "pass@16": 0.5280781674583069, + "pass@32": 0.6026351462793775, + "pass@64": 0.6678938901052593, + "pass@128": 0.73 + }, + "15": { + "pass@1": 0.1123515625, + "pass@2": 0.18843528543307092, + "pass@4": 0.28883970463067066, + "pass@8": 0.3976124523685819, + "pass@16": 0.4939233993402138, + "pass@32": 0.5725014214834312, + "pass@64": 0.6423549838394249, + "pass@128": 0.708 + }, + "16": { + "pass@1": 0.12721875, + "pass@2": 0.20742445866141734, + "pass@4": 0.3063593675478061, + "pass@8": 0.4061218939267599, + "pass@16": 0.4879354435232922, + "pass@32": 0.5483041968695981, + "pass@64": 0.5979801919405243, + "pass@128": 0.648 + }, + "17": { + "pass@1": 0.11103125, + "pass@2": 0.1865286663385827, + "pass@4": 0.28527401659167617, + "pass@8": 0.3900816215096604, + "pass@16": 0.47768656098507156, + "pass@32": 0.5406021652419373, + "pass@64": 0.5883553332453951, + "pass@128": 0.632 + }, + "18": { + "pass@1": 0.1110078125, + "pass@2": 0.18122563976377937, + "pass@4": 0.2712755418072743, + "pass@8": 0.368057092070443, + "pass@16": 0.4507021167752477, + "pass@32": 0.5089708953402846, + "pass@64": 0.5504486046107775, + "pass@128": 0.589 + }, + "19": { + "pass@1": 0.100734375, + "pass@2": 0.1696690452755905, + "pass@4": 0.2611097652793399, + "pass@8": 0.3598293124601181, + "pass@16": 0.44099038348320846, + "pass@32": 0.4914653739624523, + "pass@64": 0.5203726748494747, + "pass@128": 0.544 + }, + "20": { + "pass@1": 0.095, + "pass@2": 0.15982812499999996, + "pass@4": 0.24579961445444298, + "pass@8": 0.34111707938207964, + "pass@16": 0.425712651999865, + "pass@32": 0.4851802526772986, + "pass@64": 0.5231122620453987, + "pass@128": 0.552 + } + }, + "per_op_accuracy": { + "10": 0.190796875, + "2": 0.9806953125, + "3": 0.971625, + "4": 0.706859375, + "5": 0.3789453125, + "6": 0.42653125, + "7": 0.320046875, + "8": 0.2600234375, + "9": 0.2039453125, + "11": 0.171453125, + "12": 0.167859375, + "13": 0.147890625, + "14": 0.1368671875, + "15": 0.1123515625, + "16": 0.12721875, + "17": 0.11103125, + "18": 0.1110078125, + "19": 0.100734375, + "20": 0.095 + }, + "per_op_avg_response_len": { + "10": 272.0839765625, + "2": 115.115125, + "3": 145.4905390625, + "4": 162.5349921875, + "5": 170.5468828125, + "6": 203.4081484375, + "7": 225.40946875, + "8": 243.94565625, + "9": 253.698359375, + "11": 283.170015625, + "12": 292.4186640625, + "13": 300.93834375, + "14": 308.9383125, + "15": 305.68665625, + "16": 315.7982890625, + "17": 317.9190078125, + "18": 313.0223125, + "19": 316.4489296875, + "20": 314.5211796875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 272.0839765625, + "2": 115.115125, + "3": 145.4905390625, + "4": 162.5349921875, + "5": 170.5468828125, + "6": 203.4081484375, + "7": 225.40946875, + "8": 243.94565625, + "9": 253.698359375, + "11": 283.170015625, + "12": 292.4186640625, + "13": 300.93834375, + "14": 308.9383125, + "15": 305.68665625, + "16": 315.7982890625, + "17": 317.9190078125, + "18": 313.0223125, + "19": 316.4489296875, + "20": 314.5211796875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 239685, + "answer_accuracy": 0.2960068072241543, + "avg_response_len": 245.8270233955106, + "resp_tokens_sum": 199053024, + "pass_at_k": { + "pass@1": 0.2960068072241543, + "pass@2": 0.38263211474455283, + "pass@4": 0.480826212706066, + "pass@8": 0.5767558778808439, + "pass@16": 0.6568090850720517, + "pass@32": 0.7180689672909597, + "pass@64": 0.7652942721942446, + "pass@128": 0.8039835599114764 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.19035766961651918, + "pass@2": 0.30144277111931805, + "pass@4": 0.43816523764175513, + "pass@8": 0.5809281738806453, + "pass@16": 0.7052121639952813, + "pass@32": 0.8019947591187581, + "pass@64": 0.8754555701737227, + "pass@128": 0.9233038348082596 + }, + "2": { + "pass@1": 0.9800915109034268, + "pass@2": 0.9870491144798491, + "pass@4": 0.990911137860104, + "pass@8": 0.9931530593847345, + "pass@16": 0.9945585526843174, + "pass@32": 0.9955696135172313, + "pass@64": 0.9965045227606399, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9717842125382263, + "pass@2": 0.9858160111006766, + "pass@4": 0.991448598168348, + "pass@8": 0.994291079132131, + "pass@16": 0.995887473442639, + "pass@32": 0.9967283718547971, + "pass@64": 0.9969374773681062, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.6986328125, + "pass@2": 0.7886968657726381, + "pass@4": 0.8542420943475817, + "pass@8": 0.9004456167198726, + "pass@16": 0.9312778575005979, + "pass@32": 0.9517112440915444, + "pass@64": 0.9667046571410877, + "pass@128": 0.978125 + }, + "5": { + "pass@1": 0.33509424603174603, + "pass@2": 0.41271325459317576, + "pass@4": 0.48771068527148376, + "pass@8": 0.5572081248784712, + "pass@16": 0.6188679542444174, + "pass@32": 0.6719682822822658, + "pass@64": 0.7166542430331456, + "pass@128": 0.7523809523809524 + }, + "6": { + "pass@1": 0.46206025449101795, + "pass@2": 0.6031718551086802, + "pass@4": 0.7298749273107324, + "pass@8": 0.8273623466954299, + "pass@16": 0.8904114372774126, + "pass@32": 0.9291838638462578, + "pass@64": 0.9545298379466033, + "pass@128": 0.9730538922155688 + }, + "7": { + "pass@1": 0.30755362426035504, + "pass@2": 0.44404614918697294, + "pass@4": 0.5866653091255597, + "pass@8": 0.7100621776567952, + "pass@16": 0.8009828251974904, + "pass@32": 0.8641776511072881, + "pass@64": 0.9077531903073257, + "pass@128": 0.9408284023668639 + }, + "8": { + "pass@1": 0.25122696314102566, + "pass@2": 0.38249427430345234, + "pass@4": 0.5251050217280533, + "pass@8": 0.6536996226393975, + "pass@16": 0.7536627127551332, + "pass@32": 0.826661161406808, + "pass@64": 0.8762717660769928, + "pass@128": 0.9038461538461539 + }, + "9": { + "pass@1": 0.2078125, + "pass@2": 0.3245858517245202, + "pass@4": 0.4576318187339257, + "pass@8": 0.5851793502978136, + "pass@16": 0.6946462264181785, + "pass@32": 0.7833343799733238, + "pass@64": 0.8508485859197652, + "pass@128": 0.8985915492957747 + }, + "11": { + "pass@1": 0.18229904390934845, + "pass@2": 0.2881442946844819, + "pass@4": 0.41277183172570875, + "pass@8": 0.5372213699972903, + "pass@16": 0.6464648060956772, + "pass@32": 0.7364996012096983, + "pass@64": 0.8090871748200775, + "pass@128": 0.8725212464589235 + }, + "12": { + "pass@1": 0.167041843220339, + "pass@2": 0.267159418234797, + "pass@4": 0.38463744918537746, + "pass@8": 0.5005959279433829, + "pass@16": 0.6027360469209624, + "pass@32": 0.6906821588303728, + "pass@64": 0.7683561968655983, + "pass@128": 0.8389830508474576 + }, + "13": { + "pass@1": 0.14863342476489028, + "pass@2": 0.24290428381013504, + "pass@4": 0.3593948659043012, + "pass@8": 0.4767867703415537, + "pass@16": 0.5776974942645328, + "pass@32": 0.6625444544056447, + "pass@64": 0.7355174465136388, + "pass@128": 0.7962382445141066 + }, + "14": { + "pass@1": 0.13086545950155765, + "pass@2": 0.21606400397380243, + "pass@4": 0.3265159209070825, + "pass@8": 0.44258768544823535, + "pass@16": 0.5412763209024235, + "pass@32": 0.6182166903432107, + "pass@64": 0.6805139060593539, + "pass@128": 0.735202492211838 + }, + "15": { + "pass@1": 0.11870032514450866, + "pass@2": 0.19539072811888392, + "pass@4": 0.2952038571948737, + "pass@8": 0.4034823819223066, + "pass@16": 0.4996538273439035, + "pass@32": 0.577105706738467, + "pass@64": 0.6430681278184355, + "pass@128": 0.7052023121387283 + }, + "16": { + "pass@1": 0.12497479838709677, + "pass@2": 0.20588566802133618, + "pass@4": 0.30794752833315187, + "pass@8": 0.41409910598717037, + "pass@16": 0.503439116374473, + "pass@32": 0.5680757189322889, + "pass@64": 0.6185146367153228, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.1048139450867052, + "pass@2": 0.1729335686814438, + "pass@4": 0.26166974377480257, + "pass@8": 0.3569233003143246, + "pass@16": 0.43881371763626015, + "pass@32": 0.5017114929543303, + "pass@64": 0.5540504002245086, + "pass@128": 0.6011560693641619 + }, + "18": { + "pass@1": 0.11404017857142858, + "pass@2": 0.1839486079865017, + "pass@4": 0.27364456023354217, + "pass@8": 0.3720505437784955, + "pass@16": 0.45863649753082086, + "pass@32": 0.5215180865558517, + "pass@64": 0.5696621746276215, + "pass@128": 0.62 + }, + "19": { + "pass@1": 0.10076482732732733, + "pass@2": 0.1690570984370197, + "pass@4": 0.2604623833619896, + "pass@8": 0.3599502461168844, + "pass@16": 0.44260688970777834, + "pass@32": 0.4944624925735973, + "pass@64": 0.521548379452954, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.10095251501501501, + "pass@2": 0.16684992078692867, + "pass@4": 0.2509089626747106, + "pass@8": 0.3406114996432136, + "pass@16": 0.42006647369881606, + "pass@32": 0.47932235146072466, + "pass@64": 0.5187436798897297, + "pass@128": 0.5465465465465466 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 248622, + "answer_accuracy": 0.3027840023382697, + "avg_response_len": 254.0808444563523, + "resp_tokens_sum": 208630863, + "pass_at_k": { + "pass@1": 0.3027840023382697, + "pass@2": 0.39144283820524556, + "pass@4": 0.4912621577164558, + "pass@8": 0.5883824580273733, + "pass@16": 0.6688317187866314, + "pass@32": 0.7301793076629046, + "pass@64": 0.7783423771084799, + "pass@128": 0.8188620420888543 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1796875, + "pass@2": 0.28581424206560085, + "pass@4": 0.41622926026117907, + "pass@8": 0.5547185574304757, + "pass@16": 0.6839554801528199, + "pass@32": 0.791310676081314, + "pass@64": 0.8702003501063394, + "pass@128": 0.9171779141104295 + }, + "2": { + "pass@1": 0.9739504179331308, + "pass@2": 0.9832606113730465, + "pass@4": 0.9877440161355209, + "pass@8": 0.9903351488073417, + "pass@16": 0.9921767708249385, + "pass@32": 0.9934576683623969, + "pass@64": 0.9939005764597241, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9731060606060606, + "pass@2": 0.9891795663326179, + "pass@4": 0.9937926068900476, + "pass@8": 0.9955231703667864, + "pass@16": 0.9969587575218833, + "pass@32": 0.9984506286504, + "pass@64": 0.9996207784594809, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7112610946745562, + "pass@2": 0.8105136601360479, + "pass@4": 0.8784895620480868, + "pass@8": 0.9216631817833484, + "pass@16": 0.9464260283198788, + "pass@32": 0.9590859028287348, + "pass@64": 0.9663197612118946, + "pass@128": 0.9704142011834319 + }, + "5": { + "pass@1": 0.3811141304347826, + "pass@2": 0.4672704125299555, + "pass@4": 0.547470422990604, + "pass@8": 0.6177559162397293, + "pass@16": 0.6765391597013457, + "pass@32": 0.7272516871961043, + "pass@64": 0.7712716097104394, + "pass@128": 0.8115942028985508 + }, + "6": { + "pass@1": 0.41883058739255014, + "pass@2": 0.5677878866502717, + "pass@4": 0.7127793116118358, + "pass@8": 0.8278492564390045, + "pass@16": 0.8988014599895945, + "pass@32": 0.9373003380525965, + "pass@64": 0.9604309989332116, + "pass@128": 0.9742120343839542 + }, + "7": { + "pass@1": 0.33487626445086704, + "pass@2": 0.46564541042738106, + "pass@4": 0.5980823447466463, + "pass@8": 0.714922667139698, + "pass@16": 0.8033677613399027, + "pass@32": 0.8641866922325745, + "pass@64": 0.9019377879132182, + "pass@128": 0.9219653179190751 + }, + "8": { + "pass@1": 0.28347271126760565, + "pass@2": 0.4233832621714537, + "pass@4": 0.5744736294406855, + "pass@8": 0.7088275207620468, + "pass@16": 0.8087186748834445, + "pass@32": 0.8784898814053248, + "pass@64": 0.9269338685585622, + "pass@128": 0.9605633802816902 + }, + "9": { + "pass@1": 0.21727638473053892, + "pass@2": 0.33913595053986484, + "pass@4": 0.48270442579408074, + "pass@8": 0.6238536005858858, + "pass@16": 0.7419666380525494, + "pass@32": 0.8299157749956388, + "pass@64": 0.891739006586996, + "pass@128": 0.9341317365269461 + }, + "11": { + "pass@1": 0.1560617469879518, + "pass@2": 0.2487022400626127, + "pass@4": 0.363962768961109, + "pass@8": 0.4871624174058488, + "pass@16": 0.603968486823371, + "pass@32": 0.7099225827109938, + "pass@64": 0.7991877595200533, + "pass@128": 0.8644578313253012 + }, + "12": { + "pass@1": 0.18140184169278997, + "pass@2": 0.28670233937254724, + "pass@4": 0.41065066504226133, + "pass@8": 0.5328363958358436, + "pass@16": 0.6391938939640938, + "pass@32": 0.7286125927091673, + "pass@64": 0.8047119385141658, + "pass@128": 0.8714733542319749 + }, + "13": { + "pass@1": 0.14758403361344538, + "pass@2": 0.2368842221928142, + "pass@4": 0.34356088472134266, + "pass@8": 0.44992350437981904, + "pass@16": 0.5427658276100494, + "pass@32": 0.6231971069674507, + "pass@64": 0.6981882477135789, + "pass@128": 0.773109243697479 + }, + "14": { + "pass@1": 0.1457419590643275, + "pass@2": 0.2322633190588018, + "pass@4": 0.33790162002775964, + "pass@8": 0.44425079336957135, + "pass@16": 0.5325720105945908, + "pass@32": 0.6005776223976965, + "pass@64": 0.662597142298673, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.11176609848484849, + "pass@2": 0.1877408434741111, + "pass@4": 0.28652309569258383, + "pass@8": 0.39240725982235936, + "pass@16": 0.4878854499058535, + "pass@32": 0.5707635144848856, + "pass@64": 0.6501004572511563, + "pass@128": 0.7272727272727273 + }, + "16": { + "pass@1": 0.1274928774928775, + "pass@2": 0.20607284866635253, + "pass@4": 0.302366788019019, + "pass@8": 0.39909323245321887, + "pass@16": 0.479867770840156, + "pass@32": 0.5436816319849745, + "pass@64": 0.5996621177657357, + "pass@128": 0.6552706552706553 + }, + "17": { + "pass@1": 0.1155861801242236, + "pass@2": 0.19869586614173215, + "pass@4": 0.31046421002498914, + "pass@8": 0.43139028333994395, + "pass@16": 0.5326025992065495, + "pass@32": 0.6042294785582262, + "pass@64": 0.6602624716039003, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.11557904411764706, + "pass@2": 0.19096840261695236, + "pass@4": 0.28819167300778564, + "pass@8": 0.39090138579917816, + "pass@16": 0.4763199553646875, + "pass@32": 0.5355086068492452, + "pass@64": 0.5739343318595364, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.10209728422619048, + "pass@2": 0.1712254229940008, + "pass@4": 0.26240663471753534, + "pass@8": 0.3616555082558499, + "pass@16": 0.44495512112719054, + "pass@32": 0.5003037881586762, + "pass@64": 0.5392674835648679, + "pass@128": 0.5773809523809523 + }, + "20": { + "pass@1": 0.09110684880239521, + "pass@2": 0.15343316811259372, + "pass@4": 0.23748076793544526, + "pass@8": 0.33288966241110335, + "pass@16": 0.4179387610759886, + "pass@32": 0.4779039161554723, + "pass@64": 0.5203573952212737, + "pass@128": 0.5568862275449101 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 243966, + "answer_accuracy": 0.3045189926505832, + "avg_response_len": 267.7847087693721, + "resp_tokens_sum": 214536255, + "pass_at_k": { + "pass@1": 0.3045189926505832, + "pass@2": 0.3914387101471566, + "pass@4": 0.4888976512891585, + "pass@8": 0.5832940609223775, + "pass@16": 0.6619421974877205, + "pass@32": 0.7214642063143812, + "pass@64": 0.7664130005582344, + "pass@128": 0.8017255152580285 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.20205223880597015, + "pass@2": 0.3135319955341403, + "pass@4": 0.4415771505240951, + "pass@8": 0.569811918955316, + "pass@16": 0.6850570876181581, + "pass@32": 0.7808957342914687, + "pass@64": 0.8544946545097453, + "pass@128": 0.9044776119402985 + }, + "2": { + "pass@1": 0.9875892857142857, + "pass@2": 0.9917403683914513, + "pass@4": 0.9935481662113664, + "pass@8": 0.9943054704732391, + "pass@16": 0.9946362839316639, + "pass@32": 0.9949999953558449, + "pass@64": 0.9957142857142857, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9700482871720116, + "pass@2": 0.9877388174054772, + "pass@4": 0.994211654756712, + "pass@8": 0.9968066288226595, + "pass@16": 0.9979756293527906, + "pass@32": 0.9987791216280217, + "pass@64": 0.9996441771291553, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7102065058479532, + "pass@2": 0.8100167063820963, + "pass@4": 0.8802413790929641, + "pass@8": 0.9272992602475084, + "pass@16": 0.9585762077324128, + "pass@32": 0.9767911949187461, + "pass@64": 0.9843987458071987, + "pass@128": 0.9853801169590644 + }, + "5": { + "pass@1": 0.41737132352941175, + "pass@2": 0.513387274201019, + "pass@4": 0.6009593699684594, + "pass@8": 0.6746747650464919, + "pass@16": 0.7341430684121006, + "pass@32": 0.7796575711960942, + "pass@64": 0.814747638086809, + "pass@128": 0.8411764705882353 + }, + "6": { + "pass@1": 0.3975749211356467, + "pass@2": 0.5503043574107656, + "pass@4": 0.6984590626881418, + "pass@8": 0.8139611823607342, + "pass@16": 0.887723354151534, + "pass@32": 0.9308254188372326, + "pass@64": 0.9556506522537759, + "pass@128": 0.9684542586750788 + }, + "7": { + "pass@1": 0.3171726661392405, + "pass@2": 0.46342187967208204, + "pass@4": 0.6122294229637435, + "pass@8": 0.7384669154900192, + "pass@16": 0.8279188406130379, + "pass@32": 0.8852386895678486, + "pass@64": 0.9234833724808077, + "pass@128": 0.9493670886075949 + }, + "8": { + "pass@1": 0.2432667042042042, + "pass@2": 0.3739248598992691, + "pass@4": 0.5204787287287282, + "pass@8": 0.6566691861840236, + "pass@16": 0.7674609415621451, + "pass@32": 0.8523845738554838, + "pass@64": 0.9119541812947833, + "pass@128": 0.9519519519519519 + }, + "9": { + "pass@1": 0.18521402733118972, + "pass@2": 0.28941596830138966, + "pass@4": 0.41672779638477636, + "pass@8": 0.5506735350456974, + "pass@16": 0.6746045152270984, + "pass@32": 0.7816134842691489, + "pass@64": 0.8652622165514577, + "pass@128": 0.9163987138263665 + }, + "11": { + "pass@1": 0.17552083333333332, + "pass@2": 0.2797076146731657, + "pass@4": 0.40224519435070627, + "pass@8": 0.5271396689310396, + "pass@16": 0.644746749599154, + "pass@32": 0.7499209088304015, + "pass@64": 0.8377807881434293, + "pass@128": 0.9047619047619048 + }, + "12": { + "pass@1": 0.15553325688073394, + "pass@2": 0.24843031375665203, + "pass@4": 0.3597259003748386, + "pass@8": 0.47338739712545613, + "pass@16": 0.576015274972535, + "pass@32": 0.664281855201088, + "pass@64": 0.7428282365695322, + "pass@128": 0.8134556574923547 + }, + "13": { + "pass@1": 0.14749710648148148, + "pass@2": 0.2358992235345583, + "pass@4": 0.3428141392626847, + "pass@8": 0.4519424575970455, + "pass@16": 0.5471952037565342, + "pass@32": 0.6258993050569552, + "pass@64": 0.6955199471953205, + "pass@128": 0.7623456790123457 + }, + "14": { + "pass@1": 0.13357752225519287, + "pass@2": 0.21566179992523188, + "pass@4": 0.3157897633901105, + "pass@8": 0.41891290583411983, + "pass@16": 0.510946115208545, + "pass@32": 0.5898814297305477, + "pass@64": 0.6612483786173892, + "pass@128": 0.7240356083086054 + }, + "15": { + "pass@1": 0.10616801697530864, + "pass@2": 0.18171486159716133, + "pass@4": 0.2844029273539879, + "pass@8": 0.3966455323521176, + "pass@16": 0.4939536315132476, + "pass@32": 0.5693545900367545, + "pass@64": 0.6337045083992685, + "pass@128": 0.691358024691358 + }, + "16": { + "pass@1": 0.12898691002949852, + "pass@2": 0.21023106694074747, + "pass@4": 0.3090409769022235, + "pass@8": 0.40610456188689503, + "pass@16": 0.4821112975879393, + "pass@32": 0.5350102394509227, + "pass@64": 0.57746091806195, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.1130929969879518, + "pass@2": 0.18889633336495587, + "pass@4": 0.28544219764321627, + "pass@8": 0.38457376013687594, + "pass@16": 0.46493667993499693, + "pass@32": 0.5194219776626258, + "pass@64": 0.5543655991302966, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.10257056451612903, + "pass@2": 0.16746570993141993, + "pass@4": 0.250047667428668, + "pass@8": 0.33849332443951263, + "pass@16": 0.41364696069505363, + "pass@32": 0.4656988345709459, + "pass@64": 0.5029973247060224, + "pass@128": 0.5387096774193548 + }, + "19": { + "pass@1": 0.0993202416918429, + "pass@2": 0.16870480172229227, + "pass@4": 0.2604445992589294, + "pass@8": 0.3578538662514504, + "pass@16": 0.4353394819087087, + "pass@32": 0.47947822088860853, + "pass@64": 0.5000096375040654, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.09295232732732733, + "pass@2": 0.15922049017521456, + "pass@4": 0.24903409427425166, + "pass@8": 0.3498747830497944, + "pass@16": 0.43915606624257963, + "pass@32": 0.4983363412761255, + "pass@64": 0.5302439838985393, + "pass@128": 0.5525525525525525 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/summary.csv b/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..c6635558a90cd6b17245251ea0006f4466e177d7 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85656,0.49327430555555557,0.12814140625,0.30109909539473684,0.0,0.0,0.0,199.13701649305557,306.88617109375,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/summary.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..671fc975920d254c73ebc3399009e174e932d01e --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block32/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85656", + "id_acc": 0.49327430555555557, + "ood_acc": 0.12814140625, + "total_acc": 0.30109909539473684, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 199.13701649305557, + "ood_avg_resp_len": 306.88617109375, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 239685, + "answer_accuracy": 0.2960068072241543, + "avg_response_len": 245.8270233955106, + "resp_tokens_sum": 199053024, + "pass_at_k": { + "pass@1": 0.2960068072241543, + "pass@2": 0.38263211474455283, + "pass@4": 0.480826212706066, + "pass@8": 0.5767558778808439, + "pass@16": 0.6568090850720517, + "pass@32": 0.7180689672909597, + "pass@64": 0.7652942721942446, + "pass@128": 0.8039835599114764 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.19035766961651918, + "pass@2": 0.30144277111931805, + "pass@4": 0.43816523764175513, + "pass@8": 0.5809281738806453, + "pass@16": 0.7052121639952813, + "pass@32": 0.8019947591187581, + "pass@64": 0.8754555701737227, + "pass@128": 0.9233038348082596 + }, + "2": { + "pass@1": 0.9800915109034268, + "pass@2": 0.9870491144798491, + "pass@4": 0.990911137860104, + "pass@8": 0.9931530593847345, + "pass@16": 0.9945585526843174, + "pass@32": 0.9955696135172313, + "pass@64": 0.9965045227606399, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9717842125382263, + "pass@2": 0.9858160111006766, + "pass@4": 0.991448598168348, + "pass@8": 0.994291079132131, + "pass@16": 0.995887473442639, + "pass@32": 0.9967283718547971, + "pass@64": 0.9969374773681062, + "pass@128": 0.9969418960244648 + }, + "4": { + "pass@1": 0.6986328125, + "pass@2": 0.7886968657726381, + "pass@4": 0.8542420943475817, + "pass@8": 0.9004456167198726, + "pass@16": 0.9312778575005979, + "pass@32": 0.9517112440915444, + "pass@64": 0.9667046571410877, + "pass@128": 0.978125 + }, + "5": { + "pass@1": 0.33509424603174603, + "pass@2": 0.41271325459317576, + "pass@4": 0.48771068527148376, + "pass@8": 0.5572081248784712, + "pass@16": 0.6188679542444174, + "pass@32": 0.6719682822822658, + "pass@64": 0.7166542430331456, + "pass@128": 0.7523809523809524 + }, + "6": { + "pass@1": 0.46206025449101795, + "pass@2": 0.6031718551086802, + "pass@4": 0.7298749273107324, + "pass@8": 0.8273623466954299, + "pass@16": 0.8904114372774126, + "pass@32": 0.9291838638462578, + "pass@64": 0.9545298379466033, + "pass@128": 0.9730538922155688 + }, + "7": { + "pass@1": 0.30755362426035504, + "pass@2": 0.44404614918697294, + "pass@4": 0.5866653091255597, + "pass@8": 0.7100621776567952, + "pass@16": 0.8009828251974904, + "pass@32": 0.8641776511072881, + "pass@64": 0.9077531903073257, + "pass@128": 0.9408284023668639 + }, + "8": { + "pass@1": 0.25122696314102566, + "pass@2": 0.38249427430345234, + "pass@4": 0.5251050217280533, + "pass@8": 0.6536996226393975, + "pass@16": 0.7536627127551332, + "pass@32": 0.826661161406808, + "pass@64": 0.8762717660769928, + "pass@128": 0.9038461538461539 + }, + "9": { + "pass@1": 0.2078125, + "pass@2": 0.3245858517245202, + "pass@4": 0.4576318187339257, + "pass@8": 0.5851793502978136, + "pass@16": 0.6946462264181785, + "pass@32": 0.7833343799733238, + "pass@64": 0.8508485859197652, + "pass@128": 0.8985915492957747 + }, + "11": { + "pass@1": 0.18229904390934845, + "pass@2": 0.2881442946844819, + "pass@4": 0.41277183172570875, + "pass@8": 0.5372213699972903, + "pass@16": 0.6464648060956772, + "pass@32": 0.7364996012096983, + "pass@64": 0.8090871748200775, + "pass@128": 0.8725212464589235 + }, + "12": { + "pass@1": 0.167041843220339, + "pass@2": 0.267159418234797, + "pass@4": 0.38463744918537746, + "pass@8": 0.5005959279433829, + "pass@16": 0.6027360469209624, + "pass@32": 0.6906821588303728, + "pass@64": 0.7683561968655983, + "pass@128": 0.8389830508474576 + }, + "13": { + "pass@1": 0.14863342476489028, + "pass@2": 0.24290428381013504, + "pass@4": 0.3593948659043012, + "pass@8": 0.4767867703415537, + "pass@16": 0.5776974942645328, + "pass@32": 0.6625444544056447, + "pass@64": 0.7355174465136388, + "pass@128": 0.7962382445141066 + }, + "14": { + "pass@1": 0.13086545950155765, + "pass@2": 0.21606400397380243, + "pass@4": 0.3265159209070825, + "pass@8": 0.44258768544823535, + "pass@16": 0.5412763209024235, + "pass@32": 0.6182166903432107, + "pass@64": 0.6805139060593539, + "pass@128": 0.735202492211838 + }, + "15": { + "pass@1": 0.11870032514450866, + "pass@2": 0.19539072811888392, + "pass@4": 0.2952038571948737, + "pass@8": 0.4034823819223066, + "pass@16": 0.4996538273439035, + "pass@32": 0.577105706738467, + "pass@64": 0.6430681278184355, + "pass@128": 0.7052023121387283 + }, + "16": { + "pass@1": 0.12497479838709677, + "pass@2": 0.20588566802133618, + "pass@4": 0.30794752833315187, + "pass@8": 0.41409910598717037, + "pass@16": 0.503439116374473, + "pass@32": 0.5680757189322889, + "pass@64": 0.6185146367153228, + "pass@128": 0.667741935483871 + }, + "17": { + "pass@1": 0.1048139450867052, + "pass@2": 0.1729335686814438, + "pass@4": 0.26166974377480257, + "pass@8": 0.3569233003143246, + "pass@16": 0.43881371763626015, + "pass@32": 0.5017114929543303, + "pass@64": 0.5540504002245086, + "pass@128": 0.6011560693641619 + }, + "18": { + "pass@1": 0.11404017857142858, + "pass@2": 0.1839486079865017, + "pass@4": 0.27364456023354217, + "pass@8": 0.3720505437784955, + "pass@16": 0.45863649753082086, + "pass@32": 0.5215180865558517, + "pass@64": 0.5696621746276215, + "pass@128": 0.62 + }, + "19": { + "pass@1": 0.10076482732732733, + "pass@2": 0.1690570984370197, + "pass@4": 0.2604623833619896, + "pass@8": 0.3599502461168844, + "pass@16": 0.44260688970777834, + "pass@32": 0.4944624925735973, + "pass@64": 0.521548379452954, + "pass@128": 0.5405405405405406 + }, + "20": { + "pass@1": 0.10095251501501501, + "pass@2": 0.16684992078692867, + "pass@4": 0.2509089626747106, + "pass@8": 0.3406114996432136, + "pass@16": 0.42006647369881606, + "pass@32": 0.47932235146072466, + "pass@64": 0.5187436798897297, + "pass@128": 0.5465465465465466 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 248622, + "answer_accuracy": 0.3027840023382697, + "avg_response_len": 254.0808444563523, + "resp_tokens_sum": 208630863, + "pass_at_k": { + "pass@1": 0.3027840023382697, + "pass@2": 0.39144283820524556, + "pass@4": 0.4912621577164558, + "pass@8": 0.5883824580273733, + "pass@16": 0.6688317187866314, + "pass@32": 0.7301793076629046, + "pass@64": 0.7783423771084799, + "pass@128": 0.8188620420888543 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.1796875, + "pass@2": 0.28581424206560085, + "pass@4": 0.41622926026117907, + "pass@8": 0.5547185574304757, + "pass@16": 0.6839554801528199, + "pass@32": 0.791310676081314, + "pass@64": 0.8702003501063394, + "pass@128": 0.9171779141104295 + }, + "2": { + "pass@1": 0.9739504179331308, + "pass@2": 0.9832606113730465, + "pass@4": 0.9877440161355209, + "pass@8": 0.9903351488073417, + "pass@16": 0.9921767708249385, + "pass@32": 0.9934576683623969, + "pass@64": 0.9939005764597241, + "pass@128": 0.993920972644377 + }, + "3": { + "pass@1": 0.9731060606060606, + "pass@2": 0.9891795663326179, + "pass@4": 0.9937926068900476, + "pass@8": 0.9955231703667864, + "pass@16": 0.9969587575218833, + "pass@32": 0.9984506286504, + "pass@64": 0.9996207784594809, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7112610946745562, + "pass@2": 0.8105136601360479, + "pass@4": 0.8784895620480868, + "pass@8": 0.9216631817833484, + "pass@16": 0.9464260283198788, + "pass@32": 0.9590859028287348, + "pass@64": 0.9663197612118946, + "pass@128": 0.9704142011834319 + }, + "5": { + "pass@1": 0.3811141304347826, + "pass@2": 0.4672704125299555, + "pass@4": 0.547470422990604, + "pass@8": 0.6177559162397293, + "pass@16": 0.6765391597013457, + "pass@32": 0.7272516871961043, + "pass@64": 0.7712716097104394, + "pass@128": 0.8115942028985508 + }, + "6": { + "pass@1": 0.41883058739255014, + "pass@2": 0.5677878866502717, + "pass@4": 0.7127793116118358, + "pass@8": 0.8278492564390045, + "pass@16": 0.8988014599895945, + "pass@32": 0.9373003380525965, + "pass@64": 0.9604309989332116, + "pass@128": 0.9742120343839542 + }, + "7": { + "pass@1": 0.33487626445086704, + "pass@2": 0.46564541042738106, + "pass@4": 0.5980823447466463, + "pass@8": 0.714922667139698, + "pass@16": 0.8033677613399027, + "pass@32": 0.8641866922325745, + "pass@64": 0.9019377879132182, + "pass@128": 0.9219653179190751 + }, + "8": { + "pass@1": 0.28347271126760565, + "pass@2": 0.4233832621714537, + "pass@4": 0.5744736294406855, + "pass@8": 0.7088275207620468, + "pass@16": 0.8087186748834445, + "pass@32": 0.8784898814053248, + "pass@64": 0.9269338685585622, + "pass@128": 0.9605633802816902 + }, + "9": { + "pass@1": 0.21727638473053892, + "pass@2": 0.33913595053986484, + "pass@4": 0.48270442579408074, + "pass@8": 0.6238536005858858, + "pass@16": 0.7419666380525494, + "pass@32": 0.8299157749956388, + "pass@64": 0.891739006586996, + "pass@128": 0.9341317365269461 + }, + "11": { + "pass@1": 0.1560617469879518, + "pass@2": 0.2487022400626127, + "pass@4": 0.363962768961109, + "pass@8": 0.4871624174058488, + "pass@16": 0.603968486823371, + "pass@32": 0.7099225827109938, + "pass@64": 0.7991877595200533, + "pass@128": 0.8644578313253012 + }, + "12": { + "pass@1": 0.18140184169278997, + "pass@2": 0.28670233937254724, + "pass@4": 0.41065066504226133, + "pass@8": 0.5328363958358436, + "pass@16": 0.6391938939640938, + "pass@32": 0.7286125927091673, + "pass@64": 0.8047119385141658, + "pass@128": 0.8714733542319749 + }, + "13": { + "pass@1": 0.14758403361344538, + "pass@2": 0.2368842221928142, + "pass@4": 0.34356088472134266, + "pass@8": 0.44992350437981904, + "pass@16": 0.5427658276100494, + "pass@32": 0.6231971069674507, + "pass@64": 0.6981882477135789, + "pass@128": 0.773109243697479 + }, + "14": { + "pass@1": 0.1457419590643275, + "pass@2": 0.2322633190588018, + "pass@4": 0.33790162002775964, + "pass@8": 0.44425079336957135, + "pass@16": 0.5325720105945908, + "pass@32": 0.6005776223976965, + "pass@64": 0.662597142298673, + "pass@128": 0.7309941520467836 + }, + "15": { + "pass@1": 0.11176609848484849, + "pass@2": 0.1877408434741111, + "pass@4": 0.28652309569258383, + "pass@8": 0.39240725982235936, + "pass@16": 0.4878854499058535, + "pass@32": 0.5707635144848856, + "pass@64": 0.6501004572511563, + "pass@128": 0.7272727272727273 + }, + "16": { + "pass@1": 0.1274928774928775, + "pass@2": 0.20607284866635253, + "pass@4": 0.302366788019019, + "pass@8": 0.39909323245321887, + "pass@16": 0.479867770840156, + "pass@32": 0.5436816319849745, + "pass@64": 0.5996621177657357, + "pass@128": 0.6552706552706553 + }, + "17": { + "pass@1": 0.1155861801242236, + "pass@2": 0.19869586614173215, + "pass@4": 0.31046421002498914, + "pass@8": 0.43139028333994395, + "pass@16": 0.5326025992065495, + "pass@32": 0.6042294785582262, + "pass@64": 0.6602624716039003, + "pass@128": 0.717391304347826 + }, + "18": { + "pass@1": 0.11557904411764706, + "pass@2": 0.19096840261695236, + "pass@4": 0.28819167300778564, + "pass@8": 0.39090138579917816, + "pass@16": 0.4763199553646875, + "pass@32": 0.5355086068492452, + "pass@64": 0.5739343318595364, + "pass@128": 0.6029411764705882 + }, + "19": { + "pass@1": 0.10209728422619048, + "pass@2": 0.1712254229940008, + "pass@4": 0.26240663471753534, + "pass@8": 0.3616555082558499, + "pass@16": 0.44495512112719054, + "pass@32": 0.5003037881586762, + "pass@64": 0.5392674835648679, + "pass@128": 0.5773809523809523 + }, + "20": { + "pass@1": 0.09110684880239521, + "pass@2": 0.15343316811259372, + "pass@4": 0.23748076793544526, + "pass@8": 0.33288966241110335, + "pass@16": 0.4179387610759886, + "pass@32": 0.4779039161554723, + "pass@64": 0.5203573952212737, + "pass@128": 0.5568862275449101 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 243966, + "answer_accuracy": 0.3045189926505832, + "avg_response_len": 267.7847087693721, + "resp_tokens_sum": 214536255, + "pass_at_k": { + "pass@1": 0.3045189926505832, + "pass@2": 0.3914387101471566, + "pass@4": 0.4888976512891585, + "pass@8": 0.5832940609223775, + "pass@16": 0.6619421974877205, + "pass@32": 0.7214642063143812, + "pass@64": 0.7664130005582344, + "pass@128": 0.8017255152580285 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.20205223880597015, + "pass@2": 0.3135319955341403, + "pass@4": 0.4415771505240951, + "pass@8": 0.569811918955316, + "pass@16": 0.6850570876181581, + "pass@32": 0.7808957342914687, + "pass@64": 0.8544946545097453, + "pass@128": 0.9044776119402985 + }, + "2": { + "pass@1": 0.9875892857142857, + "pass@2": 0.9917403683914513, + "pass@4": 0.9935481662113664, + "pass@8": 0.9943054704732391, + "pass@16": 0.9946362839316639, + "pass@32": 0.9949999953558449, + "pass@64": 0.9957142857142857, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9700482871720116, + "pass@2": 0.9877388174054772, + "pass@4": 0.994211654756712, + "pass@8": 0.9968066288226595, + "pass@16": 0.9979756293527906, + "pass@32": 0.9987791216280217, + "pass@64": 0.9996441771291553, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.7102065058479532, + "pass@2": 0.8100167063820963, + "pass@4": 0.8802413790929641, + "pass@8": 0.9272992602475084, + "pass@16": 0.9585762077324128, + "pass@32": 0.9767911949187461, + "pass@64": 0.9843987458071987, + "pass@128": 0.9853801169590644 + }, + "5": { + "pass@1": 0.41737132352941175, + "pass@2": 0.513387274201019, + "pass@4": 0.6009593699684594, + "pass@8": 0.6746747650464919, + "pass@16": 0.7341430684121006, + "pass@32": 0.7796575711960942, + "pass@64": 0.814747638086809, + "pass@128": 0.8411764705882353 + }, + "6": { + "pass@1": 0.3975749211356467, + "pass@2": 0.5503043574107656, + "pass@4": 0.6984590626881418, + "pass@8": 0.8139611823607342, + "pass@16": 0.887723354151534, + "pass@32": 0.9308254188372326, + "pass@64": 0.9556506522537759, + "pass@128": 0.9684542586750788 + }, + "7": { + "pass@1": 0.3171726661392405, + "pass@2": 0.46342187967208204, + "pass@4": 0.6122294229637435, + "pass@8": 0.7384669154900192, + "pass@16": 0.8279188406130379, + "pass@32": 0.8852386895678486, + "pass@64": 0.9234833724808077, + "pass@128": 0.9493670886075949 + }, + "8": { + "pass@1": 0.2432667042042042, + "pass@2": 0.3739248598992691, + "pass@4": 0.5204787287287282, + "pass@8": 0.6566691861840236, + "pass@16": 0.7674609415621451, + "pass@32": 0.8523845738554838, + "pass@64": 0.9119541812947833, + "pass@128": 0.9519519519519519 + }, + "9": { + "pass@1": 0.18521402733118972, + "pass@2": 0.28941596830138966, + "pass@4": 0.41672779638477636, + "pass@8": 0.5506735350456974, + "pass@16": 0.6746045152270984, + "pass@32": 0.7816134842691489, + "pass@64": 0.8652622165514577, + "pass@128": 0.9163987138263665 + }, + "11": { + "pass@1": 0.17552083333333332, + "pass@2": 0.2797076146731657, + "pass@4": 0.40224519435070627, + "pass@8": 0.5271396689310396, + "pass@16": 0.644746749599154, + "pass@32": 0.7499209088304015, + "pass@64": 0.8377807881434293, + "pass@128": 0.9047619047619048 + }, + "12": { + "pass@1": 0.15553325688073394, + "pass@2": 0.24843031375665203, + "pass@4": 0.3597259003748386, + "pass@8": 0.47338739712545613, + "pass@16": 0.576015274972535, + "pass@32": 0.664281855201088, + "pass@64": 0.7428282365695322, + "pass@128": 0.8134556574923547 + }, + "13": { + "pass@1": 0.14749710648148148, + "pass@2": 0.2358992235345583, + "pass@4": 0.3428141392626847, + "pass@8": 0.4519424575970455, + "pass@16": 0.5471952037565342, + "pass@32": 0.6258993050569552, + "pass@64": 0.6955199471953205, + "pass@128": 0.7623456790123457 + }, + "14": { + "pass@1": 0.13357752225519287, + "pass@2": 0.21566179992523188, + "pass@4": 0.3157897633901105, + "pass@8": 0.41891290583411983, + "pass@16": 0.510946115208545, + "pass@32": 0.5898814297305477, + "pass@64": 0.6612483786173892, + "pass@128": 0.7240356083086054 + }, + "15": { + "pass@1": 0.10616801697530864, + "pass@2": 0.18171486159716133, + "pass@4": 0.2844029273539879, + "pass@8": 0.3966455323521176, + "pass@16": 0.4939536315132476, + "pass@32": 0.5693545900367545, + "pass@64": 0.6337045083992685, + "pass@128": 0.691358024691358 + }, + "16": { + "pass@1": 0.12898691002949852, + "pass@2": 0.21023106694074747, + "pass@4": 0.3090409769022235, + "pass@8": 0.40610456188689503, + "pass@16": 0.4821112975879393, + "pass@32": 0.5350102394509227, + "pass@64": 0.57746091806195, + "pass@128": 0.6224188790560472 + }, + "17": { + "pass@1": 0.1130929969879518, + "pass@2": 0.18889633336495587, + "pass@4": 0.28544219764321627, + "pass@8": 0.38457376013687594, + "pass@16": 0.46493667993499693, + "pass@32": 0.5194219776626258, + "pass@64": 0.5543655991302966, + "pass@128": 0.5813253012048193 + }, + "18": { + "pass@1": 0.10257056451612903, + "pass@2": 0.16746570993141993, + "pass@4": 0.250047667428668, + "pass@8": 0.33849332443951263, + "pass@16": 0.41364696069505363, + "pass@32": 0.4656988345709459, + "pass@64": 0.5029973247060224, + "pass@128": 0.5387096774193548 + }, + "19": { + "pass@1": 0.0993202416918429, + "pass@2": 0.16870480172229227, + "pass@4": 0.2604445992589294, + "pass@8": 0.3578538662514504, + "pass@16": 0.4353394819087087, + "pass@32": 0.47947822088860853, + "pass@64": 0.5000096375040654, + "pass@128": 0.513595166163142 + }, + "20": { + "pass@1": 0.09295232732732733, + "pass@2": 0.15922049017521456, + "pass@4": 0.24903409427425166, + "pass@8": 0.3498747830497944, + "pass@16": 0.43915606624257963, + "pass@32": 0.4983363412761255, + "pass@64": 0.5302439838985393, + "pass@128": 0.5525525525525525 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/checkpoint-85656_metrics.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/checkpoint-85656_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..04dc365c03cebf90ce8f9339bc8559fbe177964b --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/checkpoint-85656_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85656", + "id": { + "answer_accuracy": 0.9933975694444445, + "avg_response_len": 197.00196440972223, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.9933975694444445, + "pass@2": 0.9948879866579177, + "pass@4": 0.9958351290359537, + "pass@8": 0.996458587386234, + "pass@16": 0.9969074769932769, + "pass@32": 0.997269549362243, + "pass@64": 0.9975768415808485, + "pass@128": 0.9978888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.95190625, + "pass@2": 0.9614729330708662, + "pass@4": 0.9679910248406451, + "pass@8": 0.9721753907797562, + "pass@16": 0.9749541976253611, + "pass@32": 0.9772179007398849, + "pass@64": 0.9794459979259951, + "pass@128": 0.982 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997796875, + "pass@2": 0.998189345472441, + "pass@4": 0.9984128804836896, + "pass@8": 0.9987078420980413, + "pass@16": 0.9990747511343737, + "pass@32": 0.9994199109897796, + "pass@64": 0.9997519064525439, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998671875, + "pass@2": 0.999990280511811, + "pass@4": 0.9999999329771279, + "pass@8": 0.9999999999990998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995859375, + "pass@2": 0.9999505413385827, + "pass@4": 0.999998551743532, + "pass@8": 0.9999999989069265, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963515625, + "pass@2": 0.9979307332677166, + "pass@4": 0.9988394668166479, + "pass@8": 0.9994306833527995, + "pass@16": 0.999803536762109, + "pass@32": 0.9999736270683416, + "pass@64": 0.9999998615776508, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950703125, + "pass@2": 0.9964580462598425, + "pass@4": 0.9972743044619421, + "pass@8": 0.9978133713394822, + "pass@16": 0.9983348074176456, + "pass@32": 0.9988145054621784, + "pass@64": 0.9989938082714463, + "pass@128": 0.999 + } + }, + "per_op_accuracy": { + "10": 0.95190625, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.997796875, + "6": 0.9998671875, + "7": 0.9995859375, + "8": 0.9963515625, + "9": 0.9950703125 + }, + "per_op_avg_response_len": { + "10": 276.1677265625, + "2": 114.5604296875, + "3": 142.8816328125, + "4": 157.92240625, + "5": 170.412265625, + "6": 196.033046875, + "7": 220.4257265625, + "8": 239.5748359375, + "9": 255.039609375 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.1677265625, + "2": 114.5604296875, + "3": 142.8816328125, + "4": 157.92240625, + "5": 170.412265625, + "6": 196.033046875, + "7": 220.4257265625, + "8": 239.5748359375, + "9": 255.039609375 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 376885, + "answer_accuracy": 0.9943985351232691, + "avg_response_len": 187.54930238939548, + "resp_tokens_sum": 71082686, + "pass_at_k": { + "pass@1": 0.9943985351232691, + "pass@2": 0.9956074307200963, + "pass@4": 0.9963501826392775, + "pass@8": 0.9968237532896089, + "pass@16": 0.997181706397073, + "pass@32": 0.9974756671773004, + "pass@64": 0.9977194959490809, + "pass@128": 0.9979736575481256 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9612140486725663, + "pass@2": 0.9701686729147794, + "pass@4": 0.97611373799514, + "pass@8": 0.9795318507801245, + "pass@16": 0.9815704330658933, + "pass@32": 0.9828295616605529, + "pass@64": 0.9837715482760134, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966269841269841, + "pass@2": 0.9969132764654418, + "pass@4": 0.9970214452360122, + "pass@8": 0.9972112860892366, + "pass@16": 0.9975721784776902, + "pass@32": 0.9982189726284215, + "pass@64": 0.9992125984251968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993296967455622, + "pass@2": 0.9998813364860458, + "pass@4": 0.9999958538836491, + "pass@8": 0.9999999967670793, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997495993589743, + "pass@2": 0.9999948736876642, + "pass@4": 0.999999998497784, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941901408450704, + "pass@2": 0.9947054868581569, + "pass@4": 0.9950140287921757, + "pass@8": 0.9955275550723595, + "pass@16": 0.996246252427952, + "pass@32": 0.9969218386791731, + "pass@64": 0.9971743778471076, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 387848, + "answer_accuracy": 0.9928120904325033, + "avg_response_len": 194.97932452080602, + "resp_tokens_sum": 76169843, + "pass_at_k": { + "pass@1": 0.9928120904325033, + "pass@2": 0.994320798417973, + "pass@4": 0.9952557982287765, + "pass@8": 0.9958411640141788, + "pass@16": 0.996245046161337, + "pass@32": 0.9965925432643324, + "pass@64": 0.9968830242222334, + "pass@128": 0.997051114023591 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9406873082822086, + "pass@2": 0.9511727241437608, + "pass@4": 0.9582467286489494, + "pass@8": 0.962884319554284, + "pass@16": 0.9658664939297285, + "pass@32": 0.9683840162934146, + "pass@64": 0.9708284840200408, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996866045845272, + "pass@2": 0.9999725029894185, + "pass@4": 0.9999998079573864, + "pass@8": 0.9999999999974207, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996387283236994, + "pass@2": 0.9999765315643349, + "pass@4": 0.9999998658946533, + "pass@8": 0.9999999999989994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976012323943662, + "pass@2": 0.9993276588665854, + "pass@4": 0.9999400980687273, + "pass@8": 0.9999995286192314, + "pass@16": 0.9999999999862863, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954622005988024, + "pass@2": 0.996530438139469, + "pass@4": 0.9974659918071619, + "pass@8": 0.9982248854431038, + "pass@16": 0.9990042031981354, + "pass@32": 0.9997223135661357, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 379661, + "answer_accuracy": 0.9930035361566789, + "avg_response_len": 208.43900129728826, + "resp_tokens_sum": 79693734, + "pass_at_k": { + "pass@1": 0.9930035361566789, + "pass@2": 0.994754335717242, + "pass@4": 0.9959164962619546, + "pass@8": 0.9967274591276531, + "pass@16": 0.9973124805535174, + "pass@32": 0.9977569640794962, + "pass@64": 0.9981443444245567, + "pass@128": 0.9986608637428859 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9534048507462687, + "pass@2": 0.9626968503937007, + "pass@4": 0.9692538331962235, + "pass@8": 0.9737725528077535, + "pass@16": 0.9771025187908413, + "pass@32": 0.9801357911203106, + "pass@64": 0.9834547978205753, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966452205882353, + "pass@2": 0.9975343040759611, + "pass@4": 0.9980915448068992, + "pass@8": 0.9987831969997992, + "pass@16": 0.9995279850408865, + "pass@32": 0.9999439253289022, + "pass@64": 0.9999998174959024, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.999996106598226, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918355855855856, + "pass@2": 0.9945075488874701, + "pass@4": 0.9965787761890124, + "pass@8": 0.998290842921839, + "pass@16": 0.9994100203212533, + "pass@32": 0.9999208020070317, + "pass@64": 0.99999958431727, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956541398713826, + "pass@2": 0.9983808105678912, + "pass@4": 0.999648498318739, + "pass@8": 0.9999806352115692, + "pass@16": 0.9999999483522362, + "pass@32": 0.9999999999999448, + "pass@64": 1.0, + "pass@128": 1.0 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.48817109375, + "avg_response_len": 334.2508890625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.48817109375, + "pass@2": 0.5513962475393651, + "pass@4": 0.6065081957442733, + "pass@8": 0.6533290306899502, + "pass@16": 0.691861134708799, + "pass@32": 0.7241844942083255, + "pass@64": 0.7527495474788342, + "pass@128": 0.779 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9364765625, + "pass@2": 0.9481942667322834, + "pass@4": 0.9557262470941131, + "pass@8": 0.9609822801801745, + "pass@16": 0.9651884055858303, + "pass@32": 0.9691439571354269, + "pass@64": 0.972584853801251, + "pass@128": 0.974 + }, + "12": { + "pass@1": 0.849703125, + "pass@2": 0.8770287893700794, + "pass@4": 0.8946197364079495, + "pass@8": 0.9078932332798402, + "pass@16": 0.9189775197403667, + "pass@32": 0.9293363953132724, + "pass@64": 0.9391381770612707, + "pass@128": 0.949 + }, + "13": { + "pass@1": 0.7045625, + "pass@2": 0.7599717027559058, + "pass@4": 0.7976920028121491, + "pass@8": 0.8256908432136142, + "pass@16": 0.8478287269163802, + "pass@32": 0.8664857859411831, + "pass@64": 0.8833490874573225, + "pass@128": 0.9 + }, + "14": { + "pass@1": 0.669171875, + "pass@2": 0.7423502706692918, + "pass@4": 0.7949840661792276, + "pass@8": 0.8312132024876353, + "pass@16": 0.8559843518667011, + "pass@32": 0.8741379967011483, + "pass@64": 0.8884107001878003, + "pass@128": 0.9 + }, + "15": { + "pass@1": 0.5416640625, + "pass@2": 0.630910556102364, + "pass@4": 0.7019842690288716, + "pass@8": 0.7580656245321791, + "pass@16": 0.8010678139371163, + "pass@32": 0.834215706568113, + "pass@64": 0.8603956848455061, + "pass@128": 0.881 + }, + "16": { + "pass@1": 0.3383515625, + "pass@2": 0.42554552165354337, + "pass@4": 0.5022154736595419, + "pass@8": 0.5662979914119732, + "pass@16": 0.6184983327775875, + "pass@32": 0.6627077013391794, + "pass@64": 0.7018137813814018, + "pass@128": 0.736 + }, + "17": { + "pass@1": 0.2721171875, + "pass@2": 0.35760986712598475, + "pass@4": 0.4365985718972629, + "pass@8": 0.502066813437018, + "pass@16": 0.5550240476475563, + "pass@32": 0.6023919230168082, + "pass@64": 0.6485056246095271, + "pass@128": 0.692 + }, + "18": { + "pass@1": 0.236171875, + "pass@2": 0.31101574803149645, + "pass@4": 0.38453675609298826, + "pass@8": 0.4525474699368694, + "pass@16": 0.5103961887579852, + "pass@32": 0.5584121243707435, + "pass@64": 0.6022721866905326, + "pass@128": 0.647 + }, + "19": { + "pass@1": 0.176421875, + "pass@2": 0.24365317421259833, + "pass@4": 0.31328017444694395, + "pass@8": 0.37901701514885006, + "pass@16": 0.436620942097062, + "pass@32": 0.4851708072028206, + "pass@64": 0.5268048195838971, + "pass@128": 0.567 + }, + "20": { + "pass@1": 0.1570703125, + "pass@2": 0.21768257874015734, + "pass@4": 0.28344465982377176, + "pass@8": 0.3495158332713638, + "pass@16": 0.40902501776131367, + "pass@32": 0.4598425444945442, + "pass@64": 0.5042205591697556, + "pass@128": 0.544 + } + }, + "per_op_accuracy": { + "11": 0.9364765625, + "12": 0.849703125, + "13": 0.7045625, + "14": 0.669171875, + "15": 0.5416640625, + "16": 0.3383515625, + "17": 0.2721171875, + "18": 0.236171875, + "19": 0.176421875, + "20": 0.1570703125 + }, + "per_op_avg_response_len": { + "11": 292.843015625, + "12": 301.91553125, + "13": 317.6629765625, + "14": 338.810421875, + "15": 347.1005859375, + "16": 346.8352421875, + "17": 353.3876484375, + "18": 349.18034375, + "19": 348.2933828125, + "20": 346.4797421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 292.843015625, + "12": 301.91553125, + "13": 317.6629765625, + "14": 338.810421875, + "15": 347.1005859375, + "16": 346.8352421875, + "17": 353.3876484375, + "18": 349.18034375, + "19": 348.2933828125, + "20": 346.4797421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 209535, + "answer_accuracy": 0.48647613298662706, + "avg_response_len": 317.56532085809806, + "resp_tokens_sum": 136781735, + "pass_at_k": { + "pass@1": 0.48647613298662706, + "pass@2": 0.5500108955084195, + "pass@4": 0.6050688621760336, + "pass@8": 0.6514354275075487, + "pass@16": 0.6895260568716199, + "pass@32": 0.7216144599329469, + "pass@64": 0.7504924816052639, + "pass@128": 0.7777117384843982 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9172716005665722, + "pass@2": 0.9294245332470837, + "pass@4": 0.9375553278927952, + "pass@8": 0.9436644091134152, + "pass@16": 0.94896922068123, + "pass@32": 0.9540347827409994, + "pass@64": 0.9584288926634769, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.8369085451977402, + "pass@2": 0.8658925413719473, + "pass@4": 0.8836768536877804, + "pass@8": 0.8958159717383948, + "pass@16": 0.9059099640119125, + "pass@32": 0.9152839590889877, + "pass@64": 0.9230974982514136, + "pass@128": 0.9293785310734464 + }, + "13": { + "pass@1": 0.7135579937304075, + "pass@2": 0.7642971083355962, + "pass@4": 0.7975250877731188, + "pass@8": 0.8216605066521206, + "pass@16": 0.8408669871052371, + "pass@32": 0.8585289603051615, + "pass@64": 0.8786083453443386, + "pass@128": 0.9028213166144201 + }, + "14": { + "pass@1": 0.6607038551401869, + "pass@2": 0.7342098081781829, + "pass@4": 0.7866001411038571, + "pass@8": 0.8223529852870952, + "pass@16": 0.8463869020391322, + "pass@32": 0.8634275011957604, + "pass@64": 0.8774430319994344, + "pass@128": 0.8909657320872274 + }, + "15": { + "pass@1": 0.5573744580924855, + "pass@2": 0.6510904999772427, + "pass@4": 0.723555908654481, + "pass@8": 0.779664294770354, + "pass@16": 0.8218733604726072, + "pass@32": 0.8537991239873666, + "pass@64": 0.8786629030021994, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.32026209677419354, + "pass@2": 0.4102227266954534, + "pass@4": 0.4922931598267957, + "pass@8": 0.5608738002824728, + "pass@16": 0.6161741026713436, + "pass@32": 0.6634223589324809, + "pass@64": 0.7052034534598515, + "pass@128": 0.7419354838709677 + }, + "17": { + "pass@1": 0.26948609104046245, + "pass@2": 0.3551862398161212, + "pass@4": 0.43630361992323174, + "pass@8": 0.505574472344914, + "pass@16": 0.5626248447366223, + "pass@32": 0.612875860834388, + "pass@64": 0.6613171736457973, + "pass@128": 0.7052023121387283 + }, + "18": { + "pass@1": 0.22223214285714285, + "pass@2": 0.29839848143981995, + "pass@4": 0.37236364186619514, + "pass@8": 0.43890352397685345, + "pass@16": 0.49460411827912604, + "pass@32": 0.5408992361679525, + "pass@64": 0.5819298205174266, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.1733765015015015, + "pass@2": 0.24151267409141433, + "pass@4": 0.3106473104150269, + "pass@8": 0.3760434973464308, + "pass@16": 0.4339640400717074, + "pass@32": 0.48326526375921236, + "pass@64": 0.5279349721453175, + "pass@128": 0.5735735735735735 + }, + "20": { + "pass@1": 0.16915352852852852, + "pass@2": 0.22966285675439213, + "pass@4": 0.29331480636992446, + "pass@8": 0.3554372484491605, + "pass@16": 0.4113463579552269, + "pass@32": 0.4595926430033481, + "pass@64": 0.5028409960228297, + "pass@128": 0.5435435435435435 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 213875, + "answer_accuracy": 0.49684758771929827, + "avg_response_len": 331.1756383809099, + "resp_tokens_sum": 142559190, + "pass_at_k": { + "pass@1": 0.49684758771929827, + "pass@2": 0.5606803557589438, + "pass@4": 0.6156620822988135, + "pass@8": 0.661460716590065, + "pass@16": 0.6980869573817106, + "pass@32": 0.7283111283109683, + "pass@64": 0.755055709302105, + "pass@128": 0.7802557240559025 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9490540286144579, + "pass@2": 0.9586703111659238, + "pass@4": 0.963866821334833, + "pass@8": 0.9669982809187331, + "pass@16": 0.969628195342412, + "pass@32": 0.9726172988672541, + "pass@64": 0.9751752015368585, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.8521012931034483, + "pass@2": 0.8769631044356135, + "pass@4": 0.893521940635163, + "pass@8": 0.9075129211754909, + "pass@16": 0.919181931341527, + "pass@32": 0.9290936707076372, + "pass@64": 0.9385951394414912, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.7197785364145658, + "pass@2": 0.7765865204349457, + "pass@4": 0.8138981214533052, + "pass@8": 0.840127920923921, + "pass@16": 0.8606898149555469, + "pass@32": 0.8784119819890157, + "pass@64": 0.8928222471865029, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.6990131578947368, + "pass@2": 0.7748933727034123, + "pass@4": 0.8276604991262051, + "pass@8": 0.8608615541084115, + "pass@16": 0.8813606703486647, + "pass@32": 0.8960207950792911, + "pass@64": 0.9067741271436445, + "pass@128": 0.9152046783625731 + }, + "15": { + "pass@1": 0.5314867424242424, + "pass@2": 0.6243128877356237, + "pass@4": 0.6995647515651452, + "pass@8": 0.75847718932693, + "pass@16": 0.8041517108171061, + "pass@32": 0.8403438310227098, + "pass@64": 0.8678819774825477, + "pass@128": 0.8878787878787879 + }, + "16": { + "pass@1": 0.3511173433048433, + "pass@2": 0.4367120375978642, + "pass@4": 0.509192721636291, + "pass@8": 0.567463683171276, + "pass@16": 0.6138049647381483, + "pass@32": 0.6543388540897418, + "pass@64": 0.6920704030785279, + "pass@128": 0.7236467236467237 + }, + "17": { + "pass@1": 0.2860539596273292, + "pass@2": 0.3759124199149017, + "pass@4": 0.4613117808061256, + "pass@8": 0.5321531449577279, + "pass@16": 0.5842617561652651, + "pass@32": 0.6244032611091793, + "pass@64": 0.6603615858501944, + "pass@128": 0.6987577639751553 + }, + "18": { + "pass@1": 0.2511948529411765, + "pass@2": 0.3300544233441406, + "pass@4": 0.40649941000022055, + "pass@8": 0.4779186811576755, + "pass@16": 0.5402423482438927, + "pass@32": 0.5939876301314176, + "pass@64": 0.6458831650585033, + "pass@128": 0.6970588235294117 + }, + "19": { + "pass@1": 0.18038504464285715, + "pass@2": 0.24445003456599168, + "pass@4": 0.31143215635991933, + "pass@8": 0.3728059388521139, + "pass@16": 0.42353768253638485, + "pass@32": 0.4649215111100438, + "pass@64": 0.500723873663761, + "pass@128": 0.5386904761904762 + }, + "20": { + "pass@1": 0.1533261601796407, + "pass@2": 0.21130733120373435, + "pass@4": 0.2719651745440502, + "pass@8": 0.332414175156139, + "pass@16": 0.3859289673999605, + "pass@32": 0.4304629843267275, + "pass@64": 0.4713483469991754, + "pass@128": 0.5119760479041916 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 201449, + "answer_accuracy": 0.4809964280256724, + "avg_response_len": 354.5714896279034, + "resp_tokens_sum": 148500213, + "pass_at_k": { + "pass@1": 0.4809964280256724, + "pass@2": 0.5432786600215638, + "pass@4": 0.5985799674356888, + "pass@8": 0.6469186135220798, + "pass@16": 0.6878636088142506, + "pass@32": 0.7225861735632851, + "pass@64": 0.7527004657101296, + "pass@128": 0.7790342298288508 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.9447420634920635, + "pass@2": 0.9581868672665916, + "pass@4": 0.967509306872355, + "pass@8": 0.9740486174543461, + "pass@16": 0.9786847931164305, + "pass@32": 0.9824150336632566, + "pass@64": 0.9857183739073222, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8612146406727829, + "pass@2": 0.8891486220472444, + "pass@4": 0.9075370982870257, + "pass@8": 0.921338707735371, + "pass@16": 0.9329246372544424, + "pass@32": 0.9447859108257943, + "pass@64": 0.9570330678851205, + "pass@128": 0.9694189602446484 + }, + "13": { + "pass@1": 0.6789400077160493, + "pass@2": 0.73740597963449, + "pass@4": 0.7799996001657193, + "pass@8": 0.8137514624128047, + "pass@16": 0.8405120187983943, + "pass@32": 0.8611789198572768, + "pass@64": 0.8775786513793102, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.6469538204747775, + "pass@2": 0.7170783049837612, + "pass@4": 0.7698086949072013, + "pass@8": 0.8095645184136512, + "pass@16": 0.8393733146970238, + "pass@32": 0.8621325130569488, + "pass@64": 0.8802217668630712, + "pass@128": 0.8931750741839762 + }, + "15": { + "pass@1": 0.5352527006172839, + "pass@2": 0.6160802165354328, + "pass@4": 0.6814122117837121, + "pass@8": 0.734581191554782, + "pass@16": 0.7757085390245353, + "pass@32": 0.8070609426882411, + "pass@64": 0.8332631723379745, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.3416758849557522, + "pass@2": 0.42799572619794185, + "pass@4": 0.5040647457784591, + "pass@8": 0.5700512110067496, + "pass@16": 0.6254832399008291, + "pass@32": 0.6707192692761402, + "pass@64": 0.7088023579005431, + "pass@128": 0.7433628318584071 + }, + "17": { + "pass@1": 0.2613422439759036, + "pass@2": 0.3423844245090598, + "pass@4": 0.41293712645858993, + "pass@8": 0.46923112448581195, + "pass@16": 0.5187456803719004, + "pass@32": 0.5701175454546831, + "pass@64": 0.6236549755551729, + "pass@128": 0.6716867469879518 + }, + "18": { + "pass@1": 0.2354334677419355, + "pass@2": 0.3043799212598424, + "pass@4": 0.37419252270885495, + "pass@8": 0.4401254353269728, + "pass@16": 0.4954914482495745, + "pass@32": 0.5391664434428325, + "pass@64": 0.5774079786759414, + "pass@128": 0.6193548387096774 + }, + "19": { + "pass@1": 0.17546261329305135, + "pass@2": 0.24499771035040552, + "pass@4": 0.31780488079095853, + "pass@8": 0.38831339902772993, + "pass@16": 0.45257478979141497, + "pass@32": 0.5076430412025037, + "pass@64": 0.5521427562189815, + "pass@128": 0.5891238670694864 + }, + "20": { + "pass@1": 0.1487424924924925, + "pass@2": 0.2120966931498427, + "pass@4": 0.28508847148610916, + "pass@8": 0.3607474325394686, + "pass@16": 0.42986908543734814, + "pass@32": 0.48956023306096935, + "pass@64": 0.5385710498391255, + "pass@128": 0.5765765765765766 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.7274888980263158, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.7274888980263158, + "pass@2": 0.7614712818586641, + "pass@4": 0.7909262167771619, + "pass@8": 0.8158640838618849, + "pass@16": 0.8363567705277731, + "pass@32": 0.8535405729654564, + "pass@64": 0.8687203710008377, + "pass@128": 0.8826842105263157 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.95190625, + "pass@2": 0.9614729330708662, + "pass@4": 0.9679910248406451, + "pass@8": 0.9721753907797562, + "pass@16": 0.9749541976253611, + "pass@32": 0.9772179007398849, + "pass@64": 0.9794459979259951, + "pass@128": 0.982 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.997796875, + "pass@2": 0.998189345472441, + "pass@4": 0.9984128804836896, + "pass@8": 0.9987078420980413, + "pass@16": 0.9990747511343737, + "pass@32": 0.9994199109897796, + "pass@64": 0.9997519064525439, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9998671875, + "pass@2": 0.999990280511811, + "pass@4": 0.9999999329771279, + "pass@8": 0.9999999999990998, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9995859375, + "pass@2": 0.9999505413385827, + "pass@4": 0.999998551743532, + "pass@8": 0.9999999989069265, + "pass@16": 0.9999999999999999, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9963515625, + "pass@2": 0.9979307332677166, + "pass@4": 0.9988394668166479, + "pass@8": 0.9994306833527995, + "pass@16": 0.999803536762109, + "pass@32": 0.9999736270683416, + "pass@64": 0.9999998615776508, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9950703125, + "pass@2": 0.9964580462598425, + "pass@4": 0.9972743044619421, + "pass@8": 0.9978133713394822, + "pass@16": 0.9983348074176456, + "pass@32": 0.9988145054621784, + "pass@64": 0.9989938082714463, + "pass@128": 0.999 + }, + "11": { + "pass@1": 0.9364765625, + "pass@2": 0.9481942667322834, + "pass@4": 0.9557262470941131, + "pass@8": 0.9609822801801745, + "pass@16": 0.9651884055858303, + "pass@32": 0.9691439571354269, + "pass@64": 0.972584853801251, + "pass@128": 0.974 + }, + "12": { + "pass@1": 0.849703125, + "pass@2": 0.8770287893700794, + "pass@4": 0.8946197364079495, + "pass@8": 0.9078932332798402, + "pass@16": 0.9189775197403667, + "pass@32": 0.9293363953132724, + "pass@64": 0.9391381770612707, + "pass@128": 0.949 + }, + "13": { + "pass@1": 0.7045625, + "pass@2": 0.7599717027559058, + "pass@4": 0.7976920028121491, + "pass@8": 0.8256908432136142, + "pass@16": 0.8478287269163802, + "pass@32": 0.8664857859411831, + "pass@64": 0.8833490874573225, + "pass@128": 0.9 + }, + "14": { + "pass@1": 0.669171875, + "pass@2": 0.7423502706692918, + "pass@4": 0.7949840661792276, + "pass@8": 0.8312132024876353, + "pass@16": 0.8559843518667011, + "pass@32": 0.8741379967011483, + "pass@64": 0.8884107001878003, + "pass@128": 0.9 + }, + "15": { + "pass@1": 0.5416640625, + "pass@2": 0.630910556102364, + "pass@4": 0.7019842690288716, + "pass@8": 0.7580656245321791, + "pass@16": 0.8010678139371163, + "pass@32": 0.834215706568113, + "pass@64": 0.8603956848455061, + "pass@128": 0.881 + }, + "16": { + "pass@1": 0.3383515625, + "pass@2": 0.42554552165354337, + "pass@4": 0.5022154736595419, + "pass@8": 0.5662979914119732, + "pass@16": 0.6184983327775875, + "pass@32": 0.6627077013391794, + "pass@64": 0.7018137813814018, + "pass@128": 0.736 + }, + "17": { + "pass@1": 0.2721171875, + "pass@2": 0.35760986712598475, + "pass@4": 0.4365985718972629, + "pass@8": 0.502066813437018, + "pass@16": 0.5550240476475563, + "pass@32": 0.6023919230168082, + "pass@64": 0.6485056246095271, + "pass@128": 0.692 + }, + "18": { + "pass@1": 0.236171875, + "pass@2": 0.31101574803149645, + "pass@4": 0.38453675609298826, + "pass@8": 0.4525474699368694, + "pass@16": 0.5103961887579852, + "pass@32": 0.5584121243707435, + "pass@64": 0.6022721866905326, + "pass@128": 0.647 + }, + "19": { + "pass@1": 0.176421875, + "pass@2": 0.24365317421259833, + "pass@4": 0.31328017444694395, + "pass@8": 0.37901701514885006, + "pass@16": 0.436620942097062, + "pass@32": 0.4851708072028206, + "pass@64": 0.5268048195838971, + "pass@128": 0.567 + }, + "20": { + "pass@1": 0.1570703125, + "pass@2": 0.21768257874015734, + "pass@4": 0.28344465982377176, + "pass@8": 0.3495158332713638, + "pass@16": 0.40902501776131367, + "pass@32": 0.4598425444945442, + "pass@64": 0.5042205591697556, + "pass@128": 0.544 + } + }, + "per_op_accuracy": { + "10": 0.95190625, + "2": 1.0, + "3": 1.0, + "4": 1.0, + "5": 0.997796875, + "6": 0.9998671875, + "7": 0.9995859375, + "8": 0.9963515625, + "9": 0.9950703125, + "11": 0.9364765625, + "12": 0.849703125, + "13": 0.7045625, + "14": 0.669171875, + "15": 0.5416640625, + "16": 0.3383515625, + "17": 0.2721171875, + "18": 0.236171875, + "19": 0.176421875, + "20": 0.1570703125 + }, + "per_op_avg_response_len": { + "10": 276.1677265625, + "2": 114.5604296875, + "3": 142.8816328125, + "4": 157.92240625, + "5": 170.412265625, + "6": 196.033046875, + "7": 220.4257265625, + "8": 239.5748359375, + "9": 255.039609375, + "11": 292.843015625, + "12": 301.91553125, + "13": 317.6629765625, + "14": 338.810421875, + "15": 347.1005859375, + "16": 346.8352421875, + "17": 353.3876484375, + "18": 349.18034375, + "19": 348.2933828125, + "20": 346.4797421875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.1677265625, + "2": 114.5604296875, + "3": 142.8816328125, + "4": 157.92240625, + "5": 170.412265625, + "6": 196.033046875, + "7": 220.4257265625, + "8": 239.5748359375, + "9": 255.039609375, + "11": 292.843015625, + "12": 301.91553125, + "13": 317.6629765625, + "14": 338.810421875, + "15": 347.1005859375, + "16": 346.8352421875, + "17": 353.3876484375, + "18": 349.18034375, + "19": 348.2933828125, + "20": 346.4797421875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 586420, + "answer_accuracy": 0.7242185030034777, + "avg_response_len": 256.7089454730477, + "resp_tokens_sum": 207864421, + "pass_at_k": { + "pass@1": 0.7242185030034777, + "pass@2": 0.7585805035959495, + "pass@4": 0.7882152405970935, + "pass@8": 0.8131007504036403, + "pass@16": 0.8335299105303158, + "pass@32": 0.8507363433743848, + "pass@64": 0.8662116073517198, + "pass@128": 0.8808093582042364 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9612140486725663, + "pass@2": 0.9701686729147794, + "pass@4": 0.97611373799514, + "pass@8": 0.9795318507801245, + "pass@16": 0.9815704330658933, + "pass@32": 0.9828295616605529, + "pass@64": 0.9837715482760134, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966269841269841, + "pass@2": 0.9969132764654418, + "pass@4": 0.9970214452360122, + "pass@8": 0.9972112860892366, + "pass@16": 0.9975721784776902, + "pass@32": 0.9982189726284215, + "pass@64": 0.9992125984251968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993296967455622, + "pass@2": 0.9998813364860458, + "pass@4": 0.9999958538836491, + "pass@8": 0.9999999967670793, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997495993589743, + "pass@2": 0.9999948736876642, + "pass@4": 0.999999998497784, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941901408450704, + "pass@2": 0.9947054868581569, + "pass@4": 0.9950140287921757, + "pass@8": 0.9955275550723595, + "pass@16": 0.996246252427952, + "pass@32": 0.9969218386791731, + "pass@64": 0.9971743778471076, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9172716005665722, + "pass@2": 0.9294245332470837, + "pass@4": 0.9375553278927952, + "pass@8": 0.9436644091134152, + "pass@16": 0.94896922068123, + "pass@32": 0.9540347827409994, + "pass@64": 0.9584288926634769, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.8369085451977402, + "pass@2": 0.8658925413719473, + "pass@4": 0.8836768536877804, + "pass@8": 0.8958159717383948, + "pass@16": 0.9059099640119125, + "pass@32": 0.9152839590889877, + "pass@64": 0.9230974982514136, + "pass@128": 0.9293785310734464 + }, + "13": { + "pass@1": 0.7135579937304075, + "pass@2": 0.7642971083355962, + "pass@4": 0.7975250877731188, + "pass@8": 0.8216605066521206, + "pass@16": 0.8408669871052371, + "pass@32": 0.8585289603051615, + "pass@64": 0.8786083453443386, + "pass@128": 0.9028213166144201 + }, + "14": { + "pass@1": 0.6607038551401869, + "pass@2": 0.7342098081781829, + "pass@4": 0.7866001411038571, + "pass@8": 0.8223529852870952, + "pass@16": 0.8463869020391322, + "pass@32": 0.8634275011957604, + "pass@64": 0.8774430319994344, + "pass@128": 0.8909657320872274 + }, + "15": { + "pass@1": 0.5573744580924855, + "pass@2": 0.6510904999772427, + "pass@4": 0.723555908654481, + "pass@8": 0.779664294770354, + "pass@16": 0.8218733604726072, + "pass@32": 0.8537991239873666, + "pass@64": 0.8786629030021994, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.32026209677419354, + "pass@2": 0.4102227266954534, + "pass@4": 0.4922931598267957, + "pass@8": 0.5608738002824728, + "pass@16": 0.6161741026713436, + "pass@32": 0.6634223589324809, + "pass@64": 0.7052034534598515, + "pass@128": 0.7419354838709677 + }, + "17": { + "pass@1": 0.26948609104046245, + "pass@2": 0.3551862398161212, + "pass@4": 0.43630361992323174, + "pass@8": 0.505574472344914, + "pass@16": 0.5626248447366223, + "pass@32": 0.612875860834388, + "pass@64": 0.6613171736457973, + "pass@128": 0.7052023121387283 + }, + "18": { + "pass@1": 0.22223214285714285, + "pass@2": 0.29839848143981995, + "pass@4": 0.37236364186619514, + "pass@8": 0.43890352397685345, + "pass@16": 0.49460411827912604, + "pass@32": 0.5408992361679525, + "pass@64": 0.5819298205174266, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.1733765015015015, + "pass@2": 0.24151267409141433, + "pass@4": 0.3106473104150269, + "pass@8": 0.3760434973464308, + "pass@16": 0.4339640400717074, + "pass@32": 0.48326526375921236, + "pass@64": 0.5279349721453175, + "pass@128": 0.5735735735735735 + }, + "20": { + "pass@1": 0.16915352852852852, + "pass@2": 0.22966285675439213, + "pass@4": 0.29331480636992446, + "pass@8": 0.3554372484491605, + "pass@16": 0.4113463579552269, + "pass@32": 0.4595926430033481, + "pass@64": 0.5028409960228297, + "pass@128": 0.5435435435435435 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 601723, + "answer_accuracy": 0.7328076286048324, + "avg_response_len": 266.3788885911925, + "resp_tokens_sum": 218729033, + "pass_at_k": { + "pass@1": 0.7328076286048324, + "pass@2": 0.7669891057192438, + "pass@4": 0.7962575649205126, + "pass@8": 0.8205455374066479, + "pass@16": 0.8399386311082003, + "pass@32": 0.8559486775608024, + "pass@64": 0.8701074575696454, + "pass@128": 0.883398285268901 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9406873082822086, + "pass@2": 0.9511727241437608, + "pass@4": 0.9582467286489494, + "pass@8": 0.962884319554284, + "pass@16": 0.9658664939297285, + "pass@32": 0.9683840162934146, + "pass@64": 0.9708284840200408, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996866045845272, + "pass@2": 0.9999725029894185, + "pass@4": 0.9999998079573864, + "pass@8": 0.9999999999974207, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996387283236994, + "pass@2": 0.9999765315643349, + "pass@4": 0.9999998658946533, + "pass@8": 0.9999999999989994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976012323943662, + "pass@2": 0.9993276588665854, + "pass@4": 0.9999400980687273, + "pass@8": 0.9999995286192314, + "pass@16": 0.9999999999862863, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954622005988024, + "pass@2": 0.996530438139469, + "pass@4": 0.9974659918071619, + "pass@8": 0.9982248854431038, + "pass@16": 0.9990042031981354, + "pass@32": 0.9997223135661357, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9490540286144579, + "pass@2": 0.9586703111659238, + "pass@4": 0.963866821334833, + "pass@8": 0.9669982809187331, + "pass@16": 0.969628195342412, + "pass@32": 0.9726172988672541, + "pass@64": 0.9751752015368585, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.8521012931034483, + "pass@2": 0.8769631044356135, + "pass@4": 0.893521940635163, + "pass@8": 0.9075129211754909, + "pass@16": 0.919181931341527, + "pass@32": 0.9290936707076372, + "pass@64": 0.9385951394414912, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.7197785364145658, + "pass@2": 0.7765865204349457, + "pass@4": 0.8138981214533052, + "pass@8": 0.840127920923921, + "pass@16": 0.8606898149555469, + "pass@32": 0.8784119819890157, + "pass@64": 0.8928222471865029, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.6990131578947368, + "pass@2": 0.7748933727034123, + "pass@4": 0.8276604991262051, + "pass@8": 0.8608615541084115, + "pass@16": 0.8813606703486647, + "pass@32": 0.8960207950792911, + "pass@64": 0.9067741271436445, + "pass@128": 0.9152046783625731 + }, + "15": { + "pass@1": 0.5314867424242424, + "pass@2": 0.6243128877356237, + "pass@4": 0.6995647515651452, + "pass@8": 0.75847718932693, + "pass@16": 0.8041517108171061, + "pass@32": 0.8403438310227098, + "pass@64": 0.8678819774825477, + "pass@128": 0.8878787878787879 + }, + "16": { + "pass@1": 0.3511173433048433, + "pass@2": 0.4367120375978642, + "pass@4": 0.509192721636291, + "pass@8": 0.567463683171276, + "pass@16": 0.6138049647381483, + "pass@32": 0.6543388540897418, + "pass@64": 0.6920704030785279, + "pass@128": 0.7236467236467237 + }, + "17": { + "pass@1": 0.2860539596273292, + "pass@2": 0.3759124199149017, + "pass@4": 0.4613117808061256, + "pass@8": 0.5321531449577279, + "pass@16": 0.5842617561652651, + "pass@32": 0.6244032611091793, + "pass@64": 0.6603615858501944, + "pass@128": 0.6987577639751553 + }, + "18": { + "pass@1": 0.2511948529411765, + "pass@2": 0.3300544233441406, + "pass@4": 0.40649941000022055, + "pass@8": 0.4779186811576755, + "pass@16": 0.5402423482438927, + "pass@32": 0.5939876301314176, + "pass@64": 0.6458831650585033, + "pass@128": 0.6970588235294117 + }, + "19": { + "pass@1": 0.18038504464285715, + "pass@2": 0.24445003456599168, + "pass@4": 0.31143215635991933, + "pass@8": 0.3728059388521139, + "pass@16": 0.42353768253638485, + "pass@32": 0.4649215111100438, + "pass@64": 0.500723873663761, + "pass@128": 0.5386904761904762 + }, + "20": { + "pass@1": 0.1533261601796407, + "pass@2": 0.21130733120373435, + "pass@4": 0.2719651745440502, + "pass@8": 0.332414175156139, + "pass@16": 0.3859289673999605, + "pass@32": 0.4304629843267275, + "pass@64": 0.4713483469991754, + "pass@128": 0.5119760479041916 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581110, + "answer_accuracy": 0.7253430060712573, + "avg_response_len": 284.832275273606, + "resp_tokens_sum": 228193947, + "pass_at_k": { + "pass@1": 0.7253430060712573, + "pass@2": 0.75873765399871, + "pass@4": 0.7882019855861954, + "pass@8": 0.8138588630545672, + "pass@16": 0.8355427556244789, + "pass@32": 0.8539066962141767, + "pass@64": 0.8698343314586575, + "pass@128": 0.8838472599456783 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9534048507462687, + "pass@2": 0.9626968503937007, + "pass@4": 0.9692538331962235, + "pass@8": 0.9737725528077535, + "pass@16": 0.9771025187908413, + "pass@32": 0.9801357911203106, + "pass@64": 0.9834547978205753, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966452205882353, + "pass@2": 0.9975343040759611, + "pass@4": 0.9980915448068992, + "pass@8": 0.9987831969997992, + "pass@16": 0.9995279850408865, + "pass@32": 0.9999439253289022, + "pass@64": 0.9999998174959024, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.999996106598226, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918355855855856, + "pass@2": 0.9945075488874701, + "pass@4": 0.9965787761890124, + "pass@8": 0.998290842921839, + "pass@16": 0.9994100203212533, + "pass@32": 0.9999208020070317, + "pass@64": 0.99999958431727, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956541398713826, + "pass@2": 0.9983808105678912, + "pass@4": 0.999648498318739, + "pass@8": 0.9999806352115692, + "pass@16": 0.9999999483522362, + "pass@32": 0.9999999999999448, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9447420634920635, + "pass@2": 0.9581868672665916, + "pass@4": 0.967509306872355, + "pass@8": 0.9740486174543461, + "pass@16": 0.9786847931164305, + "pass@32": 0.9824150336632566, + "pass@64": 0.9857183739073222, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8612146406727829, + "pass@2": 0.8891486220472444, + "pass@4": 0.9075370982870257, + "pass@8": 0.921338707735371, + "pass@16": 0.9329246372544424, + "pass@32": 0.9447859108257943, + "pass@64": 0.9570330678851205, + "pass@128": 0.9694189602446484 + }, + "13": { + "pass@1": 0.6789400077160493, + "pass@2": 0.73740597963449, + "pass@4": 0.7799996001657193, + "pass@8": 0.8137514624128047, + "pass@16": 0.8405120187983943, + "pass@32": 0.8611789198572768, + "pass@64": 0.8775786513793102, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.6469538204747775, + "pass@2": 0.7170783049837612, + "pass@4": 0.7698086949072013, + "pass@8": 0.8095645184136512, + "pass@16": 0.8393733146970238, + "pass@32": 0.8621325130569488, + "pass@64": 0.8802217668630712, + "pass@128": 0.8931750741839762 + }, + "15": { + "pass@1": 0.5352527006172839, + "pass@2": 0.6160802165354328, + "pass@4": 0.6814122117837121, + "pass@8": 0.734581191554782, + "pass@16": 0.7757085390245353, + "pass@32": 0.8070609426882411, + "pass@64": 0.8332631723379745, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.3416758849557522, + "pass@2": 0.42799572619794185, + "pass@4": 0.5040647457784591, + "pass@8": 0.5700512110067496, + "pass@16": 0.6254832399008291, + "pass@32": 0.6707192692761402, + "pass@64": 0.7088023579005431, + "pass@128": 0.7433628318584071 + }, + "17": { + "pass@1": 0.2613422439759036, + "pass@2": 0.3423844245090598, + "pass@4": 0.41293712645858993, + "pass@8": 0.46923112448581195, + "pass@16": 0.5187456803719004, + "pass@32": 0.5701175454546831, + "pass@64": 0.6236549755551729, + "pass@128": 0.6716867469879518 + }, + "18": { + "pass@1": 0.2354334677419355, + "pass@2": 0.3043799212598424, + "pass@4": 0.37419252270885495, + "pass@8": 0.4401254353269728, + "pass@16": 0.4954914482495745, + "pass@32": 0.5391664434428325, + "pass@64": 0.5774079786759414, + "pass@128": 0.6193548387096774 + }, + "19": { + "pass@1": 0.17546261329305135, + "pass@2": 0.24499771035040552, + "pass@4": 0.31780488079095853, + "pass@8": 0.38831339902772993, + "pass@16": 0.45257478979141497, + "pass@32": 0.5076430412025037, + "pass@64": 0.5521427562189815, + "pass@128": 0.5891238670694864 + }, + "20": { + "pass@1": 0.1487424924924925, + "pass@2": 0.2120966931498427, + "pass@4": 0.28508847148610916, + "pass@8": 0.3607474325394686, + "pass@16": 0.42986908543734814, + "pass@32": 0.48956023306096935, + "pass@64": 0.5385710498391255, + "pass@128": 0.5765765765765766 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/summary.csv b/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..a94ee564314431e472ecf7de833a9a9df0b0d2f4 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85656,0.9933975694444445,0.48817109375,0.7274888980263158,0.0,0.0,0.0,197.00196440972223,334.2508890625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/summary.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5279c1d845c7ed41633d2b8d18464528784aaa66 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block4/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85656", + "id_acc": 0.9933975694444445, + "ood_acc": 0.48817109375, + "total_acc": 0.7274888980263158, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.00196440972223, + "ood_avg_resp_len": 334.2508890625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 586420, + "answer_accuracy": 0.7242185030034777, + "avg_response_len": 256.7089454730477, + "resp_tokens_sum": 207864421, + "pass_at_k": { + "pass@1": 0.7242185030034777, + "pass@2": 0.7585805035959495, + "pass@4": 0.7882152405970935, + "pass@8": 0.8131007504036403, + "pass@16": 0.8335299105303158, + "pass@32": 0.8507363433743848, + "pass@64": 0.8662116073517198, + "pass@128": 0.8808093582042364 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9612140486725663, + "pass@2": 0.9701686729147794, + "pass@4": 0.97611373799514, + "pass@8": 0.9795318507801245, + "pass@16": 0.9815704330658933, + "pass@32": 0.9828295616605529, + "pass@64": 0.9837715482760134, + "pass@128": 0.9852507374631269 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966269841269841, + "pass@2": 0.9969132764654418, + "pass@4": 0.9970214452360122, + "pass@8": 0.9972112860892366, + "pass@16": 0.9975721784776902, + "pass@32": 0.9982189726284215, + "pass@64": 0.9992125984251968, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999766092814372, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9993296967455622, + "pass@2": 0.9998813364860458, + "pass@4": 0.9999958538836491, + "pass@8": 0.9999999967670793, + "pass@16": 0.9999999999999998, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9997495993589743, + "pass@2": 0.9999948736876642, + "pass@4": 0.999999998497784, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9941901408450704, + "pass@2": 0.9947054868581569, + "pass@4": 0.9950140287921757, + "pass@8": 0.9955275550723595, + "pass@16": 0.996246252427952, + "pass@32": 0.9969218386791731, + "pass@64": 0.9971743778471076, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.9172716005665722, + "pass@2": 0.9294245332470837, + "pass@4": 0.9375553278927952, + "pass@8": 0.9436644091134152, + "pass@16": 0.94896922068123, + "pass@32": 0.9540347827409994, + "pass@64": 0.9584288926634769, + "pass@128": 0.9603399433427762 + }, + "12": { + "pass@1": 0.8369085451977402, + "pass@2": 0.8658925413719473, + "pass@4": 0.8836768536877804, + "pass@8": 0.8958159717383948, + "pass@16": 0.9059099640119125, + "pass@32": 0.9152839590889877, + "pass@64": 0.9230974982514136, + "pass@128": 0.9293785310734464 + }, + "13": { + "pass@1": 0.7135579937304075, + "pass@2": 0.7642971083355962, + "pass@4": 0.7975250877731188, + "pass@8": 0.8216605066521206, + "pass@16": 0.8408669871052371, + "pass@32": 0.8585289603051615, + "pass@64": 0.8786083453443386, + "pass@128": 0.9028213166144201 + }, + "14": { + "pass@1": 0.6607038551401869, + "pass@2": 0.7342098081781829, + "pass@4": 0.7866001411038571, + "pass@8": 0.8223529852870952, + "pass@16": 0.8463869020391322, + "pass@32": 0.8634275011957604, + "pass@64": 0.8774430319994344, + "pass@128": 0.8909657320872274 + }, + "15": { + "pass@1": 0.5573744580924855, + "pass@2": 0.6510904999772427, + "pass@4": 0.723555908654481, + "pass@8": 0.779664294770354, + "pass@16": 0.8218733604726072, + "pass@32": 0.8537991239873666, + "pass@64": 0.8786629030021994, + "pass@128": 0.8988439306358381 + }, + "16": { + "pass@1": 0.32026209677419354, + "pass@2": 0.4102227266954534, + "pass@4": 0.4922931598267957, + "pass@8": 0.5608738002824728, + "pass@16": 0.6161741026713436, + "pass@32": 0.6634223589324809, + "pass@64": 0.7052034534598515, + "pass@128": 0.7419354838709677 + }, + "17": { + "pass@1": 0.26948609104046245, + "pass@2": 0.3551862398161212, + "pass@4": 0.43630361992323174, + "pass@8": 0.505574472344914, + "pass@16": 0.5626248447366223, + "pass@32": 0.612875860834388, + "pass@64": 0.6613171736457973, + "pass@128": 0.7052023121387283 + }, + "18": { + "pass@1": 0.22223214285714285, + "pass@2": 0.29839848143981995, + "pass@4": 0.37236364186619514, + "pass@8": 0.43890352397685345, + "pass@16": 0.49460411827912604, + "pass@32": 0.5408992361679525, + "pass@64": 0.5819298205174266, + "pass@128": 0.6228571428571429 + }, + "19": { + "pass@1": 0.1733765015015015, + "pass@2": 0.24151267409141433, + "pass@4": 0.3106473104150269, + "pass@8": 0.3760434973464308, + "pass@16": 0.4339640400717074, + "pass@32": 0.48326526375921236, + "pass@64": 0.5279349721453175, + "pass@128": 0.5735735735735735 + }, + "20": { + "pass@1": 0.16915352852852852, + "pass@2": 0.22966285675439213, + "pass@4": 0.29331480636992446, + "pass@8": 0.3554372484491605, + "pass@16": 0.4113463579552269, + "pass@32": 0.4595926430033481, + "pass@64": 0.5028409960228297, + "pass@128": 0.5435435435435435 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 601723, + "answer_accuracy": 0.7328076286048324, + "avg_response_len": 266.3788885911925, + "resp_tokens_sum": 218729033, + "pass_at_k": { + "pass@1": 0.7328076286048324, + "pass@2": 0.7669891057192438, + "pass@4": 0.7962575649205126, + "pass@8": 0.8205455374066479, + "pass@16": 0.8399386311082003, + "pass@32": 0.8559486775608024, + "pass@64": 0.8701074575696454, + "pass@128": 0.883398285268901 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9406873082822086, + "pass@2": 0.9511727241437608, + "pass@4": 0.9582467286489494, + "pass@8": 0.962884319554284, + "pass@16": 0.9658664939297285, + "pass@32": 0.9683840162934146, + "pass@64": 0.9708284840200408, + "pass@128": 0.9723926380368099 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9996866045845272, + "pass@2": 0.9999725029894185, + "pass@4": 0.9999998079573864, + "pass@8": 0.9999999999974207, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9996387283236994, + "pass@2": 0.9999765315643349, + "pass@4": 0.9999998658946533, + "pass@8": 0.9999999999989994, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9976012323943662, + "pass@2": 0.9993276588665854, + "pass@4": 0.9999400980687273, + "pass@8": 0.9999995286192314, + "pass@16": 0.9999999999862863, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9954622005988024, + "pass@2": 0.996530438139469, + "pass@4": 0.9974659918071619, + "pass@8": 0.9982248854431038, + "pass@16": 0.9990042031981354, + "pass@32": 0.9997223135661357, + "pass@64": 0.999990730945279, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9490540286144579, + "pass@2": 0.9586703111659238, + "pass@4": 0.963866821334833, + "pass@8": 0.9669982809187331, + "pass@16": 0.969628195342412, + "pass@32": 0.9726172988672541, + "pass@64": 0.9751752015368585, + "pass@128": 0.9759036144578314 + }, + "12": { + "pass@1": 0.8521012931034483, + "pass@2": 0.8769631044356135, + "pass@4": 0.893521940635163, + "pass@8": 0.9075129211754909, + "pass@16": 0.919181931341527, + "pass@32": 0.9290936707076372, + "pass@64": 0.9385951394414912, + "pass@128": 0.9498432601880877 + }, + "13": { + "pass@1": 0.7197785364145658, + "pass@2": 0.7765865204349457, + "pass@4": 0.8138981214533052, + "pass@8": 0.840127920923921, + "pass@16": 0.8606898149555469, + "pass@32": 0.8784119819890157, + "pass@64": 0.8928222471865029, + "pass@128": 0.9047619047619048 + }, + "14": { + "pass@1": 0.6990131578947368, + "pass@2": 0.7748933727034123, + "pass@4": 0.8276604991262051, + "pass@8": 0.8608615541084115, + "pass@16": 0.8813606703486647, + "pass@32": 0.8960207950792911, + "pass@64": 0.9067741271436445, + "pass@128": 0.9152046783625731 + }, + "15": { + "pass@1": 0.5314867424242424, + "pass@2": 0.6243128877356237, + "pass@4": 0.6995647515651452, + "pass@8": 0.75847718932693, + "pass@16": 0.8041517108171061, + "pass@32": 0.8403438310227098, + "pass@64": 0.8678819774825477, + "pass@128": 0.8878787878787879 + }, + "16": { + "pass@1": 0.3511173433048433, + "pass@2": 0.4367120375978642, + "pass@4": 0.509192721636291, + "pass@8": 0.567463683171276, + "pass@16": 0.6138049647381483, + "pass@32": 0.6543388540897418, + "pass@64": 0.6920704030785279, + "pass@128": 0.7236467236467237 + }, + "17": { + "pass@1": 0.2860539596273292, + "pass@2": 0.3759124199149017, + "pass@4": 0.4613117808061256, + "pass@8": 0.5321531449577279, + "pass@16": 0.5842617561652651, + "pass@32": 0.6244032611091793, + "pass@64": 0.6603615858501944, + "pass@128": 0.6987577639751553 + }, + "18": { + "pass@1": 0.2511948529411765, + "pass@2": 0.3300544233441406, + "pass@4": 0.40649941000022055, + "pass@8": 0.4779186811576755, + "pass@16": 0.5402423482438927, + "pass@32": 0.5939876301314176, + "pass@64": 0.6458831650585033, + "pass@128": 0.6970588235294117 + }, + "19": { + "pass@1": 0.18038504464285715, + "pass@2": 0.24445003456599168, + "pass@4": 0.31143215635991933, + "pass@8": 0.3728059388521139, + "pass@16": 0.42353768253638485, + "pass@32": 0.4649215111100438, + "pass@64": 0.500723873663761, + "pass@128": 0.5386904761904762 + }, + "20": { + "pass@1": 0.1533261601796407, + "pass@2": 0.21130733120373435, + "pass@4": 0.2719651745440502, + "pass@8": 0.332414175156139, + "pass@16": 0.3859289673999605, + "pass@32": 0.4304629843267275, + "pass@64": 0.4713483469991754, + "pass@128": 0.5119760479041916 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 581110, + "answer_accuracy": 0.7253430060712573, + "avg_response_len": 284.832275273606, + "resp_tokens_sum": 228193947, + "pass_at_k": { + "pass@1": 0.7253430060712573, + "pass@2": 0.75873765399871, + "pass@4": 0.7882019855861954, + "pass@8": 0.8138588630545672, + "pass@16": 0.8355427556244789, + "pass@32": 0.8539066962141767, + "pass@64": 0.8698343314586575, + "pass@128": 0.8838472599456783 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.9534048507462687, + "pass@2": 0.9626968503937007, + "pass@4": 0.9692538331962235, + "pass@8": 0.9737725528077535, + "pass@16": 0.9771025187908413, + "pass@32": 0.9801357911203106, + "pass@64": 0.9834547978205753, + "pass@128": 0.9880597014925373 + }, + "2": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "3": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 1.0, + "pass@2": 1.0, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.9966452205882353, + "pass@2": 0.9975343040759611, + "pass@4": 0.9980915448068992, + "pass@8": 0.9987831969997992, + "pass@16": 0.9995279850408865, + "pass@32": 0.9999439253289022, + "pass@64": 0.9999998174959024, + "pass@128": 1.0 + }, + "6": { + "pass@1": 0.9999507097791798, + "pass@2": 0.999999611888025, + "pass@4": 1.0, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "7": { + "pass@1": 0.9998022151898734, + "pass@2": 0.999996106598226, + "pass@4": 0.9999999985167994, + "pass@8": 1.0, + "pass@16": 1.0, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.9918355855855856, + "pass@2": 0.9945075488874701, + "pass@4": 0.9965787761890124, + "pass@8": 0.998290842921839, + "pass@16": 0.9994100203212533, + "pass@32": 0.9999208020070317, + "pass@64": 0.99999958431727, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.9956541398713826, + "pass@2": 0.9983808105678912, + "pass@4": 0.999648498318739, + "pass@8": 0.9999806352115692, + "pass@16": 0.9999999483522362, + "pass@32": 0.9999999999999448, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "11": { + "pass@1": 0.9447420634920635, + "pass@2": 0.9581868672665916, + "pass@4": 0.967509306872355, + "pass@8": 0.9740486174543461, + "pass@16": 0.9786847931164305, + "pass@32": 0.9824150336632566, + "pass@64": 0.9857183739073222, + "pass@128": 0.9873015873015873 + }, + "12": { + "pass@1": 0.8612146406727829, + "pass@2": 0.8891486220472444, + "pass@4": 0.9075370982870257, + "pass@8": 0.921338707735371, + "pass@16": 0.9329246372544424, + "pass@32": 0.9447859108257943, + "pass@64": 0.9570330678851205, + "pass@128": 0.9694189602446484 + }, + "13": { + "pass@1": 0.6789400077160493, + "pass@2": 0.73740597963449, + "pass@4": 0.7799996001657193, + "pass@8": 0.8137514624128047, + "pass@16": 0.8405120187983943, + "pass@32": 0.8611789198572768, + "pass@64": 0.8775786513793102, + "pass@128": 0.8919753086419753 + }, + "14": { + "pass@1": 0.6469538204747775, + "pass@2": 0.7170783049837612, + "pass@4": 0.7698086949072013, + "pass@8": 0.8095645184136512, + "pass@16": 0.8393733146970238, + "pass@32": 0.8621325130569488, + "pass@64": 0.8802217668630712, + "pass@128": 0.8931750741839762 + }, + "15": { + "pass@1": 0.5352527006172839, + "pass@2": 0.6160802165354328, + "pass@4": 0.6814122117837121, + "pass@8": 0.734581191554782, + "pass@16": 0.7757085390245353, + "pass@32": 0.8070609426882411, + "pass@64": 0.8332631723379745, + "pass@128": 0.8549382716049383 + }, + "16": { + "pass@1": 0.3416758849557522, + "pass@2": 0.42799572619794185, + "pass@4": 0.5040647457784591, + "pass@8": 0.5700512110067496, + "pass@16": 0.6254832399008291, + "pass@32": 0.6707192692761402, + "pass@64": 0.7088023579005431, + "pass@128": 0.7433628318584071 + }, + "17": { + "pass@1": 0.2613422439759036, + "pass@2": 0.3423844245090598, + "pass@4": 0.41293712645858993, + "pass@8": 0.46923112448581195, + "pass@16": 0.5187456803719004, + "pass@32": 0.5701175454546831, + "pass@64": 0.6236549755551729, + "pass@128": 0.6716867469879518 + }, + "18": { + "pass@1": 0.2354334677419355, + "pass@2": 0.3043799212598424, + "pass@4": 0.37419252270885495, + "pass@8": 0.4401254353269728, + "pass@16": 0.4954914482495745, + "pass@32": 0.5391664434428325, + "pass@64": 0.5774079786759414, + "pass@128": 0.6193548387096774 + }, + "19": { + "pass@1": 0.17546261329305135, + "pass@2": 0.24499771035040552, + "pass@4": 0.31780488079095853, + "pass@8": 0.38831339902772993, + "pass@16": 0.45257478979141497, + "pass@32": 0.5076430412025037, + "pass@64": 0.5521427562189815, + "pass@128": 0.5891238670694864 + }, + "20": { + "pass@1": 0.1487424924924925, + "pass@2": 0.2120966931498427, + "pass@4": 0.28508847148610916, + "pass@8": 0.3607474325394686, + "pass@16": 0.42986908543734814, + "pass@32": 0.48956023306096935, + "pass@64": 0.5385710498391255, + "pass@128": 0.5765765765765766 + } + } + } + } + } +] \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/checkpoint-85656_metrics.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/checkpoint-85656_metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..494513789c7e88c8df85d57c3da088baacd83905 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/checkpoint-85656_metrics.json @@ -0,0 +1,1897 @@ +{ + "checkpoint": "checkpoint-85656", + "id": { + "answer_accuracy": 0.7683654513888889, + "avg_response_len": 197.30304166666667, + "avg_loss": 0.0, + "count": 1152000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.7683654513888889, + "pass@2": 0.8660864774715631, + "pass@4": 0.9312000551181093, + "pass@8": 0.9661839306069261, + "pass@16": 0.9828142168302318, + "pass@32": 0.9907125170999708, + "pass@64": 0.994819479065359, + "pass@128": 0.9968888888888889 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.54915625, + "pass@2": 0.7164767470472423, + "pass@4": 0.8426845951443562, + "pass@8": 0.9191123432543775, + "pass@16": 0.9590365811916756, + "pass@32": 0.9779281132253446, + "pass@64": 0.987908506220136, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.98878125, + "pass@2": 0.9927722687007875, + "pass@4": 0.9940767196287963, + "pass@8": 0.9946724268833085, + "pass@16": 0.9952055668049555, + "pass@32": 0.9957817319358768, + "pass@64": 0.9963772850204733, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.994609375, + "pass@2": 0.998306102362205, + "pass@4": 0.9992007952755906, + "pass@8": 0.9996691369022104, + "pass@16": 0.9999327111749879, + "pass@32": 0.999997438633278, + "pass@64": 0.9999999996028359, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.916265625, + "pass@2": 0.9581102362204724, + "pass@4": 0.9784218173978244, + "pass@8": 0.9870107311461325, + "pass@16": 0.9912463126427409, + "pass@32": 0.9939197472867504, + "pass@64": 0.9959517707728855, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.740921875, + "pass@2": 0.8353421505905507, + "pass@4": 0.8950855312148475, + "pass@8": 0.9297393408121918, + "pass@16": 0.9533335123789132, + "pass@32": 0.97132714165803, + "pass@64": 0.9839180181059016, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.8026484375, + "pass@2": 0.9024115403543312, + "pass@4": 0.9577660202474688, + "pass@8": 0.9815508879309162, + "pass@16": 0.9905558795888216, + "pass@32": 0.9944732576094857, + "pass@64": 0.9964454722777892, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.6978828125, + "pass@2": 0.8403853346456682, + "pass@4": 0.9284471146419201, + "pass@8": 0.9699026068300013, + "pass@16": 0.9873766827112286, + "pass@32": 0.9952395813054687, + "pass@64": 0.9986726234881326, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.660140625, + "pass@2": 0.8153296013779523, + "pass@4": 0.9184502872140984, + "pass@8": 0.9701473425162639, + "pass@16": 0.9900834472917458, + "pass@32": 0.9963527544566101, + "pass@64": 0.9981845960791449, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.5648828125, + "pass@2": 0.7356443159448799, + "pass@4": 0.8666676152980888, + "pass@8": 0.9438505591870303, + "pass@16": 0.9785572576870053, + "pass@32": 0.9913928877889056, + "pass@64": 0.9959170400208928, + "pass@128": 0.997 + } + }, + "per_op_accuracy": { + "10": 0.54915625, + "2": 0.98878125, + "3": 0.994609375, + "4": 0.916265625, + "5": 0.740921875, + "6": 0.8026484375, + "7": 0.6978828125, + "8": 0.660140625, + "9": 0.5648828125 + }, + "per_op_avg_response_len": { + "10": 276.4170703125, + "2": 114.74503125, + "3": 143.584171875, + "4": 158.5899765625, + "5": 171.45425, + "6": 194.59884375, + "7": 220.80815625, + "8": 240.4442578125, + "9": 255.0856171875 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.4170703125, + "2": 114.74503125, + "3": 143.584171875, + "4": 158.5899765625, + "5": 171.45425, + "6": 194.59884375, + "7": 220.80815625, + "8": 240.4442578125, + "9": 255.0856171875 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 379008, + "correct": 286428, + "answer_accuracy": 0.7557307497467072, + "avg_response_len": 187.75557771867614, + "resp_tokens_sum": 71160866, + "pass_at_k": { + "pass@1": 0.7557307497467072, + "pass@2": 0.8557669902299482, + "pass@4": 0.9239831380310022, + "pass@8": 0.9619600644692671, + "pass@16": 0.9807098355315982, + "pass@32": 0.9902347168175939, + "pass@64": 0.9955029315782938, + "pass@128": 0.9979736575481256 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5576373525073747, + "pass@2": 0.7241162056070428, + "pass@4": 0.8478356140216987, + "pass@8": 0.9210285105668401, + "pass@16": 0.9590456424231748, + "pass@32": 0.9787901271304692, + "pass@64": 0.990491462519074, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9872225467289719, + "pass@2": 0.992541071822798, + "pass@4": 0.9944339238553123, + "pass@8": 0.9952862602378583, + "pass@16": 0.9961075311262445, + "pass@32": 0.9967298286406606, + "pass@64": 0.996882655029806, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9942660550458715, + "pass@2": 0.9974246159310364, + "pass@4": 0.9983950320430129, + "pass@8": 0.9991873729741889, + "pass@16": 0.999804553543206, + "pass@32": 0.9999921788300652, + "pass@64": 0.999999998785431, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.91865234375, + "pass@2": 0.9625246062992125, + "pass@4": 0.982705639822365, + "pass@8": 0.9912215061732457, + "pass@16": 0.9953870393196059, + "pass@32": 0.9976677753977258, + "pass@64": 0.9991816802247045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7051587301587302, + "pass@2": 0.7998789213848272, + "pass@4": 0.8623743475517939, + "pass@8": 0.902292682084196, + "pass@16": 0.9335744672132729, + "pass@32": 0.961012724109928, + "pass@64": 0.9823888277410862, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.8122427020958084, + "pass@2": 0.9102172132113728, + "pass@4": 0.9629234422044544, + "pass@8": 0.9852725468337608, + "pass@16": 0.9928515702293308, + "pass@32": 0.9955259002838712, + "pass@64": 0.9966392296552004, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.6823687130177515, + "pass@2": 0.83220396729255, + "pass@4": 0.9268846605897633, + "pass@8": 0.9704636354507595, + "pass@16": 0.9878959614025046, + "pass@32": 0.9961612605932577, + "pass@64": 0.9994934131512692, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6389723557692307, + "pass@2": 0.7958146420098927, + "pass@4": 0.9043729308475859, + "pass@8": 0.9630561425068057, + "pass@16": 0.9881072533959968, + "pass@32": 0.9960452808281866, + "pass@64": 0.9983311703038567, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5331646126760563, + "pass@2": 0.7045795441942995, + "pass@4": 0.8440219997148238, + "pass@8": 0.9321960354746593, + "pass@16": 0.9737132718388507, + "pass@32": 0.9898040713041135, + "pass@64": 0.9958648530897064, + "pass@128": 0.9971830985915493 + } + } + }, + "teachers_in_school": { + "count": 390656, + "correct": 301237, + "answer_accuracy": 0.7711055250655308, + "avg_response_len": 195.18825002047836, + "resp_tokens_sum": 76251461, + "pass_at_k": { + "pass@1": 0.7711055250655308, + "pass@2": 0.8674490830203033, + "pass@4": 0.9315876602519219, + "pass@8": 0.9655659148142043, + "pass@16": 0.9814173312084932, + "pass@32": 0.9888882773076394, + "pass@64": 0.9927472467456936, + "pass@128": 0.9947575360419397 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5151457055214724, + "pass@2": 0.6830678469639145, + "pass@4": 0.8189656213410438, + "pass@8": 0.9071709306839472, + "pass@16": 0.9537003905148077, + "pass@32": 0.9732703621233486, + "pass@64": 0.9815479508426936, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 0.9870820668693009, + "pass@2": 0.9916114209128114, + "pass@4": 0.9932387670766079, + "pass@8": 0.9939459341067186, + "pass@16": 0.9942939216780183, + "pass@32": 0.9946808461232393, + "pass@64": 0.9954407294832827, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9931344696969697, + "pass@2": 0.9978040742066334, + "pass@4": 0.9991804419333946, + "pass@8": 0.9998026414887873, + "pass@16": 0.9999897641404931, + "pass@32": 0.9999999883510501, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9130917159763313, + "pass@2": 0.9530991561058564, + "pass@4": 0.9734309767307188, + "pass@8": 0.981508303989163, + "pass@16": 0.9852970136399636, + "pass@32": 0.9883859742602302, + "pass@64": 0.9910152794130401, + "pass@128": 0.9940828402366864 + }, + "5": { + "pass@1": 0.7668704710144928, + "pass@2": 0.8570691401346572, + "pass@4": 0.9105642180596992, + "pass@8": 0.9386731501468899, + "pass@16": 0.9567025653999209, + "pass@32": 0.9706738161299903, + "pass@64": 0.980097445623169, + "pass@128": 0.9855072463768116 + }, + "6": { + "pass@1": 0.8068812679083095, + "pass@2": 0.9043202738984274, + "pass@4": 0.9584084456312574, + "pass@8": 0.9802832598182536, + "pass@16": 0.9885671782379833, + "pass@32": 0.9930770664664508, + "pass@64": 0.9960588198590073, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.7005735187861272, + "pass@2": 0.8407154601520194, + "pass@4": 0.9295946147085653, + "pass@8": 0.9719966626529152, + "pass@16": 0.9888768016805433, + "pass@32": 0.9956843233080916, + "pass@64": 0.9985466054756086, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6794454225352112, + "pass@2": 0.8303634800931575, + "pass@4": 0.925194801354056, + "pass@8": 0.9693069551068682, + "pass@16": 0.9866333068822101, + "pass@32": 0.9936383088194044, + "pass@64": 0.9963589210449596, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.5826160179640718, + "pass@2": 0.7489468651280118, + "pass@4": 0.8742062343254993, + "pass@8": 0.9464091288357557, + "pass@16": 0.978296787242704, + "pass@32": 0.9903044676743343, + "pass@64": 0.995371590764158, + "pass@128": 0.9970059880239521 + } + } + }, + "movie_festival_awards": { + "count": 382336, + "correct": 297492, + "answer_accuracy": 0.7780904753933713, + "avg_response_len": 208.92821235771677, + "resp_tokens_sum": 79880777, + "pass_at_k": { + "pass@1": 0.7780904753933713, + "pass@2": 0.8749238827570425, + "pass@4": 0.9379581135802968, + "pass@8": 0.9710024949969832, + "pass@16": 0.9863275639821666, + "pass@32": 0.9930500954335235, + "pass@64": 0.9962593016796092, + "pass@128": 0.9979912956143288 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5736707089552239, + "pass@2": 0.7412574186155834, + "pass@4": 0.8605538192054352, + "pass@8": 0.9287938948335882, + "pass@16": 0.9642202420966983, + "pass@32": 0.981588424107176, + "pass@64": 0.9914843834371672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9918080357142857, + "pass@2": 0.9940755061867266, + "pass@4": 0.9945367877229631, + "pass@8": 0.9947923572167008, + "pass@16": 0.9952352830896093, + "pass@32": 0.9959470244790829, + "pass@64": 0.9967941507311587, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9963556851311953, + "pass@2": 0.9996294707421778, + "pass@4": 0.9999885538177989, + "pass@8": 0.9999999832313435, + "pass@16": 0.9999999999999907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9171692251461988, + "pass@2": 0.958932302458904, + "pass@4": 0.9793460307593129, + "pass@8": 0.9885088959718029, + "pass@16": 0.9932516650589491, + "pass@32": 0.9958818709342699, + "pass@64": 0.9978083878929018, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7477251838235294, + "pass@2": 0.8461512853172767, + "pass@4": 0.9096851661924609, + "pass@8": 0.9461026739852725, + "pass@16": 0.9682210886581221, + "pass@32": 0.9815460794075175, + "pass@64": 0.9892115253749012, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.7878795347003155, + "pass@2": 0.892085853473757, + "pass@4": 0.9516247414065356, + "pass@8": 0.9790252353686714, + "pass@16": 0.9903265297387072, + "pass@32": 0.9949012956400033, + "pass@64": 0.9966670077039709, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.7115308544303798, + "pass@2": 0.8487748243297122, + "pass@4": 0.9288619071943541, + "pass@8": 0.9670096606637208, + "pass@16": 0.9851787163788448, + "pass@32": 0.9937667701276847, + "pass@64": 0.9979326719887415, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6593937687687688, + "pass@2": 0.8175867796930787, + "pass@4": 0.9244497847115954, + "pass@8": 0.9776872582318384, + "pass@16": 0.9956130939609944, + "pass@32": 0.9995346162382196, + "pass@64": 0.9999935554756175, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5820438102893891, + "pass@2": 0.7568177646403522, + "pass@4": 0.8844209747093505, + "pass@8": 0.9544061657954517, + "pass@16": 0.9843662998236358, + "pass@32": 0.994375402806163, + "pass@64": 0.9965623982598637, + "pass@128": 0.9967845659163987 + } + } + } + } + }, + "ood": { + "answer_accuracy": 0.2703328125, + "avg_response_len": 329.4546515625, + "avg_loss": 0.0, + "count": 1280000, + "validation_metrics": {}, + "pass_at_k": { + "pass@1": 0.2703328125, + "pass@2": 0.3871927657480404, + "pass@4": 0.504655227221593, + "pass@8": 0.6075568847750559, + "pass@16": 0.6895286744785497, + "pass@32": 0.7523238515022522, + "pass@64": 0.800602604412557, + "pass@128": 0.8383 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4902265625, + "pass@2": 0.657569389763779, + "pass@4": 0.7954171744469429, + "pass@8": 0.8889310363785922, + "pass@16": 0.9432360609500965, + "pass@32": 0.9703895698123104, + "pass@64": 0.9821512430341022, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.4189609375, + "pass@2": 0.5766635088582681, + "pass@4": 0.7181645785526807, + "pass@8": 0.8265505995204228, + "pass@16": 0.8996512964671463, + "pass@32": 0.9437508071708404, + "pass@64": 0.9691049665552037, + "pass@128": 0.983 + }, + "13": { + "pass@1": 0.356453125, + "pass@2": 0.5007289616141732, + "pass@4": 0.6386766360142478, + "pass@8": 0.7547754791433346, + "pass@16": 0.8418339989360605, + "pass@32": 0.8993570639455176, + "pass@64": 0.934437073689461, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.3128203125, + "pass@2": 0.45377066929133836, + "pass@4": 0.596020788620172, + "pass@8": 0.7176963176037108, + "pass@16": 0.8092411950502408, + "pass@32": 0.872027923841253, + "pass@64": 0.9129473155162873, + "pass@128": 0.941 + }, + "15": { + "pass@1": 0.2425625, + "pass@2": 0.36528346456692967, + "pass@4": 0.49832945256842787, + "pass@8": 0.6224226498573554, + "pass@16": 0.7256435127968704, + "pass@32": 0.804989358782077, + "pass@64": 0.8603006224461872, + "pass@128": 0.892 + }, + "16": { + "pass@1": 0.21909375, + "pass@2": 0.32267691929133874, + "pass@4": 0.4338456074240717, + "pass@8": 0.5392787735555705, + "pass@16": 0.6330451872015441, + "pass@32": 0.7145189769989431, + "pass@64": 0.7837527776254032, + "pass@128": 0.839 + }, + "17": { + "pass@1": 0.18896875, + "pass@2": 0.2848443651574808, + "pass@4": 0.39093288432695933, + "pass@8": 0.4928634972974445, + "pass@16": 0.5841979883201639, + "pass@32": 0.6640798225273898, + "pass@64": 0.7313303793764716, + "pass@128": 0.789 + }, + "18": { + "pass@1": 0.1713359375, + "pass@2": 0.2532303149606297, + "pass@4": 0.3452790949568799, + "pass@8": 0.4372334878085905, + "pass@16": 0.5205739442698896, + "pass@32": 0.5946192543420282, + "pass@64": 0.660529727818769, + "pass@128": 0.718 + }, + "19": { + "pass@1": 0.16296875, + "pass@2": 0.2434144931102361, + "pass@4": 0.3309236845706785, + "pass@8": 0.41296291386471917, + "pass@16": 0.4832973009120211, + "pass@32": 0.5445339261283848, + "pass@64": 0.6018396933828155, + "pass@128": 0.657 + }, + "20": { + "pass@1": 0.1399375, + "pass@2": 0.21374557086614168, + "pass@4": 0.298962370734908, + "pass@8": 0.3828540927207293, + "pass@16": 0.4545662598812266, + "pass@32": 0.5149718114737571, + "pass@64": 0.5696322446807763, + "pass@128": 0.621 + } + }, + "per_op_accuracy": { + "11": 0.4902265625, + "12": 0.4189609375, + "13": 0.356453125, + "14": 0.3128203125, + "15": 0.2425625, + "16": 0.21909375, + "17": 0.18896875, + "18": 0.1713359375, + "19": 0.16296875, + "20": 0.1399375 + }, + "per_op_avg_response_len": { + "11": 293.0759296875, + "12": 302.2104765625, + "13": 317.1795625, + "14": 332.7867578125, + "15": 336.7595625, + "16": 341.10715625, + "17": 346.28096875, + "18": 341.680140625, + "19": 342.133359375, + "20": 341.3326015625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "11": 293.0759296875, + "12": 302.2104765625, + "13": 317.1795625, + "14": 332.7867578125, + "15": 336.7595625, + "16": 341.10715625, + "17": 346.28096875, + "18": 341.680140625, + "19": 342.133359375, + "20": 341.3326015625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 430720, + "correct": 115303, + "answer_accuracy": 0.26769827265973256, + "avg_response_len": 312.3780274888559, + "resp_tokens_sum": 134547464, + "pass_at_k": { + "pass@1": 0.26769827265973256, + "pass@2": 0.3841039285254657, + "pass@4": 0.5010747570465315, + "pass@8": 0.6037159023329047, + "pass@16": 0.6858445789485913, + "pass@32": 0.7493543898963755, + "pass@64": 0.7988351624167817, + "pass@128": 0.837741456166419 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.47826664305949007, + "pass@2": 0.6444870736766972, + "pass@4": 0.7803746312872358, + "pass@8": 0.8730347091383628, + "pass@16": 0.9291473364437194, + "pass@32": 0.9599139448333401, + "pass@64": 0.9753934096770958, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.4042858403954802, + "pass@2": 0.5600105237110191, + "pass@4": 0.7015747594580333, + "pass@8": 0.8112545401764185, + "pass@16": 0.8858790711888753, + "pass@32": 0.931855418213947, + "pass@64": 0.9582409337387682, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.3693181818181818, + "pass@2": 0.5208440037765657, + "pass@4": 0.6606441350983167, + "pass@8": 0.7731861628772339, + "pass@16": 0.8562613390543541, + "pass@32": 0.9116335367856937, + "pass@64": 0.9451118730416904, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.29650992990654207, + "pass@2": 0.4420015729634261, + "pass@4": 0.5948001467690369, + "pass@8": 0.727273434984605, + "pass@16": 0.8239426336095855, + "pass@32": 0.8852793296906377, + "pass@64": 0.921121409033376, + "pass@128": 0.940809968847352 + }, + "15": { + "pass@1": 0.23956828034682082, + "pass@2": 0.35823891436438904, + "pass@4": 0.48848129271052076, + "pass@8": 0.6136605434268556, + "pass@16": 0.7200001692318181, + "pass@32": 0.8001646916555948, + "pass@64": 0.8558966508480305, + "pass@128": 0.8930635838150289 + }, + "16": { + "pass@1": 0.21872479838709677, + "pass@2": 0.31988625539751075, + "pass@4": 0.42930173748442724, + "pass@8": 0.535122776781214, + "pass@16": 0.631811148947321, + "pass@32": 0.7182628887583782, + "pass@64": 0.7924879547356923, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.18007135115606937, + "pass@2": 0.27148997542214703, + "pass@4": 0.37336469203994, + "pass@8": 0.47328118727751295, + "pass@16": 0.564995564823014, + "pass@32": 0.6460617228269897, + "pass@64": 0.7156147681114684, + "pass@128": 0.7774566473988439 + }, + "18": { + "pass@1": 0.16145089285714287, + "pass@2": 0.24092589988751387, + "pass@4": 0.3327022473083723, + "pass@8": 0.42501536664350503, + "pass@16": 0.5082382591955663, + "pass@32": 0.5857258779369915, + "pass@64": 0.659598059312317, + "pass@128": 0.7257142857142858 + }, + "19": { + "pass@1": 0.16737049549549549, + "pass@2": 0.24731769170745538, + "pass@4": 0.3329046515581949, + "pass@8": 0.4120433778670679, + "pass@16": 0.4800331911673055, + "pass@32": 0.5407885727783139, + "pass@64": 0.5994796591091415, + "pass@128": 0.6546546546546547 + }, + "20": { + "pass@1": 0.15202702702702703, + "pass@2": 0.22522079165779937, + "pass@4": 0.3063820921596512, + "pass@8": 0.38469000130086534, + "pass@16": 0.4517700634104969, + "pass@32": 0.5096781724410925, + "pass@64": 0.5629976979045833, + "pass@128": 0.6186186186186187 + } + } + }, + "teachers_in_school": { + "count": 430464, + "correct": 118040, + "answer_accuracy": 0.27421573000297356, + "avg_response_len": 327.60069367008623, + "resp_tokens_sum": 141020305, + "pass_at_k": { + "pass@1": 0.27421573000297356, + "pass@2": 0.3925843945577281, + "pass@4": 0.5112998768775708, + "pass@8": 0.6144344249565099, + "pass@16": 0.6959205865899795, + "pass@32": 0.7575661044799963, + "pass@64": 0.8051344619684987, + "pass@128": 0.8444840915848945 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.4816453313253012, + "pass@2": 0.6459794641163077, + "pass@4": 0.7857084705526265, + "pass@8": 0.8833762506171791, + "pass@16": 0.9416448761628213, + "pass@32": 0.969734804750012, + "pass@64": 0.9807077406498477, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.43245493730407525, + "pass@2": 0.5923997389726756, + "pass@4": 0.7327143862111276, + "pass@8": 0.8373373792935619, + "pass@16": 0.9062656469918041, + "pass@32": 0.9467871615968801, + "pass@64": 0.9707780412767565, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.3606004901960784, + "pass@2": 0.5023848121925938, + "pass@4": 0.637892442016176, + "pass@8": 0.7521662568425521, + "pass@16": 0.838171767362744, + "pass@32": 0.896080984422846, + "pass@64": 0.9310394842754216, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.3340643274853801, + "pass@2": 0.47814751980015674, + "pass@4": 0.6160534801570856, + "pass@8": 0.7248460344568162, + "pass@16": 0.8047331801838976, + "pass@32": 0.8650785721351549, + "pass@64": 0.9105640305812829, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.24607007575757575, + "pass@2": 0.3718403274874728, + "pass@4": 0.5075580916021858, + "pass@8": 0.6319157591570926, + "pass@16": 0.7326543407650808, + "pass@32": 0.8093295131299464, + "pass@64": 0.862885345057764, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.22097578347578348, + "pass@2": 0.3284331045157817, + "pass@4": 0.44146620401082326, + "pass@8": 0.5452602848164503, + "pass@16": 0.6349042776932105, + "pass@32": 0.7107435949774508, + "pass@64": 0.7762919863833435, + "pass@128": 0.8347578347578347 + }, + "17": { + "pass@1": 0.19870923913043478, + "pass@2": 0.29929016175967105, + "pass@4": 0.41198660736119147, + "pass@8": 0.5218542528506436, + "pass@16": 0.6189107868220821, + "pass@32": 0.699994055890072, + "pass@64": 0.7651703780769352, + "pass@128": 0.8229813664596274 + }, + "18": { + "pass@1": 0.1825827205882353, + "pass@2": 0.2713991575961092, + "pass@4": 0.37012361791173165, + "pass@8": 0.4668118891358911, + "pass@16": 0.5509026207660206, + "pass@32": 0.6187246923957278, + "pass@64": 0.6751167026493657, + "pass@128": 0.7235294117647059 + }, + "19": { + "pass@1": 0.15364583333333334, + "pass@2": 0.232394046447319, + "pass@4": 0.3216450493911474, + "pass@8": 0.40932489549476053, + "pass@16": 0.48750145304638565, + "pass@32": 0.5560887876551346, + "pass@64": 0.6210566599372096, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.13440306886227546, + "pass@2": 0.20725723749351688, + "pass@4": 0.2918240431398169, + "pass@8": 0.3752272375730725, + "pass@16": 0.4466647823075647, + "pass@32": 0.5053889559802511, + "pass@64": 0.5591910380893472, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 418816, + "correct": 112683, + "answer_accuracy": 0.2690513256418093, + "avg_response_len": 348.92216390968827, + "resp_tokens_sum": 146134185, + "pass_at_k": { + "pass@1": 0.2690513256418093, + "pass@2": 0.3848278175716666, + "pass@4": 0.501508016141552, + "pass@8": 0.6044382228824874, + "pass@16": 0.6867478007094041, + "pass@32": 0.7499896649312134, + "pass@64": 0.797762386000275, + "pass@128": 0.8325183374083129 + }, + "per_op_pass_at_k": { + "11": { + "pass@1": 0.5126736111111111, + "pass@2": 0.6844453037120355, + "pass@4": 0.8225070392986582, + "pass@8": 0.9125995836439618, + "pass@16": 0.9607014358711359, + "pass@32": 0.9828190225686912, + "pass@64": 0.9912457127693258, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.42168386850152906, + "pass@2": 0.579340265236341, + "pass@4": 0.7219303501626515, + "pass@8": 0.8325867532211753, + "pass@16": 0.9081081464094839, + "pass@32": 0.9536663136810352, + "pass@64": 0.979233886374294, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.3392168209876543, + "pass@2": 0.4790998347428793, + "pass@4": 0.617912207154661, + "pass@8": 0.739523887323168, + "pass@16": 0.831664539596371, + "pass@32": 0.8908797971355734, + "pass@64": 0.9276706490828273, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.30679710682492584, + "pass@2": 0.44024247061847227, + "pass@4": 0.5768535646694014, + "pass@8": 0.7013181044196499, + "pass@16": 0.79981264699903, + "pass@32": 0.8664581226716184, + "pass@64": 0.9075799547708441, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.2421875, + "pass@2": 0.3661280621172356, + "pass@4": 0.49944677488230654, + "pass@8": 0.6221108065117973, + "pass@16": 0.7245293882413184, + "pass@32": 0.8057210991862905, + "pass@64": 0.8623710413077329, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.21748248525073746, + "pass@2": 0.31926890983206774, + "pass@4": 0.4301104165076712, + "pass@8": 0.5368859964094961, + "pass@16": 0.6322487597568095, + "pass@32": 0.7150043647397045, + "pass@64": 0.7834897476011347, + "pass@128": 0.8348082595870207 + }, + "17": { + "pass@1": 0.18879423945783133, + "pass@2": 0.2847512095626601, + "pass@4": 0.3888223292495063, + "pass@8": 0.4851540273539716, + "pass@16": 0.5705429202852202, + "pass@32": 0.6480252422670163, + "pass@64": 0.7148879755094221, + "pass@128": 0.7680722891566265 + }, + "18": { + "pass@1": 0.17016129032258065, + "pass@2": 0.24719527876555752, + "pass@4": 0.3322299300289077, + "pass@8": 0.4185873134747127, + "pass@16": 0.501237620938696, + "pass@32": 0.578221940804949, + "pass@64": 0.6455829940602361, + "pass@128": 0.7032258064516129 + }, + "19": { + "pass@1": 0.16800415407854985, + "pass@2": 0.2506746288983512, + "pass@4": 0.3383495438259038, + "pass@8": 0.4175809793013465, + "pass@16": 0.4823134744100392, + "pass@32": 0.5365725037494888, + "pass@64": 0.5847067346240746, + "pass@128": 0.6314199395770392 + }, + "20": { + "pass@1": 0.13339902402402404, + "pass@2": 0.20877816793171128, + "pass@4": 0.29870241332446046, + "pass@8": 0.38866794275716215, + "pass@16": 0.4652876620865009, + "pass@32": 0.5298770833437524, + "pass@64": 0.5867393530231473, + "pass@128": 0.6306306306306306 + } + } + } + } + }, + "total": { + "answer_accuracy": 0.5062430098684211, + "avg_loss": 0.0, + "count": 2432000, + "validation_metrics": { + "id": {}, + "ood": {} + }, + "pass_at_k": { + "pass@1": 0.5062430098684211, + "pass@2": 0.6140371555117888, + "pass@4": 0.7067027772778162, + "pass@8": 0.7774328538533138, + "pass@16": 0.8284534050661962, + "pass@32": 0.865244798364356, + "pass@64": 0.8926000713533586, + "pass@128": 0.9134210526315789 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.54915625, + "pass@2": 0.7164767470472423, + "pass@4": 0.8426845951443562, + "pass@8": 0.9191123432543775, + "pass@16": 0.9590365811916756, + "pass@32": 0.9779281132253446, + "pass@64": 0.987908506220136, + "pass@128": 0.993 + }, + "2": { + "pass@1": 0.98878125, + "pass@2": 0.9927722687007875, + "pass@4": 0.9940767196287963, + "pass@8": 0.9946724268833085, + "pass@16": 0.9952055668049555, + "pass@32": 0.9957817319358768, + "pass@64": 0.9963772850204733, + "pass@128": 0.997 + }, + "3": { + "pass@1": 0.994609375, + "pass@2": 0.998306102362205, + "pass@4": 0.9992007952755906, + "pass@8": 0.9996691369022104, + "pass@16": 0.9999327111749879, + "pass@32": 0.999997438633278, + "pass@64": 0.9999999996028359, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.916265625, + "pass@2": 0.9581102362204724, + "pass@4": 0.9784218173978244, + "pass@8": 0.9870107311461325, + "pass@16": 0.9912463126427409, + "pass@32": 0.9939197472867504, + "pass@64": 0.9959517707728855, + "pass@128": 0.998 + }, + "5": { + "pass@1": 0.740921875, + "pass@2": 0.8353421505905507, + "pass@4": 0.8950855312148475, + "pass@8": 0.9297393408121918, + "pass@16": 0.9533335123789132, + "pass@32": 0.97132714165803, + "pass@64": 0.9839180181059016, + "pass@128": 0.991 + }, + "6": { + "pass@1": 0.8026484375, + "pass@2": 0.9024115403543312, + "pass@4": 0.9577660202474688, + "pass@8": 0.9815508879309162, + "pass@16": 0.9905558795888216, + "pass@32": 0.9944732576094857, + "pass@64": 0.9964454722777892, + "pass@128": 0.997 + }, + "7": { + "pass@1": 0.6978828125, + "pass@2": 0.8403853346456682, + "pass@4": 0.9284471146419201, + "pass@8": 0.9699026068300013, + "pass@16": 0.9873766827112286, + "pass@32": 0.9952395813054687, + "pass@64": 0.9986726234881326, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.660140625, + "pass@2": 0.8153296013779523, + "pass@4": 0.9184502872140984, + "pass@8": 0.9701473425162639, + "pass@16": 0.9900834472917458, + "pass@32": 0.9963527544566101, + "pass@64": 0.9981845960791449, + "pass@128": 0.999 + }, + "9": { + "pass@1": 0.5648828125, + "pass@2": 0.7356443159448799, + "pass@4": 0.8666676152980888, + "pass@8": 0.9438505591870303, + "pass@16": 0.9785572576870053, + "pass@32": 0.9913928877889056, + "pass@64": 0.9959170400208928, + "pass@128": 0.997 + }, + "11": { + "pass@1": 0.4902265625, + "pass@2": 0.657569389763779, + "pass@4": 0.7954171744469429, + "pass@8": 0.8889310363785922, + "pass@16": 0.9432360609500965, + "pass@32": 0.9703895698123104, + "pass@64": 0.9821512430341022, + "pass@128": 0.987 + }, + "12": { + "pass@1": 0.4189609375, + "pass@2": 0.5766635088582681, + "pass@4": 0.7181645785526807, + "pass@8": 0.8265505995204228, + "pass@16": 0.8996512964671463, + "pass@32": 0.9437508071708404, + "pass@64": 0.9691049665552037, + "pass@128": 0.983 + }, + "13": { + "pass@1": 0.356453125, + "pass@2": 0.5007289616141732, + "pass@4": 0.6386766360142478, + "pass@8": 0.7547754791433346, + "pass@16": 0.8418339989360605, + "pass@32": 0.8993570639455176, + "pass@64": 0.934437073689461, + "pass@128": 0.956 + }, + "14": { + "pass@1": 0.3128203125, + "pass@2": 0.45377066929133836, + "pass@4": 0.596020788620172, + "pass@8": 0.7176963176037108, + "pass@16": 0.8092411950502408, + "pass@32": 0.872027923841253, + "pass@64": 0.9129473155162873, + "pass@128": 0.941 + }, + "15": { + "pass@1": 0.2425625, + "pass@2": 0.36528346456692967, + "pass@4": 0.49832945256842787, + "pass@8": 0.6224226498573554, + "pass@16": 0.7256435127968704, + "pass@32": 0.804989358782077, + "pass@64": 0.8603006224461872, + "pass@128": 0.892 + }, + "16": { + "pass@1": 0.21909375, + "pass@2": 0.32267691929133874, + "pass@4": 0.4338456074240717, + "pass@8": 0.5392787735555705, + "pass@16": 0.6330451872015441, + "pass@32": 0.7145189769989431, + "pass@64": 0.7837527776254032, + "pass@128": 0.839 + }, + "17": { + "pass@1": 0.18896875, + "pass@2": 0.2848443651574808, + "pass@4": 0.39093288432695933, + "pass@8": 0.4928634972974445, + "pass@16": 0.5841979883201639, + "pass@32": 0.6640798225273898, + "pass@64": 0.7313303793764716, + "pass@128": 0.789 + }, + "18": { + "pass@1": 0.1713359375, + "pass@2": 0.2532303149606297, + "pass@4": 0.3452790949568799, + "pass@8": 0.4372334878085905, + "pass@16": 0.5205739442698896, + "pass@32": 0.5946192543420282, + "pass@64": 0.660529727818769, + "pass@128": 0.718 + }, + "19": { + "pass@1": 0.16296875, + "pass@2": 0.2434144931102361, + "pass@4": 0.3309236845706785, + "pass@8": 0.41296291386471917, + "pass@16": 0.4832973009120211, + "pass@32": 0.5445339261283848, + "pass@64": 0.6018396933828155, + "pass@128": 0.657 + }, + "20": { + "pass@1": 0.1399375, + "pass@2": 0.21374557086614168, + "pass@4": 0.298962370734908, + "pass@8": 0.3828540927207293, + "pass@16": 0.4545662598812266, + "pass@32": 0.5149718114737571, + "pass@64": 0.5696322446807763, + "pass@128": 0.621 + } + }, + "per_op_accuracy": { + "10": 0.54915625, + "2": 0.98878125, + "3": 0.994609375, + "4": 0.916265625, + "5": 0.740921875, + "6": 0.8026484375, + "7": 0.6978828125, + "8": 0.660140625, + "9": 0.5648828125, + "11": 0.4902265625, + "12": 0.4189609375, + "13": 0.356453125, + "14": 0.3128203125, + "15": 0.2425625, + "16": 0.21909375, + "17": 0.18896875, + "18": 0.1713359375, + "19": 0.16296875, + "20": 0.1399375 + }, + "per_op_avg_response_len": { + "10": 276.4170703125, + "2": 114.74503125, + "3": 143.584171875, + "4": 158.5899765625, + "5": 171.45425, + "6": 194.59884375, + "7": 220.80815625, + "8": 240.4442578125, + "9": 255.0856171875, + "11": 293.0759296875, + "12": 302.2104765625, + "13": 317.1795625, + "14": 332.7867578125, + "15": 336.7595625, + "16": 341.10715625, + "17": 346.28096875, + "18": 341.680140625, + "19": 342.133359375, + "20": 341.3326015625 + }, + "per_op_avg_loss": {}, + "per_op_length": { + "10": 276.4170703125, + "2": 114.74503125, + "3": 143.584171875, + "4": 158.5899765625, + "5": 171.45425, + "6": 194.59884375, + "7": 220.80815625, + "8": 240.4442578125, + "9": 255.0856171875, + "11": 293.0759296875, + "12": 302.2104765625, + "13": 317.1795625, + "14": 332.7867578125, + "15": 336.7595625, + "16": 341.10715625, + "17": 346.28096875, + "18": 341.680140625, + "19": 342.133359375, + "20": 341.3326015625 + }, + "per_op_loss": {}, + "per_template": { + "crazy_zootopia": { + "count": 809728, + "correct": 401731, + "answer_accuracy": 0.4961307994783433, + "avg_response_len": 254.04621057935503, + "resp_tokens_sum": 205708330, + "pass_at_k": { + "pass@1": 0.4961307994783433, + "pass@2": 0.6048744510842731, + "pass@4": 0.6990247595907915, + "pass@8": 0.7713986345627221, + "pass@16": 0.8238616552594298, + "pass@32": 0.8621028325163171, + "pass@64": 0.8908891087473675, + "pass@128": 0.912741068605754 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5576373525073747, + "pass@2": 0.7241162056070428, + "pass@4": 0.8478356140216987, + "pass@8": 0.9210285105668401, + "pass@16": 0.9590456424231748, + "pass@32": 0.9787901271304692, + "pass@64": 0.990491462519074, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9872225467289719, + "pass@2": 0.992541071822798, + "pass@4": 0.9944339238553123, + "pass@8": 0.9952862602378583, + "pass@16": 0.9961075311262445, + "pass@32": 0.9967298286406606, + "pass@64": 0.996882655029806, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9942660550458715, + "pass@2": 0.9974246159310364, + "pass@4": 0.9983950320430129, + "pass@8": 0.9991873729741889, + "pass@16": 0.999804553543206, + "pass@32": 0.9999921788300652, + "pass@64": 0.999999998785431, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.91865234375, + "pass@2": 0.9625246062992125, + "pass@4": 0.982705639822365, + "pass@8": 0.9912215061732457, + "pass@16": 0.9953870393196059, + "pass@32": 0.9976677753977258, + "pass@64": 0.9991816802247045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7051587301587302, + "pass@2": 0.7998789213848272, + "pass@4": 0.8623743475517939, + "pass@8": 0.902292682084196, + "pass@16": 0.9335744672132729, + "pass@32": 0.961012724109928, + "pass@64": 0.9823888277410862, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.8122427020958084, + "pass@2": 0.9102172132113728, + "pass@4": 0.9629234422044544, + "pass@8": 0.9852725468337608, + "pass@16": 0.9928515702293308, + "pass@32": 0.9955259002838712, + "pass@64": 0.9966392296552004, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.6823687130177515, + "pass@2": 0.83220396729255, + "pass@4": 0.9268846605897633, + "pass@8": 0.9704636354507595, + "pass@16": 0.9878959614025046, + "pass@32": 0.9961612605932577, + "pass@64": 0.9994934131512692, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6389723557692307, + "pass@2": 0.7958146420098927, + "pass@4": 0.9043729308475859, + "pass@8": 0.9630561425068057, + "pass@16": 0.9881072533959968, + "pass@32": 0.9960452808281866, + "pass@64": 0.9983311703038567, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5331646126760563, + "pass@2": 0.7045795441942995, + "pass@4": 0.8440219997148238, + "pass@8": 0.9321960354746593, + "pass@16": 0.9737132718388507, + "pass@32": 0.9898040713041135, + "pass@64": 0.9958648530897064, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.47826664305949007, + "pass@2": 0.6444870736766972, + "pass@4": 0.7803746312872358, + "pass@8": 0.8730347091383628, + "pass@16": 0.9291473364437194, + "pass@32": 0.9599139448333401, + "pass@64": 0.9753934096770958, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.4042858403954802, + "pass@2": 0.5600105237110191, + "pass@4": 0.7015747594580333, + "pass@8": 0.8112545401764185, + "pass@16": 0.8858790711888753, + "pass@32": 0.931855418213947, + "pass@64": 0.9582409337387682, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.3693181818181818, + "pass@2": 0.5208440037765657, + "pass@4": 0.6606441350983167, + "pass@8": 0.7731861628772339, + "pass@16": 0.8562613390543541, + "pass@32": 0.9116335367856937, + "pass@64": 0.9451118730416904, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.29650992990654207, + "pass@2": 0.4420015729634261, + "pass@4": 0.5948001467690369, + "pass@8": 0.727273434984605, + "pass@16": 0.8239426336095855, + "pass@32": 0.8852793296906377, + "pass@64": 0.921121409033376, + "pass@128": 0.940809968847352 + }, + "15": { + "pass@1": 0.23956828034682082, + "pass@2": 0.35823891436438904, + "pass@4": 0.48848129271052076, + "pass@8": 0.6136605434268556, + "pass@16": 0.7200001692318181, + "pass@32": 0.8001646916555948, + "pass@64": 0.8558966508480305, + "pass@128": 0.8930635838150289 + }, + "16": { + "pass@1": 0.21872479838709677, + "pass@2": 0.31988625539751075, + "pass@4": 0.42930173748442724, + "pass@8": 0.535122776781214, + "pass@16": 0.631811148947321, + "pass@32": 0.7182628887583782, + "pass@64": 0.7924879547356923, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.18007135115606937, + "pass@2": 0.27148997542214703, + "pass@4": 0.37336469203994, + "pass@8": 0.47328118727751295, + "pass@16": 0.564995564823014, + "pass@32": 0.6460617228269897, + "pass@64": 0.7156147681114684, + "pass@128": 0.7774566473988439 + }, + "18": { + "pass@1": 0.16145089285714287, + "pass@2": 0.24092589988751387, + "pass@4": 0.3327022473083723, + "pass@8": 0.42501536664350503, + "pass@16": 0.5082382591955663, + "pass@32": 0.5857258779369915, + "pass@64": 0.659598059312317, + "pass@128": 0.7257142857142858 + }, + "19": { + "pass@1": 0.16737049549549549, + "pass@2": 0.24731769170745538, + "pass@4": 0.3329046515581949, + "pass@8": 0.4120433778670679, + "pass@16": 0.4800331911673055, + "pass@32": 0.5407885727783139, + "pass@64": 0.5994796591091415, + "pass@128": 0.6546546546546547 + }, + "20": { + "pass@1": 0.15202702702702703, + "pass@2": 0.22522079165779937, + "pass@4": 0.3063820921596512, + "pass@8": 0.38469000130086534, + "pass@16": 0.4517700634104969, + "pass@32": 0.5096781724410925, + "pass@64": 0.5629976979045833, + "pass@128": 0.6186186186186187 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 419277, + "answer_accuracy": 0.51061598791894, + "avg_response_len": 264.6041577357755, + "resp_tokens_sum": 217271766, + "pass_at_k": { + "pass@1": 0.51061598791894, + "pass@2": 0.6185059891310457, + "pass@4": 0.7112559664891815, + "pass@8": 0.7814887206768086, + "pass@16": 0.8317485000078719, + "pass@32": 0.8676199270006473, + "pass@64": 0.8943931087557251, + "pass@128": 0.9159781761496493 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5151457055214724, + "pass@2": 0.6830678469639145, + "pass@4": 0.8189656213410438, + "pass@8": 0.9071709306839472, + "pass@16": 0.9537003905148077, + "pass@32": 0.9732703621233486, + "pass@64": 0.9815479508426936, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 0.9870820668693009, + "pass@2": 0.9916114209128114, + "pass@4": 0.9932387670766079, + "pass@8": 0.9939459341067186, + "pass@16": 0.9942939216780183, + "pass@32": 0.9946808461232393, + "pass@64": 0.9954407294832827, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9931344696969697, + "pass@2": 0.9978040742066334, + "pass@4": 0.9991804419333946, + "pass@8": 0.9998026414887873, + "pass@16": 0.9999897641404931, + "pass@32": 0.9999999883510501, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9130917159763313, + "pass@2": 0.9530991561058564, + "pass@4": 0.9734309767307188, + "pass@8": 0.981508303989163, + "pass@16": 0.9852970136399636, + "pass@32": 0.9883859742602302, + "pass@64": 0.9910152794130401, + "pass@128": 0.9940828402366864 + }, + "5": { + "pass@1": 0.7668704710144928, + "pass@2": 0.8570691401346572, + "pass@4": 0.9105642180596992, + "pass@8": 0.9386731501468899, + "pass@16": 0.9567025653999209, + "pass@32": 0.9706738161299903, + "pass@64": 0.980097445623169, + "pass@128": 0.9855072463768116 + }, + "6": { + "pass@1": 0.8068812679083095, + "pass@2": 0.9043202738984274, + "pass@4": 0.9584084456312574, + "pass@8": 0.9802832598182536, + "pass@16": 0.9885671782379833, + "pass@32": 0.9930770664664508, + "pass@64": 0.9960588198590073, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.7005735187861272, + "pass@2": 0.8407154601520194, + "pass@4": 0.9295946147085653, + "pass@8": 0.9719966626529152, + "pass@16": 0.9888768016805433, + "pass@32": 0.9956843233080916, + "pass@64": 0.9985466054756086, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6794454225352112, + "pass@2": 0.8303634800931575, + "pass@4": 0.925194801354056, + "pass@8": 0.9693069551068682, + "pass@16": 0.9866333068822101, + "pass@32": 0.9936383088194044, + "pass@64": 0.9963589210449596, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.5826160179640718, + "pass@2": 0.7489468651280118, + "pass@4": 0.8742062343254993, + "pass@8": 0.9464091288357557, + "pass@16": 0.978296787242704, + "pass@32": 0.9903044676743343, + "pass@64": 0.995371590764158, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.4816453313253012, + "pass@2": 0.6459794641163077, + "pass@4": 0.7857084705526265, + "pass@8": 0.8833762506171791, + "pass@16": 0.9416448761628213, + "pass@32": 0.969734804750012, + "pass@64": 0.9807077406498477, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.43245493730407525, + "pass@2": 0.5923997389726756, + "pass@4": 0.7327143862111276, + "pass@8": 0.8373373792935619, + "pass@16": 0.9062656469918041, + "pass@32": 0.9467871615968801, + "pass@64": 0.9707780412767565, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.3606004901960784, + "pass@2": 0.5023848121925938, + "pass@4": 0.637892442016176, + "pass@8": 0.7521662568425521, + "pass@16": 0.838171767362744, + "pass@32": 0.896080984422846, + "pass@64": 0.9310394842754216, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.3340643274853801, + "pass@2": 0.47814751980015674, + "pass@4": 0.6160534801570856, + "pass@8": 0.7248460344568162, + "pass@16": 0.8047331801838976, + "pass@32": 0.8650785721351549, + "pass@64": 0.9105640305812829, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.24607007575757575, + "pass@2": 0.3718403274874728, + "pass@4": 0.5075580916021858, + "pass@8": 0.6319157591570926, + "pass@16": 0.7326543407650808, + "pass@32": 0.8093295131299464, + "pass@64": 0.862885345057764, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.22097578347578348, + "pass@2": 0.3284331045157817, + "pass@4": 0.44146620401082326, + "pass@8": 0.5452602848164503, + "pass@16": 0.6349042776932105, + "pass@32": 0.7107435949774508, + "pass@64": 0.7762919863833435, + "pass@128": 0.8347578347578347 + }, + "17": { + "pass@1": 0.19870923913043478, + "pass@2": 0.29929016175967105, + "pass@4": 0.41198660736119147, + "pass@8": 0.5218542528506436, + "pass@16": 0.6189107868220821, + "pass@32": 0.699994055890072, + "pass@64": 0.7651703780769352, + "pass@128": 0.8229813664596274 + }, + "18": { + "pass@1": 0.1825827205882353, + "pass@2": 0.2713991575961092, + "pass@4": 0.37012361791173165, + "pass@8": 0.4668118891358911, + "pass@16": 0.5509026207660206, + "pass@32": 0.6187246923957278, + "pass@64": 0.6751167026493657, + "pass@128": 0.7235294117647059 + }, + "19": { + "pass@1": 0.15364583333333334, + "pass@2": 0.232394046447319, + "pass@4": 0.3216450493911474, + "pass@8": 0.40932489549476053, + "pass@16": 0.48750145304638565, + "pass@32": 0.5560887876551346, + "pass@64": 0.6210566599372096, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.13440306886227546, + "pass@2": 0.20725723749351688, + "pass@4": 0.2918240431398169, + "pass@8": 0.3752272375730725, + "pass@16": 0.4466647823075647, + "pass@32": 0.5053889559802511, + "pass@64": 0.5591910380893472, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 410175, + "answer_accuracy": 0.5119814966448314, + "avg_response_len": 282.1124605567982, + "resp_tokens_sum": 226014962, + "pass_at_k": { + "pass@1": 0.5119814966448314, + "pass@2": 0.6187177275746641, + "pass@4": 0.7097963115640613, + "pass@8": 0.7793747112681776, + "pass@16": 0.8297170854027788, + "pass@32": 0.8659860710520697, + "pass@64": 0.8924916218421354, + "pass@128": 0.911487458060393 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5736707089552239, + "pass@2": 0.7412574186155834, + "pass@4": 0.8605538192054352, + "pass@8": 0.9287938948335882, + "pass@16": 0.9642202420966983, + "pass@32": 0.981588424107176, + "pass@64": 0.9914843834371672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9918080357142857, + "pass@2": 0.9940755061867266, + "pass@4": 0.9945367877229631, + "pass@8": 0.9947923572167008, + "pass@16": 0.9952352830896093, + "pass@32": 0.9959470244790829, + "pass@64": 0.9967941507311587, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9963556851311953, + "pass@2": 0.9996294707421778, + "pass@4": 0.9999885538177989, + "pass@8": 0.9999999832313435, + "pass@16": 0.9999999999999907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9171692251461988, + "pass@2": 0.958932302458904, + "pass@4": 0.9793460307593129, + "pass@8": 0.9885088959718029, + "pass@16": 0.9932516650589491, + "pass@32": 0.9958818709342699, + "pass@64": 0.9978083878929018, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7477251838235294, + "pass@2": 0.8461512853172767, + "pass@4": 0.9096851661924609, + "pass@8": 0.9461026739852725, + "pass@16": 0.9682210886581221, + "pass@32": 0.9815460794075175, + "pass@64": 0.9892115253749012, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.7878795347003155, + "pass@2": 0.892085853473757, + "pass@4": 0.9516247414065356, + "pass@8": 0.9790252353686714, + "pass@16": 0.9903265297387072, + "pass@32": 0.9949012956400033, + "pass@64": 0.9966670077039709, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.7115308544303798, + "pass@2": 0.8487748243297122, + "pass@4": 0.9288619071943541, + "pass@8": 0.9670096606637208, + "pass@16": 0.9851787163788448, + "pass@32": 0.9937667701276847, + "pass@64": 0.9979326719887415, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6593937687687688, + "pass@2": 0.8175867796930787, + "pass@4": 0.9244497847115954, + "pass@8": 0.9776872582318384, + "pass@16": 0.9956130939609944, + "pass@32": 0.9995346162382196, + "pass@64": 0.9999935554756175, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5820438102893891, + "pass@2": 0.7568177646403522, + "pass@4": 0.8844209747093505, + "pass@8": 0.9544061657954517, + "pass@16": 0.9843662998236358, + "pass@32": 0.994375402806163, + "pass@64": 0.9965623982598637, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.5126736111111111, + "pass@2": 0.6844453037120355, + "pass@4": 0.8225070392986582, + "pass@8": 0.9125995836439618, + "pass@16": 0.9607014358711359, + "pass@32": 0.9828190225686912, + "pass@64": 0.9912457127693258, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.42168386850152906, + "pass@2": 0.579340265236341, + "pass@4": 0.7219303501626515, + "pass@8": 0.8325867532211753, + "pass@16": 0.9081081464094839, + "pass@32": 0.9536663136810352, + "pass@64": 0.979233886374294, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.3392168209876543, + "pass@2": 0.4790998347428793, + "pass@4": 0.617912207154661, + "pass@8": 0.739523887323168, + "pass@16": 0.831664539596371, + "pass@32": 0.8908797971355734, + "pass@64": 0.9276706490828273, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.30679710682492584, + "pass@2": 0.44024247061847227, + "pass@4": 0.5768535646694014, + "pass@8": 0.7013181044196499, + "pass@16": 0.79981264699903, + "pass@32": 0.8664581226716184, + "pass@64": 0.9075799547708441, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.2421875, + "pass@2": 0.3661280621172356, + "pass@4": 0.49944677488230654, + "pass@8": 0.6221108065117973, + "pass@16": 0.7245293882413184, + "pass@32": 0.8057210991862905, + "pass@64": 0.8623710413077329, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.21748248525073746, + "pass@2": 0.31926890983206774, + "pass@4": 0.4301104165076712, + "pass@8": 0.5368859964094961, + "pass@16": 0.6322487597568095, + "pass@32": 0.7150043647397045, + "pass@64": 0.7834897476011347, + "pass@128": 0.8348082595870207 + }, + "17": { + "pass@1": 0.18879423945783133, + "pass@2": 0.2847512095626601, + "pass@4": 0.3888223292495063, + "pass@8": 0.4851540273539716, + "pass@16": 0.5705429202852202, + "pass@32": 0.6480252422670163, + "pass@64": 0.7148879755094221, + "pass@128": 0.7680722891566265 + }, + "18": { + "pass@1": 0.17016129032258065, + "pass@2": 0.24719527876555752, + "pass@4": 0.3322299300289077, + "pass@8": 0.4185873134747127, + "pass@16": 0.501237620938696, + "pass@32": 0.578221940804949, + "pass@64": 0.6455829940602361, + "pass@128": 0.7032258064516129 + }, + "19": { + "pass@1": 0.16800415407854985, + "pass@2": 0.2506746288983512, + "pass@4": 0.3383495438259038, + "pass@8": 0.4175809793013465, + "pass@16": 0.4823134744100392, + "pass@32": 0.5365725037494888, + "pass@64": 0.5847067346240746, + "pass@128": 0.6314199395770392 + }, + "20": { + "pass@1": 0.13339902402402404, + "pass@2": 0.20877816793171128, + "pass@4": 0.29870241332446046, + "pass@8": 0.38866794275716215, + "pass@16": 0.4652876620865009, + "pass@32": 0.5298770833437524, + "pass@64": 0.5867393530231473, + "pass@128": 0.6306306306306306 + } + } + } + } + } +} \ No newline at end of file diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/summary.csv b/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/summary.csv new file mode 100644 index 0000000000000000000000000000000000000000..32f855b1c1ff3284fb02b171e265acc1ae4721bc --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/summary.csv @@ -0,0 +1,2 @@ +checkpoint,id_acc,ood_acc,total_acc,id_avg_loss,ood_avg_loss,total_avg_loss,id_avg_resp_len,ood_avg_resp_len,count_id,count_ood,count_total +checkpoint-85656,0.7683654513888889,0.2703328125,0.5062430098684211,0.0,0.0,0.0,197.30304166666667,329.4546515625,1152000,1280000,2432000 diff --git a/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/summary.json b/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dc12ace2421bd65783ff27b2d2ef38ed2544dff9 --- /dev/null +++ b/SDAR-100M-ep4-block4/composition/low_confidence_static/block8/summary.json @@ -0,0 +1,645 @@ +[ + { + "checkpoint": "checkpoint-85656", + "id_acc": 0.7683654513888889, + "ood_acc": 0.2703328125, + "total_acc": 0.5062430098684211, + "id_avg_loss": 0.0, + "ood_avg_loss": 0.0, + "total_avg_loss": 0.0, + "id_avg_resp_len": 197.30304166666667, + "ood_avg_resp_len": 329.4546515625, + "count_id": 1152000, + "count_ood": 1280000, + "count_total": 2432000, + "template_metrics": { + "crazy_zootopia": { + "count": 809728, + "correct": 401731, + "answer_accuracy": 0.4961307994783433, + "avg_response_len": 254.04621057935503, + "resp_tokens_sum": 205708330, + "pass_at_k": { + "pass@1": 0.4961307994783433, + "pass@2": 0.6048744510842731, + "pass@4": 0.6990247595907915, + "pass@8": 0.7713986345627221, + "pass@16": 0.8238616552594298, + "pass@32": 0.8621028325163171, + "pass@64": 0.8908891087473675, + "pass@128": 0.912741068605754 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5576373525073747, + "pass@2": 0.7241162056070428, + "pass@4": 0.8478356140216987, + "pass@8": 0.9210285105668401, + "pass@16": 0.9590456424231748, + "pass@32": 0.9787901271304692, + "pass@64": 0.990491462519074, + "pass@128": 0.9970501474926253 + }, + "2": { + "pass@1": 0.9872225467289719, + "pass@2": 0.992541071822798, + "pass@4": 0.9944339238553123, + "pass@8": 0.9952862602378583, + "pass@16": 0.9961075311262445, + "pass@32": 0.9967298286406606, + "pass@64": 0.996882655029806, + "pass@128": 0.9968847352024922 + }, + "3": { + "pass@1": 0.9942660550458715, + "pass@2": 0.9974246159310364, + "pass@4": 0.9983950320430129, + "pass@8": 0.9991873729741889, + "pass@16": 0.999804553543206, + "pass@32": 0.9999921788300652, + "pass@64": 0.999999998785431, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.91865234375, + "pass@2": 0.9625246062992125, + "pass@4": 0.982705639822365, + "pass@8": 0.9912215061732457, + "pass@16": 0.9953870393196059, + "pass@32": 0.9976677753977258, + "pass@64": 0.9991816802247045, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7051587301587302, + "pass@2": 0.7998789213848272, + "pass@4": 0.8623743475517939, + "pass@8": 0.902292682084196, + "pass@16": 0.9335744672132729, + "pass@32": 0.961012724109928, + "pass@64": 0.9823888277410862, + "pass@128": 0.9936507936507937 + }, + "6": { + "pass@1": 0.8122427020958084, + "pass@2": 0.9102172132113728, + "pass@4": 0.9629234422044544, + "pass@8": 0.9852725468337608, + "pass@16": 0.9928515702293308, + "pass@32": 0.9955259002838712, + "pass@64": 0.9966392296552004, + "pass@128": 0.9970059880239521 + }, + "7": { + "pass@1": 0.6823687130177515, + "pass@2": 0.83220396729255, + "pass@4": 0.9268846605897633, + "pass@8": 0.9704636354507595, + "pass@16": 0.9878959614025046, + "pass@32": 0.9961612605932577, + "pass@64": 0.9994934131512692, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6389723557692307, + "pass@2": 0.7958146420098927, + "pass@4": 0.9043729308475859, + "pass@8": 0.9630561425068057, + "pass@16": 0.9881072533959968, + "pass@32": 0.9960452808281866, + "pass@64": 0.9983311703038567, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5331646126760563, + "pass@2": 0.7045795441942995, + "pass@4": 0.8440219997148238, + "pass@8": 0.9321960354746593, + "pass@16": 0.9737132718388507, + "pass@32": 0.9898040713041135, + "pass@64": 0.9958648530897064, + "pass@128": 0.9971830985915493 + }, + "11": { + "pass@1": 0.47826664305949007, + "pass@2": 0.6444870736766972, + "pass@4": 0.7803746312872358, + "pass@8": 0.8730347091383628, + "pass@16": 0.9291473364437194, + "pass@32": 0.9599139448333401, + "pass@64": 0.9753934096770958, + "pass@128": 0.9830028328611898 + }, + "12": { + "pass@1": 0.4042858403954802, + "pass@2": 0.5600105237110191, + "pass@4": 0.7015747594580333, + "pass@8": 0.8112545401764185, + "pass@16": 0.8858790711888753, + "pass@32": 0.931855418213947, + "pass@64": 0.9582409337387682, + "pass@128": 0.9717514124293786 + }, + "13": { + "pass@1": 0.3693181818181818, + "pass@2": 0.5208440037765657, + "pass@4": 0.6606441350983167, + "pass@8": 0.7731861628772339, + "pass@16": 0.8562613390543541, + "pass@32": 0.9116335367856937, + "pass@64": 0.9451118730416904, + "pass@128": 0.9623824451410659 + }, + "14": { + "pass@1": 0.29650992990654207, + "pass@2": 0.4420015729634261, + "pass@4": 0.5948001467690369, + "pass@8": 0.727273434984605, + "pass@16": 0.8239426336095855, + "pass@32": 0.8852793296906377, + "pass@64": 0.921121409033376, + "pass@128": 0.940809968847352 + }, + "15": { + "pass@1": 0.23956828034682082, + "pass@2": 0.35823891436438904, + "pass@4": 0.48848129271052076, + "pass@8": 0.6136605434268556, + "pass@16": 0.7200001692318181, + "pass@32": 0.8001646916555948, + "pass@64": 0.8558966508480305, + "pass@128": 0.8930635838150289 + }, + "16": { + "pass@1": 0.21872479838709677, + "pass@2": 0.31988625539751075, + "pass@4": 0.42930173748442724, + "pass@8": 0.535122776781214, + "pass@16": 0.631811148947321, + "pass@32": 0.7182628887583782, + "pass@64": 0.7924879547356923, + "pass@128": 0.8483870967741935 + }, + "17": { + "pass@1": 0.18007135115606937, + "pass@2": 0.27148997542214703, + "pass@4": 0.37336469203994, + "pass@8": 0.47328118727751295, + "pass@16": 0.564995564823014, + "pass@32": 0.6460617228269897, + "pass@64": 0.7156147681114684, + "pass@128": 0.7774566473988439 + }, + "18": { + "pass@1": 0.16145089285714287, + "pass@2": 0.24092589988751387, + "pass@4": 0.3327022473083723, + "pass@8": 0.42501536664350503, + "pass@16": 0.5082382591955663, + "pass@32": 0.5857258779369915, + "pass@64": 0.659598059312317, + "pass@128": 0.7257142857142858 + }, + "19": { + "pass@1": 0.16737049549549549, + "pass@2": 0.24731769170745538, + "pass@4": 0.3329046515581949, + "pass@8": 0.4120433778670679, + "pass@16": 0.4800331911673055, + "pass@32": 0.5407885727783139, + "pass@64": 0.5994796591091415, + "pass@128": 0.6546546546546547 + }, + "20": { + "pass@1": 0.15202702702702703, + "pass@2": 0.22522079165779937, + "pass@4": 0.3063820921596512, + "pass@8": 0.38469000130086534, + "pass@16": 0.4517700634104969, + "pass@32": 0.5096781724410925, + "pass@64": 0.5629976979045833, + "pass@128": 0.6186186186186187 + } + } + }, + "teachers_in_school": { + "count": 821120, + "correct": 419277, + "answer_accuracy": 0.51061598791894, + "avg_response_len": 264.6041577357755, + "resp_tokens_sum": 217271766, + "pass_at_k": { + "pass@1": 0.51061598791894, + "pass@2": 0.6185059891310457, + "pass@4": 0.7112559664891815, + "pass@8": 0.7814887206768086, + "pass@16": 0.8317485000078719, + "pass@32": 0.8676199270006473, + "pass@64": 0.8943931087557251, + "pass@128": 0.9159781761496493 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5151457055214724, + "pass@2": 0.6830678469639145, + "pass@4": 0.8189656213410438, + "pass@8": 0.9071709306839472, + "pass@16": 0.9537003905148077, + "pass@32": 0.9732703621233486, + "pass@64": 0.9815479508426936, + "pass@128": 0.9846625766871165 + }, + "2": { + "pass@1": 0.9870820668693009, + "pass@2": 0.9916114209128114, + "pass@4": 0.9932387670766079, + "pass@8": 0.9939459341067186, + "pass@16": 0.9942939216780183, + "pass@32": 0.9946808461232393, + "pass@64": 0.9954407294832827, + "pass@128": 0.9969604863221885 + }, + "3": { + "pass@1": 0.9931344696969697, + "pass@2": 0.9978040742066334, + "pass@4": 0.9991804419333946, + "pass@8": 0.9998026414887873, + "pass@16": 0.9999897641404931, + "pass@32": 0.9999999883510501, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9130917159763313, + "pass@2": 0.9530991561058564, + "pass@4": 0.9734309767307188, + "pass@8": 0.981508303989163, + "pass@16": 0.9852970136399636, + "pass@32": 0.9883859742602302, + "pass@64": 0.9910152794130401, + "pass@128": 0.9940828402366864 + }, + "5": { + "pass@1": 0.7668704710144928, + "pass@2": 0.8570691401346572, + "pass@4": 0.9105642180596992, + "pass@8": 0.9386731501468899, + "pass@16": 0.9567025653999209, + "pass@32": 0.9706738161299903, + "pass@64": 0.980097445623169, + "pass@128": 0.9855072463768116 + }, + "6": { + "pass@1": 0.8068812679083095, + "pass@2": 0.9043202738984274, + "pass@4": 0.9584084456312574, + "pass@8": 0.9802832598182536, + "pass@16": 0.9885671782379833, + "pass@32": 0.9930770664664508, + "pass@64": 0.9960588198590073, + "pass@128": 0.997134670487106 + }, + "7": { + "pass@1": 0.7005735187861272, + "pass@2": 0.8407154601520194, + "pass@4": 0.9295946147085653, + "pass@8": 0.9719966626529152, + "pass@16": 0.9888768016805433, + "pass@32": 0.9956843233080916, + "pass@64": 0.9985466054756086, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6794454225352112, + "pass@2": 0.8303634800931575, + "pass@4": 0.925194801354056, + "pass@8": 0.9693069551068682, + "pass@16": 0.9866333068822101, + "pass@32": 0.9936383088194044, + "pass@64": 0.9963589210449596, + "pass@128": 0.9971830985915493 + }, + "9": { + "pass@1": 0.5826160179640718, + "pass@2": 0.7489468651280118, + "pass@4": 0.8742062343254993, + "pass@8": 0.9464091288357557, + "pass@16": 0.978296787242704, + "pass@32": 0.9903044676743343, + "pass@64": 0.995371590764158, + "pass@128": 0.9970059880239521 + }, + "11": { + "pass@1": 0.4816453313253012, + "pass@2": 0.6459794641163077, + "pass@4": 0.7857084705526265, + "pass@8": 0.8833762506171791, + "pass@16": 0.9416448761628213, + "pass@32": 0.969734804750012, + "pass@64": 0.9807077406498477, + "pass@128": 0.9849397590361446 + }, + "12": { + "pass@1": 0.43245493730407525, + "pass@2": 0.5923997389726756, + "pass@4": 0.7327143862111276, + "pass@8": 0.8373373792935619, + "pass@16": 0.9062656469918041, + "pass@32": 0.9467871615968801, + "pass@64": 0.9707780412767565, + "pass@128": 0.9843260188087775 + }, + "13": { + "pass@1": 0.3606004901960784, + "pass@2": 0.5023848121925938, + "pass@4": 0.637892442016176, + "pass@8": 0.7521662568425521, + "pass@16": 0.838171767362744, + "pass@32": 0.896080984422846, + "pass@64": 0.9310394842754216, + "pass@128": 0.9523809523809523 + }, + "14": { + "pass@1": 0.3340643274853801, + "pass@2": 0.47814751980015674, + "pass@4": 0.6160534801570856, + "pass@8": 0.7248460344568162, + "pass@16": 0.8047331801838976, + "pass@32": 0.8650785721351549, + "pass@64": 0.9105640305812829, + "pass@128": 0.9502923976608187 + }, + "15": { + "pass@1": 0.24607007575757575, + "pass@2": 0.3718403274874728, + "pass@4": 0.5075580916021858, + "pass@8": 0.6319157591570926, + "pass@16": 0.7326543407650808, + "pass@32": 0.8093295131299464, + "pass@64": 0.862885345057764, + "pass@128": 0.8939393939393939 + }, + "16": { + "pass@1": 0.22097578347578348, + "pass@2": 0.3284331045157817, + "pass@4": 0.44146620401082326, + "pass@8": 0.5452602848164503, + "pass@16": 0.6349042776932105, + "pass@32": 0.7107435949774508, + "pass@64": 0.7762919863833435, + "pass@128": 0.8347578347578347 + }, + "17": { + "pass@1": 0.19870923913043478, + "pass@2": 0.29929016175967105, + "pass@4": 0.41198660736119147, + "pass@8": 0.5218542528506436, + "pass@16": 0.6189107868220821, + "pass@32": 0.699994055890072, + "pass@64": 0.7651703780769352, + "pass@128": 0.8229813664596274 + }, + "18": { + "pass@1": 0.1825827205882353, + "pass@2": 0.2713991575961092, + "pass@4": 0.37012361791173165, + "pass@8": 0.4668118891358911, + "pass@16": 0.5509026207660206, + "pass@32": 0.6187246923957278, + "pass@64": 0.6751167026493657, + "pass@128": 0.7235294117647059 + }, + "19": { + "pass@1": 0.15364583333333334, + "pass@2": 0.232394046447319, + "pass@4": 0.3216450493911474, + "pass@8": 0.40932489549476053, + "pass@16": 0.48750145304638565, + "pass@32": 0.5560887876551346, + "pass@64": 0.6210566599372096, + "pass@128": 0.6845238095238095 + }, + "20": { + "pass@1": 0.13440306886227546, + "pass@2": 0.20725723749351688, + "pass@4": 0.2918240431398169, + "pass@8": 0.3752272375730725, + "pass@16": 0.4466647823075647, + "pass@32": 0.5053889559802511, + "pass@64": 0.5591910380893472, + "pass@128": 0.6137724550898204 + } + } + }, + "movie_festival_awards": { + "count": 801152, + "correct": 410175, + "answer_accuracy": 0.5119814966448314, + "avg_response_len": 282.1124605567982, + "resp_tokens_sum": 226014962, + "pass_at_k": { + "pass@1": 0.5119814966448314, + "pass@2": 0.6187177275746641, + "pass@4": 0.7097963115640613, + "pass@8": 0.7793747112681776, + "pass@16": 0.8297170854027788, + "pass@32": 0.8659860710520697, + "pass@64": 0.8924916218421354, + "pass@128": 0.911487458060393 + }, + "per_op_pass_at_k": { + "10": { + "pass@1": 0.5736707089552239, + "pass@2": 0.7412574186155834, + "pass@4": 0.8605538192054352, + "pass@8": 0.9287938948335882, + "pass@16": 0.9642202420966983, + "pass@32": 0.981588424107176, + "pass@64": 0.9914843834371672, + "pass@128": 0.9970149253731343 + }, + "2": { + "pass@1": 0.9918080357142857, + "pass@2": 0.9940755061867266, + "pass@4": 0.9945367877229631, + "pass@8": 0.9947923572167008, + "pass@16": 0.9952352830896093, + "pass@32": 0.9959470244790829, + "pass@64": 0.9967941507311587, + "pass@128": 0.9971428571428571 + }, + "3": { + "pass@1": 0.9963556851311953, + "pass@2": 0.9996294707421778, + "pass@4": 0.9999885538177989, + "pass@8": 0.9999999832313435, + "pass@16": 0.9999999999999907, + "pass@32": 1.0, + "pass@64": 1.0, + "pass@128": 1.0 + }, + "4": { + "pass@1": 0.9171692251461988, + "pass@2": 0.958932302458904, + "pass@4": 0.9793460307593129, + "pass@8": 0.9885088959718029, + "pass@16": 0.9932516650589491, + "pass@32": 0.9958818709342699, + "pass@64": 0.9978083878929018, + "pass@128": 1.0 + }, + "5": { + "pass@1": 0.7477251838235294, + "pass@2": 0.8461512853172767, + "pass@4": 0.9096851661924609, + "pass@8": 0.9461026739852725, + "pass@16": 0.9682210886581221, + "pass@32": 0.9815460794075175, + "pass@64": 0.9892115253749012, + "pass@128": 0.9941176470588236 + }, + "6": { + "pass@1": 0.7878795347003155, + "pass@2": 0.892085853473757, + "pass@4": 0.9516247414065356, + "pass@8": 0.9790252353686714, + "pass@16": 0.9903265297387072, + "pass@32": 0.9949012956400033, + "pass@64": 0.9966670077039709, + "pass@128": 0.9968454258675079 + }, + "7": { + "pass@1": 0.7115308544303798, + "pass@2": 0.8487748243297122, + "pass@4": 0.9288619071943541, + "pass@8": 0.9670096606637208, + "pass@16": 0.9851787163788448, + "pass@32": 0.9937667701276847, + "pass@64": 0.9979326719887415, + "pass@128": 1.0 + }, + "8": { + "pass@1": 0.6593937687687688, + "pass@2": 0.8175867796930787, + "pass@4": 0.9244497847115954, + "pass@8": 0.9776872582318384, + "pass@16": 0.9956130939609944, + "pass@32": 0.9995346162382196, + "pass@64": 0.9999935554756175, + "pass@128": 1.0 + }, + "9": { + "pass@1": 0.5820438102893891, + "pass@2": 0.7568177646403522, + "pass@4": 0.8844209747093505, + "pass@8": 0.9544061657954517, + "pass@16": 0.9843662998236358, + "pass@32": 0.994375402806163, + "pass@64": 0.9965623982598637, + "pass@128": 0.9967845659163987 + }, + "11": { + "pass@1": 0.5126736111111111, + "pass@2": 0.6844453037120355, + "pass@4": 0.8225070392986582, + "pass@8": 0.9125995836439618, + "pass@16": 0.9607014358711359, + "pass@32": 0.9828190225686912, + "pass@64": 0.9912457127693258, + "pass@128": 0.9936507936507937 + }, + "12": { + "pass@1": 0.42168386850152906, + "pass@2": 0.579340265236341, + "pass@4": 0.7219303501626515, + "pass@8": 0.8325867532211753, + "pass@16": 0.9081081464094839, + "pass@32": 0.9536663136810352, + "pass@64": 0.979233886374294, + "pass@128": 0.9938837920489296 + }, + "13": { + "pass@1": 0.3392168209876543, + "pass@2": 0.4790998347428793, + "pass@4": 0.617912207154661, + "pass@8": 0.739523887323168, + "pass@16": 0.831664539596371, + "pass@32": 0.8908797971355734, + "pass@64": 0.9276706490828273, + "pass@128": 0.9537037037037037 + }, + "14": { + "pass@1": 0.30679710682492584, + "pass@2": 0.44024247061847227, + "pass@4": 0.5768535646694014, + "pass@8": 0.7013181044196499, + "pass@16": 0.79981264699903, + "pass@32": 0.8664581226716184, + "pass@64": 0.9075799547708441, + "pass@128": 0.9317507418397626 + }, + "15": { + "pass@1": 0.2421875, + "pass@2": 0.3661280621172356, + "pass@4": 0.49944677488230654, + "pass@8": 0.6221108065117973, + "pass@16": 0.7245293882413184, + "pass@32": 0.8057210991862905, + "pass@64": 0.8623710413077329, + "pass@128": 0.8888888888888888 + }, + "16": { + "pass@1": 0.21748248525073746, + "pass@2": 0.31926890983206774, + "pass@4": 0.4301104165076712, + "pass@8": 0.5368859964094961, + "pass@16": 0.6322487597568095, + "pass@32": 0.7150043647397045, + "pass@64": 0.7834897476011347, + "pass@128": 0.8348082595870207 + }, + "17": { + "pass@1": 0.18879423945783133, + "pass@2": 0.2847512095626601, + "pass@4": 0.3888223292495063, + "pass@8": 0.4851540273539716, + "pass@16": 0.5705429202852202, + "pass@32": 0.6480252422670163, + "pass@64": 0.7148879755094221, + "pass@128": 0.7680722891566265 + }, + "18": { + "pass@1": 0.17016129032258065, + "pass@2": 0.24719527876555752, + "pass@4": 0.3322299300289077, + "pass@8": 0.4185873134747127, + "pass@16": 0.501237620938696, + "pass@32": 0.578221940804949, + "pass@64": 0.6455829940602361, + "pass@128": 0.7032258064516129 + }, + "19": { + "pass@1": 0.16800415407854985, + "pass@2": 0.2506746288983512, + "pass@4": 0.3383495438259038, + "pass@8": 0.4175809793013465, + "pass@16": 0.4823134744100392, + "pass@32": 0.5365725037494888, + "pass@64": 0.5847067346240746, + "pass@128": 0.6314199395770392 + }, + "20": { + "pass@1": 0.13339902402402404, + "pass@2": 0.20877816793171128, + "pass@4": 0.29870241332446046, + "pass@8": 0.38866794275716215, + "pass@16": 0.4652876620865009, + "pass@32": 0.5298770833437524, + "pass@64": 0.5867393530231473, + "pass@128": 0.6306306306306306 + } + } + } + } + } +] \ No newline at end of file