{ "files": { "checkpoint-42637_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99375, "pass@2": 0.99375, "pass@4": 0.996875, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9809523809523809, "pass@2": 0.9936507936507937, "pass@4": 0.9968253968253968, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9610778443113772, "pass@2": 0.9820359281437125, "pass@4": 0.9910179640718563, "pass@8": 0.9970059880239521, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8905325443786982, "pass@2": 0.9171597633136095, "pass@4": 0.9556213017751479, "pass@8": 0.9970414201183432, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8269230769230769, "pass@2": 0.8942307692307693, "pass@4": 0.9358974358974359, "pass@8": 0.9807692307692307, "pass@16": 0.9903846153846154, "pass@32": 0.9935897435897436, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7549295774647887, "pass@2": 0.8309859154929577, "pass@4": 0.9070422535211268, "pass@8": 0.9605633802816902, "pass@16": 0.9887323943661972, "pass@32": 0.9943661971830986, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6460176991150443, "pass@2": 0.7020648967551623, "pass@4": 0.7787610619469026, "pass@8": 0.8761061946902655, "pass@16": 0.9321533923303835, "pass@32": 0.9528023598820059, "pass@64": 0.9734513274336283, "pass@128": 0.9734513274336283 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.99375, "5": 0.9809523809523809, "6": 0.9610778443113772, "7": 0.8905325443786982, "8": 0.8269230769230769, "9": 0.7549295774647887, "10": 0.6460176991150443 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.99375, "5": 0.9936507936507937, "6": 0.9820359281437125, "7": 0.9171597633136095, "8": 0.8942307692307693, "9": 0.8309859154929577, "10": 0.7020648967551623 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9968253968253968, "6": 0.9910179640718563, "7": 0.9556213017751479, "8": 0.9358974358974359, "9": 0.9070422535211268, "10": 0.7787610619469026 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9970059880239521, "7": 0.9970414201183432, "8": 0.9807692307692307, "9": 0.9605633802816902, "10": 0.8761061946902655 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9970414201183432, "8": 0.9903846153846154, "9": 0.9887323943661972, "10": 0.9321533923303835 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9935897435897436, "9": 0.9943661971830986, "10": 0.9528023598820059 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9734513274336283 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9734513274336283 } }, "total_pass_at_k": { "pass@1": 0.8922661263086795, "pass@2": 0.9216480918608578, "pass@4": 0.950016886187099, "pass@8": 0.9783856805133401, "pass@16": 0.9895305639986491, "pass@32": 0.9932455251604188, "pass@64": 0.9969604863221885, "pass@128": 0.9969604863221885 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9970760233918129, "pass@2": 0.9970760233918129, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.961764705882353, "pass@2": 0.9911764705882353, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9652996845425867, "pass@2": 0.9810725552050473, "pass@4": 0.9873817034700315, "pass@8": 0.9968454258675079, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8734177215189873, "pass@2": 0.9082278481012658, "pass@4": 0.9620253164556962, "pass@8": 0.9778481012658228, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8078078078078078, "pass@2": 0.8768768768768769, "pass@4": 0.924924924924925, "pass@8": 0.96996996996997, "pass@16": 0.990990990990991, "pass@32": 0.996996996996997, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7588424437299035, "pass@2": 0.842443729903537, "pass@4": 0.9131832797427653, "pass@8": 0.9517684887459807, "pass@16": 0.9935691318327974, "pass@32": 0.9967845659163987, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6179104477611941, "pass@2": 0.6746268656716418, "pass@4": 0.7701492537313432, "pass@8": 0.8507462686567164, "pass@16": 0.9283582089552239, "pass@32": 0.9522388059701492, "pass@64": 0.9641791044776119, "pass@128": 0.9761194029850746 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.961764705882353, "6": 0.9652996845425867, "7": 0.8734177215189873, "8": 0.8078078078078078, "9": 0.7588424437299035, "10": 0.6179104477611941 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.9911764705882353, "6": 0.9810725552050473, "7": 0.9082278481012658, "8": 0.8768768768768769, "9": 0.842443729903537, "10": 0.6746268656716418 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9873817034700315, "7": 0.9620253164556962, "8": 0.924924924924925, "9": 0.9131832797427653, "10": 0.7701492537313432 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9968454258675079, "7": 0.9778481012658228, "8": 0.96996996996997, "9": 0.9517684887459807, "10": 0.8507462686567164 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.990990990990991, "9": 0.9935691318327974, "10": 0.9283582089552239 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.996996996996997, "9": 0.9967845659163987, "10": 0.9522388059701492 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9641791044776119 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9761194029850746 } }, "total_pass_at_k": { "pass@1": 0.8888516906595246, "pass@2": 0.9203213927017074, "pass@4": 0.9514563106796117, "pass@8": 0.9722129226648811, "pass@16": 0.9902912621359223, "pass@32": 0.9939738868429863, "pass@64": 0.9959825912286575, "pass@128": 0.9973217274857716 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9822485207100592, "pass@2": 0.9940828402366864, "pass@4": 0.9970414201183432, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9536231884057971, "pass@2": 0.9797101449275363, "pass@4": 0.991304347826087, "pass@8": 0.9971014492753624, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9713467048710601, "pass@2": 0.9799426934097422, "pass@4": 0.9885386819484241, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8526011560693642, "pass@2": 0.9161849710982659, "pass@4": 0.9508670520231214, "pass@8": 0.976878612716763, "pass@16": 0.9971098265895953, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8, "pass@2": 0.8901408450704226, "pass@4": 0.9464788732394366, "pass@8": 0.9690140845070423, "pass@16": 0.9887323943661972, "pass@32": 0.9943661971830986, "pass@64": 0.9971830985915493, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.7485029940119761, "pass@2": 0.8083832335329342, "pass@4": 0.8862275449101796, "pass@8": 0.9221556886227545, "pass@16": 0.9730538922155688, "pass@32": 0.9910179640718563, "pass@64": 0.9970059880239521, "pass@128": 0.9970059880239521 }, "10": { "pass@1": 0.6656441717791411, "pass@2": 0.7331288343558282, "pass@4": 0.8159509202453987, "pass@8": 0.8865030674846626, "pass@16": 0.9263803680981595, "pass@32": 0.9478527607361963, "pass@64": 0.9662576687116564, "pass@128": 0.9754601226993865 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9822485207100592, "5": 0.9536231884057971, "6": 0.9713467048710601, "7": 0.8526011560693642, "8": 0.8, "9": 0.7485029940119761, "10": 0.6656441717791411 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9940828402366864, "5": 0.9797101449275363, "6": 0.9799426934097422, "7": 0.9161849710982659, "8": 0.8901408450704226, "9": 0.8083832335329342, "10": 0.7331288343558282 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 0.991304347826087, "6": 0.9885386819484241, "7": 0.9508670520231214, "8": 0.9464788732394366, "9": 0.8862275449101796, "10": 0.8159509202453987 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 1.0, "7": 0.976878612716763, "8": 0.9690140845070423, "9": 0.9221556886227545, "10": 0.8865030674846626 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9971098265895953, "8": 0.9887323943661972, "9": 0.9730538922155688, "10": 0.9263803680981595 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9943661971830986, "9": 0.9910179640718563, "10": 0.9478527607361963 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 0.9970059880239521, "10": 0.9662576687116564 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 0.9970059880239521, "10": 0.9754601226993865 } }, "total_pass_at_k": { "pass@1": 0.8863040629095675, "pass@2": 0.9230013106159896, "pass@4": 0.953473132372215, "pass@8": 0.9728047182175622, "pass@16": 0.9875491480996068, "pass@32": 0.9927916120576671, "pass@64": 0.995740498034076, "pass@128": 0.9967234600262124 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.991, "pass@2": 0.995, "pass@4": 0.998, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.965, "pass@2": 0.988, "pass@4": 0.996, "pass@8": 0.999, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.966, "pass@2": 0.981, "pass@4": 0.989, "pass@8": 0.998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.872, "pass@2": 0.914, "pass@4": 0.956, "pass@8": 0.984, "pass@16": 0.998, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.811, "pass@2": 0.887, "pass@4": 0.936, "pass@8": 0.973, "pass@16": 0.99, "pass@32": 0.995, "pass@64": 0.999, "pass@128": 0.999 }, "9": { "pass@1": 0.754, "pass@2": 0.827, "pass@4": 0.902, "pass@8": 0.945, "pass@16": 0.985, "pass@32": 0.994, "pass@64": 0.999, "pass@128": 0.999 }, "10": { "pass@1": 0.643, "pass@2": 0.703, "pass@4": 0.788, "pass@8": 0.871, "pass@16": 0.929, "pass@32": 0.951, "pass@64": 0.968, "pass@128": 0.975 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.991, "5": 0.965, "6": 0.966, "7": 0.872, "8": 0.811, "9": 0.754, "10": 0.643 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.995, "5": 0.988, "6": 0.981, "7": 0.914, "8": 0.887, "9": 0.827, "10": 0.703 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.998, "5": 0.996, "6": 0.989, "7": 0.956, "8": 0.936, "9": 0.902, "10": 0.788 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 0.998, "7": 0.984, "8": 0.973, "9": 0.945, "10": 0.871 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.998, "8": 0.99, "9": 0.985, "10": 0.929 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.995, "9": 0.994, "10": 0.951 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.999, "9": 0.999, "10": 0.968 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.999, "9": 0.999, "10": 0.975 } }, "total_pass_at_k": { "pass@1": 0.8891111111111111, "pass@2": 0.9216666666666666, "pass@4": 0.9516666666666667, "pass@8": 0.9744444444444444, "pass@16": 0.9891111111111112, "pass@32": 0.9933333333333333, "pass@64": 0.9962222222222222, "pass@128": 0.997 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-42637_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.5779036827195467, "pass@2": 0.6543909348441926, "pass@4": 0.7223796033994334, "pass@8": 0.8215297450424929, "pass@16": 0.8951841359773371, "pass@32": 0.9348441926345609, "pass@64": 0.9490084985835694, "pass@128": 0.9546742209631728 }, "12": { "pass@1": 0.5988700564971752, "pass@2": 0.6497175141242938, "pass@4": 0.7062146892655368, "pass@8": 0.7796610169491526, "pass@16": 0.8220338983050848, "pass@32": 0.8418079096045198, "pass@64": 0.8559322033898306, "pass@128": 0.864406779661017 }, "13": { "pass@1": 0.4608150470219436, "pass@2": 0.5203761755485894, "pass@4": 0.5768025078369906, "pass@8": 0.6238244514106583, "pass@16": 0.6739811912225705, "pass@32": 0.7241379310344828, "pass@64": 0.768025078369906, "pass@128": 0.7774294670846394 }, "14": { "pass@1": 0.3333333333333333, "pass@2": 0.3956386292834891, "pass@4": 0.46105919003115264, "pass@8": 0.5171339563862928, "pass@16": 0.5669781931464174, "pass@32": 0.6261682242990654, "pass@64": 0.67601246105919, "pass@128": 0.7165109034267912 }, "15": { "pass@1": 0.1791907514450867, "pass@2": 0.24566473988439305, "pass@4": 0.3179190751445087, "pass@8": 0.37283236994219654, "pass@16": 0.41040462427745666, "pass@32": 0.4653179190751445, "pass@64": 0.5289017341040463, "pass@128": 0.5924855491329479 }, "16": { "pass@1": 0.05161290322580645, "pass@2": 0.09032258064516129, "pass@4": 0.11935483870967742, "pass@8": 0.14516129032258066, "pass@16": 0.2032258064516129, "pass@32": 0.23548387096774193, "pass@64": 0.27419354838709675, "pass@128": 0.3225806451612903 }, "17": { "pass@1": 0.05491329479768786, "pass@2": 0.07225433526011561, "pass@4": 0.0838150289017341, "pass@8": 0.10982658959537572, "pass@16": 0.13005780346820808, "pass@32": 0.15028901734104047, "pass@64": 0.18208092485549132, "pass@128": 0.21965317919075145 }, "18": { "pass@1": 0.008571428571428572, "pass@2": 0.014285714285714285, "pass@4": 0.025714285714285714, "pass@8": 0.03142857142857143, "pass@16": 0.04285714285714286, "pass@32": 0.05142857142857143, "pass@64": 0.06285714285714286, "pass@128": 0.07714285714285714 }, "19": { "pass@1": 0.006006006006006006, "pass@2": 0.006006006006006006, "pass@4": 0.012012012012012012, "pass@8": 0.012012012012012012, "pass@16": 0.015015015015015015, "pass@32": 0.018018018018018018, "pass@64": 0.024024024024024024, "pass@128": 0.03903903903903904 }, "20": { "pass@1": 0.003003003003003003, "pass@2": 0.006006006006006006, "pass@4": 0.015015015015015015, "pass@8": 0.021021021021021023, "pass@16": 0.02702702702702703, "pass@32": 0.03003003003003003, "pass@64": 0.042042042042042045, "pass@128": 0.057057057057057055 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5779036827195467, "12": 0.5988700564971752, "13": 0.4608150470219436, "14": 0.3333333333333333, "15": 0.1791907514450867, "16": 0.05161290322580645, "17": 0.05491329479768786, "18": 0.008571428571428572, "19": 0.006006006006006006, "20": 0.003003003003003003 }, "pass@2": { "11": 0.6543909348441926, "12": 0.6497175141242938, "13": 0.5203761755485894, "14": 0.3956386292834891, "15": 0.24566473988439305, "16": 0.09032258064516129, "17": 0.07225433526011561, "18": 0.014285714285714285, "19": 0.006006006006006006, "20": 0.006006006006006006 }, "pass@4": { "11": 0.7223796033994334, "12": 0.7062146892655368, "13": 0.5768025078369906, "14": 0.46105919003115264, "15": 0.3179190751445087, "16": 0.11935483870967742, "17": 0.0838150289017341, "18": 0.025714285714285714, "19": 0.012012012012012012, "20": 0.015015015015015015 }, "pass@8": { "11": 0.8215297450424929, "12": 0.7796610169491526, "13": 0.6238244514106583, "14": 0.5171339563862928, "15": 0.37283236994219654, "16": 0.14516129032258066, "17": 0.10982658959537572, "18": 0.03142857142857143, "19": 0.012012012012012012, "20": 0.021021021021021023 }, "pass@16": { "11": 0.8951841359773371, "12": 0.8220338983050848, "13": 0.6739811912225705, "14": 0.5669781931464174, "15": 0.41040462427745666, "16": 0.2032258064516129, "17": 0.13005780346820808, "18": 0.04285714285714286, "19": 0.015015015015015015, "20": 0.02702702702702703 }, "pass@32": { "11": 0.9348441926345609, "12": 0.8418079096045198, "13": 0.7241379310344828, "14": 0.6261682242990654, "15": 0.4653179190751445, "16": 0.23548387096774193, "17": 0.15028901734104047, "18": 0.05142857142857143, "19": 0.018018018018018018, "20": 0.03003003003003003 }, "pass@64": { "11": 0.9490084985835694, "12": 0.8559322033898306, "13": 0.768025078369906, "14": 0.67601246105919, "15": 0.5289017341040463, "16": 0.27419354838709675, "17": 0.18208092485549132, "18": 0.06285714285714286, "19": 0.024024024024024024, "20": 0.042042042042042045 }, "pass@128": { "11": 0.9546742209631728, "12": 0.864406779661017, "13": 0.7774294670846394, "14": 0.7165109034267912, "15": 0.5924855491329479, "16": 0.3225806451612903, "17": 0.21965317919075145, "18": 0.07714285714285714, "19": 0.03903903903903904, "20": 0.057057057057057055 } }, "total_pass_at_k": { "pass@1": 0.22971768202080237, "pass@2": 0.2677563150074294, "pass@4": 0.30638930163447253, "pass@8": 0.3462109955423477, "pass@16": 0.3812778603268945, "pass@32": 0.4101040118870728, "pass@64": 0.43833580980683506, "pass@128": 0.46389301634472513 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.5714285714285714, "pass@2": 0.6285714285714286, "pass@4": 0.7111111111111111, "pass@8": 0.8158730158730159, "pass@16": 0.9206349206349206, "pass@32": 0.9523809523809523, "pass@64": 0.9650793650793651, "pass@128": 0.9777777777777777 }, "12": { "pass@1": 0.581039755351682, "pass@2": 0.6513761467889908, "pass@4": 0.7370030581039755, "pass@8": 0.8012232415902141, "pass@16": 0.8440366972477065, "pass@32": 0.8593272171253823, "pass@64": 0.8685015290519877, "pass@128": 0.8776758409785933 }, "13": { "pass@1": 0.4351851851851852, "pass@2": 0.4845679012345679, "pass@4": 0.558641975308642, "pass@8": 0.6265432098765432, "pass@16": 0.6820987654320988, "pass@32": 0.7222222222222222, "pass@64": 0.7530864197530864, "pass@128": 0.7746913580246914 }, "14": { "pass@1": 0.2818991097922849, "pass@2": 0.34124629080118696, "pass@4": 0.3827893175074184, "pass@8": 0.45103857566765576, "pass@16": 0.5074183976261127, "pass@32": 0.5667655786350149, "pass@64": 0.6201780415430267, "pass@128": 0.6646884272997032 }, "15": { "pass@1": 0.16358024691358025, "pass@2": 0.2345679012345679, "pass@4": 0.2716049382716049, "pass@8": 0.2962962962962963, "pass@16": 0.3611111111111111, "pass@32": 0.41975308641975306, "pass@64": 0.4783950617283951, "pass@128": 0.5308641975308642 }, "16": { "pass@1": 0.05604719764011799, "pass@2": 0.09734513274336283, "pass@4": 0.13274336283185842, "pass@8": 0.168141592920354, "pass@16": 0.19469026548672566, "pass@32": 0.22418879056047197, "pass@64": 0.26253687315634217, "pass@128": 0.30383480825958703 }, "17": { "pass@1": 0.04216867469879518, "pass@2": 0.060240963855421686, "pass@4": 0.08734939759036145, "pass@8": 0.10240963855421686, "pass@16": 0.12349397590361445, "pass@32": 0.14156626506024098, "pass@64": 0.1536144578313253, "pass@128": 0.1716867469879518 }, "18": { "pass@1": 0.02258064516129032, "pass@2": 0.04516129032258064, "pass@4": 0.04838709677419355, "pass@8": 0.05806451612903226, "pass@16": 0.06774193548387097, "pass@32": 0.08064516129032258, "pass@64": 0.09032258064516129, "pass@128": 0.0967741935483871 }, "19": { "pass@1": 0.006042296072507553, "pass@2": 0.012084592145015106, "pass@4": 0.015105740181268883, "pass@8": 0.015105740181268883, "pass@16": 0.015105740181268883, "pass@32": 0.027190332326283987, "pass@64": 0.030211480362537766, "pass@128": 0.03927492447129909 }, "20": { "pass@1": 0.006006006006006006, "pass@2": 0.006006006006006006, "pass@4": 0.009009009009009009, "pass@8": 0.009009009009009009, "pass@16": 0.009009009009009009, "pass@32": 0.015015015015015015, "pass@64": 0.015015015015015015, "pass@128": 0.015015015015015015 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5714285714285714, "12": 0.581039755351682, "13": 0.4351851851851852, "14": 0.2818991097922849, "15": 0.16358024691358025, "16": 0.05604719764011799, "17": 0.04216867469879518, "18": 0.02258064516129032, "19": 0.006042296072507553, "20": 0.006006006006006006 }, "pass@2": { "11": 0.6285714285714286, "12": 0.6513761467889908, "13": 0.4845679012345679, "14": 0.34124629080118696, "15": 0.2345679012345679, "16": 0.09734513274336283, "17": 0.060240963855421686, "18": 0.04516129032258064, "19": 0.012084592145015106, "20": 0.006006006006006006 }, "pass@4": { "11": 0.7111111111111111, "12": 0.7370030581039755, "13": 0.558641975308642, "14": 0.3827893175074184, "15": 0.2716049382716049, "16": 0.13274336283185842, "17": 0.08734939759036145, "18": 0.04838709677419355, "19": 0.015105740181268883, "20": 0.009009009009009009 }, "pass@8": { "11": 0.8158730158730159, "12": 0.8012232415902141, "13": 0.6265432098765432, "14": 0.45103857566765576, "15": 0.2962962962962963, "16": 0.168141592920354, "17": 0.10240963855421686, "18": 0.05806451612903226, "19": 0.015105740181268883, "20": 0.009009009009009009 }, "pass@16": { "11": 0.9206349206349206, "12": 0.8440366972477065, "13": 0.6820987654320988, "14": 0.5074183976261127, "15": 0.3611111111111111, "16": 0.19469026548672566, "17": 0.12349397590361445, "18": 0.06774193548387097, "19": 0.015105740181268883, "20": 0.009009009009009009 }, "pass@32": { "11": 0.9523809523809523, "12": 0.8593272171253823, "13": 0.7222222222222222, "14": 0.5667655786350149, "15": 0.41975308641975306, "16": 0.22418879056047197, "17": 0.14156626506024098, "18": 0.08064516129032258, "19": 0.027190332326283987, "20": 0.015015015015015015 }, "pass@64": { "11": 0.9650793650793651, "12": 0.8685015290519877, "13": 0.7530864197530864, "14": 0.6201780415430267, "15": 0.4783950617283951, "16": 0.26253687315634217, "17": 0.1536144578313253, "18": 0.09032258064516129, "19": 0.030211480362537766, "20": 0.015015015015015015 }, "pass@128": { "11": 0.9777777777777777, "12": 0.8776758409785933, "13": 0.7746913580246914, "14": 0.6646884272997032, "15": 0.5308641975308642, "16": 0.30383480825958703, "17": 0.1716867469879518, "18": 0.0967741935483871, "19": 0.03927492447129909, "20": 0.015015015015015015 } }, "total_pass_at_k": { "pass@1": 0.21485330073349634, "pass@2": 0.254278728606357, "pass@4": 0.293398533007335, "pass@8": 0.3322127139364303, "pass@16": 0.37011002444987773, "pass@32": 0.39853300733496333, "pass@64": 0.42145476772616136, "pass@128": 0.44315403422982885 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6144578313253012, "pass@2": 0.6867469879518072, "pass@4": 0.7469879518072289, "pass@8": 0.8313253012048193, "pass@16": 0.8975903614457831, "pass@32": 0.9337349397590361, "pass@64": 0.9608433734939759, "pass@128": 0.9698795180722891 }, "12": { "pass@1": 0.5203761755485894, "pass@2": 0.5768025078369906, "pass@4": 0.6739811912225705, "pass@8": 0.7523510971786834, "pass@16": 0.8056426332288401, "pass@32": 0.8338557993730408, "pass@64": 0.8620689655172413, "pass@128": 0.8871473354231975 }, "13": { "pass@1": 0.4425770308123249, "pass@2": 0.5266106442577031, "pass@4": 0.6190476190476191, "pass@8": 0.6722689075630253, "pass@16": 0.7226890756302521, "pass@32": 0.7591036414565826, "pass@64": 0.803921568627451, "pass@128": 0.8263305322128851 }, "14": { "pass@1": 0.30701754385964913, "pass@2": 0.3654970760233918, "pass@4": 0.41812865497076024, "pass@8": 0.48830409356725146, "pass@16": 0.5526315789473685, "pass@32": 0.6228070175438597, "pass@64": 0.6754385964912281, "pass@128": 0.7397660818713451 }, "15": { "pass@1": 0.1484848484848485, "pass@2": 0.19393939393939394, "pass@4": 0.22424242424242424, "pass@8": 0.2696969696969697, "pass@16": 0.31212121212121213, "pass@32": 0.3696969696969697, "pass@64": 0.43333333333333335, "pass@128": 0.5212121212121212 }, "16": { "pass@1": 0.08262108262108261, "pass@2": 0.1339031339031339, "pass@4": 0.15384615384615385, "pass@8": 0.1908831908831909, "pass@16": 0.2336182336182336, "pass@32": 0.2905982905982906, "pass@64": 0.32763532763532766, "pass@128": 0.3817663817663818 }, "17": { "pass@1": 0.034161490683229816, "pass@2": 0.049689440993788817, "pass@4": 0.06521739130434782, "pass@8": 0.08074534161490683, "pass@16": 0.10248447204968944, "pass@32": 0.13043478260869565, "pass@64": 0.16149068322981366, "pass@128": 0.17701863354037267 }, "18": { "pass@1": 0.023529411764705882, "pass@2": 0.041176470588235294, "pass@4": 0.052941176470588235, "pass@8": 0.06764705882352941, "pass@16": 0.08235294117647059, "pass@32": 0.09705882352941177, "pass@64": 0.11470588235294117, "pass@128": 0.1323529411764706 }, "19": { "pass@1": 0.005952380952380952, "pass@2": 0.011904761904761904, "pass@4": 0.017857142857142856, "pass@8": 0.017857142857142856, "pass@16": 0.023809523809523808, "pass@32": 0.026785714285714284, "pass@64": 0.02976190476190476, "pass@128": 0.03571428571428571 }, "20": { "pass@1": 0.008982035928143712, "pass@2": 0.008982035928143712, "pass@4": 0.011976047904191617, "pass@8": 0.020958083832335328, "pass@16": 0.023952095808383235, "pass@32": 0.03293413173652695, "pass@64": 0.03293413173652695, "pass@128": 0.03592814371257485 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6144578313253012, "12": 0.5203761755485894, "13": 0.4425770308123249, "14": 0.30701754385964913, "15": 0.1484848484848485, "16": 0.08262108262108261, "17": 0.034161490683229816, "18": 0.023529411764705882, "19": 0.005952380952380952, "20": 0.008982035928143712 }, "pass@2": { "11": 0.6867469879518072, "12": 0.5768025078369906, "13": 0.5266106442577031, "14": 0.3654970760233918, "15": 0.19393939393939394, "16": 0.1339031339031339, "17": 0.049689440993788817, "18": 0.041176470588235294, "19": 0.011904761904761904, "20": 0.008982035928143712 }, "pass@4": { "11": 0.7469879518072289, "12": 0.6739811912225705, "13": 0.6190476190476191, "14": 0.41812865497076024, "15": 0.22424242424242424, "16": 0.15384615384615385, "17": 0.06521739130434782, "18": 0.052941176470588235, "19": 0.017857142857142856, "20": 0.011976047904191617 }, "pass@8": { "11": 0.8313253012048193, "12": 0.7523510971786834, "13": 0.6722689075630253, "14": 0.48830409356725146, "15": 0.2696969696969697, "16": 0.1908831908831909, "17": 0.08074534161490683, "18": 0.06764705882352941, "19": 0.017857142857142856, "20": 0.020958083832335328 }, "pass@16": { "11": 0.8975903614457831, "12": 0.8056426332288401, "13": 0.7226890756302521, "14": 0.5526315789473685, "15": 0.31212121212121213, "16": 0.2336182336182336, "17": 0.10248447204968944, "18": 0.08235294117647059, "19": 0.023809523809523808, "20": 0.023952095808383235 }, "pass@32": { "11": 0.9337349397590361, "12": 0.8338557993730408, "13": 0.7591036414565826, "14": 0.6228070175438597, "15": 0.3696969696969697, "16": 0.2905982905982906, "17": 0.13043478260869565, "18": 0.09705882352941177, "19": 0.026785714285714284, "20": 0.03293413173652695 }, "pass@64": { "11": 0.9608433734939759, "12": 0.8620689655172413, "13": 0.803921568627451, "14": 0.6754385964912281, "15": 0.43333333333333335, "16": 0.32763532763532766, "17": 0.16149068322981366, "18": 0.11470588235294117, "19": 0.02976190476190476, "20": 0.03293413173652695 }, "pass@128": { "11": 0.9698795180722891, "12": 0.8871473354231975, "13": 0.8263305322128851, "14": 0.7397660818713451, "15": 0.5212121212121212, "16": 0.3817663817663818, "17": 0.17701863354037267, "18": 0.1323529411764706, "19": 0.03571428571428571, "20": 0.03592814371257485 } }, "total_pass_at_k": { "pass@1": 0.21855486173059768, "pass@2": 0.25958965209634255, "pass@4": 0.29854296758846266, "pass@8": 0.33928040440083257, "pass@16": 0.375854891465953, "pass@32": 0.41005055010407376, "pass@64": 0.4406779661016949, "pass@128": 0.4713053820993161 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.588, "pass@2": 0.657, "pass@4": 0.727, "pass@8": 0.823, "pass@16": 0.904, "pass@32": 0.94, "pass@64": 0.958, "pass@128": 0.967 }, "12": { "pass@1": 0.568, "pass@2": 0.627, "pass@4": 0.706, "pass@8": 0.778, "pass@16": 0.824, "pass@32": 0.845, "pass@64": 0.862, "pass@128": 0.876 }, "13": { "pass@1": 0.446, "pass@2": 0.511, "pass@4": 0.586, "pass@8": 0.642, "pass@16": 0.694, "pass@32": 0.736, "pass@64": 0.776, "pass@128": 0.794 }, "14": { "pass@1": 0.307, "pass@2": 0.367, "pass@4": 0.42, "pass@8": 0.485, "pass@16": 0.542, "pass@32": 0.605, "pass@64": 0.657, "pass@128": 0.707 }, "15": { "pass@1": 0.164, "pass@2": 0.225, "pass@4": 0.272, "pass@8": 0.314, "pass@16": 0.362, "pass@32": 0.419, "pass@64": 0.481, "pass@128": 0.549 }, "16": { "pass@1": 0.064, "pass@2": 0.108, "pass@4": 0.136, "pass@8": 0.169, "pass@16": 0.211, "pass@32": 0.251, "pass@64": 0.289, "pass@128": 0.337 }, "17": { "pass@1": 0.044, "pass@2": 0.061, "pass@4": 0.079, "pass@8": 0.098, "pass@16": 0.119, "pass@32": 0.141, "pass@64": 0.166, "pass@128": 0.19 }, "18": { "pass@1": 0.018, "pass@2": 0.033, "pass@4": 0.042, "pass@8": 0.052, "pass@16": 0.064, "pass@32": 0.076, "pass@64": 0.089, "pass@128": 0.102 }, "19": { "pass@1": 0.006, "pass@2": 0.01, "pass@4": 0.015, "pass@8": 0.015, "pass@16": 0.018, "pass@32": 0.024, "pass@64": 0.028, "pass@128": 0.038 }, "20": { "pass@1": 0.006, "pass@2": 0.007, "pass@4": 0.012, "pass@8": 0.017, "pass@16": 0.02, "pass@32": 0.026, "pass@64": 0.03, "pass@128": 0.036 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.588, "12": 0.568, "13": 0.446, "14": 0.307, "15": 0.164, "16": 0.064, "17": 0.044, "18": 0.018, "19": 0.006, "20": 0.006 }, "pass@2": { "11": 0.657, "12": 0.627, "13": 0.511, "14": 0.367, "15": 0.225, "16": 0.108, "17": 0.061, "18": 0.033, "19": 0.01, "20": 0.007 }, "pass@4": { "11": 0.727, "12": 0.706, "13": 0.586, "14": 0.42, "15": 0.272, "16": 0.136, "17": 0.079, "18": 0.042, "19": 0.015, "20": 0.012 }, "pass@8": { "11": 0.823, "12": 0.778, "13": 0.642, "14": 0.485, "15": 0.314, "16": 0.169, "17": 0.098, "18": 0.052, "19": 0.015, "20": 0.017 }, "pass@16": { "11": 0.904, "12": 0.824, "13": 0.694, "14": 0.542, "15": 0.362, "16": 0.211, "17": 0.119, "18": 0.064, "19": 0.018, "20": 0.02 }, "pass@32": { "11": 0.94, "12": 0.845, "13": 0.736, "14": 0.605, "15": 0.419, "16": 0.251, "17": 0.141, "18": 0.076, "19": 0.024, "20": 0.026 }, "pass@64": { "11": 0.958, "12": 0.862, "13": 0.776, "14": 0.657, "15": 0.481, "16": 0.289, "17": 0.166, "18": 0.089, "19": 0.028, "20": 0.03 }, "pass@128": { "11": 0.967, "12": 0.876, "13": 0.794, "14": 0.707, "15": 0.549, "16": 0.337, "17": 0.19, "18": 0.102, "19": 0.038, "20": 0.036 } }, "total_pass_at_k": { "pass@1": 0.2211, "pass@2": 0.2606, "pass@4": 0.2995, "pass@8": 0.3393, "pass@16": 0.3758, "pass@32": 0.4063, "pass@64": 0.4336, "pass@128": 0.4596 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }