{ "files": { "checkpoint-38796_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.996875, "pass@2": 0.996875, "pass@4": 0.996875, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9873015873015873, "pass@2": 0.9904761904761905, "pass@4": 0.9968253968253968, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9730538922155688, "pass@2": 0.9820359281437125, "pass@4": 0.9970059880239521, "pass@8": 0.9970059880239521, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8905325443786982, "pass@2": 0.9260355029585798, "pass@4": 0.9556213017751479, "pass@8": 0.985207100591716, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8269230769230769, "pass@2": 0.875, "pass@4": 0.9583333333333334, "pass@8": 0.9871794871794872, "pass@16": 0.9935897435897436, "pass@32": 0.9967948717948718, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7380281690140845, "pass@2": 0.7887323943661971, "pass@4": 0.8591549295774648, "pass@8": 0.9436619718309859, "pass@16": 0.9830985915492958, "pass@32": 0.9943661971830986, "pass@64": 0.9971830985915493, "pass@128": 1.0 }, "10": { "pass@1": 0.6784660766961652, "pass@2": 0.7463126843657817, "pass@4": 0.7964601769911505, "pass@8": 0.8761061946902655, "pass@16": 0.9056047197640118, "pass@32": 0.9262536873156342, "pass@64": 0.9410029498525073, "pass@128": 0.9528023598820059 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9873015873015873, "6": 0.9730538922155688, "7": 0.8905325443786982, "8": 0.8269230769230769, "9": 0.7380281690140845, "10": 0.6784660766961652 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9904761904761905, "6": 0.9820359281437125, "7": 0.9260355029585798, "8": 0.875, "9": 0.7887323943661971, "10": 0.7463126843657817 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9968253968253968, "6": 0.9970059880239521, "7": 0.9556213017751479, "8": 0.9583333333333334, "9": 0.8591549295774648, "10": 0.7964601769911505 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9970059880239521, "7": 0.985207100591716, "8": 0.9871794871794872, "9": 0.9436619718309859, "10": 0.8761061946902655 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9970414201183432, "8": 0.9935897435897436, "9": 0.9830985915492958, "10": 0.9056047197640118 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9967948717948718, "9": 0.9943661971830986, "10": 0.9262536873156342 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9410029498525073 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9528023598820059 } }, "total_pass_at_k": { "pass@1": 0.8963188112124282, "pass@2": 0.9206349206349206, "pass@4": 0.9493414387031408, "pass@8": 0.9756838905775076, "pass@16": 0.9861533265788585, "pass@32": 0.9905437352245863, "pass@64": 0.9929078014184397, "pass@128": 0.994596420128335 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9912280701754386, "pass@2": 0.9941520467836257, "pass@4": 0.9970760233918129, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9676470588235294, "pass@2": 0.9882352941176471, "pass@4": 0.9941176470588236, "pass@8": 0.9970588235294118, "pass@16": 0.9970588235294118, "pass@32": 0.9970588235294118, "pass@64": 0.9970588235294118, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9716088328075709, "pass@2": 0.9747634069400631, "pass@4": 0.9873817034700315, "pass@8": 0.9968454258675079, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8639240506329114, "pass@2": 0.9240506329113924, "pass@4": 0.9651898734177216, "pass@8": 0.9810126582278481, "pass@16": 0.9968354430379747, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8048048048048048, "pass@2": 0.8678678678678678, "pass@4": 0.9459459459459459, "pass@8": 0.972972972972973, "pass@16": 0.993993993993994, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7620578778135049, "pass@2": 0.8231511254019293, "pass@4": 0.887459807073955, "pass@8": 0.9356913183279743, "pass@16": 0.9742765273311897, "pass@32": 0.9903536977491961, "pass@64": 0.9967845659163987, "pass@128": 1.0 }, "10": { "pass@1": 0.6388059701492538, "pass@2": 0.6925373134328359, "pass@4": 0.7731343283582089, "pass@8": 0.8567164179104477, "pass@16": 0.9194029850746268, "pass@32": 0.9522388059701492, "pass@64": 0.9701492537313433, "pass@128": 0.9850746268656716 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9912280701754386, "5": 0.9676470588235294, "6": 0.9716088328075709, "7": 0.8639240506329114, "8": 0.8048048048048048, "9": 0.7620578778135049, "10": 0.6388059701492538 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9941520467836257, "5": 0.9882352941176471, "6": 0.9747634069400631, "7": 0.9240506329113924, "8": 0.8678678678678678, "9": 0.8231511254019293, "10": 0.6925373134328359 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.9941176470588236, "6": 0.9873817034700315, "7": 0.9651898734177216, "8": 0.9459459459459459, "9": 0.887459807073955, "10": 0.7731343283582089 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 0.9968454258675079, "7": 0.9810126582278481, "8": 0.972972972972973, "9": 0.9356913183279743, "10": 0.8567164179104477 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 0.993993993993994, "9": 0.9742765273311897, "10": 0.9194029850746268 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9903536977491961, "10": 0.9522388059701492 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9967845659163987, "10": 0.9701492537313433 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9850746268656716 } }, "total_pass_at_k": { "pass@1": 0.8908603950451959, "pass@2": 0.9196518245731503, "pass@4": 0.9507867425510546, "pass@8": 0.9715433545363241, "pass@16": 0.986943421493137, "pass@32": 0.9933043187144291, "pass@64": 0.9959825912286575, "pass@128": 0.9979912956143288 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.985207100591716, "pass@2": 0.9970414201183432, "pass@4": 0.9970414201183432, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9623188405797102, "pass@2": 0.9826086956521739, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9742120343839542, "pass@2": 0.9885386819484241, "pass@4": 0.994269340974212, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8526011560693642, "pass@2": 0.9132947976878613, "pass@4": 0.9624277456647399, "pass@8": 0.9913294797687862, "pass@16": 0.9971098265895953, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.7915492957746478, "pass@2": 0.8732394366197183, "pass@4": 0.9352112676056338, "pass@8": 0.9830985915492958, "pass@16": 0.9971830985915493, "pass@32": 0.9971830985915493, "pass@64": 0.9971830985915493, "pass@128": 1.0 }, "9": { "pass@1": 0.7425149700598802, "pass@2": 0.811377245508982, "pass@4": 0.8952095808383234, "pass@8": 0.9491017964071856, "pass@16": 0.9820359281437125, "pass@32": 0.9910179640718563, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6625766871165644, "pass@2": 0.7269938650306749, "pass@4": 0.7944785276073619, "pass@8": 0.8558282208588958, "pass@16": 0.8895705521472392, "pass@32": 0.9141104294478528, "pass@64": 0.9447852760736196, "pass@128": 0.9570552147239264 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.985207100591716, "5": 0.9623188405797102, "6": 0.9742120343839542, "7": 0.8526011560693642, "8": 0.7915492957746478, "9": 0.7425149700598802, "10": 0.6625766871165644 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 0.9826086956521739, "6": 0.9885386819484241, "7": 0.9132947976878613, "8": 0.8732394366197183, "9": 0.811377245508982, "10": 0.7269938650306749 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 1.0, "6": 0.994269340974212, "7": 0.9624277456647399, "8": 0.9352112676056338, "9": 0.8952095808383234, "10": 0.7944785276073619 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9913294797687862, "8": 0.9830985915492958, "9": 0.9491017964071856, "10": 0.8558282208588958 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9971098265895953, "8": 0.9971830985915493, "9": 0.9820359281437125, "10": 0.8895705521472392 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 0.9910179640718563, "10": 0.9141104294478528 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 1.0, "10": 0.9447852760736196 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9570552147239264 } }, "total_pass_at_k": { "pass@1": 0.8859764089121888, "pass@2": 0.9220183486238532, "pass@4": 0.9538007863695938, "pass@8": 0.9760812581913499, "pass@16": 0.9855832241153342, "pass@32": 0.9895150720838795, "pass@64": 0.9937745740498034, "pass@128": 0.9954128440366973 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.991, "pass@2": 0.996, "pass@4": 0.997, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.972, "pass@2": 0.987, "pass@4": 0.997, "pass@8": 0.999, "pass@16": 0.999, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 0.999 }, "6": { "pass@1": 0.973, "pass@2": 0.982, "pass@4": 0.993, "pass@8": 0.998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.869, "pass@2": 0.921, "pass@4": 0.961, "pass@8": 0.986, "pass@16": 0.997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.807, "pass@2": 0.872, "pass@4": 0.946, "pass@8": 0.981, "pass@16": 0.995, "pass@32": 0.998, "pass@64": 0.999, "pass@128": 1.0 }, "9": { "pass@1": 0.747, "pass@2": 0.807, "pass@4": 0.88, "pass@8": 0.943, "pass@16": 0.98, "pass@32": 0.992, "pass@64": 0.998, "pass@128": 1.0 }, "10": { "pass@1": 0.66, "pass@2": 0.722, "pass@4": 0.788, "pass@8": 0.863, "pass@16": 0.905, "pass@32": 0.931, "pass@64": 0.952, "pass@128": 0.965 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.991, "5": 0.972, "6": 0.973, "7": 0.869, "8": 0.807, "9": 0.747, "10": 0.66 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.996, "5": 0.987, "6": 0.982, "7": 0.921, "8": 0.872, "9": 0.807, "10": 0.722 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.997, "5": 0.997, "6": 0.993, "7": 0.961, "8": 0.946, "9": 0.88, "10": 0.788 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 0.998, "7": 0.986, "8": 0.981, "9": 0.943, "10": 0.863 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 0.997, "8": 0.995, "9": 0.98, "10": 0.905 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.998, "9": 0.992, "10": 0.931 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.999, "9": 0.998, "10": 0.952 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.965 } }, "total_pass_at_k": { "pass@1": 0.891, "pass@2": 0.9207777777777778, "pass@4": 0.9513333333333334, "pass@8": 0.9744444444444444, "pass@16": 0.9862222222222222, "pass@32": 0.9911111111111112, "pass@64": 0.9942222222222222, "pass@128": 0.996 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-38796_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.5807365439093485, "pass@2": 0.6430594900849859, "pass@4": 0.7620396600566572, "pass@8": 0.8555240793201133, "pass@16": 0.9263456090651558, "pass@32": 0.9660056657223796, "pass@64": 0.9745042492917847, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.635593220338983, "pass@2": 0.672316384180791, "pass@4": 0.748587570621469, "pass@8": 0.844632768361582, "pass@16": 0.8926553672316384, "pass@32": 0.9378531073446328, "pass@64": 0.9548022598870056, "pass@128": 0.9745762711864406 }, "13": { "pass@1": 0.567398119122257, "pass@2": 0.6238244514106583, "pass@4": 0.6865203761755486, "pass@8": 0.7648902821316614, "pass@16": 0.8557993730407524, "pass@32": 0.9122257053291536, "pass@64": 0.9373040752351097, "pass@128": 0.9529780564263323 }, "14": { "pass@1": 0.5420560747663551, "pass@2": 0.5794392523364486, "pass@4": 0.6292834890965732, "pass@8": 0.6978193146417445, "pass@16": 0.7445482866043613, "pass@32": 0.8099688473520249, "pass@64": 0.8878504672897196, "pass@128": 0.9283489096573209 }, "15": { "pass@1": 0.49710982658959535, "pass@2": 0.5346820809248555, "pass@4": 0.5867052023121387, "pass@8": 0.6734104046242775, "pass@16": 0.7369942196531792, "pass@32": 0.8208092485549133, "pass@64": 0.884393063583815, "pass@128": 0.930635838150289 }, "16": { "pass@1": 0.4870967741935484, "pass@2": 0.5193548387096775, "pass@4": 0.5645161290322581, "pass@8": 0.6129032258064516, "pass@16": 0.6548387096774193, "pass@32": 0.7129032258064516, "pass@64": 0.7645161290322581, "pass@128": 0.7870967741935484 }, "17": { "pass@1": 0.5289017341040463, "pass@2": 0.5635838150289018, "pass@4": 0.5953757225433526, "pass@8": 0.638728323699422, "pass@16": 0.6907514450867052, "pass@32": 0.7630057803468208, "pass@64": 0.8063583815028902, "pass@128": 0.8439306358381503 }, "18": { "pass@1": 0.4657142857142857, "pass@2": 0.48857142857142855, "pass@4": 0.5085714285714286, "pass@8": 0.5657142857142857, "pass@16": 0.62, "pass@32": 0.6571428571428571, "pass@64": 0.7285714285714285, "pass@128": 0.7771428571428571 }, "19": { "pass@1": 0.4024024024024024, "pass@2": 0.43843843843843844, "pass@4": 0.4624624624624625, "pass@8": 0.4954954954954955, "pass@16": 0.5525525525525525, "pass@32": 0.5975975975975976, "pass@64": 0.6516516516516516, "pass@128": 0.6966966966966966 }, "20": { "pass@1": 0.37237237237237236, "pass@2": 0.4144144144144144, "pass@4": 0.44144144144144143, "pass@8": 0.4864864864864865, "pass@16": 0.5225225225225225, "pass@32": 0.5945945945945946, "pass@64": 0.6486486486486487, "pass@128": 0.6966966966966966 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5807365439093485, "12": 0.635593220338983, "13": 0.567398119122257, "14": 0.5420560747663551, "15": 0.49710982658959535, "16": 0.4870967741935484, "17": 0.5289017341040463, "18": 0.4657142857142857, "19": 0.4024024024024024, "20": 0.37237237237237236 }, "pass@2": { "11": 0.6430594900849859, "12": 0.672316384180791, "13": 0.6238244514106583, "14": 0.5794392523364486, "15": 0.5346820809248555, "16": 0.5193548387096775, "17": 0.5635838150289018, "18": 0.48857142857142855, "19": 0.43843843843843844, "20": 0.4144144144144144 }, "pass@4": { "11": 0.7620396600566572, "12": 0.748587570621469, "13": 0.6865203761755486, "14": 0.6292834890965732, "15": 0.5867052023121387, "16": 0.5645161290322581, "17": 0.5953757225433526, "18": 0.5085714285714286, "19": 0.4624624624624625, "20": 0.44144144144144143 }, "pass@8": { "11": 0.8555240793201133, "12": 0.844632768361582, "13": 0.7648902821316614, "14": 0.6978193146417445, "15": 0.6734104046242775, "16": 0.6129032258064516, "17": 0.638728323699422, "18": 0.5657142857142857, "19": 0.4954954954954955, "20": 0.4864864864864865 }, "pass@16": { "11": 0.9263456090651558, "12": 0.8926553672316384, "13": 0.8557993730407524, "14": 0.7445482866043613, "15": 0.7369942196531792, "16": 0.6548387096774193, "17": 0.6907514450867052, "18": 0.62, "19": 0.5525525525525525, "20": 0.5225225225225225 }, "pass@32": { "11": 0.9660056657223796, "12": 0.9378531073446328, "13": 0.9122257053291536, "14": 0.8099688473520249, "15": 0.8208092485549133, "16": 0.7129032258064516, "17": 0.7630057803468208, "18": 0.6571428571428571, "19": 0.5975975975975976, "20": 0.5945945945945946 }, "pass@64": { "11": 0.9745042492917847, "12": 0.9548022598870056, "13": 0.9373040752351097, "14": 0.8878504672897196, "15": 0.884393063583815, "16": 0.7645161290322581, "17": 0.8063583815028902, "18": 0.7285714285714285, "19": 0.6516516516516516, "20": 0.6486486486486487 }, "pass@128": { "11": 0.9830028328611898, "12": 0.9745762711864406, "13": 0.9529780564263323, "14": 0.9283489096573209, "15": 0.930635838150289, "16": 0.7870967741935484, "17": 0.8439306358381503, "18": 0.7771428571428571, "19": 0.6966966966966966, "20": 0.6966966966966966 } }, "total_pass_at_k": { "pass@1": 0.5087667161961367, "pass@2": 0.5485884101040119, "pass@4": 0.599702823179792, "pass@8": 0.6650817236255572, "pass@16": 0.7212481426448737, "pass@32": 0.7786032689450223, "pass@64": 0.824962852897474, "pass@128": 0.8582466567607726 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.5777777777777777, "pass@2": 0.6666666666666666, "pass@4": 0.7587301587301587, "pass@8": 0.8444444444444444, "pass@16": 0.9238095238095239, "pass@32": 0.9555555555555556, "pass@64": 0.9777777777777777, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.6146788990825688, "pass@2": 0.6697247706422018, "pass@4": 0.7767584097859327, "pass@8": 0.8776758409785933, "pass@16": 0.9327217125382263, "pass@32": 0.9602446483180428, "pass@64": 0.981651376146789, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.5925925925925926, "pass@2": 0.6481481481481481, "pass@4": 0.7283950617283951, "pass@8": 0.7932098765432098, "pass@16": 0.845679012345679, "pass@32": 0.8919753086419753, "pass@64": 0.9290123456790124, "pass@128": 0.9506172839506173 }, "14": { "pass@1": 0.543026706231454, "pass@2": 0.5816023738872403, "pass@4": 0.6379821958456974, "pass@8": 0.6943620178041543, "pass@16": 0.7655786350148368, "pass@32": 0.8338278931750742, "pass@64": 0.8961424332344213, "pass@128": 0.9376854599406528 }, "15": { "pass@1": 0.5061728395061729, "pass@2": 0.5370370370370371, "pass@4": 0.5709876543209876, "pass@8": 0.6358024691358025, "pass@16": 0.7283950617283951, "pass@32": 0.7932098765432098, "pass@64": 0.8858024691358025, "pass@128": 0.9259259259259259 }, "16": { "pass@1": 0.5427728613569321, "pass@2": 0.584070796460177, "pass@4": 0.6253687315634219, "pass@8": 0.672566371681416, "pass@16": 0.7374631268436578, "pass@32": 0.7817109144542773, "pass@64": 0.8141592920353983, "pass@128": 0.8348082595870207 }, "17": { "pass@1": 0.4879518072289157, "pass@2": 0.5090361445783133, "pass@4": 0.5451807228915663, "pass@8": 0.5813253012048193, "pass@16": 0.6265060240963856, "pass@32": 0.7048192771084337, "pass@64": 0.7680722891566265, "pass@128": 0.8253012048192772 }, "18": { "pass@1": 0.47096774193548385, "pass@2": 0.5, "pass@4": 0.5258064516129032, "pass@8": 0.567741935483871, "pass@16": 0.6193548387096774, "pass@32": 0.7032258064516129, "pass@64": 0.7774193548387097, "pass@128": 0.8354838709677419 }, "19": { "pass@1": 0.39274924471299094, "pass@2": 0.44108761329305135, "pass@4": 0.47129909365558914, "pass@8": 0.4984894259818731, "pass@16": 0.5468277945619335, "pass@32": 0.595166163141994, "pass@64": 0.6374622356495468, "pass@128": 0.6858006042296072 }, "20": { "pass@1": 0.32732732732732733, "pass@2": 0.3663663663663664, "pass@4": 0.3993993993993994, "pass@8": 0.43543543543543545, "pass@16": 0.4984984984984985, "pass@32": 0.5495495495495496, "pass@64": 0.6246246246246246, "pass@128": 0.6816816816816816 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5777777777777777, "12": 0.6146788990825688, "13": 0.5925925925925926, "14": 0.543026706231454, "15": 0.5061728395061729, "16": 0.5427728613569321, "17": 0.4879518072289157, "18": 0.47096774193548385, "19": 0.39274924471299094, "20": 0.32732732732732733 }, "pass@2": { "11": 0.6666666666666666, "12": 0.6697247706422018, "13": 0.6481481481481481, "14": 0.5816023738872403, "15": 0.5370370370370371, "16": 0.584070796460177, "17": 0.5090361445783133, "18": 0.5, "19": 0.44108761329305135, "20": 0.3663663663663664 }, "pass@4": { "11": 0.7587301587301587, "12": 0.7767584097859327, "13": 0.7283950617283951, "14": 0.6379821958456974, "15": 0.5709876543209876, "16": 0.6253687315634219, "17": 0.5451807228915663, "18": 0.5258064516129032, "19": 0.47129909365558914, "20": 0.3993993993993994 }, "pass@8": { "11": 0.8444444444444444, "12": 0.8776758409785933, "13": 0.7932098765432098, "14": 0.6943620178041543, "15": 0.6358024691358025, "16": 0.672566371681416, "17": 0.5813253012048193, "18": 0.567741935483871, "19": 0.4984894259818731, "20": 0.43543543543543545 }, "pass@16": { "11": 0.9238095238095239, "12": 0.9327217125382263, "13": 0.845679012345679, "14": 0.7655786350148368, "15": 0.7283950617283951, "16": 0.7374631268436578, "17": 0.6265060240963856, "18": 0.6193548387096774, "19": 0.5468277945619335, "20": 0.4984984984984985 }, "pass@32": { "11": 0.9555555555555556, "12": 0.9602446483180428, "13": 0.8919753086419753, "14": 0.8338278931750742, "15": 0.7932098765432098, "16": 0.7817109144542773, "17": 0.7048192771084337, "18": 0.7032258064516129, "19": 0.595166163141994, "20": 0.5495495495495496 }, "pass@64": { "11": 0.9777777777777777, "12": 0.981651376146789, "13": 0.9290123456790124, "14": 0.8961424332344213, "15": 0.8858024691358025, "16": 0.8141592920353983, "17": 0.7680722891566265, "18": 0.7774193548387097, "19": 0.6374622356495468, "20": 0.6246246246246246 }, "pass@128": { "11": 0.9904761904761905, "12": 0.9847094801223242, "13": 0.9506172839506173, "14": 0.9376854599406528, "15": 0.9259259259259259, "16": 0.8348082595870207, "17": 0.8253012048192772, "18": 0.8354838709677419, "19": 0.6858006042296072, "20": 0.6816816816816816 } }, "total_pass_at_k": { "pass@1": 0.5051955990220048, "pass@2": 0.5498166259168704, "pass@4": 0.6033007334963325, "pass@8": 0.6592298288508558, "pass@16": 0.7215770171149144, "pass@32": 0.7759779951100244, "pass@64": 0.828239608801956, "pass@128": 0.8643031784841075 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6596385542168675, "pass@2": 0.7289156626506024, "pass@4": 0.7951807228915663, "pass@8": 0.8524096385542169, "pass@16": 0.9096385542168675, "pass@32": 0.9548192771084337, "pass@64": 0.9728915662650602, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.5924764890282131, "pass@2": 0.677115987460815, "pass@4": 0.7366771159874608, "pass@8": 0.8495297805642633, "pass@16": 0.9122257053291536, "pass@32": 0.9467084639498433, "pass@64": 0.9811912225705329, "pass@128": 0.9843260188087775 }, "13": { "pass@1": 0.5994397759103641, "pass@2": 0.6526610644257703, "pass@4": 0.7507002801120448, "pass@8": 0.8403361344537815, "pass@16": 0.8907563025210085, "pass@32": 0.9355742296918768, "pass@64": 0.9635854341736695, "pass@128": 0.9747899159663865 }, "14": { "pass@1": 0.5497076023391813, "pass@2": 0.5760233918128655, "pass@4": 0.652046783625731, "pass@8": 0.7602339181286549, "pass@16": 0.8128654970760234, "pass@32": 0.8596491228070176, "pass@64": 0.9064327485380117, "pass@128": 0.935672514619883 }, "15": { "pass@1": 0.5303030303030303, "pass@2": 0.5818181818181818, "pass@4": 0.6393939393939394, "pass@8": 0.696969696969697, "pass@16": 0.7787878787878788, "pass@32": 0.8393939393939394, "pass@64": 0.9, "pass@128": 0.9454545454545454 }, "16": { "pass@1": 0.5441595441595442, "pass@2": 0.5641025641025641, "pass@4": 0.5897435897435898, "pass@8": 0.6552706552706553, "pass@16": 0.7094017094017094, "pass@32": 0.7720797720797721, "pass@64": 0.8148148148148148, "pass@128": 0.8461538461538461 }, "17": { "pass@1": 0.422360248447205, "pass@2": 0.453416149068323, "pass@4": 0.4906832298136646, "pass@8": 0.5527950310559007, "pass@16": 0.6180124223602484, "pass@32": 0.7142857142857143, "pass@64": 0.7857142857142857, "pass@128": 0.8198757763975155 }, "18": { "pass@1": 0.4411764705882353, "pass@2": 0.4764705882352941, "pass@4": 0.5147058823529411, "pass@8": 0.5323529411764706, "pass@16": 0.5823529411764706, "pass@32": 0.6411764705882353, "pass@64": 0.7264705882352941, "pass@128": 0.7911764705882353 }, "19": { "pass@1": 0.4226190476190476, "pass@2": 0.46130952380952384, "pass@4": 0.4791666666666667, "pass@8": 0.5059523809523809, "pass@16": 0.5535714285714286, "pass@32": 0.6190476190476191, "pass@64": 0.6726190476190477, "pass@128": 0.7291666666666666 }, "20": { "pass@1": 0.38922155688622756, "pass@2": 0.40718562874251496, "pass@4": 0.41916167664670656, "pass@8": 0.44610778443113774, "pass@16": 0.49700598802395207, "pass@32": 0.5449101796407185, "pass@64": 0.6047904191616766, "pass@128": 0.6616766467065869 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6596385542168675, "12": 0.5924764890282131, "13": 0.5994397759103641, "14": 0.5497076023391813, "15": 0.5303030303030303, "16": 0.5441595441595442, "17": 0.422360248447205, "18": 0.4411764705882353, "19": 0.4226190476190476, "20": 0.38922155688622756 }, "pass@2": { "11": 0.7289156626506024, "12": 0.677115987460815, "13": 0.6526610644257703, "14": 0.5760233918128655, "15": 0.5818181818181818, "16": 0.5641025641025641, "17": 0.453416149068323, "18": 0.4764705882352941, "19": 0.46130952380952384, "20": 0.40718562874251496 }, "pass@4": { "11": 0.7951807228915663, "12": 0.7366771159874608, "13": 0.7507002801120448, "14": 0.652046783625731, "15": 0.6393939393939394, "16": 0.5897435897435898, "17": 0.4906832298136646, "18": 0.5147058823529411, "19": 0.4791666666666667, "20": 0.41916167664670656 }, "pass@8": { "11": 0.8524096385542169, "12": 0.8495297805642633, "13": 0.8403361344537815, "14": 0.7602339181286549, "15": 0.696969696969697, "16": 0.6552706552706553, "17": 0.5527950310559007, "18": 0.5323529411764706, "19": 0.5059523809523809, "20": 0.44610778443113774 }, "pass@16": { "11": 0.9096385542168675, "12": 0.9122257053291536, "13": 0.8907563025210085, "14": 0.8128654970760234, "15": 0.7787878787878788, "16": 0.7094017094017094, "17": 0.6180124223602484, "18": 0.5823529411764706, "19": 0.5535714285714286, "20": 0.49700598802395207 }, "pass@32": { "11": 0.9548192771084337, "12": 0.9467084639498433, "13": 0.9355742296918768, "14": 0.8596491228070176, "15": 0.8393939393939394, "16": 0.7720797720797721, "17": 0.7142857142857143, "18": 0.6411764705882353, "19": 0.6190476190476191, "20": 0.5449101796407185 }, "pass@64": { "11": 0.9728915662650602, "12": 0.9811912225705329, "13": 0.9635854341736695, "14": 0.9064327485380117, "15": 0.9, "16": 0.8148148148148148, "17": 0.7857142857142857, "18": 0.7264705882352941, "19": 0.6726190476190477, "20": 0.6047904191616766 }, "pass@128": { "11": 0.9849397590361446, "12": 0.9843260188087775, "13": 0.9747899159663865, "14": 0.935672514619883, "15": 0.9454545454545454, "16": 0.8461538461538461, "17": 0.8198757763975155, "18": 0.7911764705882353, "19": 0.7291666666666666, "20": 0.6616766467065869 } }, "total_pass_at_k": { "pass@1": 0.5156110615521855, "pass@2": 0.5581326196848052, "pass@4": 0.6071959559916741, "pass@8": 0.6696402022004163, "pass@16": 0.7267320844484092, "pass@32": 0.7829319060362772, "pass@64": 0.8328873030032709, "pass@128": 0.8673803151947665 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.606, "pass@2": 0.679, "pass@4": 0.772, "pass@8": 0.851, "pass@16": 0.92, "pass@32": 0.959, "pass@64": 0.975, "pass@128": 0.986 }, "12": { "pass@1": 0.615, "pass@2": 0.673, "pass@4": 0.754, "pass@8": 0.857, "pass@16": 0.912, "pass@32": 0.948, "pass@64": 0.972, "pass@128": 0.981 }, "13": { "pass@1": 0.587, "pass@2": 0.642, "pass@4": 0.723, "pass@8": 0.801, "pass@16": 0.865, "pass@32": 0.914, "pass@64": 0.944, "pass@128": 0.96 }, "14": { "pass@1": 0.545, "pass@2": 0.579, "pass@4": 0.64, "pass@8": 0.718, "pass@16": 0.775, "pass@32": 0.835, "pass@64": 0.897, "pass@128": 0.934 }, "15": { "pass@1": 0.511, "pass@2": 0.551, "pass@4": 0.599, "pass@8": 0.669, "pass@16": 0.748, "pass@32": 0.818, "pass@64": 0.89, "pass@128": 0.934 }, "16": { "pass@1": 0.526, "pass@2": 0.557, "pass@4": 0.594, "pass@8": 0.648, "pass@16": 0.702, "pass@32": 0.757, "pass@64": 0.799, "pass@128": 0.824 }, "17": { "pass@1": 0.481, "pass@2": 0.51, "pass@4": 0.545, "pass@8": 0.592, "pass@16": 0.646, "pass@32": 0.728, "pass@64": 0.787, "pass@128": 0.83 }, "18": { "pass@1": 0.459, "pass@2": 0.488, "pass@4": 0.516, "pass@8": 0.555, "pass@16": 0.607, "pass@32": 0.666, "pass@64": 0.743, "pass@128": 0.8 }, "19": { "pass@1": 0.406, "pass@2": 0.447, "pass@4": 0.471, "pass@8": 0.5, "pass@16": 0.551, "pass@32": 0.604, "pass@64": 0.654, "pass@128": 0.704 }, "20": { "pass@1": 0.363, "pass@2": 0.396, "pass@4": 0.42, "pass@8": 0.456, "pass@16": 0.506, "pass@32": 0.563, "pass@64": 0.626, "pass@128": 0.68 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.606, "12": 0.615, "13": 0.587, "14": 0.545, "15": 0.511, "16": 0.526, "17": 0.481, "18": 0.459, "19": 0.406, "20": 0.363 }, "pass@2": { "11": 0.679, "12": 0.673, "13": 0.642, "14": 0.579, "15": 0.551, "16": 0.557, "17": 0.51, "18": 0.488, "19": 0.447, "20": 0.396 }, "pass@4": { "11": 0.772, "12": 0.754, "13": 0.723, "14": 0.64, "15": 0.599, "16": 0.594, "17": 0.545, "18": 0.516, "19": 0.471, "20": 0.42 }, "pass@8": { "11": 0.851, "12": 0.857, "13": 0.801, "14": 0.718, "15": 0.669, "16": 0.648, "17": 0.592, "18": 0.555, "19": 0.5, "20": 0.456 }, "pass@16": { "11": 0.92, "12": 0.912, "13": 0.865, "14": 0.775, "15": 0.748, "16": 0.702, "17": 0.646, "18": 0.607, "19": 0.551, "20": 0.506 }, "pass@32": { "11": 0.959, "12": 0.948, "13": 0.914, "14": 0.835, "15": 0.818, "16": 0.757, "17": 0.728, "18": 0.666, "19": 0.604, "20": 0.563 }, "pass@64": { "11": 0.975, "12": 0.972, "13": 0.944, "14": 0.897, "15": 0.89, "16": 0.799, "17": 0.787, "18": 0.743, "19": 0.654, "20": 0.626 }, "pass@128": { "11": 0.986, "12": 0.981, "13": 0.96, "14": 0.934, "15": 0.934, "16": 0.824, "17": 0.83, "18": 0.8, "19": 0.704, "20": 0.68 } }, "total_pass_at_k": { "pass@1": 0.5099, "pass@2": 0.5522, "pass@4": 0.6034, "pass@8": 0.6647, "pass@16": 0.7232, "pass@32": 0.7792, "pass@64": 0.8287, "pass@128": 0.8633 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }