| { |
| "files": { |
| "checkpoint-38796_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.996875, |
| "pass@2": 0.996875, |
| "pass@4": 0.996875, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9873015873015873, |
| "pass@2": 0.9904761904761905, |
| "pass@4": 0.9968253968253968, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9730538922155688, |
| "pass@2": 0.9820359281437125, |
| "pass@4": 0.9970059880239521, |
| "pass@8": 0.9970059880239521, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8905325443786982, |
| "pass@2": 0.9260355029585798, |
| "pass@4": 0.9556213017751479, |
| "pass@8": 0.985207100591716, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8269230769230769, |
| "pass@2": 0.875, |
| "pass@4": 0.9583333333333334, |
| "pass@8": 0.9871794871794872, |
| "pass@16": 0.9935897435897436, |
| "pass@32": 0.9967948717948718, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7380281690140845, |
| "pass@2": 0.7887323943661971, |
| "pass@4": 0.8591549295774648, |
| "pass@8": 0.9436619718309859, |
| "pass@16": 0.9830985915492958, |
| "pass@32": 0.9943661971830986, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6784660766961652, |
| "pass@2": 0.7463126843657817, |
| "pass@4": 0.7964601769911505, |
| "pass@8": 0.8761061946902655, |
| "pass@16": 0.9056047197640118, |
| "pass@32": 0.9262536873156342, |
| "pass@64": 0.9410029498525073, |
| "pass@128": 0.9528023598820059 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9873015873015873, |
| "6": 0.9730538922155688, |
| "7": 0.8905325443786982, |
| "8": 0.8269230769230769, |
| "9": 0.7380281690140845, |
| "10": 0.6784660766961652 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9904761904761905, |
| "6": 0.9820359281437125, |
| "7": 0.9260355029585798, |
| "8": 0.875, |
| "9": 0.7887323943661971, |
| "10": 0.7463126843657817 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9968253968253968, |
| "6": 0.9970059880239521, |
| "7": 0.9556213017751479, |
| "8": 0.9583333333333334, |
| "9": 0.8591549295774648, |
| "10": 0.7964601769911505 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9970059880239521, |
| "7": 0.985207100591716, |
| "8": 0.9871794871794872, |
| "9": 0.9436619718309859, |
| "10": 0.8761061946902655 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 0.9935897435897436, |
| "9": 0.9830985915492958, |
| "10": 0.9056047197640118 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9967948717948718, |
| "9": 0.9943661971830986, |
| "10": 0.9262536873156342 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9410029498525073 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9528023598820059 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8963188112124282, |
| "pass@2": 0.9206349206349206, |
| "pass@4": 0.9493414387031408, |
| "pass@8": 0.9756838905775076, |
| "pass@16": 0.9861533265788585, |
| "pass@32": 0.9905437352245863, |
| "pass@64": 0.9929078014184397, |
| "pass@128": 0.994596420128335 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9912280701754386, |
| "pass@2": 0.9941520467836257, |
| "pass@4": 0.9970760233918129, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9676470588235294, |
| "pass@2": 0.9882352941176471, |
| "pass@4": 0.9941176470588236, |
| "pass@8": 0.9970588235294118, |
| "pass@16": 0.9970588235294118, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 0.9970588235294118, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9716088328075709, |
| "pass@2": 0.9747634069400631, |
| "pass@4": 0.9873817034700315, |
| "pass@8": 0.9968454258675079, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8639240506329114, |
| "pass@2": 0.9240506329113924, |
| "pass@4": 0.9651898734177216, |
| "pass@8": 0.9810126582278481, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8048048048048048, |
| "pass@2": 0.8678678678678678, |
| "pass@4": 0.9459459459459459, |
| "pass@8": 0.972972972972973, |
| "pass@16": 0.993993993993994, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7620578778135049, |
| "pass@2": 0.8231511254019293, |
| "pass@4": 0.887459807073955, |
| "pass@8": 0.9356913183279743, |
| "pass@16": 0.9742765273311897, |
| "pass@32": 0.9903536977491961, |
| "pass@64": 0.9967845659163987, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6388059701492538, |
| "pass@2": 0.6925373134328359, |
| "pass@4": 0.7731343283582089, |
| "pass@8": 0.8567164179104477, |
| "pass@16": 0.9194029850746268, |
| "pass@32": 0.9522388059701492, |
| "pass@64": 0.9701492537313433, |
| "pass@128": 0.9850746268656716 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9912280701754386, |
| "5": 0.9676470588235294, |
| "6": 0.9716088328075709, |
| "7": 0.8639240506329114, |
| "8": 0.8048048048048048, |
| "9": 0.7620578778135049, |
| "10": 0.6388059701492538 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9941520467836257, |
| "5": 0.9882352941176471, |
| "6": 0.9747634069400631, |
| "7": 0.9240506329113924, |
| "8": 0.8678678678678678, |
| "9": 0.8231511254019293, |
| "10": 0.6925373134328359 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.9941176470588236, |
| "6": 0.9873817034700315, |
| "7": 0.9651898734177216, |
| "8": 0.9459459459459459, |
| "9": 0.887459807073955, |
| "10": 0.7731343283582089 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 0.9968454258675079, |
| "7": 0.9810126582278481, |
| "8": 0.972972972972973, |
| "9": 0.9356913183279743, |
| "10": 0.8567164179104477 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.993993993993994, |
| "9": 0.9742765273311897, |
| "10": 0.9194029850746268 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9903536977491961, |
| "10": 0.9522388059701492 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.9701492537313433 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9850746268656716 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8908603950451959, |
| "pass@2": 0.9196518245731503, |
| "pass@4": 0.9507867425510546, |
| "pass@8": 0.9715433545363241, |
| "pass@16": 0.986943421493137, |
| "pass@32": 0.9933043187144291, |
| "pass@64": 0.9959825912286575, |
| "pass@128": 0.9979912956143288 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.985207100591716, |
| "pass@2": 0.9970414201183432, |
| "pass@4": 0.9970414201183432, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9623188405797102, |
| "pass@2": 0.9826086956521739, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9742120343839542, |
| "pass@2": 0.9885386819484241, |
| "pass@4": 0.994269340974212, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8526011560693642, |
| "pass@2": 0.9132947976878613, |
| "pass@4": 0.9624277456647399, |
| "pass@8": 0.9913294797687862, |
| "pass@16": 0.9971098265895953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7915492957746478, |
| "pass@2": 0.8732394366197183, |
| "pass@4": 0.9352112676056338, |
| "pass@8": 0.9830985915492958, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7425149700598802, |
| "pass@2": 0.811377245508982, |
| "pass@4": 0.8952095808383234, |
| "pass@8": 0.9491017964071856, |
| "pass@16": 0.9820359281437125, |
| "pass@32": 0.9910179640718563, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6625766871165644, |
| "pass@2": 0.7269938650306749, |
| "pass@4": 0.7944785276073619, |
| "pass@8": 0.8558282208588958, |
| "pass@16": 0.8895705521472392, |
| "pass@32": 0.9141104294478528, |
| "pass@64": 0.9447852760736196, |
| "pass@128": 0.9570552147239264 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.985207100591716, |
| "5": 0.9623188405797102, |
| "6": 0.9742120343839542, |
| "7": 0.8526011560693642, |
| "8": 0.7915492957746478, |
| "9": 0.7425149700598802, |
| "10": 0.6625766871165644 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 0.9826086956521739, |
| "6": 0.9885386819484241, |
| "7": 0.9132947976878613, |
| "8": 0.8732394366197183, |
| "9": 0.811377245508982, |
| "10": 0.7269938650306749 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 1.0, |
| "6": 0.994269340974212, |
| "7": 0.9624277456647399, |
| "8": 0.9352112676056338, |
| "9": 0.8952095808383234, |
| "10": 0.7944785276073619 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9913294797687862, |
| "8": 0.9830985915492958, |
| "9": 0.9491017964071856, |
| "10": 0.8558282208588958 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9971098265895953, |
| "8": 0.9971830985915493, |
| "9": 0.9820359281437125, |
| "10": 0.8895705521472392 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9910179640718563, |
| "10": 0.9141104294478528 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 1.0, |
| "10": 0.9447852760736196 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9570552147239264 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8859764089121888, |
| "pass@2": 0.9220183486238532, |
| "pass@4": 0.9538007863695938, |
| "pass@8": 0.9760812581913499, |
| "pass@16": 0.9855832241153342, |
| "pass@32": 0.9895150720838795, |
| "pass@64": 0.9937745740498034, |
| "pass@128": 0.9954128440366973 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.991, |
| "pass@2": 0.996, |
| "pass@4": 0.997, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.972, |
| "pass@2": 0.987, |
| "pass@4": 0.997, |
| "pass@8": 0.999, |
| "pass@16": 0.999, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.973, |
| "pass@2": 0.982, |
| "pass@4": 0.993, |
| "pass@8": 0.998, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.869, |
| "pass@2": 0.921, |
| "pass@4": 0.961, |
| "pass@8": 0.986, |
| "pass@16": 0.997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.807, |
| "pass@2": 0.872, |
| "pass@4": 0.946, |
| "pass@8": 0.981, |
| "pass@16": 0.995, |
| "pass@32": 0.998, |
| "pass@64": 0.999, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.747, |
| "pass@2": 0.807, |
| "pass@4": 0.88, |
| "pass@8": 0.943, |
| "pass@16": 0.98, |
| "pass@32": 0.992, |
| "pass@64": 0.998, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.66, |
| "pass@2": 0.722, |
| "pass@4": 0.788, |
| "pass@8": 0.863, |
| "pass@16": 0.905, |
| "pass@32": 0.931, |
| "pass@64": 0.952, |
| "pass@128": 0.965 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.991, |
| "5": 0.972, |
| "6": 0.973, |
| "7": 0.869, |
| "8": 0.807, |
| "9": 0.747, |
| "10": 0.66 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996, |
| "5": 0.987, |
| "6": 0.982, |
| "7": 0.921, |
| "8": 0.872, |
| "9": 0.807, |
| "10": 0.722 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.997, |
| "5": 0.997, |
| "6": 0.993, |
| "7": 0.961, |
| "8": 0.946, |
| "9": 0.88, |
| "10": 0.788 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.998, |
| "7": 0.986, |
| "8": 0.981, |
| "9": 0.943, |
| "10": 0.863 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 0.997, |
| "8": 0.995, |
| "9": 0.98, |
| "10": 0.905 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.998, |
| "9": 0.992, |
| "10": 0.931 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 0.998, |
| "10": 0.952 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.965 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.891, |
| "pass@2": 0.9207777777777778, |
| "pass@4": 0.9513333333333334, |
| "pass@8": 0.9744444444444444, |
| "pass@16": 0.9862222222222222, |
| "pass@32": 0.9911111111111112, |
| "pass@64": 0.9942222222222222, |
| "pass@128": 0.996 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-38796_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5807365439093485, |
| "pass@2": 0.6430594900849859, |
| "pass@4": 0.7620396600566572, |
| "pass@8": 0.8555240793201133, |
| "pass@16": 0.9263456090651558, |
| "pass@32": 0.9660056657223796, |
| "pass@64": 0.9745042492917847, |
| "pass@128": 0.9830028328611898 |
| }, |
| "12": { |
| "pass@1": 0.635593220338983, |
| "pass@2": 0.672316384180791, |
| "pass@4": 0.748587570621469, |
| "pass@8": 0.844632768361582, |
| "pass@16": 0.8926553672316384, |
| "pass@32": 0.9378531073446328, |
| "pass@64": 0.9548022598870056, |
| "pass@128": 0.9745762711864406 |
| }, |
| "13": { |
| "pass@1": 0.567398119122257, |
| "pass@2": 0.6238244514106583, |
| "pass@4": 0.6865203761755486, |
| "pass@8": 0.7648902821316614, |
| "pass@16": 0.8557993730407524, |
| "pass@32": 0.9122257053291536, |
| "pass@64": 0.9373040752351097, |
| "pass@128": 0.9529780564263323 |
| }, |
| "14": { |
| "pass@1": 0.5420560747663551, |
| "pass@2": 0.5794392523364486, |
| "pass@4": 0.6292834890965732, |
| "pass@8": 0.6978193146417445, |
| "pass@16": 0.7445482866043613, |
| "pass@32": 0.8099688473520249, |
| "pass@64": 0.8878504672897196, |
| "pass@128": 0.9283489096573209 |
| }, |
| "15": { |
| "pass@1": 0.49710982658959535, |
| "pass@2": 0.5346820809248555, |
| "pass@4": 0.5867052023121387, |
| "pass@8": 0.6734104046242775, |
| "pass@16": 0.7369942196531792, |
| "pass@32": 0.8208092485549133, |
| "pass@64": 0.884393063583815, |
| "pass@128": 0.930635838150289 |
| }, |
| "16": { |
| "pass@1": 0.4870967741935484, |
| "pass@2": 0.5193548387096775, |
| "pass@4": 0.5645161290322581, |
| "pass@8": 0.6129032258064516, |
| "pass@16": 0.6548387096774193, |
| "pass@32": 0.7129032258064516, |
| "pass@64": 0.7645161290322581, |
| "pass@128": 0.7870967741935484 |
| }, |
| "17": { |
| "pass@1": 0.5289017341040463, |
| "pass@2": 0.5635838150289018, |
| "pass@4": 0.5953757225433526, |
| "pass@8": 0.638728323699422, |
| "pass@16": 0.6907514450867052, |
| "pass@32": 0.7630057803468208, |
| "pass@64": 0.8063583815028902, |
| "pass@128": 0.8439306358381503 |
| }, |
| "18": { |
| "pass@1": 0.4657142857142857, |
| "pass@2": 0.48857142857142855, |
| "pass@4": 0.5085714285714286, |
| "pass@8": 0.5657142857142857, |
| "pass@16": 0.62, |
| "pass@32": 0.6571428571428571, |
| "pass@64": 0.7285714285714285, |
| "pass@128": 0.7771428571428571 |
| }, |
| "19": { |
| "pass@1": 0.4024024024024024, |
| "pass@2": 0.43843843843843844, |
| "pass@4": 0.4624624624624625, |
| "pass@8": 0.4954954954954955, |
| "pass@16": 0.5525525525525525, |
| "pass@32": 0.5975975975975976, |
| "pass@64": 0.6516516516516516, |
| "pass@128": 0.6966966966966966 |
| }, |
| "20": { |
| "pass@1": 0.37237237237237236, |
| "pass@2": 0.4144144144144144, |
| "pass@4": 0.44144144144144143, |
| "pass@8": 0.4864864864864865, |
| "pass@16": 0.5225225225225225, |
| "pass@32": 0.5945945945945946, |
| "pass@64": 0.6486486486486487, |
| "pass@128": 0.6966966966966966 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5807365439093485, |
| "12": 0.635593220338983, |
| "13": 0.567398119122257, |
| "14": 0.5420560747663551, |
| "15": 0.49710982658959535, |
| "16": 0.4870967741935484, |
| "17": 0.5289017341040463, |
| "18": 0.4657142857142857, |
| "19": 0.4024024024024024, |
| "20": 0.37237237237237236 |
| }, |
| "pass@2": { |
| "11": 0.6430594900849859, |
| "12": 0.672316384180791, |
| "13": 0.6238244514106583, |
| "14": 0.5794392523364486, |
| "15": 0.5346820809248555, |
| "16": 0.5193548387096775, |
| "17": 0.5635838150289018, |
| "18": 0.48857142857142855, |
| "19": 0.43843843843843844, |
| "20": 0.4144144144144144 |
| }, |
| "pass@4": { |
| "11": 0.7620396600566572, |
| "12": 0.748587570621469, |
| "13": 0.6865203761755486, |
| "14": 0.6292834890965732, |
| "15": 0.5867052023121387, |
| "16": 0.5645161290322581, |
| "17": 0.5953757225433526, |
| "18": 0.5085714285714286, |
| "19": 0.4624624624624625, |
| "20": 0.44144144144144143 |
| }, |
| "pass@8": { |
| "11": 0.8555240793201133, |
| "12": 0.844632768361582, |
| "13": 0.7648902821316614, |
| "14": 0.6978193146417445, |
| "15": 0.6734104046242775, |
| "16": 0.6129032258064516, |
| "17": 0.638728323699422, |
| "18": 0.5657142857142857, |
| "19": 0.4954954954954955, |
| "20": 0.4864864864864865 |
| }, |
| "pass@16": { |
| "11": 0.9263456090651558, |
| "12": 0.8926553672316384, |
| "13": 0.8557993730407524, |
| "14": 0.7445482866043613, |
| "15": 0.7369942196531792, |
| "16": 0.6548387096774193, |
| "17": 0.6907514450867052, |
| "18": 0.62, |
| "19": 0.5525525525525525, |
| "20": 0.5225225225225225 |
| }, |
| "pass@32": { |
| "11": 0.9660056657223796, |
| "12": 0.9378531073446328, |
| "13": 0.9122257053291536, |
| "14": 0.8099688473520249, |
| "15": 0.8208092485549133, |
| "16": 0.7129032258064516, |
| "17": 0.7630057803468208, |
| "18": 0.6571428571428571, |
| "19": 0.5975975975975976, |
| "20": 0.5945945945945946 |
| }, |
| "pass@64": { |
| "11": 0.9745042492917847, |
| "12": 0.9548022598870056, |
| "13": 0.9373040752351097, |
| "14": 0.8878504672897196, |
| "15": 0.884393063583815, |
| "16": 0.7645161290322581, |
| "17": 0.8063583815028902, |
| "18": 0.7285714285714285, |
| "19": 0.6516516516516516, |
| "20": 0.6486486486486487 |
| }, |
| "pass@128": { |
| "11": 0.9830028328611898, |
| "12": 0.9745762711864406, |
| "13": 0.9529780564263323, |
| "14": 0.9283489096573209, |
| "15": 0.930635838150289, |
| "16": 0.7870967741935484, |
| "17": 0.8439306358381503, |
| "18": 0.7771428571428571, |
| "19": 0.6966966966966966, |
| "20": 0.6966966966966966 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5087667161961367, |
| "pass@2": 0.5485884101040119, |
| "pass@4": 0.599702823179792, |
| "pass@8": 0.6650817236255572, |
| "pass@16": 0.7212481426448737, |
| "pass@32": 0.7786032689450223, |
| "pass@64": 0.824962852897474, |
| "pass@128": 0.8582466567607726 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5777777777777777, |
| "pass@2": 0.6666666666666666, |
| "pass@4": 0.7587301587301587, |
| "pass@8": 0.8444444444444444, |
| "pass@16": 0.9238095238095239, |
| "pass@32": 0.9555555555555556, |
| "pass@64": 0.9777777777777777, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.6146788990825688, |
| "pass@2": 0.6697247706422018, |
| "pass@4": 0.7767584097859327, |
| "pass@8": 0.8776758409785933, |
| "pass@16": 0.9327217125382263, |
| "pass@32": 0.9602446483180428, |
| "pass@64": 0.981651376146789, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.5925925925925926, |
| "pass@2": 0.6481481481481481, |
| "pass@4": 0.7283950617283951, |
| "pass@8": 0.7932098765432098, |
| "pass@16": 0.845679012345679, |
| "pass@32": 0.8919753086419753, |
| "pass@64": 0.9290123456790124, |
| "pass@128": 0.9506172839506173 |
| }, |
| "14": { |
| "pass@1": 0.543026706231454, |
| "pass@2": 0.5816023738872403, |
| "pass@4": 0.6379821958456974, |
| "pass@8": 0.6943620178041543, |
| "pass@16": 0.7655786350148368, |
| "pass@32": 0.8338278931750742, |
| "pass@64": 0.8961424332344213, |
| "pass@128": 0.9376854599406528 |
| }, |
| "15": { |
| "pass@1": 0.5061728395061729, |
| "pass@2": 0.5370370370370371, |
| "pass@4": 0.5709876543209876, |
| "pass@8": 0.6358024691358025, |
| "pass@16": 0.7283950617283951, |
| "pass@32": 0.7932098765432098, |
| "pass@64": 0.8858024691358025, |
| "pass@128": 0.9259259259259259 |
| }, |
| "16": { |
| "pass@1": 0.5427728613569321, |
| "pass@2": 0.584070796460177, |
| "pass@4": 0.6253687315634219, |
| "pass@8": 0.672566371681416, |
| "pass@16": 0.7374631268436578, |
| "pass@32": 0.7817109144542773, |
| "pass@64": 0.8141592920353983, |
| "pass@128": 0.8348082595870207 |
| }, |
| "17": { |
| "pass@1": 0.4879518072289157, |
| "pass@2": 0.5090361445783133, |
| "pass@4": 0.5451807228915663, |
| "pass@8": 0.5813253012048193, |
| "pass@16": 0.6265060240963856, |
| "pass@32": 0.7048192771084337, |
| "pass@64": 0.7680722891566265, |
| "pass@128": 0.8253012048192772 |
| }, |
| "18": { |
| "pass@1": 0.47096774193548385, |
| "pass@2": 0.5, |
| "pass@4": 0.5258064516129032, |
| "pass@8": 0.567741935483871, |
| "pass@16": 0.6193548387096774, |
| "pass@32": 0.7032258064516129, |
| "pass@64": 0.7774193548387097, |
| "pass@128": 0.8354838709677419 |
| }, |
| "19": { |
| "pass@1": 0.39274924471299094, |
| "pass@2": 0.44108761329305135, |
| "pass@4": 0.47129909365558914, |
| "pass@8": 0.4984894259818731, |
| "pass@16": 0.5468277945619335, |
| "pass@32": 0.595166163141994, |
| "pass@64": 0.6374622356495468, |
| "pass@128": 0.6858006042296072 |
| }, |
| "20": { |
| "pass@1": 0.32732732732732733, |
| "pass@2": 0.3663663663663664, |
| "pass@4": 0.3993993993993994, |
| "pass@8": 0.43543543543543545, |
| "pass@16": 0.4984984984984985, |
| "pass@32": 0.5495495495495496, |
| "pass@64": 0.6246246246246246, |
| "pass@128": 0.6816816816816816 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5777777777777777, |
| "12": 0.6146788990825688, |
| "13": 0.5925925925925926, |
| "14": 0.543026706231454, |
| "15": 0.5061728395061729, |
| "16": 0.5427728613569321, |
| "17": 0.4879518072289157, |
| "18": 0.47096774193548385, |
| "19": 0.39274924471299094, |
| "20": 0.32732732732732733 |
| }, |
| "pass@2": { |
| "11": 0.6666666666666666, |
| "12": 0.6697247706422018, |
| "13": 0.6481481481481481, |
| "14": 0.5816023738872403, |
| "15": 0.5370370370370371, |
| "16": 0.584070796460177, |
| "17": 0.5090361445783133, |
| "18": 0.5, |
| "19": 0.44108761329305135, |
| "20": 0.3663663663663664 |
| }, |
| "pass@4": { |
| "11": 0.7587301587301587, |
| "12": 0.7767584097859327, |
| "13": 0.7283950617283951, |
| "14": 0.6379821958456974, |
| "15": 0.5709876543209876, |
| "16": 0.6253687315634219, |
| "17": 0.5451807228915663, |
| "18": 0.5258064516129032, |
| "19": 0.47129909365558914, |
| "20": 0.3993993993993994 |
| }, |
| "pass@8": { |
| "11": 0.8444444444444444, |
| "12": 0.8776758409785933, |
| "13": 0.7932098765432098, |
| "14": 0.6943620178041543, |
| "15": 0.6358024691358025, |
| "16": 0.672566371681416, |
| "17": 0.5813253012048193, |
| "18": 0.567741935483871, |
| "19": 0.4984894259818731, |
| "20": 0.43543543543543545 |
| }, |
| "pass@16": { |
| "11": 0.9238095238095239, |
| "12": 0.9327217125382263, |
| "13": 0.845679012345679, |
| "14": 0.7655786350148368, |
| "15": 0.7283950617283951, |
| "16": 0.7374631268436578, |
| "17": 0.6265060240963856, |
| "18": 0.6193548387096774, |
| "19": 0.5468277945619335, |
| "20": 0.4984984984984985 |
| }, |
| "pass@32": { |
| "11": 0.9555555555555556, |
| "12": 0.9602446483180428, |
| "13": 0.8919753086419753, |
| "14": 0.8338278931750742, |
| "15": 0.7932098765432098, |
| "16": 0.7817109144542773, |
| "17": 0.7048192771084337, |
| "18": 0.7032258064516129, |
| "19": 0.595166163141994, |
| "20": 0.5495495495495496 |
| }, |
| "pass@64": { |
| "11": 0.9777777777777777, |
| "12": 0.981651376146789, |
| "13": 0.9290123456790124, |
| "14": 0.8961424332344213, |
| "15": 0.8858024691358025, |
| "16": 0.8141592920353983, |
| "17": 0.7680722891566265, |
| "18": 0.7774193548387097, |
| "19": 0.6374622356495468, |
| "20": 0.6246246246246246 |
| }, |
| "pass@128": { |
| "11": 0.9904761904761905, |
| "12": 0.9847094801223242, |
| "13": 0.9506172839506173, |
| "14": 0.9376854599406528, |
| "15": 0.9259259259259259, |
| "16": 0.8348082595870207, |
| "17": 0.8253012048192772, |
| "18": 0.8354838709677419, |
| "19": 0.6858006042296072, |
| "20": 0.6816816816816816 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5051955990220048, |
| "pass@2": 0.5498166259168704, |
| "pass@4": 0.6033007334963325, |
| "pass@8": 0.6592298288508558, |
| "pass@16": 0.7215770171149144, |
| "pass@32": 0.7759779951100244, |
| "pass@64": 0.828239608801956, |
| "pass@128": 0.8643031784841075 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6596385542168675, |
| "pass@2": 0.7289156626506024, |
| "pass@4": 0.7951807228915663, |
| "pass@8": 0.8524096385542169, |
| "pass@16": 0.9096385542168675, |
| "pass@32": 0.9548192771084337, |
| "pass@64": 0.9728915662650602, |
| "pass@128": 0.9849397590361446 |
| }, |
| "12": { |
| "pass@1": 0.5924764890282131, |
| "pass@2": 0.677115987460815, |
| "pass@4": 0.7366771159874608, |
| "pass@8": 0.8495297805642633, |
| "pass@16": 0.9122257053291536, |
| "pass@32": 0.9467084639498433, |
| "pass@64": 0.9811912225705329, |
| "pass@128": 0.9843260188087775 |
| }, |
| "13": { |
| "pass@1": 0.5994397759103641, |
| "pass@2": 0.6526610644257703, |
| "pass@4": 0.7507002801120448, |
| "pass@8": 0.8403361344537815, |
| "pass@16": 0.8907563025210085, |
| "pass@32": 0.9355742296918768, |
| "pass@64": 0.9635854341736695, |
| "pass@128": 0.9747899159663865 |
| }, |
| "14": { |
| "pass@1": 0.5497076023391813, |
| "pass@2": 0.5760233918128655, |
| "pass@4": 0.652046783625731, |
| "pass@8": 0.7602339181286549, |
| "pass@16": 0.8128654970760234, |
| "pass@32": 0.8596491228070176, |
| "pass@64": 0.9064327485380117, |
| "pass@128": 0.935672514619883 |
| }, |
| "15": { |
| "pass@1": 0.5303030303030303, |
| "pass@2": 0.5818181818181818, |
| "pass@4": 0.6393939393939394, |
| "pass@8": 0.696969696969697, |
| "pass@16": 0.7787878787878788, |
| "pass@32": 0.8393939393939394, |
| "pass@64": 0.9, |
| "pass@128": 0.9454545454545454 |
| }, |
| "16": { |
| "pass@1": 0.5441595441595442, |
| "pass@2": 0.5641025641025641, |
| "pass@4": 0.5897435897435898, |
| "pass@8": 0.6552706552706553, |
| "pass@16": 0.7094017094017094, |
| "pass@32": 0.7720797720797721, |
| "pass@64": 0.8148148148148148, |
| "pass@128": 0.8461538461538461 |
| }, |
| "17": { |
| "pass@1": 0.422360248447205, |
| "pass@2": 0.453416149068323, |
| "pass@4": 0.4906832298136646, |
| "pass@8": 0.5527950310559007, |
| "pass@16": 0.6180124223602484, |
| "pass@32": 0.7142857142857143, |
| "pass@64": 0.7857142857142857, |
| "pass@128": 0.8198757763975155 |
| }, |
| "18": { |
| "pass@1": 0.4411764705882353, |
| "pass@2": 0.4764705882352941, |
| "pass@4": 0.5147058823529411, |
| "pass@8": 0.5323529411764706, |
| "pass@16": 0.5823529411764706, |
| "pass@32": 0.6411764705882353, |
| "pass@64": 0.7264705882352941, |
| "pass@128": 0.7911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.4226190476190476, |
| "pass@2": 0.46130952380952384, |
| "pass@4": 0.4791666666666667, |
| "pass@8": 0.5059523809523809, |
| "pass@16": 0.5535714285714286, |
| "pass@32": 0.6190476190476191, |
| "pass@64": 0.6726190476190477, |
| "pass@128": 0.7291666666666666 |
| }, |
| "20": { |
| "pass@1": 0.38922155688622756, |
| "pass@2": 0.40718562874251496, |
| "pass@4": 0.41916167664670656, |
| "pass@8": 0.44610778443113774, |
| "pass@16": 0.49700598802395207, |
| "pass@32": 0.5449101796407185, |
| "pass@64": 0.6047904191616766, |
| "pass@128": 0.6616766467065869 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6596385542168675, |
| "12": 0.5924764890282131, |
| "13": 0.5994397759103641, |
| "14": 0.5497076023391813, |
| "15": 0.5303030303030303, |
| "16": 0.5441595441595442, |
| "17": 0.422360248447205, |
| "18": 0.4411764705882353, |
| "19": 0.4226190476190476, |
| "20": 0.38922155688622756 |
| }, |
| "pass@2": { |
| "11": 0.7289156626506024, |
| "12": 0.677115987460815, |
| "13": 0.6526610644257703, |
| "14": 0.5760233918128655, |
| "15": 0.5818181818181818, |
| "16": 0.5641025641025641, |
| "17": 0.453416149068323, |
| "18": 0.4764705882352941, |
| "19": 0.46130952380952384, |
| "20": 0.40718562874251496 |
| }, |
| "pass@4": { |
| "11": 0.7951807228915663, |
| "12": 0.7366771159874608, |
| "13": 0.7507002801120448, |
| "14": 0.652046783625731, |
| "15": 0.6393939393939394, |
| "16": 0.5897435897435898, |
| "17": 0.4906832298136646, |
| "18": 0.5147058823529411, |
| "19": 0.4791666666666667, |
| "20": 0.41916167664670656 |
| }, |
| "pass@8": { |
| "11": 0.8524096385542169, |
| "12": 0.8495297805642633, |
| "13": 0.8403361344537815, |
| "14": 0.7602339181286549, |
| "15": 0.696969696969697, |
| "16": 0.6552706552706553, |
| "17": 0.5527950310559007, |
| "18": 0.5323529411764706, |
| "19": 0.5059523809523809, |
| "20": 0.44610778443113774 |
| }, |
| "pass@16": { |
| "11": 0.9096385542168675, |
| "12": 0.9122257053291536, |
| "13": 0.8907563025210085, |
| "14": 0.8128654970760234, |
| "15": 0.7787878787878788, |
| "16": 0.7094017094017094, |
| "17": 0.6180124223602484, |
| "18": 0.5823529411764706, |
| "19": 0.5535714285714286, |
| "20": 0.49700598802395207 |
| }, |
| "pass@32": { |
| "11": 0.9548192771084337, |
| "12": 0.9467084639498433, |
| "13": 0.9355742296918768, |
| "14": 0.8596491228070176, |
| "15": 0.8393939393939394, |
| "16": 0.7720797720797721, |
| "17": 0.7142857142857143, |
| "18": 0.6411764705882353, |
| "19": 0.6190476190476191, |
| "20": 0.5449101796407185 |
| }, |
| "pass@64": { |
| "11": 0.9728915662650602, |
| "12": 0.9811912225705329, |
| "13": 0.9635854341736695, |
| "14": 0.9064327485380117, |
| "15": 0.9, |
| "16": 0.8148148148148148, |
| "17": 0.7857142857142857, |
| "18": 0.7264705882352941, |
| "19": 0.6726190476190477, |
| "20": 0.6047904191616766 |
| }, |
| "pass@128": { |
| "11": 0.9849397590361446, |
| "12": 0.9843260188087775, |
| "13": 0.9747899159663865, |
| "14": 0.935672514619883, |
| "15": 0.9454545454545454, |
| "16": 0.8461538461538461, |
| "17": 0.8198757763975155, |
| "18": 0.7911764705882353, |
| "19": 0.7291666666666666, |
| "20": 0.6616766467065869 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5156110615521855, |
| "pass@2": 0.5581326196848052, |
| "pass@4": 0.6071959559916741, |
| "pass@8": 0.6696402022004163, |
| "pass@16": 0.7267320844484092, |
| "pass@32": 0.7829319060362772, |
| "pass@64": 0.8328873030032709, |
| "pass@128": 0.8673803151947665 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.606, |
| "pass@2": 0.679, |
| "pass@4": 0.772, |
| "pass@8": 0.851, |
| "pass@16": 0.92, |
| "pass@32": 0.959, |
| "pass@64": 0.975, |
| "pass@128": 0.986 |
| }, |
| "12": { |
| "pass@1": 0.615, |
| "pass@2": 0.673, |
| "pass@4": 0.754, |
| "pass@8": 0.857, |
| "pass@16": 0.912, |
| "pass@32": 0.948, |
| "pass@64": 0.972, |
| "pass@128": 0.981 |
| }, |
| "13": { |
| "pass@1": 0.587, |
| "pass@2": 0.642, |
| "pass@4": 0.723, |
| "pass@8": 0.801, |
| "pass@16": 0.865, |
| "pass@32": 0.914, |
| "pass@64": 0.944, |
| "pass@128": 0.96 |
| }, |
| "14": { |
| "pass@1": 0.545, |
| "pass@2": 0.579, |
| "pass@4": 0.64, |
| "pass@8": 0.718, |
| "pass@16": 0.775, |
| "pass@32": 0.835, |
| "pass@64": 0.897, |
| "pass@128": 0.934 |
| }, |
| "15": { |
| "pass@1": 0.511, |
| "pass@2": 0.551, |
| "pass@4": 0.599, |
| "pass@8": 0.669, |
| "pass@16": 0.748, |
| "pass@32": 0.818, |
| "pass@64": 0.89, |
| "pass@128": 0.934 |
| }, |
| "16": { |
| "pass@1": 0.526, |
| "pass@2": 0.557, |
| "pass@4": 0.594, |
| "pass@8": 0.648, |
| "pass@16": 0.702, |
| "pass@32": 0.757, |
| "pass@64": 0.799, |
| "pass@128": 0.824 |
| }, |
| "17": { |
| "pass@1": 0.481, |
| "pass@2": 0.51, |
| "pass@4": 0.545, |
| "pass@8": 0.592, |
| "pass@16": 0.646, |
| "pass@32": 0.728, |
| "pass@64": 0.787, |
| "pass@128": 0.83 |
| }, |
| "18": { |
| "pass@1": 0.459, |
| "pass@2": 0.488, |
| "pass@4": 0.516, |
| "pass@8": 0.555, |
| "pass@16": 0.607, |
| "pass@32": 0.666, |
| "pass@64": 0.743, |
| "pass@128": 0.8 |
| }, |
| "19": { |
| "pass@1": 0.406, |
| "pass@2": 0.447, |
| "pass@4": 0.471, |
| "pass@8": 0.5, |
| "pass@16": 0.551, |
| "pass@32": 0.604, |
| "pass@64": 0.654, |
| "pass@128": 0.704 |
| }, |
| "20": { |
| "pass@1": 0.363, |
| "pass@2": 0.396, |
| "pass@4": 0.42, |
| "pass@8": 0.456, |
| "pass@16": 0.506, |
| "pass@32": 0.563, |
| "pass@64": 0.626, |
| "pass@128": 0.68 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.606, |
| "12": 0.615, |
| "13": 0.587, |
| "14": 0.545, |
| "15": 0.511, |
| "16": 0.526, |
| "17": 0.481, |
| "18": 0.459, |
| "19": 0.406, |
| "20": 0.363 |
| }, |
| "pass@2": { |
| "11": 0.679, |
| "12": 0.673, |
| "13": 0.642, |
| "14": 0.579, |
| "15": 0.551, |
| "16": 0.557, |
| "17": 0.51, |
| "18": 0.488, |
| "19": 0.447, |
| "20": 0.396 |
| }, |
| "pass@4": { |
| "11": 0.772, |
| "12": 0.754, |
| "13": 0.723, |
| "14": 0.64, |
| "15": 0.599, |
| "16": 0.594, |
| "17": 0.545, |
| "18": 0.516, |
| "19": 0.471, |
| "20": 0.42 |
| }, |
| "pass@8": { |
| "11": 0.851, |
| "12": 0.857, |
| "13": 0.801, |
| "14": 0.718, |
| "15": 0.669, |
| "16": 0.648, |
| "17": 0.592, |
| "18": 0.555, |
| "19": 0.5, |
| "20": 0.456 |
| }, |
| "pass@16": { |
| "11": 0.92, |
| "12": 0.912, |
| "13": 0.865, |
| "14": 0.775, |
| "15": 0.748, |
| "16": 0.702, |
| "17": 0.646, |
| "18": 0.607, |
| "19": 0.551, |
| "20": 0.506 |
| }, |
| "pass@32": { |
| "11": 0.959, |
| "12": 0.948, |
| "13": 0.914, |
| "14": 0.835, |
| "15": 0.818, |
| "16": 0.757, |
| "17": 0.728, |
| "18": 0.666, |
| "19": 0.604, |
| "20": 0.563 |
| }, |
| "pass@64": { |
| "11": 0.975, |
| "12": 0.972, |
| "13": 0.944, |
| "14": 0.897, |
| "15": 0.89, |
| "16": 0.799, |
| "17": 0.787, |
| "18": 0.743, |
| "19": 0.654, |
| "20": 0.626 |
| }, |
| "pass@128": { |
| "11": 0.986, |
| "12": 0.981, |
| "13": 0.96, |
| "14": 0.934, |
| "15": 0.934, |
| "16": 0.824, |
| "17": 0.83, |
| "18": 0.8, |
| "19": 0.704, |
| "20": 0.68 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5099, |
| "pass@2": 0.5522, |
| "pass@4": 0.6034, |
| "pass@8": 0.6647, |
| "pass@16": 0.7232, |
| "pass@32": 0.7792, |
| "pass@64": 0.8287, |
| "pass@128": 0.8633 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
