| { |
| "files": { |
| "checkpoint-38645_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.990625, |
| "pass@2": 0.996875, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9619047619047619, |
| "pass@2": 0.9746031746031746, |
| "pass@4": 0.9873015873015873, |
| "pass@8": 0.9968253968253968, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9550898203592815, |
| "pass@2": 0.9700598802395209, |
| "pass@4": 0.9820359281437125, |
| "pass@8": 0.9880239520958084, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9053254437869822, |
| "pass@2": 0.9289940828402367, |
| "pass@4": 0.9615384615384616, |
| "pass@8": 0.9881656804733728, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8141025641025641, |
| "pass@2": 0.875, |
| "pass@4": 0.9391025641025641, |
| "pass@8": 0.9743589743589743, |
| "pass@16": 0.9935897435897436, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7577464788732394, |
| "pass@2": 0.8422535211267606, |
| "pass@4": 0.9183098591549296, |
| "pass@8": 0.9661971830985916, |
| "pass@16": 0.9887323943661972, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6578171091445427, |
| "pass@2": 0.7109144542772862, |
| "pass@4": 0.8082595870206489, |
| "pass@8": 0.8672566371681416, |
| "pass@16": 0.9203539823008849, |
| "pass@32": 0.9469026548672567, |
| "pass@64": 0.9587020648967551, |
| "pass@128": 0.976401179941003 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.990625, |
| "5": 0.9619047619047619, |
| "6": 0.9550898203592815, |
| "7": 0.9053254437869822, |
| "8": 0.8141025641025641, |
| "9": 0.7577464788732394, |
| "10": 0.6578171091445427 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9746031746031746, |
| "6": 0.9700598802395209, |
| "7": 0.9289940828402367, |
| "8": 0.875, |
| "9": 0.8422535211267606, |
| "10": 0.7109144542772862 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9873015873015873, |
| "6": 0.9820359281437125, |
| "7": 0.9615384615384616, |
| "8": 0.9391025641025641, |
| "9": 0.9183098591549296, |
| "10": 0.8082595870206489 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 0.9880239520958084, |
| "7": 0.9881656804733728, |
| "8": 0.9743589743589743, |
| "9": 0.9661971830985916, |
| "10": 0.8672566371681416 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 0.9935897435897436, |
| "9": 0.9887323943661972, |
| "10": 0.9203539823008849 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9469026548672567 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9587020648967551 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.976401179941003 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8912529550827423, |
| "pass@2": 0.9202971968929415, |
| "pass@4": 0.9540695710908477, |
| "pass@8": 0.9750084430935495, |
| "pass@16": 0.9885173927727119, |
| "pass@32": 0.9935832489023978, |
| "pass@64": 0.9952718676122931, |
| "pass@128": 0.9972982100641675 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9883040935672515, |
| "pass@2": 0.9941520467836257, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9382352941176471, |
| "pass@2": 0.9647058823529412, |
| "pass@4": 0.9852941176470589, |
| "pass@8": 0.9911764705882353, |
| "pass@16": 0.9970588235294118, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 0.9970588235294118, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9684542586750788, |
| "pass@2": 0.9716088328075709, |
| "pass@4": 0.9842271293375394, |
| "pass@8": 0.9873817034700315, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.870253164556962, |
| "pass@2": 0.9050632911392406, |
| "pass@4": 0.9525316455696202, |
| "pass@8": 0.9810126582278481, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 0.9968354430379747, |
| "pass@64": 0.9968354430379747, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7927927927927928, |
| "pass@2": 0.8738738738738738, |
| "pass@4": 0.9429429429429429, |
| "pass@8": 0.9819819819819819, |
| "pass@16": 0.993993993993994, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.77491961414791, |
| "pass@2": 0.864951768488746, |
| "pass@4": 0.9131832797427653, |
| "pass@8": 0.9646302250803859, |
| "pass@16": 0.9839228295819936, |
| "pass@32": 0.9935691318327974, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.5880597014925373, |
| "pass@2": 0.6686567164179105, |
| "pass@4": 0.7820895522388059, |
| "pass@8": 0.8597014925373134, |
| "pass@16": 0.9164179104477612, |
| "pass@32": 0.9492537313432836, |
| "pass@64": 0.9761194029850746, |
| "pass@128": 0.9880597014925373 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9883040935672515, |
| "5": 0.9382352941176471, |
| "6": 0.9684542586750788, |
| "7": 0.870253164556962, |
| "8": 0.7927927927927928, |
| "9": 0.77491961414791, |
| "10": 0.5880597014925373 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9941520467836257, |
| "5": 0.9647058823529412, |
| "6": 0.9716088328075709, |
| "7": 0.9050632911392406, |
| "8": 0.8738738738738738, |
| "9": 0.864951768488746, |
| "10": 0.6686567164179105 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9852941176470589, |
| "6": 0.9842271293375394, |
| "7": 0.9525316455696202, |
| "8": 0.9429429429429429, |
| "9": 0.9131832797427653, |
| "10": 0.7820895522388059 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9911764705882353, |
| "6": 0.9873817034700315, |
| "7": 0.9810126582278481, |
| "8": 0.9819819819819819, |
| "9": 0.9646302250803859, |
| "10": 0.8597014925373134 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.993993993993994, |
| "9": 0.9839228295819936, |
| "10": 0.9164179104477612 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 0.9935691318327974, |
| "10": 0.9492537313432836 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9761194029850746 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9880597014925373 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8818212253096752, |
| "pass@2": 0.916973552058922, |
| "pass@4": 0.9517910947438902, |
| "pass@8": 0.9742216270505524, |
| "pass@16": 0.987612989621694, |
| "pass@32": 0.9929695346501507, |
| "pass@64": 0.9966521593572146, |
| "pass@128": 0.9983260796786073 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9763313609467456, |
| "pass@2": 0.9822485207100592, |
| "pass@4": 0.9940828402366864, |
| "pass@8": 0.9970414201183432, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9478260869565217, |
| "pass@2": 0.9681159420289855, |
| "pass@4": 0.9884057971014493, |
| "pass@8": 0.9942028985507246, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9799426934097422, |
| "pass@2": 0.9914040114613181, |
| "pass@4": 0.9914040114613181, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8670520231213873, |
| "pass@2": 0.9132947976878613, |
| "pass@4": 0.9450867052023122, |
| "pass@8": 0.9797687861271677, |
| "pass@16": 0.9971098265895953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8169014084507042, |
| "pass@2": 0.8591549295774648, |
| "pass@4": 0.9211267605633803, |
| "pass@8": 0.9887323943661972, |
| "pass@16": 0.9943661971830986, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7395209580838323, |
| "pass@2": 0.8023952095808383, |
| "pass@4": 0.8772455089820359, |
| "pass@8": 0.9491017964071856, |
| "pass@16": 0.9820359281437125, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6625766871165644, |
| "pass@2": 0.7055214723926381, |
| "pass@4": 0.7975460122699386, |
| "pass@8": 0.8773006134969326, |
| "pass@16": 0.9202453987730062, |
| "pass@32": 0.941717791411043, |
| "pass@64": 0.941717791411043, |
| "pass@128": 0.9478527607361963 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9763313609467456, |
| "5": 0.9478260869565217, |
| "6": 0.9799426934097422, |
| "7": 0.8670520231213873, |
| "8": 0.8169014084507042, |
| "9": 0.7395209580838323, |
| "10": 0.6625766871165644 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9822485207100592, |
| "5": 0.9681159420289855, |
| "6": 0.9914040114613181, |
| "7": 0.9132947976878613, |
| "8": 0.8591549295774648, |
| "9": 0.8023952095808383, |
| "10": 0.7055214723926381 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9940828402366864, |
| "5": 0.9884057971014493, |
| "6": 0.9914040114613181, |
| "7": 0.9450867052023122, |
| "8": 0.9211267605633803, |
| "9": 0.8772455089820359, |
| "10": 0.7975460122699386 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 0.9942028985507246, |
| "6": 1.0, |
| "7": 0.9797687861271677, |
| "8": 0.9887323943661972, |
| "9": 0.9491017964071856, |
| "10": 0.8773006134969326 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 1.0, |
| "7": 0.9971098265895953, |
| "8": 0.9943661971830986, |
| "9": 0.9820359281437125, |
| "10": 0.9202453987730062 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9940119760479041, |
| "10": 0.941717791411043 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.941717791411043 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9478527607361963 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8882699868938401, |
| "pass@2": 0.9141546526867628, |
| "pass@4": 0.9465923984272608, |
| "pass@8": 0.9767365661861075, |
| "pass@16": 0.9882044560943644, |
| "pass@32": 0.9927916120576671, |
| "pass@64": 0.9934469200524246, |
| "pass@128": 0.994429882044561 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.985, |
| "pass@2": 0.991, |
| "pass@4": 0.998, |
| "pass@8": 0.999, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.949, |
| "pass@2": 0.969, |
| "pass@4": 0.987, |
| "pass@8": 0.994, |
| "pass@16": 0.998, |
| "pass@32": 0.998, |
| "pass@64": 0.998, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.968, |
| "pass@2": 0.978, |
| "pass@4": 0.986, |
| "pass@8": 0.992, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.881, |
| "pass@2": 0.916, |
| "pass@4": 0.953, |
| "pass@8": 0.983, |
| "pass@16": 0.997, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.808, |
| "pass@2": 0.869, |
| "pass@4": 0.934, |
| "pass@8": 0.982, |
| "pass@16": 0.994, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.757, |
| "pass@2": 0.836, |
| "pass@4": 0.903, |
| "pass@8": 0.96, |
| "pass@16": 0.985, |
| "pass@32": 0.995, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.636, |
| "pass@2": 0.695, |
| "pass@4": 0.796, |
| "pass@8": 0.868, |
| "pass@16": 0.919, |
| "pass@32": 0.946, |
| "pass@64": 0.959, |
| "pass@128": 0.971 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.985, |
| "5": 0.949, |
| "6": 0.968, |
| "7": 0.881, |
| "8": 0.808, |
| "9": 0.757, |
| "10": 0.636 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.991, |
| "5": 0.969, |
| "6": 0.978, |
| "7": 0.916, |
| "8": 0.869, |
| "9": 0.836, |
| "10": 0.695 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.998, |
| "5": 0.987, |
| "6": 0.986, |
| "7": 0.953, |
| "8": 0.934, |
| "9": 0.903, |
| "10": 0.796 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.999, |
| "5": 0.994, |
| "6": 0.992, |
| "7": 0.983, |
| "8": 0.982, |
| "9": 0.96, |
| "10": 0.868 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 1.0, |
| "7": 0.997, |
| "8": 0.994, |
| "9": 0.985, |
| "10": 0.919 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 1.0, |
| "9": 0.995, |
| "10": 0.946 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.959 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.971 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8871111111111111, |
| "pass@2": 0.9171111111111111, |
| "pass@4": 0.9507777777777778, |
| "pass@8": 0.9753333333333334, |
| "pass@16": 0.9881111111111112, |
| "pass@32": 0.9931111111111111, |
| "pass@64": 0.9951111111111111, |
| "pass@128": 0.9966666666666667 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-38645_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6147308781869688, |
| "pass@2": 0.6940509915014165, |
| "pass@4": 0.7903682719546742, |
| "pass@8": 0.8753541076487252, |
| "pass@16": 0.9405099150141643, |
| "pass@32": 0.9688385269121813, |
| "pass@64": 0.9801699716713881, |
| "pass@128": 0.9801699716713881 |
| }, |
| "12": { |
| "pass@1": 0.6610169491525424, |
| "pass@2": 0.7344632768361582, |
| "pass@4": 0.7796610169491526, |
| "pass@8": 0.867231638418079, |
| "pass@16": 0.9265536723163842, |
| "pass@32": 0.9519774011299436, |
| "pass@64": 0.9689265536723164, |
| "pass@128": 0.9717514124293786 |
| }, |
| "13": { |
| "pass@1": 0.5862068965517241, |
| "pass@2": 0.658307210031348, |
| "pass@4": 0.7272727272727273, |
| "pass@8": 0.7931034482758621, |
| "pass@16": 0.8683385579937304, |
| "pass@32": 0.9216300940438872, |
| "pass@64": 0.9435736677115988, |
| "pass@128": 0.9498432601880877 |
| }, |
| "14": { |
| "pass@1": 0.5638629283489096, |
| "pass@2": 0.5950155763239875, |
| "pass@4": 0.6355140186915887, |
| "pass@8": 0.7040498442367601, |
| "pass@16": 0.7757009345794392, |
| "pass@32": 0.8473520249221184, |
| "pass@64": 0.8940809968847352, |
| "pass@128": 0.9376947040498442 |
| }, |
| "15": { |
| "pass@1": 0.5028901734104047, |
| "pass@2": 0.5346820809248555, |
| "pass@4": 0.5780346820809249, |
| "pass@8": 0.6560693641618497, |
| "pass@16": 0.7225433526011561, |
| "pass@32": 0.815028901734104, |
| "pass@64": 0.8930635838150289, |
| "pass@128": 0.9335260115606936 |
| }, |
| "16": { |
| "pass@1": 0.4935483870967742, |
| "pass@2": 0.5193548387096775, |
| "pass@4": 0.5612903225806452, |
| "pass@8": 0.6258064516129033, |
| "pass@16": 0.6516129032258065, |
| "pass@32": 0.7129032258064516, |
| "pass@64": 0.7580645161290323, |
| "pass@128": 0.7870967741935484 |
| }, |
| "17": { |
| "pass@1": 0.5057803468208093, |
| "pass@2": 0.546242774566474, |
| "pass@4": 0.5982658959537572, |
| "pass@8": 0.6416184971098265, |
| "pass@16": 0.6965317919075145, |
| "pass@32": 0.7630057803468208, |
| "pass@64": 0.7832369942196532, |
| "pass@128": 0.8352601156069365 |
| }, |
| "18": { |
| "pass@1": 0.40285714285714286, |
| "pass@2": 0.4342857142857143, |
| "pass@4": 0.4657142857142857, |
| "pass@8": 0.52, |
| "pass@16": 0.5714285714285714, |
| "pass@32": 0.6371428571428571, |
| "pass@64": 0.7028571428571428, |
| "pass@128": 0.7542857142857143 |
| }, |
| "19": { |
| "pass@1": 0.3213213213213213, |
| "pass@2": 0.37237237237237236, |
| "pass@4": 0.4144144144144144, |
| "pass@8": 0.43843843843843844, |
| "pass@16": 0.4744744744744745, |
| "pass@32": 0.5255255255255256, |
| "pass@64": 0.5825825825825826, |
| "pass@128": 0.6336336336336337 |
| }, |
| "20": { |
| "pass@1": 0.1981981981981982, |
| "pass@2": 0.25225225225225223, |
| "pass@4": 0.2912912912912913, |
| "pass@8": 0.32432432432432434, |
| "pass@16": 0.3663663663663664, |
| "pass@32": 0.4294294294294294, |
| "pass@64": 0.48348348348348347, |
| "pass@128": 0.5315315315315315 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6147308781869688, |
| "12": 0.6610169491525424, |
| "13": 0.5862068965517241, |
| "14": 0.5638629283489096, |
| "15": 0.5028901734104047, |
| "16": 0.4935483870967742, |
| "17": 0.5057803468208093, |
| "18": 0.40285714285714286, |
| "19": 0.3213213213213213, |
| "20": 0.1981981981981982 |
| }, |
| "pass@2": { |
| "11": 0.6940509915014165, |
| "12": 0.7344632768361582, |
| "13": 0.658307210031348, |
| "14": 0.5950155763239875, |
| "15": 0.5346820809248555, |
| "16": 0.5193548387096775, |
| "17": 0.546242774566474, |
| "18": 0.4342857142857143, |
| "19": 0.37237237237237236, |
| "20": 0.25225225225225223 |
| }, |
| "pass@4": { |
| "11": 0.7903682719546742, |
| "12": 0.7796610169491526, |
| "13": 0.7272727272727273, |
| "14": 0.6355140186915887, |
| "15": 0.5780346820809249, |
| "16": 0.5612903225806452, |
| "17": 0.5982658959537572, |
| "18": 0.4657142857142857, |
| "19": 0.4144144144144144, |
| "20": 0.2912912912912913 |
| }, |
| "pass@8": { |
| "11": 0.8753541076487252, |
| "12": 0.867231638418079, |
| "13": 0.7931034482758621, |
| "14": 0.7040498442367601, |
| "15": 0.6560693641618497, |
| "16": 0.6258064516129033, |
| "17": 0.6416184971098265, |
| "18": 0.52, |
| "19": 0.43843843843843844, |
| "20": 0.32432432432432434 |
| }, |
| "pass@16": { |
| "11": 0.9405099150141643, |
| "12": 0.9265536723163842, |
| "13": 0.8683385579937304, |
| "14": 0.7757009345794392, |
| "15": 0.7225433526011561, |
| "16": 0.6516129032258065, |
| "17": 0.6965317919075145, |
| "18": 0.5714285714285714, |
| "19": 0.4744744744744745, |
| "20": 0.3663663663663664 |
| }, |
| "pass@32": { |
| "11": 0.9688385269121813, |
| "12": 0.9519774011299436, |
| "13": 0.9216300940438872, |
| "14": 0.8473520249221184, |
| "15": 0.815028901734104, |
| "16": 0.7129032258064516, |
| "17": 0.7630057803468208, |
| "18": 0.6371428571428571, |
| "19": 0.5255255255255256, |
| "20": 0.4294294294294294 |
| }, |
| "pass@64": { |
| "11": 0.9801699716713881, |
| "12": 0.9689265536723164, |
| "13": 0.9435736677115988, |
| "14": 0.8940809968847352, |
| "15": 0.8930635838150289, |
| "16": 0.7580645161290323, |
| "17": 0.7832369942196532, |
| "18": 0.7028571428571428, |
| "19": 0.5825825825825826, |
| "20": 0.48348348348348347 |
| }, |
| "pass@128": { |
| "11": 0.9801699716713881, |
| "12": 0.9717514124293786, |
| "13": 0.9498432601880877, |
| "14": 0.9376947040498442, |
| "15": 0.9335260115606936, |
| "16": 0.7870967741935484, |
| "17": 0.8352601156069365, |
| "18": 0.7542857142857143, |
| "19": 0.6336336336336337, |
| "20": 0.5315315315315315 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.48588410104011887, |
| "pass@2": 0.5352154531946508, |
| "pass@4": 0.5854383358098069, |
| "pass@8": 0.6460624071322437, |
| "pass@16": 0.701040118870728, |
| "pass@32": 0.7586924219910847, |
| "pass@64": 0.8002971768202081, |
| "pass@128": 0.8326894502228827 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5968253968253968, |
| "pass@2": 0.653968253968254, |
| "pass@4": 0.7492063492063492, |
| "pass@8": 0.8507936507936508, |
| "pass@16": 0.9079365079365079, |
| "pass@32": 0.9682539682539683, |
| "pass@64": 0.9873015873015873, |
| "pass@128": 0.9968253968253968 |
| }, |
| "12": { |
| "pass@1": 0.6697247706422018, |
| "pass@2": 0.72782874617737, |
| "pass@4": 0.7706422018348624, |
| "pass@8": 0.8776758409785933, |
| "pass@16": 0.9327217125382263, |
| "pass@32": 0.9785932721712538, |
| "pass@64": 0.9877675840978594, |
| "pass@128": 0.9908256880733946 |
| }, |
| "13": { |
| "pass@1": 0.6172839506172839, |
| "pass@2": 0.6820987654320988, |
| "pass@4": 0.7623456790123457, |
| "pass@8": 0.8518518518518519, |
| "pass@16": 0.9012345679012346, |
| "pass@32": 0.9320987654320988, |
| "pass@64": 0.9629629629629629, |
| "pass@128": 0.9722222222222222 |
| }, |
| "14": { |
| "pass@1": 0.5637982195845698, |
| "pass@2": 0.599406528189911, |
| "pass@4": 0.6528189910979229, |
| "pass@8": 0.7002967359050445, |
| "pass@16": 0.7566765578635015, |
| "pass@32": 0.8160237388724035, |
| "pass@64": 0.8991097922848664, |
| "pass@128": 0.9228486646884273 |
| }, |
| "15": { |
| "pass@1": 0.49382716049382713, |
| "pass@2": 0.5277777777777778, |
| "pass@4": 0.5802469135802469, |
| "pass@8": 0.6450617283950617, |
| "pass@16": 0.7283950617283951, |
| "pass@32": 0.808641975308642, |
| "pass@64": 0.8950617283950617, |
| "pass@128": 0.9290123456790124 |
| }, |
| "16": { |
| "pass@1": 0.528023598820059, |
| "pass@2": 0.551622418879056, |
| "pass@4": 0.6047197640117994, |
| "pass@8": 0.6430678466076696, |
| "pass@16": 0.6902654867256637, |
| "pass@32": 0.7492625368731564, |
| "pass@64": 0.8023598820058997, |
| "pass@128": 0.8259587020648967 |
| }, |
| "17": { |
| "pass@1": 0.4819277108433735, |
| "pass@2": 0.49698795180722893, |
| "pass@4": 0.5271084337349398, |
| "pass@8": 0.572289156626506, |
| "pass@16": 0.6325301204819277, |
| "pass@32": 0.6837349397590361, |
| "pass@64": 0.7439759036144579, |
| "pass@128": 0.8072289156626506 |
| }, |
| "18": { |
| "pass@1": 0.41935483870967744, |
| "pass@2": 0.4774193548387097, |
| "pass@4": 0.5161290322580645, |
| "pass@8": 0.5516129032258065, |
| "pass@16": 0.6, |
| "pass@32": 0.6580645161290323, |
| "pass@64": 0.7419354838709677, |
| "pass@128": 0.7838709677419354 |
| }, |
| "19": { |
| "pass@1": 0.31722054380664655, |
| "pass@2": 0.3806646525679758, |
| "pass@4": 0.42900302114803623, |
| "pass@8": 0.46827794561933533, |
| "pass@16": 0.5045317220543807, |
| "pass@32": 0.5468277945619335, |
| "pass@64": 0.5921450151057401, |
| "pass@128": 0.6283987915407855 |
| }, |
| "20": { |
| "pass@1": 0.2222222222222222, |
| "pass@2": 0.2672672672672673, |
| "pass@4": 0.29429429429429427, |
| "pass@8": 0.34234234234234234, |
| "pass@16": 0.4024024024024024, |
| "pass@32": 0.44744744744744747, |
| "pass@64": 0.5015015015015015, |
| "pass@128": 0.5735735735735735 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5968253968253968, |
| "12": 0.6697247706422018, |
| "13": 0.6172839506172839, |
| "14": 0.5637982195845698, |
| "15": 0.49382716049382713, |
| "16": 0.528023598820059, |
| "17": 0.4819277108433735, |
| "18": 0.41935483870967744, |
| "19": 0.31722054380664655, |
| "20": 0.2222222222222222 |
| }, |
| "pass@2": { |
| "11": 0.653968253968254, |
| "12": 0.72782874617737, |
| "13": 0.6820987654320988, |
| "14": 0.599406528189911, |
| "15": 0.5277777777777778, |
| "16": 0.551622418879056, |
| "17": 0.49698795180722893, |
| "18": 0.4774193548387097, |
| "19": 0.3806646525679758, |
| "20": 0.2672672672672673 |
| }, |
| "pass@4": { |
| "11": 0.7492063492063492, |
| "12": 0.7706422018348624, |
| "13": 0.7623456790123457, |
| "14": 0.6528189910979229, |
| "15": 0.5802469135802469, |
| "16": 0.6047197640117994, |
| "17": 0.5271084337349398, |
| "18": 0.5161290322580645, |
| "19": 0.42900302114803623, |
| "20": 0.29429429429429427 |
| }, |
| "pass@8": { |
| "11": 0.8507936507936508, |
| "12": 0.8776758409785933, |
| "13": 0.8518518518518519, |
| "14": 0.7002967359050445, |
| "15": 0.6450617283950617, |
| "16": 0.6430678466076696, |
| "17": 0.572289156626506, |
| "18": 0.5516129032258065, |
| "19": 0.46827794561933533, |
| "20": 0.34234234234234234 |
| }, |
| "pass@16": { |
| "11": 0.9079365079365079, |
| "12": 0.9327217125382263, |
| "13": 0.9012345679012346, |
| "14": 0.7566765578635015, |
| "15": 0.7283950617283951, |
| "16": 0.6902654867256637, |
| "17": 0.6325301204819277, |
| "18": 0.6, |
| "19": 0.5045317220543807, |
| "20": 0.4024024024024024 |
| }, |
| "pass@32": { |
| "11": 0.9682539682539683, |
| "12": 0.9785932721712538, |
| "13": 0.9320987654320988, |
| "14": 0.8160237388724035, |
| "15": 0.808641975308642, |
| "16": 0.7492625368731564, |
| "17": 0.6837349397590361, |
| "18": 0.6580645161290323, |
| "19": 0.5468277945619335, |
| "20": 0.44744744744744747 |
| }, |
| "pass@64": { |
| "11": 0.9873015873015873, |
| "12": 0.9877675840978594, |
| "13": 0.9629629629629629, |
| "14": 0.8991097922848664, |
| "15": 0.8950617283950617, |
| "16": 0.8023598820058997, |
| "17": 0.7439759036144579, |
| "18": 0.7419354838709677, |
| "19": 0.5921450151057401, |
| "20": 0.5015015015015015 |
| }, |
| "pass@128": { |
| "11": 0.9968253968253968, |
| "12": 0.9908256880733946, |
| "13": 0.9722222222222222, |
| "14": 0.9228486646884273, |
| "15": 0.9290123456790124, |
| "16": 0.8259587020648967, |
| "17": 0.8072289156626506, |
| "18": 0.7838709677419354, |
| "19": 0.6283987915407855, |
| "20": 0.5735735735735735 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.49052567237163813, |
| "pass@2": 0.535757946210269, |
| "pass@4": 0.5877139364303179, |
| "pass@8": 0.6491442542787286, |
| "pass@16": 0.7044621026894865, |
| "pass@32": 0.757640586797066, |
| "pass@64": 0.8102078239608802, |
| "pass@128": 0.8419926650366748 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6566265060240963, |
| "pass@2": 0.6957831325301205, |
| "pass@4": 0.7620481927710844, |
| "pass@8": 0.8524096385542169, |
| "pass@16": 0.9096385542168675, |
| "pass@32": 0.9457831325301205, |
| "pass@64": 0.9668674698795181, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.5987460815047022, |
| "pass@2": 0.6551724137931034, |
| "pass@4": 0.7210031347962382, |
| "pass@8": 0.8181818181818182, |
| "pass@16": 0.9090909090909091, |
| "pass@32": 0.9498432601880877, |
| "pass@64": 0.9686520376175548, |
| "pass@128": 0.9843260188087775 |
| }, |
| "13": { |
| "pass@1": 0.6246498599439776, |
| "pass@2": 0.6890756302521008, |
| "pass@4": 0.7759103641456583, |
| "pass@8": 0.8487394957983193, |
| "pass@16": 0.9019607843137255, |
| "pass@32": 0.938375350140056, |
| "pass@64": 0.9607843137254902, |
| "pass@128": 0.9747899159663865 |
| }, |
| "14": { |
| "pass@1": 0.5818713450292398, |
| "pass@2": 0.6198830409356725, |
| "pass@4": 0.6666666666666666, |
| "pass@8": 0.7309941520467836, |
| "pass@16": 0.7953216374269005, |
| "pass@32": 0.8567251461988304, |
| "pass@64": 0.9152046783625731, |
| "pass@128": 0.9678362573099415 |
| }, |
| "15": { |
| "pass@1": 0.5787878787878787, |
| "pass@2": 0.6060606060606061, |
| "pass@4": 0.6393939393939394, |
| "pass@8": 0.693939393939394, |
| "pass@16": 0.7484848484848485, |
| "pass@32": 0.8181818181818182, |
| "pass@64": 0.8848484848484849, |
| "pass@128": 0.9515151515151515 |
| }, |
| "16": { |
| "pass@1": 0.5071225071225072, |
| "pass@2": 0.5584045584045584, |
| "pass@4": 0.5925925925925926, |
| "pass@8": 0.6438746438746439, |
| "pass@16": 0.7008547008547008, |
| "pass@32": 0.7720797720797721, |
| "pass@64": 0.8176638176638177, |
| "pass@128": 0.8347578347578347 |
| }, |
| "17": { |
| "pass@1": 0.40993788819875776, |
| "pass@2": 0.45652173913043476, |
| "pass@4": 0.4937888198757764, |
| "pass@8": 0.5434782608695652, |
| "pass@16": 0.6242236024844721, |
| "pass@32": 0.7018633540372671, |
| "pass@64": 0.7391304347826086, |
| "pass@128": 0.8043478260869565 |
| }, |
| "18": { |
| "pass@1": 0.3911764705882353, |
| "pass@2": 0.4294117647058823, |
| "pass@4": 0.4647058823529412, |
| "pass@8": 0.49411764705882355, |
| "pass@16": 0.5294117647058824, |
| "pass@32": 0.5941176470588235, |
| "pass@64": 0.6705882352941176, |
| "pass@128": 0.7382352941176471 |
| }, |
| "19": { |
| "pass@1": 0.31845238095238093, |
| "pass@2": 0.38392857142857145, |
| "pass@4": 0.43452380952380953, |
| "pass@8": 0.46130952380952384, |
| "pass@16": 0.5148809523809523, |
| "pass@32": 0.5505952380952381, |
| "pass@64": 0.6130952380952381, |
| "pass@128": 0.6577380952380952 |
| }, |
| "20": { |
| "pass@1": 0.19461077844311378, |
| "pass@2": 0.24550898203592814, |
| "pass@4": 0.2844311377245509, |
| "pass@8": 0.31736526946107785, |
| "pass@16": 0.3562874251497006, |
| "pass@32": 0.40119760479041916, |
| "pass@64": 0.45808383233532934, |
| "pass@128": 0.5239520958083832 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6566265060240963, |
| "12": 0.5987460815047022, |
| "13": 0.6246498599439776, |
| "14": 0.5818713450292398, |
| "15": 0.5787878787878787, |
| "16": 0.5071225071225072, |
| "17": 0.40993788819875776, |
| "18": 0.3911764705882353, |
| "19": 0.31845238095238093, |
| "20": 0.19461077844311378 |
| }, |
| "pass@2": { |
| "11": 0.6957831325301205, |
| "12": 0.6551724137931034, |
| "13": 0.6890756302521008, |
| "14": 0.6198830409356725, |
| "15": 0.6060606060606061, |
| "16": 0.5584045584045584, |
| "17": 0.45652173913043476, |
| "18": 0.4294117647058823, |
| "19": 0.38392857142857145, |
| "20": 0.24550898203592814 |
| }, |
| "pass@4": { |
| "11": 0.7620481927710844, |
| "12": 0.7210031347962382, |
| "13": 0.7759103641456583, |
| "14": 0.6666666666666666, |
| "15": 0.6393939393939394, |
| "16": 0.5925925925925926, |
| "17": 0.4937888198757764, |
| "18": 0.4647058823529412, |
| "19": 0.43452380952380953, |
| "20": 0.2844311377245509 |
| }, |
| "pass@8": { |
| "11": 0.8524096385542169, |
| "12": 0.8181818181818182, |
| "13": 0.8487394957983193, |
| "14": 0.7309941520467836, |
| "15": 0.693939393939394, |
| "16": 0.6438746438746439, |
| "17": 0.5434782608695652, |
| "18": 0.49411764705882355, |
| "19": 0.46130952380952384, |
| "20": 0.31736526946107785 |
| }, |
| "pass@16": { |
| "11": 0.9096385542168675, |
| "12": 0.9090909090909091, |
| "13": 0.9019607843137255, |
| "14": 0.7953216374269005, |
| "15": 0.7484848484848485, |
| "16": 0.7008547008547008, |
| "17": 0.6242236024844721, |
| "18": 0.5294117647058824, |
| "19": 0.5148809523809523, |
| "20": 0.3562874251497006 |
| }, |
| "pass@32": { |
| "11": 0.9457831325301205, |
| "12": 0.9498432601880877, |
| "13": 0.938375350140056, |
| "14": 0.8567251461988304, |
| "15": 0.8181818181818182, |
| "16": 0.7720797720797721, |
| "17": 0.7018633540372671, |
| "18": 0.5941176470588235, |
| "19": 0.5505952380952381, |
| "20": 0.40119760479041916 |
| }, |
| "pass@64": { |
| "11": 0.9668674698795181, |
| "12": 0.9686520376175548, |
| "13": 0.9607843137254902, |
| "14": 0.9152046783625731, |
| "15": 0.8848484848484849, |
| "16": 0.8176638176638177, |
| "17": 0.7391304347826086, |
| "18": 0.6705882352941176, |
| "19": 0.6130952380952381, |
| "20": 0.45808383233532934 |
| }, |
| "pass@128": { |
| "11": 0.9789156626506024, |
| "12": 0.9843260188087775, |
| "13": 0.9747899159663865, |
| "14": 0.9678362573099415, |
| "15": 0.9515151515151515, |
| "16": 0.8347578347578347, |
| "17": 0.8043478260869565, |
| "18": 0.7382352941176471, |
| "19": 0.6577380952380952, |
| "20": 0.5239520958083832 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.48676776687481416, |
| "pass@2": 0.5346416889681832, |
| "pass@4": 0.584299732381802, |
| "pass@8": 0.6410942610764199, |
| "pass@16": 0.6993755575379126, |
| "pass@32": 0.7531965506987809, |
| "pass@64": 0.79988105857865, |
| "pass@128": 0.8418079096045198 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.623, |
| "pass@2": 0.682, |
| "pass@4": 0.768, |
| "pass@8": 0.86, |
| "pass@16": 0.92, |
| "pass@32": 0.961, |
| "pass@64": 0.978, |
| "pass@128": 0.985 |
| }, |
| "12": { |
| "pass@1": 0.644, |
| "pass@2": 0.707, |
| "pass@4": 0.758, |
| "pass@8": 0.855, |
| "pass@16": 0.923, |
| "pass@32": 0.96, |
| "pass@64": 0.975, |
| "pass@128": 0.982 |
| }, |
| "13": { |
| "pass@1": 0.61, |
| "pass@2": 0.677, |
| "pass@4": 0.756, |
| "pass@8": 0.832, |
| "pass@16": 0.891, |
| "pass@32": 0.931, |
| "pass@64": 0.956, |
| "pass@128": 0.966 |
| }, |
| "14": { |
| "pass@1": 0.57, |
| "pass@2": 0.605, |
| "pass@4": 0.652, |
| "pass@8": 0.712, |
| "pass@16": 0.776, |
| "pass@32": 0.84, |
| "pass@64": 0.903, |
| "pass@128": 0.943 |
| }, |
| "15": { |
| "pass@1": 0.525, |
| "pass@2": 0.556, |
| "pass@4": 0.599, |
| "pass@8": 0.665, |
| "pass@16": 0.733, |
| "pass@32": 0.814, |
| "pass@64": 0.891, |
| "pass@128": 0.938 |
| }, |
| "16": { |
| "pass@1": 0.51, |
| "pass@2": 0.544, |
| "pass@4": 0.587, |
| "pass@8": 0.638, |
| "pass@16": 0.682, |
| "pass@32": 0.746, |
| "pass@64": 0.794, |
| "pass@128": 0.817 |
| }, |
| "17": { |
| "pass@1": 0.467, |
| "pass@2": 0.501, |
| "pass@4": 0.541, |
| "pass@8": 0.587, |
| "pass@16": 0.652, |
| "pass@32": 0.717, |
| "pass@64": 0.756, |
| "pass@128": 0.816 |
| }, |
| "18": { |
| "pass@1": 0.404, |
| "pass@2": 0.446, |
| "pass@4": 0.481, |
| "pass@8": 0.521, |
| "pass@16": 0.566, |
| "pass@32": 0.629, |
| "pass@64": 0.704, |
| "pass@128": 0.758 |
| }, |
| "19": { |
| "pass@1": 0.319, |
| "pass@2": 0.379, |
| "pass@4": 0.426, |
| "pass@8": 0.456, |
| "pass@16": 0.498, |
| "pass@32": 0.541, |
| "pass@64": 0.596, |
| "pass@128": 0.64 |
| }, |
| "20": { |
| "pass@1": 0.205, |
| "pass@2": 0.255, |
| "pass@4": 0.29, |
| "pass@8": 0.328, |
| "pass@16": 0.375, |
| "pass@32": 0.426, |
| "pass@64": 0.481, |
| "pass@128": 0.543 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.623, |
| "12": 0.644, |
| "13": 0.61, |
| "14": 0.57, |
| "15": 0.525, |
| "16": 0.51, |
| "17": 0.467, |
| "18": 0.404, |
| "19": 0.319, |
| "20": 0.205 |
| }, |
| "pass@2": { |
| "11": 0.682, |
| "12": 0.707, |
| "13": 0.677, |
| "14": 0.605, |
| "15": 0.556, |
| "16": 0.544, |
| "17": 0.501, |
| "18": 0.446, |
| "19": 0.379, |
| "20": 0.255 |
| }, |
| "pass@4": { |
| "11": 0.768, |
| "12": 0.758, |
| "13": 0.756, |
| "14": 0.652, |
| "15": 0.599, |
| "16": 0.587, |
| "17": 0.541, |
| "18": 0.481, |
| "19": 0.426, |
| "20": 0.29 |
| }, |
| "pass@8": { |
| "11": 0.86, |
| "12": 0.855, |
| "13": 0.832, |
| "14": 0.712, |
| "15": 0.665, |
| "16": 0.638, |
| "17": 0.587, |
| "18": 0.521, |
| "19": 0.456, |
| "20": 0.328 |
| }, |
| "pass@16": { |
| "11": 0.92, |
| "12": 0.923, |
| "13": 0.891, |
| "14": 0.776, |
| "15": 0.733, |
| "16": 0.682, |
| "17": 0.652, |
| "18": 0.566, |
| "19": 0.498, |
| "20": 0.375 |
| }, |
| "pass@32": { |
| "11": 0.961, |
| "12": 0.96, |
| "13": 0.931, |
| "14": 0.84, |
| "15": 0.814, |
| "16": 0.746, |
| "17": 0.717, |
| "18": 0.629, |
| "19": 0.541, |
| "20": 0.426 |
| }, |
| "pass@64": { |
| "11": 0.978, |
| "12": 0.975, |
| "13": 0.956, |
| "14": 0.903, |
| "15": 0.891, |
| "16": 0.794, |
| "17": 0.756, |
| "18": 0.704, |
| "19": 0.596, |
| "20": 0.481 |
| }, |
| "pass@128": { |
| "11": 0.985, |
| "12": 0.982, |
| "13": 0.966, |
| "14": 0.943, |
| "15": 0.938, |
| "16": 0.817, |
| "17": 0.816, |
| "18": 0.758, |
| "19": 0.64, |
| "20": 0.543 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4877, |
| "pass@2": 0.5352, |
| "pass@4": 0.5858, |
| "pass@8": 0.6454, |
| "pass@16": 0.7016, |
| "pass@32": 0.7565, |
| "pass@64": 0.8034, |
| "pass@128": 0.8388 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
