{ "files": { "checkpoint-38645_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.990625, "pass@2": 0.996875, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9619047619047619, "pass@2": 0.9746031746031746, "pass@4": 0.9873015873015873, "pass@8": 0.9968253968253968, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9550898203592815, "pass@2": 0.9700598802395209, "pass@4": 0.9820359281437125, "pass@8": 0.9880239520958084, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9053254437869822, "pass@2": 0.9289940828402367, "pass@4": 0.9615384615384616, "pass@8": 0.9881656804733728, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8141025641025641, "pass@2": 0.875, "pass@4": 0.9391025641025641, "pass@8": 0.9743589743589743, "pass@16": 0.9935897435897436, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7577464788732394, "pass@2": 0.8422535211267606, "pass@4": 0.9183098591549296, "pass@8": 0.9661971830985916, "pass@16": 0.9887323943661972, "pass@32": 0.9971830985915493, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6578171091445427, "pass@2": 0.7109144542772862, "pass@4": 0.8082595870206489, "pass@8": 0.8672566371681416, "pass@16": 0.9203539823008849, "pass@32": 0.9469026548672567, "pass@64": 0.9587020648967551, "pass@128": 0.976401179941003 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.990625, "5": 0.9619047619047619, "6": 0.9550898203592815, "7": 0.9053254437869822, "8": 0.8141025641025641, "9": 0.7577464788732394, "10": 0.6578171091445427 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9746031746031746, "6": 0.9700598802395209, "7": 0.9289940828402367, "8": 0.875, "9": 0.8422535211267606, "10": 0.7109144542772862 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9873015873015873, "6": 0.9820359281437125, "7": 0.9615384615384616, "8": 0.9391025641025641, "9": 0.9183098591549296, "10": 0.8082595870206489 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 0.9880239520958084, "7": 0.9881656804733728, "8": 0.9743589743589743, "9": 0.9661971830985916, "10": 0.8672566371681416 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9970414201183432, "8": 0.9935897435897436, "9": 0.9887323943661972, "10": 0.9203539823008849 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9469026548672567 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9587020648967551 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.976401179941003 } }, "total_pass_at_k": { "pass@1": 0.8912529550827423, "pass@2": 0.9202971968929415, "pass@4": 0.9540695710908477, "pass@8": 0.9750084430935495, "pass@16": 0.9885173927727119, "pass@32": 0.9935832489023978, "pass@64": 0.9952718676122931, "pass@128": 0.9972982100641675 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9883040935672515, "pass@2": 0.9941520467836257, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9382352941176471, "pass@2": 0.9647058823529412, "pass@4": 0.9852941176470589, "pass@8": 0.9911764705882353, "pass@16": 0.9970588235294118, "pass@32": 0.9970588235294118, "pass@64": 0.9970588235294118, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9684542586750788, "pass@2": 0.9716088328075709, "pass@4": 0.9842271293375394, "pass@8": 0.9873817034700315, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.870253164556962, "pass@2": 0.9050632911392406, "pass@4": 0.9525316455696202, "pass@8": 0.9810126582278481, "pass@16": 0.9968354430379747, "pass@32": 0.9968354430379747, "pass@64": 0.9968354430379747, "pass@128": 1.0 }, "8": { "pass@1": 0.7927927927927928, "pass@2": 0.8738738738738738, "pass@4": 0.9429429429429429, "pass@8": 0.9819819819819819, "pass@16": 0.993993993993994, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.77491961414791, "pass@2": 0.864951768488746, "pass@4": 0.9131832797427653, "pass@8": 0.9646302250803859, "pass@16": 0.9839228295819936, "pass@32": 0.9935691318327974, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.5880597014925373, "pass@2": 0.6686567164179105, "pass@4": 0.7820895522388059, "pass@8": 0.8597014925373134, "pass@16": 0.9164179104477612, "pass@32": 0.9492537313432836, "pass@64": 0.9761194029850746, "pass@128": 0.9880597014925373 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9883040935672515, "5": 0.9382352941176471, "6": 0.9684542586750788, "7": 0.870253164556962, "8": 0.7927927927927928, "9": 0.77491961414791, "10": 0.5880597014925373 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9941520467836257, "5": 0.9647058823529412, "6": 0.9716088328075709, "7": 0.9050632911392406, "8": 0.8738738738738738, "9": 0.864951768488746, "10": 0.6686567164179105 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9852941176470589, "6": 0.9842271293375394, "7": 0.9525316455696202, "8": 0.9429429429429429, "9": 0.9131832797427653, "10": 0.7820895522388059 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9911764705882353, "6": 0.9873817034700315, "7": 0.9810126582278481, "8": 0.9819819819819819, "9": 0.9646302250803859, "10": 0.8597014925373134 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 0.993993993993994, "9": 0.9839228295819936, "10": 0.9164179104477612 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 1.0, "9": 0.9935691318327974, "10": 0.9492537313432836 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 1.0, "9": 1.0, "10": 0.9761194029850746 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9880597014925373 } }, "total_pass_at_k": { "pass@1": 0.8818212253096752, "pass@2": 0.916973552058922, "pass@4": 0.9517910947438902, "pass@8": 0.9742216270505524, "pass@16": 0.987612989621694, "pass@32": 0.9929695346501507, "pass@64": 0.9966521593572146, "pass@128": 0.9983260796786073 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9763313609467456, "pass@2": 0.9822485207100592, "pass@4": 0.9940828402366864, "pass@8": 0.9970414201183432, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9478260869565217, "pass@2": 0.9681159420289855, "pass@4": 0.9884057971014493, "pass@8": 0.9942028985507246, "pass@16": 0.9971014492753624, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 1.0 }, "6": { "pass@1": 0.9799426934097422, "pass@2": 0.9914040114613181, "pass@4": 0.9914040114613181, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8670520231213873, "pass@2": 0.9132947976878613, "pass@4": 0.9450867052023122, "pass@8": 0.9797687861271677, "pass@16": 0.9971098265895953, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8169014084507042, "pass@2": 0.8591549295774648, "pass@4": 0.9211267605633803, "pass@8": 0.9887323943661972, "pass@16": 0.9943661971830986, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7395209580838323, "pass@2": 0.8023952095808383, "pass@4": 0.8772455089820359, "pass@8": 0.9491017964071856, "pass@16": 0.9820359281437125, "pass@32": 0.9940119760479041, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6625766871165644, "pass@2": 0.7055214723926381, "pass@4": 0.7975460122699386, "pass@8": 0.8773006134969326, "pass@16": 0.9202453987730062, "pass@32": 0.941717791411043, "pass@64": 0.941717791411043, "pass@128": 0.9478527607361963 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9763313609467456, "5": 0.9478260869565217, "6": 0.9799426934097422, "7": 0.8670520231213873, "8": 0.8169014084507042, "9": 0.7395209580838323, "10": 0.6625766871165644 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9822485207100592, "5": 0.9681159420289855, "6": 0.9914040114613181, "7": 0.9132947976878613, "8": 0.8591549295774648, "9": 0.8023952095808383, "10": 0.7055214723926381 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9940828402366864, "5": 0.9884057971014493, "6": 0.9914040114613181, "7": 0.9450867052023122, "8": 0.9211267605633803, "9": 0.8772455089820359, "10": 0.7975460122699386 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 0.9942028985507246, "6": 1.0, "7": 0.9797687861271677, "8": 0.9887323943661972, "9": 0.9491017964071856, "10": 0.8773006134969326 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 1.0, "7": 0.9971098265895953, "8": 0.9943661971830986, "9": 0.9820359281437125, "10": 0.9202453987730062 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9940119760479041, "10": 0.941717791411043 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.941717791411043 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9478527607361963 } }, "total_pass_at_k": { "pass@1": 0.8882699868938401, "pass@2": 0.9141546526867628, "pass@4": 0.9465923984272608, "pass@8": 0.9767365661861075, "pass@16": 0.9882044560943644, "pass@32": 0.9927916120576671, "pass@64": 0.9934469200524246, "pass@128": 0.994429882044561 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.985, "pass@2": 0.991, "pass@4": 0.998, "pass@8": 0.999, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.949, "pass@2": 0.969, "pass@4": 0.987, "pass@8": 0.994, "pass@16": 0.998, "pass@32": 0.998, "pass@64": 0.998, "pass@128": 0.999 }, "6": { "pass@1": 0.968, "pass@2": 0.978, "pass@4": 0.986, "pass@8": 0.992, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.881, "pass@2": 0.916, "pass@4": 0.953, "pass@8": 0.983, "pass@16": 0.997, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 1.0 }, "8": { "pass@1": 0.808, "pass@2": 0.869, "pass@4": 0.934, "pass@8": 0.982, "pass@16": 0.994, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.757, "pass@2": 0.836, "pass@4": 0.903, "pass@8": 0.96, "pass@16": 0.985, "pass@32": 0.995, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.636, "pass@2": 0.695, "pass@4": 0.796, "pass@8": 0.868, "pass@16": 0.919, "pass@32": 0.946, "pass@64": 0.959, "pass@128": 0.971 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.985, "5": 0.949, "6": 0.968, "7": 0.881, "8": 0.808, "9": 0.757, "10": 0.636 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.991, "5": 0.969, "6": 0.978, "7": 0.916, "8": 0.869, "9": 0.836, "10": 0.695 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.998, "5": 0.987, "6": 0.986, "7": 0.953, "8": 0.934, "9": 0.903, "10": 0.796 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 0.999, "5": 0.994, "6": 0.992, "7": 0.983, "8": 0.982, "9": 0.96, "10": 0.868 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 1.0, "7": 0.997, "8": 0.994, "9": 0.985, "10": 0.919 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 1.0, "7": 0.999, "8": 1.0, "9": 0.995, "10": 0.946 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 1.0, "7": 0.999, "8": 1.0, "9": 1.0, "10": 0.959 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.971 } }, "total_pass_at_k": { "pass@1": 0.8871111111111111, "pass@2": 0.9171111111111111, "pass@4": 0.9507777777777778, "pass@8": 0.9753333333333334, "pass@16": 0.9881111111111112, "pass@32": 0.9931111111111111, "pass@64": 0.9951111111111111, "pass@128": 0.9966666666666667 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-38645_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.6147308781869688, "pass@2": 0.6940509915014165, "pass@4": 0.7903682719546742, "pass@8": 0.8753541076487252, "pass@16": 0.9405099150141643, "pass@32": 0.9688385269121813, "pass@64": 0.9801699716713881, "pass@128": 0.9801699716713881 }, "12": { "pass@1": 0.6610169491525424, "pass@2": 0.7344632768361582, "pass@4": 0.7796610169491526, "pass@8": 0.867231638418079, "pass@16": 0.9265536723163842, "pass@32": 0.9519774011299436, "pass@64": 0.9689265536723164, "pass@128": 0.9717514124293786 }, "13": { "pass@1": 0.5862068965517241, "pass@2": 0.658307210031348, "pass@4": 0.7272727272727273, "pass@8": 0.7931034482758621, "pass@16": 0.8683385579937304, "pass@32": 0.9216300940438872, "pass@64": 0.9435736677115988, "pass@128": 0.9498432601880877 }, "14": { "pass@1": 0.5638629283489096, "pass@2": 0.5950155763239875, "pass@4": 0.6355140186915887, "pass@8": 0.7040498442367601, "pass@16": 0.7757009345794392, "pass@32": 0.8473520249221184, "pass@64": 0.8940809968847352, "pass@128": 0.9376947040498442 }, "15": { "pass@1": 0.5028901734104047, "pass@2": 0.5346820809248555, "pass@4": 0.5780346820809249, "pass@8": 0.6560693641618497, "pass@16": 0.7225433526011561, "pass@32": 0.815028901734104, "pass@64": 0.8930635838150289, "pass@128": 0.9335260115606936 }, "16": { "pass@1": 0.4935483870967742, "pass@2": 0.5193548387096775, "pass@4": 0.5612903225806452, "pass@8": 0.6258064516129033, "pass@16": 0.6516129032258065, "pass@32": 0.7129032258064516, "pass@64": 0.7580645161290323, "pass@128": 0.7870967741935484 }, "17": { "pass@1": 0.5057803468208093, "pass@2": 0.546242774566474, "pass@4": 0.5982658959537572, "pass@8": 0.6416184971098265, "pass@16": 0.6965317919075145, "pass@32": 0.7630057803468208, "pass@64": 0.7832369942196532, "pass@128": 0.8352601156069365 }, "18": { "pass@1": 0.40285714285714286, "pass@2": 0.4342857142857143, "pass@4": 0.4657142857142857, "pass@8": 0.52, "pass@16": 0.5714285714285714, "pass@32": 0.6371428571428571, "pass@64": 0.7028571428571428, "pass@128": 0.7542857142857143 }, "19": { "pass@1": 0.3213213213213213, "pass@2": 0.37237237237237236, "pass@4": 0.4144144144144144, "pass@8": 0.43843843843843844, "pass@16": 0.4744744744744745, "pass@32": 0.5255255255255256, "pass@64": 0.5825825825825826, "pass@128": 0.6336336336336337 }, "20": { "pass@1": 0.1981981981981982, "pass@2": 0.25225225225225223, "pass@4": 0.2912912912912913, "pass@8": 0.32432432432432434, "pass@16": 0.3663663663663664, "pass@32": 0.4294294294294294, "pass@64": 0.48348348348348347, "pass@128": 0.5315315315315315 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6147308781869688, "12": 0.6610169491525424, "13": 0.5862068965517241, "14": 0.5638629283489096, "15": 0.5028901734104047, "16": 0.4935483870967742, "17": 0.5057803468208093, "18": 0.40285714285714286, "19": 0.3213213213213213, "20": 0.1981981981981982 }, "pass@2": { "11": 0.6940509915014165, "12": 0.7344632768361582, "13": 0.658307210031348, "14": 0.5950155763239875, "15": 0.5346820809248555, "16": 0.5193548387096775, "17": 0.546242774566474, "18": 0.4342857142857143, "19": 0.37237237237237236, "20": 0.25225225225225223 }, "pass@4": { "11": 0.7903682719546742, "12": 0.7796610169491526, "13": 0.7272727272727273, "14": 0.6355140186915887, "15": 0.5780346820809249, "16": 0.5612903225806452, "17": 0.5982658959537572, "18": 0.4657142857142857, "19": 0.4144144144144144, "20": 0.2912912912912913 }, "pass@8": { "11": 0.8753541076487252, "12": 0.867231638418079, "13": 0.7931034482758621, "14": 0.7040498442367601, "15": 0.6560693641618497, "16": 0.6258064516129033, "17": 0.6416184971098265, "18": 0.52, "19": 0.43843843843843844, "20": 0.32432432432432434 }, "pass@16": { "11": 0.9405099150141643, "12": 0.9265536723163842, "13": 0.8683385579937304, "14": 0.7757009345794392, "15": 0.7225433526011561, "16": 0.6516129032258065, "17": 0.6965317919075145, "18": 0.5714285714285714, "19": 0.4744744744744745, "20": 0.3663663663663664 }, "pass@32": { "11": 0.9688385269121813, "12": 0.9519774011299436, "13": 0.9216300940438872, "14": 0.8473520249221184, "15": 0.815028901734104, "16": 0.7129032258064516, "17": 0.7630057803468208, "18": 0.6371428571428571, "19": 0.5255255255255256, "20": 0.4294294294294294 }, "pass@64": { "11": 0.9801699716713881, "12": 0.9689265536723164, "13": 0.9435736677115988, "14": 0.8940809968847352, "15": 0.8930635838150289, "16": 0.7580645161290323, "17": 0.7832369942196532, "18": 0.7028571428571428, "19": 0.5825825825825826, "20": 0.48348348348348347 }, "pass@128": { "11": 0.9801699716713881, "12": 0.9717514124293786, "13": 0.9498432601880877, "14": 0.9376947040498442, "15": 0.9335260115606936, "16": 0.7870967741935484, "17": 0.8352601156069365, "18": 0.7542857142857143, "19": 0.6336336336336337, "20": 0.5315315315315315 } }, "total_pass_at_k": { "pass@1": 0.48588410104011887, "pass@2": 0.5352154531946508, "pass@4": 0.5854383358098069, "pass@8": 0.6460624071322437, "pass@16": 0.701040118870728, "pass@32": 0.7586924219910847, "pass@64": 0.8002971768202081, "pass@128": 0.8326894502228827 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.5968253968253968, "pass@2": 0.653968253968254, "pass@4": 0.7492063492063492, "pass@8": 0.8507936507936508, "pass@16": 0.9079365079365079, "pass@32": 0.9682539682539683, "pass@64": 0.9873015873015873, "pass@128": 0.9968253968253968 }, "12": { "pass@1": 0.6697247706422018, "pass@2": 0.72782874617737, "pass@4": 0.7706422018348624, "pass@8": 0.8776758409785933, "pass@16": 0.9327217125382263, "pass@32": 0.9785932721712538, "pass@64": 0.9877675840978594, "pass@128": 0.9908256880733946 }, "13": { "pass@1": 0.6172839506172839, "pass@2": 0.6820987654320988, "pass@4": 0.7623456790123457, "pass@8": 0.8518518518518519, "pass@16": 0.9012345679012346, "pass@32": 0.9320987654320988, "pass@64": 0.9629629629629629, "pass@128": 0.9722222222222222 }, "14": { "pass@1": 0.5637982195845698, "pass@2": 0.599406528189911, "pass@4": 0.6528189910979229, "pass@8": 0.7002967359050445, "pass@16": 0.7566765578635015, "pass@32": 0.8160237388724035, "pass@64": 0.8991097922848664, "pass@128": 0.9228486646884273 }, "15": { "pass@1": 0.49382716049382713, "pass@2": 0.5277777777777778, "pass@4": 0.5802469135802469, "pass@8": 0.6450617283950617, "pass@16": 0.7283950617283951, "pass@32": 0.808641975308642, "pass@64": 0.8950617283950617, "pass@128": 0.9290123456790124 }, "16": { "pass@1": 0.528023598820059, "pass@2": 0.551622418879056, "pass@4": 0.6047197640117994, "pass@8": 0.6430678466076696, "pass@16": 0.6902654867256637, "pass@32": 0.7492625368731564, "pass@64": 0.8023598820058997, "pass@128": 0.8259587020648967 }, "17": { "pass@1": 0.4819277108433735, "pass@2": 0.49698795180722893, "pass@4": 0.5271084337349398, "pass@8": 0.572289156626506, "pass@16": 0.6325301204819277, "pass@32": 0.6837349397590361, "pass@64": 0.7439759036144579, "pass@128": 0.8072289156626506 }, "18": { "pass@1": 0.41935483870967744, "pass@2": 0.4774193548387097, "pass@4": 0.5161290322580645, "pass@8": 0.5516129032258065, "pass@16": 0.6, "pass@32": 0.6580645161290323, "pass@64": 0.7419354838709677, "pass@128": 0.7838709677419354 }, "19": { "pass@1": 0.31722054380664655, "pass@2": 0.3806646525679758, "pass@4": 0.42900302114803623, "pass@8": 0.46827794561933533, "pass@16": 0.5045317220543807, "pass@32": 0.5468277945619335, "pass@64": 0.5921450151057401, "pass@128": 0.6283987915407855 }, "20": { "pass@1": 0.2222222222222222, "pass@2": 0.2672672672672673, "pass@4": 0.29429429429429427, "pass@8": 0.34234234234234234, "pass@16": 0.4024024024024024, "pass@32": 0.44744744744744747, "pass@64": 0.5015015015015015, "pass@128": 0.5735735735735735 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5968253968253968, "12": 0.6697247706422018, "13": 0.6172839506172839, "14": 0.5637982195845698, "15": 0.49382716049382713, "16": 0.528023598820059, "17": 0.4819277108433735, "18": 0.41935483870967744, "19": 0.31722054380664655, "20": 0.2222222222222222 }, "pass@2": { "11": 0.653968253968254, "12": 0.72782874617737, "13": 0.6820987654320988, "14": 0.599406528189911, "15": 0.5277777777777778, "16": 0.551622418879056, "17": 0.49698795180722893, "18": 0.4774193548387097, "19": 0.3806646525679758, "20": 0.2672672672672673 }, "pass@4": { "11": 0.7492063492063492, "12": 0.7706422018348624, "13": 0.7623456790123457, "14": 0.6528189910979229, "15": 0.5802469135802469, "16": 0.6047197640117994, "17": 0.5271084337349398, "18": 0.5161290322580645, "19": 0.42900302114803623, "20": 0.29429429429429427 }, "pass@8": { "11": 0.8507936507936508, "12": 0.8776758409785933, "13": 0.8518518518518519, "14": 0.7002967359050445, "15": 0.6450617283950617, "16": 0.6430678466076696, "17": 0.572289156626506, "18": 0.5516129032258065, "19": 0.46827794561933533, "20": 0.34234234234234234 }, "pass@16": { "11": 0.9079365079365079, "12": 0.9327217125382263, "13": 0.9012345679012346, "14": 0.7566765578635015, "15": 0.7283950617283951, "16": 0.6902654867256637, "17": 0.6325301204819277, "18": 0.6, "19": 0.5045317220543807, "20": 0.4024024024024024 }, "pass@32": { "11": 0.9682539682539683, "12": 0.9785932721712538, "13": 0.9320987654320988, "14": 0.8160237388724035, "15": 0.808641975308642, "16": 0.7492625368731564, "17": 0.6837349397590361, "18": 0.6580645161290323, "19": 0.5468277945619335, "20": 0.44744744744744747 }, "pass@64": { "11": 0.9873015873015873, "12": 0.9877675840978594, "13": 0.9629629629629629, "14": 0.8991097922848664, "15": 0.8950617283950617, "16": 0.8023598820058997, "17": 0.7439759036144579, "18": 0.7419354838709677, "19": 0.5921450151057401, "20": 0.5015015015015015 }, "pass@128": { "11": 0.9968253968253968, "12": 0.9908256880733946, "13": 0.9722222222222222, "14": 0.9228486646884273, "15": 0.9290123456790124, "16": 0.8259587020648967, "17": 0.8072289156626506, "18": 0.7838709677419354, "19": 0.6283987915407855, "20": 0.5735735735735735 } }, "total_pass_at_k": { "pass@1": 0.49052567237163813, "pass@2": 0.535757946210269, "pass@4": 0.5877139364303179, "pass@8": 0.6491442542787286, "pass@16": 0.7044621026894865, "pass@32": 0.757640586797066, "pass@64": 0.8102078239608802, "pass@128": 0.8419926650366748 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6566265060240963, "pass@2": 0.6957831325301205, "pass@4": 0.7620481927710844, "pass@8": 0.8524096385542169, "pass@16": 0.9096385542168675, "pass@32": 0.9457831325301205, "pass@64": 0.9668674698795181, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.5987460815047022, "pass@2": 0.6551724137931034, "pass@4": 0.7210031347962382, "pass@8": 0.8181818181818182, "pass@16": 0.9090909090909091, "pass@32": 0.9498432601880877, "pass@64": 0.9686520376175548, "pass@128": 0.9843260188087775 }, "13": { "pass@1": 0.6246498599439776, "pass@2": 0.6890756302521008, "pass@4": 0.7759103641456583, "pass@8": 0.8487394957983193, "pass@16": 0.9019607843137255, "pass@32": 0.938375350140056, "pass@64": 0.9607843137254902, "pass@128": 0.9747899159663865 }, "14": { "pass@1": 0.5818713450292398, "pass@2": 0.6198830409356725, "pass@4": 0.6666666666666666, "pass@8": 0.7309941520467836, "pass@16": 0.7953216374269005, "pass@32": 0.8567251461988304, "pass@64": 0.9152046783625731, "pass@128": 0.9678362573099415 }, "15": { "pass@1": 0.5787878787878787, "pass@2": 0.6060606060606061, "pass@4": 0.6393939393939394, "pass@8": 0.693939393939394, "pass@16": 0.7484848484848485, "pass@32": 0.8181818181818182, "pass@64": 0.8848484848484849, "pass@128": 0.9515151515151515 }, "16": { "pass@1": 0.5071225071225072, "pass@2": 0.5584045584045584, "pass@4": 0.5925925925925926, "pass@8": 0.6438746438746439, "pass@16": 0.7008547008547008, "pass@32": 0.7720797720797721, "pass@64": 0.8176638176638177, "pass@128": 0.8347578347578347 }, "17": { "pass@1": 0.40993788819875776, "pass@2": 0.45652173913043476, "pass@4": 0.4937888198757764, "pass@8": 0.5434782608695652, "pass@16": 0.6242236024844721, "pass@32": 0.7018633540372671, "pass@64": 0.7391304347826086, "pass@128": 0.8043478260869565 }, "18": { "pass@1": 0.3911764705882353, "pass@2": 0.4294117647058823, "pass@4": 0.4647058823529412, "pass@8": 0.49411764705882355, "pass@16": 0.5294117647058824, "pass@32": 0.5941176470588235, "pass@64": 0.6705882352941176, "pass@128": 0.7382352941176471 }, "19": { "pass@1": 0.31845238095238093, "pass@2": 0.38392857142857145, "pass@4": 0.43452380952380953, "pass@8": 0.46130952380952384, "pass@16": 0.5148809523809523, "pass@32": 0.5505952380952381, "pass@64": 0.6130952380952381, "pass@128": 0.6577380952380952 }, "20": { "pass@1": 0.19461077844311378, "pass@2": 0.24550898203592814, "pass@4": 0.2844311377245509, "pass@8": 0.31736526946107785, "pass@16": 0.3562874251497006, "pass@32": 0.40119760479041916, "pass@64": 0.45808383233532934, "pass@128": 0.5239520958083832 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6566265060240963, "12": 0.5987460815047022, "13": 0.6246498599439776, "14": 0.5818713450292398, "15": 0.5787878787878787, "16": 0.5071225071225072, "17": 0.40993788819875776, "18": 0.3911764705882353, "19": 0.31845238095238093, "20": 0.19461077844311378 }, "pass@2": { "11": 0.6957831325301205, "12": 0.6551724137931034, "13": 0.6890756302521008, "14": 0.6198830409356725, "15": 0.6060606060606061, "16": 0.5584045584045584, "17": 0.45652173913043476, "18": 0.4294117647058823, "19": 0.38392857142857145, "20": 0.24550898203592814 }, "pass@4": { "11": 0.7620481927710844, "12": 0.7210031347962382, "13": 0.7759103641456583, "14": 0.6666666666666666, "15": 0.6393939393939394, "16": 0.5925925925925926, "17": 0.4937888198757764, "18": 0.4647058823529412, "19": 0.43452380952380953, "20": 0.2844311377245509 }, "pass@8": { "11": 0.8524096385542169, "12": 0.8181818181818182, "13": 0.8487394957983193, "14": 0.7309941520467836, "15": 0.693939393939394, "16": 0.6438746438746439, "17": 0.5434782608695652, "18": 0.49411764705882355, "19": 0.46130952380952384, "20": 0.31736526946107785 }, "pass@16": { "11": 0.9096385542168675, "12": 0.9090909090909091, "13": 0.9019607843137255, "14": 0.7953216374269005, "15": 0.7484848484848485, "16": 0.7008547008547008, "17": 0.6242236024844721, "18": 0.5294117647058824, "19": 0.5148809523809523, "20": 0.3562874251497006 }, "pass@32": { "11": 0.9457831325301205, "12": 0.9498432601880877, "13": 0.938375350140056, "14": 0.8567251461988304, "15": 0.8181818181818182, "16": 0.7720797720797721, "17": 0.7018633540372671, "18": 0.5941176470588235, "19": 0.5505952380952381, "20": 0.40119760479041916 }, "pass@64": { "11": 0.9668674698795181, "12": 0.9686520376175548, "13": 0.9607843137254902, "14": 0.9152046783625731, "15": 0.8848484848484849, "16": 0.8176638176638177, "17": 0.7391304347826086, "18": 0.6705882352941176, "19": 0.6130952380952381, "20": 0.45808383233532934 }, "pass@128": { "11": 0.9789156626506024, "12": 0.9843260188087775, "13": 0.9747899159663865, "14": 0.9678362573099415, "15": 0.9515151515151515, "16": 0.8347578347578347, "17": 0.8043478260869565, "18": 0.7382352941176471, "19": 0.6577380952380952, "20": 0.5239520958083832 } }, "total_pass_at_k": { "pass@1": 0.48676776687481416, "pass@2": 0.5346416889681832, "pass@4": 0.584299732381802, "pass@8": 0.6410942610764199, "pass@16": 0.6993755575379126, "pass@32": 0.7531965506987809, "pass@64": 0.79988105857865, "pass@128": 0.8418079096045198 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.623, "pass@2": 0.682, "pass@4": 0.768, "pass@8": 0.86, "pass@16": 0.92, "pass@32": 0.961, "pass@64": 0.978, "pass@128": 0.985 }, "12": { "pass@1": 0.644, "pass@2": 0.707, "pass@4": 0.758, "pass@8": 0.855, "pass@16": 0.923, "pass@32": 0.96, "pass@64": 0.975, "pass@128": 0.982 }, "13": { "pass@1": 0.61, "pass@2": 0.677, "pass@4": 0.756, "pass@8": 0.832, "pass@16": 0.891, "pass@32": 0.931, "pass@64": 0.956, "pass@128": 0.966 }, "14": { "pass@1": 0.57, "pass@2": 0.605, "pass@4": 0.652, "pass@8": 0.712, "pass@16": 0.776, "pass@32": 0.84, "pass@64": 0.903, "pass@128": 0.943 }, "15": { "pass@1": 0.525, "pass@2": 0.556, "pass@4": 0.599, "pass@8": 0.665, "pass@16": 0.733, "pass@32": 0.814, "pass@64": 0.891, "pass@128": 0.938 }, "16": { "pass@1": 0.51, "pass@2": 0.544, "pass@4": 0.587, "pass@8": 0.638, "pass@16": 0.682, "pass@32": 0.746, "pass@64": 0.794, "pass@128": 0.817 }, "17": { "pass@1": 0.467, "pass@2": 0.501, "pass@4": 0.541, "pass@8": 0.587, "pass@16": 0.652, "pass@32": 0.717, "pass@64": 0.756, "pass@128": 0.816 }, "18": { "pass@1": 0.404, "pass@2": 0.446, "pass@4": 0.481, "pass@8": 0.521, "pass@16": 0.566, "pass@32": 0.629, "pass@64": 0.704, "pass@128": 0.758 }, "19": { "pass@1": 0.319, "pass@2": 0.379, "pass@4": 0.426, "pass@8": 0.456, "pass@16": 0.498, "pass@32": 0.541, "pass@64": 0.596, "pass@128": 0.64 }, "20": { "pass@1": 0.205, "pass@2": 0.255, "pass@4": 0.29, "pass@8": 0.328, "pass@16": 0.375, "pass@32": 0.426, "pass@64": 0.481, "pass@128": 0.543 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.623, "12": 0.644, "13": 0.61, "14": 0.57, "15": 0.525, "16": 0.51, "17": 0.467, "18": 0.404, "19": 0.319, "20": 0.205 }, "pass@2": { "11": 0.682, "12": 0.707, "13": 0.677, "14": 0.605, "15": 0.556, "16": 0.544, "17": 0.501, "18": 0.446, "19": 0.379, "20": 0.255 }, "pass@4": { "11": 0.768, "12": 0.758, "13": 0.756, "14": 0.652, "15": 0.599, "16": 0.587, "17": 0.541, "18": 0.481, "19": 0.426, "20": 0.29 }, "pass@8": { "11": 0.86, "12": 0.855, "13": 0.832, "14": 0.712, "15": 0.665, "16": 0.638, "17": 0.587, "18": 0.521, "19": 0.456, "20": 0.328 }, "pass@16": { "11": 0.92, "12": 0.923, "13": 0.891, "14": 0.776, "15": 0.733, "16": 0.682, "17": 0.652, "18": 0.566, "19": 0.498, "20": 0.375 }, "pass@32": { "11": 0.961, "12": 0.96, "13": 0.931, "14": 0.84, "15": 0.814, "16": 0.746, "17": 0.717, "18": 0.629, "19": 0.541, "20": 0.426 }, "pass@64": { "11": 0.978, "12": 0.975, "13": 0.956, "14": 0.903, "15": 0.891, "16": 0.794, "17": 0.756, "18": 0.704, "19": 0.596, "20": 0.481 }, "pass@128": { "11": 0.985, "12": 0.982, "13": 0.966, "14": 0.943, "15": 0.938, "16": 0.817, "17": 0.816, "18": 0.758, "19": 0.64, "20": 0.543 } }, "total_pass_at_k": { "pass@1": 0.4877, "pass@2": 0.5352, "pass@4": 0.5858, "pass@8": 0.6454, "pass@16": 0.7016, "pass@32": 0.7565, "pass@64": 0.8034, "pass@128": 0.8388 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }