{ "files": { "checkpoint-21414_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.990625, "pass@2": 0.990625, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9587301587301588, "pass@2": 0.9746031746031746, "pass@4": 0.9873015873015873, "pass@8": 0.9936507936507937, "pass@16": 0.9936507936507937, "pass@32": 0.9936507936507937, "pass@64": 0.9968253968253968, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9610778443113772, "pass@2": 0.9700598802395209, "pass@4": 0.9820359281437125, "pass@8": 0.9970059880239521, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8816568047337278, "pass@2": 0.9437869822485208, "pass@4": 0.9644970414201184, "pass@8": 0.9881656804733728, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.7852564102564102, "pass@2": 0.8717948717948718, "pass@4": 0.9391025641025641, "pass@8": 0.9839743589743589, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.6732394366197183, "pass@2": 0.7915492957746478, "pass@4": 0.8676056338028169, "pass@8": 0.9492957746478873, "pass@16": 0.9802816901408451, "pass@32": 0.9943661971830986, "pass@64": 0.9971830985915493, "pass@128": 1.0 }, "10": { "pass@1": 0.5693215339233039, "pass@2": 0.6696165191740413, "pass@4": 0.7463126843657817, "pass@8": 0.831858407079646, "pass@16": 0.8731563421828908, "pass@32": 0.9056047197640118, "pass@64": 0.9262536873156342, "pass@128": 0.9321533923303835 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.990625, "5": 0.9587301587301588, "6": 0.9610778443113772, "7": 0.8816568047337278, "8": 0.7852564102564102, "9": 0.6732394366197183, "10": 0.5693215339233039 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.990625, "5": 0.9746031746031746, "6": 0.9700598802395209, "7": 0.9437869822485208, "8": 0.8717948717948718, "9": 0.7915492957746478, "10": 0.6696165191740413 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9873015873015873, "6": 0.9820359281437125, "7": 0.9644970414201184, "8": 0.9391025641025641, "9": 0.8676056338028169, "10": 0.7463126843657817 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9936507936507937, "6": 0.9970059880239521, "7": 0.9881656804733728, "8": 0.9839743589743589, "9": 0.9492957746478873, "10": 0.831858407079646 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9936507936507937, "6": 1.0, "7": 0.9970414201183432, "8": 1.0, "9": 0.9802816901408451, "10": 0.8731563421828908 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9936507936507937, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9943661971830986, "10": 0.9056047197640118 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9262536873156342 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9321533923303835 } }, "total_pass_at_k": { "pass@1": 0.8655859506923337, "pass@2": 0.9101654846335697, "pass@4": 0.9412360688956434, "pass@8": 0.9706180344478217, "pass@16": 0.9821006416751098, "pass@32": 0.9878419452887538, "pass@64": 0.9908814589665653, "pass@128": 0.9918946301925026 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9883040935672515, "pass@2": 0.9941520467836257, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.95, "pass@2": 0.9794117647058823, "pass@4": 0.9882352941176471, "pass@8": 0.9941176470588236, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9621451104100947, "pass@2": 0.9842271293375394, "pass@4": 0.9936908517350158, "pass@8": 0.9968454258675079, "pass@16": 0.9968454258675079, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8639240506329114, "pass@2": 0.9177215189873418, "pass@4": 0.9620253164556962, "pass@8": 0.9778481012658228, "pass@16": 0.9936708860759493, "pass@32": 0.9968354430379747, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.7567567567567568, "pass@2": 0.8288288288288288, "pass@4": 0.9159159159159159, "pass@8": 0.975975975975976, "pass@16": 0.996996996996997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.729903536977492, "pass@2": 0.819935691318328, "pass@4": 0.8713826366559485, "pass@8": 0.9228295819935691, "pass@16": 0.9710610932475884, "pass@32": 0.9903536977491961, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.5552238805970149, "pass@2": 0.6507462686567164, "pass@4": 0.7432835820895523, "pass@8": 0.8477611940298507, "pass@16": 0.8865671641791045, "pass@32": 0.9283582089552239, "pass@64": 0.9462686567164179, "pass@128": 0.9522388059701492 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9883040935672515, "5": 0.95, "6": 0.9621451104100947, "7": 0.8639240506329114, "8": 0.7567567567567568, "9": 0.729903536977492, "10": 0.5552238805970149 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9941520467836257, "5": 0.9794117647058823, "6": 0.9842271293375394, "7": 0.9177215189873418, "8": 0.8288288288288288, "9": 0.819935691318328, "10": 0.6507462686567164 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9882352941176471, "6": 0.9936908517350158, "7": 0.9620253164556962, "8": 0.9159159159159159, "9": 0.8713826366559485, "10": 0.7432835820895523 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9941176470588236, "6": 0.9968454258675079, "7": 0.9778481012658228, "8": 0.975975975975976, "9": 0.9228295819935691, "10": 0.8477611940298507 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9968454258675079, "7": 0.9936708860759493, "8": 0.996996996996997, "9": 0.9710610932475884, "10": 0.8865671641791045 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9968354430379747, "8": 1.0, "9": 0.9903536977491961, "10": 0.9283582089552239 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9462686567164179 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9522388059701492 } }, "total_pass_at_k": { "pass@1": 0.8694342149313693, "pass@2": 0.9096083026447941, "pass@4": 0.942417140944091, "pass@8": 0.9688650820220958, "pass@16": 0.9829260127217945, "pass@32": 0.9906260462002009, "pass@64": 0.9939738868429863, "pass@128": 0.9946434549715434 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.996969696969697, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.985207100591716, "pass@2": 0.9940828402366864, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9478260869565217, "pass@2": 0.9855072463768116, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9627507163323782, "pass@2": 0.9885386819484241, "pass@4": 0.9885386819484241, "pass@8": 0.994269340974212, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8352601156069365, "pass@2": 0.930635838150289, "pass@4": 0.9682080924855492, "pass@8": 0.9971098265895953, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.7690140845070422, "pass@2": 0.8535211267605634, "pass@4": 0.9323943661971831, "pass@8": 0.9661971830985916, "pass@16": 0.9887323943661972, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.6796407185628742, "pass@2": 0.7664670658682635, "pass@4": 0.8502994011976048, "pass@8": 0.9401197604790419, "pass@16": 0.9820359281437125, "pass@32": 0.9940119760479041, "pass@64": 0.9970059880239521, "pass@128": 0.9970059880239521 }, "10": { "pass@1": 0.6319018404907976, "pass@2": 0.7147239263803681, "pass@4": 0.7883435582822086, "pass@8": 0.8374233128834356, "pass@16": 0.8834355828220859, "pass@32": 0.9049079754601227, "pass@64": 0.9325153374233128, "pass@128": 0.9447852760736196 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 0.996969696969697, "4": 0.985207100591716, "5": 0.9478260869565217, "6": 0.9627507163323782, "7": 0.8352601156069365, "8": 0.7690140845070422, "9": 0.6796407185628742, "10": 0.6319018404907976 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9940828402366864, "5": 0.9855072463768116, "6": 0.9885386819484241, "7": 0.930635838150289, "8": 0.8535211267605634, "9": 0.7664670658682635, "10": 0.7147239263803681 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9885386819484241, "7": 0.9682080924855492, "8": 0.9323943661971831, "9": 0.8502994011976048, "10": 0.7883435582822086 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.994269340974212, "7": 0.9971098265895953, "8": 0.9661971830985916, "9": 0.9401197604790419, "10": 0.8374233128834356 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9887323943661972, "9": 0.9820359281437125, "10": 0.8834355828220859 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9940119760479041, "10": 0.9049079754601227 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.9325153374233128 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.9447852760736196 } }, "total_pass_at_k": { "pass@1": 0.8679554390563565, "pass@2": 0.9154652686762779, "pass@4": 0.9482306684141546, "pass@8": 0.9711664482306684, "pass@16": 0.9842726081258192, "pass@32": 0.9891874180865007, "pass@64": 0.9924639580602883, "pass@128": 0.9937745740498034 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.999, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.988, "pass@2": 0.993, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.952, "pass@2": 0.98, "pass@4": 0.992, "pass@8": 0.996, "pass@16": 0.998, "pass@32": 0.998, "pass@64": 0.999, "pass@128": 0.999 }, "6": { "pass@1": 0.962, "pass@2": 0.981, "pass@4": 0.988, "pass@8": 0.996, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.86, "pass@2": 0.931, "pass@4": 0.965, "pass@8": 0.988, "pass@16": 0.997, "pass@32": 0.999, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.77, "pass@2": 0.851, "pass@4": 0.929, "pass@8": 0.975, "pass@16": 0.995, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.693, "pass@2": 0.792, "pass@4": 0.863, "pass@8": 0.938, "pass@16": 0.978, "pass@32": 0.993, "pass@64": 0.998, "pass@128": 0.999 }, "10": { "pass@1": 0.585, "pass@2": 0.678, "pass@4": 0.759, "pass@8": 0.839, "pass@16": 0.881, "pass@32": 0.913, "pass@64": 0.935, "pass@128": 0.943 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 0.999, "4": 0.988, "5": 0.952, "6": 0.962, "7": 0.86, "8": 0.77, "9": 0.693, "10": 0.585 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.993, "5": 0.98, "6": 0.981, "7": 0.931, "8": 0.851, "9": 0.792, "10": 0.678 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.992, "6": 0.988, "7": 0.965, "8": 0.929, "9": 0.863, "10": 0.759 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.996, "6": 0.996, "7": 0.988, "8": 0.975, "9": 0.938, "10": 0.839 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 0.999, "7": 0.997, "8": 0.995, "9": 0.978, "10": 0.881 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 1.0, "7": 0.999, "8": 1.0, "9": 0.993, "10": 0.913 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.998, "10": 0.935 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.999, "10": 0.943 } }, "total_pass_at_k": { "pass@1": 0.8676666666666667, "pass@2": 0.9117777777777778, "pass@4": 0.944, "pass@8": 0.9702222222222222, "pass@16": 0.9831111111111112, "pass@32": 0.9892222222222222, "pass@64": 0.9924444444444445, "pass@128": 0.9934444444444445 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-21414_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.4475920679886686, "pass@2": 0.5694050991501416, "pass@4": 0.6713881019830028, "pass@8": 0.7620396600566572, "pass@16": 0.8470254957507082, "pass@32": 0.8980169971671388, "pass@64": 0.9235127478753541, "pass@128": 0.9291784702549575 }, "12": { "pass@1": 0.3898305084745763, "pass@2": 0.5084745762711864, "pass@4": 0.5847457627118644, "pass@8": 0.6864406779661016, "pass@16": 0.748587570621469, "pass@32": 0.788135593220339, "pass@64": 0.8163841807909604, "pass@128": 0.8333333333333334 }, "13": { "pass@1": 0.22884012539184953, "pass@2": 0.30407523510971785, "pass@4": 0.3761755485893417, "pass@8": 0.45454545454545453, "pass@16": 0.54858934169279, "pass@32": 0.6144200626959248, "pass@64": 0.658307210031348, "pass@128": 0.6739811912225705 }, "14": { "pass@1": 0.12149532710280374, "pass@2": 0.1806853582554517, "pass@4": 0.2554517133956386, "pass@8": 0.32710280373831774, "pass@16": 0.3956386292834891, "pass@32": 0.4797507788161994, "pass@64": 0.5202492211838006, "pass@128": 0.5451713395638629 }, "15": { "pass@1": 0.06936416184971098, "pass@2": 0.11271676300578035, "pass@4": 0.15028901734104047, "pass@8": 0.17052023121387283, "pass@16": 0.20809248554913296, "pass@32": 0.2630057803468208, "pass@64": 0.29190751445086704, "pass@128": 0.3786127167630058 }, "16": { "pass@1": 0.012903225806451613, "pass@2": 0.025806451612903226, "pass@4": 0.03870967741935484, "pass@8": 0.05161290322580645, "pass@16": 0.05806451612903226, "pass@32": 0.06451612903225806, "pass@64": 0.08064516129032258, "pass@128": 0.1064516129032258 }, "17": { "pass@1": 0.002890173410404624, "pass@2": 0.008670520231213872, "pass@4": 0.017341040462427744, "pass@8": 0.023121387283236993, "pass@16": 0.031791907514450865, "pass@32": 0.046242774566473986, "pass@64": 0.057803468208092484, "pass@128": 0.07225433526011561 }, "18": { "pass@1": 0.002857142857142857, "pass@2": 0.002857142857142857, "pass@4": 0.002857142857142857, "pass@8": 0.005714285714285714, "pass@16": 0.008571428571428572, "pass@32": 0.011428571428571429, "pass@64": 0.017142857142857144, "pass@128": 0.022857142857142857 }, "19": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.0, "pass@8": 0.003003003003003003, "pass@16": 0.003003003003003003, "pass@32": 0.003003003003003003, "pass@64": 0.006006006006006006, "pass@128": 0.009009009009009009 }, "20": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.003003003003003003, "pass@8": 0.003003003003003003, "pass@16": 0.003003003003003003, "pass@32": 0.006006006006006006, "pass@64": 0.006006006006006006, "pass@128": 0.009009009009009009 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.4475920679886686, "12": 0.3898305084745763, "13": 0.22884012539184953, "14": 0.12149532710280374, "15": 0.06936416184971098, "16": 0.012903225806451613, "17": 0.002890173410404624, "18": 0.002857142857142857, "19": 0.0, "20": 0.0 }, "pass@2": { "11": 0.5694050991501416, "12": 0.5084745762711864, "13": 0.30407523510971785, "14": 0.1806853582554517, "15": 0.11271676300578035, "16": 0.025806451612903226, "17": 0.008670520231213872, "18": 0.002857142857142857, "19": 0.0, "20": 0.0 }, "pass@4": { "11": 0.6713881019830028, "12": 0.5847457627118644, "13": 0.3761755485893417, "14": 0.2554517133956386, "15": 0.15028901734104047, "16": 0.03870967741935484, "17": 0.017341040462427744, "18": 0.002857142857142857, "19": 0.0, "20": 0.003003003003003003 }, "pass@8": { "11": 0.7620396600566572, "12": 0.6864406779661016, "13": 0.45454545454545453, "14": 0.32710280373831774, "15": 0.17052023121387283, "16": 0.05161290322580645, "17": 0.023121387283236993, "18": 0.005714285714285714, "19": 0.003003003003003003, "20": 0.003003003003003003 }, "pass@16": { "11": 0.8470254957507082, "12": 0.748587570621469, "13": 0.54858934169279, "14": 0.3956386292834891, "15": 0.20809248554913296, "16": 0.05806451612903226, "17": 0.031791907514450865, "18": 0.008571428571428572, "19": 0.003003003003003003, "20": 0.003003003003003003 }, "pass@32": { "11": 0.8980169971671388, "12": 0.788135593220339, "13": 0.6144200626959248, "14": 0.4797507788161994, "15": 0.2630057803468208, "16": 0.06451612903225806, "17": 0.046242774566473986, "18": 0.011428571428571429, "19": 0.003003003003003003, "20": 0.006006006006006006 }, "pass@64": { "11": 0.9235127478753541, "12": 0.8163841807909604, "13": 0.658307210031348, "14": 0.5202492211838006, "15": 0.29190751445086704, "16": 0.08064516129032258, "17": 0.057803468208092484, "18": 0.017142857142857144, "19": 0.006006006006006006, "20": 0.006006006006006006 }, "pass@128": { "11": 0.9291784702549575, "12": 0.8333333333333334, "13": 0.6739811912225705, "14": 0.5451713395638629, "15": 0.3786127167630058, "16": 0.1064516129032258, "17": 0.07225433526011561, "18": 0.022857142857142857, "19": 0.009009009009009009, "20": 0.009009009009009009 } }, "total_pass_at_k": { "pass@1": 0.13016344725111442, "pass@2": 0.17444279346210995, "pass@4": 0.21337295690936106, "pass@8": 0.2523031203566122, "pass@16": 0.2888558692421991, "pass@32": 0.3209509658246657, "pass@64": 0.3411589895988113, "pass@128": 0.3613670133729569 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.5015873015873016, "pass@2": 0.6253968253968254, "pass@4": 0.7396825396825397, "pass@8": 0.8285714285714286, "pass@16": 0.8793650793650793, "pass@32": 0.9142857142857143, "pass@64": 0.946031746031746, "pass@128": 0.9619047619047619 }, "12": { "pass@1": 0.41896024464831805, "pass@2": 0.5412844036697247, "pass@4": 0.6391437308868502, "pass@8": 0.7155963302752294, "pass@16": 0.7706422018348624, "pass@32": 0.8287461773700305, "pass@64": 0.8470948012232415, "pass@128": 0.8654434250764526 }, "13": { "pass@1": 0.2345679012345679, "pass@2": 0.3395061728395062, "pass@4": 0.41975308641975306, "pass@8": 0.4876543209876543, "pass@16": 0.5462962962962963, "pass@32": 0.5833333333333334, "pass@64": 0.6327160493827161, "pass@128": 0.6666666666666666 }, "14": { "pass@1": 0.14540059347181009, "pass@2": 0.19881305637982197, "pass@4": 0.23738872403560832, "pass@8": 0.2818991097922849, "pass@16": 0.3353115727002967, "pass@32": 0.4035608308605341, "pass@64": 0.45103857566765576, "pass@128": 0.5103857566765578 }, "15": { "pass@1": 0.08024691358024691, "pass@2": 0.14506172839506173, "pass@4": 0.1697530864197531, "pass@8": 0.20987654320987653, "pass@16": 0.26851851851851855, "pass@32": 0.3117283950617284, "pass@64": 0.37962962962962965, "pass@128": 0.4228395061728395 }, "16": { "pass@1": 0.038348082595870206, "pass@2": 0.0471976401179941, "pass@4": 0.06489675516224189, "pass@8": 0.07079646017699115, "pass@16": 0.07964601769911504, "pass@32": 0.08849557522123894, "pass@64": 0.12389380530973451, "pass@128": 0.15339233038348082 }, "17": { "pass@1": 0.015060240963855422, "pass@2": 0.024096385542168676, "pass@4": 0.03313253012048193, "pass@8": 0.045180722891566265, "pass@16": 0.05120481927710843, "pass@32": 0.060240963855421686, "pass@64": 0.06325301204819277, "pass@128": 0.06927710843373494 }, "18": { "pass@1": 0.012903225806451613, "pass@2": 0.016129032258064516, "pass@4": 0.01935483870967742, "pass@8": 0.02258064516129032, "pass@16": 0.03225806451612903, "pass@32": 0.03225806451612903, "pass@64": 0.03870967741935484, "pass@128": 0.06129032258064516 }, "19": { "pass@1": 0.0030211480362537764, "pass@2": 0.0030211480362537764, "pass@4": 0.0030211480362537764, "pass@8": 0.006042296072507553, "pass@16": 0.012084592145015106, "pass@32": 0.012084592145015106, "pass@64": 0.012084592145015106, "pass@128": 0.012084592145015106 }, "20": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.003003003003003003, "pass@8": 0.003003003003003003, "pass@16": 0.009009009009009009, "pass@32": 0.009009009009009009, "pass@64": 0.009009009009009009, "pass@128": 0.012012012012012012 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5015873015873016, "12": 0.41896024464831805, "13": 0.2345679012345679, "14": 0.14540059347181009, "15": 0.08024691358024691, "16": 0.038348082595870206, "17": 0.015060240963855422, "18": 0.012903225806451613, "19": 0.0030211480362537764, "20": 0.0 }, "pass@2": { "11": 0.6253968253968254, "12": 0.5412844036697247, "13": 0.3395061728395062, "14": 0.19881305637982197, "15": 0.14506172839506173, "16": 0.0471976401179941, "17": 0.024096385542168676, "18": 0.016129032258064516, "19": 0.0030211480362537764, "20": 0.0 }, "pass@4": { "11": 0.7396825396825397, "12": 0.6391437308868502, "13": 0.41975308641975306, "14": 0.23738872403560832, "15": 0.1697530864197531, "16": 0.06489675516224189, "17": 0.03313253012048193, "18": 0.01935483870967742, "19": 0.0030211480362537764, "20": 0.003003003003003003 }, "pass@8": { "11": 0.8285714285714286, "12": 0.7155963302752294, "13": 0.4876543209876543, "14": 0.2818991097922849, "15": 0.20987654320987653, "16": 0.07079646017699115, "17": 0.045180722891566265, "18": 0.02258064516129032, "19": 0.006042296072507553, "20": 0.003003003003003003 }, "pass@16": { "11": 0.8793650793650793, "12": 0.7706422018348624, "13": 0.5462962962962963, "14": 0.3353115727002967, "15": 0.26851851851851855, "16": 0.07964601769911504, "17": 0.05120481927710843, "18": 0.03225806451612903, "19": 0.012084592145015106, "20": 0.009009009009009009 }, "pass@32": { "11": 0.9142857142857143, "12": 0.8287461773700305, "13": 0.5833333333333334, "14": 0.4035608308605341, "15": 0.3117283950617284, "16": 0.08849557522123894, "17": 0.060240963855421686, "18": 0.03225806451612903, "19": 0.012084592145015106, "20": 0.009009009009009009 }, "pass@64": { "11": 0.946031746031746, "12": 0.8470948012232415, "13": 0.6327160493827161, "14": 0.45103857566765576, "15": 0.37962962962962965, "16": 0.12389380530973451, "17": 0.06325301204819277, "18": 0.03870967741935484, "19": 0.012084592145015106, "20": 0.009009009009009009 }, "pass@128": { "11": 0.9619047619047619, "12": 0.8654434250764526, "13": 0.6666666666666666, "14": 0.5103857566765578, "15": 0.4228395061728395, "16": 0.15339233038348082, "17": 0.06927710843373494, "18": 0.06129032258064516, "19": 0.012084592145015106, "20": 0.012012012012012012 } }, "total_pass_at_k": { "pass@1": 0.14333740831295844, "pass@2": 0.19193154034229828, "pass@4": 0.230440097799511, "pass@8": 0.2643643031784841, "pass@16": 0.29553789731051344, "pass@32": 0.3215158924205379, "pass@64": 0.3474938875305624, "pass@128": 0.370721271393643 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.5421686746987951, "pass@2": 0.6295180722891566, "pass@4": 0.7289156626506024, "pass@8": 0.8042168674698795, "pass@16": 0.8704819277108434, "pass@32": 0.9246987951807228, "pass@64": 0.9487951807228916, "pass@128": 0.9668674698795181 }, "12": { "pass@1": 0.3949843260188088, "pass@2": 0.4890282131661442, "pass@4": 0.5987460815047022, "pass@8": 0.6677115987460815, "pass@16": 0.7429467084639498, "pass@32": 0.7836990595611285, "pass@64": 0.8119122257053292, "pass@128": 0.8369905956112853 }, "13": { "pass@1": 0.23809523809523808, "pass@2": 0.31092436974789917, "pass@4": 0.4257703081232493, "pass@8": 0.5126050420168067, "pass@16": 0.5574229691876751, "pass@32": 0.6274509803921569, "pass@64": 0.6554621848739496, "pass@128": 0.6974789915966386 }, "14": { "pass@1": 0.1783625730994152, "pass@2": 0.23976608187134502, "pass@4": 0.28362573099415206, "pass@8": 0.34502923976608185, "pass@16": 0.3830409356725146, "pass@32": 0.4590643274853801, "pass@64": 0.5058479532163743, "pass@128": 0.5467836257309941 }, "15": { "pass@1": 0.06060606060606061, "pass@2": 0.08181818181818182, "pass@4": 0.1393939393939394, "pass@8": 0.18787878787878787, "pass@16": 0.23333333333333334, "pass@32": 0.2787878787878788, "pass@64": 0.3212121212121212, "pass@128": 0.38484848484848483 }, "16": { "pass@1": 0.017094017094017096, "pass@2": 0.02564102564102564, "pass@4": 0.02564102564102564, "pass@8": 0.04843304843304843, "pass@16": 0.06552706552706553, "pass@32": 0.07692307692307693, "pass@64": 0.09116809116809117, "pass@128": 0.11965811965811966 }, "17": { "pass@1": 0.012422360248447204, "pass@2": 0.018633540372670808, "pass@4": 0.021739130434782608, "pass@8": 0.027950310559006212, "pass@16": 0.040372670807453416, "pass@32": 0.055900621118012424, "pass@64": 0.06832298136645963, "pass@128": 0.08385093167701864 }, "18": { "pass@1": 0.0058823529411764705, "pass@2": 0.011764705882352941, "pass@4": 0.014705882352941176, "pass@8": 0.023529411764705882, "pass@16": 0.023529411764705882, "pass@32": 0.03235294117647059, "pass@64": 0.047058823529411764, "pass@128": 0.058823529411764705 }, "19": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.0, "pass@8": 0.0, "pass@16": 0.0, "pass@32": 0.0, "pass@64": 0.0, "pass@128": 0.011904761904761904 }, "20": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.0, "pass@8": 0.0, "pass@16": 0.0029940119760479044, "pass@32": 0.0029940119760479044, "pass@64": 0.0029940119760479044, "pass@128": 0.0029940119760479044 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5421686746987951, "12": 0.3949843260188088, "13": 0.23809523809523808, "14": 0.1783625730994152, "15": 0.06060606060606061, "16": 0.017094017094017096, "17": 0.012422360248447204, "18": 0.0058823529411764705, "19": 0.0, "20": 0.0 }, "pass@2": { "11": 0.6295180722891566, "12": 0.4890282131661442, "13": 0.31092436974789917, "14": 0.23976608187134502, "15": 0.08181818181818182, "16": 0.02564102564102564, "17": 0.018633540372670808, "18": 0.011764705882352941, "19": 0.0, "20": 0.0 }, "pass@4": { "11": 0.7289156626506024, "12": 0.5987460815047022, "13": 0.4257703081232493, "14": 0.28362573099415206, "15": 0.1393939393939394, "16": 0.02564102564102564, "17": 0.021739130434782608, "18": 0.014705882352941176, "19": 0.0, "20": 0.0 }, "pass@8": { "11": 0.8042168674698795, "12": 0.6677115987460815, "13": 0.5126050420168067, "14": 0.34502923976608185, "15": 0.18787878787878787, "16": 0.04843304843304843, "17": 0.027950310559006212, "18": 0.023529411764705882, "19": 0.0, "20": 0.0 }, "pass@16": { "11": 0.8704819277108434, "12": 0.7429467084639498, "13": 0.5574229691876751, "14": 0.3830409356725146, "15": 0.23333333333333334, "16": 0.06552706552706553, "17": 0.040372670807453416, "18": 0.023529411764705882, "19": 0.0, "20": 0.0029940119760479044 }, "pass@32": { "11": 0.9246987951807228, "12": 0.7836990595611285, "13": 0.6274509803921569, "14": 0.4590643274853801, "15": 0.2787878787878788, "16": 0.07692307692307693, "17": 0.055900621118012424, "18": 0.03235294117647059, "19": 0.0, "20": 0.0029940119760479044 }, "pass@64": { "11": 0.9487951807228916, "12": 0.8119122257053292, "13": 0.6554621848739496, "14": 0.5058479532163743, "15": 0.3212121212121212, "16": 0.09116809116809117, "17": 0.06832298136645963, "18": 0.047058823529411764, "19": 0.0, "20": 0.0029940119760479044 }, "pass@128": { "11": 0.9668674698795181, "12": 0.8369905956112853, "13": 0.6974789915966386, "14": 0.5467836257309941, "15": 0.38484848484848483, "16": 0.11965811965811966, "17": 0.08385093167701864, "18": 0.058823529411764705, "19": 0.011904761904761904, "20": 0.0029940119760479044 } }, "total_pass_at_k": { "pass@1": 0.14391911983348202, "pass@2": 0.17960154623847754, "pass@4": 0.22271781147784717, "pass@8": 0.2607790663098424, "pass@16": 0.29081177520071366, "pass@32": 0.3232233125185846, "pass@64": 0.34433541480820695, "pass@128": 0.3702051739518287 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.496, "pass@2": 0.607, "pass@4": 0.712, "pass@8": 0.797, "pass@16": 0.865, "pass@32": 0.912, "pass@64": 0.939, "pass@128": 0.952 }, "12": { "pass@1": 0.401, "pass@2": 0.513, "pass@4": 0.607, "pass@8": 0.69, "pass@16": 0.754, "pass@32": 0.8, "pass@64": 0.825, "pass@128": 0.845 }, "13": { "pass@1": 0.234, "pass@2": 0.318, "pass@4": 0.408, "pass@8": 0.486, "pass@16": 0.551, "pass@32": 0.609, "pass@64": 0.649, "pass@128": 0.68 }, "14": { "pass@1": 0.149, "pass@2": 0.207, "pass@4": 0.259, "pass@8": 0.318, "pass@16": 0.371, "pass@32": 0.447, "pass@64": 0.492, "pass@128": 0.534 }, "15": { "pass@1": 0.07, "pass@2": 0.113, "pass@4": 0.153, "pass@8": 0.189, "pass@16": 0.236, "pass@32": 0.284, "pass@64": 0.33, "pass@128": 0.395 }, "16": { "pass@1": 0.023, "pass@2": 0.033, "pass@4": 0.043, "pass@8": 0.057, "pass@16": 0.068, "pass@32": 0.077, "pass@64": 0.099, "pass@128": 0.127 }, "17": { "pass@1": 0.01, "pass@2": 0.017, "pass@4": 0.024, "pass@8": 0.032, "pass@16": 0.041, "pass@32": 0.054, "pass@64": 0.063, "pass@128": 0.075 }, "18": { "pass@1": 0.007, "pass@2": 0.01, "pass@4": 0.012, "pass@8": 0.017, "pass@16": 0.021, "pass@32": 0.025, "pass@64": 0.034, "pass@128": 0.047 }, "19": { "pass@1": 0.001, "pass@2": 0.001, "pass@4": 0.001, "pass@8": 0.003, "pass@16": 0.005, "pass@32": 0.005, "pass@64": 0.006, "pass@128": 0.011 }, "20": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.002, "pass@8": 0.002, "pass@16": 0.005, "pass@32": 0.006, "pass@64": 0.006, "pass@128": 0.008 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.496, "12": 0.401, "13": 0.234, "14": 0.149, "15": 0.07, "16": 0.023, "17": 0.01, "18": 0.007, "19": 0.001, "20": 0.0 }, "pass@2": { "11": 0.607, "12": 0.513, "13": 0.318, "14": 0.207, "15": 0.113, "16": 0.033, "17": 0.017, "18": 0.01, "19": 0.001, "20": 0.0 }, "pass@4": { "11": 0.712, "12": 0.607, "13": 0.408, "14": 0.259, "15": 0.153, "16": 0.043, "17": 0.024, "18": 0.012, "19": 0.001, "20": 0.002 }, "pass@8": { "11": 0.797, "12": 0.69, "13": 0.486, "14": 0.318, "15": 0.189, "16": 0.057, "17": 0.032, "18": 0.017, "19": 0.003, "20": 0.002 }, "pass@16": { "11": 0.865, "12": 0.754, "13": 0.551, "14": 0.371, "15": 0.236, "16": 0.068, "17": 0.041, "18": 0.021, "19": 0.005, "20": 0.005 }, "pass@32": { "11": 0.912, "12": 0.8, "13": 0.609, "14": 0.447, "15": 0.284, "16": 0.077, "17": 0.054, "18": 0.025, "19": 0.005, "20": 0.006 }, "pass@64": { "11": 0.939, "12": 0.825, "13": 0.649, "14": 0.492, "15": 0.33, "16": 0.099, "17": 0.063, "18": 0.034, "19": 0.006, "20": 0.006 }, "pass@128": { "11": 0.952, "12": 0.845, "13": 0.68, "14": 0.534, "15": 0.395, "16": 0.127, "17": 0.075, "18": 0.047, "19": 0.011, "20": 0.008 } }, "total_pass_at_k": { "pass@1": 0.1391, "pass@2": 0.1819, "pass@4": 0.2221, "pass@8": 0.2591, "pass@16": 0.2917, "pass@32": 0.3219, "pass@64": 0.3443, "pass@128": 0.3674 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }