{ "files": { "checkpoint-38797_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99375, "pass@2": 0.996875, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9587301587301588, "pass@2": 0.9746031746031746, "pass@4": 0.9936507936507937, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9580838323353293, "pass@2": 0.9760479041916168, "pass@4": 0.9850299401197605, "pass@8": 0.9970059880239521, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8846153846153846, "pass@2": 0.9289940828402367, "pass@4": 0.9526627218934911, "pass@8": 0.9911242603550295, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8269230769230769, "pass@2": 0.8846153846153846, "pass@4": 0.9583333333333334, "pass@8": 0.9871794871794872, "pass@16": 0.9935897435897436, "pass@32": 0.9967948717948718, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7408450704225352, "pass@2": 0.8140845070422535, "pass@4": 0.8929577464788733, "pass@8": 0.9464788732394366, "pass@16": 0.9746478873239437, "pass@32": 0.9971830985915493, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6312684365781711, "pass@2": 0.7079646017699115, "pass@4": 0.8023598820058997, "pass@8": 0.8643067846607669, "pass@16": 0.911504424778761, "pass@32": 0.9410029498525073, "pass@64": 0.9469026548672567, "pass@128": 0.9557522123893806 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.99375, "5": 0.9587301587301588, "6": 0.9580838323353293, "7": 0.8846153846153846, "8": 0.8269230769230769, "9": 0.7408450704225352, "10": 0.6312684365781711 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9746031746031746, "6": 0.9760479041916168, "7": 0.9289940828402367, "8": 0.8846153846153846, "9": 0.8140845070422535, "10": 0.7079646017699115 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9936507936507937, "6": 0.9850299401197605, "7": 0.9526627218934911, "8": 0.9583333333333334, "9": 0.8929577464788733, "10": 0.8023598820058997 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9970059880239521, "7": 0.9911242603550295, "8": 0.9871794871794872, "9": 0.9464788732394366, "10": 0.8643067846607669 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9970414201183432, "8": 0.9935897435897436, "9": 0.9746478873239437, "10": 0.911504424778761 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9967948717948718, "9": 0.9971830985915493, "10": 0.9410029498525073 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9469026548672567 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9557522123893806 } }, "total_pass_at_k": { "pass@1": 0.8855116514690983, "pass@2": 0.9182708544410672, "pass@4": 0.9523809523809523, "pass@8": 0.9753461668355285, "pass@16": 0.9858156028368794, "pass@32": 0.9925700776764607, "pass@64": 0.993920972644377, "pass@128": 0.9949341438703141 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9883040935672515, "pass@2": 0.9970760233918129, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9441176470588235, "pass@2": 0.9705882352941176, "pass@4": 0.9911764705882353, "pass@8": 0.9970588235294118, "pass@16": 0.9970588235294118, "pass@32": 0.9970588235294118, "pass@64": 0.9970588235294118, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9716088328075709, "pass@2": 0.9842271293375394, "pass@4": 0.9936908517350158, "pass@8": 0.9968454258675079, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9050632911392406, "pass@2": 0.939873417721519, "pass@4": 0.9620253164556962, "pass@8": 0.9936708860759493, "pass@16": 0.9968354430379747, "pass@32": 0.9968354430379747, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8288288288288288, "pass@2": 0.8588588588588588, "pass@4": 0.9279279279279279, "pass@8": 0.9819819819819819, "pass@16": 0.996996996996997, "pass@32": 0.996996996996997, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7909967845659164, "pass@2": 0.8456591639871383, "pass@4": 0.9003215434083601, "pass@8": 0.954983922829582, "pass@16": 0.9839228295819936, "pass@32": 0.9967845659163987, "pass@64": 0.9967845659163987, "pass@128": 0.9967845659163987 }, "10": { "pass@1": 0.6059701492537314, "pass@2": 0.6447761194029851, "pass@4": 0.7313432835820896, "pass@8": 0.8388059701492537, "pass@16": 0.9223880597014925, "pass@32": 0.9731343283582089, "pass@64": 0.9880597014925373, "pass@128": 0.991044776119403 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9883040935672515, "5": 0.9441176470588235, "6": 0.9716088328075709, "7": 0.9050632911392406, "8": 0.8288288288288288, "9": 0.7909967845659164, "10": 0.6059701492537314 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.9705882352941176, "6": 0.9842271293375394, "7": 0.939873417721519, "8": 0.8588588588588588, "9": 0.8456591639871383, "10": 0.6447761194029851 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9911764705882353, "6": 0.9936908517350158, "7": 0.9620253164556962, "8": 0.9279279279279279, "9": 0.9003215434083601, "10": 0.7313432835820896 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 0.9968454258675079, "7": 0.9936708860759493, "8": 0.9819819819819819, "9": 0.954983922829582, "10": 0.8388059701492537 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 0.996996996996997, "9": 0.9839228295819936, "10": 0.9223880597014925 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 0.996996996996997, "9": 0.9967845659163987, "10": 0.9731343283582089 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9967845659163987, "10": 0.9880597014925373 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9967845659163987, "10": 0.991044776119403 } }, "total_pass_at_k": { "pass@1": 0.8942082356879812, "pass@2": 0.9166387679946435, "pass@4": 0.9457649815868765, "pass@8": 0.9738868429862738, "pass@16": 0.9886173418145296, "pass@32": 0.995647807164379, "pass@64": 0.9979912956143288, "pass@128": 0.9983260796786073 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9881656804733728, "pass@2": 0.9911242603550295, "pass@4": 0.9970414201183432, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9478260869565217, "pass@2": 0.9826086956521739, "pass@4": 0.9971014492753624, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9770773638968482, "pass@2": 0.9828080229226361, "pass@4": 0.9885386819484241, "pass@8": 0.994269340974212, "pass@16": 0.997134670487106, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8872832369942196, "pass@2": 0.9248554913294798, "pass@4": 0.9739884393063584, "pass@8": 0.9884393063583815, "pass@16": 0.9971098265895953, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8169014084507042, "pass@2": 0.8901408450704226, "pass@4": 0.9464788732394366, "pass@8": 0.9802816901408451, "pass@16": 0.9971830985915493, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7455089820359282, "pass@2": 0.8203592814371258, "pass@4": 0.8922155688622755, "pass@8": 0.9341317365269461, "pass@16": 0.9760479041916168, "pass@32": 0.9940119760479041, "pass@64": 0.9970059880239521, "pass@128": 1.0 }, "10": { "pass@1": 0.6411042944785276, "pass@2": 0.6871165644171779, "pass@4": 0.7760736196319018, "pass@8": 0.8496932515337423, "pass@16": 0.9049079754601227, "pass@32": 0.9294478527607362, "pass@64": 0.9447852760736196, "pass@128": 0.9570552147239264 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9881656804733728, "5": 0.9478260869565217, "6": 0.9770773638968482, "7": 0.8872832369942196, "8": 0.8169014084507042, "9": 0.7455089820359282, "10": 0.6411042944785276 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9911242603550295, "5": 0.9826086956521739, "6": 0.9828080229226361, "7": 0.9248554913294798, "8": 0.8901408450704226, "9": 0.8203592814371258, "10": 0.6871165644171779 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 0.9971014492753624, "6": 0.9885386819484241, "7": 0.9739884393063584, "8": 0.9464788732394366, "9": 0.8922155688622755, "10": 0.7760736196319018 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.994269340974212, "7": 0.9884393063583815, "8": 0.9802816901408451, "9": 0.9341317365269461, "10": 0.8496932515337423 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.997134670487106, "7": 0.9971098265895953, "8": 0.9971830985915493, "9": 0.9760479041916168, "10": 0.9049079754601227 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9940119760479041, "10": 0.9294478527607362 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.9447852760736196 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9570552147239264 } }, "total_pass_at_k": { "pass@1": 0.8899082568807339, "pass@2": 0.9207077326343381, "pass@4": 0.9531454783748362, "pass@8": 0.9724770642201835, "pass@16": 0.9862385321100917, "pass@32": 0.9918086500655308, "pass@64": 0.9937745740498034, "pass@128": 0.9954128440366973 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99, "pass@2": 0.995, "pass@4": 0.999, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.95, "pass@2": 0.976, "pass@4": 0.994, "pass@8": 0.999, "pass@16": 0.999, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 0.999 }, "6": { "pass@1": 0.969, "pass@2": 0.981, "pass@4": 0.989, "pass@8": 0.996, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.892, "pass@2": 0.931, "pass@4": 0.963, "pass@8": 0.991, "pass@16": 0.997, "pass@32": 0.999, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.824, "pass@2": 0.878, "pass@4": 0.944, "pass@8": 0.983, "pass@16": 0.996, "pass@32": 0.998, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.758, "pass@2": 0.826, "pass@4": 0.895, "pass@8": 0.945, "pass@16": 0.978, "pass@32": 0.996, "pass@64": 0.998, "pass@128": 0.999 }, "10": { "pass@1": 0.626, "pass@2": 0.68, "pass@4": 0.77, "pass@8": 0.851, "pass@16": 0.913, "pass@32": 0.948, "pass@64": 0.96, "pass@128": 0.968 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.99, "5": 0.95, "6": 0.969, "7": 0.892, "8": 0.824, "9": 0.758, "10": 0.626 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.995, "5": 0.976, "6": 0.981, "7": 0.931, "8": 0.878, "9": 0.826, "10": 0.68 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.999, "5": 0.994, "6": 0.989, "7": 0.963, "8": 0.944, "9": 0.895, "10": 0.77 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 0.996, "7": 0.991, "8": 0.983, "9": 0.945, "10": 0.851 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 0.999, "7": 0.997, "8": 0.996, "9": 0.978, "10": 0.913 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 0.999, "8": 0.998, "9": 0.996, "10": 0.948 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.998, "10": 0.96 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.999, "10": 0.968 } }, "total_pass_at_k": { "pass@1": 0.8898888888888888, "pass@2": 0.9185555555555556, "pass@4": 0.9504444444444444, "pass@8": 0.9738888888888889, "pass@16": 0.9868888888888889, "pass@32": 0.9933333333333333, "pass@64": 0.9952222222222222, "pass@128": 0.9962222222222222 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-38797_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.5750708215297451, "pass@2": 0.6458923512747875, "pass@4": 0.7252124645892352, "pass@8": 0.8498583569405099, "pass@16": 0.9008498583569405, "pass@32": 0.9490084985835694, "pass@64": 0.9631728045325779, "pass@128": 0.9745042492917847 }, "12": { "pass@1": 0.652542372881356, "pass@2": 0.7175141242937854, "pass@4": 0.7966101694915254, "pass@8": 0.8757062146892656, "pass@16": 0.9293785310734464, "pass@32": 0.943502824858757, "pass@64": 0.9689265536723164, "pass@128": 0.9745762711864406 }, "13": { "pass@1": 0.6081504702194357, "pass@2": 0.6489028213166145, "pass@4": 0.7178683385579937, "pass@8": 0.8087774294670846, "pass@16": 0.8746081504702194, "pass@32": 0.9153605015673981, "pass@64": 0.9404388714733543, "pass@128": 0.9655172413793104 }, "14": { "pass@1": 0.557632398753894, "pass@2": 0.5825545171339563, "pass@4": 0.6137071651090342, "pass@8": 0.6822429906542056, "pass@16": 0.7570093457943925, "pass@32": 0.8193146417445483, "pass@64": 0.8722741433021807, "pass@128": 0.9252336448598131 }, "15": { "pass@1": 0.5260115606936416, "pass@2": 0.5549132947976878, "pass@4": 0.5867052023121387, "pass@8": 0.6560693641618497, "pass@16": 0.7254335260115607, "pass@32": 0.8034682080924855, "pass@64": 0.884393063583815, "pass@128": 0.9393063583815029 }, "16": { "pass@1": 0.5, "pass@2": 0.5258064516129032, "pass@4": 0.567741935483871, "pass@8": 0.6161290322580645, "pass@16": 0.6419354838709678, "pass@32": 0.7, "pass@64": 0.7483870967741936, "pass@128": 0.7935483870967742 }, "17": { "pass@1": 0.5433526011560693, "pass@2": 0.5751445086705202, "pass@4": 0.6069364161849711, "pass@8": 0.653179190751445, "pass@16": 0.7023121387283237, "pass@32": 0.7485549132947977, "pass@64": 0.8063583815028902, "pass@128": 0.8526011560693642 }, "18": { "pass@1": 0.4257142857142857, "pass@2": 0.48, "pass@4": 0.5228571428571429, "pass@8": 0.54, "pass@16": 0.5742857142857143, "pass@32": 0.62, "pass@64": 0.7114285714285714, "pass@128": 0.78 }, "19": { "pass@1": 0.38738738738738737, "pass@2": 0.42042042042042044, "pass@4": 0.4594594594594595, "pass@8": 0.4864864864864865, "pass@16": 0.5375375375375375, "pass@32": 0.5885885885885885, "pass@64": 0.6486486486486487, "pass@128": 0.6906906906906907 }, "20": { "pass@1": 0.2882882882882883, "pass@2": 0.35135135135135137, "pass@4": 0.3993993993993994, "pass@8": 0.4264264264264264, "pass@16": 0.48348348348348347, "pass@32": 0.5495495495495496, "pass@64": 0.6036036036036037, "pass@128": 0.6456456456456456 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5750708215297451, "12": 0.652542372881356, "13": 0.6081504702194357, "14": 0.557632398753894, "15": 0.5260115606936416, "16": 0.5, "17": 0.5433526011560693, "18": 0.4257142857142857, "19": 0.38738738738738737, "20": 0.2882882882882883 }, "pass@2": { "11": 0.6458923512747875, "12": 0.7175141242937854, "13": 0.6489028213166145, "14": 0.5825545171339563, "15": 0.5549132947976878, "16": 0.5258064516129032, "17": 0.5751445086705202, "18": 0.48, "19": 0.42042042042042044, "20": 0.35135135135135137 }, "pass@4": { "11": 0.7252124645892352, "12": 0.7966101694915254, "13": 0.7178683385579937, "14": 0.6137071651090342, "15": 0.5867052023121387, "16": 0.567741935483871, "17": 0.6069364161849711, "18": 0.5228571428571429, "19": 0.4594594594594595, "20": 0.3993993993993994 }, "pass@8": { "11": 0.8498583569405099, "12": 0.8757062146892656, "13": 0.8087774294670846, "14": 0.6822429906542056, "15": 0.6560693641618497, "16": 0.6161290322580645, "17": 0.653179190751445, "18": 0.54, "19": 0.4864864864864865, "20": 0.4264264264264264 }, "pass@16": { "11": 0.9008498583569405, "12": 0.9293785310734464, "13": 0.8746081504702194, "14": 0.7570093457943925, "15": 0.7254335260115607, "16": 0.6419354838709678, "17": 0.7023121387283237, "18": 0.5742857142857143, "19": 0.5375375375375375, "20": 0.48348348348348347 }, "pass@32": { "11": 0.9490084985835694, "12": 0.943502824858757, "13": 0.9153605015673981, "14": 0.8193146417445483, "15": 0.8034682080924855, "16": 0.7, "17": 0.7485549132947977, "18": 0.62, "19": 0.5885885885885885, "20": 0.5495495495495496 }, "pass@64": { "11": 0.9631728045325779, "12": 0.9689265536723164, "13": 0.9404388714733543, "14": 0.8722741433021807, "15": 0.884393063583815, "16": 0.7483870967741936, "17": 0.8063583815028902, "18": 0.7114285714285714, "19": 0.6486486486486487, "20": 0.6036036036036037 }, "pass@128": { "11": 0.9745042492917847, "12": 0.9745762711864406, "13": 0.9655172413793104, "14": 0.9252336448598131, "15": 0.9393063583815029, "16": 0.7935483870967742, "17": 0.8526011560693642, "18": 0.78, "19": 0.6906906906906907, "20": 0.6456456456456456 } }, "total_pass_at_k": { "pass@1": 0.5069836552748885, "pass@2": 0.5512630014858841, "pass@4": 0.600891530460624, "pass@8": 0.6609212481426449, "pass@16": 0.7141158989598811, "pass@32": 0.7649331352154531, "pass@64": 0.8160475482912333, "pass@128": 0.8552748885586924 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.6, "pass@2": 0.6666666666666666, "pass@4": 0.7619047619047619, "pass@8": 0.8571428571428571, "pass@16": 0.9174603174603174, "pass@32": 0.9587301587301588, "pass@64": 0.9841269841269841, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.6391437308868502, "pass@2": 0.7003058103975535, "pass@4": 0.7981651376146789, "pass@8": 0.8807339449541285, "pass@16": 0.9235474006116208, "pass@32": 0.9663608562691132, "pass@64": 0.9785932721712538, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.6080246913580247, "pass@2": 0.6759259259259259, "pass@4": 0.7654320987654321, "pass@8": 0.8518518518518519, "pass@16": 0.8919753086419753, "pass@32": 0.9290123456790124, "pass@64": 0.9567901234567902, "pass@128": 0.9629629629629629 }, "14": { "pass@1": 0.5608308605341247, "pass@2": 0.5875370919881305, "pass@4": 0.6468842729970327, "pass@8": 0.7002967359050445, "pass@16": 0.7626112759643917, "pass@32": 0.8486646884272997, "pass@64": 0.9080118694362018, "pass@128": 0.9406528189910979 }, "15": { "pass@1": 0.5092592592592593, "pass@2": 0.5401234567901234, "pass@4": 0.5709876543209876, "pass@8": 0.6450617283950617, "pass@16": 0.7283950617283951, "pass@32": 0.7870370370370371, "pass@64": 0.8858024691358025, "pass@128": 0.9320987654320988 }, "16": { "pass@1": 0.5368731563421829, "pass@2": 0.5752212389380531, "pass@4": 0.6194690265486725, "pass@8": 0.6607669616519174, "pass@16": 0.7138643067846607, "pass@32": 0.7492625368731564, "pass@64": 0.7876106194690266, "pass@128": 0.8171091445427728 }, "17": { "pass@1": 0.48493975903614456, "pass@2": 0.5030120481927711, "pass@4": 0.5572289156626506, "pass@8": 0.5903614457831325, "pass@16": 0.6475903614457831, "pass@32": 0.6897590361445783, "pass@64": 0.7680722891566265, "pass@128": 0.8042168674698795 }, "18": { "pass@1": 0.4645161290322581, "pass@2": 0.5064516129032258, "pass@4": 0.5451612903225806, "pass@8": 0.5870967741935483, "pass@16": 0.632258064516129, "pass@32": 0.6806451612903226, "pass@64": 0.7483870967741936, "pass@128": 0.7935483870967742 }, "19": { "pass@1": 0.3806646525679758, "pass@2": 0.4380664652567976, "pass@4": 0.48338368580060426, "pass@8": 0.5166163141993958, "pass@16": 0.5498489425981873, "pass@32": 0.5891238670694864, "pass@64": 0.6404833836858006, "pass@128": 0.6918429003021148 }, "20": { "pass@1": 0.3003003003003003, "pass@2": 0.34234234234234234, "pass@4": 0.3933933933933934, "pass@8": 0.43543543543543545, "pass@16": 0.46846846846846846, "pass@32": 0.5105105105105106, "pass@64": 0.5825825825825826, "pass@128": 0.6366366366366366 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6, "12": 0.6391437308868502, "13": 0.6080246913580247, "14": 0.5608308605341247, "15": 0.5092592592592593, "16": 0.5368731563421829, "17": 0.48493975903614456, "18": 0.4645161290322581, "19": 0.3806646525679758, "20": 0.3003003003003003 }, "pass@2": { "11": 0.6666666666666666, "12": 0.7003058103975535, "13": 0.6759259259259259, "14": 0.5875370919881305, "15": 0.5401234567901234, "16": 0.5752212389380531, "17": 0.5030120481927711, "18": 0.5064516129032258, "19": 0.4380664652567976, "20": 0.34234234234234234 }, "pass@4": { "11": 0.7619047619047619, "12": 0.7981651376146789, "13": 0.7654320987654321, "14": 0.6468842729970327, "15": 0.5709876543209876, "16": 0.6194690265486725, "17": 0.5572289156626506, "18": 0.5451612903225806, "19": 0.48338368580060426, "20": 0.3933933933933934 }, "pass@8": { "11": 0.8571428571428571, "12": 0.8807339449541285, "13": 0.8518518518518519, "14": 0.7002967359050445, "15": 0.6450617283950617, "16": 0.6607669616519174, "17": 0.5903614457831325, "18": 0.5870967741935483, "19": 0.5166163141993958, "20": 0.43543543543543545 }, "pass@16": { "11": 0.9174603174603174, "12": 0.9235474006116208, "13": 0.8919753086419753, "14": 0.7626112759643917, "15": 0.7283950617283951, "16": 0.7138643067846607, "17": 0.6475903614457831, "18": 0.632258064516129, "19": 0.5498489425981873, "20": 0.46846846846846846 }, "pass@32": { "11": 0.9587301587301588, "12": 0.9663608562691132, "13": 0.9290123456790124, "14": 0.8486646884272997, "15": 0.7870370370370371, "16": 0.7492625368731564, "17": 0.6897590361445783, "18": 0.6806451612903226, "19": 0.5891238670694864, "20": 0.5105105105105106 }, "pass@64": { "11": 0.9841269841269841, "12": 0.9785932721712538, "13": 0.9567901234567902, "14": 0.9080118694362018, "15": 0.8858024691358025, "16": 0.7876106194690266, "17": 0.7680722891566265, "18": 0.7483870967741936, "19": 0.6404833836858006, "20": 0.5825825825825826 }, "pass@128": { "11": 0.9936507936507937, "12": 0.981651376146789, "13": 0.9629629629629629, "14": 0.9406528189910979, "15": 0.9320987654320988, "16": 0.8171091445427728, "17": 0.8042168674698795, "18": 0.7935483870967742, "19": 0.6918429003021148, "20": 0.6366366366366366 } }, "total_pass_at_k": { "pass@1": 0.5079462102689487, "pass@2": 0.5528728606356969, "pass@4": 0.6133863080684596, "pass@8": 0.6714547677261614, "pass@16": 0.7224938875305623, "pass@32": 0.7698655256723717, "pass@64": 0.8230440097799511, "pass@128": 0.8545232273838631 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.677710843373494, "pass@2": 0.7078313253012049, "pass@4": 0.7740963855421686, "pass@8": 0.8644578313253012, "pass@16": 0.927710843373494, "pass@32": 0.963855421686747, "pass@64": 0.9698795180722891, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.5705329153605015, "pass@2": 0.6489028213166145, "pass@4": 0.7460815047021944, "pass@8": 0.8432601880877743, "pass@16": 0.9090909090909091, "pass@32": 0.9623824451410659, "pass@64": 0.9655172413793104, "pass@128": 0.9686520376175548 }, "13": { "pass@1": 0.6330532212885154, "pass@2": 0.680672268907563, "pass@4": 0.7563025210084033, "pass@8": 0.8431372549019608, "pass@16": 0.9103641456582633, "pass@32": 0.9551820728291317, "pass@64": 0.9859943977591037, "pass@128": 0.988795518207283 }, "14": { "pass@1": 0.5584795321637427, "pass@2": 0.6228070175438597, "pass@4": 0.6666666666666666, "pass@8": 0.7251461988304093, "pass@16": 0.7748538011695907, "pass@32": 0.8450292397660819, "pass@64": 0.9122807017543859, "pass@128": 0.9502923976608187 }, "15": { "pass@1": 0.5636363636363636, "pass@2": 0.6060606060606061, "pass@4": 0.6393939393939394, "pass@8": 0.7181818181818181, "pass@16": 0.7787878787878788, "pass@32": 0.8606060606060606, "pass@64": 0.9090909090909091, "pass@128": 0.9424242424242424 }, "16": { "pass@1": 0.5213675213675214, "pass@2": 0.5584045584045584, "pass@4": 0.6125356125356125, "pass@8": 0.6752136752136753, "pass@16": 0.7122507122507122, "pass@32": 0.7435897435897436, "pass@64": 0.8062678062678063, "pass@128": 0.8404558404558404 }, "17": { "pass@1": 0.4440993788819876, "pass@2": 0.48757763975155277, "pass@4": 0.5372670807453416, "pass@8": 0.5807453416149069, "pass@16": 0.6428571428571429, "pass@32": 0.7267080745341615, "pass@64": 0.7919254658385093, "pass@128": 0.8167701863354038 }, "18": { "pass@1": 0.4323529411764706, "pass@2": 0.47058823529411764, "pass@4": 0.4970588235294118, "pass@8": 0.5352941176470588, "pass@16": 0.5794117647058824, "pass@32": 0.6441176470588236, "pass@64": 0.7147058823529412, "pass@128": 0.7794117647058824 }, "19": { "pass@1": 0.41369047619047616, "pass@2": 0.44047619047619047, "pass@4": 0.47619047619047616, "pass@8": 0.5327380952380952, "pass@16": 0.5625, "pass@32": 0.6041666666666666, "pass@64": 0.6577380952380952, "pass@128": 0.6904761904761905 }, "20": { "pass@1": 0.32335329341317365, "pass@2": 0.3712574850299401, "pass@4": 0.40119760479041916, "pass@8": 0.4341317365269461, "pass@16": 0.47604790419161674, "pass@32": 0.5299401197604791, "pass@64": 0.5808383233532934, "pass@128": 0.6377245508982036 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.677710843373494, "12": 0.5705329153605015, "13": 0.6330532212885154, "14": 0.5584795321637427, "15": 0.5636363636363636, "16": 0.5213675213675214, "17": 0.4440993788819876, "18": 0.4323529411764706, "19": 0.41369047619047616, "20": 0.32335329341317365 }, "pass@2": { "11": 0.7078313253012049, "12": 0.6489028213166145, "13": 0.680672268907563, "14": 0.6228070175438597, "15": 0.6060606060606061, "16": 0.5584045584045584, "17": 0.48757763975155277, "18": 0.47058823529411764, "19": 0.44047619047619047, "20": 0.3712574850299401 }, "pass@4": { "11": 0.7740963855421686, "12": 0.7460815047021944, "13": 0.7563025210084033, "14": 0.6666666666666666, "15": 0.6393939393939394, "16": 0.6125356125356125, "17": 0.5372670807453416, "18": 0.4970588235294118, "19": 0.47619047619047616, "20": 0.40119760479041916 }, "pass@8": { "11": 0.8644578313253012, "12": 0.8432601880877743, "13": 0.8431372549019608, "14": 0.7251461988304093, "15": 0.7181818181818181, "16": 0.6752136752136753, "17": 0.5807453416149069, "18": 0.5352941176470588, "19": 0.5327380952380952, "20": 0.4341317365269461 }, "pass@16": { "11": 0.927710843373494, "12": 0.9090909090909091, "13": 0.9103641456582633, "14": 0.7748538011695907, "15": 0.7787878787878788, "16": 0.7122507122507122, "17": 0.6428571428571429, "18": 0.5794117647058824, "19": 0.5625, "20": 0.47604790419161674 }, "pass@32": { "11": 0.963855421686747, "12": 0.9623824451410659, "13": 0.9551820728291317, "14": 0.8450292397660819, "15": 0.8606060606060606, "16": 0.7435897435897436, "17": 0.7267080745341615, "18": 0.6441176470588236, "19": 0.6041666666666666, "20": 0.5299401197604791 }, "pass@64": { "11": 0.9698795180722891, "12": 0.9655172413793104, "13": 0.9859943977591037, "14": 0.9122807017543859, "15": 0.9090909090909091, "16": 0.8062678062678063, "17": 0.7919254658385093, "18": 0.7147058823529412, "19": 0.6577380952380952, "20": 0.5808383233532934 }, "pass@128": { "11": 0.9879518072289156, "12": 0.9686520376175548, "13": 0.988795518207283, "14": 0.9502923976608187, "15": 0.9424242424242424, "16": 0.8404558404558404, "17": 0.8167701863354038, "18": 0.7794117647058824, "19": 0.6904761904761905, "20": 0.6377245508982036 } }, "total_pass_at_k": { "pass@1": 0.5144216473386857, "pass@2": 0.5599167410050551, "pass@4": 0.6110615521855486, "pass@8": 0.6755872732679156, "pass@16": 0.727624145108534, "pass@32": 0.7835266131430271, "pass@64": 0.8296164139161463, "pass@128": 0.8605411834671425 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.617, "pass@2": 0.673, "pass@4": 0.753, "pass@8": 0.857, "pass@16": 0.915, "pass@32": 0.957, "pass@64": 0.972, "pass@128": 0.985 }, "12": { "pass@1": 0.622, "pass@2": 0.69, "pass@4": 0.781, "pass@8": 0.867, "pass@16": 0.921, "pass@32": 0.957, "pass@64": 0.971, "pass@128": 0.975 }, "13": { "pass@1": 0.617, "pass@2": 0.669, "pass@4": 0.747, "pass@8": 0.835, "pass@16": 0.893, "pass@32": 0.934, "pass@64": 0.962, "pass@128": 0.973 }, "14": { "pass@1": 0.559, "pass@2": 0.598, "pass@4": 0.643, "pass@8": 0.703, "pass@16": 0.765, "pass@32": 0.838, "pass@64": 0.898, "pass@128": 0.939 }, "15": { "pass@1": 0.533, "pass@2": 0.567, "pass@4": 0.599, "pass@8": 0.673, "pass@16": 0.744, "pass@32": 0.817, "pass@64": 0.893, "pass@128": 0.938 }, "16": { "pass@1": 0.52, "pass@2": 0.554, "pass@4": 0.601, "pass@8": 0.652, "pass@16": 0.691, "pass@32": 0.732, "pass@64": 0.782, "pass@128": 0.818 }, "17": { "pass@1": 0.492, "pass@2": 0.523, "pass@4": 0.568, "pass@8": 0.609, "pass@16": 0.665, "pass@32": 0.722, "pass@64": 0.789, "pass@128": 0.825 }, "18": { "pass@1": 0.44, "pass@2": 0.485, "pass@4": 0.521, "pass@8": 0.553, "pass@16": 0.594, "pass@32": 0.647, "pass@64": 0.724, "pass@128": 0.784 }, "19": { "pass@1": 0.394, "pass@2": 0.433, "pass@4": 0.473, "pass@8": 0.512, "pass@16": 0.55, "pass@32": 0.594, "pass@64": 0.649, "pass@128": 0.691 }, "20": { "pass@1": 0.304, "pass@2": 0.355, "pass@4": 0.398, "pass@8": 0.432, "pass@16": 0.476, "pass@32": 0.53, "pass@64": 0.589, "pass@128": 0.64 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.617, "12": 0.622, "13": 0.617, "14": 0.559, "15": 0.533, "16": 0.52, "17": 0.492, "18": 0.44, "19": 0.394, "20": 0.304 }, "pass@2": { "11": 0.673, "12": 0.69, "13": 0.669, "14": 0.598, "15": 0.567, "16": 0.554, "17": 0.523, "18": 0.485, "19": 0.433, "20": 0.355 }, "pass@4": { "11": 0.753, "12": 0.781, "13": 0.747, "14": 0.643, "15": 0.599, "16": 0.601, "17": 0.568, "18": 0.521, "19": 0.473, "20": 0.398 }, "pass@8": { "11": 0.857, "12": 0.867, "13": 0.835, "14": 0.703, "15": 0.673, "16": 0.652, "17": 0.609, "18": 0.553, "19": 0.512, "20": 0.432 }, "pass@16": { "11": 0.915, "12": 0.921, "13": 0.893, "14": 0.765, "15": 0.744, "16": 0.691, "17": 0.665, "18": 0.594, "19": 0.55, "20": 0.476 }, "pass@32": { "11": 0.957, "12": 0.957, "13": 0.934, "14": 0.838, "15": 0.817, "16": 0.732, "17": 0.722, "18": 0.647, "19": 0.594, "20": 0.53 }, "pass@64": { "11": 0.972, "12": 0.971, "13": 0.962, "14": 0.898, "15": 0.893, "16": 0.782, "17": 0.789, "18": 0.724, "19": 0.649, "20": 0.589 }, "pass@128": { "11": 0.985, "12": 0.975, "13": 0.973, "14": 0.939, "15": 0.938, "16": 0.818, "17": 0.825, "18": 0.784, "19": 0.691, "20": 0.64 } }, "total_pass_at_k": { "pass@1": 0.5098, "pass@2": 0.5547, "pass@4": 0.6084, "pass@8": 0.6693, "pass@16": 0.7214, "pass@32": 0.7728, "pass@64": 0.8229, "pass@128": 0.8568 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }