{ "files": { "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.996875, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9619047619047619, "pass@2": 0.9809523809523809, "pass@4": 0.9968253968253968, "pass@8": 0.9968253968253968, "pass@16": 0.9968253968253968, "pass@32": 0.9968253968253968, "pass@64": 0.9968253968253968, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9580838323353293, "pass@2": 0.9700598802395209, "pass@4": 0.9850299401197605, "pass@8": 0.9910179640718563, "pass@16": 0.9970059880239521, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8875739644970414, "pass@2": 0.9556213017751479, "pass@4": 0.9733727810650887, "pass@8": 0.9970414201183432, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8076923076923077, "pass@2": 0.8782051282051282, "pass@4": 0.9358974358974359, "pass@8": 0.9775641025641025, "pass@16": 0.9871794871794872, "pass@32": 0.9967948717948718, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7098591549295775, "pass@2": 0.7887323943661971, "pass@4": 0.8901408450704226, "pass@8": 0.9323943661971831, "pass@16": 0.9774647887323944, "pass@32": 0.9971830985915493, "pass@64": 0.9971830985915493, "pass@128": 0.9971830985915493 }, "10": { "pass@1": 0.6430678466076696, "pass@2": 0.696165191740413, "pass@4": 0.7905604719764012, "pass@8": 0.8790560471976401, "pass@16": 0.9351032448377581, "pass@32": 0.9528023598820059, "pass@64": 0.9587020648967551, "pass@128": 0.9646017699115044 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9619047619047619, "6": 0.9580838323353293, "7": 0.8875739644970414, "8": 0.8076923076923077, "9": 0.7098591549295775, "10": 0.6430678466076696 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9809523809523809, "6": 0.9700598802395209, "7": 0.9556213017751479, "8": 0.8782051282051282, "9": 0.7887323943661971, "10": 0.696165191740413 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 0.9850299401197605, "7": 0.9733727810650887, "8": 0.9358974358974359, "9": 0.8901408450704226, "10": 0.7905604719764012 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 0.9910179640718563, "7": 0.9970414201183432, "8": 0.9775641025641025, "9": 0.9323943661971831, "10": 0.8790560471976401 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 0.9970059880239521, "7": 1.0, "8": 0.9871794871794872, "9": 0.9774647887323944, "10": 0.9351032448377581 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 0.9967948717948718, "9": 0.9971830985915493, "10": 0.9528023598820059 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9587020648967551 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9646017699115044 } }, "total_pass_at_k": { "pass@1": 0.8821344140493077, "pass@2": 0.9165822357311719, "pass@4": 0.9510300574130361, "pass@8": 0.9739952718676123, "pass@16": 0.9878419452887538, "pass@32": 0.9935832489023978, "pass@64": 0.994596420128335, "pass@128": 0.9952718676122931 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9912280701754386, "pass@2": 0.9912280701754386, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9470588235294117, "pass@2": 0.9823529411764705, "pass@4": 0.9970588235294118, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9716088328075709, "pass@2": 0.9779179810725552, "pass@4": 0.9905362776025236, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.879746835443038, "pass@2": 0.930379746835443, "pass@4": 0.9683544303797469, "pass@8": 0.9841772151898734, "pass@16": 0.9936708860759493, "pass@32": 0.9968354430379747, "pass@64": 0.9968354430379747, "pass@128": 1.0 }, "8": { "pass@1": 0.8018018018018018, "pass@2": 0.8498498498498499, "pass@4": 0.918918918918919, "pass@8": 0.9669669669669669, "pass@16": 0.990990990990991, "pass@32": 0.996996996996997, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7620578778135049, "pass@2": 0.8231511254019293, "pass@4": 0.9067524115755627, "pass@8": 0.9678456591639871, "pass@16": 0.9807073954983923, "pass@32": 0.9967845659163987, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6, "pass@2": 0.6597014925373135, "pass@4": 0.7492537313432835, "pass@8": 0.8597014925373134, "pass@16": 0.9194029850746268, "pass@32": 0.9432835820895522, "pass@64": 0.9641791044776119, "pass@128": 0.9731343283582089 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9912280701754386, "5": 0.9470588235294117, "6": 0.9716088328075709, "7": 0.879746835443038, "8": 0.8018018018018018, "9": 0.7620578778135049, "10": 0.6 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9912280701754386, "5": 0.9823529411764705, "6": 0.9779179810725552, "7": 0.930379746835443, "8": 0.8498498498498499, "9": 0.8231511254019293, "10": 0.6597014925373135 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 0.9905362776025236, "7": 0.9683544303797469, "8": 0.918918918918919, "9": 0.9067524115755627, "10": 0.7492537313432835 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9841772151898734, "8": 0.9669669669669669, "9": 0.9678456591639871, "10": 0.8597014925373134 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9936708860759493, "8": 0.990990990990991, "9": 0.9807073954983923, "10": 0.9194029850746268 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9968354430379747, "8": 0.996996996996997, "9": 0.9967845659163987, "10": 0.9432835820895522 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9968354430379747, "8": 1.0, "9": 1.0, "10": 0.9641791044776119 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9731343283582089 } }, "total_pass_at_k": { "pass@1": 0.8855038500167393, "pass@2": 0.9139604954804151, "pass@4": 0.9484432541011047, "pass@8": 0.9755607633076665, "pass@16": 0.9872782055574155, "pass@32": 0.9926347505858721, "pass@64": 0.995647807164379, "pass@128": 0.9969869434214932 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9792899408284024, "pass@2": 0.9881656804733728, "pass@4": 0.9940828402366864, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9681159420289855, "pass@2": 0.9884057971014493, "pass@4": 0.991304347826087, "pass@8": 0.9971014492753624, "pass@16": 0.9971014492753624, "pass@32": 0.9971014492753624, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9742120343839542, "pass@2": 0.9799426934097422, "pass@4": 0.9885386819484241, "pass@8": 0.9914040114613181, "pass@16": 0.997134670487106, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8583815028901735, "pass@2": 0.9132947976878613, "pass@4": 0.9508670520231214, "pass@8": 0.9884393063583815, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.7971830985915493, "pass@2": 0.8732394366197183, "pass@4": 0.9577464788732394, "pass@8": 0.9887323943661972, "pass@16": 0.9971830985915493, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7425149700598802, "pass@2": 0.8053892215568862, "pass@4": 0.8922155688622755, "pass@8": 0.9730538922155688, "pass@16": 0.9970059880239521, "pass@32": 0.9970059880239521, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.656441717791411, "pass@2": 0.7208588957055214, "pass@4": 0.8006134969325154, "pass@8": 0.8650306748466258, "pass@16": 0.9079754601226994, "pass@32": 0.9447852760736196, "pass@64": 0.9478527607361963, "pass@128": 0.950920245398773 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9792899408284024, "5": 0.9681159420289855, "6": 0.9742120343839542, "7": 0.8583815028901735, "8": 0.7971830985915493, "9": 0.7425149700598802, "10": 0.656441717791411 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9881656804733728, "5": 0.9884057971014493, "6": 0.9799426934097422, "7": 0.9132947976878613, "8": 0.8732394366197183, "9": 0.8053892215568862, "10": 0.7208588957055214 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9940828402366864, "5": 0.991304347826087, "6": 0.9885386819484241, "7": 0.9508670520231214, "8": 0.9577464788732394, "9": 0.8922155688622755, "10": 0.8006134969325154 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 0.9914040114613181, "7": 0.9884393063583815, "8": 0.9887323943661972, "9": 0.9730538922155688, "10": 0.8650306748466258 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 0.997134670487106, "7": 1.0, "8": 0.9971830985915493, "9": 0.9970059880239521, "10": 0.9079754601226994 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.9447852760736196 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9478527607361963 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.950920245398773 } }, "total_pass_at_k": { "pass@1": 0.8866317169069463, "pass@2": 0.919397116644823, "pass@4": 0.953473132372215, "pass@8": 0.97870249017038, "pass@16": 0.9888597640891219, "pass@32": 0.9934469200524246, "pass@64": 0.994429882044561, "pass@128": 0.9947575360419397 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.989, "pass@2": 0.993, "pass@4": 0.998, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.959, "pass@2": 0.984, "pass@4": 0.995, "pass@8": 0.998, "pass@16": 0.998, "pass@32": 0.998, "pass@64": 0.999, "pass@128": 0.999 }, "6": { "pass@1": 0.968, "pass@2": 0.976, "pass@4": 0.988, "pass@8": 0.994, "pass@16": 0.998, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.875, "pass@2": 0.933, "pass@4": 0.964, "pass@8": 0.99, "pass@16": 0.998, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 1.0 }, "8": { "pass@1": 0.802, "pass@2": 0.867, "pass@4": 0.938, "pass@8": 0.978, "pass@16": 0.992, "pass@32": 0.998, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.737, "pass@2": 0.805, "pass@4": 0.896, "pass@8": 0.957, "pass@16": 0.985, "pass@32": 0.997, "pass@64": 0.999, "pass@128": 0.999 }, "10": { "pass@1": 0.633, "pass@2": 0.692, "pass@4": 0.78, "pass@8": 0.868, "pass@16": 0.921, "pass@32": 0.947, "pass@64": 0.957, "pass@128": 0.963 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.989, "5": 0.959, "6": 0.968, "7": 0.875, "8": 0.802, "9": 0.737, "10": 0.633 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.993, "5": 0.984, "6": 0.976, "7": 0.933, "8": 0.867, "9": 0.805, "10": 0.692 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.998, "5": 0.995, "6": 0.988, "7": 0.964, "8": 0.938, "9": 0.896, "10": 0.78 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 0.994, "7": 0.99, "8": 0.978, "9": 0.957, "10": 0.868 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 0.998, "7": 0.998, "8": 0.992, "9": 0.985, "10": 0.921 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.998, "6": 1.0, "7": 0.999, "8": 0.998, "9": 0.997, "10": 0.947 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 0.999, "8": 1.0, "9": 0.999, "10": 0.957 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.999, "10": 0.963 } }, "total_pass_at_k": { "pass@1": 0.8847777777777778, "pass@2": 0.9166666666666666, "pass@4": 0.951, "pass@8": 0.9761111111111112, "pass@16": 0.988, "pass@32": 0.9932222222222222, "pass@64": 0.9948888888888889, "pass@128": 0.9956666666666667 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.6005665722379604, "pass@2": 0.6742209631728046, "pass@4": 0.773371104815864, "pass@8": 0.8640226628895185, "pass@16": 0.9178470254957507, "pass@32": 0.9518413597733711, "pass@64": 0.9745042492917847, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.6638418079096046, "pass@2": 0.7231638418079096, "pass@4": 0.8022598870056498, "pass@8": 0.8813559322033898, "pass@16": 0.9293785310734464, "pass@32": 0.9491525423728814, "pass@64": 0.963276836158192, "pass@128": 0.9745762711864406 }, "13": { "pass@1": 0.5893416927899686, "pass@2": 0.6394984326018809, "pass@4": 0.7272727272727273, "pass@8": 0.8056426332288401, "pass@16": 0.8714733542319749, "pass@32": 0.9122257053291536, "pass@64": 0.9467084639498433, "pass@128": 0.9717868338557993 }, "14": { "pass@1": 0.5607476635514018, "pass@2": 0.5919003115264797, "pass@4": 0.6417445482866043, "pass@8": 0.7227414330218068, "pass@16": 0.7757009345794392, "pass@32": 0.8286604361370716, "pass@64": 0.8847352024922118, "pass@128": 0.9283489096573209 }, "15": { "pass@1": 0.5057803468208093, "pass@2": 0.5549132947976878, "pass@4": 0.5953757225433526, "pass@8": 0.6647398843930635, "pass@16": 0.7456647398843931, "pass@32": 0.8179190751445087, "pass@64": 0.9104046242774566, "pass@128": 0.9450867052023122 }, "16": { "pass@1": 0.4935483870967742, "pass@2": 0.5193548387096775, "pass@4": 0.5548387096774193, "pass@8": 0.5870967741935483, "pass@16": 0.6548387096774193, "pass@32": 0.7032258064516129, "pass@64": 0.7354838709677419, "pass@128": 0.7709677419354839 }, "17": { "pass@1": 0.5289017341040463, "pass@2": 0.546242774566474, "pass@4": 0.5809248554913294, "pass@8": 0.6184971098265896, "pass@16": 0.6705202312138728, "pass@32": 0.7167630057803468, "pass@64": 0.7803468208092486, "pass@128": 0.8294797687861272 }, "18": { "pass@1": 0.4, "pass@2": 0.4142857142857143, "pass@4": 0.4514285714285714, "pass@8": 0.49714285714285716, "pass@16": 0.5457142857142857, "pass@32": 0.6057142857142858, "pass@64": 0.6714285714285714, "pass@128": 0.7314285714285714 }, "19": { "pass@1": 0.3063063063063063, "pass@2": 0.33633633633633636, "pass@4": 0.39039039039039036, "pass@8": 0.4174174174174174, "pass@16": 0.45645645645645644, "pass@32": 0.5015015015015015, "pass@64": 0.5555555555555556, "pass@128": 0.5825825825825826 }, "20": { "pass@1": 0.21621621621621623, "pass@2": 0.24924924924924924, "pass@4": 0.2822822822822823, "pass@8": 0.3153153153153153, "pass@16": 0.34234234234234234, "pass@32": 0.39039039039039036, "pass@64": 0.43843843843843844, "pass@128": 0.4924924924924925 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6005665722379604, "12": 0.6638418079096046, "13": 0.5893416927899686, "14": 0.5607476635514018, "15": 0.5057803468208093, "16": 0.4935483870967742, "17": 0.5289017341040463, "18": 0.4, "19": 0.3063063063063063, "20": 0.21621621621621623 }, "pass@2": { "11": 0.6742209631728046, "12": 0.7231638418079096, "13": 0.6394984326018809, "14": 0.5919003115264797, "15": 0.5549132947976878, "16": 0.5193548387096775, "17": 0.546242774566474, "18": 0.4142857142857143, "19": 0.33633633633633636, "20": 0.24924924924924924 }, "pass@4": { "11": 0.773371104815864, "12": 0.8022598870056498, "13": 0.7272727272727273, "14": 0.6417445482866043, "15": 0.5953757225433526, "16": 0.5548387096774193, "17": 0.5809248554913294, "18": 0.4514285714285714, "19": 0.39039039039039036, "20": 0.2822822822822823 }, "pass@8": { "11": 0.8640226628895185, "12": 0.8813559322033898, "13": 0.8056426332288401, "14": 0.7227414330218068, "15": 0.6647398843930635, "16": 0.5870967741935483, "17": 0.6184971098265896, "18": 0.49714285714285716, "19": 0.4174174174174174, "20": 0.3153153153153153 }, "pass@16": { "11": 0.9178470254957507, "12": 0.9293785310734464, "13": 0.8714733542319749, "14": 0.7757009345794392, "15": 0.7456647398843931, "16": 0.6548387096774193, "17": 0.6705202312138728, "18": 0.5457142857142857, "19": 0.45645645645645644, "20": 0.34234234234234234 }, "pass@32": { "11": 0.9518413597733711, "12": 0.9491525423728814, "13": 0.9122257053291536, "14": 0.8286604361370716, "15": 0.8179190751445087, "16": 0.7032258064516129, "17": 0.7167630057803468, "18": 0.6057142857142858, "19": 0.5015015015015015, "20": 0.39039039039039036 }, "pass@64": { "11": 0.9745042492917847, "12": 0.963276836158192, "13": 0.9467084639498433, "14": 0.8847352024922118, "15": 0.9104046242774566, "16": 0.7354838709677419, "17": 0.7803468208092486, "18": 0.6714285714285714, "19": 0.5555555555555556, "20": 0.43843843843843844 }, "pass@128": { "11": 0.9830028328611898, "12": 0.9745762711864406, "13": 0.9717868338557993, "14": 0.9283489096573209, "15": 0.9450867052023122, "16": 0.7709677419354839, "17": 0.8294797687861272, "18": 0.7314285714285714, "19": 0.5825825825825826, "20": 0.4924924924924925 } }, "total_pass_at_k": { "pass@1": 0.487369985141159, "pass@2": 0.5260029717682021, "pass@4": 0.5812778603268945, "pass@8": 0.6389301634472511, "pass@16": 0.6924219910846954, "pass@32": 0.7390787518573552, "pass@64": 0.787518573551263, "pass@128": 0.8222882615156017 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.6095238095238096, "pass@2": 0.6952380952380952, "pass@4": 0.8095238095238095, "pass@8": 0.873015873015873, "pass@16": 0.9333333333333333, "pass@32": 0.9619047619047619, "pass@64": 0.9841269841269841, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.617737003058104, "pass@2": 0.6788990825688074, "pass@4": 0.7675840978593272, "pass@8": 0.8837920489296636, "pass@16": 0.9327217125382263, "pass@32": 0.9724770642201835, "pass@64": 0.981651376146789, "pass@128": 0.9938837920489296 }, "13": { "pass@1": 0.5864197530864198, "pass@2": 0.6851851851851852, "pass@4": 0.7623456790123457, "pass@8": 0.8302469135802469, "pass@16": 0.8796296296296297, "pass@32": 0.9197530864197531, "pass@64": 0.9506172839506173, "pass@128": 0.9629629629629629 }, "14": { "pass@1": 0.56973293768546, "pass@2": 0.599406528189911, "pass@4": 0.6528189910979229, "pass@8": 0.7151335311572701, "pass@16": 0.7833827893175074, "pass@32": 0.8486646884272997, "pass@64": 0.9287833827893175, "pass@128": 0.9584569732937686 }, "15": { "pass@1": 0.5, "pass@2": 0.5308641975308642, "pass@4": 0.5802469135802469, "pass@8": 0.6481481481481481, "pass@16": 0.7314814814814815, "pass@32": 0.7962962962962963, "pass@64": 0.8765432098765432, "pass@128": 0.9320987654320988 }, "16": { "pass@1": 0.5398230088495575, "pass@2": 0.5781710914454278, "pass@4": 0.6106194690265486, "pass@8": 0.6460176991150443, "pass@16": 0.7050147492625368, "pass@32": 0.7463126843657817, "pass@64": 0.7935103244837758, "pass@128": 0.8230088495575221 }, "17": { "pass@1": 0.4759036144578313, "pass@2": 0.5090361445783133, "pass@4": 0.536144578313253, "pass@8": 0.5662650602409639, "pass@16": 0.6234939759036144, "pass@32": 0.7018072289156626, "pass@64": 0.7680722891566265, "pass@128": 0.7891566265060241 }, "18": { "pass@1": 0.4290322580645161, "pass@2": 0.45483870967741935, "pass@4": 0.4870967741935484, "pass@8": 0.535483870967742, "pass@16": 0.5709677419354838, "pass@32": 0.6387096774193548, "pass@64": 0.7258064516129032, "pass@128": 0.7741935483870968 }, "19": { "pass@1": 0.2990936555891239, "pass@2": 0.3564954682779456, "pass@4": 0.3867069486404834, "pass@8": 0.4199395770392749, "pass@16": 0.459214501510574, "pass@32": 0.4743202416918429, "pass@64": 0.525679758308157, "pass@128": 0.5770392749244713 }, "20": { "pass@1": 0.1921921921921922, "pass@2": 0.24324324324324326, "pass@4": 0.2822822822822823, "pass@8": 0.30930930930930933, "pass@16": 0.3333333333333333, "pass@32": 0.36036036036036034, "pass@64": 0.4144144144144144, "pass@128": 0.46846846846846846 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6095238095238096, "12": 0.617737003058104, "13": 0.5864197530864198, "14": 0.56973293768546, "15": 0.5, "16": 0.5398230088495575, "17": 0.4759036144578313, "18": 0.4290322580645161, "19": 0.2990936555891239, "20": 0.1921921921921922 }, "pass@2": { "11": 0.6952380952380952, "12": 0.6788990825688074, "13": 0.6851851851851852, "14": 0.599406528189911, "15": 0.5308641975308642, "16": 0.5781710914454278, "17": 0.5090361445783133, "18": 0.45483870967741935, "19": 0.3564954682779456, "20": 0.24324324324324326 }, "pass@4": { "11": 0.8095238095238095, "12": 0.7675840978593272, "13": 0.7623456790123457, "14": 0.6528189910979229, "15": 0.5802469135802469, "16": 0.6106194690265486, "17": 0.536144578313253, "18": 0.4870967741935484, "19": 0.3867069486404834, "20": 0.2822822822822823 }, "pass@8": { "11": 0.873015873015873, "12": 0.8837920489296636, "13": 0.8302469135802469, "14": 0.7151335311572701, "15": 0.6481481481481481, "16": 0.6460176991150443, "17": 0.5662650602409639, "18": 0.535483870967742, "19": 0.4199395770392749, "20": 0.30930930930930933 }, "pass@16": { "11": 0.9333333333333333, "12": 0.9327217125382263, "13": 0.8796296296296297, "14": 0.7833827893175074, "15": 0.7314814814814815, "16": 0.7050147492625368, "17": 0.6234939759036144, "18": 0.5709677419354838, "19": 0.459214501510574, "20": 0.3333333333333333 }, "pass@32": { "11": 0.9619047619047619, "12": 0.9724770642201835, "13": 0.9197530864197531, "14": 0.8486646884272997, "15": 0.7962962962962963, "16": 0.7463126843657817, "17": 0.7018072289156626, "18": 0.6387096774193548, "19": 0.4743202416918429, "20": 0.36036036036036034 }, "pass@64": { "11": 0.9841269841269841, "12": 0.981651376146789, "13": 0.9506172839506173, "14": 0.9287833827893175, "15": 0.8765432098765432, "16": 0.7935103244837758, "17": 0.7680722891566265, "18": 0.7258064516129032, "19": 0.525679758308157, "20": 0.4144144144144144 }, "pass@128": { "11": 0.9904761904761905, "12": 0.9938837920489296, "13": 0.9629629629629629, "14": 0.9584569732937686, "15": 0.9320987654320988, "16": 0.8230088495575221, "17": 0.7891566265060241, "18": 0.7741935483870968, "19": 0.5770392749244713, "20": 0.46846846846846846 } }, "total_pass_at_k": { "pass@1": 0.48135696821515894, "pass@2": 0.53239608801956, "pass@4": 0.5864914425427873, "pass@8": 0.6415036674816625, "pass@16": 0.6940709046454768, "pass@32": 0.7408312958435208, "pass@64": 0.7937041564792175, "pass@128": 0.8257946210268948 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6686746987951807, "pass@2": 0.7228915662650602, "pass@4": 0.8042168674698795, "pass@8": 0.8825301204819277, "pass@16": 0.9216867469879518, "pass@32": 0.9578313253012049, "pass@64": 0.9819277108433735, "pass@128": 0.9879518072289156 }, "12": { "pass@1": 0.5987460815047022, "pass@2": 0.658307210031348, "pass@4": 0.7711598746081505, "pass@8": 0.8369905956112853, "pass@16": 0.9122257053291536, "pass@32": 0.9467084639498433, "pass@64": 0.9717868338557993, "pass@128": 0.9811912225705329 }, "13": { "pass@1": 0.6162464985994398, "pass@2": 0.6750700280112045, "pass@4": 0.7675070028011205, "pass@8": 0.8487394957983193, "pass@16": 0.9131652661064426, "pass@32": 0.938375350140056, "pass@64": 0.9635854341736695, "pass@128": 0.9831932773109243 }, "14": { "pass@1": 0.5672514619883041, "pass@2": 0.6023391812865497, "pass@4": 0.6374269005847953, "pass@8": 0.716374269005848, "pass@16": 0.7923976608187134, "pass@32": 0.8713450292397661, "pass@64": 0.9385964912280702, "pass@128": 0.9590643274853801 }, "15": { "pass@1": 0.5545454545454546, "pass@2": 0.5848484848484848, "pass@4": 0.6242424242424243, "pass@8": 0.6818181818181818, "pass@16": 0.7545454545454545, "pass@32": 0.8121212121212121, "pass@64": 0.8727272727272727, "pass@128": 0.9272727272727272 }, "16": { "pass@1": 0.5156695156695157, "pass@2": 0.5384615384615384, "pass@4": 0.5726495726495726, "pass@8": 0.6296296296296297, "pass@16": 0.6837606837606838, "pass@32": 0.7492877492877493, "pass@64": 0.7977207977207977, "pass@128": 0.8319088319088319 }, "17": { "pass@1": 0.43478260869565216, "pass@2": 0.45652173913043476, "pass@4": 0.5186335403726708, "pass@8": 0.5403726708074534, "pass@16": 0.6180124223602484, "pass@32": 0.6770186335403726, "pass@64": 0.7298136645962733, "pass@128": 0.7888198757763976 }, "18": { "pass@1": 0.39705882352941174, "pass@2": 0.4323529411764706, "pass@4": 0.45294117647058824, "pass@8": 0.5, "pass@16": 0.5323529411764706, "pass@32": 0.5823529411764706, "pass@64": 0.6529411764705882, "pass@128": 0.7205882352941176 }, "19": { "pass@1": 0.2976190476190476, "pass@2": 0.3601190476190476, "pass@4": 0.39880952380952384, "pass@8": 0.4375, "pass@16": 0.4732142857142857, "pass@32": 0.5148809523809523, "pass@64": 0.5625, "pass@128": 0.6101190476190477 }, "20": { "pass@1": 0.19760479041916168, "pass@2": 0.24251497005988024, "pass@4": 0.2934131736526946, "pass@8": 0.32934131736526945, "pass@16": 0.3532934131736527, "pass@32": 0.3772455089820359, "pass@64": 0.4251497005988024, "pass@128": 0.46407185628742514 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6686746987951807, "12": 0.5987460815047022, "13": 0.6162464985994398, "14": 0.5672514619883041, "15": 0.5545454545454546, "16": 0.5156695156695157, "17": 0.43478260869565216, "18": 0.39705882352941174, "19": 0.2976190476190476, "20": 0.19760479041916168 }, "pass@2": { "11": 0.7228915662650602, "12": 0.658307210031348, "13": 0.6750700280112045, "14": 0.6023391812865497, "15": 0.5848484848484848, "16": 0.5384615384615384, "17": 0.45652173913043476, "18": 0.4323529411764706, "19": 0.3601190476190476, "20": 0.24251497005988024 }, "pass@4": { "11": 0.8042168674698795, "12": 0.7711598746081505, "13": 0.7675070028011205, "14": 0.6374269005847953, "15": 0.6242424242424243, "16": 0.5726495726495726, "17": 0.5186335403726708, "18": 0.45294117647058824, "19": 0.39880952380952384, "20": 0.2934131736526946 }, "pass@8": { "11": 0.8825301204819277, "12": 0.8369905956112853, "13": 0.8487394957983193, "14": 0.716374269005848, "15": 0.6818181818181818, "16": 0.6296296296296297, "17": 0.5403726708074534, "18": 0.5, "19": 0.4375, "20": 0.32934131736526945 }, "pass@16": { "11": 0.9216867469879518, "12": 0.9122257053291536, "13": 0.9131652661064426, "14": 0.7923976608187134, "15": 0.7545454545454545, "16": 0.6837606837606838, "17": 0.6180124223602484, "18": 0.5323529411764706, "19": 0.4732142857142857, "20": 0.3532934131736527 }, "pass@32": { "11": 0.9578313253012049, "12": 0.9467084639498433, "13": 0.938375350140056, "14": 0.8713450292397661, "15": 0.8121212121212121, "16": 0.7492877492877493, "17": 0.6770186335403726, "18": 0.5823529411764706, "19": 0.5148809523809523, "20": 0.3772455089820359 }, "pass@64": { "11": 0.9819277108433735, "12": 0.9717868338557993, "13": 0.9635854341736695, "14": 0.9385964912280702, "15": 0.8727272727272727, "16": 0.7977207977207977, "17": 0.7298136645962733, "18": 0.6529411764705882, "19": 0.5625, "20": 0.4251497005988024 }, "pass@128": { "11": 0.9879518072289156, "12": 0.9811912225705329, "13": 0.9831932773109243, "14": 0.9590643274853801, "15": 0.9272727272727272, "16": 0.8319088319088319, "17": 0.7888198757763976, "18": 0.7205882352941176, "19": 0.6101190476190477, "20": 0.46407185628742514 } }, "total_pass_at_k": { "pass@1": 0.4852809991079393, "pass@2": 0.527802557240559, "pass@4": 0.584299732381802, "pass@8": 0.6407969075230449, "pass@16": 0.695807314897413, "pass@32": 0.7430865298840321, "pass@64": 0.7900683913172762, "pass@128": 0.8257508177222718 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.626, "pass@2": 0.697, "pass@4": 0.795, "pass@8": 0.873, "pass@16": 0.924, "pass@32": 0.957, "pass@64": 0.98, "pass@128": 0.987 }, "12": { "pass@1": 0.628, "pass@2": 0.688, "pass@4": 0.781, "pass@8": 0.868, "pass@16": 0.925, "pass@32": 0.956, "pass@64": 0.972, "pass@128": 0.983 }, "13": { "pass@1": 0.598, "pass@2": 0.667, "pass@4": 0.753, "pass@8": 0.829, "pass@16": 0.889, "pass@32": 0.924, "pass@64": 0.954, "pass@128": 0.973 }, "14": { "pass@1": 0.566, "pass@2": 0.598, "pass@4": 0.644, "pass@8": 0.718, "pass@16": 0.784, "pass@32": 0.85, "pass@64": 0.918, "pass@128": 0.949 }, "15": { "pass@1": 0.52, "pass@2": 0.557, "pass@4": 0.6, "pass@8": 0.665, "pass@16": 0.744, "pass@32": 0.809, "pass@64": 0.887, "pass@128": 0.935 }, "16": { "pass@1": 0.517, "pass@2": 0.546, "pass@4": 0.58, "pass@8": 0.622, "pass@16": 0.682, "pass@32": 0.734, "pass@64": 0.777, "pass@128": 0.81 }, "17": { "pass@1": 0.481, "pass@2": 0.505, "pass@4": 0.546, "pass@8": 0.576, "pass@16": 0.638, "pass@32": 0.699, "pass@64": 0.76, "pass@128": 0.803 }, "18": { "pass@1": 0.408, "pass@2": 0.433, "pass@4": 0.463, "pass@8": 0.51, "pass@16": 0.549, "pass@32": 0.608, "pass@64": 0.682, "pass@128": 0.741 }, "19": { "pass@1": 0.301, "pass@2": 0.351, "pass@4": 0.392, "pass@8": 0.425, "pass@16": 0.463, "pass@32": 0.497, "pass@64": 0.548, "pass@128": 0.59 }, "20": { "pass@1": 0.202, "pass@2": 0.245, "pass@4": 0.286, "pass@8": 0.318, "pass@16": 0.343, "pass@32": 0.376, "pass@64": 0.426, "pass@128": 0.475 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.626, "12": 0.628, "13": 0.598, "14": 0.566, "15": 0.52, "16": 0.517, "17": 0.481, "18": 0.408, "19": 0.301, "20": 0.202 }, "pass@2": { "11": 0.697, "12": 0.688, "13": 0.667, "14": 0.598, "15": 0.557, "16": 0.546, "17": 0.505, "18": 0.433, "19": 0.351, "20": 0.245 }, "pass@4": { "11": 0.795, "12": 0.781, "13": 0.753, "14": 0.644, "15": 0.6, "16": 0.58, "17": 0.546, "18": 0.463, "19": 0.392, "20": 0.286 }, "pass@8": { "11": 0.873, "12": 0.868, "13": 0.829, "14": 0.718, "15": 0.665, "16": 0.622, "17": 0.576, "18": 0.51, "19": 0.425, "20": 0.318 }, "pass@16": { "11": 0.924, "12": 0.925, "13": 0.889, "14": 0.784, "15": 0.744, "16": 0.682, "17": 0.638, "18": 0.549, "19": 0.463, "20": 0.343 }, "pass@32": { "11": 0.957, "12": 0.956, "13": 0.924, "14": 0.85, "15": 0.809, "16": 0.734, "17": 0.699, "18": 0.608, "19": 0.497, "20": 0.376 }, "pass@64": { "11": 0.98, "12": 0.972, "13": 0.954, "14": 0.918, "15": 0.887, "16": 0.777, "17": 0.76, "18": 0.682, "19": 0.548, "20": 0.426 }, "pass@128": { "11": 0.987, "12": 0.983, "13": 0.973, "14": 0.949, "15": 0.935, "16": 0.81, "17": 0.803, "18": 0.741, "19": 0.59, "20": 0.475 } }, "total_pass_at_k": { "pass@1": 0.4847, "pass@2": 0.5287, "pass@4": 0.584, "pass@8": 0.6404, "pass@16": 0.6941, "pass@32": 0.741, "pass@64": 0.7904, "pass@128": 0.8246 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }