{ "files": { "checkpoint-39008_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 0.9968847352024922, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9969418960244648, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.990625, "pass@2": 0.99375, "pass@4": 0.996875, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9492063492063492, "pass@2": 0.9777777777777777, "pass@4": 0.9904761904761905, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9670658682634731, "pass@2": 0.9790419161676647, "pass@4": 0.9820359281437125, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8816568047337278, "pass@2": 0.9289940828402367, "pass@4": 0.9585798816568047, "pass@8": 0.9881656804733728, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8301282051282052, "pass@2": 0.9102564102564102, "pass@4": 0.9423076923076923, "pass@8": 0.9711538461538461, "pass@16": 0.9935897435897436, "pass@32": 0.9935897435897436, "pass@64": 0.9967948717948718, "pass@128": 1.0 }, "9": { "pass@1": 0.7492957746478873, "pass@2": 0.8394366197183099, "pass@4": 0.8929577464788733, "pass@8": 0.9492957746478873, "pass@16": 0.9859154929577465, "pass@32": 0.9915492957746479, "pass@64": 0.9971830985915493, "pass@128": 0.9971830985915493 }, "10": { "pass@1": 0.6194690265486725, "pass@2": 0.6991150442477876, "pass@4": 0.7876106194690266, "pass@8": 0.8466076696165191, "pass@16": 0.8908554572271387, "pass@32": 0.9233038348082596, "pass@64": 0.9380530973451328, "pass@128": 0.9528023598820059 } }, "per_op_pass_at_columns": { "pass@1": { "2": 0.9968847352024922, "3": 0.9969418960244648, "4": 0.990625, "5": 0.9492063492063492, "6": 0.9670658682634731, "7": 0.8816568047337278, "8": 0.8301282051282052, "9": 0.7492957746478873, "10": 0.6194690265486725 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.99375, "5": 0.9777777777777777, "6": 0.9790419161676647, "7": 0.9289940828402367, "8": 0.9102564102564102, "9": 0.8394366197183099, "10": 0.6991150442477876 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9904761904761905, "6": 0.9820359281437125, "7": 0.9585798816568047, "8": 0.9423076923076923, "9": 0.8929577464788733, "10": 0.7876106194690266 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9881656804733728, "8": 0.9711538461538461, "9": 0.9492957746478873, "10": 0.8466076696165191 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9970414201183432, "8": 0.9935897435897436, "9": 0.9859154929577465, "10": 0.8908554572271387 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9935897435897436, "9": 0.9915492957746479, "10": 0.9233038348082596 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9967948717948718, "9": 0.9971830985915493, "10": 0.9380530973451328 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9528023598820059 } }, "total_pass_at_k": { "pass@1": 0.8841607565011821, "pass@2": 0.9233367105707532, "pass@4": 0.9486659912191827, "pass@8": 0.971968929415738, "pass@16": 0.9848024316109423, "pass@32": 0.9895305639986491, "pass@64": 0.9922323539344816, "pass@128": 0.994258696386356 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9970845481049563, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9883040935672515, "pass@2": 0.9970760233918129, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9470588235294117, "pass@2": 0.961764705882353, "pass@4": 0.9882352941176471, "pass@8": 0.9970588235294118, "pass@16": 0.9970588235294118, "pass@32": 0.9970588235294118, "pass@64": 0.9970588235294118, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9716088328075709, "pass@2": 0.9747634069400631, "pass@4": 0.9905362776025236, "pass@8": 0.9968454258675079, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8386075949367089, "pass@2": 0.8955696202531646, "pass@4": 0.9525316455696202, "pass@8": 0.9841772151898734, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8048048048048048, "pass@2": 0.8828828828828829, "pass@4": 0.9279279279279279, "pass@8": 0.960960960960961, "pass@16": 0.987987987987988, "pass@32": 0.996996996996997, "pass@64": 0.996996996996997, "pass@128": 0.996996996996997 }, "9": { "pass@1": 0.7845659163987139, "pass@2": 0.8585209003215434, "pass@4": 0.9196141479099679, "pass@8": 0.9485530546623794, "pass@16": 0.9807073954983923, "pass@32": 0.9839228295819936, "pass@64": 0.9935691318327974, "pass@128": 0.9935691318327974 }, "10": { "pass@1": 0.5880597014925373, "pass@2": 0.6686567164179105, "pass@4": 0.7343283582089553, "pass@8": 0.8388059701492537, "pass@16": 0.9134328358208955, "pass@32": 0.9402985074626866, "pass@64": 0.9641791044776119, "pass@128": 0.9731343283582089 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 0.9970845481049563, "4": 0.9883040935672515, "5": 0.9470588235294117, "6": 0.9716088328075709, "7": 0.8386075949367089, "8": 0.8048048048048048, "9": 0.7845659163987139, "10": 0.5880597014925373 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.961764705882353, "6": 0.9747634069400631, "7": 0.8955696202531646, "8": 0.8828828828828829, "9": 0.8585209003215434, "10": 0.6686567164179105 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9882352941176471, "6": 0.9905362776025236, "7": 0.9525316455696202, "8": 0.9279279279279279, "9": 0.9196141479099679, "10": 0.7343283582089553 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 0.9968454258675079, "7": 0.9841772151898734, "8": 0.960960960960961, "9": 0.9485530546623794, "10": 0.8388059701492537 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 0.987987987987988, "9": 0.9807073954983923, "10": 0.9134328358208955 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 0.996996996996997, "9": 0.9839228295819936, "10": 0.9402985074626866 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 0.996996996996997, "9": 0.9935691318327974, "10": 0.9641791044776119 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 0.996996996996997, "9": 0.9935691318327974, "10": 0.9731343283582089 } }, "total_pass_at_k": { "pass@1": 0.8818212253096752, "pass@2": 0.9166387679946435, "pass@4": 0.9464345497154335, "pass@8": 0.9698694342149313, "pass@16": 0.9866086374288584, "pass@32": 0.9909608302644795, "pass@64": 0.9946434549715434, "pass@128": 0.995647807164379 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.996969696969697, "pass@2": 0.996969696969697, "pass@4": 0.996969696969697, "pass@8": 0.996969696969697, "pass@16": 0.996969696969697, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9822485207100592, "pass@2": 0.9881656804733728, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9391304347826087, "pass@2": 0.9681159420289855, "pass@4": 0.9884057971014493, "pass@8": 0.9971014492753624, "pass@16": 0.9971014492753624, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9656160458452722, "pass@2": 0.9713467048710601, "pass@4": 0.9828080229226361, "pass@8": 0.9914040114613181, "pass@16": 0.997134670487106, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8757225433526011, "pass@2": 0.9393063583815029, "pass@4": 0.9682080924855492, "pass@8": 0.9884393063583815, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.7971830985915493, "pass@2": 0.8647887323943662, "pass@4": 0.9380281690140845, "pass@8": 0.9661971830985916, "pass@16": 0.9943661971830986, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7365269461077845, "pass@2": 0.7934131736526946, "pass@4": 0.8772455089820359, "pass@8": 0.9341317365269461, "pass@16": 0.9730538922155688, "pass@32": 0.9970059880239521, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6625766871165644, "pass@2": 0.7116564417177914, "pass@4": 0.7852760736196319, "pass@8": 0.8282208588957055, "pass@16": 0.8773006134969326, "pass@32": 0.9233128834355828, "pass@64": 0.9478527607361963, "pass@128": 0.950920245398773 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 0.996969696969697, "4": 0.9822485207100592, "5": 0.9391304347826087, "6": 0.9656160458452722, "7": 0.8757225433526011, "8": 0.7971830985915493, "9": 0.7365269461077845, "10": 0.6625766871165644 }, "pass@2": { "2": 1.0, "3": 0.996969696969697, "4": 0.9881656804733728, "5": 0.9681159420289855, "6": 0.9713467048710601, "7": 0.9393063583815029, "8": 0.8647887323943662, "9": 0.7934131736526946, "10": 0.7116564417177914 }, "pass@4": { "2": 1.0, "3": 0.996969696969697, "4": 1.0, "5": 0.9884057971014493, "6": 0.9828080229226361, "7": 0.9682080924855492, "8": 0.9380281690140845, "9": 0.8772455089820359, "10": 0.7852760736196319 }, "pass@8": { "2": 1.0, "3": 0.996969696969697, "4": 1.0, "5": 0.9971014492753624, "6": 0.9914040114613181, "7": 0.9884393063583815, "8": 0.9661971830985916, "9": 0.9341317365269461, "10": 0.8282208588957055 }, "pass@16": { "2": 1.0, "3": 0.996969696969697, "4": 1.0, "5": 0.9971014492753624, "6": 0.997134670487106, "7": 1.0, "8": 0.9943661971830986, "9": 0.9730538922155688, "10": 0.8773006134969326 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.9233128834355828 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9478527607361963 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.950920245398773 } }, "total_pass_at_k": { "pass@1": 0.8843381389252949, "pass@2": 0.9154652686762779, "pass@4": 0.9492136304062909, "pass@8": 0.9675622542595019, "pass@16": 0.9823066841415465, "pass@32": 0.991480996068152, "pass@64": 0.994429882044561, "pass@128": 0.9947575360419397 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 0.999, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.997, "pass@2": 0.999, "pass@4": 0.999, "pass@8": 0.999, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.987, "pass@2": 0.993, "pass@4": 0.999, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.945, "pass@2": 0.969, "pass@4": 0.989, "pass@8": 0.998, "pass@16": 0.998, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 0.999 }, "6": { "pass@1": 0.968, "pass@2": 0.975, "pass@4": 0.985, "pass@8": 0.996, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.866, "pass@2": 0.922, "pass@4": 0.96, "pass@8": 0.987, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.81, "pass@2": 0.885, "pass@4": 0.936, "pass@8": 0.966, "pass@16": 0.992, "pass@32": 0.997, "pass@64": 0.998, "pass@128": 0.999 }, "9": { "pass@1": 0.756, "pass@2": 0.83, "pass@4": 0.896, "pass@8": 0.944, "pass@16": 0.98, "pass@32": 0.991, "pass@64": 0.997, "pass@128": 0.997 }, "10": { "pass@1": 0.623, "pass@2": 0.693, "pass@4": 0.769, "pass@8": 0.838, "pass@16": 0.894, "pass@32": 0.929, "pass@64": 0.95, "pass@128": 0.959 } }, "per_op_pass_at_columns": { "pass@1": { "2": 0.999, "3": 0.997, "4": 0.987, "5": 0.945, "6": 0.968, "7": 0.866, "8": 0.81, "9": 0.756, "10": 0.623 }, "pass@2": { "2": 1.0, "3": 0.999, "4": 0.993, "5": 0.969, "6": 0.975, "7": 0.922, "8": 0.885, "9": 0.83, "10": 0.693 }, "pass@4": { "2": 1.0, "3": 0.999, "4": 0.999, "5": 0.989, "6": 0.985, "7": 0.96, "8": 0.936, "9": 0.896, "10": 0.769 }, "pass@8": { "2": 1.0, "3": 0.999, "4": 1.0, "5": 0.998, "6": 0.996, "7": 0.987, "8": 0.966, "9": 0.944, "10": 0.838 }, "pass@16": { "2": 1.0, "3": 0.999, "4": 1.0, "5": 0.998, "6": 0.999, "7": 0.999, "8": 0.992, "9": 0.98, "10": 0.894 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.997, "9": 0.991, "10": 0.929 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.998, "9": 0.997, "10": 0.95 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.999, "9": 0.997, "10": 0.959 } }, "total_pass_at_k": { "pass@1": 0.8834444444444445, "pass@2": 0.9184444444444444, "pass@4": 0.9481111111111111, "pass@8": 0.9697777777777777, "pass@16": 0.9845555555555555, "pass@32": 0.9906666666666667, "pass@64": 0.9937777777777778, "pass@128": 0.9948888888888889 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-39008_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.56657223796034, "pass@2": 0.6487252124645893, "pass@4": 0.7648725212464589, "pass@8": 0.8441926345609065, "pass@16": 0.9065155807365439, "pass@32": 0.9376770538243626, "pass@64": 0.9660056657223796, "pass@128": 0.9801699716713881 }, "12": { "pass@1": 0.6271186440677966, "pass@2": 0.6977401129943502, "pass@4": 0.751412429378531, "pass@8": 0.8418079096045198, "pass@16": 0.9067796610169492, "pass@32": 0.9350282485875706, "pass@64": 0.963276836158192, "pass@128": 0.9774011299435028 }, "13": { "pass@1": 0.5768025078369906, "pass@2": 0.6238244514106583, "pass@4": 0.7021943573667712, "pass@8": 0.8056426332288401, "pass@16": 0.8620689655172413, "pass@32": 0.890282131661442, "pass@64": 0.9216300940438872, "pass@128": 0.9498432601880877 }, "14": { "pass@1": 0.5264797507788161, "pass@2": 0.5763239875389408, "pass@4": 0.6199376947040498, "pass@8": 0.6915887850467289, "pass@16": 0.7538940809968847, "pass@32": 0.8286604361370716, "pass@64": 0.8785046728971962, "pass@128": 0.9065420560747663 }, "15": { "pass@1": 0.5028901734104047, "pass@2": 0.5549132947976878, "pass@4": 0.5924855491329479, "pass@8": 0.653179190751445, "pass@16": 0.7196531791907514, "pass@32": 0.815028901734104, "pass@64": 0.8901734104046243, "pass@128": 0.9508670520231214 }, "16": { "pass@1": 0.4645161290322581, "pass@2": 0.5193548387096775, "pass@4": 0.5451612903225806, "pass@8": 0.5903225806451613, "pass@16": 0.6451612903225806, "pass@32": 0.7225806451612903, "pass@64": 0.7741935483870968, "pass@128": 0.7935483870967742 }, "17": { "pass@1": 0.5, "pass@2": 0.5404624277456648, "pass@4": 0.5838150289017341, "pass@8": 0.6213872832369942, "pass@16": 0.6936416184971098, "pass@32": 0.7485549132947977, "pass@64": 0.8092485549132948, "pass@128": 0.8554913294797688 }, "18": { "pass@1": 0.44, "pass@2": 0.4942857142857143, "pass@4": 0.5114285714285715, "pass@8": 0.56, "pass@16": 0.6171428571428571, "pass@32": 0.6657142857142857, "pass@64": 0.7371428571428571, "pass@128": 0.7885714285714286 }, "19": { "pass@1": 0.37237237237237236, "pass@2": 0.4264264264264264, "pass@4": 0.46846846846846846, "pass@8": 0.5105105105105106, "pass@16": 0.5375375375375375, "pass@32": 0.5705705705705706, "pass@64": 0.6186186186186187, "pass@128": 0.6846846846846847 }, "20": { "pass@1": 0.36936936936936937, "pass@2": 0.42042042042042044, "pass@4": 0.4444444444444444, "pass@8": 0.4774774774774775, "pass@16": 0.5255255255255256, "pass@32": 0.5825825825825826, "pass@64": 0.6516516516516516, "pass@128": 0.6996996996996997 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.56657223796034, "12": 0.6271186440677966, "13": 0.5768025078369906, "14": 0.5264797507788161, "15": 0.5028901734104047, "16": 0.4645161290322581, "17": 0.5, "18": 0.44, "19": 0.37237237237237236, "20": 0.36936936936936937 }, "pass@2": { "11": 0.6487252124645893, "12": 0.6977401129943502, "13": 0.6238244514106583, "14": 0.5763239875389408, "15": 0.5549132947976878, "16": 0.5193548387096775, "17": 0.5404624277456648, "18": 0.4942857142857143, "19": 0.4264264264264264, "20": 0.42042042042042044 }, "pass@4": { "11": 0.7648725212464589, "12": 0.751412429378531, "13": 0.7021943573667712, "14": 0.6199376947040498, "15": 0.5924855491329479, "16": 0.5451612903225806, "17": 0.5838150289017341, "18": 0.5114285714285715, "19": 0.46846846846846846, "20": 0.4444444444444444 }, "pass@8": { "11": 0.8441926345609065, "12": 0.8418079096045198, "13": 0.8056426332288401, "14": 0.6915887850467289, "15": 0.653179190751445, "16": 0.5903225806451613, "17": 0.6213872832369942, "18": 0.56, "19": 0.5105105105105106, "20": 0.4774774774774775 }, "pass@16": { "11": 0.9065155807365439, "12": 0.9067796610169492, "13": 0.8620689655172413, "14": 0.7538940809968847, "15": 0.7196531791907514, "16": 0.6451612903225806, "17": 0.6936416184971098, "18": 0.6171428571428571, "19": 0.5375375375375375, "20": 0.5255255255255256 }, "pass@32": { "11": 0.9376770538243626, "12": 0.9350282485875706, "13": 0.890282131661442, "14": 0.8286604361370716, "15": 0.815028901734104, "16": 0.7225806451612903, "17": 0.7485549132947977, "18": 0.6657142857142857, "19": 0.5705705705705706, "20": 0.5825825825825826 }, "pass@64": { "11": 0.9660056657223796, "12": 0.963276836158192, "13": 0.9216300940438872, "14": 0.8785046728971962, "15": 0.8901734104046243, "16": 0.7741935483870968, "17": 0.8092485549132948, "18": 0.7371428571428571, "19": 0.6186186186186187, "20": 0.6516516516516516 }, "pass@128": { "11": 0.9801699716713881, "12": 0.9774011299435028, "13": 0.9498432601880877, "14": 0.9065420560747663, "15": 0.9508670520231214, "16": 0.7935483870967742, "17": 0.8554913294797688, "18": 0.7885714285714286, "19": 0.6846846846846847, "20": 0.6996996996996997 } }, "total_pass_at_k": { "pass@1": 0.49539375928677565, "pass@2": 0.5512630014858841, "pass@4": 0.599702823179792, "pass@8": 0.6609212481426449, "pass@16": 0.7182763744427935, "pass@32": 0.7708766716196137, "pass@64": 0.8222882615156017, "pass@128": 0.8600297176820209 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.6126984126984127, "pass@2": 0.6888888888888889, "pass@4": 0.7746031746031746, "pass@8": 0.8476190476190476, "pass@16": 0.8984126984126984, "pass@32": 0.946031746031746, "pass@64": 0.9809523809523809, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.6330275229357798, "pass@2": 0.7155963302752294, "pass@4": 0.7737003058103975, "pass@8": 0.8654434250764526, "pass@16": 0.9051987767584098, "pass@32": 0.9541284403669725, "pass@64": 0.9755351681957186, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.5864197530864198, "pass@2": 0.6203703703703703, "pass@4": 0.7067901234567902, "pass@8": 0.8055555555555556, "pass@16": 0.8549382716049383, "pass@32": 0.9012345679012346, "pass@64": 0.9382716049382716, "pass@128": 0.9537037037037037 }, "14": { "pass@1": 0.5459940652818991, "pass@2": 0.5786350148367952, "pass@4": 0.6112759643916914, "pass@8": 0.6646884272997032, "pass@16": 0.7418397626112759, "pass@32": 0.8456973293768546, "pass@64": 0.9020771513353115, "pass@128": 0.9347181008902077 }, "15": { "pass@1": 0.4783950617283951, "pass@2": 0.5277777777777778, "pass@4": 0.5679012345679012, "pass@8": 0.6172839506172839, "pass@16": 0.691358024691358, "pass@32": 0.7870370370370371, "pass@64": 0.8888888888888888, "pass@128": 0.941358024691358 }, "16": { "pass@1": 0.5103244837758112, "pass@2": 0.5663716814159292, "pass@4": 0.5958702064896755, "pass@8": 0.6637168141592921, "pass@16": 0.7109144542772862, "pass@32": 0.7551622418879056, "pass@64": 0.8082595870206489, "pass@128": 0.8436578171091446 }, "17": { "pass@1": 0.5030120481927711, "pass@2": 0.5421686746987951, "pass@4": 0.5933734939759037, "pass@8": 0.6295180722891566, "pass@16": 0.6656626506024096, "pass@32": 0.7018072289156626, "pass@64": 0.7560240963855421, "pass@128": 0.8102409638554217 }, "18": { "pass@1": 0.4806451612903226, "pass@2": 0.5193548387096775, "pass@4": 0.5387096774193548, "pass@8": 0.6064516129032258, "pass@16": 0.6548387096774193, "pass@32": 0.6838709677419355, "pass@64": 0.7677419354838709, "pass@128": 0.8354838709677419 }, "19": { "pass@1": 0.38368580060422963, "pass@2": 0.43202416918429004, "pass@4": 0.48036253776435045, "pass@8": 0.513595166163142, "pass@16": 0.5558912386706949, "pass@32": 0.6042296072507553, "pass@64": 0.6646525679758308, "pass@128": 0.7009063444108762 }, "20": { "pass@1": 0.34234234234234234, "pass@2": 0.3933933933933934, "pass@4": 0.4264264264264264, "pass@8": 0.45045045045045046, "pass@16": 0.4924924924924925, "pass@32": 0.5465465465465466, "pass@64": 0.6036036036036037, "pass@128": 0.6636636636636637 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6126984126984127, "12": 0.6330275229357798, "13": 0.5864197530864198, "14": 0.5459940652818991, "15": 0.4783950617283951, "16": 0.5103244837758112, "17": 0.5030120481927711, "18": 0.4806451612903226, "19": 0.38368580060422963, "20": 0.34234234234234234 }, "pass@2": { "11": 0.6888888888888889, "12": 0.7155963302752294, "13": 0.6203703703703703, "14": 0.5786350148367952, "15": 0.5277777777777778, "16": 0.5663716814159292, "17": 0.5421686746987951, "18": 0.5193548387096775, "19": 0.43202416918429004, "20": 0.3933933933933934 }, "pass@4": { "11": 0.7746031746031746, "12": 0.7737003058103975, "13": 0.7067901234567902, "14": 0.6112759643916914, "15": 0.5679012345679012, "16": 0.5958702064896755, "17": 0.5933734939759037, "18": 0.5387096774193548, "19": 0.48036253776435045, "20": 0.4264264264264264 }, "pass@8": { "11": 0.8476190476190476, "12": 0.8654434250764526, "13": 0.8055555555555556, "14": 0.6646884272997032, "15": 0.6172839506172839, "16": 0.6637168141592921, "17": 0.6295180722891566, "18": 0.6064516129032258, "19": 0.513595166163142, "20": 0.45045045045045046 }, "pass@16": { "11": 0.8984126984126984, "12": 0.9051987767584098, "13": 0.8549382716049383, "14": 0.7418397626112759, "15": 0.691358024691358, "16": 0.7109144542772862, "17": 0.6656626506024096, "18": 0.6548387096774193, "19": 0.5558912386706949, "20": 0.4924924924924925 }, "pass@32": { "11": 0.946031746031746, "12": 0.9541284403669725, "13": 0.9012345679012346, "14": 0.8456973293768546, "15": 0.7870370370370371, "16": 0.7551622418879056, "17": 0.7018072289156626, "18": 0.6838709677419355, "19": 0.6042296072507553, "20": 0.5465465465465466 }, "pass@64": { "11": 0.9809523809523809, "12": 0.9755351681957186, "13": 0.9382716049382716, "14": 0.9020771513353115, "15": 0.8888888888888888, "16": 0.8082595870206489, "17": 0.7560240963855421, "18": 0.7677419354838709, "19": 0.6646525679758308, "20": 0.6036036036036037 }, "pass@128": { "11": 0.9904761904761905, "12": 0.981651376146789, "13": 0.9537037037037037, "14": 0.9347181008902077, "15": 0.941358024691358, "16": 0.8436578171091446, "17": 0.8102409638554217, "18": 0.8354838709677419, "19": 0.7009063444108762, "20": 0.6636636636636637 } }, "total_pass_at_k": { "pass@1": 0.5070293398533008, "pass@2": 0.5577628361858191, "pass@4": 0.6060513447432763, "pass@8": 0.6653422982885085, "pass@16": 0.7160757946210269, "pass@32": 0.7716992665036675, "pass@64": 0.8276283618581907, "pass@128": 0.8646088019559902 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6596385542168675, "pass@2": 0.713855421686747, "pass@4": 0.7981927710843374, "pass@8": 0.8554216867469879, "pass@16": 0.9126506024096386, "pass@32": 0.9457831325301205, "pass@64": 0.9668674698795181, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.5956112852664577, "pass@2": 0.6520376175548589, "pass@4": 0.7304075235109718, "pass@8": 0.8432601880877743, "pass@16": 0.9122257053291536, "pass@32": 0.9373040752351097, "pass@64": 0.9529780564263323, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.6078431372549019, "pass@2": 0.6470588235294118, "pass@4": 0.7563025210084033, "pass@8": 0.8291316526610645, "pass@16": 0.8935574229691877, "pass@32": 0.9243697478991597, "pass@64": 0.9523809523809523, "pass@128": 0.969187675070028 }, "14": { "pass@1": 0.5584795321637427, "pass@2": 0.5964912280701754, "pass@4": 0.652046783625731, "pass@8": 0.7309941520467836, "pass@16": 0.8187134502923976, "pass@32": 0.8888888888888888, "pass@64": 0.935672514619883, "pass@128": 0.956140350877193 }, "15": { "pass@1": 0.5393939393939394, "pass@2": 0.5696969696969697, "pass@4": 0.6212121212121212, "pass@8": 0.6787878787878788, "pass@16": 0.7606060606060606, "pass@32": 0.8333333333333334, "pass@64": 0.8878787878787879, "pass@128": 0.9333333333333333 }, "16": { "pass@1": 0.5213675213675214, "pass@2": 0.5555555555555556, "pass@4": 0.5811965811965812, "pass@8": 0.6381766381766382, "pass@16": 0.6837606837606838, "pass@32": 0.7435897435897436, "pass@64": 0.8176638176638177, "pass@128": 0.8547008547008547 }, "17": { "pass@1": 0.42857142857142855, "pass@2": 0.468944099378882, "pass@4": 0.5186335403726708, "pass@8": 0.5714285714285714, "pass@16": 0.6366459627329193, "pass@32": 0.7204968944099379, "pass@64": 0.7950310559006211, "pass@128": 0.8385093167701864 }, "18": { "pass@1": 0.4294117647058823, "pass@2": 0.4676470588235294, "pass@4": 0.4852941176470588, "pass@8": 0.5176470588235295, "pass@16": 0.5764705882352941, "pass@32": 0.6617647058823529, "pass@64": 0.7382352941176471, "pass@128": 0.8058823529411765 }, "19": { "pass@1": 0.40773809523809523, "pass@2": 0.4523809523809524, "pass@4": 0.4851190476190476, "pass@8": 0.5178571428571429, "pass@16": 0.5476190476190477, "pass@32": 0.5952380952380952, "pass@64": 0.6488095238095238, "pass@128": 0.6845238095238095 }, "20": { "pass@1": 0.3712574850299401, "pass@2": 0.4041916167664671, "pass@4": 0.45209580838323354, "pass@8": 0.4820359281437126, "pass@16": 0.5239520958083832, "pass@32": 0.5748502994011976, "pass@64": 0.5898203592814372, "pass@128": 0.6287425149700598 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6596385542168675, "12": 0.5956112852664577, "13": 0.6078431372549019, "14": 0.5584795321637427, "15": 0.5393939393939394, "16": 0.5213675213675214, "17": 0.42857142857142855, "18": 0.4294117647058823, "19": 0.40773809523809523, "20": 0.3712574850299401 }, "pass@2": { "11": 0.713855421686747, "12": 0.6520376175548589, "13": 0.6470588235294118, "14": 0.5964912280701754, "15": 0.5696969696969697, "16": 0.5555555555555556, "17": 0.468944099378882, "18": 0.4676470588235294, "19": 0.4523809523809524, "20": 0.4041916167664671 }, "pass@4": { "11": 0.7981927710843374, "12": 0.7304075235109718, "13": 0.7563025210084033, "14": 0.652046783625731, "15": 0.6212121212121212, "16": 0.5811965811965812, "17": 0.5186335403726708, "18": 0.4852941176470588, "19": 0.4851190476190476, "20": 0.45209580838323354 }, "pass@8": { "11": 0.8554216867469879, "12": 0.8432601880877743, "13": 0.8291316526610645, "14": 0.7309941520467836, "15": 0.6787878787878788, "16": 0.6381766381766382, "17": 0.5714285714285714, "18": 0.5176470588235295, "19": 0.5178571428571429, "20": 0.4820359281437126 }, "pass@16": { "11": 0.9126506024096386, "12": 0.9122257053291536, "13": 0.8935574229691877, "14": 0.8187134502923976, "15": 0.7606060606060606, "16": 0.6837606837606838, "17": 0.6366459627329193, "18": 0.5764705882352941, "19": 0.5476190476190477, "20": 0.5239520958083832 }, "pass@32": { "11": 0.9457831325301205, "12": 0.9373040752351097, "13": 0.9243697478991597, "14": 0.8888888888888888, "15": 0.8333333333333334, "16": 0.7435897435897436, "17": 0.7204968944099379, "18": 0.6617647058823529, "19": 0.5952380952380952, "20": 0.5748502994011976 }, "pass@64": { "11": 0.9668674698795181, "12": 0.9529780564263323, "13": 0.9523809523809523, "14": 0.935672514619883, "15": 0.8878787878787879, "16": 0.8176638176638177, "17": 0.7950310559006211, "18": 0.7382352941176471, "19": 0.6488095238095238, "20": 0.5898203592814372 }, "pass@128": { "11": 0.9789156626506024, "12": 0.9655172413793104, "13": 0.969187675070028, "14": 0.956140350877193, "15": 0.9333333333333333, "16": 0.8547008547008547, "17": 0.8385093167701864, "18": 0.8058823529411765, "19": 0.6845238095238095, "20": 0.6287425149700598 } }, "total_pass_at_k": { "pass@1": 0.5123401724650609, "pass@2": 0.5530776092774309, "pass@4": 0.6083853702051739, "pass@8": 0.6666666666666666, "pass@16": 0.7267320844484092, "pass@32": 0.7826345524829021, "pass@64": 0.8287243532560215, "pass@128": 0.8617305976806423 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.612, "pass@2": 0.683, "pass@4": 0.779, "pass@8": 0.849, "pass@16": 0.906, "pass@32": 0.943, "pass@64": 0.971, "pass@128": 0.983 }, "12": { "pass@1": 0.619, "pass@2": 0.689, "pass@4": 0.752, "pass@8": 0.85, "pass@16": 0.908, "pass@32": 0.942, "pass@64": 0.964, "pass@128": 0.975 }, "13": { "pass@1": 0.591, "pass@2": 0.631, "pass@4": 0.723, "pass@8": 0.814, "pass@16": 0.871, "pass@32": 0.906, "pass@64": 0.938, "pass@128": 0.958 }, "14": { "pass@1": 0.544, "pass@2": 0.584, "pass@4": 0.628, "pass@8": 0.696, "pass@16": 0.772, "pass@32": 0.855, "pass@64": 0.906, "pass@128": 0.933 }, "15": { "pass@1": 0.507, "pass@2": 0.551, "pass@4": 0.594, "pass@8": 0.65, "pass@16": 0.724, "pass@32": 0.812, "pass@64": 0.889, "pass@128": 0.942 }, "16": { "pass@1": 0.5, "pass@2": 0.548, "pass@4": 0.575, "pass@8": 0.632, "pass@16": 0.681, "pass@32": 0.741, "pass@64": 0.801, "pass@128": 0.832 }, "17": { "pass@1": 0.478, "pass@2": 0.518, "pass@4": 0.566, "pass@8": 0.608, "pass@16": 0.666, "pass@32": 0.724, "pass@64": 0.787, "pass@128": 0.835 }, "18": { "pass@1": 0.449, "pass@2": 0.493, "pass@4": 0.511, "pass@8": 0.56, "pass@16": 0.615, "pass@32": 0.67, "pass@64": 0.747, "pass@128": 0.809 }, "19": { "pass@1": 0.388, "pass@2": 0.437, "pass@4": 0.478, "pass@8": 0.514, "pass@16": 0.547, "pass@32": 0.59, "pass@64": 0.644, "pass@128": 0.69 }, "20": { "pass@1": 0.361, "pass@2": 0.406, "pass@4": 0.441, "pass@8": 0.47, "pass@16": 0.514, "pass@32": 0.568, "pass@64": 0.615, "pass@128": 0.664 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.612, "12": 0.619, "13": 0.591, "14": 0.544, "15": 0.507, "16": 0.5, "17": 0.478, "18": 0.449, "19": 0.388, "20": 0.361 }, "pass@2": { "11": 0.683, "12": 0.689, "13": 0.631, "14": 0.584, "15": 0.551, "16": 0.548, "17": 0.518, "18": 0.493, "19": 0.437, "20": 0.406 }, "pass@4": { "11": 0.779, "12": 0.752, "13": 0.723, "14": 0.628, "15": 0.594, "16": 0.575, "17": 0.566, "18": 0.511, "19": 0.478, "20": 0.441 }, "pass@8": { "11": 0.849, "12": 0.85, "13": 0.814, "14": 0.696, "15": 0.65, "16": 0.632, "17": 0.608, "18": 0.56, "19": 0.514, "20": 0.47 }, "pass@16": { "11": 0.906, "12": 0.908, "13": 0.871, "14": 0.772, "15": 0.724, "16": 0.681, "17": 0.666, "18": 0.615, "19": 0.547, "20": 0.514 }, "pass@32": { "11": 0.943, "12": 0.942, "13": 0.906, "14": 0.855, "15": 0.812, "16": 0.741, "17": 0.724, "18": 0.67, "19": 0.59, "20": 0.568 }, "pass@64": { "11": 0.971, "12": 0.964, "13": 0.938, "14": 0.906, "15": 0.889, "16": 0.801, "17": 0.787, "18": 0.747, "19": 0.644, "20": 0.615 }, "pass@128": { "11": 0.983, "12": 0.975, "13": 0.958, "14": 0.933, "15": 0.942, "16": 0.832, "17": 0.835, "18": 0.809, "19": 0.69, "20": 0.664 } }, "total_pass_at_k": { "pass@1": 0.5049, "pass@2": 0.554, "pass@4": 0.6047, "pass@8": 0.6643, "pass@16": 0.7204, "pass@32": 0.7751, "pass@64": 0.8262, "pass@128": 0.8621 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }