| { |
| "files": { |
| "checkpoint-39008_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 0.9968847352024922, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9969418960244648, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.990625, |
| "pass@2": 0.99375, |
| "pass@4": 0.996875, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9492063492063492, |
| "pass@2": 0.9777777777777777, |
| "pass@4": 0.9904761904761905, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9670658682634731, |
| "pass@2": 0.9790419161676647, |
| "pass@4": 0.9820359281437125, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8816568047337278, |
| "pass@2": 0.9289940828402367, |
| "pass@4": 0.9585798816568047, |
| "pass@8": 0.9881656804733728, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8301282051282052, |
| "pass@2": 0.9102564102564102, |
| "pass@4": 0.9423076923076923, |
| "pass@8": 0.9711538461538461, |
| "pass@16": 0.9935897435897436, |
| "pass@32": 0.9935897435897436, |
| "pass@64": 0.9967948717948718, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7492957746478873, |
| "pass@2": 0.8394366197183099, |
| "pass@4": 0.8929577464788733, |
| "pass@8": 0.9492957746478873, |
| "pass@16": 0.9859154929577465, |
| "pass@32": 0.9915492957746479, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 0.9971830985915493 |
| }, |
| "10": { |
| "pass@1": 0.6194690265486725, |
| "pass@2": 0.6991150442477876, |
| "pass@4": 0.7876106194690266, |
| "pass@8": 0.8466076696165191, |
| "pass@16": 0.8908554572271387, |
| "pass@32": 0.9233038348082596, |
| "pass@64": 0.9380530973451328, |
| "pass@128": 0.9528023598820059 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 0.9968847352024922, |
| "3": 0.9969418960244648, |
| "4": 0.990625, |
| "5": 0.9492063492063492, |
| "6": 0.9670658682634731, |
| "7": 0.8816568047337278, |
| "8": 0.8301282051282052, |
| "9": 0.7492957746478873, |
| "10": 0.6194690265486725 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9777777777777777, |
| "6": 0.9790419161676647, |
| "7": 0.9289940828402367, |
| "8": 0.9102564102564102, |
| "9": 0.8394366197183099, |
| "10": 0.6991150442477876 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9904761904761905, |
| "6": 0.9820359281437125, |
| "7": 0.9585798816568047, |
| "8": 0.9423076923076923, |
| "9": 0.8929577464788733, |
| "10": 0.7876106194690266 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9881656804733728, |
| "8": 0.9711538461538461, |
| "9": 0.9492957746478873, |
| "10": 0.8466076696165191 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 0.9935897435897436, |
| "9": 0.9859154929577465, |
| "10": 0.8908554572271387 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9935897435897436, |
| "9": 0.9915492957746479, |
| "10": 0.9233038348082596 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9967948717948718, |
| "9": 0.9971830985915493, |
| "10": 0.9380530973451328 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9528023598820059 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8841607565011821, |
| "pass@2": 0.9233367105707532, |
| "pass@4": 0.9486659912191827, |
| "pass@8": 0.971968929415738, |
| "pass@16": 0.9848024316109423, |
| "pass@32": 0.9895305639986491, |
| "pass@64": 0.9922323539344816, |
| "pass@128": 0.994258696386356 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9970845481049563, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9883040935672515, |
| "pass@2": 0.9970760233918129, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9470588235294117, |
| "pass@2": 0.961764705882353, |
| "pass@4": 0.9882352941176471, |
| "pass@8": 0.9970588235294118, |
| "pass@16": 0.9970588235294118, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 0.9970588235294118, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9716088328075709, |
| "pass@2": 0.9747634069400631, |
| "pass@4": 0.9905362776025236, |
| "pass@8": 0.9968454258675079, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8386075949367089, |
| "pass@2": 0.8955696202531646, |
| "pass@4": 0.9525316455696202, |
| "pass@8": 0.9841772151898734, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8048048048048048, |
| "pass@2": 0.8828828828828829, |
| "pass@4": 0.9279279279279279, |
| "pass@8": 0.960960960960961, |
| "pass@16": 0.987987987987988, |
| "pass@32": 0.996996996996997, |
| "pass@64": 0.996996996996997, |
| "pass@128": 0.996996996996997 |
| }, |
| "9": { |
| "pass@1": 0.7845659163987139, |
| "pass@2": 0.8585209003215434, |
| "pass@4": 0.9196141479099679, |
| "pass@8": 0.9485530546623794, |
| "pass@16": 0.9807073954983923, |
| "pass@32": 0.9839228295819936, |
| "pass@64": 0.9935691318327974, |
| "pass@128": 0.9935691318327974 |
| }, |
| "10": { |
| "pass@1": 0.5880597014925373, |
| "pass@2": 0.6686567164179105, |
| "pass@4": 0.7343283582089553, |
| "pass@8": 0.8388059701492537, |
| "pass@16": 0.9134328358208955, |
| "pass@32": 0.9402985074626866, |
| "pass@64": 0.9641791044776119, |
| "pass@128": 0.9731343283582089 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.9970845481049563, |
| "4": 0.9883040935672515, |
| "5": 0.9470588235294117, |
| "6": 0.9716088328075709, |
| "7": 0.8386075949367089, |
| "8": 0.8048048048048048, |
| "9": 0.7845659163987139, |
| "10": 0.5880597014925373 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.961764705882353, |
| "6": 0.9747634069400631, |
| "7": 0.8955696202531646, |
| "8": 0.8828828828828829, |
| "9": 0.8585209003215434, |
| "10": 0.6686567164179105 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9882352941176471, |
| "6": 0.9905362776025236, |
| "7": 0.9525316455696202, |
| "8": 0.9279279279279279, |
| "9": 0.9196141479099679, |
| "10": 0.7343283582089553 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 0.9968454258675079, |
| "7": 0.9841772151898734, |
| "8": 0.960960960960961, |
| "9": 0.9485530546623794, |
| "10": 0.8388059701492537 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.987987987987988, |
| "9": 0.9807073954983923, |
| "10": 0.9134328358208955 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.996996996996997, |
| "9": 0.9839228295819936, |
| "10": 0.9402985074626866 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.996996996996997, |
| "9": 0.9935691318327974, |
| "10": 0.9641791044776119 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.996996996996997, |
| "9": 0.9935691318327974, |
| "10": 0.9731343283582089 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8818212253096752, |
| "pass@2": 0.9166387679946435, |
| "pass@4": 0.9464345497154335, |
| "pass@8": 0.9698694342149313, |
| "pass@16": 0.9866086374288584, |
| "pass@32": 0.9909608302644795, |
| "pass@64": 0.9946434549715434, |
| "pass@128": 0.995647807164379 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.996969696969697, |
| "pass@2": 0.996969696969697, |
| "pass@4": 0.996969696969697, |
| "pass@8": 0.996969696969697, |
| "pass@16": 0.996969696969697, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9822485207100592, |
| "pass@2": 0.9881656804733728, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9391304347826087, |
| "pass@2": 0.9681159420289855, |
| "pass@4": 0.9884057971014493, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9656160458452722, |
| "pass@2": 0.9713467048710601, |
| "pass@4": 0.9828080229226361, |
| "pass@8": 0.9914040114613181, |
| "pass@16": 0.997134670487106, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8757225433526011, |
| "pass@2": 0.9393063583815029, |
| "pass@4": 0.9682080924855492, |
| "pass@8": 0.9884393063583815, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7971830985915493, |
| "pass@2": 0.8647887323943662, |
| "pass@4": 0.9380281690140845, |
| "pass@8": 0.9661971830985916, |
| "pass@16": 0.9943661971830986, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7365269461077845, |
| "pass@2": 0.7934131736526946, |
| "pass@4": 0.8772455089820359, |
| "pass@8": 0.9341317365269461, |
| "pass@16": 0.9730538922155688, |
| "pass@32": 0.9970059880239521, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6625766871165644, |
| "pass@2": 0.7116564417177914, |
| "pass@4": 0.7852760736196319, |
| "pass@8": 0.8282208588957055, |
| "pass@16": 0.8773006134969326, |
| "pass@32": 0.9233128834355828, |
| "pass@64": 0.9478527607361963, |
| "pass@128": 0.950920245398773 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 0.9822485207100592, |
| "5": 0.9391304347826087, |
| "6": 0.9656160458452722, |
| "7": 0.8757225433526011, |
| "8": 0.7971830985915493, |
| "9": 0.7365269461077845, |
| "10": 0.6625766871165644 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 0.9881656804733728, |
| "5": 0.9681159420289855, |
| "6": 0.9713467048710601, |
| "7": 0.9393063583815029, |
| "8": 0.8647887323943662, |
| "9": 0.7934131736526946, |
| "10": 0.7116564417177914 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 1.0, |
| "5": 0.9884057971014493, |
| "6": 0.9828080229226361, |
| "7": 0.9682080924855492, |
| "8": 0.9380281690140845, |
| "9": 0.8772455089820359, |
| "10": 0.7852760736196319 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 0.9914040114613181, |
| "7": 0.9884393063583815, |
| "8": 0.9661971830985916, |
| "9": 0.9341317365269461, |
| "10": 0.8282208588957055 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 0.997134670487106, |
| "7": 1.0, |
| "8": 0.9943661971830986, |
| "9": 0.9730538922155688, |
| "10": 0.8773006134969326 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.9233128834355828 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9478527607361963 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.950920245398773 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8843381389252949, |
| "pass@2": 0.9154652686762779, |
| "pass@4": 0.9492136304062909, |
| "pass@8": 0.9675622542595019, |
| "pass@16": 0.9823066841415465, |
| "pass@32": 0.991480996068152, |
| "pass@64": 0.994429882044561, |
| "pass@128": 0.9947575360419397 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 0.999, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.997, |
| "pass@2": 0.999, |
| "pass@4": 0.999, |
| "pass@8": 0.999, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.987, |
| "pass@2": 0.993, |
| "pass@4": 0.999, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.945, |
| "pass@2": 0.969, |
| "pass@4": 0.989, |
| "pass@8": 0.998, |
| "pass@16": 0.998, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.968, |
| "pass@2": 0.975, |
| "pass@4": 0.985, |
| "pass@8": 0.996, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.866, |
| "pass@2": 0.922, |
| "pass@4": 0.96, |
| "pass@8": 0.987, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.81, |
| "pass@2": 0.885, |
| "pass@4": 0.936, |
| "pass@8": 0.966, |
| "pass@16": 0.992, |
| "pass@32": 0.997, |
| "pass@64": 0.998, |
| "pass@128": 0.999 |
| }, |
| "9": { |
| "pass@1": 0.756, |
| "pass@2": 0.83, |
| "pass@4": 0.896, |
| "pass@8": 0.944, |
| "pass@16": 0.98, |
| "pass@32": 0.991, |
| "pass@64": 0.997, |
| "pass@128": 0.997 |
| }, |
| "10": { |
| "pass@1": 0.623, |
| "pass@2": 0.693, |
| "pass@4": 0.769, |
| "pass@8": 0.838, |
| "pass@16": 0.894, |
| "pass@32": 0.929, |
| "pass@64": 0.95, |
| "pass@128": 0.959 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 0.999, |
| "3": 0.997, |
| "4": 0.987, |
| "5": 0.945, |
| "6": 0.968, |
| "7": 0.866, |
| "8": 0.81, |
| "9": 0.756, |
| "10": 0.623 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 0.993, |
| "5": 0.969, |
| "6": 0.975, |
| "7": 0.922, |
| "8": 0.885, |
| "9": 0.83, |
| "10": 0.693 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 0.999, |
| "5": 0.989, |
| "6": 0.985, |
| "7": 0.96, |
| "8": 0.936, |
| "9": 0.896, |
| "10": 0.769 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 0.996, |
| "7": 0.987, |
| "8": 0.966, |
| "9": 0.944, |
| "10": 0.838 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 0.999, |
| "7": 0.999, |
| "8": 0.992, |
| "9": 0.98, |
| "10": 0.894 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.997, |
| "9": 0.991, |
| "10": 0.929 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.998, |
| "9": 0.997, |
| "10": 0.95 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 0.997, |
| "10": 0.959 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8834444444444445, |
| "pass@2": 0.9184444444444444, |
| "pass@4": 0.9481111111111111, |
| "pass@8": 0.9697777777777777, |
| "pass@16": 0.9845555555555555, |
| "pass@32": 0.9906666666666667, |
| "pass@64": 0.9937777777777778, |
| "pass@128": 0.9948888888888889 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-39008_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.56657223796034, |
| "pass@2": 0.6487252124645893, |
| "pass@4": 0.7648725212464589, |
| "pass@8": 0.8441926345609065, |
| "pass@16": 0.9065155807365439, |
| "pass@32": 0.9376770538243626, |
| "pass@64": 0.9660056657223796, |
| "pass@128": 0.9801699716713881 |
| }, |
| "12": { |
| "pass@1": 0.6271186440677966, |
| "pass@2": 0.6977401129943502, |
| "pass@4": 0.751412429378531, |
| "pass@8": 0.8418079096045198, |
| "pass@16": 0.9067796610169492, |
| "pass@32": 0.9350282485875706, |
| "pass@64": 0.963276836158192, |
| "pass@128": 0.9774011299435028 |
| }, |
| "13": { |
| "pass@1": 0.5768025078369906, |
| "pass@2": 0.6238244514106583, |
| "pass@4": 0.7021943573667712, |
| "pass@8": 0.8056426332288401, |
| "pass@16": 0.8620689655172413, |
| "pass@32": 0.890282131661442, |
| "pass@64": 0.9216300940438872, |
| "pass@128": 0.9498432601880877 |
| }, |
| "14": { |
| "pass@1": 0.5264797507788161, |
| "pass@2": 0.5763239875389408, |
| "pass@4": 0.6199376947040498, |
| "pass@8": 0.6915887850467289, |
| "pass@16": 0.7538940809968847, |
| "pass@32": 0.8286604361370716, |
| "pass@64": 0.8785046728971962, |
| "pass@128": 0.9065420560747663 |
| }, |
| "15": { |
| "pass@1": 0.5028901734104047, |
| "pass@2": 0.5549132947976878, |
| "pass@4": 0.5924855491329479, |
| "pass@8": 0.653179190751445, |
| "pass@16": 0.7196531791907514, |
| "pass@32": 0.815028901734104, |
| "pass@64": 0.8901734104046243, |
| "pass@128": 0.9508670520231214 |
| }, |
| "16": { |
| "pass@1": 0.4645161290322581, |
| "pass@2": 0.5193548387096775, |
| "pass@4": 0.5451612903225806, |
| "pass@8": 0.5903225806451613, |
| "pass@16": 0.6451612903225806, |
| "pass@32": 0.7225806451612903, |
| "pass@64": 0.7741935483870968, |
| "pass@128": 0.7935483870967742 |
| }, |
| "17": { |
| "pass@1": 0.5, |
| "pass@2": 0.5404624277456648, |
| "pass@4": 0.5838150289017341, |
| "pass@8": 0.6213872832369942, |
| "pass@16": 0.6936416184971098, |
| "pass@32": 0.7485549132947977, |
| "pass@64": 0.8092485549132948, |
| "pass@128": 0.8554913294797688 |
| }, |
| "18": { |
| "pass@1": 0.44, |
| "pass@2": 0.4942857142857143, |
| "pass@4": 0.5114285714285715, |
| "pass@8": 0.56, |
| "pass@16": 0.6171428571428571, |
| "pass@32": 0.6657142857142857, |
| "pass@64": 0.7371428571428571, |
| "pass@128": 0.7885714285714286 |
| }, |
| "19": { |
| "pass@1": 0.37237237237237236, |
| "pass@2": 0.4264264264264264, |
| "pass@4": 0.46846846846846846, |
| "pass@8": 0.5105105105105106, |
| "pass@16": 0.5375375375375375, |
| "pass@32": 0.5705705705705706, |
| "pass@64": 0.6186186186186187, |
| "pass@128": 0.6846846846846847 |
| }, |
| "20": { |
| "pass@1": 0.36936936936936937, |
| "pass@2": 0.42042042042042044, |
| "pass@4": 0.4444444444444444, |
| "pass@8": 0.4774774774774775, |
| "pass@16": 0.5255255255255256, |
| "pass@32": 0.5825825825825826, |
| "pass@64": 0.6516516516516516, |
| "pass@128": 0.6996996996996997 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.56657223796034, |
| "12": 0.6271186440677966, |
| "13": 0.5768025078369906, |
| "14": 0.5264797507788161, |
| "15": 0.5028901734104047, |
| "16": 0.4645161290322581, |
| "17": 0.5, |
| "18": 0.44, |
| "19": 0.37237237237237236, |
| "20": 0.36936936936936937 |
| }, |
| "pass@2": { |
| "11": 0.6487252124645893, |
| "12": 0.6977401129943502, |
| "13": 0.6238244514106583, |
| "14": 0.5763239875389408, |
| "15": 0.5549132947976878, |
| "16": 0.5193548387096775, |
| "17": 0.5404624277456648, |
| "18": 0.4942857142857143, |
| "19": 0.4264264264264264, |
| "20": 0.42042042042042044 |
| }, |
| "pass@4": { |
| "11": 0.7648725212464589, |
| "12": 0.751412429378531, |
| "13": 0.7021943573667712, |
| "14": 0.6199376947040498, |
| "15": 0.5924855491329479, |
| "16": 0.5451612903225806, |
| "17": 0.5838150289017341, |
| "18": 0.5114285714285715, |
| "19": 0.46846846846846846, |
| "20": 0.4444444444444444 |
| }, |
| "pass@8": { |
| "11": 0.8441926345609065, |
| "12": 0.8418079096045198, |
| "13": 0.8056426332288401, |
| "14": 0.6915887850467289, |
| "15": 0.653179190751445, |
| "16": 0.5903225806451613, |
| "17": 0.6213872832369942, |
| "18": 0.56, |
| "19": 0.5105105105105106, |
| "20": 0.4774774774774775 |
| }, |
| "pass@16": { |
| "11": 0.9065155807365439, |
| "12": 0.9067796610169492, |
| "13": 0.8620689655172413, |
| "14": 0.7538940809968847, |
| "15": 0.7196531791907514, |
| "16": 0.6451612903225806, |
| "17": 0.6936416184971098, |
| "18": 0.6171428571428571, |
| "19": 0.5375375375375375, |
| "20": 0.5255255255255256 |
| }, |
| "pass@32": { |
| "11": 0.9376770538243626, |
| "12": 0.9350282485875706, |
| "13": 0.890282131661442, |
| "14": 0.8286604361370716, |
| "15": 0.815028901734104, |
| "16": 0.7225806451612903, |
| "17": 0.7485549132947977, |
| "18": 0.6657142857142857, |
| "19": 0.5705705705705706, |
| "20": 0.5825825825825826 |
| }, |
| "pass@64": { |
| "11": 0.9660056657223796, |
| "12": 0.963276836158192, |
| "13": 0.9216300940438872, |
| "14": 0.8785046728971962, |
| "15": 0.8901734104046243, |
| "16": 0.7741935483870968, |
| "17": 0.8092485549132948, |
| "18": 0.7371428571428571, |
| "19": 0.6186186186186187, |
| "20": 0.6516516516516516 |
| }, |
| "pass@128": { |
| "11": 0.9801699716713881, |
| "12": 0.9774011299435028, |
| "13": 0.9498432601880877, |
| "14": 0.9065420560747663, |
| "15": 0.9508670520231214, |
| "16": 0.7935483870967742, |
| "17": 0.8554913294797688, |
| "18": 0.7885714285714286, |
| "19": 0.6846846846846847, |
| "20": 0.6996996996996997 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.49539375928677565, |
| "pass@2": 0.5512630014858841, |
| "pass@4": 0.599702823179792, |
| "pass@8": 0.6609212481426449, |
| "pass@16": 0.7182763744427935, |
| "pass@32": 0.7708766716196137, |
| "pass@64": 0.8222882615156017, |
| "pass@128": 0.8600297176820209 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6126984126984127, |
| "pass@2": 0.6888888888888889, |
| "pass@4": 0.7746031746031746, |
| "pass@8": 0.8476190476190476, |
| "pass@16": 0.8984126984126984, |
| "pass@32": 0.946031746031746, |
| "pass@64": 0.9809523809523809, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.6330275229357798, |
| "pass@2": 0.7155963302752294, |
| "pass@4": 0.7737003058103975, |
| "pass@8": 0.8654434250764526, |
| "pass@16": 0.9051987767584098, |
| "pass@32": 0.9541284403669725, |
| "pass@64": 0.9755351681957186, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.5864197530864198, |
| "pass@2": 0.6203703703703703, |
| "pass@4": 0.7067901234567902, |
| "pass@8": 0.8055555555555556, |
| "pass@16": 0.8549382716049383, |
| "pass@32": 0.9012345679012346, |
| "pass@64": 0.9382716049382716, |
| "pass@128": 0.9537037037037037 |
| }, |
| "14": { |
| "pass@1": 0.5459940652818991, |
| "pass@2": 0.5786350148367952, |
| "pass@4": 0.6112759643916914, |
| "pass@8": 0.6646884272997032, |
| "pass@16": 0.7418397626112759, |
| "pass@32": 0.8456973293768546, |
| "pass@64": 0.9020771513353115, |
| "pass@128": 0.9347181008902077 |
| }, |
| "15": { |
| "pass@1": 0.4783950617283951, |
| "pass@2": 0.5277777777777778, |
| "pass@4": 0.5679012345679012, |
| "pass@8": 0.6172839506172839, |
| "pass@16": 0.691358024691358, |
| "pass@32": 0.7870370370370371, |
| "pass@64": 0.8888888888888888, |
| "pass@128": 0.941358024691358 |
| }, |
| "16": { |
| "pass@1": 0.5103244837758112, |
| "pass@2": 0.5663716814159292, |
| "pass@4": 0.5958702064896755, |
| "pass@8": 0.6637168141592921, |
| "pass@16": 0.7109144542772862, |
| "pass@32": 0.7551622418879056, |
| "pass@64": 0.8082595870206489, |
| "pass@128": 0.8436578171091446 |
| }, |
| "17": { |
| "pass@1": 0.5030120481927711, |
| "pass@2": 0.5421686746987951, |
| "pass@4": 0.5933734939759037, |
| "pass@8": 0.6295180722891566, |
| "pass@16": 0.6656626506024096, |
| "pass@32": 0.7018072289156626, |
| "pass@64": 0.7560240963855421, |
| "pass@128": 0.8102409638554217 |
| }, |
| "18": { |
| "pass@1": 0.4806451612903226, |
| "pass@2": 0.5193548387096775, |
| "pass@4": 0.5387096774193548, |
| "pass@8": 0.6064516129032258, |
| "pass@16": 0.6548387096774193, |
| "pass@32": 0.6838709677419355, |
| "pass@64": 0.7677419354838709, |
| "pass@128": 0.8354838709677419 |
| }, |
| "19": { |
| "pass@1": 0.38368580060422963, |
| "pass@2": 0.43202416918429004, |
| "pass@4": 0.48036253776435045, |
| "pass@8": 0.513595166163142, |
| "pass@16": 0.5558912386706949, |
| "pass@32": 0.6042296072507553, |
| "pass@64": 0.6646525679758308, |
| "pass@128": 0.7009063444108762 |
| }, |
| "20": { |
| "pass@1": 0.34234234234234234, |
| "pass@2": 0.3933933933933934, |
| "pass@4": 0.4264264264264264, |
| "pass@8": 0.45045045045045046, |
| "pass@16": 0.4924924924924925, |
| "pass@32": 0.5465465465465466, |
| "pass@64": 0.6036036036036037, |
| "pass@128": 0.6636636636636637 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6126984126984127, |
| "12": 0.6330275229357798, |
| "13": 0.5864197530864198, |
| "14": 0.5459940652818991, |
| "15": 0.4783950617283951, |
| "16": 0.5103244837758112, |
| "17": 0.5030120481927711, |
| "18": 0.4806451612903226, |
| "19": 0.38368580060422963, |
| "20": 0.34234234234234234 |
| }, |
| "pass@2": { |
| "11": 0.6888888888888889, |
| "12": 0.7155963302752294, |
| "13": 0.6203703703703703, |
| "14": 0.5786350148367952, |
| "15": 0.5277777777777778, |
| "16": 0.5663716814159292, |
| "17": 0.5421686746987951, |
| "18": 0.5193548387096775, |
| "19": 0.43202416918429004, |
| "20": 0.3933933933933934 |
| }, |
| "pass@4": { |
| "11": 0.7746031746031746, |
| "12": 0.7737003058103975, |
| "13": 0.7067901234567902, |
| "14": 0.6112759643916914, |
| "15": 0.5679012345679012, |
| "16": 0.5958702064896755, |
| "17": 0.5933734939759037, |
| "18": 0.5387096774193548, |
| "19": 0.48036253776435045, |
| "20": 0.4264264264264264 |
| }, |
| "pass@8": { |
| "11": 0.8476190476190476, |
| "12": 0.8654434250764526, |
| "13": 0.8055555555555556, |
| "14": 0.6646884272997032, |
| "15": 0.6172839506172839, |
| "16": 0.6637168141592921, |
| "17": 0.6295180722891566, |
| "18": 0.6064516129032258, |
| "19": 0.513595166163142, |
| "20": 0.45045045045045046 |
| }, |
| "pass@16": { |
| "11": 0.8984126984126984, |
| "12": 0.9051987767584098, |
| "13": 0.8549382716049383, |
| "14": 0.7418397626112759, |
| "15": 0.691358024691358, |
| "16": 0.7109144542772862, |
| "17": 0.6656626506024096, |
| "18": 0.6548387096774193, |
| "19": 0.5558912386706949, |
| "20": 0.4924924924924925 |
| }, |
| "pass@32": { |
| "11": 0.946031746031746, |
| "12": 0.9541284403669725, |
| "13": 0.9012345679012346, |
| "14": 0.8456973293768546, |
| "15": 0.7870370370370371, |
| "16": 0.7551622418879056, |
| "17": 0.7018072289156626, |
| "18": 0.6838709677419355, |
| "19": 0.6042296072507553, |
| "20": 0.5465465465465466 |
| }, |
| "pass@64": { |
| "11": 0.9809523809523809, |
| "12": 0.9755351681957186, |
| "13": 0.9382716049382716, |
| "14": 0.9020771513353115, |
| "15": 0.8888888888888888, |
| "16": 0.8082595870206489, |
| "17": 0.7560240963855421, |
| "18": 0.7677419354838709, |
| "19": 0.6646525679758308, |
| "20": 0.6036036036036037 |
| }, |
| "pass@128": { |
| "11": 0.9904761904761905, |
| "12": 0.981651376146789, |
| "13": 0.9537037037037037, |
| "14": 0.9347181008902077, |
| "15": 0.941358024691358, |
| "16": 0.8436578171091446, |
| "17": 0.8102409638554217, |
| "18": 0.8354838709677419, |
| "19": 0.7009063444108762, |
| "20": 0.6636636636636637 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5070293398533008, |
| "pass@2": 0.5577628361858191, |
| "pass@4": 0.6060513447432763, |
| "pass@8": 0.6653422982885085, |
| "pass@16": 0.7160757946210269, |
| "pass@32": 0.7716992665036675, |
| "pass@64": 0.8276283618581907, |
| "pass@128": 0.8646088019559902 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6596385542168675, |
| "pass@2": 0.713855421686747, |
| "pass@4": 0.7981927710843374, |
| "pass@8": 0.8554216867469879, |
| "pass@16": 0.9126506024096386, |
| "pass@32": 0.9457831325301205, |
| "pass@64": 0.9668674698795181, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.5956112852664577, |
| "pass@2": 0.6520376175548589, |
| "pass@4": 0.7304075235109718, |
| "pass@8": 0.8432601880877743, |
| "pass@16": 0.9122257053291536, |
| "pass@32": 0.9373040752351097, |
| "pass@64": 0.9529780564263323, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.6078431372549019, |
| "pass@2": 0.6470588235294118, |
| "pass@4": 0.7563025210084033, |
| "pass@8": 0.8291316526610645, |
| "pass@16": 0.8935574229691877, |
| "pass@32": 0.9243697478991597, |
| "pass@64": 0.9523809523809523, |
| "pass@128": 0.969187675070028 |
| }, |
| "14": { |
| "pass@1": 0.5584795321637427, |
| "pass@2": 0.5964912280701754, |
| "pass@4": 0.652046783625731, |
| "pass@8": 0.7309941520467836, |
| "pass@16": 0.8187134502923976, |
| "pass@32": 0.8888888888888888, |
| "pass@64": 0.935672514619883, |
| "pass@128": 0.956140350877193 |
| }, |
| "15": { |
| "pass@1": 0.5393939393939394, |
| "pass@2": 0.5696969696969697, |
| "pass@4": 0.6212121212121212, |
| "pass@8": 0.6787878787878788, |
| "pass@16": 0.7606060606060606, |
| "pass@32": 0.8333333333333334, |
| "pass@64": 0.8878787878787879, |
| "pass@128": 0.9333333333333333 |
| }, |
| "16": { |
| "pass@1": 0.5213675213675214, |
| "pass@2": 0.5555555555555556, |
| "pass@4": 0.5811965811965812, |
| "pass@8": 0.6381766381766382, |
| "pass@16": 0.6837606837606838, |
| "pass@32": 0.7435897435897436, |
| "pass@64": 0.8176638176638177, |
| "pass@128": 0.8547008547008547 |
| }, |
| "17": { |
| "pass@1": 0.42857142857142855, |
| "pass@2": 0.468944099378882, |
| "pass@4": 0.5186335403726708, |
| "pass@8": 0.5714285714285714, |
| "pass@16": 0.6366459627329193, |
| "pass@32": 0.7204968944099379, |
| "pass@64": 0.7950310559006211, |
| "pass@128": 0.8385093167701864 |
| }, |
| "18": { |
| "pass@1": 0.4294117647058823, |
| "pass@2": 0.4676470588235294, |
| "pass@4": 0.4852941176470588, |
| "pass@8": 0.5176470588235295, |
| "pass@16": 0.5764705882352941, |
| "pass@32": 0.6617647058823529, |
| "pass@64": 0.7382352941176471, |
| "pass@128": 0.8058823529411765 |
| }, |
| "19": { |
| "pass@1": 0.40773809523809523, |
| "pass@2": 0.4523809523809524, |
| "pass@4": 0.4851190476190476, |
| "pass@8": 0.5178571428571429, |
| "pass@16": 0.5476190476190477, |
| "pass@32": 0.5952380952380952, |
| "pass@64": 0.6488095238095238, |
| "pass@128": 0.6845238095238095 |
| }, |
| "20": { |
| "pass@1": 0.3712574850299401, |
| "pass@2": 0.4041916167664671, |
| "pass@4": 0.45209580838323354, |
| "pass@8": 0.4820359281437126, |
| "pass@16": 0.5239520958083832, |
| "pass@32": 0.5748502994011976, |
| "pass@64": 0.5898203592814372, |
| "pass@128": 0.6287425149700598 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6596385542168675, |
| "12": 0.5956112852664577, |
| "13": 0.6078431372549019, |
| "14": 0.5584795321637427, |
| "15": 0.5393939393939394, |
| "16": 0.5213675213675214, |
| "17": 0.42857142857142855, |
| "18": 0.4294117647058823, |
| "19": 0.40773809523809523, |
| "20": 0.3712574850299401 |
| }, |
| "pass@2": { |
| "11": 0.713855421686747, |
| "12": 0.6520376175548589, |
| "13": 0.6470588235294118, |
| "14": 0.5964912280701754, |
| "15": 0.5696969696969697, |
| "16": 0.5555555555555556, |
| "17": 0.468944099378882, |
| "18": 0.4676470588235294, |
| "19": 0.4523809523809524, |
| "20": 0.4041916167664671 |
| }, |
| "pass@4": { |
| "11": 0.7981927710843374, |
| "12": 0.7304075235109718, |
| "13": 0.7563025210084033, |
| "14": 0.652046783625731, |
| "15": 0.6212121212121212, |
| "16": 0.5811965811965812, |
| "17": 0.5186335403726708, |
| "18": 0.4852941176470588, |
| "19": 0.4851190476190476, |
| "20": 0.45209580838323354 |
| }, |
| "pass@8": { |
| "11": 0.8554216867469879, |
| "12": 0.8432601880877743, |
| "13": 0.8291316526610645, |
| "14": 0.7309941520467836, |
| "15": 0.6787878787878788, |
| "16": 0.6381766381766382, |
| "17": 0.5714285714285714, |
| "18": 0.5176470588235295, |
| "19": 0.5178571428571429, |
| "20": 0.4820359281437126 |
| }, |
| "pass@16": { |
| "11": 0.9126506024096386, |
| "12": 0.9122257053291536, |
| "13": 0.8935574229691877, |
| "14": 0.8187134502923976, |
| "15": 0.7606060606060606, |
| "16": 0.6837606837606838, |
| "17": 0.6366459627329193, |
| "18": 0.5764705882352941, |
| "19": 0.5476190476190477, |
| "20": 0.5239520958083832 |
| }, |
| "pass@32": { |
| "11": 0.9457831325301205, |
| "12": 0.9373040752351097, |
| "13": 0.9243697478991597, |
| "14": 0.8888888888888888, |
| "15": 0.8333333333333334, |
| "16": 0.7435897435897436, |
| "17": 0.7204968944099379, |
| "18": 0.6617647058823529, |
| "19": 0.5952380952380952, |
| "20": 0.5748502994011976 |
| }, |
| "pass@64": { |
| "11": 0.9668674698795181, |
| "12": 0.9529780564263323, |
| "13": 0.9523809523809523, |
| "14": 0.935672514619883, |
| "15": 0.8878787878787879, |
| "16": 0.8176638176638177, |
| "17": 0.7950310559006211, |
| "18": 0.7382352941176471, |
| "19": 0.6488095238095238, |
| "20": 0.5898203592814372 |
| }, |
| "pass@128": { |
| "11": 0.9789156626506024, |
| "12": 0.9655172413793104, |
| "13": 0.969187675070028, |
| "14": 0.956140350877193, |
| "15": 0.9333333333333333, |
| "16": 0.8547008547008547, |
| "17": 0.8385093167701864, |
| "18": 0.8058823529411765, |
| "19": 0.6845238095238095, |
| "20": 0.6287425149700598 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5123401724650609, |
| "pass@2": 0.5530776092774309, |
| "pass@4": 0.6083853702051739, |
| "pass@8": 0.6666666666666666, |
| "pass@16": 0.7267320844484092, |
| "pass@32": 0.7826345524829021, |
| "pass@64": 0.8287243532560215, |
| "pass@128": 0.8617305976806423 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.612, |
| "pass@2": 0.683, |
| "pass@4": 0.779, |
| "pass@8": 0.849, |
| "pass@16": 0.906, |
| "pass@32": 0.943, |
| "pass@64": 0.971, |
| "pass@128": 0.983 |
| }, |
| "12": { |
| "pass@1": 0.619, |
| "pass@2": 0.689, |
| "pass@4": 0.752, |
| "pass@8": 0.85, |
| "pass@16": 0.908, |
| "pass@32": 0.942, |
| "pass@64": 0.964, |
| "pass@128": 0.975 |
| }, |
| "13": { |
| "pass@1": 0.591, |
| "pass@2": 0.631, |
| "pass@4": 0.723, |
| "pass@8": 0.814, |
| "pass@16": 0.871, |
| "pass@32": 0.906, |
| "pass@64": 0.938, |
| "pass@128": 0.958 |
| }, |
| "14": { |
| "pass@1": 0.544, |
| "pass@2": 0.584, |
| "pass@4": 0.628, |
| "pass@8": 0.696, |
| "pass@16": 0.772, |
| "pass@32": 0.855, |
| "pass@64": 0.906, |
| "pass@128": 0.933 |
| }, |
| "15": { |
| "pass@1": 0.507, |
| "pass@2": 0.551, |
| "pass@4": 0.594, |
| "pass@8": 0.65, |
| "pass@16": 0.724, |
| "pass@32": 0.812, |
| "pass@64": 0.889, |
| "pass@128": 0.942 |
| }, |
| "16": { |
| "pass@1": 0.5, |
| "pass@2": 0.548, |
| "pass@4": 0.575, |
| "pass@8": 0.632, |
| "pass@16": 0.681, |
| "pass@32": 0.741, |
| "pass@64": 0.801, |
| "pass@128": 0.832 |
| }, |
| "17": { |
| "pass@1": 0.478, |
| "pass@2": 0.518, |
| "pass@4": 0.566, |
| "pass@8": 0.608, |
| "pass@16": 0.666, |
| "pass@32": 0.724, |
| "pass@64": 0.787, |
| "pass@128": 0.835 |
| }, |
| "18": { |
| "pass@1": 0.449, |
| "pass@2": 0.493, |
| "pass@4": 0.511, |
| "pass@8": 0.56, |
| "pass@16": 0.615, |
| "pass@32": 0.67, |
| "pass@64": 0.747, |
| "pass@128": 0.809 |
| }, |
| "19": { |
| "pass@1": 0.388, |
| "pass@2": 0.437, |
| "pass@4": 0.478, |
| "pass@8": 0.514, |
| "pass@16": 0.547, |
| "pass@32": 0.59, |
| "pass@64": 0.644, |
| "pass@128": 0.69 |
| }, |
| "20": { |
| "pass@1": 0.361, |
| "pass@2": 0.406, |
| "pass@4": 0.441, |
| "pass@8": 0.47, |
| "pass@16": 0.514, |
| "pass@32": 0.568, |
| "pass@64": 0.615, |
| "pass@128": 0.664 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.612, |
| "12": 0.619, |
| "13": 0.591, |
| "14": 0.544, |
| "15": 0.507, |
| "16": 0.5, |
| "17": 0.478, |
| "18": 0.449, |
| "19": 0.388, |
| "20": 0.361 |
| }, |
| "pass@2": { |
| "11": 0.683, |
| "12": 0.689, |
| "13": 0.631, |
| "14": 0.584, |
| "15": 0.551, |
| "16": 0.548, |
| "17": 0.518, |
| "18": 0.493, |
| "19": 0.437, |
| "20": 0.406 |
| }, |
| "pass@4": { |
| "11": 0.779, |
| "12": 0.752, |
| "13": 0.723, |
| "14": 0.628, |
| "15": 0.594, |
| "16": 0.575, |
| "17": 0.566, |
| "18": 0.511, |
| "19": 0.478, |
| "20": 0.441 |
| }, |
| "pass@8": { |
| "11": 0.849, |
| "12": 0.85, |
| "13": 0.814, |
| "14": 0.696, |
| "15": 0.65, |
| "16": 0.632, |
| "17": 0.608, |
| "18": 0.56, |
| "19": 0.514, |
| "20": 0.47 |
| }, |
| "pass@16": { |
| "11": 0.906, |
| "12": 0.908, |
| "13": 0.871, |
| "14": 0.772, |
| "15": 0.724, |
| "16": 0.681, |
| "17": 0.666, |
| "18": 0.615, |
| "19": 0.547, |
| "20": 0.514 |
| }, |
| "pass@32": { |
| "11": 0.943, |
| "12": 0.942, |
| "13": 0.906, |
| "14": 0.855, |
| "15": 0.812, |
| "16": 0.741, |
| "17": 0.724, |
| "18": 0.67, |
| "19": 0.59, |
| "20": 0.568 |
| }, |
| "pass@64": { |
| "11": 0.971, |
| "12": 0.964, |
| "13": 0.938, |
| "14": 0.906, |
| "15": 0.889, |
| "16": 0.801, |
| "17": 0.787, |
| "18": 0.747, |
| "19": 0.644, |
| "20": 0.615 |
| }, |
| "pass@128": { |
| "11": 0.983, |
| "12": 0.975, |
| "13": 0.958, |
| "14": 0.933, |
| "15": 0.942, |
| "16": 0.832, |
| "17": 0.835, |
| "18": 0.809, |
| "19": 0.69, |
| "20": 0.664 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5049, |
| "pass@2": 0.554, |
| "pass@4": 0.6047, |
| "pass@8": 0.6643, |
| "pass@16": 0.7204, |
| "pass@32": 0.7751, |
| "pass@64": 0.8262, |
| "pass@128": 0.8621 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
