| { |
| "files": { |
| "checkpoint-42637_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99375, |
| "pass@2": 0.99375, |
| "pass@4": 0.996875, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9809523809523809, |
| "pass@2": 0.9936507936507937, |
| "pass@4": 0.9968253968253968, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9610778443113772, |
| "pass@2": 0.9820359281437125, |
| "pass@4": 0.9910179640718563, |
| "pass@8": 0.9970059880239521, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8905325443786982, |
| "pass@2": 0.9171597633136095, |
| "pass@4": 0.9556213017751479, |
| "pass@8": 0.9970414201183432, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8269230769230769, |
| "pass@2": 0.8942307692307693, |
| "pass@4": 0.9358974358974359, |
| "pass@8": 0.9807692307692307, |
| "pass@16": 0.9903846153846154, |
| "pass@32": 0.9935897435897436, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7549295774647887, |
| "pass@2": 0.8309859154929577, |
| "pass@4": 0.9070422535211268, |
| "pass@8": 0.9605633802816902, |
| "pass@16": 0.9887323943661972, |
| "pass@32": 0.9943661971830986, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6460176991150443, |
| "pass@2": 0.7020648967551623, |
| "pass@4": 0.7787610619469026, |
| "pass@8": 0.8761061946902655, |
| "pass@16": 0.9321533923303835, |
| "pass@32": 0.9528023598820059, |
| "pass@64": 0.9734513274336283, |
| "pass@128": 0.9734513274336283 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9809523809523809, |
| "6": 0.9610778443113772, |
| "7": 0.8905325443786982, |
| "8": 0.8269230769230769, |
| "9": 0.7549295774647887, |
| "10": 0.6460176991150443 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9936507936507937, |
| "6": 0.9820359281437125, |
| "7": 0.9171597633136095, |
| "8": 0.8942307692307693, |
| "9": 0.8309859154929577, |
| "10": 0.7020648967551623 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9968253968253968, |
| "6": 0.9910179640718563, |
| "7": 0.9556213017751479, |
| "8": 0.9358974358974359, |
| "9": 0.9070422535211268, |
| "10": 0.7787610619469026 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9970059880239521, |
| "7": 0.9970414201183432, |
| "8": 0.9807692307692307, |
| "9": 0.9605633802816902, |
| "10": 0.8761061946902655 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 0.9903846153846154, |
| "9": 0.9887323943661972, |
| "10": 0.9321533923303835 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9935897435897436, |
| "9": 0.9943661971830986, |
| "10": 0.9528023598820059 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9734513274336283 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9734513274336283 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8922661263086795, |
| "pass@2": 0.9216480918608578, |
| "pass@4": 0.950016886187099, |
| "pass@8": 0.9783856805133401, |
| "pass@16": 0.9895305639986491, |
| "pass@32": 0.9932455251604188, |
| "pass@64": 0.9969604863221885, |
| "pass@128": 0.9969604863221885 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9970760233918129, |
| "pass@2": 0.9970760233918129, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.961764705882353, |
| "pass@2": 0.9911764705882353, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9652996845425867, |
| "pass@2": 0.9810725552050473, |
| "pass@4": 0.9873817034700315, |
| "pass@8": 0.9968454258675079, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8734177215189873, |
| "pass@2": 0.9082278481012658, |
| "pass@4": 0.9620253164556962, |
| "pass@8": 0.9778481012658228, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8078078078078078, |
| "pass@2": 0.8768768768768769, |
| "pass@4": 0.924924924924925, |
| "pass@8": 0.96996996996997, |
| "pass@16": 0.990990990990991, |
| "pass@32": 0.996996996996997, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7588424437299035, |
| "pass@2": 0.842443729903537, |
| "pass@4": 0.9131832797427653, |
| "pass@8": 0.9517684887459807, |
| "pass@16": 0.9935691318327974, |
| "pass@32": 0.9967845659163987, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6179104477611941, |
| "pass@2": 0.6746268656716418, |
| "pass@4": 0.7701492537313432, |
| "pass@8": 0.8507462686567164, |
| "pass@16": 0.9283582089552239, |
| "pass@32": 0.9522388059701492, |
| "pass@64": 0.9641791044776119, |
| "pass@128": 0.9761194029850746 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.961764705882353, |
| "6": 0.9652996845425867, |
| "7": 0.8734177215189873, |
| "8": 0.8078078078078078, |
| "9": 0.7588424437299035, |
| "10": 0.6179104477611941 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.9911764705882353, |
| "6": 0.9810725552050473, |
| "7": 0.9082278481012658, |
| "8": 0.8768768768768769, |
| "9": 0.842443729903537, |
| "10": 0.6746268656716418 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9873817034700315, |
| "7": 0.9620253164556962, |
| "8": 0.924924924924925, |
| "9": 0.9131832797427653, |
| "10": 0.7701492537313432 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9968454258675079, |
| "7": 0.9778481012658228, |
| "8": 0.96996996996997, |
| "9": 0.9517684887459807, |
| "10": 0.8507462686567164 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.990990990990991, |
| "9": 0.9935691318327974, |
| "10": 0.9283582089552239 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.996996996996997, |
| "9": 0.9967845659163987, |
| "10": 0.9522388059701492 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9641791044776119 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9761194029850746 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8888516906595246, |
| "pass@2": 0.9203213927017074, |
| "pass@4": 0.9514563106796117, |
| "pass@8": 0.9722129226648811, |
| "pass@16": 0.9902912621359223, |
| "pass@32": 0.9939738868429863, |
| "pass@64": 0.9959825912286575, |
| "pass@128": 0.9973217274857716 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9822485207100592, |
| "pass@2": 0.9940828402366864, |
| "pass@4": 0.9970414201183432, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9536231884057971, |
| "pass@2": 0.9797101449275363, |
| "pass@4": 0.991304347826087, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9713467048710601, |
| "pass@2": 0.9799426934097422, |
| "pass@4": 0.9885386819484241, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8526011560693642, |
| "pass@2": 0.9161849710982659, |
| "pass@4": 0.9508670520231214, |
| "pass@8": 0.976878612716763, |
| "pass@16": 0.9971098265895953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8, |
| "pass@2": 0.8901408450704226, |
| "pass@4": 0.9464788732394366, |
| "pass@8": 0.9690140845070423, |
| "pass@16": 0.9887323943661972, |
| "pass@32": 0.9943661971830986, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.7485029940119761, |
| "pass@2": 0.8083832335329342, |
| "pass@4": 0.8862275449101796, |
| "pass@8": 0.9221556886227545, |
| "pass@16": 0.9730538922155688, |
| "pass@32": 0.9910179640718563, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 0.9970059880239521 |
| }, |
| "10": { |
| "pass@1": 0.6656441717791411, |
| "pass@2": 0.7331288343558282, |
| "pass@4": 0.8159509202453987, |
| "pass@8": 0.8865030674846626, |
| "pass@16": 0.9263803680981595, |
| "pass@32": 0.9478527607361963, |
| "pass@64": 0.9662576687116564, |
| "pass@128": 0.9754601226993865 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9822485207100592, |
| "5": 0.9536231884057971, |
| "6": 0.9713467048710601, |
| "7": 0.8526011560693642, |
| "8": 0.8, |
| "9": 0.7485029940119761, |
| "10": 0.6656441717791411 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9940828402366864, |
| "5": 0.9797101449275363, |
| "6": 0.9799426934097422, |
| "7": 0.9161849710982659, |
| "8": 0.8901408450704226, |
| "9": 0.8083832335329342, |
| "10": 0.7331288343558282 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 0.991304347826087, |
| "6": 0.9885386819484241, |
| "7": 0.9508670520231214, |
| "8": 0.9464788732394366, |
| "9": 0.8862275449101796, |
| "10": 0.8159509202453987 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 1.0, |
| "7": 0.976878612716763, |
| "8": 0.9690140845070423, |
| "9": 0.9221556886227545, |
| "10": 0.8865030674846626 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9971098265895953, |
| "8": 0.9887323943661972, |
| "9": 0.9730538922155688, |
| "10": 0.9263803680981595 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9943661971830986, |
| "9": 0.9910179640718563, |
| "10": 0.9478527607361963 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9970059880239521, |
| "10": 0.9662576687116564 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9970059880239521, |
| "10": 0.9754601226993865 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8863040629095675, |
| "pass@2": 0.9230013106159896, |
| "pass@4": 0.953473132372215, |
| "pass@8": 0.9728047182175622, |
| "pass@16": 0.9875491480996068, |
| "pass@32": 0.9927916120576671, |
| "pass@64": 0.995740498034076, |
| "pass@128": 0.9967234600262124 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.991, |
| "pass@2": 0.995, |
| "pass@4": 0.998, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.965, |
| "pass@2": 0.988, |
| "pass@4": 0.996, |
| "pass@8": 0.999, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.966, |
| "pass@2": 0.981, |
| "pass@4": 0.989, |
| "pass@8": 0.998, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.872, |
| "pass@2": 0.914, |
| "pass@4": 0.956, |
| "pass@8": 0.984, |
| "pass@16": 0.998, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.811, |
| "pass@2": 0.887, |
| "pass@4": 0.936, |
| "pass@8": 0.973, |
| "pass@16": 0.99, |
| "pass@32": 0.995, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "9": { |
| "pass@1": 0.754, |
| "pass@2": 0.827, |
| "pass@4": 0.902, |
| "pass@8": 0.945, |
| "pass@16": 0.985, |
| "pass@32": 0.994, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "10": { |
| "pass@1": 0.643, |
| "pass@2": 0.703, |
| "pass@4": 0.788, |
| "pass@8": 0.871, |
| "pass@16": 0.929, |
| "pass@32": 0.951, |
| "pass@64": 0.968, |
| "pass@128": 0.975 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.991, |
| "5": 0.965, |
| "6": 0.966, |
| "7": 0.872, |
| "8": 0.811, |
| "9": 0.754, |
| "10": 0.643 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.995, |
| "5": 0.988, |
| "6": 0.981, |
| "7": 0.914, |
| "8": 0.887, |
| "9": 0.827, |
| "10": 0.703 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.998, |
| "5": 0.996, |
| "6": 0.989, |
| "7": 0.956, |
| "8": 0.936, |
| "9": 0.902, |
| "10": 0.788 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.998, |
| "7": 0.984, |
| "8": 0.973, |
| "9": 0.945, |
| "10": 0.871 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.998, |
| "8": 0.99, |
| "9": 0.985, |
| "10": 0.929 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.995, |
| "9": 0.994, |
| "10": 0.951 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 0.999, |
| "10": 0.968 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 0.999, |
| "10": 0.975 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8891111111111111, |
| "pass@2": 0.9216666666666666, |
| "pass@4": 0.9516666666666667, |
| "pass@8": 0.9744444444444444, |
| "pass@16": 0.9891111111111112, |
| "pass@32": 0.9933333333333333, |
| "pass@64": 0.9962222222222222, |
| "pass@128": 0.997 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-42637_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5779036827195467, |
| "pass@2": 0.6543909348441926, |
| "pass@4": 0.7223796033994334, |
| "pass@8": 0.8215297450424929, |
| "pass@16": 0.8951841359773371, |
| "pass@32": 0.9348441926345609, |
| "pass@64": 0.9490084985835694, |
| "pass@128": 0.9546742209631728 |
| }, |
| "12": { |
| "pass@1": 0.5988700564971752, |
| "pass@2": 0.6497175141242938, |
| "pass@4": 0.7062146892655368, |
| "pass@8": 0.7796610169491526, |
| "pass@16": 0.8220338983050848, |
| "pass@32": 0.8418079096045198, |
| "pass@64": 0.8559322033898306, |
| "pass@128": 0.864406779661017 |
| }, |
| "13": { |
| "pass@1": 0.4608150470219436, |
| "pass@2": 0.5203761755485894, |
| "pass@4": 0.5768025078369906, |
| "pass@8": 0.6238244514106583, |
| "pass@16": 0.6739811912225705, |
| "pass@32": 0.7241379310344828, |
| "pass@64": 0.768025078369906, |
| "pass@128": 0.7774294670846394 |
| }, |
| "14": { |
| "pass@1": 0.3333333333333333, |
| "pass@2": 0.3956386292834891, |
| "pass@4": 0.46105919003115264, |
| "pass@8": 0.5171339563862928, |
| "pass@16": 0.5669781931464174, |
| "pass@32": 0.6261682242990654, |
| "pass@64": 0.67601246105919, |
| "pass@128": 0.7165109034267912 |
| }, |
| "15": { |
| "pass@1": 0.1791907514450867, |
| "pass@2": 0.24566473988439305, |
| "pass@4": 0.3179190751445087, |
| "pass@8": 0.37283236994219654, |
| "pass@16": 0.41040462427745666, |
| "pass@32": 0.4653179190751445, |
| "pass@64": 0.5289017341040463, |
| "pass@128": 0.5924855491329479 |
| }, |
| "16": { |
| "pass@1": 0.05161290322580645, |
| "pass@2": 0.09032258064516129, |
| "pass@4": 0.11935483870967742, |
| "pass@8": 0.14516129032258066, |
| "pass@16": 0.2032258064516129, |
| "pass@32": 0.23548387096774193, |
| "pass@64": 0.27419354838709675, |
| "pass@128": 0.3225806451612903 |
| }, |
| "17": { |
| "pass@1": 0.05491329479768786, |
| "pass@2": 0.07225433526011561, |
| "pass@4": 0.0838150289017341, |
| "pass@8": 0.10982658959537572, |
| "pass@16": 0.13005780346820808, |
| "pass@32": 0.15028901734104047, |
| "pass@64": 0.18208092485549132, |
| "pass@128": 0.21965317919075145 |
| }, |
| "18": { |
| "pass@1": 0.008571428571428572, |
| "pass@2": 0.014285714285714285, |
| "pass@4": 0.025714285714285714, |
| "pass@8": 0.03142857142857143, |
| "pass@16": 0.04285714285714286, |
| "pass@32": 0.05142857142857143, |
| "pass@64": 0.06285714285714286, |
| "pass@128": 0.07714285714285714 |
| }, |
| "19": { |
| "pass@1": 0.006006006006006006, |
| "pass@2": 0.006006006006006006, |
| "pass@4": 0.012012012012012012, |
| "pass@8": 0.012012012012012012, |
| "pass@16": 0.015015015015015015, |
| "pass@32": 0.018018018018018018, |
| "pass@64": 0.024024024024024024, |
| "pass@128": 0.03903903903903904 |
| }, |
| "20": { |
| "pass@1": 0.003003003003003003, |
| "pass@2": 0.006006006006006006, |
| "pass@4": 0.015015015015015015, |
| "pass@8": 0.021021021021021023, |
| "pass@16": 0.02702702702702703, |
| "pass@32": 0.03003003003003003, |
| "pass@64": 0.042042042042042045, |
| "pass@128": 0.057057057057057055 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5779036827195467, |
| "12": 0.5988700564971752, |
| "13": 0.4608150470219436, |
| "14": 0.3333333333333333, |
| "15": 0.1791907514450867, |
| "16": 0.05161290322580645, |
| "17": 0.05491329479768786, |
| "18": 0.008571428571428572, |
| "19": 0.006006006006006006, |
| "20": 0.003003003003003003 |
| }, |
| "pass@2": { |
| "11": 0.6543909348441926, |
| "12": 0.6497175141242938, |
| "13": 0.5203761755485894, |
| "14": 0.3956386292834891, |
| "15": 0.24566473988439305, |
| "16": 0.09032258064516129, |
| "17": 0.07225433526011561, |
| "18": 0.014285714285714285, |
| "19": 0.006006006006006006, |
| "20": 0.006006006006006006 |
| }, |
| "pass@4": { |
| "11": 0.7223796033994334, |
| "12": 0.7062146892655368, |
| "13": 0.5768025078369906, |
| "14": 0.46105919003115264, |
| "15": 0.3179190751445087, |
| "16": 0.11935483870967742, |
| "17": 0.0838150289017341, |
| "18": 0.025714285714285714, |
| "19": 0.012012012012012012, |
| "20": 0.015015015015015015 |
| }, |
| "pass@8": { |
| "11": 0.8215297450424929, |
| "12": 0.7796610169491526, |
| "13": 0.6238244514106583, |
| "14": 0.5171339563862928, |
| "15": 0.37283236994219654, |
| "16": 0.14516129032258066, |
| "17": 0.10982658959537572, |
| "18": 0.03142857142857143, |
| "19": 0.012012012012012012, |
| "20": 0.021021021021021023 |
| }, |
| "pass@16": { |
| "11": 0.8951841359773371, |
| "12": 0.8220338983050848, |
| "13": 0.6739811912225705, |
| "14": 0.5669781931464174, |
| "15": 0.41040462427745666, |
| "16": 0.2032258064516129, |
| "17": 0.13005780346820808, |
| "18": 0.04285714285714286, |
| "19": 0.015015015015015015, |
| "20": 0.02702702702702703 |
| }, |
| "pass@32": { |
| "11": 0.9348441926345609, |
| "12": 0.8418079096045198, |
| "13": 0.7241379310344828, |
| "14": 0.6261682242990654, |
| "15": 0.4653179190751445, |
| "16": 0.23548387096774193, |
| "17": 0.15028901734104047, |
| "18": 0.05142857142857143, |
| "19": 0.018018018018018018, |
| "20": 0.03003003003003003 |
| }, |
| "pass@64": { |
| "11": 0.9490084985835694, |
| "12": 0.8559322033898306, |
| "13": 0.768025078369906, |
| "14": 0.67601246105919, |
| "15": 0.5289017341040463, |
| "16": 0.27419354838709675, |
| "17": 0.18208092485549132, |
| "18": 0.06285714285714286, |
| "19": 0.024024024024024024, |
| "20": 0.042042042042042045 |
| }, |
| "pass@128": { |
| "11": 0.9546742209631728, |
| "12": 0.864406779661017, |
| "13": 0.7774294670846394, |
| "14": 0.7165109034267912, |
| "15": 0.5924855491329479, |
| "16": 0.3225806451612903, |
| "17": 0.21965317919075145, |
| "18": 0.07714285714285714, |
| "19": 0.03903903903903904, |
| "20": 0.057057057057057055 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.22971768202080237, |
| "pass@2": 0.2677563150074294, |
| "pass@4": 0.30638930163447253, |
| "pass@8": 0.3462109955423477, |
| "pass@16": 0.3812778603268945, |
| "pass@32": 0.4101040118870728, |
| "pass@64": 0.43833580980683506, |
| "pass@128": 0.46389301634472513 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5714285714285714, |
| "pass@2": 0.6285714285714286, |
| "pass@4": 0.7111111111111111, |
| "pass@8": 0.8158730158730159, |
| "pass@16": 0.9206349206349206, |
| "pass@32": 0.9523809523809523, |
| "pass@64": 0.9650793650793651, |
| "pass@128": 0.9777777777777777 |
| }, |
| "12": { |
| "pass@1": 0.581039755351682, |
| "pass@2": 0.6513761467889908, |
| "pass@4": 0.7370030581039755, |
| "pass@8": 0.8012232415902141, |
| "pass@16": 0.8440366972477065, |
| "pass@32": 0.8593272171253823, |
| "pass@64": 0.8685015290519877, |
| "pass@128": 0.8776758409785933 |
| }, |
| "13": { |
| "pass@1": 0.4351851851851852, |
| "pass@2": 0.4845679012345679, |
| "pass@4": 0.558641975308642, |
| "pass@8": 0.6265432098765432, |
| "pass@16": 0.6820987654320988, |
| "pass@32": 0.7222222222222222, |
| "pass@64": 0.7530864197530864, |
| "pass@128": 0.7746913580246914 |
| }, |
| "14": { |
| "pass@1": 0.2818991097922849, |
| "pass@2": 0.34124629080118696, |
| "pass@4": 0.3827893175074184, |
| "pass@8": 0.45103857566765576, |
| "pass@16": 0.5074183976261127, |
| "pass@32": 0.5667655786350149, |
| "pass@64": 0.6201780415430267, |
| "pass@128": 0.6646884272997032 |
| }, |
| "15": { |
| "pass@1": 0.16358024691358025, |
| "pass@2": 0.2345679012345679, |
| "pass@4": 0.2716049382716049, |
| "pass@8": 0.2962962962962963, |
| "pass@16": 0.3611111111111111, |
| "pass@32": 0.41975308641975306, |
| "pass@64": 0.4783950617283951, |
| "pass@128": 0.5308641975308642 |
| }, |
| "16": { |
| "pass@1": 0.05604719764011799, |
| "pass@2": 0.09734513274336283, |
| "pass@4": 0.13274336283185842, |
| "pass@8": 0.168141592920354, |
| "pass@16": 0.19469026548672566, |
| "pass@32": 0.22418879056047197, |
| "pass@64": 0.26253687315634217, |
| "pass@128": 0.30383480825958703 |
| }, |
| "17": { |
| "pass@1": 0.04216867469879518, |
| "pass@2": 0.060240963855421686, |
| "pass@4": 0.08734939759036145, |
| "pass@8": 0.10240963855421686, |
| "pass@16": 0.12349397590361445, |
| "pass@32": 0.14156626506024098, |
| "pass@64": 0.1536144578313253, |
| "pass@128": 0.1716867469879518 |
| }, |
| "18": { |
| "pass@1": 0.02258064516129032, |
| "pass@2": 0.04516129032258064, |
| "pass@4": 0.04838709677419355, |
| "pass@8": 0.05806451612903226, |
| "pass@16": 0.06774193548387097, |
| "pass@32": 0.08064516129032258, |
| "pass@64": 0.09032258064516129, |
| "pass@128": 0.0967741935483871 |
| }, |
| "19": { |
| "pass@1": 0.006042296072507553, |
| "pass@2": 0.012084592145015106, |
| "pass@4": 0.015105740181268883, |
| "pass@8": 0.015105740181268883, |
| "pass@16": 0.015105740181268883, |
| "pass@32": 0.027190332326283987, |
| "pass@64": 0.030211480362537766, |
| "pass@128": 0.03927492447129909 |
| }, |
| "20": { |
| "pass@1": 0.006006006006006006, |
| "pass@2": 0.006006006006006006, |
| "pass@4": 0.009009009009009009, |
| "pass@8": 0.009009009009009009, |
| "pass@16": 0.009009009009009009, |
| "pass@32": 0.015015015015015015, |
| "pass@64": 0.015015015015015015, |
| "pass@128": 0.015015015015015015 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5714285714285714, |
| "12": 0.581039755351682, |
| "13": 0.4351851851851852, |
| "14": 0.2818991097922849, |
| "15": 0.16358024691358025, |
| "16": 0.05604719764011799, |
| "17": 0.04216867469879518, |
| "18": 0.02258064516129032, |
| "19": 0.006042296072507553, |
| "20": 0.006006006006006006 |
| }, |
| "pass@2": { |
| "11": 0.6285714285714286, |
| "12": 0.6513761467889908, |
| "13": 0.4845679012345679, |
| "14": 0.34124629080118696, |
| "15": 0.2345679012345679, |
| "16": 0.09734513274336283, |
| "17": 0.060240963855421686, |
| "18": 0.04516129032258064, |
| "19": 0.012084592145015106, |
| "20": 0.006006006006006006 |
| }, |
| "pass@4": { |
| "11": 0.7111111111111111, |
| "12": 0.7370030581039755, |
| "13": 0.558641975308642, |
| "14": 0.3827893175074184, |
| "15": 0.2716049382716049, |
| "16": 0.13274336283185842, |
| "17": 0.08734939759036145, |
| "18": 0.04838709677419355, |
| "19": 0.015105740181268883, |
| "20": 0.009009009009009009 |
| }, |
| "pass@8": { |
| "11": 0.8158730158730159, |
| "12": 0.8012232415902141, |
| "13": 0.6265432098765432, |
| "14": 0.45103857566765576, |
| "15": 0.2962962962962963, |
| "16": 0.168141592920354, |
| "17": 0.10240963855421686, |
| "18": 0.05806451612903226, |
| "19": 0.015105740181268883, |
| "20": 0.009009009009009009 |
| }, |
| "pass@16": { |
| "11": 0.9206349206349206, |
| "12": 0.8440366972477065, |
| "13": 0.6820987654320988, |
| "14": 0.5074183976261127, |
| "15": 0.3611111111111111, |
| "16": 0.19469026548672566, |
| "17": 0.12349397590361445, |
| "18": 0.06774193548387097, |
| "19": 0.015105740181268883, |
| "20": 0.009009009009009009 |
| }, |
| "pass@32": { |
| "11": 0.9523809523809523, |
| "12": 0.8593272171253823, |
| "13": 0.7222222222222222, |
| "14": 0.5667655786350149, |
| "15": 0.41975308641975306, |
| "16": 0.22418879056047197, |
| "17": 0.14156626506024098, |
| "18": 0.08064516129032258, |
| "19": 0.027190332326283987, |
| "20": 0.015015015015015015 |
| }, |
| "pass@64": { |
| "11": 0.9650793650793651, |
| "12": 0.8685015290519877, |
| "13": 0.7530864197530864, |
| "14": 0.6201780415430267, |
| "15": 0.4783950617283951, |
| "16": 0.26253687315634217, |
| "17": 0.1536144578313253, |
| "18": 0.09032258064516129, |
| "19": 0.030211480362537766, |
| "20": 0.015015015015015015 |
| }, |
| "pass@128": { |
| "11": 0.9777777777777777, |
| "12": 0.8776758409785933, |
| "13": 0.7746913580246914, |
| "14": 0.6646884272997032, |
| "15": 0.5308641975308642, |
| "16": 0.30383480825958703, |
| "17": 0.1716867469879518, |
| "18": 0.0967741935483871, |
| "19": 0.03927492447129909, |
| "20": 0.015015015015015015 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.21485330073349634, |
| "pass@2": 0.254278728606357, |
| "pass@4": 0.293398533007335, |
| "pass@8": 0.3322127139364303, |
| "pass@16": 0.37011002444987773, |
| "pass@32": 0.39853300733496333, |
| "pass@64": 0.42145476772616136, |
| "pass@128": 0.44315403422982885 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6144578313253012, |
| "pass@2": 0.6867469879518072, |
| "pass@4": 0.7469879518072289, |
| "pass@8": 0.8313253012048193, |
| "pass@16": 0.8975903614457831, |
| "pass@32": 0.9337349397590361, |
| "pass@64": 0.9608433734939759, |
| "pass@128": 0.9698795180722891 |
| }, |
| "12": { |
| "pass@1": 0.5203761755485894, |
| "pass@2": 0.5768025078369906, |
| "pass@4": 0.6739811912225705, |
| "pass@8": 0.7523510971786834, |
| "pass@16": 0.8056426332288401, |
| "pass@32": 0.8338557993730408, |
| "pass@64": 0.8620689655172413, |
| "pass@128": 0.8871473354231975 |
| }, |
| "13": { |
| "pass@1": 0.4425770308123249, |
| "pass@2": 0.5266106442577031, |
| "pass@4": 0.6190476190476191, |
| "pass@8": 0.6722689075630253, |
| "pass@16": 0.7226890756302521, |
| "pass@32": 0.7591036414565826, |
| "pass@64": 0.803921568627451, |
| "pass@128": 0.8263305322128851 |
| }, |
| "14": { |
| "pass@1": 0.30701754385964913, |
| "pass@2": 0.3654970760233918, |
| "pass@4": 0.41812865497076024, |
| "pass@8": 0.48830409356725146, |
| "pass@16": 0.5526315789473685, |
| "pass@32": 0.6228070175438597, |
| "pass@64": 0.6754385964912281, |
| "pass@128": 0.7397660818713451 |
| }, |
| "15": { |
| "pass@1": 0.1484848484848485, |
| "pass@2": 0.19393939393939394, |
| "pass@4": 0.22424242424242424, |
| "pass@8": 0.2696969696969697, |
| "pass@16": 0.31212121212121213, |
| "pass@32": 0.3696969696969697, |
| "pass@64": 0.43333333333333335, |
| "pass@128": 0.5212121212121212 |
| }, |
| "16": { |
| "pass@1": 0.08262108262108261, |
| "pass@2": 0.1339031339031339, |
| "pass@4": 0.15384615384615385, |
| "pass@8": 0.1908831908831909, |
| "pass@16": 0.2336182336182336, |
| "pass@32": 0.2905982905982906, |
| "pass@64": 0.32763532763532766, |
| "pass@128": 0.3817663817663818 |
| }, |
| "17": { |
| "pass@1": 0.034161490683229816, |
| "pass@2": 0.049689440993788817, |
| "pass@4": 0.06521739130434782, |
| "pass@8": 0.08074534161490683, |
| "pass@16": 0.10248447204968944, |
| "pass@32": 0.13043478260869565, |
| "pass@64": 0.16149068322981366, |
| "pass@128": 0.17701863354037267 |
| }, |
| "18": { |
| "pass@1": 0.023529411764705882, |
| "pass@2": 0.041176470588235294, |
| "pass@4": 0.052941176470588235, |
| "pass@8": 0.06764705882352941, |
| "pass@16": 0.08235294117647059, |
| "pass@32": 0.09705882352941177, |
| "pass@64": 0.11470588235294117, |
| "pass@128": 0.1323529411764706 |
| }, |
| "19": { |
| "pass@1": 0.005952380952380952, |
| "pass@2": 0.011904761904761904, |
| "pass@4": 0.017857142857142856, |
| "pass@8": 0.017857142857142856, |
| "pass@16": 0.023809523809523808, |
| "pass@32": 0.026785714285714284, |
| "pass@64": 0.02976190476190476, |
| "pass@128": 0.03571428571428571 |
| }, |
| "20": { |
| "pass@1": 0.008982035928143712, |
| "pass@2": 0.008982035928143712, |
| "pass@4": 0.011976047904191617, |
| "pass@8": 0.020958083832335328, |
| "pass@16": 0.023952095808383235, |
| "pass@32": 0.03293413173652695, |
| "pass@64": 0.03293413173652695, |
| "pass@128": 0.03592814371257485 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6144578313253012, |
| "12": 0.5203761755485894, |
| "13": 0.4425770308123249, |
| "14": 0.30701754385964913, |
| "15": 0.1484848484848485, |
| "16": 0.08262108262108261, |
| "17": 0.034161490683229816, |
| "18": 0.023529411764705882, |
| "19": 0.005952380952380952, |
| "20": 0.008982035928143712 |
| }, |
| "pass@2": { |
| "11": 0.6867469879518072, |
| "12": 0.5768025078369906, |
| "13": 0.5266106442577031, |
| "14": 0.3654970760233918, |
| "15": 0.19393939393939394, |
| "16": 0.1339031339031339, |
| "17": 0.049689440993788817, |
| "18": 0.041176470588235294, |
| "19": 0.011904761904761904, |
| "20": 0.008982035928143712 |
| }, |
| "pass@4": { |
| "11": 0.7469879518072289, |
| "12": 0.6739811912225705, |
| "13": 0.6190476190476191, |
| "14": 0.41812865497076024, |
| "15": 0.22424242424242424, |
| "16": 0.15384615384615385, |
| "17": 0.06521739130434782, |
| "18": 0.052941176470588235, |
| "19": 0.017857142857142856, |
| "20": 0.011976047904191617 |
| }, |
| "pass@8": { |
| "11": 0.8313253012048193, |
| "12": 0.7523510971786834, |
| "13": 0.6722689075630253, |
| "14": 0.48830409356725146, |
| "15": 0.2696969696969697, |
| "16": 0.1908831908831909, |
| "17": 0.08074534161490683, |
| "18": 0.06764705882352941, |
| "19": 0.017857142857142856, |
| "20": 0.020958083832335328 |
| }, |
| "pass@16": { |
| "11": 0.8975903614457831, |
| "12": 0.8056426332288401, |
| "13": 0.7226890756302521, |
| "14": 0.5526315789473685, |
| "15": 0.31212121212121213, |
| "16": 0.2336182336182336, |
| "17": 0.10248447204968944, |
| "18": 0.08235294117647059, |
| "19": 0.023809523809523808, |
| "20": 0.023952095808383235 |
| }, |
| "pass@32": { |
| "11": 0.9337349397590361, |
| "12": 0.8338557993730408, |
| "13": 0.7591036414565826, |
| "14": 0.6228070175438597, |
| "15": 0.3696969696969697, |
| "16": 0.2905982905982906, |
| "17": 0.13043478260869565, |
| "18": 0.09705882352941177, |
| "19": 0.026785714285714284, |
| "20": 0.03293413173652695 |
| }, |
| "pass@64": { |
| "11": 0.9608433734939759, |
| "12": 0.8620689655172413, |
| "13": 0.803921568627451, |
| "14": 0.6754385964912281, |
| "15": 0.43333333333333335, |
| "16": 0.32763532763532766, |
| "17": 0.16149068322981366, |
| "18": 0.11470588235294117, |
| "19": 0.02976190476190476, |
| "20": 0.03293413173652695 |
| }, |
| "pass@128": { |
| "11": 0.9698795180722891, |
| "12": 0.8871473354231975, |
| "13": 0.8263305322128851, |
| "14": 0.7397660818713451, |
| "15": 0.5212121212121212, |
| "16": 0.3817663817663818, |
| "17": 0.17701863354037267, |
| "18": 0.1323529411764706, |
| "19": 0.03571428571428571, |
| "20": 0.03592814371257485 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.21855486173059768, |
| "pass@2": 0.25958965209634255, |
| "pass@4": 0.29854296758846266, |
| "pass@8": 0.33928040440083257, |
| "pass@16": 0.375854891465953, |
| "pass@32": 0.41005055010407376, |
| "pass@64": 0.4406779661016949, |
| "pass@128": 0.4713053820993161 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.588, |
| "pass@2": 0.657, |
| "pass@4": 0.727, |
| "pass@8": 0.823, |
| "pass@16": 0.904, |
| "pass@32": 0.94, |
| "pass@64": 0.958, |
| "pass@128": 0.967 |
| }, |
| "12": { |
| "pass@1": 0.568, |
| "pass@2": 0.627, |
| "pass@4": 0.706, |
| "pass@8": 0.778, |
| "pass@16": 0.824, |
| "pass@32": 0.845, |
| "pass@64": 0.862, |
| "pass@128": 0.876 |
| }, |
| "13": { |
| "pass@1": 0.446, |
| "pass@2": 0.511, |
| "pass@4": 0.586, |
| "pass@8": 0.642, |
| "pass@16": 0.694, |
| "pass@32": 0.736, |
| "pass@64": 0.776, |
| "pass@128": 0.794 |
| }, |
| "14": { |
| "pass@1": 0.307, |
| "pass@2": 0.367, |
| "pass@4": 0.42, |
| "pass@8": 0.485, |
| "pass@16": 0.542, |
| "pass@32": 0.605, |
| "pass@64": 0.657, |
| "pass@128": 0.707 |
| }, |
| "15": { |
| "pass@1": 0.164, |
| "pass@2": 0.225, |
| "pass@4": 0.272, |
| "pass@8": 0.314, |
| "pass@16": 0.362, |
| "pass@32": 0.419, |
| "pass@64": 0.481, |
| "pass@128": 0.549 |
| }, |
| "16": { |
| "pass@1": 0.064, |
| "pass@2": 0.108, |
| "pass@4": 0.136, |
| "pass@8": 0.169, |
| "pass@16": 0.211, |
| "pass@32": 0.251, |
| "pass@64": 0.289, |
| "pass@128": 0.337 |
| }, |
| "17": { |
| "pass@1": 0.044, |
| "pass@2": 0.061, |
| "pass@4": 0.079, |
| "pass@8": 0.098, |
| "pass@16": 0.119, |
| "pass@32": 0.141, |
| "pass@64": 0.166, |
| "pass@128": 0.19 |
| }, |
| "18": { |
| "pass@1": 0.018, |
| "pass@2": 0.033, |
| "pass@4": 0.042, |
| "pass@8": 0.052, |
| "pass@16": 0.064, |
| "pass@32": 0.076, |
| "pass@64": 0.089, |
| "pass@128": 0.102 |
| }, |
| "19": { |
| "pass@1": 0.006, |
| "pass@2": 0.01, |
| "pass@4": 0.015, |
| "pass@8": 0.015, |
| "pass@16": 0.018, |
| "pass@32": 0.024, |
| "pass@64": 0.028, |
| "pass@128": 0.038 |
| }, |
| "20": { |
| "pass@1": 0.006, |
| "pass@2": 0.007, |
| "pass@4": 0.012, |
| "pass@8": 0.017, |
| "pass@16": 0.02, |
| "pass@32": 0.026, |
| "pass@64": 0.03, |
| "pass@128": 0.036 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.588, |
| "12": 0.568, |
| "13": 0.446, |
| "14": 0.307, |
| "15": 0.164, |
| "16": 0.064, |
| "17": 0.044, |
| "18": 0.018, |
| "19": 0.006, |
| "20": 0.006 |
| }, |
| "pass@2": { |
| "11": 0.657, |
| "12": 0.627, |
| "13": 0.511, |
| "14": 0.367, |
| "15": 0.225, |
| "16": 0.108, |
| "17": 0.061, |
| "18": 0.033, |
| "19": 0.01, |
| "20": 0.007 |
| }, |
| "pass@4": { |
| "11": 0.727, |
| "12": 0.706, |
| "13": 0.586, |
| "14": 0.42, |
| "15": 0.272, |
| "16": 0.136, |
| "17": 0.079, |
| "18": 0.042, |
| "19": 0.015, |
| "20": 0.012 |
| }, |
| "pass@8": { |
| "11": 0.823, |
| "12": 0.778, |
| "13": 0.642, |
| "14": 0.485, |
| "15": 0.314, |
| "16": 0.169, |
| "17": 0.098, |
| "18": 0.052, |
| "19": 0.015, |
| "20": 0.017 |
| }, |
| "pass@16": { |
| "11": 0.904, |
| "12": 0.824, |
| "13": 0.694, |
| "14": 0.542, |
| "15": 0.362, |
| "16": 0.211, |
| "17": 0.119, |
| "18": 0.064, |
| "19": 0.018, |
| "20": 0.02 |
| }, |
| "pass@32": { |
| "11": 0.94, |
| "12": 0.845, |
| "13": 0.736, |
| "14": 0.605, |
| "15": 0.419, |
| "16": 0.251, |
| "17": 0.141, |
| "18": 0.076, |
| "19": 0.024, |
| "20": 0.026 |
| }, |
| "pass@64": { |
| "11": 0.958, |
| "12": 0.862, |
| "13": 0.776, |
| "14": 0.657, |
| "15": 0.481, |
| "16": 0.289, |
| "17": 0.166, |
| "18": 0.089, |
| "19": 0.028, |
| "20": 0.03 |
| }, |
| "pass@128": { |
| "11": 0.967, |
| "12": 0.876, |
| "13": 0.794, |
| "14": 0.707, |
| "15": 0.549, |
| "16": 0.337, |
| "17": 0.19, |
| "18": 0.102, |
| "19": 0.038, |
| "20": 0.036 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.2211, |
| "pass@2": 0.2606, |
| "pass@4": 0.2995, |
| "pass@8": 0.3393, |
| "pass@16": 0.3758, |
| "pass@32": 0.4063, |
| "pass@64": 0.4336, |
| "pass@128": 0.4596 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
