| { |
| "files": { |
| "checkpoint-38796_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.990625, |
| "pass@2": 0.99375, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9619047619047619, |
| "pass@2": 0.9873015873015873, |
| "pass@4": 0.9904761904761905, |
| "pass@8": 0.9968253968253968, |
| "pass@16": 0.9968253968253968, |
| "pass@32": 0.9968253968253968, |
| "pass@64": 0.9968253968253968, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9700598802395209, |
| "pass@2": 0.9790419161676647, |
| "pass@4": 0.9940119760479041, |
| "pass@8": 0.9970059880239521, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.878698224852071, |
| "pass@2": 0.9319526627218935, |
| "pass@4": 0.9644970414201184, |
| "pass@8": 0.985207100591716, |
| "pass@16": 0.9911242603550295, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8365384615384616, |
| "pass@2": 0.8942307692307693, |
| "pass@4": 0.9423076923076923, |
| "pass@8": 0.9775641025641025, |
| "pass@16": 0.9967948717948718, |
| "pass@32": 0.9967948717948718, |
| "pass@64": 0.9967948717948718, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7633802816901408, |
| "pass@2": 0.8225352112676056, |
| "pass@4": 0.9042253521126761, |
| "pass@8": 0.9464788732394366, |
| "pass@16": 0.9859154929577465, |
| "pass@32": 0.9943661971830986, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6548672566371682, |
| "pass@2": 0.7020648967551623, |
| "pass@4": 0.799410029498525, |
| "pass@8": 0.8584070796460177, |
| "pass@16": 0.8938053097345132, |
| "pass@32": 0.9292035398230089, |
| "pass@64": 0.9498525073746312, |
| "pass@128": 0.9705014749262537 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.990625, |
| "5": 0.9619047619047619, |
| "6": 0.9700598802395209, |
| "7": 0.878698224852071, |
| "8": 0.8365384615384616, |
| "9": 0.7633802816901408, |
| "10": 0.6548672566371682 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9873015873015873, |
| "6": 0.9790419161676647, |
| "7": 0.9319526627218935, |
| "8": 0.8942307692307693, |
| "9": 0.8225352112676056, |
| "10": 0.7020648967551623 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9904761904761905, |
| "6": 0.9940119760479041, |
| "7": 0.9644970414201184, |
| "8": 0.9423076923076923, |
| "9": 0.9042253521126761, |
| "10": 0.799410029498525 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 0.9970059880239521, |
| "7": 0.985207100591716, |
| "8": 0.9775641025641025, |
| "9": 0.9464788732394366, |
| "10": 0.8584070796460177 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 0.9911242603550295, |
| "8": 0.9967948717948718, |
| "9": 0.9859154929577465, |
| "10": 0.8938053097345132 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9967948717948718, |
| "9": 0.9943661971830986, |
| "10": 0.9292035398230089 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9967948717948718, |
| "9": 1.0, |
| "10": 0.9498525073746312 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9705014749262537 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8926038500506586, |
| "pass@2": 0.9213103681188788, |
| "pass@4": 0.9537318473488686, |
| "pass@8": 0.9726443768996961, |
| "pass@16": 0.9844647078689632, |
| "pass@32": 0.9905437352245863, |
| "pass@64": 0.9935832489023978, |
| "pass@128": 0.9966227625802094 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9970845481049563, |
| "pass@2": 0.9970845481049563, |
| "pass@4": 0.9970845481049563, |
| "pass@8": 0.9970845481049563, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9912280701754386, |
| "pass@2": 0.9941520467836257, |
| "pass@4": 0.9970760233918129, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9558823529411765, |
| "pass@2": 0.9823529411764705, |
| "pass@4": 0.9970588235294118, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9589905362776026, |
| "pass@2": 0.9747634069400631, |
| "pass@4": 0.9810725552050473, |
| "pass@8": 0.9873817034700315, |
| "pass@16": 0.9936908517350158, |
| "pass@32": 0.9968454258675079, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8765822784810127, |
| "pass@2": 0.9113924050632911, |
| "pass@4": 0.9620253164556962, |
| "pass@8": 0.990506329113924, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 0.9968354430379747, |
| "pass@64": 0.9968354430379747, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.8198198198198198, |
| "pass@2": 0.8858858858858859, |
| "pass@4": 0.9369369369369369, |
| "pass@8": 0.963963963963964, |
| "pass@16": 0.987987987987988, |
| "pass@32": 0.996996996996997, |
| "pass@64": 0.996996996996997, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7620578778135049, |
| "pass@2": 0.8231511254019293, |
| "pass@4": 0.8971061093247589, |
| "pass@8": 0.9581993569131833, |
| "pass@16": 0.9935691318327974, |
| "pass@32": 0.9935691318327974, |
| "pass@64": 0.9967845659163987, |
| "pass@128": 0.9967845659163987 |
| }, |
| "10": { |
| "pass@1": 0.6179104477611941, |
| "pass@2": 0.6776119402985075, |
| "pass@4": 0.764179104477612, |
| "pass@8": 0.8477611940298507, |
| "pass@16": 0.9194029850746268, |
| "pass@32": 0.9582089552238806, |
| "pass@64": 0.9761194029850746, |
| "pass@128": 0.9791044776119403 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.9970845481049563, |
| "4": 0.9912280701754386, |
| "5": 0.9558823529411765, |
| "6": 0.9589905362776026, |
| "7": 0.8765822784810127, |
| "8": 0.8198198198198198, |
| "9": 0.7620578778135049, |
| "10": 0.6179104477611941 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 0.9970845481049563, |
| "4": 0.9941520467836257, |
| "5": 0.9823529411764705, |
| "6": 0.9747634069400631, |
| "7": 0.9113924050632911, |
| "8": 0.8858858858858859, |
| "9": 0.8231511254019293, |
| "10": 0.6776119402985075 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 0.9970845481049563, |
| "4": 0.9970760233918129, |
| "5": 0.9970588235294118, |
| "6": 0.9810725552050473, |
| "7": 0.9620253164556962, |
| "8": 0.9369369369369369, |
| "9": 0.8971061093247589, |
| "10": 0.764179104477612 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 0.9970845481049563, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9873817034700315, |
| "7": 0.990506329113924, |
| "8": 0.963963963963964, |
| "9": 0.9581993569131833, |
| "10": 0.8477611940298507 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9936908517350158, |
| "7": 0.9968354430379747, |
| "8": 0.987987987987988, |
| "9": 0.9935691318327974, |
| "10": 0.9194029850746268 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9968454258675079, |
| "7": 0.9968354430379747, |
| "8": 0.996996996996997, |
| "9": 0.9935691318327974, |
| "10": 0.9582089552238806 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.996996996996997, |
| "9": 0.9967845659163987, |
| "10": 0.9761194029850746 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.9791044776119403 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8885169065952461, |
| "pass@2": 0.9176431201874791, |
| "pass@4": 0.9487780381653833, |
| "pass@8": 0.9718781386006026, |
| "pass@16": 0.9879477736859725, |
| "pass@32": 0.9936391027787077, |
| "pass@64": 0.9963173752929361, |
| "pass@128": 0.9969869434214932 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.996969696969697, |
| "pass@2": 0.996969696969697, |
| "pass@4": 0.996969696969697, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9792899408284024, |
| "pass@2": 0.9911242603550295, |
| "pass@4": 0.9970414201183432, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9652173913043478, |
| "pass@2": 0.9739130434782609, |
| "pass@4": 0.9971014492753624, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9627507163323782, |
| "pass@2": 0.9742120343839542, |
| "pass@4": 0.9828080229226361, |
| "pass@8": 0.994269340974212, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.869942196531792, |
| "pass@2": 0.9219653179190751, |
| "pass@4": 0.9710982658959537, |
| "pass@8": 0.9855491329479769, |
| "pass@16": 0.9971098265895953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8225352112676056, |
| "pass@2": 0.8816901408450705, |
| "pass@4": 0.9352112676056338, |
| "pass@8": 0.971830985915493, |
| "pass@16": 0.9943661971830986, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7514970059880239, |
| "pass@2": 0.8083832335329342, |
| "pass@4": 0.8982035928143712, |
| "pass@8": 0.9520958083832335, |
| "pass@16": 0.9850299401197605, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.656441717791411, |
| "pass@2": 0.7208588957055214, |
| "pass@4": 0.7576687116564417, |
| "pass@8": 0.8374233128834356, |
| "pass@16": 0.8895705521472392, |
| "pass@32": 0.9294478527607362, |
| "pass@64": 0.9386503067484663, |
| "pass@128": 0.9631901840490797 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 0.9792899408284024, |
| "5": 0.9652173913043478, |
| "6": 0.9627507163323782, |
| "7": 0.869942196531792, |
| "8": 0.8225352112676056, |
| "9": 0.7514970059880239, |
| "10": 0.656441717791411 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 0.9911242603550295, |
| "5": 0.9739130434782609, |
| "6": 0.9742120343839542, |
| "7": 0.9219653179190751, |
| "8": 0.8816901408450705, |
| "9": 0.8083832335329342, |
| "10": 0.7208588957055214 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 0.9970414201183432, |
| "5": 0.9971014492753624, |
| "6": 0.9828080229226361, |
| "7": 0.9710982658959537, |
| "8": 0.9352112676056338, |
| "9": 0.8982035928143712, |
| "10": 0.7576687116564417 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.994269340974212, |
| "7": 0.9855491329479769, |
| "8": 0.971830985915493, |
| "9": 0.9520958083832335, |
| "10": 0.8374233128834356 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9971098265895953, |
| "8": 0.9943661971830986, |
| "9": 0.9850299401197605, |
| "10": 0.8895705521472392 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9940119760479041, |
| "10": 0.9294478527607362 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9386503067484663 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9631901840490797 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8899082568807339, |
| "pass@2": 0.919397116644823, |
| "pass@4": 0.9492136304062909, |
| "pass@8": 0.971821756225426, |
| "pass@16": 0.9855832241153342, |
| "pass@32": 0.991480996068152, |
| "pass@64": 0.9934469200524246, |
| "pass@128": 0.9960681520314548 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.998, |
| "pass@2": 0.998, |
| "pass@4": 0.998, |
| "pass@8": 0.999, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.987, |
| "pass@2": 0.993, |
| "pass@4": 0.998, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.961, |
| "pass@2": 0.981, |
| "pass@4": 0.995, |
| "pass@8": 0.999, |
| "pass@16": 0.999, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.964, |
| "pass@2": 0.976, |
| "pass@4": 0.986, |
| "pass@8": 0.993, |
| "pass@16": 0.998, |
| "pass@32": 0.999, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.875, |
| "pass@2": 0.922, |
| "pass@4": 0.966, |
| "pass@8": 0.987, |
| "pass@16": 0.995, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "8": { |
| "pass@1": 0.826, |
| "pass@2": 0.887, |
| "pass@4": 0.938, |
| "pass@8": 0.971, |
| "pass@16": 0.993, |
| "pass@32": 0.997, |
| "pass@64": 0.998, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.759, |
| "pass@2": 0.818, |
| "pass@4": 0.9, |
| "pass@8": 0.952, |
| "pass@16": 0.988, |
| "pass@32": 0.994, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "10": { |
| "pass@1": 0.643, |
| "pass@2": 0.7, |
| "pass@4": 0.774, |
| "pass@8": 0.848, |
| "pass@16": 0.901, |
| "pass@32": 0.939, |
| "pass@64": 0.955, |
| "pass@128": 0.971 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.998, |
| "4": 0.987, |
| "5": 0.961, |
| "6": 0.964, |
| "7": 0.875, |
| "8": 0.826, |
| "9": 0.759, |
| "10": 0.643 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 0.998, |
| "4": 0.993, |
| "5": 0.981, |
| "6": 0.976, |
| "7": 0.922, |
| "8": 0.887, |
| "9": 0.818, |
| "10": 0.7 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 0.998, |
| "4": 0.998, |
| "5": 0.995, |
| "6": 0.986, |
| "7": 0.966, |
| "8": 0.938, |
| "9": 0.9, |
| "10": 0.774 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.993, |
| "7": 0.987, |
| "8": 0.971, |
| "9": 0.952, |
| "10": 0.848 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.998, |
| "7": 0.995, |
| "8": 0.993, |
| "9": 0.988, |
| "10": 0.901 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.999, |
| "7": 0.999, |
| "8": 0.997, |
| "9": 0.994, |
| "10": 0.939 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 0.998, |
| "9": 0.999, |
| "10": 0.955 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.971 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8903333333333333, |
| "pass@2": 0.9194444444444444, |
| "pass@4": 0.9505555555555556, |
| "pass@8": 0.9721111111111111, |
| "pass@16": 0.986, |
| "pass@32": 0.9918888888888889, |
| "pass@64": 0.9944444444444445, |
| "pass@128": 0.9965555555555555 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-38796_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5807365439093485, |
| "pass@2": 0.6742209631728046, |
| "pass@4": 0.7478753541076487, |
| "pass@8": 0.8583569405099151, |
| "pass@16": 0.9263456090651558, |
| "pass@32": 0.9546742209631728, |
| "pass@64": 0.9688385269121813, |
| "pass@128": 0.9745042492917847 |
| }, |
| "12": { |
| "pass@1": 0.632768361581921, |
| "pass@2": 0.6864406779661016, |
| "pass@4": 0.7627118644067796, |
| "pass@8": 0.8531073446327684, |
| "pass@16": 0.9124293785310734, |
| "pass@32": 0.940677966101695, |
| "pass@64": 0.96045197740113, |
| "pass@128": 0.9745762711864406 |
| }, |
| "13": { |
| "pass@1": 0.5862068965517241, |
| "pass@2": 0.664576802507837, |
| "pass@4": 0.7115987460815048, |
| "pass@8": 0.780564263322884, |
| "pass@16": 0.8401253918495298, |
| "pass@32": 0.896551724137931, |
| "pass@64": 0.9310344827586207, |
| "pass@128": 0.9529780564263323 |
| }, |
| "14": { |
| "pass@1": 0.5420560747663551, |
| "pass@2": 0.5825545171339563, |
| "pass@4": 0.6448598130841121, |
| "pass@8": 0.6978193146417445, |
| "pass@16": 0.7757009345794392, |
| "pass@32": 0.822429906542056, |
| "pass@64": 0.8691588785046729, |
| "pass@128": 0.9252336448598131 |
| }, |
| "15": { |
| "pass@1": 0.4797687861271676, |
| "pass@2": 0.5173410404624278, |
| "pass@4": 0.5635838150289018, |
| "pass@8": 0.6242774566473989, |
| "pass@16": 0.7109826589595376, |
| "pass@32": 0.7947976878612717, |
| "pass@64": 0.8930635838150289, |
| "pass@128": 0.9421965317919075 |
| }, |
| "16": { |
| "pass@1": 0.44193548387096776, |
| "pass@2": 0.5, |
| "pass@4": 0.5387096774193548, |
| "pass@8": 0.5645161290322581, |
| "pass@16": 0.6419354838709678, |
| "pass@32": 0.7032258064516129, |
| "pass@64": 0.7612903225806451, |
| "pass@128": 0.7903225806451613 |
| }, |
| "17": { |
| "pass@1": 0.49421965317919075, |
| "pass@2": 0.5375722543352601, |
| "pass@4": 0.5867052023121387, |
| "pass@8": 0.6329479768786127, |
| "pass@16": 0.7023121387283237, |
| "pass@32": 0.7485549132947977, |
| "pass@64": 0.7976878612716763, |
| "pass@128": 0.846820809248555 |
| }, |
| "18": { |
| "pass@1": 0.4142857142857143, |
| "pass@2": 0.4742857142857143, |
| "pass@4": 0.5028571428571429, |
| "pass@8": 0.5457142857142857, |
| "pass@16": 0.5914285714285714, |
| "pass@32": 0.64, |
| "pass@64": 0.7114285714285714, |
| "pass@128": 0.7571428571428571 |
| }, |
| "19": { |
| "pass@1": 0.3303303303303303, |
| "pass@2": 0.3963963963963964, |
| "pass@4": 0.45345345345345345, |
| "pass@8": 0.4774774774774775, |
| "pass@16": 0.5165165165165165, |
| "pass@32": 0.5525525525525525, |
| "pass@64": 0.6186186186186187, |
| "pass@128": 0.6606606606606606 |
| }, |
| "20": { |
| "pass@1": 0.3003003003003003, |
| "pass@2": 0.34234234234234234, |
| "pass@4": 0.3963963963963964, |
| "pass@8": 0.45645645645645644, |
| "pass@16": 0.4864864864864865, |
| "pass@32": 0.5435435435435435, |
| "pass@64": 0.5795795795795796, |
| "pass@128": 0.6396396396396397 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5807365439093485, |
| "12": 0.632768361581921, |
| "13": 0.5862068965517241, |
| "14": 0.5420560747663551, |
| "15": 0.4797687861271676, |
| "16": 0.44193548387096776, |
| "17": 0.49421965317919075, |
| "18": 0.4142857142857143, |
| "19": 0.3303303303303303, |
| "20": 0.3003003003003003 |
| }, |
| "pass@2": { |
| "11": 0.6742209631728046, |
| "12": 0.6864406779661016, |
| "13": 0.664576802507837, |
| "14": 0.5825545171339563, |
| "15": 0.5173410404624278, |
| "16": 0.5, |
| "17": 0.5375722543352601, |
| "18": 0.4742857142857143, |
| "19": 0.3963963963963964, |
| "20": 0.34234234234234234 |
| }, |
| "pass@4": { |
| "11": 0.7478753541076487, |
| "12": 0.7627118644067796, |
| "13": 0.7115987460815048, |
| "14": 0.6448598130841121, |
| "15": 0.5635838150289018, |
| "16": 0.5387096774193548, |
| "17": 0.5867052023121387, |
| "18": 0.5028571428571429, |
| "19": 0.45345345345345345, |
| "20": 0.3963963963963964 |
| }, |
| "pass@8": { |
| "11": 0.8583569405099151, |
| "12": 0.8531073446327684, |
| "13": 0.780564263322884, |
| "14": 0.6978193146417445, |
| "15": 0.6242774566473989, |
| "16": 0.5645161290322581, |
| "17": 0.6329479768786127, |
| "18": 0.5457142857142857, |
| "19": 0.4774774774774775, |
| "20": 0.45645645645645644 |
| }, |
| "pass@16": { |
| "11": 0.9263456090651558, |
| "12": 0.9124293785310734, |
| "13": 0.8401253918495298, |
| "14": 0.7757009345794392, |
| "15": 0.7109826589595376, |
| "16": 0.6419354838709678, |
| "17": 0.7023121387283237, |
| "18": 0.5914285714285714, |
| "19": 0.5165165165165165, |
| "20": 0.4864864864864865 |
| }, |
| "pass@32": { |
| "11": 0.9546742209631728, |
| "12": 0.940677966101695, |
| "13": 0.896551724137931, |
| "14": 0.822429906542056, |
| "15": 0.7947976878612717, |
| "16": 0.7032258064516129, |
| "17": 0.7485549132947977, |
| "18": 0.64, |
| "19": 0.5525525525525525, |
| "20": 0.5435435435435435 |
| }, |
| "pass@64": { |
| "11": 0.9688385269121813, |
| "12": 0.96045197740113, |
| "13": 0.9310344827586207, |
| "14": 0.8691588785046729, |
| "15": 0.8930635838150289, |
| "16": 0.7612903225806451, |
| "17": 0.7976878612716763, |
| "18": 0.7114285714285714, |
| "19": 0.6186186186186187, |
| "20": 0.5795795795795796 |
| }, |
| "pass@128": { |
| "11": 0.9745042492917847, |
| "12": 0.9745762711864406, |
| "13": 0.9529780564263323, |
| "14": 0.9252336448598131, |
| "15": 0.9421965317919075, |
| "16": 0.7903225806451613, |
| "17": 0.846820809248555, |
| "18": 0.7571428571428571, |
| "19": 0.6606606606606606, |
| "20": 0.6396396396396397 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4811292719167905, |
| "pass@2": 0.538484398216939, |
| "pass@4": 0.5919762258543834, |
| "pass@8": 0.6508172362555721, |
| "pass@16": 0.712035661218425, |
| "pass@32": 0.7610698365527488, |
| "pass@64": 0.8104011887072808, |
| "pass@128": 0.8475482912332838 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6031746031746031, |
| "pass@2": 0.6888888888888889, |
| "pass@4": 0.7777777777777778, |
| "pass@8": 0.8571428571428571, |
| "pass@16": 0.9333333333333333, |
| "pass@32": 0.9714285714285714, |
| "pass@64": 0.9777777777777777, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.6269113149847095, |
| "pass@2": 0.6880733944954128, |
| "pass@4": 0.7614678899082569, |
| "pass@8": 0.8532110091743119, |
| "pass@16": 0.9113149847094801, |
| "pass@32": 0.944954128440367, |
| "pass@64": 0.9694189602446484, |
| "pass@128": 0.9785932721712538 |
| }, |
| "13": { |
| "pass@1": 0.595679012345679, |
| "pass@2": 0.654320987654321, |
| "pass@4": 0.7407407407407407, |
| "pass@8": 0.8179012345679012, |
| "pass@16": 0.8703703703703703, |
| "pass@32": 0.8981481481481481, |
| "pass@64": 0.9320987654320988, |
| "pass@128": 0.9475308641975309 |
| }, |
| "14": { |
| "pass@1": 0.5400593471810089, |
| "pass@2": 0.5845697329376854, |
| "pass@4": 0.6320474777448071, |
| "pass@8": 0.6913946587537092, |
| "pass@16": 0.7655786350148368, |
| "pass@32": 0.8308605341246291, |
| "pass@64": 0.9109792284866469, |
| "pass@128": 0.9317507418397626 |
| }, |
| "15": { |
| "pass@1": 0.4783950617283951, |
| "pass@2": 0.5154320987654321, |
| "pass@4": 0.5617283950617284, |
| "pass@8": 0.6203703703703703, |
| "pass@16": 0.7253086419753086, |
| "pass@32": 0.8024691358024691, |
| "pass@64": 0.8888888888888888, |
| "pass@128": 0.9320987654320988 |
| }, |
| "16": { |
| "pass@1": 0.49557522123893805, |
| "pass@2": 0.5457227138643068, |
| "pass@4": 0.6017699115044248, |
| "pass@8": 0.6430678466076696, |
| "pass@16": 0.6991150442477876, |
| "pass@32": 0.7551622418879056, |
| "pass@64": 0.799410029498525, |
| "pass@128": 0.831858407079646 |
| }, |
| "17": { |
| "pass@1": 0.45180722891566266, |
| "pass@2": 0.48493975903614456, |
| "pass@4": 0.5120481927710844, |
| "pass@8": 0.5512048192771084, |
| "pass@16": 0.5963855421686747, |
| "pass@32": 0.6837349397590361, |
| "pass@64": 0.7680722891566265, |
| "pass@128": 0.8102409638554217 |
| }, |
| "18": { |
| "pass@1": 0.3935483870967742, |
| "pass@2": 0.45806451612903226, |
| "pass@4": 0.5161290322580645, |
| "pass@8": 0.5451612903225806, |
| "pass@16": 0.603225806451613, |
| "pass@32": 0.6645161290322581, |
| "pass@64": 0.7258064516129032, |
| "pass@128": 0.8096774193548387 |
| }, |
| "19": { |
| "pass@1": 0.3474320241691843, |
| "pass@2": 0.4108761329305136, |
| "pass@4": 0.45317220543806647, |
| "pass@8": 0.486404833836858, |
| "pass@16": 0.5377643504531722, |
| "pass@32": 0.5800604229607251, |
| "pass@64": 0.6253776435045317, |
| "pass@128": 0.6616314199395771 |
| }, |
| "20": { |
| "pass@1": 0.2882882882882883, |
| "pass@2": 0.3663663663663664, |
| "pass@4": 0.4114114114114114, |
| "pass@8": 0.44144144144144143, |
| "pass@16": 0.4804804804804805, |
| "pass@32": 0.5255255255255256, |
| "pass@64": 0.5765765765765766, |
| "pass@128": 0.6456456456456456 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6031746031746031, |
| "12": 0.6269113149847095, |
| "13": 0.595679012345679, |
| "14": 0.5400593471810089, |
| "15": 0.4783950617283951, |
| "16": 0.49557522123893805, |
| "17": 0.45180722891566266, |
| "18": 0.3935483870967742, |
| "19": 0.3474320241691843, |
| "20": 0.2882882882882883 |
| }, |
| "pass@2": { |
| "11": 0.6888888888888889, |
| "12": 0.6880733944954128, |
| "13": 0.654320987654321, |
| "14": 0.5845697329376854, |
| "15": 0.5154320987654321, |
| "16": 0.5457227138643068, |
| "17": 0.48493975903614456, |
| "18": 0.45806451612903226, |
| "19": 0.4108761329305136, |
| "20": 0.3663663663663664 |
| }, |
| "pass@4": { |
| "11": 0.7777777777777778, |
| "12": 0.7614678899082569, |
| "13": 0.7407407407407407, |
| "14": 0.6320474777448071, |
| "15": 0.5617283950617284, |
| "16": 0.6017699115044248, |
| "17": 0.5120481927710844, |
| "18": 0.5161290322580645, |
| "19": 0.45317220543806647, |
| "20": 0.4114114114114114 |
| }, |
| "pass@8": { |
| "11": 0.8571428571428571, |
| "12": 0.8532110091743119, |
| "13": 0.8179012345679012, |
| "14": 0.6913946587537092, |
| "15": 0.6203703703703703, |
| "16": 0.6430678466076696, |
| "17": 0.5512048192771084, |
| "18": 0.5451612903225806, |
| "19": 0.486404833836858, |
| "20": 0.44144144144144143 |
| }, |
| "pass@16": { |
| "11": 0.9333333333333333, |
| "12": 0.9113149847094801, |
| "13": 0.8703703703703703, |
| "14": 0.7655786350148368, |
| "15": 0.7253086419753086, |
| "16": 0.6991150442477876, |
| "17": 0.5963855421686747, |
| "18": 0.603225806451613, |
| "19": 0.5377643504531722, |
| "20": 0.4804804804804805 |
| }, |
| "pass@32": { |
| "11": 0.9714285714285714, |
| "12": 0.944954128440367, |
| "13": 0.8981481481481481, |
| "14": 0.8308605341246291, |
| "15": 0.8024691358024691, |
| "16": 0.7551622418879056, |
| "17": 0.6837349397590361, |
| "18": 0.6645161290322581, |
| "19": 0.5800604229607251, |
| "20": 0.5255255255255256 |
| }, |
| "pass@64": { |
| "11": 0.9777777777777777, |
| "12": 0.9694189602446484, |
| "13": 0.9320987654320988, |
| "14": 0.9109792284866469, |
| "15": 0.8888888888888888, |
| "16": 0.799410029498525, |
| "17": 0.7680722891566265, |
| "18": 0.7258064516129032, |
| "19": 0.6253776435045317, |
| "20": 0.5765765765765766 |
| }, |
| "pass@128": { |
| "11": 0.9936507936507937, |
| "12": 0.9785932721712538, |
| "13": 0.9475308641975309, |
| "14": 0.9317507418397626, |
| "15": 0.9320987654320988, |
| "16": 0.831858407079646, |
| "17": 0.8102409638554217, |
| "18": 0.8096774193548387, |
| "19": 0.6616314199395771, |
| "20": 0.6456456456456456 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.48166259168704156, |
| "pass@2": 0.539119804400978, |
| "pass@4": 0.5959657701711492, |
| "pass@8": 0.6497555012224939, |
| "pass@16": 0.7111858190709046, |
| "pass@32": 0.7646699266503667, |
| "pass@64": 0.8166259168704156, |
| "pass@128": 0.8533007334963325 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6385542168674698, |
| "pass@2": 0.6837349397590361, |
| "pass@4": 0.7771084337349398, |
| "pass@8": 0.8373493975903614, |
| "pass@16": 0.8855421686746988, |
| "pass@32": 0.9367469879518072, |
| "pass@64": 0.9728915662650602, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.5705329153605015, |
| "pass@2": 0.6394984326018809, |
| "pass@4": 0.7429467084639498, |
| "pass@8": 0.8338557993730408, |
| "pass@16": 0.896551724137931, |
| "pass@32": 0.9498432601880877, |
| "pass@64": 0.9623824451410659, |
| "pass@128": 0.9811912225705329 |
| }, |
| "13": { |
| "pass@1": 0.6190476190476191, |
| "pass@2": 0.6694677871148459, |
| "pass@4": 0.7535014005602241, |
| "pass@8": 0.8487394957983193, |
| "pass@16": 0.9019607843137255, |
| "pass@32": 0.927170868347339, |
| "pass@64": 0.9439775910364145, |
| "pass@128": 0.969187675070028 |
| }, |
| "14": { |
| "pass@1": 0.5467836257309941, |
| "pass@2": 0.5847953216374269, |
| "pass@4": 0.631578947368421, |
| "pass@8": 0.6783625730994152, |
| "pass@16": 0.7865497076023392, |
| "pass@32": 0.8713450292397661, |
| "pass@64": 0.9210526315789473, |
| "pass@128": 0.9590643274853801 |
| }, |
| "15": { |
| "pass@1": 0.509090909090909, |
| "pass@2": 0.5545454545454546, |
| "pass@4": 0.6181818181818182, |
| "pass@8": 0.6848484848484848, |
| "pass@16": 0.7454545454545455, |
| "pass@32": 0.8181818181818182, |
| "pass@64": 0.8909090909090909, |
| "pass@128": 0.9393939393939394 |
| }, |
| "16": { |
| "pass@1": 0.5042735042735043, |
| "pass@2": 0.5356125356125356, |
| "pass@4": 0.5641025641025641, |
| "pass@8": 0.5925925925925926, |
| "pass@16": 0.6495726495726496, |
| "pass@32": 0.7207977207977208, |
| "pass@64": 0.7977207977207977, |
| "pass@128": 0.8376068376068376 |
| }, |
| "17": { |
| "pass@1": 0.39751552795031053, |
| "pass@2": 0.4440993788819876, |
| "pass@4": 0.48757763975155277, |
| "pass@8": 0.5279503105590062, |
| "pass@16": 0.5807453416149069, |
| "pass@32": 0.6708074534161491, |
| "pass@64": 0.7391304347826086, |
| "pass@128": 0.8074534161490683 |
| }, |
| "18": { |
| "pass@1": 0.3735294117647059, |
| "pass@2": 0.4147058823529412, |
| "pass@4": 0.4647058823529412, |
| "pass@8": 0.5, |
| "pass@16": 0.5617647058823529, |
| "pass@32": 0.6147058823529412, |
| "pass@64": 0.6911764705882353, |
| "pass@128": 0.7617647058823529 |
| }, |
| "19": { |
| "pass@1": 0.3482142857142857, |
| "pass@2": 0.40773809523809523, |
| "pass@4": 0.4494047619047619, |
| "pass@8": 0.49702380952380953, |
| "pass@16": 0.5297619047619048, |
| "pass@32": 0.5714285714285714, |
| "pass@64": 0.6190476190476191, |
| "pass@128": 0.6547619047619048 |
| }, |
| "20": { |
| "pass@1": 0.31736526946107785, |
| "pass@2": 0.3772455089820359, |
| "pass@4": 0.4161676646706587, |
| "pass@8": 0.44011976047904194, |
| "pass@16": 0.4880239520958084, |
| "pass@32": 0.5239520958083832, |
| "pass@64": 0.5748502994011976, |
| "pass@128": 0.6317365269461078 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6385542168674698, |
| "12": 0.5705329153605015, |
| "13": 0.6190476190476191, |
| "14": 0.5467836257309941, |
| "15": 0.509090909090909, |
| "16": 0.5042735042735043, |
| "17": 0.39751552795031053, |
| "18": 0.3735294117647059, |
| "19": 0.3482142857142857, |
| "20": 0.31736526946107785 |
| }, |
| "pass@2": { |
| "11": 0.6837349397590361, |
| "12": 0.6394984326018809, |
| "13": 0.6694677871148459, |
| "14": 0.5847953216374269, |
| "15": 0.5545454545454546, |
| "16": 0.5356125356125356, |
| "17": 0.4440993788819876, |
| "18": 0.4147058823529412, |
| "19": 0.40773809523809523, |
| "20": 0.3772455089820359 |
| }, |
| "pass@4": { |
| "11": 0.7771084337349398, |
| "12": 0.7429467084639498, |
| "13": 0.7535014005602241, |
| "14": 0.631578947368421, |
| "15": 0.6181818181818182, |
| "16": 0.5641025641025641, |
| "17": 0.48757763975155277, |
| "18": 0.4647058823529412, |
| "19": 0.4494047619047619, |
| "20": 0.4161676646706587 |
| }, |
| "pass@8": { |
| "11": 0.8373493975903614, |
| "12": 0.8338557993730408, |
| "13": 0.8487394957983193, |
| "14": 0.6783625730994152, |
| "15": 0.6848484848484848, |
| "16": 0.5925925925925926, |
| "17": 0.5279503105590062, |
| "18": 0.5, |
| "19": 0.49702380952380953, |
| "20": 0.44011976047904194 |
| }, |
| "pass@16": { |
| "11": 0.8855421686746988, |
| "12": 0.896551724137931, |
| "13": 0.9019607843137255, |
| "14": 0.7865497076023392, |
| "15": 0.7454545454545455, |
| "16": 0.6495726495726496, |
| "17": 0.5807453416149069, |
| "18": 0.5617647058823529, |
| "19": 0.5297619047619048, |
| "20": 0.4880239520958084 |
| }, |
| "pass@32": { |
| "11": 0.9367469879518072, |
| "12": 0.9498432601880877, |
| "13": 0.927170868347339, |
| "14": 0.8713450292397661, |
| "15": 0.8181818181818182, |
| "16": 0.7207977207977208, |
| "17": 0.6708074534161491, |
| "18": 0.6147058823529412, |
| "19": 0.5714285714285714, |
| "20": 0.5239520958083832 |
| }, |
| "pass@64": { |
| "11": 0.9728915662650602, |
| "12": 0.9623824451410659, |
| "13": 0.9439775910364145, |
| "14": 0.9210526315789473, |
| "15": 0.8909090909090909, |
| "16": 0.7977207977207977, |
| "17": 0.7391304347826086, |
| "18": 0.6911764705882353, |
| "19": 0.6190476190476191, |
| "20": 0.5748502994011976 |
| }, |
| "pass@128": { |
| "11": 0.9879518072289156, |
| "12": 0.9811912225705329, |
| "13": 0.969187675070028, |
| "14": 0.9590643274853801, |
| "15": 0.9393939393939394, |
| "16": 0.8376068376068376, |
| "17": 0.8074534161490683, |
| "18": 0.7617647058823529, |
| "19": 0.6547619047619048, |
| "20": 0.6317365269461078 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4831995242343146, |
| "pass@2": 0.5316681534344335, |
| "pass@4": 0.5908415105560512, |
| "pass@8": 0.6443651501635445, |
| "pass@16": 0.7029438001784122, |
| "pass@32": 0.7606303895331549, |
| "pass@64": 0.8114778471602736, |
| "pass@128": 0.8531073446327684 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.607, |
| "pass@2": 0.682, |
| "pass@4": 0.767, |
| "pass@8": 0.851, |
| "pass@16": 0.915, |
| "pass@32": 0.954, |
| "pass@64": 0.973, |
| "pass@128": 0.985 |
| }, |
| "12": { |
| "pass@1": 0.611, |
| "pass@2": 0.672, |
| "pass@4": 0.756, |
| "pass@8": 0.847, |
| "pass@16": 0.907, |
| "pass@32": 0.945, |
| "pass@64": 0.964, |
| "pass@128": 0.978 |
| }, |
| "13": { |
| "pass@1": 0.601, |
| "pass@2": 0.663, |
| "pass@4": 0.736, |
| "pass@8": 0.817, |
| "pass@16": 0.872, |
| "pass@32": 0.908, |
| "pass@64": 0.936, |
| "pass@128": 0.957 |
| }, |
| "14": { |
| "pass@1": 0.543, |
| "pass@2": 0.584, |
| "pass@4": 0.636, |
| "pass@8": 0.689, |
| "pass@16": 0.776, |
| "pass@32": 0.842, |
| "pass@64": 0.901, |
| "pass@128": 0.939 |
| }, |
| "15": { |
| "pass@1": 0.489, |
| "pass@2": 0.529, |
| "pass@4": 0.581, |
| "pass@8": 0.643, |
| "pass@16": 0.727, |
| "pass@32": 0.805, |
| "pass@64": 0.891, |
| "pass@128": 0.938 |
| }, |
| "16": { |
| "pass@1": 0.482, |
| "pass@2": 0.528, |
| "pass@4": 0.569, |
| "pass@8": 0.601, |
| "pass@16": 0.664, |
| "pass@32": 0.727, |
| "pass@64": 0.787, |
| "pass@128": 0.821 |
| }, |
| "17": { |
| "pass@1": 0.449, |
| "pass@2": 0.49, |
| "pass@4": 0.53, |
| "pass@8": 0.572, |
| "pass@16": 0.628, |
| "pass@32": 0.702, |
| "pass@64": 0.769, |
| "pass@128": 0.822 |
| }, |
| "18": { |
| "pass@1": 0.394, |
| "pass@2": 0.449, |
| "pass@4": 0.494, |
| "pass@8": 0.53, |
| "pass@16": 0.585, |
| "pass@32": 0.639, |
| "pass@64": 0.709, |
| "pass@128": 0.775 |
| }, |
| "19": { |
| "pass@1": 0.342, |
| "pass@2": 0.405, |
| "pass@4": 0.452, |
| "pass@8": 0.487, |
| "pass@16": 0.528, |
| "pass@32": 0.568, |
| "pass@64": 0.621, |
| "pass@128": 0.659 |
| }, |
| "20": { |
| "pass@1": 0.302, |
| "pass@2": 0.362, |
| "pass@4": 0.408, |
| "pass@8": 0.446, |
| "pass@16": 0.485, |
| "pass@32": 0.531, |
| "pass@64": 0.577, |
| "pass@128": 0.639 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.607, |
| "12": 0.611, |
| "13": 0.601, |
| "14": 0.543, |
| "15": 0.489, |
| "16": 0.482, |
| "17": 0.449, |
| "18": 0.394, |
| "19": 0.342, |
| "20": 0.302 |
| }, |
| "pass@2": { |
| "11": 0.682, |
| "12": 0.672, |
| "13": 0.663, |
| "14": 0.584, |
| "15": 0.529, |
| "16": 0.528, |
| "17": 0.49, |
| "18": 0.449, |
| "19": 0.405, |
| "20": 0.362 |
| }, |
| "pass@4": { |
| "11": 0.767, |
| "12": 0.756, |
| "13": 0.736, |
| "14": 0.636, |
| "15": 0.581, |
| "16": 0.569, |
| "17": 0.53, |
| "18": 0.494, |
| "19": 0.452, |
| "20": 0.408 |
| }, |
| "pass@8": { |
| "11": 0.851, |
| "12": 0.847, |
| "13": 0.817, |
| "14": 0.689, |
| "15": 0.643, |
| "16": 0.601, |
| "17": 0.572, |
| "18": 0.53, |
| "19": 0.487, |
| "20": 0.446 |
| }, |
| "pass@16": { |
| "11": 0.915, |
| "12": 0.907, |
| "13": 0.872, |
| "14": 0.776, |
| "15": 0.727, |
| "16": 0.664, |
| "17": 0.628, |
| "18": 0.585, |
| "19": 0.528, |
| "20": 0.485 |
| }, |
| "pass@32": { |
| "11": 0.954, |
| "12": 0.945, |
| "13": 0.908, |
| "14": 0.842, |
| "15": 0.805, |
| "16": 0.727, |
| "17": 0.702, |
| "18": 0.639, |
| "19": 0.568, |
| "20": 0.531 |
| }, |
| "pass@64": { |
| "11": 0.973, |
| "12": 0.964, |
| "13": 0.936, |
| "14": 0.901, |
| "15": 0.891, |
| "16": 0.787, |
| "17": 0.769, |
| "18": 0.709, |
| "19": 0.621, |
| "20": 0.577 |
| }, |
| "pass@128": { |
| "11": 0.985, |
| "12": 0.978, |
| "13": 0.957, |
| "14": 0.939, |
| "15": 0.938, |
| "16": 0.821, |
| "17": 0.822, |
| "18": 0.775, |
| "19": 0.659, |
| "20": 0.639 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.482, |
| "pass@2": 0.5364, |
| "pass@4": 0.5929, |
| "pass@8": 0.6483, |
| "pass@16": 0.7087, |
| "pass@32": 0.7621, |
| "pass@64": 0.8128, |
| "pass@128": 0.8513 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
