| { |
| "files": { |
| "checkpoint-21414_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.990625, |
| "pass@2": 0.990625, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9587301587301588, |
| "pass@2": 0.9746031746031746, |
| "pass@4": 0.9873015873015873, |
| "pass@8": 0.9936507936507937, |
| "pass@16": 0.9936507936507937, |
| "pass@32": 0.9936507936507937, |
| "pass@64": 0.9968253968253968, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9610778443113772, |
| "pass@2": 0.9700598802395209, |
| "pass@4": 0.9820359281437125, |
| "pass@8": 0.9970059880239521, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8816568047337278, |
| "pass@2": 0.9437869822485208, |
| "pass@4": 0.9644970414201184, |
| "pass@8": 0.9881656804733728, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7852564102564102, |
| "pass@2": 0.8717948717948718, |
| "pass@4": 0.9391025641025641, |
| "pass@8": 0.9839743589743589, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.6732394366197183, |
| "pass@2": 0.7915492957746478, |
| "pass@4": 0.8676056338028169, |
| "pass@8": 0.9492957746478873, |
| "pass@16": 0.9802816901408451, |
| "pass@32": 0.9943661971830986, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.5693215339233039, |
| "pass@2": 0.6696165191740413, |
| "pass@4": 0.7463126843657817, |
| "pass@8": 0.831858407079646, |
| "pass@16": 0.8731563421828908, |
| "pass@32": 0.9056047197640118, |
| "pass@64": 0.9262536873156342, |
| "pass@128": 0.9321533923303835 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.990625, |
| "5": 0.9587301587301588, |
| "6": 0.9610778443113772, |
| "7": 0.8816568047337278, |
| "8": 0.7852564102564102, |
| "9": 0.6732394366197183, |
| "10": 0.5693215339233039 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.990625, |
| "5": 0.9746031746031746, |
| "6": 0.9700598802395209, |
| "7": 0.9437869822485208, |
| "8": 0.8717948717948718, |
| "9": 0.7915492957746478, |
| "10": 0.6696165191740413 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9873015873015873, |
| "6": 0.9820359281437125, |
| "7": 0.9644970414201184, |
| "8": 0.9391025641025641, |
| "9": 0.8676056338028169, |
| "10": 0.7463126843657817 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9936507936507937, |
| "6": 0.9970059880239521, |
| "7": 0.9881656804733728, |
| "8": 0.9839743589743589, |
| "9": 0.9492957746478873, |
| "10": 0.831858407079646 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9936507936507937, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 1.0, |
| "9": 0.9802816901408451, |
| "10": 0.8731563421828908 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9936507936507937, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9943661971830986, |
| "10": 0.9056047197640118 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9262536873156342 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9321533923303835 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8655859506923337, |
| "pass@2": 0.9101654846335697, |
| "pass@4": 0.9412360688956434, |
| "pass@8": 0.9706180344478217, |
| "pass@16": 0.9821006416751098, |
| "pass@32": 0.9878419452887538, |
| "pass@64": 0.9908814589665653, |
| "pass@128": 0.9918946301925026 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9883040935672515, |
| "pass@2": 0.9941520467836257, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.95, |
| "pass@2": 0.9794117647058823, |
| "pass@4": 0.9882352941176471, |
| "pass@8": 0.9941176470588236, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9621451104100947, |
| "pass@2": 0.9842271293375394, |
| "pass@4": 0.9936908517350158, |
| "pass@8": 0.9968454258675079, |
| "pass@16": 0.9968454258675079, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8639240506329114, |
| "pass@2": 0.9177215189873418, |
| "pass@4": 0.9620253164556962, |
| "pass@8": 0.9778481012658228, |
| "pass@16": 0.9936708860759493, |
| "pass@32": 0.9968354430379747, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7567567567567568, |
| "pass@2": 0.8288288288288288, |
| "pass@4": 0.9159159159159159, |
| "pass@8": 0.975975975975976, |
| "pass@16": 0.996996996996997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.729903536977492, |
| "pass@2": 0.819935691318328, |
| "pass@4": 0.8713826366559485, |
| "pass@8": 0.9228295819935691, |
| "pass@16": 0.9710610932475884, |
| "pass@32": 0.9903536977491961, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.5552238805970149, |
| "pass@2": 0.6507462686567164, |
| "pass@4": 0.7432835820895523, |
| "pass@8": 0.8477611940298507, |
| "pass@16": 0.8865671641791045, |
| "pass@32": 0.9283582089552239, |
| "pass@64": 0.9462686567164179, |
| "pass@128": 0.9522388059701492 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9883040935672515, |
| "5": 0.95, |
| "6": 0.9621451104100947, |
| "7": 0.8639240506329114, |
| "8": 0.7567567567567568, |
| "9": 0.729903536977492, |
| "10": 0.5552238805970149 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9941520467836257, |
| "5": 0.9794117647058823, |
| "6": 0.9842271293375394, |
| "7": 0.9177215189873418, |
| "8": 0.8288288288288288, |
| "9": 0.819935691318328, |
| "10": 0.6507462686567164 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9882352941176471, |
| "6": 0.9936908517350158, |
| "7": 0.9620253164556962, |
| "8": 0.9159159159159159, |
| "9": 0.8713826366559485, |
| "10": 0.7432835820895523 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9941176470588236, |
| "6": 0.9968454258675079, |
| "7": 0.9778481012658228, |
| "8": 0.975975975975976, |
| "9": 0.9228295819935691, |
| "10": 0.8477611940298507 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9968454258675079, |
| "7": 0.9936708860759493, |
| "8": 0.996996996996997, |
| "9": 0.9710610932475884, |
| "10": 0.8865671641791045 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 0.9903536977491961, |
| "10": 0.9283582089552239 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9462686567164179 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9522388059701492 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8694342149313693, |
| "pass@2": 0.9096083026447941, |
| "pass@4": 0.942417140944091, |
| "pass@8": 0.9688650820220958, |
| "pass@16": 0.9829260127217945, |
| "pass@32": 0.9906260462002009, |
| "pass@64": 0.9939738868429863, |
| "pass@128": 0.9946434549715434 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.996969696969697, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.985207100591716, |
| "pass@2": 0.9940828402366864, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9478260869565217, |
| "pass@2": 0.9855072463768116, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9627507163323782, |
| "pass@2": 0.9885386819484241, |
| "pass@4": 0.9885386819484241, |
| "pass@8": 0.994269340974212, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8352601156069365, |
| "pass@2": 0.930635838150289, |
| "pass@4": 0.9682080924855492, |
| "pass@8": 0.9971098265895953, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7690140845070422, |
| "pass@2": 0.8535211267605634, |
| "pass@4": 0.9323943661971831, |
| "pass@8": 0.9661971830985916, |
| "pass@16": 0.9887323943661972, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.6796407185628742, |
| "pass@2": 0.7664670658682635, |
| "pass@4": 0.8502994011976048, |
| "pass@8": 0.9401197604790419, |
| "pass@16": 0.9820359281437125, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 0.9970059880239521 |
| }, |
| "10": { |
| "pass@1": 0.6319018404907976, |
| "pass@2": 0.7147239263803681, |
| "pass@4": 0.7883435582822086, |
| "pass@8": 0.8374233128834356, |
| "pass@16": 0.8834355828220859, |
| "pass@32": 0.9049079754601227, |
| "pass@64": 0.9325153374233128, |
| "pass@128": 0.9447852760736196 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.996969696969697, |
| "4": 0.985207100591716, |
| "5": 0.9478260869565217, |
| "6": 0.9627507163323782, |
| "7": 0.8352601156069365, |
| "8": 0.7690140845070422, |
| "9": 0.6796407185628742, |
| "10": 0.6319018404907976 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9940828402366864, |
| "5": 0.9855072463768116, |
| "6": 0.9885386819484241, |
| "7": 0.930635838150289, |
| "8": 0.8535211267605634, |
| "9": 0.7664670658682635, |
| "10": 0.7147239263803681 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9885386819484241, |
| "7": 0.9682080924855492, |
| "8": 0.9323943661971831, |
| "9": 0.8502994011976048, |
| "10": 0.7883435582822086 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.994269340974212, |
| "7": 0.9971098265895953, |
| "8": 0.9661971830985916, |
| "9": 0.9401197604790419, |
| "10": 0.8374233128834356 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9887323943661972, |
| "9": 0.9820359281437125, |
| "10": 0.8834355828220859 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9940119760479041, |
| "10": 0.9049079754601227 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.9325153374233128 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.9447852760736196 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8679554390563565, |
| "pass@2": 0.9154652686762779, |
| "pass@4": 0.9482306684141546, |
| "pass@8": 0.9711664482306684, |
| "pass@16": 0.9842726081258192, |
| "pass@32": 0.9891874180865007, |
| "pass@64": 0.9924639580602883, |
| "pass@128": 0.9937745740498034 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.999, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.988, |
| "pass@2": 0.993, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.952, |
| "pass@2": 0.98, |
| "pass@4": 0.992, |
| "pass@8": 0.996, |
| "pass@16": 0.998, |
| "pass@32": 0.998, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.962, |
| "pass@2": 0.981, |
| "pass@4": 0.988, |
| "pass@8": 0.996, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.86, |
| "pass@2": 0.931, |
| "pass@4": 0.965, |
| "pass@8": 0.988, |
| "pass@16": 0.997, |
| "pass@32": 0.999, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.77, |
| "pass@2": 0.851, |
| "pass@4": 0.929, |
| "pass@8": 0.975, |
| "pass@16": 0.995, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.693, |
| "pass@2": 0.792, |
| "pass@4": 0.863, |
| "pass@8": 0.938, |
| "pass@16": 0.978, |
| "pass@32": 0.993, |
| "pass@64": 0.998, |
| "pass@128": 0.999 |
| }, |
| "10": { |
| "pass@1": 0.585, |
| "pass@2": 0.678, |
| "pass@4": 0.759, |
| "pass@8": 0.839, |
| "pass@16": 0.881, |
| "pass@32": 0.913, |
| "pass@64": 0.935, |
| "pass@128": 0.943 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 0.988, |
| "5": 0.952, |
| "6": 0.962, |
| "7": 0.86, |
| "8": 0.77, |
| "9": 0.693, |
| "10": 0.585 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.993, |
| "5": 0.98, |
| "6": 0.981, |
| "7": 0.931, |
| "8": 0.851, |
| "9": 0.792, |
| "10": 0.678 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.992, |
| "6": 0.988, |
| "7": 0.965, |
| "8": 0.929, |
| "9": 0.863, |
| "10": 0.759 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.996, |
| "6": 0.996, |
| "7": 0.988, |
| "8": 0.975, |
| "9": 0.938, |
| "10": 0.839 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 0.999, |
| "7": 0.997, |
| "8": 0.995, |
| "9": 0.978, |
| "10": 0.881 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 1.0, |
| "9": 0.993, |
| "10": 0.913 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.998, |
| "10": 0.935 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.943 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8676666666666667, |
| "pass@2": 0.9117777777777778, |
| "pass@4": 0.944, |
| "pass@8": 0.9702222222222222, |
| "pass@16": 0.9831111111111112, |
| "pass@32": 0.9892222222222222, |
| "pass@64": 0.9924444444444445, |
| "pass@128": 0.9934444444444445 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-21414_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.4475920679886686, |
| "pass@2": 0.5694050991501416, |
| "pass@4": 0.6713881019830028, |
| "pass@8": 0.7620396600566572, |
| "pass@16": 0.8470254957507082, |
| "pass@32": 0.8980169971671388, |
| "pass@64": 0.9235127478753541, |
| "pass@128": 0.9291784702549575 |
| }, |
| "12": { |
| "pass@1": 0.3898305084745763, |
| "pass@2": 0.5084745762711864, |
| "pass@4": 0.5847457627118644, |
| "pass@8": 0.6864406779661016, |
| "pass@16": 0.748587570621469, |
| "pass@32": 0.788135593220339, |
| "pass@64": 0.8163841807909604, |
| "pass@128": 0.8333333333333334 |
| }, |
| "13": { |
| "pass@1": 0.22884012539184953, |
| "pass@2": 0.30407523510971785, |
| "pass@4": 0.3761755485893417, |
| "pass@8": 0.45454545454545453, |
| "pass@16": 0.54858934169279, |
| "pass@32": 0.6144200626959248, |
| "pass@64": 0.658307210031348, |
| "pass@128": 0.6739811912225705 |
| }, |
| "14": { |
| "pass@1": 0.12149532710280374, |
| "pass@2": 0.1806853582554517, |
| "pass@4": 0.2554517133956386, |
| "pass@8": 0.32710280373831774, |
| "pass@16": 0.3956386292834891, |
| "pass@32": 0.4797507788161994, |
| "pass@64": 0.5202492211838006, |
| "pass@128": 0.5451713395638629 |
| }, |
| "15": { |
| "pass@1": 0.06936416184971098, |
| "pass@2": 0.11271676300578035, |
| "pass@4": 0.15028901734104047, |
| "pass@8": 0.17052023121387283, |
| "pass@16": 0.20809248554913296, |
| "pass@32": 0.2630057803468208, |
| "pass@64": 0.29190751445086704, |
| "pass@128": 0.3786127167630058 |
| }, |
| "16": { |
| "pass@1": 0.012903225806451613, |
| "pass@2": 0.025806451612903226, |
| "pass@4": 0.03870967741935484, |
| "pass@8": 0.05161290322580645, |
| "pass@16": 0.05806451612903226, |
| "pass@32": 0.06451612903225806, |
| "pass@64": 0.08064516129032258, |
| "pass@128": 0.1064516129032258 |
| }, |
| "17": { |
| "pass@1": 0.002890173410404624, |
| "pass@2": 0.008670520231213872, |
| "pass@4": 0.017341040462427744, |
| "pass@8": 0.023121387283236993, |
| "pass@16": 0.031791907514450865, |
| "pass@32": 0.046242774566473986, |
| "pass@64": 0.057803468208092484, |
| "pass@128": 0.07225433526011561 |
| }, |
| "18": { |
| "pass@1": 0.002857142857142857, |
| "pass@2": 0.002857142857142857, |
| "pass@4": 0.002857142857142857, |
| "pass@8": 0.005714285714285714, |
| "pass@16": 0.008571428571428572, |
| "pass@32": 0.011428571428571429, |
| "pass@64": 0.017142857142857144, |
| "pass@128": 0.022857142857142857 |
| }, |
| "19": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.0, |
| "pass@8": 0.003003003003003003, |
| "pass@16": 0.003003003003003003, |
| "pass@32": 0.003003003003003003, |
| "pass@64": 0.006006006006006006, |
| "pass@128": 0.009009009009009009 |
| }, |
| "20": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.003003003003003003, |
| "pass@8": 0.003003003003003003, |
| "pass@16": 0.003003003003003003, |
| "pass@32": 0.006006006006006006, |
| "pass@64": 0.006006006006006006, |
| "pass@128": 0.009009009009009009 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.4475920679886686, |
| "12": 0.3898305084745763, |
| "13": 0.22884012539184953, |
| "14": 0.12149532710280374, |
| "15": 0.06936416184971098, |
| "16": 0.012903225806451613, |
| "17": 0.002890173410404624, |
| "18": 0.002857142857142857, |
| "19": 0.0, |
| "20": 0.0 |
| }, |
| "pass@2": { |
| "11": 0.5694050991501416, |
| "12": 0.5084745762711864, |
| "13": 0.30407523510971785, |
| "14": 0.1806853582554517, |
| "15": 0.11271676300578035, |
| "16": 0.025806451612903226, |
| "17": 0.008670520231213872, |
| "18": 0.002857142857142857, |
| "19": 0.0, |
| "20": 0.0 |
| }, |
| "pass@4": { |
| "11": 0.6713881019830028, |
| "12": 0.5847457627118644, |
| "13": 0.3761755485893417, |
| "14": 0.2554517133956386, |
| "15": 0.15028901734104047, |
| "16": 0.03870967741935484, |
| "17": 0.017341040462427744, |
| "18": 0.002857142857142857, |
| "19": 0.0, |
| "20": 0.003003003003003003 |
| }, |
| "pass@8": { |
| "11": 0.7620396600566572, |
| "12": 0.6864406779661016, |
| "13": 0.45454545454545453, |
| "14": 0.32710280373831774, |
| "15": 0.17052023121387283, |
| "16": 0.05161290322580645, |
| "17": 0.023121387283236993, |
| "18": 0.005714285714285714, |
| "19": 0.003003003003003003, |
| "20": 0.003003003003003003 |
| }, |
| "pass@16": { |
| "11": 0.8470254957507082, |
| "12": 0.748587570621469, |
| "13": 0.54858934169279, |
| "14": 0.3956386292834891, |
| "15": 0.20809248554913296, |
| "16": 0.05806451612903226, |
| "17": 0.031791907514450865, |
| "18": 0.008571428571428572, |
| "19": 0.003003003003003003, |
| "20": 0.003003003003003003 |
| }, |
| "pass@32": { |
| "11": 0.8980169971671388, |
| "12": 0.788135593220339, |
| "13": 0.6144200626959248, |
| "14": 0.4797507788161994, |
| "15": 0.2630057803468208, |
| "16": 0.06451612903225806, |
| "17": 0.046242774566473986, |
| "18": 0.011428571428571429, |
| "19": 0.003003003003003003, |
| "20": 0.006006006006006006 |
| }, |
| "pass@64": { |
| "11": 0.9235127478753541, |
| "12": 0.8163841807909604, |
| "13": 0.658307210031348, |
| "14": 0.5202492211838006, |
| "15": 0.29190751445086704, |
| "16": 0.08064516129032258, |
| "17": 0.057803468208092484, |
| "18": 0.017142857142857144, |
| "19": 0.006006006006006006, |
| "20": 0.006006006006006006 |
| }, |
| "pass@128": { |
| "11": 0.9291784702549575, |
| "12": 0.8333333333333334, |
| "13": 0.6739811912225705, |
| "14": 0.5451713395638629, |
| "15": 0.3786127167630058, |
| "16": 0.1064516129032258, |
| "17": 0.07225433526011561, |
| "18": 0.022857142857142857, |
| "19": 0.009009009009009009, |
| "20": 0.009009009009009009 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.13016344725111442, |
| "pass@2": 0.17444279346210995, |
| "pass@4": 0.21337295690936106, |
| "pass@8": 0.2523031203566122, |
| "pass@16": 0.2888558692421991, |
| "pass@32": 0.3209509658246657, |
| "pass@64": 0.3411589895988113, |
| "pass@128": 0.3613670133729569 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5015873015873016, |
| "pass@2": 0.6253968253968254, |
| "pass@4": 0.7396825396825397, |
| "pass@8": 0.8285714285714286, |
| "pass@16": 0.8793650793650793, |
| "pass@32": 0.9142857142857143, |
| "pass@64": 0.946031746031746, |
| "pass@128": 0.9619047619047619 |
| }, |
| "12": { |
| "pass@1": 0.41896024464831805, |
| "pass@2": 0.5412844036697247, |
| "pass@4": 0.6391437308868502, |
| "pass@8": 0.7155963302752294, |
| "pass@16": 0.7706422018348624, |
| "pass@32": 0.8287461773700305, |
| "pass@64": 0.8470948012232415, |
| "pass@128": 0.8654434250764526 |
| }, |
| "13": { |
| "pass@1": 0.2345679012345679, |
| "pass@2": 0.3395061728395062, |
| "pass@4": 0.41975308641975306, |
| "pass@8": 0.4876543209876543, |
| "pass@16": 0.5462962962962963, |
| "pass@32": 0.5833333333333334, |
| "pass@64": 0.6327160493827161, |
| "pass@128": 0.6666666666666666 |
| }, |
| "14": { |
| "pass@1": 0.14540059347181009, |
| "pass@2": 0.19881305637982197, |
| "pass@4": 0.23738872403560832, |
| "pass@8": 0.2818991097922849, |
| "pass@16": 0.3353115727002967, |
| "pass@32": 0.4035608308605341, |
| "pass@64": 0.45103857566765576, |
| "pass@128": 0.5103857566765578 |
| }, |
| "15": { |
| "pass@1": 0.08024691358024691, |
| "pass@2": 0.14506172839506173, |
| "pass@4": 0.1697530864197531, |
| "pass@8": 0.20987654320987653, |
| "pass@16": 0.26851851851851855, |
| "pass@32": 0.3117283950617284, |
| "pass@64": 0.37962962962962965, |
| "pass@128": 0.4228395061728395 |
| }, |
| "16": { |
| "pass@1": 0.038348082595870206, |
| "pass@2": 0.0471976401179941, |
| "pass@4": 0.06489675516224189, |
| "pass@8": 0.07079646017699115, |
| "pass@16": 0.07964601769911504, |
| "pass@32": 0.08849557522123894, |
| "pass@64": 0.12389380530973451, |
| "pass@128": 0.15339233038348082 |
| }, |
| "17": { |
| "pass@1": 0.015060240963855422, |
| "pass@2": 0.024096385542168676, |
| "pass@4": 0.03313253012048193, |
| "pass@8": 0.045180722891566265, |
| "pass@16": 0.05120481927710843, |
| "pass@32": 0.060240963855421686, |
| "pass@64": 0.06325301204819277, |
| "pass@128": 0.06927710843373494 |
| }, |
| "18": { |
| "pass@1": 0.012903225806451613, |
| "pass@2": 0.016129032258064516, |
| "pass@4": 0.01935483870967742, |
| "pass@8": 0.02258064516129032, |
| "pass@16": 0.03225806451612903, |
| "pass@32": 0.03225806451612903, |
| "pass@64": 0.03870967741935484, |
| "pass@128": 0.06129032258064516 |
| }, |
| "19": { |
| "pass@1": 0.0030211480362537764, |
| "pass@2": 0.0030211480362537764, |
| "pass@4": 0.0030211480362537764, |
| "pass@8": 0.006042296072507553, |
| "pass@16": 0.012084592145015106, |
| "pass@32": 0.012084592145015106, |
| "pass@64": 0.012084592145015106, |
| "pass@128": 0.012084592145015106 |
| }, |
| "20": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.003003003003003003, |
| "pass@8": 0.003003003003003003, |
| "pass@16": 0.009009009009009009, |
| "pass@32": 0.009009009009009009, |
| "pass@64": 0.009009009009009009, |
| "pass@128": 0.012012012012012012 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5015873015873016, |
| "12": 0.41896024464831805, |
| "13": 0.2345679012345679, |
| "14": 0.14540059347181009, |
| "15": 0.08024691358024691, |
| "16": 0.038348082595870206, |
| "17": 0.015060240963855422, |
| "18": 0.012903225806451613, |
| "19": 0.0030211480362537764, |
| "20": 0.0 |
| }, |
| "pass@2": { |
| "11": 0.6253968253968254, |
| "12": 0.5412844036697247, |
| "13": 0.3395061728395062, |
| "14": 0.19881305637982197, |
| "15": 0.14506172839506173, |
| "16": 0.0471976401179941, |
| "17": 0.024096385542168676, |
| "18": 0.016129032258064516, |
| "19": 0.0030211480362537764, |
| "20": 0.0 |
| }, |
| "pass@4": { |
| "11": 0.7396825396825397, |
| "12": 0.6391437308868502, |
| "13": 0.41975308641975306, |
| "14": 0.23738872403560832, |
| "15": 0.1697530864197531, |
| "16": 0.06489675516224189, |
| "17": 0.03313253012048193, |
| "18": 0.01935483870967742, |
| "19": 0.0030211480362537764, |
| "20": 0.003003003003003003 |
| }, |
| "pass@8": { |
| "11": 0.8285714285714286, |
| "12": 0.7155963302752294, |
| "13": 0.4876543209876543, |
| "14": 0.2818991097922849, |
| "15": 0.20987654320987653, |
| "16": 0.07079646017699115, |
| "17": 0.045180722891566265, |
| "18": 0.02258064516129032, |
| "19": 0.006042296072507553, |
| "20": 0.003003003003003003 |
| }, |
| "pass@16": { |
| "11": 0.8793650793650793, |
| "12": 0.7706422018348624, |
| "13": 0.5462962962962963, |
| "14": 0.3353115727002967, |
| "15": 0.26851851851851855, |
| "16": 0.07964601769911504, |
| "17": 0.05120481927710843, |
| "18": 0.03225806451612903, |
| "19": 0.012084592145015106, |
| "20": 0.009009009009009009 |
| }, |
| "pass@32": { |
| "11": 0.9142857142857143, |
| "12": 0.8287461773700305, |
| "13": 0.5833333333333334, |
| "14": 0.4035608308605341, |
| "15": 0.3117283950617284, |
| "16": 0.08849557522123894, |
| "17": 0.060240963855421686, |
| "18": 0.03225806451612903, |
| "19": 0.012084592145015106, |
| "20": 0.009009009009009009 |
| }, |
| "pass@64": { |
| "11": 0.946031746031746, |
| "12": 0.8470948012232415, |
| "13": 0.6327160493827161, |
| "14": 0.45103857566765576, |
| "15": 0.37962962962962965, |
| "16": 0.12389380530973451, |
| "17": 0.06325301204819277, |
| "18": 0.03870967741935484, |
| "19": 0.012084592145015106, |
| "20": 0.009009009009009009 |
| }, |
| "pass@128": { |
| "11": 0.9619047619047619, |
| "12": 0.8654434250764526, |
| "13": 0.6666666666666666, |
| "14": 0.5103857566765578, |
| "15": 0.4228395061728395, |
| "16": 0.15339233038348082, |
| "17": 0.06927710843373494, |
| "18": 0.06129032258064516, |
| "19": 0.012084592145015106, |
| "20": 0.012012012012012012 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.14333740831295844, |
| "pass@2": 0.19193154034229828, |
| "pass@4": 0.230440097799511, |
| "pass@8": 0.2643643031784841, |
| "pass@16": 0.29553789731051344, |
| "pass@32": 0.3215158924205379, |
| "pass@64": 0.3474938875305624, |
| "pass@128": 0.370721271393643 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5421686746987951, |
| "pass@2": 0.6295180722891566, |
| "pass@4": 0.7289156626506024, |
| "pass@8": 0.8042168674698795, |
| "pass@16": 0.8704819277108434, |
| "pass@32": 0.9246987951807228, |
| "pass@64": 0.9487951807228916, |
| "pass@128": 0.9668674698795181 |
| }, |
| "12": { |
| "pass@1": 0.3949843260188088, |
| "pass@2": 0.4890282131661442, |
| "pass@4": 0.5987460815047022, |
| "pass@8": 0.6677115987460815, |
| "pass@16": 0.7429467084639498, |
| "pass@32": 0.7836990595611285, |
| "pass@64": 0.8119122257053292, |
| "pass@128": 0.8369905956112853 |
| }, |
| "13": { |
| "pass@1": 0.23809523809523808, |
| "pass@2": 0.31092436974789917, |
| "pass@4": 0.4257703081232493, |
| "pass@8": 0.5126050420168067, |
| "pass@16": 0.5574229691876751, |
| "pass@32": 0.6274509803921569, |
| "pass@64": 0.6554621848739496, |
| "pass@128": 0.6974789915966386 |
| }, |
| "14": { |
| "pass@1": 0.1783625730994152, |
| "pass@2": 0.23976608187134502, |
| "pass@4": 0.28362573099415206, |
| "pass@8": 0.34502923976608185, |
| "pass@16": 0.3830409356725146, |
| "pass@32": 0.4590643274853801, |
| "pass@64": 0.5058479532163743, |
| "pass@128": 0.5467836257309941 |
| }, |
| "15": { |
| "pass@1": 0.06060606060606061, |
| "pass@2": 0.08181818181818182, |
| "pass@4": 0.1393939393939394, |
| "pass@8": 0.18787878787878787, |
| "pass@16": 0.23333333333333334, |
| "pass@32": 0.2787878787878788, |
| "pass@64": 0.3212121212121212, |
| "pass@128": 0.38484848484848483 |
| }, |
| "16": { |
| "pass@1": 0.017094017094017096, |
| "pass@2": 0.02564102564102564, |
| "pass@4": 0.02564102564102564, |
| "pass@8": 0.04843304843304843, |
| "pass@16": 0.06552706552706553, |
| "pass@32": 0.07692307692307693, |
| "pass@64": 0.09116809116809117, |
| "pass@128": 0.11965811965811966 |
| }, |
| "17": { |
| "pass@1": 0.012422360248447204, |
| "pass@2": 0.018633540372670808, |
| "pass@4": 0.021739130434782608, |
| "pass@8": 0.027950310559006212, |
| "pass@16": 0.040372670807453416, |
| "pass@32": 0.055900621118012424, |
| "pass@64": 0.06832298136645963, |
| "pass@128": 0.08385093167701864 |
| }, |
| "18": { |
| "pass@1": 0.0058823529411764705, |
| "pass@2": 0.011764705882352941, |
| "pass@4": 0.014705882352941176, |
| "pass@8": 0.023529411764705882, |
| "pass@16": 0.023529411764705882, |
| "pass@32": 0.03235294117647059, |
| "pass@64": 0.047058823529411764, |
| "pass@128": 0.058823529411764705 |
| }, |
| "19": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.0, |
| "pass@8": 0.0, |
| "pass@16": 0.0, |
| "pass@32": 0.0, |
| "pass@64": 0.0, |
| "pass@128": 0.011904761904761904 |
| }, |
| "20": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.0, |
| "pass@8": 0.0, |
| "pass@16": 0.0029940119760479044, |
| "pass@32": 0.0029940119760479044, |
| "pass@64": 0.0029940119760479044, |
| "pass@128": 0.0029940119760479044 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5421686746987951, |
| "12": 0.3949843260188088, |
| "13": 0.23809523809523808, |
| "14": 0.1783625730994152, |
| "15": 0.06060606060606061, |
| "16": 0.017094017094017096, |
| "17": 0.012422360248447204, |
| "18": 0.0058823529411764705, |
| "19": 0.0, |
| "20": 0.0 |
| }, |
| "pass@2": { |
| "11": 0.6295180722891566, |
| "12": 0.4890282131661442, |
| "13": 0.31092436974789917, |
| "14": 0.23976608187134502, |
| "15": 0.08181818181818182, |
| "16": 0.02564102564102564, |
| "17": 0.018633540372670808, |
| "18": 0.011764705882352941, |
| "19": 0.0, |
| "20": 0.0 |
| }, |
| "pass@4": { |
| "11": 0.7289156626506024, |
| "12": 0.5987460815047022, |
| "13": 0.4257703081232493, |
| "14": 0.28362573099415206, |
| "15": 0.1393939393939394, |
| "16": 0.02564102564102564, |
| "17": 0.021739130434782608, |
| "18": 0.014705882352941176, |
| "19": 0.0, |
| "20": 0.0 |
| }, |
| "pass@8": { |
| "11": 0.8042168674698795, |
| "12": 0.6677115987460815, |
| "13": 0.5126050420168067, |
| "14": 0.34502923976608185, |
| "15": 0.18787878787878787, |
| "16": 0.04843304843304843, |
| "17": 0.027950310559006212, |
| "18": 0.023529411764705882, |
| "19": 0.0, |
| "20": 0.0 |
| }, |
| "pass@16": { |
| "11": 0.8704819277108434, |
| "12": 0.7429467084639498, |
| "13": 0.5574229691876751, |
| "14": 0.3830409356725146, |
| "15": 0.23333333333333334, |
| "16": 0.06552706552706553, |
| "17": 0.040372670807453416, |
| "18": 0.023529411764705882, |
| "19": 0.0, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@32": { |
| "11": 0.9246987951807228, |
| "12": 0.7836990595611285, |
| "13": 0.6274509803921569, |
| "14": 0.4590643274853801, |
| "15": 0.2787878787878788, |
| "16": 0.07692307692307693, |
| "17": 0.055900621118012424, |
| "18": 0.03235294117647059, |
| "19": 0.0, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@64": { |
| "11": 0.9487951807228916, |
| "12": 0.8119122257053292, |
| "13": 0.6554621848739496, |
| "14": 0.5058479532163743, |
| "15": 0.3212121212121212, |
| "16": 0.09116809116809117, |
| "17": 0.06832298136645963, |
| "18": 0.047058823529411764, |
| "19": 0.0, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@128": { |
| "11": 0.9668674698795181, |
| "12": 0.8369905956112853, |
| "13": 0.6974789915966386, |
| "14": 0.5467836257309941, |
| "15": 0.38484848484848483, |
| "16": 0.11965811965811966, |
| "17": 0.08385093167701864, |
| "18": 0.058823529411764705, |
| "19": 0.011904761904761904, |
| "20": 0.0029940119760479044 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.14391911983348202, |
| "pass@2": 0.17960154623847754, |
| "pass@4": 0.22271781147784717, |
| "pass@8": 0.2607790663098424, |
| "pass@16": 0.29081177520071366, |
| "pass@32": 0.3232233125185846, |
| "pass@64": 0.34433541480820695, |
| "pass@128": 0.3702051739518287 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.496, |
| "pass@2": 0.607, |
| "pass@4": 0.712, |
| "pass@8": 0.797, |
| "pass@16": 0.865, |
| "pass@32": 0.912, |
| "pass@64": 0.939, |
| "pass@128": 0.952 |
| }, |
| "12": { |
| "pass@1": 0.401, |
| "pass@2": 0.513, |
| "pass@4": 0.607, |
| "pass@8": 0.69, |
| "pass@16": 0.754, |
| "pass@32": 0.8, |
| "pass@64": 0.825, |
| "pass@128": 0.845 |
| }, |
| "13": { |
| "pass@1": 0.234, |
| "pass@2": 0.318, |
| "pass@4": 0.408, |
| "pass@8": 0.486, |
| "pass@16": 0.551, |
| "pass@32": 0.609, |
| "pass@64": 0.649, |
| "pass@128": 0.68 |
| }, |
| "14": { |
| "pass@1": 0.149, |
| "pass@2": 0.207, |
| "pass@4": 0.259, |
| "pass@8": 0.318, |
| "pass@16": 0.371, |
| "pass@32": 0.447, |
| "pass@64": 0.492, |
| "pass@128": 0.534 |
| }, |
| "15": { |
| "pass@1": 0.07, |
| "pass@2": 0.113, |
| "pass@4": 0.153, |
| "pass@8": 0.189, |
| "pass@16": 0.236, |
| "pass@32": 0.284, |
| "pass@64": 0.33, |
| "pass@128": 0.395 |
| }, |
| "16": { |
| "pass@1": 0.023, |
| "pass@2": 0.033, |
| "pass@4": 0.043, |
| "pass@8": 0.057, |
| "pass@16": 0.068, |
| "pass@32": 0.077, |
| "pass@64": 0.099, |
| "pass@128": 0.127 |
| }, |
| "17": { |
| "pass@1": 0.01, |
| "pass@2": 0.017, |
| "pass@4": 0.024, |
| "pass@8": 0.032, |
| "pass@16": 0.041, |
| "pass@32": 0.054, |
| "pass@64": 0.063, |
| "pass@128": 0.075 |
| }, |
| "18": { |
| "pass@1": 0.007, |
| "pass@2": 0.01, |
| "pass@4": 0.012, |
| "pass@8": 0.017, |
| "pass@16": 0.021, |
| "pass@32": 0.025, |
| "pass@64": 0.034, |
| "pass@128": 0.047 |
| }, |
| "19": { |
| "pass@1": 0.001, |
| "pass@2": 0.001, |
| "pass@4": 0.001, |
| "pass@8": 0.003, |
| "pass@16": 0.005, |
| "pass@32": 0.005, |
| "pass@64": 0.006, |
| "pass@128": 0.011 |
| }, |
| "20": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.002, |
| "pass@8": 0.002, |
| "pass@16": 0.005, |
| "pass@32": 0.006, |
| "pass@64": 0.006, |
| "pass@128": 0.008 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.496, |
| "12": 0.401, |
| "13": 0.234, |
| "14": 0.149, |
| "15": 0.07, |
| "16": 0.023, |
| "17": 0.01, |
| "18": 0.007, |
| "19": 0.001, |
| "20": 0.0 |
| }, |
| "pass@2": { |
| "11": 0.607, |
| "12": 0.513, |
| "13": 0.318, |
| "14": 0.207, |
| "15": 0.113, |
| "16": 0.033, |
| "17": 0.017, |
| "18": 0.01, |
| "19": 0.001, |
| "20": 0.0 |
| }, |
| "pass@4": { |
| "11": 0.712, |
| "12": 0.607, |
| "13": 0.408, |
| "14": 0.259, |
| "15": 0.153, |
| "16": 0.043, |
| "17": 0.024, |
| "18": 0.012, |
| "19": 0.001, |
| "20": 0.002 |
| }, |
| "pass@8": { |
| "11": 0.797, |
| "12": 0.69, |
| "13": 0.486, |
| "14": 0.318, |
| "15": 0.189, |
| "16": 0.057, |
| "17": 0.032, |
| "18": 0.017, |
| "19": 0.003, |
| "20": 0.002 |
| }, |
| "pass@16": { |
| "11": 0.865, |
| "12": 0.754, |
| "13": 0.551, |
| "14": 0.371, |
| "15": 0.236, |
| "16": 0.068, |
| "17": 0.041, |
| "18": 0.021, |
| "19": 0.005, |
| "20": 0.005 |
| }, |
| "pass@32": { |
| "11": 0.912, |
| "12": 0.8, |
| "13": 0.609, |
| "14": 0.447, |
| "15": 0.284, |
| "16": 0.077, |
| "17": 0.054, |
| "18": 0.025, |
| "19": 0.005, |
| "20": 0.006 |
| }, |
| "pass@64": { |
| "11": 0.939, |
| "12": 0.825, |
| "13": 0.649, |
| "14": 0.492, |
| "15": 0.33, |
| "16": 0.099, |
| "17": 0.063, |
| "18": 0.034, |
| "19": 0.006, |
| "20": 0.006 |
| }, |
| "pass@128": { |
| "11": 0.952, |
| "12": 0.845, |
| "13": 0.68, |
| "14": 0.534, |
| "15": 0.395, |
| "16": 0.127, |
| "17": 0.075, |
| "18": 0.047, |
| "19": 0.011, |
| "20": 0.008 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.1391, |
| "pass@2": 0.1819, |
| "pass@4": 0.2221, |
| "pass@8": 0.2591, |
| "pass@16": 0.2917, |
| "pass@32": 0.3219, |
| "pass@64": 0.3443, |
| "pass@128": 0.3674 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
