| { |
| "files": { |
| "checkpoint-85276_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99375, |
| "pass@2": 0.996875, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9555555555555556, |
| "pass@2": 0.9746031746031746, |
| "pass@4": 0.9904761904761905, |
| "pass@8": 0.9968253968253968, |
| "pass@16": 0.9968253968253968, |
| "pass@32": 0.9968253968253968, |
| "pass@64": 0.9968253968253968, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9640718562874252, |
| "pass@2": 0.9850299401197605, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8994082840236687, |
| "pass@2": 0.9437869822485208, |
| "pass@4": 0.9644970414201184, |
| "pass@8": 0.985207100591716, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8301282051282052, |
| "pass@2": 0.8910256410256411, |
| "pass@4": 0.9391025641025641, |
| "pass@8": 0.9743589743589743, |
| "pass@16": 0.9903846153846154, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7436619718309859, |
| "pass@2": 0.819718309859155, |
| "pass@4": 0.9014084507042254, |
| "pass@8": 0.9577464788732394, |
| "pass@16": 0.9915492957746479, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6666666666666666, |
| "pass@2": 0.7374631268436578, |
| "pass@4": 0.8082595870206489, |
| "pass@8": 0.8790560471976401, |
| "pass@16": 0.9262536873156342, |
| "pass@32": 0.9557522123893806, |
| "pass@64": 0.9646017699115044, |
| "pass@128": 0.9734513274336283 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9555555555555556, |
| "6": 0.9640718562874252, |
| "7": 0.8994082840236687, |
| "8": 0.8301282051282052, |
| "9": 0.7436619718309859, |
| "10": 0.6666666666666666 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9746031746031746, |
| "6": 0.9850299401197605, |
| "7": 0.9437869822485208, |
| "8": 0.8910256410256411, |
| "9": 0.819718309859155, |
| "10": 0.7374631268436578 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9904761904761905, |
| "6": 1.0, |
| "7": 0.9644970414201184, |
| "8": 0.9391025641025641, |
| "9": 0.9014084507042254, |
| "10": 0.8082595870206489 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 0.985207100591716, |
| "8": 0.9743589743589743, |
| "9": 0.9577464788732394, |
| "10": 0.8790560471976401 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 0.9903846153846154, |
| "9": 0.9915492957746479, |
| "10": 0.9262536873156342 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9557522123893806 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9646017699115044 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9734513274336283 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8922661263086795, |
| "pass@2": 0.9257007767646066, |
| "pass@4": 0.9547450185748058, |
| "pass@8": 0.9763593380614657, |
| "pass@16": 0.9888551165146909, |
| "pass@32": 0.994258696386356, |
| "pass@64": 0.9956095913542722, |
| "pass@128": 0.9966227625802094 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9912280701754386, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9558823529411765, |
| "pass@2": 0.9705882352941176, |
| "pass@4": 0.9941176470588236, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9684542586750788, |
| "pass@2": 0.9747634069400631, |
| "pass@4": 0.9873817034700315, |
| "pass@8": 0.9936908517350158, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.879746835443038, |
| "pass@2": 0.9145569620253164, |
| "pass@4": 0.9493670886075949, |
| "pass@8": 0.9841772151898734, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8048048048048048, |
| "pass@2": 0.8678678678678678, |
| "pass@4": 0.93993993993994, |
| "pass@8": 0.96996996996997, |
| "pass@16": 0.996996996996997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7620578778135049, |
| "pass@2": 0.8392282958199357, |
| "pass@4": 0.9003215434083601, |
| "pass@8": 0.9453376205787781, |
| "pass@16": 0.9678456591639871, |
| "pass@32": 0.9935691318327974, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.608955223880597, |
| "pass@2": 0.6746268656716418, |
| "pass@4": 0.7671641791044777, |
| "pass@8": 0.8567164179104477, |
| "pass@16": 0.9074626865671642, |
| "pass@32": 0.9432835820895522, |
| "pass@64": 0.9641791044776119, |
| "pass@128": 0.9671641791044776 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9912280701754386, |
| "5": 0.9558823529411765, |
| "6": 0.9684542586750788, |
| "7": 0.879746835443038, |
| "8": 0.8048048048048048, |
| "9": 0.7620578778135049, |
| "10": 0.608955223880597 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9705882352941176, |
| "6": 0.9747634069400631, |
| "7": 0.9145569620253164, |
| "8": 0.8678678678678678, |
| "9": 0.8392282958199357, |
| "10": 0.6746268656716418 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9941176470588236, |
| "6": 0.9873817034700315, |
| "7": 0.9493670886075949, |
| "8": 0.93993993993994, |
| "9": 0.9003215434083601, |
| "10": 0.7671641791044777 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9936908517350158, |
| "7": 0.9841772151898734, |
| "8": 0.96996996996997, |
| "9": 0.9453376205787781, |
| "10": 0.8567164179104477 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.996996996996997, |
| "9": 0.9678456591639871, |
| "10": 0.9074626865671642 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9935691318327974, |
| "10": 0.9432835820895522 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9641791044776119 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9671641791044776 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8875125544024104, |
| "pass@2": 0.916973552058922, |
| "pass@4": 0.9494476062939404, |
| "pass@8": 0.9725477067291597, |
| "pass@16": 0.9856042852360227, |
| "pass@32": 0.9929695346501507, |
| "pass@64": 0.9959825912286575, |
| "pass@128": 0.9963173752929361 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9822485207100592, |
| "pass@2": 0.985207100591716, |
| "pass@4": 0.9970414201183432, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9594202898550724, |
| "pass@2": 0.9797101449275363, |
| "pass@4": 0.991304347826087, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9684813753581661, |
| "pass@2": 0.9770773638968482, |
| "pass@4": 0.9856733524355301, |
| "pass@8": 0.997134670487106, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8728323699421965, |
| "pass@2": 0.9132947976878613, |
| "pass@4": 0.9566473988439307, |
| "pass@8": 0.9913294797687862, |
| "pass@16": 0.9971098265895953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8112676056338028, |
| "pass@2": 0.8816901408450705, |
| "pass@4": 0.9605633802816902, |
| "pass@8": 0.9859154929577465, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 0.9971830985915493 |
| }, |
| "9": { |
| "pass@1": 0.7305389221556886, |
| "pass@2": 0.7724550898203593, |
| "pass@4": 0.8622754491017964, |
| "pass@8": 0.9401197604790419, |
| "pass@16": 0.9670658682634731, |
| "pass@32": 0.9910179640718563, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6717791411042945, |
| "pass@2": 0.7392638036809815, |
| "pass@4": 0.8159509202453987, |
| "pass@8": 0.8926380368098159, |
| "pass@16": 0.9325153374233128, |
| "pass@32": 0.9478527607361963, |
| "pass@64": 0.9631901840490797, |
| "pass@128": 0.9662576687116564 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9822485207100592, |
| "5": 0.9594202898550724, |
| "6": 0.9684813753581661, |
| "7": 0.8728323699421965, |
| "8": 0.8112676056338028, |
| "9": 0.7305389221556886, |
| "10": 0.6717791411042945 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.985207100591716, |
| "5": 0.9797101449275363, |
| "6": 0.9770773638968482, |
| "7": 0.9132947976878613, |
| "8": 0.8816901408450705, |
| "9": 0.7724550898203593, |
| "10": 0.7392638036809815 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 0.991304347826087, |
| "6": 0.9856733524355301, |
| "7": 0.9566473988439307, |
| "8": 0.9605633802816902, |
| "9": 0.8622754491017964, |
| "10": 0.8159509202453987 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.997134670487106, |
| "7": 0.9913294797687862, |
| "8": 0.9859154929577465, |
| "9": 0.9401197604790419, |
| "10": 0.8926380368098159 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9971098265895953, |
| "8": 0.9971830985915493, |
| "9": 0.9670658682634731, |
| "10": 0.9325153374233128 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9910179640718563, |
| "10": 0.9478527607361963 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9970059880239521, |
| "10": 0.9631901840490797 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 1.0, |
| "10": 0.9662576687116564 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8889252948885976, |
| "pass@2": 0.9171035386631717, |
| "pass@4": 0.9528178243774574, |
| "pass@8": 0.9790301441677588, |
| "pass@16": 0.9885321100917431, |
| "pass@32": 0.9931192660550459, |
| "pass@64": 0.9954128440366973, |
| "pass@128": 0.9960681520314548 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.989, |
| "pass@2": 0.994, |
| "pass@4": 0.999, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.957, |
| "pass@2": 0.975, |
| "pass@4": 0.992, |
| "pass@8": 0.999, |
| "pass@16": 0.999, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.967, |
| "pass@2": 0.979, |
| "pass@4": 0.991, |
| "pass@8": 0.997, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.884, |
| "pass@2": 0.924, |
| "pass@4": 0.957, |
| "pass@8": 0.987, |
| "pass@16": 0.997, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.815, |
| "pass@2": 0.88, |
| "pass@4": 0.947, |
| "pass@8": 0.977, |
| "pass@16": 0.995, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "9": { |
| "pass@1": 0.745, |
| "pass@2": 0.81, |
| "pass@4": 0.888, |
| "pass@8": 0.948, |
| "pass@16": 0.976, |
| "pass@32": 0.994, |
| "pass@64": 0.999, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.649, |
| "pass@2": 0.717, |
| "pass@4": 0.797, |
| "pass@8": 0.876, |
| "pass@16": 0.922, |
| "pass@32": 0.949, |
| "pass@64": 0.964, |
| "pass@128": 0.969 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.989, |
| "5": 0.957, |
| "6": 0.967, |
| "7": 0.884, |
| "8": 0.815, |
| "9": 0.745, |
| "10": 0.649 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.994, |
| "5": 0.975, |
| "6": 0.979, |
| "7": 0.924, |
| "8": 0.88, |
| "9": 0.81, |
| "10": 0.717 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.999, |
| "5": 0.992, |
| "6": 0.991, |
| "7": 0.957, |
| "8": 0.947, |
| "9": 0.888, |
| "10": 0.797 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.997, |
| "7": 0.987, |
| "8": 0.977, |
| "9": 0.948, |
| "10": 0.876 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 0.997, |
| "8": 0.995, |
| "9": 0.976, |
| "10": 0.922 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 0.994, |
| "10": 0.949 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 0.999, |
| "10": 0.964 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.999, |
| "9": 1.0, |
| "10": 0.969 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8895555555555555, |
| "pass@2": 0.9198888888888889, |
| "pass@4": 0.9523333333333334, |
| "pass@8": 0.976, |
| "pass@16": 0.9876666666666667, |
| "pass@32": 0.9934444444444445, |
| "pass@64": 0.9956666666666667, |
| "pass@128": 0.9963333333333333 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-85276_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5835694050991501, |
| "pass@2": 0.6458923512747875, |
| "pass@4": 0.71671388101983, |
| "pass@8": 0.8158640226628895, |
| "pass@16": 0.8923512747875354, |
| "pass@32": 0.9206798866855525, |
| "pass@64": 0.943342776203966, |
| "pass@128": 0.9461756373937678 |
| }, |
| "12": { |
| "pass@1": 0.5790960451977402, |
| "pass@2": 0.6497175141242938, |
| "pass@4": 0.7033898305084746, |
| "pass@8": 0.7598870056497176, |
| "pass@16": 0.8107344632768362, |
| "pass@32": 0.8361581920903954, |
| "pass@64": 0.8502824858757062, |
| "pass@128": 0.867231638418079 |
| }, |
| "13": { |
| "pass@1": 0.4231974921630094, |
| "pass@2": 0.48589341692789967, |
| "pass@4": 0.5642633228840125, |
| "pass@8": 0.6238244514106583, |
| "pass@16": 0.6865203761755486, |
| "pass@32": 0.7366771159874608, |
| "pass@64": 0.774294670846395, |
| "pass@128": 0.7962382445141066 |
| }, |
| "14": { |
| "pass@1": 0.26479750778816197, |
| "pass@2": 0.3426791277258567, |
| "pass@4": 0.40186915887850466, |
| "pass@8": 0.4797507788161994, |
| "pass@16": 0.5545171339563862, |
| "pass@32": 0.6074766355140186, |
| "pass@64": 0.6697819314641744, |
| "pass@128": 0.7040498442367601 |
| }, |
| "15": { |
| "pass@1": 0.15895953757225434, |
| "pass@2": 0.1936416184971098, |
| "pass@4": 0.23699421965317918, |
| "pass@8": 0.2658959537572254, |
| "pass@16": 0.33236994219653176, |
| "pass@32": 0.3901734104046243, |
| "pass@64": 0.45375722543352603, |
| "pass@128": 0.49710982658959535 |
| }, |
| "16": { |
| "pass@1": 0.04838709677419355, |
| "pass@2": 0.08387096774193549, |
| "pass@4": 0.1064516129032258, |
| "pass@8": 0.12903225806451613, |
| "pass@16": 0.14838709677419354, |
| "pass@32": 0.16451612903225807, |
| "pass@64": 0.19032258064516128, |
| "pass@128": 0.2129032258064516 |
| }, |
| "17": { |
| "pass@1": 0.031791907514450865, |
| "pass@2": 0.04046242774566474, |
| "pass@4": 0.057803468208092484, |
| "pass@8": 0.06069364161849711, |
| "pass@16": 0.08092485549132948, |
| "pass@32": 0.11560693641618497, |
| "pass@64": 0.1329479768786127, |
| "pass@128": 0.15895953757225434 |
| }, |
| "18": { |
| "pass@1": 0.008571428571428572, |
| "pass@2": 0.011428571428571429, |
| "pass@4": 0.014285714285714285, |
| "pass@8": 0.02, |
| "pass@16": 0.02857142857142857, |
| "pass@32": 0.037142857142857144, |
| "pass@64": 0.04285714285714286, |
| "pass@128": 0.05142857142857143 |
| }, |
| "19": { |
| "pass@1": 0.003003003003003003, |
| "pass@2": 0.009009009009009009, |
| "pass@4": 0.009009009009009009, |
| "pass@8": 0.012012012012012012, |
| "pass@16": 0.012012012012012012, |
| "pass@32": 0.015015015015015015, |
| "pass@64": 0.018018018018018018, |
| "pass@128": 0.018018018018018018 |
| }, |
| "20": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.0, |
| "pass@8": 0.003003003003003003, |
| "pass@16": 0.003003003003003003, |
| "pass@32": 0.003003003003003003, |
| "pass@64": 0.003003003003003003, |
| "pass@128": 0.003003003003003003 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5835694050991501, |
| "12": 0.5790960451977402, |
| "13": 0.4231974921630094, |
| "14": 0.26479750778816197, |
| "15": 0.15895953757225434, |
| "16": 0.04838709677419355, |
| "17": 0.031791907514450865, |
| "18": 0.008571428571428572, |
| "19": 0.003003003003003003, |
| "20": 0.0 |
| }, |
| "pass@2": { |
| "11": 0.6458923512747875, |
| "12": 0.6497175141242938, |
| "13": 0.48589341692789967, |
| "14": 0.3426791277258567, |
| "15": 0.1936416184971098, |
| "16": 0.08387096774193549, |
| "17": 0.04046242774566474, |
| "18": 0.011428571428571429, |
| "19": 0.009009009009009009, |
| "20": 0.0 |
| }, |
| "pass@4": { |
| "11": 0.71671388101983, |
| "12": 0.7033898305084746, |
| "13": 0.5642633228840125, |
| "14": 0.40186915887850466, |
| "15": 0.23699421965317918, |
| "16": 0.1064516129032258, |
| "17": 0.057803468208092484, |
| "18": 0.014285714285714285, |
| "19": 0.009009009009009009, |
| "20": 0.0 |
| }, |
| "pass@8": { |
| "11": 0.8158640226628895, |
| "12": 0.7598870056497176, |
| "13": 0.6238244514106583, |
| "14": 0.4797507788161994, |
| "15": 0.2658959537572254, |
| "16": 0.12903225806451613, |
| "17": 0.06069364161849711, |
| "18": 0.02, |
| "19": 0.012012012012012012, |
| "20": 0.003003003003003003 |
| }, |
| "pass@16": { |
| "11": 0.8923512747875354, |
| "12": 0.8107344632768362, |
| "13": 0.6865203761755486, |
| "14": 0.5545171339563862, |
| "15": 0.33236994219653176, |
| "16": 0.14838709677419354, |
| "17": 0.08092485549132948, |
| "18": 0.02857142857142857, |
| "19": 0.012012012012012012, |
| "20": 0.003003003003003003 |
| }, |
| "pass@32": { |
| "11": 0.9206798866855525, |
| "12": 0.8361581920903954, |
| "13": 0.7366771159874608, |
| "14": 0.6074766355140186, |
| "15": 0.3901734104046243, |
| "16": 0.16451612903225807, |
| "17": 0.11560693641618497, |
| "18": 0.037142857142857144, |
| "19": 0.015015015015015015, |
| "20": 0.003003003003003003 |
| }, |
| "pass@64": { |
| "11": 0.943342776203966, |
| "12": 0.8502824858757062, |
| "13": 0.774294670846395, |
| "14": 0.6697819314641744, |
| "15": 0.45375722543352603, |
| "16": 0.19032258064516128, |
| "17": 0.1329479768786127, |
| "18": 0.04285714285714286, |
| "19": 0.018018018018018018, |
| "20": 0.003003003003003003 |
| }, |
| "pass@128": { |
| "11": 0.9461756373937678, |
| "12": 0.867231638418079, |
| "13": 0.7962382445141066, |
| "14": 0.7040498442367601, |
| "15": 0.49710982658959535, |
| "16": 0.2129032258064516, |
| "17": 0.15895953757225434, |
| "18": 0.05142857142857143, |
| "19": 0.018018018018018018, |
| "20": 0.003003003003003003 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.21277860326894502, |
| "pass@2": 0.2487369985141159, |
| "pass@4": 0.2835066864784547, |
| "pass@8": 0.31946508172362553, |
| "pass@16": 0.3575037147102526, |
| "pass@32": 0.38514115898959883, |
| "pass@64": 0.4101040118870728, |
| "pass@128": 0.4276374442793462 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5873015873015873, |
| "pass@2": 0.6634920634920635, |
| "pass@4": 0.7555555555555555, |
| "pass@8": 0.8539682539682539, |
| "pass@16": 0.926984126984127, |
| "pass@32": 0.9619047619047619, |
| "pass@64": 0.9714285714285714, |
| "pass@128": 0.9746031746031746 |
| }, |
| "12": { |
| "pass@1": 0.5902140672782875, |
| "pass@2": 0.6238532110091743, |
| "pass@4": 0.6880733944954128, |
| "pass@8": 0.7706422018348624, |
| "pass@16": 0.8348623853211009, |
| "pass@32": 0.8746177370030581, |
| "pass@64": 0.8868501529051988, |
| "pass@128": 0.8929663608562691 |
| }, |
| "13": { |
| "pass@1": 0.41358024691358025, |
| "pass@2": 0.4537037037037037, |
| "pass@4": 0.5339506172839507, |
| "pass@8": 0.5925925925925926, |
| "pass@16": 0.6419753086419753, |
| "pass@32": 0.6666666666666666, |
| "pass@64": 0.7006172839506173, |
| "pass@128": 0.7160493827160493 |
| }, |
| "14": { |
| "pass@1": 0.21364985163204747, |
| "pass@2": 0.2789317507418398, |
| "pass@4": 0.3353115727002967, |
| "pass@8": 0.4035608308605341, |
| "pass@16": 0.4658753709198813, |
| "pass@32": 0.516320474777448, |
| "pass@64": 0.5548961424332344, |
| "pass@128": 0.6320474777448071 |
| }, |
| "15": { |
| "pass@1": 0.08024691358024691, |
| "pass@2": 0.11728395061728394, |
| "pass@4": 0.15123456790123457, |
| "pass@8": 0.19135802469135801, |
| "pass@16": 0.24074074074074073, |
| "pass@32": 0.2808641975308642, |
| "pass@64": 0.3549382716049383, |
| "pass@128": 0.404320987654321 |
| }, |
| "16": { |
| "pass@1": 0.038348082595870206, |
| "pass@2": 0.061946902654867256, |
| "pass@4": 0.06489675516224189, |
| "pass@8": 0.07669616519174041, |
| "pass@16": 0.09144542772861357, |
| "pass@32": 0.11504424778761062, |
| "pass@64": 0.13274336283185842, |
| "pass@128": 0.14749262536873156 |
| }, |
| "17": { |
| "pass@1": 0.02108433734939759, |
| "pass@2": 0.030120481927710843, |
| "pass@4": 0.03614457831325301, |
| "pass@8": 0.045180722891566265, |
| "pass@16": 0.05120481927710843, |
| "pass@32": 0.060240963855421686, |
| "pass@64": 0.06927710843373494, |
| "pass@128": 0.08433734939759036 |
| }, |
| "18": { |
| "pass@1": 0.012903225806451613, |
| "pass@2": 0.012903225806451613, |
| "pass@4": 0.016129032258064516, |
| "pass@8": 0.025806451612903226, |
| "pass@16": 0.03225806451612903, |
| "pass@32": 0.041935483870967745, |
| "pass@64": 0.04838709677419355, |
| "pass@128": 0.06451612903225806 |
| }, |
| "19": { |
| "pass@1": 0.0, |
| "pass@2": 0.0, |
| "pass@4": 0.0030211480362537764, |
| "pass@8": 0.006042296072507553, |
| "pass@16": 0.00906344410876133, |
| "pass@32": 0.00906344410876133, |
| "pass@64": 0.00906344410876133, |
| "pass@128": 0.012084592145015106 |
| }, |
| "20": { |
| "pass@1": 0.003003003003003003, |
| "pass@2": 0.003003003003003003, |
| "pass@4": 0.006006006006006006, |
| "pass@8": 0.009009009009009009, |
| "pass@16": 0.012012012012012012, |
| "pass@32": 0.015015015015015015, |
| "pass@64": 0.015015015015015015, |
| "pass@128": 0.018018018018018018 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5873015873015873, |
| "12": 0.5902140672782875, |
| "13": 0.41358024691358025, |
| "14": 0.21364985163204747, |
| "15": 0.08024691358024691, |
| "16": 0.038348082595870206, |
| "17": 0.02108433734939759, |
| "18": 0.012903225806451613, |
| "19": 0.0, |
| "20": 0.003003003003003003 |
| }, |
| "pass@2": { |
| "11": 0.6634920634920635, |
| "12": 0.6238532110091743, |
| "13": 0.4537037037037037, |
| "14": 0.2789317507418398, |
| "15": 0.11728395061728394, |
| "16": 0.061946902654867256, |
| "17": 0.030120481927710843, |
| "18": 0.012903225806451613, |
| "19": 0.0, |
| "20": 0.003003003003003003 |
| }, |
| "pass@4": { |
| "11": 0.7555555555555555, |
| "12": 0.6880733944954128, |
| "13": 0.5339506172839507, |
| "14": 0.3353115727002967, |
| "15": 0.15123456790123457, |
| "16": 0.06489675516224189, |
| "17": 0.03614457831325301, |
| "18": 0.016129032258064516, |
| "19": 0.0030211480362537764, |
| "20": 0.006006006006006006 |
| }, |
| "pass@8": { |
| "11": 0.8539682539682539, |
| "12": 0.7706422018348624, |
| "13": 0.5925925925925926, |
| "14": 0.4035608308605341, |
| "15": 0.19135802469135801, |
| "16": 0.07669616519174041, |
| "17": 0.045180722891566265, |
| "18": 0.025806451612903226, |
| "19": 0.006042296072507553, |
| "20": 0.009009009009009009 |
| }, |
| "pass@16": { |
| "11": 0.926984126984127, |
| "12": 0.8348623853211009, |
| "13": 0.6419753086419753, |
| "14": 0.4658753709198813, |
| "15": 0.24074074074074073, |
| "16": 0.09144542772861357, |
| "17": 0.05120481927710843, |
| "18": 0.03225806451612903, |
| "19": 0.00906344410876133, |
| "20": 0.012012012012012012 |
| }, |
| "pass@32": { |
| "11": 0.9619047619047619, |
| "12": 0.8746177370030581, |
| "13": 0.6666666666666666, |
| "14": 0.516320474777448, |
| "15": 0.2808641975308642, |
| "16": 0.11504424778761062, |
| "17": 0.060240963855421686, |
| "18": 0.041935483870967745, |
| "19": 0.00906344410876133, |
| "20": 0.015015015015015015 |
| }, |
| "pass@64": { |
| "11": 0.9714285714285714, |
| "12": 0.8868501529051988, |
| "13": 0.7006172839506173, |
| "14": 0.5548961424332344, |
| "15": 0.3549382716049383, |
| "16": 0.13274336283185842, |
| "17": 0.06927710843373494, |
| "18": 0.04838709677419355, |
| "19": 0.00906344410876133, |
| "20": 0.015015015015015015 |
| }, |
| "pass@128": { |
| "11": 0.9746031746031746, |
| "12": 0.8929663608562691, |
| "13": 0.7160493827160493, |
| "14": 0.6320474777448071, |
| "15": 0.404320987654321, |
| "16": 0.14749262536873156, |
| "17": 0.08433734939759036, |
| "18": 0.06451612903225806, |
| "19": 0.012084592145015106, |
| "20": 0.018018018018018018 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.19407090464547677, |
| "pass@2": 0.22249388753056235, |
| "pass@4": 0.2567237163814181, |
| "pass@8": 0.29492665036674814, |
| "pass@16": 0.32793398533007334, |
| "pass@32": 0.35146699266503667, |
| "pass@64": 0.37163814180929094, |
| "pass@128": 0.3921149144254279 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6506024096385542, |
| "pass@2": 0.6987951807228916, |
| "pass@4": 0.7801204819277109, |
| "pass@8": 0.8283132530120482, |
| "pass@16": 0.9006024096385542, |
| "pass@32": 0.9337349397590361, |
| "pass@64": 0.9578313253012049, |
| "pass@128": 0.9698795180722891 |
| }, |
| "12": { |
| "pass@1": 0.5454545454545454, |
| "pass@2": 0.5987460815047022, |
| "pass@4": 0.6614420062695925, |
| "pass@8": 0.7492163009404389, |
| "pass@16": 0.8181818181818182, |
| "pass@32": 0.8432601880877743, |
| "pass@64": 0.8557993730407524, |
| "pass@128": 0.8589341692789969 |
| }, |
| "13": { |
| "pass@1": 0.40896358543417366, |
| "pass@2": 0.49019607843137253, |
| "pass@4": 0.5686274509803921, |
| "pass@8": 0.6386554621848739, |
| "pass@16": 0.680672268907563, |
| "pass@32": 0.7394957983193278, |
| "pass@64": 0.7787114845938375, |
| "pass@128": 0.8151260504201681 |
| }, |
| "14": { |
| "pass@1": 0.22807017543859648, |
| "pass@2": 0.26900584795321636, |
| "pass@4": 0.30994152046783624, |
| "pass@8": 0.35964912280701755, |
| "pass@16": 0.40350877192982454, |
| "pass@32": 0.4649122807017544, |
| "pass@64": 0.5321637426900585, |
| "pass@128": 0.5877192982456141 |
| }, |
| "15": { |
| "pass@1": 0.12121212121212122, |
| "pass@2": 0.1484848484848485, |
| "pass@4": 0.19090909090909092, |
| "pass@8": 0.23636363636363636, |
| "pass@16": 0.27575757575757576, |
| "pass@32": 0.32727272727272727, |
| "pass@64": 0.36666666666666664, |
| "pass@128": 0.4090909090909091 |
| }, |
| "16": { |
| "pass@1": 0.05128205128205128, |
| "pass@2": 0.07977207977207977, |
| "pass@4": 0.09401709401709402, |
| "pass@8": 0.1111111111111111, |
| "pass@16": 0.1225071225071225, |
| "pass@32": 0.1396011396011396, |
| "pass@64": 0.1623931623931624, |
| "pass@128": 0.19658119658119658 |
| }, |
| "17": { |
| "pass@1": 0.021739130434782608, |
| "pass@2": 0.037267080745341616, |
| "pass@4": 0.046583850931677016, |
| "pass@8": 0.06521739130434782, |
| "pass@16": 0.06832298136645963, |
| "pass@32": 0.06832298136645963, |
| "pass@64": 0.07763975155279502, |
| "pass@128": 0.09937888198757763 |
| }, |
| "18": { |
| "pass@1": 0.008823529411764706, |
| "pass@2": 0.008823529411764706, |
| "pass@4": 0.011764705882352941, |
| "pass@8": 0.011764705882352941, |
| "pass@16": 0.023529411764705882, |
| "pass@32": 0.029411764705882353, |
| "pass@64": 0.03235294117647059, |
| "pass@128": 0.041176470588235294 |
| }, |
| "19": { |
| "pass@1": 0.002976190476190476, |
| "pass@2": 0.002976190476190476, |
| "pass@4": 0.008928571428571428, |
| "pass@8": 0.01488095238095238, |
| "pass@16": 0.01488095238095238, |
| "pass@32": 0.01488095238095238, |
| "pass@64": 0.017857142857142856, |
| "pass@128": 0.023809523809523808 |
| }, |
| "20": { |
| "pass@1": 0.0, |
| "pass@2": 0.0029940119760479044, |
| "pass@4": 0.0029940119760479044, |
| "pass@8": 0.0029940119760479044, |
| "pass@16": 0.0029940119760479044, |
| "pass@32": 0.0029940119760479044, |
| "pass@64": 0.0029940119760479044, |
| "pass@128": 0.0029940119760479044 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6506024096385542, |
| "12": 0.5454545454545454, |
| "13": 0.40896358543417366, |
| "14": 0.22807017543859648, |
| "15": 0.12121212121212122, |
| "16": 0.05128205128205128, |
| "17": 0.021739130434782608, |
| "18": 0.008823529411764706, |
| "19": 0.002976190476190476, |
| "20": 0.0 |
| }, |
| "pass@2": { |
| "11": 0.6987951807228916, |
| "12": 0.5987460815047022, |
| "13": 0.49019607843137253, |
| "14": 0.26900584795321636, |
| "15": 0.1484848484848485, |
| "16": 0.07977207977207977, |
| "17": 0.037267080745341616, |
| "18": 0.008823529411764706, |
| "19": 0.002976190476190476, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@4": { |
| "11": 0.7801204819277109, |
| "12": 0.6614420062695925, |
| "13": 0.5686274509803921, |
| "14": 0.30994152046783624, |
| "15": 0.19090909090909092, |
| "16": 0.09401709401709402, |
| "17": 0.046583850931677016, |
| "18": 0.011764705882352941, |
| "19": 0.008928571428571428, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@8": { |
| "11": 0.8283132530120482, |
| "12": 0.7492163009404389, |
| "13": 0.6386554621848739, |
| "14": 0.35964912280701755, |
| "15": 0.23636363636363636, |
| "16": 0.1111111111111111, |
| "17": 0.06521739130434782, |
| "18": 0.011764705882352941, |
| "19": 0.01488095238095238, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@16": { |
| "11": 0.9006024096385542, |
| "12": 0.8181818181818182, |
| "13": 0.680672268907563, |
| "14": 0.40350877192982454, |
| "15": 0.27575757575757576, |
| "16": 0.1225071225071225, |
| "17": 0.06832298136645963, |
| "18": 0.023529411764705882, |
| "19": 0.01488095238095238, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@32": { |
| "11": 0.9337349397590361, |
| "12": 0.8432601880877743, |
| "13": 0.7394957983193278, |
| "14": 0.4649122807017544, |
| "15": 0.32727272727272727, |
| "16": 0.1396011396011396, |
| "17": 0.06832298136645963, |
| "18": 0.029411764705882353, |
| "19": 0.01488095238095238, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@64": { |
| "11": 0.9578313253012049, |
| "12": 0.8557993730407524, |
| "13": 0.7787114845938375, |
| "14": 0.5321637426900585, |
| "15": 0.36666666666666664, |
| "16": 0.1623931623931624, |
| "17": 0.07763975155279502, |
| "18": 0.03235294117647059, |
| "19": 0.017857142857142856, |
| "20": 0.0029940119760479044 |
| }, |
| "pass@128": { |
| "11": 0.9698795180722891, |
| "12": 0.8589341692789969, |
| "13": 0.8151260504201681, |
| "14": 0.5877192982456141, |
| "15": 0.4090909090909091, |
| "16": 0.19658119658119658, |
| "17": 0.09937888198757763, |
| "18": 0.041176470588235294, |
| "19": 0.023809523809523808, |
| "20": 0.0029940119760479044 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.2030924769550996, |
| "pass@2": 0.23312518584597086, |
| "pass@4": 0.26702349093071664, |
| "pass@8": 0.30121914956883733, |
| "pass@16": 0.3303597977995837, |
| "pass@32": 0.3559322033898305, |
| "pass@64": 0.37823371989295274, |
| "pass@128": 0.4005352363960749 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.607, |
| "pass@2": 0.669, |
| "pass@4": 0.75, |
| "pass@8": 0.832, |
| "pass@16": 0.906, |
| "pass@32": 0.938, |
| "pass@64": 0.957, |
| "pass@128": 0.963 |
| }, |
| "12": { |
| "pass@1": 0.572, |
| "pass@2": 0.625, |
| "pass@4": 0.685, |
| "pass@8": 0.76, |
| "pass@16": 0.821, |
| "pass@32": 0.851, |
| "pass@64": 0.864, |
| "pass@128": 0.873 |
| }, |
| "13": { |
| "pass@1": 0.415, |
| "pass@2": 0.477, |
| "pass@4": 0.556, |
| "pass@8": 0.619, |
| "pass@16": 0.67, |
| "pass@32": 0.715, |
| "pass@64": 0.752, |
| "pass@128": 0.777 |
| }, |
| "14": { |
| "pass@1": 0.235, |
| "pass@2": 0.296, |
| "pass@4": 0.348, |
| "pass@8": 0.413, |
| "pass@16": 0.473, |
| "pass@32": 0.528, |
| "pass@64": 0.584, |
| "pass@128": 0.64 |
| }, |
| "15": { |
| "pass@1": 0.121, |
| "pass@2": 0.154, |
| "pass@4": 0.194, |
| "pass@8": 0.232, |
| "pass@16": 0.284, |
| "pass@32": 0.334, |
| "pass@64": 0.393, |
| "pass@128": 0.438 |
| }, |
| "16": { |
| "pass@1": 0.046, |
| "pass@2": 0.075, |
| "pass@4": 0.088, |
| "pass@8": 0.105, |
| "pass@16": 0.12, |
| "pass@32": 0.139, |
| "pass@64": 0.161, |
| "pass@128": 0.185 |
| }, |
| "17": { |
| "pass@1": 0.025, |
| "pass@2": 0.036, |
| "pass@4": 0.047, |
| "pass@8": 0.057, |
| "pass@16": 0.067, |
| "pass@32": 0.082, |
| "pass@64": 0.094, |
| "pass@128": 0.115 |
| }, |
| "18": { |
| "pass@1": 0.01, |
| "pass@2": 0.011, |
| "pass@4": 0.014, |
| "pass@8": 0.019, |
| "pass@16": 0.028, |
| "pass@32": 0.036, |
| "pass@64": 0.041, |
| "pass@128": 0.052 |
| }, |
| "19": { |
| "pass@1": 0.002, |
| "pass@2": 0.004, |
| "pass@4": 0.007, |
| "pass@8": 0.011, |
| "pass@16": 0.012, |
| "pass@32": 0.013, |
| "pass@64": 0.015, |
| "pass@128": 0.018 |
| }, |
| "20": { |
| "pass@1": 0.001, |
| "pass@2": 0.002, |
| "pass@4": 0.003, |
| "pass@8": 0.005, |
| "pass@16": 0.006, |
| "pass@32": 0.007, |
| "pass@64": 0.007, |
| "pass@128": 0.008 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.607, |
| "12": 0.572, |
| "13": 0.415, |
| "14": 0.235, |
| "15": 0.121, |
| "16": 0.046, |
| "17": 0.025, |
| "18": 0.01, |
| "19": 0.002, |
| "20": 0.001 |
| }, |
| "pass@2": { |
| "11": 0.669, |
| "12": 0.625, |
| "13": 0.477, |
| "14": 0.296, |
| "15": 0.154, |
| "16": 0.075, |
| "17": 0.036, |
| "18": 0.011, |
| "19": 0.004, |
| "20": 0.002 |
| }, |
| "pass@4": { |
| "11": 0.75, |
| "12": 0.685, |
| "13": 0.556, |
| "14": 0.348, |
| "15": 0.194, |
| "16": 0.088, |
| "17": 0.047, |
| "18": 0.014, |
| "19": 0.007, |
| "20": 0.003 |
| }, |
| "pass@8": { |
| "11": 0.832, |
| "12": 0.76, |
| "13": 0.619, |
| "14": 0.413, |
| "15": 0.232, |
| "16": 0.105, |
| "17": 0.057, |
| "18": 0.019, |
| "19": 0.011, |
| "20": 0.005 |
| }, |
| "pass@16": { |
| "11": 0.906, |
| "12": 0.821, |
| "13": 0.67, |
| "14": 0.473, |
| "15": 0.284, |
| "16": 0.12, |
| "17": 0.067, |
| "18": 0.028, |
| "19": 0.012, |
| "20": 0.006 |
| }, |
| "pass@32": { |
| "11": 0.938, |
| "12": 0.851, |
| "13": 0.715, |
| "14": 0.528, |
| "15": 0.334, |
| "16": 0.139, |
| "17": 0.082, |
| "18": 0.036, |
| "19": 0.013, |
| "20": 0.007 |
| }, |
| "pass@64": { |
| "11": 0.957, |
| "12": 0.864, |
| "13": 0.752, |
| "14": 0.584, |
| "15": 0.393, |
| "16": 0.161, |
| "17": 0.094, |
| "18": 0.041, |
| "19": 0.015, |
| "20": 0.007 |
| }, |
| "pass@128": { |
| "11": 0.963, |
| "12": 0.873, |
| "13": 0.777, |
| "14": 0.64, |
| "15": 0.438, |
| "16": 0.185, |
| "17": 0.115, |
| "18": 0.052, |
| "19": 0.018, |
| "20": 0.008 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.2034, |
| "pass@2": 0.2349, |
| "pass@4": 0.2692, |
| "pass@8": 0.3053, |
| "pass@16": 0.3387, |
| "pass@32": 0.3643, |
| "pass@64": 0.3868, |
| "pass@128": 0.4069 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
