{ "files": { "checkpoint-85276_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99375, "pass@2": 0.996875, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9555555555555556, "pass@2": 0.9746031746031746, "pass@4": 0.9904761904761905, "pass@8": 0.9968253968253968, "pass@16": 0.9968253968253968, "pass@32": 0.9968253968253968, "pass@64": 0.9968253968253968, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9640718562874252, "pass@2": 0.9850299401197605, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8994082840236687, "pass@2": 0.9437869822485208, "pass@4": 0.9644970414201184, "pass@8": 0.985207100591716, "pass@16": 0.9970414201183432, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8301282051282052, "pass@2": 0.8910256410256411, "pass@4": 0.9391025641025641, "pass@8": 0.9743589743589743, "pass@16": 0.9903846153846154, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7436619718309859, "pass@2": 0.819718309859155, "pass@4": 0.9014084507042254, "pass@8": 0.9577464788732394, "pass@16": 0.9915492957746479, "pass@32": 0.9971830985915493, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6666666666666666, "pass@2": 0.7374631268436578, "pass@4": 0.8082595870206489, "pass@8": 0.8790560471976401, "pass@16": 0.9262536873156342, "pass@32": 0.9557522123893806, "pass@64": 0.9646017699115044, "pass@128": 0.9734513274336283 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.99375, "5": 0.9555555555555556, "6": 0.9640718562874252, "7": 0.8994082840236687, "8": 0.8301282051282052, "9": 0.7436619718309859, "10": 0.6666666666666666 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.996875, "5": 0.9746031746031746, "6": 0.9850299401197605, "7": 0.9437869822485208, "8": 0.8910256410256411, "9": 0.819718309859155, "10": 0.7374631268436578 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9904761904761905, "6": 1.0, "7": 0.9644970414201184, "8": 0.9391025641025641, "9": 0.9014084507042254, "10": 0.8082595870206489 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 0.985207100591716, "8": 0.9743589743589743, "9": 0.9577464788732394, "10": 0.8790560471976401 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 0.9970414201183432, "8": 0.9903846153846154, "9": 0.9915492957746479, "10": 0.9262536873156342 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9971830985915493, "10": 0.9557522123893806 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9646017699115044 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9968253968253968, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9734513274336283 } }, "total_pass_at_k": { "pass@1": 0.8922661263086795, "pass@2": 0.9257007767646066, "pass@4": 0.9547450185748058, "pass@8": 0.9763593380614657, "pass@16": 0.9888551165146909, "pass@32": 0.994258696386356, "pass@64": 0.9956095913542722, "pass@128": 0.9966227625802094 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9912280701754386, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9558823529411765, "pass@2": 0.9705882352941176, "pass@4": 0.9941176470588236, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9684542586750788, "pass@2": 0.9747634069400631, "pass@4": 0.9873817034700315, "pass@8": 0.9936908517350158, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.879746835443038, "pass@2": 0.9145569620253164, "pass@4": 0.9493670886075949, "pass@8": 0.9841772151898734, "pass@16": 0.9968354430379747, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8048048048048048, "pass@2": 0.8678678678678678, "pass@4": 0.93993993993994, "pass@8": 0.96996996996997, "pass@16": 0.996996996996997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7620578778135049, "pass@2": 0.8392282958199357, "pass@4": 0.9003215434083601, "pass@8": 0.9453376205787781, "pass@16": 0.9678456591639871, "pass@32": 0.9935691318327974, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.608955223880597, "pass@2": 0.6746268656716418, "pass@4": 0.7671641791044777, "pass@8": 0.8567164179104477, "pass@16": 0.9074626865671642, "pass@32": 0.9432835820895522, "pass@64": 0.9641791044776119, "pass@128": 0.9671641791044776 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9912280701754386, "5": 0.9558823529411765, "6": 0.9684542586750788, "7": 0.879746835443038, "8": 0.8048048048048048, "9": 0.7620578778135049, "10": 0.608955223880597 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9705882352941176, "6": 0.9747634069400631, "7": 0.9145569620253164, "8": 0.8678678678678678, "9": 0.8392282958199357, "10": 0.6746268656716418 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9941176470588236, "6": 0.9873817034700315, "7": 0.9493670886075949, "8": 0.93993993993994, "9": 0.9003215434083601, "10": 0.7671641791044777 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9936908517350158, "7": 0.9841772151898734, "8": 0.96996996996997, "9": 0.9453376205787781, "10": 0.8567164179104477 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9968354430379747, "8": 0.996996996996997, "9": 0.9678456591639871, "10": 0.9074626865671642 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9935691318327974, "10": 0.9432835820895522 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9641791044776119 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9671641791044776 } }, "total_pass_at_k": { "pass@1": 0.8875125544024104, "pass@2": 0.916973552058922, "pass@4": 0.9494476062939404, "pass@8": 0.9725477067291597, "pass@16": 0.9856042852360227, "pass@32": 0.9929695346501507, "pass@64": 0.9959825912286575, "pass@128": 0.9963173752929361 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9822485207100592, "pass@2": 0.985207100591716, "pass@4": 0.9970414201183432, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9594202898550724, "pass@2": 0.9797101449275363, "pass@4": 0.991304347826087, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9684813753581661, "pass@2": 0.9770773638968482, "pass@4": 0.9856733524355301, "pass@8": 0.997134670487106, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8728323699421965, "pass@2": 0.9132947976878613, "pass@4": 0.9566473988439307, "pass@8": 0.9913294797687862, "pass@16": 0.9971098265895953, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8112676056338028, "pass@2": 0.8816901408450705, "pass@4": 0.9605633802816902, "pass@8": 0.9859154929577465, "pass@16": 0.9971830985915493, "pass@32": 0.9971830985915493, "pass@64": 0.9971830985915493, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.7305389221556886, "pass@2": 0.7724550898203593, "pass@4": 0.8622754491017964, "pass@8": 0.9401197604790419, "pass@16": 0.9670658682634731, "pass@32": 0.9910179640718563, "pass@64": 0.9970059880239521, "pass@128": 1.0 }, "10": { "pass@1": 0.6717791411042945, "pass@2": 0.7392638036809815, "pass@4": 0.8159509202453987, "pass@8": 0.8926380368098159, "pass@16": 0.9325153374233128, "pass@32": 0.9478527607361963, "pass@64": 0.9631901840490797, "pass@128": 0.9662576687116564 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9822485207100592, "5": 0.9594202898550724, "6": 0.9684813753581661, "7": 0.8728323699421965, "8": 0.8112676056338028, "9": 0.7305389221556886, "10": 0.6717791411042945 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.985207100591716, "5": 0.9797101449275363, "6": 0.9770773638968482, "7": 0.9132947976878613, "8": 0.8816901408450705, "9": 0.7724550898203593, "10": 0.7392638036809815 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 0.991304347826087, "6": 0.9856733524355301, "7": 0.9566473988439307, "8": 0.9605633802816902, "9": 0.8622754491017964, "10": 0.8159509202453987 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.997134670487106, "7": 0.9913294797687862, "8": 0.9859154929577465, "9": 0.9401197604790419, "10": 0.8926380368098159 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9971098265895953, "8": 0.9971830985915493, "9": 0.9670658682634731, "10": 0.9325153374233128 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 0.9910179640718563, "10": 0.9478527607361963 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 0.9970059880239521, "10": 0.9631901840490797 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9971830985915493, "9": 1.0, "10": 0.9662576687116564 } }, "total_pass_at_k": { "pass@1": 0.8889252948885976, "pass@2": 0.9171035386631717, "pass@4": 0.9528178243774574, "pass@8": 0.9790301441677588, "pass@16": 0.9885321100917431, "pass@32": 0.9931192660550459, "pass@64": 0.9954128440366973, "pass@128": 0.9960681520314548 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.989, "pass@2": 0.994, "pass@4": 0.999, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.957, "pass@2": 0.975, "pass@4": 0.992, "pass@8": 0.999, "pass@16": 0.999, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 0.999 }, "6": { "pass@1": 0.967, "pass@2": 0.979, "pass@4": 0.991, "pass@8": 0.997, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.884, "pass@2": 0.924, "pass@4": 0.957, "pass@8": 0.987, "pass@16": 0.997, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.815, "pass@2": 0.88, "pass@4": 0.947, "pass@8": 0.977, "pass@16": 0.995, "pass@32": 0.999, "pass@64": 0.999, "pass@128": 0.999 }, "9": { "pass@1": 0.745, "pass@2": 0.81, "pass@4": 0.888, "pass@8": 0.948, "pass@16": 0.976, "pass@32": 0.994, "pass@64": 0.999, "pass@128": 1.0 }, "10": { "pass@1": 0.649, "pass@2": 0.717, "pass@4": 0.797, "pass@8": 0.876, "pass@16": 0.922, "pass@32": 0.949, "pass@64": 0.964, "pass@128": 0.969 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.989, "5": 0.957, "6": 0.967, "7": 0.884, "8": 0.815, "9": 0.745, "10": 0.649 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.994, "5": 0.975, "6": 0.979, "7": 0.924, "8": 0.88, "9": 0.81, "10": 0.717 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.999, "5": 0.992, "6": 0.991, "7": 0.957, "8": 0.947, "9": 0.888, "10": 0.797 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 0.997, "7": 0.987, "8": 0.977, "9": 0.948, "10": 0.876 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 0.997, "8": 0.995, "9": 0.976, "10": 0.922 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.999, "9": 0.994, "10": 0.949 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.999, "9": 0.999, "10": 0.964 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 1.0, "8": 0.999, "9": 1.0, "10": 0.969 } }, "total_pass_at_k": { "pass@1": 0.8895555555555555, "pass@2": 0.9198888888888889, "pass@4": 0.9523333333333334, "pass@8": 0.976, "pass@16": 0.9876666666666667, "pass@32": 0.9934444444444445, "pass@64": 0.9956666666666667, "pass@128": 0.9963333333333333 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-85276_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.5835694050991501, "pass@2": 0.6458923512747875, "pass@4": 0.71671388101983, "pass@8": 0.8158640226628895, "pass@16": 0.8923512747875354, "pass@32": 0.9206798866855525, "pass@64": 0.943342776203966, "pass@128": 0.9461756373937678 }, "12": { "pass@1": 0.5790960451977402, "pass@2": 0.6497175141242938, "pass@4": 0.7033898305084746, "pass@8": 0.7598870056497176, "pass@16": 0.8107344632768362, "pass@32": 0.8361581920903954, "pass@64": 0.8502824858757062, "pass@128": 0.867231638418079 }, "13": { "pass@1": 0.4231974921630094, "pass@2": 0.48589341692789967, "pass@4": 0.5642633228840125, "pass@8": 0.6238244514106583, "pass@16": 0.6865203761755486, "pass@32": 0.7366771159874608, "pass@64": 0.774294670846395, "pass@128": 0.7962382445141066 }, "14": { "pass@1": 0.26479750778816197, "pass@2": 0.3426791277258567, "pass@4": 0.40186915887850466, "pass@8": 0.4797507788161994, "pass@16": 0.5545171339563862, "pass@32": 0.6074766355140186, "pass@64": 0.6697819314641744, "pass@128": 0.7040498442367601 }, "15": { "pass@1": 0.15895953757225434, "pass@2": 0.1936416184971098, "pass@4": 0.23699421965317918, "pass@8": 0.2658959537572254, "pass@16": 0.33236994219653176, "pass@32": 0.3901734104046243, "pass@64": 0.45375722543352603, "pass@128": 0.49710982658959535 }, "16": { "pass@1": 0.04838709677419355, "pass@2": 0.08387096774193549, "pass@4": 0.1064516129032258, "pass@8": 0.12903225806451613, "pass@16": 0.14838709677419354, "pass@32": 0.16451612903225807, "pass@64": 0.19032258064516128, "pass@128": 0.2129032258064516 }, "17": { "pass@1": 0.031791907514450865, "pass@2": 0.04046242774566474, "pass@4": 0.057803468208092484, "pass@8": 0.06069364161849711, "pass@16": 0.08092485549132948, "pass@32": 0.11560693641618497, "pass@64": 0.1329479768786127, "pass@128": 0.15895953757225434 }, "18": { "pass@1": 0.008571428571428572, "pass@2": 0.011428571428571429, "pass@4": 0.014285714285714285, "pass@8": 0.02, "pass@16": 0.02857142857142857, "pass@32": 0.037142857142857144, "pass@64": 0.04285714285714286, "pass@128": 0.05142857142857143 }, "19": { "pass@1": 0.003003003003003003, "pass@2": 0.009009009009009009, "pass@4": 0.009009009009009009, "pass@8": 0.012012012012012012, "pass@16": 0.012012012012012012, "pass@32": 0.015015015015015015, "pass@64": 0.018018018018018018, "pass@128": 0.018018018018018018 }, "20": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.0, "pass@8": 0.003003003003003003, "pass@16": 0.003003003003003003, "pass@32": 0.003003003003003003, "pass@64": 0.003003003003003003, "pass@128": 0.003003003003003003 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5835694050991501, "12": 0.5790960451977402, "13": 0.4231974921630094, "14": 0.26479750778816197, "15": 0.15895953757225434, "16": 0.04838709677419355, "17": 0.031791907514450865, "18": 0.008571428571428572, "19": 0.003003003003003003, "20": 0.0 }, "pass@2": { "11": 0.6458923512747875, "12": 0.6497175141242938, "13": 0.48589341692789967, "14": 0.3426791277258567, "15": 0.1936416184971098, "16": 0.08387096774193549, "17": 0.04046242774566474, "18": 0.011428571428571429, "19": 0.009009009009009009, "20": 0.0 }, "pass@4": { "11": 0.71671388101983, "12": 0.7033898305084746, "13": 0.5642633228840125, "14": 0.40186915887850466, "15": 0.23699421965317918, "16": 0.1064516129032258, "17": 0.057803468208092484, "18": 0.014285714285714285, "19": 0.009009009009009009, "20": 0.0 }, "pass@8": { "11": 0.8158640226628895, "12": 0.7598870056497176, "13": 0.6238244514106583, "14": 0.4797507788161994, "15": 0.2658959537572254, "16": 0.12903225806451613, "17": 0.06069364161849711, "18": 0.02, "19": 0.012012012012012012, "20": 0.003003003003003003 }, "pass@16": { "11": 0.8923512747875354, "12": 0.8107344632768362, "13": 0.6865203761755486, "14": 0.5545171339563862, "15": 0.33236994219653176, "16": 0.14838709677419354, "17": 0.08092485549132948, "18": 0.02857142857142857, "19": 0.012012012012012012, "20": 0.003003003003003003 }, "pass@32": { "11": 0.9206798866855525, "12": 0.8361581920903954, "13": 0.7366771159874608, "14": 0.6074766355140186, "15": 0.3901734104046243, "16": 0.16451612903225807, "17": 0.11560693641618497, "18": 0.037142857142857144, "19": 0.015015015015015015, "20": 0.003003003003003003 }, "pass@64": { "11": 0.943342776203966, "12": 0.8502824858757062, "13": 0.774294670846395, "14": 0.6697819314641744, "15": 0.45375722543352603, "16": 0.19032258064516128, "17": 0.1329479768786127, "18": 0.04285714285714286, "19": 0.018018018018018018, "20": 0.003003003003003003 }, "pass@128": { "11": 0.9461756373937678, "12": 0.867231638418079, "13": 0.7962382445141066, "14": 0.7040498442367601, "15": 0.49710982658959535, "16": 0.2129032258064516, "17": 0.15895953757225434, "18": 0.05142857142857143, "19": 0.018018018018018018, "20": 0.003003003003003003 } }, "total_pass_at_k": { "pass@1": 0.21277860326894502, "pass@2": 0.2487369985141159, "pass@4": 0.2835066864784547, "pass@8": 0.31946508172362553, "pass@16": 0.3575037147102526, "pass@32": 0.38514115898959883, "pass@64": 0.4101040118870728, "pass@128": 0.4276374442793462 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.5873015873015873, "pass@2": 0.6634920634920635, "pass@4": 0.7555555555555555, "pass@8": 0.8539682539682539, "pass@16": 0.926984126984127, "pass@32": 0.9619047619047619, "pass@64": 0.9714285714285714, "pass@128": 0.9746031746031746 }, "12": { "pass@1": 0.5902140672782875, "pass@2": 0.6238532110091743, "pass@4": 0.6880733944954128, "pass@8": 0.7706422018348624, "pass@16": 0.8348623853211009, "pass@32": 0.8746177370030581, "pass@64": 0.8868501529051988, "pass@128": 0.8929663608562691 }, "13": { "pass@1": 0.41358024691358025, "pass@2": 0.4537037037037037, "pass@4": 0.5339506172839507, "pass@8": 0.5925925925925926, "pass@16": 0.6419753086419753, "pass@32": 0.6666666666666666, "pass@64": 0.7006172839506173, "pass@128": 0.7160493827160493 }, "14": { "pass@1": 0.21364985163204747, "pass@2": 0.2789317507418398, "pass@4": 0.3353115727002967, "pass@8": 0.4035608308605341, "pass@16": 0.4658753709198813, "pass@32": 0.516320474777448, "pass@64": 0.5548961424332344, "pass@128": 0.6320474777448071 }, "15": { "pass@1": 0.08024691358024691, "pass@2": 0.11728395061728394, "pass@4": 0.15123456790123457, "pass@8": 0.19135802469135801, "pass@16": 0.24074074074074073, "pass@32": 0.2808641975308642, "pass@64": 0.3549382716049383, "pass@128": 0.404320987654321 }, "16": { "pass@1": 0.038348082595870206, "pass@2": 0.061946902654867256, "pass@4": 0.06489675516224189, "pass@8": 0.07669616519174041, "pass@16": 0.09144542772861357, "pass@32": 0.11504424778761062, "pass@64": 0.13274336283185842, "pass@128": 0.14749262536873156 }, "17": { "pass@1": 0.02108433734939759, "pass@2": 0.030120481927710843, "pass@4": 0.03614457831325301, "pass@8": 0.045180722891566265, "pass@16": 0.05120481927710843, "pass@32": 0.060240963855421686, "pass@64": 0.06927710843373494, "pass@128": 0.08433734939759036 }, "18": { "pass@1": 0.012903225806451613, "pass@2": 0.012903225806451613, "pass@4": 0.016129032258064516, "pass@8": 0.025806451612903226, "pass@16": 0.03225806451612903, "pass@32": 0.041935483870967745, "pass@64": 0.04838709677419355, "pass@128": 0.06451612903225806 }, "19": { "pass@1": 0.0, "pass@2": 0.0, "pass@4": 0.0030211480362537764, "pass@8": 0.006042296072507553, "pass@16": 0.00906344410876133, "pass@32": 0.00906344410876133, "pass@64": 0.00906344410876133, "pass@128": 0.012084592145015106 }, "20": { "pass@1": 0.003003003003003003, "pass@2": 0.003003003003003003, "pass@4": 0.006006006006006006, "pass@8": 0.009009009009009009, "pass@16": 0.012012012012012012, "pass@32": 0.015015015015015015, "pass@64": 0.015015015015015015, "pass@128": 0.018018018018018018 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5873015873015873, "12": 0.5902140672782875, "13": 0.41358024691358025, "14": 0.21364985163204747, "15": 0.08024691358024691, "16": 0.038348082595870206, "17": 0.02108433734939759, "18": 0.012903225806451613, "19": 0.0, "20": 0.003003003003003003 }, "pass@2": { "11": 0.6634920634920635, "12": 0.6238532110091743, "13": 0.4537037037037037, "14": 0.2789317507418398, "15": 0.11728395061728394, "16": 0.061946902654867256, "17": 0.030120481927710843, "18": 0.012903225806451613, "19": 0.0, "20": 0.003003003003003003 }, "pass@4": { "11": 0.7555555555555555, "12": 0.6880733944954128, "13": 0.5339506172839507, "14": 0.3353115727002967, "15": 0.15123456790123457, "16": 0.06489675516224189, "17": 0.03614457831325301, "18": 0.016129032258064516, "19": 0.0030211480362537764, "20": 0.006006006006006006 }, "pass@8": { "11": 0.8539682539682539, "12": 0.7706422018348624, "13": 0.5925925925925926, "14": 0.4035608308605341, "15": 0.19135802469135801, "16": 0.07669616519174041, "17": 0.045180722891566265, "18": 0.025806451612903226, "19": 0.006042296072507553, "20": 0.009009009009009009 }, "pass@16": { "11": 0.926984126984127, "12": 0.8348623853211009, "13": 0.6419753086419753, "14": 0.4658753709198813, "15": 0.24074074074074073, "16": 0.09144542772861357, "17": 0.05120481927710843, "18": 0.03225806451612903, "19": 0.00906344410876133, "20": 0.012012012012012012 }, "pass@32": { "11": 0.9619047619047619, "12": 0.8746177370030581, "13": 0.6666666666666666, "14": 0.516320474777448, "15": 0.2808641975308642, "16": 0.11504424778761062, "17": 0.060240963855421686, "18": 0.041935483870967745, "19": 0.00906344410876133, "20": 0.015015015015015015 }, "pass@64": { "11": 0.9714285714285714, "12": 0.8868501529051988, "13": 0.7006172839506173, "14": 0.5548961424332344, "15": 0.3549382716049383, "16": 0.13274336283185842, "17": 0.06927710843373494, "18": 0.04838709677419355, "19": 0.00906344410876133, "20": 0.015015015015015015 }, "pass@128": { "11": 0.9746031746031746, "12": 0.8929663608562691, "13": 0.7160493827160493, "14": 0.6320474777448071, "15": 0.404320987654321, "16": 0.14749262536873156, "17": 0.08433734939759036, "18": 0.06451612903225806, "19": 0.012084592145015106, "20": 0.018018018018018018 } }, "total_pass_at_k": { "pass@1": 0.19407090464547677, "pass@2": 0.22249388753056235, "pass@4": 0.2567237163814181, "pass@8": 0.29492665036674814, "pass@16": 0.32793398533007334, "pass@32": 0.35146699266503667, "pass@64": 0.37163814180929094, "pass@128": 0.3921149144254279 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6506024096385542, "pass@2": 0.6987951807228916, "pass@4": 0.7801204819277109, "pass@8": 0.8283132530120482, "pass@16": 0.9006024096385542, "pass@32": 0.9337349397590361, "pass@64": 0.9578313253012049, "pass@128": 0.9698795180722891 }, "12": { "pass@1": 0.5454545454545454, "pass@2": 0.5987460815047022, "pass@4": 0.6614420062695925, "pass@8": 0.7492163009404389, "pass@16": 0.8181818181818182, "pass@32": 0.8432601880877743, "pass@64": 0.8557993730407524, "pass@128": 0.8589341692789969 }, "13": { "pass@1": 0.40896358543417366, "pass@2": 0.49019607843137253, "pass@4": 0.5686274509803921, "pass@8": 0.6386554621848739, "pass@16": 0.680672268907563, "pass@32": 0.7394957983193278, "pass@64": 0.7787114845938375, "pass@128": 0.8151260504201681 }, "14": { "pass@1": 0.22807017543859648, "pass@2": 0.26900584795321636, "pass@4": 0.30994152046783624, "pass@8": 0.35964912280701755, "pass@16": 0.40350877192982454, "pass@32": 0.4649122807017544, "pass@64": 0.5321637426900585, "pass@128": 0.5877192982456141 }, "15": { "pass@1": 0.12121212121212122, "pass@2": 0.1484848484848485, "pass@4": 0.19090909090909092, "pass@8": 0.23636363636363636, "pass@16": 0.27575757575757576, "pass@32": 0.32727272727272727, "pass@64": 0.36666666666666664, "pass@128": 0.4090909090909091 }, "16": { "pass@1": 0.05128205128205128, "pass@2": 0.07977207977207977, "pass@4": 0.09401709401709402, "pass@8": 0.1111111111111111, "pass@16": 0.1225071225071225, "pass@32": 0.1396011396011396, "pass@64": 0.1623931623931624, "pass@128": 0.19658119658119658 }, "17": { "pass@1": 0.021739130434782608, "pass@2": 0.037267080745341616, "pass@4": 0.046583850931677016, "pass@8": 0.06521739130434782, "pass@16": 0.06832298136645963, "pass@32": 0.06832298136645963, "pass@64": 0.07763975155279502, "pass@128": 0.09937888198757763 }, "18": { "pass@1": 0.008823529411764706, "pass@2": 0.008823529411764706, "pass@4": 0.011764705882352941, "pass@8": 0.011764705882352941, "pass@16": 0.023529411764705882, "pass@32": 0.029411764705882353, "pass@64": 0.03235294117647059, "pass@128": 0.041176470588235294 }, "19": { "pass@1": 0.002976190476190476, "pass@2": 0.002976190476190476, "pass@4": 0.008928571428571428, "pass@8": 0.01488095238095238, "pass@16": 0.01488095238095238, "pass@32": 0.01488095238095238, "pass@64": 0.017857142857142856, "pass@128": 0.023809523809523808 }, "20": { "pass@1": 0.0, "pass@2": 0.0029940119760479044, "pass@4": 0.0029940119760479044, "pass@8": 0.0029940119760479044, "pass@16": 0.0029940119760479044, "pass@32": 0.0029940119760479044, "pass@64": 0.0029940119760479044, "pass@128": 0.0029940119760479044 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6506024096385542, "12": 0.5454545454545454, "13": 0.40896358543417366, "14": 0.22807017543859648, "15": 0.12121212121212122, "16": 0.05128205128205128, "17": 0.021739130434782608, "18": 0.008823529411764706, "19": 0.002976190476190476, "20": 0.0 }, "pass@2": { "11": 0.6987951807228916, "12": 0.5987460815047022, "13": 0.49019607843137253, "14": 0.26900584795321636, "15": 0.1484848484848485, "16": 0.07977207977207977, "17": 0.037267080745341616, "18": 0.008823529411764706, "19": 0.002976190476190476, "20": 0.0029940119760479044 }, "pass@4": { "11": 0.7801204819277109, "12": 0.6614420062695925, "13": 0.5686274509803921, "14": 0.30994152046783624, "15": 0.19090909090909092, "16": 0.09401709401709402, "17": 0.046583850931677016, "18": 0.011764705882352941, "19": 0.008928571428571428, "20": 0.0029940119760479044 }, "pass@8": { "11": 0.8283132530120482, "12": 0.7492163009404389, "13": 0.6386554621848739, "14": 0.35964912280701755, "15": 0.23636363636363636, "16": 0.1111111111111111, "17": 0.06521739130434782, "18": 0.011764705882352941, "19": 0.01488095238095238, "20": 0.0029940119760479044 }, "pass@16": { "11": 0.9006024096385542, "12": 0.8181818181818182, "13": 0.680672268907563, "14": 0.40350877192982454, "15": 0.27575757575757576, "16": 0.1225071225071225, "17": 0.06832298136645963, "18": 0.023529411764705882, "19": 0.01488095238095238, "20": 0.0029940119760479044 }, "pass@32": { "11": 0.9337349397590361, "12": 0.8432601880877743, "13": 0.7394957983193278, "14": 0.4649122807017544, "15": 0.32727272727272727, "16": 0.1396011396011396, "17": 0.06832298136645963, "18": 0.029411764705882353, "19": 0.01488095238095238, "20": 0.0029940119760479044 }, "pass@64": { "11": 0.9578313253012049, "12": 0.8557993730407524, "13": 0.7787114845938375, "14": 0.5321637426900585, "15": 0.36666666666666664, "16": 0.1623931623931624, "17": 0.07763975155279502, "18": 0.03235294117647059, "19": 0.017857142857142856, "20": 0.0029940119760479044 }, "pass@128": { "11": 0.9698795180722891, "12": 0.8589341692789969, "13": 0.8151260504201681, "14": 0.5877192982456141, "15": 0.4090909090909091, "16": 0.19658119658119658, "17": 0.09937888198757763, "18": 0.041176470588235294, "19": 0.023809523809523808, "20": 0.0029940119760479044 } }, "total_pass_at_k": { "pass@1": 0.2030924769550996, "pass@2": 0.23312518584597086, "pass@4": 0.26702349093071664, "pass@8": 0.30121914956883733, "pass@16": 0.3303597977995837, "pass@32": 0.3559322033898305, "pass@64": 0.37823371989295274, "pass@128": 0.4005352363960749 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.607, "pass@2": 0.669, "pass@4": 0.75, "pass@8": 0.832, "pass@16": 0.906, "pass@32": 0.938, "pass@64": 0.957, "pass@128": 0.963 }, "12": { "pass@1": 0.572, "pass@2": 0.625, "pass@4": 0.685, "pass@8": 0.76, "pass@16": 0.821, "pass@32": 0.851, "pass@64": 0.864, "pass@128": 0.873 }, "13": { "pass@1": 0.415, "pass@2": 0.477, "pass@4": 0.556, "pass@8": 0.619, "pass@16": 0.67, "pass@32": 0.715, "pass@64": 0.752, "pass@128": 0.777 }, "14": { "pass@1": 0.235, "pass@2": 0.296, "pass@4": 0.348, "pass@8": 0.413, "pass@16": 0.473, "pass@32": 0.528, "pass@64": 0.584, "pass@128": 0.64 }, "15": { "pass@1": 0.121, "pass@2": 0.154, "pass@4": 0.194, "pass@8": 0.232, "pass@16": 0.284, "pass@32": 0.334, "pass@64": 0.393, "pass@128": 0.438 }, "16": { "pass@1": 0.046, "pass@2": 0.075, "pass@4": 0.088, "pass@8": 0.105, "pass@16": 0.12, "pass@32": 0.139, "pass@64": 0.161, "pass@128": 0.185 }, "17": { "pass@1": 0.025, "pass@2": 0.036, "pass@4": 0.047, "pass@8": 0.057, "pass@16": 0.067, "pass@32": 0.082, "pass@64": 0.094, "pass@128": 0.115 }, "18": { "pass@1": 0.01, "pass@2": 0.011, "pass@4": 0.014, "pass@8": 0.019, "pass@16": 0.028, "pass@32": 0.036, "pass@64": 0.041, "pass@128": 0.052 }, "19": { "pass@1": 0.002, "pass@2": 0.004, "pass@4": 0.007, "pass@8": 0.011, "pass@16": 0.012, "pass@32": 0.013, "pass@64": 0.015, "pass@128": 0.018 }, "20": { "pass@1": 0.001, "pass@2": 0.002, "pass@4": 0.003, "pass@8": 0.005, "pass@16": 0.006, "pass@32": 0.007, "pass@64": 0.007, "pass@128": 0.008 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.607, "12": 0.572, "13": 0.415, "14": 0.235, "15": 0.121, "16": 0.046, "17": 0.025, "18": 0.01, "19": 0.002, "20": 0.001 }, "pass@2": { "11": 0.669, "12": 0.625, "13": 0.477, "14": 0.296, "15": 0.154, "16": 0.075, "17": 0.036, "18": 0.011, "19": 0.004, "20": 0.002 }, "pass@4": { "11": 0.75, "12": 0.685, "13": 0.556, "14": 0.348, "15": 0.194, "16": 0.088, "17": 0.047, "18": 0.014, "19": 0.007, "20": 0.003 }, "pass@8": { "11": 0.832, "12": 0.76, "13": 0.619, "14": 0.413, "15": 0.232, "16": 0.105, "17": 0.057, "18": 0.019, "19": 0.011, "20": 0.005 }, "pass@16": { "11": 0.906, "12": 0.821, "13": 0.67, "14": 0.473, "15": 0.284, "16": 0.12, "17": 0.067, "18": 0.028, "19": 0.012, "20": 0.006 }, "pass@32": { "11": 0.938, "12": 0.851, "13": 0.715, "14": 0.528, "15": 0.334, "16": 0.139, "17": 0.082, "18": 0.036, "19": 0.013, "20": 0.007 }, "pass@64": { "11": 0.957, "12": 0.864, "13": 0.752, "14": 0.584, "15": 0.393, "16": 0.161, "17": 0.094, "18": 0.041, "19": 0.015, "20": 0.007 }, "pass@128": { "11": 0.963, "12": 0.873, "13": 0.777, "14": 0.64, "15": 0.438, "16": 0.185, "17": 0.115, "18": 0.052, "19": 0.018, "20": 0.008 } }, "total_pass_at_k": { "pass@1": 0.2034, "pass@2": 0.2349, "pass@4": 0.2692, "pass@8": 0.3053, "pass@16": 0.3387, "pass@32": 0.3643, "pass@64": 0.3868, "pass@128": 0.4069 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }