{ "files": { "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.99375, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9587301587301588, "pass@2": 0.9809523809523809, "pass@4": 0.9936507936507937, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9550898203592815, "pass@2": 0.9640718562874252, "pass@4": 0.9940119760479041, "pass@8": 0.9940119760479041, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.893491124260355, "pass@2": 0.9230769230769231, "pass@4": 0.9644970414201184, "pass@8": 0.985207100591716, "pass@16": 0.9940828402366864, "pass@32": 0.9970414201183432, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8301282051282052, "pass@2": 0.8782051282051282, "pass@4": 0.9423076923076923, "pass@8": 0.9807692307692307, "pass@16": 0.9935897435897436, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7633802816901408, "pass@2": 0.8338028169014085, "pass@4": 0.9070422535211268, "pass@8": 0.9690140845070423, "pass@16": 0.9887323943661972, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6371681415929203, "pass@2": 0.7109144542772862, "pass@4": 0.7905604719764012, "pass@8": 0.8584070796460177, "pass@16": 0.9056047197640118, "pass@32": 0.9410029498525073, "pass@64": 0.9557522123893806, "pass@128": 0.9616519174041298 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.99375, "5": 0.9587301587301588, "6": 0.9550898203592815, "7": 0.893491124260355, "8": 0.8301282051282052, "9": 0.7633802816901408, "10": 0.6371681415929203 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9809523809523809, "6": 0.9640718562874252, "7": 0.9230769230769231, "8": 0.8782051282051282, "9": 0.8338028169014085, "10": 0.7109144542772862 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9936507936507937, "6": 0.9940119760479041, "7": 0.9644970414201184, "8": 0.9423076923076923, "9": 0.9070422535211268, "10": 0.7905604719764012 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.9940119760479041, "7": 0.985207100591716, "8": 0.9807692307692307, "9": 0.9690140845070423, "10": 0.8584070796460177 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9940828402366864, "8": 0.9935897435897436, "9": 0.9887323943661972, "10": 0.9056047197640118 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9970414201183432, "8": 1.0, "9": 1.0, "10": 0.9410029498525073 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9557522123893806 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9616519174041298 } }, "total_pass_at_k": { "pass@1": 0.8899020601148261, "pass@2": 0.9192840256670044, "pass@4": 0.9533941236068896, "pass@8": 0.9756838905775076, "pass@16": 0.9864910503208375, "pass@32": 0.9929078014184397, "pass@64": 0.9949341438703141, "pass@128": 0.9956095913542722 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9883040935672515, "pass@2": 0.9941520467836257, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.95, "pass@2": 0.9735294117647059, "pass@4": 0.9911764705882353, "pass@8": 0.9941176470588236, "pass@16": 0.9970588235294118, "pass@32": 0.9970588235294118, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9684542586750788, "pass@2": 0.9810725552050473, "pass@4": 0.9842271293375394, "pass@8": 0.9936908517350158, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8575949367088608, "pass@2": 0.9145569620253164, "pass@4": 0.9651898734177216, "pass@8": 0.9936708860759493, "pass@16": 0.9968354430379747, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8168168168168168, "pass@2": 0.8708708708708709, "pass@4": 0.9129129129129129, "pass@8": 0.960960960960961, "pass@16": 0.993993993993994, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7652733118971061, "pass@2": 0.8263665594855305, "pass@4": 0.9035369774919614, "pass@8": 0.9453376205787781, "pass@16": 0.9646302250803859, "pass@32": 0.9935691318327974, "pass@64": 0.9967845659163987, "pass@128": 0.9967845659163987 }, "10": { "pass@1": 0.6149253731343284, "pass@2": 0.6895522388059702, "pass@4": 0.7701492537313432, "pass@8": 0.8716417910447761, "pass@16": 0.9343283582089552, "pass@32": 0.9552238805970149, "pass@64": 0.9791044776119403, "pass@128": 0.9880597014925373 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.9883040935672515, "5": 0.95, "6": 0.9684542586750788, "7": 0.8575949367088608, "8": 0.8168168168168168, "9": 0.7652733118971061, "10": 0.6149253731343284 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9941520467836257, "5": 0.9735294117647059, "6": 0.9810725552050473, "7": 0.9145569620253164, "8": 0.8708708708708709, "9": 0.8263665594855305, "10": 0.6895522388059702 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9911764705882353, "6": 0.9842271293375394, "7": 0.9651898734177216, "8": 0.9129129129129129, "9": 0.9035369774919614, "10": 0.7701492537313432 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9941176470588236, "6": 0.9936908517350158, "7": 0.9936708860759493, "8": 0.960960960960961, "9": 0.9453376205787781, "10": 0.8716417910447761 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 0.993993993993994, "9": 0.9646302250803859, "10": 0.9343283582089552 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9935691318327974, "10": 0.9552238805970149 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9967845659163987, "10": 0.9791044776119403 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9967845659163987, "10": 0.9880597014925373 } }, "total_pass_at_k": { "pass@1": 0.8865082022095748, "pass@2": 0.9179779042517576, "pass@4": 0.9481084700368263, "pass@8": 0.9735520589219954, "pass@16": 0.987612989621694, "pass@32": 0.9939738868429863, "pass@64": 0.9973217274857716, "pass@128": 0.9983260796786073 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.985207100591716, "pass@2": 0.9881656804733728, "pass@4": 0.9970414201183432, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9478260869565217, "pass@2": 0.9826086956521739, "pass@4": 0.991304347826087, "pass@8": 0.9971014492753624, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9770773638968482, "pass@2": 0.9856733524355301, "pass@4": 0.9914040114613181, "pass@8": 0.997134670487106, "pass@16": 0.997134670487106, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8554913294797688, "pass@2": 0.9017341040462428, "pass@4": 0.9595375722543352, "pass@8": 0.9826589595375722, "pass@16": 0.9913294797687862, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.819718309859155, "pass@2": 0.8788732394366198, "pass@4": 0.952112676056338, "pass@8": 0.9830985915492958, "pass@16": 0.9971830985915493, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7455089820359282, "pass@2": 0.8023952095808383, "pass@4": 0.8952095808383234, "pass@8": 0.9580838323353293, "pass@16": 0.9850299401197605, "pass@32": 0.9910179640718563, "pass@64": 1.0, "pass@128": 1.0 }, "10": { "pass@1": 0.6319018404907976, "pass@2": 0.6932515337423313, "pass@4": 0.7668711656441718, "pass@8": 0.8312883435582822, "pass@16": 0.8957055214723927, "pass@32": 0.9478527607361963, "pass@64": 0.9570552147239264, "pass@128": 0.9693251533742331 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.985207100591716, "5": 0.9478260869565217, "6": 0.9770773638968482, "7": 0.8554913294797688, "8": 0.819718309859155, "9": 0.7455089820359282, "10": 0.6319018404907976 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9881656804733728, "5": 0.9826086956521739, "6": 0.9856733524355301, "7": 0.9017341040462428, "8": 0.8788732394366198, "9": 0.8023952095808383, "10": 0.6932515337423313 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970414201183432, "5": 0.991304347826087, "6": 0.9914040114613181, "7": 0.9595375722543352, "8": 0.952112676056338, "9": 0.8952095808383234, "10": 0.7668711656441718 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 0.997134670487106, "7": 0.9826589595375722, "8": 0.9830985915492958, "9": 0.9580838323353293, "10": 0.8312883435582822 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 0.997134670487106, "7": 0.9913294797687862, "8": 0.9971830985915493, "9": 0.9850299401197605, "10": 0.8957055214723927 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9910179640718563, "10": 0.9478527607361963 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9570552147239264 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 0.9693251533742331 } }, "total_pass_at_k": { "pass@1": 0.8853211009174312, "pass@2": 0.9154652686762779, "pass@4": 0.9511795543905636, "pass@8": 0.9728047182175622, "pass@16": 0.9855832241153342, "pass@32": 0.9934469200524246, "pass@64": 0.9954128440366973, "pass@128": 0.9967234600262124 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.989, "pass@2": 0.994, "pass@4": 0.999, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.952, "pass@2": 0.979, "pass@4": 0.992, "pass@8": 0.997, "pass@16": 0.999, "pass@32": 0.999, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.967, "pass@2": 0.977, "pass@4": 0.99, "pass@8": 0.995, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.869, "pass@2": 0.913, "pass@4": 0.963, "pass@8": 0.987, "pass@16": 0.994, "pass@32": 0.999, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.822, "pass@2": 0.876, "pass@4": 0.936, "pass@8": 0.975, "pass@16": 0.995, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.758, "pass@2": 0.821, "pass@4": 0.902, "pass@8": 0.958, "pass@16": 0.98, "pass@32": 0.995, "pass@64": 0.999, "pass@128": 0.999 }, "10": { "pass@1": 0.628, "pass@2": 0.698, "pass@4": 0.776, "pass@8": 0.854, "pass@16": 0.912, "pass@32": 0.948, "pass@64": 0.964, "pass@128": 0.973 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 1.0, "4": 0.989, "5": 0.952, "6": 0.967, "7": 0.869, "8": 0.822, "9": 0.758, "10": 0.628 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.994, "5": 0.979, "6": 0.977, "7": 0.913, "8": 0.876, "9": 0.821, "10": 0.698 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.999, "5": 0.992, "6": 0.99, "7": 0.963, "8": 0.936, "9": 0.902, "10": 0.776 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.997, "6": 0.995, "7": 0.987, "8": 0.975, "9": 0.958, "10": 0.854 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 0.999, "7": 0.994, "8": 0.995, "9": 0.98, "10": 0.912 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.999, "6": 1.0, "7": 0.999, "8": 1.0, "9": 0.995, "10": 0.948 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.999, "10": 0.964 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.999, "10": 0.973 } }, "total_pass_at_k": { "pass@1": 0.8872222222222222, "pass@2": 0.9175555555555556, "pass@4": 0.9508888888888889, "pass@8": 0.974, "pass@16": 0.9865555555555555, "pass@32": 0.9934444444444445, "pass@64": 0.9958888888888889, "pass@128": 0.9968888888888889 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.5920679886685553, "pass@2": 0.6798866855524079, "pass@4": 0.7648725212464589, "pass@8": 0.8470254957507082, "pass@16": 0.9206798866855525, "pass@32": 0.9603399433427762, "pass@64": 0.9801699716713881, "pass@128": 0.9858356940509915 }, "12": { "pass@1": 0.6384180790960452, "pass@2": 0.6836158192090396, "pass@4": 0.768361581920904, "pass@8": 0.867231638418079, "pass@16": 0.923728813559322, "pass@32": 0.9491525423728814, "pass@64": 0.9717514124293786, "pass@128": 0.9774011299435028 }, "13": { "pass@1": 0.5862068965517241, "pass@2": 0.6426332288401254, "pass@4": 0.7241379310344828, "pass@8": 0.8119122257053292, "pass@16": 0.8746081504702194, "pass@32": 0.9090909090909091, "pass@64": 0.9373040752351097, "pass@128": 0.9592476489028213 }, "14": { "pass@1": 0.5514018691588785, "pass@2": 0.5856697819314641, "pass@4": 0.6261682242990654, "pass@8": 0.6853582554517134, "pass@16": 0.7476635514018691, "pass@32": 0.8473520249221184, "pass@64": 0.9003115264797508, "pass@128": 0.9283489096573209 }, "15": { "pass@1": 0.5028901734104047, "pass@2": 0.5404624277456648, "pass@4": 0.5867052023121387, "pass@8": 0.6763005780346821, "pass@16": 0.7485549132947977, "pass@32": 0.838150289017341, "pass@64": 0.9104046242774566, "pass@128": 0.9508670520231214 }, "16": { "pass@1": 0.4967741935483871, "pass@2": 0.535483870967742, "pass@4": 0.5838709677419355, "pass@8": 0.6225806451612903, "pass@16": 0.6838709677419355, "pass@32": 0.7129032258064516, "pass@64": 0.7645161290322581, "pass@128": 0.7935483870967742 }, "17": { "pass@1": 0.5433526011560693, "pass@2": 0.5780346820809249, "pass@4": 0.615606936416185, "pass@8": 0.6445086705202312, "pass@16": 0.6936416184971098, "pass@32": 0.7312138728323699, "pass@64": 0.7832369942196532, "pass@128": 0.8352601156069365 }, "18": { "pass@1": 0.43714285714285717, "pass@2": 0.46, "pass@4": 0.4828571428571429, "pass@8": 0.5142857142857142, "pass@16": 0.5742857142857143, "pass@32": 0.6371428571428571, "pass@64": 0.6971428571428572, "pass@128": 0.7714285714285715 }, "19": { "pass@1": 0.34234234234234234, "pass@2": 0.4084084084084084, "pass@4": 0.44144144144144143, "pass@8": 0.4804804804804805, "pass@16": 0.5225225225225225, "pass@32": 0.5585585585585585, "pass@64": 0.5975975975975976, "pass@128": 0.6276276276276276 }, "20": { "pass@1": 0.27627627627627627, "pass@2": 0.3303303303303303, "pass@4": 0.35135135135135137, "pass@8": 0.4084084084084084, "pass@16": 0.44744744744744747, "pass@32": 0.5105105105105106, "pass@64": 0.5705705705705706, "pass@128": 0.6246246246246246 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.5920679886685553, "12": 0.6384180790960452, "13": 0.5862068965517241, "14": 0.5514018691588785, "15": 0.5028901734104047, "16": 0.4967741935483871, "17": 0.5433526011560693, "18": 0.43714285714285717, "19": 0.34234234234234234, "20": 0.27627627627627627 }, "pass@2": { "11": 0.6798866855524079, "12": 0.6836158192090396, "13": 0.6426332288401254, "14": 0.5856697819314641, "15": 0.5404624277456648, "16": 0.535483870967742, "17": 0.5780346820809249, "18": 0.46, "19": 0.4084084084084084, "20": 0.3303303303303303 }, "pass@4": { "11": 0.7648725212464589, "12": 0.768361581920904, "13": 0.7241379310344828, "14": 0.6261682242990654, "15": 0.5867052023121387, "16": 0.5838709677419355, "17": 0.615606936416185, "18": 0.4828571428571429, "19": 0.44144144144144143, "20": 0.35135135135135137 }, "pass@8": { "11": 0.8470254957507082, "12": 0.867231638418079, "13": 0.8119122257053292, "14": 0.6853582554517134, "15": 0.6763005780346821, "16": 0.6225806451612903, "17": 0.6445086705202312, "18": 0.5142857142857142, "19": 0.4804804804804805, "20": 0.4084084084084084 }, "pass@16": { "11": 0.9206798866855525, "12": 0.923728813559322, "13": 0.8746081504702194, "14": 0.7476635514018691, "15": 0.7485549132947977, "16": 0.6838709677419355, "17": 0.6936416184971098, "18": 0.5742857142857143, "19": 0.5225225225225225, "20": 0.44744744744744747 }, "pass@32": { "11": 0.9603399433427762, "12": 0.9491525423728814, "13": 0.9090909090909091, "14": 0.8473520249221184, "15": 0.838150289017341, "16": 0.7129032258064516, "17": 0.7312138728323699, "18": 0.6371428571428571, "19": 0.5585585585585585, "20": 0.5105105105105106 }, "pass@64": { "11": 0.9801699716713881, "12": 0.9717514124293786, "13": 0.9373040752351097, "14": 0.9003115264797508, "15": 0.9104046242774566, "16": 0.7645161290322581, "17": 0.7832369942196532, "18": 0.6971428571428572, "19": 0.5975975975975976, "20": 0.5705705705705706 }, "pass@128": { "11": 0.9858356940509915, "12": 0.9774011299435028, "13": 0.9592476489028213, "14": 0.9283489096573209, "15": 0.9508670520231214, "16": 0.7935483870967742, "17": 0.8352601156069365, "18": 0.7714285714285715, "19": 0.6276276276276276, "20": 0.6246246246246246 } }, "total_pass_at_k": { "pass@1": 0.4974739970282318, "pass@2": 0.5453194650817236, "pass@4": 0.5955423476968796, "pass@8": 0.6570579494799406, "pass@16": 0.7150074294205052, "pass@32": 0.7667161961367014, "pass@64": 0.812481426448737, "pass@128": 0.8466567607726597 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.6, "pass@2": 0.6666666666666666, "pass@4": 0.7619047619047619, "pass@8": 0.8698412698412699, "pass@16": 0.926984126984127, "pass@32": 0.9619047619047619, "pass@64": 0.9682539682539683, "pass@128": 0.9873015873015873 }, "12": { "pass@1": 0.636085626911315, "pass@2": 0.709480122324159, "pass@4": 0.7981651376146789, "pass@8": 0.8776758409785933, "pass@16": 0.9296636085626911, "pass@32": 0.963302752293578, "pass@64": 0.9755351681957186, "pass@128": 0.9877675840978594 }, "13": { "pass@1": 0.5802469135802469, "pass@2": 0.6820987654320988, "pass@4": 0.7561728395061729, "pass@8": 0.8240740740740741, "pass@16": 0.8827160493827161, "pass@32": 0.9320987654320988, "pass@64": 0.9629629629629629, "pass@128": 0.9722222222222222 }, "14": { "pass@1": 0.5608308605341247, "pass@2": 0.5964391691394659, "pass@4": 0.6617210682492581, "pass@8": 0.7388724035608308, "pass@16": 0.8011869436201781, "pass@32": 0.8545994065281899, "pass@64": 0.9109792284866469, "pass@128": 0.9376854599406528 }, "15": { "pass@1": 0.5, "pass@2": 0.5462962962962963, "pass@4": 0.5895061728395061, "pass@8": 0.6574074074074074, "pass@16": 0.7253086419753086, "pass@32": 0.7870370370370371, "pass@64": 0.8765432098765432, "pass@128": 0.9259259259259259 }, "16": { "pass@1": 0.5368731563421829, "pass@2": 0.584070796460177, "pass@4": 0.6135693215339233, "pass@8": 0.6666666666666666, "pass@16": 0.7168141592920354, "pass@32": 0.7699115044247787, "pass@64": 0.8082595870206489, "pass@128": 0.8348082595870207 }, "17": { "pass@1": 0.4939759036144578, "pass@2": 0.5120481927710844, "pass@4": 0.536144578313253, "pass@8": 0.5933734939759037, "pass@16": 0.6566265060240963, "pass@32": 0.7198795180722891, "pass@64": 0.7740963855421686, "pass@128": 0.8192771084337349 }, "18": { "pass@1": 0.44516129032258067, "pass@2": 0.5064516129032258, "pass@4": 0.532258064516129, "pass@8": 0.5645161290322581, "pass@16": 0.6258064516129033, "pass@32": 0.6774193548387096, "pass@64": 0.7483870967741936, "pass@128": 0.7935483870967742 }, "19": { "pass@1": 0.3776435045317221, "pass@2": 0.4501510574018127, "pass@4": 0.4652567975830816, "pass@8": 0.5105740181268882, "pass@16": 0.5317220543806647, "pass@32": 0.5800604229607251, "pass@64": 0.622356495468278, "pass@128": 0.6737160120845922 }, "20": { "pass@1": 0.2912912912912913, "pass@2": 0.34534534534534533, "pass@4": 0.37537537537537535, "pass@8": 0.4084084084084084, "pass@16": 0.4624624624624625, "pass@32": 0.5165165165165165, "pass@64": 0.5645645645645646, "pass@128": 0.6096096096096096 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6, "12": 0.636085626911315, "13": 0.5802469135802469, "14": 0.5608308605341247, "15": 0.5, "16": 0.5368731563421829, "17": 0.4939759036144578, "18": 0.44516129032258067, "19": 0.3776435045317221, "20": 0.2912912912912913 }, "pass@2": { "11": 0.6666666666666666, "12": 0.709480122324159, "13": 0.6820987654320988, "14": 0.5964391691394659, "15": 0.5462962962962963, "16": 0.584070796460177, "17": 0.5120481927710844, "18": 0.5064516129032258, "19": 0.4501510574018127, "20": 0.34534534534534533 }, "pass@4": { "11": 0.7619047619047619, "12": 0.7981651376146789, "13": 0.7561728395061729, "14": 0.6617210682492581, "15": 0.5895061728395061, "16": 0.6135693215339233, "17": 0.536144578313253, "18": 0.532258064516129, "19": 0.4652567975830816, "20": 0.37537537537537535 }, "pass@8": { "11": 0.8698412698412699, "12": 0.8776758409785933, "13": 0.8240740740740741, "14": 0.7388724035608308, "15": 0.6574074074074074, "16": 0.6666666666666666, "17": 0.5933734939759037, "18": 0.5645161290322581, "19": 0.5105740181268882, "20": 0.4084084084084084 }, "pass@16": { "11": 0.926984126984127, "12": 0.9296636085626911, "13": 0.8827160493827161, "14": 0.8011869436201781, "15": 0.7253086419753086, "16": 0.7168141592920354, "17": 0.6566265060240963, "18": 0.6258064516129033, "19": 0.5317220543806647, "20": 0.4624624624624625 }, "pass@32": { "11": 0.9619047619047619, "12": 0.963302752293578, "13": 0.9320987654320988, "14": 0.8545994065281899, "15": 0.7870370370370371, "16": 0.7699115044247787, "17": 0.7198795180722891, "18": 0.6774193548387096, "19": 0.5800604229607251, "20": 0.5165165165165165 }, "pass@64": { "11": 0.9682539682539683, "12": 0.9755351681957186, "13": 0.9629629629629629, "14": 0.9109792284866469, "15": 0.8765432098765432, "16": 0.8082595870206489, "17": 0.7740963855421686, "18": 0.7483870967741936, "19": 0.622356495468278, "20": 0.5645645645645646 }, "pass@128": { "11": 0.9873015873015873, "12": 0.9877675840978594, "13": 0.9722222222222222, "14": 0.9376854599406528, "15": 0.9259259259259259, "16": 0.8348082595870207, "17": 0.8192771084337349, "18": 0.7935483870967742, "19": 0.6737160120845922, "20": 0.6096096096096096 } }, "total_pass_at_k": { "pass@1": 0.5018337408312958, "pass@2": 0.5592909535452323, "pass@4": 0.6081907090464548, "pass@8": 0.6702322738386308, "pass@16": 0.7249388753056235, "pass@32": 0.7753667481662592, "pass@64": 0.8202933985330073, "pass@128": 0.8533007334963325 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6626506024096386, "pass@2": 0.6987951807228916, "pass@4": 0.7921686746987951, "pass@8": 0.8524096385542169, "pass@16": 0.927710843373494, "pass@32": 0.963855421686747, "pass@64": 0.9728915662650602, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.5862068965517241, "pass@2": 0.6551724137931034, "pass@4": 0.7492163009404389, "pass@8": 0.8401253918495298, "pass@16": 0.9090909090909091, "pass@32": 0.9561128526645768, "pass@64": 0.9780564263322884, "pass@128": 0.9811912225705329 }, "13": { "pass@1": 0.6106442577030813, "pass@2": 0.6694677871148459, "pass@4": 0.7675070028011205, "pass@8": 0.8711484593837535, "pass@16": 0.927170868347339, "pass@32": 0.9523809523809523, "pass@64": 0.969187675070028, "pass@128": 0.9915966386554622 }, "14": { "pass@1": 0.5760233918128655, "pass@2": 0.631578947368421, "pass@4": 0.6900584795321637, "pass@8": 0.7543859649122807, "pass@16": 0.8099415204678363, "pass@32": 0.8859649122807017, "pass@64": 0.9502923976608187, "pass@128": 0.9736842105263158 }, "15": { "pass@1": 0.5484848484848485, "pass@2": 0.5848484848484848, "pass@4": 0.6090909090909091, "pass@8": 0.6666666666666666, "pass@16": 0.7333333333333333, "pass@32": 0.8272727272727273, "pass@64": 0.896969696969697, "pass@128": 0.9454545454545454 }, "16": { "pass@1": 0.5384615384615384, "pass@2": 0.5641025641025641, "pass@4": 0.5982905982905983, "pass@8": 0.6524216524216524, "pass@16": 0.6866096866096866, "pass@32": 0.7606837606837606, "pass@64": 0.8034188034188035, "pass@128": 0.8376068376068376 }, "17": { "pass@1": 0.4254658385093168, "pass@2": 0.4472049689440994, "pass@4": 0.5093167701863354, "pass@8": 0.5683229813664596, "pass@16": 0.6242236024844721, "pass@32": 0.6956521739130435, "pass@64": 0.7795031055900621, "pass@128": 0.8136645962732919 }, "18": { "pass@1": 0.4176470588235294, "pass@2": 0.45, "pass@4": 0.47941176470588237, "pass@8": 0.5205882352941177, "pass@16": 0.5647058823529412, "pass@32": 0.5794117647058824, "pass@64": 0.6823529411764706, "pass@128": 0.7529411764705882 }, "19": { "pass@1": 0.375, "pass@2": 0.40773809523809523, "pass@4": 0.44047619047619047, "pass@8": 0.4851190476190476, "pass@16": 0.5208333333333334, "pass@32": 0.5744047619047619, "pass@64": 0.6190476190476191, "pass@128": 0.6755952380952381 }, "20": { "pass@1": 0.2844311377245509, "pass@2": 0.33532934131736525, "pass@4": 0.38323353293413176, "pass@8": 0.4281437125748503, "pass@16": 0.47005988023952094, "pass@32": 0.5209580838323353, "pass@64": 0.5658682634730539, "pass@128": 0.6287425149700598 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6626506024096386, "12": 0.5862068965517241, "13": 0.6106442577030813, "14": 0.5760233918128655, "15": 0.5484848484848485, "16": 0.5384615384615384, "17": 0.4254658385093168, "18": 0.4176470588235294, "19": 0.375, "20": 0.2844311377245509 }, "pass@2": { "11": 0.6987951807228916, "12": 0.6551724137931034, "13": 0.6694677871148459, "14": 0.631578947368421, "15": 0.5848484848484848, "16": 0.5641025641025641, "17": 0.4472049689440994, "18": 0.45, "19": 0.40773809523809523, "20": 0.33532934131736525 }, "pass@4": { "11": 0.7921686746987951, "12": 0.7492163009404389, "13": 0.7675070028011205, "14": 0.6900584795321637, "15": 0.6090909090909091, "16": 0.5982905982905983, "17": 0.5093167701863354, "18": 0.47941176470588237, "19": 0.44047619047619047, "20": 0.38323353293413176 }, "pass@8": { "11": 0.8524096385542169, "12": 0.8401253918495298, "13": 0.8711484593837535, "14": 0.7543859649122807, "15": 0.6666666666666666, "16": 0.6524216524216524, "17": 0.5683229813664596, "18": 0.5205882352941177, "19": 0.4851190476190476, "20": 0.4281437125748503 }, "pass@16": { "11": 0.927710843373494, "12": 0.9090909090909091, "13": 0.927170868347339, "14": 0.8099415204678363, "15": 0.7333333333333333, "16": 0.6866096866096866, "17": 0.6242236024844721, "18": 0.5647058823529412, "19": 0.5208333333333334, "20": 0.47005988023952094 }, "pass@32": { "11": 0.963855421686747, "12": 0.9561128526645768, "13": 0.9523809523809523, "14": 0.8859649122807017, "15": 0.8272727272727273, "16": 0.7606837606837606, "17": 0.6956521739130435, "18": 0.5794117647058824, "19": 0.5744047619047619, "20": 0.5209580838323353 }, "pass@64": { "11": 0.9728915662650602, "12": 0.9780564263322884, "13": 0.969187675070028, "14": 0.9502923976608187, "15": 0.896969696969697, "16": 0.8034188034188035, "17": 0.7795031055900621, "18": 0.6823529411764706, "19": 0.6190476190476191, "20": 0.5658682634730539 }, "pass@128": { "11": 0.9789156626506024, "12": 0.9811912225705329, "13": 0.9915966386554622, "14": 0.9736842105263158, "15": 0.9454545454545454, "16": 0.8376068376068376, "17": 0.8136645962732919, "18": 0.7529411764705882, "19": 0.6755952380952381, "20": 0.6287425149700598 } }, "total_pass_at_k": { "pass@1": 0.5031222123104371, "pass@2": 0.5450490633363069, "pass@4": 0.6024382991376747, "pass@8": 0.6645851917930419, "pass@16": 0.7178114778471603, "pass@32": 0.7719298245614035, "pass@64": 0.8218852215283973, "pass@128": 0.8581623550401427 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.618, "pass@2": 0.682, "pass@4": 0.773, "pass@8": 0.856, "pass@16": 0.925, "pass@32": 0.962, "pass@64": 0.974, "pass@128": 0.984 }, "12": { "pass@1": 0.621, "pass@2": 0.683, "pass@4": 0.772, "pass@8": 0.862, "pass@16": 0.921, "pass@32": 0.956, "pass@64": 0.975, "pass@128": 0.982 }, "13": { "pass@1": 0.593, "pass@2": 0.665, "pass@4": 0.75, "pass@8": 0.837, "pass@16": 0.896, "pass@32": 0.932, "pass@64": 0.957, "pass@128": 0.975 }, "14": { "pass@1": 0.563, "pass@2": 0.605, "pass@4": 0.66, "pass@8": 0.727, "pass@16": 0.787, "pass@32": 0.863, "pass@64": 0.921, "pass@128": 0.947 }, "15": { "pass@1": 0.517, "pass@2": 0.557, "pass@4": 0.595, "pass@8": 0.667, "pass@16": 0.736, "pass@32": 0.818, "pass@64": 0.895, "pass@128": 0.941 }, "16": { "pass@1": 0.525, "pass@2": 0.562, "pass@4": 0.599, "pass@8": 0.648, "pass@16": 0.696, "pass@32": 0.749, "pass@64": 0.793, "pass@128": 0.823 }, "17": { "pass@1": 0.489, "pass@2": 0.514, "pass@4": 0.555, "pass@8": 0.603, "pass@16": 0.659, "pass@32": 0.716, "pass@64": 0.779, "pass@128": 0.823 }, "18": { "pass@1": 0.433, "pass@2": 0.471, "pass@4": 0.497, "pass@8": 0.532, "pass@16": 0.587, "pass@32": 0.63, "pass@64": 0.708, "pass@128": 0.772 }, "19": { "pass@1": 0.365, "pass@2": 0.422, "pass@4": 0.449, "pass@8": 0.492, "pass@16": 0.525, "pass@32": 0.571, "pass@64": 0.613, "pass@128": 0.659 }, "20": { "pass@1": 0.284, "pass@2": 0.337, "pass@4": 0.37, "pass@8": 0.415, "pass@16": 0.46, "pass@32": 0.516, "pass@64": 0.567, "pass@128": 0.621 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.618, "12": 0.621, "13": 0.593, "14": 0.563, "15": 0.517, "16": 0.525, "17": 0.489, "18": 0.433, "19": 0.365, "20": 0.284 }, "pass@2": { "11": 0.682, "12": 0.683, "13": 0.665, "14": 0.605, "15": 0.557, "16": 0.562, "17": 0.514, "18": 0.471, "19": 0.422, "20": 0.337 }, "pass@4": { "11": 0.773, "12": 0.772, "13": 0.75, "14": 0.66, "15": 0.595, "16": 0.599, "17": 0.555, "18": 0.497, "19": 0.449, "20": 0.37 }, "pass@8": { "11": 0.856, "12": 0.862, "13": 0.837, "14": 0.727, "15": 0.667, "16": 0.648, "17": 0.603, "18": 0.532, "19": 0.492, "20": 0.415 }, "pass@16": { "11": 0.925, "12": 0.921, "13": 0.896, "14": 0.787, "15": 0.736, "16": 0.696, "17": 0.659, "18": 0.587, "19": 0.525, "20": 0.46 }, "pass@32": { "11": 0.962, "12": 0.956, "13": 0.932, "14": 0.863, "15": 0.818, "16": 0.749, "17": 0.716, "18": 0.63, "19": 0.571, "20": 0.516 }, "pass@64": { "11": 0.974, "12": 0.975, "13": 0.957, "14": 0.921, "15": 0.895, "16": 0.793, "17": 0.779, "18": 0.708, "19": 0.613, "20": 0.567 }, "pass@128": { "11": 0.984, "12": 0.982, "13": 0.975, "14": 0.947, "15": 0.941, "16": 0.823, "17": 0.823, "18": 0.772, "19": 0.659, "20": 0.621 } }, "total_pass_at_k": { "pass@1": 0.5008, "pass@2": 0.5498, "pass@4": 0.602, "pass@8": 0.6639, "pass@16": 0.7192, "pass@32": 0.7713, "pass@64": 0.8182, "pass@128": 0.8527 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }