| { |
| "files": { |
| "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99375, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9587301587301588, |
| "pass@2": 0.9809523809523809, |
| "pass@4": 0.9936507936507937, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9550898203592815, |
| "pass@2": 0.9640718562874252, |
| "pass@4": 0.9940119760479041, |
| "pass@8": 0.9940119760479041, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.893491124260355, |
| "pass@2": 0.9230769230769231, |
| "pass@4": 0.9644970414201184, |
| "pass@8": 0.985207100591716, |
| "pass@16": 0.9940828402366864, |
| "pass@32": 0.9970414201183432, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8301282051282052, |
| "pass@2": 0.8782051282051282, |
| "pass@4": 0.9423076923076923, |
| "pass@8": 0.9807692307692307, |
| "pass@16": 0.9935897435897436, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7633802816901408, |
| "pass@2": 0.8338028169014085, |
| "pass@4": 0.9070422535211268, |
| "pass@8": 0.9690140845070423, |
| "pass@16": 0.9887323943661972, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6371681415929203, |
| "pass@2": 0.7109144542772862, |
| "pass@4": 0.7905604719764012, |
| "pass@8": 0.8584070796460177, |
| "pass@16": 0.9056047197640118, |
| "pass@32": 0.9410029498525073, |
| "pass@64": 0.9557522123893806, |
| "pass@128": 0.9616519174041298 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9587301587301588, |
| "6": 0.9550898203592815, |
| "7": 0.893491124260355, |
| "8": 0.8301282051282052, |
| "9": 0.7633802816901408, |
| "10": 0.6371681415929203 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9809523809523809, |
| "6": 0.9640718562874252, |
| "7": 0.9230769230769231, |
| "8": 0.8782051282051282, |
| "9": 0.8338028169014085, |
| "10": 0.7109144542772862 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9936507936507937, |
| "6": 0.9940119760479041, |
| "7": 0.9644970414201184, |
| "8": 0.9423076923076923, |
| "9": 0.9070422535211268, |
| "10": 0.7905604719764012 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9940119760479041, |
| "7": 0.985207100591716, |
| "8": 0.9807692307692307, |
| "9": 0.9690140845070423, |
| "10": 0.8584070796460177 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9940828402366864, |
| "8": 0.9935897435897436, |
| "9": 0.9887323943661972, |
| "10": 0.9056047197640118 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9410029498525073 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9557522123893806 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9616519174041298 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8899020601148261, |
| "pass@2": 0.9192840256670044, |
| "pass@4": 0.9533941236068896, |
| "pass@8": 0.9756838905775076, |
| "pass@16": 0.9864910503208375, |
| "pass@32": 0.9929078014184397, |
| "pass@64": 0.9949341438703141, |
| "pass@128": 0.9956095913542722 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9883040935672515, |
| "pass@2": 0.9941520467836257, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.95, |
| "pass@2": 0.9735294117647059, |
| "pass@4": 0.9911764705882353, |
| "pass@8": 0.9941176470588236, |
| "pass@16": 0.9970588235294118, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9684542586750788, |
| "pass@2": 0.9810725552050473, |
| "pass@4": 0.9842271293375394, |
| "pass@8": 0.9936908517350158, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8575949367088608, |
| "pass@2": 0.9145569620253164, |
| "pass@4": 0.9651898734177216, |
| "pass@8": 0.9936708860759493, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8168168168168168, |
| "pass@2": 0.8708708708708709, |
| "pass@4": 0.9129129129129129, |
| "pass@8": 0.960960960960961, |
| "pass@16": 0.993993993993994, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7652733118971061, |
| "pass@2": 0.8263665594855305, |
| "pass@4": 0.9035369774919614, |
| "pass@8": 0.9453376205787781, |
| "pass@16": 0.9646302250803859, |
| "pass@32": 0.9935691318327974, |
| "pass@64": 0.9967845659163987, |
| "pass@128": 0.9967845659163987 |
| }, |
| "10": { |
| "pass@1": 0.6149253731343284, |
| "pass@2": 0.6895522388059702, |
| "pass@4": 0.7701492537313432, |
| "pass@8": 0.8716417910447761, |
| "pass@16": 0.9343283582089552, |
| "pass@32": 0.9552238805970149, |
| "pass@64": 0.9791044776119403, |
| "pass@128": 0.9880597014925373 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9883040935672515, |
| "5": 0.95, |
| "6": 0.9684542586750788, |
| "7": 0.8575949367088608, |
| "8": 0.8168168168168168, |
| "9": 0.7652733118971061, |
| "10": 0.6149253731343284 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9941520467836257, |
| "5": 0.9735294117647059, |
| "6": 0.9810725552050473, |
| "7": 0.9145569620253164, |
| "8": 0.8708708708708709, |
| "9": 0.8263665594855305, |
| "10": 0.6895522388059702 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9911764705882353, |
| "6": 0.9842271293375394, |
| "7": 0.9651898734177216, |
| "8": 0.9129129129129129, |
| "9": 0.9035369774919614, |
| "10": 0.7701492537313432 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9941176470588236, |
| "6": 0.9936908517350158, |
| "7": 0.9936708860759493, |
| "8": 0.960960960960961, |
| "9": 0.9453376205787781, |
| "10": 0.8716417910447761 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.993993993993994, |
| "9": 0.9646302250803859, |
| "10": 0.9343283582089552 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9935691318327974, |
| "10": 0.9552238805970149 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.9791044776119403 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.9880597014925373 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8865082022095748, |
| "pass@2": 0.9179779042517576, |
| "pass@4": 0.9481084700368263, |
| "pass@8": 0.9735520589219954, |
| "pass@16": 0.987612989621694, |
| "pass@32": 0.9939738868429863, |
| "pass@64": 0.9973217274857716, |
| "pass@128": 0.9983260796786073 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.985207100591716, |
| "pass@2": 0.9881656804733728, |
| "pass@4": 0.9970414201183432, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9478260869565217, |
| "pass@2": 0.9826086956521739, |
| "pass@4": 0.991304347826087, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9770773638968482, |
| "pass@2": 0.9856733524355301, |
| "pass@4": 0.9914040114613181, |
| "pass@8": 0.997134670487106, |
| "pass@16": 0.997134670487106, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8554913294797688, |
| "pass@2": 0.9017341040462428, |
| "pass@4": 0.9595375722543352, |
| "pass@8": 0.9826589595375722, |
| "pass@16": 0.9913294797687862, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.819718309859155, |
| "pass@2": 0.8788732394366198, |
| "pass@4": 0.952112676056338, |
| "pass@8": 0.9830985915492958, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7455089820359282, |
| "pass@2": 0.8023952095808383, |
| "pass@4": 0.8952095808383234, |
| "pass@8": 0.9580838323353293, |
| "pass@16": 0.9850299401197605, |
| "pass@32": 0.9910179640718563, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6319018404907976, |
| "pass@2": 0.6932515337423313, |
| "pass@4": 0.7668711656441718, |
| "pass@8": 0.8312883435582822, |
| "pass@16": 0.8957055214723927, |
| "pass@32": 0.9478527607361963, |
| "pass@64": 0.9570552147239264, |
| "pass@128": 0.9693251533742331 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.985207100591716, |
| "5": 0.9478260869565217, |
| "6": 0.9770773638968482, |
| "7": 0.8554913294797688, |
| "8": 0.819718309859155, |
| "9": 0.7455089820359282, |
| "10": 0.6319018404907976 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9881656804733728, |
| "5": 0.9826086956521739, |
| "6": 0.9856733524355301, |
| "7": 0.9017341040462428, |
| "8": 0.8788732394366198, |
| "9": 0.8023952095808383, |
| "10": 0.6932515337423313 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 0.991304347826087, |
| "6": 0.9914040114613181, |
| "7": 0.9595375722543352, |
| "8": 0.952112676056338, |
| "9": 0.8952095808383234, |
| "10": 0.7668711656441718 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 0.997134670487106, |
| "7": 0.9826589595375722, |
| "8": 0.9830985915492958, |
| "9": 0.9580838323353293, |
| "10": 0.8312883435582822 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.997134670487106, |
| "7": 0.9913294797687862, |
| "8": 0.9971830985915493, |
| "9": 0.9850299401197605, |
| "10": 0.8957055214723927 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9910179640718563, |
| "10": 0.9478527607361963 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9570552147239264 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9693251533742331 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8853211009174312, |
| "pass@2": 0.9154652686762779, |
| "pass@4": 0.9511795543905636, |
| "pass@8": 0.9728047182175622, |
| "pass@16": 0.9855832241153342, |
| "pass@32": 0.9934469200524246, |
| "pass@64": 0.9954128440366973, |
| "pass@128": 0.9967234600262124 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.989, |
| "pass@2": 0.994, |
| "pass@4": 0.999, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.952, |
| "pass@2": 0.979, |
| "pass@4": 0.992, |
| "pass@8": 0.997, |
| "pass@16": 0.999, |
| "pass@32": 0.999, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.967, |
| "pass@2": 0.977, |
| "pass@4": 0.99, |
| "pass@8": 0.995, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.869, |
| "pass@2": 0.913, |
| "pass@4": 0.963, |
| "pass@8": 0.987, |
| "pass@16": 0.994, |
| "pass@32": 0.999, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.822, |
| "pass@2": 0.876, |
| "pass@4": 0.936, |
| "pass@8": 0.975, |
| "pass@16": 0.995, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.758, |
| "pass@2": 0.821, |
| "pass@4": 0.902, |
| "pass@8": 0.958, |
| "pass@16": 0.98, |
| "pass@32": 0.995, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "10": { |
| "pass@1": 0.628, |
| "pass@2": 0.698, |
| "pass@4": 0.776, |
| "pass@8": 0.854, |
| "pass@16": 0.912, |
| "pass@32": 0.948, |
| "pass@64": 0.964, |
| "pass@128": 0.973 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.989, |
| "5": 0.952, |
| "6": 0.967, |
| "7": 0.869, |
| "8": 0.822, |
| "9": 0.758, |
| "10": 0.628 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.994, |
| "5": 0.979, |
| "6": 0.977, |
| "7": 0.913, |
| "8": 0.876, |
| "9": 0.821, |
| "10": 0.698 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.999, |
| "5": 0.992, |
| "6": 0.99, |
| "7": 0.963, |
| "8": 0.936, |
| "9": 0.902, |
| "10": 0.776 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.997, |
| "6": 0.995, |
| "7": 0.987, |
| "8": 0.975, |
| "9": 0.958, |
| "10": 0.854 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.999, |
| "7": 0.994, |
| "8": 0.995, |
| "9": 0.98, |
| "10": 0.912 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 1.0, |
| "9": 0.995, |
| "10": 0.948 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.964 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.973 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8872222222222222, |
| "pass@2": 0.9175555555555556, |
| "pass@4": 0.9508888888888889, |
| "pass@8": 0.974, |
| "pass@16": 0.9865555555555555, |
| "pass@32": 0.9934444444444445, |
| "pass@64": 0.9958888888888889, |
| "pass@128": 0.9968888888888889 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5920679886685553, |
| "pass@2": 0.6798866855524079, |
| "pass@4": 0.7648725212464589, |
| "pass@8": 0.8470254957507082, |
| "pass@16": 0.9206798866855525, |
| "pass@32": 0.9603399433427762, |
| "pass@64": 0.9801699716713881, |
| "pass@128": 0.9858356940509915 |
| }, |
| "12": { |
| "pass@1": 0.6384180790960452, |
| "pass@2": 0.6836158192090396, |
| "pass@4": 0.768361581920904, |
| "pass@8": 0.867231638418079, |
| "pass@16": 0.923728813559322, |
| "pass@32": 0.9491525423728814, |
| "pass@64": 0.9717514124293786, |
| "pass@128": 0.9774011299435028 |
| }, |
| "13": { |
| "pass@1": 0.5862068965517241, |
| "pass@2": 0.6426332288401254, |
| "pass@4": 0.7241379310344828, |
| "pass@8": 0.8119122257053292, |
| "pass@16": 0.8746081504702194, |
| "pass@32": 0.9090909090909091, |
| "pass@64": 0.9373040752351097, |
| "pass@128": 0.9592476489028213 |
| }, |
| "14": { |
| "pass@1": 0.5514018691588785, |
| "pass@2": 0.5856697819314641, |
| "pass@4": 0.6261682242990654, |
| "pass@8": 0.6853582554517134, |
| "pass@16": 0.7476635514018691, |
| "pass@32": 0.8473520249221184, |
| "pass@64": 0.9003115264797508, |
| "pass@128": 0.9283489096573209 |
| }, |
| "15": { |
| "pass@1": 0.5028901734104047, |
| "pass@2": 0.5404624277456648, |
| "pass@4": 0.5867052023121387, |
| "pass@8": 0.6763005780346821, |
| "pass@16": 0.7485549132947977, |
| "pass@32": 0.838150289017341, |
| "pass@64": 0.9104046242774566, |
| "pass@128": 0.9508670520231214 |
| }, |
| "16": { |
| "pass@1": 0.4967741935483871, |
| "pass@2": 0.535483870967742, |
| "pass@4": 0.5838709677419355, |
| "pass@8": 0.6225806451612903, |
| "pass@16": 0.6838709677419355, |
| "pass@32": 0.7129032258064516, |
| "pass@64": 0.7645161290322581, |
| "pass@128": 0.7935483870967742 |
| }, |
| "17": { |
| "pass@1": 0.5433526011560693, |
| "pass@2": 0.5780346820809249, |
| "pass@4": 0.615606936416185, |
| "pass@8": 0.6445086705202312, |
| "pass@16": 0.6936416184971098, |
| "pass@32": 0.7312138728323699, |
| "pass@64": 0.7832369942196532, |
| "pass@128": 0.8352601156069365 |
| }, |
| "18": { |
| "pass@1": 0.43714285714285717, |
| "pass@2": 0.46, |
| "pass@4": 0.4828571428571429, |
| "pass@8": 0.5142857142857142, |
| "pass@16": 0.5742857142857143, |
| "pass@32": 0.6371428571428571, |
| "pass@64": 0.6971428571428572, |
| "pass@128": 0.7714285714285715 |
| }, |
| "19": { |
| "pass@1": 0.34234234234234234, |
| "pass@2": 0.4084084084084084, |
| "pass@4": 0.44144144144144143, |
| "pass@8": 0.4804804804804805, |
| "pass@16": 0.5225225225225225, |
| "pass@32": 0.5585585585585585, |
| "pass@64": 0.5975975975975976, |
| "pass@128": 0.6276276276276276 |
| }, |
| "20": { |
| "pass@1": 0.27627627627627627, |
| "pass@2": 0.3303303303303303, |
| "pass@4": 0.35135135135135137, |
| "pass@8": 0.4084084084084084, |
| "pass@16": 0.44744744744744747, |
| "pass@32": 0.5105105105105106, |
| "pass@64": 0.5705705705705706, |
| "pass@128": 0.6246246246246246 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5920679886685553, |
| "12": 0.6384180790960452, |
| "13": 0.5862068965517241, |
| "14": 0.5514018691588785, |
| "15": 0.5028901734104047, |
| "16": 0.4967741935483871, |
| "17": 0.5433526011560693, |
| "18": 0.43714285714285717, |
| "19": 0.34234234234234234, |
| "20": 0.27627627627627627 |
| }, |
| "pass@2": { |
| "11": 0.6798866855524079, |
| "12": 0.6836158192090396, |
| "13": 0.6426332288401254, |
| "14": 0.5856697819314641, |
| "15": 0.5404624277456648, |
| "16": 0.535483870967742, |
| "17": 0.5780346820809249, |
| "18": 0.46, |
| "19": 0.4084084084084084, |
| "20": 0.3303303303303303 |
| }, |
| "pass@4": { |
| "11": 0.7648725212464589, |
| "12": 0.768361581920904, |
| "13": 0.7241379310344828, |
| "14": 0.6261682242990654, |
| "15": 0.5867052023121387, |
| "16": 0.5838709677419355, |
| "17": 0.615606936416185, |
| "18": 0.4828571428571429, |
| "19": 0.44144144144144143, |
| "20": 0.35135135135135137 |
| }, |
| "pass@8": { |
| "11": 0.8470254957507082, |
| "12": 0.867231638418079, |
| "13": 0.8119122257053292, |
| "14": 0.6853582554517134, |
| "15": 0.6763005780346821, |
| "16": 0.6225806451612903, |
| "17": 0.6445086705202312, |
| "18": 0.5142857142857142, |
| "19": 0.4804804804804805, |
| "20": 0.4084084084084084 |
| }, |
| "pass@16": { |
| "11": 0.9206798866855525, |
| "12": 0.923728813559322, |
| "13": 0.8746081504702194, |
| "14": 0.7476635514018691, |
| "15": 0.7485549132947977, |
| "16": 0.6838709677419355, |
| "17": 0.6936416184971098, |
| "18": 0.5742857142857143, |
| "19": 0.5225225225225225, |
| "20": 0.44744744744744747 |
| }, |
| "pass@32": { |
| "11": 0.9603399433427762, |
| "12": 0.9491525423728814, |
| "13": 0.9090909090909091, |
| "14": 0.8473520249221184, |
| "15": 0.838150289017341, |
| "16": 0.7129032258064516, |
| "17": 0.7312138728323699, |
| "18": 0.6371428571428571, |
| "19": 0.5585585585585585, |
| "20": 0.5105105105105106 |
| }, |
| "pass@64": { |
| "11": 0.9801699716713881, |
| "12": 0.9717514124293786, |
| "13": 0.9373040752351097, |
| "14": 0.9003115264797508, |
| "15": 0.9104046242774566, |
| "16": 0.7645161290322581, |
| "17": 0.7832369942196532, |
| "18": 0.6971428571428572, |
| "19": 0.5975975975975976, |
| "20": 0.5705705705705706 |
| }, |
| "pass@128": { |
| "11": 0.9858356940509915, |
| "12": 0.9774011299435028, |
| "13": 0.9592476489028213, |
| "14": 0.9283489096573209, |
| "15": 0.9508670520231214, |
| "16": 0.7935483870967742, |
| "17": 0.8352601156069365, |
| "18": 0.7714285714285715, |
| "19": 0.6276276276276276, |
| "20": 0.6246246246246246 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4974739970282318, |
| "pass@2": 0.5453194650817236, |
| "pass@4": 0.5955423476968796, |
| "pass@8": 0.6570579494799406, |
| "pass@16": 0.7150074294205052, |
| "pass@32": 0.7667161961367014, |
| "pass@64": 0.812481426448737, |
| "pass@128": 0.8466567607726597 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6, |
| "pass@2": 0.6666666666666666, |
| "pass@4": 0.7619047619047619, |
| "pass@8": 0.8698412698412699, |
| "pass@16": 0.926984126984127, |
| "pass@32": 0.9619047619047619, |
| "pass@64": 0.9682539682539683, |
| "pass@128": 0.9873015873015873 |
| }, |
| "12": { |
| "pass@1": 0.636085626911315, |
| "pass@2": 0.709480122324159, |
| "pass@4": 0.7981651376146789, |
| "pass@8": 0.8776758409785933, |
| "pass@16": 0.9296636085626911, |
| "pass@32": 0.963302752293578, |
| "pass@64": 0.9755351681957186, |
| "pass@128": 0.9877675840978594 |
| }, |
| "13": { |
| "pass@1": 0.5802469135802469, |
| "pass@2": 0.6820987654320988, |
| "pass@4": 0.7561728395061729, |
| "pass@8": 0.8240740740740741, |
| "pass@16": 0.8827160493827161, |
| "pass@32": 0.9320987654320988, |
| "pass@64": 0.9629629629629629, |
| "pass@128": 0.9722222222222222 |
| }, |
| "14": { |
| "pass@1": 0.5608308605341247, |
| "pass@2": 0.5964391691394659, |
| "pass@4": 0.6617210682492581, |
| "pass@8": 0.7388724035608308, |
| "pass@16": 0.8011869436201781, |
| "pass@32": 0.8545994065281899, |
| "pass@64": 0.9109792284866469, |
| "pass@128": 0.9376854599406528 |
| }, |
| "15": { |
| "pass@1": 0.5, |
| "pass@2": 0.5462962962962963, |
| "pass@4": 0.5895061728395061, |
| "pass@8": 0.6574074074074074, |
| "pass@16": 0.7253086419753086, |
| "pass@32": 0.7870370370370371, |
| "pass@64": 0.8765432098765432, |
| "pass@128": 0.9259259259259259 |
| }, |
| "16": { |
| "pass@1": 0.5368731563421829, |
| "pass@2": 0.584070796460177, |
| "pass@4": 0.6135693215339233, |
| "pass@8": 0.6666666666666666, |
| "pass@16": 0.7168141592920354, |
| "pass@32": 0.7699115044247787, |
| "pass@64": 0.8082595870206489, |
| "pass@128": 0.8348082595870207 |
| }, |
| "17": { |
| "pass@1": 0.4939759036144578, |
| "pass@2": 0.5120481927710844, |
| "pass@4": 0.536144578313253, |
| "pass@8": 0.5933734939759037, |
| "pass@16": 0.6566265060240963, |
| "pass@32": 0.7198795180722891, |
| "pass@64": 0.7740963855421686, |
| "pass@128": 0.8192771084337349 |
| }, |
| "18": { |
| "pass@1": 0.44516129032258067, |
| "pass@2": 0.5064516129032258, |
| "pass@4": 0.532258064516129, |
| "pass@8": 0.5645161290322581, |
| "pass@16": 0.6258064516129033, |
| "pass@32": 0.6774193548387096, |
| "pass@64": 0.7483870967741936, |
| "pass@128": 0.7935483870967742 |
| }, |
| "19": { |
| "pass@1": 0.3776435045317221, |
| "pass@2": 0.4501510574018127, |
| "pass@4": 0.4652567975830816, |
| "pass@8": 0.5105740181268882, |
| "pass@16": 0.5317220543806647, |
| "pass@32": 0.5800604229607251, |
| "pass@64": 0.622356495468278, |
| "pass@128": 0.6737160120845922 |
| }, |
| "20": { |
| "pass@1": 0.2912912912912913, |
| "pass@2": 0.34534534534534533, |
| "pass@4": 0.37537537537537535, |
| "pass@8": 0.4084084084084084, |
| "pass@16": 0.4624624624624625, |
| "pass@32": 0.5165165165165165, |
| "pass@64": 0.5645645645645646, |
| "pass@128": 0.6096096096096096 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6, |
| "12": 0.636085626911315, |
| "13": 0.5802469135802469, |
| "14": 0.5608308605341247, |
| "15": 0.5, |
| "16": 0.5368731563421829, |
| "17": 0.4939759036144578, |
| "18": 0.44516129032258067, |
| "19": 0.3776435045317221, |
| "20": 0.2912912912912913 |
| }, |
| "pass@2": { |
| "11": 0.6666666666666666, |
| "12": 0.709480122324159, |
| "13": 0.6820987654320988, |
| "14": 0.5964391691394659, |
| "15": 0.5462962962962963, |
| "16": 0.584070796460177, |
| "17": 0.5120481927710844, |
| "18": 0.5064516129032258, |
| "19": 0.4501510574018127, |
| "20": 0.34534534534534533 |
| }, |
| "pass@4": { |
| "11": 0.7619047619047619, |
| "12": 0.7981651376146789, |
| "13": 0.7561728395061729, |
| "14": 0.6617210682492581, |
| "15": 0.5895061728395061, |
| "16": 0.6135693215339233, |
| "17": 0.536144578313253, |
| "18": 0.532258064516129, |
| "19": 0.4652567975830816, |
| "20": 0.37537537537537535 |
| }, |
| "pass@8": { |
| "11": 0.8698412698412699, |
| "12": 0.8776758409785933, |
| "13": 0.8240740740740741, |
| "14": 0.7388724035608308, |
| "15": 0.6574074074074074, |
| "16": 0.6666666666666666, |
| "17": 0.5933734939759037, |
| "18": 0.5645161290322581, |
| "19": 0.5105740181268882, |
| "20": 0.4084084084084084 |
| }, |
| "pass@16": { |
| "11": 0.926984126984127, |
| "12": 0.9296636085626911, |
| "13": 0.8827160493827161, |
| "14": 0.8011869436201781, |
| "15": 0.7253086419753086, |
| "16": 0.7168141592920354, |
| "17": 0.6566265060240963, |
| "18": 0.6258064516129033, |
| "19": 0.5317220543806647, |
| "20": 0.4624624624624625 |
| }, |
| "pass@32": { |
| "11": 0.9619047619047619, |
| "12": 0.963302752293578, |
| "13": 0.9320987654320988, |
| "14": 0.8545994065281899, |
| "15": 0.7870370370370371, |
| "16": 0.7699115044247787, |
| "17": 0.7198795180722891, |
| "18": 0.6774193548387096, |
| "19": 0.5800604229607251, |
| "20": 0.5165165165165165 |
| }, |
| "pass@64": { |
| "11": 0.9682539682539683, |
| "12": 0.9755351681957186, |
| "13": 0.9629629629629629, |
| "14": 0.9109792284866469, |
| "15": 0.8765432098765432, |
| "16": 0.8082595870206489, |
| "17": 0.7740963855421686, |
| "18": 0.7483870967741936, |
| "19": 0.622356495468278, |
| "20": 0.5645645645645646 |
| }, |
| "pass@128": { |
| "11": 0.9873015873015873, |
| "12": 0.9877675840978594, |
| "13": 0.9722222222222222, |
| "14": 0.9376854599406528, |
| "15": 0.9259259259259259, |
| "16": 0.8348082595870207, |
| "17": 0.8192771084337349, |
| "18": 0.7935483870967742, |
| "19": 0.6737160120845922, |
| "20": 0.6096096096096096 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5018337408312958, |
| "pass@2": 0.5592909535452323, |
| "pass@4": 0.6081907090464548, |
| "pass@8": 0.6702322738386308, |
| "pass@16": 0.7249388753056235, |
| "pass@32": 0.7753667481662592, |
| "pass@64": 0.8202933985330073, |
| "pass@128": 0.8533007334963325 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6626506024096386, |
| "pass@2": 0.6987951807228916, |
| "pass@4": 0.7921686746987951, |
| "pass@8": 0.8524096385542169, |
| "pass@16": 0.927710843373494, |
| "pass@32": 0.963855421686747, |
| "pass@64": 0.9728915662650602, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.5862068965517241, |
| "pass@2": 0.6551724137931034, |
| "pass@4": 0.7492163009404389, |
| "pass@8": 0.8401253918495298, |
| "pass@16": 0.9090909090909091, |
| "pass@32": 0.9561128526645768, |
| "pass@64": 0.9780564263322884, |
| "pass@128": 0.9811912225705329 |
| }, |
| "13": { |
| "pass@1": 0.6106442577030813, |
| "pass@2": 0.6694677871148459, |
| "pass@4": 0.7675070028011205, |
| "pass@8": 0.8711484593837535, |
| "pass@16": 0.927170868347339, |
| "pass@32": 0.9523809523809523, |
| "pass@64": 0.969187675070028, |
| "pass@128": 0.9915966386554622 |
| }, |
| "14": { |
| "pass@1": 0.5760233918128655, |
| "pass@2": 0.631578947368421, |
| "pass@4": 0.6900584795321637, |
| "pass@8": 0.7543859649122807, |
| "pass@16": 0.8099415204678363, |
| "pass@32": 0.8859649122807017, |
| "pass@64": 0.9502923976608187, |
| "pass@128": 0.9736842105263158 |
| }, |
| "15": { |
| "pass@1": 0.5484848484848485, |
| "pass@2": 0.5848484848484848, |
| "pass@4": 0.6090909090909091, |
| "pass@8": 0.6666666666666666, |
| "pass@16": 0.7333333333333333, |
| "pass@32": 0.8272727272727273, |
| "pass@64": 0.896969696969697, |
| "pass@128": 0.9454545454545454 |
| }, |
| "16": { |
| "pass@1": 0.5384615384615384, |
| "pass@2": 0.5641025641025641, |
| "pass@4": 0.5982905982905983, |
| "pass@8": 0.6524216524216524, |
| "pass@16": 0.6866096866096866, |
| "pass@32": 0.7606837606837606, |
| "pass@64": 0.8034188034188035, |
| "pass@128": 0.8376068376068376 |
| }, |
| "17": { |
| "pass@1": 0.4254658385093168, |
| "pass@2": 0.4472049689440994, |
| "pass@4": 0.5093167701863354, |
| "pass@8": 0.5683229813664596, |
| "pass@16": 0.6242236024844721, |
| "pass@32": 0.6956521739130435, |
| "pass@64": 0.7795031055900621, |
| "pass@128": 0.8136645962732919 |
| }, |
| "18": { |
| "pass@1": 0.4176470588235294, |
| "pass@2": 0.45, |
| "pass@4": 0.47941176470588237, |
| "pass@8": 0.5205882352941177, |
| "pass@16": 0.5647058823529412, |
| "pass@32": 0.5794117647058824, |
| "pass@64": 0.6823529411764706, |
| "pass@128": 0.7529411764705882 |
| }, |
| "19": { |
| "pass@1": 0.375, |
| "pass@2": 0.40773809523809523, |
| "pass@4": 0.44047619047619047, |
| "pass@8": 0.4851190476190476, |
| "pass@16": 0.5208333333333334, |
| "pass@32": 0.5744047619047619, |
| "pass@64": 0.6190476190476191, |
| "pass@128": 0.6755952380952381 |
| }, |
| "20": { |
| "pass@1": 0.2844311377245509, |
| "pass@2": 0.33532934131736525, |
| "pass@4": 0.38323353293413176, |
| "pass@8": 0.4281437125748503, |
| "pass@16": 0.47005988023952094, |
| "pass@32": 0.5209580838323353, |
| "pass@64": 0.5658682634730539, |
| "pass@128": 0.6287425149700598 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6626506024096386, |
| "12": 0.5862068965517241, |
| "13": 0.6106442577030813, |
| "14": 0.5760233918128655, |
| "15": 0.5484848484848485, |
| "16": 0.5384615384615384, |
| "17": 0.4254658385093168, |
| "18": 0.4176470588235294, |
| "19": 0.375, |
| "20": 0.2844311377245509 |
| }, |
| "pass@2": { |
| "11": 0.6987951807228916, |
| "12": 0.6551724137931034, |
| "13": 0.6694677871148459, |
| "14": 0.631578947368421, |
| "15": 0.5848484848484848, |
| "16": 0.5641025641025641, |
| "17": 0.4472049689440994, |
| "18": 0.45, |
| "19": 0.40773809523809523, |
| "20": 0.33532934131736525 |
| }, |
| "pass@4": { |
| "11": 0.7921686746987951, |
| "12": 0.7492163009404389, |
| "13": 0.7675070028011205, |
| "14": 0.6900584795321637, |
| "15": 0.6090909090909091, |
| "16": 0.5982905982905983, |
| "17": 0.5093167701863354, |
| "18": 0.47941176470588237, |
| "19": 0.44047619047619047, |
| "20": 0.38323353293413176 |
| }, |
| "pass@8": { |
| "11": 0.8524096385542169, |
| "12": 0.8401253918495298, |
| "13": 0.8711484593837535, |
| "14": 0.7543859649122807, |
| "15": 0.6666666666666666, |
| "16": 0.6524216524216524, |
| "17": 0.5683229813664596, |
| "18": 0.5205882352941177, |
| "19": 0.4851190476190476, |
| "20": 0.4281437125748503 |
| }, |
| "pass@16": { |
| "11": 0.927710843373494, |
| "12": 0.9090909090909091, |
| "13": 0.927170868347339, |
| "14": 0.8099415204678363, |
| "15": 0.7333333333333333, |
| "16": 0.6866096866096866, |
| "17": 0.6242236024844721, |
| "18": 0.5647058823529412, |
| "19": 0.5208333333333334, |
| "20": 0.47005988023952094 |
| }, |
| "pass@32": { |
| "11": 0.963855421686747, |
| "12": 0.9561128526645768, |
| "13": 0.9523809523809523, |
| "14": 0.8859649122807017, |
| "15": 0.8272727272727273, |
| "16": 0.7606837606837606, |
| "17": 0.6956521739130435, |
| "18": 0.5794117647058824, |
| "19": 0.5744047619047619, |
| "20": 0.5209580838323353 |
| }, |
| "pass@64": { |
| "11": 0.9728915662650602, |
| "12": 0.9780564263322884, |
| "13": 0.969187675070028, |
| "14": 0.9502923976608187, |
| "15": 0.896969696969697, |
| "16": 0.8034188034188035, |
| "17": 0.7795031055900621, |
| "18": 0.6823529411764706, |
| "19": 0.6190476190476191, |
| "20": 0.5658682634730539 |
| }, |
| "pass@128": { |
| "11": 0.9789156626506024, |
| "12": 0.9811912225705329, |
| "13": 0.9915966386554622, |
| "14": 0.9736842105263158, |
| "15": 0.9454545454545454, |
| "16": 0.8376068376068376, |
| "17": 0.8136645962732919, |
| "18": 0.7529411764705882, |
| "19": 0.6755952380952381, |
| "20": 0.6287425149700598 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5031222123104371, |
| "pass@2": 0.5450490633363069, |
| "pass@4": 0.6024382991376747, |
| "pass@8": 0.6645851917930419, |
| "pass@16": 0.7178114778471603, |
| "pass@32": 0.7719298245614035, |
| "pass@64": 0.8218852215283973, |
| "pass@128": 0.8581623550401427 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.618, |
| "pass@2": 0.682, |
| "pass@4": 0.773, |
| "pass@8": 0.856, |
| "pass@16": 0.925, |
| "pass@32": 0.962, |
| "pass@64": 0.974, |
| "pass@128": 0.984 |
| }, |
| "12": { |
| "pass@1": 0.621, |
| "pass@2": 0.683, |
| "pass@4": 0.772, |
| "pass@8": 0.862, |
| "pass@16": 0.921, |
| "pass@32": 0.956, |
| "pass@64": 0.975, |
| "pass@128": 0.982 |
| }, |
| "13": { |
| "pass@1": 0.593, |
| "pass@2": 0.665, |
| "pass@4": 0.75, |
| "pass@8": 0.837, |
| "pass@16": 0.896, |
| "pass@32": 0.932, |
| "pass@64": 0.957, |
| "pass@128": 0.975 |
| }, |
| "14": { |
| "pass@1": 0.563, |
| "pass@2": 0.605, |
| "pass@4": 0.66, |
| "pass@8": 0.727, |
| "pass@16": 0.787, |
| "pass@32": 0.863, |
| "pass@64": 0.921, |
| "pass@128": 0.947 |
| }, |
| "15": { |
| "pass@1": 0.517, |
| "pass@2": 0.557, |
| "pass@4": 0.595, |
| "pass@8": 0.667, |
| "pass@16": 0.736, |
| "pass@32": 0.818, |
| "pass@64": 0.895, |
| "pass@128": 0.941 |
| }, |
| "16": { |
| "pass@1": 0.525, |
| "pass@2": 0.562, |
| "pass@4": 0.599, |
| "pass@8": 0.648, |
| "pass@16": 0.696, |
| "pass@32": 0.749, |
| "pass@64": 0.793, |
| "pass@128": 0.823 |
| }, |
| "17": { |
| "pass@1": 0.489, |
| "pass@2": 0.514, |
| "pass@4": 0.555, |
| "pass@8": 0.603, |
| "pass@16": 0.659, |
| "pass@32": 0.716, |
| "pass@64": 0.779, |
| "pass@128": 0.823 |
| }, |
| "18": { |
| "pass@1": 0.433, |
| "pass@2": 0.471, |
| "pass@4": 0.497, |
| "pass@8": 0.532, |
| "pass@16": 0.587, |
| "pass@32": 0.63, |
| "pass@64": 0.708, |
| "pass@128": 0.772 |
| }, |
| "19": { |
| "pass@1": 0.365, |
| "pass@2": 0.422, |
| "pass@4": 0.449, |
| "pass@8": 0.492, |
| "pass@16": 0.525, |
| "pass@32": 0.571, |
| "pass@64": 0.613, |
| "pass@128": 0.659 |
| }, |
| "20": { |
| "pass@1": 0.284, |
| "pass@2": 0.337, |
| "pass@4": 0.37, |
| "pass@8": 0.415, |
| "pass@16": 0.46, |
| "pass@32": 0.516, |
| "pass@64": 0.567, |
| "pass@128": 0.621 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.618, |
| "12": 0.621, |
| "13": 0.593, |
| "14": 0.563, |
| "15": 0.517, |
| "16": 0.525, |
| "17": 0.489, |
| "18": 0.433, |
| "19": 0.365, |
| "20": 0.284 |
| }, |
| "pass@2": { |
| "11": 0.682, |
| "12": 0.683, |
| "13": 0.665, |
| "14": 0.605, |
| "15": 0.557, |
| "16": 0.562, |
| "17": 0.514, |
| "18": 0.471, |
| "19": 0.422, |
| "20": 0.337 |
| }, |
| "pass@4": { |
| "11": 0.773, |
| "12": 0.772, |
| "13": 0.75, |
| "14": 0.66, |
| "15": 0.595, |
| "16": 0.599, |
| "17": 0.555, |
| "18": 0.497, |
| "19": 0.449, |
| "20": 0.37 |
| }, |
| "pass@8": { |
| "11": 0.856, |
| "12": 0.862, |
| "13": 0.837, |
| "14": 0.727, |
| "15": 0.667, |
| "16": 0.648, |
| "17": 0.603, |
| "18": 0.532, |
| "19": 0.492, |
| "20": 0.415 |
| }, |
| "pass@16": { |
| "11": 0.925, |
| "12": 0.921, |
| "13": 0.896, |
| "14": 0.787, |
| "15": 0.736, |
| "16": 0.696, |
| "17": 0.659, |
| "18": 0.587, |
| "19": 0.525, |
| "20": 0.46 |
| }, |
| "pass@32": { |
| "11": 0.962, |
| "12": 0.956, |
| "13": 0.932, |
| "14": 0.863, |
| "15": 0.818, |
| "16": 0.749, |
| "17": 0.716, |
| "18": 0.63, |
| "19": 0.571, |
| "20": 0.516 |
| }, |
| "pass@64": { |
| "11": 0.974, |
| "12": 0.975, |
| "13": 0.957, |
| "14": 0.921, |
| "15": 0.895, |
| "16": 0.793, |
| "17": 0.779, |
| "18": 0.708, |
| "19": 0.613, |
| "20": 0.567 |
| }, |
| "pass@128": { |
| "11": 0.984, |
| "12": 0.982, |
| "13": 0.975, |
| "14": 0.947, |
| "15": 0.941, |
| "16": 0.823, |
| "17": 0.823, |
| "18": 0.772, |
| "19": 0.659, |
| "20": 0.621 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5008, |
| "pass@2": 0.5498, |
| "pass@4": 0.602, |
| "pass@8": 0.6639, |
| "pass@16": 0.7192, |
| "pass@32": 0.7713, |
| "pass@64": 0.8182, |
| "pass@128": 0.8527 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
