| { |
| "files": { |
| "checkpoint-38797_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99375, |
| "pass@2": 0.996875, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9587301587301588, |
| "pass@2": 0.9746031746031746, |
| "pass@4": 0.9936507936507937, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9580838323353293, |
| "pass@2": 0.9760479041916168, |
| "pass@4": 0.9850299401197605, |
| "pass@8": 0.9970059880239521, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8846153846153846, |
| "pass@2": 0.9289940828402367, |
| "pass@4": 0.9526627218934911, |
| "pass@8": 0.9911242603550295, |
| "pass@16": 0.9970414201183432, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8269230769230769, |
| "pass@2": 0.8846153846153846, |
| "pass@4": 0.9583333333333334, |
| "pass@8": 0.9871794871794872, |
| "pass@16": 0.9935897435897436, |
| "pass@32": 0.9967948717948718, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7408450704225352, |
| "pass@2": 0.8140845070422535, |
| "pass@4": 0.8929577464788733, |
| "pass@8": 0.9464788732394366, |
| "pass@16": 0.9746478873239437, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6312684365781711, |
| "pass@2": 0.7079646017699115, |
| "pass@4": 0.8023598820058997, |
| "pass@8": 0.8643067846607669, |
| "pass@16": 0.911504424778761, |
| "pass@32": 0.9410029498525073, |
| "pass@64": 0.9469026548672567, |
| "pass@128": 0.9557522123893806 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99375, |
| "5": 0.9587301587301588, |
| "6": 0.9580838323353293, |
| "7": 0.8846153846153846, |
| "8": 0.8269230769230769, |
| "9": 0.7408450704225352, |
| "10": 0.6312684365781711 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9746031746031746, |
| "6": 0.9760479041916168, |
| "7": 0.9289940828402367, |
| "8": 0.8846153846153846, |
| "9": 0.8140845070422535, |
| "10": 0.7079646017699115 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9936507936507937, |
| "6": 0.9850299401197605, |
| "7": 0.9526627218934911, |
| "8": 0.9583333333333334, |
| "9": 0.8929577464788733, |
| "10": 0.8023598820058997 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.9970059880239521, |
| "7": 0.9911242603550295, |
| "8": 0.9871794871794872, |
| "9": 0.9464788732394366, |
| "10": 0.8643067846607669 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9970414201183432, |
| "8": 0.9935897435897436, |
| "9": 0.9746478873239437, |
| "10": 0.911504424778761 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9967948717948718, |
| "9": 0.9971830985915493, |
| "10": 0.9410029498525073 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9469026548672567 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9557522123893806 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8855116514690983, |
| "pass@2": 0.9182708544410672, |
| "pass@4": 0.9523809523809523, |
| "pass@8": 0.9753461668355285, |
| "pass@16": 0.9858156028368794, |
| "pass@32": 0.9925700776764607, |
| "pass@64": 0.993920972644377, |
| "pass@128": 0.9949341438703141 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9883040935672515, |
| "pass@2": 0.9970760233918129, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9441176470588235, |
| "pass@2": 0.9705882352941176, |
| "pass@4": 0.9911764705882353, |
| "pass@8": 0.9970588235294118, |
| "pass@16": 0.9970588235294118, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 0.9970588235294118, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9716088328075709, |
| "pass@2": 0.9842271293375394, |
| "pass@4": 0.9936908517350158, |
| "pass@8": 0.9968454258675079, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9050632911392406, |
| "pass@2": 0.939873417721519, |
| "pass@4": 0.9620253164556962, |
| "pass@8": 0.9936708860759493, |
| "pass@16": 0.9968354430379747, |
| "pass@32": 0.9968354430379747, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8288288288288288, |
| "pass@2": 0.8588588588588588, |
| "pass@4": 0.9279279279279279, |
| "pass@8": 0.9819819819819819, |
| "pass@16": 0.996996996996997, |
| "pass@32": 0.996996996996997, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7909967845659164, |
| "pass@2": 0.8456591639871383, |
| "pass@4": 0.9003215434083601, |
| "pass@8": 0.954983922829582, |
| "pass@16": 0.9839228295819936, |
| "pass@32": 0.9967845659163987, |
| "pass@64": 0.9967845659163987, |
| "pass@128": 0.9967845659163987 |
| }, |
| "10": { |
| "pass@1": 0.6059701492537314, |
| "pass@2": 0.6447761194029851, |
| "pass@4": 0.7313432835820896, |
| "pass@8": 0.8388059701492537, |
| "pass@16": 0.9223880597014925, |
| "pass@32": 0.9731343283582089, |
| "pass@64": 0.9880597014925373, |
| "pass@128": 0.991044776119403 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9883040935672515, |
| "5": 0.9441176470588235, |
| "6": 0.9716088328075709, |
| "7": 0.9050632911392406, |
| "8": 0.8288288288288288, |
| "9": 0.7909967845659164, |
| "10": 0.6059701492537314 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.9705882352941176, |
| "6": 0.9842271293375394, |
| "7": 0.939873417721519, |
| "8": 0.8588588588588588, |
| "9": 0.8456591639871383, |
| "10": 0.6447761194029851 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9911764705882353, |
| "6": 0.9936908517350158, |
| "7": 0.9620253164556962, |
| "8": 0.9279279279279279, |
| "9": 0.9003215434083601, |
| "10": 0.7313432835820896 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 0.9968454258675079, |
| "7": 0.9936708860759493, |
| "8": 0.9819819819819819, |
| "9": 0.954983922829582, |
| "10": 0.8388059701492537 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.996996996996997, |
| "9": 0.9839228295819936, |
| "10": 0.9223880597014925 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.996996996996997, |
| "9": 0.9967845659163987, |
| "10": 0.9731343283582089 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.9880597014925373 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.991044776119403 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8942082356879812, |
| "pass@2": 0.9166387679946435, |
| "pass@4": 0.9457649815868765, |
| "pass@8": 0.9738868429862738, |
| "pass@16": 0.9886173418145296, |
| "pass@32": 0.995647807164379, |
| "pass@64": 0.9979912956143288, |
| "pass@128": 0.9983260796786073 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9881656804733728, |
| "pass@2": 0.9911242603550295, |
| "pass@4": 0.9970414201183432, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9478260869565217, |
| "pass@2": 0.9826086956521739, |
| "pass@4": 0.9971014492753624, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9770773638968482, |
| "pass@2": 0.9828080229226361, |
| "pass@4": 0.9885386819484241, |
| "pass@8": 0.994269340974212, |
| "pass@16": 0.997134670487106, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8872832369942196, |
| "pass@2": 0.9248554913294798, |
| "pass@4": 0.9739884393063584, |
| "pass@8": 0.9884393063583815, |
| "pass@16": 0.9971098265895953, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8169014084507042, |
| "pass@2": 0.8901408450704226, |
| "pass@4": 0.9464788732394366, |
| "pass@8": 0.9802816901408451, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7455089820359282, |
| "pass@2": 0.8203592814371258, |
| "pass@4": 0.8922155688622755, |
| "pass@8": 0.9341317365269461, |
| "pass@16": 0.9760479041916168, |
| "pass@32": 0.9940119760479041, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6411042944785276, |
| "pass@2": 0.6871165644171779, |
| "pass@4": 0.7760736196319018, |
| "pass@8": 0.8496932515337423, |
| "pass@16": 0.9049079754601227, |
| "pass@32": 0.9294478527607362, |
| "pass@64": 0.9447852760736196, |
| "pass@128": 0.9570552147239264 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9881656804733728, |
| "5": 0.9478260869565217, |
| "6": 0.9770773638968482, |
| "7": 0.8872832369942196, |
| "8": 0.8169014084507042, |
| "9": 0.7455089820359282, |
| "10": 0.6411042944785276 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9911242603550295, |
| "5": 0.9826086956521739, |
| "6": 0.9828080229226361, |
| "7": 0.9248554913294798, |
| "8": 0.8901408450704226, |
| "9": 0.8203592814371258, |
| "10": 0.6871165644171779 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970414201183432, |
| "5": 0.9971014492753624, |
| "6": 0.9885386819484241, |
| "7": 0.9739884393063584, |
| "8": 0.9464788732394366, |
| "9": 0.8922155688622755, |
| "10": 0.7760736196319018 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.994269340974212, |
| "7": 0.9884393063583815, |
| "8": 0.9802816901408451, |
| "9": 0.9341317365269461, |
| "10": 0.8496932515337423 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 0.997134670487106, |
| "7": 0.9971098265895953, |
| "8": 0.9971830985915493, |
| "9": 0.9760479041916168, |
| "10": 0.9049079754601227 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9940119760479041, |
| "10": 0.9294478527607362 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.9447852760736196 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9570552147239264 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8899082568807339, |
| "pass@2": 0.9207077326343381, |
| "pass@4": 0.9531454783748362, |
| "pass@8": 0.9724770642201835, |
| "pass@16": 0.9862385321100917, |
| "pass@32": 0.9918086500655308, |
| "pass@64": 0.9937745740498034, |
| "pass@128": 0.9954128440366973 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99, |
| "pass@2": 0.995, |
| "pass@4": 0.999, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.95, |
| "pass@2": 0.976, |
| "pass@4": 0.994, |
| "pass@8": 0.999, |
| "pass@16": 0.999, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.969, |
| "pass@2": 0.981, |
| "pass@4": 0.989, |
| "pass@8": 0.996, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.892, |
| "pass@2": 0.931, |
| "pass@4": 0.963, |
| "pass@8": 0.991, |
| "pass@16": 0.997, |
| "pass@32": 0.999, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.824, |
| "pass@2": 0.878, |
| "pass@4": 0.944, |
| "pass@8": 0.983, |
| "pass@16": 0.996, |
| "pass@32": 0.998, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.758, |
| "pass@2": 0.826, |
| "pass@4": 0.895, |
| "pass@8": 0.945, |
| "pass@16": 0.978, |
| "pass@32": 0.996, |
| "pass@64": 0.998, |
| "pass@128": 0.999 |
| }, |
| "10": { |
| "pass@1": 0.626, |
| "pass@2": 0.68, |
| "pass@4": 0.77, |
| "pass@8": 0.851, |
| "pass@16": 0.913, |
| "pass@32": 0.948, |
| "pass@64": 0.96, |
| "pass@128": 0.968 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.99, |
| "5": 0.95, |
| "6": 0.969, |
| "7": 0.892, |
| "8": 0.824, |
| "9": 0.758, |
| "10": 0.626 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.995, |
| "5": 0.976, |
| "6": 0.981, |
| "7": 0.931, |
| "8": 0.878, |
| "9": 0.826, |
| "10": 0.68 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.999, |
| "5": 0.994, |
| "6": 0.989, |
| "7": 0.963, |
| "8": 0.944, |
| "9": 0.895, |
| "10": 0.77 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.996, |
| "7": 0.991, |
| "8": 0.983, |
| "9": 0.945, |
| "10": 0.851 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 0.999, |
| "7": 0.997, |
| "8": 0.996, |
| "9": 0.978, |
| "10": 0.913 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 0.998, |
| "9": 0.996, |
| "10": 0.948 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.998, |
| "10": 0.96 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.968 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8898888888888888, |
| "pass@2": 0.9185555555555556, |
| "pass@4": 0.9504444444444444, |
| "pass@8": 0.9738888888888889, |
| "pass@16": 0.9868888888888889, |
| "pass@32": 0.9933333333333333, |
| "pass@64": 0.9952222222222222, |
| "pass@128": 0.9962222222222222 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-38797_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.5750708215297451, |
| "pass@2": 0.6458923512747875, |
| "pass@4": 0.7252124645892352, |
| "pass@8": 0.8498583569405099, |
| "pass@16": 0.9008498583569405, |
| "pass@32": 0.9490084985835694, |
| "pass@64": 0.9631728045325779, |
| "pass@128": 0.9745042492917847 |
| }, |
| "12": { |
| "pass@1": 0.652542372881356, |
| "pass@2": 0.7175141242937854, |
| "pass@4": 0.7966101694915254, |
| "pass@8": 0.8757062146892656, |
| "pass@16": 0.9293785310734464, |
| "pass@32": 0.943502824858757, |
| "pass@64": 0.9689265536723164, |
| "pass@128": 0.9745762711864406 |
| }, |
| "13": { |
| "pass@1": 0.6081504702194357, |
| "pass@2": 0.6489028213166145, |
| "pass@4": 0.7178683385579937, |
| "pass@8": 0.8087774294670846, |
| "pass@16": 0.8746081504702194, |
| "pass@32": 0.9153605015673981, |
| "pass@64": 0.9404388714733543, |
| "pass@128": 0.9655172413793104 |
| }, |
| "14": { |
| "pass@1": 0.557632398753894, |
| "pass@2": 0.5825545171339563, |
| "pass@4": 0.6137071651090342, |
| "pass@8": 0.6822429906542056, |
| "pass@16": 0.7570093457943925, |
| "pass@32": 0.8193146417445483, |
| "pass@64": 0.8722741433021807, |
| "pass@128": 0.9252336448598131 |
| }, |
| "15": { |
| "pass@1": 0.5260115606936416, |
| "pass@2": 0.5549132947976878, |
| "pass@4": 0.5867052023121387, |
| "pass@8": 0.6560693641618497, |
| "pass@16": 0.7254335260115607, |
| "pass@32": 0.8034682080924855, |
| "pass@64": 0.884393063583815, |
| "pass@128": 0.9393063583815029 |
| }, |
| "16": { |
| "pass@1": 0.5, |
| "pass@2": 0.5258064516129032, |
| "pass@4": 0.567741935483871, |
| "pass@8": 0.6161290322580645, |
| "pass@16": 0.6419354838709678, |
| "pass@32": 0.7, |
| "pass@64": 0.7483870967741936, |
| "pass@128": 0.7935483870967742 |
| }, |
| "17": { |
| "pass@1": 0.5433526011560693, |
| "pass@2": 0.5751445086705202, |
| "pass@4": 0.6069364161849711, |
| "pass@8": 0.653179190751445, |
| "pass@16": 0.7023121387283237, |
| "pass@32": 0.7485549132947977, |
| "pass@64": 0.8063583815028902, |
| "pass@128": 0.8526011560693642 |
| }, |
| "18": { |
| "pass@1": 0.4257142857142857, |
| "pass@2": 0.48, |
| "pass@4": 0.5228571428571429, |
| "pass@8": 0.54, |
| "pass@16": 0.5742857142857143, |
| "pass@32": 0.62, |
| "pass@64": 0.7114285714285714, |
| "pass@128": 0.78 |
| }, |
| "19": { |
| "pass@1": 0.38738738738738737, |
| "pass@2": 0.42042042042042044, |
| "pass@4": 0.4594594594594595, |
| "pass@8": 0.4864864864864865, |
| "pass@16": 0.5375375375375375, |
| "pass@32": 0.5885885885885885, |
| "pass@64": 0.6486486486486487, |
| "pass@128": 0.6906906906906907 |
| }, |
| "20": { |
| "pass@1": 0.2882882882882883, |
| "pass@2": 0.35135135135135137, |
| "pass@4": 0.3993993993993994, |
| "pass@8": 0.4264264264264264, |
| "pass@16": 0.48348348348348347, |
| "pass@32": 0.5495495495495496, |
| "pass@64": 0.6036036036036037, |
| "pass@128": 0.6456456456456456 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.5750708215297451, |
| "12": 0.652542372881356, |
| "13": 0.6081504702194357, |
| "14": 0.557632398753894, |
| "15": 0.5260115606936416, |
| "16": 0.5, |
| "17": 0.5433526011560693, |
| "18": 0.4257142857142857, |
| "19": 0.38738738738738737, |
| "20": 0.2882882882882883 |
| }, |
| "pass@2": { |
| "11": 0.6458923512747875, |
| "12": 0.7175141242937854, |
| "13": 0.6489028213166145, |
| "14": 0.5825545171339563, |
| "15": 0.5549132947976878, |
| "16": 0.5258064516129032, |
| "17": 0.5751445086705202, |
| "18": 0.48, |
| "19": 0.42042042042042044, |
| "20": 0.35135135135135137 |
| }, |
| "pass@4": { |
| "11": 0.7252124645892352, |
| "12": 0.7966101694915254, |
| "13": 0.7178683385579937, |
| "14": 0.6137071651090342, |
| "15": 0.5867052023121387, |
| "16": 0.567741935483871, |
| "17": 0.6069364161849711, |
| "18": 0.5228571428571429, |
| "19": 0.4594594594594595, |
| "20": 0.3993993993993994 |
| }, |
| "pass@8": { |
| "11": 0.8498583569405099, |
| "12": 0.8757062146892656, |
| "13": 0.8087774294670846, |
| "14": 0.6822429906542056, |
| "15": 0.6560693641618497, |
| "16": 0.6161290322580645, |
| "17": 0.653179190751445, |
| "18": 0.54, |
| "19": 0.4864864864864865, |
| "20": 0.4264264264264264 |
| }, |
| "pass@16": { |
| "11": 0.9008498583569405, |
| "12": 0.9293785310734464, |
| "13": 0.8746081504702194, |
| "14": 0.7570093457943925, |
| "15": 0.7254335260115607, |
| "16": 0.6419354838709678, |
| "17": 0.7023121387283237, |
| "18": 0.5742857142857143, |
| "19": 0.5375375375375375, |
| "20": 0.48348348348348347 |
| }, |
| "pass@32": { |
| "11": 0.9490084985835694, |
| "12": 0.943502824858757, |
| "13": 0.9153605015673981, |
| "14": 0.8193146417445483, |
| "15": 0.8034682080924855, |
| "16": 0.7, |
| "17": 0.7485549132947977, |
| "18": 0.62, |
| "19": 0.5885885885885885, |
| "20": 0.5495495495495496 |
| }, |
| "pass@64": { |
| "11": 0.9631728045325779, |
| "12": 0.9689265536723164, |
| "13": 0.9404388714733543, |
| "14": 0.8722741433021807, |
| "15": 0.884393063583815, |
| "16": 0.7483870967741936, |
| "17": 0.8063583815028902, |
| "18": 0.7114285714285714, |
| "19": 0.6486486486486487, |
| "20": 0.6036036036036037 |
| }, |
| "pass@128": { |
| "11": 0.9745042492917847, |
| "12": 0.9745762711864406, |
| "13": 0.9655172413793104, |
| "14": 0.9252336448598131, |
| "15": 0.9393063583815029, |
| "16": 0.7935483870967742, |
| "17": 0.8526011560693642, |
| "18": 0.78, |
| "19": 0.6906906906906907, |
| "20": 0.6456456456456456 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5069836552748885, |
| "pass@2": 0.5512630014858841, |
| "pass@4": 0.600891530460624, |
| "pass@8": 0.6609212481426449, |
| "pass@16": 0.7141158989598811, |
| "pass@32": 0.7649331352154531, |
| "pass@64": 0.8160475482912333, |
| "pass@128": 0.8552748885586924 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6, |
| "pass@2": 0.6666666666666666, |
| "pass@4": 0.7619047619047619, |
| "pass@8": 0.8571428571428571, |
| "pass@16": 0.9174603174603174, |
| "pass@32": 0.9587301587301588, |
| "pass@64": 0.9841269841269841, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.6391437308868502, |
| "pass@2": 0.7003058103975535, |
| "pass@4": 0.7981651376146789, |
| "pass@8": 0.8807339449541285, |
| "pass@16": 0.9235474006116208, |
| "pass@32": 0.9663608562691132, |
| "pass@64": 0.9785932721712538, |
| "pass@128": 0.981651376146789 |
| }, |
| "13": { |
| "pass@1": 0.6080246913580247, |
| "pass@2": 0.6759259259259259, |
| "pass@4": 0.7654320987654321, |
| "pass@8": 0.8518518518518519, |
| "pass@16": 0.8919753086419753, |
| "pass@32": 0.9290123456790124, |
| "pass@64": 0.9567901234567902, |
| "pass@128": 0.9629629629629629 |
| }, |
| "14": { |
| "pass@1": 0.5608308605341247, |
| "pass@2": 0.5875370919881305, |
| "pass@4": 0.6468842729970327, |
| "pass@8": 0.7002967359050445, |
| "pass@16": 0.7626112759643917, |
| "pass@32": 0.8486646884272997, |
| "pass@64": 0.9080118694362018, |
| "pass@128": 0.9406528189910979 |
| }, |
| "15": { |
| "pass@1": 0.5092592592592593, |
| "pass@2": 0.5401234567901234, |
| "pass@4": 0.5709876543209876, |
| "pass@8": 0.6450617283950617, |
| "pass@16": 0.7283950617283951, |
| "pass@32": 0.7870370370370371, |
| "pass@64": 0.8858024691358025, |
| "pass@128": 0.9320987654320988 |
| }, |
| "16": { |
| "pass@1": 0.5368731563421829, |
| "pass@2": 0.5752212389380531, |
| "pass@4": 0.6194690265486725, |
| "pass@8": 0.6607669616519174, |
| "pass@16": 0.7138643067846607, |
| "pass@32": 0.7492625368731564, |
| "pass@64": 0.7876106194690266, |
| "pass@128": 0.8171091445427728 |
| }, |
| "17": { |
| "pass@1": 0.48493975903614456, |
| "pass@2": 0.5030120481927711, |
| "pass@4": 0.5572289156626506, |
| "pass@8": 0.5903614457831325, |
| "pass@16": 0.6475903614457831, |
| "pass@32": 0.6897590361445783, |
| "pass@64": 0.7680722891566265, |
| "pass@128": 0.8042168674698795 |
| }, |
| "18": { |
| "pass@1": 0.4645161290322581, |
| "pass@2": 0.5064516129032258, |
| "pass@4": 0.5451612903225806, |
| "pass@8": 0.5870967741935483, |
| "pass@16": 0.632258064516129, |
| "pass@32": 0.6806451612903226, |
| "pass@64": 0.7483870967741936, |
| "pass@128": 0.7935483870967742 |
| }, |
| "19": { |
| "pass@1": 0.3806646525679758, |
| "pass@2": 0.4380664652567976, |
| "pass@4": 0.48338368580060426, |
| "pass@8": 0.5166163141993958, |
| "pass@16": 0.5498489425981873, |
| "pass@32": 0.5891238670694864, |
| "pass@64": 0.6404833836858006, |
| "pass@128": 0.6918429003021148 |
| }, |
| "20": { |
| "pass@1": 0.3003003003003003, |
| "pass@2": 0.34234234234234234, |
| "pass@4": 0.3933933933933934, |
| "pass@8": 0.43543543543543545, |
| "pass@16": 0.46846846846846846, |
| "pass@32": 0.5105105105105106, |
| "pass@64": 0.5825825825825826, |
| "pass@128": 0.6366366366366366 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6, |
| "12": 0.6391437308868502, |
| "13": 0.6080246913580247, |
| "14": 0.5608308605341247, |
| "15": 0.5092592592592593, |
| "16": 0.5368731563421829, |
| "17": 0.48493975903614456, |
| "18": 0.4645161290322581, |
| "19": 0.3806646525679758, |
| "20": 0.3003003003003003 |
| }, |
| "pass@2": { |
| "11": 0.6666666666666666, |
| "12": 0.7003058103975535, |
| "13": 0.6759259259259259, |
| "14": 0.5875370919881305, |
| "15": 0.5401234567901234, |
| "16": 0.5752212389380531, |
| "17": 0.5030120481927711, |
| "18": 0.5064516129032258, |
| "19": 0.4380664652567976, |
| "20": 0.34234234234234234 |
| }, |
| "pass@4": { |
| "11": 0.7619047619047619, |
| "12": 0.7981651376146789, |
| "13": 0.7654320987654321, |
| "14": 0.6468842729970327, |
| "15": 0.5709876543209876, |
| "16": 0.6194690265486725, |
| "17": 0.5572289156626506, |
| "18": 0.5451612903225806, |
| "19": 0.48338368580060426, |
| "20": 0.3933933933933934 |
| }, |
| "pass@8": { |
| "11": 0.8571428571428571, |
| "12": 0.8807339449541285, |
| "13": 0.8518518518518519, |
| "14": 0.7002967359050445, |
| "15": 0.6450617283950617, |
| "16": 0.6607669616519174, |
| "17": 0.5903614457831325, |
| "18": 0.5870967741935483, |
| "19": 0.5166163141993958, |
| "20": 0.43543543543543545 |
| }, |
| "pass@16": { |
| "11": 0.9174603174603174, |
| "12": 0.9235474006116208, |
| "13": 0.8919753086419753, |
| "14": 0.7626112759643917, |
| "15": 0.7283950617283951, |
| "16": 0.7138643067846607, |
| "17": 0.6475903614457831, |
| "18": 0.632258064516129, |
| "19": 0.5498489425981873, |
| "20": 0.46846846846846846 |
| }, |
| "pass@32": { |
| "11": 0.9587301587301588, |
| "12": 0.9663608562691132, |
| "13": 0.9290123456790124, |
| "14": 0.8486646884272997, |
| "15": 0.7870370370370371, |
| "16": 0.7492625368731564, |
| "17": 0.6897590361445783, |
| "18": 0.6806451612903226, |
| "19": 0.5891238670694864, |
| "20": 0.5105105105105106 |
| }, |
| "pass@64": { |
| "11": 0.9841269841269841, |
| "12": 0.9785932721712538, |
| "13": 0.9567901234567902, |
| "14": 0.9080118694362018, |
| "15": 0.8858024691358025, |
| "16": 0.7876106194690266, |
| "17": 0.7680722891566265, |
| "18": 0.7483870967741936, |
| "19": 0.6404833836858006, |
| "20": 0.5825825825825826 |
| }, |
| "pass@128": { |
| "11": 0.9936507936507937, |
| "12": 0.981651376146789, |
| "13": 0.9629629629629629, |
| "14": 0.9406528189910979, |
| "15": 0.9320987654320988, |
| "16": 0.8171091445427728, |
| "17": 0.8042168674698795, |
| "18": 0.7935483870967742, |
| "19": 0.6918429003021148, |
| "20": 0.6366366366366366 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5079462102689487, |
| "pass@2": 0.5528728606356969, |
| "pass@4": 0.6133863080684596, |
| "pass@8": 0.6714547677261614, |
| "pass@16": 0.7224938875305623, |
| "pass@32": 0.7698655256723717, |
| "pass@64": 0.8230440097799511, |
| "pass@128": 0.8545232273838631 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.677710843373494, |
| "pass@2": 0.7078313253012049, |
| "pass@4": 0.7740963855421686, |
| "pass@8": 0.8644578313253012, |
| "pass@16": 0.927710843373494, |
| "pass@32": 0.963855421686747, |
| "pass@64": 0.9698795180722891, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.5705329153605015, |
| "pass@2": 0.6489028213166145, |
| "pass@4": 0.7460815047021944, |
| "pass@8": 0.8432601880877743, |
| "pass@16": 0.9090909090909091, |
| "pass@32": 0.9623824451410659, |
| "pass@64": 0.9655172413793104, |
| "pass@128": 0.9686520376175548 |
| }, |
| "13": { |
| "pass@1": 0.6330532212885154, |
| "pass@2": 0.680672268907563, |
| "pass@4": 0.7563025210084033, |
| "pass@8": 0.8431372549019608, |
| "pass@16": 0.9103641456582633, |
| "pass@32": 0.9551820728291317, |
| "pass@64": 0.9859943977591037, |
| "pass@128": 0.988795518207283 |
| }, |
| "14": { |
| "pass@1": 0.5584795321637427, |
| "pass@2": 0.6228070175438597, |
| "pass@4": 0.6666666666666666, |
| "pass@8": 0.7251461988304093, |
| "pass@16": 0.7748538011695907, |
| "pass@32": 0.8450292397660819, |
| "pass@64": 0.9122807017543859, |
| "pass@128": 0.9502923976608187 |
| }, |
| "15": { |
| "pass@1": 0.5636363636363636, |
| "pass@2": 0.6060606060606061, |
| "pass@4": 0.6393939393939394, |
| "pass@8": 0.7181818181818181, |
| "pass@16": 0.7787878787878788, |
| "pass@32": 0.8606060606060606, |
| "pass@64": 0.9090909090909091, |
| "pass@128": 0.9424242424242424 |
| }, |
| "16": { |
| "pass@1": 0.5213675213675214, |
| "pass@2": 0.5584045584045584, |
| "pass@4": 0.6125356125356125, |
| "pass@8": 0.6752136752136753, |
| "pass@16": 0.7122507122507122, |
| "pass@32": 0.7435897435897436, |
| "pass@64": 0.8062678062678063, |
| "pass@128": 0.8404558404558404 |
| }, |
| "17": { |
| "pass@1": 0.4440993788819876, |
| "pass@2": 0.48757763975155277, |
| "pass@4": 0.5372670807453416, |
| "pass@8": 0.5807453416149069, |
| "pass@16": 0.6428571428571429, |
| "pass@32": 0.7267080745341615, |
| "pass@64": 0.7919254658385093, |
| "pass@128": 0.8167701863354038 |
| }, |
| "18": { |
| "pass@1": 0.4323529411764706, |
| "pass@2": 0.47058823529411764, |
| "pass@4": 0.4970588235294118, |
| "pass@8": 0.5352941176470588, |
| "pass@16": 0.5794117647058824, |
| "pass@32": 0.6441176470588236, |
| "pass@64": 0.7147058823529412, |
| "pass@128": 0.7794117647058824 |
| }, |
| "19": { |
| "pass@1": 0.41369047619047616, |
| "pass@2": 0.44047619047619047, |
| "pass@4": 0.47619047619047616, |
| "pass@8": 0.5327380952380952, |
| "pass@16": 0.5625, |
| "pass@32": 0.6041666666666666, |
| "pass@64": 0.6577380952380952, |
| "pass@128": 0.6904761904761905 |
| }, |
| "20": { |
| "pass@1": 0.32335329341317365, |
| "pass@2": 0.3712574850299401, |
| "pass@4": 0.40119760479041916, |
| "pass@8": 0.4341317365269461, |
| "pass@16": 0.47604790419161674, |
| "pass@32": 0.5299401197604791, |
| "pass@64": 0.5808383233532934, |
| "pass@128": 0.6377245508982036 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.677710843373494, |
| "12": 0.5705329153605015, |
| "13": 0.6330532212885154, |
| "14": 0.5584795321637427, |
| "15": 0.5636363636363636, |
| "16": 0.5213675213675214, |
| "17": 0.4440993788819876, |
| "18": 0.4323529411764706, |
| "19": 0.41369047619047616, |
| "20": 0.32335329341317365 |
| }, |
| "pass@2": { |
| "11": 0.7078313253012049, |
| "12": 0.6489028213166145, |
| "13": 0.680672268907563, |
| "14": 0.6228070175438597, |
| "15": 0.6060606060606061, |
| "16": 0.5584045584045584, |
| "17": 0.48757763975155277, |
| "18": 0.47058823529411764, |
| "19": 0.44047619047619047, |
| "20": 0.3712574850299401 |
| }, |
| "pass@4": { |
| "11": 0.7740963855421686, |
| "12": 0.7460815047021944, |
| "13": 0.7563025210084033, |
| "14": 0.6666666666666666, |
| "15": 0.6393939393939394, |
| "16": 0.6125356125356125, |
| "17": 0.5372670807453416, |
| "18": 0.4970588235294118, |
| "19": 0.47619047619047616, |
| "20": 0.40119760479041916 |
| }, |
| "pass@8": { |
| "11": 0.8644578313253012, |
| "12": 0.8432601880877743, |
| "13": 0.8431372549019608, |
| "14": 0.7251461988304093, |
| "15": 0.7181818181818181, |
| "16": 0.6752136752136753, |
| "17": 0.5807453416149069, |
| "18": 0.5352941176470588, |
| "19": 0.5327380952380952, |
| "20": 0.4341317365269461 |
| }, |
| "pass@16": { |
| "11": 0.927710843373494, |
| "12": 0.9090909090909091, |
| "13": 0.9103641456582633, |
| "14": 0.7748538011695907, |
| "15": 0.7787878787878788, |
| "16": 0.7122507122507122, |
| "17": 0.6428571428571429, |
| "18": 0.5794117647058824, |
| "19": 0.5625, |
| "20": 0.47604790419161674 |
| }, |
| "pass@32": { |
| "11": 0.963855421686747, |
| "12": 0.9623824451410659, |
| "13": 0.9551820728291317, |
| "14": 0.8450292397660819, |
| "15": 0.8606060606060606, |
| "16": 0.7435897435897436, |
| "17": 0.7267080745341615, |
| "18": 0.6441176470588236, |
| "19": 0.6041666666666666, |
| "20": 0.5299401197604791 |
| }, |
| "pass@64": { |
| "11": 0.9698795180722891, |
| "12": 0.9655172413793104, |
| "13": 0.9859943977591037, |
| "14": 0.9122807017543859, |
| "15": 0.9090909090909091, |
| "16": 0.8062678062678063, |
| "17": 0.7919254658385093, |
| "18": 0.7147058823529412, |
| "19": 0.6577380952380952, |
| "20": 0.5808383233532934 |
| }, |
| "pass@128": { |
| "11": 0.9879518072289156, |
| "12": 0.9686520376175548, |
| "13": 0.988795518207283, |
| "14": 0.9502923976608187, |
| "15": 0.9424242424242424, |
| "16": 0.8404558404558404, |
| "17": 0.8167701863354038, |
| "18": 0.7794117647058824, |
| "19": 0.6904761904761905, |
| "20": 0.6377245508982036 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5144216473386857, |
| "pass@2": 0.5599167410050551, |
| "pass@4": 0.6110615521855486, |
| "pass@8": 0.6755872732679156, |
| "pass@16": 0.727624145108534, |
| "pass@32": 0.7835266131430271, |
| "pass@64": 0.8296164139161463, |
| "pass@128": 0.8605411834671425 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.617, |
| "pass@2": 0.673, |
| "pass@4": 0.753, |
| "pass@8": 0.857, |
| "pass@16": 0.915, |
| "pass@32": 0.957, |
| "pass@64": 0.972, |
| "pass@128": 0.985 |
| }, |
| "12": { |
| "pass@1": 0.622, |
| "pass@2": 0.69, |
| "pass@4": 0.781, |
| "pass@8": 0.867, |
| "pass@16": 0.921, |
| "pass@32": 0.957, |
| "pass@64": 0.971, |
| "pass@128": 0.975 |
| }, |
| "13": { |
| "pass@1": 0.617, |
| "pass@2": 0.669, |
| "pass@4": 0.747, |
| "pass@8": 0.835, |
| "pass@16": 0.893, |
| "pass@32": 0.934, |
| "pass@64": 0.962, |
| "pass@128": 0.973 |
| }, |
| "14": { |
| "pass@1": 0.559, |
| "pass@2": 0.598, |
| "pass@4": 0.643, |
| "pass@8": 0.703, |
| "pass@16": 0.765, |
| "pass@32": 0.838, |
| "pass@64": 0.898, |
| "pass@128": 0.939 |
| }, |
| "15": { |
| "pass@1": 0.533, |
| "pass@2": 0.567, |
| "pass@4": 0.599, |
| "pass@8": 0.673, |
| "pass@16": 0.744, |
| "pass@32": 0.817, |
| "pass@64": 0.893, |
| "pass@128": 0.938 |
| }, |
| "16": { |
| "pass@1": 0.52, |
| "pass@2": 0.554, |
| "pass@4": 0.601, |
| "pass@8": 0.652, |
| "pass@16": 0.691, |
| "pass@32": 0.732, |
| "pass@64": 0.782, |
| "pass@128": 0.818 |
| }, |
| "17": { |
| "pass@1": 0.492, |
| "pass@2": 0.523, |
| "pass@4": 0.568, |
| "pass@8": 0.609, |
| "pass@16": 0.665, |
| "pass@32": 0.722, |
| "pass@64": 0.789, |
| "pass@128": 0.825 |
| }, |
| "18": { |
| "pass@1": 0.44, |
| "pass@2": 0.485, |
| "pass@4": 0.521, |
| "pass@8": 0.553, |
| "pass@16": 0.594, |
| "pass@32": 0.647, |
| "pass@64": 0.724, |
| "pass@128": 0.784 |
| }, |
| "19": { |
| "pass@1": 0.394, |
| "pass@2": 0.433, |
| "pass@4": 0.473, |
| "pass@8": 0.512, |
| "pass@16": 0.55, |
| "pass@32": 0.594, |
| "pass@64": 0.649, |
| "pass@128": 0.691 |
| }, |
| "20": { |
| "pass@1": 0.304, |
| "pass@2": 0.355, |
| "pass@4": 0.398, |
| "pass@8": 0.432, |
| "pass@16": 0.476, |
| "pass@32": 0.53, |
| "pass@64": 0.589, |
| "pass@128": 0.64 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.617, |
| "12": 0.622, |
| "13": 0.617, |
| "14": 0.559, |
| "15": 0.533, |
| "16": 0.52, |
| "17": 0.492, |
| "18": 0.44, |
| "19": 0.394, |
| "20": 0.304 |
| }, |
| "pass@2": { |
| "11": 0.673, |
| "12": 0.69, |
| "13": 0.669, |
| "14": 0.598, |
| "15": 0.567, |
| "16": 0.554, |
| "17": 0.523, |
| "18": 0.485, |
| "19": 0.433, |
| "20": 0.355 |
| }, |
| "pass@4": { |
| "11": 0.753, |
| "12": 0.781, |
| "13": 0.747, |
| "14": 0.643, |
| "15": 0.599, |
| "16": 0.601, |
| "17": 0.568, |
| "18": 0.521, |
| "19": 0.473, |
| "20": 0.398 |
| }, |
| "pass@8": { |
| "11": 0.857, |
| "12": 0.867, |
| "13": 0.835, |
| "14": 0.703, |
| "15": 0.673, |
| "16": 0.652, |
| "17": 0.609, |
| "18": 0.553, |
| "19": 0.512, |
| "20": 0.432 |
| }, |
| "pass@16": { |
| "11": 0.915, |
| "12": 0.921, |
| "13": 0.893, |
| "14": 0.765, |
| "15": 0.744, |
| "16": 0.691, |
| "17": 0.665, |
| "18": 0.594, |
| "19": 0.55, |
| "20": 0.476 |
| }, |
| "pass@32": { |
| "11": 0.957, |
| "12": 0.957, |
| "13": 0.934, |
| "14": 0.838, |
| "15": 0.817, |
| "16": 0.732, |
| "17": 0.722, |
| "18": 0.647, |
| "19": 0.594, |
| "20": 0.53 |
| }, |
| "pass@64": { |
| "11": 0.972, |
| "12": 0.971, |
| "13": 0.962, |
| "14": 0.898, |
| "15": 0.893, |
| "16": 0.782, |
| "17": 0.789, |
| "18": 0.724, |
| "19": 0.649, |
| "20": 0.589 |
| }, |
| "pass@128": { |
| "11": 0.985, |
| "12": 0.975, |
| "13": 0.973, |
| "14": 0.939, |
| "15": 0.938, |
| "16": 0.818, |
| "17": 0.825, |
| "18": 0.784, |
| "19": 0.691, |
| "20": 0.64 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.5098, |
| "pass@2": 0.5547, |
| "pass@4": 0.6084, |
| "pass@8": 0.6693, |
| "pass@16": 0.7214, |
| "pass@32": 0.7728, |
| "pass@64": 0.8229, |
| "pass@128": 0.8568 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
