| { |
| "files": { |
| "checkpoint-38646_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.996875, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9619047619047619, |
| "pass@2": 0.9809523809523809, |
| "pass@4": 0.9968253968253968, |
| "pass@8": 0.9968253968253968, |
| "pass@16": 0.9968253968253968, |
| "pass@32": 0.9968253968253968, |
| "pass@64": 0.9968253968253968, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9580838323353293, |
| "pass@2": 0.9700598802395209, |
| "pass@4": 0.9850299401197605, |
| "pass@8": 0.9910179640718563, |
| "pass@16": 0.9970059880239521, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8875739644970414, |
| "pass@2": 0.9556213017751479, |
| "pass@4": 0.9733727810650887, |
| "pass@8": 0.9970414201183432, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8076923076923077, |
| "pass@2": 0.8782051282051282, |
| "pass@4": 0.9358974358974359, |
| "pass@8": 0.9775641025641025, |
| "pass@16": 0.9871794871794872, |
| "pass@32": 0.9967948717948718, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7098591549295775, |
| "pass@2": 0.7887323943661971, |
| "pass@4": 0.8901408450704226, |
| "pass@8": 0.9323943661971831, |
| "pass@16": 0.9774647887323944, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 0.9971830985915493 |
| }, |
| "10": { |
| "pass@1": 0.6430678466076696, |
| "pass@2": 0.696165191740413, |
| "pass@4": 0.7905604719764012, |
| "pass@8": 0.8790560471976401, |
| "pass@16": 0.9351032448377581, |
| "pass@32": 0.9528023598820059, |
| "pass@64": 0.9587020648967551, |
| "pass@128": 0.9646017699115044 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.996875, |
| "5": 0.9619047619047619, |
| "6": 0.9580838323353293, |
| "7": 0.8875739644970414, |
| "8": 0.8076923076923077, |
| "9": 0.7098591549295775, |
| "10": 0.6430678466076696 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9809523809523809, |
| "6": 0.9700598802395209, |
| "7": 0.9556213017751479, |
| "8": 0.8782051282051282, |
| "9": 0.7887323943661971, |
| "10": 0.696165191740413 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 0.9850299401197605, |
| "7": 0.9733727810650887, |
| "8": 0.9358974358974359, |
| "9": 0.8901408450704226, |
| "10": 0.7905604719764012 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 0.9910179640718563, |
| "7": 0.9970414201183432, |
| "8": 0.9775641025641025, |
| "9": 0.9323943661971831, |
| "10": 0.8790560471976401 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 0.9970059880239521, |
| "7": 1.0, |
| "8": 0.9871794871794872, |
| "9": 0.9774647887323944, |
| "10": 0.9351032448377581 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9967948717948718, |
| "9": 0.9971830985915493, |
| "10": 0.9528023598820059 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9587020648967551 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9968253968253968, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9971830985915493, |
| "10": 0.9646017699115044 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8821344140493077, |
| "pass@2": 0.9165822357311719, |
| "pass@4": 0.9510300574130361, |
| "pass@8": 0.9739952718676123, |
| "pass@16": 0.9878419452887538, |
| "pass@32": 0.9935832489023978, |
| "pass@64": 0.994596420128335, |
| "pass@128": 0.9952718676122931 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9912280701754386, |
| "pass@2": 0.9912280701754386, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9470588235294117, |
| "pass@2": 0.9823529411764705, |
| "pass@4": 0.9970588235294118, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9716088328075709, |
| "pass@2": 0.9779179810725552, |
| "pass@4": 0.9905362776025236, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.879746835443038, |
| "pass@2": 0.930379746835443, |
| "pass@4": 0.9683544303797469, |
| "pass@8": 0.9841772151898734, |
| "pass@16": 0.9936708860759493, |
| "pass@32": 0.9968354430379747, |
| "pass@64": 0.9968354430379747, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8018018018018018, |
| "pass@2": 0.8498498498498499, |
| "pass@4": 0.918918918918919, |
| "pass@8": 0.9669669669669669, |
| "pass@16": 0.990990990990991, |
| "pass@32": 0.996996996996997, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7620578778135049, |
| "pass@2": 0.8231511254019293, |
| "pass@4": 0.9067524115755627, |
| "pass@8": 0.9678456591639871, |
| "pass@16": 0.9807073954983923, |
| "pass@32": 0.9967845659163987, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6, |
| "pass@2": 0.6597014925373135, |
| "pass@4": 0.7492537313432835, |
| "pass@8": 0.8597014925373134, |
| "pass@16": 0.9194029850746268, |
| "pass@32": 0.9432835820895522, |
| "pass@64": 0.9641791044776119, |
| "pass@128": 0.9731343283582089 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9912280701754386, |
| "5": 0.9470588235294117, |
| "6": 0.9716088328075709, |
| "7": 0.879746835443038, |
| "8": 0.8018018018018018, |
| "9": 0.7620578778135049, |
| "10": 0.6 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9912280701754386, |
| "5": 0.9823529411764705, |
| "6": 0.9779179810725552, |
| "7": 0.930379746835443, |
| "8": 0.8498498498498499, |
| "9": 0.8231511254019293, |
| "10": 0.6597014925373135 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 0.9905362776025236, |
| "7": 0.9683544303797469, |
| "8": 0.918918918918919, |
| "9": 0.9067524115755627, |
| "10": 0.7492537313432835 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9841772151898734, |
| "8": 0.9669669669669669, |
| "9": 0.9678456591639871, |
| "10": 0.8597014925373134 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9936708860759493, |
| "8": 0.990990990990991, |
| "9": 0.9807073954983923, |
| "10": 0.9194029850746268 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 0.996996996996997, |
| "9": 0.9967845659163987, |
| "10": 0.9432835820895522 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9641791044776119 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9731343283582089 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8855038500167393, |
| "pass@2": 0.9139604954804151, |
| "pass@4": 0.9484432541011047, |
| "pass@8": 0.9755607633076665, |
| "pass@16": 0.9872782055574155, |
| "pass@32": 0.9926347505858721, |
| "pass@64": 0.995647807164379, |
| "pass@128": 0.9969869434214932 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9792899408284024, |
| "pass@2": 0.9881656804733728, |
| "pass@4": 0.9940828402366864, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9681159420289855, |
| "pass@2": 0.9884057971014493, |
| "pass@4": 0.991304347826087, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9742120343839542, |
| "pass@2": 0.9799426934097422, |
| "pass@4": 0.9885386819484241, |
| "pass@8": 0.9914040114613181, |
| "pass@16": 0.997134670487106, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8583815028901735, |
| "pass@2": 0.9132947976878613, |
| "pass@4": 0.9508670520231214, |
| "pass@8": 0.9884393063583815, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.7971830985915493, |
| "pass@2": 0.8732394366197183, |
| "pass@4": 0.9577464788732394, |
| "pass@8": 0.9887323943661972, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7425149700598802, |
| "pass@2": 0.8053892215568862, |
| "pass@4": 0.8922155688622755, |
| "pass@8": 0.9730538922155688, |
| "pass@16": 0.9970059880239521, |
| "pass@32": 0.9970059880239521, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.656441717791411, |
| "pass@2": 0.7208588957055214, |
| "pass@4": 0.8006134969325154, |
| "pass@8": 0.8650306748466258, |
| "pass@16": 0.9079754601226994, |
| "pass@32": 0.9447852760736196, |
| "pass@64": 0.9478527607361963, |
| "pass@128": 0.950920245398773 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9792899408284024, |
| "5": 0.9681159420289855, |
| "6": 0.9742120343839542, |
| "7": 0.8583815028901735, |
| "8": 0.7971830985915493, |
| "9": 0.7425149700598802, |
| "10": 0.656441717791411 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9881656804733728, |
| "5": 0.9884057971014493, |
| "6": 0.9799426934097422, |
| "7": 0.9132947976878613, |
| "8": 0.8732394366197183, |
| "9": 0.8053892215568862, |
| "10": 0.7208588957055214 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9940828402366864, |
| "5": 0.991304347826087, |
| "6": 0.9885386819484241, |
| "7": 0.9508670520231214, |
| "8": 0.9577464788732394, |
| "9": 0.8922155688622755, |
| "10": 0.8006134969325154 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 0.9914040114613181, |
| "7": 0.9884393063583815, |
| "8": 0.9887323943661972, |
| "9": 0.9730538922155688, |
| "10": 0.8650306748466258 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 0.997134670487106, |
| "7": 1.0, |
| "8": 0.9971830985915493, |
| "9": 0.9970059880239521, |
| "10": 0.9079754601226994 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.9447852760736196 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9478527607361963 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.950920245398773 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8866317169069463, |
| "pass@2": 0.919397116644823, |
| "pass@4": 0.953473132372215, |
| "pass@8": 0.97870249017038, |
| "pass@16": 0.9888597640891219, |
| "pass@32": 0.9934469200524246, |
| "pass@64": 0.994429882044561, |
| "pass@128": 0.9947575360419397 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.989, |
| "pass@2": 0.993, |
| "pass@4": 0.998, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.959, |
| "pass@2": 0.984, |
| "pass@4": 0.995, |
| "pass@8": 0.998, |
| "pass@16": 0.998, |
| "pass@32": 0.998, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "6": { |
| "pass@1": 0.968, |
| "pass@2": 0.976, |
| "pass@4": 0.988, |
| "pass@8": 0.994, |
| "pass@16": 0.998, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.875, |
| "pass@2": 0.933, |
| "pass@4": 0.964, |
| "pass@8": 0.99, |
| "pass@16": 0.998, |
| "pass@32": 0.999, |
| "pass@64": 0.999, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.802, |
| "pass@2": 0.867, |
| "pass@4": 0.938, |
| "pass@8": 0.978, |
| "pass@16": 0.992, |
| "pass@32": 0.998, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.737, |
| "pass@2": 0.805, |
| "pass@4": 0.896, |
| "pass@8": 0.957, |
| "pass@16": 0.985, |
| "pass@32": 0.997, |
| "pass@64": 0.999, |
| "pass@128": 0.999 |
| }, |
| "10": { |
| "pass@1": 0.633, |
| "pass@2": 0.692, |
| "pass@4": 0.78, |
| "pass@8": 0.868, |
| "pass@16": 0.921, |
| "pass@32": 0.947, |
| "pass@64": 0.957, |
| "pass@128": 0.963 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.989, |
| "5": 0.959, |
| "6": 0.968, |
| "7": 0.875, |
| "8": 0.802, |
| "9": 0.737, |
| "10": 0.633 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.993, |
| "5": 0.984, |
| "6": 0.976, |
| "7": 0.933, |
| "8": 0.867, |
| "9": 0.805, |
| "10": 0.692 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.998, |
| "5": 0.995, |
| "6": 0.988, |
| "7": 0.964, |
| "8": 0.938, |
| "9": 0.896, |
| "10": 0.78 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 0.994, |
| "7": 0.99, |
| "8": 0.978, |
| "9": 0.957, |
| "10": 0.868 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 0.998, |
| "7": 0.998, |
| "8": 0.992, |
| "9": 0.985, |
| "10": 0.921 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.998, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 0.998, |
| "9": 0.997, |
| "10": 0.947 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 0.999, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.957 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.999, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.999, |
| "10": 0.963 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8847777777777778, |
| "pass@2": 0.9166666666666666, |
| "pass@4": 0.951, |
| "pass@8": 0.9761111111111112, |
| "pass@16": 0.988, |
| "pass@32": 0.9932222222222222, |
| "pass@64": 0.9948888888888889, |
| "pass@128": 0.9956666666666667 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-38646_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6005665722379604, |
| "pass@2": 0.6742209631728046, |
| "pass@4": 0.773371104815864, |
| "pass@8": 0.8640226628895185, |
| "pass@16": 0.9178470254957507, |
| "pass@32": 0.9518413597733711, |
| "pass@64": 0.9745042492917847, |
| "pass@128": 0.9830028328611898 |
| }, |
| "12": { |
| "pass@1": 0.6638418079096046, |
| "pass@2": 0.7231638418079096, |
| "pass@4": 0.8022598870056498, |
| "pass@8": 0.8813559322033898, |
| "pass@16": 0.9293785310734464, |
| "pass@32": 0.9491525423728814, |
| "pass@64": 0.963276836158192, |
| "pass@128": 0.9745762711864406 |
| }, |
| "13": { |
| "pass@1": 0.5893416927899686, |
| "pass@2": 0.6394984326018809, |
| "pass@4": 0.7272727272727273, |
| "pass@8": 0.8056426332288401, |
| "pass@16": 0.8714733542319749, |
| "pass@32": 0.9122257053291536, |
| "pass@64": 0.9467084639498433, |
| "pass@128": 0.9717868338557993 |
| }, |
| "14": { |
| "pass@1": 0.5607476635514018, |
| "pass@2": 0.5919003115264797, |
| "pass@4": 0.6417445482866043, |
| "pass@8": 0.7227414330218068, |
| "pass@16": 0.7757009345794392, |
| "pass@32": 0.8286604361370716, |
| "pass@64": 0.8847352024922118, |
| "pass@128": 0.9283489096573209 |
| }, |
| "15": { |
| "pass@1": 0.5057803468208093, |
| "pass@2": 0.5549132947976878, |
| "pass@4": 0.5953757225433526, |
| "pass@8": 0.6647398843930635, |
| "pass@16": 0.7456647398843931, |
| "pass@32": 0.8179190751445087, |
| "pass@64": 0.9104046242774566, |
| "pass@128": 0.9450867052023122 |
| }, |
| "16": { |
| "pass@1": 0.4935483870967742, |
| "pass@2": 0.5193548387096775, |
| "pass@4": 0.5548387096774193, |
| "pass@8": 0.5870967741935483, |
| "pass@16": 0.6548387096774193, |
| "pass@32": 0.7032258064516129, |
| "pass@64": 0.7354838709677419, |
| "pass@128": 0.7709677419354839 |
| }, |
| "17": { |
| "pass@1": 0.5289017341040463, |
| "pass@2": 0.546242774566474, |
| "pass@4": 0.5809248554913294, |
| "pass@8": 0.6184971098265896, |
| "pass@16": 0.6705202312138728, |
| "pass@32": 0.7167630057803468, |
| "pass@64": 0.7803468208092486, |
| "pass@128": 0.8294797687861272 |
| }, |
| "18": { |
| "pass@1": 0.4, |
| "pass@2": 0.4142857142857143, |
| "pass@4": 0.4514285714285714, |
| "pass@8": 0.49714285714285716, |
| "pass@16": 0.5457142857142857, |
| "pass@32": 0.6057142857142858, |
| "pass@64": 0.6714285714285714, |
| "pass@128": 0.7314285714285714 |
| }, |
| "19": { |
| "pass@1": 0.3063063063063063, |
| "pass@2": 0.33633633633633636, |
| "pass@4": 0.39039039039039036, |
| "pass@8": 0.4174174174174174, |
| "pass@16": 0.45645645645645644, |
| "pass@32": 0.5015015015015015, |
| "pass@64": 0.5555555555555556, |
| "pass@128": 0.5825825825825826 |
| }, |
| "20": { |
| "pass@1": 0.21621621621621623, |
| "pass@2": 0.24924924924924924, |
| "pass@4": 0.2822822822822823, |
| "pass@8": 0.3153153153153153, |
| "pass@16": 0.34234234234234234, |
| "pass@32": 0.39039039039039036, |
| "pass@64": 0.43843843843843844, |
| "pass@128": 0.4924924924924925 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6005665722379604, |
| "12": 0.6638418079096046, |
| "13": 0.5893416927899686, |
| "14": 0.5607476635514018, |
| "15": 0.5057803468208093, |
| "16": 0.4935483870967742, |
| "17": 0.5289017341040463, |
| "18": 0.4, |
| "19": 0.3063063063063063, |
| "20": 0.21621621621621623 |
| }, |
| "pass@2": { |
| "11": 0.6742209631728046, |
| "12": 0.7231638418079096, |
| "13": 0.6394984326018809, |
| "14": 0.5919003115264797, |
| "15": 0.5549132947976878, |
| "16": 0.5193548387096775, |
| "17": 0.546242774566474, |
| "18": 0.4142857142857143, |
| "19": 0.33633633633633636, |
| "20": 0.24924924924924924 |
| }, |
| "pass@4": { |
| "11": 0.773371104815864, |
| "12": 0.8022598870056498, |
| "13": 0.7272727272727273, |
| "14": 0.6417445482866043, |
| "15": 0.5953757225433526, |
| "16": 0.5548387096774193, |
| "17": 0.5809248554913294, |
| "18": 0.4514285714285714, |
| "19": 0.39039039039039036, |
| "20": 0.2822822822822823 |
| }, |
| "pass@8": { |
| "11": 0.8640226628895185, |
| "12": 0.8813559322033898, |
| "13": 0.8056426332288401, |
| "14": 0.7227414330218068, |
| "15": 0.6647398843930635, |
| "16": 0.5870967741935483, |
| "17": 0.6184971098265896, |
| "18": 0.49714285714285716, |
| "19": 0.4174174174174174, |
| "20": 0.3153153153153153 |
| }, |
| "pass@16": { |
| "11": 0.9178470254957507, |
| "12": 0.9293785310734464, |
| "13": 0.8714733542319749, |
| "14": 0.7757009345794392, |
| "15": 0.7456647398843931, |
| "16": 0.6548387096774193, |
| "17": 0.6705202312138728, |
| "18": 0.5457142857142857, |
| "19": 0.45645645645645644, |
| "20": 0.34234234234234234 |
| }, |
| "pass@32": { |
| "11": 0.9518413597733711, |
| "12": 0.9491525423728814, |
| "13": 0.9122257053291536, |
| "14": 0.8286604361370716, |
| "15": 0.8179190751445087, |
| "16": 0.7032258064516129, |
| "17": 0.7167630057803468, |
| "18": 0.6057142857142858, |
| "19": 0.5015015015015015, |
| "20": 0.39039039039039036 |
| }, |
| "pass@64": { |
| "11": 0.9745042492917847, |
| "12": 0.963276836158192, |
| "13": 0.9467084639498433, |
| "14": 0.8847352024922118, |
| "15": 0.9104046242774566, |
| "16": 0.7354838709677419, |
| "17": 0.7803468208092486, |
| "18": 0.6714285714285714, |
| "19": 0.5555555555555556, |
| "20": 0.43843843843843844 |
| }, |
| "pass@128": { |
| "11": 0.9830028328611898, |
| "12": 0.9745762711864406, |
| "13": 0.9717868338557993, |
| "14": 0.9283489096573209, |
| "15": 0.9450867052023122, |
| "16": 0.7709677419354839, |
| "17": 0.8294797687861272, |
| "18": 0.7314285714285714, |
| "19": 0.5825825825825826, |
| "20": 0.4924924924924925 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.487369985141159, |
| "pass@2": 0.5260029717682021, |
| "pass@4": 0.5812778603268945, |
| "pass@8": 0.6389301634472511, |
| "pass@16": 0.6924219910846954, |
| "pass@32": 0.7390787518573552, |
| "pass@64": 0.787518573551263, |
| "pass@128": 0.8222882615156017 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6095238095238096, |
| "pass@2": 0.6952380952380952, |
| "pass@4": 0.8095238095238095, |
| "pass@8": 0.873015873015873, |
| "pass@16": 0.9333333333333333, |
| "pass@32": 0.9619047619047619, |
| "pass@64": 0.9841269841269841, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.617737003058104, |
| "pass@2": 0.6788990825688074, |
| "pass@4": 0.7675840978593272, |
| "pass@8": 0.8837920489296636, |
| "pass@16": 0.9327217125382263, |
| "pass@32": 0.9724770642201835, |
| "pass@64": 0.981651376146789, |
| "pass@128": 0.9938837920489296 |
| }, |
| "13": { |
| "pass@1": 0.5864197530864198, |
| "pass@2": 0.6851851851851852, |
| "pass@4": 0.7623456790123457, |
| "pass@8": 0.8302469135802469, |
| "pass@16": 0.8796296296296297, |
| "pass@32": 0.9197530864197531, |
| "pass@64": 0.9506172839506173, |
| "pass@128": 0.9629629629629629 |
| }, |
| "14": { |
| "pass@1": 0.56973293768546, |
| "pass@2": 0.599406528189911, |
| "pass@4": 0.6528189910979229, |
| "pass@8": 0.7151335311572701, |
| "pass@16": 0.7833827893175074, |
| "pass@32": 0.8486646884272997, |
| "pass@64": 0.9287833827893175, |
| "pass@128": 0.9584569732937686 |
| }, |
| "15": { |
| "pass@1": 0.5, |
| "pass@2": 0.5308641975308642, |
| "pass@4": 0.5802469135802469, |
| "pass@8": 0.6481481481481481, |
| "pass@16": 0.7314814814814815, |
| "pass@32": 0.7962962962962963, |
| "pass@64": 0.8765432098765432, |
| "pass@128": 0.9320987654320988 |
| }, |
| "16": { |
| "pass@1": 0.5398230088495575, |
| "pass@2": 0.5781710914454278, |
| "pass@4": 0.6106194690265486, |
| "pass@8": 0.6460176991150443, |
| "pass@16": 0.7050147492625368, |
| "pass@32": 0.7463126843657817, |
| "pass@64": 0.7935103244837758, |
| "pass@128": 0.8230088495575221 |
| }, |
| "17": { |
| "pass@1": 0.4759036144578313, |
| "pass@2": 0.5090361445783133, |
| "pass@4": 0.536144578313253, |
| "pass@8": 0.5662650602409639, |
| "pass@16": 0.6234939759036144, |
| "pass@32": 0.7018072289156626, |
| "pass@64": 0.7680722891566265, |
| "pass@128": 0.7891566265060241 |
| }, |
| "18": { |
| "pass@1": 0.4290322580645161, |
| "pass@2": 0.45483870967741935, |
| "pass@4": 0.4870967741935484, |
| "pass@8": 0.535483870967742, |
| "pass@16": 0.5709677419354838, |
| "pass@32": 0.6387096774193548, |
| "pass@64": 0.7258064516129032, |
| "pass@128": 0.7741935483870968 |
| }, |
| "19": { |
| "pass@1": 0.2990936555891239, |
| "pass@2": 0.3564954682779456, |
| "pass@4": 0.3867069486404834, |
| "pass@8": 0.4199395770392749, |
| "pass@16": 0.459214501510574, |
| "pass@32": 0.4743202416918429, |
| "pass@64": 0.525679758308157, |
| "pass@128": 0.5770392749244713 |
| }, |
| "20": { |
| "pass@1": 0.1921921921921922, |
| "pass@2": 0.24324324324324326, |
| "pass@4": 0.2822822822822823, |
| "pass@8": 0.30930930930930933, |
| "pass@16": 0.3333333333333333, |
| "pass@32": 0.36036036036036034, |
| "pass@64": 0.4144144144144144, |
| "pass@128": 0.46846846846846846 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6095238095238096, |
| "12": 0.617737003058104, |
| "13": 0.5864197530864198, |
| "14": 0.56973293768546, |
| "15": 0.5, |
| "16": 0.5398230088495575, |
| "17": 0.4759036144578313, |
| "18": 0.4290322580645161, |
| "19": 0.2990936555891239, |
| "20": 0.1921921921921922 |
| }, |
| "pass@2": { |
| "11": 0.6952380952380952, |
| "12": 0.6788990825688074, |
| "13": 0.6851851851851852, |
| "14": 0.599406528189911, |
| "15": 0.5308641975308642, |
| "16": 0.5781710914454278, |
| "17": 0.5090361445783133, |
| "18": 0.45483870967741935, |
| "19": 0.3564954682779456, |
| "20": 0.24324324324324326 |
| }, |
| "pass@4": { |
| "11": 0.8095238095238095, |
| "12": 0.7675840978593272, |
| "13": 0.7623456790123457, |
| "14": 0.6528189910979229, |
| "15": 0.5802469135802469, |
| "16": 0.6106194690265486, |
| "17": 0.536144578313253, |
| "18": 0.4870967741935484, |
| "19": 0.3867069486404834, |
| "20": 0.2822822822822823 |
| }, |
| "pass@8": { |
| "11": 0.873015873015873, |
| "12": 0.8837920489296636, |
| "13": 0.8302469135802469, |
| "14": 0.7151335311572701, |
| "15": 0.6481481481481481, |
| "16": 0.6460176991150443, |
| "17": 0.5662650602409639, |
| "18": 0.535483870967742, |
| "19": 0.4199395770392749, |
| "20": 0.30930930930930933 |
| }, |
| "pass@16": { |
| "11": 0.9333333333333333, |
| "12": 0.9327217125382263, |
| "13": 0.8796296296296297, |
| "14": 0.7833827893175074, |
| "15": 0.7314814814814815, |
| "16": 0.7050147492625368, |
| "17": 0.6234939759036144, |
| "18": 0.5709677419354838, |
| "19": 0.459214501510574, |
| "20": 0.3333333333333333 |
| }, |
| "pass@32": { |
| "11": 0.9619047619047619, |
| "12": 0.9724770642201835, |
| "13": 0.9197530864197531, |
| "14": 0.8486646884272997, |
| "15": 0.7962962962962963, |
| "16": 0.7463126843657817, |
| "17": 0.7018072289156626, |
| "18": 0.6387096774193548, |
| "19": 0.4743202416918429, |
| "20": 0.36036036036036034 |
| }, |
| "pass@64": { |
| "11": 0.9841269841269841, |
| "12": 0.981651376146789, |
| "13": 0.9506172839506173, |
| "14": 0.9287833827893175, |
| "15": 0.8765432098765432, |
| "16": 0.7935103244837758, |
| "17": 0.7680722891566265, |
| "18": 0.7258064516129032, |
| "19": 0.525679758308157, |
| "20": 0.4144144144144144 |
| }, |
| "pass@128": { |
| "11": 0.9904761904761905, |
| "12": 0.9938837920489296, |
| "13": 0.9629629629629629, |
| "14": 0.9584569732937686, |
| "15": 0.9320987654320988, |
| "16": 0.8230088495575221, |
| "17": 0.7891566265060241, |
| "18": 0.7741935483870968, |
| "19": 0.5770392749244713, |
| "20": 0.46846846846846846 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.48135696821515894, |
| "pass@2": 0.53239608801956, |
| "pass@4": 0.5864914425427873, |
| "pass@8": 0.6415036674816625, |
| "pass@16": 0.6940709046454768, |
| "pass@32": 0.7408312958435208, |
| "pass@64": 0.7937041564792175, |
| "pass@128": 0.8257946210268948 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6686746987951807, |
| "pass@2": 0.7228915662650602, |
| "pass@4": 0.8042168674698795, |
| "pass@8": 0.8825301204819277, |
| "pass@16": 0.9216867469879518, |
| "pass@32": 0.9578313253012049, |
| "pass@64": 0.9819277108433735, |
| "pass@128": 0.9879518072289156 |
| }, |
| "12": { |
| "pass@1": 0.5987460815047022, |
| "pass@2": 0.658307210031348, |
| "pass@4": 0.7711598746081505, |
| "pass@8": 0.8369905956112853, |
| "pass@16": 0.9122257053291536, |
| "pass@32": 0.9467084639498433, |
| "pass@64": 0.9717868338557993, |
| "pass@128": 0.9811912225705329 |
| }, |
| "13": { |
| "pass@1": 0.6162464985994398, |
| "pass@2": 0.6750700280112045, |
| "pass@4": 0.7675070028011205, |
| "pass@8": 0.8487394957983193, |
| "pass@16": 0.9131652661064426, |
| "pass@32": 0.938375350140056, |
| "pass@64": 0.9635854341736695, |
| "pass@128": 0.9831932773109243 |
| }, |
| "14": { |
| "pass@1": 0.5672514619883041, |
| "pass@2": 0.6023391812865497, |
| "pass@4": 0.6374269005847953, |
| "pass@8": 0.716374269005848, |
| "pass@16": 0.7923976608187134, |
| "pass@32": 0.8713450292397661, |
| "pass@64": 0.9385964912280702, |
| "pass@128": 0.9590643274853801 |
| }, |
| "15": { |
| "pass@1": 0.5545454545454546, |
| "pass@2": 0.5848484848484848, |
| "pass@4": 0.6242424242424243, |
| "pass@8": 0.6818181818181818, |
| "pass@16": 0.7545454545454545, |
| "pass@32": 0.8121212121212121, |
| "pass@64": 0.8727272727272727, |
| "pass@128": 0.9272727272727272 |
| }, |
| "16": { |
| "pass@1": 0.5156695156695157, |
| "pass@2": 0.5384615384615384, |
| "pass@4": 0.5726495726495726, |
| "pass@8": 0.6296296296296297, |
| "pass@16": 0.6837606837606838, |
| "pass@32": 0.7492877492877493, |
| "pass@64": 0.7977207977207977, |
| "pass@128": 0.8319088319088319 |
| }, |
| "17": { |
| "pass@1": 0.43478260869565216, |
| "pass@2": 0.45652173913043476, |
| "pass@4": 0.5186335403726708, |
| "pass@8": 0.5403726708074534, |
| "pass@16": 0.6180124223602484, |
| "pass@32": 0.6770186335403726, |
| "pass@64": 0.7298136645962733, |
| "pass@128": 0.7888198757763976 |
| }, |
| "18": { |
| "pass@1": 0.39705882352941174, |
| "pass@2": 0.4323529411764706, |
| "pass@4": 0.45294117647058824, |
| "pass@8": 0.5, |
| "pass@16": 0.5323529411764706, |
| "pass@32": 0.5823529411764706, |
| "pass@64": 0.6529411764705882, |
| "pass@128": 0.7205882352941176 |
| }, |
| "19": { |
| "pass@1": 0.2976190476190476, |
| "pass@2": 0.3601190476190476, |
| "pass@4": 0.39880952380952384, |
| "pass@8": 0.4375, |
| "pass@16": 0.4732142857142857, |
| "pass@32": 0.5148809523809523, |
| "pass@64": 0.5625, |
| "pass@128": 0.6101190476190477 |
| }, |
| "20": { |
| "pass@1": 0.19760479041916168, |
| "pass@2": 0.24251497005988024, |
| "pass@4": 0.2934131736526946, |
| "pass@8": 0.32934131736526945, |
| "pass@16": 0.3532934131736527, |
| "pass@32": 0.3772455089820359, |
| "pass@64": 0.4251497005988024, |
| "pass@128": 0.46407185628742514 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6686746987951807, |
| "12": 0.5987460815047022, |
| "13": 0.6162464985994398, |
| "14": 0.5672514619883041, |
| "15": 0.5545454545454546, |
| "16": 0.5156695156695157, |
| "17": 0.43478260869565216, |
| "18": 0.39705882352941174, |
| "19": 0.2976190476190476, |
| "20": 0.19760479041916168 |
| }, |
| "pass@2": { |
| "11": 0.7228915662650602, |
| "12": 0.658307210031348, |
| "13": 0.6750700280112045, |
| "14": 0.6023391812865497, |
| "15": 0.5848484848484848, |
| "16": 0.5384615384615384, |
| "17": 0.45652173913043476, |
| "18": 0.4323529411764706, |
| "19": 0.3601190476190476, |
| "20": 0.24251497005988024 |
| }, |
| "pass@4": { |
| "11": 0.8042168674698795, |
| "12": 0.7711598746081505, |
| "13": 0.7675070028011205, |
| "14": 0.6374269005847953, |
| "15": 0.6242424242424243, |
| "16": 0.5726495726495726, |
| "17": 0.5186335403726708, |
| "18": 0.45294117647058824, |
| "19": 0.39880952380952384, |
| "20": 0.2934131736526946 |
| }, |
| "pass@8": { |
| "11": 0.8825301204819277, |
| "12": 0.8369905956112853, |
| "13": 0.8487394957983193, |
| "14": 0.716374269005848, |
| "15": 0.6818181818181818, |
| "16": 0.6296296296296297, |
| "17": 0.5403726708074534, |
| "18": 0.5, |
| "19": 0.4375, |
| "20": 0.32934131736526945 |
| }, |
| "pass@16": { |
| "11": 0.9216867469879518, |
| "12": 0.9122257053291536, |
| "13": 0.9131652661064426, |
| "14": 0.7923976608187134, |
| "15": 0.7545454545454545, |
| "16": 0.6837606837606838, |
| "17": 0.6180124223602484, |
| "18": 0.5323529411764706, |
| "19": 0.4732142857142857, |
| "20": 0.3532934131736527 |
| }, |
| "pass@32": { |
| "11": 0.9578313253012049, |
| "12": 0.9467084639498433, |
| "13": 0.938375350140056, |
| "14": 0.8713450292397661, |
| "15": 0.8121212121212121, |
| "16": 0.7492877492877493, |
| "17": 0.6770186335403726, |
| "18": 0.5823529411764706, |
| "19": 0.5148809523809523, |
| "20": 0.3772455089820359 |
| }, |
| "pass@64": { |
| "11": 0.9819277108433735, |
| "12": 0.9717868338557993, |
| "13": 0.9635854341736695, |
| "14": 0.9385964912280702, |
| "15": 0.8727272727272727, |
| "16": 0.7977207977207977, |
| "17": 0.7298136645962733, |
| "18": 0.6529411764705882, |
| "19": 0.5625, |
| "20": 0.4251497005988024 |
| }, |
| "pass@128": { |
| "11": 0.9879518072289156, |
| "12": 0.9811912225705329, |
| "13": 0.9831932773109243, |
| "14": 0.9590643274853801, |
| "15": 0.9272727272727272, |
| "16": 0.8319088319088319, |
| "17": 0.7888198757763976, |
| "18": 0.7205882352941176, |
| "19": 0.6101190476190477, |
| "20": 0.46407185628742514 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4852809991079393, |
| "pass@2": 0.527802557240559, |
| "pass@4": 0.584299732381802, |
| "pass@8": 0.6407969075230449, |
| "pass@16": 0.695807314897413, |
| "pass@32": 0.7430865298840321, |
| "pass@64": 0.7900683913172762, |
| "pass@128": 0.8257508177222718 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.626, |
| "pass@2": 0.697, |
| "pass@4": 0.795, |
| "pass@8": 0.873, |
| "pass@16": 0.924, |
| "pass@32": 0.957, |
| "pass@64": 0.98, |
| "pass@128": 0.987 |
| }, |
| "12": { |
| "pass@1": 0.628, |
| "pass@2": 0.688, |
| "pass@4": 0.781, |
| "pass@8": 0.868, |
| "pass@16": 0.925, |
| "pass@32": 0.956, |
| "pass@64": 0.972, |
| "pass@128": 0.983 |
| }, |
| "13": { |
| "pass@1": 0.598, |
| "pass@2": 0.667, |
| "pass@4": 0.753, |
| "pass@8": 0.829, |
| "pass@16": 0.889, |
| "pass@32": 0.924, |
| "pass@64": 0.954, |
| "pass@128": 0.973 |
| }, |
| "14": { |
| "pass@1": 0.566, |
| "pass@2": 0.598, |
| "pass@4": 0.644, |
| "pass@8": 0.718, |
| "pass@16": 0.784, |
| "pass@32": 0.85, |
| "pass@64": 0.918, |
| "pass@128": 0.949 |
| }, |
| "15": { |
| "pass@1": 0.52, |
| "pass@2": 0.557, |
| "pass@4": 0.6, |
| "pass@8": 0.665, |
| "pass@16": 0.744, |
| "pass@32": 0.809, |
| "pass@64": 0.887, |
| "pass@128": 0.935 |
| }, |
| "16": { |
| "pass@1": 0.517, |
| "pass@2": 0.546, |
| "pass@4": 0.58, |
| "pass@8": 0.622, |
| "pass@16": 0.682, |
| "pass@32": 0.734, |
| "pass@64": 0.777, |
| "pass@128": 0.81 |
| }, |
| "17": { |
| "pass@1": 0.481, |
| "pass@2": 0.505, |
| "pass@4": 0.546, |
| "pass@8": 0.576, |
| "pass@16": 0.638, |
| "pass@32": 0.699, |
| "pass@64": 0.76, |
| "pass@128": 0.803 |
| }, |
| "18": { |
| "pass@1": 0.408, |
| "pass@2": 0.433, |
| "pass@4": 0.463, |
| "pass@8": 0.51, |
| "pass@16": 0.549, |
| "pass@32": 0.608, |
| "pass@64": 0.682, |
| "pass@128": 0.741 |
| }, |
| "19": { |
| "pass@1": 0.301, |
| "pass@2": 0.351, |
| "pass@4": 0.392, |
| "pass@8": 0.425, |
| "pass@16": 0.463, |
| "pass@32": 0.497, |
| "pass@64": 0.548, |
| "pass@128": 0.59 |
| }, |
| "20": { |
| "pass@1": 0.202, |
| "pass@2": 0.245, |
| "pass@4": 0.286, |
| "pass@8": 0.318, |
| "pass@16": 0.343, |
| "pass@32": 0.376, |
| "pass@64": 0.426, |
| "pass@128": 0.475 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.626, |
| "12": 0.628, |
| "13": 0.598, |
| "14": 0.566, |
| "15": 0.52, |
| "16": 0.517, |
| "17": 0.481, |
| "18": 0.408, |
| "19": 0.301, |
| "20": 0.202 |
| }, |
| "pass@2": { |
| "11": 0.697, |
| "12": 0.688, |
| "13": 0.667, |
| "14": 0.598, |
| "15": 0.557, |
| "16": 0.546, |
| "17": 0.505, |
| "18": 0.433, |
| "19": 0.351, |
| "20": 0.245 |
| }, |
| "pass@4": { |
| "11": 0.795, |
| "12": 0.781, |
| "13": 0.753, |
| "14": 0.644, |
| "15": 0.6, |
| "16": 0.58, |
| "17": 0.546, |
| "18": 0.463, |
| "19": 0.392, |
| "20": 0.286 |
| }, |
| "pass@8": { |
| "11": 0.873, |
| "12": 0.868, |
| "13": 0.829, |
| "14": 0.718, |
| "15": 0.665, |
| "16": 0.622, |
| "17": 0.576, |
| "18": 0.51, |
| "19": 0.425, |
| "20": 0.318 |
| }, |
| "pass@16": { |
| "11": 0.924, |
| "12": 0.925, |
| "13": 0.889, |
| "14": 0.784, |
| "15": 0.744, |
| "16": 0.682, |
| "17": 0.638, |
| "18": 0.549, |
| "19": 0.463, |
| "20": 0.343 |
| }, |
| "pass@32": { |
| "11": 0.957, |
| "12": 0.956, |
| "13": 0.924, |
| "14": 0.85, |
| "15": 0.809, |
| "16": 0.734, |
| "17": 0.699, |
| "18": 0.608, |
| "19": 0.497, |
| "20": 0.376 |
| }, |
| "pass@64": { |
| "11": 0.98, |
| "12": 0.972, |
| "13": 0.954, |
| "14": 0.918, |
| "15": 0.887, |
| "16": 0.777, |
| "17": 0.76, |
| "18": 0.682, |
| "19": 0.548, |
| "20": 0.426 |
| }, |
| "pass@128": { |
| "11": 0.987, |
| "12": 0.983, |
| "13": 0.973, |
| "14": 0.949, |
| "15": 0.935, |
| "16": 0.81, |
| "17": 0.803, |
| "18": 0.741, |
| "19": 0.59, |
| "20": 0.475 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4847, |
| "pass@2": 0.5287, |
| "pass@4": 0.584, |
| "pass@8": 0.6404, |
| "pass@16": 0.6941, |
| "pass@32": 0.741, |
| "pass@64": 0.7904, |
| "pass@128": 0.8246 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
