| { |
| "files": { |
| "checkpoint-39470_id_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 0.9190031152647975, |
| "pass@2": 0.9470404984423676, |
| "pass@4": 0.9750778816199377, |
| "pass@8": 0.9875389408099688, |
| "pass@16": 0.9968847352024922, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.8532110091743119, |
| "pass@2": 0.908256880733945, |
| "pass@4": 0.9510703363914373, |
| "pass@8": 0.9847094801223242, |
| "pass@16": 0.9877675840978594, |
| "pass@32": 0.9908256880733946, |
| "pass@64": 0.9938837920489296, |
| "pass@128": 0.9969418960244648 |
| }, |
| "4": { |
| "pass@1": 0.765625, |
| "pass@2": 0.8625, |
| "pass@4": 0.928125, |
| "pass@8": 0.9625, |
| "pass@16": 0.975, |
| "pass@32": 0.978125, |
| "pass@64": 0.99375, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.7587301587301587, |
| "pass@2": 0.8412698412698413, |
| "pass@4": 0.9047619047619048, |
| "pass@8": 0.946031746031746, |
| "pass@16": 0.9619047619047619, |
| "pass@32": 0.9746031746031746, |
| "pass@64": 0.9809523809523809, |
| "pass@128": 0.9904761904761905 |
| }, |
| "6": { |
| "pass@1": 0.7994011976047904, |
| "pass@2": 0.8682634730538922, |
| "pass@4": 0.9161676646706587, |
| "pass@8": 0.9431137724550899, |
| "pass@16": 0.9700598802395209, |
| "pass@32": 0.9850299401197605, |
| "pass@64": 0.9880239520958084, |
| "pass@128": 0.9910179640718563 |
| }, |
| "7": { |
| "pass@1": 0.6686390532544378, |
| "pass@2": 0.7662721893491125, |
| "pass@4": 0.863905325443787, |
| "pass@8": 0.9112426035502958, |
| "pass@16": 0.9526627218934911, |
| "pass@32": 0.9674556213017751, |
| "pass@64": 0.9822485207100592, |
| "pass@128": 0.9911242603550295 |
| }, |
| "8": { |
| "pass@1": 0.6025641025641025, |
| "pass@2": 0.7051282051282052, |
| "pass@4": 0.8012820512820513, |
| "pass@8": 0.8878205128205128, |
| "pass@16": 0.9166666666666666, |
| "pass@32": 0.9391025641025641, |
| "pass@64": 0.9615384615384616, |
| "pass@128": 0.9743589743589743 |
| }, |
| "9": { |
| "pass@1": 0.5746478873239437, |
| "pass@2": 0.6816901408450704, |
| "pass@4": 0.7774647887323943, |
| "pass@8": 0.8366197183098592, |
| "pass@16": 0.8873239436619719, |
| "pass@32": 0.9380281690140845, |
| "pass@64": 0.9633802816901409, |
| "pass@128": 0.9661971830985916 |
| }, |
| "10": { |
| "pass@1": 0.471976401179941, |
| "pass@2": 0.5545722713864307, |
| "pass@4": 0.6548672566371682, |
| "pass@8": 0.7404129793510325, |
| "pass@16": 0.7935103244837758, |
| "pass@32": 0.8436578171091446, |
| "pass@64": 0.8761061946902655, |
| "pass@128": 0.9026548672566371 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 0.9190031152647975, |
| "3": 0.8532110091743119, |
| "4": 0.765625, |
| "5": 0.7587301587301587, |
| "6": 0.7994011976047904, |
| "7": 0.6686390532544378, |
| "8": 0.6025641025641025, |
| "9": 0.5746478873239437, |
| "10": 0.471976401179941 |
| }, |
| "pass@2": { |
| "2": 0.9470404984423676, |
| "3": 0.908256880733945, |
| "4": 0.8625, |
| "5": 0.8412698412698413, |
| "6": 0.8682634730538922, |
| "7": 0.7662721893491125, |
| "8": 0.7051282051282052, |
| "9": 0.6816901408450704, |
| "10": 0.5545722713864307 |
| }, |
| "pass@4": { |
| "2": 0.9750778816199377, |
| "3": 0.9510703363914373, |
| "4": 0.928125, |
| "5": 0.9047619047619048, |
| "6": 0.9161676646706587, |
| "7": 0.863905325443787, |
| "8": 0.8012820512820513, |
| "9": 0.7774647887323943, |
| "10": 0.6548672566371682 |
| }, |
| "pass@8": { |
| "2": 0.9875389408099688, |
| "3": 0.9847094801223242, |
| "4": 0.9625, |
| "5": 0.946031746031746, |
| "6": 0.9431137724550899, |
| "7": 0.9112426035502958, |
| "8": 0.8878205128205128, |
| "9": 0.8366197183098592, |
| "10": 0.7404129793510325 |
| }, |
| "pass@16": { |
| "2": 0.9968847352024922, |
| "3": 0.9877675840978594, |
| "4": 0.975, |
| "5": 0.9619047619047619, |
| "6": 0.9700598802395209, |
| "7": 0.9526627218934911, |
| "8": 0.9166666666666666, |
| "9": 0.8873239436619719, |
| "10": 0.7935103244837758 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 0.9908256880733946, |
| "4": 0.978125, |
| "5": 0.9746031746031746, |
| "6": 0.9850299401197605, |
| "7": 0.9674556213017751, |
| "8": 0.9391025641025641, |
| "9": 0.9380281690140845, |
| "10": 0.8436578171091446 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 0.9938837920489296, |
| "4": 0.99375, |
| "5": 0.9809523809523809, |
| "6": 0.9880239520958084, |
| "7": 0.9822485207100592, |
| "8": 0.9615384615384616, |
| "9": 0.9633802816901409, |
| "10": 0.8761061946902655 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 0.9969418960244648, |
| "4": 0.996875, |
| "5": 0.9904761904761905, |
| "6": 0.9910179640718563, |
| "7": 0.9911242603550295, |
| "8": 0.9743589743589743, |
| "9": 0.9661971830985916, |
| "10": 0.9026548672566371 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.7102330293819655, |
| "pass@2": 0.7906112799729821, |
| "pass@4": 0.861870989530564, |
| "pass@8": 0.9094900371496116, |
| "pass@16": 0.9368456602499156, |
| "pass@32": 0.9567713610266801, |
| "pass@64": 0.9706180344478217, |
| "pass@128": 0.9783856805133401 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9970845481049563, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9941520467836257, |
| "pass@2": 0.9970760233918129, |
| "pass@4": 0.9970760233918129, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9470588235294117, |
| "pass@2": 0.9735294117647059, |
| "pass@4": 0.9911764705882353, |
| "pass@8": 0.9970588235294118, |
| "pass@16": 0.9970588235294118, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 0.9970588235294118, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9621451104100947, |
| "pass@2": 0.9684542586750788, |
| "pass@4": 0.9842271293375394, |
| "pass@8": 0.9873817034700315, |
| "pass@16": 0.9968454258675079, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8449367088607594, |
| "pass@2": 0.9177215189873418, |
| "pass@4": 0.9493670886075949, |
| "pass@8": 0.9778481012658228, |
| "pass@16": 0.990506329113924, |
| "pass@32": 0.9936708860759493, |
| "pass@64": 0.9968354430379747, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.8018018018018018, |
| "pass@2": 0.8708708708708709, |
| "pass@4": 0.933933933933934, |
| "pass@8": 0.972972972972973, |
| "pass@16": 0.984984984984985, |
| "pass@32": 0.996996996996997, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.7684887459807074, |
| "pass@2": 0.819935691318328, |
| "pass@4": 0.887459807073955, |
| "pass@8": 0.9485530546623794, |
| "pass@16": 0.9839228295819936, |
| "pass@32": 0.9903536977491961, |
| "pass@64": 0.9967845659163987, |
| "pass@128": 1.0 |
| }, |
| "10": { |
| "pass@1": 0.6059701492537314, |
| "pass@2": 0.6805970149253732, |
| "pass@4": 0.7492537313432835, |
| "pass@8": 0.8507462686567164, |
| "pass@16": 0.9104477611940298, |
| "pass@32": 0.9522388059701492, |
| "pass@64": 0.9582089552238806, |
| "pass@128": 0.9671641791044776 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.9970845481049563, |
| "4": 0.9941520467836257, |
| "5": 0.9470588235294117, |
| "6": 0.9621451104100947, |
| "7": 0.8449367088607594, |
| "8": 0.8018018018018018, |
| "9": 0.7684887459807074, |
| "10": 0.6059701492537314 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.9735294117647059, |
| "6": 0.9684542586750788, |
| "7": 0.9177215189873418, |
| "8": 0.8708708708708709, |
| "9": 0.819935691318328, |
| "10": 0.6805970149253732 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 0.9970760233918129, |
| "5": 0.9911764705882353, |
| "6": 0.9842271293375394, |
| "7": 0.9493670886075949, |
| "8": 0.933933933933934, |
| "9": 0.887459807073955, |
| "10": 0.7492537313432835 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 0.9873817034700315, |
| "7": 0.9778481012658228, |
| "8": 0.972972972972973, |
| "9": 0.9485530546623794, |
| "10": 0.8507462686567164 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 0.9968454258675079, |
| "7": 0.990506329113924, |
| "8": 0.984984984984985, |
| "9": 0.9839228295819936, |
| "10": 0.9104477611940298 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9936708860759493, |
| "8": 0.996996996996997, |
| "9": 0.9903536977491961, |
| "10": 0.9522388059701492 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 0.9967845659163987, |
| "10": 0.9582089552238806 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9970588235294118, |
| "6": 1.0, |
| "7": 0.9968354430379747, |
| "8": 1.0, |
| "9": 1.0, |
| "10": 0.9671641791044776 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8821560093739538, |
| "pass@2": 0.9156344158018078, |
| "pass@4": 0.9444258453297623, |
| "pass@8": 0.970873786407767, |
| "pass@16": 0.9849347171074657, |
| "pass@32": 0.9922999665215936, |
| "pass@64": 0.9943086709072648, |
| "pass@128": 0.995647807164379 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9939393939393939, |
| "pass@2": 0.9939393939393939, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9733727810650887, |
| "pass@2": 0.9822485207100592, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9478260869565217, |
| "pass@2": 0.9884057971014493, |
| "pass@4": 0.9971014492753624, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9828080229226361, |
| "pass@2": 0.9856733524355301, |
| "pass@4": 0.9914040114613181, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.8728323699421965, |
| "pass@2": 0.9219653179190751, |
| "pass@4": 0.9710982658959537, |
| "pass@8": 0.9884393063583815, |
| "pass@16": 0.9942196531791907, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.8, |
| "pass@2": 0.8760563380281691, |
| "pass@4": 0.9267605633802817, |
| "pass@8": 0.9661971830985916, |
| "pass@16": 0.9859154929577465, |
| "pass@32": 0.9943661971830986, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.718562874251497, |
| "pass@2": 0.8053892215568862, |
| "pass@4": 0.8682634730538922, |
| "pass@8": 0.9311377245508982, |
| "pass@16": 0.9670658682634731, |
| "pass@32": 0.9850299401197605, |
| "pass@64": 0.9970059880239521, |
| "pass@128": 0.9970059880239521 |
| }, |
| "10": { |
| "pass@1": 0.6595092024539877, |
| "pass@2": 0.7208588957055214, |
| "pass@4": 0.7852760736196319, |
| "pass@8": 0.8404907975460123, |
| "pass@16": 0.8926380368098159, |
| "pass@32": 0.9202453987730062, |
| "pass@64": 0.950920245398773, |
| "pass@128": 0.9631901840490797 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 1.0, |
| "3": 0.9939393939393939, |
| "4": 0.9733727810650887, |
| "5": 0.9478260869565217, |
| "6": 0.9828080229226361, |
| "7": 0.8728323699421965, |
| "8": 0.8, |
| "9": 0.718562874251497, |
| "10": 0.6595092024539877 |
| }, |
| "pass@2": { |
| "2": 1.0, |
| "3": 0.9939393939393939, |
| "4": 0.9822485207100592, |
| "5": 0.9884057971014493, |
| "6": 0.9856733524355301, |
| "7": 0.9219653179190751, |
| "8": 0.8760563380281691, |
| "9": 0.8053892215568862, |
| "10": 0.7208588957055214 |
| }, |
| "pass@4": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 0.9971014492753624, |
| "6": 0.9914040114613181, |
| "7": 0.9710982658959537, |
| "8": 0.9267605633802817, |
| "9": 0.8682634730538922, |
| "10": 0.7852760736196319 |
| }, |
| "pass@8": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9884393063583815, |
| "8": 0.9661971830985916, |
| "9": 0.9311377245508982, |
| "10": 0.8404907975460123 |
| }, |
| "pass@16": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 0.9942196531791907, |
| "8": 0.9859154929577465, |
| "9": 0.9670658682634731, |
| "10": 0.8926380368098159 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 0.9943661971830986, |
| "9": 0.9850299401197605, |
| "10": 0.9202453987730062 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.950920245398773 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 1.0, |
| "4": 1.0, |
| "5": 1.0, |
| "6": 1.0, |
| "7": 1.0, |
| "8": 1.0, |
| "9": 0.9970059880239521, |
| "10": 0.9631901840490797 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8836828309305373, |
| "pass@2": 0.9200524246395806, |
| "pass@4": 0.9495412844036697, |
| "pass@8": 0.9701834862385321, |
| "pass@16": 0.9826343381389253, |
| "pass@32": 0.9891874180865007, |
| "pass@64": 0.994429882044561, |
| "pass@128": 0.995740498034076 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "2": { |
| "pass@1": 0.974, |
| "pass@2": 0.983, |
| "pass@4": 0.992, |
| "pass@8": 0.996, |
| "pass@16": 0.999, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.949, |
| "pass@2": 0.968, |
| "pass@4": 0.984, |
| "pass@8": 0.995, |
| "pass@16": 0.996, |
| "pass@32": 0.997, |
| "pass@64": 0.998, |
| "pass@128": 0.999 |
| }, |
| "4": { |
| "pass@1": 0.914, |
| "pass@2": 0.949, |
| "pass@4": 0.976, |
| "pass@8": 0.988, |
| "pass@16": 0.992, |
| "pass@32": 0.993, |
| "pass@64": 0.998, |
| "pass@128": 0.999 |
| }, |
| "5": { |
| "pass@1": 0.888, |
| "pass@2": 0.937, |
| "pass@4": 0.966, |
| "pass@8": 0.982, |
| "pass@16": 0.987, |
| "pass@32": 0.991, |
| "pass@64": 0.993, |
| "pass@128": 0.996 |
| }, |
| "6": { |
| "pass@1": 0.915, |
| "pass@2": 0.941, |
| "pass@4": 0.964, |
| "pass@8": 0.977, |
| "pass@16": 0.989, |
| "pass@32": 0.995, |
| "pass@64": 0.996, |
| "pass@128": 0.997 |
| }, |
| "7": { |
| "pass@1": 0.795, |
| "pass@2": 0.868, |
| "pass@4": 0.928, |
| "pass@8": 0.959, |
| "pass@16": 0.979, |
| "pass@32": 0.987, |
| "pass@64": 0.993, |
| "pass@128": 0.996 |
| }, |
| "8": { |
| "pass@1": 0.739, |
| "pass@2": 0.821, |
| "pass@4": 0.89, |
| "pass@8": 0.944, |
| "pass@16": 0.964, |
| "pass@32": 0.978, |
| "pass@64": 0.988, |
| "pass@128": 0.992 |
| }, |
| "9": { |
| "pass@1": 0.683, |
| "pass@2": 0.766, |
| "pass@4": 0.842, |
| "pass@8": 0.903, |
| "pass@16": 0.944, |
| "pass@32": 0.97, |
| "pass@64": 0.985, |
| "pass@128": 0.987 |
| }, |
| "10": { |
| "pass@1": 0.578, |
| "pass@2": 0.651, |
| "pass@4": 0.729, |
| "pass@8": 0.81, |
| "pass@16": 0.865, |
| "pass@32": 0.905, |
| "pass@64": 0.928, |
| "pass@128": 0.944 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "2": 0.974, |
| "3": 0.949, |
| "4": 0.914, |
| "5": 0.888, |
| "6": 0.915, |
| "7": 0.795, |
| "8": 0.739, |
| "9": 0.683, |
| "10": 0.578 |
| }, |
| "pass@2": { |
| "2": 0.983, |
| "3": 0.968, |
| "4": 0.949, |
| "5": 0.937, |
| "6": 0.941, |
| "7": 0.868, |
| "8": 0.821, |
| "9": 0.766, |
| "10": 0.651 |
| }, |
| "pass@4": { |
| "2": 0.992, |
| "3": 0.984, |
| "4": 0.976, |
| "5": 0.966, |
| "6": 0.964, |
| "7": 0.928, |
| "8": 0.89, |
| "9": 0.842, |
| "10": 0.729 |
| }, |
| "pass@8": { |
| "2": 0.996, |
| "3": 0.995, |
| "4": 0.988, |
| "5": 0.982, |
| "6": 0.977, |
| "7": 0.959, |
| "8": 0.944, |
| "9": 0.903, |
| "10": 0.81 |
| }, |
| "pass@16": { |
| "2": 0.999, |
| "3": 0.996, |
| "4": 0.992, |
| "5": 0.987, |
| "6": 0.989, |
| "7": 0.979, |
| "8": 0.964, |
| "9": 0.944, |
| "10": 0.865 |
| }, |
| "pass@32": { |
| "2": 1.0, |
| "3": 0.997, |
| "4": 0.993, |
| "5": 0.991, |
| "6": 0.995, |
| "7": 0.987, |
| "8": 0.978, |
| "9": 0.97, |
| "10": 0.905 |
| }, |
| "pass@64": { |
| "2": 1.0, |
| "3": 0.998, |
| "4": 0.998, |
| "5": 0.993, |
| "6": 0.996, |
| "7": 0.993, |
| "8": 0.988, |
| "9": 0.985, |
| "10": 0.928 |
| }, |
| "pass@128": { |
| "2": 1.0, |
| "3": 0.999, |
| "4": 0.999, |
| "5": 0.996, |
| "6": 0.997, |
| "7": 0.996, |
| "8": 0.992, |
| "9": 0.987, |
| "10": 0.944 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.8261111111111111, |
| "pass@2": 0.876, |
| "pass@4": 0.919, |
| "pass@8": 0.9504444444444444, |
| "pass@16": 0.9683333333333334, |
| "pass@32": 0.9795555555555555, |
| "pass@64": 0.9865555555555555, |
| "pass@128": 0.99 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "2": 321, |
| "3": 327, |
| "4": 320, |
| "5": 315, |
| "6": 334, |
| "7": 338, |
| "8": 312, |
| "9": 355, |
| "10": 339 |
| }, |
| "total": 2961 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "2": 350, |
| "3": 343, |
| "4": 342, |
| "5": 340, |
| "6": 317, |
| "7": 316, |
| "8": 333, |
| "9": 311, |
| "10": 335 |
| }, |
| "total": 2987 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "2": 329, |
| "3": 330, |
| "4": 338, |
| "5": 345, |
| "6": 349, |
| "7": 346, |
| "8": 355, |
| "9": 334, |
| "10": 326 |
| }, |
| "total": 3052 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "2": 1000, |
| "3": 1000, |
| "4": 1000, |
| "5": 1000, |
| "6": 1000, |
| "7": 1000, |
| "8": 1000, |
| "9": 1000, |
| "10": 1000 |
| }, |
| "total": 9000 |
| } |
| } |
| }, |
| "checkpoint-39470_ood_generations.jsonl.graph_results.jsonl": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.42209631728045327, |
| "pass@2": 0.5155807365439093, |
| "pass@4": 0.5892351274787535, |
| "pass@8": 0.6883852691218131, |
| "pass@16": 0.773371104815864, |
| "pass@32": 0.8215297450424929, |
| "pass@64": 0.8725212464589235, |
| "pass@128": 0.9178470254957507 |
| }, |
| "12": { |
| "pass@1": 0.4519774011299435, |
| "pass@2": 0.559322033898305, |
| "pass@4": 0.632768361581921, |
| "pass@8": 0.7288135593220338, |
| "pass@16": 0.788135593220339, |
| "pass@32": 0.844632768361582, |
| "pass@64": 0.8870056497175142, |
| "pass@128": 0.9124293785310734 |
| }, |
| "13": { |
| "pass@1": 0.43260188087774293, |
| "pass@2": 0.5172413793103449, |
| "pass@4": 0.5862068965517241, |
| "pass@8": 0.6520376175548589, |
| "pass@16": 0.7210031347962382, |
| "pass@32": 0.8025078369905956, |
| "pass@64": 0.8369905956112853, |
| "pass@128": 0.877742946708464 |
| }, |
| "14": { |
| "pass@1": 0.38317757009345793, |
| "pass@2": 0.4485981308411215, |
| "pass@4": 0.48286604361370716, |
| "pass@8": 0.5451713395638629, |
| "pass@16": 0.632398753894081, |
| "pass@32": 0.7071651090342679, |
| "pass@64": 0.8006230529595015, |
| "pass@128": 0.881619937694704 |
| }, |
| "15": { |
| "pass@1": 0.3670520231213873, |
| "pass@2": 0.4421965317919075, |
| "pass@4": 0.49421965317919075, |
| "pass@8": 0.5549132947976878, |
| "pass@16": 0.638728323699422, |
| "pass@32": 0.7254335260115607, |
| "pass@64": 0.8034682080924855, |
| "pass@128": 0.8670520231213873 |
| }, |
| "16": { |
| "pass@1": 0.32903225806451614, |
| "pass@2": 0.4032258064516129, |
| "pass@4": 0.4483870967741935, |
| "pass@8": 0.4967741935483871, |
| "pass@16": 0.5870967741935483, |
| "pass@32": 0.6580645161290323, |
| "pass@64": 0.7032258064516129, |
| "pass@128": 0.7354838709677419 |
| }, |
| "17": { |
| "pass@1": 0.3468208092485549, |
| "pass@2": 0.42485549132947975, |
| "pass@4": 0.47398843930635837, |
| "pass@8": 0.5317919075144508, |
| "pass@16": 0.6040462427745664, |
| "pass@32": 0.6647398843930635, |
| "pass@64": 0.7254335260115607, |
| "pass@128": 0.7716763005780347 |
| }, |
| "18": { |
| "pass@1": 0.2857142857142857, |
| "pass@2": 0.3657142857142857, |
| "pass@4": 0.44, |
| "pass@8": 0.4657142857142857, |
| "pass@16": 0.5342857142857143, |
| "pass@32": 0.58, |
| "pass@64": 0.6485714285714286, |
| "pass@128": 0.7028571428571428 |
| }, |
| "19": { |
| "pass@1": 0.24324324324324326, |
| "pass@2": 0.3063063063063063, |
| "pass@4": 0.34534534534534533, |
| "pass@8": 0.4084084084084084, |
| "pass@16": 0.4594594594594595, |
| "pass@32": 0.4984984984984985, |
| "pass@64": 0.5375375375375375, |
| "pass@128": 0.6006006006006006 |
| }, |
| "20": { |
| "pass@1": 0.1891891891891892, |
| "pass@2": 0.2672672672672673, |
| "pass@4": 0.33933933933933935, |
| "pass@8": 0.39039039039039036, |
| "pass@16": 0.44744744744744747, |
| "pass@32": 0.4984984984984985, |
| "pass@64": 0.5465465465465466, |
| "pass@128": 0.5915915915915916 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.42209631728045327, |
| "12": 0.4519774011299435, |
| "13": 0.43260188087774293, |
| "14": 0.38317757009345793, |
| "15": 0.3670520231213873, |
| "16": 0.32903225806451614, |
| "17": 0.3468208092485549, |
| "18": 0.2857142857142857, |
| "19": 0.24324324324324326, |
| "20": 0.1891891891891892 |
| }, |
| "pass@2": { |
| "11": 0.5155807365439093, |
| "12": 0.559322033898305, |
| "13": 0.5172413793103449, |
| "14": 0.4485981308411215, |
| "15": 0.4421965317919075, |
| "16": 0.4032258064516129, |
| "17": 0.42485549132947975, |
| "18": 0.3657142857142857, |
| "19": 0.3063063063063063, |
| "20": 0.2672672672672673 |
| }, |
| "pass@4": { |
| "11": 0.5892351274787535, |
| "12": 0.632768361581921, |
| "13": 0.5862068965517241, |
| "14": 0.48286604361370716, |
| "15": 0.49421965317919075, |
| "16": 0.4483870967741935, |
| "17": 0.47398843930635837, |
| "18": 0.44, |
| "19": 0.34534534534534533, |
| "20": 0.33933933933933935 |
| }, |
| "pass@8": { |
| "11": 0.6883852691218131, |
| "12": 0.7288135593220338, |
| "13": 0.6520376175548589, |
| "14": 0.5451713395638629, |
| "15": 0.5549132947976878, |
| "16": 0.4967741935483871, |
| "17": 0.5317919075144508, |
| "18": 0.4657142857142857, |
| "19": 0.4084084084084084, |
| "20": 0.39039039039039036 |
| }, |
| "pass@16": { |
| "11": 0.773371104815864, |
| "12": 0.788135593220339, |
| "13": 0.7210031347962382, |
| "14": 0.632398753894081, |
| "15": 0.638728323699422, |
| "16": 0.5870967741935483, |
| "17": 0.6040462427745664, |
| "18": 0.5342857142857143, |
| "19": 0.4594594594594595, |
| "20": 0.44744744744744747 |
| }, |
| "pass@32": { |
| "11": 0.8215297450424929, |
| "12": 0.844632768361582, |
| "13": 0.8025078369905956, |
| "14": 0.7071651090342679, |
| "15": 0.7254335260115607, |
| "16": 0.6580645161290323, |
| "17": 0.6647398843930635, |
| "18": 0.58, |
| "19": 0.4984984984984985, |
| "20": 0.4984984984984985 |
| }, |
| "pass@64": { |
| "11": 0.8725212464589235, |
| "12": 0.8870056497175142, |
| "13": 0.8369905956112853, |
| "14": 0.8006230529595015, |
| "15": 0.8034682080924855, |
| "16": 0.7032258064516129, |
| "17": 0.7254335260115607, |
| "18": 0.6485714285714286, |
| "19": 0.5375375375375375, |
| "20": 0.5465465465465466 |
| }, |
| "pass@128": { |
| "11": 0.9178470254957507, |
| "12": 0.9124293785310734, |
| "13": 0.877742946708464, |
| "14": 0.881619937694704, |
| "15": 0.8670520231213873, |
| "16": 0.7354838709677419, |
| "17": 0.7716763005780347, |
| "18": 0.7028571428571428, |
| "19": 0.6006006006006006, |
| "20": 0.5915915915915916 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.34561664190193164, |
| "pass@2": 0.42585438335809805, |
| "pass@4": 0.48439821693907875, |
| "pass@8": 0.5476968796433879, |
| "pass@16": 0.6199108469539376, |
| "pass@32": 0.6811292719167905, |
| "pass@64": 0.737295690936107, |
| "pass@128": 0.7869242199108469 |
| } |
| }, |
| "movie_festival_awards": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.580952380952381, |
| "pass@2": 0.6571428571428571, |
| "pass@4": 0.7396825396825397, |
| "pass@8": 0.8444444444444444, |
| "pass@16": 0.9111111111111111, |
| "pass@32": 0.9587301587301588, |
| "pass@64": 0.9746031746031746, |
| "pass@128": 0.9936507936507937 |
| }, |
| "12": { |
| "pass@1": 0.6207951070336392, |
| "pass@2": 0.6850152905198776, |
| "pass@4": 0.7614678899082569, |
| "pass@8": 0.8379204892966361, |
| "pass@16": 0.8990825688073395, |
| "pass@32": 0.9388379204892966, |
| "pass@64": 0.9724770642201835, |
| "pass@128": 0.9755351681957186 |
| }, |
| "13": { |
| "pass@1": 0.5648148148148148, |
| "pass@2": 0.6234567901234568, |
| "pass@4": 0.7191358024691358, |
| "pass@8": 0.7932098765432098, |
| "pass@16": 0.8703703703703703, |
| "pass@32": 0.9074074074074074, |
| "pass@64": 0.9259259259259259, |
| "pass@128": 0.9567901234567902 |
| }, |
| "14": { |
| "pass@1": 0.516320474777448, |
| "pass@2": 0.5786350148367952, |
| "pass@4": 0.6350148367952523, |
| "pass@8": 0.6943620178041543, |
| "pass@16": 0.7655786350148368, |
| "pass@32": 0.8664688427299704, |
| "pass@64": 0.9258160237388724, |
| "pass@128": 0.9465875370919882 |
| }, |
| "15": { |
| "pass@1": 0.4722222222222222, |
| "pass@2": 0.5, |
| "pass@4": 0.5555555555555556, |
| "pass@8": 0.6203703703703703, |
| "pass@16": 0.7283950617283951, |
| "pass@32": 0.8055555555555556, |
| "pass@64": 0.8672839506172839, |
| "pass@128": 0.9074074074074074 |
| }, |
| "16": { |
| "pass@1": 0.5162241887905604, |
| "pass@2": 0.5634218289085545, |
| "pass@4": 0.5958702064896755, |
| "pass@8": 0.6460176991150443, |
| "pass@16": 0.6991150442477876, |
| "pass@32": 0.7610619469026548, |
| "pass@64": 0.8082595870206489, |
| "pass@128": 0.8377581120943953 |
| }, |
| "17": { |
| "pass@1": 0.4819277108433735, |
| "pass@2": 0.5090361445783133, |
| "pass@4": 0.5421686746987951, |
| "pass@8": 0.5873493975903614, |
| "pass@16": 0.6536144578313253, |
| "pass@32": 0.7198795180722891, |
| "pass@64": 0.7801204819277109, |
| "pass@128": 0.8132530120481928 |
| }, |
| "18": { |
| "pass@1": 0.43548387096774194, |
| "pass@2": 0.5, |
| "pass@4": 0.5451612903225806, |
| "pass@8": 0.5870967741935483, |
| "pass@16": 0.635483870967742, |
| "pass@32": 0.7, |
| "pass@64": 0.7580645161290323, |
| "pass@128": 0.8225806451612904 |
| }, |
| "19": { |
| "pass@1": 0.3504531722054381, |
| "pass@2": 0.40483383685800606, |
| "pass@4": 0.43202416918429004, |
| "pass@8": 0.486404833836858, |
| "pass@16": 0.5287009063444109, |
| "pass@32": 0.5740181268882175, |
| "pass@64": 0.6193353474320241, |
| "pass@128": 0.6888217522658611 |
| }, |
| "20": { |
| "pass@1": 0.2912912912912913, |
| "pass@2": 0.35735735735735735, |
| "pass@4": 0.39039039039039036, |
| "pass@8": 0.42342342342342343, |
| "pass@16": 0.46846846846846846, |
| "pass@32": 0.5135135135135135, |
| "pass@64": 0.5615615615615616, |
| "pass@128": 0.6306306306306306 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.580952380952381, |
| "12": 0.6207951070336392, |
| "13": 0.5648148148148148, |
| "14": 0.516320474777448, |
| "15": 0.4722222222222222, |
| "16": 0.5162241887905604, |
| "17": 0.4819277108433735, |
| "18": 0.43548387096774194, |
| "19": 0.3504531722054381, |
| "20": 0.2912912912912913 |
| }, |
| "pass@2": { |
| "11": 0.6571428571428571, |
| "12": 0.6850152905198776, |
| "13": 0.6234567901234568, |
| "14": 0.5786350148367952, |
| "15": 0.5, |
| "16": 0.5634218289085545, |
| "17": 0.5090361445783133, |
| "18": 0.5, |
| "19": 0.40483383685800606, |
| "20": 0.35735735735735735 |
| }, |
| "pass@4": { |
| "11": 0.7396825396825397, |
| "12": 0.7614678899082569, |
| "13": 0.7191358024691358, |
| "14": 0.6350148367952523, |
| "15": 0.5555555555555556, |
| "16": 0.5958702064896755, |
| "17": 0.5421686746987951, |
| "18": 0.5451612903225806, |
| "19": 0.43202416918429004, |
| "20": 0.39039039039039036 |
| }, |
| "pass@8": { |
| "11": 0.8444444444444444, |
| "12": 0.8379204892966361, |
| "13": 0.7932098765432098, |
| "14": 0.6943620178041543, |
| "15": 0.6203703703703703, |
| "16": 0.6460176991150443, |
| "17": 0.5873493975903614, |
| "18": 0.5870967741935483, |
| "19": 0.486404833836858, |
| "20": 0.42342342342342343 |
| }, |
| "pass@16": { |
| "11": 0.9111111111111111, |
| "12": 0.8990825688073395, |
| "13": 0.8703703703703703, |
| "14": 0.7655786350148368, |
| "15": 0.7283950617283951, |
| "16": 0.6991150442477876, |
| "17": 0.6536144578313253, |
| "18": 0.635483870967742, |
| "19": 0.5287009063444109, |
| "20": 0.46846846846846846 |
| }, |
| "pass@32": { |
| "11": 0.9587301587301588, |
| "12": 0.9388379204892966, |
| "13": 0.9074074074074074, |
| "14": 0.8664688427299704, |
| "15": 0.8055555555555556, |
| "16": 0.7610619469026548, |
| "17": 0.7198795180722891, |
| "18": 0.7, |
| "19": 0.5740181268882175, |
| "20": 0.5135135135135135 |
| }, |
| "pass@64": { |
| "11": 0.9746031746031746, |
| "12": 0.9724770642201835, |
| "13": 0.9259259259259259, |
| "14": 0.9258160237388724, |
| "15": 0.8672839506172839, |
| "16": 0.8082595870206489, |
| "17": 0.7801204819277109, |
| "18": 0.7580645161290323, |
| "19": 0.6193353474320241, |
| "20": 0.5615615615615616 |
| }, |
| "pass@128": { |
| "11": 0.9936507936507937, |
| "12": 0.9755351681957186, |
| "13": 0.9567901234567902, |
| "14": 0.9465875370919882, |
| "15": 0.9074074074074074, |
| "16": 0.8377581120943953, |
| "17": 0.8132530120481928, |
| "18": 0.8225806451612904, |
| "19": 0.6888217522658611, |
| "20": 0.6306306306306306 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4825794621026895, |
| "pass@2": 0.5372860635696821, |
| "pass@4": 0.5907701711491442, |
| "pass@8": 0.6509779951100244, |
| "pass@16": 0.7148533007334963, |
| "pass@32": 0.7735330073349633, |
| "pass@64": 0.8184596577017115, |
| "pass@128": 0.8563569682151589 |
| } |
| }, |
| "teachers_in_school": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.6716867469879518, |
| "pass@2": 0.7198795180722891, |
| "pass@4": 0.8132530120481928, |
| "pass@8": 0.8765060240963856, |
| "pass@16": 0.9186746987951807, |
| "pass@32": 0.9518072289156626, |
| "pass@64": 0.9728915662650602, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.554858934169279, |
| "pass@2": 0.5987460815047022, |
| "pass@4": 0.7084639498432602, |
| "pass@8": 0.786833855799373, |
| "pass@16": 0.8746081504702194, |
| "pass@32": 0.9216300940438872, |
| "pass@64": 0.9561128526645768, |
| "pass@128": 0.9749216300940439 |
| }, |
| "13": { |
| "pass@1": 0.6190476190476191, |
| "pass@2": 0.680672268907563, |
| "pass@4": 0.7647058823529411, |
| "pass@8": 0.8431372549019608, |
| "pass@16": 0.896358543417367, |
| "pass@32": 0.9355742296918768, |
| "pass@64": 0.9495798319327731, |
| "pass@128": 0.969187675070028 |
| }, |
| "14": { |
| "pass@1": 0.5263157894736842, |
| "pass@2": 0.5964912280701754, |
| "pass@4": 0.652046783625731, |
| "pass@8": 0.7076023391812866, |
| "pass@16": 0.7719298245614035, |
| "pass@32": 0.8421052631578947, |
| "pass@64": 0.9035087719298246, |
| "pass@128": 0.9532163742690059 |
| }, |
| "15": { |
| "pass@1": 0.5272727272727272, |
| "pass@2": 0.5727272727272728, |
| "pass@4": 0.6212121212121212, |
| "pass@8": 0.6818181818181818, |
| "pass@16": 0.7545454545454545, |
| "pass@32": 0.8272727272727273, |
| "pass@64": 0.9090909090909091, |
| "pass@128": 0.9424242424242424 |
| }, |
| "16": { |
| "pass@1": 0.5128205128205128, |
| "pass@2": 0.5441595441595442, |
| "pass@4": 0.5698005698005698, |
| "pass@8": 0.6267806267806267, |
| "pass@16": 0.6837606837606838, |
| "pass@32": 0.7578347578347578, |
| "pass@64": 0.8005698005698005, |
| "pass@128": 0.8490028490028491 |
| }, |
| "17": { |
| "pass@1": 0.4161490683229814, |
| "pass@2": 0.4503105590062112, |
| "pass@4": 0.4968944099378882, |
| "pass@8": 0.5434782608695652, |
| "pass@16": 0.6242236024844721, |
| "pass@32": 0.7049689440993789, |
| "pass@64": 0.7888198757763976, |
| "pass@128": 0.8322981366459627 |
| }, |
| "18": { |
| "pass@1": 0.40294117647058825, |
| "pass@2": 0.45588235294117646, |
| "pass@4": 0.4764705882352941, |
| "pass@8": 0.5117647058823529, |
| "pass@16": 0.5705882352941176, |
| "pass@32": 0.6235294117647059, |
| "pass@64": 0.7029411764705882, |
| "pass@128": 0.7617647058823529 |
| }, |
| "19": { |
| "pass@1": 0.35119047619047616, |
| "pass@2": 0.3958333333333333, |
| "pass@4": 0.44047619047619047, |
| "pass@8": 0.4791666666666667, |
| "pass@16": 0.5238095238095238, |
| "pass@32": 0.6011904761904762, |
| "pass@64": 0.6607142857142857, |
| "pass@128": 0.7142857142857143 |
| }, |
| "20": { |
| "pass@1": 0.3203592814371258, |
| "pass@2": 0.36826347305389223, |
| "pass@4": 0.4161676646706587, |
| "pass@8": 0.45209580838323354, |
| "pass@16": 0.5029940119760479, |
| "pass@32": 0.5389221556886228, |
| "pass@64": 0.5718562874251497, |
| "pass@128": 0.5988023952095808 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.6716867469879518, |
| "12": 0.554858934169279, |
| "13": 0.6190476190476191, |
| "14": 0.5263157894736842, |
| "15": 0.5272727272727272, |
| "16": 0.5128205128205128, |
| "17": 0.4161490683229814, |
| "18": 0.40294117647058825, |
| "19": 0.35119047619047616, |
| "20": 0.3203592814371258 |
| }, |
| "pass@2": { |
| "11": 0.7198795180722891, |
| "12": 0.5987460815047022, |
| "13": 0.680672268907563, |
| "14": 0.5964912280701754, |
| "15": 0.5727272727272728, |
| "16": 0.5441595441595442, |
| "17": 0.4503105590062112, |
| "18": 0.45588235294117646, |
| "19": 0.3958333333333333, |
| "20": 0.36826347305389223 |
| }, |
| "pass@4": { |
| "11": 0.8132530120481928, |
| "12": 0.7084639498432602, |
| "13": 0.7647058823529411, |
| "14": 0.652046783625731, |
| "15": 0.6212121212121212, |
| "16": 0.5698005698005698, |
| "17": 0.4968944099378882, |
| "18": 0.4764705882352941, |
| "19": 0.44047619047619047, |
| "20": 0.4161676646706587 |
| }, |
| "pass@8": { |
| "11": 0.8765060240963856, |
| "12": 0.786833855799373, |
| "13": 0.8431372549019608, |
| "14": 0.7076023391812866, |
| "15": 0.6818181818181818, |
| "16": 0.6267806267806267, |
| "17": 0.5434782608695652, |
| "18": 0.5117647058823529, |
| "19": 0.4791666666666667, |
| "20": 0.45209580838323354 |
| }, |
| "pass@16": { |
| "11": 0.9186746987951807, |
| "12": 0.8746081504702194, |
| "13": 0.896358543417367, |
| "14": 0.7719298245614035, |
| "15": 0.7545454545454545, |
| "16": 0.6837606837606838, |
| "17": 0.6242236024844721, |
| "18": 0.5705882352941176, |
| "19": 0.5238095238095238, |
| "20": 0.5029940119760479 |
| }, |
| "pass@32": { |
| "11": 0.9518072289156626, |
| "12": 0.9216300940438872, |
| "13": 0.9355742296918768, |
| "14": 0.8421052631578947, |
| "15": 0.8272727272727273, |
| "16": 0.7578347578347578, |
| "17": 0.7049689440993789, |
| "18": 0.6235294117647059, |
| "19": 0.6011904761904762, |
| "20": 0.5389221556886228 |
| }, |
| "pass@64": { |
| "11": 0.9728915662650602, |
| "12": 0.9561128526645768, |
| "13": 0.9495798319327731, |
| "14": 0.9035087719298246, |
| "15": 0.9090909090909091, |
| "16": 0.8005698005698005, |
| "17": 0.7888198757763976, |
| "18": 0.7029411764705882, |
| "19": 0.6607142857142857, |
| "20": 0.5718562874251497 |
| }, |
| "pass@128": { |
| "11": 0.9819277108433735, |
| "12": 0.9749216300940439, |
| "13": 0.969187675070028, |
| "14": 0.9532163742690059, |
| "15": 0.9424242424242424, |
| "16": 0.8490028490028491, |
| "17": 0.8322981366459627, |
| "18": 0.7617647058823529, |
| "19": 0.7142857142857143, |
| "20": 0.5988023952095808 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4909307166220636, |
| "pass@2": 0.5391019922688076, |
| "pass@4": 0.5964912280701754, |
| "pass@8": 0.6515016354445435, |
| "pass@16": 0.7124591138864109, |
| "pass@32": 0.7707404103479036, |
| "pass@64": 0.8215878679750223, |
| "pass@128": 0.8578650014867678 |
| } |
| } |
| }, |
| "total": { |
| "per_op_pass_at_k": { |
| "11": { |
| "pass@1": 0.555, |
| "pass@2": 0.628, |
| "pass@4": 0.711, |
| "pass@8": 0.8, |
| "pass@16": 0.865, |
| "pass@32": 0.908, |
| "pass@64": 0.938, |
| "pass@128": 0.963 |
| }, |
| "12": { |
| "pass@1": 0.54, |
| "pass@2": 0.613, |
| "pass@4": 0.699, |
| "pass@8": 0.783, |
| "pass@16": 0.852, |
| "pass@32": 0.9, |
| "pass@64": 0.937, |
| "pass@128": 0.953 |
| }, |
| "13": { |
| "pass@1": 0.542, |
| "pass@2": 0.61, |
| "pass@4": 0.693, |
| "pass@8": 0.766, |
| "pass@16": 0.832, |
| "pass@32": 0.884, |
| "pass@64": 0.906, |
| "pass@128": 0.936 |
| }, |
| "14": { |
| "pass@1": 0.477, |
| "pass@2": 0.543, |
| "pass@4": 0.592, |
| "pass@8": 0.651, |
| "pass@16": 0.725, |
| "pass@32": 0.807, |
| "pass@64": 0.878, |
| "pass@128": 0.928 |
| }, |
| "15": { |
| "pass@1": 0.454, |
| "pass@2": 0.504, |
| "pass@4": 0.556, |
| "pass@8": 0.618, |
| "pass@16": 0.706, |
| "pass@32": 0.785, |
| "pass@64": 0.859, |
| "pass@128": 0.905 |
| }, |
| "16": { |
| "pass@1": 0.457, |
| "pass@2": 0.507, |
| "pass@4": 0.541, |
| "pass@8": 0.593, |
| "pass@16": 0.659, |
| "pass@32": 0.728, |
| "pass@64": 0.773, |
| "pass@128": 0.81 |
| }, |
| "17": { |
| "pass@1": 0.414, |
| "pass@2": 0.461, |
| "pass@4": 0.504, |
| "pass@8": 0.554, |
| "pass@16": 0.627, |
| "pass@32": 0.696, |
| "pass@64": 0.764, |
| "pass@128": 0.805 |
| }, |
| "18": { |
| "pass@1": 0.372, |
| "pass@2": 0.438, |
| "pass@4": 0.485, |
| "pass@8": 0.519, |
| "pass@16": 0.578, |
| "pass@32": 0.632, |
| "pass@64": 0.701, |
| "pass@128": 0.76 |
| }, |
| "19": { |
| "pass@1": 0.315, |
| "pass@2": 0.369, |
| "pass@4": 0.406, |
| "pass@8": 0.458, |
| "pass@16": 0.504, |
| "pass@32": 0.558, |
| "pass@64": 0.606, |
| "pass@128": 0.668 |
| }, |
| "20": { |
| "pass@1": 0.267, |
| "pass@2": 0.331, |
| "pass@4": 0.382, |
| "pass@8": 0.422, |
| "pass@16": 0.473, |
| "pass@32": 0.517, |
| "pass@64": 0.56, |
| "pass@128": 0.607 |
| } |
| }, |
| "per_op_pass_at_columns": { |
| "pass@1": { |
| "11": 0.555, |
| "12": 0.54, |
| "13": 0.542, |
| "14": 0.477, |
| "15": 0.454, |
| "16": 0.457, |
| "17": 0.414, |
| "18": 0.372, |
| "19": 0.315, |
| "20": 0.267 |
| }, |
| "pass@2": { |
| "11": 0.628, |
| "12": 0.613, |
| "13": 0.61, |
| "14": 0.543, |
| "15": 0.504, |
| "16": 0.507, |
| "17": 0.461, |
| "18": 0.438, |
| "19": 0.369, |
| "20": 0.331 |
| }, |
| "pass@4": { |
| "11": 0.711, |
| "12": 0.699, |
| "13": 0.693, |
| "14": 0.592, |
| "15": 0.556, |
| "16": 0.541, |
| "17": 0.504, |
| "18": 0.485, |
| "19": 0.406, |
| "20": 0.382 |
| }, |
| "pass@8": { |
| "11": 0.8, |
| "12": 0.783, |
| "13": 0.766, |
| "14": 0.651, |
| "15": 0.618, |
| "16": 0.593, |
| "17": 0.554, |
| "18": 0.519, |
| "19": 0.458, |
| "20": 0.422 |
| }, |
| "pass@16": { |
| "11": 0.865, |
| "12": 0.852, |
| "13": 0.832, |
| "14": 0.725, |
| "15": 0.706, |
| "16": 0.659, |
| "17": 0.627, |
| "18": 0.578, |
| "19": 0.504, |
| "20": 0.473 |
| }, |
| "pass@32": { |
| "11": 0.908, |
| "12": 0.9, |
| "13": 0.884, |
| "14": 0.807, |
| "15": 0.785, |
| "16": 0.728, |
| "17": 0.696, |
| "18": 0.632, |
| "19": 0.558, |
| "20": 0.517 |
| }, |
| "pass@64": { |
| "11": 0.938, |
| "12": 0.937, |
| "13": 0.906, |
| "14": 0.878, |
| "15": 0.859, |
| "16": 0.773, |
| "17": 0.764, |
| "18": 0.701, |
| "19": 0.606, |
| "20": 0.56 |
| }, |
| "pass@128": { |
| "11": 0.963, |
| "12": 0.953, |
| "13": 0.936, |
| "14": 0.928, |
| "15": 0.905, |
| "16": 0.81, |
| "17": 0.805, |
| "18": 0.76, |
| "19": 0.668, |
| "20": 0.607 |
| } |
| }, |
| "total_pass_at_k": { |
| "pass@1": 0.4393, |
| "pass@2": 0.5004, |
| "pass@4": 0.5569, |
| "pass@8": 0.6164, |
| "pass@16": 0.6821, |
| "pass@32": 0.7415, |
| "pass@64": 0.7922, |
| "pass@128": 0.8335 |
| } |
| }, |
| "ks": [ |
| 1, |
| 2, |
| 4, |
| 8, |
| 16, |
| 32, |
| 64, |
| 128 |
| ], |
| "prompt_counts": { |
| "per_template": { |
| "crazy_zootopia": { |
| "per_op": { |
| "11": 353, |
| "12": 354, |
| "13": 319, |
| "14": 321, |
| "15": 346, |
| "16": 310, |
| "17": 346, |
| "18": 350, |
| "19": 333, |
| "20": 333 |
| }, |
| "total": 3365 |
| }, |
| "movie_festival_awards": { |
| "per_op": { |
| "11": 315, |
| "12": 327, |
| "13": 324, |
| "14": 337, |
| "15": 324, |
| "16": 339, |
| "17": 332, |
| "18": 310, |
| "19": 331, |
| "20": 333 |
| }, |
| "total": 3272 |
| }, |
| "teachers_in_school": { |
| "per_op": { |
| "11": 332, |
| "12": 319, |
| "13": 357, |
| "14": 342, |
| "15": 330, |
| "16": 351, |
| "17": 322, |
| "18": 340, |
| "19": 336, |
| "20": 334 |
| }, |
| "total": 3363 |
| } |
| }, |
| "total": { |
| "per_op": { |
| "11": 1000, |
| "12": 1000, |
| "13": 1000, |
| "14": 1000, |
| "15": 1000, |
| "16": 1000, |
| "17": 1000, |
| "18": 1000, |
| "19": 1000, |
| "20": 1000 |
| }, |
| "total": 10000 |
| } |
| } |
| } |
| } |
| } |
|
|
