{ "files": { "checkpoint-39470_id_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "2": { "pass@1": 0.9190031152647975, "pass@2": 0.9470404984423676, "pass@4": 0.9750778816199377, "pass@8": 0.9875389408099688, "pass@16": 0.9968847352024922, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.8532110091743119, "pass@2": 0.908256880733945, "pass@4": 0.9510703363914373, "pass@8": 0.9847094801223242, "pass@16": 0.9877675840978594, "pass@32": 0.9908256880733946, "pass@64": 0.9938837920489296, "pass@128": 0.9969418960244648 }, "4": { "pass@1": 0.765625, "pass@2": 0.8625, "pass@4": 0.928125, "pass@8": 0.9625, "pass@16": 0.975, "pass@32": 0.978125, "pass@64": 0.99375, "pass@128": 0.996875 }, "5": { "pass@1": 0.7587301587301587, "pass@2": 0.8412698412698413, "pass@4": 0.9047619047619048, "pass@8": 0.946031746031746, "pass@16": 0.9619047619047619, "pass@32": 0.9746031746031746, "pass@64": 0.9809523809523809, "pass@128": 0.9904761904761905 }, "6": { "pass@1": 0.7994011976047904, "pass@2": 0.8682634730538922, "pass@4": 0.9161676646706587, "pass@8": 0.9431137724550899, "pass@16": 0.9700598802395209, "pass@32": 0.9850299401197605, "pass@64": 0.9880239520958084, "pass@128": 0.9910179640718563 }, "7": { "pass@1": 0.6686390532544378, "pass@2": 0.7662721893491125, "pass@4": 0.863905325443787, "pass@8": 0.9112426035502958, "pass@16": 0.9526627218934911, "pass@32": 0.9674556213017751, "pass@64": 0.9822485207100592, "pass@128": 0.9911242603550295 }, "8": { "pass@1": 0.6025641025641025, "pass@2": 0.7051282051282052, "pass@4": 0.8012820512820513, "pass@8": 0.8878205128205128, "pass@16": 0.9166666666666666, "pass@32": 0.9391025641025641, "pass@64": 0.9615384615384616, "pass@128": 0.9743589743589743 }, "9": { "pass@1": 0.5746478873239437, "pass@2": 0.6816901408450704, "pass@4": 0.7774647887323943, "pass@8": 0.8366197183098592, "pass@16": 0.8873239436619719, "pass@32": 0.9380281690140845, "pass@64": 0.9633802816901409, "pass@128": 0.9661971830985916 }, "10": { "pass@1": 0.471976401179941, "pass@2": 0.5545722713864307, "pass@4": 0.6548672566371682, "pass@8": 0.7404129793510325, "pass@16": 0.7935103244837758, "pass@32": 0.8436578171091446, "pass@64": 0.8761061946902655, "pass@128": 0.9026548672566371 } }, "per_op_pass_at_columns": { "pass@1": { "2": 0.9190031152647975, "3": 0.8532110091743119, "4": 0.765625, "5": 0.7587301587301587, "6": 0.7994011976047904, "7": 0.6686390532544378, "8": 0.6025641025641025, "9": 0.5746478873239437, "10": 0.471976401179941 }, "pass@2": { "2": 0.9470404984423676, "3": 0.908256880733945, "4": 0.8625, "5": 0.8412698412698413, "6": 0.8682634730538922, "7": 0.7662721893491125, "8": 0.7051282051282052, "9": 0.6816901408450704, "10": 0.5545722713864307 }, "pass@4": { "2": 0.9750778816199377, "3": 0.9510703363914373, "4": 0.928125, "5": 0.9047619047619048, "6": 0.9161676646706587, "7": 0.863905325443787, "8": 0.8012820512820513, "9": 0.7774647887323943, "10": 0.6548672566371682 }, "pass@8": { "2": 0.9875389408099688, "3": 0.9847094801223242, "4": 0.9625, "5": 0.946031746031746, "6": 0.9431137724550899, "7": 0.9112426035502958, "8": 0.8878205128205128, "9": 0.8366197183098592, "10": 0.7404129793510325 }, "pass@16": { "2": 0.9968847352024922, "3": 0.9877675840978594, "4": 0.975, "5": 0.9619047619047619, "6": 0.9700598802395209, "7": 0.9526627218934911, "8": 0.9166666666666666, "9": 0.8873239436619719, "10": 0.7935103244837758 }, "pass@32": { "2": 1.0, "3": 0.9908256880733946, "4": 0.978125, "5": 0.9746031746031746, "6": 0.9850299401197605, "7": 0.9674556213017751, "8": 0.9391025641025641, "9": 0.9380281690140845, "10": 0.8436578171091446 }, "pass@64": { "2": 1.0, "3": 0.9938837920489296, "4": 0.99375, "5": 0.9809523809523809, "6": 0.9880239520958084, "7": 0.9822485207100592, "8": 0.9615384615384616, "9": 0.9633802816901409, "10": 0.8761061946902655 }, "pass@128": { "2": 1.0, "3": 0.9969418960244648, "4": 0.996875, "5": 0.9904761904761905, "6": 0.9910179640718563, "7": 0.9911242603550295, "8": 0.9743589743589743, "9": 0.9661971830985916, "10": 0.9026548672566371 } }, "total_pass_at_k": { "pass@1": 0.7102330293819655, "pass@2": 0.7906112799729821, "pass@4": 0.861870989530564, "pass@8": 0.9094900371496116, "pass@16": 0.9368456602499156, "pass@32": 0.9567713610266801, "pass@64": 0.9706180344478217, "pass@128": 0.9783856805133401 } }, "movie_festival_awards": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9970845481049563, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9941520467836257, "pass@2": 0.9970760233918129, "pass@4": 0.9970760233918129, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9470588235294117, "pass@2": 0.9735294117647059, "pass@4": 0.9911764705882353, "pass@8": 0.9970588235294118, "pass@16": 0.9970588235294118, "pass@32": 0.9970588235294118, "pass@64": 0.9970588235294118, "pass@128": 0.9970588235294118 }, "6": { "pass@1": 0.9621451104100947, "pass@2": 0.9684542586750788, "pass@4": 0.9842271293375394, "pass@8": 0.9873817034700315, "pass@16": 0.9968454258675079, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8449367088607594, "pass@2": 0.9177215189873418, "pass@4": 0.9493670886075949, "pass@8": 0.9778481012658228, "pass@16": 0.990506329113924, "pass@32": 0.9936708860759493, "pass@64": 0.9968354430379747, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.8018018018018018, "pass@2": 0.8708708708708709, "pass@4": 0.933933933933934, "pass@8": 0.972972972972973, "pass@16": 0.984984984984985, "pass@32": 0.996996996996997, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.7684887459807074, "pass@2": 0.819935691318328, "pass@4": 0.887459807073955, "pass@8": 0.9485530546623794, "pass@16": 0.9839228295819936, "pass@32": 0.9903536977491961, "pass@64": 0.9967845659163987, "pass@128": 1.0 }, "10": { "pass@1": 0.6059701492537314, "pass@2": 0.6805970149253732, "pass@4": 0.7492537313432835, "pass@8": 0.8507462686567164, "pass@16": 0.9104477611940298, "pass@32": 0.9522388059701492, "pass@64": 0.9582089552238806, "pass@128": 0.9671641791044776 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 0.9970845481049563, "4": 0.9941520467836257, "5": 0.9470588235294117, "6": 0.9621451104100947, "7": 0.8449367088607594, "8": 0.8018018018018018, "9": 0.7684887459807074, "10": 0.6059701492537314 }, "pass@2": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.9735294117647059, "6": 0.9684542586750788, "7": 0.9177215189873418, "8": 0.8708708708708709, "9": 0.819935691318328, "10": 0.6805970149253732 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 0.9970760233918129, "5": 0.9911764705882353, "6": 0.9842271293375394, "7": 0.9493670886075949, "8": 0.933933933933934, "9": 0.887459807073955, "10": 0.7492537313432835 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 0.9873817034700315, "7": 0.9778481012658228, "8": 0.972972972972973, "9": 0.9485530546623794, "10": 0.8507462686567164 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 0.9968454258675079, "7": 0.990506329113924, "8": 0.984984984984985, "9": 0.9839228295819936, "10": 0.9104477611940298 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9936708860759493, "8": 0.996996996996997, "9": 0.9903536977491961, "10": 0.9522388059701492 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 1.0, "9": 0.9967845659163987, "10": 0.9582089552238806 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9970588235294118, "6": 1.0, "7": 0.9968354430379747, "8": 1.0, "9": 1.0, "10": 0.9671641791044776 } }, "total_pass_at_k": { "pass@1": 0.8821560093739538, "pass@2": 0.9156344158018078, "pass@4": 0.9444258453297623, "pass@8": 0.970873786407767, "pass@16": 0.9849347171074657, "pass@32": 0.9922999665215936, "pass@64": 0.9943086709072648, "pass@128": 0.995647807164379 } }, "teachers_in_school": { "per_op_pass_at_k": { "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9939393939393939, "pass@2": 0.9939393939393939, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9733727810650887, "pass@2": 0.9822485207100592, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9478260869565217, "pass@2": 0.9884057971014493, "pass@4": 0.9971014492753624, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9828080229226361, "pass@2": 0.9856733524355301, "pass@4": 0.9914040114613181, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.8728323699421965, "pass@2": 0.9219653179190751, "pass@4": 0.9710982658959537, "pass@8": 0.9884393063583815, "pass@16": 0.9942196531791907, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.8, "pass@2": 0.8760563380281691, "pass@4": 0.9267605633802817, "pass@8": 0.9661971830985916, "pass@16": 0.9859154929577465, "pass@32": 0.9943661971830986, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.718562874251497, "pass@2": 0.8053892215568862, "pass@4": 0.8682634730538922, "pass@8": 0.9311377245508982, "pass@16": 0.9670658682634731, "pass@32": 0.9850299401197605, "pass@64": 0.9970059880239521, "pass@128": 0.9970059880239521 }, "10": { "pass@1": 0.6595092024539877, "pass@2": 0.7208588957055214, "pass@4": 0.7852760736196319, "pass@8": 0.8404907975460123, "pass@16": 0.8926380368098159, "pass@32": 0.9202453987730062, "pass@64": 0.950920245398773, "pass@128": 0.9631901840490797 } }, "per_op_pass_at_columns": { "pass@1": { "2": 1.0, "3": 0.9939393939393939, "4": 0.9733727810650887, "5": 0.9478260869565217, "6": 0.9828080229226361, "7": 0.8728323699421965, "8": 0.8, "9": 0.718562874251497, "10": 0.6595092024539877 }, "pass@2": { "2": 1.0, "3": 0.9939393939393939, "4": 0.9822485207100592, "5": 0.9884057971014493, "6": 0.9856733524355301, "7": 0.9219653179190751, "8": 0.8760563380281691, "9": 0.8053892215568862, "10": 0.7208588957055214 }, "pass@4": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 0.9971014492753624, "6": 0.9914040114613181, "7": 0.9710982658959537, "8": 0.9267605633802817, "9": 0.8682634730538922, "10": 0.7852760736196319 }, "pass@8": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9884393063583815, "8": 0.9661971830985916, "9": 0.9311377245508982, "10": 0.8404907975460123 }, "pass@16": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 0.9942196531791907, "8": 0.9859154929577465, "9": 0.9670658682634731, "10": 0.8926380368098159 }, "pass@32": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 0.9943661971830986, "9": 0.9850299401197605, "10": 0.9202453987730062 }, "pass@64": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.950920245398773 }, "pass@128": { "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 0.9970059880239521, "10": 0.9631901840490797 } }, "total_pass_at_k": { "pass@1": 0.8836828309305373, "pass@2": 0.9200524246395806, "pass@4": 0.9495412844036697, "pass@8": 0.9701834862385321, "pass@16": 0.9826343381389253, "pass@32": 0.9891874180865007, "pass@64": 0.994429882044561, "pass@128": 0.995740498034076 } } }, "total": { "per_op_pass_at_k": { "2": { "pass@1": 0.974, "pass@2": 0.983, "pass@4": 0.992, "pass@8": 0.996, "pass@16": 0.999, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.949, "pass@2": 0.968, "pass@4": 0.984, "pass@8": 0.995, "pass@16": 0.996, "pass@32": 0.997, "pass@64": 0.998, "pass@128": 0.999 }, "4": { "pass@1": 0.914, "pass@2": 0.949, "pass@4": 0.976, "pass@8": 0.988, "pass@16": 0.992, "pass@32": 0.993, "pass@64": 0.998, "pass@128": 0.999 }, "5": { "pass@1": 0.888, "pass@2": 0.937, "pass@4": 0.966, "pass@8": 0.982, "pass@16": 0.987, "pass@32": 0.991, "pass@64": 0.993, "pass@128": 0.996 }, "6": { "pass@1": 0.915, "pass@2": 0.941, "pass@4": 0.964, "pass@8": 0.977, "pass@16": 0.989, "pass@32": 0.995, "pass@64": 0.996, "pass@128": 0.997 }, "7": { "pass@1": 0.795, "pass@2": 0.868, "pass@4": 0.928, "pass@8": 0.959, "pass@16": 0.979, "pass@32": 0.987, "pass@64": 0.993, "pass@128": 0.996 }, "8": { "pass@1": 0.739, "pass@2": 0.821, "pass@4": 0.89, "pass@8": 0.944, "pass@16": 0.964, "pass@32": 0.978, "pass@64": 0.988, "pass@128": 0.992 }, "9": { "pass@1": 0.683, "pass@2": 0.766, "pass@4": 0.842, "pass@8": 0.903, "pass@16": 0.944, "pass@32": 0.97, "pass@64": 0.985, "pass@128": 0.987 }, "10": { "pass@1": 0.578, "pass@2": 0.651, "pass@4": 0.729, "pass@8": 0.81, "pass@16": 0.865, "pass@32": 0.905, "pass@64": 0.928, "pass@128": 0.944 } }, "per_op_pass_at_columns": { "pass@1": { "2": 0.974, "3": 0.949, "4": 0.914, "5": 0.888, "6": 0.915, "7": 0.795, "8": 0.739, "9": 0.683, "10": 0.578 }, "pass@2": { "2": 0.983, "3": 0.968, "4": 0.949, "5": 0.937, "6": 0.941, "7": 0.868, "8": 0.821, "9": 0.766, "10": 0.651 }, "pass@4": { "2": 0.992, "3": 0.984, "4": 0.976, "5": 0.966, "6": 0.964, "7": 0.928, "8": 0.89, "9": 0.842, "10": 0.729 }, "pass@8": { "2": 0.996, "3": 0.995, "4": 0.988, "5": 0.982, "6": 0.977, "7": 0.959, "8": 0.944, "9": 0.903, "10": 0.81 }, "pass@16": { "2": 0.999, "3": 0.996, "4": 0.992, "5": 0.987, "6": 0.989, "7": 0.979, "8": 0.964, "9": 0.944, "10": 0.865 }, "pass@32": { "2": 1.0, "3": 0.997, "4": 0.993, "5": 0.991, "6": 0.995, "7": 0.987, "8": 0.978, "9": 0.97, "10": 0.905 }, "pass@64": { "2": 1.0, "3": 0.998, "4": 0.998, "5": 0.993, "6": 0.996, "7": 0.993, "8": 0.988, "9": 0.985, "10": 0.928 }, "pass@128": { "2": 1.0, "3": 0.999, "4": 0.999, "5": 0.996, "6": 0.997, "7": 0.996, "8": 0.992, "9": 0.987, "10": 0.944 } }, "total_pass_at_k": { "pass@1": 0.8261111111111111, "pass@2": 0.876, "pass@4": 0.919, "pass@8": 0.9504444444444444, "pass@16": 0.9683333333333334, "pass@32": 0.9795555555555555, "pass@64": 0.9865555555555555, "pass@128": 0.99 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "2": 321, "3": 327, "4": 320, "5": 315, "6": 334, "7": 338, "8": 312, "9": 355, "10": 339 }, "total": 2961 }, "movie_festival_awards": { "per_op": { "2": 350, "3": 343, "4": 342, "5": 340, "6": 317, "7": 316, "8": 333, "9": 311, "10": 335 }, "total": 2987 }, "teachers_in_school": { "per_op": { "2": 329, "3": 330, "4": 338, "5": 345, "6": 349, "7": 346, "8": 355, "9": 334, "10": 326 }, "total": 3052 } }, "total": { "per_op": { "2": 1000, "3": 1000, "4": 1000, "5": 1000, "6": 1000, "7": 1000, "8": 1000, "9": 1000, "10": 1000 }, "total": 9000 } } }, "checkpoint-39470_ood_generations.jsonl.graph_results.jsonl": { "per_template": { "crazy_zootopia": { "per_op_pass_at_k": { "11": { "pass@1": 0.42209631728045327, "pass@2": 0.5155807365439093, "pass@4": 0.5892351274787535, "pass@8": 0.6883852691218131, "pass@16": 0.773371104815864, "pass@32": 0.8215297450424929, "pass@64": 0.8725212464589235, "pass@128": 0.9178470254957507 }, "12": { "pass@1": 0.4519774011299435, "pass@2": 0.559322033898305, "pass@4": 0.632768361581921, "pass@8": 0.7288135593220338, "pass@16": 0.788135593220339, "pass@32": 0.844632768361582, "pass@64": 0.8870056497175142, "pass@128": 0.9124293785310734 }, "13": { "pass@1": 0.43260188087774293, "pass@2": 0.5172413793103449, "pass@4": 0.5862068965517241, "pass@8": 0.6520376175548589, "pass@16": 0.7210031347962382, "pass@32": 0.8025078369905956, "pass@64": 0.8369905956112853, "pass@128": 0.877742946708464 }, "14": { "pass@1": 0.38317757009345793, "pass@2": 0.4485981308411215, "pass@4": 0.48286604361370716, "pass@8": 0.5451713395638629, "pass@16": 0.632398753894081, "pass@32": 0.7071651090342679, "pass@64": 0.8006230529595015, "pass@128": 0.881619937694704 }, "15": { "pass@1": 0.3670520231213873, "pass@2": 0.4421965317919075, "pass@4": 0.49421965317919075, "pass@8": 0.5549132947976878, "pass@16": 0.638728323699422, "pass@32": 0.7254335260115607, "pass@64": 0.8034682080924855, "pass@128": 0.8670520231213873 }, "16": { "pass@1": 0.32903225806451614, "pass@2": 0.4032258064516129, "pass@4": 0.4483870967741935, "pass@8": 0.4967741935483871, "pass@16": 0.5870967741935483, "pass@32": 0.6580645161290323, "pass@64": 0.7032258064516129, "pass@128": 0.7354838709677419 }, "17": { "pass@1": 0.3468208092485549, "pass@2": 0.42485549132947975, "pass@4": 0.47398843930635837, "pass@8": 0.5317919075144508, "pass@16": 0.6040462427745664, "pass@32": 0.6647398843930635, "pass@64": 0.7254335260115607, "pass@128": 0.7716763005780347 }, "18": { "pass@1": 0.2857142857142857, "pass@2": 0.3657142857142857, "pass@4": 0.44, "pass@8": 0.4657142857142857, "pass@16": 0.5342857142857143, "pass@32": 0.58, "pass@64": 0.6485714285714286, "pass@128": 0.7028571428571428 }, "19": { "pass@1": 0.24324324324324326, "pass@2": 0.3063063063063063, "pass@4": 0.34534534534534533, "pass@8": 0.4084084084084084, "pass@16": 0.4594594594594595, "pass@32": 0.4984984984984985, "pass@64": 0.5375375375375375, "pass@128": 0.6006006006006006 }, "20": { "pass@1": 0.1891891891891892, "pass@2": 0.2672672672672673, "pass@4": 0.33933933933933935, "pass@8": 0.39039039039039036, "pass@16": 0.44744744744744747, "pass@32": 0.4984984984984985, "pass@64": 0.5465465465465466, "pass@128": 0.5915915915915916 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.42209631728045327, "12": 0.4519774011299435, "13": 0.43260188087774293, "14": 0.38317757009345793, "15": 0.3670520231213873, "16": 0.32903225806451614, "17": 0.3468208092485549, "18": 0.2857142857142857, "19": 0.24324324324324326, "20": 0.1891891891891892 }, "pass@2": { "11": 0.5155807365439093, "12": 0.559322033898305, "13": 0.5172413793103449, "14": 0.4485981308411215, "15": 0.4421965317919075, "16": 0.4032258064516129, "17": 0.42485549132947975, "18": 0.3657142857142857, "19": 0.3063063063063063, "20": 0.2672672672672673 }, "pass@4": { "11": 0.5892351274787535, "12": 0.632768361581921, "13": 0.5862068965517241, "14": 0.48286604361370716, "15": 0.49421965317919075, "16": 0.4483870967741935, "17": 0.47398843930635837, "18": 0.44, "19": 0.34534534534534533, "20": 0.33933933933933935 }, "pass@8": { "11": 0.6883852691218131, "12": 0.7288135593220338, "13": 0.6520376175548589, "14": 0.5451713395638629, "15": 0.5549132947976878, "16": 0.4967741935483871, "17": 0.5317919075144508, "18": 0.4657142857142857, "19": 0.4084084084084084, "20": 0.39039039039039036 }, "pass@16": { "11": 0.773371104815864, "12": 0.788135593220339, "13": 0.7210031347962382, "14": 0.632398753894081, "15": 0.638728323699422, "16": 0.5870967741935483, "17": 0.6040462427745664, "18": 0.5342857142857143, "19": 0.4594594594594595, "20": 0.44744744744744747 }, "pass@32": { "11": 0.8215297450424929, "12": 0.844632768361582, "13": 0.8025078369905956, "14": 0.7071651090342679, "15": 0.7254335260115607, "16": 0.6580645161290323, "17": 0.6647398843930635, "18": 0.58, "19": 0.4984984984984985, "20": 0.4984984984984985 }, "pass@64": { "11": 0.8725212464589235, "12": 0.8870056497175142, "13": 0.8369905956112853, "14": 0.8006230529595015, "15": 0.8034682080924855, "16": 0.7032258064516129, "17": 0.7254335260115607, "18": 0.6485714285714286, "19": 0.5375375375375375, "20": 0.5465465465465466 }, "pass@128": { "11": 0.9178470254957507, "12": 0.9124293785310734, "13": 0.877742946708464, "14": 0.881619937694704, "15": 0.8670520231213873, "16": 0.7354838709677419, "17": 0.7716763005780347, "18": 0.7028571428571428, "19": 0.6006006006006006, "20": 0.5915915915915916 } }, "total_pass_at_k": { "pass@1": 0.34561664190193164, "pass@2": 0.42585438335809805, "pass@4": 0.48439821693907875, "pass@8": 0.5476968796433879, "pass@16": 0.6199108469539376, "pass@32": 0.6811292719167905, "pass@64": 0.737295690936107, "pass@128": 0.7869242199108469 } }, "movie_festival_awards": { "per_op_pass_at_k": { "11": { "pass@1": 0.580952380952381, "pass@2": 0.6571428571428571, "pass@4": 0.7396825396825397, "pass@8": 0.8444444444444444, "pass@16": 0.9111111111111111, "pass@32": 0.9587301587301588, "pass@64": 0.9746031746031746, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.6207951070336392, "pass@2": 0.6850152905198776, "pass@4": 0.7614678899082569, "pass@8": 0.8379204892966361, "pass@16": 0.8990825688073395, "pass@32": 0.9388379204892966, "pass@64": 0.9724770642201835, "pass@128": 0.9755351681957186 }, "13": { "pass@1": 0.5648148148148148, "pass@2": 0.6234567901234568, "pass@4": 0.7191358024691358, "pass@8": 0.7932098765432098, "pass@16": 0.8703703703703703, "pass@32": 0.9074074074074074, "pass@64": 0.9259259259259259, "pass@128": 0.9567901234567902 }, "14": { "pass@1": 0.516320474777448, "pass@2": 0.5786350148367952, "pass@4": 0.6350148367952523, "pass@8": 0.6943620178041543, "pass@16": 0.7655786350148368, "pass@32": 0.8664688427299704, "pass@64": 0.9258160237388724, "pass@128": 0.9465875370919882 }, "15": { "pass@1": 0.4722222222222222, "pass@2": 0.5, "pass@4": 0.5555555555555556, "pass@8": 0.6203703703703703, "pass@16": 0.7283950617283951, "pass@32": 0.8055555555555556, "pass@64": 0.8672839506172839, "pass@128": 0.9074074074074074 }, "16": { "pass@1": 0.5162241887905604, "pass@2": 0.5634218289085545, "pass@4": 0.5958702064896755, "pass@8": 0.6460176991150443, "pass@16": 0.6991150442477876, "pass@32": 0.7610619469026548, "pass@64": 0.8082595870206489, "pass@128": 0.8377581120943953 }, "17": { "pass@1": 0.4819277108433735, "pass@2": 0.5090361445783133, "pass@4": 0.5421686746987951, "pass@8": 0.5873493975903614, "pass@16": 0.6536144578313253, "pass@32": 0.7198795180722891, "pass@64": 0.7801204819277109, "pass@128": 0.8132530120481928 }, "18": { "pass@1": 0.43548387096774194, "pass@2": 0.5, "pass@4": 0.5451612903225806, "pass@8": 0.5870967741935483, "pass@16": 0.635483870967742, "pass@32": 0.7, "pass@64": 0.7580645161290323, "pass@128": 0.8225806451612904 }, "19": { "pass@1": 0.3504531722054381, "pass@2": 0.40483383685800606, "pass@4": 0.43202416918429004, "pass@8": 0.486404833836858, "pass@16": 0.5287009063444109, "pass@32": 0.5740181268882175, "pass@64": 0.6193353474320241, "pass@128": 0.6888217522658611 }, "20": { "pass@1": 0.2912912912912913, "pass@2": 0.35735735735735735, "pass@4": 0.39039039039039036, "pass@8": 0.42342342342342343, "pass@16": 0.46846846846846846, "pass@32": 0.5135135135135135, "pass@64": 0.5615615615615616, "pass@128": 0.6306306306306306 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.580952380952381, "12": 0.6207951070336392, "13": 0.5648148148148148, "14": 0.516320474777448, "15": 0.4722222222222222, "16": 0.5162241887905604, "17": 0.4819277108433735, "18": 0.43548387096774194, "19": 0.3504531722054381, "20": 0.2912912912912913 }, "pass@2": { "11": 0.6571428571428571, "12": 0.6850152905198776, "13": 0.6234567901234568, "14": 0.5786350148367952, "15": 0.5, "16": 0.5634218289085545, "17": 0.5090361445783133, "18": 0.5, "19": 0.40483383685800606, "20": 0.35735735735735735 }, "pass@4": { "11": 0.7396825396825397, "12": 0.7614678899082569, "13": 0.7191358024691358, "14": 0.6350148367952523, "15": 0.5555555555555556, "16": 0.5958702064896755, "17": 0.5421686746987951, "18": 0.5451612903225806, "19": 0.43202416918429004, "20": 0.39039039039039036 }, "pass@8": { "11": 0.8444444444444444, "12": 0.8379204892966361, "13": 0.7932098765432098, "14": 0.6943620178041543, "15": 0.6203703703703703, "16": 0.6460176991150443, "17": 0.5873493975903614, "18": 0.5870967741935483, "19": 0.486404833836858, "20": 0.42342342342342343 }, "pass@16": { "11": 0.9111111111111111, "12": 0.8990825688073395, "13": 0.8703703703703703, "14": 0.7655786350148368, "15": 0.7283950617283951, "16": 0.6991150442477876, "17": 0.6536144578313253, "18": 0.635483870967742, "19": 0.5287009063444109, "20": 0.46846846846846846 }, "pass@32": { "11": 0.9587301587301588, "12": 0.9388379204892966, "13": 0.9074074074074074, "14": 0.8664688427299704, "15": 0.8055555555555556, "16": 0.7610619469026548, "17": 0.7198795180722891, "18": 0.7, "19": 0.5740181268882175, "20": 0.5135135135135135 }, "pass@64": { "11": 0.9746031746031746, "12": 0.9724770642201835, "13": 0.9259259259259259, "14": 0.9258160237388724, "15": 0.8672839506172839, "16": 0.8082595870206489, "17": 0.7801204819277109, "18": 0.7580645161290323, "19": 0.6193353474320241, "20": 0.5615615615615616 }, "pass@128": { "11": 0.9936507936507937, "12": 0.9755351681957186, "13": 0.9567901234567902, "14": 0.9465875370919882, "15": 0.9074074074074074, "16": 0.8377581120943953, "17": 0.8132530120481928, "18": 0.8225806451612904, "19": 0.6888217522658611, "20": 0.6306306306306306 } }, "total_pass_at_k": { "pass@1": 0.4825794621026895, "pass@2": 0.5372860635696821, "pass@4": 0.5907701711491442, "pass@8": 0.6509779951100244, "pass@16": 0.7148533007334963, "pass@32": 0.7735330073349633, "pass@64": 0.8184596577017115, "pass@128": 0.8563569682151589 } }, "teachers_in_school": { "per_op_pass_at_k": { "11": { "pass@1": 0.6716867469879518, "pass@2": 0.7198795180722891, "pass@4": 0.8132530120481928, "pass@8": 0.8765060240963856, "pass@16": 0.9186746987951807, "pass@32": 0.9518072289156626, "pass@64": 0.9728915662650602, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.554858934169279, "pass@2": 0.5987460815047022, "pass@4": 0.7084639498432602, "pass@8": 0.786833855799373, "pass@16": 0.8746081504702194, "pass@32": 0.9216300940438872, "pass@64": 0.9561128526645768, "pass@128": 0.9749216300940439 }, "13": { "pass@1": 0.6190476190476191, "pass@2": 0.680672268907563, "pass@4": 0.7647058823529411, "pass@8": 0.8431372549019608, "pass@16": 0.896358543417367, "pass@32": 0.9355742296918768, "pass@64": 0.9495798319327731, "pass@128": 0.969187675070028 }, "14": { "pass@1": 0.5263157894736842, "pass@2": 0.5964912280701754, "pass@4": 0.652046783625731, "pass@8": 0.7076023391812866, "pass@16": 0.7719298245614035, "pass@32": 0.8421052631578947, "pass@64": 0.9035087719298246, "pass@128": 0.9532163742690059 }, "15": { "pass@1": 0.5272727272727272, "pass@2": 0.5727272727272728, "pass@4": 0.6212121212121212, "pass@8": 0.6818181818181818, "pass@16": 0.7545454545454545, "pass@32": 0.8272727272727273, "pass@64": 0.9090909090909091, "pass@128": 0.9424242424242424 }, "16": { "pass@1": 0.5128205128205128, "pass@2": 0.5441595441595442, "pass@4": 0.5698005698005698, "pass@8": 0.6267806267806267, "pass@16": 0.6837606837606838, "pass@32": 0.7578347578347578, "pass@64": 0.8005698005698005, "pass@128": 0.8490028490028491 }, "17": { "pass@1": 0.4161490683229814, "pass@2": 0.4503105590062112, "pass@4": 0.4968944099378882, "pass@8": 0.5434782608695652, "pass@16": 0.6242236024844721, "pass@32": 0.7049689440993789, "pass@64": 0.7888198757763976, "pass@128": 0.8322981366459627 }, "18": { "pass@1": 0.40294117647058825, "pass@2": 0.45588235294117646, "pass@4": 0.4764705882352941, "pass@8": 0.5117647058823529, "pass@16": 0.5705882352941176, "pass@32": 0.6235294117647059, "pass@64": 0.7029411764705882, "pass@128": 0.7617647058823529 }, "19": { "pass@1": 0.35119047619047616, "pass@2": 0.3958333333333333, "pass@4": 0.44047619047619047, "pass@8": 0.4791666666666667, "pass@16": 0.5238095238095238, "pass@32": 0.6011904761904762, "pass@64": 0.6607142857142857, "pass@128": 0.7142857142857143 }, "20": { "pass@1": 0.3203592814371258, "pass@2": 0.36826347305389223, "pass@4": 0.4161676646706587, "pass@8": 0.45209580838323354, "pass@16": 0.5029940119760479, "pass@32": 0.5389221556886228, "pass@64": 0.5718562874251497, "pass@128": 0.5988023952095808 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.6716867469879518, "12": 0.554858934169279, "13": 0.6190476190476191, "14": 0.5263157894736842, "15": 0.5272727272727272, "16": 0.5128205128205128, "17": 0.4161490683229814, "18": 0.40294117647058825, "19": 0.35119047619047616, "20": 0.3203592814371258 }, "pass@2": { "11": 0.7198795180722891, "12": 0.5987460815047022, "13": 0.680672268907563, "14": 0.5964912280701754, "15": 0.5727272727272728, "16": 0.5441595441595442, "17": 0.4503105590062112, "18": 0.45588235294117646, "19": 0.3958333333333333, "20": 0.36826347305389223 }, "pass@4": { "11": 0.8132530120481928, "12": 0.7084639498432602, "13": 0.7647058823529411, "14": 0.652046783625731, "15": 0.6212121212121212, "16": 0.5698005698005698, "17": 0.4968944099378882, "18": 0.4764705882352941, "19": 0.44047619047619047, "20": 0.4161676646706587 }, "pass@8": { "11": 0.8765060240963856, "12": 0.786833855799373, "13": 0.8431372549019608, "14": 0.7076023391812866, "15": 0.6818181818181818, "16": 0.6267806267806267, "17": 0.5434782608695652, "18": 0.5117647058823529, "19": 0.4791666666666667, "20": 0.45209580838323354 }, "pass@16": { "11": 0.9186746987951807, "12": 0.8746081504702194, "13": 0.896358543417367, "14": 0.7719298245614035, "15": 0.7545454545454545, "16": 0.6837606837606838, "17": 0.6242236024844721, "18": 0.5705882352941176, "19": 0.5238095238095238, "20": 0.5029940119760479 }, "pass@32": { "11": 0.9518072289156626, "12": 0.9216300940438872, "13": 0.9355742296918768, "14": 0.8421052631578947, "15": 0.8272727272727273, "16": 0.7578347578347578, "17": 0.7049689440993789, "18": 0.6235294117647059, "19": 0.6011904761904762, "20": 0.5389221556886228 }, "pass@64": { "11": 0.9728915662650602, "12": 0.9561128526645768, "13": 0.9495798319327731, "14": 0.9035087719298246, "15": 0.9090909090909091, "16": 0.8005698005698005, "17": 0.7888198757763976, "18": 0.7029411764705882, "19": 0.6607142857142857, "20": 0.5718562874251497 }, "pass@128": { "11": 0.9819277108433735, "12": 0.9749216300940439, "13": 0.969187675070028, "14": 0.9532163742690059, "15": 0.9424242424242424, "16": 0.8490028490028491, "17": 0.8322981366459627, "18": 0.7617647058823529, "19": 0.7142857142857143, "20": 0.5988023952095808 } }, "total_pass_at_k": { "pass@1": 0.4909307166220636, "pass@2": 0.5391019922688076, "pass@4": 0.5964912280701754, "pass@8": 0.6515016354445435, "pass@16": 0.7124591138864109, "pass@32": 0.7707404103479036, "pass@64": 0.8215878679750223, "pass@128": 0.8578650014867678 } } }, "total": { "per_op_pass_at_k": { "11": { "pass@1": 0.555, "pass@2": 0.628, "pass@4": 0.711, "pass@8": 0.8, "pass@16": 0.865, "pass@32": 0.908, "pass@64": 0.938, "pass@128": 0.963 }, "12": { "pass@1": 0.54, "pass@2": 0.613, "pass@4": 0.699, "pass@8": 0.783, "pass@16": 0.852, "pass@32": 0.9, "pass@64": 0.937, "pass@128": 0.953 }, "13": { "pass@1": 0.542, "pass@2": 0.61, "pass@4": 0.693, "pass@8": 0.766, "pass@16": 0.832, "pass@32": 0.884, "pass@64": 0.906, "pass@128": 0.936 }, "14": { "pass@1": 0.477, "pass@2": 0.543, "pass@4": 0.592, "pass@8": 0.651, "pass@16": 0.725, "pass@32": 0.807, "pass@64": 0.878, "pass@128": 0.928 }, "15": { "pass@1": 0.454, "pass@2": 0.504, "pass@4": 0.556, "pass@8": 0.618, "pass@16": 0.706, "pass@32": 0.785, "pass@64": 0.859, "pass@128": 0.905 }, "16": { "pass@1": 0.457, "pass@2": 0.507, "pass@4": 0.541, "pass@8": 0.593, "pass@16": 0.659, "pass@32": 0.728, "pass@64": 0.773, "pass@128": 0.81 }, "17": { "pass@1": 0.414, "pass@2": 0.461, "pass@4": 0.504, "pass@8": 0.554, "pass@16": 0.627, "pass@32": 0.696, "pass@64": 0.764, "pass@128": 0.805 }, "18": { "pass@1": 0.372, "pass@2": 0.438, "pass@4": 0.485, "pass@8": 0.519, "pass@16": 0.578, "pass@32": 0.632, "pass@64": 0.701, "pass@128": 0.76 }, "19": { "pass@1": 0.315, "pass@2": 0.369, "pass@4": 0.406, "pass@8": 0.458, "pass@16": 0.504, "pass@32": 0.558, "pass@64": 0.606, "pass@128": 0.668 }, "20": { "pass@1": 0.267, "pass@2": 0.331, "pass@4": 0.382, "pass@8": 0.422, "pass@16": 0.473, "pass@32": 0.517, "pass@64": 0.56, "pass@128": 0.607 } }, "per_op_pass_at_columns": { "pass@1": { "11": 0.555, "12": 0.54, "13": 0.542, "14": 0.477, "15": 0.454, "16": 0.457, "17": 0.414, "18": 0.372, "19": 0.315, "20": 0.267 }, "pass@2": { "11": 0.628, "12": 0.613, "13": 0.61, "14": 0.543, "15": 0.504, "16": 0.507, "17": 0.461, "18": 0.438, "19": 0.369, "20": 0.331 }, "pass@4": { "11": 0.711, "12": 0.699, "13": 0.693, "14": 0.592, "15": 0.556, "16": 0.541, "17": 0.504, "18": 0.485, "19": 0.406, "20": 0.382 }, "pass@8": { "11": 0.8, "12": 0.783, "13": 0.766, "14": 0.651, "15": 0.618, "16": 0.593, "17": 0.554, "18": 0.519, "19": 0.458, "20": 0.422 }, "pass@16": { "11": 0.865, "12": 0.852, "13": 0.832, "14": 0.725, "15": 0.706, "16": 0.659, "17": 0.627, "18": 0.578, "19": 0.504, "20": 0.473 }, "pass@32": { "11": 0.908, "12": 0.9, "13": 0.884, "14": 0.807, "15": 0.785, "16": 0.728, "17": 0.696, "18": 0.632, "19": 0.558, "20": 0.517 }, "pass@64": { "11": 0.938, "12": 0.937, "13": 0.906, "14": 0.878, "15": 0.859, "16": 0.773, "17": 0.764, "18": 0.701, "19": 0.606, "20": 0.56 }, "pass@128": { "11": 0.963, "12": 0.953, "13": 0.936, "14": 0.928, "15": 0.905, "16": 0.81, "17": 0.805, "18": 0.76, "19": 0.668, "20": 0.607 } }, "total_pass_at_k": { "pass@1": 0.4393, "pass@2": 0.5004, "pass@4": 0.5569, "pass@8": 0.6164, "pass@16": 0.6821, "pass@32": 0.7415, "pass@64": 0.7922, "pass@128": 0.8335 } }, "ks": [ 1, 2, 4, 8, 16, 32, 64, 128 ], "prompt_counts": { "per_template": { "crazy_zootopia": { "per_op": { "11": 353, "12": 354, "13": 319, "14": 321, "15": 346, "16": 310, "17": 346, "18": 350, "19": 333, "20": 333 }, "total": 3365 }, "movie_festival_awards": { "per_op": { "11": 315, "12": 327, "13": 324, "14": 337, "15": 324, "16": 339, "17": 332, "18": 310, "19": 331, "20": 333 }, "total": 3272 }, "teachers_in_school": { "per_op": { "11": 332, "12": 319, "13": 357, "14": 342, "15": 330, "16": 351, "17": 322, "18": 340, "19": 336, "20": 334 }, "total": 3363 } }, "total": { "per_op": { "11": 1000, "12": 1000, "13": 1000, "14": 1000, "15": 1000, "16": 1000, "17": 1000, "18": 1000, "19": 1000, "20": 1000 }, "total": 10000 } } } } }