| [ |
| { |
| "checkpoint": "checkpoint-39008", |
| "id_acc": 0.9902135416666666, |
| "ood_acc": 0.87520703125, |
| "total_acc": 0.9296837993421052, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 200.16405208333333, |
| "ood_avg_resp_len": 381.9587625, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 750630, |
| "answer_accuracy": 0.9270149976288334, |
| "avg_response_len": 282.90025539440404, |
| "resp_tokens_sum": 229072258, |
| "pass_at_k": { |
| "pass@1": 0.9270149976288334, |
| "pass@2": 0.9493477619236584, |
| "pass@4": 0.96401552496224, |
| "pass@8": 0.9738049633109391, |
| "pass@16": 0.980471830554453, |
| "pass@32": 0.9851893630710853, |
| "pass@64": 0.9886710958308421, |
| "pass@128": 0.9914638001896934 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9435149336283186, |
| "pass@2": 0.954635492880868, |
| "pass@4": 0.9636101311451112, |
| "pass@8": 0.9714900775767249, |
| "pass@16": 0.9780677121400383, |
| "pass@32": 0.9842094464543852, |
| "pass@64": 0.9901773717362713, |
| "pass@128": 0.9941002949852508 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997371941896025, |
| "pass@2": 0.9999939801102844, |
| "pass@4": 0.9999999982800315, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9909474206349206, |
| "pass@2": 0.994433508311461, |
| "pass@4": 0.9964954672332624, |
| "pass@8": 0.9978625121315736, |
| "pass@16": 0.9989254195211111, |
| "pass@32": 0.9997055216891618, |
| "pass@64": 0.9999901718594374, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9992281062874252, |
| "pass@2": 0.9999447463341034, |
| "pass@4": 0.9999994695648073, |
| "pass@8": 0.9999999999730462, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.997133875739645, |
| "pass@2": 0.9989840801612077, |
| "pass@4": 0.9997141855973625, |
| "pass@8": 0.9999752317035955, |
| "pass@16": 0.9999998695305009, |
| "pass@32": 0.999999999999429, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9951923076923077, |
| "pass@2": 0.9986281199525542, |
| "pass@4": 0.9998137005710825, |
| "pass@8": 0.9999957663923499, |
| "pass@16": 0.9999999980003716, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.991637323943662, |
| "pass@2": 0.9929615587224134, |
| "pass@4": 0.9940380704172542, |
| "pass@8": 0.9954732852979431, |
| "pass@16": 0.9970755009921725, |
| "pass@32": 0.9986975516024429, |
| "pass@64": 0.9998376400088083, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9542315864022662, |
| "pass@2": 0.969464140327006, |
| "pass@4": 0.9810940441934841, |
| "pass@8": 0.9893951399204369, |
| "pass@16": 0.9945360284144443, |
| "pass@32": 0.9977323646041456, |
| "pass@64": 0.9996420458747602, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9427304025423728, |
| "pass@2": 0.9646079396325459, |
| "pass@4": 0.979820588898845, |
| "pass@8": 0.989370272818801, |
| "pass@16": 0.9949116559332181, |
| "pass@32": 0.9979107888230833, |
| "pass@64": 0.9992908315443338, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.914576802507837, |
| "pass@2": 0.9446635956359689, |
| "pass@4": 0.9665589881202152, |
| "pass@8": 0.979981660826714, |
| "pass@16": 0.9874232215331469, |
| "pass@32": 0.9921527270385655, |
| "pass@64": 0.996205173271306, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9100467289719626, |
| "pass@2": 0.9379875266759877, |
| "pass@4": 0.958589766816531, |
| "pass@8": 0.9731059061694559, |
| "pass@16": 0.9830554969430491, |
| "pass@32": 0.9893467916867982, |
| "pass@64": 0.992726298871782, |
| "pass@128": 0.9937694704049844 |
| }, |
| "15": { |
| "pass@1": 0.9557216401734104, |
| "pass@2": 0.9786739480451507, |
| "pass@4": 0.988442469889963, |
| "pass@8": 0.9914905405416587, |
| "pass@16": 0.992362600034752, |
| "pass@32": 0.9933207386637914, |
| "pass@64": 0.9949478858495289, |
| "pass@128": 0.9971098265895953 |
| }, |
| "16": { |
| "pass@1": 0.8498739919354839, |
| "pass@2": 0.8913004826009657, |
| "pass@4": 0.9222687491684507, |
| "pass@8": 0.9436176461985871, |
| "pass@16": 0.9585810148724633, |
| "pass@32": 0.9695494764061836, |
| "pass@64": 0.9774985888021867, |
| "pass@128": 0.9838709677419355 |
| }, |
| "17": { |
| "pass@1": 0.8969698338150289, |
| "pass@2": 0.9344089225570076, |
| "pass@4": 0.956744095300948, |
| "pass@8": 0.97040257318339, |
| "pass@16": 0.9791461578401852, |
| "pass@32": 0.9845951312505338, |
| "pass@64": 0.9879973643102052, |
| "pass@128": 0.9913294797687862 |
| }, |
| "18": { |
| "pass@1": 0.8265625, |
| "pass@2": 0.8718394966254224, |
| "pass@4": 0.9008116189940538, |
| "pass@8": 0.9218754716646158, |
| "pass@16": 0.9370514134261215, |
| "pass@32": 0.9485340476411356, |
| "pass@64": 0.9583897263711767, |
| "pass@128": 0.9685714285714285 |
| }, |
| "19": { |
| "pass@1": 0.7230902777777778, |
| "pass@2": 0.7983750236456933, |
| "pass@4": 0.8509800430351607, |
| "pass@8": 0.8887121617215229, |
| "pass@16": 0.9156199204084648, |
| "pass@32": 0.9346616186743513, |
| "pass@64": 0.9488236564767019, |
| "pass@128": 0.960960960960961 |
| }, |
| "20": { |
| "pass@1": 0.7268909534534534, |
| "pass@2": 0.8097002021706746, |
| "pass@4": 0.8596189023669333, |
| "pass@8": 0.8914002622308476, |
| "pass@16": 0.9137724953284576, |
| "pass@32": 0.9295212910338141, |
| "pass@64": 0.9403257272551829, |
| "pass@128": 0.948948948948949 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 766541, |
| "answer_accuracy": 0.9335310307872174, |
| "avg_response_len": 292.15439521628997, |
| "resp_tokens_sum": 239893817, |
| "pass_at_k": { |
| "pass@1": 0.9335310307872174, |
| "pass@2": 0.953042742465052, |
| "pass@4": 0.9665239741758695, |
| "pass@8": 0.9762235468442136, |
| "pass@16": 0.9828267648985132, |
| "pass@32": 0.9872906061783097, |
| "pass@64": 0.9907222191039753, |
| "pass@128": 0.9937646141855028 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9211320935582822, |
| "pass@2": 0.9365198994009952, |
| "pass@4": 0.9526557933134124, |
| "pass@8": 0.9657088477738169, |
| "pass@16": 0.9746686601446526, |
| "pass@32": 0.9816210687882091, |
| "pass@64": 0.9878415021526776, |
| "pass@128": 0.9938650306748467 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9968513257575757, |
| "pass@2": 0.9974964954664758, |
| "pass@4": 0.9979660738430424, |
| "pass@8": 0.9986534604223928, |
| "pass@16": 0.9994358706064412, |
| "pass@32": 0.9999202169903283, |
| "pass@64": 0.9999995805383678, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999537721893491, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9962635869565217, |
| "pass@2": 0.9972030839895015, |
| "pass@4": 0.9974499779918814, |
| "pass@8": 0.9977680587306922, |
| "pass@16": 0.9983124881128989, |
| "pass@32": 0.999097406302473, |
| "pass@64": 0.9998273650576287, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9998209169054442, |
| "pass@2": 0.9999901292782528, |
| "pass@4": 0.9999999811986252, |
| "pass@8": 0.999999999999998, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9987129696531792, |
| "pass@2": 0.9996746421419143, |
| "pass@4": 0.9999665655051787, |
| "pass@8": 0.9999997068622991, |
| "pass@16": 0.9999999999917892, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9898767605633803, |
| "pass@2": 0.9936876178329822, |
| "pass@4": 0.9957979374937288, |
| "pass@8": 0.9972295980502696, |
| "pass@16": 0.9982648879987673, |
| "pass@32": 0.9991210789903305, |
| "pass@64": 0.9998322279837888, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9966785179640718, |
| "pass@2": 0.9970903419538876, |
| "pass@4": 0.9971908938029453, |
| "pass@8": 0.997369925503324, |
| "pass@16": 0.9977102880852468, |
| "pass@32": 0.9983202885567448, |
| "pass@64": 0.9992573907303504, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.973621046686747, |
| "pass@2": 0.9814229852480792, |
| "pass@4": 0.9863388201286587, |
| "pass@8": 0.9898787999691451, |
| "pass@16": 0.993064840318199, |
| "pass@32": 0.9960242380201543, |
| "pass@64": 0.9987056869995018, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9421532131661442, |
| "pass@2": 0.9639698985510823, |
| "pass@4": 0.9794393557388399, |
| "pass@8": 0.9891352473786128, |
| "pass@16": 0.99374911404456, |
| "pass@32": 0.9954805067246403, |
| "pass@64": 0.9964824145461987, |
| "pass@128": 0.9968652037617555 |
| }, |
| "13": { |
| "pass@1": 0.9447435224089635, |
| "pass@2": 0.9655367068087075, |
| "pass@4": 0.9791388313855723, |
| "pass@8": 0.9881338330670343, |
| "pass@16": 0.9938460585401995, |
| "pass@32": 0.9969037090102838, |
| "pass@64": 0.9985099383695012, |
| "pass@128": 1.0 |
| }, |
| "14": { |
| "pass@1": 0.9357638888888888, |
| "pass@2": 0.9592812514389654, |
| "pass@4": 0.976130010064531, |
| "pass@8": 0.9864184088906494, |
| "pass@16": 0.9913667647399901, |
| "pass@32": 0.9940353616501455, |
| "pass@64": 0.9955747091064813, |
| "pass@128": 0.9970760233918129 |
| }, |
| "15": { |
| "pass@1": 0.9528645833333333, |
| "pass@2": 0.9747278394178001, |
| "pass@4": 0.9865421262683073, |
| "pass@8": 0.993495451097799, |
| "pass@16": 0.9974149003571077, |
| "pass@32": 0.99942427013389, |
| "pass@64": 0.999978739438185, |
| "pass@128": 1.0 |
| }, |
| "16": { |
| "pass@1": 0.8898459757834758, |
| "pass@2": 0.918867913946654, |
| "pass@4": 0.940958512130428, |
| "pass@8": 0.9576799933576514, |
| "pass@16": 0.9685457081346035, |
| "pass@32": 0.9757692082156921, |
| "pass@64": 0.9819115581804977, |
| "pass@128": 0.9886039886039886 |
| }, |
| "17": { |
| "pass@1": 0.8886354813664596, |
| "pass@2": 0.9250734368122461, |
| "pass@4": 0.9469407821499012, |
| "pass@8": 0.9609358541243732, |
| "pass@16": 0.9696919174554715, |
| "pass@32": 0.9768628206172025, |
| "pass@64": 0.9848878654881961, |
| "pass@128": 0.9937888198757764 |
| }, |
| "18": { |
| "pass@1": 0.865234375, |
| "pass@2": 0.9126758626679023, |
| "pass@4": 0.9434961421182645, |
| "pass@8": 0.9639815917003957, |
| "pass@16": 0.9758078148025395, |
| "pass@32": 0.9821697623332426, |
| "pass@64": 0.9862873674165625, |
| "pass@128": 0.9882352941176471 |
| }, |
| "19": { |
| "pass@1": 0.7456752232142857, |
| "pass@2": 0.8095109937429702, |
| "pass@4": 0.8518101247388715, |
| "pass@8": 0.8841118867969516, |
| "pass@16": 0.9092287569498064, |
| "pass@32": 0.9274672406486854, |
| "pass@64": 0.9428158675268603, |
| "pass@128": 0.9583333333333334 |
| }, |
| "20": { |
| "pass@1": 0.6919910179640718, |
| "pass@2": 0.7697962834174169, |
| "pass@4": 0.8282520268798735, |
| "pass@8": 0.8748525418555514, |
| "pass@16": 0.9103714017754003, |
| "pass@32": 0.9346016347947679, |
| "pass@64": 0.9506273862852723, |
| "pass@128": 0.9640718562874252 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 743820, |
| "answer_accuracy": 0.9284380492091389, |
| "avg_response_len": 312.7123554581403, |
| "resp_tokens_sum": 250530129, |
| "pass_at_k": { |
| "pass@1": 0.9284380492091389, |
| "pass@2": 0.9505996797682092, |
| "pass@4": 0.9653567733704562, |
| "pass@8": 0.9753979459646634, |
| "pass@16": 0.98229150543528, |
| "pass@32": 0.9872608599596581, |
| "pass@64": 0.9909840482108517, |
| "pass@128": 0.9936092027480428 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9478311567164179, |
| "pass@2": 0.9607738864731464, |
| "pass@4": 0.9701930362249498, |
| "pass@8": 0.9777165144485844, |
| "pass@16": 0.9850490284905359, |
| "pass@32": 0.9921181247105286, |
| "pass@64": 0.9976074742078478, |
| "pass@128": 1.0 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999772230320699, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9994745979532164, |
| "pass@2": 0.9999316491688539, |
| "pass@4": 0.9999986720409949, |
| "pass@8": 0.9999999997423706, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9941865808823529, |
| "pass@2": 0.9964205071792498, |
| "pass@4": 0.9969975253093362, |
| "pass@8": 0.9970582244883547, |
| "pass@16": 0.9970588234979403, |
| "pass@32": 0.9970588235294118, |
| "pass@64": 0.9970588235294118, |
| "pass@128": 0.9970588235294118 |
| }, |
| "6": { |
| "pass@1": 0.9979544558359621, |
| "pass@2": 0.9992621991355971, |
| "pass@4": 0.9998455858435678, |
| "pass@8": 0.9999935023705825, |
| "pass@16": 0.9999999931461893, |
| "pass@32": 0.9999999999999997, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9960195806962026, |
| "pass@2": 0.9982615961078442, |
| "pass@4": 0.9994898540174567, |
| "pass@8": 0.9999461399811752, |
| "pass@16": 0.9999994164199608, |
| "pass@32": 0.9999999999832518, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9913898273273273, |
| "pass@2": 0.9939999054172283, |
| "pass@4": 0.9949980785059523, |
| "pass@8": 0.9957539554064397, |
| "pass@16": 0.9965150597531441, |
| "pass@32": 0.9969397435790592, |
| "pass@64": 0.9969968106564769, |
| "pass@128": 0.996996996996997 |
| }, |
| "9": { |
| "pass@1": 0.9915846061093248, |
| "pass@2": 0.9950103014152974, |
| "pass@4": 0.9964494795867558, |
| "pass@8": 0.9972464958966785, |
| "pass@16": 0.9978505020760328, |
| "pass@32": 0.9986542070682007, |
| "pass@64": 0.999607565131529, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9640625, |
| "pass@2": 0.9797177696537934, |
| "pass@4": 0.9911180554811603, |
| "pass@8": 0.9973277243916092, |
| "pass@16": 0.9995487596571645, |
| "pass@32": 0.9999809674873567, |
| "pass@64": 0.99999999279412, |
| "pass@128": 1.0 |
| }, |
| "12": { |
| "pass@1": 0.9453363914373089, |
| "pass@2": 0.9667115148450481, |
| "pass@4": 0.9831256795308844, |
| "pass@8": 0.9930450861150011, |
| "pass@16": 0.9978774666448946, |
| "pass@32": 0.9996537102787295, |
| "pass@64": 0.999990478361063, |
| "pass@128": 1.0 |
| }, |
| "13": { |
| "pass@1": 0.9026572145061729, |
| "pass@2": 0.9364265122241666, |
| "pass@4": 0.9617531060932193, |
| "pass@8": 0.9778678586981546, |
| "pass@16": 0.9868936266865624, |
| "pass@32": 0.9917170187367842, |
| "pass@64": 0.9945546397415536, |
| "pass@128": 0.9969135802469136 |
| }, |
| "14": { |
| "pass@1": 0.9428319732937686, |
| "pass@2": 0.9654668479403722, |
| "pass@4": 0.9797224478987603, |
| "pass@8": 0.9890457880872057, |
| "pass@16": 0.994350926646944, |
| "pass@32": 0.9976499071092195, |
| "pass@64": 0.9996521834093685, |
| "pass@128": 1.0 |
| }, |
| "15": { |
| "pass@1": 0.9220196759259259, |
| "pass@2": 0.9517492496597652, |
| "pass@4": 0.9698473547288068, |
| "pass@8": 0.9799816784659696, |
| "pass@16": 0.9860164971157882, |
| "pass@32": 0.9894908053800399, |
| "pass@64": 0.9919161725258471, |
| "pass@128": 0.9938271604938271 |
| }, |
| "16": { |
| "pass@1": 0.8983222713864307, |
| "pass@2": 0.9338663972313197, |
| "pass@4": 0.9579245663982268, |
| "pass@8": 0.9727914706482771, |
| "pass@16": 0.9807054356298547, |
| "pass@32": 0.9851133092513668, |
| "pass@64": 0.9889024393806544, |
| "pass@128": 0.9941002949852508 |
| }, |
| "17": { |
| "pass@1": 0.8501035391566265, |
| "pass@2": 0.8853027760648893, |
| "pass@4": 0.9105946367020382, |
| "pass@8": 0.9327234486879625, |
| "pass@16": 0.9505718186178311, |
| "pass@32": 0.9637574221371371, |
| "pass@64": 0.9733407462659791, |
| "pass@128": 0.9819277108433735 |
| }, |
| "18": { |
| "pass@1": 0.8662550403225806, |
| "pass@2": 0.9099980949961902, |
| "pass@4": 0.9349534958331817, |
| "pass@8": 0.9524300861471281, |
| "pass@16": 0.9665201701946451, |
| "pass@32": 0.9776554085806844, |
| "pass@64": 0.9853613753348571, |
| "pass@128": 0.9903225806451613 |
| }, |
| "19": { |
| "pass@1": 0.7413378021148036, |
| "pass@2": 0.8133043509289437, |
| "pass@4": 0.8603601790146319, |
| "pass@8": 0.8925875144159812, |
| "pass@16": 0.9169766051361591, |
| "pass@32": 0.9370711730923081, |
| "pass@64": 0.9531285297469417, |
| "pass@128": 0.9637462235649547 |
| }, |
| "20": { |
| "pass@1": 0.6867961711711712, |
| "pass@2": 0.7743428714147219, |
| "pass@4": 0.8342244167677225, |
| "pass@8": 0.8771601288105502, |
| "pass@16": 0.9078914256137317, |
| "pass@32": 0.9314685436519544, |
| "pass@64": 0.9509367136317464, |
| "pass@128": 0.963963963963964 |
| } |
| } |
| } |
| } |
| } |
| ] |
