[ { "checkpoint": "checkpoint-21350", "id_acc": 0.9823862847222222, "ood_acc": 0.3994484375, "total_acc": 0.6755768914473684, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.05145833333333, "ood_avg_resp_len": 314.61243671875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 546547, "answer_accuracy": 0.674976041337338, "avg_response_len": 247.53832521538098, "resp_tokens_sum": 200438713, "pass_at_k": { "pass@1": 0.674976041337338, "pass@2": 0.7101091864658026, "pass@4": 0.7386124625565469, "pass@8": 0.762491115602194, "pass@16": 0.7830723815661093, "pass@32": 0.8011891747753505, "pass@64": 0.817504962251469, "pass@128": 0.8321214037306355 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9228890117994101, "pass@2": 0.9435989507119136, "pass@4": 0.9556432768912737, "pass@8": 0.9621737909507128, "pass@16": 0.9668383416955398, "pass@32": 0.9704871371383337, "pass@64": 0.972724070641377, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9974675076452599, "pass@2": 0.9980608430253559, "pass@4": 0.9987818108172257, "pass@8": 0.9995309324501443, "pass@16": 0.999937859513187, "pass@32": 0.9999993687767768, "pass@64": 0.9999999999981503, "pass@128": 1.0 }, "4": { "pass@1": 0.99677734375, "pass@2": 0.9968726931594489, "pass@4": 0.9968749997070677, "pass@8": 0.996875, "pass@16": 0.996875, "pass@32": 0.996875, "pass@64": 0.996875, "pass@128": 0.996875 }, "5": { "pass@1": 0.9848958333333333, "pass@2": 0.9886990688663916, "pass@4": 0.9909946988769263, "pass@8": 0.9925728488812268, "pass@16": 0.9940615103834403, "pass@32": 0.9956209222387189, "pass@64": 0.9966317339943471, "pass@128": 0.9968253968253968 }, "6": { "pass@1": 0.9984562125748503, "pass@2": 0.9996393777405818, "pass@4": 0.9999618897750056, "pass@8": 0.9999996288482736, "pass@16": 0.9999999999866336, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9819018121301775, "pass@2": 0.9881998963332246, "pass@4": 0.9908525150147651, "pass@8": 0.992921263269219, "pass@16": 0.9953543225386476, "pass@32": 0.9973493282786543, "pass@64": 0.9985121256129645, "pass@128": 1.0 }, "8": { "pass@1": 0.9764372996794872, "pass@2": 0.9853718705834849, "pass@4": 0.9896878509176735, "pass@8": 0.9922069966524123, "pass@16": 0.99332484085682, "pass@32": 0.9935781371491155, "pass@64": 0.9935897405457036, "pass@128": 0.9935897435897436 }, "9": { "pass@1": 0.9763204225352112, "pass@2": 0.9888221553731841, "pass@4": 0.9937658911474093, "pass@8": 0.9966332180416213, "pass@16": 0.9987499819877586, "pass@32": 0.9998218884821831, "pass@64": 0.9999980823812246, "pass@128": 1.0 }, "11": { "pass@1": 0.8874601628895185, "pass@2": 0.9190749007383285, "pass@4": 0.9379697790963095, "pass@8": 0.9499868038901691, "pass@16": 0.958601731222962, "pass@32": 0.9659637243092551, "pass@64": 0.9738305540462164, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.797801906779661, "pass@2": 0.8524762000088971, "pass@4": 0.8844551449077335, "pass@8": 0.9041495170093695, "pass@16": 0.9175524746444642, "pass@32": 0.9286310489660135, "pass@64": 0.9398662723540929, "pass@128": 0.9519774011299436 }, "13": { "pass@1": 0.6121424373040752, "pass@2": 0.6958881254165329, "pass@4": 0.7590125529959217, "pass@8": 0.8064926681604645, "pass@16": 0.8440489366042815, "pass@32": 0.8767117774857617, "pass@64": 0.9045546474803343, "pass@128": 0.9247648902821317 }, "14": { "pass@1": 0.4789476246105919, "pass@2": 0.5674093782961706, "pass@4": 0.6386814332203795, "pass@8": 0.699237479706666, "pass@16": 0.749983464432357, "pass@32": 0.7912858007134622, "pass@64": 0.8267369070020342, "pass@128": 0.8535825545171339 }, "15": { "pass@1": 0.3295926661849711, "pass@2": 0.4034149418551725, "pass@4": 0.4654226851520722, "pass@8": 0.5206955188961653, "pass@16": 0.5724889979926816, "pass@32": 0.6207619416448199, "pass@64": 0.6668436877333662, "pass@128": 0.7138728323699421 }, "16": { "pass@1": 0.2501260080645161, "pass@2": 0.3187746062992125, "pass@4": 0.3809942934552536, "pass@8": 0.43700423620641776, "pass@16": 0.48804601495912603, "pass@32": 0.5356938779931577, "pass@64": 0.5791105338215622, "pass@128": 0.6129032258064516 }, "17": { "pass@1": 0.17241690751445088, "pass@2": 0.2274190979017795, "pass@4": 0.27921864887915004, "pass@8": 0.32568698302405064, "pass@16": 0.3676062818519709, "pass@32": 0.4061846663090747, "pass@64": 0.44490948479031045, "pass@128": 0.4884393063583815 }, "18": { "pass@1": 0.17680803571428572, "pass@2": 0.2299905089988751, "pass@4": 0.2822858747120897, "pass@8": 0.33053922417649595, "pass@16": 0.3754431840988442, "pass@32": 0.4195090787833997, "pass@64": 0.45981741367360784, "pass@128": 0.49142857142857144 }, "19": { "pass@1": 0.15892454954954954, "pass@2": 0.21205235747558576, "pass@4": 0.2627885883633915, "pass@8": 0.30732869937392576, "pass@16": 0.34537864261948176, "pass@32": 0.37712504930713686, "pass@64": 0.40499634950234564, "pass@128": 0.43543543543543545 }, "20": { "pass@1": 0.16005067567567569, "pass@2": 0.2089211504811898, "pass@4": 0.2578512809772651, "pass@8": 0.30439842135885803, "pass@16": 0.34464130107609786, "pass@32": 0.37680522912423126, "pass@64": 0.4022415923768691, "pass@128": 0.42042042042042044 } } }, "teachers_in_school": { "count": 821120, "correct": 557409, "answer_accuracy": 0.6788398772408418, "avg_response_len": 256.3155385327358, "resp_tokens_sum": 210465815, "pass_at_k": { "pass@1": 0.6788398772408418, "pass@2": 0.7117769813920345, "pass@4": 0.7388926999284842, "pass@8": 0.7618370788125385, "pass@16": 0.7817088024003805, "pass@32": 0.7994336680502945, "pass@64": 0.8155949686648367, "pass@128": 0.8308651597817615 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9202214340490797, "pass@2": 0.9360723062895516, "pass@4": 0.9465917125581695, "pass@8": 0.9537278944651678, "pass@16": 0.959031497936665, "pass@32": 0.9641086020857509, "pass@64": 0.9686631103780736, "pass@128": 0.9723926380368099 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9985795454545454, "pass@2": 0.9997170275590551, "pass@4": 0.999981120257695, "pass@8": 0.9999999181688074, "pass@16": 0.9999999999995821, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9999768860946746, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.991304347826087, "pass@2": 0.9935520797672031, "pass@4": 0.994298946055656, "pass@8": 0.9945538934608948, "pass@16": 0.9948847424451219, "pass@32": 0.9954752938491385, "pass@64": 0.996382517402716, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9971570558739254, "pass@2": 0.9976052923989802, "pass@4": 0.9980087450565815, "pass@8": 0.9986333194949814, "pass@16": 0.999382080743133, "pass@32": 0.9998961597167226, "pass@64": 0.9999991244302526, "pass@128": 1.0 }, "7": { "pass@1": 0.9897940751445087, "pass@2": 0.9925530386646032, "pass@4": 0.9948359450010945, "pass@8": 0.997232438770044, "pass@16": 0.9991944808243975, "pass@32": 0.9999485653009884, "pass@64": 0.9999999670026358, "pass@128": 1.0 }, "8": { "pass@1": 0.9779709507042254, "pass@2": 0.9868096373516692, "pass@4": 0.9909287493992825, "pass@8": 0.9933035854016835, "pass@16": 0.9949393747728471, "pass@32": 0.9965682470179622, "pass@64": 0.9982458732971077, "pass@128": 1.0 }, "9": { "pass@1": 0.9639549026946108, "pass@2": 0.9735508068508654, "pass@4": 0.9791762661403849, "pass@8": 0.9836002268475236, "pass@16": 0.9872007390250478, "pass@32": 0.990516785177483, "pass@64": 0.9936008151224044, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.9093561746987951, "pass@2": 0.9346984394269996, "pass@4": 0.9480427477627043, "pass@8": 0.9566085144358756, "pass@16": 0.9621387470688224, "pass@32": 0.9660424121012906, "pass@64": 0.9682693163975254, "pass@128": 0.9698795180722891 }, "12": { "pass@1": 0.8195532915360502, "pass@2": 0.8715327487473156, "pass@4": 0.9028534996996849, "pass@8": 0.9224261190045239, "pass@16": 0.9357743493401, "pass@32": 0.9465884364452, "pass@64": 0.9558694712461404, "pass@128": 0.9655172413793104 }, "13": { "pass@1": 0.6233368347338936, "pass@2": 0.7012609177529281, "pass@4": 0.752510632074352, "pass@8": 0.7909648856753403, "pass@16": 0.825167615439675, "pass@32": 0.856456023376782, "pass@64": 0.8833283014284938, "pass@128": 0.9019607843137255 }, "14": { "pass@1": 0.484078033625731, "pass@2": 0.5705107174103236, "pass@4": 0.6397225598993105, "pass@8": 0.6942476151472764, "pass@16": 0.735376785198006, "pass@32": 0.7658550027742763, "pass@64": 0.7913686429763481, "pass@128": 0.8128654970760234 }, "15": { "pass@1": 0.332717803030303, "pass@2": 0.4114877863278452, "pass@4": 0.4812904997670743, "pass@8": 0.5437209905176837, "pass@16": 0.5981587135360744, "pass@32": 0.6433320324324602, "pass@64": 0.6817106148445318, "pass@128": 0.7181818181818181 }, "16": { "pass@1": 0.22081997863247863, "pass@2": 0.2871509550889474, "pass@4": 0.3512183749467215, "pass@8": 0.4076236800542498, "pass@16": 0.4546544514041406, "pass@32": 0.4964979651009504, "pass@64": 0.5380652864044299, "pass@128": 0.5811965811965812 }, "17": { "pass@1": 0.18730590062111802, "pass@2": 0.23642870897931226, "pass@4": 0.2845957679203141, "pass@8": 0.3326317237752575, "pass@16": 0.38087277944396986, "pass@32": 0.42657265186991555, "pass@64": 0.46936251256103506, "pass@128": 0.5186335403726708 }, "18": { "pass@1": 0.18276654411764706, "pass@2": 0.2352152327466419, "pass@4": 0.2849862334303801, "pass@8": 0.3288812893596294, "pass@16": 0.3685753443013062, "pass@32": 0.40794190067953773, "pass@64": 0.4466121363445853, "pass@128": 0.4823529411764706 }, "19": { "pass@1": 0.14267113095238096, "pass@2": 0.19041760404949387, "pass@4": 0.2374986904538717, "pass@8": 0.28168255992725094, "pass@16": 0.3226338126666716, "pass@32": 0.3610534610549758, "pass@64": 0.39768084772182344, "pass@128": 0.43452380952380953 }, "20": { "pass@1": 0.14221556886227546, "pass@2": 0.19062441062756372, "pass@4": 0.2377481939944634, "pass@8": 0.280558264608587, "pass@16": 0.32031383167398947, "pass@32": 0.35856987955107245, "pass@64": 0.3935714032182891, "pass@128": 0.4221556886227545 } } }, "movie_festival_awards": { "count": 801152, "correct": 539047, "answer_accuracy": 0.672839860600735, "avg_response_len": 273.110060263221, "resp_tokens_sum": 218802671, "pass_at_k": { "pass@1": 0.672839860600735, "pass@2": 0.7047058338354943, "pass@4": 0.7315443418620015, "pass@8": 0.7548757040045201, "pass@16": 0.7751949735171257, "pass@32": 0.7929837040886027, "pass@64": 0.8085162439976868, "pass@128": 0.8223358363955904 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9311100746268657, "pass@2": 0.9520261634739686, "pass@4": 0.9662647566442252, "pass@8": 0.9755282876597258, "pass@16": 0.9810774588415156, "pass@32": 0.9848828094725318, "pass@64": 0.9880498176506736, "pass@128": 0.991044776119403 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9997494533527697, "pass@2": 0.9999838588416243, "pass@4": 0.9999999426092147, "pass@8": 0.9999999999999082, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9996801900584795, "pass@2": 0.9999672635492931, "pass@4": 0.9999997256373656, "pass@8": 0.9999999999938582, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9776194852941177, "pass@2": 0.9819042380731822, "pass@4": 0.9844617613974724, "pass@8": 0.9863978237239305, "pass@16": 0.9878696643403634, "pass@32": 0.988862169457123, "pass@64": 0.9897049835750625, "pass@128": 0.9911764705882353 }, "6": { "pass@1": 0.9965496845425867, "pass@2": 0.9971636776869769, "pass@4": 0.9974878323569174, "pass@8": 0.99801474353403, "pass@16": 0.9987909801124433, "pass@32": 0.9996022252317682, "pass@64": 0.9999792683835579, "pass@128": 1.0 }, "7": { "pass@1": 0.9878362341772152, "pass@2": 0.9902992001395396, "pass@4": 0.9915792477087515, "pass@8": 0.9928315670329757, "pass@16": 0.9943870274755153, "pass@32": 0.9960010268780787, "pass@64": 0.9967819001166339, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.9844219219219219, "pass@2": 0.9916711741268824, "pass@4": 0.9939474885459137, "pass@8": 0.9951080518725748, "pass@16": 0.9959944462170828, "pass@32": 0.9967184753879357, "pass@64": 0.9969877001072768, "pass@128": 0.996996996996997 }, "9": { "pass@1": 0.9726688102893891, "pass@2": 0.9816675158872827, "pass@4": 0.9860481763533578, "pass@8": 0.9891498765884253, "pass@16": 0.9913841449354583, "pass@32": 0.9929023262538196, "pass@64": 0.9935237000858083, "pass@128": 0.9935691318327974 }, "11": { "pass@1": 0.893625992063492, "pass@2": 0.923719691288589, "pass@4": 0.9415772218948818, "pass@8": 0.9541297662181599, "pass@16": 0.9648180622646247, "pass@32": 0.9740996623349433, "pass@64": 0.9801861811374231, "pass@128": 0.9841269841269841 }, "12": { "pass@1": 0.8082473241590215, "pass@2": 0.8621445255122921, "pass@4": 0.8994472826791146, "pass@8": 0.9270114252522809, "pass@16": 0.9465701531750612, "pass@32": 0.9611674288837075, "pass@64": 0.9732524253679298, "pass@128": 0.9847094801223242 }, "13": { "pass@1": 0.5974874614197531, "pass@2": 0.6732845460289684, "pass@4": 0.7297426236072337, "pass@8": 0.7771452699041174, "pass@16": 0.8178942459211255, "pass@32": 0.8521100811585546, "pass@64": 0.879897410083746, "pass@128": 0.8981481481481481 }, "14": { "pass@1": 0.45252225519287836, "pass@2": 0.5335529159559803, "pass@4": 0.6004611306005145, "pass@8": 0.6562780500283442, "pass@16": 0.7023598202623946, "pass@32": 0.7390769682434493, "pass@64": 0.7694374359811129, "pass@128": 0.798219584569733 }, "15": { "pass@1": 0.29366801697530864, "pass@2": 0.3638971669339944, "pass@4": 0.42763749178343446, "pass@8": 0.48427563508057453, "pass@16": 0.534451744441426, "pass@32": 0.5792335638578042, "pass@64": 0.6187090353473133, "pass@128": 0.654320987654321 }, "16": { "pass@1": 0.20404682890855458, "pass@2": 0.253627795972406, "pass@4": 0.30170414815404706, "pass@8": 0.34745819229398955, "pass@16": 0.3904366019786063, "pass@32": 0.43057834381205445, "pass@64": 0.4704313191110292, "pass@128": 0.5162241887905604 }, "17": { "pass@1": 0.18211125753012047, "pass@2": 0.23127216049236302, "pass@4": 0.27671165821892746, "pass@8": 0.3184606778754773, "pass@16": 0.3563930038046845, "pass@32": 0.3931867515264538, "pass@64": 0.43110471739026307, "pass@128": 0.46987951807228917 }, "18": { "pass@1": 0.18402217741935484, "pass@2": 0.24017811785623552, "pass@4": 0.29430360469054273, "pass@8": 0.3408792289278583, "pass@16": 0.3782742791909101, "pass@32": 0.41051421704477314, "pass@64": 0.43918690774918834, "pass@128": 0.4612903225806452 }, "19": { "pass@1": 0.15525868580060423, "pass@2": 0.20456532340557113, "pass@4": 0.2538143409943849, "pass@8": 0.301296925041148, "pass@16": 0.3447051650683001, "pass@32": 0.3817948403445567, "pass@64": 0.41078464647666285, "pass@128": 0.43202416918429004 }, "20": { "pass@1": 0.14630255255255256, "pass@2": 0.19446882906528568, "pass@4": 0.24227324146418622, "pass@8": 0.2883451930018625, "pass@16": 0.3340379341883104, "pass@32": 0.37754065399385844, "pass@64": 0.41581311513220076, "pass@128": 0.44744744744744747 } } } } } ]