| [ |
| { |
| "checkpoint": "checkpoint-21350", |
| "id_acc": 0.9823862847222222, |
| "ood_acc": 0.3994484375, |
| "total_acc": 0.6755768914473684, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.05145833333333, |
| "ood_avg_resp_len": 314.61243671875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 546547, |
| "answer_accuracy": 0.674976041337338, |
| "avg_response_len": 247.53832521538098, |
| "resp_tokens_sum": 200438713, |
| "pass_at_k": { |
| "pass@1": 0.674976041337338, |
| "pass@2": 0.7101091864658026, |
| "pass@4": 0.7386124625565469, |
| "pass@8": 0.762491115602194, |
| "pass@16": 0.7830723815661093, |
| "pass@32": 0.8011891747753505, |
| "pass@64": 0.817504962251469, |
| "pass@128": 0.8321214037306355 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9228890117994101, |
| "pass@2": 0.9435989507119136, |
| "pass@4": 0.9556432768912737, |
| "pass@8": 0.9621737909507128, |
| "pass@16": 0.9668383416955398, |
| "pass@32": 0.9704871371383337, |
| "pass@64": 0.972724070641377, |
| "pass@128": 0.9734513274336283 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9974675076452599, |
| "pass@2": 0.9980608430253559, |
| "pass@4": 0.9987818108172257, |
| "pass@8": 0.9995309324501443, |
| "pass@16": 0.999937859513187, |
| "pass@32": 0.9999993687767768, |
| "pass@64": 0.9999999999981503, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.99677734375, |
| "pass@2": 0.9968726931594489, |
| "pass@4": 0.9968749997070677, |
| "pass@8": 0.996875, |
| "pass@16": 0.996875, |
| "pass@32": 0.996875, |
| "pass@64": 0.996875, |
| "pass@128": 0.996875 |
| }, |
| "5": { |
| "pass@1": 0.9848958333333333, |
| "pass@2": 0.9886990688663916, |
| "pass@4": 0.9909946988769263, |
| "pass@8": 0.9925728488812268, |
| "pass@16": 0.9940615103834403, |
| "pass@32": 0.9956209222387189, |
| "pass@64": 0.9966317339943471, |
| "pass@128": 0.9968253968253968 |
| }, |
| "6": { |
| "pass@1": 0.9984562125748503, |
| "pass@2": 0.9996393777405818, |
| "pass@4": 0.9999618897750056, |
| "pass@8": 0.9999996288482736, |
| "pass@16": 0.9999999999866336, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9819018121301775, |
| "pass@2": 0.9881998963332246, |
| "pass@4": 0.9908525150147651, |
| "pass@8": 0.992921263269219, |
| "pass@16": 0.9953543225386476, |
| "pass@32": 0.9973493282786543, |
| "pass@64": 0.9985121256129645, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9764372996794872, |
| "pass@2": 0.9853718705834849, |
| "pass@4": 0.9896878509176735, |
| "pass@8": 0.9922069966524123, |
| "pass@16": 0.99332484085682, |
| "pass@32": 0.9935781371491155, |
| "pass@64": 0.9935897405457036, |
| "pass@128": 0.9935897435897436 |
| }, |
| "9": { |
| "pass@1": 0.9763204225352112, |
| "pass@2": 0.9888221553731841, |
| "pass@4": 0.9937658911474093, |
| "pass@8": 0.9966332180416213, |
| "pass@16": 0.9987499819877586, |
| "pass@32": 0.9998218884821831, |
| "pass@64": 0.9999980823812246, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.8874601628895185, |
| "pass@2": 0.9190749007383285, |
| "pass@4": 0.9379697790963095, |
| "pass@8": 0.9499868038901691, |
| "pass@16": 0.958601731222962, |
| "pass@32": 0.9659637243092551, |
| "pass@64": 0.9738305540462164, |
| "pass@128": 0.9830028328611898 |
| }, |
| "12": { |
| "pass@1": 0.797801906779661, |
| "pass@2": 0.8524762000088971, |
| "pass@4": 0.8844551449077335, |
| "pass@8": 0.9041495170093695, |
| "pass@16": 0.9175524746444642, |
| "pass@32": 0.9286310489660135, |
| "pass@64": 0.9398662723540929, |
| "pass@128": 0.9519774011299436 |
| }, |
| "13": { |
| "pass@1": 0.6121424373040752, |
| "pass@2": 0.6958881254165329, |
| "pass@4": 0.7590125529959217, |
| "pass@8": 0.8064926681604645, |
| "pass@16": 0.8440489366042815, |
| "pass@32": 0.8767117774857617, |
| "pass@64": 0.9045546474803343, |
| "pass@128": 0.9247648902821317 |
| }, |
| "14": { |
| "pass@1": 0.4789476246105919, |
| "pass@2": 0.5674093782961706, |
| "pass@4": 0.6386814332203795, |
| "pass@8": 0.699237479706666, |
| "pass@16": 0.749983464432357, |
| "pass@32": 0.7912858007134622, |
| "pass@64": 0.8267369070020342, |
| "pass@128": 0.8535825545171339 |
| }, |
| "15": { |
| "pass@1": 0.3295926661849711, |
| "pass@2": 0.4034149418551725, |
| "pass@4": 0.4654226851520722, |
| "pass@8": 0.5206955188961653, |
| "pass@16": 0.5724889979926816, |
| "pass@32": 0.6207619416448199, |
| "pass@64": 0.6668436877333662, |
| "pass@128": 0.7138728323699421 |
| }, |
| "16": { |
| "pass@1": 0.2501260080645161, |
| "pass@2": 0.3187746062992125, |
| "pass@4": 0.3809942934552536, |
| "pass@8": 0.43700423620641776, |
| "pass@16": 0.48804601495912603, |
| "pass@32": 0.5356938779931577, |
| "pass@64": 0.5791105338215622, |
| "pass@128": 0.6129032258064516 |
| }, |
| "17": { |
| "pass@1": 0.17241690751445088, |
| "pass@2": 0.2274190979017795, |
| "pass@4": 0.27921864887915004, |
| "pass@8": 0.32568698302405064, |
| "pass@16": 0.3676062818519709, |
| "pass@32": 0.4061846663090747, |
| "pass@64": 0.44490948479031045, |
| "pass@128": 0.4884393063583815 |
| }, |
| "18": { |
| "pass@1": 0.17680803571428572, |
| "pass@2": 0.2299905089988751, |
| "pass@4": 0.2822858747120897, |
| "pass@8": 0.33053922417649595, |
| "pass@16": 0.3754431840988442, |
| "pass@32": 0.4195090787833997, |
| "pass@64": 0.45981741367360784, |
| "pass@128": 0.49142857142857144 |
| }, |
| "19": { |
| "pass@1": 0.15892454954954954, |
| "pass@2": 0.21205235747558576, |
| "pass@4": 0.2627885883633915, |
| "pass@8": 0.30732869937392576, |
| "pass@16": 0.34537864261948176, |
| "pass@32": 0.37712504930713686, |
| "pass@64": 0.40499634950234564, |
| "pass@128": 0.43543543543543545 |
| }, |
| "20": { |
| "pass@1": 0.16005067567567569, |
| "pass@2": 0.2089211504811898, |
| "pass@4": 0.2578512809772651, |
| "pass@8": 0.30439842135885803, |
| "pass@16": 0.34464130107609786, |
| "pass@32": 0.37680522912423126, |
| "pass@64": 0.4022415923768691, |
| "pass@128": 0.42042042042042044 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 557409, |
| "answer_accuracy": 0.6788398772408418, |
| "avg_response_len": 256.3155385327358, |
| "resp_tokens_sum": 210465815, |
| "pass_at_k": { |
| "pass@1": 0.6788398772408418, |
| "pass@2": 0.7117769813920345, |
| "pass@4": 0.7388926999284842, |
| "pass@8": 0.7618370788125385, |
| "pass@16": 0.7817088024003805, |
| "pass@32": 0.7994336680502945, |
| "pass@64": 0.8155949686648367, |
| "pass@128": 0.8308651597817615 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9202214340490797, |
| "pass@2": 0.9360723062895516, |
| "pass@4": 0.9465917125581695, |
| "pass@8": 0.9537278944651678, |
| "pass@16": 0.959031497936665, |
| "pass@32": 0.9641086020857509, |
| "pass@64": 0.9686631103780736, |
| "pass@128": 0.9723926380368099 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9985795454545454, |
| "pass@2": 0.9997170275590551, |
| "pass@4": 0.999981120257695, |
| "pass@8": 0.9999999181688074, |
| "pass@16": 0.9999999999995821, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9999768860946746, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.991304347826087, |
| "pass@2": 0.9935520797672031, |
| "pass@4": 0.994298946055656, |
| "pass@8": 0.9945538934608948, |
| "pass@16": 0.9948847424451219, |
| "pass@32": 0.9954752938491385, |
| "pass@64": 0.996382517402716, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9971570558739254, |
| "pass@2": 0.9976052923989802, |
| "pass@4": 0.9980087450565815, |
| "pass@8": 0.9986333194949814, |
| "pass@16": 0.999382080743133, |
| "pass@32": 0.9998961597167226, |
| "pass@64": 0.9999991244302526, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9897940751445087, |
| "pass@2": 0.9925530386646032, |
| "pass@4": 0.9948359450010945, |
| "pass@8": 0.997232438770044, |
| "pass@16": 0.9991944808243975, |
| "pass@32": 0.9999485653009884, |
| "pass@64": 0.9999999670026358, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9779709507042254, |
| "pass@2": 0.9868096373516692, |
| "pass@4": 0.9909287493992825, |
| "pass@8": 0.9933035854016835, |
| "pass@16": 0.9949393747728471, |
| "pass@32": 0.9965682470179622, |
| "pass@64": 0.9982458732971077, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9639549026946108, |
| "pass@2": 0.9735508068508654, |
| "pass@4": 0.9791762661403849, |
| "pass@8": 0.9836002268475236, |
| "pass@16": 0.9872007390250478, |
| "pass@32": 0.990516785177483, |
| "pass@64": 0.9936008151224044, |
| "pass@128": 0.9970059880239521 |
| }, |
| "11": { |
| "pass@1": 0.9093561746987951, |
| "pass@2": 0.9346984394269996, |
| "pass@4": 0.9480427477627043, |
| "pass@8": 0.9566085144358756, |
| "pass@16": 0.9621387470688224, |
| "pass@32": 0.9660424121012906, |
| "pass@64": 0.9682693163975254, |
| "pass@128": 0.9698795180722891 |
| }, |
| "12": { |
| "pass@1": 0.8195532915360502, |
| "pass@2": 0.8715327487473156, |
| "pass@4": 0.9028534996996849, |
| "pass@8": 0.9224261190045239, |
| "pass@16": 0.9357743493401, |
| "pass@32": 0.9465884364452, |
| "pass@64": 0.9558694712461404, |
| "pass@128": 0.9655172413793104 |
| }, |
| "13": { |
| "pass@1": 0.6233368347338936, |
| "pass@2": 0.7012609177529281, |
| "pass@4": 0.752510632074352, |
| "pass@8": 0.7909648856753403, |
| "pass@16": 0.825167615439675, |
| "pass@32": 0.856456023376782, |
| "pass@64": 0.8833283014284938, |
| "pass@128": 0.9019607843137255 |
| }, |
| "14": { |
| "pass@1": 0.484078033625731, |
| "pass@2": 0.5705107174103236, |
| "pass@4": 0.6397225598993105, |
| "pass@8": 0.6942476151472764, |
| "pass@16": 0.735376785198006, |
| "pass@32": 0.7658550027742763, |
| "pass@64": 0.7913686429763481, |
| "pass@128": 0.8128654970760234 |
| }, |
| "15": { |
| "pass@1": 0.332717803030303, |
| "pass@2": 0.4114877863278452, |
| "pass@4": 0.4812904997670743, |
| "pass@8": 0.5437209905176837, |
| "pass@16": 0.5981587135360744, |
| "pass@32": 0.6433320324324602, |
| "pass@64": 0.6817106148445318, |
| "pass@128": 0.7181818181818181 |
| }, |
| "16": { |
| "pass@1": 0.22081997863247863, |
| "pass@2": 0.2871509550889474, |
| "pass@4": 0.3512183749467215, |
| "pass@8": 0.4076236800542498, |
| "pass@16": 0.4546544514041406, |
| "pass@32": 0.4964979651009504, |
| "pass@64": 0.5380652864044299, |
| "pass@128": 0.5811965811965812 |
| }, |
| "17": { |
| "pass@1": 0.18730590062111802, |
| "pass@2": 0.23642870897931226, |
| "pass@4": 0.2845957679203141, |
| "pass@8": 0.3326317237752575, |
| "pass@16": 0.38087277944396986, |
| "pass@32": 0.42657265186991555, |
| "pass@64": 0.46936251256103506, |
| "pass@128": 0.5186335403726708 |
| }, |
| "18": { |
| "pass@1": 0.18276654411764706, |
| "pass@2": 0.2352152327466419, |
| "pass@4": 0.2849862334303801, |
| "pass@8": 0.3288812893596294, |
| "pass@16": 0.3685753443013062, |
| "pass@32": 0.40794190067953773, |
| "pass@64": 0.4466121363445853, |
| "pass@128": 0.4823529411764706 |
| }, |
| "19": { |
| "pass@1": 0.14267113095238096, |
| "pass@2": 0.19041760404949387, |
| "pass@4": 0.2374986904538717, |
| "pass@8": 0.28168255992725094, |
| "pass@16": 0.3226338126666716, |
| "pass@32": 0.3610534610549758, |
| "pass@64": 0.39768084772182344, |
| "pass@128": 0.43452380952380953 |
| }, |
| "20": { |
| "pass@1": 0.14221556886227546, |
| "pass@2": 0.19062441062756372, |
| "pass@4": 0.2377481939944634, |
| "pass@8": 0.280558264608587, |
| "pass@16": 0.32031383167398947, |
| "pass@32": 0.35856987955107245, |
| "pass@64": 0.3935714032182891, |
| "pass@128": 0.4221556886227545 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 539047, |
| "answer_accuracy": 0.672839860600735, |
| "avg_response_len": 273.110060263221, |
| "resp_tokens_sum": 218802671, |
| "pass_at_k": { |
| "pass@1": 0.672839860600735, |
| "pass@2": 0.7047058338354943, |
| "pass@4": 0.7315443418620015, |
| "pass@8": 0.7548757040045201, |
| "pass@16": 0.7751949735171257, |
| "pass@32": 0.7929837040886027, |
| "pass@64": 0.8085162439976868, |
| "pass@128": 0.8223358363955904 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9311100746268657, |
| "pass@2": 0.9520261634739686, |
| "pass@4": 0.9662647566442252, |
| "pass@8": 0.9755282876597258, |
| "pass@16": 0.9810774588415156, |
| "pass@32": 0.9848828094725318, |
| "pass@64": 0.9880498176506736, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9997494533527697, |
| "pass@2": 0.9999838588416243, |
| "pass@4": 0.9999999426092147, |
| "pass@8": 0.9999999999999082, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.9996801900584795, |
| "pass@2": 0.9999672635492931, |
| "pass@4": 0.9999997256373656, |
| "pass@8": 0.9999999999938582, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9776194852941177, |
| "pass@2": 0.9819042380731822, |
| "pass@4": 0.9844617613974724, |
| "pass@8": 0.9863978237239305, |
| "pass@16": 0.9878696643403634, |
| "pass@32": 0.988862169457123, |
| "pass@64": 0.9897049835750625, |
| "pass@128": 0.9911764705882353 |
| }, |
| "6": { |
| "pass@1": 0.9965496845425867, |
| "pass@2": 0.9971636776869769, |
| "pass@4": 0.9974878323569174, |
| "pass@8": 0.99801474353403, |
| "pass@16": 0.9987909801124433, |
| "pass@32": 0.9996022252317682, |
| "pass@64": 0.9999792683835579, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9878362341772152, |
| "pass@2": 0.9902992001395396, |
| "pass@4": 0.9915792477087515, |
| "pass@8": 0.9928315670329757, |
| "pass@16": 0.9943870274755153, |
| "pass@32": 0.9960010268780787, |
| "pass@64": 0.9967819001166339, |
| "pass@128": 0.9968354430379747 |
| }, |
| "8": { |
| "pass@1": 0.9844219219219219, |
| "pass@2": 0.9916711741268824, |
| "pass@4": 0.9939474885459137, |
| "pass@8": 0.9951080518725748, |
| "pass@16": 0.9959944462170828, |
| "pass@32": 0.9967184753879357, |
| "pass@64": 0.9969877001072768, |
| "pass@128": 0.996996996996997 |
| }, |
| "9": { |
| "pass@1": 0.9726688102893891, |
| "pass@2": 0.9816675158872827, |
| "pass@4": 0.9860481763533578, |
| "pass@8": 0.9891498765884253, |
| "pass@16": 0.9913841449354583, |
| "pass@32": 0.9929023262538196, |
| "pass@64": 0.9935237000858083, |
| "pass@128": 0.9935691318327974 |
| }, |
| "11": { |
| "pass@1": 0.893625992063492, |
| "pass@2": 0.923719691288589, |
| "pass@4": 0.9415772218948818, |
| "pass@8": 0.9541297662181599, |
| "pass@16": 0.9648180622646247, |
| "pass@32": 0.9740996623349433, |
| "pass@64": 0.9801861811374231, |
| "pass@128": 0.9841269841269841 |
| }, |
| "12": { |
| "pass@1": 0.8082473241590215, |
| "pass@2": 0.8621445255122921, |
| "pass@4": 0.8994472826791146, |
| "pass@8": 0.9270114252522809, |
| "pass@16": 0.9465701531750612, |
| "pass@32": 0.9611674288837075, |
| "pass@64": 0.9732524253679298, |
| "pass@128": 0.9847094801223242 |
| }, |
| "13": { |
| "pass@1": 0.5974874614197531, |
| "pass@2": 0.6732845460289684, |
| "pass@4": 0.7297426236072337, |
| "pass@8": 0.7771452699041174, |
| "pass@16": 0.8178942459211255, |
| "pass@32": 0.8521100811585546, |
| "pass@64": 0.879897410083746, |
| "pass@128": 0.8981481481481481 |
| }, |
| "14": { |
| "pass@1": 0.45252225519287836, |
| "pass@2": 0.5335529159559803, |
| "pass@4": 0.6004611306005145, |
| "pass@8": 0.6562780500283442, |
| "pass@16": 0.7023598202623946, |
| "pass@32": 0.7390769682434493, |
| "pass@64": 0.7694374359811129, |
| "pass@128": 0.798219584569733 |
| }, |
| "15": { |
| "pass@1": 0.29366801697530864, |
| "pass@2": 0.3638971669339944, |
| "pass@4": 0.42763749178343446, |
| "pass@8": 0.48427563508057453, |
| "pass@16": 0.534451744441426, |
| "pass@32": 0.5792335638578042, |
| "pass@64": 0.6187090353473133, |
| "pass@128": 0.654320987654321 |
| }, |
| "16": { |
| "pass@1": 0.20404682890855458, |
| "pass@2": 0.253627795972406, |
| "pass@4": 0.30170414815404706, |
| "pass@8": 0.34745819229398955, |
| "pass@16": 0.3904366019786063, |
| "pass@32": 0.43057834381205445, |
| "pass@64": 0.4704313191110292, |
| "pass@128": 0.5162241887905604 |
| }, |
| "17": { |
| "pass@1": 0.18211125753012047, |
| "pass@2": 0.23127216049236302, |
| "pass@4": 0.27671165821892746, |
| "pass@8": 0.3184606778754773, |
| "pass@16": 0.3563930038046845, |
| "pass@32": 0.3931867515264538, |
| "pass@64": 0.43110471739026307, |
| "pass@128": 0.46987951807228917 |
| }, |
| "18": { |
| "pass@1": 0.18402217741935484, |
| "pass@2": 0.24017811785623552, |
| "pass@4": 0.29430360469054273, |
| "pass@8": 0.3408792289278583, |
| "pass@16": 0.3782742791909101, |
| "pass@32": 0.41051421704477314, |
| "pass@64": 0.43918690774918834, |
| "pass@128": 0.4612903225806452 |
| }, |
| "19": { |
| "pass@1": 0.15525868580060423, |
| "pass@2": 0.20456532340557113, |
| "pass@4": 0.2538143409943849, |
| "pass@8": 0.301296925041148, |
| "pass@16": 0.3447051650683001, |
| "pass@32": 0.3817948403445567, |
| "pass@64": 0.41078464647666285, |
| "pass@128": 0.43202416918429004 |
| }, |
| "20": { |
| "pass@1": 0.14630255255255256, |
| "pass@2": 0.19446882906528568, |
| "pass@4": 0.24227324146418622, |
| "pass@8": 0.2883451930018625, |
| "pass@16": 0.3340379341883104, |
| "pass@32": 0.37754065399385844, |
| "pass@64": 0.41581311513220076, |
| "pass@128": 0.44744744744744747 |
| } |
| } |
| } |
| } |
| } |
| ] |
