[ { "checkpoint": "checkpoint-21414", "id_acc": 0.5163628472222223, "ood_acc": 0.14568359375, "total_acc": 0.3212685032894737, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.46446354166667, "ood_avg_resp_len": 302.6413921875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 256705, "answer_accuracy": 0.3170262112709453, "avg_response_len": 240.0941365495574, "resp_tokens_sum": 194410945, "pass_at_k": { "pass@1": 0.3170262112709453, "pass@2": 0.41333096242603934, "pass@4": 0.5188438225967936, "pass@8": 0.6212990365018248, "pass@16": 0.7097565650424552, "pass@32": 0.7780028734739614, "pass@64": 0.8260987824256207, "pass@128": 0.8594688586784698 }, "per_op_pass_at_k": { "10": { "pass@1": 0.25410213864306785, "pass@2": 0.38541848129050216, "pass@4": 0.5397663588511608, "pass@8": 0.6911220142903763, "pass@16": 0.8144716731548343, "pass@32": 0.9007734251817849, "pass@64": 0.9547486096517519, "pass@128": 0.9852507374631269 }, "2": { "pass@1": 0.9106795171339563, "pass@2": 0.9424638034439621, "pass@4": 0.9664033990494175, "pass@8": 0.9818446668613697, "pass@16": 0.9903789463115695, "pass@32": 0.994020154983159, "pass@64": 0.9959263337220218, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.8564124617737003, "pass@2": 0.8995242782152231, "pass@4": 0.9379065466587319, "pass@8": 0.9688289456978036, "pass@16": 0.988510504308861, "pass@32": 0.9967707993454415, "pass@64": 0.999420725755249, "pass@128": 1.0 }, "4": { "pass@1": 0.7214599609375, "pass@2": 0.8123266024852361, "pass@4": 0.8851466221995687, "pass@8": 0.9351398158085544, "pass@16": 0.9643246165797228, "pass@32": 0.9816108955922624, "pass@64": 0.9923375102880352, "pass@128": 1.0 }, "5": { "pass@1": 0.4746031746031746, "pass@2": 0.6117000999875012, "pass@4": 0.7313267404074488, "pass@8": 0.8257029432030616, "pass@16": 0.8926087586123375, "pass@32": 0.936348525554623, "pass@64": 0.9630246909720346, "pass@128": 0.9777777777777777 }, "6": { "pass@1": 0.4722118263473054, "pass@2": 0.6342759854307132, "pass@4": 0.7775939507374454, "pass@8": 0.8825466618292312, "pass@16": 0.9464171107787874, "pass@32": 0.9786132080116102, "pass@64": 0.9937169652841435, "pass@128": 1.0 }, "7": { "pass@1": 0.3753004807692308, "pass@2": 0.5153170572613338, "pass@4": 0.6561292772944801, "pass@8": 0.7761565488757287, "pass@16": 0.8626775371075649, "pass@32": 0.9154326493792188, "pass@64": 0.9457564239431862, "pass@128": 0.9615384615384616 }, "8": { "pass@1": 0.3215144230769231, "pass@2": 0.46476094033918824, "pass@4": 0.6165568053993246, "pass@8": 0.7534847622641432, "pass@16": 0.8589420967295084, "pass@32": 0.9294160107049367, "pass@64": 0.968232374522714, "pass@128": 0.9839743589743589 }, "9": { "pass@1": 0.2617517605633803, "pass@2": 0.39134794000221806, "pass@4": 0.5420022101110598, "pass@8": 0.6911958532785958, "pass@16": 0.8130187441216671, "pass@32": 0.8937970226619533, "pass@64": 0.9389010514280172, "pass@128": 0.9633802816901409 }, "11": { "pass@1": 0.23074539660056656, "pass@2": 0.35009535812272746, "pass@4": 0.4882828888598553, "pass@8": 0.6247398677588178, "pass@16": 0.7429827393881699, "pass@32": 0.8357673540792772, "pass@64": 0.9005930374066281, "pass@128": 0.943342776203966 }, "12": { "pass@1": 0.19639389124293785, "pass@2": 0.3063894356955379, "pass@4": 0.440758561959416, "pass@8": 0.5857390531049586, "pass@16": 0.7220385954008949, "pass@32": 0.8282172555034433, "pass@64": 0.8965112418457605, "pass@128": 0.9378531073446328 }, "13": { "pass@1": 0.17932014106583072, "pass@2": 0.2773797145360748, "pass@4": 0.3952843552275407, "pass@8": 0.5244591507479613, "pass@16": 0.6515171712008817, "pass@32": 0.7609219998509404, "pass@64": 0.8391465004588685, "pass@128": 0.8871473354231975 }, "14": { "pass@1": 0.14987344236760125, "pass@2": 0.24167831824760222, "pass@4": 0.3558844169011584, "pass@8": 0.48062360747413807, "pass@16": 0.6031438802092132, "pass@32": 0.7101192293253078, "pass@64": 0.791392407450985, "pass@128": 0.8442367601246106 }, "15": { "pass@1": 0.1279127528901734, "pass@2": 0.19832997189477028, "pass@4": 0.2873977274697598, "pass@8": 0.39199371493524304, "pass@16": 0.5038338391917309, "pass@32": 0.6065491304457705, "pass@64": 0.6887834435073522, "pass@128": 0.7572254335260116 }, "16": { "pass@1": 0.12358870967741936, "pass@2": 0.19655353060706127, "pass@4": 0.28869364908015527, "pass@8": 0.38926244247020075, "pass@16": 0.48742943166589325, "pass@32": 0.5751892983598913, "pass@64": 0.6502279989309359, "pass@128": 0.7161290322580646 }, "17": { "pass@1": 0.12741600433526012, "pass@2": 0.19630173012152363, "pass@4": 0.2774420586335666, "pass@8": 0.36357971010177026, "pass@16": 0.4478440310684387, "pass@32": 0.5246952747803684, "pass@64": 0.5915975129658583, "pass@128": 0.6502890173410405 }, "18": { "pass@1": 0.10390625, "pass@2": 0.16584434758155223, "pass@4": 0.24453465691788526, "pass@8": 0.33183968973437294, "pass@16": 0.4167819865707351, "pass@32": 0.49231203177580335, "pass@64": 0.5558379435748657, "pass@128": 0.6057142857142858 }, "19": { "pass@1": 0.10846002252252253, "pass@2": 0.17541778983708112, "pass@4": 0.2587067981930187, "pass@8": 0.34694823379659046, "pass@16": 0.42791437185014847, "pass@32": 0.4934379723821667, "pass@64": 0.5438193608394496, "pass@128": 0.5915915915915916 }, "20": { "pass@1": 0.10027214714714715, "pass@2": 0.15846235014542095, "pass@4": 0.23135669668543687, "pass@8": 0.31203855130831787, "pass@16": 0.39199227760299216, "pass@32": 0.4606754660817298, "pass@64": 0.5124782566863452, "pass@128": 0.5495495495495496 } } }, "teachers_in_school": { "count": 821120, "correct": 262888, "answer_accuracy": 0.32015783320342944, "avg_response_len": 249.93761204208886, "resp_tokens_sum": 205228772, "pass_at_k": { "pass@1": 0.32015783320342944, "pass@2": 0.4156968626680902, "pass@4": 0.5208065187194584, "pass@8": 0.6237177484598363, "pass@16": 0.7124685902904693, "pass@32": 0.7799126469709882, "pass@64": 0.8275756037833933, "pass@128": 0.8618862042088854 }, "per_op_pass_at_k": { "10": { "pass@1": 0.26540931748466257, "pass@2": 0.39662335152891176, "pass@4": 0.5456323869217269, "pass@8": 0.6920447324686265, "pass@16": 0.8146591405440967, "pass@32": 0.9003614283097305, "pass@64": 0.9499486824301553, "pass@128": 0.9754601226993865 }, "2": { "pass@1": 0.9039228723404256, "pass@2": 0.936999273029701, "pass@4": 0.9640325722012707, "pass@8": 0.9822922126341371, "pass@16": 0.9921029236893512, "pass@32": 0.9961241979706833, "pass@64": 0.9969365657746452, "pass@128": 0.9969604863221885 }, "3": { "pass@1": 0.8807528409090909, "pass@2": 0.9155679581245529, "pass@4": 0.9469561969526535, "pass@8": 0.9723452248893673, "pass@16": 0.9888141155481235, "pass@32": 0.9966414290903808, "pass@64": 0.9994041833835215, "pass@128": 1.0 }, "4": { "pass@1": 0.6924232618343196, "pass@2": 0.7907765689791734, "pass@4": 0.8758891791632553, "pass@8": 0.936007704442523, "pass@16": 0.9686286192064628, "pass@32": 0.9837451220106904, "pass@64": 0.9917408064315401, "pass@128": 0.9970414201183432 }, "5": { "pass@1": 0.44732789855072463, "pass@2": 0.578949132717106, "pass@4": 0.7002346964781571, "pass@8": 0.7998659410224632, "pass@16": 0.8748176184318499, "pass@32": 0.9272979570411638, "pass@64": 0.9597548745587382, "pass@128": 0.9768115942028985 }, "6": { "pass@1": 0.4549606017191977, "pass@2": 0.611524702186224, "pass@4": 0.748689749383046, "pass@8": 0.8528006981800831, "pass@16": 0.9231486355342031, "pass@32": 0.9622876414728101, "pass@64": 0.9801267017916749, "pass@128": 0.9885386819484241 }, "7": { "pass@1": 0.4048274927745665, "pass@2": 0.5412742222702657, "pass@4": 0.674239169977307, "pass@8": 0.787814474108333, "pass@16": 0.8702585537862398, "pass@32": 0.9193176128192155, "pass@64": 0.9434355530262379, "pass@128": 0.9566473988439307 }, "8": { "pass@1": 0.34401408450704224, "pass@2": 0.48745494621270946, "pass@4": 0.635357102985366, "pass@8": 0.7675013657375953, "pass@16": 0.8688231467359476, "pass@32": 0.9331341673035908, "pass@64": 0.9671424869278781, "pass@128": 0.9802816901408451 }, "9": { "pass@1": 0.277905127245509, "pass@2": 0.40908488141826543, "pass@4": 0.5569567148043622, "pass@8": 0.7026706835230918, "pass@16": 0.8241962201311076, "pass@32": 0.9040788025953181, "pass@64": 0.9458140176070623, "pass@128": 0.9700598802395209 }, "11": { "pass@1": 0.19147684487951808, "pass@2": 0.2949832054596335, "pass@4": 0.4249811888160067, "pass@8": 0.5684708001105898, "pass@16": 0.7033851584127203, "pass@32": 0.8127286622022419, "pass@64": 0.8939142475671594, "pass@128": 0.9487951807228916 }, "12": { "pass@1": 0.2179418103448276, "pass@2": 0.33208283452225207, "pass@4": 0.47159283704348853, "pass@8": 0.6208105407618292, "pass@16": 0.7542302648948708, "pass@32": 0.8506687264615499, "pass@64": 0.9116059188654317, "pass@128": 0.9529780564263323 }, "13": { "pass@1": 0.16940213585434175, "pass@2": 0.2634938325723991, "pass@4": 0.3793841751923866, "pass@8": 0.5050673959610599, "pass@16": 0.6265313995042749, "pass@32": 0.7333003831218914, "pass@64": 0.8201189132557245, "pass@128": 0.8795518207282913 }, "14": { "pass@1": 0.14487390350877194, "pass@2": 0.2287026004973063, "pass@4": 0.3352070892454231, "pass@8": 0.4521805285221764, "pass@16": 0.5621008322504216, "pass@32": 0.6559120049884942, "pass@64": 0.7329353333346605, "pass@128": 0.7953216374269005 }, "15": { "pass@1": 0.12080965909090909, "pass@2": 0.19719637318062513, "pass@4": 0.2982539980797855, "pass@8": 0.4146947642198226, "pass@16": 0.5306220264944798, "pass@32": 0.632068217942676, "pass@64": 0.713947397576443, "pass@128": 0.7818181818181819 }, "16": { "pass@1": 0.13054220085470086, "pass@2": 0.20016067703075577, "pass@4": 0.28242962231216834, "pass@8": 0.37245942247155317, "pass@16": 0.4633370433089357, "pass@32": 0.5449004689861886, "pass@64": 0.6146582022395269, "pass@128": 0.6723646723646723 }, "17": { "pass@1": 0.12473311335403726, "pass@2": 0.19863702499144123, "pass@4": 0.2904211032238981, "pass@8": 0.39107439466974725, "pass@16": 0.48594380207333926, "pass@32": 0.565189798800697, "pass@64": 0.6291464350799738, "pass@128": 0.6832298136645962 }, "18": { "pass@1": 0.11567095588235295, "pass@2": 0.1858296665122742, "pass@4": 0.2744835213613003, "pass@8": 0.3731498178040868, "pass@16": 0.47119057826231775, "pass@32": 0.5583474575059797, "pass@64": 0.6331969527222595, "pass@128": 0.6970588235294117 }, "19": { "pass@1": 0.11228143601190477, "pass@2": 0.1775589818850769, "pass@4": 0.2564224229225811, "pass@8": 0.33700818889898293, "pass@16": 0.4081891566185268, "pass@32": 0.46506532057111194, "pass@64": 0.5090195808415485, "pass@128": 0.5505952380952381 }, "20": { "pass@1": 0.09300149700598802, "pass@2": 0.15357940614833324, "pass@4": 0.23198116028909568, "pass@8": 0.3187552978474917, "pass@16": 0.40235214790860857, "pass@32": 0.47365640144881727, "pass@64": 0.5272035572429615, "pass@128": 0.5688622754491018 } } }, "movie_festival_awards": { "count": 801152, "correct": 261732, "answer_accuracy": 0.32669455983383927, "avg_response_len": 272.9523573554082, "resp_tokens_sum": 218676327, "pass_at_k": { "pass@1": 0.32669455983383927, "pass@2": 0.423339878763563, "pass@4": 0.5288904041172903, "pass@8": 0.629584657101439, "pass@16": 0.7138453133463799, "pass@32": 0.7778640458840944, "pass@64": 0.824266594191614, "pass@128": 0.8586036108004473 }, "per_op_pass_at_k": { "10": { "pass@1": 0.2988106343283582, "pass@2": 0.43811001586555404, "pass@4": 0.592827761361919, "pass@8": 0.7389906797871061, "pass@16": 0.8525613765094223, "pass@32": 0.9236767364419983, "pass@64": 0.9630391911668797, "pass@128": 0.982089552238806 }, "2": { "pass@1": 0.8912946428571429, "pass@2": 0.9280339566929132, "pass@4": 0.9595391234131446, "pass@8": 0.9812508611442692, "pass@16": 0.9925072517907899, "pass@32": 0.9966186521833359, "pass@64": 0.9983968451130126, "pass@128": 1.0 }, "3": { "pass@1": 0.8721073250728864, "pass@2": 0.9140296796446361, "pass@4": 0.9490690140591022, "pass@8": 0.9730759865091045, "pass@16": 0.9857702689101124, "pass@32": 0.9920553316855298, "pass@64": 0.9955026148242196, "pass@128": 0.9970845481049563 }, "4": { "pass@1": 0.7161686769005848, "pass@2": 0.8083079356034446, "pass@4": 0.8882740408545416, "pass@8": 0.9440402847981656, "pass@16": 0.9733790763978329, "pass@32": 0.9852732332074049, "pass@64": 0.9894172655511996, "pass@128": 0.9912280701754386 }, "5": { "pass@1": 0.4732766544117647, "pass@2": 0.6120990620657709, "pass@4": 0.7379644536160923, "pass@8": 0.8328578936686588, "pass@16": 0.8941166055439531, "pass@32": 0.9329952890031863, "pass@64": 0.9585346672880898, "pass@128": 0.9764705882352941 }, "6": { "pass@1": 0.4296382097791798, "pass@2": 0.5824524485208274, "pass@4": 0.7232217291844827, "pass@8": 0.831538757783547, "pass@16": 0.9022300063004003, "pass@32": 0.9425257163922585, "pass@64": 0.9631168749883116, "pass@128": 0.9747634069400631 }, "7": { "pass@1": 0.38983386075949367, "pass@2": 0.5293648148609583, "pass@4": 0.665921777776196, "pass@8": 0.7814408029242328, "pass@16": 0.8668908924653281, "pass@32": 0.924388953311442, "pass@64": 0.9611535974011868, "pass@128": 0.9841772151898734 }, "8": { "pass@1": 0.3031625375375375, "pass@2": 0.44241460949138073, "pass@4": 0.5955823466886454, "pass@8": 0.736405779811164, "pass@16": 0.8433245202487523, "pass@32": 0.9120078894889249, "pass@64": 0.9486454274049398, "pass@128": 0.960960960960961 }, "9": { "pass@1": 0.24593046623794212, "pass@2": 0.37445249006253595, "pass@4": 0.5259498379623767, "pass@8": 0.6801062895678363, "pass@16": 0.8112116977754945, "pass@32": 0.9045034497592848, "pass@64": 0.9614410982686454, "pass@128": 0.9903536977491961 }, "11": { "pass@1": 0.2273313492063492, "pass@2": 0.34733900449943744, "pass@4": 0.48797233292267006, "pass@8": 0.6312245582865806, "pass@16": 0.755738770752698, "pass@32": 0.8506597987618932, "pass@64": 0.9176150703846887, "pass@128": 0.9619047619047619 }, "12": { "pass@1": 0.20637423547400613, "pass@2": 0.31808833887163207, "pass@4": 0.45083233947820794, "pass@8": 0.5880314127552496, "pass@16": 0.7108223889541407, "pass@32": 0.8044281622631726, "pass@64": 0.8670414062590106, "pass@128": 0.908256880733945 }, "13": { "pass@1": 0.18993537808641975, "pass@2": 0.2897349962331098, "pass@4": 0.4045277045809091, "pass@8": 0.5189595494585346, "pass@16": 0.6230551493144086, "pass@32": 0.7152991866459357, "pass@64": 0.7936777363002678, "pass@128": 0.8549382716049383 }, "14": { "pass@1": 0.16450296735905046, "pass@2": 0.2570091444893571, "pass@4": 0.3680840732260101, "pass@8": 0.4827612360891505, "pass@16": 0.5865991822960027, "pass@32": 0.6762563749288109, "pass@64": 0.7509042429339469, "pass@128": 0.8100890207715133 }, "15": { "pass@1": 0.11622299382716049, "pass@2": 0.19063654369592675, "pass@4": 0.29270876470302304, "pass@8": 0.4155669062840721, "pass@16": 0.5390361333862039, "pass@32": 0.6411043054391663, "pass@64": 0.7135058242529809, "pass@128": 0.7685185185185185 }, "16": { "pass@1": 0.13880438790560473, "pass@2": 0.20914410726314084, "pass@4": 0.29200838169565074, "pass@8": 0.38077294241430154, "pass@16": 0.46737511475798427, "pass@32": 0.5497080689560159, "pass@64": 0.6309065509538583, "pass@128": 0.7109144542772862 }, "17": { "pass@1": 0.12768260542168675, "pass@2": 0.19840273989659413, "pass@4": 0.2831356568004299, "pass@8": 0.3738445163007728, "pass@16": 0.4613568303598599, "pass@32": 0.5385335841890195, "pass@64": 0.6024723538594325, "pass@128": 0.6536144578313253 }, "18": { "pass@1": 0.10819052419354838, "pass@2": 0.17318469011938017, "pass@4": 0.25503082659022464, "pass@8": 0.34397619629010034, "pass@16": 0.42672772692874866, "pass@32": 0.4951718346534467, "pass@64": 0.5523887072386519, "pass@128": 0.6129032258064516 }, "19": { "pass@1": 0.12157760574018127, "pass@2": 0.1908757463662964, "pass@4": 0.27040493408867705, "pass@8": 0.3496266345867802, "pass@16": 0.42369818096696776, "pass@32": 0.48808375812634547, "pass@64": 0.5379706412146679, "pass@128": 0.5770392749244713 }, "20": { "pass@1": 0.10203171921921922, "pass@2": 0.16524644723463625, "pass@4": 0.24453744633272184, "pass@8": 0.33137040399755197, "pass@16": 0.4145973832634668, "pass@32": 0.48491618782761853, "pass@64": 0.5411038749238117, "pass@128": 0.5885885885885885 } } } } } ]