[ { "checkpoint": "checkpoint-21350", "id_acc": 0.9110954861111111, "ood_acc": 0.35171796875, "total_acc": 0.6166862664473685, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 199.92764930555555, "ood_avg_resp_len": 320.44338671875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 499909, "answer_accuracy": 0.617378922304774, "avg_response_len": 251.66101456291494, "resp_tokens_sum": 203776970, "pass_at_k": { "pass@1": 0.617378922304774, "pass@2": 0.6702401358535359, "pass@4": 0.7126881271914993, "pass@8": 0.7469478039476964, "pass@16": 0.7751029216952456, "pass@32": 0.7984062550092523, "pass@64": 0.8175423265721407, "pass@128": 0.833386025924755 }, "per_op_pass_at_k": { "10": { "pass@1": 0.8190680309734514, "pass@2": 0.8758038783592319, "pass@4": 0.9097118844657693, "pass@8": 0.9301279923292359, "pass@16": 0.945463319652103, "pass@32": 0.9597276818955355, "pass@64": 0.971984946225041, "pass@128": 0.9793510324483776 }, "2": { "pass@1": 0.9998539719626168, "pass@2": 0.9999996167243114, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9974197247706422, "pass@2": 0.9979062071082859, "pass@4": 0.9985759549551719, "pass@8": 0.9993549275868031, "pass@16": 0.9998791891503424, "pass@32": 0.999997274769305, "pass@64": 0.999999999917904, "pass@128": 1.0 }, "4": { "pass@1": 0.9796875, "pass@2": 0.9833226962352362, "pass@4": 0.9855182915612108, "pass@8": 0.9877684577554199, "pass@16": 0.9898908624050529, "pass@32": 0.9915912787745874, "pass@64": 0.9929543613492345, "pass@128": 0.99375 }, "5": { "pass@1": 0.8453621031746031, "pass@2": 0.878759686289214, "pass@4": 0.9038273906237906, "pass@8": 0.9236315572169654, "pass@16": 0.9401206361112525, "pass@32": 0.9527244490497119, "pass@64": 0.9623853430676196, "pass@128": 0.9682539682539683 }, "6": { "pass@1": 0.9475580089820359, "pass@2": 0.9595694192324015, "pass@4": 0.9684871766590554, "pass@8": 0.9760311732040033, "pass@16": 0.9819037036422803, "pass@32": 0.9859049285559663, "pass@64": 0.9889859412882993, "pass@128": 0.9910179640718563 }, "7": { "pass@1": 0.9068047337278107, "pass@2": 0.932142305945115, "pass@4": 0.947185020234157, "pass@8": 0.9582908289110658, "pass@16": 0.9681797653463373, "pass@32": 0.9767144305121371, "pass@64": 0.9830296516855794, "pass@128": 0.9881656804733728 }, "8": { "pass@1": 0.8812349759615384, "pass@2": 0.9134710024227741, "pass@4": 0.9344750253574073, "pass@8": 0.9487608950939497, "pass@16": 0.9589510177536014, "pass@32": 0.9657383512461981, "pass@64": 0.9703603198365051, "pass@128": 0.9743589743589743 }, "9": { "pass@1": 0.8417913732394366, "pass@2": 0.8856545275590555, "pass@4": 0.9142673059353488, "pass@8": 0.9349954551526746, "pass@16": 0.9509292121287449, "pass@32": 0.9633821177411638, "pass@64": 0.973017756199603, "pass@128": 0.9802816901408451 }, "11": { "pass@1": 0.7452195467422096, "pass@2": 0.8180329320113316, "pass@4": 0.8652162037641893, "pass@8": 0.8969106210163632, "pass@16": 0.9210920894565872, "pass@32": 0.9403227057391875, "pass@64": 0.9548937056898483, "pass@128": 0.9660056657223796 }, "12": { "pass@1": 0.6948490466101694, "pass@2": 0.7856954685484225, "pass@4": 0.8424545043628017, "pass@8": 0.8799306789509234, "pass@16": 0.9062520096316939, "pass@32": 0.924809518611065, "pass@64": 0.938702946618994, "pass@128": 0.9491525423728814 }, "13": { "pass@1": 0.5179516065830722, "pass@2": 0.6325184200133289, "pass@4": 0.7205692302764656, "pass@8": 0.7828447072240617, "pass@16": 0.8262471661892159, "pass@32": 0.8598153482049776, "pass@64": 0.8862313514669503, "pass@128": 0.9059561128526645 }, "14": { "pass@1": 0.3964417834890966, "pass@2": 0.5047361376848922, "pass@4": 0.5946405522323733, "pass@8": 0.6680429180152108, "pass@16": 0.7301454909544141, "pass@32": 0.7804329625460306, "pass@64": 0.8173909998798203, "pass@128": 0.8411214953271028 }, "15": { "pass@1": 0.29612987716763006, "pass@2": 0.3869219653179187, "pass@4": 0.46993725170191875, "pass@8": 0.5421051675527985, "pass@16": 0.6023771645269487, "pass@32": 0.6529647288435331, "pass@64": 0.6944795629555712, "pass@128": 0.7283236994219653 }, "16": { "pass@1": 0.24740423387096774, "pass@2": 0.32904813309626596, "pass@4": 0.40458356405046153, "pass@8": 0.4702012137774771, "pass@16": 0.5247288669662102, "pass@32": 0.5706876527594704, "pass@64": 0.6115942057418606, "pass@128": 0.6451612903225806 }, "17": { "pass@1": 0.16986542630057805, "pass@2": 0.22985803729916698, "pass@4": 0.28688736695124073, "pass@8": 0.33843467932243226, "pass@16": 0.38700181611143314, "pass@32": 0.43410878184626833, "pass@64": 0.47804135764926775, "pass@128": 0.5202312138728323 }, "18": { "pass@1": 0.1679017857142857, "pass@2": 0.2257529527559055, "pass@4": 0.2848023300658846, "pass@8": 0.34394438573386016, "pass@16": 0.3988663230247612, "pass@32": 0.4435387309873241, "pass@64": 0.47824795716449336, "pass@128": 0.5085714285714286 }, "19": { "pass@1": 0.16246715465465467, "pass@2": 0.2220851510959779, "pass@4": 0.2795086211070464, "pass@8": 0.3279673225850375, "pass@16": 0.36686001138513913, "pass@32": 0.4010357183857945, "pass@64": 0.4330668055174505, "pass@128": 0.46546546546546547 }, "20": { "pass@1": 0.15582770270270271, "pass@2": 0.21083792946489788, "pass@4": 0.2651404334593312, "pass@8": 0.31510224505908696, "pass@16": 0.3582141742580615, "pass@32": 0.3942354522194376, "pass@64": 0.4242069841105825, "pass@128": 0.45345345345345345 } } }, "teachers_in_school": { "count": 821120, "correct": 507526, "answer_accuracy": 0.6180899259547935, "avg_response_len": 260.83856196414655, "resp_tokens_sum": 214179760, "pass_at_k": { "pass@1": 0.6180899259547935, "pass@2": 0.6704741094936216, "pass@4": 0.714109435375526, "pass@8": 0.7502174513426444, "pass@16": 0.7800515230208226, "pass@32": 0.804595822219313, "pass@64": 0.8246596996146075, "pass@128": 0.8419329696024942 }, "per_op_pass_at_k": { "10": { "pass@1": 0.7672785659509203, "pass@2": 0.8221017402540941, "pass@4": 0.8567607347586148, "pass@8": 0.8803886350813609, "pass@16": 0.8990823514930147, "pass@32": 0.9155576005344653, "pass@64": 0.9292128475444902, "pass@128": 0.9386503067484663 }, "2": { "pass@1": 0.9999762537993921, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9975378787878788, "pass@2": 0.999380368647101, "pass@4": 0.9999463317085363, "pass@8": 0.999999574463101, "pass@16": 0.99999999998634, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9803069526627219, "pass@2": 0.9856821186460419, "pass@4": 0.9884984804968907, "pass@8": 0.9901388748274397, "pass@16": 0.9915489433224962, "pass@32": 0.993055431506491, "pass@64": 0.9948274051186248, "pass@128": 0.9970414201183432 }, "5": { "pass@1": 0.8750452898550725, "pass@2": 0.8991081108068009, "pass@4": 0.91609362552507, "pass@8": 0.9283145339273063, "pass@16": 0.9387362873527924, "pass@32": 0.9495945434273302, "pass@64": 0.9611962026508659, "pass@128": 0.9739130434782609 }, "6": { "pass@1": 0.9597063037249284, "pass@2": 0.9688247354646573, "pass@4": 0.9755671450810769, "pass@8": 0.9809583841351045, "pass@16": 0.9856212779919914, "pass@32": 0.9895090347929678, "pass@64": 0.9923046542590954, "pass@128": 0.994269340974212 }, "7": { "pass@1": 0.9184429190751445, "pass@2": 0.9380771101679485, "pass@4": 0.9515284368355686, "pass@8": 0.9620686520464642, "pass@16": 0.9694759351290418, "pass@32": 0.9729727131601149, "pass@64": 0.9739292569091923, "pass@128": 0.9739884393063584 }, "8": { "pass@1": 0.8584507042253521, "pass@2": 0.8973130614395034, "pass@4": 0.9233392773966632, "pass@8": 0.9433777973806029, "pass@16": 0.9601626523567398, "pass@32": 0.9730733897077097, "pass@64": 0.9801915294569885, "pass@128": 0.9830985915492958 }, "9": { "pass@1": 0.8166401571856288, "pass@2": 0.8604763897402046, "pass@4": 0.8899476901090955, "pass@8": 0.9137484797589711, "pass@16": 0.9336248046731362, "pass@32": 0.9489223240336991, "pass@64": 0.9611196746977648, "pass@128": 0.9760479041916168 }, "11": { "pass@1": 0.759765625, "pass@2": 0.8237484880466749, "pass@4": 0.8661105835038693, "pass@8": 0.8974093782721269, "pass@16": 0.9220586483640653, "pass@32": 0.9400776540050912, "pass@64": 0.9516687417338571, "pass@128": 0.9578313253012049 }, "12": { "pass@1": 0.6983248432601881, "pass@2": 0.792604765754202, "pass@4": 0.8538179523210068, "pass@8": 0.8929835881620504, "pass@16": 0.9185320929903009, "pass@32": 0.9352178761761952, "pass@64": 0.9478804552757559, "pass@128": 0.9592476489028213 }, "13": { "pass@1": 0.5278361344537815, "pass@2": 0.63536910827323, "pass@4": 0.7126288210822387, "pass@8": 0.7701978399669224, "pass@16": 0.8166211744373886, "pass@32": 0.8550839822738148, "pass@64": 0.8865038689796082, "pass@128": 0.9131652661064426 }, "14": { "pass@1": 0.41721491228070173, "pass@2": 0.5287199400239444, "pass@4": 0.6211705283001904, "pass@8": 0.6923516248529362, "pass@16": 0.7462194130033619, "pass@32": 0.7883736937027563, "pass@64": 0.8222691206866892, "pass@128": 0.8508771929824561 }, "15": { "pass@1": 0.28927556818181815, "pass@2": 0.38632150739680254, "pass@4": 0.477739166411017, "pass@8": 0.5566535711234109, "pass@16": 0.6215893466194053, "pass@32": 0.6719853954938052, "pass@64": 0.7093504805012789, "pass@128": 0.7393939393939394 }, "16": { "pass@1": 0.21276264245014245, "pass@2": 0.28995263813177213, "pass@4": 0.3666882563824822, "pass@8": 0.4346854612065274, "pass@16": 0.490842220216734, "pass@32": 0.5378512803345692, "pass@64": 0.5810806305675018, "pass@128": 0.6239316239316239 }, "17": { "pass@1": 0.19004755434782608, "pass@2": 0.2589851200665132, "pass@4": 0.3332288832684735, "pass@8": 0.4048084183288024, "pass@16": 0.46686980557788155, "pass@32": 0.5180633036985619, "pass@64": 0.5611210047877057, "pass@128": 0.5962732919254659 }, "18": { "pass@1": 0.17541360294117647, "pass@2": 0.238078248031496, "pass@4": 0.3021544674562736, "pass@8": 0.36484250871024415, "pass@16": 0.42552854294852027, "pass@32": 0.4830248444609159, "pass@64": 0.532842036580717, "pass@128": 0.5705882352941176 }, "19": { "pass@1": 0.14276413690476192, "pass@2": 0.19880974350862385, "pass@4": 0.25782905819808244, "pass@8": 0.3139917285710976, "pass@16": 0.3642751043986496, "pass@32": 0.4084618548346345, "pass@64": 0.44438723572602956, "pass@128": 0.47619047619047616 }, "20": { "pass@1": 0.14429734281437126, "pass@2": 0.2033662743410817, "pass@4": 0.26359721810597025, "pass@8": 0.3163735906613064, "pass@16": 0.359637297892788, "pass@32": 0.395871809493765, "pass@64": 0.42783318947128957, "pass@128": 0.46107784431137727 } } }, "movie_festival_awards": { "count": 801152, "correct": 492346, "answer_accuracy": 0.6145475515258029, "avg_response_len": 277.75934778918355, "resp_tokens_sum": 222527457, "pass_at_k": { "pass@1": 0.6145475515258029, "pass@2": 0.66511725084383, "pass@4": 0.7072032517044973, "pass@8": 0.7420143982533723, "pass@16": 0.7702823493585079, "pass@32": 0.7934652060341785, "pass@64": 0.8130399718603196, "pass@128": 0.8304841028918357 }, "per_op_pass_at_k": { "10": { "pass@1": 0.837080223880597, "pass@2": 0.8841825567046653, "pass@4": 0.9129999748165799, "pass@8": 0.9340309207425049, "pass@16": 0.9503269298903634, "pass@32": 0.9633252118626652, "pass@64": 0.9742339636240699, "pass@128": 0.9850746268656716 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9994989067055393, "pass@2": 0.9999314897500058, "pass@4": 0.9999986759125955, "pass@8": 0.9999999997431218, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9685215643274854, "pass@2": 0.9750720921397983, "pass@4": 0.9805761952453314, "pass@8": 0.9847048683048196, "pass@16": 0.9872249490269424, "pass@32": 0.988723427016145, "pass@64": 0.9897569491732896, "pass@128": 0.9912280701754386 }, "5": { "pass@1": 0.8611213235294117, "pass@2": 0.8884335919407136, "pass@4": 0.9081887436864504, "pass@8": 0.924253573036253, "pass@16": 0.9389244449350933, "pass@32": 0.9523447840529764, "pass@64": 0.9630246066646815, "pass@128": 0.9705882352941176 }, "6": { "pass@1": 0.9733832807570978, "pass@2": 0.9828349716833503, "pass@4": 0.9887145172508008, "pass@8": 0.9916570561272308, "pass@16": 0.9930273048146369, "pass@32": 0.9941869150417751, "pass@64": 0.9952583726678965, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.8899080300632911, "pass@2": 0.9240950176916176, "pass@4": 0.945723327898886, "pass@8": 0.9597153776968665, "pass@16": 0.9702119965930145, "pass@32": 0.9779051128332572, "pass@64": 0.9826542051427243, "pass@128": 0.9841772151898734 }, "8": { "pass@1": 0.8529466966966966, "pass@2": 0.8928994200893807, "pass@4": 0.9212113263432158, "pass@8": 0.94219890076564, "pass@16": 0.956245171195436, "pass@32": 0.9650723844889141, "pass@64": 0.9708403647709795, "pass@128": 0.972972972972973 }, "9": { "pass@1": 0.8221463022508039, "pass@2": 0.8635371040838545, "pass@4": 0.8914274667635996, "pass@8": 0.9129749581292425, "pass@16": 0.9306763023858616, "pass@32": 0.9479524332552338, "pass@64": 0.9652302587641902, "pass@128": 0.977491961414791 }, "11": { "pass@1": 0.7507688492063492, "pass@2": 0.8152867610298712, "pass@4": 0.8574687646187082, "pass@8": 0.8888661836516896, "pass@16": 0.9135599558410991, "pass@32": 0.9314423496372826, "pass@64": 0.9434417545138772, "pass@128": 0.9523809523809523 }, "12": { "pass@1": 0.7076404816513762, "pass@2": 0.8042358201497741, "pass@4": 0.8671578274619333, "pass@8": 0.9085258038760612, "pass@16": 0.935189808714328, "pass@32": 0.9534243395021395, "pass@64": 0.9653087455773289, "pass@128": 0.9724770642201835 }, "13": { "pass@1": 0.5027488425925926, "pass@2": 0.6106045555069503, "pass@4": 0.6959111670647646, "pass@8": 0.7622771166009414, "pass@16": 0.8123751233047263, "pass@32": 0.8480814715984645, "pass@64": 0.8730092960227223, "pass@128": 0.8919753086419753 }, "14": { "pass@1": 0.3861044139465875, "pass@2": 0.4967668637117687, "pass@4": 0.5935214991672034, "pass@8": 0.6672018859137244, "pass@16": 0.71946447521043, "pass@32": 0.7584769248935936, "pass@64": 0.7924551518791305, "pass@128": 0.827893175074184 }, "15": { "pass@1": 0.24127121913580246, "pass@2": 0.32664900177408374, "pass@4": 0.4125610458993551, "pass@8": 0.49270826498349163, "pass@16": 0.5618038483266853, "pass@32": 0.6195859693933924, "pass@64": 0.6711680233603406, "pass@128": 0.7191358024691358 }, "16": { "pass@1": 0.19690265486725664, "pass@2": 0.2587497532111583, "pass@4": 0.3189182358842312, "pass@8": 0.374695087684852, "pass@16": 0.4241129705890384, "pass@32": 0.4690045235965585, "pass@64": 0.5130468430129036, "pass@128": 0.5634218289085545 }, "17": { "pass@1": 0.17613422439759036, "pass@2": 0.23685675279859578, "pass@4": 0.2954108272234041, "pass@8": 0.34783661316669684, "pass@16": 0.3928556096830005, "pass@32": 0.43333668541543363, "pass@64": 0.4700079164554424, "pass@128": 0.5060240963855421 }, "18": { "pass@1": 0.1748991935483871, "pass@2": 0.23574501524003058, "pass@4": 0.2958681713172949, "pass@8": 0.34993805557826135, "pass@16": 0.395412438432603, "pass@32": 0.43308274288586, "pass@64": 0.4660027797221985, "pass@128": 0.4935483870967742 }, "19": { "pass@1": 0.15717050604229607, "pass@2": 0.21232374158479428, "pass@4": 0.2676054354610507, "pass@8": 0.31955934750096615, "pass@16": 0.3666705109285142, "pass@32": 0.4077764639729569, "pass@64": 0.44331074246388014, "pass@128": 0.4773413897280967 }, "20": { "pass@1": 0.14761636636636635, "pass@2": 0.20476615887541083, "pass@4": 0.2636930164134889, "pass@8": 0.32053535130427724, "pass@16": 0.3732904435639585, "pass@32": 0.4200946391152685, "pass@64": 0.4584333411955923, "pass@128": 0.4864864864864865 } } } } } ]