[ { "checkpoint": "checkpoint-85656", "id_acc": 0.7683654513888889, "ood_acc": 0.2703328125, "total_acc": 0.5062430098684211, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.30304166666667, "ood_avg_resp_len": 329.4546515625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 401731, "answer_accuracy": 0.4961307994783433, "avg_response_len": 254.04621057935503, "resp_tokens_sum": 205708330, "pass_at_k": { "pass@1": 0.4961307994783433, "pass@2": 0.6048744510842731, "pass@4": 0.6990247595907915, "pass@8": 0.7713986345627221, "pass@16": 0.8238616552594298, "pass@32": 0.8621028325163171, "pass@64": 0.8908891087473675, "pass@128": 0.912741068605754 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5576373525073747, "pass@2": 0.7241162056070428, "pass@4": 0.8478356140216987, "pass@8": 0.9210285105668401, "pass@16": 0.9590456424231748, "pass@32": 0.9787901271304692, "pass@64": 0.990491462519074, "pass@128": 0.9970501474926253 }, "2": { "pass@1": 0.9872225467289719, "pass@2": 0.992541071822798, "pass@4": 0.9944339238553123, "pass@8": 0.9952862602378583, "pass@16": 0.9961075311262445, "pass@32": 0.9967298286406606, "pass@64": 0.996882655029806, "pass@128": 0.9968847352024922 }, "3": { "pass@1": 0.9942660550458715, "pass@2": 0.9974246159310364, "pass@4": 0.9983950320430129, "pass@8": 0.9991873729741889, "pass@16": 0.999804553543206, "pass@32": 0.9999921788300652, "pass@64": 0.999999998785431, "pass@128": 1.0 }, "4": { "pass@1": 0.91865234375, "pass@2": 0.9625246062992125, "pass@4": 0.982705639822365, "pass@8": 0.9912215061732457, "pass@16": 0.9953870393196059, "pass@32": 0.9976677753977258, "pass@64": 0.9991816802247045, "pass@128": 1.0 }, "5": { "pass@1": 0.7051587301587302, "pass@2": 0.7998789213848272, "pass@4": 0.8623743475517939, "pass@8": 0.902292682084196, "pass@16": 0.9335744672132729, "pass@32": 0.961012724109928, "pass@64": 0.9823888277410862, "pass@128": 0.9936507936507937 }, "6": { "pass@1": 0.8122427020958084, "pass@2": 0.9102172132113728, "pass@4": 0.9629234422044544, "pass@8": 0.9852725468337608, "pass@16": 0.9928515702293308, "pass@32": 0.9955259002838712, "pass@64": 0.9966392296552004, "pass@128": 0.9970059880239521 }, "7": { "pass@1": 0.6823687130177515, "pass@2": 0.83220396729255, "pass@4": 0.9268846605897633, "pass@8": 0.9704636354507595, "pass@16": 0.9878959614025046, "pass@32": 0.9961612605932577, "pass@64": 0.9994934131512692, "pass@128": 1.0 }, "8": { "pass@1": 0.6389723557692307, "pass@2": 0.7958146420098927, "pass@4": 0.9043729308475859, "pass@8": 0.9630561425068057, "pass@16": 0.9881072533959968, "pass@32": 0.9960452808281866, "pass@64": 0.9983311703038567, "pass@128": 1.0 }, "9": { "pass@1": 0.5331646126760563, "pass@2": 0.7045795441942995, "pass@4": 0.8440219997148238, "pass@8": 0.9321960354746593, "pass@16": 0.9737132718388507, "pass@32": 0.9898040713041135, "pass@64": 0.9958648530897064, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.47826664305949007, "pass@2": 0.6444870736766972, "pass@4": 0.7803746312872358, "pass@8": 0.8730347091383628, "pass@16": 0.9291473364437194, "pass@32": 0.9599139448333401, "pass@64": 0.9753934096770958, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.4042858403954802, "pass@2": 0.5600105237110191, "pass@4": 0.7015747594580333, "pass@8": 0.8112545401764185, "pass@16": 0.8858790711888753, "pass@32": 0.931855418213947, "pass@64": 0.9582409337387682, "pass@128": 0.9717514124293786 }, "13": { "pass@1": 0.3693181818181818, "pass@2": 0.5208440037765657, "pass@4": 0.6606441350983167, "pass@8": 0.7731861628772339, "pass@16": 0.8562613390543541, "pass@32": 0.9116335367856937, "pass@64": 0.9451118730416904, "pass@128": 0.9623824451410659 }, "14": { "pass@1": 0.29650992990654207, "pass@2": 0.4420015729634261, "pass@4": 0.5948001467690369, "pass@8": 0.727273434984605, "pass@16": 0.8239426336095855, "pass@32": 0.8852793296906377, "pass@64": 0.921121409033376, "pass@128": 0.940809968847352 }, "15": { "pass@1": 0.23956828034682082, "pass@2": 0.35823891436438904, "pass@4": 0.48848129271052076, "pass@8": 0.6136605434268556, "pass@16": 0.7200001692318181, "pass@32": 0.8001646916555948, "pass@64": 0.8558966508480305, "pass@128": 0.8930635838150289 }, "16": { "pass@1": 0.21872479838709677, "pass@2": 0.31988625539751075, "pass@4": 0.42930173748442724, "pass@8": 0.535122776781214, "pass@16": 0.631811148947321, "pass@32": 0.7182628887583782, "pass@64": 0.7924879547356923, "pass@128": 0.8483870967741935 }, "17": { "pass@1": 0.18007135115606937, "pass@2": 0.27148997542214703, "pass@4": 0.37336469203994, "pass@8": 0.47328118727751295, "pass@16": 0.564995564823014, "pass@32": 0.6460617228269897, "pass@64": 0.7156147681114684, "pass@128": 0.7774566473988439 }, "18": { "pass@1": 0.16145089285714287, "pass@2": 0.24092589988751387, "pass@4": 0.3327022473083723, "pass@8": 0.42501536664350503, "pass@16": 0.5082382591955663, "pass@32": 0.5857258779369915, "pass@64": 0.659598059312317, "pass@128": 0.7257142857142858 }, "19": { "pass@1": 0.16737049549549549, "pass@2": 0.24731769170745538, "pass@4": 0.3329046515581949, "pass@8": 0.4120433778670679, "pass@16": 0.4800331911673055, "pass@32": 0.5407885727783139, "pass@64": 0.5994796591091415, "pass@128": 0.6546546546546547 }, "20": { "pass@1": 0.15202702702702703, "pass@2": 0.22522079165779937, "pass@4": 0.3063820921596512, "pass@8": 0.38469000130086534, "pass@16": 0.4517700634104969, "pass@32": 0.5096781724410925, "pass@64": 0.5629976979045833, "pass@128": 0.6186186186186187 } } }, "teachers_in_school": { "count": 821120, "correct": 419277, "answer_accuracy": 0.51061598791894, "avg_response_len": 264.6041577357755, "resp_tokens_sum": 217271766, "pass_at_k": { "pass@1": 0.51061598791894, "pass@2": 0.6185059891310457, "pass@4": 0.7112559664891815, "pass@8": 0.7814887206768086, "pass@16": 0.8317485000078719, "pass@32": 0.8676199270006473, "pass@64": 0.8943931087557251, "pass@128": 0.9159781761496493 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5151457055214724, "pass@2": 0.6830678469639145, "pass@4": 0.8189656213410438, "pass@8": 0.9071709306839472, "pass@16": 0.9537003905148077, "pass@32": 0.9732703621233486, "pass@64": 0.9815479508426936, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 0.9870820668693009, "pass@2": 0.9916114209128114, "pass@4": 0.9932387670766079, "pass@8": 0.9939459341067186, "pass@16": 0.9942939216780183, "pass@32": 0.9946808461232393, "pass@64": 0.9954407294832827, "pass@128": 0.9969604863221885 }, "3": { "pass@1": 0.9931344696969697, "pass@2": 0.9978040742066334, "pass@4": 0.9991804419333946, "pass@8": 0.9998026414887873, "pass@16": 0.9999897641404931, "pass@32": 0.9999999883510501, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9130917159763313, "pass@2": 0.9530991561058564, "pass@4": 0.9734309767307188, "pass@8": 0.981508303989163, "pass@16": 0.9852970136399636, "pass@32": 0.9883859742602302, "pass@64": 0.9910152794130401, "pass@128": 0.9940828402366864 }, "5": { "pass@1": 0.7668704710144928, "pass@2": 0.8570691401346572, "pass@4": 0.9105642180596992, "pass@8": 0.9386731501468899, "pass@16": 0.9567025653999209, "pass@32": 0.9706738161299903, "pass@64": 0.980097445623169, "pass@128": 0.9855072463768116 }, "6": { "pass@1": 0.8068812679083095, "pass@2": 0.9043202738984274, "pass@4": 0.9584084456312574, "pass@8": 0.9802832598182536, "pass@16": 0.9885671782379833, "pass@32": 0.9930770664664508, "pass@64": 0.9960588198590073, "pass@128": 0.997134670487106 }, "7": { "pass@1": 0.7005735187861272, "pass@2": 0.8407154601520194, "pass@4": 0.9295946147085653, "pass@8": 0.9719966626529152, "pass@16": 0.9888768016805433, "pass@32": 0.9956843233080916, "pass@64": 0.9985466054756086, "pass@128": 1.0 }, "8": { "pass@1": 0.6794454225352112, "pass@2": 0.8303634800931575, "pass@4": 0.925194801354056, "pass@8": 0.9693069551068682, "pass@16": 0.9866333068822101, "pass@32": 0.9936383088194044, "pass@64": 0.9963589210449596, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.5826160179640718, "pass@2": 0.7489468651280118, "pass@4": 0.8742062343254993, "pass@8": 0.9464091288357557, "pass@16": 0.978296787242704, "pass@32": 0.9903044676743343, "pass@64": 0.995371590764158, "pass@128": 0.9970059880239521 }, "11": { "pass@1": 0.4816453313253012, "pass@2": 0.6459794641163077, "pass@4": 0.7857084705526265, "pass@8": 0.8833762506171791, "pass@16": 0.9416448761628213, "pass@32": 0.969734804750012, "pass@64": 0.9807077406498477, "pass@128": 0.9849397590361446 }, "12": { "pass@1": 0.43245493730407525, "pass@2": 0.5923997389726756, "pass@4": 0.7327143862111276, "pass@8": 0.8373373792935619, "pass@16": 0.9062656469918041, "pass@32": 0.9467871615968801, "pass@64": 0.9707780412767565, "pass@128": 0.9843260188087775 }, "13": { "pass@1": 0.3606004901960784, "pass@2": 0.5023848121925938, "pass@4": 0.637892442016176, "pass@8": 0.7521662568425521, "pass@16": 0.838171767362744, "pass@32": 0.896080984422846, "pass@64": 0.9310394842754216, "pass@128": 0.9523809523809523 }, "14": { "pass@1": 0.3340643274853801, "pass@2": 0.47814751980015674, "pass@4": 0.6160534801570856, "pass@8": 0.7248460344568162, "pass@16": 0.8047331801838976, "pass@32": 0.8650785721351549, "pass@64": 0.9105640305812829, "pass@128": 0.9502923976608187 }, "15": { "pass@1": 0.24607007575757575, "pass@2": 0.3718403274874728, "pass@4": 0.5075580916021858, "pass@8": 0.6319157591570926, "pass@16": 0.7326543407650808, "pass@32": 0.8093295131299464, "pass@64": 0.862885345057764, "pass@128": 0.8939393939393939 }, "16": { "pass@1": 0.22097578347578348, "pass@2": 0.3284331045157817, "pass@4": 0.44146620401082326, "pass@8": 0.5452602848164503, "pass@16": 0.6349042776932105, "pass@32": 0.7107435949774508, "pass@64": 0.7762919863833435, "pass@128": 0.8347578347578347 }, "17": { "pass@1": 0.19870923913043478, "pass@2": 0.29929016175967105, "pass@4": 0.41198660736119147, "pass@8": 0.5218542528506436, "pass@16": 0.6189107868220821, "pass@32": 0.699994055890072, "pass@64": 0.7651703780769352, "pass@128": 0.8229813664596274 }, "18": { "pass@1": 0.1825827205882353, "pass@2": 0.2713991575961092, "pass@4": 0.37012361791173165, "pass@8": 0.4668118891358911, "pass@16": 0.5509026207660206, "pass@32": 0.6187246923957278, "pass@64": 0.6751167026493657, "pass@128": 0.7235294117647059 }, "19": { "pass@1": 0.15364583333333334, "pass@2": 0.232394046447319, "pass@4": 0.3216450493911474, "pass@8": 0.40932489549476053, "pass@16": 0.48750145304638565, "pass@32": 0.5560887876551346, "pass@64": 0.6210566599372096, "pass@128": 0.6845238095238095 }, "20": { "pass@1": 0.13440306886227546, "pass@2": 0.20725723749351688, "pass@4": 0.2918240431398169, "pass@8": 0.3752272375730725, "pass@16": 0.4466647823075647, "pass@32": 0.5053889559802511, "pass@64": 0.5591910380893472, "pass@128": 0.6137724550898204 } } }, "movie_festival_awards": { "count": 801152, "correct": 410175, "answer_accuracy": 0.5119814966448314, "avg_response_len": 282.1124605567982, "resp_tokens_sum": 226014962, "pass_at_k": { "pass@1": 0.5119814966448314, "pass@2": 0.6187177275746641, "pass@4": 0.7097963115640613, "pass@8": 0.7793747112681776, "pass@16": 0.8297170854027788, "pass@32": 0.8659860710520697, "pass@64": 0.8924916218421354, "pass@128": 0.911487458060393 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5736707089552239, "pass@2": 0.7412574186155834, "pass@4": 0.8605538192054352, "pass@8": 0.9287938948335882, "pass@16": 0.9642202420966983, "pass@32": 0.981588424107176, "pass@64": 0.9914843834371672, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 0.9918080357142857, "pass@2": 0.9940755061867266, "pass@4": 0.9945367877229631, "pass@8": 0.9947923572167008, "pass@16": 0.9952352830896093, "pass@32": 0.9959470244790829, "pass@64": 0.9967941507311587, "pass@128": 0.9971428571428571 }, "3": { "pass@1": 0.9963556851311953, "pass@2": 0.9996294707421778, "pass@4": 0.9999885538177989, "pass@8": 0.9999999832313435, "pass@16": 0.9999999999999907, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9171692251461988, "pass@2": 0.958932302458904, "pass@4": 0.9793460307593129, "pass@8": 0.9885088959718029, "pass@16": 0.9932516650589491, "pass@32": 0.9958818709342699, "pass@64": 0.9978083878929018, "pass@128": 1.0 }, "5": { "pass@1": 0.7477251838235294, "pass@2": 0.8461512853172767, "pass@4": 0.9096851661924609, "pass@8": 0.9461026739852725, "pass@16": 0.9682210886581221, "pass@32": 0.9815460794075175, "pass@64": 0.9892115253749012, "pass@128": 0.9941176470588236 }, "6": { "pass@1": 0.7878795347003155, "pass@2": 0.892085853473757, "pass@4": 0.9516247414065356, "pass@8": 0.9790252353686714, "pass@16": 0.9903265297387072, "pass@32": 0.9949012956400033, "pass@64": 0.9966670077039709, "pass@128": 0.9968454258675079 }, "7": { "pass@1": 0.7115308544303798, "pass@2": 0.8487748243297122, "pass@4": 0.9288619071943541, "pass@8": 0.9670096606637208, "pass@16": 0.9851787163788448, "pass@32": 0.9937667701276847, "pass@64": 0.9979326719887415, "pass@128": 1.0 }, "8": { "pass@1": 0.6593937687687688, "pass@2": 0.8175867796930787, "pass@4": 0.9244497847115954, "pass@8": 0.9776872582318384, "pass@16": 0.9956130939609944, "pass@32": 0.9995346162382196, "pass@64": 0.9999935554756175, "pass@128": 1.0 }, "9": { "pass@1": 0.5820438102893891, "pass@2": 0.7568177646403522, "pass@4": 0.8844209747093505, "pass@8": 0.9544061657954517, "pass@16": 0.9843662998236358, "pass@32": 0.994375402806163, "pass@64": 0.9965623982598637, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.5126736111111111, "pass@2": 0.6844453037120355, "pass@4": 0.8225070392986582, "pass@8": 0.9125995836439618, "pass@16": 0.9607014358711359, "pass@32": 0.9828190225686912, "pass@64": 0.9912457127693258, "pass@128": 0.9936507936507937 }, "12": { "pass@1": 0.42168386850152906, "pass@2": 0.579340265236341, "pass@4": 0.7219303501626515, "pass@8": 0.8325867532211753, "pass@16": 0.9081081464094839, "pass@32": 0.9536663136810352, "pass@64": 0.979233886374294, "pass@128": 0.9938837920489296 }, "13": { "pass@1": 0.3392168209876543, "pass@2": 0.4790998347428793, "pass@4": 0.617912207154661, "pass@8": 0.739523887323168, "pass@16": 0.831664539596371, "pass@32": 0.8908797971355734, "pass@64": 0.9276706490828273, "pass@128": 0.9537037037037037 }, "14": { "pass@1": 0.30679710682492584, "pass@2": 0.44024247061847227, "pass@4": 0.5768535646694014, "pass@8": 0.7013181044196499, "pass@16": 0.79981264699903, "pass@32": 0.8664581226716184, "pass@64": 0.9075799547708441, "pass@128": 0.9317507418397626 }, "15": { "pass@1": 0.2421875, "pass@2": 0.3661280621172356, "pass@4": 0.49944677488230654, "pass@8": 0.6221108065117973, "pass@16": 0.7245293882413184, "pass@32": 0.8057210991862905, "pass@64": 0.8623710413077329, "pass@128": 0.8888888888888888 }, "16": { "pass@1": 0.21748248525073746, "pass@2": 0.31926890983206774, "pass@4": 0.4301104165076712, "pass@8": 0.5368859964094961, "pass@16": 0.6322487597568095, "pass@32": 0.7150043647397045, "pass@64": 0.7834897476011347, "pass@128": 0.8348082595870207 }, "17": { "pass@1": 0.18879423945783133, "pass@2": 0.2847512095626601, "pass@4": 0.3888223292495063, "pass@8": 0.4851540273539716, "pass@16": 0.5705429202852202, "pass@32": 0.6480252422670163, "pass@64": 0.7148879755094221, "pass@128": 0.7680722891566265 }, "18": { "pass@1": 0.17016129032258065, "pass@2": 0.24719527876555752, "pass@4": 0.3322299300289077, "pass@8": 0.4185873134747127, "pass@16": 0.501237620938696, "pass@32": 0.578221940804949, "pass@64": 0.6455829940602361, "pass@128": 0.7032258064516129 }, "19": { "pass@1": 0.16800415407854985, "pass@2": 0.2506746288983512, "pass@4": 0.3383495438259038, "pass@8": 0.4175809793013465, "pass@16": 0.4823134744100392, "pass@32": 0.5365725037494888, "pass@64": 0.5847067346240746, "pass@128": 0.6314199395770392 }, "20": { "pass@1": 0.13339902402402404, "pass@2": 0.20877816793171128, "pass@4": 0.29870241332446046, "pass@8": 0.38866794275716215, "pass@16": 0.4652876620865009, "pass@32": 0.5298770833437524, "pass@64": 0.5867393530231473, "pass@128": 0.6306306306306306 } } } } } ]