[ { "checkpoint": "checkpoint-38796", "id_acc": 0.7004678819444444, "ood_acc": 0.331778125, "total_acc": 0.5064206414473684, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 198.95842534722223, "ood_avg_resp_len": 371.70684140625, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 401802, "answer_accuracy": 0.49621848324375595, "avg_response_len": 276.35492288768575, "resp_tokens_sum": 223772319, "pass_at_k": { "pass@1": 0.49621848324375595, "pass@2": 0.6231529911240028, "pass@4": 0.740067849446409, "pass@8": 0.8317378367298127, "pass@16": 0.8949596717478773, "pass@32": 0.9350559220713367, "pass@64": 0.9600439120925355, "pass@128": 0.9753398672146696 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5095409292035398, "pass@2": 0.6676066418135783, "pass@4": 0.792585876710101, "pass@8": 0.8760817636863799, "pass@16": 0.9283330831148673, "pass@32": 0.9602328298448352, "pass@64": 0.9791876634794598, "pass@128": 0.9911504424778761 }, "2": { "pass@1": 0.9343360591900312, "pass@2": 0.9592853901439887, "pass@4": 0.9779000282674944, "pass@8": 0.9878251114374832, "pass@16": 0.9922969056587208, "pass@32": 0.9936688924295337, "pass@64": 0.9937693490770984, "pass@128": 0.9937694704049844 }, "3": { "pass@1": 0.9195575305810397, "pass@2": 0.9484343771822101, "pass@4": 0.9708153361563746, "pass@8": 0.984238780440124, "pass@16": 0.9897953555580769, "pass@32": 0.990792748570468, "pass@64": 0.9908256717392108, "pass@128": 0.9908256880733946 }, "4": { "pass@1": 0.7866455078125, "pass@2": 0.8459907111220479, "pass@4": 0.8906744697928376, "pass@8": 0.9215507509441558, "pass@16": 0.9437763487973065, "pass@32": 0.9611323349556897, "pass@64": 0.9735555459387797, "pass@128": 0.98125 }, "5": { "pass@1": 0.6709573412698413, "pass@2": 0.7741130014998127, "pass@4": 0.8445359746698321, "pass@8": 0.8894758925880082, "pass@16": 0.9211739208974751, "pass@32": 0.9452852732182677, "pass@64": 0.964295583575878, "pass@128": 0.9777777777777777 }, "6": { "pass@1": 0.7365503368263473, "pass@2": 0.8420378730727522, "pass@4": 0.9124382115970529, "pass@8": 0.9505378525143745, "pass@16": 0.9685954065179547, "pass@32": 0.97972634565819, "pass@64": 0.9880378752295537, "pass@128": 0.9940119760479041 }, "7": { "pass@1": 0.6175110946745562, "pass@2": 0.7572351983646276, "pass@4": 0.8602960354807709, "pass@8": 0.9199085087926454, "pass@16": 0.9515635930824008, "pass@32": 0.9699748728805815, "pass@64": 0.9825938231042854, "pass@128": 0.9911242603550295 }, "8": { "pass@1": 0.5553635817307693, "pass@2": 0.705602191853422, "pass@4": 0.821673709956928, "pass@8": 0.8943235864636814, "pass@16": 0.9343372725706356, "pass@32": 0.9560323516975169, "pass@64": 0.9697434965886538, "pass@128": 0.9807692307692307 }, "9": { "pass@1": 0.479137323943662, "pass@2": 0.6481472496395699, "pass@4": 0.7957091023833289, "pass@8": 0.8944526189705332, "pass@16": 0.9475698851465509, "pass@32": 0.9725689666299432, "pass@64": 0.9838555505224045, "pass@128": 0.9887323943661972 }, "11": { "pass@1": 0.4631506728045326, "pass@2": 0.6252648836742432, "pass@4": 0.767624272798053, "pass@8": 0.871077584229325, "pass@16": 0.9329364153366072, "pass@32": 0.9639094963103593, "pass@64": 0.9780315644820233, "pass@128": 0.9858356940509915 }, "12": { "pass@1": 0.4075962217514124, "pass@2": 0.5693932392455183, "pass@4": 0.7213146841814255, "pass@8": 0.8371093473559977, "pass@16": 0.9086267449422752, "pass@32": 0.9469433243538209, "pass@64": 0.9686512377816593, "pass@128": 0.9830508474576272 }, "13": { "pass@1": 0.39882934952978055, "pass@2": 0.5654986516673662, "pass@4": 0.7186619650600091, "pass@8": 0.8329572629488781, "pass@16": 0.9032880963070734, "pass@32": 0.9407221789825072, "pass@64": 0.9613466943723601, "pass@128": 0.9749216300940439 }, "14": { "pass@1": 0.35604069314641745, "pass@2": 0.5193354919420117, "pass@4": 0.684745257018106, "pass@8": 0.8188914601096546, "pass@16": 0.9064022768205324, "pass@32": 0.9520687996104888, "pass@64": 0.9726630069903832, "pass@128": 0.9813084112149533 }, "15": { "pass@1": 0.33921152456647397, "pass@2": 0.4931290820854757, "pass@4": 0.650143435556827, "pass@8": 0.7837146243177703, "pass@16": 0.8784636507395034, "pass@32": 0.9341449136359978, "pass@64": 0.9638817286290657, "pass@128": 0.9797687861271677 }, "16": { "pass@1": 0.3211945564516129, "pass@2": 0.47427451104902224, "pass@4": 0.6338670295245349, "pass@8": 0.7704789743204787, "pass@16": 0.8674678402262296, "pass@32": 0.9269334345131763, "pass@64": 0.95867323469044, "pass@128": 0.9774193548387097 }, "17": { "pass@1": 0.2943686777456647, "pass@2": 0.43410561080515186, "pass@4": 0.5843995006404546, "pass@8": 0.7225977142959461, "pass@16": 0.8319339455322081, "pass@32": 0.9085076669393349, "pass@64": 0.9564128585594077, "pass@128": 0.976878612716763 }, "18": { "pass@1": 0.2702678571428571, "pass@2": 0.40575260123734536, "pass@4": 0.5552940701162348, "pass@8": 0.6981102462106233, "pass@16": 0.8141778003682741, "pass@32": 0.894818854884761, "pass@64": 0.9449149724164841, "pass@128": 0.9685714285714285 }, "19": { "pass@1": 0.23252158408408408, "pass@2": 0.34998618204819004, "pass@4": 0.48075028543729326, "pass@8": 0.6082667201968596, "pass@16": 0.7195758859617793, "pass@32": 0.8085267342570868, "pass@64": 0.8769663601857598, "pass@128": 0.9279279279279279 }, "20": { "pass@1": 0.19352946696696696, "pass@2": 0.29999401468397513, "pass@4": 0.42863513513513496, "pass@8": 0.5593568000768633, "pass@16": 0.6727429737905991, "pass@32": 0.763426098355866, "pass@64": 0.8337823046158129, "pass@128": 0.8858858858858859 } } }, "teachers_in_school": { "count": 821120, "correct": 420450, "answer_accuracy": 0.5120445245518317, "avg_response_len": 286.99597988113794, "resp_tokens_sum": 235658139, "pass_at_k": { "pass@1": 0.5120445245518317, "pass@2": 0.6380669421753976, "pass@4": 0.7534741674827613, "pass@8": 0.8433371855207411, "pass@16": 0.904921209981468, "pass@32": 0.9434274968633595, "pass@64": 0.9665472920084971, "pass@128": 0.9800467653936087 }, "per_op_pass_at_k": { "10": { "pass@1": 0.4727521088957055, "pass@2": 0.6345907957345057, "pass@4": 0.77542852455943, "pass@8": 0.8752167459860357, "pass@16": 0.9336091179643451, "pass@32": 0.9627348671897347, "pass@64": 0.9780142800175672, "pass@128": 0.9877300613496932 }, "2": { "pass@1": 0.9381648936170213, "pass@2": 0.9569980614125363, "pass@4": 0.971669399322805, "pass@8": 0.9823670230919718, "pass@16": 0.9902618175090221, "pass@32": 0.9948116541729152, "pass@64": 0.9965635996372417, "pass@128": 0.9969604863221885 }, "3": { "pass@1": 0.950686553030303, "pass@2": 0.9672959914101645, "pass@4": 0.9803795730079196, "pass@8": 0.9893317365210512, "pass@16": 0.9939045470160155, "pass@32": 0.9959745105782435, "pass@64": 0.9968729838687728, "pass@128": 0.996969696969697 }, "4": { "pass@1": 0.8468703772189349, "pass@2": 0.9065033429623076, "pass@4": 0.9489256611044922, "pass@8": 0.9743967832425964, "pass@16": 0.9872230174014897, "pass@32": 0.9930567852310904, "pass@64": 0.9965815370797454, "pass@128": 1.0 }, "5": { "pass@1": 0.7758605072463768, "pass@2": 0.8736209774049982, "pass@4": 0.9324389600756424, "pass@8": 0.9618868319100985, "pass@16": 0.9760629639219559, "pass@32": 0.9825935008857577, "pass@64": 0.9869077034509552, "pass@128": 0.991304347826087 }, "6": { "pass@1": 0.6769340974212035, "pass@2": 0.7943660740473344, "pass@4": 0.8786454614018506, "pass@8": 0.9272978475082277, "pass@16": 0.9543528148099992, "pass@32": 0.9711407927719327, "pass@64": 0.9821259539329217, "pass@128": 0.9885386819484241 }, "7": { "pass@1": 0.6066428829479769, "pass@2": 0.7398836818988667, "pass@4": 0.8397892167380809, "pass@8": 0.9042434643738783, "pass@16": 0.9441212894901903, "pass@32": 0.96651931919803, "pass@64": 0.9778479793557311, "pass@128": 0.9826589595375722 }, "8": { "pass@1": 0.5825044014084507, "pass@2": 0.7344803565487412, "pass@4": 0.8506169114424071, "pass@8": 0.9214648743029239, "pass@16": 0.9596192806093881, "pass@32": 0.9801233610769453, "pass@64": 0.9908749402461413, "pass@128": 0.9971830985915493 }, "9": { "pass@1": 0.5183383233532934, "pass@2": 0.6750163550851055, "pass@4": 0.8066507348071007, "pass@8": 0.8964758234235036, "pass@16": 0.9496126362729699, "pass@32": 0.9775149761319307, "pass@64": 0.9920008501197732, "pass@128": 1.0 }, "11": { "pass@1": 0.4303934487951807, "pass@2": 0.5927442545773641, "pass@4": 0.7420111311387282, "pass@8": 0.851233626656646, "pass@16": 0.9172232851276387, "pass@32": 0.9531123039455416, "pass@64": 0.9715619962195704, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.41724627742946707, "pass@2": 0.577324331078913, "pass@4": 0.7282632558861173, "pass@8": 0.8475615268998387, "pass@16": 0.9247642940425248, "pass@32": 0.9665189848115742, "pass@64": 0.9858146148658699, "pass@128": 0.9937304075235109 }, "13": { "pass@1": 0.3948266806722689, "pass@2": 0.5571427882176495, "pass@4": 0.7088043327567247, "pass@8": 0.8258372669093875, "pass@16": 0.9026431000892041, "pass@32": 0.9463133172020857, "pass@64": 0.9691138082317794, "pass@128": 0.9831932773109243 }, "14": { "pass@1": 0.41015625, "pass@2": 0.5688616636736198, "pass@4": 0.7169494355968666, "pass@8": 0.831644961695096, "pass@16": 0.9084661438307753, "pass@32": 0.9549772171257439, "pass@64": 0.9812690835166658, "pass@128": 0.9941520467836257 }, "15": { "pass@1": 0.33780776515151517, "pass@2": 0.4990828561202573, "pass@4": 0.6665392868505067, "pass@8": 0.807065398160388, "pass@16": 0.8994904127312967, "pass@32": 0.9489088718389327, "pass@64": 0.9717879055264091, "pass@128": 0.9787878787878788 }, "16": { "pass@1": 0.32298344017094016, "pass@2": 0.4738822711263657, "pass@4": 0.6295656763524213, "pass@8": 0.7637132042842444, "pass@16": 0.8619682864031663, "pass@32": 0.9226160339647616, "pass@64": 0.9550762847040853, "pass@128": 0.9743589743589743 }, "17": { "pass@1": 0.32038528726708076, "pass@2": 0.4652963301462315, "pass@4": 0.6150197741198496, "pass@8": 0.7468171297086604, "pass@16": 0.8433732629472633, "pass@32": 0.9032106328795099, "pass@64": 0.9403173018208404, "pass@128": 0.968944099378882 }, "18": { "pass@1": 0.2850873161764706, "pass@2": 0.4292677454840204, "pass@4": 0.5829545498908223, "pass@8": 0.7183212578697612, "pass@16": 0.8203104066913215, "pass@32": 0.8889514779071954, "pass@64": 0.932731144763619, "pass@128": 0.9558823529411765 }, "19": { "pass@1": 0.24262927827380953, "pass@2": 0.3626660925196849, "pass@4": 0.4944864068330747, "pass@8": 0.621680543117198, "pass@16": 0.7348198506994026, "pass@32": 0.8285702067226429, "pass@64": 0.8994453819696483, "pass@128": 0.9434523809523809 }, "20": { "pass@1": 0.19344124251497005, "pass@2": 0.3052857130227736, "pass@4": 0.43797066861777034, "pass@8": 0.5705444770411404, "pass@16": 0.6877959075104455, "pass@32": 0.7850349116495576, "pass@64": 0.8579923825438611, "pass@128": 0.907185628742515 } } }, "movie_festival_awards": { "count": 801152, "correct": 409363, "answer_accuracy": 0.5109679561431538, "avg_response_len": 306.50164388280876, "resp_tokens_sum": 245554405, "pass_at_k": { "pass@1": 0.5109679561431538, "pass@2": 0.6347774558965905, "pass@4": 0.7470422575742857, "pass@8": 0.8347069618219127, "pass@16": 0.8956642572378208, "pass@32": 0.9350350049432259, "pass@64": 0.9601454722895479, "pass@128": 0.975235660648666 }, "per_op_pass_at_k": { "10": { "pass@1": 0.5484141791044777, "pass@2": 0.7068614848983433, "pass@4": 0.8289243484489802, "pass@8": 0.9026957799440277, "pass@16": 0.9424595065592135, "pass@32": 0.9669448671204803, "pass@64": 0.9840754473966935, "pass@128": 0.9970149253731343 }, "2": { "pass@1": 0.9351785714285714, "pass@2": 0.9614377109111365, "pass@4": 0.9807448331458566, "pass@8": 0.9900186920873438, "pass@16": 0.9932031478280128, "pass@32": 0.994165464267091, "pass@64": 0.994284838023708, "pass@128": 0.9942857142857143 }, "3": { "pass@1": 0.9484784985422741, "pass@2": 0.9667359421271322, "pass@4": 0.9812006491171111, "pass@8": 0.9911249057844892, "pass@16": 0.9963589159034837, "pass@32": 0.9986878666696775, "pass@64": 0.9998071628884445, "pass@128": 1.0 }, "4": { "pass@1": 0.7850420321637427, "pass@2": 0.8495130542892672, "pass@4": 0.8935691234867573, "pass@8": 0.9234471775450245, "pass@16": 0.9452483337671119, "pass@32": 0.961587906311577, "pass@64": 0.9729863836696224, "pass@128": 0.97953216374269 }, "5": { "pass@1": 0.7065027573529412, "pass@2": 0.8072114549559978, "pass@4": 0.8746589903284145, "pass@8": 0.9123061051506803, "pass@16": 0.9350173413401953, "pass@32": 0.9502013517528385, "pass@64": 0.9608592051459295, "pass@128": 0.9705882352941176 }, "6": { "pass@1": 0.6834828470031545, "pass@2": 0.7923822157778388, "pass@4": 0.8667003048120562, "pass@8": 0.9103088471153025, "pass@16": 0.9358442713049175, "pass@32": 0.9527734551605721, "pass@64": 0.9654646330828386, "pass@128": 0.9747634069400631 }, "7": { "pass@1": 0.6329855617088608, "pass@2": 0.7641914494667598, "pass@4": 0.8574156207135493, "pass@8": 0.9165753219580909, "pass@16": 0.9548959002581976, "pass@32": 0.9780368342011752, "pass@64": 0.990102927075729, "pass@128": 0.9968354430379747 }, "8": { "pass@1": 0.5770457957957958, "pass@2": 0.7276407805443237, "pass@4": 0.8416383075989373, "pass@8": 0.911326672634802, "pass@16": 0.949508092764589, "pass@32": 0.9704760825523712, "pass@64": 0.98276801411379, "pass@128": 0.987987987987988 }, "9": { "pass@1": 0.5122588424437299, "pass@2": 0.6717535509026002, "pass@4": 0.8050184239429874, "pass@8": 0.8946679048869003, "pass@16": 0.9452181368847813, "pass@32": 0.9728178175087213, "pass@64": 0.9876158326332224, "pass@128": 0.9935691318327974 }, "11": { "pass@1": 0.4708829365079365, "pass@2": 0.6231510123734529, "pass@4": 0.7544366772605811, "pass@8": 0.851528774724346, "pass@16": 0.9152272927693456, "pass@32": 0.9535655892565871, "pass@64": 0.9750103454894454, "pass@128": 0.9841269841269841 }, "12": { "pass@1": 0.43018922018348627, "pass@2": 0.593299260757543, "pass@4": 0.7401128262063567, "pass@8": 0.849012069087901, "pass@16": 0.916627492993055, "pass@32": 0.9528701633670399, "pass@64": 0.9721467280169921, "pass@128": 0.981651376146789 }, "13": { "pass@1": 0.39754533179012347, "pass@2": 0.5523874945319335, "pass@4": 0.6953536343258018, "pass@8": 0.8065616697672022, "pass@16": 0.8807671612053861, "pass@32": 0.9246210517968243, "pass@64": 0.9498011940741536, "pass@128": 0.9660493827160493 }, "14": { "pass@1": 0.38318341988130566, "pass@2": 0.5392996185658544, "pass@4": 0.6927449403579738, "pass@8": 0.8184434802350886, "pass@16": 0.9013070276922327, "pass@32": 0.9460098896212308, "pass@64": 0.9683616043843487, "pass@128": 0.9792284866468842 }, "15": { "pass@1": 0.3361304012345679, "pass@2": 0.4982395863711481, "pass@4": 0.6646675548658267, "pass@8": 0.8041232007332922, "pass@16": 0.8965907609063177, "pass@32": 0.9461555988848293, "pass@64": 0.9736658319254623, "pass@128": 0.9907407407407407 }, "16": { "pass@1": 0.3105872050147493, "pass@2": 0.4608186421387594, "pass@4": 0.6136408792374404, "pass@8": 0.7452051705964161, "pass@16": 0.844733132645075, "pass@32": 0.9102423990647945, "pass@64": 0.9475723541577755, "pass@128": 0.9646017699115044 }, "17": { "pass@1": 0.28572100903614456, "pass@2": 0.4279750646285927, "pass@4": 0.5799197509949807, "pass@8": 0.7145075723252433, "pass@16": 0.8179494250317864, "pass@32": 0.8902762163574444, "pass@64": 0.9388825184567935, "pass@128": 0.9698795180722891 }, "18": { "pass@1": 0.26066028225806454, "pass@2": 0.39178943357886703, "pass@4": 0.5395992352770422, "pass@8": 0.6798815082751558, "pass@16": 0.7896970588371004, "pass@32": 0.8658843755573179, "pass@64": 0.9173571118838663, "pass@128": 0.9451612903225807 }, "19": { "pass@1": 0.24596393504531722, "pass@2": 0.37129121369269935, "pass@4": 0.5090978352970231, "pass@8": 0.6405254066302252, "pass@16": 0.7512558089644261, "pass@32": 0.8355988987470783, "pass@64": 0.8976115860640348, "pass@128": 0.9395770392749244 }, "20": { "pass@1": 0.20038006756756757, "pass@2": 0.3129104744508288, "pass@4": 0.44692457558796145, "pass@8": 0.5810482888254881, "pass@16": 0.6969763264168647, "pass@32": 0.7902839304149739, "pass@64": 0.8623890297183104, "pass@128": 0.9129129129129129 } } } } } ]