[ { "checkpoint": "checkpoint-85656", "id_acc": 0.99334375, "ood_acc": 0.4865421875, "total_acc": 0.7266060855263158, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.87361805555557, "ood_avg_resp_len": 336.17079375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 585874, "answer_accuracy": 0.7235442024976289, "avg_response_len": 258.13694228185267, "resp_tokens_sum": 209020710, "pass_at_k": { "pass@1": 0.7235442024976289, "pass@2": 0.7601259044351346, "pass@4": 0.7915020775787805, "pass@8": 0.8180794280332532, "pass@16": 0.8402727252631793, "pass@32": 0.8592362336703517, "pass@64": 0.8759766756894397, "pass@128": 0.8910844135314575 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9606148598820059, "pass@2": 0.970110604952036, "pass@4": 0.9766451804143947, "pass@8": 0.9812420969145403, "pass@16": 0.984712123290507, "pass@32": 0.9877358738401864, "pass@64": 0.9911736613911921, "pass@128": 0.9941002949852508 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999951171875, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9963293650793651, "pass@2": 0.9968914041994751, "pass@4": 0.9970213660792401, "pass@8": 0.997211286089119, "pass@16": 0.9975721784776902, "pass@32": 0.9982189726284215, "pass@64": 0.9992125984251968, "pass@128": 1.0 }, "6": { "pass@1": 0.9999532185628742, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9983357988165681, "pass@2": 0.9996622093835906, "pass@4": 0.999980712377669, "pass@8": 0.9999999358582831, "pass@16": 0.9999999999997683, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9995993589743589, "pass@2": 0.9999917190339189, "pass@4": 0.9999999969955679, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9967209507042254, "pass@2": 0.9971629976710658, "pass@4": 0.9971830114545541, "pass@8": 0.9971830985912242, "pass@16": 0.9971830985915493, "pass@32": 0.9971830985915493, "pass@64": 0.9971830985915493, "pass@128": 0.9971830985915493 }, "11": { "pass@1": 0.9175593130311614, "pass@2": 0.9301508721643506, "pass@4": 0.9383426105022994, "pass@8": 0.94506009266078, "pass@16": 0.9523354989438444, "pass@32": 0.9605310994724667, "pass@64": 0.9676012340007842, "pass@128": 0.9716713881019831 }, "12": { "pass@1": 0.8390271892655368, "pass@2": 0.8704061290537837, "pass@4": 0.8879430150680314, "pass@8": 0.8998900279443047, "pass@16": 0.9101521093680011, "pass@32": 0.9205997134915045, "pass@64": 0.9318575609376857, "pass@128": 0.943502824858757 }, "13": { "pass@1": 0.7098354231974922, "pass@2": 0.7657214196677613, "pass@4": 0.8028330453599253, "pass@8": 0.8308064695801197, "pass@16": 0.8536985149124111, "pass@32": 0.8754225625963826, "pass@64": 0.8965705154324992, "pass@128": 0.9153605015673981 }, "14": { "pass@1": 0.6602414330218068, "pass@2": 0.742746507591925, "pass@4": 0.7996196558958166, "pass@8": 0.8367759768205015, "pass@16": 0.8619997000724972, "pass@32": 0.8821281369592076, "pass@64": 0.8977516145766892, "pass@128": 0.9096573208722741 }, "15": { "pass@1": 0.5442557803468208, "pass@2": 0.6445744532565655, "pass@4": 0.7226312514765129, "pass@8": 0.7833987025767928, "pass@16": 0.8303731247677375, "pass@32": 0.8669906303856226, "pass@64": 0.896652764809943, "pass@128": 0.9190751445086706 }, "16": { "pass@1": 0.32200100806451615, "pass@2": 0.41802014541529064, "pass@4": 0.5057130259927185, "pass@8": 0.579146486298523, "pass@16": 0.6372956308256016, "pass@32": 0.6853502169400502, "pass@64": 0.7263549298081786, "pass@128": 0.7612903225806451 }, "17": { "pass@1": 0.266573338150289, "pass@2": 0.3552971815119928, "pass@4": 0.44027677365835044, "pass@8": 0.5145766287777432, "pass@16": 0.5764773175385587, "pass@32": 0.6297498942971839, "pass@64": 0.6792948234940933, "pass@128": 0.7283236994219653 }, "18": { "pass@1": 0.22410714285714287, "pass@2": 0.3040772637795276, "pass@4": 0.3812210715624831, "pass@8": 0.45209828595059937, "pass@16": 0.5148515801438031, "pass@32": 0.5696556136715276, "pass@64": 0.6196317222927293, "pass@128": 0.6714285714285714 }, "19": { "pass@1": 0.17450262762762764, "pass@2": 0.24425742179187057, "pass@4": 0.3161038088144387, "pass@8": 0.38331538160776063, "pass@16": 0.44096227160841417, "pass@32": 0.48847477835581016, "pass@64": 0.5293757971334146, "pass@128": 0.5675675675675675 }, "20": { "pass@1": 0.16915352852852852, "pass@2": 0.2322273060462035, "pass@4": 0.29948549364887955, "pass@8": 0.36665142621617874, "pass@16": 0.42865530956555087, "pass@32": 0.4819577047980443, "pass@64": 0.526854050384028, "pass@128": 0.5645645645645646 } } }, "teachers_in_school": { "count": 821120, "correct": 600278, "answer_accuracy": 0.7310478371005456, "avg_response_len": 267.8073351032736, "resp_tokens_sum": 219901959, "pass_at_k": { "pass@1": 0.7310478371005456, "pass@2": 0.7680041011777181, "pass@4": 0.7996512798526805, "pass@8": 0.8257688373431301, "pass@16": 0.846615880063671, "pass@32": 0.8637151935821454, "pass@64": 0.8785491835920611, "pass@128": 0.8916601714731099 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9403278374233128, "pass@2": 0.952113575431139, "pass@4": 0.9607398866169331, "pass@8": 0.9662308127069366, "pass@16": 0.9701134353995957, "pass@32": 0.9738990670749782, "pass@64": 0.9783020822793281, "pass@128": 0.9846625766871165 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999526515151516, "pass@2": 0.9999996271772846, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9997735507246377, "pass@2": 0.9999942941914869, "pass@4": 0.9999999959244225, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9994851361031518, "pass@2": 0.9999626322676716, "pass@4": 0.999999727111475, "pass@8": 0.9999999999939815, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9995032514450867, "pass@2": 0.9999455958991397, "pass@4": 0.9999991709851297, "pass@8": 0.9999999999115423, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9968089788732394, "pass@2": 0.9990167877342796, "pass@4": 0.9998786036428545, "pass@8": 0.9999981102949981, "pass@16": 0.9999999997381142, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9952984655688623, "pass@2": 0.9967109334480646, "pass@4": 0.9979339827094966, "pass@8": 0.9989726884046817, "pass@16": 0.999694545217648, "pass@32": 0.9999787580173054, "pass@64": 0.9999999843303174, "pass@128": 1.0 }, "11": { "pass@1": 0.9503247364457831, "pass@2": 0.9617861208613987, "pass@4": 0.9686446664046513, "pass@8": 0.9728691216772047, "pass@16": 0.9758402586096212, "pass@32": 0.9778266280336569, "pass@64": 0.9788090621047877, "pass@128": 0.9789156626506024 }, "12": { "pass@1": 0.8506073667711599, "pass@2": 0.8813039178782122, "pass@4": 0.9007288483532033, "pass@8": 0.9156088842870286, "pass@16": 0.92726192452425, "pass@32": 0.9371659829216429, "pass@64": 0.9472018257825825, "pass@128": 0.9592476489028213 }, "13": { "pass@1": 0.7220325630252101, "pass@2": 0.7833849583140341, "pass@4": 0.8213891459470926, "pass@8": 0.8472609166913442, "pass@16": 0.8663873407991339, "pass@32": 0.8816572307661266, "pass@64": 0.8938391501339014, "pass@128": 0.9019607843137255 }, "14": { "pass@1": 0.6827713815789473, "pass@2": 0.7667430757010633, "pass@4": 0.8250682096755444, "pass@8": 0.8622464010462382, "pass@16": 0.8840158404678026, "pass@32": 0.8985840968140026, "pass@64": 0.911342591360829, "pass@128": 0.9239766081871345 }, "15": { "pass@1": 0.5276751893939394, "pass@2": 0.6309297452875214, "pass@4": 0.7133767940939197, "pass@8": 0.7782515948523318, "pass@16": 0.8285203684912462, "pass@32": 0.8654085228745337, "pass@64": 0.8911681455877384, "pass@128": 0.906060606060606 }, "16": { "pass@1": 0.3476451210826211, "pass@2": 0.4371421220584603, "pass@4": 0.5146949160735249, "pass@8": 0.5768380160414723, "pass@16": 0.6278066916101048, "pass@32": 0.6740581101475652, "pass@64": 0.7166717203668512, "pass@128": 0.7549857549857549 }, "17": { "pass@1": 0.28217197204968947, "pass@2": 0.37584135203208296, "pass@4": 0.4652980295164967, "pass@8": 0.5403919264533021, "pass@16": 0.6006332682410064, "pass@32": 0.6515994259098172, "pass@64": 0.6960613563926029, "pass@128": 0.7360248447204969 }, "18": { "pass@1": 0.2483685661764706, "pass@2": 0.33491054886521515, "pass@4": 0.41869472667019536, "pass@8": 0.4939067588373051, "pass@16": 0.5575300786689851, "pass@32": 0.6101799014188762, "pass@64": 0.6536175854405368, "pass@128": 0.6882352941176471 }, "19": { "pass@1": 0.18145461309523808, "pass@2": 0.2484313484251969, "pass@4": 0.32077272958067743, "pass@8": 0.3879525679017902, "pass@16": 0.4426682347327378, "pass@32": 0.4877902642399231, "pass@64": 0.5304925094750792, "pass@128": 0.5744047619047619 }, "20": { "pass@1": 0.14885853293413173, "pass@2": 0.20804846998915552, "pass@4": 0.2722438149572623, "pass@8": 0.3372685915989853, "pass@16": 0.39522203120650934, "pass@32": 0.44381516350834477, "pass@64": 0.48748600114513696, "pass@128": 0.5269461077844312 } } }, "movie_festival_awards": { "count": 801152, "correct": 580954, "answer_accuracy": 0.7251482864674869, "avg_response_len": 286.24574987018696, "resp_tokens_sum": 229326355, "pass_at_k": { "pass@1": 0.7251482864674869, "pass@2": 0.7622002466055128, "pass@4": 0.7944293377641134, "pass@8": 0.8216631401631663, "pass@16": 0.8435584542301595, "pass@32": 0.8610946532121718, "pass@64": 0.8756452647380498, "pass@128": 0.8876817382968525 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9548507462686567, "pass@2": 0.9641611088259492, "pass@4": 0.9702759092613421, "pass@8": 0.9743496102469225, "pass@16": 0.9778077564445188, "pass@32": 0.9818317881305241, "pass@64": 0.9868807117022572, "pass@128": 0.991044776119403 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9959788602941176, "pass@2": 0.9978610612552107, "pass@4": 0.9987736642294712, "pass@8": 0.9995108704580657, "pass@16": 0.9999292442955922, "pass@32": 0.9999991192945575, "pass@64": 0.9999999999953015, "pass@128": 1.0 }, "6": { "pass@1": 0.9995563880126183, "pass@2": 0.999959248242629, "pass@4": 0.999999596363546, "pass@8": 0.9999999999858014, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9997033227848101, "pass@2": 0.9999859837536131, "pass@4": 0.999999962623343, "pass@8": 0.99999999999998, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9920701951951952, "pass@2": 0.9952568217824125, "pass@4": 0.9976451983479543, "pass@8": 0.999336102227142, "pass@16": 0.9999393042339394, "pass@32": 0.9999995670396389, "pass@64": 0.9999999999993233, "pass@128": 1.0 }, "9": { "pass@1": 0.9950261254019293, "pass@2": 0.9979658265184698, "pass@4": 0.999369864438167, "pass@8": 0.9999233509024527, "pass@16": 0.9999988400129937, "pass@32": 0.9999999999211941, "pass@64": 1.0, "pass@128": 1.0 }, "11": { "pass@1": 0.9424107142857143, "pass@2": 0.9582763092113488, "pass@4": 0.9704186628457155, "pass@8": 0.979249727923491, "pass@16": 0.9846760641101544, "pass@32": 0.9875064744371793, "pass@64": 0.9888742737578513, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.859375, "pass@2": 0.8942583796864841, "pass@4": 0.916091897343107, "pass@8": 0.9319538736797466, "pass@16": 0.9446446168477055, "pass@32": 0.9559976044880033, "pass@64": 0.9657581898039411, "pass@128": 0.9724770642201835 }, "13": { "pass@1": 0.6749373070987654, "pass@2": 0.7393748632983379, "pass@4": 0.7844400370092625, "pass@8": 0.8188400380665399, "pass@16": 0.8458120470339987, "pass@32": 0.8672366071285914, "pass@64": 0.885291263140771, "pass@128": 0.9012345679012346 }, "14": { "pass@1": 0.6472783753709199, "pass@2": 0.7265900634360616, "pass@4": 0.7834193592084369, "pass@8": 0.8229248873711785, "pass@16": 0.850694602949872, "pass@32": 0.8729859244399738, "pass@64": 0.8929176328904539, "pass@128": 0.9109792284866469 }, "15": { "pass@1": 0.5328655478395061, "pass@2": 0.6268868626664722, "pass@4": 0.7019003683567334, "pass@8": 0.7591171970498471, "pass@16": 0.7976979518896169, "pass@32": 0.8230547091767454, "pass@64": 0.8429850160241235, "pass@128": 0.8611111111111112 }, "16": { "pass@1": 0.34462573746312686, "pass@2": 0.4405050170719808, "pass@4": 0.5239615202966885, "pass@8": 0.591901375563439, "pass@16": 0.6466237250958037, "pass@32": 0.6925296333526205, "pass@64": 0.7317750204679877, "pass@128": 0.7640117994100295 }, "17": { "pass@1": 0.26136577560240964, "pass@2": 0.3482866011289251, "pass@4": 0.42627636925655377, "pass@8": 0.49088887967339034, "pass@16": 0.5467325363614678, "pass@32": 0.5977798190482726, "pass@64": 0.6434842851816812, "pass@128": 0.6837349397590361 }, "18": { "pass@1": 0.23865927419354838, "pass@2": 0.31305284798069594, "pass@4": 0.3894739970003749, "pass@8": 0.4619313302764042, "pass@16": 0.5196857133240281, "pass@32": 0.5624058343339355, "pass@64": 0.5987371793517974, "pass@128": 0.6290322580645161 }, "19": { "pass@1": 0.17529739425981872, "pass@2": 0.2498134084259105, "pass@4": 0.3267649176972214, "pass@8": 0.39822689720832727, "pass@16": 0.46005277817344775, "pass@32": 0.5109431951797943, "pass@64": 0.5520361190642347, "pass@128": 0.5861027190332326 }, "20": { "pass@1": 0.14916478978978978, "pass@2": 0.216420160317798, "pass@4": 0.29373762485770377, "pass@8": 0.3734408191736654, "pass@16": 0.44431723624192304, "pass@32": 0.499859994169132, "pass@64": 0.5400738646113595, "pass@128": 0.5675675675675675 } } } } } ]