| [ |
| { |
| "checkpoint": "checkpoint-85656", |
| "id_acc": 0.99334375, |
| "ood_acc": 0.4865421875, |
| "total_acc": 0.7266060855263158, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.87361805555557, |
| "ood_avg_resp_len": 336.17079375, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 585874, |
| "answer_accuracy": 0.7235442024976289, |
| "avg_response_len": 258.13694228185267, |
| "resp_tokens_sum": 209020710, |
| "pass_at_k": { |
| "pass@1": 0.7235442024976289, |
| "pass@2": 0.7601259044351346, |
| "pass@4": 0.7915020775787805, |
| "pass@8": 0.8180794280332532, |
| "pass@16": 0.8402727252631793, |
| "pass@32": 0.8592362336703517, |
| "pass@64": 0.8759766756894397, |
| "pass@128": 0.8910844135314575 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9606148598820059, |
| "pass@2": 0.970110604952036, |
| "pass@4": 0.9766451804143947, |
| "pass@8": 0.9812420969145403, |
| "pass@16": 0.984712123290507, |
| "pass@32": 0.9877358738401864, |
| "pass@64": 0.9911736613911921, |
| "pass@128": 0.9941002949852508 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 0.999951171875, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9963293650793651, |
| "pass@2": 0.9968914041994751, |
| "pass@4": 0.9970213660792401, |
| "pass@8": 0.997211286089119, |
| "pass@16": 0.9975721784776902, |
| "pass@32": 0.9982189726284215, |
| "pass@64": 0.9992125984251968, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9999532185628742, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9983357988165681, |
| "pass@2": 0.9996622093835906, |
| "pass@4": 0.999980712377669, |
| "pass@8": 0.9999999358582831, |
| "pass@16": 0.9999999999997683, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9995993589743589, |
| "pass@2": 0.9999917190339189, |
| "pass@4": 0.9999999969955679, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9967209507042254, |
| "pass@2": 0.9971629976710658, |
| "pass@4": 0.9971830114545541, |
| "pass@8": 0.9971830985912242, |
| "pass@16": 0.9971830985915493, |
| "pass@32": 0.9971830985915493, |
| "pass@64": 0.9971830985915493, |
| "pass@128": 0.9971830985915493 |
| }, |
| "11": { |
| "pass@1": 0.9175593130311614, |
| "pass@2": 0.9301508721643506, |
| "pass@4": 0.9383426105022994, |
| "pass@8": 0.94506009266078, |
| "pass@16": 0.9523354989438444, |
| "pass@32": 0.9605310994724667, |
| "pass@64": 0.9676012340007842, |
| "pass@128": 0.9716713881019831 |
| }, |
| "12": { |
| "pass@1": 0.8390271892655368, |
| "pass@2": 0.8704061290537837, |
| "pass@4": 0.8879430150680314, |
| "pass@8": 0.8998900279443047, |
| "pass@16": 0.9101521093680011, |
| "pass@32": 0.9205997134915045, |
| "pass@64": 0.9318575609376857, |
| "pass@128": 0.943502824858757 |
| }, |
| "13": { |
| "pass@1": 0.7098354231974922, |
| "pass@2": 0.7657214196677613, |
| "pass@4": 0.8028330453599253, |
| "pass@8": 0.8308064695801197, |
| "pass@16": 0.8536985149124111, |
| "pass@32": 0.8754225625963826, |
| "pass@64": 0.8965705154324992, |
| "pass@128": 0.9153605015673981 |
| }, |
| "14": { |
| "pass@1": 0.6602414330218068, |
| "pass@2": 0.742746507591925, |
| "pass@4": 0.7996196558958166, |
| "pass@8": 0.8367759768205015, |
| "pass@16": 0.8619997000724972, |
| "pass@32": 0.8821281369592076, |
| "pass@64": 0.8977516145766892, |
| "pass@128": 0.9096573208722741 |
| }, |
| "15": { |
| "pass@1": 0.5442557803468208, |
| "pass@2": 0.6445744532565655, |
| "pass@4": 0.7226312514765129, |
| "pass@8": 0.7833987025767928, |
| "pass@16": 0.8303731247677375, |
| "pass@32": 0.8669906303856226, |
| "pass@64": 0.896652764809943, |
| "pass@128": 0.9190751445086706 |
| }, |
| "16": { |
| "pass@1": 0.32200100806451615, |
| "pass@2": 0.41802014541529064, |
| "pass@4": 0.5057130259927185, |
| "pass@8": 0.579146486298523, |
| "pass@16": 0.6372956308256016, |
| "pass@32": 0.6853502169400502, |
| "pass@64": 0.7263549298081786, |
| "pass@128": 0.7612903225806451 |
| }, |
| "17": { |
| "pass@1": 0.266573338150289, |
| "pass@2": 0.3552971815119928, |
| "pass@4": 0.44027677365835044, |
| "pass@8": 0.5145766287777432, |
| "pass@16": 0.5764773175385587, |
| "pass@32": 0.6297498942971839, |
| "pass@64": 0.6792948234940933, |
| "pass@128": 0.7283236994219653 |
| }, |
| "18": { |
| "pass@1": 0.22410714285714287, |
| "pass@2": 0.3040772637795276, |
| "pass@4": 0.3812210715624831, |
| "pass@8": 0.45209828595059937, |
| "pass@16": 0.5148515801438031, |
| "pass@32": 0.5696556136715276, |
| "pass@64": 0.6196317222927293, |
| "pass@128": 0.6714285714285714 |
| }, |
| "19": { |
| "pass@1": 0.17450262762762764, |
| "pass@2": 0.24425742179187057, |
| "pass@4": 0.3161038088144387, |
| "pass@8": 0.38331538160776063, |
| "pass@16": 0.44096227160841417, |
| "pass@32": 0.48847477835581016, |
| "pass@64": 0.5293757971334146, |
| "pass@128": 0.5675675675675675 |
| }, |
| "20": { |
| "pass@1": 0.16915352852852852, |
| "pass@2": 0.2322273060462035, |
| "pass@4": 0.29948549364887955, |
| "pass@8": 0.36665142621617874, |
| "pass@16": 0.42865530956555087, |
| "pass@32": 0.4819577047980443, |
| "pass@64": 0.526854050384028, |
| "pass@128": 0.5645645645645646 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 600278, |
| "answer_accuracy": 0.7310478371005456, |
| "avg_response_len": 267.8073351032736, |
| "resp_tokens_sum": 219901959, |
| "pass_at_k": { |
| "pass@1": 0.7310478371005456, |
| "pass@2": 0.7680041011777181, |
| "pass@4": 0.7996512798526805, |
| "pass@8": 0.8257688373431301, |
| "pass@16": 0.846615880063671, |
| "pass@32": 0.8637151935821454, |
| "pass@64": 0.8785491835920611, |
| "pass@128": 0.8916601714731099 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9403278374233128, |
| "pass@2": 0.952113575431139, |
| "pass@4": 0.9607398866169331, |
| "pass@8": 0.9662308127069366, |
| "pass@16": 0.9701134353995957, |
| "pass@32": 0.9738990670749782, |
| "pass@64": 0.9783020822793281, |
| "pass@128": 0.9846625766871165 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999526515151516, |
| "pass@2": 0.9999996271772846, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9997735507246377, |
| "pass@2": 0.9999942941914869, |
| "pass@4": 0.9999999959244225, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9994851361031518, |
| "pass@2": 0.9999626322676716, |
| "pass@4": 0.999999727111475, |
| "pass@8": 0.9999999999939815, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9995032514450867, |
| "pass@2": 0.9999455958991397, |
| "pass@4": 0.9999991709851297, |
| "pass@8": 0.9999999999115423, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9968089788732394, |
| "pass@2": 0.9990167877342796, |
| "pass@4": 0.9998786036428545, |
| "pass@8": 0.9999981102949981, |
| "pass@16": 0.9999999997381142, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9952984655688623, |
| "pass@2": 0.9967109334480646, |
| "pass@4": 0.9979339827094966, |
| "pass@8": 0.9989726884046817, |
| "pass@16": 0.999694545217648, |
| "pass@32": 0.9999787580173054, |
| "pass@64": 0.9999999843303174, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9503247364457831, |
| "pass@2": 0.9617861208613987, |
| "pass@4": 0.9686446664046513, |
| "pass@8": 0.9728691216772047, |
| "pass@16": 0.9758402586096212, |
| "pass@32": 0.9778266280336569, |
| "pass@64": 0.9788090621047877, |
| "pass@128": 0.9789156626506024 |
| }, |
| "12": { |
| "pass@1": 0.8506073667711599, |
| "pass@2": 0.8813039178782122, |
| "pass@4": 0.9007288483532033, |
| "pass@8": 0.9156088842870286, |
| "pass@16": 0.92726192452425, |
| "pass@32": 0.9371659829216429, |
| "pass@64": 0.9472018257825825, |
| "pass@128": 0.9592476489028213 |
| }, |
| "13": { |
| "pass@1": 0.7220325630252101, |
| "pass@2": 0.7833849583140341, |
| "pass@4": 0.8213891459470926, |
| "pass@8": 0.8472609166913442, |
| "pass@16": 0.8663873407991339, |
| "pass@32": 0.8816572307661266, |
| "pass@64": 0.8938391501339014, |
| "pass@128": 0.9019607843137255 |
| }, |
| "14": { |
| "pass@1": 0.6827713815789473, |
| "pass@2": 0.7667430757010633, |
| "pass@4": 0.8250682096755444, |
| "pass@8": 0.8622464010462382, |
| "pass@16": 0.8840158404678026, |
| "pass@32": 0.8985840968140026, |
| "pass@64": 0.911342591360829, |
| "pass@128": 0.9239766081871345 |
| }, |
| "15": { |
| "pass@1": 0.5276751893939394, |
| "pass@2": 0.6309297452875214, |
| "pass@4": 0.7133767940939197, |
| "pass@8": 0.7782515948523318, |
| "pass@16": 0.8285203684912462, |
| "pass@32": 0.8654085228745337, |
| "pass@64": 0.8911681455877384, |
| "pass@128": 0.906060606060606 |
| }, |
| "16": { |
| "pass@1": 0.3476451210826211, |
| "pass@2": 0.4371421220584603, |
| "pass@4": 0.5146949160735249, |
| "pass@8": 0.5768380160414723, |
| "pass@16": 0.6278066916101048, |
| "pass@32": 0.6740581101475652, |
| "pass@64": 0.7166717203668512, |
| "pass@128": 0.7549857549857549 |
| }, |
| "17": { |
| "pass@1": 0.28217197204968947, |
| "pass@2": 0.37584135203208296, |
| "pass@4": 0.4652980295164967, |
| "pass@8": 0.5403919264533021, |
| "pass@16": 0.6006332682410064, |
| "pass@32": 0.6515994259098172, |
| "pass@64": 0.6960613563926029, |
| "pass@128": 0.7360248447204969 |
| }, |
| "18": { |
| "pass@1": 0.2483685661764706, |
| "pass@2": 0.33491054886521515, |
| "pass@4": 0.41869472667019536, |
| "pass@8": 0.4939067588373051, |
| "pass@16": 0.5575300786689851, |
| "pass@32": 0.6101799014188762, |
| "pass@64": 0.6536175854405368, |
| "pass@128": 0.6882352941176471 |
| }, |
| "19": { |
| "pass@1": 0.18145461309523808, |
| "pass@2": 0.2484313484251969, |
| "pass@4": 0.32077272958067743, |
| "pass@8": 0.3879525679017902, |
| "pass@16": 0.4426682347327378, |
| "pass@32": 0.4877902642399231, |
| "pass@64": 0.5304925094750792, |
| "pass@128": 0.5744047619047619 |
| }, |
| "20": { |
| "pass@1": 0.14885853293413173, |
| "pass@2": 0.20804846998915552, |
| "pass@4": 0.2722438149572623, |
| "pass@8": 0.3372685915989853, |
| "pass@16": 0.39522203120650934, |
| "pass@32": 0.44381516350834477, |
| "pass@64": 0.48748600114513696, |
| "pass@128": 0.5269461077844312 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 580954, |
| "answer_accuracy": 0.7251482864674869, |
| "avg_response_len": 286.24574987018696, |
| "resp_tokens_sum": 229326355, |
| "pass_at_k": { |
| "pass@1": 0.7251482864674869, |
| "pass@2": 0.7622002466055128, |
| "pass@4": 0.7944293377641134, |
| "pass@8": 0.8216631401631663, |
| "pass@16": 0.8435584542301595, |
| "pass@32": 0.8610946532121718, |
| "pass@64": 0.8756452647380498, |
| "pass@128": 0.8876817382968525 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9548507462686567, |
| "pass@2": 0.9641611088259492, |
| "pass@4": 0.9702759092613421, |
| "pass@8": 0.9743496102469225, |
| "pass@16": 0.9778077564445188, |
| "pass@32": 0.9818317881305241, |
| "pass@64": 0.9868807117022572, |
| "pass@128": 0.991044776119403 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9959788602941176, |
| "pass@2": 0.9978610612552107, |
| "pass@4": 0.9987736642294712, |
| "pass@8": 0.9995108704580657, |
| "pass@16": 0.9999292442955922, |
| "pass@32": 0.9999991192945575, |
| "pass@64": 0.9999999999953015, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9995563880126183, |
| "pass@2": 0.999959248242629, |
| "pass@4": 0.999999596363546, |
| "pass@8": 0.9999999999858014, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9997033227848101, |
| "pass@2": 0.9999859837536131, |
| "pass@4": 0.999999962623343, |
| "pass@8": 0.99999999999998, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9920701951951952, |
| "pass@2": 0.9952568217824125, |
| "pass@4": 0.9976451983479543, |
| "pass@8": 0.999336102227142, |
| "pass@16": 0.9999393042339394, |
| "pass@32": 0.9999995670396389, |
| "pass@64": 0.9999999999993233, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9950261254019293, |
| "pass@2": 0.9979658265184698, |
| "pass@4": 0.999369864438167, |
| "pass@8": 0.9999233509024527, |
| "pass@16": 0.9999988400129937, |
| "pass@32": 0.9999999999211941, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9424107142857143, |
| "pass@2": 0.9582763092113488, |
| "pass@4": 0.9704186628457155, |
| "pass@8": 0.979249727923491, |
| "pass@16": 0.9846760641101544, |
| "pass@32": 0.9875064744371793, |
| "pass@64": 0.9888742737578513, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.859375, |
| "pass@2": 0.8942583796864841, |
| "pass@4": 0.916091897343107, |
| "pass@8": 0.9319538736797466, |
| "pass@16": 0.9446446168477055, |
| "pass@32": 0.9559976044880033, |
| "pass@64": 0.9657581898039411, |
| "pass@128": 0.9724770642201835 |
| }, |
| "13": { |
| "pass@1": 0.6749373070987654, |
| "pass@2": 0.7393748632983379, |
| "pass@4": 0.7844400370092625, |
| "pass@8": 0.8188400380665399, |
| "pass@16": 0.8458120470339987, |
| "pass@32": 0.8672366071285914, |
| "pass@64": 0.885291263140771, |
| "pass@128": 0.9012345679012346 |
| }, |
| "14": { |
| "pass@1": 0.6472783753709199, |
| "pass@2": 0.7265900634360616, |
| "pass@4": 0.7834193592084369, |
| "pass@8": 0.8229248873711785, |
| "pass@16": 0.850694602949872, |
| "pass@32": 0.8729859244399738, |
| "pass@64": 0.8929176328904539, |
| "pass@128": 0.9109792284866469 |
| }, |
| "15": { |
| "pass@1": 0.5328655478395061, |
| "pass@2": 0.6268868626664722, |
| "pass@4": 0.7019003683567334, |
| "pass@8": 0.7591171970498471, |
| "pass@16": 0.7976979518896169, |
| "pass@32": 0.8230547091767454, |
| "pass@64": 0.8429850160241235, |
| "pass@128": 0.8611111111111112 |
| }, |
| "16": { |
| "pass@1": 0.34462573746312686, |
| "pass@2": 0.4405050170719808, |
| "pass@4": 0.5239615202966885, |
| "pass@8": 0.591901375563439, |
| "pass@16": 0.6466237250958037, |
| "pass@32": 0.6925296333526205, |
| "pass@64": 0.7317750204679877, |
| "pass@128": 0.7640117994100295 |
| }, |
| "17": { |
| "pass@1": 0.26136577560240964, |
| "pass@2": 0.3482866011289251, |
| "pass@4": 0.42627636925655377, |
| "pass@8": 0.49088887967339034, |
| "pass@16": 0.5467325363614678, |
| "pass@32": 0.5977798190482726, |
| "pass@64": 0.6434842851816812, |
| "pass@128": 0.6837349397590361 |
| }, |
| "18": { |
| "pass@1": 0.23865927419354838, |
| "pass@2": 0.31305284798069594, |
| "pass@4": 0.3894739970003749, |
| "pass@8": 0.4619313302764042, |
| "pass@16": 0.5196857133240281, |
| "pass@32": 0.5624058343339355, |
| "pass@64": 0.5987371793517974, |
| "pass@128": 0.6290322580645161 |
| }, |
| "19": { |
| "pass@1": 0.17529739425981872, |
| "pass@2": 0.2498134084259105, |
| "pass@4": 0.3267649176972214, |
| "pass@8": 0.39822689720832727, |
| "pass@16": 0.46005277817344775, |
| "pass@32": 0.5109431951797943, |
| "pass@64": 0.5520361190642347, |
| "pass@128": 0.5861027190332326 |
| }, |
| "20": { |
| "pass@1": 0.14916478978978978, |
| "pass@2": 0.216420160317798, |
| "pass@4": 0.29373762485770377, |
| "pass@8": 0.3734408191736654, |
| "pass@16": 0.44431723624192304, |
| "pass@32": 0.499859994169132, |
| "pass@64": 0.5400738646113595, |
| "pass@128": 0.5675675675675675 |
| } |
| } |
| } |
| } |
| } |
| ] |
