[ { "checkpoint": "checkpoint-39008", "id_acc": 0.9907135416666667, "ood_acc": 0.885690625, "total_acc": 0.9354383223684211, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 200.3214800347222, "ood_avg_resp_len": 383.00255859375, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 756616, "answer_accuracy": 0.9344076035409421, "avg_response_len": 283.59606312243125, "resp_tokens_sum": 229635673, "pass_at_k": { "pass@1": 0.9344076035409421, "pass@2": 0.9533806316762856, "pass@4": 0.9661068553298521, "pass@8": 0.9750141706759139, "pass@16": 0.9814565557468075, "pass@32": 0.9862101820285257, "pass@64": 0.9896272451161039, "pass@128": 0.9920961112867531 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9496220501474927, "pass@2": 0.9594972330615753, "pass@4": 0.966436561856759, "pass@8": 0.9727340543343289, "pass@16": 0.9784618657970596, "pass@32": 0.9825567911391251, "pass@64": 0.985620534223565, "pass@128": 0.9882005899705014 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9996177370030581, "pass@2": 0.9999838215463894, "pass@4": 0.9999999635940003, "pass@8": 0.9999999999999807, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.999951171875, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9942460317460318, "pass@2": 0.9975178883889514, "pass@4": 0.9992110941489457, "pass@8": 0.9998670726976857, "pass@16": 0.9999959700663595, "pass@32": 0.9999999986446861, "pass@64": 1.0, "pass@128": 1.0 }, "6": { "pass@1": 0.9999532185628742, "pass@2": 0.9999996316422274, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9993528106508875, "pass@2": 0.9999286562922239, "pass@4": 0.9999986563255037, "pass@8": 0.9999999997393219, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9939903846153846, "pass@2": 0.997067355012114, "pass@4": 0.9988529817186313, "pass@8": 0.9997549809210944, "pass@16": 0.9999868735089701, "pass@32": 0.9999999812154626, "pass@64": 0.9999999999999998, "pass@128": 1.0 }, "9": { "pass@1": 0.9912852112676056, "pass@2": 0.9926787595652655, "pass@4": 0.9935059028712959, "pass@8": 0.9944525916293162, "pass@16": 0.9953237078346044, "pass@32": 0.9963063942759401, "pass@64": 0.9978928690234709, "pass@128": 1.0 }, "11": { "pass@1": 0.9613358711048159, "pass@2": 0.9751445010149226, "pass@4": 0.9847932027264298, "pass@8": 0.9914273628571794, "pass@16": 0.9961681616814383, "pass@32": 0.99904074927096, "pass@64": 0.9999499987724876, "pass@128": 1.0 }, "12": { "pass@1": 0.9385151836158192, "pass@2": 0.9597627924952182, "pass@4": 0.9762562224743094, "pass@8": 0.9865212132430582, "pass@16": 0.9922087450214933, "pass@32": 0.9959870309300201, "pass@64": 0.9983853008294012, "pass@128": 1.0 }, "13": { "pass@1": 0.9068867554858934, "pass@2": 0.9370576265643127, "pass@4": 0.9592523463955722, "pass@8": 0.9740163899882339, "pass@16": 0.9836944065507172, "pass@32": 0.9896518391446572, "pass@64": 0.9926987166516155, "pass@128": 0.9937304075235109 }, "14": { "pass@1": 0.9166179906542056, "pass@2": 0.941457704761204, "pass@4": 0.9603239408158093, "pass@8": 0.9735008083823662, "pass@16": 0.9829294702078442, "pass@32": 0.9901209671439047, "pass@64": 0.9952071961226423, "pass@128": 0.9968847352024922 }, "15": { "pass@1": 0.9691112716763006, "pass@2": 0.9872729962222933, "pass@4": 0.9933811493397139, "pass@8": 0.9952925662134263, "pass@16": 0.9963578351065164, "pass@32": 0.9974679316498488, "pass@64": 0.9985359192993869, "pass@128": 1.0 }, "16": { "pass@1": 0.8643397177419355, "pass@2": 0.9029233870967743, "pass@4": 0.9338913727517938, "pass@8": 0.9545205921905574, "pass@16": 0.9671956269598453, "pass@32": 0.9761510160969632, "pass@64": 0.9823101727503963, "pass@128": 0.9838709677419355 }, "17": { "pass@1": 0.8985503973988439, "pass@2": 0.9290570524782674, "pass@4": 0.9477699960554062, "pass@8": 0.9613079311183532, "pass@16": 0.9720480116375539, "pass@32": 0.9786693409502875, "pass@64": 0.9811048717975289, "pass@128": 0.9826589595375722 }, "18": { "pass@1": 0.8410714285714286, "pass@2": 0.8791718222722164, "pass@4": 0.9039378402699658, "pass@8": 0.9225013658392592, "pass@16": 0.9380212263183387, "pass@32": 0.9522564152117664, "pass@64": 0.9651583391167925, "pass@128": 0.9771428571428571 }, "19": { "pass@1": 0.7519707207207207, "pass@2": 0.8127601026223072, "pass@4": 0.8581806291668495, "pass@8": 0.8929533160295472, "pass@16": 0.9184558230351982, "pass@32": 0.9365729445914778, "pass@64": 0.9498045174907821, "pass@128": 0.960960960960961 }, "20": { "pass@1": 0.7815080705705706, "pass@2": 0.8459900599418323, "pass@4": 0.8831622091720508, "pass@8": 0.9091561650991454, "pass@16": 0.9291310201543674, "pass@32": 0.9450531167298203, "pass@64": 0.9576555137668181, "pass@128": 0.9669669669669669 } } }, "teachers_in_school": { "count": 821120, "correct": 770509, "answer_accuracy": 0.9383634547934528, "avg_response_len": 292.753721745908, "resp_tokens_sum": 240385936, "pass_at_k": { "pass@1": 0.9383634547934528, "pass@2": 0.9567760147844829, "pass@4": 0.9693254594052584, "pass@8": 0.9780153737379047, "pass@16": 0.9838214676849862, "pass@32": 0.9877374039694216, "pass@64": 0.9905971086523647, "pass@128": 0.9929851909586905 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9213238113496932, "pass@2": 0.9363610151683497, "pass@4": 0.9515447243749438, "pass@8": 0.9635215251668017, "pass@16": 0.9717501171588053, "pass@32": 0.9789974182409743, "pass@64": 0.9873278512974969, "pass@128": 0.9969325153374233 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.99765625, "pass@2": 0.9993058041040325, "pass@4": 0.9999122283010079, "pass@8": 0.9999981196920988, "pass@16": 0.9999999995180833, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9998844304733728, "pass@2": 0.9999978160089457, "pass@4": 0.9999999997226678, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9952445652173914, "pass@2": 0.9969466792194455, "pass@4": 0.997354996114616, "pass@8": 0.9976154255526752, "pass@16": 0.9980647799459662, "pass@32": 0.9987868364280551, "pass@64": 0.9996462398721899, "pass@128": 1.0 }, "6": { "pass@1": 0.9998656876790831, "pass@2": 0.9999964747422333, "pass@4": 0.9999999986570447, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9968388728323699, "pass@2": 0.9980816331755495, "pass@4": 0.998801978690525, "pass@8": 0.9995193591669413, "pass@16": 0.9999304712731262, "pass@32": 0.9999991345669063, "pass@64": 0.999999999995383, "pass@128": 1.0 }, "8": { "pass@1": 0.9913072183098591, "pass@2": 0.99444105578352, "pass@4": 0.9964716870778477, "pass@8": 0.9975479530508959, "pass@16": 0.9981170196665599, "pass@32": 0.9988210098675391, "pass@64": 0.9996562049462128, "pass@128": 1.0 }, "9": { "pass@1": 0.9941757110778443, "pass@2": 0.9960987228299307, "pass@4": 0.9969899691617888, "pass@8": 0.9975118492787143, "pass@16": 0.9980009131217787, "pass@32": 0.998746881938581, "pass@64": 0.9996345890895375, "pass@128": 1.0 }, "11": { "pass@1": 0.9735504518072289, "pass@2": 0.9807889283512, "pass@4": 0.9859694000975779, "pass@8": 0.9903034578261026, "pass@16": 0.9939776137045641, "pass@32": 0.9972317237104557, "pass@64": 0.9994343404291871, "pass@128": 1.0 }, "12": { "pass@1": 0.943255289968652, "pass@2": 0.9643270370004688, "pass@4": 0.9795041327122507, "pass@8": 0.9901062501212899, "pass@16": 0.9963812763120088, "pass@32": 0.999159903580148, "pass@64": 0.9999519319520963, "pass@128": 1.0 }, "13": { "pass@1": 0.9459252450980392, "pass@2": 0.969042242881405, "pass@4": 0.9843180109314063, "pass@8": 0.9930887640011501, "pass@16": 0.9972316564809265, "pass@32": 0.9991371398635513, "pass@64": 0.9999152127956612, "pass@128": 1.0 }, "14": { "pass@1": 0.9421829312865497, "pass@2": 0.9626221681171436, "pass@4": 0.9766211556340545, "pass@8": 0.9860053568807828, "pass@16": 0.9912234750641322, "pass@32": 0.9933893500190114, "pass@64": 0.994067684369298, "pass@128": 0.9941520467836257 }, "15": { "pass@1": 0.9615056818181819, "pass@2": 0.9816444464328327, "pass@4": 0.9924044471713761, "pass@8": 0.9970935786701942, "pass@16": 0.9988697871371285, "pass@32": 0.9997170217125875, "pass@64": 0.9999906185608666, "pass@128": 1.0 }, "16": { "pass@1": 0.9047364672364673, "pass@2": 0.9336404564012831, "pass@4": 0.9554778179436971, "pass@8": 0.9713046676762731, "pass@16": 0.981129930647535, "pass@32": 0.9863700316854713, "pass@64": 0.9898460871084831, "pass@128": 0.9943019943019943 }, "17": { "pass@1": 0.8905764751552795, "pass@2": 0.9193115891084263, "pass@4": 0.9371282252373729, "pass@8": 0.9495247038246433, "pass@16": 0.9600510648716486, "pass@32": 0.9696963337203404, "pass@64": 0.9783483636870083, "pass@128": 0.9875776397515528 }, "18": { "pass@1": 0.8783318014705882, "pass@2": 0.9212120773506257, "pass@4": 0.9446521532234942, "pass@8": 0.9603982535156668, "pass@16": 0.971534461743965, "pass@32": 0.9800680670995429, "pass@64": 0.986866489127435, "pass@128": 0.9911764705882353 }, "19": { "pass@1": 0.7672526041666666, "pass@2": 0.8213970812242222, "pass@4": 0.8611331446292425, "pass@8": 0.8925303692099814, "pass@16": 0.9155570589381155, "pass@32": 0.9306472438901101, "pass@64": 0.9397489108469029, "pass@128": 0.9464285714285714 }, "20": { "pass@1": 0.7179079341317365, "pass@2": 0.797639710736008, "pass@4": 0.8541305002480968, "pass@8": 0.8923328681460105, "pass@16": 0.9176848689576249, "pass@32": 0.9338587403043053, "pass@64": 0.9451756698292793, "pass@128": 0.9550898203592815 } } }, "movie_festival_awards": { "count": 801152, "correct": 747861, "answer_accuracy": 0.9334820358683495, "avg_response_len": 313.2888777660169, "resp_tokens_sum": 250992011, "pass_at_k": { "pass@1": 0.9334820358683495, "pass@2": 0.9537701733126235, "pass@4": 0.9670815015054736, "pass@8": 0.9760319147552091, "pass@16": 0.9821892780315375, "pass@32": 0.9865542675416312, "pass@64": 0.9896955198919034, "pass@128": 0.9918517335037546 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9504897388059701, "pass@2": 0.9637266423786578, "pass@4": 0.9740664414149726, "pass@8": 0.9809101215824065, "pass@16": 0.9856060890227171, "pass@32": 0.9899456026975152, "pass@64": 0.9932744178001089, "pass@128": 0.9940298507462687 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 0.9996573464912281, "pass@2": 0.9999622271722615, "pass@4": 0.9999996258691347, "pass@8": 0.9999999999868394, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9943244485294118, "pass@2": 0.9970045449282076, "pass@4": 0.9982432999183926, "pass@8": 0.9991491885263906, "pass@16": 0.999780019461051, "pass@32": 0.9999893918747133, "pass@64": 0.9999999972066469, "pass@128": 1.0 }, "6": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9955992879746836, "pass@2": 0.996529421658527, "pass@4": 0.9969824845865787, "pass@8": 0.9972195335748744, "pass@16": 0.9975798614235771, "pass@32": 0.9982246087909897, "pass@64": 0.9992150902023323, "pass@128": 1.0 }, "8": { "pass@1": 0.9915540540540541, "pass@2": 0.9943531129554752, "pass@4": 0.9959725785965943, "pass@8": 0.9974309201820005, "pass@16": 0.9988093286723544, "pass@32": 0.9997137994250378, "pass@64": 0.9999907019175951, "pass@128": 1.0 }, "9": { "pass@1": 0.9928155144694534, "pass@2": 0.9952599248044157, "pass@4": 0.9962277089760885, "pass@8": 0.9966958564196817, "pass@16": 0.9967827847936206, "pass@32": 0.9967845657060646, "pass@64": 0.9967845659163987, "pass@128": 0.9967845659163987 }, "11": { "pass@1": 0.9673611111111111, "pass@2": 0.9821323115860517, "pass@4": 0.9926942709542262, "pass@8": 0.9980986789972592, "pass@16": 0.9997494773091639, "pass@32": 0.9999937908674831, "pass@64": 0.9999999994787548, "pass@128": 1.0 }, "12": { "pass@1": 0.950759747706422, "pass@2": 0.9713404337932532, "pass@4": 0.9855429708327742, "pass@8": 0.9928148852591774, "pass@16": 0.995613443895582, "pass@32": 0.9967029940509836, "pass@64": 0.9969374409579194, "pass@128": 0.9969418960244648 }, "13": { "pass@1": 0.8924334490740741, "pass@2": 0.9274767303392636, "pass@4": 0.9551435529470846, "pass@8": 0.9738012340666778, "pass@16": 0.9852240935964495, "pass@32": 0.9924642039682082, "pass@64": 0.9960781632221886, "pass@128": 0.9969135802469136 }, "14": { "pass@1": 0.9461007047477745, "pass@2": 0.9686999842286034, "pass@4": 0.9829498372236113, "pass@8": 0.9910325386471672, "pass@16": 0.995148432923748, "pass@32": 0.9973034567467718, "pass@64": 0.9984965964597918, "pass@128": 1.0 }, "15": { "pass@1": 0.9357397762345679, "pass@2": 0.9589940580344121, "pass@4": 0.9725759832682954, "pass@8": 0.981825320220379, "pass@16": 0.987668589719086, "pass@32": 0.9901856943660934, "pass@64": 0.9907199216941467, "pass@128": 0.9907407407407407 }, "16": { "pass@1": 0.9002581120943953, "pass@2": 0.9337901830302185, "pass@4": 0.9539900026324142, "pass@8": 0.9649273623901918, "pass@16": 0.971576240013085, "pass@32": 0.9763674951513314, "pass@64": 0.9798234041567667, "pass@128": 0.9823008849557522 }, "17": { "pass@1": 0.8558923192771084, "pass@2": 0.8873935703443695, "pass@4": 0.9093786171118668, "pass@8": 0.9264764595802648, "pass@16": 0.9403468570920928, "pass@32": 0.9517275677742213, "pass@64": 0.963140868562918, "pass@128": 0.9759036144578314 }, "18": { "pass@1": 0.8797631048387097, "pass@2": 0.9189547879095755, "pass@4": 0.9408445722510488, "pass@8": 0.9570493000821247, "pass@16": 0.9697178547928714, "pass@32": 0.9798583276175898, "pass@64": 0.9874463456793682, "pass@128": 0.9903225806451613 }, "19": { "pass@1": 0.7480173716012085, "pass@2": 0.8153925410947498, "pass@4": 0.8618116894187314, "pass@8": 0.8955726691120511, "pass@16": 0.921278465008222, "pass@32": 0.9415106476448225, "pass@64": 0.9571645244414844, "pass@128": 0.9667673716012085 }, "20": { "pass@1": 0.7336007882882883, "pass@2": 0.8098284061620676, "pass@4": 0.8578734074344312, "pass@8": 0.8917625881142276, "pass@16": 0.9171084516911536, "pass@32": 0.9343042775669052, "pass@64": 0.9457695347626426, "pass@128": 0.954954954954955 } } } } } ]