[ { "checkpoint": "checkpoint-42828", "id_acc": 0.9906362847222222, "ood_acc": 0.482271875, "total_acc": 0.7230760690789474, "id_avg_loss": 0.0, "ood_avg_loss": 0.0, "total_avg_loss": 0.0, "id_avg_resp_len": 197.01928211805554, "ood_avg_resp_len": 336.27141796875, "count_id": 1152000, "count_ood": 1280000, "count_total": 2432000, "template_metrics": { "crazy_zootopia": { "count": 809728, "correct": 578902, "answer_accuracy": 0.7149339037306355, "avg_response_len": 257.2338896518337, "resp_tokens_sum": 208289483, "pass_at_k": { "pass@1": 0.7149339037306355, "pass@2": 0.7528802711780616, "pass@4": 0.7853499391160436, "pass@8": 0.8123146802770955, "pass@16": 0.8344963841133981, "pass@32": 0.8535153920866395, "pass@64": 0.8706372430698933, "pass@128": 0.8866582358520392 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9403346238938053, "pass@2": 0.9509430962999099, "pass@4": 0.9565902188111443, "pass@8": 0.9595371010736143, "pass@16": 0.9614777765607152, "pass@32": 0.9640615389013077, "pass@64": 0.9683065059344581, "pass@128": 0.9734513274336283 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999761085626911, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9942212301587302, "pass@2": 0.9963039776277965, "pass@4": 0.9979051413216204, "pass@8": 0.9991469167239885, "pass@16": 0.9998257477435663, "pass@32": 0.999994264397576, "pass@64": 0.9999999994794706, "pass@128": 1.0 }, "6": { "pass@1": 0.9983626497005988, "pass@2": 0.9992098725776793, "pass@4": 0.9997977273798858, "pass@8": 0.9999879720977759, "pass@16": 0.9999999725620663, "pass@32": 0.9999999999999857, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9966715976331361, "pass@2": 0.9970745439826679, "pass@4": 0.997133861873035, "pass@8": 0.9972263313609467, "pass@16": 0.9974112426035503, "pass@32": 0.9977810650887574, "pass@64": 0.9985207100591716, "pass@128": 1.0 }, "8": { "pass@1": 0.9923377403846154, "pass@2": 0.9947195039622452, "pass@4": 0.9961377279763107, "pass@8": 0.9976819311023585, "pass@16": 0.9991340276202069, "pass@32": 0.9998780546502605, "pass@64": 0.9999990200711261, "pass@128": 1.0 }, "9": { "pass@1": 0.9925176056338029, "pass@2": 0.9967980619940113, "pass@4": 0.998693938433752, "pass@8": 0.9996446951320699, "pass@16": 0.9999658250813326, "pass@32": 0.9999998146310128, "pass@64": 0.9999999999999163, "pass@128": 1.0 }, "11": { "pass@1": 0.9313473796033994, "pass@2": 0.9494340551181104, "pass@4": 0.9594840918433353, "pass@8": 0.9656293634436289, "pass@16": 0.9699144094325778, "pass@32": 0.9738884592658764, "pass@64": 0.9786071170542627, "pass@128": 0.9830028328611898 }, "12": { "pass@1": 0.8317002118644068, "pass@2": 0.8737276319008856, "pass@4": 0.8998880984686232, "pass@8": 0.9179671212613043, "pass@16": 0.9325065069008127, "pass@32": 0.9452756292304754, "pass@64": 0.9557585734361933, "pass@128": 0.963276836158192 }, "13": { "pass@1": 0.7110109717868338, "pass@2": 0.7740023264137438, "pass@4": 0.8133029303938883, "pass@8": 0.8424745465775301, "pass@16": 0.8657274734514422, "pass@32": 0.883408745813764, "pass@64": 0.8965575001318108, "pass@128": 0.9090909090909091 }, "14": { "pass@1": 0.6173091900311527, "pass@2": 0.6968503937007875, "pass@4": 0.755002373827103, "pass@8": 0.8013279156768645, "pass@16": 0.8399074981743214, "pass@32": 0.8702874981026382, "pass@64": 0.8921680924393293, "pass@128": 0.9034267912772586 }, "15": { "pass@1": 0.4992097182080925, "pass@2": 0.5920616949615403, "pass@4": 0.6689661642728186, "pass@8": 0.7308171485662596, "pass@16": 0.7809849819776262, "pass@32": 0.8218656781610865, "pass@64": 0.8537746171902934, "pass@128": 0.8786127167630058 }, "16": { "pass@1": 0.3229586693548387, "pass@2": 0.42145907416814815, "pass@4": 0.5108256428631905, "pass@8": 0.5849112297661468, "pass@16": 0.6464023440647372, "pass@32": 0.7016135981530378, "pass@64": 0.7537077441857907, "pass@128": 0.8032258064516129 }, "17": { "pass@1": 0.25645773121387283, "pass@2": 0.3438627196076645, "pass@4": 0.42732528484517446, "pass@8": 0.4971233048281026, "pass@16": 0.5524004532069977, "pass@32": 0.5981978550504864, "pass@64": 0.6375194136444414, "pass@128": 0.6734104046242775 }, "18": { "pass@1": 0.20622767857142857, "pass@2": 0.28793061023622046, "pass@4": 0.3699408173978249, "pass@8": 0.44184211629728665, "pass@16": 0.4989635409044062, "pass@32": 0.5479358885222821, "pass@64": 0.5944936051842626, "pass@128": 0.6428571428571429 }, "19": { "pass@1": 0.16169294294294295, "pass@2": 0.23235661842945318, "pass@4": 0.30415678501898985, "pass@8": 0.3676896908172229, "pass@16": 0.4226789379330411, "pass@32": 0.47296546637220566, "pass@64": 0.5220829393578104, "pass@128": 0.5705705705705706 }, "20": { "pass@1": 0.16422672672672672, "pass@2": 0.2278495276772835, "pass@4": 0.29342507918492167, "pass@8": 0.3555577960470933, "pass@16": 0.4111276274809168, "pass@32": 0.4613041839190401, "pass@64": 0.5109832928872265, "pass@128": 0.5645645645645646 } } }, "teachers_in_school": { "count": 821120, "correct": 598262, "answer_accuracy": 0.7285926539360873, "avg_response_len": 267.6364964925955, "resp_tokens_sum": 219761680, "pass_at_k": { "pass@1": 0.7285926539360873, "pass@2": 0.7669765820143414, "pass@4": 0.7992143296098074, "pass@8": 0.8257590652483338, "pass@16": 0.8472570108859162, "pass@32": 0.8648530083401398, "pass@64": 0.8796294875653305, "pass@128": 0.8922837100545596 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9299750766871165, "pass@2": 0.9450815328969617, "pass@4": 0.9548209196243105, "pass@8": 0.9609402334817647, "pass@16": 0.9646918199515472, "pass@32": 0.9680913386644568, "pass@64": 0.9717931593100866, "pass@128": 0.9754601226993865 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 0.9999763257575758, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9970108695652173, "pass@2": 0.9971003794362662, "pass@4": 0.9971014492753624, "pass@8": 0.9971014492753624, "pass@16": 0.9971014492753624, "pass@32": 0.9971014492753624, "pass@64": 0.9971014492753624, "pass@128": 0.9971014492753624 }, "6": { "pass@1": 0.9996194484240688, "pass@2": 0.9999802585565056, "pass@4": 0.9999999113649476, "pass@8": 0.9999999999996694, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9967937138728323, "pass@2": 0.9971763204906468, "pass@4": 0.9972881948282477, "pass@8": 0.9974611419588555, "pass@16": 0.9977897000591688, "pass@32": 0.9983785444449502, "pass@64": 0.9992831459651359, "pass@128": 1.0 }, "8": { "pass@1": 0.9960607394366198, "pass@2": 0.997819743262726, "pass@4": 0.9991270069234303, "pass@8": 0.9998443747974352, "pass@16": 0.9999946157287308, "pass@32": 0.9999999970491635, "pass@64": 1.0, "pass@128": 1.0 }, "9": { "pass@1": 0.9898016467065869, "pass@2": 0.9924092513791315, "pass@4": 0.9937748347511949, "pass@8": 0.9950757224434577, "pass@16": 0.9963683581836216, "pass@32": 0.9978084671909977, "pass@64": 0.9992160024103857, "pass@128": 1.0 }, "11": { "pass@1": 0.9445359563253012, "pass@2": 0.960735165069728, "pass@4": 0.9692484578358429, "pass@8": 0.9744137411819622, "pass@16": 0.9774220287888713, "pass@32": 0.9791276976341268, "pass@64": 0.9803800331296995, "pass@128": 0.9819277108433735 }, "12": { "pass@1": 0.8528605015673981, "pass@2": 0.8893576598869498, "pass@4": 0.9101586607120814, "pass@8": 0.9224588859306808, "pass@16": 0.9310670110053562, "pass@32": 0.939270483161552, "pass@64": 0.9473029142231101, "pass@128": 0.9529780564263323 }, "13": { "pass@1": 0.7229297969187675, "pass@2": 0.7983782587838287, "pass@4": 0.8442330002867283, "pass@8": 0.8754003024489906, "pass@16": 0.8995318657586917, "pass@32": 0.9187236439009093, "pass@64": 0.9338720887441335, "pass@128": 0.9467787114845938 }, "14": { "pass@1": 0.6482775950292398, "pass@2": 0.7274618530413961, "pass@4": 0.7836484260739333, "pass@8": 0.8230514235910104, "pass@16": 0.8486700010918876, "pass@32": 0.8653014217564913, "pass@64": 0.8789849352201726, "pass@128": 0.8918128654970761 }, "15": { "pass@1": 0.5319365530303031, "pass@2": 0.6206174689811499, "pass@4": 0.6868207306473046, "pass@8": 0.7367855877874147, "pass@16": 0.7752766588096213, "pass@32": 0.806337169755552, "pass@64": 0.8352289076502452, "pass@128": 0.8606060606060606 }, "16": { "pass@1": 0.32307247150997154, "pass@2": 0.42357676043699655, "pass@4": 0.5158068435889954, "pass@8": 0.5908012273052607, "pass@16": 0.6477279950868452, "pass@32": 0.6922022409236969, "pass@64": 0.7305376737607546, "pass@128": 0.7663817663817664 }, "17": { "pass@1": 0.3075504658385093, "pass@2": 0.40160269538318555, "pass@4": 0.48713984820220435, "pass@8": 0.5596349682703199, "pass@16": 0.6222102488736226, "pass@32": 0.6744260839145514, "pass@64": 0.7144136967050305, "pass@128": 0.7484472049689441 }, "18": { "pass@1": 0.25282628676470587, "pass@2": 0.33365743689207966, "pass@4": 0.4128379664858067, "pass@8": 0.48697562456319365, "pass@16": 0.552866663966737, "pass@32": 0.6086135381508534, "pass@64": 0.6542544109542544, "pass@128": 0.6911764705882353 }, "19": { "pass@1": 0.17978050595238096, "pass@2": 0.24771110154199458, "pass@4": 0.3194336779331155, "pass@8": 0.386929726317321, "pass@16": 0.446349013761704, "pass@32": 0.5002298543667417, "pass@64": 0.5475897643199032, "pass@128": 0.5863095238095238 }, "20": { "pass@1": 0.15585235778443113, "pass@2": 0.2257480609646848, "pass@4": 0.30009878920449323, "pass@8": 0.3697365614620309, "pass@16": 0.4289157795835793, "pass@32": 0.47570921025814755, "pass@64": 0.5129501839362879, "pass@128": 0.5449101796407185 } } }, "movie_festival_awards": { "count": 801152, "correct": 581357, "answer_accuracy": 0.7256513121105608, "avg_response_len": 286.265858413884, "resp_tokens_sum": 229342465, "pass_at_k": { "pass@1": 0.7256513121105608, "pass@2": 0.7625552668723906, "pass@4": 0.7936511745695828, "pass@8": 0.8199106880345169, "pass@16": 0.8416984025434961, "pass@32": 0.859703225160211, "pass@64": 0.8748839546594552, "pass@128": 0.8881610480907494 }, "per_op_pass_at_k": { "10": { "pass@1": 0.9540345149253732, "pass@2": 0.9661670730990719, "pass@4": 0.9737339595610252, "pass@8": 0.9792955903839986, "pass@16": 0.9827165338826096, "pass@32": 0.9843298489148303, "pass@64": 0.9849885152450082, "pass@128": 0.9850746268656716 }, "2": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "3": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "4": { "pass@1": 1.0, "pass@2": 1.0, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "5": { "pass@1": 0.9884420955882353, "pass@2": 0.9918386695229272, "pass@4": 0.9951052884933501, "pass@8": 0.9974676076953616, "pass@16": 0.9986750762012426, "pass@32": 0.9994972316545585, "pass@64": 0.9999593420621558, "pass@128": 1.0 }, "6": { "pass@1": 0.9998274842271293, "pass@2": 0.9999984475521002, "pass@4": 1.0, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "7": { "pass@1": 0.9995302610759493, "pass@2": 0.9999871517741453, "pass@4": 0.9999999881343946, "pass@8": 1.0, "pass@16": 1.0, "pass@32": 1.0, "pass@64": 1.0, "pass@128": 1.0 }, "8": { "pass@1": 0.9923986486486487, "pass@2": 0.9960404548249037, "pass@4": 0.9979970237391498, "pass@8": 0.9993233078415068, "pass@16": 0.9999099641930826, "pass@32": 0.9999986991137875, "pass@64": 0.9999999999875272, "pass@128": 1.0 }, "9": { "pass@1": 0.9886706189710611, "pass@2": 0.9933001240600552, "pass@4": 0.9959347885975668, "pass@8": 0.9979804126552496, "pass@16": 0.999431623703106, "pass@32": 0.9999655897612711, "pass@64": 0.9999999801176399, "pass@128": 1.0 }, "11": { "pass@1": 0.9533978174603175, "pass@2": 0.9724440694913137, "pass@4": 0.9814224727861399, "pass@8": 0.9859808984729452, "pass@16": 0.9886611665135104, "pass@32": 0.9900384257613478, "pass@64": 0.9904552883937954, "pass@128": 0.9904761904761905 }, "12": { "pass@1": 0.8652522935779816, "pass@2": 0.9085191221796822, "pass@4": 0.9335249151768868, "pass@8": 0.9493285515894258, "pass@16": 0.9598282138920922, "pass@32": 0.9673255156709447, "pass@64": 0.9731453224447834, "pass@128": 0.9785932721712538 }, "13": { "pass@1": 0.6766734182098766, "pass@2": 0.7418051150724216, "pass@4": 0.7869648880695465, "pass@8": 0.8206838073852996, "pass@16": 0.8472708154414211, "pass@32": 0.8691403156138889, "pass@64": 0.8887575482814627, "pass@128": 0.9104938271604939 }, "14": { "pass@1": 0.6343657270029673, "pass@2": 0.710647079955139, "pass@4": 0.76384462420684, "pass@8": 0.8053589968983749, "pass@16": 0.8376984322875355, "pass@32": 0.8624379829234405, "pass@64": 0.8842262313961666, "pass@128": 0.9050445103857567 }, "15": { "pass@1": 0.5200858410493827, "pass@2": 0.5983652000097209, "pass@4": 0.660162098950594, "pass@8": 0.7098186955100998, "pass@16": 0.7494732224712638, "pass@32": 0.782183749337275, "pass@64": 0.8108561559010913, "pass@128": 0.8333333333333334 }, "16": { "pass@1": 0.3369284660766962, "pass@2": 0.4336185196153577, "pass@4": 0.5248042432195976, "pass@8": 0.6073500575462162, "pass@16": 0.6767348416897403, "pass@32": 0.7298902304694688, "pass@64": 0.7681849356854631, "pass@128": 0.799410029498525 }, "17": { "pass@1": 0.2506588855421687, "pass@2": 0.3325241171378427, "pass@4": 0.4093735848907438, "pass@8": 0.47527240531329096, "pass@16": 0.5328122572752206, "pass@32": 0.5861045627721161, "pass@64": 0.6353681803030131, "pass@128": 0.6807228915662651 }, "18": { "pass@1": 0.26272681451612906, "pass@2": 0.33898669672339343, "pass@4": 0.40957589746846157, "pass@8": 0.46941617022340476, "pass@16": 0.516339879142077, "pass@32": 0.5533118163588501, "pass@64": 0.585479609144845, "pass@128": 0.6161290322580645 }, "19": { "pass@1": 0.19115842145015105, "pass@2": 0.26258526714560976, "pass@4": 0.3307034777752478, "pass@8": 0.3926879790877576, "pass@16": 0.44833979573114197, "pass@32": 0.49538652663741045, "pass@64": 0.5320829769264548, "pass@128": 0.5619335347432024 }, "20": { "pass@1": 0.1606371996996997, "pass@2": 0.230762381475964, "pass@4": 0.3060352743069278, "pass@8": 0.3783404293240473, "pass@16": 0.44421421827119956, "pass@32": 0.5043447793446393, "pass@64": 0.558844624078598, "pass@128": 0.6036036036036037 } } } } } ]