| [ |
| { |
| "checkpoint": "checkpoint-42828", |
| "id_acc": 0.9906362847222222, |
| "ood_acc": 0.482271875, |
| "total_acc": 0.7230760690789474, |
| "id_avg_loss": 0.0, |
| "ood_avg_loss": 0.0, |
| "total_avg_loss": 0.0, |
| "id_avg_resp_len": 197.01928211805554, |
| "ood_avg_resp_len": 336.27141796875, |
| "count_id": 1152000, |
| "count_ood": 1280000, |
| "count_total": 2432000, |
| "template_metrics": { |
| "crazy_zootopia": { |
| "count": 809728, |
| "correct": 578902, |
| "answer_accuracy": 0.7149339037306355, |
| "avg_response_len": 257.2338896518337, |
| "resp_tokens_sum": 208289483, |
| "pass_at_k": { |
| "pass@1": 0.7149339037306355, |
| "pass@2": 0.7528802711780616, |
| "pass@4": 0.7853499391160436, |
| "pass@8": 0.8123146802770955, |
| "pass@16": 0.8344963841133981, |
| "pass@32": 0.8535153920866395, |
| "pass@64": 0.8706372430698933, |
| "pass@128": 0.8866582358520392 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9403346238938053, |
| "pass@2": 0.9509430962999099, |
| "pass@4": 0.9565902188111443, |
| "pass@8": 0.9595371010736143, |
| "pass@16": 0.9614777765607152, |
| "pass@32": 0.9640615389013077, |
| "pass@64": 0.9683065059344581, |
| "pass@128": 0.9734513274336283 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999761085626911, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9942212301587302, |
| "pass@2": 0.9963039776277965, |
| "pass@4": 0.9979051413216204, |
| "pass@8": 0.9991469167239885, |
| "pass@16": 0.9998257477435663, |
| "pass@32": 0.999994264397576, |
| "pass@64": 0.9999999994794706, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9983626497005988, |
| "pass@2": 0.9992098725776793, |
| "pass@4": 0.9997977273798858, |
| "pass@8": 0.9999879720977759, |
| "pass@16": 0.9999999725620663, |
| "pass@32": 0.9999999999999857, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9966715976331361, |
| "pass@2": 0.9970745439826679, |
| "pass@4": 0.997133861873035, |
| "pass@8": 0.9972263313609467, |
| "pass@16": 0.9974112426035503, |
| "pass@32": 0.9977810650887574, |
| "pass@64": 0.9985207100591716, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9923377403846154, |
| "pass@2": 0.9947195039622452, |
| "pass@4": 0.9961377279763107, |
| "pass@8": 0.9976819311023585, |
| "pass@16": 0.9991340276202069, |
| "pass@32": 0.9998780546502605, |
| "pass@64": 0.9999990200711261, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9925176056338029, |
| "pass@2": 0.9967980619940113, |
| "pass@4": 0.998693938433752, |
| "pass@8": 0.9996446951320699, |
| "pass@16": 0.9999658250813326, |
| "pass@32": 0.9999998146310128, |
| "pass@64": 0.9999999999999163, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9313473796033994, |
| "pass@2": 0.9494340551181104, |
| "pass@4": 0.9594840918433353, |
| "pass@8": 0.9656293634436289, |
| "pass@16": 0.9699144094325778, |
| "pass@32": 0.9738884592658764, |
| "pass@64": 0.9786071170542627, |
| "pass@128": 0.9830028328611898 |
| }, |
| "12": { |
| "pass@1": 0.8317002118644068, |
| "pass@2": 0.8737276319008856, |
| "pass@4": 0.8998880984686232, |
| "pass@8": 0.9179671212613043, |
| "pass@16": 0.9325065069008127, |
| "pass@32": 0.9452756292304754, |
| "pass@64": 0.9557585734361933, |
| "pass@128": 0.963276836158192 |
| }, |
| "13": { |
| "pass@1": 0.7110109717868338, |
| "pass@2": 0.7740023264137438, |
| "pass@4": 0.8133029303938883, |
| "pass@8": 0.8424745465775301, |
| "pass@16": 0.8657274734514422, |
| "pass@32": 0.883408745813764, |
| "pass@64": 0.8965575001318108, |
| "pass@128": 0.9090909090909091 |
| }, |
| "14": { |
| "pass@1": 0.6173091900311527, |
| "pass@2": 0.6968503937007875, |
| "pass@4": 0.755002373827103, |
| "pass@8": 0.8013279156768645, |
| "pass@16": 0.8399074981743214, |
| "pass@32": 0.8702874981026382, |
| "pass@64": 0.8921680924393293, |
| "pass@128": 0.9034267912772586 |
| }, |
| "15": { |
| "pass@1": 0.4992097182080925, |
| "pass@2": 0.5920616949615403, |
| "pass@4": 0.6689661642728186, |
| "pass@8": 0.7308171485662596, |
| "pass@16": 0.7809849819776262, |
| "pass@32": 0.8218656781610865, |
| "pass@64": 0.8537746171902934, |
| "pass@128": 0.8786127167630058 |
| }, |
| "16": { |
| "pass@1": 0.3229586693548387, |
| "pass@2": 0.42145907416814815, |
| "pass@4": 0.5108256428631905, |
| "pass@8": 0.5849112297661468, |
| "pass@16": 0.6464023440647372, |
| "pass@32": 0.7016135981530378, |
| "pass@64": 0.7537077441857907, |
| "pass@128": 0.8032258064516129 |
| }, |
| "17": { |
| "pass@1": 0.25645773121387283, |
| "pass@2": 0.3438627196076645, |
| "pass@4": 0.42732528484517446, |
| "pass@8": 0.4971233048281026, |
| "pass@16": 0.5524004532069977, |
| "pass@32": 0.5981978550504864, |
| "pass@64": 0.6375194136444414, |
| "pass@128": 0.6734104046242775 |
| }, |
| "18": { |
| "pass@1": 0.20622767857142857, |
| "pass@2": 0.28793061023622046, |
| "pass@4": 0.3699408173978249, |
| "pass@8": 0.44184211629728665, |
| "pass@16": 0.4989635409044062, |
| "pass@32": 0.5479358885222821, |
| "pass@64": 0.5944936051842626, |
| "pass@128": 0.6428571428571429 |
| }, |
| "19": { |
| "pass@1": 0.16169294294294295, |
| "pass@2": 0.23235661842945318, |
| "pass@4": 0.30415678501898985, |
| "pass@8": 0.3676896908172229, |
| "pass@16": 0.4226789379330411, |
| "pass@32": 0.47296546637220566, |
| "pass@64": 0.5220829393578104, |
| "pass@128": 0.5705705705705706 |
| }, |
| "20": { |
| "pass@1": 0.16422672672672672, |
| "pass@2": 0.2278495276772835, |
| "pass@4": 0.29342507918492167, |
| "pass@8": 0.3555577960470933, |
| "pass@16": 0.4111276274809168, |
| "pass@32": 0.4613041839190401, |
| "pass@64": 0.5109832928872265, |
| "pass@128": 0.5645645645645646 |
| } |
| } |
| }, |
| "teachers_in_school": { |
| "count": 821120, |
| "correct": 598262, |
| "answer_accuracy": 0.7285926539360873, |
| "avg_response_len": 267.6364964925955, |
| "resp_tokens_sum": 219761680, |
| "pass_at_k": { |
| "pass@1": 0.7285926539360873, |
| "pass@2": 0.7669765820143414, |
| "pass@4": 0.7992143296098074, |
| "pass@8": 0.8257590652483338, |
| "pass@16": 0.8472570108859162, |
| "pass@32": 0.8648530083401398, |
| "pass@64": 0.8796294875653305, |
| "pass@128": 0.8922837100545596 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9299750766871165, |
| "pass@2": 0.9450815328969617, |
| "pass@4": 0.9548209196243105, |
| "pass@8": 0.9609402334817647, |
| "pass@16": 0.9646918199515472, |
| "pass@32": 0.9680913386644568, |
| "pass@64": 0.9717931593100866, |
| "pass@128": 0.9754601226993865 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 0.9999763257575758, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9970108695652173, |
| "pass@2": 0.9971003794362662, |
| "pass@4": 0.9971014492753624, |
| "pass@8": 0.9971014492753624, |
| "pass@16": 0.9971014492753624, |
| "pass@32": 0.9971014492753624, |
| "pass@64": 0.9971014492753624, |
| "pass@128": 0.9971014492753624 |
| }, |
| "6": { |
| "pass@1": 0.9996194484240688, |
| "pass@2": 0.9999802585565056, |
| "pass@4": 0.9999999113649476, |
| "pass@8": 0.9999999999996694, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9967937138728323, |
| "pass@2": 0.9971763204906468, |
| "pass@4": 0.9972881948282477, |
| "pass@8": 0.9974611419588555, |
| "pass@16": 0.9977897000591688, |
| "pass@32": 0.9983785444449502, |
| "pass@64": 0.9992831459651359, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9960607394366198, |
| "pass@2": 0.997819743262726, |
| "pass@4": 0.9991270069234303, |
| "pass@8": 0.9998443747974352, |
| "pass@16": 0.9999946157287308, |
| "pass@32": 0.9999999970491635, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9898016467065869, |
| "pass@2": 0.9924092513791315, |
| "pass@4": 0.9937748347511949, |
| "pass@8": 0.9950757224434577, |
| "pass@16": 0.9963683581836216, |
| "pass@32": 0.9978084671909977, |
| "pass@64": 0.9992160024103857, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9445359563253012, |
| "pass@2": 0.960735165069728, |
| "pass@4": 0.9692484578358429, |
| "pass@8": 0.9744137411819622, |
| "pass@16": 0.9774220287888713, |
| "pass@32": 0.9791276976341268, |
| "pass@64": 0.9803800331296995, |
| "pass@128": 0.9819277108433735 |
| }, |
| "12": { |
| "pass@1": 0.8528605015673981, |
| "pass@2": 0.8893576598869498, |
| "pass@4": 0.9101586607120814, |
| "pass@8": 0.9224588859306808, |
| "pass@16": 0.9310670110053562, |
| "pass@32": 0.939270483161552, |
| "pass@64": 0.9473029142231101, |
| "pass@128": 0.9529780564263323 |
| }, |
| "13": { |
| "pass@1": 0.7229297969187675, |
| "pass@2": 0.7983782587838287, |
| "pass@4": 0.8442330002867283, |
| "pass@8": 0.8754003024489906, |
| "pass@16": 0.8995318657586917, |
| "pass@32": 0.9187236439009093, |
| "pass@64": 0.9338720887441335, |
| "pass@128": 0.9467787114845938 |
| }, |
| "14": { |
| "pass@1": 0.6482775950292398, |
| "pass@2": 0.7274618530413961, |
| "pass@4": 0.7836484260739333, |
| "pass@8": 0.8230514235910104, |
| "pass@16": 0.8486700010918876, |
| "pass@32": 0.8653014217564913, |
| "pass@64": 0.8789849352201726, |
| "pass@128": 0.8918128654970761 |
| }, |
| "15": { |
| "pass@1": 0.5319365530303031, |
| "pass@2": 0.6206174689811499, |
| "pass@4": 0.6868207306473046, |
| "pass@8": 0.7367855877874147, |
| "pass@16": 0.7752766588096213, |
| "pass@32": 0.806337169755552, |
| "pass@64": 0.8352289076502452, |
| "pass@128": 0.8606060606060606 |
| }, |
| "16": { |
| "pass@1": 0.32307247150997154, |
| "pass@2": 0.42357676043699655, |
| "pass@4": 0.5158068435889954, |
| "pass@8": 0.5908012273052607, |
| "pass@16": 0.6477279950868452, |
| "pass@32": 0.6922022409236969, |
| "pass@64": 0.7305376737607546, |
| "pass@128": 0.7663817663817664 |
| }, |
| "17": { |
| "pass@1": 0.3075504658385093, |
| "pass@2": 0.40160269538318555, |
| "pass@4": 0.48713984820220435, |
| "pass@8": 0.5596349682703199, |
| "pass@16": 0.6222102488736226, |
| "pass@32": 0.6744260839145514, |
| "pass@64": 0.7144136967050305, |
| "pass@128": 0.7484472049689441 |
| }, |
| "18": { |
| "pass@1": 0.25282628676470587, |
| "pass@2": 0.33365743689207966, |
| "pass@4": 0.4128379664858067, |
| "pass@8": 0.48697562456319365, |
| "pass@16": 0.552866663966737, |
| "pass@32": 0.6086135381508534, |
| "pass@64": 0.6542544109542544, |
| "pass@128": 0.6911764705882353 |
| }, |
| "19": { |
| "pass@1": 0.17978050595238096, |
| "pass@2": 0.24771110154199458, |
| "pass@4": 0.3194336779331155, |
| "pass@8": 0.386929726317321, |
| "pass@16": 0.446349013761704, |
| "pass@32": 0.5002298543667417, |
| "pass@64": 0.5475897643199032, |
| "pass@128": 0.5863095238095238 |
| }, |
| "20": { |
| "pass@1": 0.15585235778443113, |
| "pass@2": 0.2257480609646848, |
| "pass@4": 0.30009878920449323, |
| "pass@8": 0.3697365614620309, |
| "pass@16": 0.4289157795835793, |
| "pass@32": 0.47570921025814755, |
| "pass@64": 0.5129501839362879, |
| "pass@128": 0.5449101796407185 |
| } |
| } |
| }, |
| "movie_festival_awards": { |
| "count": 801152, |
| "correct": 581357, |
| "answer_accuracy": 0.7256513121105608, |
| "avg_response_len": 286.265858413884, |
| "resp_tokens_sum": 229342465, |
| "pass_at_k": { |
| "pass@1": 0.7256513121105608, |
| "pass@2": 0.7625552668723906, |
| "pass@4": 0.7936511745695828, |
| "pass@8": 0.8199106880345169, |
| "pass@16": 0.8416984025434961, |
| "pass@32": 0.859703225160211, |
| "pass@64": 0.8748839546594552, |
| "pass@128": 0.8881610480907494 |
| }, |
| "per_op_pass_at_k": { |
| "10": { |
| "pass@1": 0.9540345149253732, |
| "pass@2": 0.9661670730990719, |
| "pass@4": 0.9737339595610252, |
| "pass@8": 0.9792955903839986, |
| "pass@16": 0.9827165338826096, |
| "pass@32": 0.9843298489148303, |
| "pass@64": 0.9849885152450082, |
| "pass@128": 0.9850746268656716 |
| }, |
| "2": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "3": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "4": { |
| "pass@1": 1.0, |
| "pass@2": 1.0, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "5": { |
| "pass@1": 0.9884420955882353, |
| "pass@2": 0.9918386695229272, |
| "pass@4": 0.9951052884933501, |
| "pass@8": 0.9974676076953616, |
| "pass@16": 0.9986750762012426, |
| "pass@32": 0.9994972316545585, |
| "pass@64": 0.9999593420621558, |
| "pass@128": 1.0 |
| }, |
| "6": { |
| "pass@1": 0.9998274842271293, |
| "pass@2": 0.9999984475521002, |
| "pass@4": 1.0, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "7": { |
| "pass@1": 0.9995302610759493, |
| "pass@2": 0.9999871517741453, |
| "pass@4": 0.9999999881343946, |
| "pass@8": 1.0, |
| "pass@16": 1.0, |
| "pass@32": 1.0, |
| "pass@64": 1.0, |
| "pass@128": 1.0 |
| }, |
| "8": { |
| "pass@1": 0.9923986486486487, |
| "pass@2": 0.9960404548249037, |
| "pass@4": 0.9979970237391498, |
| "pass@8": 0.9993233078415068, |
| "pass@16": 0.9999099641930826, |
| "pass@32": 0.9999986991137875, |
| "pass@64": 0.9999999999875272, |
| "pass@128": 1.0 |
| }, |
| "9": { |
| "pass@1": 0.9886706189710611, |
| "pass@2": 0.9933001240600552, |
| "pass@4": 0.9959347885975668, |
| "pass@8": 0.9979804126552496, |
| "pass@16": 0.999431623703106, |
| "pass@32": 0.9999655897612711, |
| "pass@64": 0.9999999801176399, |
| "pass@128": 1.0 |
| }, |
| "11": { |
| "pass@1": 0.9533978174603175, |
| "pass@2": 0.9724440694913137, |
| "pass@4": 0.9814224727861399, |
| "pass@8": 0.9859808984729452, |
| "pass@16": 0.9886611665135104, |
| "pass@32": 0.9900384257613478, |
| "pass@64": 0.9904552883937954, |
| "pass@128": 0.9904761904761905 |
| }, |
| "12": { |
| "pass@1": 0.8652522935779816, |
| "pass@2": 0.9085191221796822, |
| "pass@4": 0.9335249151768868, |
| "pass@8": 0.9493285515894258, |
| "pass@16": 0.9598282138920922, |
| "pass@32": 0.9673255156709447, |
| "pass@64": 0.9731453224447834, |
| "pass@128": 0.9785932721712538 |
| }, |
| "13": { |
| "pass@1": 0.6766734182098766, |
| "pass@2": 0.7418051150724216, |
| "pass@4": 0.7869648880695465, |
| "pass@8": 0.8206838073852996, |
| "pass@16": 0.8472708154414211, |
| "pass@32": 0.8691403156138889, |
| "pass@64": 0.8887575482814627, |
| "pass@128": 0.9104938271604939 |
| }, |
| "14": { |
| "pass@1": 0.6343657270029673, |
| "pass@2": 0.710647079955139, |
| "pass@4": 0.76384462420684, |
| "pass@8": 0.8053589968983749, |
| "pass@16": 0.8376984322875355, |
| "pass@32": 0.8624379829234405, |
| "pass@64": 0.8842262313961666, |
| "pass@128": 0.9050445103857567 |
| }, |
| "15": { |
| "pass@1": 0.5200858410493827, |
| "pass@2": 0.5983652000097209, |
| "pass@4": 0.660162098950594, |
| "pass@8": 0.7098186955100998, |
| "pass@16": 0.7494732224712638, |
| "pass@32": 0.782183749337275, |
| "pass@64": 0.8108561559010913, |
| "pass@128": 0.8333333333333334 |
| }, |
| "16": { |
| "pass@1": 0.3369284660766962, |
| "pass@2": 0.4336185196153577, |
| "pass@4": 0.5248042432195976, |
| "pass@8": 0.6073500575462162, |
| "pass@16": 0.6767348416897403, |
| "pass@32": 0.7298902304694688, |
| "pass@64": 0.7681849356854631, |
| "pass@128": 0.799410029498525 |
| }, |
| "17": { |
| "pass@1": 0.2506588855421687, |
| "pass@2": 0.3325241171378427, |
| "pass@4": 0.4093735848907438, |
| "pass@8": 0.47527240531329096, |
| "pass@16": 0.5328122572752206, |
| "pass@32": 0.5861045627721161, |
| "pass@64": 0.6353681803030131, |
| "pass@128": 0.6807228915662651 |
| }, |
| "18": { |
| "pass@1": 0.26272681451612906, |
| "pass@2": 0.33898669672339343, |
| "pass@4": 0.40957589746846157, |
| "pass@8": 0.46941617022340476, |
| "pass@16": 0.516339879142077, |
| "pass@32": 0.5533118163588501, |
| "pass@64": 0.585479609144845, |
| "pass@128": 0.6161290322580645 |
| }, |
| "19": { |
| "pass@1": 0.19115842145015105, |
| "pass@2": 0.26258526714560976, |
| "pass@4": 0.3307034777752478, |
| "pass@8": 0.3926879790877576, |
| "pass@16": 0.44833979573114197, |
| "pass@32": 0.49538652663741045, |
| "pass@64": 0.5320829769264548, |
| "pass@128": 0.5619335347432024 |
| }, |
| "20": { |
| "pass@1": 0.1606371996996997, |
| "pass@2": 0.230762381475964, |
| "pass@4": 0.3060352743069278, |
| "pass@8": 0.3783404293240473, |
| "pass@16": 0.44421421827119956, |
| "pass@32": 0.5043447793446393, |
| "pass@64": 0.558844624078598, |
| "pass@128": 0.6036036036036037 |
| } |
| } |
| } |
| } |
| } |
| ] |
