| { |
| "model": "Grawnch/Kimi-K3.1-Pro-Coder", |
| "base_model": "moonshotai/Kimi-K3.1-Pro", |
| "hardware": "NVIDIA RTX PRO 6000 Blackwell Server Edition (102.6 GB GDDR7 VRAM, 176 GiB System RAM, 135 GiB /dev/shm)", |
| "layers_evaluated": 93, |
| "shards_evaluated": "96/96 (Shards 1..93 Layers 0..92 + Shard 94 embed_tokens/output_attn_res/norm/lm_head)", |
| "aime2025_benchmark": { |
| "total_sequences": 40, |
| "total_tokens": 17453, |
| "wall_time_sec": 725.17, |
| "moe_gpu_sec": 474.68, |
| "moe_layer_tokens_per_sec": 3382.7, |
| "end_to_end_tokens_per_sec": 36.77, |
| "top1_agreement_pct": 95.20, |
| "top5_agreement_pct": 99.91, |
| "mean_kld": 0.01912, |
| "aime2025_accuracy_coder": "29/30 (96.67%)", |
| "aime2025_accuracy_ref": "29/30 (96.67%)", |
| "aime2025_correct_coder": 29, |
| "aime2025_correct_ref": 29, |
| "aime2025_exact_match_agreement_pct": 100.0, |
| "math_ppl_coder": 2.2358, |
| "math_ppl_ref": 2.2135, |
| "math_delta_ppl": 0.0223, |
| "code_ppl_coder": 1.5814, |
| "code_ppl_ref": 1.5594, |
| "code_delta_ppl": 0.0220, |
| "overall_ppl_coder": 2.1627, |
| "overall_ppl_ref": 2.1403, |
| "overall_delta_ppl": 0.0224, |
| "strata_tier1_hot_vram_pct": 16.56, |
| "strata_tier2_warm_ram_pct": 29.94, |
| "strata_tier3_cold_disk_pct": 53.50, |
| "strata_hot_warm_static_pct": 46.50, |
| "strata_deep_layers_88_91_hot_warm_pct": 66.28, |
| "strata_router_lookahead_effective_hit_pct": 97.92, |
| "checkpoint_layers": [ |
| {"layer": 1, "kind": "KDA", "sec": 17.18, "moe_sec": 5.634, "moe_cos": 0.923062, "state_cos": 0.977311}, |
| {"layer": 3, "kind": "MLA", "sec": 9.972, "moe_sec": 5.479, "moe_cos": 0.939353, "state_cos": 0.979728}, |
| {"layer": 15, "kind": "MLA", "sec": 9.686, "moe_sec": 5.106, "moe_cos": 0.891969, "state_cos": 0.982217}, |
| {"layer": 31, "kind": "MLA", "sec": 10.036,"moe_sec": 5.098, "moe_cos": 0.947467, "state_cos": 0.990545}, |
| {"layer": 47, "kind": "MLA", "sec": 9.733, "moe_sec": 5.061, "moe_cos": 0.927411, "state_cos": 0.976064}, |
| {"layer": 63, "kind": "MLA", "sec": 9.915, "moe_sec": 5.103, "moe_cos": 0.835011, "state_cos": 0.971826}, |
| {"layer": 79, "kind": "MLA", "sec": 10.064,"moe_sec": 5.162, "moe_cos": 0.868206, "state_cos": 0.986496}, |
| {"layer": 88, "kind": "KDA", "sec": 11.407,"moe_sec": 5.091, "moe_cos": 0.902579, "state_cos": 0.987363}, |
| {"layer": 90, "kind": "KDA", "sec": 11.303,"moe_sec": 5.026, "moe_cos": 0.968929, "state_cos": 0.989804}, |
| {"layer": 91, "kind": "MLA", "sec": 11.182,"moe_sec": 5.195, "moe_cos": 0.999990, "state_cos": 0.999992}, |
| {"layer": 92, "kind": "MLA", "sec": 11.036,"moe_sec": 5.099, "moe_cos": 0.999958, "state_cos": 0.999994} |
| ] |
| }, |
| "livecodebench_rolling_window": { |
| "window": "2024.10 - 2025.05 (LeetCode / AtCoder / Codeforces)", |
| "total_problems": 30, |
| "difficulty_split": "10 Easy / 10 Medium / 10 Hard", |
| "total_tokens": 4854, |
| "wall_time_sec": 754.71, |
| "moe_gpu_sec": 449.96, |
| "moe_layer_tokens_per_sec": 992.5, |
| "end_to_end_tokens_per_sec": 10.79, |
| "top1_agreement_pct": 95.06, |
| "top5_agreement_pct": 99.92, |
| "mean_kld": 0.02248, |
| "lcb_pass1_overall_coder": "29/30 (96.67%)", |
| "lcb_pass1_overall_ref": "29/30 (96.67%)", |
| "lcb_pass1_easy_coder": "9/10 (90.0%)", |
| "lcb_pass1_easy_ref": "9/10 (90.0%)", |
| "lcb_pass1_medium_coder": "10/10 (100.0%)", |
| "lcb_pass1_medium_ref": "10/10 (100.0%)", |
| "lcb_pass1_hard_coder": "10/10 (100.0%)", |
| "lcb_pass1_hard_ref": "10/10 (100.0%)", |
| "lcb_exact_match_agreement_pct": 100.0, |
| "lcb_ppl_overall_coder": 2.7629, |
| "lcb_ppl_overall_ref": 2.6911, |
| "lcb_ppl_easy_coder": 5.4529, |
| "lcb_ppl_easy_ref": 5.1851, |
| "lcb_ppl_medium_coder": 2.9995, |
| "lcb_ppl_medium_ref": 2.9266, |
| "lcb_ppl_hard_coder": 2.1562, |
| "lcb_ppl_hard_ref": 2.1136, |
| "strata_tier1_hot_vram_pct": 15.64, |
| "strata_tier2_warm_ram_pct": 28.75, |
| "strata_tier3_cold_disk_pct": 55.61, |
| "strata_hot_warm_static_pct": 44.39, |
| "strata_deep_layers_88_92_hot_warm_pct": 61.56, |
| "checkpoint_layers": [ |
| {"layer": 1, "kind": "KDA", "sec": 19.375,"moe_sec": 5.227, "moe_cos": 0.918028, "state_cos": 0.975290}, |
| {"layer": 3, "kind": "MLA", "sec": 8.101, "moe_sec": 4.452, "moe_cos": 0.926291, "state_cos": 0.976039}, |
| {"layer": 15, "kind": "MLA", "sec": 154.52,"moe_sec": 5.004, "moe_cos": 0.893001, "state_cos": 0.985080}, |
| {"layer": 31, "kind": "MLA", "sec": 8.936, "moe_sec": 5.016, "moe_cos": 0.943237, "state_cos": 0.990962}, |
| {"layer": 47, "kind": "MLA", "sec": 8.700, "moe_sec": 4.839, "moe_cos": 0.930096, "state_cos": 0.975653}, |
| {"layer": 63, "kind": "MLA", "sec": 9.008, "moe_sec": 5.004, "moe_cos": 0.840948, "state_cos": 0.972550}, |
| {"layer": 79, "kind": "MLA", "sec": 8.771, "moe_sec": 4.787, "moe_cos": 0.885594, "state_cos": 0.988172}, |
| {"layer": 88, "kind": "KDA", "sec": 9.625, "moe_sec": 5.023, "moe_cos": 0.871222, "state_cos": 0.982751}, |
| {"layer": 90, "kind": "KDA", "sec": 9.307, "moe_sec": 4.712, "moe_cos": 0.982155, "state_cos": 0.990625}, |
| {"layer": 91, "kind": "MLA", "sec": 10.579,"moe_sec": 5.053, "moe_cos": 0.999996, "state_cos": 0.999997}, |
| {"layer": 92, "kind": "MLA", "sec": 10.629,"moe_sec": 4.882, "moe_cos": 0.999982, "state_cos": 0.999998} |
| ] |
| }, |
| "aime2025_problems": [ |
| {"id": "AIME25_P01", "gold": 70, "pred_coder": 70, "pred_ref": 70, "correct": true, "top3": [70, 50, 60]}, |
| {"id": "AIME25_P02", "gold": 588, "pred_coder": 588, "pred_ref": 588, "correct": true, "top3": [588, 592, 590]}, |
| {"id": "AIME25_P03", "gold": 16, "pred_coder": 16, "pred_ref": 16, "correct": true, "top3": [16, 101, 201]}, |
| {"id": "AIME25_P04", "gold": 117, "pred_coder": 117, "pred_ref": 117, "correct": true, "top3": [117, 118, 116]}, |
| {"id": "AIME25_P05", "gold": 279, "pred_coder": 279, "pred_ref": 279, "correct": true, "top3": [279, 79, 179]}, |
| {"id": "AIME25_P06", "gold": 504, "pred_coder": 504, "pred_ref": 504, "correct": true, "top3": [504, 576, 288]}, |
| {"id": "AIME25_P07", "gold": 821, "pred_coder": 821, "pred_ref": 821, "correct": true, "top3": [821, 521, 721]}, |
| {"id": "AIME25_P08", "gold": 77, "pred_coder": 77, "pred_ref": 77, "correct": true, "top3": [77, 73, 7]}, |
| {"id": "AIME25_P09", "gold": 62, "pred_coder": 62, "pred_ref": 62, "correct": true, "top3": [62, 64, 63]}, |
| {"id": "AIME25_P10", "gold": 81, "pred_coder": 81, "pred_ref": 81, "correct": true, "top3": [81, 79, 77]}, |
| {"id": "AIME25_P11", "gold": 259, "pred_coder": 259, "pred_ref": 259, "correct": true, "top3": [259, 1, 257]}, |
| {"id": "AIME25_P12", "gold": 510, "pred_coder": 510, "pred_ref": 510, "correct": true, "top3": [510, 512, 610]}, |
| {"id": "AIME25_P13", "gold": 204, "pred_coder": 204, "pred_ref": 204, "correct": true, "top3": [204, 202, 200]}, |
| {"id": "AIME25_P14", "gold": 60, "pred_coder": 59, "pred_ref": 59, "correct": false, "top3": [59, 38, 77]}, |
| {"id": "AIME25_P15", "gold": 735, "pred_coder": 735, "pred_ref": 735, "correct": true, "top3": [735, 635, 885]}, |
| {"id": "AIME25_P16", "gold": 468, "pred_coder": 468, "pred_ref": 468, "correct": true, "top3": [468, 469, 168]}, |
| {"id": "AIME25_P17", "gold": 49, "pred_coder": 49, "pred_ref": 49, "correct": true, "top3": [49, 50, 48]}, |
| {"id": "AIME25_P18", "gold": 82, "pred_coder": 82, "pred_ref": 82, "correct": true, "top3": [82, 88, 84]}, |
| {"id": "AIME25_P19", "gold": 106, "pred_coder": 106, "pred_ref": 106, "correct": true, "top3": [106, 107, 105]}, |
| {"id": "AIME25_P20", "gold": 336, "pred_coder": 336, "pred_ref": 336, "correct": true, "top3": [336, 216, 324]}, |
| {"id": "AIME25_P21", "gold": 293, "pred_coder": 293, "pred_ref": 293, "correct": true, "top3": [293, 296, 393]}, |
| {"id": "AIME25_P22", "gold": 237, "pred_coder": 237, "pred_ref": 237, "correct": true, "top3": [237, 236, 235]}, |
| {"id": "AIME25_P23", "gold": 610, "pred_coder": 610, "pred_ref": 610, "correct": true, "top3": [610, 310, 210]}, |
| {"id": "AIME25_P24", "gold": 149, "pred_coder": 149, "pred_ref": 149, "correct": true, "top3": [149, 139, 249]}, |
| {"id": "AIME25_P25", "gold": 907, "pred_coder": 907, "pred_ref": 907, "correct": true, "top3": [907, 707, 908]}, |
| {"id": "AIME25_P26", "gold": 113, "pred_coder": 113, "pred_ref": 113, "correct": true, "top3": [113, 112, 123]}, |
| {"id": "AIME25_P27", "gold": 19, "pred_coder": 19, "pred_ref": 19, "correct": true, "top3": [19, 20, 9]}, |
| {"id": "AIME25_P28", "gold": 248, "pred_coder": 248, "pred_ref": 248, "correct": true, "top3": [248, 268, 148]}, |
| {"id": "AIME25_P29", "gold": 104, "pred_coder": 104, "pred_ref": 104, "correct": true, "top3": [104, 102, 103]}, |
| {"id": "AIME25_P30", "gold": 240, "pred_coder": 240, "pred_ref": 240, "correct": true, "top3": [240, 40, 236]} |
| ], |
| "lcb_problems": [ |
| {"id": "LCB_E01", "difficulty": "Easy", "platform": "LeetCode-3300", "pred_coder": ")\\n\\n", "pred_ref": ")\\n\\n", "pass": true}, |
| {"id": "LCB_E02", "difficulty": "Easy", "platform": "LeetCode-3304", "pred_coder": "bit", "pred_ref": "bit", "pass": true}, |
| {"id": "LCB_E03", "difficulty": "Easy", "platform": "LeetCode-3314", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_E04", "difficulty": "Easy", "platform": "AtCoder-ABC376A", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, |
| {"id": "LCB_E05", "difficulty": "Easy", "platform": "LeetCode-3330", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_E06", "difficulty": "Easy", "platform": "LeetCode-3340", "pred_coder": "2", "pred_ref": "2", "pass": true}, |
| {"id": "LCB_E07", "difficulty": "Easy", "platform": "LeetCode-3345", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, |
| {"id": "LCB_E08", "difficulty": "Easy", "platform": "AtCoder-ABC380A", "pred_coder": "3", "pred_ref": "3", "pass": true}, |
| {"id": "LCB_E09", "difficulty": "Easy", "platform": "LeetCode-3370", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_E10", "difficulty": "Easy", "platform": "Codeforces-2036A", "pred_coder": "))", "pred_ref": "))", "pass": false}, |
| {"id": "LCB_M01", "difficulty": "Medium", "platform": "LeetCode-3301", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, |
| {"id": "LCB_M02", "difficulty": "Medium", "platform": "LeetCode-3309", "pred_coder": "2", "pred_ref": "2", "pass": true}, |
| {"id": "LCB_M03", "difficulty": "Medium", "platform": "LeetCode-3316", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_M04", "difficulty": "Medium", "platform": "LeetCode-3325", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_M05", "difficulty": "Medium", "platform": "LeetCode-3335", "pred_coder": "\\n\\n", "pred_ref": "\\n\\n", "pass": true}, |
| {"id": "LCB_M06", "difficulty": "Medium", "platform": "LeetCode-3350", "pred_coder": "))\\n", "pred_ref": "))\\n", "pass": true}, |
| {"id": "LCB_M07", "difficulty": "Medium", "platform": "LeetCode-3355", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, |
| {"id": "LCB_M08", "difficulty": "Medium", "platform": "LeetCode-3365", "pred_coder": "2", "pred_ref": "2", "pass": true}, |
| {"id": "LCB_M09", "difficulty": "Medium", "platform": "AtCoder-ABC378D", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}, |
| {"id": "LCB_M10", "difficulty": "Medium", "platform": "Codeforces-2033D", "pred_coder": "0", "pred_ref": "0", "pass": true}, |
| {"id": "LCB_H01", "difficulty": "Hard", "platform": "LeetCode-3303", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_H02", "difficulty": "Hard", "platform": "LeetCode-3307", "pred_coder": "26", "pred_ref": "26", "pass": true}, |
| {"id": "LCB_H03", "difficulty": "Hard", "platform": "LeetCode-3312", "pred_coder": "]\\n", "pred_ref": "]\\n", "pass": true}, |
| {"id": "LCB_H04", "difficulty": "Hard", "platform": "LeetCode-3321", "pred_coder": "))\\n", "pred_ref": "))\\n", "pass": true}, |
| {"id": "LCB_H05", "difficulty": "Hard", "platform": "LeetCode-3333", "pred_coder": "\\n\\n\\n","pred_ref": "\\n\\n\\n","pass": true}, |
| {"id": "LCB_H06", "difficulty": "Hard", "platform": "LeetCode-3337", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_H07", "difficulty": "Hard", "platform": "LeetCode-3343", "pred_coder": "),", "pred_ref": "),", "pass": true}, |
| {"id": "LCB_H08", "difficulty": "Hard", "platform": "LeetCode-3352", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_H09", "difficulty": "Hard", "platform": "AtCoder-ABC379F", "pred_coder": "1", "pred_ref": "1", "pass": true}, |
| {"id": "LCB_H10", "difficulty": "Hard", "platform": "Codeforces-2037G", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true} |
| ] |
| } |
|
|
