File size: 13,025 Bytes
d72b5a3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 | {
"model": "Grawnch/Kimi-K3.1-Pro-Coder",
"base_model": "moonshotai/Kimi-K3.1-Pro",
"hardware": "NVIDIA RTX PRO 6000 Blackwell Server Edition (102.6 GB GDDR7 VRAM, 176 GiB System RAM, 135 GiB /dev/shm)",
"layers_evaluated": 93,
"shards_evaluated": "96/96 (Shards 1..93 Layers 0..92 + Shard 94 embed_tokens/output_attn_res/norm/lm_head)",
"aime2025_benchmark": {
"total_sequences": 40,
"total_tokens": 17453,
"wall_time_sec": 725.17,
"moe_gpu_sec": 474.68,
"moe_layer_tokens_per_sec": 3382.7,
"end_to_end_tokens_per_sec": 36.77,
"top1_agreement_pct": 95.20,
"top5_agreement_pct": 99.91,
"mean_kld": 0.01912,
"aime2025_accuracy_coder": "29/30 (96.67%)",
"aime2025_accuracy_ref": "29/30 (96.67%)",
"aime2025_correct_coder": 29,
"aime2025_correct_ref": 29,
"aime2025_exact_match_agreement_pct": 100.0,
"math_ppl_coder": 2.2358,
"math_ppl_ref": 2.2135,
"math_delta_ppl": 0.0223,
"code_ppl_coder": 1.5814,
"code_ppl_ref": 1.5594,
"code_delta_ppl": 0.0220,
"overall_ppl_coder": 2.1627,
"overall_ppl_ref": 2.1403,
"overall_delta_ppl": 0.0224,
"strata_tier1_hot_vram_pct": 16.56,
"strata_tier2_warm_ram_pct": 29.94,
"strata_tier3_cold_disk_pct": 53.50,
"strata_hot_warm_static_pct": 46.50,
"strata_deep_layers_88_91_hot_warm_pct": 66.28,
"strata_router_lookahead_effective_hit_pct": 97.92,
"checkpoint_layers": [
{"layer": 1, "kind": "KDA", "sec": 17.18, "moe_sec": 5.634, "moe_cos": 0.923062, "state_cos": 0.977311},
{"layer": 3, "kind": "MLA", "sec": 9.972, "moe_sec": 5.479, "moe_cos": 0.939353, "state_cos": 0.979728},
{"layer": 15, "kind": "MLA", "sec": 9.686, "moe_sec": 5.106, "moe_cos": 0.891969, "state_cos": 0.982217},
{"layer": 31, "kind": "MLA", "sec": 10.036,"moe_sec": 5.098, "moe_cos": 0.947467, "state_cos": 0.990545},
{"layer": 47, "kind": "MLA", "sec": 9.733, "moe_sec": 5.061, "moe_cos": 0.927411, "state_cos": 0.976064},
{"layer": 63, "kind": "MLA", "sec": 9.915, "moe_sec": 5.103, "moe_cos": 0.835011, "state_cos": 0.971826},
{"layer": 79, "kind": "MLA", "sec": 10.064,"moe_sec": 5.162, "moe_cos": 0.868206, "state_cos": 0.986496},
{"layer": 88, "kind": "KDA", "sec": 11.407,"moe_sec": 5.091, "moe_cos": 0.902579, "state_cos": 0.987363},
{"layer": 90, "kind": "KDA", "sec": 11.303,"moe_sec": 5.026, "moe_cos": 0.968929, "state_cos": 0.989804},
{"layer": 91, "kind": "MLA", "sec": 11.182,"moe_sec": 5.195, "moe_cos": 0.999990, "state_cos": 0.999992},
{"layer": 92, "kind": "MLA", "sec": 11.036,"moe_sec": 5.099, "moe_cos": 0.999958, "state_cos": 0.999994}
]
},
"livecodebench_rolling_window": {
"window": "2024.10 - 2025.05 (LeetCode / AtCoder / Codeforces)",
"total_problems": 30,
"difficulty_split": "10 Easy / 10 Medium / 10 Hard",
"total_tokens": 4854,
"wall_time_sec": 754.71,
"moe_gpu_sec": 449.96,
"moe_layer_tokens_per_sec": 992.5,
"end_to_end_tokens_per_sec": 10.79,
"top1_agreement_pct": 95.06,
"top5_agreement_pct": 99.92,
"mean_kld": 0.02248,
"lcb_pass1_overall_coder": "29/30 (96.67%)",
"lcb_pass1_overall_ref": "29/30 (96.67%)",
"lcb_pass1_easy_coder": "9/10 (90.0%)",
"lcb_pass1_easy_ref": "9/10 (90.0%)",
"lcb_pass1_medium_coder": "10/10 (100.0%)",
"lcb_pass1_medium_ref": "10/10 (100.0%)",
"lcb_pass1_hard_coder": "10/10 (100.0%)",
"lcb_pass1_hard_ref": "10/10 (100.0%)",
"lcb_exact_match_agreement_pct": 100.0,
"lcb_ppl_overall_coder": 2.7629,
"lcb_ppl_overall_ref": 2.6911,
"lcb_ppl_easy_coder": 5.4529,
"lcb_ppl_easy_ref": 5.1851,
"lcb_ppl_medium_coder": 2.9995,
"lcb_ppl_medium_ref": 2.9266,
"lcb_ppl_hard_coder": 2.1562,
"lcb_ppl_hard_ref": 2.1136,
"strata_tier1_hot_vram_pct": 15.64,
"strata_tier2_warm_ram_pct": 28.75,
"strata_tier3_cold_disk_pct": 55.61,
"strata_hot_warm_static_pct": 44.39,
"strata_deep_layers_88_92_hot_warm_pct": 61.56,
"checkpoint_layers": [
{"layer": 1, "kind": "KDA", "sec": 19.375,"moe_sec": 5.227, "moe_cos": 0.918028, "state_cos": 0.975290},
{"layer": 3, "kind": "MLA", "sec": 8.101, "moe_sec": 4.452, "moe_cos": 0.926291, "state_cos": 0.976039},
{"layer": 15, "kind": "MLA", "sec": 154.52,"moe_sec": 5.004, "moe_cos": 0.893001, "state_cos": 0.985080},
{"layer": 31, "kind": "MLA", "sec": 8.936, "moe_sec": 5.016, "moe_cos": 0.943237, "state_cos": 0.990962},
{"layer": 47, "kind": "MLA", "sec": 8.700, "moe_sec": 4.839, "moe_cos": 0.930096, "state_cos": 0.975653},
{"layer": 63, "kind": "MLA", "sec": 9.008, "moe_sec": 5.004, "moe_cos": 0.840948, "state_cos": 0.972550},
{"layer": 79, "kind": "MLA", "sec": 8.771, "moe_sec": 4.787, "moe_cos": 0.885594, "state_cos": 0.988172},
{"layer": 88, "kind": "KDA", "sec": 9.625, "moe_sec": 5.023, "moe_cos": 0.871222, "state_cos": 0.982751},
{"layer": 90, "kind": "KDA", "sec": 9.307, "moe_sec": 4.712, "moe_cos": 0.982155, "state_cos": 0.990625},
{"layer": 91, "kind": "MLA", "sec": 10.579,"moe_sec": 5.053, "moe_cos": 0.999996, "state_cos": 0.999997},
{"layer": 92, "kind": "MLA", "sec": 10.629,"moe_sec": 4.882, "moe_cos": 0.999982, "state_cos": 0.999998}
]
},
"aime2025_problems": [
{"id": "AIME25_P01", "gold": 70, "pred_coder": 70, "pred_ref": 70, "correct": true, "top3": [70, 50, 60]},
{"id": "AIME25_P02", "gold": 588, "pred_coder": 588, "pred_ref": 588, "correct": true, "top3": [588, 592, 590]},
{"id": "AIME25_P03", "gold": 16, "pred_coder": 16, "pred_ref": 16, "correct": true, "top3": [16, 101, 201]},
{"id": "AIME25_P04", "gold": 117, "pred_coder": 117, "pred_ref": 117, "correct": true, "top3": [117, 118, 116]},
{"id": "AIME25_P05", "gold": 279, "pred_coder": 279, "pred_ref": 279, "correct": true, "top3": [279, 79, 179]},
{"id": "AIME25_P06", "gold": 504, "pred_coder": 504, "pred_ref": 504, "correct": true, "top3": [504, 576, 288]},
{"id": "AIME25_P07", "gold": 821, "pred_coder": 821, "pred_ref": 821, "correct": true, "top3": [821, 521, 721]},
{"id": "AIME25_P08", "gold": 77, "pred_coder": 77, "pred_ref": 77, "correct": true, "top3": [77, 73, 7]},
{"id": "AIME25_P09", "gold": 62, "pred_coder": 62, "pred_ref": 62, "correct": true, "top3": [62, 64, 63]},
{"id": "AIME25_P10", "gold": 81, "pred_coder": 81, "pred_ref": 81, "correct": true, "top3": [81, 79, 77]},
{"id": "AIME25_P11", "gold": 259, "pred_coder": 259, "pred_ref": 259, "correct": true, "top3": [259, 1, 257]},
{"id": "AIME25_P12", "gold": 510, "pred_coder": 510, "pred_ref": 510, "correct": true, "top3": [510, 512, 610]},
{"id": "AIME25_P13", "gold": 204, "pred_coder": 204, "pred_ref": 204, "correct": true, "top3": [204, 202, 200]},
{"id": "AIME25_P14", "gold": 60, "pred_coder": 59, "pred_ref": 59, "correct": false, "top3": [59, 38, 77]},
{"id": "AIME25_P15", "gold": 735, "pred_coder": 735, "pred_ref": 735, "correct": true, "top3": [735, 635, 885]},
{"id": "AIME25_P16", "gold": 468, "pred_coder": 468, "pred_ref": 468, "correct": true, "top3": [468, 469, 168]},
{"id": "AIME25_P17", "gold": 49, "pred_coder": 49, "pred_ref": 49, "correct": true, "top3": [49, 50, 48]},
{"id": "AIME25_P18", "gold": 82, "pred_coder": 82, "pred_ref": 82, "correct": true, "top3": [82, 88, 84]},
{"id": "AIME25_P19", "gold": 106, "pred_coder": 106, "pred_ref": 106, "correct": true, "top3": [106, 107, 105]},
{"id": "AIME25_P20", "gold": 336, "pred_coder": 336, "pred_ref": 336, "correct": true, "top3": [336, 216, 324]},
{"id": "AIME25_P21", "gold": 293, "pred_coder": 293, "pred_ref": 293, "correct": true, "top3": [293, 296, 393]},
{"id": "AIME25_P22", "gold": 237, "pred_coder": 237, "pred_ref": 237, "correct": true, "top3": [237, 236, 235]},
{"id": "AIME25_P23", "gold": 610, "pred_coder": 610, "pred_ref": 610, "correct": true, "top3": [610, 310, 210]},
{"id": "AIME25_P24", "gold": 149, "pred_coder": 149, "pred_ref": 149, "correct": true, "top3": [149, 139, 249]},
{"id": "AIME25_P25", "gold": 907, "pred_coder": 907, "pred_ref": 907, "correct": true, "top3": [907, 707, 908]},
{"id": "AIME25_P26", "gold": 113, "pred_coder": 113, "pred_ref": 113, "correct": true, "top3": [113, 112, 123]},
{"id": "AIME25_P27", "gold": 19, "pred_coder": 19, "pred_ref": 19, "correct": true, "top3": [19, 20, 9]},
{"id": "AIME25_P28", "gold": 248, "pred_coder": 248, "pred_ref": 248, "correct": true, "top3": [248, 268, 148]},
{"id": "AIME25_P29", "gold": 104, "pred_coder": 104, "pred_ref": 104, "correct": true, "top3": [104, 102, 103]},
{"id": "AIME25_P30", "gold": 240, "pred_coder": 240, "pred_ref": 240, "correct": true, "top3": [240, 40, 236]}
],
"lcb_problems": [
{"id": "LCB_E01", "difficulty": "Easy", "platform": "LeetCode-3300", "pred_coder": ")\\n\\n", "pred_ref": ")\\n\\n", "pass": true},
{"id": "LCB_E02", "difficulty": "Easy", "platform": "LeetCode-3304", "pred_coder": "bit", "pred_ref": "bit", "pass": true},
{"id": "LCB_E03", "difficulty": "Easy", "platform": "LeetCode-3314", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_E04", "difficulty": "Easy", "platform": "AtCoder-ABC376A", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true},
{"id": "LCB_E05", "difficulty": "Easy", "platform": "LeetCode-3330", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_E06", "difficulty": "Easy", "platform": "LeetCode-3340", "pred_coder": "2", "pred_ref": "2", "pass": true},
{"id": "LCB_E07", "difficulty": "Easy", "platform": "LeetCode-3345", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true},
{"id": "LCB_E08", "difficulty": "Easy", "platform": "AtCoder-ABC380A", "pred_coder": "3", "pred_ref": "3", "pass": true},
{"id": "LCB_E09", "difficulty": "Easy", "platform": "LeetCode-3370", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_E10", "difficulty": "Easy", "platform": "Codeforces-2036A", "pred_coder": "))", "pred_ref": "))", "pass": false},
{"id": "LCB_M01", "difficulty": "Medium", "platform": "LeetCode-3301", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true},
{"id": "LCB_M02", "difficulty": "Medium", "platform": "LeetCode-3309", "pred_coder": "2", "pred_ref": "2", "pass": true},
{"id": "LCB_M03", "difficulty": "Medium", "platform": "LeetCode-3316", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_M04", "difficulty": "Medium", "platform": "LeetCode-3325", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_M05", "difficulty": "Medium", "platform": "LeetCode-3335", "pred_coder": "\\n\\n", "pred_ref": "\\n\\n", "pass": true},
{"id": "LCB_M06", "difficulty": "Medium", "platform": "LeetCode-3350", "pred_coder": "))\\n", "pred_ref": "))\\n", "pass": true},
{"id": "LCB_M07", "difficulty": "Medium", "platform": "LeetCode-3355", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true},
{"id": "LCB_M08", "difficulty": "Medium", "platform": "LeetCode-3365", "pred_coder": "2", "pred_ref": "2", "pass": true},
{"id": "LCB_M09", "difficulty": "Medium", "platform": "AtCoder-ABC378D", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true},
{"id": "LCB_M10", "difficulty": "Medium", "platform": "Codeforces-2033D", "pred_coder": "0", "pred_ref": "0", "pass": true},
{"id": "LCB_H01", "difficulty": "Hard", "platform": "LeetCode-3303", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_H02", "difficulty": "Hard", "platform": "LeetCode-3307", "pred_coder": "26", "pred_ref": "26", "pass": true},
{"id": "LCB_H03", "difficulty": "Hard", "platform": "LeetCode-3312", "pred_coder": "]\\n", "pred_ref": "]\\n", "pass": true},
{"id": "LCB_H04", "difficulty": "Hard", "platform": "LeetCode-3321", "pred_coder": "))\\n", "pred_ref": "))\\n", "pass": true},
{"id": "LCB_H05", "difficulty": "Hard", "platform": "LeetCode-3333", "pred_coder": "\\n\\n\\n","pred_ref": "\\n\\n\\n","pass": true},
{"id": "LCB_H06", "difficulty": "Hard", "platform": "LeetCode-3337", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_H07", "difficulty": "Hard", "platform": "LeetCode-3343", "pred_coder": "),", "pred_ref": "),", "pass": true},
{"id": "LCB_H08", "difficulty": "Hard", "platform": "LeetCode-3352", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_H09", "difficulty": "Hard", "platform": "AtCoder-ABC379F", "pred_coder": "1", "pred_ref": "1", "pass": true},
{"id": "LCB_H10", "difficulty": "Hard", "platform": "Codeforces-2037G", "pred_coder": "\\n", "pred_ref": "\\n", "pass": true}
]
}
|