cm00cm commited on
Commit
d0d8bc2
·
0 Parent(s):

Kimi-K3.1-Dflash2-Preview: DFlash2 draft for Kimi-K3.1-Pro (weights, config, model card)

Browse files
Files changed (4) hide show
  1. .gitattributes +35 -0
  2. README.md +134 -0
  3. config.json +88 -0
  4. model.safetensors +3 -0
.gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
README.md ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ pipeline_tag: text-generation
3
+ tags:
4
+ - speculative-decoding
5
+ - dflash
6
+ - sglang
7
+ - safetensors
8
+ base_model: moonshotai/Kimi-K3.1-Pro
9
+ inference: false
10
+ ---
11
+
12
+ # Kimi-K3.1-Dflash2-Preview
13
+
14
+ ## Overview
15
+
16
+ A DFlash2 speculator for
17
+ [Kimi-K3.1-Pro](https://huggingface.co/moonshotai/Kimi-K3.1-Pro), designed to
18
+ accelerate inference through speculative decoding with SGLang. In each step the
19
+ draft proposes a block of tokens in parallel from the target's hidden states, and
20
+ the target verifies the block in one forward pass.
21
+
22
+ This repository contains the BF16 draft weights and configuration. The target
23
+ model supplies the embeddings, LM head, tokenizer, and native chat template.
24
+ The draft is used alongside the target and is not a standalone language model.
25
+
26
+ The draft is a five-layer MLA stack. It reads hidden states from six target
27
+ layers, fuses them through one projection, and runs block-causal sliding-window
28
+ attention over the target's context. A low-rank candidate selector then picks
29
+ each proposed token from the draft's top-k candidates, conditioned on the token
30
+ before it, so the block is sampled as a path rather than position by position.
31
+
32
+ ## Model Specifications
33
+
34
+ - **Base model:** `moonshotai/Kimi-K3.1-Pro` (revision `535833b1`)
35
+ - **Format:** Single-file BF16 Safetensors, 2,784,811,008 parameters
36
+ - **Draft:** 5 MLA layers (DeepSeek-style multi-head latent attention), hidden
37
+ size 7168, 96 heads, q LoRA rank 1536, KV LoRA rank 512, QK head dim 192
38
+ (128 + 64 RoPE), V head dim 128, SwiGLU MLP 14336
39
+ - **Block convolutions:** a learned convolution over the block (kernel 2, group
40
+ size 16) in the attention and MLP branches of every layer
41
+ - **Attention:** Block-causal sliding-window attention with a 1,024-token window
42
+ - **Verification width:** 5 tokens (1 current token + 4 draft tokens) as
43
+ configured; the same weights also serve at 8 (see below)
44
+ - **Selector:** Rank 256, top-k 32
45
+ - **Auxiliary target layers:** `[7, 23, 51, 67, 83, 91]`
46
+ - **Configured context length:** 1,048,576 tokens (YaRN, factor 32), aligned
47
+ with the target
48
+ - **Target weights:** Embedding and unembedding weights are loaded from the target
49
+
50
+ The configured context limit is distinct from the attention window. The config
51
+ alignment does not establish draft acceptance quality across the full 1M context.
52
+
53
+ ## Evaluation Results
54
+
55
+ `acc_len` is the per-request acceptance length `L = C / V`, where `C` is the
56
+ number of completion tokens and `V` is the number of verification steps, averaged
57
+ equally across successful requests. Completion-token-weighted AL is
58
+ `sum(C * L) / sum(C)`; pooled AL is `sum(C) / sum(V)`. Every verification step
59
+ commits the accepted draft tokens plus one token from the target, so AL is at most
60
+ the verification width.
61
+
62
+ ### Verification width 5 (1 current token + 4 draft tokens, as configured)
63
+
64
+ | Dataset | Samples | acc_len | Completion-token-weighted AL | Pooled AL |
65
+ |---|---:|---:|---:|---:|
66
+ | GSM8K | 1,319 | **4.259405** | 4.103988 | 4.063391 |
67
+ | AIME26 | 240 (30 × 8) | **3.231564** | 3.039857 | 3.019893 |
68
+ | GPQA Diamond | 198 | **3.275916** | 2.979262 | 2.960371 |
69
+
70
+ ### Verification width 8 (1 current token + 7 draft tokens)
71
+
72
+ | Dataset | Samples | acc_len | Completion-token-weighted AL | Pooled AL |
73
+ |---|---:|---:|---:|---:|
74
+ | GSM8K | 1,319 | **5.780277** | 5.442283 | 5.287548 |
75
+ | AIME26 | 240 (30 × 8) | **3.696036** | 3.367764 | 3.320980 |
76
+ | GPQA Diamond | 198 | **3.768280** | 3.268278 | 3.227970 |
77
+
78
+ Measured with real DFlash2 and **block verification enabled**, `thinking_effort`
79
+ max, `temperature=1.0` and `top_p=0.95`, on one 8x B300 node (TP8) with FP8
80
+ target KV cache, FP8 draft KV cache and `cutedsl_mla` draft attention, using the
81
+ command below. The three suites ran concurrently against one server, with 32
82
+ client workers per suite and at most 64 running requests at 5 tokens (32 at 8
83
+ tokens). These results are not from a single-request-at-a-time evaluation.
84
+
85
+ All 3,514 requests (1,757 per width) finished with an end-of-turn token. None was
86
+ truncated (the longest used 104,938 of the 131,072-token context), and none needed
87
+ a retry. Mean completion length at width 5: GSM8K 344, AIME26 9,806 and GPQA
88
+ Diamond 5,761 tokens.
89
+
90
+ Prompts: each sample is one user turn rendered with the target's chat template.
91
+ GSM8K and AIME26 append "Please reason step by step, and put your final answer
92
+ within \boxed{}."; GPQA Diamond uses the simple-evals multiple-choice template
93
+ with options shuffled per question. AIME26 is the 30 AIME 2026 problems from
94
+ `MathArena/aime_2026`, sampled 8 times each with different seeds. Only acceptance
95
+ length is measured here, not answer accuracy.
96
+
97
+ ## Serving with SGLang
98
+
99
+ The following example launches one TP8 replica with block verification, FP8 KV
100
+ caches for the target and the draft, and `cutedsl_mla` draft attention:
101
+
102
+ ```bash
103
+ python -m sglang.launch_server \
104
+ --model-path moonshotai/Kimi-K3.1-Pro \
105
+ --trust-remote-code \
106
+ --tp-size 8 \
107
+ --mem-fraction-static 0.85 \
108
+ --mamba-full-memory-ratio 1.0 \
109
+ --reasoning-parser kimi_k3 \
110
+ --tool-call-parser kimi_k3 \
111
+ --kv-cache-dtype fp8_e4m3 \
112
+ --disable-radix-cache \
113
+ --max-running-requests 64 \
114
+ --context-length 131072 \
115
+ --speculative-algorithm DFLASH \
116
+ --speculative-draft-model-path RadixArk/Kimi-K3.1-Dflash2-Preview \
117
+ --speculative-draft-attention-backend cutedsl_mla \
118
+ --speculative-draft-kv-cache-dtype fp8_e4m3 \
119
+ --speculative-num-draft-tokens 5 \
120
+ --speculative-use-block-verification \
121
+ --host 127.0.0.1 \
122
+ --port 30000
123
+ ```
124
+
125
+ For an 8-token verification width, pass `--speculative-num-draft-tokens 8`. It
126
+ overrides the config's `block_size` of 5, and SGLang logs a warning. Also lower
127
+ `--max-running-requests` to 32: the target's speculative Mamba state cache grows
128
+ with verification width × running requests, and at 8 × 64 it leaves no memory for
129
+ the KV cache at `--mem-fraction-static 0.85` on B300.
130
+
131
+ Use the target's native chat template with `thinking_effort` max for the
132
+ evaluation settings above. On first launch, SGLang builds its Rust TreeCore with
133
+ the toolchain pinned in `rust/rust-toolchain.toml` (1.92), so install that
134
+ toolchain beforehand.
config.json ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "DFlash2DraftModel"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "base_model": "moonshotai/Kimi-K3.1-Pro",
8
+ "base_model_revision": "535833b1b6099ea85659e047561ed0f22ab38db1",
9
+ "block_size": 5,
10
+ "bos_token_id": 163584,
11
+ "dflash_config": {
12
+ "attention_mode": "mla",
13
+ "aux_hidden_stream": "attn_res",
14
+ "block_size": 5,
15
+ "conv_group_size": 16,
16
+ "conv_kernel_size": 2,
17
+ "mask_token_id": 163824,
18
+ "selector_rank": 256,
19
+ "selector_top_k": 32,
20
+ "target_layer_ids": [
21
+ 7,
22
+ 23,
23
+ 51,
24
+ 67,
25
+ 83,
26
+ 91
27
+ ]
28
+ },
29
+ "draft_vocab_size": 163840,
30
+ "dtype": "bfloat16",
31
+ "eos_token_id": 163586,
32
+ "head_dim": 64,
33
+ "hidden_act": "silu",
34
+ "hidden_size": 7168,
35
+ "initializer_range": 0.02,
36
+ "intermediate_size": 14336,
37
+ "is_causal": true,
38
+ "kv_lora_rank": 512,
39
+ "layer_types": [
40
+ "sliding_attention",
41
+ "sliding_attention",
42
+ "sliding_attention",
43
+ "sliding_attention",
44
+ "sliding_attention"
45
+ ],
46
+ "max_position_embeddings": 1048576,
47
+ "max_window_layers": 5,
48
+ "mla_use_nope": false,
49
+ "model_type": "qwen3",
50
+ "num_attention_heads": 96,
51
+ "num_hidden_layers": 5,
52
+ "num_key_value_heads": 96,
53
+ "num_target_layers": 93,
54
+ "pad_token_id": 163839,
55
+ "q_lora_rank": 1536,
56
+ "qk_head_dim": 192,
57
+ "qk_nope_head_dim": 128,
58
+ "qk_rope_head_dim": 64,
59
+ "rms_norm_eps": 1e-05,
60
+ "rope_interleave": true,
61
+ "rope_parameters": {
62
+ "beta_fast": 32,
63
+ "beta_slow": 1,
64
+ "factor": 32.0,
65
+ "mscale": 1.0,
66
+ "mscale_all_dim": 1.0,
67
+ "original_max_position_embeddings": 32768,
68
+ "rope_theta": 50000.0,
69
+ "rope_type": "yarn"
70
+ },
71
+ "rope_scaling": {
72
+ "beta_fast": 32,
73
+ "beta_slow": 1,
74
+ "factor": 32.0,
75
+ "mscale": 1.0,
76
+ "mscale_all_dim": 1.0,
77
+ "original_max_position_embeddings": 32768,
78
+ "rope_type": "yarn"
79
+ },
80
+ "rope_theta": 50000.0,
81
+ "sliding_window": 1024,
82
+ "tie_word_embeddings": false,
83
+ "transformers_version": "5.19.0",
84
+ "use_cache": true,
85
+ "use_sliding_window": true,
86
+ "v_head_dim": 128,
87
+ "vocab_size": 163840
88
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e07d2352fc44ea3967862309abeaa670faa3cf386c37adc35b8c97396f27b3f
3
+ size 5569631720