liangsu9988 commited on
Commit
8c25c71
·
verified ·
1 Parent(s): 54f6324

Promote latest kernel artifacts to main

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +3 -35
  2. CARD.md +223 -0
  3. README.md +220 -6
  4. VALIDATION.md +206 -0
  5. benchmarks/benchmark_decode.py +76 -0
  6. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +110 -0
  7. build/torch211-cxx11-cu128-x86_64-linux/_compat.py +51 -0
  8. build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so +3 -0
  9. build/torch211-cxx11-cu128-x86_64-linux/_native.py +259 -0
  10. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
  11. build/torch211-cxx11-cu128-x86_64-linux/api_status.py +146 -0
  12. build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py +1 -0
  13. build/torch211-cxx11-cu128-x86_64-linux/common/utils.py +144 -0
  14. build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py +1 -0
  15. build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py +1150 -0
  16. build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py +452 -0
  17. build/torch211-cxx11-cu128-x86_64-linux/interface.py +679 -0
  18. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +38 -0
  19. build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py +26 -0
  20. build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py +1 -0
  21. build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py +106 -0
  22. build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py +88 -0
  23. build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py +4 -0
  24. build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py +371 -0
  25. build/torch211-cxx11-cu128-x86_64-linux/quantize.py +227 -0
  26. build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py +174 -0
  27. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +110 -0
  28. build/torch211-cxx11-cu130-x86_64-linux/_compat.py +51 -0
  29. build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so +3 -0
  30. build/torch211-cxx11-cu130-x86_64-linux/_native.py +259 -0
  31. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
  32. build/torch211-cxx11-cu130-x86_64-linux/api_status.py +146 -0
  33. build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py +1 -0
  34. build/torch211-cxx11-cu130-x86_64-linux/common/utils.py +144 -0
  35. build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py +1 -0
  36. build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py +1150 -0
  37. build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py +452 -0
  38. build/torch211-cxx11-cu130-x86_64-linux/interface.py +679 -0
  39. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +39 -0
  40. build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py +26 -0
  41. build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py +1 -0
  42. build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py +106 -0
  43. build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py +88 -0
  44. build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py +4 -0
  45. build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py +371 -0
  46. build/torch211-cxx11-cu130-x86_64-linux/quantize.py +227 -0
  47. build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py +174 -0
  48. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +110 -0
  49. build/torch212-cxx11-cu130-x86_64-linux/_compat.py +51 -0
  50. build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so +3 -0
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.so filter=lfs diff=lfs merge=lfs -text
2
+ *.pyd filter=lfs diff=lfs merge=lfs -text
3
+ *.dylib filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
CARD.md ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: kernels
3
+ license: apache-2.0
4
+ tags:
5
+ - cuda
6
+ - triton
7
+ - native-cuda
8
+ - minimax
9
+ - sparse-attention
10
+ - blackwell
11
+ ---
12
+
13
+ # MiniMaxAI MSA Blackwell
14
+
15
+ Blackwell-family package for MiniMax MSA decode sparse attention, maintained by
16
+ FlashRT. The upstream package is
17
+ [`MiniMaxAI/msa`](https://huggingface.co/kernels/MiniMaxAI/msa), which targets
18
+ SM100. This package extends the decode-sparse path to NVIDIA Blackwell
19
+ compute capability 12.x and has been validated in the FlashRT MiniMax-Spark
20
+ runtime on DGX Spark / GB10 / SM121.
21
+
22
+ ## Load
23
+
24
+ ```python
25
+ from kernels import get_kernel
26
+
27
+ msa = get_kernel(
28
+ "flashrt/MiniMaxAI-msa-blackwell",
29
+ version=1,
30
+ trust_remote_code=True,
31
+ )
32
+ ```
33
+
34
+ ## What You Can Call
35
+
36
+ ### Official MiniMaxAI/msa names
37
+
38
+ | Function/class | Status in this Blackwell package |
39
+ |---|---|
40
+ | `sparse_decode_atten_func` | Available. Blackwell paged BF16/FP16 single-token decode wrapper. |
41
+ | `SparseDecodePagedAttentionWrapper` | Available. `plan(...).run(...)` wrapper for the same decode path. |
42
+ | `build_k2q_csr` | Available. CSR construction helper for the official prefill API. |
43
+ | `SparseK2qCsrBuilderSm100` | Available compatibility class; `build()` delegates to `build_k2q_csr`. |
44
+ | `Nvfp4QuantizedTensor` | Available metadata dataclass. |
45
+ | `quantize_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
46
+ | `quantize_kv_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
47
+ | `dequantize_nvfp4_128x4_to_bf16` | Available reference dequantizer. |
48
+ | `swizzle_nvfp4_scale_to_128x4` | Available scale-layout helper. |
49
+ | `nvfp4_global_scale_from_amax` | Available scale helper. |
50
+ | `sparse_atten_func` | Available. Official CSR sparse prefill API backed by the Blackwell Triton BF16/FP16 prefill kernel. |
51
+ | `sparse_atten_nvfp4_kv_func` | Available. Built artifacts use native CUDA swizzled NVFP4 -> BF16 dequantization, then call Blackwell sparse prefill. |
52
+ | `fp4_indexer_block_scores` | Available. Built artifacts use the native CUDA Blackwell block-score kernel and return the official `[Hq, ceil(max_seqlen_k/128), total_q]` score layout. |
53
+
54
+ ### FlashRT Blackwell helper names
55
+
56
+ These are the direct low-level APIs used by the FlashRT MiniMax-Spark decode
57
+ path:
58
+
59
+ - `flash_decode_with_topk_idx`
60
+ - `flash_decode_with_gqa_share_sparse`
61
+ - `native_topk_from_scores`
62
+ - `native_nvfp4_dequant_swizzled_to_bf16`
63
+ - `has_native_ops`
64
+ - `naive_flash_decode_with_topk_idx`
65
+ - `naive_flash_decode_with_gqa_share_sparse`
66
+ - `get_cu_seqblocks`
67
+ - `robust_allocator`
68
+
69
+ ## Decode Example
70
+
71
+ This example uses the official MiniMax decode-facing name
72
+ `sparse_decode_atten_func`.
73
+
74
+ ```python
75
+ import torch
76
+ from kernels import get_kernel
77
+
78
+ msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
79
+
80
+ B, Hq, Hkv, D = 1, 64, 4, 128
81
+ page_size = 128
82
+ num_pages = 32
83
+ topk = 16
84
+ device = "cuda"
85
+ dtype = torch.bfloat16
86
+
87
+ q = torch.randn(B, Hq, D, device=device, dtype=dtype)
88
+ k = torch.randn(num_pages, Hkv, page_size, D, device=device, dtype=dtype)
89
+ v = torch.randn_like(k)
90
+ page_table = torch.arange(num_pages, device=device, dtype=torch.int32).view(B, -1)
91
+ seqused_k = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
92
+ q2k_indices = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
93
+ q2k_indices = q2k_indices.expand(Hkv, B, topk).contiguous()
94
+
95
+ out = msa.sparse_decode_atten_func(
96
+ q,
97
+ k,
98
+ v,
99
+ q2k_indices,
100
+ page_table=page_table,
101
+ seqused_k=seqused_k,
102
+ seqlen_q=1,
103
+ max_seqlen_k=num_pages * page_size,
104
+ blk_kv=page_size,
105
+ )
106
+ ```
107
+
108
+ ## Wrapper Example
109
+
110
+ ```python
111
+ wrapper = msa.SparseDecodePagedAttentionWrapper(blk_kv=128)
112
+ wrapper.plan(
113
+ page_table=page_table,
114
+ seqused_k=seqused_k,
115
+ seqlen_q=1,
116
+ max_seqlen_k=num_pages * page_size,
117
+ q2k_indices=q2k_indices,
118
+ num_qo_heads=Hq,
119
+ num_kv_heads=Hkv,
120
+ head_dim=D,
121
+ )
122
+ out = wrapper.run(q, k, v)
123
+ ```
124
+
125
+ ## Prefill Example
126
+
127
+ This example uses the official MiniMax CSR prefill-facing name
128
+ `sparse_atten_func`.
129
+
130
+ ```python
131
+ import torch
132
+ from kernels import get_kernel
133
+
134
+ msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
135
+
136
+ T, Hq, Hkv, D = 512, 64, 4, 128
137
+ page_size = 128
138
+ topk = 16
139
+ device = "cuda"
140
+ dtype = torch.bfloat16
141
+
142
+ q = torch.randn(T, Hq, D, device=device, dtype=dtype)
143
+ k = torch.randn(T, Hkv, D, device=device, dtype=dtype)
144
+ v = torch.randn_like(k)
145
+ cu = torch.tensor([0, T], device=device, dtype=torch.int32)
146
+
147
+ q2k = torch.full((Hkv, T, topk), -1, device=device, dtype=torch.int32)
148
+ for qi in range(T):
149
+ blocks = torch.arange(qi // page_size + 1, device=device, dtype=torch.int32)
150
+ q2k[:, qi, : min(topk, blocks.numel())] = blocks[:topk]
151
+
152
+ k2q_row_ptr, k2q_q_indices = msa.build_k2q_csr(
153
+ q2k, cu, cu, page_size, total_k=T
154
+ )
155
+ out = msa.sparse_atten_func(
156
+ q,
157
+ k,
158
+ v,
159
+ k2q_row_ptr,
160
+ k2q_q_indices,
161
+ topk,
162
+ cu_seqlens_q=cu,
163
+ cu_seqlens_k=cu,
164
+ max_seqlen_q=T,
165
+ max_seqlen_k=T,
166
+ blk_kv=page_size,
167
+ )
168
+ ```
169
+
170
+ ## Direct FlashRT Decode Path
171
+
172
+ Use this lower-level path if you already have `topk_idx` in FlashRT's paged KV
173
+ layout.
174
+
175
+ ```python
176
+ q = torch.randn(B, Hq, D, device=device, dtype=dtype)
177
+ k_cache = torch.randn(num_pages * page_size, Hkv, D, device=device, dtype=dtype)
178
+ v_cache = torch.randn_like(k_cache)
179
+ req_to_token = torch.arange(num_pages * page_size, device=device, dtype=torch.int32).view(B, -1)
180
+ seq_lens = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
181
+ slot_ids = torch.zeros(B, device=device, dtype=torch.int64)
182
+ topk_idx = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
183
+ topk_idx = topk_idx.expand(Hkv, B, topk).contiguous()
184
+
185
+ out = msa.flash_decode_with_gqa_share_sparse(
186
+ q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, page_size, topk_idx
187
+ )
188
+ ```
189
+
190
+ ## Scope
191
+
192
+ - Target family: NVIDIA Blackwell CUDA compute capability 12.x.
193
+ - Builder target: CUDA 12.8+ with `cuda-capabilities = ["12.0", "12.1"]`.
194
+ - Validated hardware: DGX Spark / GB10 / SM121.
195
+ - Validated MiniMax shape: query heads `64`, KV heads `4`, head dim `128`,
196
+ sparse block/page size `128`, top-k blocks `16`.
197
+ - End-to-end model validation: MiniMax-Spark runtime on GB10 through `32768`
198
+ context length.
199
+ - Standalone kernel long-context validation: `128`, `2048`, `4096`, `32768`,
200
+ `65536`, `131072`.
201
+ - Correctness gate: cosine similarity `>= 0.999` against paged FP32 PyTorch
202
+ references; official decode wrapper output matches the direct Blackwell
203
+ decode kernel.
204
+
205
+ ## Implementation Notes
206
+
207
+ This package contains:
208
+
209
+ - native CUDA score-to-top-k helper;
210
+ - native CUDA tensor-core sparse decode route for the MiniMax-M3 Blackwell shape;
211
+ - native CUDA FP4 block-score indexer;
212
+ - native CUDA swizzled NVFP4 -> BF16 dequantization for the W4A16 quality path;
213
+ - Blackwell-validated sparse prefill attention wrapper;
214
+ - MiniMaxAI/msa-compatible Python API layer for decode, prefill, CSR, NVFP4,
215
+ and FP4 block-score helpers.
216
+
217
+ When loaded from Hub built artifacts, the decode, FP4 indexer, and NVFP4
218
+ dequant hot paths use compiled CUDA ops. The source-tree mode keeps reference
219
+ paths so the API and correctness tests remain runnable before a wheel/shared
220
+ object has been built.
221
+
222
+ Source provenance and validation details are documented in `SYNC.md` and
223
+ `VALIDATION.md`.
README.md CHANGED
@@ -1,9 +1,223 @@
1
- # flashrt/MiniMaxAI-msa-blackwell
 
 
 
 
 
 
 
 
 
 
2
 
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
 
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/MiniMaxAI-msa-blackwell
 
 
 
 
 
7
 
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: kernels
3
+ license: apache-2.0
4
+ tags:
5
+ - cuda
6
+ - triton
7
+ - native-cuda
8
+ - minimax
9
+ - sparse-attention
10
+ - blackwell
11
+ ---
12
 
13
+ # MiniMaxAI MSA Blackwell
 
14
 
15
+ Blackwell-family package for MiniMax MSA decode sparse attention, maintained by
16
+ FlashRT. The upstream package is
17
+ [`MiniMaxAI/msa`](https://huggingface.co/kernels/MiniMaxAI/msa), which targets
18
+ SM100. This package extends the decode-sparse path to NVIDIA Blackwell
19
+ compute capability 12.x and has been validated in the FlashRT MiniMax-Spark
20
+ runtime on DGX Spark / GB10 / SM121.
21
 
22
+ ## Load
23
+
24
+ ```python
25
+ from kernels import get_kernel
26
+
27
+ msa = get_kernel(
28
+ "flashrt/MiniMaxAI-msa-blackwell",
29
+ version=1,
30
+ trust_remote_code=True,
31
+ )
32
+ ```
33
+
34
+ ## What You Can Call
35
+
36
+ ### Official MiniMaxAI/msa names
37
+
38
+ | Function/class | Status in this Blackwell package |
39
+ |---|---|
40
+ | `sparse_decode_atten_func` | Available. Blackwell paged BF16/FP16 single-token decode wrapper. |
41
+ | `SparseDecodePagedAttentionWrapper` | Available. `plan(...).run(...)` wrapper for the same decode path. |
42
+ | `build_k2q_csr` | Available. CSR construction helper for the official prefill API. |
43
+ | `SparseK2qCsrBuilderSm100` | Available compatibility class; `build()` delegates to `build_k2q_csr`. |
44
+ | `Nvfp4QuantizedTensor` | Available metadata dataclass. |
45
+ | `quantize_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
46
+ | `quantize_kv_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
47
+ | `dequantize_nvfp4_128x4_to_bf16` | Available reference dequantizer. |
48
+ | `swizzle_nvfp4_scale_to_128x4` | Available scale-layout helper. |
49
+ | `nvfp4_global_scale_from_amax` | Available scale helper. |
50
+ | `sparse_atten_func` | Available. Official CSR sparse prefill API backed by the Blackwell Triton BF16/FP16 prefill kernel. |
51
+ | `sparse_atten_nvfp4_kv_func` | Available. Built artifacts use native CUDA swizzled NVFP4 -> BF16 dequantization, then call Blackwell sparse prefill. |
52
+ | `fp4_indexer_block_scores` | Available. Built artifacts use the native CUDA Blackwell block-score kernel and return the official `[Hq, ceil(max_seqlen_k/128), total_q]` score layout. |
53
+
54
+ ### FlashRT Blackwell helper names
55
+
56
+ These are the direct low-level APIs used by the FlashRT MiniMax-Spark decode
57
+ path:
58
+
59
+ - `flash_decode_with_topk_idx`
60
+ - `flash_decode_with_gqa_share_sparse`
61
+ - `native_topk_from_scores`
62
+ - `native_nvfp4_dequant_swizzled_to_bf16`
63
+ - `has_native_ops`
64
+ - `naive_flash_decode_with_topk_idx`
65
+ - `naive_flash_decode_with_gqa_share_sparse`
66
+ - `get_cu_seqblocks`
67
+ - `robust_allocator`
68
+
69
+ ## Decode Example
70
+
71
+ This example uses the official MiniMax decode-facing name
72
+ `sparse_decode_atten_func`.
73
+
74
+ ```python
75
+ import torch
76
+ from kernels import get_kernel
77
+
78
+ msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
79
+
80
+ B, Hq, Hkv, D = 1, 64, 4, 128
81
+ page_size = 128
82
+ num_pages = 32
83
+ topk = 16
84
+ device = "cuda"
85
+ dtype = torch.bfloat16
86
+
87
+ q = torch.randn(B, Hq, D, device=device, dtype=dtype)
88
+ k = torch.randn(num_pages, Hkv, page_size, D, device=device, dtype=dtype)
89
+ v = torch.randn_like(k)
90
+ page_table = torch.arange(num_pages, device=device, dtype=torch.int32).view(B, -1)
91
+ seqused_k = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
92
+ q2k_indices = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
93
+ q2k_indices = q2k_indices.expand(Hkv, B, topk).contiguous()
94
+
95
+ out = msa.sparse_decode_atten_func(
96
+ q,
97
+ k,
98
+ v,
99
+ q2k_indices,
100
+ page_table=page_table,
101
+ seqused_k=seqused_k,
102
+ seqlen_q=1,
103
+ max_seqlen_k=num_pages * page_size,
104
+ blk_kv=page_size,
105
+ )
106
+ ```
107
+
108
+ ## Wrapper Example
109
+
110
+ ```python
111
+ wrapper = msa.SparseDecodePagedAttentionWrapper(blk_kv=128)
112
+ wrapper.plan(
113
+ page_table=page_table,
114
+ seqused_k=seqused_k,
115
+ seqlen_q=1,
116
+ max_seqlen_k=num_pages * page_size,
117
+ q2k_indices=q2k_indices,
118
+ num_qo_heads=Hq,
119
+ num_kv_heads=Hkv,
120
+ head_dim=D,
121
+ )
122
+ out = wrapper.run(q, k, v)
123
+ ```
124
+
125
+ ## Prefill Example
126
+
127
+ This example uses the official MiniMax CSR prefill-facing name
128
+ `sparse_atten_func`.
129
+
130
+ ```python
131
+ import torch
132
+ from kernels import get_kernel
133
+
134
+ msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
135
+
136
+ T, Hq, Hkv, D = 512, 64, 4, 128
137
+ page_size = 128
138
+ topk = 16
139
+ device = "cuda"
140
+ dtype = torch.bfloat16
141
+
142
+ q = torch.randn(T, Hq, D, device=device, dtype=dtype)
143
+ k = torch.randn(T, Hkv, D, device=device, dtype=dtype)
144
+ v = torch.randn_like(k)
145
+ cu = torch.tensor([0, T], device=device, dtype=torch.int32)
146
+
147
+ q2k = torch.full((Hkv, T, topk), -1, device=device, dtype=torch.int32)
148
+ for qi in range(T):
149
+ blocks = torch.arange(qi // page_size + 1, device=device, dtype=torch.int32)
150
+ q2k[:, qi, : min(topk, blocks.numel())] = blocks[:topk]
151
+
152
+ k2q_row_ptr, k2q_q_indices = msa.build_k2q_csr(
153
+ q2k, cu, cu, page_size, total_k=T
154
+ )
155
+ out = msa.sparse_atten_func(
156
+ q,
157
+ k,
158
+ v,
159
+ k2q_row_ptr,
160
+ k2q_q_indices,
161
+ topk,
162
+ cu_seqlens_q=cu,
163
+ cu_seqlens_k=cu,
164
+ max_seqlen_q=T,
165
+ max_seqlen_k=T,
166
+ blk_kv=page_size,
167
+ )
168
+ ```
169
+
170
+ ## Direct FlashRT Decode Path
171
+
172
+ Use this lower-level path if you already have `topk_idx` in FlashRT's paged KV
173
+ layout.
174
+
175
+ ```python
176
+ q = torch.randn(B, Hq, D, device=device, dtype=dtype)
177
+ k_cache = torch.randn(num_pages * page_size, Hkv, D, device=device, dtype=dtype)
178
+ v_cache = torch.randn_like(k_cache)
179
+ req_to_token = torch.arange(num_pages * page_size, device=device, dtype=torch.int32).view(B, -1)
180
+ seq_lens = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
181
+ slot_ids = torch.zeros(B, device=device, dtype=torch.int64)
182
+ topk_idx = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
183
+ topk_idx = topk_idx.expand(Hkv, B, topk).contiguous()
184
+
185
+ out = msa.flash_decode_with_gqa_share_sparse(
186
+ q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, page_size, topk_idx
187
+ )
188
+ ```
189
+
190
+ ## Scope
191
+
192
+ - Target family: NVIDIA Blackwell CUDA compute capability 12.x.
193
+ - Builder target: CUDA 12.8+ with `cuda-capabilities = ["12.0", "12.1"]`.
194
+ - Validated hardware: DGX Spark / GB10 / SM121.
195
+ - Validated MiniMax shape: query heads `64`, KV heads `4`, head dim `128`,
196
+ sparse block/page size `128`, top-k blocks `16`.
197
+ - End-to-end model validation: MiniMax-Spark runtime on GB10 through `32768`
198
+ context length.
199
+ - Standalone kernel long-context validation: `128`, `2048`, `4096`, `32768`,
200
+ `65536`, `131072`.
201
+ - Correctness gate: cosine similarity `>= 0.999` against paged FP32 PyTorch
202
+ references; official decode wrapper output matches the direct Blackwell
203
+ decode kernel.
204
+
205
+ ## Implementation Notes
206
+
207
+ This package contains:
208
+
209
+ - native CUDA score-to-top-k helper;
210
+ - native CUDA tensor-core sparse decode route for the MiniMax-M3 Blackwell shape;
211
+ - native CUDA FP4 block-score indexer;
212
+ - native CUDA swizzled NVFP4 -> BF16 dequantization for the W4A16 quality path;
213
+ - Blackwell-validated sparse prefill attention wrapper;
214
+ - MiniMaxAI/msa-compatible Python API layer for decode, prefill, CSR, NVFP4,
215
+ and FP4 block-score helpers.
216
+
217
+ When loaded from Hub built artifacts, the decode, FP4 indexer, and NVFP4
218
+ dequant hot paths use compiled CUDA ops. The source-tree mode keeps reference
219
+ paths so the API and correctness tests remain runnable before a wheel/shared
220
+ object has been built.
221
+
222
+ Source provenance and validation details are documented in `SYNC.md` and
223
+ `VALIDATION.md`.
VALIDATION.md ADDED
@@ -0,0 +1,206 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Validation
2
+
3
+ ## Target
4
+
5
+ - Kernel family: MiniMax M3 sparse attention (MSA)
6
+ - Package: `flashrt/MiniMaxAI-msa-blackwell`
7
+ - HF Jobs package selector: `MiniMaxAI-msa-blackwell`
8
+ - Package version: v1 Blackwell native-helper package
9
+ - Target GPU family: Blackwell CUDA compute capability 12.x
10
+ - Validated GPU: SM121 / GB10 / DGX Spark
11
+ - Dtype: BF16 inputs with FP32 accumulation references
12
+ - Layout: paged KV cache
13
+ - Model path: FlashRT MiniMax-Spark runtime on DGX Spark / GB10
14
+
15
+ ## Correctness Gate
16
+
17
+ Run quick validation:
18
+
19
+ ```bash
20
+ PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
21
+ python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py --quick
22
+ ```
23
+
24
+ Run full validation:
25
+
26
+ ```bash
27
+ PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
28
+ python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py
29
+ ```
30
+
31
+ Run standalone long-context validation:
32
+
33
+ ```bash
34
+ PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
35
+ python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py --long-context
36
+ ```
37
+
38
+ Expected full coverage:
39
+
40
+ | Area | Shapes | Reference | Required |
41
+ |---|---:|---|---|
42
+ | API surface | official `MiniMaxAI/msa` public names | `api_status.py` | all official root names exported; no unsupported public root API entries |
43
+ | Native CUDA top-k helper | heads 64, batch 1-2, blocks 1-256 | PyTorch top-k over valid blocks | exact set match |
44
+ | Decode sparse GQA attention | ctx 128, 2048, 4096, 32768 | paged FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 |
45
+ | Prefill sparse GQA attention | ctx 512, 4096 | paged causal FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 |
46
+ | Decode sparse GQA attention with sink | ctx 2048, 32768 | paged FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 |
47
+ | Official decode API wrapper | ctx 2048, 4096 | direct Blackwell decode kernel | cos = 1.0, max_abs = 0 |
48
+ | Official CSR prefill API wrapper | ctx 512, 2048 | direct Blackwell prefill kernel | cos = 1.0, max_abs = 0 under CSR-preserved block order |
49
+ | Official NVFP4 prefill API wrapper | ctx 512 BF16 dispatch path | `sparse_atten_func` | cos = 1.0, max_abs = 0 |
50
+ | Native CUDA NVFP4 dequant | rows/cols `(1,128)`, `(257,128)`, `(64,4096)` | Python NVFP4 reference | exact BF16 match |
51
+ | Official FP4 indexer API | tiny FP4 packed tensors; native artifact path when built | PyTorch block-score reference | returns official score layout |
52
+ | Decode lightning indexer | ctx 2048, 4096, 32768 | PyTorch blockmax top-k set | overlap >= 0.99 |
53
+ | Standalone long-context decode | ctx 65536, 131072 | paged FP32 PyTorch / direct kernel | cos >= 0.999; wrapper max_abs = 0 |
54
+ | Installed-artifact native long top-k | blocks 512, 1024 | PyTorch top-k over valid blocks | exact set match |
55
+
56
+ API surface validation:
57
+
58
+ ```bash
59
+ PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
60
+ python -m pytest MiniMaxAI-msa-blackwell/tests/test_api_surface.py -q
61
+ ```
62
+
63
+ The test tracks every official `MiniMaxAI/msa` public API name:
64
+
65
+ - `sparse_atten_func`
66
+ - `sparse_atten_nvfp4_kv_func`
67
+ - `sparse_decode_atten_func`
68
+ - `SparseDecodePagedAttentionWrapper`
69
+ - `fp4_indexer_block_scores`
70
+ - `build_k2q_csr`
71
+ - `SparseK2qCsrBuilderSm100`
72
+ - `Nvfp4QuantizedTensor`
73
+ - `quantize_bf16_to_nvfp4_128x4`
74
+ - `quantize_kv_bf16_to_nvfp4_128x4`
75
+ - `dequantize_nvfp4_128x4_to_bf16`
76
+ - `swizzle_nvfp4_scale_to_128x4`
77
+ - `nvfp4_global_scale_from_amax`
78
+
79
+ The root module exports every official public name. Decode, CSR prefill, NVFP4
80
+ prefill compatibility, FP4 block scoring, CSR, and NVFP4 helper names are all
81
+ callable. Hub built artifacts use compiled CUDA ops for the MiniMax-M3
82
+ Blackwell decode route, FP4 block-score indexer, and swizzled NVFP4 -> BF16
83
+ dequantization path. Source-tree mode keeps reference paths so the API remains
84
+ testable before the extension is built.
85
+
86
+ ## FlashRT Integration Note
87
+
88
+ FlashRT has validated the decode sparse path on SM121 over context lengths
89
+ 128 to 32768 with cosine similarity >= 0.999. The 32768 context length has
90
+ also been exercised in the FlashRT MiniMax-Spark model runtime on DGX Spark /
91
+ GB10, so it is the current end-to-end model validation boundary.
92
+
93
+ The standalone package kernel tests additionally cover 65536 and 131072
94
+ context lengths. These long-context rows validate the kernel and API wrapper
95
+ contract outside the full model runtime; they should not be described as
96
+ MiniMax-Spark end-to-end model validation until the full runtime path is rerun
97
+ at those lengths.
98
+
99
+ The same decode sparse path has also been exercised in FlashRT's MiniMax-Spark
100
+ model runtime on DGX Spark / GB10. That end-to-end validation is intentionally
101
+ kept as a FlashRT runtime validation item, while this Hub package exposes the
102
+ standalone kernel API for community use.
103
+
104
+ ## Native Helper Compile Smoke
105
+
106
+ Before HF Jobs publish, the native helper was compiled locally as a PyTorch
107
+ extension using the same source files:
108
+
109
+ - `torch-ext/torch_binding.cpp`
110
+ - `csrc/msa_topk_from_scores.cu`
111
+ - `csrc/msa_decode_attn.cu`
112
+ - `csrc/msa_decode_attn_mma.cu`
113
+ - `csrc/msa_indexer_block_scores.cu`
114
+ - `csrc/msa_nvfp4_dequant.cu`
115
+
116
+ Environment:
117
+
118
+ | Field | Value |
119
+ |---|---|
120
+ | GPU | NVIDIA GeForce RTX 5090 |
121
+ | PyTorch | 2.9.1+cu128 |
122
+ | nvcc | CUDA 13.0 |
123
+ | Target arch | sm_120 |
124
+
125
+ Result:
126
+
127
+ | Check | Shape | Reference | Verdict |
128
+ |---|---:|---|---|
129
+ | Native score -> top-k | heads 64, batch 1, blocks 256, topk 16 | PyTorch top-k set | PASS |
130
+ | Native FP4 block-score indexer | official `[Hq, blocks, total_q]` score layout | PyTorch block-score reference | PASS |
131
+ | Native NVFP4 swizzled -> BF16 dequant | rows/cols `(1,128)`, `(257,128)`, `(64,4096)` | Python NVFP4 reference | PASS |
132
+
133
+ ## Blackwell Package Validation
134
+
135
+ Remote Blackwell validation environment:
136
+
137
+ | Field | Value |
138
+ |---|---|
139
+ | Host | `spark-f517` |
140
+ | GPU | NVIDIA GB10 |
141
+ | Compute capability | 12.1 |
142
+ | Driver | 580.159.03 |
143
+ | Python | 3.12.3 |
144
+ | PyTorch | 2.12.0+cu130 |
145
+ | Triton | 3.7.0 |
146
+
147
+ Command:
148
+
149
+ ```bash
150
+ PY=/home/leadtek/jax/bin/python
151
+ PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
152
+ $PY MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py
153
+ ```
154
+
155
+ Result:
156
+
157
+ | Check | Shape | Cosine | Max abs / overlap | Verdict |
158
+ |---|---|---:|---:|---|
159
+ | Decode sparse GQA | ctx128_b1 | 0.999998 | 1.6032e-03 | PASS |
160
+ | Decode sparse GQA | ctx2048_b1 | 0.999996 | 4.9090e-04 | PASS |
161
+ | Decode sparse GQA | ctx2048_b2_sink | 0.999996 | 6.8302e-04 | PASS |
162
+ | Decode sparse GQA | ctx4096_b1 | 0.999996 | 4.5899e-04 | PASS |
163
+ | Decode sparse GQA | ctx4096_b2_mixed | 0.999996 | 7.3129e-04 | PASS |
164
+ | Decode sparse GQA | ctx32768_b1 | 0.999996 | 6.9451e-04 | PASS |
165
+ | Decode sparse GQA | ctx32768_b1_sink | 0.999996 | 5.6115e-04 | PASS |
166
+ | Decode sparse GQA | ctx65536_b1 | 0.999996 | 4.3470e-04 | PASS |
167
+ | Decode sparse GQA | ctx131072_b1 | 0.999996 | 7.1825e-04 | PASS |
168
+ | Decode top-k indexer | ctx2048 | n/a | overlap 1.000 | PASS |
169
+ | Decode top-k indexer | ctx4096 | n/a | overlap 1.000 | PASS |
170
+ | Decode top-k indexer | ctx32768 | n/a | overlap 1.000 | PASS |
171
+ | Decode top-k indexer | ctx65536 | n/a | overlap 1.000 | PASS |
172
+ | Decode top-k indexer | ctx131072 | n/a | overlap 1.000 | PASS |
173
+ | Official decode wrapper | ctx2048 | 1.000000 | 0.0000e+00 | PASS |
174
+ | Official decode wrapper | ctx4096 | 1.000000 | 0.0000e+00 | PASS |
175
+ | Official decode wrapper | ctx65536 | 1.000000 | 0.0000e+00 | PASS |
176
+ | Official decode wrapper | ctx131072 | 1.000000 | 0.0000e+00 | PASS |
177
+ | Native CUDA NVFP4 dequant | rows1_cols128 | 1.000000 | 0.0000e+00 | PASS |
178
+ | Native CUDA NVFP4 dequant | rows257_cols128 | 1.000000 | 0.0000e+00 | PASS |
179
+ | Native CUDA NVFP4 dequant | rows64_cols4096 | 1.000000 | 0.0000e+00 | PASS |
180
+
181
+ Installed-artifact native top-k validation on RTX 5090 / torch 2.11 / CUDA
182
+ 12.8:
183
+
184
+ | Context | Blocks | Overlap | Verdict |
185
+ |---:|---:|---:|---|
186
+ | 32768 | 256 | 1.000 | PASS |
187
+ | 65536 | 512 | 1.000 | PASS |
188
+ | 131072 | 1024 | 1.000 | PASS |
189
+
190
+ The warning `tl.make_block_ptr is deprecated` appears with Triton 3.7.0. It is
191
+ a deprecation warning, not a correctness failure.
192
+
193
+ ## Native Alignment Status
194
+
195
+ The upstream `MiniMaxAI/msa` package targets SM100. This Blackwell package
196
+ keeps the same public API surface where practical and provides native CUDA
197
+ implementations for the hot paths needed by the FlashRT MiniMax-Spark runtime:
198
+
199
+ - score-to-top-k sparse block selection;
200
+ - tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
201
+ - FP4 block-score indexing;
202
+ - swizzled NVFP4 -> BF16 dequantization for the W4A16 path.
203
+
204
+ The CSR prefill wrapper remains part of the public compatibility surface and is
205
+ validated against the package reference path. Shape and parameter restrictions
206
+ are explicit errors rather than silent wrong results.
benchmarks/benchmark_decode.py ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import argparse
2
+ import time
3
+
4
+ import torch
5
+
6
+ from minimaxai_msa_blackwell import (
7
+ flash_decode_with_gqa_share_sparse,
8
+ has_native_ops,
9
+ native_topk_from_scores,
10
+ )
11
+
12
+
13
+ def make_case(ctx: int, seed: int = 0):
14
+ g = torch.Generator(device="cuda").manual_seed(seed)
15
+ batch, hq, hkv, d = 1, 64, 4, 128
16
+ block, topk = 128, 16
17
+ q = torch.randn(batch, hq, d, device="cuda", dtype=torch.bfloat16, generator=g)
18
+ k_cache = torch.randn(ctx, hkv, d, device="cuda", dtype=torch.bfloat16, generator=g)
19
+ v_cache = torch.randn(ctx, hkv, d, device="cuda", dtype=torch.bfloat16, generator=g)
20
+ req_to_token = torch.arange(ctx, device="cuda", dtype=torch.int32).view(1, -1)
21
+ seq_lens = torch.tensor([ctx], device="cuda", dtype=torch.int32)
22
+ slot_ids = torch.zeros(batch, device="cuda", dtype=torch.int64)
23
+ nb = (ctx + block - 1) // block
24
+ n = min(topk, nb)
25
+ topk_idx = torch.full((hkv, batch, topk), -1, device="cuda", dtype=torch.int32)
26
+ topk_idx[:, :, :n] = torch.arange(n, device="cuda", dtype=torch.int32).view(1, 1, n)
27
+ return q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, block, topk_idx
28
+
29
+
30
+ def bench(ctx: int, warmup: int, iters: int) -> float:
31
+ args = make_case(ctx)
32
+ for _ in range(warmup):
33
+ flash_decode_with_gqa_share_sparse(*args)
34
+ torch.cuda.synchronize()
35
+ start = time.perf_counter()
36
+ for _ in range(iters):
37
+ flash_decode_with_gqa_share_sparse(*args)
38
+ torch.cuda.synchronize()
39
+ return (time.perf_counter() - start) * 1e6 / iters
40
+
41
+
42
+ def bench_native_topk(ctx: int, warmup: int, iters: int) -> float | None:
43
+ if not has_native_ops():
44
+ return None
45
+ heads, batch, block, topk = 64, 1, 128, 16
46
+ blocks = (ctx + block - 1) // block
47
+ score = torch.randn(heads, batch, blocks, device="cuda", dtype=torch.float32)
48
+ seq_lens = torch.tensor([ctx], device="cuda", dtype=torch.int32)
49
+ for _ in range(warmup):
50
+ native_topk_from_scores(score, seq_lens, block, topk)
51
+ torch.cuda.synchronize()
52
+ start = time.perf_counter()
53
+ for _ in range(iters):
54
+ native_topk_from_scores(score, seq_lens, block, topk)
55
+ torch.cuda.synchronize()
56
+ return (time.perf_counter() - start) * 1e6 / iters
57
+
58
+
59
+ def main() -> None:
60
+ ap = argparse.ArgumentParser()
61
+ ap.add_argument("--ctx", type=int, nargs="+", default=[2048, 4096, 32768, 65536, 131072])
62
+ ap.add_argument("--warmup", type=int, default=10)
63
+ ap.add_argument("--iters", type=int, default=50)
64
+ args = ap.parse_args()
65
+
66
+ print("gpu:", torch.cuda.get_device_name())
67
+ print("ctx,attention_mean_us,native_topk_mean_us")
68
+ for ctx in args.ctx:
69
+ attn_us = bench(ctx, args.warmup, args.iters)
70
+ topk_us = bench_native_topk(ctx, args.warmup, args.iters)
71
+ topk_text = "NA" if topk_us is None else f"{topk_us:.3f}"
72
+ print(f"{ctx},{attn_us:.3f},{topk_text}")
73
+
74
+
75
+ if __name__ == "__main__":
76
+ main()
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """FlashRT Blackwell extension of MiniMax MSA decode sparse attention.
3
+
4
+ This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on
5
+ DGX Spark / GB10 / SM121. It exposes the
6
+ MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime.
7
+
8
+ Implementation status:
9
+ * native CUDA helper: score -> top-k sparse block ids;
10
+ * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
11
+ * native CUDA FP4 block-score indexer;
12
+ * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path;
13
+ * Blackwell-validated sparse prefill path;
14
+ * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers,
15
+ and FP4 block-score helpers.
16
+
17
+ The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell
18
+ extension path. The public API is Tensor-oriented and independent from
19
+ FlashRT's serving runtime.
20
+
21
+ Public API
22
+ ----------
23
+ Lightning indexer (q.k blockmax score -> top-k block select), paged KV:
24
+ flash_decode_with_topk_idx (single-token decode, split-K)
25
+
26
+ Block-sparse GQA attention (consumes block_indices from the indexer):
27
+ flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks)
28
+ sparse_atten_func (official CSR prefill wrapper)
29
+ sparse_decode_atten_func (official paged decode wrapper)
30
+
31
+ PyTorch naive references (for correctness checks):
32
+ naive_flash_decode_with_gqa_share_sparse (attention given topk_idx)
33
+ naive_flash_decode_with_topk_idx (joint indexer + attention)
34
+
35
+ All Triton entrypoints expect a paged KV cache:
36
+ k_cache / v_cache : [max_slots, num_kv_heads, head_dim]
37
+ req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot)
38
+ slot_ids : [batch] (per-request row in req_to_token)
39
+ topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed,
40
+ -1 right-padding (the M3 indexer contract).
41
+ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
42
+ """
43
+
44
+ from .common.utils import get_cu_seqblocks, robust_allocator
45
+ from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
46
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
47
+ from .interface import (
48
+ SparseDecodePagedAttentionWrapper,
49
+ fp4_indexer_block_scores,
50
+ sparse_atten_func,
51
+ sparse_atten_nvfp4_kv_func,
52
+ sparse_decode_atten_func,
53
+ )
54
+ from ._native import (
55
+ has_native_ops,
56
+ native_nvfp4_dequant_swizzled_to_bf16,
57
+ native_topk_from_scores,
58
+ )
59
+ from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
60
+ from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
61
+ from .quantize import (
62
+ Nvfp4QuantizedTensor,
63
+ dequantize_nvfp4_128x4_to_bf16,
64
+ nvfp4_global_scale_from_amax,
65
+ quantize_bf16_to_nvfp4_128x4,
66
+ quantize_kv_bf16_to_nvfp4_128x4,
67
+ swizzle_nvfp4_scale_to_128x4,
68
+ )
69
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
70
+ from .api_status import (
71
+ OFFICIAL_API_STATUS,
72
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
73
+ V1_AVAILABLE_FUNCTIONS,
74
+ available_functions,
75
+ official_api_status,
76
+ official_minimax_msa_functions,
77
+ unsupported_official_functions,
78
+ )
79
+
80
+ __all__ = [
81
+ "sparse_atten_func",
82
+ "sparse_atten_nvfp4_kv_func",
83
+ "sparse_decode_atten_func",
84
+ "SparseDecodePagedAttentionWrapper",
85
+ "fp4_indexer_block_scores",
86
+ "build_k2q_csr",
87
+ "SparseK2qCsrBuilderSm100",
88
+ "Nvfp4QuantizedTensor",
89
+ "quantize_bf16_to_nvfp4_128x4",
90
+ "quantize_kv_bf16_to_nvfp4_128x4",
91
+ "dequantize_nvfp4_128x4_to_bf16",
92
+ "swizzle_nvfp4_scale_to_128x4",
93
+ "nvfp4_global_scale_from_amax",
94
+ "flash_decode_with_topk_idx",
95
+ "flash_decode_with_gqa_share_sparse",
96
+ "has_native_ops",
97
+ "native_topk_from_scores",
98
+ "native_nvfp4_dequant_swizzled_to_bf16",
99
+ "naive_flash_decode_with_gqa_share_sparse",
100
+ "naive_flash_decode_with_topk_idx",
101
+ "get_cu_seqblocks",
102
+ "robust_allocator",
103
+ "OFFICIAL_API_STATUS",
104
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
105
+ "V1_AVAILABLE_FUNCTIONS",
106
+ "available_functions",
107
+ "official_api_status",
108
+ "official_minimax_msa_functions",
109
+ "unsupported_official_functions",
110
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_compat.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton
3
+ # kernels. The upstream SGLang modules pull two tiny things from the framework:
4
+ #
5
+ # from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path)
6
+ # from sglang.srt.environ import envs -> a feature flag for an optional
7
+ # JIT-CUDA radix top-k fast path
8
+ #
9
+ # Both are replaced here so the kernels import with only torch + triton. The
10
+ # defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel
11
+ # radix select), which is what we want on sm_121 (GB10).
12
+
13
+
14
+ def is_hip() -> bool:
15
+ """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121).
16
+
17
+ Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV
18
+ widening branch (HIP-only). With this False, the CUDA dtype contract
19
+ (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA.
20
+ """
21
+ try:
22
+ import torch
23
+
24
+ return bool(getattr(torch.version, "hip", None))
25
+ except Exception:
26
+ return False
27
+
28
+
29
+ class _Flag:
30
+ """Minimal stand-in for an sglang.srt.environ boolean flag (always off)."""
31
+
32
+ def __init__(self, default: bool = False):
33
+ self._default = default
34
+
35
+ def get(self) -> bool:
36
+ return self._default
37
+
38
+
39
+ class _Envs:
40
+ """Stub for `sglang.srt.environ.envs`.
41
+
42
+ Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode
43
+ indexer. Keeping it False routes top-k selection through the 2-stage Triton
44
+ fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the
45
+ decode indexer stays pure-Triton.
46
+ """
47
+
48
+ SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False)
49
+
50
+
51
+ envs = _Envs()
build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:528883873873d9a482cbe93eb0ea0ec5be7f0eac009dd8b103a0609841c032d5
3
+ size 446928
build/torch211-cxx11-cu128-x86_64-linux/_native.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Optional native CUDA helpers for the FlashRT Blackwell MSA package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import torch
7
+
8
+ try:
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+ _HAS_NATIVE_OPS = True
12
+ except Exception:
13
+ add_op_namespace_prefix = None
14
+ ops = None
15
+ _HAS_NATIVE_OPS = False
16
+
17
+
18
+ def has_native_ops() -> bool:
19
+ return _HAS_NATIVE_OPS
20
+
21
+
22
+ if _HAS_NATIVE_OPS:
23
+
24
+ @torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores"))
25
+ def _msa_topk_from_scores_fake(
26
+ score: torch.Tensor,
27
+ seq_lens: torch.Tensor,
28
+ block_size: int,
29
+ topk: int,
30
+ topk_idx: torch.Tensor,
31
+ ) -> None:
32
+ if score.dim() != 3:
33
+ raise RuntimeError("score must have shape (heads, batch, max_blocks)")
34
+ heads, batch, _ = score.shape
35
+ if seq_lens.shape != (batch,):
36
+ raise RuntimeError("seq_lens must have shape (batch,)")
37
+ if topk_idx.shape != (heads, batch, topk):
38
+ raise RuntimeError("topk_idx must have shape (heads, batch, topk)")
39
+ return None
40
+
41
+
42
+ def native_topk_from_scores(
43
+ score: torch.Tensor,
44
+ seq_lens: torch.Tensor,
45
+ block_size: int,
46
+ topk: int,
47
+ ) -> torch.Tensor:
48
+ """Return top-k sparse block ids from a score tensor."""
49
+
50
+ if not _HAS_NATIVE_OPS:
51
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
52
+ if not score.is_contiguous():
53
+ score = score.contiguous()
54
+ if not seq_lens.is_contiguous():
55
+ seq_lens = seq_lens.contiguous()
56
+ out = torch.empty(
57
+ (score.shape[0], score.shape[1], int(topk)),
58
+ device=score.device,
59
+ dtype=torch.int32,
60
+ )
61
+ ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out)
62
+ return out
63
+
64
+
65
+ if _HAS_NATIVE_OPS:
66
+
67
+ @torch.library.register_fake(
68
+ add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged")
69
+ )
70
+ def _msa_decode_sparse_attn_mma_paged_fake(
71
+ q: torch.Tensor,
72
+ k_cache: torch.Tensor,
73
+ v_cache: torch.Tensor,
74
+ req_to_token: torch.Tensor,
75
+ seq_lens: torch.Tensor,
76
+ slot_ids: torch.Tensor,
77
+ topk_idx: torch.Tensor,
78
+ block_size: int,
79
+ sm_scale: float,
80
+ out: torch.Tensor,
81
+ ) -> None:
82
+ return None
83
+
84
+
85
+ def native_decode_mma_supported(
86
+ q: torch.Tensor,
87
+ k_cache: torch.Tensor,
88
+ *,
89
+ block_size: int,
90
+ ) -> bool:
91
+ """Whether the tensor-core decode kernel applies to these inputs.
92
+
93
+ The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA
94
+ group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64.
95
+ """
96
+
97
+ if not _HAS_NATIVE_OPS:
98
+ return False
99
+ if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16:
100
+ return False
101
+ if q.dim() != 3 or k_cache.dim() != 3:
102
+ return False
103
+ head_dim = int(q.shape[-1])
104
+ num_q_heads = int(q.shape[1])
105
+ num_kv_heads = int(k_cache.shape[1])
106
+ if head_dim != 128 or num_kv_heads <= 0:
107
+ return False
108
+ if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16:
109
+ return False
110
+ if int(block_size) % 64 != 0:
111
+ return False
112
+ return True
113
+
114
+
115
+ def native_decode_sparse_attn_mma_paged(
116
+ q: torch.Tensor,
117
+ k_cache: torch.Tensor,
118
+ v_cache: torch.Tensor,
119
+ req_to_token: torch.Tensor,
120
+ seq_lens: torch.Tensor,
121
+ slot_ids: torch.Tensor,
122
+ block_size: int,
123
+ topk_idx: torch.Tensor,
124
+ sm_scale: float,
125
+ ) -> torch.Tensor:
126
+ """Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16."""
127
+
128
+ if not _HAS_NATIVE_OPS:
129
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
130
+ out = torch.empty(
131
+ (int(q.shape[0]), int(q.shape[1]), int(q.shape[2])),
132
+ device=q.device,
133
+ dtype=q.dtype,
134
+ )
135
+ ops.msa_decode_sparse_attn_mma_paged(
136
+ q.contiguous(),
137
+ k_cache.contiguous(),
138
+ v_cache.contiguous(),
139
+ req_to_token.contiguous(),
140
+ seq_lens.contiguous(),
141
+ slot_ids.contiguous(),
142
+ topk_idx.contiguous(),
143
+ int(block_size),
144
+ float(sm_scale),
145
+ out,
146
+ )
147
+ return out
148
+
149
+
150
+ if _HAS_NATIVE_OPS:
151
+
152
+ @torch.library.register_fake(
153
+ add_op_namespace_prefix("msa_indexer_block_scores")
154
+ )
155
+ def _msa_indexer_block_scores_fake(
156
+ q: torch.Tensor,
157
+ k_pages: torch.Tensor,
158
+ batch_of_q: torch.Tensor,
159
+ cu_q: torch.Tensor,
160
+ cu_k: torch.Tensor,
161
+ cu_pages: torch.Tensor,
162
+ kv_indices: torch.Tensor,
163
+ causal: int,
164
+ scores: torch.Tensor,
165
+ ) -> None:
166
+ return None
167
+
168
+
169
+ def native_indexer_block_scores(
170
+ q: torch.Tensor,
171
+ k_pages: torch.Tensor,
172
+ batch_of_q: torch.Tensor,
173
+ cu_q: torch.Tensor,
174
+ cu_k: torch.Tensor,
175
+ cu_pages: torch.Tensor,
176
+ kv_indices: torch.Tensor,
177
+ *,
178
+ max_blocks: int,
179
+ causal: bool,
180
+ ) -> torch.Tensor:
181
+ """Block-max QK indexer scores. q/k_pages are bf16 (already dequantized).
182
+
183
+ Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible).
184
+ """
185
+
186
+ if not _HAS_NATIVE_OPS:
187
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
188
+ total_q = int(q.shape[0])
189
+ hq = int(q.shape[1])
190
+ scores = torch.full(
191
+ (hq, int(max_blocks), total_q),
192
+ float("-inf"),
193
+ dtype=torch.float32,
194
+ device=q.device,
195
+ )
196
+ ops.msa_indexer_block_scores(
197
+ q.contiguous(),
198
+ k_pages.contiguous(),
199
+ batch_of_q.contiguous(),
200
+ cu_q.contiguous(),
201
+ cu_k.contiguous(),
202
+ cu_pages.contiguous(),
203
+ kv_indices.contiguous(),
204
+ 1 if causal else 0,
205
+ scores,
206
+ )
207
+ return scores
208
+
209
+
210
+ if _HAS_NATIVE_OPS:
211
+
212
+ @torch.library.register_fake(
213
+ add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16")
214
+ )
215
+ def _msa_nvfp4_dequant_swizzled_to_bf16_fake(
216
+ packed: torch.Tensor,
217
+ scale_128x4: torch.Tensor,
218
+ global_scale: float,
219
+ out: torch.Tensor,
220
+ ) -> None:
221
+ return None
222
+
223
+
224
+ def native_nvfp4_dequant_swizzled_to_bf16(
225
+ packed: torch.Tensor,
226
+ scale_128x4: torch.Tensor,
227
+ global_scale: torch.Tensor | float,
228
+ ) -> torch.Tensor:
229
+ """Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path."""
230
+
231
+ if not _HAS_NATIVE_OPS:
232
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
233
+ if packed.dtype is not torch.uint8:
234
+ raise TypeError("packed must be torch.uint8")
235
+ if scale_128x4.dtype is not torch.uint8:
236
+ raise TypeError("scale_128x4 must be torch.uint8")
237
+ original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
238
+ out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16)
239
+ if isinstance(global_scale, torch.Tensor):
240
+ global_scale_value = float(global_scale.reshape(-1)[0].item())
241
+ else:
242
+ global_scale_value = float(global_scale)
243
+ ops.msa_nvfp4_dequant_swizzled_to_bf16(
244
+ packed.contiguous(),
245
+ scale_128x4.contiguous(),
246
+ global_scale_value,
247
+ out,
248
+ )
249
+ return out
250
+
251
+
252
+ __all__ = [
253
+ "has_native_ops",
254
+ "native_topk_from_scores",
255
+ "native_decode_mma_supported",
256
+ "native_decode_sparse_attn_mma_paged",
257
+ "native_indexer_block_scores",
258
+ "native_nvfp4_dequant_swizzled_to_bf16",
259
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _minimaxai_msa_blackwell_cuda_25b0067
3
+ ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/api_status.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Public API status for the FlashRT MiniMax MSA Blackwell package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from types import MappingProxyType
7
+
8
+
9
+ V1_AVAILABLE_FUNCTIONS = (
10
+ "sparse_atten_func",
11
+ "sparse_atten_nvfp4_kv_func",
12
+ "sparse_decode_atten_func",
13
+ "SparseDecodePagedAttentionWrapper",
14
+ "fp4_indexer_block_scores",
15
+ "build_k2q_csr",
16
+ "SparseK2qCsrBuilderSm100",
17
+ "Nvfp4QuantizedTensor",
18
+ "quantize_bf16_to_nvfp4_128x4",
19
+ "quantize_kv_bf16_to_nvfp4_128x4",
20
+ "dequantize_nvfp4_128x4_to_bf16",
21
+ "swizzle_nvfp4_scale_to_128x4",
22
+ "nvfp4_global_scale_from_amax",
23
+ "has_native_ops",
24
+ "native_topk_from_scores",
25
+ "native_nvfp4_dequant_swizzled_to_bf16",
26
+ "flash_decode_with_topk_idx",
27
+ "flash_decode_with_gqa_share_sparse",
28
+ "naive_flash_decode_with_topk_idx",
29
+ "naive_flash_decode_with_gqa_share_sparse",
30
+ "get_cu_seqblocks",
31
+ "robust_allocator",
32
+ )
33
+
34
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
35
+ "sparse_atten_func",
36
+ "sparse_atten_nvfp4_kv_func",
37
+ "sparse_decode_atten_func",
38
+ "SparseDecodePagedAttentionWrapper",
39
+ "fp4_indexer_block_scores",
40
+ "build_k2q_csr",
41
+ "SparseK2qCsrBuilderSm100",
42
+ "Nvfp4QuantizedTensor",
43
+ "quantize_bf16_to_nvfp4_128x4",
44
+ "quantize_kv_bf16_to_nvfp4_128x4",
45
+ "dequantize_nvfp4_128x4_to_bf16",
46
+ "swizzle_nvfp4_scale_to_128x4",
47
+ "nvfp4_global_scale_from_amax",
48
+ )
49
+
50
+ _OFFICIAL_API_STATUS = {
51
+ "sparse_atten_func": {
52
+ "status": "available",
53
+ "target": "blackwell-prefill",
54
+ "reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.",
55
+ },
56
+ "sparse_atten_nvfp4_kv_func": {
57
+ "status": "available",
58
+ "target": "blackwell-prefill-nvfp4",
59
+ "reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.",
60
+ },
61
+ "sparse_decode_atten_func": {
62
+ "status": "available",
63
+ "target": "v1",
64
+ "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
65
+ },
66
+ "SparseDecodePagedAttentionWrapper": {
67
+ "status": "available",
68
+ "target": "v1",
69
+ "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
70
+ },
71
+ "fp4_indexer_block_scores": {
72
+ "status": "available",
73
+ "target": "blackwell-fp4-indexer",
74
+ "reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.",
75
+ },
76
+ "build_k2q_csr": {
77
+ "status": "available",
78
+ "target": "v1",
79
+ "reason": "CSR construction helper is available for the official prefill API.",
80
+ },
81
+ "SparseK2qCsrBuilderSm100": {
82
+ "status": "available",
83
+ "target": "v1",
84
+ "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
85
+ },
86
+ "Nvfp4QuantizedTensor": {
87
+ "status": "available",
88
+ "target": "v1",
89
+ "reason": "NVFP4 metadata dataclass is available.",
90
+ },
91
+ "quantize_bf16_to_nvfp4_128x4": {
92
+ "status": "available_optional_te",
93
+ "target": "v1",
94
+ "reason": "Function is available and uses Transformer Engine when installed.",
95
+ },
96
+ "quantize_kv_bf16_to_nvfp4_128x4": {
97
+ "status": "available_optional_te",
98
+ "target": "v1",
99
+ "reason": "Function is available and uses Transformer Engine when installed.",
100
+ },
101
+ "dequantize_nvfp4_128x4_to_bf16": {
102
+ "status": "available",
103
+ "target": "v1",
104
+ "reason": "Reference dequant helper is available.",
105
+ },
106
+ "swizzle_nvfp4_scale_to_128x4": {
107
+ "status": "available",
108
+ "target": "v1",
109
+ "reason": "Scale swizzle helper is available.",
110
+ },
111
+ "nvfp4_global_scale_from_amax": {
112
+ "status": "available",
113
+ "target": "v1",
114
+ "reason": "Global scale helper is available.",
115
+ },
116
+ }
117
+
118
+ OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
119
+
120
+
121
+ def available_functions() -> tuple[str, ...]:
122
+ """Return the functions/classes intentionally available in v1."""
123
+
124
+ return V1_AVAILABLE_FUNCTIONS
125
+
126
+
127
+ def official_minimax_msa_functions() -> tuple[str, ...]:
128
+ """Return the upstream MiniMaxAI/msa public function/class names tracked here."""
129
+
130
+ return OFFICIAL_MINIMAX_MSA_FUNCTIONS
131
+
132
+
133
+ def official_api_status() -> dict[str, dict[str, str]]:
134
+ """Return a copy of the official API compatibility status table."""
135
+
136
+ return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
137
+
138
+
139
+ def unsupported_official_functions() -> tuple[str, ...]:
140
+ """Return official MiniMaxAI/msa names that are not exported by this v1 package."""
141
+
142
+ return tuple(
143
+ name
144
+ for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
145
+ if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
146
+ )
build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
build/torch211-cxx11-cu128-x86_64-linux/common/utils.py ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ import functools
4
+ from collections import deque
5
+ from typing import Any, Callable, Tuple
6
+
7
+ import torch
8
+ import triton.language as tl
9
+
10
+ _tma_keep_alive_buf = deque(maxlen=200)
11
+
12
+ try:
13
+ make_tensor_descriptor = tl.make_tensor_descriptor
14
+ except Exception:
15
+ make_tensor_descriptor = tl._experimental_make_tensor_descriptor
16
+
17
+
18
+ def robust_allocator(size: int, alignment: int, stream: int = None):
19
+ """Allocator for Triton TMA descriptors.
20
+
21
+ We keep reference in deque to prevent GC from collecting the buffer.
22
+ """
23
+ tensor = torch.empty(size, device="cuda", dtype=torch.uint8)
24
+ _tma_keep_alive_buf.append(tensor)
25
+ return tensor
26
+
27
+
28
+ def tensor_cache(maxsize: int = 8):
29
+ """
30
+ Cache function results using identity comparison.
31
+ Zero-overhead cache hit: no hash, no DtoH, just pointer comparison.
32
+
33
+ Args:
34
+ maxsize: Maximum number of cached entries. Supports multi-GPU scenarios
35
+ where different devices have different tensor arguments.
36
+ """
37
+
38
+ def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
39
+ # LRU-style cache: list of (args, kwargs, result) tuples
40
+ # Most recently used at the end
41
+ _cache: list = []
42
+
43
+ def _args_match(args: tuple, cached_args: tuple) -> bool:
44
+ if len(args) != len(cached_args):
45
+ return False
46
+ for i in range(len(args)):
47
+ if args[i] is not cached_args[i]:
48
+ return False
49
+ return True
50
+
51
+ def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool:
52
+ if not kwargs and not cached_kwargs:
53
+ return True
54
+ if kwargs.keys() != cached_kwargs.keys():
55
+ return False
56
+ for k, v in kwargs.items():
57
+ if v is not cached_kwargs[k]:
58
+ return False
59
+ return True
60
+
61
+ @functools.wraps(fn)
62
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
63
+ # Search cache (most recent first for better hit rate)
64
+ for i in range(len(_cache) - 1, -1, -1):
65
+ cached_args, cached_kwargs, cached_result = _cache[i]
66
+ if _args_match(args, cached_args) and _kwargs_match(
67
+ kwargs, cached_kwargs
68
+ ):
69
+ # Move to end (most recently used)
70
+ if i != len(_cache) - 1:
71
+ _cache.append(_cache.pop(i))
72
+ return cached_result
73
+
74
+ # Cache miss
75
+ result = fn(*args, **kwargs)
76
+
77
+ # Add to cache
78
+ if len(_cache) >= maxsize:
79
+ _cache.pop(0) # Remove oldest
80
+ _cache.append((args, kwargs, result))
81
+ return result
82
+
83
+ # Expose cache for manual clearing if needed
84
+ wrapper.cache_clear = lambda: _cache.clear()
85
+ wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize}
86
+
87
+ return wrapper
88
+
89
+ return decorator
90
+
91
+
92
+ @tensor_cache(maxsize=8)
93
+ def get_cu_seqblocks(
94
+ cu_seqlens: torch.Tensor,
95
+ max_seqlen: int,
96
+ block_size_q: int,
97
+ block_size_k: int,
98
+ ) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]:
99
+ """Compute cumulative sequence block indices for blocked sparse attention.
100
+
101
+ Converts token-level cumulative sequence lengths to block-level indices,
102
+ which are needed for block-sparse attention kernels.
103
+
104
+ Note:
105
+ Results are cached (maxsize=8) based on input arguments. Repeated calls
106
+ with the same cu_seqlens, max_seqlen, and block sizes will return cached
107
+ results without recomputation.
108
+
109
+ Args:
110
+ cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32.
111
+ max_seqlen: Maximum sequence length in the batch.
112
+ block_size_q: Query block size.
113
+ block_size_k: Key-value block size.
114
+
115
+ Returns:
116
+ A tuple of 6 values:
117
+ - cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1]
118
+ - max_seqblock_q: Maximum number of query blocks per sequence.
119
+ - all_seqblock_q: Total number of query blocks across all sequences.
120
+ - cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1]
121
+ - max_seqblock_k: Maximum number of key blocks per sequence.
122
+ - all_seqblock_k: Total number of key blocks across all sequences.
123
+ """
124
+ cu_seqblocks_q = torch.zeros_like(cu_seqlens)
125
+ cu_seqblocks_k = torch.zeros_like(cu_seqlens)
126
+ seq_lens = torch.diff(cu_seqlens)
127
+ seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q
128
+ seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k
129
+ max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q
130
+ max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k
131
+ cu_seqblocks_q[1:] = seqblocks_q
132
+ cu_seqblocks_k[1:] = seqblocks_k
133
+ cu_seqblocks_q.cumsum_(0)
134
+ cu_seqblocks_k.cumsum_(0)
135
+ all_seqblock_q = seqblocks_q.sum().item()
136
+ all_seqblock_k = seqblocks_k.sum().item()
137
+ return (
138
+ cu_seqblocks_q,
139
+ max_seqblock_q,
140
+ all_seqblock_q,
141
+ cu_seqblocks_k,
142
+ max_seqblock_k,
143
+ all_seqblock_k,
144
+ )
build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py ADDED
@@ -0,0 +1,1150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .._compat import envs # vendored: was `from sglang.srt.environ import envs`
10
+
11
+ from ..common.utils import robust_allocator
12
+ from .._native import has_native_ops, native_topk_from_scores
13
+
14
+
15
+ @triton.heuristics(
16
+ {
17
+ "BLOCK_SIZE_H": lambda args: max(
18
+ 16, triton.next_power_of_2(args["gqa_group_size"])
19
+ ),
20
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
21
+ "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
22
+ }
23
+ )
24
+ @triton.autotune(
25
+ configs=[
26
+ triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
27
+ for BN in [64, 128, 256, 512]
28
+ for nw in [4, 8, 16]
29
+ for ns in [1, 2, 3]
30
+ ],
31
+ key=[
32
+ "BATCH_SIZE_BUCKET",
33
+ "gqa_group_size",
34
+ "head_dim",
35
+ "block_size",
36
+ "SCORE_TYPE",
37
+ ],
38
+ )
39
+ @triton.jit
40
+ def _decode_score_kernel(
41
+ q_ptr, # Q: b x qh x d
42
+ k_cache_ptr, # K paged: max_slots x kh x d
43
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
44
+ score_ptr, # Score: qh x b x max_seqblock
45
+ seq_lens,
46
+ slot_ids,
47
+ # shape
48
+ max_slots,
49
+ batch_size,
50
+ gqa_group_size,
51
+ head_dim,
52
+ # block size
53
+ block_size: tl.constexpr,
54
+ topk: tl.constexpr,
55
+ # sm_scale
56
+ sm_scale,
57
+ # init and local blocks
58
+ init_blocks,
59
+ local_blocks,
60
+ # stride
61
+ stride_q_b,
62
+ stride_q_h,
63
+ stride_q_d,
64
+ stride_k_s,
65
+ stride_k_h,
66
+ stride_k_d,
67
+ stride_r2t_b,
68
+ stride_s_h,
69
+ stride_s_b,
70
+ stride_s_n,
71
+ # META parameters
72
+ BATCH_SIZE_BUCKET: tl.constexpr,
73
+ BLOCK_SIZE_H: tl.constexpr,
74
+ BLOCK_SIZE_N: tl.constexpr,
75
+ BLOCK_SIZE_D: tl.constexpr,
76
+ NUM_KV_CHUNKS: tl.constexpr,
77
+ SCORE_TYPE: tl.constexpr,
78
+ SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
79
+ ):
80
+ tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
81
+ sm_scale_log2e = sm_scale * 1.4426950409
82
+ tl.static_assert(BLOCK_SIZE_N >= block_size)
83
+ BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
84
+ # get batch id and head id
85
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
86
+ pid_b = pid_bc % batch_size
87
+ pid_c = pid_bc // batch_size
88
+ pid_h = pid_kh * gqa_group_size
89
+ # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
90
+ seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
91
+ num_blocks = (seq_len + block_size - 1) // block_size
92
+ if SKIP_TRIVIAL_TOPK_SCORE:
93
+ if num_blocks <= topk:
94
+ return
95
+ chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
96
+ chunk_start_block = pid_c * chunk_size_blocks
97
+ chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
98
+ chunk_start = chunk_start_block * block_size
99
+ chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
100
+ if chunk_start_block >= chunk_end_block:
101
+ return
102
+ sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots
103
+ # init qkv pointer
104
+ q_ptrs = tl.make_block_ptr(
105
+ base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
106
+ shape=(gqa_group_size, head_dim),
107
+ strides=(stride_q_h, stride_q_d),
108
+ offsets=(0, 0),
109
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
110
+ order=(1, 0),
111
+ )
112
+ s_ptrs = tl.make_block_ptr(
113
+ base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
114
+ shape=(gqa_group_size, chunk_end_block),
115
+ strides=(stride_s_h, stride_s_n),
116
+ offsets=(0, chunk_start_block),
117
+ block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
118
+ order=(1, 0),
119
+ )
120
+ # load q
121
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
122
+ # init statistics
123
+ off_n = tl.arange(0, BLOCK_SIZE_N)
124
+ off_d = tl.arange(0, BLOCK_SIZE_D)
125
+ off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
126
+ dim_mask = off_d < head_dim
127
+ # pre-compute local_start outside the loop (constant for entire batch)
128
+ local_start = tl.maximum(0, num_blocks - local_blocks)
129
+ # prefetch first iteration's slots
130
+ r2t_base = req_to_token_ptr + sid * stride_r2t_b
131
+ prefetch_pos = chunk_start + off_n
132
+ prefetch_mask = prefetch_pos < seq_len
133
+ prefetched_slots = tl.load(
134
+ r2t_base + prefetch_pos,
135
+ mask=prefetch_mask,
136
+ other=0,
137
+ ).to(tl.int64)
138
+ # score-only: compute block scores without loading V
139
+ for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
140
+ pos_mask = prefetch_mask
141
+ slots = prefetched_slots
142
+ # prefetch next iteration's slots
143
+ next_i = i + BLOCK_SIZE_N
144
+ if next_i < chunk_end:
145
+ next_pos = next_i + off_n
146
+ prefetch_mask = next_pos < seq_len
147
+ prefetched_slots = tl.load(
148
+ r2t_base + next_pos,
149
+ mask=prefetch_mask,
150
+ other=0,
151
+ ).to(tl.int64)
152
+ slots = (slots + max_slots) % max_slots
153
+ # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
154
+ k_off = (
155
+ slots[None, :] * stride_k_s
156
+ + pid_kh * stride_k_h
157
+ + off_d[:, None] * stride_k_d
158
+ )
159
+ k = tl.load(
160
+ k_cache_ptr + k_off,
161
+ mask=dim_mask[:, None] & pos_mask[None, :],
162
+ other=0.0,
163
+ )
164
+ # compute qk
165
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
166
+ qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
167
+ # [H, D], [D, N] -> [H, N]
168
+ qk += tl.dot(q, k) * sm_scale_log2e
169
+ # save qk to score
170
+ score = tl.reshape(
171
+ qk,
172
+ (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
173
+ can_reorder=False,
174
+ )
175
+ sub_max = tl.max(score, axis=2)
176
+ if SCORE_TYPE == "max":
177
+ score = sub_max
178
+ else: # "lse"
179
+ score = sub_max + tl.log2(
180
+ tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
181
+ )
182
+ score = tl.where(score != score, float("-inf"), score)
183
+ # apply init_blocks and local_blocks efficiently
184
+ curr_block_idx = i // block_size + off_bpk
185
+ is_init = curr_block_idx < init_blocks
186
+ is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
187
+ score = tl.where(
188
+ is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
189
+ )
190
+ tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1))
191
+ # update ptrs
192
+ s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
193
+
194
+
195
+ @triton.heuristics(
196
+ {
197
+ "BLOCK_SIZE_H": lambda args: max(
198
+ 16, triton.next_power_of_2(args["gqa_group_size"])
199
+ ),
200
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
201
+ "HAS_SINK": lambda args: args["sink_ptr"] is not None,
202
+ }
203
+ )
204
+ @triton.autotune(
205
+ configs=[
206
+ triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
207
+ for BN in [64, 128, 256, 512]
208
+ for nw in [4, 8, 16]
209
+ for ns in [1, 2, 3]
210
+ ],
211
+ key=[
212
+ "gqa_group_size",
213
+ "head_dim",
214
+ "block_size",
215
+ "HAS_SINK",
216
+ "SCORE_TYPE",
217
+ ],
218
+ )
219
+ @triton.jit
220
+ def _decode_score_attn_kernel(
221
+ q_ptr, # Q: b x qh x d
222
+ sink_ptr, # Sink: qh x d
223
+ k_cache_ptr, # K paged: max_slots x kh x d
224
+ v_cache_ptr, # V paged: max_slots x kh x d
225
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
226
+ o_ptr, # O: c x b x qh x d
227
+ lse_ptr, # lse: c x b x qh
228
+ score_ptr, # Score: qh x b x max_seqblock
229
+ seq_lens,
230
+ slot_ids,
231
+ # shape
232
+ max_slots,
233
+ batch_size,
234
+ gqa_group_size,
235
+ head_dim,
236
+ # block size
237
+ block_size: tl.constexpr,
238
+ topk: tl.constexpr,
239
+ # sm_scale
240
+ sm_scale,
241
+ # init and local blocks
242
+ init_blocks,
243
+ local_blocks,
244
+ # stride
245
+ stride_q_b,
246
+ stride_q_h,
247
+ stride_q_d,
248
+ stride_sink_h,
249
+ stride_sink_d,
250
+ stride_k_s,
251
+ stride_k_h,
252
+ stride_k_d,
253
+ stride_v_s,
254
+ stride_v_h,
255
+ stride_v_d,
256
+ stride_r2t_b,
257
+ stride_o_c,
258
+ stride_o_b,
259
+ stride_o_h,
260
+ stride_o_d,
261
+ stride_l_c,
262
+ stride_l_b,
263
+ stride_l_h,
264
+ stride_s_h,
265
+ stride_s_b,
266
+ stride_s_n,
267
+ # META parameters
268
+ BLOCK_SIZE_H: tl.constexpr,
269
+ BLOCK_SIZE_N: tl.constexpr,
270
+ BLOCK_SIZE_D: tl.constexpr,
271
+ NUM_KV_CHUNKS: tl.constexpr,
272
+ HAS_SINK: tl.constexpr,
273
+ SCORE_TYPE: tl.constexpr,
274
+ SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
275
+ ):
276
+ tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
277
+ sm_scale_log2e = sm_scale * 1.4426950409
278
+ tl.static_assert(BLOCK_SIZE_N >= block_size)
279
+ BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
280
+ # get batch id and head id
281
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
282
+ pid_b = pid_bc % batch_size
283
+ pid_c = pid_bc // batch_size
284
+ pid_h = pid_kh * gqa_group_size
285
+ # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
286
+ seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
287
+ num_blocks = (seq_len + block_size - 1) // block_size
288
+ chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
289
+ chunk_start_block = pid_c * chunk_size_blocks
290
+ chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
291
+ chunk_start = chunk_start_block * block_size
292
+ chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
293
+ if chunk_start_block >= chunk_end_block:
294
+ return
295
+ sid = (
296
+ tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
297
+ ) % max_slots # to avoid bugs when slot_ids is negative
298
+ # init qkv pointer
299
+ q_ptrs = tl.make_block_ptr(
300
+ base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
301
+ shape=(gqa_group_size, head_dim),
302
+ strides=(stride_q_h, stride_q_d),
303
+ offsets=(0, 0),
304
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
305
+ order=(1, 0),
306
+ )
307
+ s_ptrs = tl.make_block_ptr(
308
+ base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
309
+ shape=(gqa_group_size, chunk_end_block),
310
+ strides=(stride_s_h, stride_s_n),
311
+ offsets=(0, chunk_start_block),
312
+ block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
313
+ order=(1, 0),
314
+ )
315
+ # load q
316
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
317
+ # init statistics
318
+ off_n = tl.arange(0, BLOCK_SIZE_N)
319
+ off_d = tl.arange(0, BLOCK_SIZE_D)
320
+ off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
321
+ dim_mask = off_d < head_dim
322
+ if HAS_SINK:
323
+ if pid_c == 0:
324
+ sink_ptrs = tl.make_block_ptr(
325
+ base=sink_ptr + pid_h * stride_sink_h,
326
+ shape=(gqa_group_size, head_dim),
327
+ strides=(stride_sink_h, stride_sink_d),
328
+ offsets=(0, 0),
329
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
330
+ order=(1, 0),
331
+ )
332
+ sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
333
+ tl.float32
334
+ )
335
+ qsink = (
336
+ tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e
337
+ ) # (BLOCK_SIZE_H,)
338
+ m_i = qsink
339
+ l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32)
340
+ else:
341
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
342
+ l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
343
+ else:
344
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
345
+ l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
346
+ acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
347
+ # pre-compute local_start outside the loop (constant for entire batch)
348
+ local_start = tl.maximum(0, num_blocks - local_blocks)
349
+ # full attention or causal attention
350
+ for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
351
+ pos = i + off_n
352
+ pos_mask = pos < seq_len
353
+ # resolve slot per position via req_to_token
354
+ slots = tl.load(
355
+ req_to_token_ptr + sid * stride_r2t_b + pos,
356
+ mask=pos_mask,
357
+ other=0,
358
+ ).to(tl.int64)
359
+ slots = (slots + max_slots) % max_slots # safety against negative
360
+ # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
361
+ k_off = (
362
+ slots[None, :] * stride_k_s
363
+ + pid_kh * stride_k_h
364
+ + off_d[:, None] * stride_k_d
365
+ )
366
+ k = tl.load(
367
+ k_cache_ptr + k_off,
368
+ mask=dim_mask[:, None] & pos_mask[None, :],
369
+ other=0.0,
370
+ )
371
+ # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
372
+ v_off = (
373
+ slots[:, None] * stride_v_s
374
+ + pid_kh * stride_v_h
375
+ + off_d[None, :] * stride_v_d
376
+ )
377
+ v = tl.load(
378
+ v_cache_ptr + v_off,
379
+ mask=pos_mask[:, None] & dim_mask[None, :],
380
+ other=0.0,
381
+ )
382
+ # compute qk
383
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
384
+ qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
385
+ # [H, D], [D, N] -> [H, N]
386
+ qk += tl.dot(q, k) * sm_scale_log2e
387
+ # save qk to score
388
+ score = tl.reshape(
389
+ qk,
390
+ (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
391
+ can_reorder=False,
392
+ )
393
+ sub_max = tl.max(score, axis=2)
394
+ if SCORE_TYPE == "max":
395
+ score = sub_max
396
+ else: # "lse"
397
+ score = sub_max + tl.log2(
398
+ tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
399
+ )
400
+ score = tl.where(score != score, float("-inf"), score)
401
+ # apply init_blocks and local_blocks efficiently
402
+ # current block indices: [BLOCKS_PER_K_BLOCK]
403
+ curr_block_idx = i // block_size + off_bpk
404
+ # Combined condition with single nested tl.where:
405
+ # - local_blocks (1e29) takes priority over init_blocks (1e30)
406
+ # - When init_blocks=0: curr_block_idx < 0 is always False
407
+ # - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False
408
+ is_init = curr_block_idx < init_blocks
409
+ is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
410
+ # Apply in one fused operation: local > init > original
411
+ score = tl.where(
412
+ is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
413
+ )
414
+ if SKIP_TRIVIAL_TOPK_SCORE:
415
+ if num_blocks > topk:
416
+ tl.store(
417
+ s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
418
+ )
419
+ else:
420
+ tl.store(
421
+ s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
422
+ )
423
+ # max-of-max == max(qk), avoids re-scanning qk
424
+ m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1))
425
+ p = tl.exp2(qk - m_ij[:, None])
426
+ l_ij = tl.sum(p, axis=1)
427
+ acc_o_scale = tl.exp2(m_i - m_ij)
428
+ acc_o = acc_o * acc_o_scale[:, None]
429
+ # [H, N], [N, D] -> [H, D]
430
+ acc_o += tl.dot(p.to(v.dtype), v)
431
+ m_i = m_ij
432
+ l_i = l_i * acc_o_scale + l_ij
433
+ # update ptrs
434
+ s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
435
+ # final scale
436
+ acc_o = acc_o / l_i[:, None]
437
+ # compute lse in log2 scale for merge kernel
438
+ lse_i = m_i + tl.log2(l_i)
439
+ # save output
440
+ o_ptrs = tl.make_block_ptr(
441
+ base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
442
+ shape=(gqa_group_size, head_dim),
443
+ strides=(stride_o_h, stride_o_d),
444
+ offsets=(0, 0),
445
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
446
+ order=(1, 0),
447
+ )
448
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
449
+ # save lse (in log2 scale, will be used directly in merge kernel)
450
+ lse_ptrs = tl.make_block_ptr(
451
+ base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
452
+ shape=(gqa_group_size,),
453
+ strides=(stride_l_h,),
454
+ offsets=(0,),
455
+ block_shape=(BLOCK_SIZE_H,),
456
+ order=(0,),
457
+ )
458
+ tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
459
+
460
+
461
+ @triton.heuristics(
462
+ {
463
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
464
+ }
465
+ )
466
+ @triton.autotune(
467
+ configs=[
468
+ triton.Config({}, num_warps=nw, num_stages=ns)
469
+ for nw in [4, 8]
470
+ for ns in [2, 3, 4]
471
+ ],
472
+ key=["BLOCK_SIZE_D"],
473
+ restore_value=["o_ptr"],
474
+ )
475
+ @triton.jit
476
+ def _merge_attn_out_kernel(
477
+ o_ptr,
478
+ lse_ptr,
479
+ # shape
480
+ seq_lens,
481
+ head_dim,
482
+ block_size: tl.constexpr,
483
+ # stride
484
+ stride_o_c,
485
+ stride_o_b,
486
+ stride_o_h,
487
+ stride_o_d,
488
+ stride_l_c,
489
+ stride_l_b,
490
+ stride_l_h,
491
+ NUM_KV_CHUNKS: tl.constexpr,
492
+ BLOCK_SIZE_D: tl.constexpr,
493
+ ):
494
+ pid_b, pid_h = tl.program_id(0), tl.program_id(1)
495
+ # recompute chunk layout (must match _decode_score_attn_kernel)
496
+ seq_len = tl.load(seq_lens + pid_b)
497
+ num_blocks = (seq_len + block_size - 1) // block_size
498
+ chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
499
+ valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks)
500
+ # ptrs
501
+ off_c = tl.arange(0, NUM_KV_CHUNKS)
502
+ off_d = tl.arange(0, BLOCK_SIZE_D)
503
+ o_ptrs = tl.make_block_ptr(
504
+ base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
505
+ shape=(valid_chunks, head_dim),
506
+ strides=(stride_o_c, stride_o_d),
507
+ offsets=(0, 0),
508
+ block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D),
509
+ order=(1, 0),
510
+ )
511
+ lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
512
+ # load o and lse
513
+ o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
514
+ lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf"))
515
+ # merge o (lse is in log2 scale, use exp2/log2 for efficiency)
516
+ lse_shifted = lse - tl.max(lse, axis=0)
517
+ log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0))
518
+ scale = tl.exp2(lse_shifted - log_sum)
519
+ o = tl.sum(o * scale[:, None], axis=0)
520
+ # save o
521
+ o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
522
+ tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
523
+
524
+
525
+ @triton.jit
526
+ def _compare_and_swap(
527
+ x,
528
+ ids,
529
+ flip,
530
+ i: tl.constexpr,
531
+ n_dims: tl.constexpr,
532
+ ):
533
+ n_outer: tl.constexpr = x.numel >> n_dims
534
+ shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
535
+ y = tl.reshape(x, shape)
536
+ # slice left/right with 'stride' 2**(n_dims - i - 1)
537
+ mask = tl.arange(0, 2)[None, :, None]
538
+ left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
539
+ right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
540
+ left = tl.reshape(left, x.shape)
541
+ right = tl.reshape(right, x.shape)
542
+ # idx
543
+ y_idx = tl.reshape(ids, shape)
544
+ left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
545
+ right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
546
+ left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
547
+ right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
548
+ # actual compare-and-swap
549
+ idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
550
+ ileft = left.to(idtype, bitcast=True)
551
+ iright = right.to(idtype, bitcast=True)
552
+ ix = x.to(idtype, bitcast=True)
553
+
554
+ cond = (left > right) != flip
555
+ ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
556
+ new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
557
+ return ret.to(x.dtype, bitcast=True), new_ids
558
+
559
+
560
+ @triton.jit
561
+ def _bitonic_merge(
562
+ x,
563
+ ids,
564
+ stage: tl.constexpr,
565
+ order: tl.constexpr,
566
+ n_dims: tl.constexpr,
567
+ ):
568
+ n_outer: tl.constexpr = x.numel >> n_dims
569
+ tl.static_assert(stage <= n_dims)
570
+ # flip denotes whether to re-arrange sub-sequences of elements in ascending or
571
+ # descending order.
572
+ # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage
573
+ # if flip = 00110011... then all the elements will be re-arranged alternatingly (with
574
+ # a stride of 2) at this stage
575
+ if order == 2:
576
+ shape: tl.constexpr = [
577
+ n_outer * 2 ** (n_dims - 1 - stage),
578
+ 2,
579
+ 2**stage,
580
+ ]
581
+ flip = tl.reshape(
582
+ tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
583
+ )
584
+ else:
585
+ flip = order
586
+ # perform `stage` rounds of `compare-and-swap`
587
+ for i in tl.static_range(stage):
588
+ x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
589
+ return x, ids
590
+
591
+
592
+ @triton.heuristics(
593
+ {
594
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
595
+ }
596
+ )
597
+ @triton.autotune(
598
+ configs=[
599
+ triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
600
+ triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2),
601
+ triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
602
+ triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3),
603
+ triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
604
+ ],
605
+ key=["topk"],
606
+ )
607
+ @triton.jit
608
+ def _topk_index_partial_kernel(
609
+ s_ptr, # Score: [num_q_heads, BS, max_seqblock]
610
+ ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
611
+ ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape
612
+ seq_lens,
613
+ block_size: tl.constexpr,
614
+ topk: tl.constexpr,
615
+ chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns
616
+ # strides
617
+ stride_s_h,
618
+ stride_s_b,
619
+ stride_s_k,
620
+ stride_ts_c,
621
+ stride_ts_h,
622
+ stride_ts_b,
623
+ stride_ts_t,
624
+ stride_ti_c,
625
+ stride_ti_h,
626
+ stride_ti_b,
627
+ stride_ti_t,
628
+ # META
629
+ BLOCK_SIZE_K: tl.constexpr,
630
+ BLOCK_SIZE_T: tl.constexpr,
631
+ ):
632
+ tl.static_assert(BLOCK_SIZE_K > topk)
633
+ pid_b = tl.program_id(0)
634
+ pid_h = tl.program_id(1)
635
+ pid_chunk = tl.program_id(2)
636
+
637
+ seq_len = tl.load(seq_lens + pid_b)
638
+ num_blocks = (seq_len + block_size - 1) // block_size
639
+
640
+ # Slice this chunk owns within [0, num_blocks).
641
+ chunk_start = pid_chunk * chunk_blocks
642
+ chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
643
+ chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
644
+
645
+ off_k = tl.arange(0, BLOCK_SIZE_K)
646
+ off_t = tl.arange(0, BLOCK_SIZE_T)
647
+
648
+ # Pointer starts at chunk_start.
649
+ s_ptrs = (
650
+ s_ptr
651
+ + pid_b * stride_s_b
652
+ + pid_h * stride_s_h
653
+ + (chunk_start + off_k) * stride_s_k
654
+ )
655
+
656
+ topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
657
+ topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
658
+ left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
659
+
660
+ # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
661
+ # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
662
+ for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
663
+ mask = off_k < chunk_actual - i
664
+ score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
665
+ score = tl.where(score != score, -1e30, score)
666
+ s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
667
+ topk_score, last_topk_score = score, topk_score
668
+ topk_idx, last_topk_idx = (
669
+ tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
670
+ topk_idx,
671
+ )
672
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
673
+ for j in tl.static_range(1, n_dims):
674
+ topk_score, topk_idx = _bitonic_merge(
675
+ topk_score, topk_idx.to(tl.int32), j, 2, n_dims
676
+ )
677
+ if i != 0:
678
+ topk_score, topk_idx = _bitonic_merge(
679
+ topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
680
+ )
681
+ topk_score_new = last_topk_score * left_half_mask + topk_score * (
682
+ 1 - left_half_mask
683
+ )
684
+ topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
685
+ 1 - left_half_mask
686
+ )
687
+ topk_score, topk_idx = _bitonic_merge(
688
+ topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
689
+ )
690
+ else:
691
+ topk_score, topk_idx = _bitonic_merge(
692
+ topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
693
+ )
694
+
695
+ # Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort).
696
+ topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
697
+ final_score = tl.sum(
698
+ topk_mask_extract[:, None]
699
+ * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
700
+ axis=0,
701
+ )
702
+ final_idx = tl.sum(
703
+ topk_mask_extract[:, None]
704
+ * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
705
+ axis=0,
706
+ )
707
+
708
+ # Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry
709
+ # -1e30 / 0 sentinels and lose to real scores in the merge stage.
710
+ ts_ptrs = (
711
+ ts_partial_ptr
712
+ + pid_chunk * stride_ts_c
713
+ + pid_b * stride_ts_b
714
+ + pid_h * stride_ts_h
715
+ + off_t * stride_ts_t
716
+ )
717
+ ti_ptrs = (
718
+ ti_partial_ptr
719
+ + pid_chunk * stride_ti_c
720
+ + pid_b * stride_ti_b
721
+ + pid_h * stride_ti_h
722
+ + off_t * stride_ti_t
723
+ )
724
+ tl.store(ts_ptrs, final_score)
725
+ tl.store(ti_ptrs, final_idx)
726
+
727
+
728
+ @triton.heuristics(
729
+ {
730
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
731
+ "BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
732
+ args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"])
733
+ ),
734
+ }
735
+ )
736
+ @triton.jit
737
+ def _topk_index_merge_kernel(
738
+ ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
739
+ ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape
740
+ ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk]
741
+ seq_lens,
742
+ block_size: tl.constexpr,
743
+ topk: tl.constexpr,
744
+ # strides
745
+ stride_ts_c,
746
+ stride_ts_h,
747
+ stride_ts_b,
748
+ stride_ts_t,
749
+ stride_ti_c,
750
+ stride_ti_h,
751
+ stride_ti_b,
752
+ stride_ti_t,
753
+ stride_tif_h,
754
+ stride_tif_b,
755
+ stride_tif_t,
756
+ # META
757
+ NUM_TOPK_CHUNKS: tl.constexpr,
758
+ BLOCK_SIZE_K: tl.constexpr,
759
+ BLOCK_SIZE_T: tl.constexpr,
760
+ ):
761
+ pid_b = tl.program_id(0)
762
+ pid_h = tl.program_id(1)
763
+
764
+ seq_len = tl.load(seq_lens + pid_b)
765
+ num_blocks = (seq_len + block_size - 1) // block_size
766
+
767
+ # Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K.
768
+ # candidate at flat position p comes from chunk = p // BLOCK_SIZE_T,
769
+ # in_chunk = p % BLOCK_SIZE_T.
770
+ off = tl.arange(0, BLOCK_SIZE_K)
771
+ chunk_idx = off // BLOCK_SIZE_T
772
+ in_chunk_idx = off % BLOCK_SIZE_T
773
+ valid = chunk_idx < NUM_TOPK_CHUNKS
774
+
775
+ score_offset = (
776
+ chunk_idx * stride_ts_c
777
+ + pid_h * stride_ts_h
778
+ + pid_b * stride_ts_b
779
+ + in_chunk_idx * stride_ts_t
780
+ )
781
+ idx_offset = (
782
+ chunk_idx * stride_ti_c
783
+ + pid_h * stride_ti_h
784
+ + pid_b * stride_ti_b
785
+ + in_chunk_idx * stride_ti_t
786
+ )
787
+
788
+ score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
789
+ tl.float32
790
+ )
791
+ score = tl.where(score != score, -1e30, score)
792
+ idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
793
+
794
+ # Full bitonic descending sort of BLOCK_SIZE_K items.
795
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
796
+ for j in tl.static_range(1, n_dims):
797
+ score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
798
+ score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
799
+
800
+ # Extract first BLOCK_SIZE_T positions — these are the global top-K.
801
+ extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
802
+ topk_idx_final = tl.sum(
803
+ extract_mask[:, None]
804
+ * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
805
+ axis=0,
806
+ )
807
+
808
+ off_t = tl.arange(0, BLOCK_SIZE_T)
809
+ tif_ptrs = (
810
+ ti_final_ptr
811
+ + pid_h * stride_tif_h
812
+ + pid_b * stride_tif_b
813
+ + off_t * stride_tif_t
814
+ )
815
+ topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
816
+ tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty))
817
+
818
+
819
+ @torch.no_grad()
820
+ def flash_decode_with_topk_idx(
821
+ q: torch.Tensor, # [batch_size, num_heads, head_dim]
822
+ sink: Optional[torch.Tensor],
823
+ k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
824
+ v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True
825
+ req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
826
+ seq_lens: torch.Tensor, # [batch_size, ]
827
+ max_seqlen: int,
828
+ slot_ids: torch.Tensor, # [batch_size, ]
829
+ block_size: int,
830
+ topk: int,
831
+ init_blocks: int,
832
+ local_blocks: int,
833
+ sm_scale: Optional[float] = None,
834
+ use_tma: bool = True,
835
+ score_type: str = "max",
836
+ disable_index_value: bool = False,
837
+ use_dense_main_attn: bool = False, # NOTE: need transform idx in this case
838
+ page_size: int = 1,
839
+ ) -> torch.Tensor:
840
+ assert score_type in (
841
+ "max",
842
+ "lse",
843
+ ), f"score_type must be 'max' or 'lse', got {score_type!r}"
844
+ triton.set_allocator(robust_allocator)
845
+ # dtype check
846
+ assert (
847
+ q.dtype == torch.bfloat16
848
+ or q.dtype == torch.float16
849
+ and k_cache.dtype == q.dtype
850
+ )
851
+ if not disable_index_value:
852
+ assert v_cache is not None
853
+ # shape
854
+ batch_size, num_q_heads, head_dim = q.shape
855
+ max_slots, num_kv_heads, _ = k_cache.shape
856
+ max_kv_len = req_to_token.shape[1]
857
+ assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
858
+ # gqa
859
+ assert num_q_heads % num_kv_heads == 0
860
+ gqa_group_size = num_q_heads // num_kv_heads
861
+ # sm scale
862
+ if sm_scale is None:
863
+ sm_scale = head_dim**-0.5
864
+ # NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets.
865
+ # Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads.
866
+ # TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by:
867
+ # NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS)
868
+ # then rounded down to the nearest power of 2 (merge kernel requires it).
869
+ # Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len.
870
+ # Empty chunks early-return cheaply, so over-chunking is nearly free.
871
+ # E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096.
872
+ TARGET_GRID = 4096
873
+ MAX_NUM_KV_CHUNKS = 256
874
+ target = max(
875
+ 1,
876
+ min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)),
877
+ )
878
+ NUM_KV_CHUNKS = 1 << (target.bit_length() - 1)
879
+ score_kv_len = min(max_seqlen, max_kv_len)
880
+ # The score producers below write every valid block column
881
+ # [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers
882
+ # clamp their scan to the same per-row valid block count, so columns beyond
883
+ # seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph
884
+ # capture the static score shape can be much larger than the live context.
885
+ score = torch.empty(
886
+ (num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)),
887
+ dtype=torch.float32,
888
+ device=q.device,
889
+ )
890
+ use_jit_topk = (
891
+ envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get()
892
+ and score.shape[2] <= 4096
893
+ and topk <= 32
894
+ )
895
+ # If the live context has <= topk sparse blocks, the downstream dense
896
+ # page-table/JIT top-k kernels select every block from seq_lens directly
897
+ # without reading score. Keep this gate in sync with the consumers below:
898
+ # _skip_block_topk/use_dense_main_attn and use_jit_topk special-case
899
+ # num_blocks <= topk, while the Triton fallback reads score and must not
900
+ # skip these writes.
901
+ skip_trivial_topk_score = use_dense_main_attn or use_jit_topk
902
+
903
+ grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads)
904
+ if disable_index_value:
905
+ _decode_score_kernel[grid](
906
+ q,
907
+ k_cache,
908
+ req_to_token,
909
+ score,
910
+ seq_lens,
911
+ slot_ids,
912
+ max_slots,
913
+ batch_size,
914
+ gqa_group_size,
915
+ head_dim,
916
+ block_size,
917
+ topk,
918
+ sm_scale,
919
+ init_blocks,
920
+ local_blocks,
921
+ q.stride(0),
922
+ q.stride(1),
923
+ q.stride(2),
924
+ k_cache.stride(0),
925
+ k_cache.stride(1),
926
+ k_cache.stride(2),
927
+ req_to_token.stride(0),
928
+ score.stride(0),
929
+ score.stride(1),
930
+ score.stride(2),
931
+ NUM_KV_CHUNKS=NUM_KV_CHUNKS,
932
+ SCORE_TYPE=score_type,
933
+ SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
934
+ )
935
+ else:
936
+ assert v_cache is not None
937
+ o = torch.empty(
938
+ NUM_KV_CHUNKS,
939
+ batch_size,
940
+ num_q_heads,
941
+ head_dim,
942
+ dtype=q.dtype,
943
+ device=q.device,
944
+ )
945
+ lse = torch.empty(
946
+ NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device
947
+ )
948
+ _decode_score_attn_kernel[grid](
949
+ q,
950
+ sink,
951
+ k_cache,
952
+ v_cache,
953
+ req_to_token,
954
+ o,
955
+ lse,
956
+ score,
957
+ seq_lens,
958
+ slot_ids,
959
+ max_slots,
960
+ batch_size,
961
+ gqa_group_size,
962
+ head_dim,
963
+ block_size,
964
+ topk,
965
+ sm_scale,
966
+ init_blocks,
967
+ local_blocks,
968
+ q.stride(0),
969
+ q.stride(1),
970
+ q.stride(2),
971
+ sink.stride(0) if sink is not None else 0,
972
+ sink.stride(1) if sink is not None else 0,
973
+ k_cache.stride(0),
974
+ k_cache.stride(1),
975
+ k_cache.stride(2),
976
+ v_cache.stride(0),
977
+ v_cache.stride(1),
978
+ v_cache.stride(2),
979
+ req_to_token.stride(0),
980
+ o.stride(0),
981
+ o.stride(1),
982
+ o.stride(2),
983
+ o.stride(3),
984
+ lse.stride(0),
985
+ lse.stride(1),
986
+ lse.stride(2),
987
+ score.stride(0),
988
+ score.stride(1),
989
+ score.stride(2),
990
+ NUM_KV_CHUNKS=NUM_KV_CHUNKS,
991
+ SCORE_TYPE=score_type,
992
+ SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
993
+ )
994
+ # Fused top-k + page-table transform: emit the dense backend's page table
995
+ # directly (page-size-aware) instead of block ids, skipping a separate gather.
996
+ # The page table + per-query effective KV length are allocated and returned.
997
+ real_seq_lens = None
998
+ if use_dense_main_attn:
999
+ # VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel
1000
+ # `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The
1001
+ # standalone vendor only ships the pure-Triton path, so this is gated off
1002
+ # by default (use_dense_main_attn=False). If you opt in, vendor that .cuh
1003
+ # too. Original line: `from sglang.jit_kernel.minimax_decode_topk import (
1004
+ # minimax_decode_topk_page_table)`.
1005
+ raise NotImplementedError(
1006
+ "use_dense_main_attn=True requires the un-vendored CUDA JIT kernel "
1007
+ "sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; "
1008
+ "the standalone Triton vendor only supports use_dense_main_attn=False."
1009
+ )
1010
+ # num_q_heads (= num index/kv heads) may be > 1 for DP attention; the
1011
+ # kernel emits a flattened [batch*num_q_heads] page table + seq_lens with
1012
+ # the kv head head-encoded (head-minor) into the page index.
1013
+ topk_idx, real_seq_lens = minimax_decode_topk_page_table(
1014
+ score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size
1015
+ )
1016
+ if disable_index_value:
1017
+ return None, topk_idx, real_seq_lens
1018
+ # fall through to the idx_o (index value) merge, then return
1019
+ _skip_block_topk = True
1020
+ else:
1021
+ _skip_block_topk = False
1022
+
1023
+ # get topk index
1024
+ if not _skip_block_topk:
1025
+ topk_idx = torch.empty(
1026
+ (num_q_heads, batch_size, topk),
1027
+ device=score.device,
1028
+ dtype=torch.int32,
1029
+ )
1030
+
1031
+ if _skip_block_topk:
1032
+ pass
1033
+ elif use_jit_topk:
1034
+ # Single-stage JIT radix-select: one kernel, no intermediate buffers.
1035
+ # Equivalent output to the 2-stage path (set of block ids, front-packed,
1036
+ # -1 padded); ~2-16x faster for long context.
1037
+ # VENDOR NOTE: this needs the un-vendored CUDA JIT kernel
1038
+ # `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the
1039
+ # `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so
1040
+ # this branch is never taken in the standalone vendor (the 2-stage Triton
1041
+ # `else` fallback below runs instead). Original line:
1042
+ # `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`.
1043
+ raise NotImplementedError(
1044
+ "use_jit_topk requires the un-vendored CUDA JIT kernel "
1045
+ "sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor "
1046
+ "uses the 2-stage Triton fallback (keep "
1047
+ "SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)."
1048
+ )
1049
+ elif has_native_ops() and topk <= 64:
1050
+ topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk)
1051
+ else:
1052
+ # 2-stage split-K Triton fallback.
1053
+ # Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim.
1054
+ # Same constraints as flash_decode: must be deterministic from values
1055
+ # constant within a cuda graph (BS, num_q_heads), pow2, capped so the
1056
+ # merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk))
1057
+ # stays reasonable. With topk=32 and cap 16, merge sorts 512 items.
1058
+ TOPK_TARGET_GRID = 64
1059
+ MAX_NUM_TOPK_CHUNKS = 16
1060
+ topk_target = max(
1061
+ 1,
1062
+ min(
1063
+ MAX_NUM_TOPK_CHUNKS,
1064
+ TOPK_TARGET_GRID // max(1, batch_size * num_q_heads),
1065
+ ),
1066
+ )
1067
+ NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1)
1068
+ BLOCK_SIZE_T = triton.next_power_of_2(topk)
1069
+ max_seqblock = score.shape[2]
1070
+ chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
1071
+ topk_score_partial = torch.empty(
1072
+ NUM_TOPK_CHUNKS,
1073
+ num_q_heads,
1074
+ batch_size,
1075
+ BLOCK_SIZE_T,
1076
+ dtype=torch.float32,
1077
+ device=score.device,
1078
+ )
1079
+ topk_idx_partial = torch.empty(
1080
+ NUM_TOPK_CHUNKS,
1081
+ num_q_heads,
1082
+ batch_size,
1083
+ BLOCK_SIZE_T,
1084
+ dtype=torch.int32,
1085
+ device=score.device,
1086
+ )
1087
+ # stream 0 (default): topk partial → topk merge
1088
+ grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS)
1089
+ _topk_index_partial_kernel[grid](
1090
+ score,
1091
+ topk_score_partial,
1092
+ topk_idx_partial,
1093
+ seq_lens,
1094
+ block_size,
1095
+ topk,
1096
+ chunk_blocks,
1097
+ score.stride(0),
1098
+ score.stride(1),
1099
+ score.stride(2),
1100
+ topk_score_partial.stride(0),
1101
+ topk_score_partial.stride(1),
1102
+ topk_score_partial.stride(2),
1103
+ topk_score_partial.stride(3),
1104
+ topk_idx_partial.stride(0),
1105
+ topk_idx_partial.stride(1),
1106
+ topk_idx_partial.stride(2),
1107
+ topk_idx_partial.stride(3),
1108
+ )
1109
+ grid = (batch_size, num_q_heads)
1110
+ _topk_index_merge_kernel[grid](
1111
+ topk_score_partial,
1112
+ topk_idx_partial,
1113
+ topk_idx,
1114
+ seq_lens,
1115
+ block_size,
1116
+ topk,
1117
+ topk_score_partial.stride(0),
1118
+ topk_score_partial.stride(1),
1119
+ topk_score_partial.stride(2),
1120
+ topk_score_partial.stride(3),
1121
+ topk_idx_partial.stride(0),
1122
+ topk_idx_partial.stride(1),
1123
+ topk_idx_partial.stride(2),
1124
+ topk_idx_partial.stride(3),
1125
+ topk_idx.stride(0),
1126
+ topk_idx.stride(1),
1127
+ topk_idx.stride(2),
1128
+ NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
1129
+ )
1130
+ if disable_index_value:
1131
+ return None, topk_idx, real_seq_lens
1132
+ # attn output merge (default stream)
1133
+ grid = (batch_size, num_q_heads)
1134
+ _merge_attn_out_kernel[grid](
1135
+ o,
1136
+ lse,
1137
+ seq_lens,
1138
+ head_dim,
1139
+ block_size,
1140
+ o.stride(0),
1141
+ o.stride(1),
1142
+ o.stride(2),
1143
+ o.stride(3),
1144
+ lse.stride(0),
1145
+ lse.stride(1),
1146
+ lse.stride(2),
1147
+ NUM_KV_CHUNKS=NUM_KV_CHUNKS,
1148
+ )
1149
+ o = o[0].contiguous()
1150
+ return o, topk_idx, real_seq_lens
build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py ADDED
@@ -0,0 +1,452 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
10
+
11
+ from .._native import (
12
+ native_decode_mma_supported,
13
+ native_decode_sparse_attn_mma_paged,
14
+ )
15
+ from ..common.utils import robust_allocator
16
+
17
+ _is_hip = is_hip()
18
+
19
+
20
+ @triton.heuristics(
21
+ {
22
+ "BLOCK_SIZE_H": lambda args: max(
23
+ 16, triton.next_power_of_2(args["gqa_group_size"])
24
+ ),
25
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
26
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
27
+ "HAS_SINK": lambda args: args["sink_ptr"] is not None,
28
+ "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
29
+ }
30
+ )
31
+ @triton.autotune(
32
+ configs=[
33
+ triton.Config({}, num_warps=nw, num_stages=ns)
34
+ for nw in [4, 8]
35
+ for ns in [2, 3, 4, 5]
36
+ ],
37
+ key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"],
38
+ )
39
+ @triton.jit
40
+ def _gqa_share_sparse_decode_kernel(
41
+ q_ptr, # Q: b x qh x d
42
+ sink_ptr, # Sink: qh x d
43
+ k_cache_ptr, # K paged: max_slots x kh x d
44
+ v_cache_ptr, # V paged: max_slots x kh x d
45
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
46
+ idx_ptr, # topk index: qh x b x topk
47
+ o_ptr, # O partial: c x b x qh x d
48
+ lse_ptr, # lse partial: c x b x qh
49
+ seq_lens,
50
+ slot_ids,
51
+ # shape
52
+ max_slots,
53
+ batch_size,
54
+ gqa_group_size,
55
+ head_dim,
56
+ max_topk,
57
+ max_kv_len,
58
+ # sm_scale
59
+ sm_scale,
60
+ # stride
61
+ stride_q_b,
62
+ stride_q_h,
63
+ stride_q_d,
64
+ stride_sink_h,
65
+ stride_sink_d,
66
+ stride_k_s,
67
+ stride_k_h,
68
+ stride_k_d,
69
+ stride_v_s,
70
+ stride_v_h,
71
+ stride_v_d,
72
+ stride_r2t_b,
73
+ stride_ti_h,
74
+ stride_ti_b,
75
+ stride_ti_t,
76
+ stride_o_c,
77
+ stride_o_b,
78
+ stride_o_h,
79
+ stride_o_d,
80
+ stride_l_c,
81
+ stride_l_b,
82
+ stride_l_h,
83
+ # META parameters
84
+ BATCH_SIZE_BUCKET: tl.constexpr,
85
+ BLOCK_SIZE_H: tl.constexpr,
86
+ BLOCK_SIZE_N: tl.constexpr,
87
+ BLOCK_SIZE_D: tl.constexpr,
88
+ BLOCK_SIZE_T: tl.constexpr,
89
+ NUM_TOPK_CHUNKS: tl.constexpr,
90
+ HAS_SINK: tl.constexpr,
91
+ IS_FP8: tl.constexpr,
92
+ ):
93
+ # decode program ids: split-K over the topk dimension to give every SM
94
+ # something to do at small batch. pid(0) folds (batch, chunk) together so
95
+ # the grid size = batch_size * NUM_TOPK_CHUNKS.
96
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
97
+ pid_b = pid_bc % batch_size
98
+ pid_c = pid_bc // batch_size
99
+ pid_h = pid_kh * gqa_group_size
100
+ # per-chunk topk range. chunk_size is *runtime* (depends on max_topk which
101
+ # is a runtime arg, not constexpr), so don't annotate as tl.constexpr —
102
+ # doing so produces undefined behavior in Triton.
103
+ chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
104
+ chunk_start_topk = pid_c * chunk_size_topk
105
+ chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk
106
+ # get q k start and len after rmpad
107
+ seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len)
108
+ sid = (
109
+ tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
110
+ ) % max_slots # to avoid bugs when slot_ids is negative
111
+ # get real topk
112
+ off_t = tl.arange(0, BLOCK_SIZE_T)
113
+ idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b
114
+ topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1)
115
+ valid_idx = tl.where(topk_idx >= 0, off_t, -1)
116
+ real_topk = tl.sum(valid_idx != -1, axis=0)
117
+ chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk)
118
+ # init pointer
119
+ off_n = tl.arange(0, BLOCK_SIZE_N)
120
+ off_d = tl.arange(0, BLOCK_SIZE_D)
121
+ dim_mask = off_d < head_dim
122
+ # init statistics — kept at -inf so empty chunks (chunk_start >= real_topk)
123
+ # naturally fall out as weight=0 in the merge step.
124
+ # q is always needed; load it unconditionally.
125
+ q_ptrs = tl.make_block_ptr(
126
+ base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
127
+ shape=(gqa_group_size, head_dim),
128
+ strides=(stride_q_h, stride_q_d),
129
+ offsets=(0, 0),
130
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
131
+ order=(1, 0),
132
+ )
133
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
134
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
135
+ lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
136
+ # Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code-
137
+ # eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is
138
+ # then None). The original compound `if HAS_SINK and pid_c == 0` traced
139
+ # the body even when False -> NoneType make_block_ptr compile error on
140
+ # sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always
141
+ # False for us, but this keeps the sink path correct too.
142
+ if HAS_SINK:
143
+ if pid_c == 0:
144
+ sink_ptrs = tl.make_block_ptr(
145
+ base=sink_ptr + pid_h * stride_sink_h,
146
+ shape=(gqa_group_size, head_dim),
147
+ strides=(stride_sink_h, stride_sink_d),
148
+ offsets=(0, 0),
149
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
150
+ order=(1, 0),
151
+ )
152
+ sink = tl.load(
153
+ sink_ptrs, boundary_check=(0, 1), padding_option="zero"
154
+ ).to(tl.float32)
155
+ qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale
156
+ m_i = qsink
157
+ lse_i = qsink
158
+ acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
159
+ # only iterate over this chunk's topk slice. the load must respect the
160
+ # per-chunk start offset.
161
+ cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t
162
+ for _ in tl.range(chunk_start_topk, chunk_end_topk):
163
+ # load index
164
+ c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N
165
+ cur_idx_ptr = cur_idx_ptr + stride_ti_t
166
+ # resolve slots for this block via req_to_token
167
+ pos = c + off_n
168
+ pos_mask = pos < seq_len
169
+ slots = tl.load(
170
+ req_to_token_ptr + sid * stride_r2t_b + pos,
171
+ mask=pos_mask,
172
+ other=0,
173
+ ).to(tl.int64)
174
+ slots = (slots + max_slots) % max_slots # safety against negative
175
+ # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
176
+ k_off = (
177
+ slots[None, :] * stride_k_s
178
+ + pid_kh * stride_k_h
179
+ + off_d[:, None] * stride_k_d
180
+ )
181
+ k = tl.load(
182
+ k_cache_ptr + k_off,
183
+ mask=dim_mask[:, None] & pos_mask[None, :],
184
+ other=0.0,
185
+ )
186
+ if IS_FP8:
187
+ # fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no
188
+ # scale), so dequant is just a widening cast to the Q compute dtype
189
+ # before the tl.dot. Matches the bf16 path bit-for-bit when the cache
190
+ # is bf16 (IS_FP8 False -> this branch is compiled out).
191
+ k = k.to(q.dtype)
192
+ # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
193
+ v_off = (
194
+ slots[:, None] * stride_v_s
195
+ + pid_kh * stride_v_h
196
+ + off_d[None, :] * stride_v_d
197
+ )
198
+ v = tl.load(
199
+ v_cache_ptr + v_off,
200
+ mask=pos_mask[:, None] & dim_mask[None, :],
201
+ other=0.0,
202
+ )
203
+ if IS_FP8:
204
+ # Widen V before the P@V dot. This also makes the `p.to(v.dtype)`
205
+ # below cast P to the compute dtype (not to fp8, which would be
206
+ # catastrophic precision loss on the attention weights).
207
+ v = v.to(q.dtype)
208
+ # compute qk
209
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
210
+ qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf"))
211
+ # [H, D], [D, N] -> [H, N]
212
+ qk += tl.dot(q, k) * sm_scale
213
+ # compute m_ij and l_ij
214
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
215
+ p = tl.exp(qk - m_ij[:, None])
216
+ l_ij = tl.sum(p, axis=1)
217
+ # scale acc_o
218
+ acc_o_scale = tl.exp(m_i - m_ij)
219
+ acc_o = acc_o * acc_o_scale[:, None]
220
+ # load v and update acc_o
221
+ p = p.to(v.dtype)
222
+ # [H, N], [N, D] -> [H, D]
223
+ acc_o += tl.dot(p.to(v.dtype), v)
224
+ # update statistics
225
+ m_i = m_ij
226
+ lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij)
227
+ # final scale (matches the old non-split kernel for chunks where lse_i>-inf).
228
+ # For empty chunks (chunk_start_topk >= real_topk) the inner loop never
229
+ # runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute
230
+ # exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial
231
+ # and the merge result. Gate the scale with tl.where so empty chunks emit a
232
+ # clean zero (lse_i stays -inf which the merge correctly turns into weight=0).
233
+ scale = tl.where(
234
+ lse_i > float("-inf"),
235
+ tl.exp(m_i - lse_i),
236
+ tl.zeros_like(lse_i),
237
+ )
238
+ acc_o = acc_o * scale[:, None]
239
+ # save partial output and lse for the merge step
240
+ o_ptrs = tl.make_block_ptr(
241
+ base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
242
+ shape=(gqa_group_size, head_dim),
243
+ strides=(stride_o_h, stride_o_d),
244
+ offsets=(0, 0),
245
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
246
+ order=(1, 0),
247
+ )
248
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
249
+ lse_ptrs = tl.make_block_ptr(
250
+ base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
251
+ shape=(gqa_group_size,),
252
+ strides=(stride_l_h,),
253
+ offsets=(0,),
254
+ block_shape=(BLOCK_SIZE_H,),
255
+ order=(0,),
256
+ )
257
+ tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
258
+
259
+
260
+ @triton.heuristics(
261
+ {
262
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
263
+ }
264
+ )
265
+ @triton.jit
266
+ def _merge_topk_attn_out_kernel(
267
+ o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0
268
+ lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH]
269
+ head_dim,
270
+ stride_o_c,
271
+ stride_o_b,
272
+ stride_o_h,
273
+ stride_o_d,
274
+ stride_l_c,
275
+ stride_l_b,
276
+ stride_l_h,
277
+ NUM_TOPK_CHUNKS: tl.constexpr,
278
+ BLOCK_SIZE_D: tl.constexpr,
279
+ ):
280
+ pid_b, pid_h = tl.program_id(0), tl.program_id(1)
281
+ off_c = tl.arange(0, NUM_TOPK_CHUNKS)
282
+ off_d = tl.arange(0, BLOCK_SIZE_D)
283
+ o_ptrs = tl.make_block_ptr(
284
+ base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
285
+ shape=(NUM_TOPK_CHUNKS, head_dim),
286
+ strides=(stride_o_c, stride_o_d),
287
+ offsets=(0, 0),
288
+ block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
289
+ order=(1, 0),
290
+ )
291
+ lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
292
+ o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
293
+ lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
294
+ # standard flash-decoding merge in linear (not log2) space, matching the
295
+ # decode kernel which uses tl.exp / tl.log.
296
+ lse_max = tl.max(lse, axis=0)
297
+ weights = tl.exp(lse - lse_max)
298
+ weights = weights / tl.sum(weights, axis=0)
299
+ o_merged = tl.sum(o * weights[:, None], axis=0)
300
+ o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
301
+ tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
302
+
303
+
304
+ @torch.no_grad()
305
+ def flash_decode_with_gqa_share_sparse(
306
+ q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
307
+ sink: Optional[torch.Tensor],
308
+ k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
309
+ v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
310
+ req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
311
+ seq_lens: torch.Tensor, # [batch_size, ]
312
+ slot_ids: torch.Tensor, # [batch_size, ]
313
+ block_size: int,
314
+ topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
315
+ sm_scale: Optional[float] = None,
316
+ use_tma: bool = True,
317
+ ) -> torch.Tensor:
318
+ triton.set_allocator(robust_allocator)
319
+ # dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may
320
+ # be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel
321
+ # widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing
322
+ # sparse decode dtype contract unchanged by accepting fp8 only on HIP.
323
+ assert q.dtype in (torch.bfloat16, torch.float16)
324
+ _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
325
+ is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
326
+ assert k_cache.dtype == q.dtype or is_fp8, (
327
+ f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) "
328
+ f"or fp8 on HIP, got {k_cache.dtype}"
329
+ )
330
+ assert v_cache.dtype == k_cache.dtype
331
+ # shape
332
+ batch_size, num_q_heads, head_dim = q.shape
333
+ max_slots, num_kv_heads, _ = k_cache.shape
334
+ assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
335
+ assert topk_idx.shape[0] == num_kv_heads
336
+ assert (
337
+ triton.next_power_of_2(block_size) == block_size
338
+ ), f"block_size must be a power of 2, but got {block_size}"
339
+ # assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}"
340
+ max_kv_len = req_to_token.shape[1]
341
+ # gqa
342
+ assert num_q_heads % num_kv_heads == 0
343
+ gqa_group_size = num_q_heads // num_kv_heads
344
+ max_topk = topk_idx.shape[2]
345
+ # sm scale
346
+ if sm_scale is None:
347
+ sm_scale = head_dim**-0.5
348
+ # Fast path: native tensor-core decode for the M3 MSA shape. Falls through
349
+ # to the Triton kernel for other shapes, sink, or fp8 caches.
350
+ if sink is None and not is_fp8 and native_decode_mma_supported(
351
+ q, k_cache, block_size=block_size
352
+ ):
353
+ return native_decode_sparse_attn_mma_paged(
354
+ q,
355
+ k_cache,
356
+ v_cache,
357
+ req_to_token,
358
+ seq_lens,
359
+ slot_ids,
360
+ block_size,
361
+ topk_idx,
362
+ sm_scale,
363
+ )
364
+ # Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as
365
+ # flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must
366
+ # only depend on shape constants (so grid is fixed within a cuda graph).
367
+ # Capped by max_topk because chunks beyond real_topk early-fall-through to
368
+ # the merge-as-zero path; capping avoids wasting blocks at tiny topk.
369
+ TARGET_GRID = 256
370
+ target = max(
371
+ 1,
372
+ min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)),
373
+ )
374
+ NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1)
375
+ # output tensor: split-K partials, merged into chunk 0 by the merge kernel
376
+ o_partial = torch.empty(
377
+ NUM_TOPK_CHUNKS,
378
+ batch_size,
379
+ num_q_heads,
380
+ head_dim,
381
+ dtype=q.dtype,
382
+ device=q.device,
383
+ )
384
+ lse_partial = torch.empty(
385
+ NUM_TOPK_CHUNKS,
386
+ batch_size,
387
+ num_q_heads,
388
+ dtype=torch.float32,
389
+ device=q.device,
390
+ )
391
+ # launch attention kernel
392
+ grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads)
393
+ _gqa_share_sparse_decode_kernel[grid](
394
+ q,
395
+ sink,
396
+ k_cache,
397
+ v_cache,
398
+ req_to_token,
399
+ topk_idx,
400
+ o_partial,
401
+ lse_partial,
402
+ seq_lens,
403
+ slot_ids,
404
+ max_slots,
405
+ batch_size,
406
+ gqa_group_size,
407
+ head_dim,
408
+ max_topk,
409
+ max_kv_len,
410
+ sm_scale,
411
+ q.stride(0),
412
+ q.stride(1),
413
+ q.stride(2),
414
+ sink.stride(0) if sink is not None else 0,
415
+ sink.stride(1) if sink is not None else 0,
416
+ k_cache.stride(0),
417
+ k_cache.stride(1),
418
+ k_cache.stride(2),
419
+ v_cache.stride(0),
420
+ v_cache.stride(1),
421
+ v_cache.stride(2),
422
+ req_to_token.stride(0),
423
+ topk_idx.stride(0),
424
+ topk_idx.stride(1),
425
+ topk_idx.stride(2),
426
+ o_partial.stride(0),
427
+ o_partial.stride(1),
428
+ o_partial.stride(2),
429
+ o_partial.stride(3),
430
+ lse_partial.stride(0),
431
+ lse_partial.stride(1),
432
+ lse_partial.stride(2),
433
+ BLOCK_SIZE_N=block_size,
434
+ NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
435
+ IS_FP8=is_fp8,
436
+ )
437
+ # merge partials into chunk 0
438
+ merge_grid = (batch_size, num_q_heads)
439
+ _merge_topk_attn_out_kernel[merge_grid](
440
+ o_partial,
441
+ lse_partial,
442
+ head_dim,
443
+ o_partial.stride(0),
444
+ o_partial.stride(1),
445
+ o_partial.stride(2),
446
+ o_partial.stride(3),
447
+ lse_partial.stride(0),
448
+ lse_partial.stride(1),
449
+ lse_partial.stride(2),
450
+ NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
451
+ )
452
+ return o_partial[0].contiguous()
build/torch211-cxx11-cu128-x86_64-linux/interface.py ADDED
@@ -0,0 +1,679 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from ._native import (
11
+ has_native_ops,
12
+ native_indexer_block_scores,
13
+ native_nvfp4_dequant_swizzled_to_bf16,
14
+ )
15
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
16
+ from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse
17
+ from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16
18
+
19
+
20
+ def _page_table_to_req_to_token(
21
+ page_table: torch.Tensor,
22
+ seqused_k: torch.Tensor,
23
+ max_seqlen_k: int,
24
+ blk_kv: int,
25
+ ) -> torch.Tensor:
26
+ batch = int(page_table.shape[0])
27
+ max_seqlen_k = int(max_seqlen_k)
28
+ blk_kv = int(blk_kv)
29
+ logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
30
+ page = logical_pos // blk_kv
31
+ offset = logical_pos % blk_kv
32
+ page = page.clamp(max=int(page_table.shape[1]) - 1)
33
+ physical_page = page_table[:, page].to(torch.int64)
34
+ physical_token = physical_page * blk_kv + offset[None, :]
35
+ valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
36
+ return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
37
+
38
+
39
+ def _dense_q2k_indices(
40
+ *,
41
+ head_kv: int,
42
+ batch: int,
43
+ topk: int,
44
+ seq_lens: torch.Tensor,
45
+ blk_kv: int,
46
+ ) -> torch.Tensor:
47
+ idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
48
+ blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
49
+ valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
50
+ for b in range(batch):
51
+ valid = blocks < valid_blocks[b]
52
+ idx[:, b, valid] = blocks[valid]
53
+ return idx
54
+
55
+
56
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]:
57
+ seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist()
58
+ return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens]
59
+
60
+
61
+ def _packed_row_to_batch_block(
62
+ cu_seqlens_k: torch.Tensor,
63
+ blk_kv: int,
64
+ ) -> list[tuple[int, int]]:
65
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv))
66
+ max_rows = max(rows_per_batch, default=0)
67
+ row_map: list[tuple[int, int]] = []
68
+ for block_idx in range(max_rows):
69
+ for batch_idx, rows in enumerate(rows_per_batch):
70
+ if block_idx < rows:
71
+ row_map.append((batch_idx, block_idx))
72
+ return row_map
73
+
74
+
75
+ def _csr_to_q2k_indices(
76
+ k2q_row_ptr: torch.Tensor,
77
+ k2q_q_indices: torch.Tensor,
78
+ cu_seqlens_q: torch.Tensor,
79
+ cu_seqlens_k: torch.Tensor,
80
+ blk_kv: int,
81
+ topk: int,
82
+ ) -> torch.Tensor:
83
+ if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32:
84
+ raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32")
85
+ if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2:
86
+ raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2")
87
+ head_kv = int(k2q_row_ptr.shape[0])
88
+ total_q = int(cu_seqlens_q[-1].item())
89
+ topk = int(topk)
90
+ if topk <= 0:
91
+ raise ValueError(f"topK must be positive, got {topk}")
92
+
93
+ row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv))
94
+ if int(k2q_row_ptr.shape[1]) != len(row_map) + 1:
95
+ raise ValueError(
96
+ "k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: "
97
+ f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}"
98
+ )
99
+
100
+ cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
101
+ row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False)
102
+ q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False)
103
+ q2k = torch.full(
104
+ (head_kv, total_q, topk),
105
+ -1,
106
+ dtype=torch.int32,
107
+ device=k2q_row_ptr.device,
108
+ )
109
+
110
+ next_slot: dict[tuple[int, int], int] = {}
111
+ seen: set[tuple[int, int, int]] = set()
112
+ for h in range(head_kv):
113
+ for row, (batch_idx, kv_block_idx) in enumerate(row_map):
114
+ start = int(row_ptr_cpu[h, row].item())
115
+ end = int(row_ptr_cpu[h, row + 1].item())
116
+ q_start = int(cu_q_cpu[batch_idx])
117
+ q_end = int(cu_q_cpu[batch_idx + 1])
118
+ q_len = q_end - q_start
119
+ for pos in range(start, end):
120
+ q_val = int(q_idx_cpu[h, pos].item())
121
+ if q_val < 0:
122
+ continue
123
+ if q_start <= q_val < q_end:
124
+ q_global = q_val
125
+ elif q_val < q_len:
126
+ q_global = q_start + q_val
127
+ else:
128
+ continue
129
+ key = (h, q_global, kv_block_idx)
130
+ if key in seen:
131
+ continue
132
+ seen.add(key)
133
+ slot_key = (h, q_global)
134
+ slot = next_slot.get(slot_key, 0)
135
+ if slot >= topk:
136
+ continue
137
+ q2k[h, q_global, slot] = int(kv_block_idx)
138
+ next_slot[slot_key] = slot + 1
139
+ return q2k.contiguous()
140
+
141
+
142
+ def _make_prefill_cache_and_metadata(
143
+ q: torch.Tensor,
144
+ k: torch.Tensor,
145
+ v: torch.Tensor,
146
+ cu_seqlens_q: torch.Tensor,
147
+ cu_seqlens_k: torch.Tensor,
148
+ max_seqlen_k: int,
149
+ blk_kv: int,
150
+ *,
151
+ page_table: Optional[torch.Tensor],
152
+ seqused_k: Optional[torch.Tensor],
153
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
154
+ if page_table is None:
155
+ if k.ndim != 3 or v.ndim != 3:
156
+ raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]")
157
+ if k.shape != v.shape:
158
+ raise ValueError("k and v must have the same shape")
159
+ k_cache = k.contiguous()
160
+ v_cache = v.contiguous()
161
+ batch = int(cu_seqlens_q.numel()) - 1
162
+ max_seqlen_k = int(max_seqlen_k)
163
+ logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64)
164
+ cu_k = cu_seqlens_k.to(torch.int64)
165
+ seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous()
166
+ valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
167
+ req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32)
168
+ req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
169
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
170
+ else:
171
+ if k.ndim != 4 or v.ndim != 4:
172
+ raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]")
173
+ if k.shape != v.shape:
174
+ raise ValueError("k and v must have the same shape")
175
+ if int(k.shape[2]) != int(blk_kv):
176
+ raise ValueError("paged k/v page dimension must equal blk_kv")
177
+ if page_table.dtype != torch.int32:
178
+ raise TypeError("page_table must be torch.int32")
179
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
180
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
181
+ batch = int(page_table.shape[0])
182
+ if seqused_k is None:
183
+ seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous()
184
+ else:
185
+ seq_lens = seqused_k.to(torch.int32).contiguous()
186
+ logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64)
187
+ page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1)
188
+ offset = logical % int(blk_kv)
189
+ physical_page = page_table[:, page].to(torch.int64)
190
+ req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32)
191
+ valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
192
+ req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
193
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
194
+
195
+ q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32)
196
+ prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous()
197
+ return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens
198
+
199
+
200
+ def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int:
201
+ if kv_heads <= 0 or q_heads % kv_heads != 0:
202
+ raise ValueError("number of query heads must be divisible by KV heads")
203
+ gqa_group = q_heads // kv_heads
204
+ limit = max(1, 128 // max(1, gqa_group))
205
+ for candidate in (64, 32, 16, 8, 4, 2, 1):
206
+ if candidate <= limit:
207
+ return candidate
208
+ return 1
209
+
210
+
211
+ def _normalize_decode_q2k(
212
+ q2k_indices: Optional[torch.Tensor],
213
+ *,
214
+ batch: int,
215
+ seqlen_q: int,
216
+ head_kv: int,
217
+ seqused_k: torch.Tensor,
218
+ blk_kv: int,
219
+ ) -> torch.Tensor:
220
+ if q2k_indices is None:
221
+ topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
222
+ return _dense_q2k_indices(
223
+ head_kv=head_kv,
224
+ batch=batch,
225
+ topk=topk,
226
+ seq_lens=seqused_k,
227
+ blk_kv=blk_kv,
228
+ )
229
+ if q2k_indices.dtype != torch.int32:
230
+ raise TypeError("q2k_indices must be torch.int32")
231
+ if q2k_indices.ndim != 3:
232
+ raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
233
+ if q2k_indices.shape[0] != head_kv:
234
+ raise ValueError("q2k_indices first dimension must match Hkv")
235
+ if int(q2k_indices.shape[1]) != batch * seqlen_q:
236
+ raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
237
+ if seqlen_q != 1:
238
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
239
+ return q2k_indices[:, ::seqlen_q, :].contiguous()
240
+
241
+
242
+ def sparse_decode_atten_func(
243
+ q: torch.Tensor,
244
+ k: torch.Tensor,
245
+ v: torch.Tensor,
246
+ q2k_indices: Optional[torch.Tensor] = None,
247
+ *,
248
+ page_table: torch.Tensor,
249
+ seqused_k: torch.Tensor,
250
+ seqlen_q: int,
251
+ max_seqlen_k: int,
252
+ blk_kv: int = 128,
253
+ causal: bool = True,
254
+ softmax_scale: Optional[float] = None,
255
+ return_softmax_lse: bool = False,
256
+ schedule: Optional[object] = None,
257
+ O_partial: Optional[torch.Tensor] = None,
258
+ LSE_partial: Optional[torch.Tensor] = None,
259
+ ):
260
+ """Run MiniMax paged sparse decode on Blackwell.
261
+
262
+ This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
263
+ but accepts the Blackwell package's validated BF16/FP16 decode path. It
264
+ supports single-token decode (`seqlen_q=1`) with paged K/V
265
+ `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
266
+ """
267
+
268
+ if schedule is not None or O_partial is not None or LSE_partial is not None:
269
+ raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
270
+ if return_softmax_lse:
271
+ raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
272
+ if not causal:
273
+ raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
274
+ if int(seqlen_q) != 1:
275
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
276
+ if q.ndim != 3:
277
+ raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
278
+ if k.ndim != 4 or v.ndim != 4:
279
+ raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
280
+ if k.shape != v.shape:
281
+ raise ValueError("k and v must have the same shape")
282
+ if q.device != k.device or q.device != v.device or q.device != page_table.device:
283
+ raise ValueError("q, k, v, and page_table must be on the same device")
284
+ if q.dtype not in (torch.bfloat16, torch.float16):
285
+ raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
286
+ if k.dtype != q.dtype or v.dtype != q.dtype:
287
+ raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
288
+ if q.shape[-1] != 128 or k.shape[-1] != 128:
289
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
290
+ if int(k.shape[2]) != int(blk_kv):
291
+ raise ValueError("k.shape[2] must equal blk_kv")
292
+ if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
293
+ raise TypeError("page_table and seqused_k must be torch.int32")
294
+ if page_table.ndim != 2:
295
+ raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
296
+ batch = int(page_table.shape[0])
297
+ if int(q.shape[0]) != batch:
298
+ raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
299
+
300
+ head_kv = int(k.shape[1])
301
+ q2k = _normalize_decode_q2k(
302
+ q2k_indices,
303
+ batch=batch,
304
+ seqlen_q=int(seqlen_q),
305
+ head_kv=head_kv,
306
+ seqused_k=seqused_k,
307
+ blk_kv=int(blk_kv),
308
+ )
309
+ req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
310
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
311
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
312
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
313
+ return flash_decode_with_gqa_share_sparse(
314
+ q.contiguous(),
315
+ None,
316
+ k_cache,
317
+ v_cache,
318
+ req_to_token,
319
+ seqused_k.contiguous(),
320
+ slot_ids,
321
+ int(blk_kv),
322
+ q2k,
323
+ sm_scale=softmax_scale,
324
+ )
325
+
326
+
327
+ class SparseDecodePagedAttentionWrapper:
328
+ def __init__(self, *, blk_kv: int = 128, causal: bool = True):
329
+ self.blk_kv = int(blk_kv)
330
+ self.causal = bool(causal)
331
+ self.page_table: Optional[torch.Tensor] = None
332
+ self.seqused_k: Optional[torch.Tensor] = None
333
+ self.seqlen_q: Optional[int] = None
334
+ self.max_seqlen_k: Optional[int] = None
335
+ self.q2k_indices: Optional[torch.Tensor] = None
336
+
337
+ def plan(
338
+ self,
339
+ *,
340
+ page_table: torch.Tensor,
341
+ seqused_k: torch.Tensor,
342
+ seqlen_q: int,
343
+ max_seqlen_k: int,
344
+ q2k_indices: Optional[torch.Tensor] = None,
345
+ num_qo_heads: Optional[int] = None,
346
+ num_kv_heads: Optional[int] = None,
347
+ head_dim: Optional[int] = 128,
348
+ enable_cuda_graph: bool = False,
349
+ max_grid_size: Optional[int] = None,
350
+ fixed_split_size: Optional[int] = None,
351
+ disable_split_kv: bool = False,
352
+ ) -> "SparseDecodePagedAttentionWrapper":
353
+ if head_dim is not None and int(head_dim) != 128:
354
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
355
+ self.page_table = page_table.contiguous()
356
+ self.seqused_k = seqused_k.contiguous()
357
+ self.seqlen_q = int(seqlen_q)
358
+ self.max_seqlen_k = int(max_seqlen_k)
359
+ self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
360
+ return self
361
+
362
+ def run(
363
+ self,
364
+ q: torch.Tensor,
365
+ k: torch.Tensor,
366
+ v: torch.Tensor,
367
+ *,
368
+ softmax_scale: Optional[float] = None,
369
+ return_softmax_lse: bool = False,
370
+ out: Optional[torch.Tensor] = None,
371
+ lse: Optional[torch.Tensor] = None,
372
+ ):
373
+ if self.page_table is None or self.seqused_k is None:
374
+ raise RuntimeError("decode wrapper must be planned before run")
375
+ result = sparse_decode_atten_func(
376
+ q,
377
+ k,
378
+ v,
379
+ self.q2k_indices,
380
+ page_table=self.page_table,
381
+ seqused_k=self.seqused_k,
382
+ seqlen_q=self.seqlen_q,
383
+ max_seqlen_k=self.max_seqlen_k,
384
+ blk_kv=self.blk_kv,
385
+ causal=self.causal,
386
+ softmax_scale=softmax_scale,
387
+ return_softmax_lse=return_softmax_lse,
388
+ )
389
+ if out is not None:
390
+ out.copy_(result)
391
+ return out
392
+ return result
393
+
394
+
395
+ def sparse_atten_func(
396
+ q: torch.Tensor,
397
+ k: torch.Tensor,
398
+ v: torch.Tensor,
399
+ k2q_row_ptr: torch.Tensor,
400
+ k2q_q_indices: torch.Tensor,
401
+ topK: int,
402
+ *,
403
+ cu_seqlens_q: torch.Tensor,
404
+ cu_seqlens_k: torch.Tensor,
405
+ max_seqlen_q: int,
406
+ max_seqlen_k: int,
407
+ blk_kv: int = 128,
408
+ causal: bool = True,
409
+ softmax_scale: Optional[float] = None,
410
+ lse_temperature_scale: float = 1.0,
411
+ return_temperature_lse: bool = False,
412
+ partial_dtype: torch.dtype = torch.bfloat16,
413
+ return_softmax_lse: bool = False,
414
+ page_table: Optional[torch.Tensor] = None,
415
+ seqused_k: Optional[torch.Tensor] = None,
416
+ schedule: Optional[object] = None,
417
+ usable_SM_count: int = -1,
418
+ qk_dtype: Optional[torch.dtype] = None,
419
+ pv_dtype: Optional[torch.dtype] = None,
420
+ ):
421
+ """Run the official MiniMax CSR sparse prefill API on Blackwell.
422
+
423
+ The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list
424
+ consumed by the Blackwell Triton sparse prefill kernel. The default output
425
+ path is supported for BF16/FP16 tensors with head_dim=128.
426
+ """
427
+
428
+ del partial_dtype, usable_SM_count, qk_dtype, pv_dtype
429
+ if schedule is not None:
430
+ raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper")
431
+ if return_temperature_lse or return_softmax_lse:
432
+ raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper")
433
+ if float(lse_temperature_scale) != 1.0:
434
+ raise ValueError("lse_temperature_scale values other than 1.0 are not supported")
435
+ if not causal:
436
+ raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True")
437
+ if q.ndim != 3:
438
+ raise ValueError("q must have shape [total_q, Hq, D]")
439
+ if q.dtype not in (torch.bfloat16, torch.float16):
440
+ raise TypeError("q must be BF16 or FP16")
441
+ if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128:
442
+ raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128")
443
+ if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32:
444
+ raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32")
445
+ if q.device != k.device or q.device != v.device:
446
+ raise ValueError("q, k, and v must be on the same device")
447
+
448
+ if k.dtype != q.dtype or v.dtype != q.dtype:
449
+ raise TypeError("q, k, and v must have matching BF16/FP16 dtype")
450
+ kv_heads = int(k.shape[1])
451
+ q2k_indices = _csr_to_q2k_indices(
452
+ k2q_row_ptr.contiguous(),
453
+ k2q_q_indices.contiguous(),
454
+ cu_seqlens_q.contiguous(),
455
+ cu_seqlens_k.contiguous(),
456
+ int(blk_kv),
457
+ int(topK),
458
+ )
459
+ k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata(
460
+ q,
461
+ k,
462
+ v,
463
+ cu_seqlens_q.contiguous(),
464
+ cu_seqlens_k.contiguous(),
465
+ int(max_seqlen_k),
466
+ int(blk_kv),
467
+ page_table=None if page_table is None else page_table.contiguous(),
468
+ seqused_k=None if seqused_k is None else seqused_k.contiguous(),
469
+ )
470
+ block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads)
471
+ return flash_prefill_with_gqa_share_sparse(
472
+ q.contiguous(),
473
+ k_cache,
474
+ v_cache,
475
+ None,
476
+ req_to_token,
477
+ slot_ids,
478
+ q2k_indices,
479
+ block_size_q=block_size_q,
480
+ block_size_k=int(blk_kv),
481
+ cu_seqlens=cu_seqlens_q.contiguous(),
482
+ seq_lens=seq_lens,
483
+ prefix_lens=prefix_lens,
484
+ max_seqlen_q=int(max_seqlen_q),
485
+ sm_scale=softmax_scale,
486
+ )
487
+
488
+
489
+ def _nvfp4_dequant_kv(
490
+ packed: torch.Tensor,
491
+ scale_128x4: torch.Tensor,
492
+ global_scale: torch.Tensor,
493
+ ) -> torch.Tensor:
494
+ if packed.dtype is not torch.uint8:
495
+ return packed
496
+ if has_native_ops() and packed.is_cuda:
497
+ return native_nvfp4_dequant_swizzled_to_bf16(
498
+ packed,
499
+ scale_128x4,
500
+ global_scale,
501
+ )
502
+ original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
503
+ rows = 1
504
+ for dim in original_shape[:-1]:
505
+ rows *= int(dim)
506
+ qx = Nvfp4QuantizedTensor(
507
+ data=packed,
508
+ scale_128x4=scale_128x4,
509
+ global_scale=global_scale,
510
+ logical_scale_shape=(rows, int(original_shape[-1]) // 16),
511
+ original_shape=original_shape,
512
+ )
513
+ return dequantize_nvfp4_128x4_to_bf16(qx)
514
+
515
+
516
+ def sparse_atten_nvfp4_kv_func(
517
+ q: torch.Tensor,
518
+ k: torch.Tensor,
519
+ v: torch.Tensor,
520
+ k_scale_128x4: torch.Tensor,
521
+ v_scale_128x4: torch.Tensor,
522
+ k_global_scale: torch.Tensor,
523
+ v_global_scale: torch.Tensor,
524
+ k2q_row_ptr: torch.Tensor,
525
+ k2q_q_indices: torch.Tensor,
526
+ topK: int,
527
+ **kwargs,
528
+ ):
529
+ """NVFP4-KV official prefill API wrapper.
530
+
531
+ The Blackwell compatibility path dequantizes NVFP4 K/V with the package's
532
+ 128x4 metadata helper, then dispatches to `sparse_atten_func`.
533
+ """
534
+
535
+ k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale)
536
+ v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale)
537
+ return sparse_atten_func(
538
+ q,
539
+ k_bf16,
540
+ v_bf16,
541
+ k2q_row_ptr,
542
+ k2q_q_indices,
543
+ topK,
544
+ **kwargs,
545
+ )
546
+
547
+
548
+ def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor:
549
+ data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8)
550
+ lut = torch.tensor(
551
+ [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
552
+ dtype=torch.float32,
553
+ device=data.device,
554
+ )
555
+ out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device)
556
+ out[..., 0::2] = lut[(data & 0x0F).long()]
557
+ out[..., 1::2] = lut[(data >> 4).long()]
558
+ return out
559
+
560
+
561
+ def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
562
+ if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]):
563
+ return values * scale.to(torch.float32).repeat_interleave(16, dim=-1)
564
+ if scale.ndim == values.ndim - 1:
565
+ return values * scale.to(torch.float32).unsqueeze(-1)
566
+ return values
567
+
568
+
569
+ def fp4_indexer_block_scores(
570
+ q_fp4: torch.Tensor,
571
+ k_fp4: torch.Tensor,
572
+ q_scale: torch.Tensor,
573
+ k_scale: torch.Tensor,
574
+ cu_seqlens_q: torch.Tensor,
575
+ cu_seqlens_k: torch.Tensor,
576
+ cu_page_offsets: torch.Tensor,
577
+ *,
578
+ max_seqlen_q: int,
579
+ max_seqlen_k: int,
580
+ kv_indices: torch.Tensor,
581
+ fp4_format: str,
582
+ causal: bool = False,
583
+ qo_offset: Optional[torch.Tensor] = None,
584
+ scale_layout: str = "preordered_mma",
585
+ ) -> torch.Tensor:
586
+ """FP4 block-score API for Blackwell.
587
+
588
+ Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`.
589
+ Built artifacts route CUDA tensors through the native Blackwell block-score
590
+ kernel. Source-tree mode uses the reference loop below so the API remains
591
+ testable before the extension is built.
592
+ """
593
+
594
+ del qo_offset
595
+ if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}:
596
+ raise ValueError(f"unsupported fp4_format: {fp4_format}")
597
+ if scale_layout not in {"public", "preordered_mma"}:
598
+ raise ValueError(f"unsupported scale_layout: {scale_layout}")
599
+ if q_fp4.ndim != 3:
600
+ raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]")
601
+ if k_fp4.ndim != 4:
602
+ raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]")
603
+ total_q, hq, _ = q_fp4.shape
604
+ num_pages, hkv, page_size, _ = k_fp4.shape
605
+ if int(page_size) != 128:
606
+ raise ValueError("fp4_indexer_block_scores currently expects 128-token pages")
607
+ if hq % hkv != 0:
608
+ raise ValueError("Hq must be divisible by Hkv")
609
+ max_blocks = (int(max_seqlen_k) + 127) // 128
610
+ scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device)
611
+
612
+ q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale)
613
+ k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale)
614
+
615
+ # Native block-max QK kernel for the cubic scoring loop when available.
616
+ # Same dequant numerics as the reference (bf16 compute); falls back to the
617
+ # Python reference below in source-tree mode.
618
+ if has_native_ops() and q_fp4.is_cuda:
619
+ batch = int(cu_seqlens_q.numel()) - 1
620
+ q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64)
621
+ batch_of_q = torch.repeat_interleave(
622
+ torch.arange(batch, device=q_fp4.device, dtype=torch.int32),
623
+ q_lens,
624
+ )
625
+ return native_indexer_block_scores(
626
+ q.to(torch.bfloat16),
627
+ k.to(torch.bfloat16),
628
+ batch_of_q,
629
+ cu_seqlens_q.to(torch.int32),
630
+ cu_seqlens_k.to(torch.int32),
631
+ cu_page_offsets.to(torch.int32),
632
+ kv_indices.to(torch.int32),
633
+ max_blocks=max_blocks,
634
+ causal=causal,
635
+ )
636
+
637
+ gqa = hq // hkv
638
+ batch = int(cu_seqlens_q.numel()) - 1
639
+ cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
640
+ cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist()
641
+ cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist()
642
+ kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist()
643
+ for b in range(batch):
644
+ q_start, q_end = int(cu_q[b]), int(cu_q[b + 1])
645
+ k_len = int(cu_k[b + 1]) - int(cu_k[b])
646
+ page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1])
647
+ for logical_block, page_pos in enumerate(range(page_start, page_end)):
648
+ if logical_block >= max_blocks:
649
+ break
650
+ physical_page = int(kv_idx_cpu[page_pos])
651
+ if physical_page < 0 or physical_page >= num_pages:
652
+ continue
653
+ valid_tokens = min(128, max(0, k_len - logical_block * 128))
654
+ if valid_tokens <= 0:
655
+ continue
656
+ for qi in range(q_start, q_end):
657
+ local_q = qi - q_start
658
+ if causal and logical_block * 128 > local_q:
659
+ continue
660
+ for qh in range(hq):
661
+ kh = qh // gqa
662
+ key_block = k[physical_page, kh, :valid_tokens]
663
+ if causal:
664
+ visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1))
665
+ if visible <= 0:
666
+ continue
667
+ key_block = key_block[:visible]
668
+ block_scores = torch.matmul(key_block, q[qi, qh])
669
+ scores[qh, logical_block, qi] = block_scores.max()
670
+ return scores
671
+
672
+
673
+ __all__ = [
674
+ "sparse_atten_func",
675
+ "sparse_atten_nvfp4_kv_func",
676
+ "sparse_decode_atten_func",
677
+ "SparseDecodePagedAttentionWrapper",
678
+ "fp4_indexer_block_scores",
679
+ ]
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "minimaxai-msa-blackwell",
3
+ "id": "_minimaxai_msa_blackwell_cuda_25b0067",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0"
11
+ ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=",
17
+ "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
18
+ "_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "UoiDhzhz2aSCy+k+sOoOxb5/DqwAndixA6BgmEHAMtU=",
19
+ "_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=",
20
+ "_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=",
21
+ "api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=",
22
+ "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
23
+ "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
24
+ "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
25
+ "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
26
+ "decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=",
27
+ "interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=",
28
+ "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
29
+ "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
30
+ "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
31
+ "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
32
+ "prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=",
33
+ "prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=",
34
+ "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
35
+ "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
36
+ }
37
+ }
38
+ }
build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+ #
3
+ # VENDOR NOTE: rewritten to drop the `einops` dependency (the original used
4
+ # einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream
5
+ # `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced
6
+ # with native torch.einsum / torch.reshape so the standalone vendor needs only
7
+ # torch (+ triton for the kernels). Original einops expressions are kept in
8
+ # comments next to each replacement for auditability.
9
+
10
+ from typing import Optional
11
+
12
+ import torch
13
+
14
+
15
+ def naive_flash_decode_with_topk_idx(
16
+ q: torch.Tensor, # [batch_size, num_heads, head_dim]
17
+ sink: Optional[torch.Tensor], # [num_heads, head_dim]
18
+ kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim]
19
+ seq_lens: torch.Tensor, # [batch_size, ]
20
+ max_seqlen: int,
21
+ slot_ids: torch.Tensor, # [batch_size, ]
22
+ block_size: int,
23
+ topk: int,
24
+ sm_scale: Optional[float] = None,
25
+ init_blocks: int = 0,
26
+ local_blocks: int = 0,
27
+ ):
28
+ """Joint reference: lightning-indexer block top-k + sparse attention.
29
+
30
+ This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense
31
+ 5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by
32
+ `slot_ids`. It both (a) selects the top-k key blocks via per-block max score
33
+ (with optional forced init/local blocks) and (b) computes the resulting
34
+ sparse attention, returning (o, topk_idx).
35
+ """
36
+ assert (
37
+ kv_cache.shape[2] % block_size == 0
38
+ ), "max cache len must be divisible by block size"
39
+ if sm_scale is None:
40
+ sm_scale = q.shape[-1] ** -0.5
41
+ original_dtype = q.dtype
42
+ batch_size = q.shape[0]
43
+ num_q_heads = q.shape[1]
44
+ num_kv_heads = kv_cache.shape[3]
45
+ gqa_group_size = num_q_heads // num_kv_heads
46
+ head_dim = q.shape[-1]
47
+ # rearrange q: "b (h g) d -> b h g d"
48
+ q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim)
49
+ kv_cache_float = kv_cache.float()
50
+ k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d]
51
+ v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d]
52
+ # einsum(q, k, "b h g d, b n h d -> b h g n")
53
+ qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale
54
+ mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None]
55
+ qk = qk.masked_fill(~mask[:, None, None, :], float("-inf"))
56
+ # get score
57
+ score = qk.clone().to(torch.float32)
58
+ # rearrange "b h g (n s) -> b (h g) n s", s=block_size
59
+ n_blocks = score.shape[-1] // block_size
60
+ score = score.reshape(
61
+ batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size
62
+ ).reshape(batch_size, num_q_heads, n_blocks, block_size)
63
+ score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks]
64
+ # post-process score for init_blocks and local_blocks
65
+ INIT_SCORE = 1e30
66
+ LOCAL_SCORE = 1e29
67
+ if init_blocks > 0:
68
+ score[:, :, :init_blocks] = INIT_SCORE
69
+ if local_blocks > 0:
70
+ num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
71
+ for b in range(batch_size):
72
+ num_blks = num_blocks_per_batch[b].item()
73
+ local_start = max(0, num_blks - local_blocks)
74
+ score[b, :, local_start:num_blks] = LOCAL_SCORE
75
+ # compute topk indices per (batch, head)
76
+ # score shape: [batch_size, num_q_heads, num_blocks]
77
+ topk_idx = torch.full(
78
+ (num_q_heads, batch_size, topk),
79
+ fill_value=-1,
80
+ device=score.device,
81
+ dtype=torch.int32,
82
+ )
83
+ num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
84
+ for b in range(batch_size):
85
+ num_blks = num_blocks_per_batch[b].item()
86
+ actual_topk = min(topk, num_blks)
87
+ for h in range(num_q_heads):
88
+ # get topk indices for this (batch, head)
89
+ _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1)
90
+ topk_idx[h, b, :actual_topk] = indices.to(torch.int32)
91
+ # compute attention output with sink
92
+ if sink is not None:
93
+ # sink: [num_q_heads, head_dim] -> [h, g, d]
94
+ sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim)
95
+ # einsum(q, sink, "b h g d, h g d -> b h g")
96
+ qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale
97
+ qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1]
98
+ attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32)
99
+ attn = attn[..., 1:] # remove sink score
100
+ else:
101
+ attn = qk.softmax(dim=-1, dtype=torch.float32)
102
+ # einsum(attn, v, "b h g n, b n h d -> b h g d")
103
+ o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel)
104
+ # rearrange "b h g d -> b (h g) d"
105
+ o = o.reshape(batch_size, num_q_heads, head_dim)
106
+ return o.to(original_dtype), topk_idx
build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+
7
+
8
+ def naive_flash_decode_with_gqa_share_sparse(
9
+ q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
10
+ sink: Optional[torch.Tensor], # [num_q_heads, head_dim]
11
+ kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim]
12
+ seq_lens: torch.Tensor, # [batch_size, ]
13
+ slot_ids: torch.Tensor, # [batch_size, ]
14
+ block_size: int,
15
+ topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
16
+ sm_scale: Optional[float] = None,
17
+ ) -> torch.Tensor:
18
+ """
19
+ Naive implementation of sparse attention with GQA group sharing.
20
+ Each GQA group (all q heads sharing the same kv head) uses the same topk_idx.
21
+ """
22
+ if sm_scale is None:
23
+ sm_scale = q.shape[-1] ** -0.5
24
+
25
+ original_dtype = q.dtype
26
+ batch_size, num_q_heads, head_dim = q.shape
27
+ max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape
28
+ gqa_group_size = num_q_heads // num_kv_heads
29
+ # Output tensor
30
+ o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device)
31
+
32
+ for b in range(batch_size):
33
+ seq_len = seq_lens[b].item()
34
+ sid = slot_ids[b].item() % max_slots
35
+
36
+ for kh in range(num_kv_heads):
37
+ # Get topk indices for this (kv_head, batch)
38
+ block_indices = topk_idx[kh, b, :].tolist()
39
+
40
+ # Collect selected K and V blocks
41
+ selected_k_blocks = []
42
+ selected_v_blocks = []
43
+ for block_idx in block_indices:
44
+ if block_idx < 0:
45
+ continue # Invalid index
46
+ start = block_idx * block_size
47
+ end = min(start + block_size, seq_len)
48
+ if start >= seq_len:
49
+ continue
50
+ # K: [block_len, head_dim]
51
+ k_block = kv_cache[sid, 0, start:end, kh, :]
52
+ # V: [block_len, head_dim]
53
+ v_block = kv_cache[sid, 1, start:end, kh, :]
54
+ selected_k_blocks.append(k_block)
55
+ selected_v_blocks.append(v_block)
56
+
57
+ if len(selected_k_blocks) == 0:
58
+ continue
59
+
60
+ # Concatenate selected blocks: [total_selected_len, head_dim]
61
+ k_selected = torch.cat(selected_k_blocks, dim=0)
62
+ v_selected = torch.cat(selected_v_blocks, dim=0)
63
+
64
+ # Compute attention for all q heads in this GQA group
65
+ for g in range(gqa_group_size):
66
+ qh = kh * gqa_group_size + g
67
+ # q_vec: [head_dim]
68
+ q_vec = q[b, qh, :].float()
69
+
70
+ # Compute attention scores: [total_selected_len]
71
+ scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale
72
+
73
+ # Add sink to softmax normalization if present
74
+ if sink is not None:
75
+ sink_vec = sink[qh, :].float()
76
+ qsink = torch.dot(q_vec, sink_vec) * sm_scale
77
+ # Concatenate sink score with regular scores for softmax
78
+ scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0)
79
+ attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1)
80
+ # Remove sink weight (only used for normalization)
81
+ attn_weights = attn_weights_with_sink[1:]
82
+ else:
83
+ attn_weights = torch.softmax(scores, dim=-1)
84
+
85
+ # Compute output: [head_dim]
86
+ o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected)
87
+
88
+ return o
build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ from .topk_sparse import flash_prefill_with_gqa_share_sparse
3
+
4
+ __all__ = ["flash_prefill_with_gqa_share_sparse"]
build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py ADDED
@@ -0,0 +1,371 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
10
+
11
+ from ..common.utils import get_cu_seqblocks, robust_allocator
12
+
13
+ _is_hip = is_hip()
14
+
15
+
16
+ @triton.heuristics(
17
+ {
18
+ "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]),
19
+ "BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]),
20
+ "BLOCK_SIZE_H": lambda args: triton.next_power_of_2(
21
+ max(
22
+ 16 // args["BLOCK_SIZE_Q"],
23
+ triton.next_power_of_2(args["gqa_group_size"]),
24
+ )
25
+ ),
26
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
27
+ "BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"],
28
+ "HAS_SINK": lambda args: args["sink_ptr"] is not None,
29
+ }
30
+ )
31
+ @triton.autotune(
32
+ # Configs that fail to compile on the target arch are skipped, so widening
33
+ # the num_warps x num_stages grid only adds candidates, never a bad kernel.
34
+ configs=[
35
+ triton.Config({}, num_warps=nw, num_stages=ns)
36
+ for nw in (2, 4, 8)
37
+ for ns in (2, 3, 4)
38
+ ],
39
+ key=[
40
+ "BLOCK_SIZE_Q",
41
+ "BLOCK_SIZE_K",
42
+ "qk_head_dim",
43
+ "v_head_dim",
44
+ "gqa_group_size",
45
+ ],
46
+ )
47
+ @triton.jit
48
+ def _gqa_share_sparse_fwd_kernel(
49
+ q_ptr, # Q: n x h x d
50
+ k_cache_ptr, # K paged: max_slots x kh x d
51
+ v_cache_ptr, # V paged: max_slots x kh x d
52
+ sink_ptr, # Sink: h x d
53
+ t_ptr, # topk_idx: kh x n x k
54
+ o_ptr, # O: n x h x d
55
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
56
+ # seqlens
57
+ cu_seqlens_q,
58
+ cu_seqblocks_q,
59
+ seq_lens,
60
+ prefix_lens,
61
+ slot_ids,
62
+ # shape
63
+ max_slots,
64
+ num_kv_heads,
65
+ gqa_group_size,
66
+ qk_head_dim,
67
+ v_head_dim,
68
+ max_topk,
69
+ # q loop num
70
+ num_q_loop,
71
+ # sm_scale
72
+ sm_scale,
73
+ # stride
74
+ stride_qn,
75
+ stride_qh,
76
+ stride_qd,
77
+ stride_ks,
78
+ stride_kh,
79
+ stride_kd,
80
+ stride_vs,
81
+ stride_vh,
82
+ stride_vd,
83
+ stride_sh,
84
+ stride_sd,
85
+ stride_th,
86
+ stride_tn,
87
+ stride_tk,
88
+ stride_on,
89
+ stride_oh,
90
+ stride_od,
91
+ stride_r2t_b,
92
+ # META parameters
93
+ BLOCK_SIZE_Q: tl.constexpr, # q block size
94
+ BLOCK_SIZE_K: tl.constexpr, # k block size
95
+ BLOCK_SIZE_KD: tl.constexpr,
96
+ BLOCK_SIZE_VD: tl.constexpr,
97
+ BLOCK_SIZE_H: tl.constexpr,
98
+ BLOCK_SIZE_T: tl.constexpr,
99
+ BLOCK_SIZE_QH: tl.constexpr,
100
+ # has sink
101
+ HAS_SINK: tl.constexpr,
102
+ USE_TMA: tl.constexpr,
103
+ IS_FP8: tl.constexpr,
104
+ ):
105
+ sm_scale_log2e = sm_scale * 1.4426950409
106
+ # get batch id and head id
107
+ pid_q = tl.program_id(0)
108
+ pid_kh = tl.program_id(1)
109
+ pid_b = tl.program_id(2)
110
+ pid_h = pid_kh * gqa_group_size
111
+ # get q k start and len after rmpad
112
+ q_start = tl.load(cu_seqlens_q + pid_b)
113
+ q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
114
+ q_block_start = tl.load(cu_seqblocks_q + pid_b)
115
+ q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start
116
+ seq_len = tl.load(seq_lens + pid_b)
117
+ prefix_len = tl.load(prefix_lens + pid_b)
118
+ sid = (
119
+ tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
120
+ ) % max_slots # safety against negative
121
+ if pid_q * num_q_loop >= q_block_len:
122
+ return
123
+ real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop)
124
+ if HAS_SINK:
125
+ sink_ptrs = tl.make_block_ptr(
126
+ base=sink_ptr + pid_h * stride_sh,
127
+ shape=(gqa_group_size, qk_head_dim),
128
+ strides=(stride_sh, stride_sd),
129
+ offsets=(0, 0),
130
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD),
131
+ order=(1, 0),
132
+ )
133
+ sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
134
+ tl.float32
135
+ )
136
+ # offsets for paged K/V load
137
+ off_n = tl.arange(0, BLOCK_SIZE_K)
138
+ off_kd = tl.arange(0, BLOCK_SIZE_KD)
139
+ off_vd = tl.arange(0, BLOCK_SIZE_VD)
140
+ kd_mask = off_kd < qk_head_dim
141
+ vd_mask = off_vd < v_head_dim
142
+ for j in range(real_q_loop):
143
+ pid_q_j = pid_q * num_q_loop + j
144
+ # init topk idx pointer
145
+ t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
146
+ # we assume that the topk_idx is right padded with -1
147
+ off_t = tl.arange(0, BLOCK_SIZE_T)
148
+ topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
149
+ valid_idx = tl.where(topk_idx >= 0, off_t, -1)
150
+ real_topk = tl.sum(valid_idx != -1, axis=0)
151
+ # init qkv pointer
152
+ q_ptrs = tl.make_block_ptr(
153
+ base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
154
+ shape=(q_len, gqa_group_size, qk_head_dim),
155
+ strides=(stride_qn, stride_qh, stride_qd),
156
+ offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
157
+ block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD),
158
+ order=(2, 1, 0),
159
+ )
160
+ # load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D]
161
+ q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero")
162
+ # init statistics
163
+ off_q_k = (
164
+ tl.arange(0, BLOCK_SIZE_Q)[:, None]
165
+ + pid_q_j * BLOCK_SIZE_Q
166
+ + prefix_len
167
+ - tl.arange(0, BLOCK_SIZE_K)[None, :]
168
+ )
169
+ if HAS_SINK:
170
+ m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
171
+ lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
172
+ qsink = (
173
+ tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e
174
+ ) # (BLOCK_SIZE_Q, BLOCK_SIZE_H)
175
+ m_i += qsink
176
+ lse_i += qsink
177
+ m_i = tl.reshape(m_i, BLOCK_SIZE_QH)
178
+ lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH)
179
+ else:
180
+ m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
181
+ lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
182
+ acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32)
183
+ q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD)
184
+ # sparse attention
185
+ for i in range(real_topk):
186
+ # get current block start index (absolute K position)
187
+ c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
188
+ t_ptr_j = t_ptr_j + stride_tk
189
+ # paged load K via req_to_token: pos -> slot -> k_cache
190
+ pos = c + off_n
191
+ pos_mask = pos < seq_len
192
+ slots = tl.load(
193
+ req_to_token_ptr + sid * stride_r2t_b + pos,
194
+ mask=pos_mask,
195
+ other=0,
196
+ ).to(tl.int64)
197
+ slots = (slots + max_slots) % max_slots # safety against negative
198
+ # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
199
+ k = tl.load(
200
+ k_cache_ptr
201
+ + slots[None, :] * stride_ks
202
+ + pid_kh * stride_kh
203
+ + off_kd[:, None] * stride_kd,
204
+ mask=kd_mask[:, None] & pos_mask[None, :],
205
+ other=0.0,
206
+ )
207
+ if IS_FP8:
208
+ # fp8 main K cache is unit-scaled; widen to the Q compute dtype
209
+ # before the tl.dot (compiled out when the cache is bf16).
210
+ k = k.to(q.dtype)
211
+ # compute qk
212
+ qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
213
+ # causal mask
214
+ qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
215
+ qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
216
+ # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
217
+ # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
218
+ qk += tl.dot(q, k) * sm_scale_log2e
219
+ # K boundary mask: positions beyond seq_len contribute -inf
220
+ qk += tl.where(pos_mask[None, :], 0, float("-inf"))
221
+ # compute m_ij and l_ij
222
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
223
+ p = tl.exp2(qk - m_ij[:, None])
224
+ l_ij = tl.sum(p, axis=1)
225
+ # scale acc_o
226
+ acc_o_scale = tl.exp2(m_i - m_ij)
227
+ acc_o = acc_o * acc_o_scale[:, None]
228
+ # paged load V
229
+ v = tl.load(
230
+ v_cache_ptr
231
+ + slots[:, None] * stride_vs
232
+ + pid_kh * stride_vh
233
+ + off_vd[None, :] * stride_vd,
234
+ mask=pos_mask[:, None] & vd_mask[None, :],
235
+ other=0.0,
236
+ )
237
+ if IS_FP8:
238
+ # Widen V so `p.to(v.dtype)` casts P to the compute dtype rather
239
+ # than to fp8 (which would wreck attention-weight precision).
240
+ v = v.to(q.dtype)
241
+ p = p.to(v.dtype)
242
+ acc_o += tl.dot(p, v)
243
+ # update statistics
244
+ m_i = m_ij
245
+ lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
246
+ # final scale
247
+ acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
248
+ # save output
249
+ acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD)
250
+ o_ptrs = tl.make_block_ptr(
251
+ base=o_ptr + q_start * stride_on + pid_h * stride_oh,
252
+ shape=(q_len, gqa_group_size, v_head_dim),
253
+ strides=(stride_on, stride_oh, stride_od),
254
+ offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
255
+ block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD),
256
+ order=(2, 1, 0),
257
+ )
258
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2))
259
+
260
+
261
+ @torch.no_grad()
262
+ def flash_prefill_with_gqa_share_sparse(
263
+ q: torch.Tensor,
264
+ k_cache: torch.Tensor,
265
+ v_cache: torch.Tensor,
266
+ sink: Optional[torch.Tensor],
267
+ req_to_token: torch.Tensor,
268
+ slot_ids: torch.Tensor,
269
+ topk_idx: torch.Tensor,
270
+ block_size_q: int,
271
+ block_size_k: int,
272
+ cu_seqlens: torch.Tensor,
273
+ seq_lens: torch.Tensor,
274
+ prefix_lens: torch.Tensor,
275
+ max_seqlen_q: int,
276
+ sm_scale: Optional[float] = None,
277
+ use_tma: bool = True,
278
+ cu_seqblocks_q: Optional[torch.Tensor] = None,
279
+ max_seqblock_q: Optional[int] = None,
280
+ ) -> torch.Tensor:
281
+ triton.set_allocator(robust_allocator)
282
+ # dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled)
283
+ # under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load.
284
+ # Keep CUDA's pre-existing sparse prefill dtype contract unchanged by
285
+ # accepting fp8 only on HIP.
286
+ assert q.dtype in (torch.bfloat16, torch.float16)
287
+ _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
288
+ is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
289
+ assert k_cache.dtype == q.dtype or is_fp8, (
290
+ f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) "
291
+ f"or fp8 on HIP, got {k_cache.dtype}"
292
+ )
293
+ assert v_cache.dtype == k_cache.dtype
294
+ assert block_size_q in {1, 2, 4, 8, 16, 32, 64}
295
+ assert block_size_k in {16, 32, 64, 128}
296
+ # shape
297
+ total_q, num_q_heads, qk_head_dim = q.shape
298
+ max_slots, num_k_heads, _ = k_cache.shape
299
+ _, num_v_heads, v_head_dim = v_cache.shape
300
+ batch_size = cu_seqlens.shape[0] - 1
301
+ topk = topk_idx.shape[-1]
302
+ assert topk_idx.shape[0] == num_k_heads
303
+ # gqa
304
+ assert num_k_heads == num_v_heads
305
+ assert num_q_heads % num_k_heads == 0
306
+ gqa_group_size = num_q_heads // num_k_heads
307
+ assert gqa_group_size * block_size_q <= 128
308
+ if sm_scale is None:
309
+ sm_scale = qk_head_dim**-0.5
310
+ if cu_seqblocks_q is None or max_seqblock_q is None:
311
+ cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks(
312
+ cu_seqlens, max_seqlen_q, block_size_q, block_size_k
313
+ )
314
+ # output tensor
315
+ o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype)
316
+ # launch kernel
317
+ num_q_loop = (
318
+ max_seqblock_q // 131072 + 1
319
+ ) # calculate multiple queries in one kernel if seqlence length is too long
320
+ BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q)
321
+ BLOCK_SIZE_K = triton.next_power_of_2(block_size_k)
322
+ grid = (
323
+ triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop),
324
+ num_k_heads,
325
+ batch_size,
326
+ )
327
+ _gqa_share_sparse_fwd_kernel[grid](
328
+ q,
329
+ k_cache,
330
+ v_cache,
331
+ sink,
332
+ topk_idx,
333
+ o,
334
+ req_to_token,
335
+ cu_seqlens,
336
+ cu_seqblocks_q,
337
+ seq_lens,
338
+ prefix_lens,
339
+ slot_ids,
340
+ max_slots,
341
+ num_k_heads,
342
+ gqa_group_size,
343
+ qk_head_dim,
344
+ v_head_dim,
345
+ topk,
346
+ num_q_loop,
347
+ sm_scale,
348
+ q.stride(0),
349
+ q.stride(1),
350
+ q.stride(2),
351
+ k_cache.stride(0),
352
+ k_cache.stride(1),
353
+ k_cache.stride(2),
354
+ v_cache.stride(0),
355
+ v_cache.stride(1),
356
+ v_cache.stride(2),
357
+ sink.stride(0) if sink is not None else 0,
358
+ sink.stride(1) if sink is not None else 0,
359
+ topk_idx.stride(0),
360
+ topk_idx.stride(1),
361
+ topk_idx.stride(2),
362
+ o.stride(0),
363
+ o.stride(1),
364
+ o.stride(2),
365
+ req_to_token.stride(0),
366
+ BLOCK_SIZE_Q=BLOCK_SIZE_Q,
367
+ BLOCK_SIZE_K=BLOCK_SIZE_K,
368
+ USE_TMA=use_tma,
369
+ IS_FP8=is_fp8,
370
+ )
371
+ return o
build/torch211-cxx11-cu128-x86_64-linux/quantize.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """NVFP4 helper API compatible with MiniMaxAI/msa.
3
+
4
+ The quantization entrypoints use Transformer Engine when it is installed. The
5
+ layout helpers and reference dequantizer are pure PyTorch and are included so
6
+ callers can prepare and validate NVFP4 metadata with the same public API names
7
+ as the upstream MiniMaxAI package.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Tuple
14
+
15
+ import torch
16
+
17
+
18
+ NVFP4_BLOCK_SIZE = 16
19
+ NVFP4_FP4_MAX = 6.0
20
+ NVFP4_FP8_E4M3_MAX = 448.0
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Nvfp4QuantizedTensor:
25
+ data: torch.Tensor
26
+ scale_128x4: torch.Tensor
27
+ global_scale: torch.Tensor
28
+ logical_scale_shape: Tuple[int, int]
29
+ original_shape: Tuple[int, ...]
30
+
31
+
32
+ def _round_up(x: int, multiple: int) -> int:
33
+ return ((int(x) + multiple - 1) // multiple) * multiple
34
+
35
+
36
+ def nvfp4_scale_128x4_offset(
37
+ row: torch.Tensor,
38
+ col: torch.Tensor,
39
+ scale_cols: int,
40
+ ) -> torch.Tensor:
41
+ tiles_n = _round_up(scale_cols, 4) // 4
42
+ tile_m = row // 128
43
+ tile_n = col // 4
44
+ outer = row % 128
45
+ inner = col % 4
46
+ return (
47
+ (tile_m * tiles_n + tile_n) * 512
48
+ + (outer % 32) * 16
49
+ + (outer // 32) * 4
50
+ + inner
51
+ )
52
+
53
+
54
+ def swizzle_nvfp4_scale_to_128x4(
55
+ scale: torch.Tensor,
56
+ *,
57
+ rows: int,
58
+ cols: int,
59
+ ) -> torch.Tensor:
60
+ if scale.ndim != 2:
61
+ raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
62
+
63
+ rows = int(rows)
64
+ cols = int(cols)
65
+ padded_rows = _round_up(rows, 128)
66
+ padded_cols = _round_up(cols, 4)
67
+ if scale.shape[0] < rows or scale.shape[1] < cols:
68
+ raise ValueError(
69
+ "scale is smaller than the requested logical shape: "
70
+ f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
71
+ )
72
+
73
+ logical = scale[:rows, :cols].contiguous()
74
+ if logical.shape != (padded_rows, padded_cols):
75
+ logical = torch.nn.functional.pad(
76
+ logical.to(torch.float32),
77
+ (0, padded_cols - cols, 0, padded_rows - rows),
78
+ ).to(scale.dtype)
79
+ swizzled = torch.empty_like(logical)
80
+
81
+ row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
82
+ col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
83
+ offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
84
+ swizzled.reshape(-1)[offset] = logical.reshape(-1)
85
+ return swizzled
86
+
87
+
88
+ def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
89
+ return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
90
+
91
+
92
+ def _import_te_nvfp4_quantizer():
93
+ try:
94
+ from transformer_engine.pytorch.tensor import NVFP4Quantizer
95
+ except Exception as exc:
96
+ raise RuntimeError(
97
+ "Transformer Engine NVFP4 quantization is unavailable. Install a "
98
+ "Transformer Engine build with NVFP4 PyTorch support to use "
99
+ "quantize_bf16_to_nvfp4_128x4."
100
+ ) from exc
101
+ return NVFP4Quantizer
102
+
103
+
104
+ def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
105
+ if not x.is_cuda:
106
+ raise ValueError("NVFP4 quantization requires a CUDA tensor")
107
+ if x.dtype not in (torch.bfloat16, torch.float16):
108
+ raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
109
+ if x.ndim < 2:
110
+ raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
111
+ if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
112
+ raise ValueError(
113
+ f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
114
+ )
115
+
116
+ rows = 1
117
+ for dim in x.shape[:-1]:
118
+ rows *= int(dim)
119
+ if rows % NVFP4_BLOCK_SIZE != 0:
120
+ raise ValueError(
121
+ "flattened row dimension must be divisible by "
122
+ f"{NVFP4_BLOCK_SIZE}, got {rows}"
123
+ )
124
+
125
+ NVFP4Quantizer = _import_te_nvfp4_quantizer()
126
+ quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
127
+ qx = quantizer.quantize(x.contiguous())
128
+ meta = qx.get_metadata()
129
+
130
+ data = meta["rowwise_data"]
131
+ if data.dtype is not torch.uint8:
132
+ data = data.view(torch.uint8)
133
+ logical_scale = meta["rowwise_scale_inv"]
134
+ amax = meta["amax_rowwise"]
135
+ scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
136
+ scale_128x4 = swizzle_nvfp4_scale_to_128x4(
137
+ logical_scale,
138
+ rows=rows,
139
+ cols=scale_cols,
140
+ )
141
+ global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
142
+
143
+ return Nvfp4QuantizedTensor(
144
+ data=data,
145
+ scale_128x4=scale_128x4,
146
+ global_scale=global_scale,
147
+ logical_scale_shape=(rows, scale_cols),
148
+ original_shape=tuple(int(v) for v in x.shape),
149
+ )
150
+
151
+
152
+ def quantize_kv_bf16_to_nvfp4_128x4(
153
+ k: torch.Tensor,
154
+ v: torch.Tensor,
155
+ ) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
156
+ return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
157
+
158
+
159
+ def dequantize_nvfp4_128x4_to_bf16(
160
+ qx: Nvfp4QuantizedTensor,
161
+ *,
162
+ include_global_scale: bool = True,
163
+ ) -> torch.Tensor:
164
+ data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
165
+ if data.shape[-1] * 2 != qx.original_shape[-1]:
166
+ raise ValueError(
167
+ "packed data last dimension does not match original shape: "
168
+ f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
169
+ )
170
+
171
+ rows, scale_cols = qx.logical_scale_shape
172
+ logical_dim = int(qx.original_shape[-1])
173
+ if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
174
+ raise ValueError(
175
+ "logical scale columns do not match original last dimension: "
176
+ f"{scale_cols} scale cols vs dim {logical_dim}"
177
+ )
178
+
179
+ fp4_lut = torch.tensor(
180
+ [
181
+ 0.0,
182
+ 0.5,
183
+ 1.0,
184
+ 1.5,
185
+ 2.0,
186
+ 3.0,
187
+ 4.0,
188
+ 6.0,
189
+ -0.0,
190
+ -0.5,
191
+ -1.0,
192
+ -1.5,
193
+ -2.0,
194
+ -3.0,
195
+ -4.0,
196
+ -6.0,
197
+ ],
198
+ dtype=torch.float32,
199
+ device=data.device,
200
+ )
201
+ packed = data.reshape(rows, logical_dim // 2)
202
+ lo = packed & 0x0F
203
+ hi = packed >> 4
204
+ values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
205
+ values[:, 0::2] = fp4_lut[lo.long()]
206
+ values[:, 1::2] = fp4_lut[hi.long()]
207
+
208
+ row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
209
+ col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
210
+ offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
211
+ scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
212
+ scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
213
+ scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
214
+ out = values * scale
215
+ if include_global_scale:
216
+ out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
217
+ return out.reshape(qx.original_shape).to(torch.bfloat16)
218
+
219
+
220
+ __all__ = [
221
+ "Nvfp4QuantizedTensor",
222
+ "quantize_bf16_to_nvfp4_128x4",
223
+ "quantize_kv_bf16_to_nvfp4_128x4",
224
+ "dequantize_nvfp4_128x4_to_bf16",
225
+ "swizzle_nvfp4_scale_to_128x4",
226
+ "nvfp4_global_scale_from_amax",
227
+ ]
build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Sparse-index helpers compatible with MiniMaxAI/msa public names."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+
11
+ def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
12
+ if cu_seqlens.dtype != torch.int32:
13
+ raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
14
+ if cu_seqlens.ndim != 1:
15
+ raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
16
+ if cu_seqlens.numel() < 1:
17
+ raise ValueError(f"{name} must have at least one element")
18
+ if not cu_seqlens.is_contiguous():
19
+ raise ValueError(f"{name} must be contiguous")
20
+
21
+
22
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
23
+ seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
24
+ return (seqlens_k + kv_block_size - 1) // kv_block_size
25
+
26
+
27
+ def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
28
+ rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
29
+ batch = len(rows_per_batch_cpu)
30
+ max_rows = max(rows_per_batch_cpu, default=0)
31
+ row_dtype = (
32
+ torch.int32
33
+ if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
34
+ else torch.int64
35
+ )
36
+ row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
37
+ row_linear = 0
38
+ for kv_block_idx in range(max_rows):
39
+ for batch_idx, row_count in enumerate(rows_per_batch_cpu):
40
+ if kv_block_idx < row_count:
41
+ row_map_cpu[batch_idx, kv_block_idx] = row_linear
42
+ row_linear += 1
43
+ return row_map_cpu.to(rows_per_batch.device), row_linear
44
+
45
+
46
+ def build_k2q_csr_torch_reference(
47
+ q2k_indices: torch.Tensor,
48
+ cu_seqlens_q: torch.Tensor,
49
+ cu_seqlens_k: torch.Tensor,
50
+ kv_block_size: int,
51
+ ) -> tuple[torch.Tensor, torch.Tensor]:
52
+ if kv_block_size <= 0:
53
+ raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
54
+ if q2k_indices.dtype != torch.int32:
55
+ raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
56
+ if q2k_indices.ndim != 3:
57
+ raise ValueError(
58
+ "q2k_indices must have shape [head_kv, total_q, topK], "
59
+ f"got {tuple(q2k_indices.shape)}"
60
+ )
61
+ _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
62
+ _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
63
+ if cu_seqlens_q.shape != cu_seqlens_k.shape:
64
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
65
+ if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
66
+ raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
67
+
68
+ head_kv, total_q, topk = q2k_indices.shape
69
+ if total_q != int(cu_seqlens_q[-1].item()):
70
+ raise ValueError(
71
+ f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
72
+ f"({int(cu_seqlens_q[-1].item())})"
73
+ )
74
+
75
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
76
+ row_map, total_rows = _build_packed_row_map(rows_per_batch)
77
+ nnz_upper_bound = total_q * topk
78
+
79
+ k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
80
+ k2q_q_indices = torch.full(
81
+ (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
82
+ )
83
+ if total_rows == 0 or total_q == 0 or topk == 0:
84
+ return k2q_row_ptr, k2q_q_indices
85
+
86
+ counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
87
+ total_entries = total_q * topk
88
+ row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
89
+ row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
90
+ q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
91
+ valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
92
+
93
+ for batch_idx in range(cu_seqlens_q.numel() - 1):
94
+ q_start = int(cu_seqlens_q[batch_idx].item())
95
+ q_end = int(cu_seqlens_q[batch_idx + 1].item())
96
+ if q_end <= q_start:
97
+ continue
98
+ q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
99
+ q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
100
+ kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
101
+ valid = kv_local >= 0
102
+ safe_kv = kv_local.clamp_min(0).long()
103
+ rows = row_map[batch_idx, safe_kv].to(row_dtype)
104
+ start = q_start * topk
105
+ end = q_end * topk
106
+ row_all[:, start:end] = rows
107
+ q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
108
+ valid_all[:, start:end] = valid & (rows >= 0)
109
+ counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
110
+
111
+ k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
112
+ offsets = torch.zeros_like(counts)
113
+ offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
114
+ cursor = torch.zeros_like(counts)
115
+ for h in range(head_kv):
116
+ for entry in range(total_entries):
117
+ if not bool(valid_all[h, entry]):
118
+ continue
119
+ row = int(row_all[h, entry].item())
120
+ pos = int(offsets[h, row].item() + cursor[h, row].item())
121
+ k2q_q_indices[h, pos] = q_all[h, entry]
122
+ cursor[h, row] += 1
123
+
124
+ return k2q_row_ptr, k2q_q_indices
125
+
126
+
127
+ def build_k2q_csr(
128
+ q2k_indices: torch.Tensor,
129
+ cu_seqlens_q: torch.Tensor,
130
+ cu_seqlens_k: torch.Tensor,
131
+ kv_block_size: int,
132
+ *,
133
+ total_k: Optional[int] = None,
134
+ max_seqlen_k: Optional[int] = None,
135
+ max_seqlen_q: Optional[int] = None,
136
+ total_rows: Optional[int] = None,
137
+ qhead_per_kv: int = 1,
138
+ return_schedule: bool = False,
139
+ ):
140
+ if return_schedule:
141
+ raise NotImplementedError(
142
+ "Blackwell build_k2q_csr currently returns CSR tensors only; "
143
+ "SM100 schedule objects are not part of this package."
144
+ )
145
+ if total_k is None:
146
+ raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
147
+ return build_k2q_csr_torch_reference(
148
+ q2k_indices.contiguous(),
149
+ cu_seqlens_q.contiguous(),
150
+ cu_seqlens_k.contiguous(),
151
+ int(kv_block_size),
152
+ )
153
+
154
+
155
+ class SparseK2qCsrBuilderSm100:
156
+ """Compatibility placeholder for the upstream SM100 class name.
157
+
158
+ The name is exported so code can feature-detect it, but schedule-producing
159
+ SM100 behavior is intentionally not claimed for this Blackwell package.
160
+ """
161
+
162
+ def __init__(self, *args, **kwargs):
163
+ self.args = args
164
+ self.kwargs = kwargs
165
+
166
+ def build(self, *args, **kwargs):
167
+ return build_k2q_csr(*args, **kwargs)
168
+
169
+
170
+ __all__ = [
171
+ "build_k2q_csr",
172
+ "build_k2q_csr_torch_reference",
173
+ "SparseK2qCsrBuilderSm100",
174
+ ]
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """FlashRT Blackwell extension of MiniMax MSA decode sparse attention.
3
+
4
+ This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on
5
+ DGX Spark / GB10 / SM121. It exposes the
6
+ MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime.
7
+
8
+ Implementation status:
9
+ * native CUDA helper: score -> top-k sparse block ids;
10
+ * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
11
+ * native CUDA FP4 block-score indexer;
12
+ * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path;
13
+ * Blackwell-validated sparse prefill path;
14
+ * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers,
15
+ and FP4 block-score helpers.
16
+
17
+ The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell
18
+ extension path. The public API is Tensor-oriented and independent from
19
+ FlashRT's serving runtime.
20
+
21
+ Public API
22
+ ----------
23
+ Lightning indexer (q.k blockmax score -> top-k block select), paged KV:
24
+ flash_decode_with_topk_idx (single-token decode, split-K)
25
+
26
+ Block-sparse GQA attention (consumes block_indices from the indexer):
27
+ flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks)
28
+ sparse_atten_func (official CSR prefill wrapper)
29
+ sparse_decode_atten_func (official paged decode wrapper)
30
+
31
+ PyTorch naive references (for correctness checks):
32
+ naive_flash_decode_with_gqa_share_sparse (attention given topk_idx)
33
+ naive_flash_decode_with_topk_idx (joint indexer + attention)
34
+
35
+ All Triton entrypoints expect a paged KV cache:
36
+ k_cache / v_cache : [max_slots, num_kv_heads, head_dim]
37
+ req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot)
38
+ slot_ids : [batch] (per-request row in req_to_token)
39
+ topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed,
40
+ -1 right-padding (the M3 indexer contract).
41
+ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
42
+ """
43
+
44
+ from .common.utils import get_cu_seqblocks, robust_allocator
45
+ from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
46
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
47
+ from .interface import (
48
+ SparseDecodePagedAttentionWrapper,
49
+ fp4_indexer_block_scores,
50
+ sparse_atten_func,
51
+ sparse_atten_nvfp4_kv_func,
52
+ sparse_decode_atten_func,
53
+ )
54
+ from ._native import (
55
+ has_native_ops,
56
+ native_nvfp4_dequant_swizzled_to_bf16,
57
+ native_topk_from_scores,
58
+ )
59
+ from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
60
+ from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
61
+ from .quantize import (
62
+ Nvfp4QuantizedTensor,
63
+ dequantize_nvfp4_128x4_to_bf16,
64
+ nvfp4_global_scale_from_amax,
65
+ quantize_bf16_to_nvfp4_128x4,
66
+ quantize_kv_bf16_to_nvfp4_128x4,
67
+ swizzle_nvfp4_scale_to_128x4,
68
+ )
69
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
70
+ from .api_status import (
71
+ OFFICIAL_API_STATUS,
72
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
73
+ V1_AVAILABLE_FUNCTIONS,
74
+ available_functions,
75
+ official_api_status,
76
+ official_minimax_msa_functions,
77
+ unsupported_official_functions,
78
+ )
79
+
80
+ __all__ = [
81
+ "sparse_atten_func",
82
+ "sparse_atten_nvfp4_kv_func",
83
+ "sparse_decode_atten_func",
84
+ "SparseDecodePagedAttentionWrapper",
85
+ "fp4_indexer_block_scores",
86
+ "build_k2q_csr",
87
+ "SparseK2qCsrBuilderSm100",
88
+ "Nvfp4QuantizedTensor",
89
+ "quantize_bf16_to_nvfp4_128x4",
90
+ "quantize_kv_bf16_to_nvfp4_128x4",
91
+ "dequantize_nvfp4_128x4_to_bf16",
92
+ "swizzle_nvfp4_scale_to_128x4",
93
+ "nvfp4_global_scale_from_amax",
94
+ "flash_decode_with_topk_idx",
95
+ "flash_decode_with_gqa_share_sparse",
96
+ "has_native_ops",
97
+ "native_topk_from_scores",
98
+ "native_nvfp4_dequant_swizzled_to_bf16",
99
+ "naive_flash_decode_with_gqa_share_sparse",
100
+ "naive_flash_decode_with_topk_idx",
101
+ "get_cu_seqblocks",
102
+ "robust_allocator",
103
+ "OFFICIAL_API_STATUS",
104
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
105
+ "V1_AVAILABLE_FUNCTIONS",
106
+ "available_functions",
107
+ "official_api_status",
108
+ "official_minimax_msa_functions",
109
+ "unsupported_official_functions",
110
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_compat.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton
3
+ # kernels. The upstream SGLang modules pull two tiny things from the framework:
4
+ #
5
+ # from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path)
6
+ # from sglang.srt.environ import envs -> a feature flag for an optional
7
+ # JIT-CUDA radix top-k fast path
8
+ #
9
+ # Both are replaced here so the kernels import with only torch + triton. The
10
+ # defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel
11
+ # radix select), which is what we want on sm_121 (GB10).
12
+
13
+
14
+ def is_hip() -> bool:
15
+ """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121).
16
+
17
+ Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV
18
+ widening branch (HIP-only). With this False, the CUDA dtype contract
19
+ (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA.
20
+ """
21
+ try:
22
+ import torch
23
+
24
+ return bool(getattr(torch.version, "hip", None))
25
+ except Exception:
26
+ return False
27
+
28
+
29
+ class _Flag:
30
+ """Minimal stand-in for an sglang.srt.environ boolean flag (always off)."""
31
+
32
+ def __init__(self, default: bool = False):
33
+ self._default = default
34
+
35
+ def get(self) -> bool:
36
+ return self._default
37
+
38
+
39
+ class _Envs:
40
+ """Stub for `sglang.srt.environ.envs`.
41
+
42
+ Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode
43
+ indexer. Keeping it False routes top-k selection through the 2-stage Triton
44
+ fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the
45
+ decode indexer stays pure-Triton.
46
+ """
47
+
48
+ SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False)
49
+
50
+
51
+ envs = _Envs()
build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a053c1fd69c4df3f94e566a7c4d88ba943f8d5cece714ade52789f6819a961ac
3
+ size 729824
build/torch211-cxx11-cu130-x86_64-linux/_native.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Optional native CUDA helpers for the FlashRT Blackwell MSA package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import torch
7
+
8
+ try:
9
+ from ._ops import add_op_namespace_prefix, ops
10
+
11
+ _HAS_NATIVE_OPS = True
12
+ except Exception:
13
+ add_op_namespace_prefix = None
14
+ ops = None
15
+ _HAS_NATIVE_OPS = False
16
+
17
+
18
+ def has_native_ops() -> bool:
19
+ return _HAS_NATIVE_OPS
20
+
21
+
22
+ if _HAS_NATIVE_OPS:
23
+
24
+ @torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores"))
25
+ def _msa_topk_from_scores_fake(
26
+ score: torch.Tensor,
27
+ seq_lens: torch.Tensor,
28
+ block_size: int,
29
+ topk: int,
30
+ topk_idx: torch.Tensor,
31
+ ) -> None:
32
+ if score.dim() != 3:
33
+ raise RuntimeError("score must have shape (heads, batch, max_blocks)")
34
+ heads, batch, _ = score.shape
35
+ if seq_lens.shape != (batch,):
36
+ raise RuntimeError("seq_lens must have shape (batch,)")
37
+ if topk_idx.shape != (heads, batch, topk):
38
+ raise RuntimeError("topk_idx must have shape (heads, batch, topk)")
39
+ return None
40
+
41
+
42
+ def native_topk_from_scores(
43
+ score: torch.Tensor,
44
+ seq_lens: torch.Tensor,
45
+ block_size: int,
46
+ topk: int,
47
+ ) -> torch.Tensor:
48
+ """Return top-k sparse block ids from a score tensor."""
49
+
50
+ if not _HAS_NATIVE_OPS:
51
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
52
+ if not score.is_contiguous():
53
+ score = score.contiguous()
54
+ if not seq_lens.is_contiguous():
55
+ seq_lens = seq_lens.contiguous()
56
+ out = torch.empty(
57
+ (score.shape[0], score.shape[1], int(topk)),
58
+ device=score.device,
59
+ dtype=torch.int32,
60
+ )
61
+ ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out)
62
+ return out
63
+
64
+
65
+ if _HAS_NATIVE_OPS:
66
+
67
+ @torch.library.register_fake(
68
+ add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged")
69
+ )
70
+ def _msa_decode_sparse_attn_mma_paged_fake(
71
+ q: torch.Tensor,
72
+ k_cache: torch.Tensor,
73
+ v_cache: torch.Tensor,
74
+ req_to_token: torch.Tensor,
75
+ seq_lens: torch.Tensor,
76
+ slot_ids: torch.Tensor,
77
+ topk_idx: torch.Tensor,
78
+ block_size: int,
79
+ sm_scale: float,
80
+ out: torch.Tensor,
81
+ ) -> None:
82
+ return None
83
+
84
+
85
+ def native_decode_mma_supported(
86
+ q: torch.Tensor,
87
+ k_cache: torch.Tensor,
88
+ *,
89
+ block_size: int,
90
+ ) -> bool:
91
+ """Whether the tensor-core decode kernel applies to these inputs.
92
+
93
+ The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA
94
+ group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64.
95
+ """
96
+
97
+ if not _HAS_NATIVE_OPS:
98
+ return False
99
+ if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16:
100
+ return False
101
+ if q.dim() != 3 or k_cache.dim() != 3:
102
+ return False
103
+ head_dim = int(q.shape[-1])
104
+ num_q_heads = int(q.shape[1])
105
+ num_kv_heads = int(k_cache.shape[1])
106
+ if head_dim != 128 or num_kv_heads <= 0:
107
+ return False
108
+ if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16:
109
+ return False
110
+ if int(block_size) % 64 != 0:
111
+ return False
112
+ return True
113
+
114
+
115
+ def native_decode_sparse_attn_mma_paged(
116
+ q: torch.Tensor,
117
+ k_cache: torch.Tensor,
118
+ v_cache: torch.Tensor,
119
+ req_to_token: torch.Tensor,
120
+ seq_lens: torch.Tensor,
121
+ slot_ids: torch.Tensor,
122
+ block_size: int,
123
+ topk_idx: torch.Tensor,
124
+ sm_scale: float,
125
+ ) -> torch.Tensor:
126
+ """Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16."""
127
+
128
+ if not _HAS_NATIVE_OPS:
129
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
130
+ out = torch.empty(
131
+ (int(q.shape[0]), int(q.shape[1]), int(q.shape[2])),
132
+ device=q.device,
133
+ dtype=q.dtype,
134
+ )
135
+ ops.msa_decode_sparse_attn_mma_paged(
136
+ q.contiguous(),
137
+ k_cache.contiguous(),
138
+ v_cache.contiguous(),
139
+ req_to_token.contiguous(),
140
+ seq_lens.contiguous(),
141
+ slot_ids.contiguous(),
142
+ topk_idx.contiguous(),
143
+ int(block_size),
144
+ float(sm_scale),
145
+ out,
146
+ )
147
+ return out
148
+
149
+
150
+ if _HAS_NATIVE_OPS:
151
+
152
+ @torch.library.register_fake(
153
+ add_op_namespace_prefix("msa_indexer_block_scores")
154
+ )
155
+ def _msa_indexer_block_scores_fake(
156
+ q: torch.Tensor,
157
+ k_pages: torch.Tensor,
158
+ batch_of_q: torch.Tensor,
159
+ cu_q: torch.Tensor,
160
+ cu_k: torch.Tensor,
161
+ cu_pages: torch.Tensor,
162
+ kv_indices: torch.Tensor,
163
+ causal: int,
164
+ scores: torch.Tensor,
165
+ ) -> None:
166
+ return None
167
+
168
+
169
+ def native_indexer_block_scores(
170
+ q: torch.Tensor,
171
+ k_pages: torch.Tensor,
172
+ batch_of_q: torch.Tensor,
173
+ cu_q: torch.Tensor,
174
+ cu_k: torch.Tensor,
175
+ cu_pages: torch.Tensor,
176
+ kv_indices: torch.Tensor,
177
+ *,
178
+ max_blocks: int,
179
+ causal: bool,
180
+ ) -> torch.Tensor:
181
+ """Block-max QK indexer scores. q/k_pages are bf16 (already dequantized).
182
+
183
+ Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible).
184
+ """
185
+
186
+ if not _HAS_NATIVE_OPS:
187
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
188
+ total_q = int(q.shape[0])
189
+ hq = int(q.shape[1])
190
+ scores = torch.full(
191
+ (hq, int(max_blocks), total_q),
192
+ float("-inf"),
193
+ dtype=torch.float32,
194
+ device=q.device,
195
+ )
196
+ ops.msa_indexer_block_scores(
197
+ q.contiguous(),
198
+ k_pages.contiguous(),
199
+ batch_of_q.contiguous(),
200
+ cu_q.contiguous(),
201
+ cu_k.contiguous(),
202
+ cu_pages.contiguous(),
203
+ kv_indices.contiguous(),
204
+ 1 if causal else 0,
205
+ scores,
206
+ )
207
+ return scores
208
+
209
+
210
+ if _HAS_NATIVE_OPS:
211
+
212
+ @torch.library.register_fake(
213
+ add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16")
214
+ )
215
+ def _msa_nvfp4_dequant_swizzled_to_bf16_fake(
216
+ packed: torch.Tensor,
217
+ scale_128x4: torch.Tensor,
218
+ global_scale: float,
219
+ out: torch.Tensor,
220
+ ) -> None:
221
+ return None
222
+
223
+
224
+ def native_nvfp4_dequant_swizzled_to_bf16(
225
+ packed: torch.Tensor,
226
+ scale_128x4: torch.Tensor,
227
+ global_scale: torch.Tensor | float,
228
+ ) -> torch.Tensor:
229
+ """Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path."""
230
+
231
+ if not _HAS_NATIVE_OPS:
232
+ raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
233
+ if packed.dtype is not torch.uint8:
234
+ raise TypeError("packed must be torch.uint8")
235
+ if scale_128x4.dtype is not torch.uint8:
236
+ raise TypeError("scale_128x4 must be torch.uint8")
237
+ original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
238
+ out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16)
239
+ if isinstance(global_scale, torch.Tensor):
240
+ global_scale_value = float(global_scale.reshape(-1)[0].item())
241
+ else:
242
+ global_scale_value = float(global_scale)
243
+ ops.msa_nvfp4_dequant_swizzled_to_bf16(
244
+ packed.contiguous(),
245
+ scale_128x4.contiguous(),
246
+ global_scale_value,
247
+ out,
248
+ )
249
+ return out
250
+
251
+
252
+ __all__ = [
253
+ "has_native_ops",
254
+ "native_topk_from_scores",
255
+ "native_decode_mma_supported",
256
+ "native_decode_sparse_attn_mma_paged",
257
+ "native_indexer_block_scores",
258
+ "native_nvfp4_dequant_swizzled_to_bf16",
259
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _minimaxai_msa_blackwell_cuda_25b0067
3
+ ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/api_status.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Public API status for the FlashRT MiniMax MSA Blackwell package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from types import MappingProxyType
7
+
8
+
9
+ V1_AVAILABLE_FUNCTIONS = (
10
+ "sparse_atten_func",
11
+ "sparse_atten_nvfp4_kv_func",
12
+ "sparse_decode_atten_func",
13
+ "SparseDecodePagedAttentionWrapper",
14
+ "fp4_indexer_block_scores",
15
+ "build_k2q_csr",
16
+ "SparseK2qCsrBuilderSm100",
17
+ "Nvfp4QuantizedTensor",
18
+ "quantize_bf16_to_nvfp4_128x4",
19
+ "quantize_kv_bf16_to_nvfp4_128x4",
20
+ "dequantize_nvfp4_128x4_to_bf16",
21
+ "swizzle_nvfp4_scale_to_128x4",
22
+ "nvfp4_global_scale_from_amax",
23
+ "has_native_ops",
24
+ "native_topk_from_scores",
25
+ "native_nvfp4_dequant_swizzled_to_bf16",
26
+ "flash_decode_with_topk_idx",
27
+ "flash_decode_with_gqa_share_sparse",
28
+ "naive_flash_decode_with_topk_idx",
29
+ "naive_flash_decode_with_gqa_share_sparse",
30
+ "get_cu_seqblocks",
31
+ "robust_allocator",
32
+ )
33
+
34
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
35
+ "sparse_atten_func",
36
+ "sparse_atten_nvfp4_kv_func",
37
+ "sparse_decode_atten_func",
38
+ "SparseDecodePagedAttentionWrapper",
39
+ "fp4_indexer_block_scores",
40
+ "build_k2q_csr",
41
+ "SparseK2qCsrBuilderSm100",
42
+ "Nvfp4QuantizedTensor",
43
+ "quantize_bf16_to_nvfp4_128x4",
44
+ "quantize_kv_bf16_to_nvfp4_128x4",
45
+ "dequantize_nvfp4_128x4_to_bf16",
46
+ "swizzle_nvfp4_scale_to_128x4",
47
+ "nvfp4_global_scale_from_amax",
48
+ )
49
+
50
+ _OFFICIAL_API_STATUS = {
51
+ "sparse_atten_func": {
52
+ "status": "available",
53
+ "target": "blackwell-prefill",
54
+ "reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.",
55
+ },
56
+ "sparse_atten_nvfp4_kv_func": {
57
+ "status": "available",
58
+ "target": "blackwell-prefill-nvfp4",
59
+ "reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.",
60
+ },
61
+ "sparse_decode_atten_func": {
62
+ "status": "available",
63
+ "target": "v1",
64
+ "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
65
+ },
66
+ "SparseDecodePagedAttentionWrapper": {
67
+ "status": "available",
68
+ "target": "v1",
69
+ "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
70
+ },
71
+ "fp4_indexer_block_scores": {
72
+ "status": "available",
73
+ "target": "blackwell-fp4-indexer",
74
+ "reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.",
75
+ },
76
+ "build_k2q_csr": {
77
+ "status": "available",
78
+ "target": "v1",
79
+ "reason": "CSR construction helper is available for the official prefill API.",
80
+ },
81
+ "SparseK2qCsrBuilderSm100": {
82
+ "status": "available",
83
+ "target": "v1",
84
+ "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
85
+ },
86
+ "Nvfp4QuantizedTensor": {
87
+ "status": "available",
88
+ "target": "v1",
89
+ "reason": "NVFP4 metadata dataclass is available.",
90
+ },
91
+ "quantize_bf16_to_nvfp4_128x4": {
92
+ "status": "available_optional_te",
93
+ "target": "v1",
94
+ "reason": "Function is available and uses Transformer Engine when installed.",
95
+ },
96
+ "quantize_kv_bf16_to_nvfp4_128x4": {
97
+ "status": "available_optional_te",
98
+ "target": "v1",
99
+ "reason": "Function is available and uses Transformer Engine when installed.",
100
+ },
101
+ "dequantize_nvfp4_128x4_to_bf16": {
102
+ "status": "available",
103
+ "target": "v1",
104
+ "reason": "Reference dequant helper is available.",
105
+ },
106
+ "swizzle_nvfp4_scale_to_128x4": {
107
+ "status": "available",
108
+ "target": "v1",
109
+ "reason": "Scale swizzle helper is available.",
110
+ },
111
+ "nvfp4_global_scale_from_amax": {
112
+ "status": "available",
113
+ "target": "v1",
114
+ "reason": "Global scale helper is available.",
115
+ },
116
+ }
117
+
118
+ OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
119
+
120
+
121
+ def available_functions() -> tuple[str, ...]:
122
+ """Return the functions/classes intentionally available in v1."""
123
+
124
+ return V1_AVAILABLE_FUNCTIONS
125
+
126
+
127
+ def official_minimax_msa_functions() -> tuple[str, ...]:
128
+ """Return the upstream MiniMaxAI/msa public function/class names tracked here."""
129
+
130
+ return OFFICIAL_MINIMAX_MSA_FUNCTIONS
131
+
132
+
133
+ def official_api_status() -> dict[str, dict[str, str]]:
134
+ """Return a copy of the official API compatibility status table."""
135
+
136
+ return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
137
+
138
+
139
+ def unsupported_official_functions() -> tuple[str, ...]:
140
+ """Return official MiniMaxAI/msa names that are not exported by this v1 package."""
141
+
142
+ return tuple(
143
+ name
144
+ for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
145
+ if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
146
+ )
build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
build/torch211-cxx11-cu130-x86_64-linux/common/utils.py ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ import functools
4
+ from collections import deque
5
+ from typing import Any, Callable, Tuple
6
+
7
+ import torch
8
+ import triton.language as tl
9
+
10
+ _tma_keep_alive_buf = deque(maxlen=200)
11
+
12
+ try:
13
+ make_tensor_descriptor = tl.make_tensor_descriptor
14
+ except Exception:
15
+ make_tensor_descriptor = tl._experimental_make_tensor_descriptor
16
+
17
+
18
+ def robust_allocator(size: int, alignment: int, stream: int = None):
19
+ """Allocator for Triton TMA descriptors.
20
+
21
+ We keep reference in deque to prevent GC from collecting the buffer.
22
+ """
23
+ tensor = torch.empty(size, device="cuda", dtype=torch.uint8)
24
+ _tma_keep_alive_buf.append(tensor)
25
+ return tensor
26
+
27
+
28
+ def tensor_cache(maxsize: int = 8):
29
+ """
30
+ Cache function results using identity comparison.
31
+ Zero-overhead cache hit: no hash, no DtoH, just pointer comparison.
32
+
33
+ Args:
34
+ maxsize: Maximum number of cached entries. Supports multi-GPU scenarios
35
+ where different devices have different tensor arguments.
36
+ """
37
+
38
+ def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
39
+ # LRU-style cache: list of (args, kwargs, result) tuples
40
+ # Most recently used at the end
41
+ _cache: list = []
42
+
43
+ def _args_match(args: tuple, cached_args: tuple) -> bool:
44
+ if len(args) != len(cached_args):
45
+ return False
46
+ for i in range(len(args)):
47
+ if args[i] is not cached_args[i]:
48
+ return False
49
+ return True
50
+
51
+ def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool:
52
+ if not kwargs and not cached_kwargs:
53
+ return True
54
+ if kwargs.keys() != cached_kwargs.keys():
55
+ return False
56
+ for k, v in kwargs.items():
57
+ if v is not cached_kwargs[k]:
58
+ return False
59
+ return True
60
+
61
+ @functools.wraps(fn)
62
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
63
+ # Search cache (most recent first for better hit rate)
64
+ for i in range(len(_cache) - 1, -1, -1):
65
+ cached_args, cached_kwargs, cached_result = _cache[i]
66
+ if _args_match(args, cached_args) and _kwargs_match(
67
+ kwargs, cached_kwargs
68
+ ):
69
+ # Move to end (most recently used)
70
+ if i != len(_cache) - 1:
71
+ _cache.append(_cache.pop(i))
72
+ return cached_result
73
+
74
+ # Cache miss
75
+ result = fn(*args, **kwargs)
76
+
77
+ # Add to cache
78
+ if len(_cache) >= maxsize:
79
+ _cache.pop(0) # Remove oldest
80
+ _cache.append((args, kwargs, result))
81
+ return result
82
+
83
+ # Expose cache for manual clearing if needed
84
+ wrapper.cache_clear = lambda: _cache.clear()
85
+ wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize}
86
+
87
+ return wrapper
88
+
89
+ return decorator
90
+
91
+
92
+ @tensor_cache(maxsize=8)
93
+ def get_cu_seqblocks(
94
+ cu_seqlens: torch.Tensor,
95
+ max_seqlen: int,
96
+ block_size_q: int,
97
+ block_size_k: int,
98
+ ) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]:
99
+ """Compute cumulative sequence block indices for blocked sparse attention.
100
+
101
+ Converts token-level cumulative sequence lengths to block-level indices,
102
+ which are needed for block-sparse attention kernels.
103
+
104
+ Note:
105
+ Results are cached (maxsize=8) based on input arguments. Repeated calls
106
+ with the same cu_seqlens, max_seqlen, and block sizes will return cached
107
+ results without recomputation.
108
+
109
+ Args:
110
+ cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32.
111
+ max_seqlen: Maximum sequence length in the batch.
112
+ block_size_q: Query block size.
113
+ block_size_k: Key-value block size.
114
+
115
+ Returns:
116
+ A tuple of 6 values:
117
+ - cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1]
118
+ - max_seqblock_q: Maximum number of query blocks per sequence.
119
+ - all_seqblock_q: Total number of query blocks across all sequences.
120
+ - cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1]
121
+ - max_seqblock_k: Maximum number of key blocks per sequence.
122
+ - all_seqblock_k: Total number of key blocks across all sequences.
123
+ """
124
+ cu_seqblocks_q = torch.zeros_like(cu_seqlens)
125
+ cu_seqblocks_k = torch.zeros_like(cu_seqlens)
126
+ seq_lens = torch.diff(cu_seqlens)
127
+ seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q
128
+ seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k
129
+ max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q
130
+ max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k
131
+ cu_seqblocks_q[1:] = seqblocks_q
132
+ cu_seqblocks_k[1:] = seqblocks_k
133
+ cu_seqblocks_q.cumsum_(0)
134
+ cu_seqblocks_k.cumsum_(0)
135
+ all_seqblock_q = seqblocks_q.sum().item()
136
+ all_seqblock_k = seqblocks_k.sum().item()
137
+ return (
138
+ cu_seqblocks_q,
139
+ max_seqblock_q,
140
+ all_seqblock_q,
141
+ cu_seqblocks_k,
142
+ max_seqblock_k,
143
+ all_seqblock_k,
144
+ )
build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py ADDED
@@ -0,0 +1,1150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .._compat import envs # vendored: was `from sglang.srt.environ import envs`
10
+
11
+ from ..common.utils import robust_allocator
12
+ from .._native import has_native_ops, native_topk_from_scores
13
+
14
+
15
+ @triton.heuristics(
16
+ {
17
+ "BLOCK_SIZE_H": lambda args: max(
18
+ 16, triton.next_power_of_2(args["gqa_group_size"])
19
+ ),
20
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
21
+ "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
22
+ }
23
+ )
24
+ @triton.autotune(
25
+ configs=[
26
+ triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
27
+ for BN in [64, 128, 256, 512]
28
+ for nw in [4, 8, 16]
29
+ for ns in [1, 2, 3]
30
+ ],
31
+ key=[
32
+ "BATCH_SIZE_BUCKET",
33
+ "gqa_group_size",
34
+ "head_dim",
35
+ "block_size",
36
+ "SCORE_TYPE",
37
+ ],
38
+ )
39
+ @triton.jit
40
+ def _decode_score_kernel(
41
+ q_ptr, # Q: b x qh x d
42
+ k_cache_ptr, # K paged: max_slots x kh x d
43
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
44
+ score_ptr, # Score: qh x b x max_seqblock
45
+ seq_lens,
46
+ slot_ids,
47
+ # shape
48
+ max_slots,
49
+ batch_size,
50
+ gqa_group_size,
51
+ head_dim,
52
+ # block size
53
+ block_size: tl.constexpr,
54
+ topk: tl.constexpr,
55
+ # sm_scale
56
+ sm_scale,
57
+ # init and local blocks
58
+ init_blocks,
59
+ local_blocks,
60
+ # stride
61
+ stride_q_b,
62
+ stride_q_h,
63
+ stride_q_d,
64
+ stride_k_s,
65
+ stride_k_h,
66
+ stride_k_d,
67
+ stride_r2t_b,
68
+ stride_s_h,
69
+ stride_s_b,
70
+ stride_s_n,
71
+ # META parameters
72
+ BATCH_SIZE_BUCKET: tl.constexpr,
73
+ BLOCK_SIZE_H: tl.constexpr,
74
+ BLOCK_SIZE_N: tl.constexpr,
75
+ BLOCK_SIZE_D: tl.constexpr,
76
+ NUM_KV_CHUNKS: tl.constexpr,
77
+ SCORE_TYPE: tl.constexpr,
78
+ SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
79
+ ):
80
+ tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
81
+ sm_scale_log2e = sm_scale * 1.4426950409
82
+ tl.static_assert(BLOCK_SIZE_N >= block_size)
83
+ BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
84
+ # get batch id and head id
85
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
86
+ pid_b = pid_bc % batch_size
87
+ pid_c = pid_bc // batch_size
88
+ pid_h = pid_kh * gqa_group_size
89
+ # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
90
+ seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
91
+ num_blocks = (seq_len + block_size - 1) // block_size
92
+ if SKIP_TRIVIAL_TOPK_SCORE:
93
+ if num_blocks <= topk:
94
+ return
95
+ chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
96
+ chunk_start_block = pid_c * chunk_size_blocks
97
+ chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
98
+ chunk_start = chunk_start_block * block_size
99
+ chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
100
+ if chunk_start_block >= chunk_end_block:
101
+ return
102
+ sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots
103
+ # init qkv pointer
104
+ q_ptrs = tl.make_block_ptr(
105
+ base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
106
+ shape=(gqa_group_size, head_dim),
107
+ strides=(stride_q_h, stride_q_d),
108
+ offsets=(0, 0),
109
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
110
+ order=(1, 0),
111
+ )
112
+ s_ptrs = tl.make_block_ptr(
113
+ base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
114
+ shape=(gqa_group_size, chunk_end_block),
115
+ strides=(stride_s_h, stride_s_n),
116
+ offsets=(0, chunk_start_block),
117
+ block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
118
+ order=(1, 0),
119
+ )
120
+ # load q
121
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
122
+ # init statistics
123
+ off_n = tl.arange(0, BLOCK_SIZE_N)
124
+ off_d = tl.arange(0, BLOCK_SIZE_D)
125
+ off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
126
+ dim_mask = off_d < head_dim
127
+ # pre-compute local_start outside the loop (constant for entire batch)
128
+ local_start = tl.maximum(0, num_blocks - local_blocks)
129
+ # prefetch first iteration's slots
130
+ r2t_base = req_to_token_ptr + sid * stride_r2t_b
131
+ prefetch_pos = chunk_start + off_n
132
+ prefetch_mask = prefetch_pos < seq_len
133
+ prefetched_slots = tl.load(
134
+ r2t_base + prefetch_pos,
135
+ mask=prefetch_mask,
136
+ other=0,
137
+ ).to(tl.int64)
138
+ # score-only: compute block scores without loading V
139
+ for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
140
+ pos_mask = prefetch_mask
141
+ slots = prefetched_slots
142
+ # prefetch next iteration's slots
143
+ next_i = i + BLOCK_SIZE_N
144
+ if next_i < chunk_end:
145
+ next_pos = next_i + off_n
146
+ prefetch_mask = next_pos < seq_len
147
+ prefetched_slots = tl.load(
148
+ r2t_base + next_pos,
149
+ mask=prefetch_mask,
150
+ other=0,
151
+ ).to(tl.int64)
152
+ slots = (slots + max_slots) % max_slots
153
+ # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
154
+ k_off = (
155
+ slots[None, :] * stride_k_s
156
+ + pid_kh * stride_k_h
157
+ + off_d[:, None] * stride_k_d
158
+ )
159
+ k = tl.load(
160
+ k_cache_ptr + k_off,
161
+ mask=dim_mask[:, None] & pos_mask[None, :],
162
+ other=0.0,
163
+ )
164
+ # compute qk
165
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
166
+ qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
167
+ # [H, D], [D, N] -> [H, N]
168
+ qk += tl.dot(q, k) * sm_scale_log2e
169
+ # save qk to score
170
+ score = tl.reshape(
171
+ qk,
172
+ (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
173
+ can_reorder=False,
174
+ )
175
+ sub_max = tl.max(score, axis=2)
176
+ if SCORE_TYPE == "max":
177
+ score = sub_max
178
+ else: # "lse"
179
+ score = sub_max + tl.log2(
180
+ tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
181
+ )
182
+ score = tl.where(score != score, float("-inf"), score)
183
+ # apply init_blocks and local_blocks efficiently
184
+ curr_block_idx = i // block_size + off_bpk
185
+ is_init = curr_block_idx < init_blocks
186
+ is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
187
+ score = tl.where(
188
+ is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
189
+ )
190
+ tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1))
191
+ # update ptrs
192
+ s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
193
+
194
+
195
+ @triton.heuristics(
196
+ {
197
+ "BLOCK_SIZE_H": lambda args: max(
198
+ 16, triton.next_power_of_2(args["gqa_group_size"])
199
+ ),
200
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
201
+ "HAS_SINK": lambda args: args["sink_ptr"] is not None,
202
+ }
203
+ )
204
+ @triton.autotune(
205
+ configs=[
206
+ triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
207
+ for BN in [64, 128, 256, 512]
208
+ for nw in [4, 8, 16]
209
+ for ns in [1, 2, 3]
210
+ ],
211
+ key=[
212
+ "gqa_group_size",
213
+ "head_dim",
214
+ "block_size",
215
+ "HAS_SINK",
216
+ "SCORE_TYPE",
217
+ ],
218
+ )
219
+ @triton.jit
220
+ def _decode_score_attn_kernel(
221
+ q_ptr, # Q: b x qh x d
222
+ sink_ptr, # Sink: qh x d
223
+ k_cache_ptr, # K paged: max_slots x kh x d
224
+ v_cache_ptr, # V paged: max_slots x kh x d
225
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
226
+ o_ptr, # O: c x b x qh x d
227
+ lse_ptr, # lse: c x b x qh
228
+ score_ptr, # Score: qh x b x max_seqblock
229
+ seq_lens,
230
+ slot_ids,
231
+ # shape
232
+ max_slots,
233
+ batch_size,
234
+ gqa_group_size,
235
+ head_dim,
236
+ # block size
237
+ block_size: tl.constexpr,
238
+ topk: tl.constexpr,
239
+ # sm_scale
240
+ sm_scale,
241
+ # init and local blocks
242
+ init_blocks,
243
+ local_blocks,
244
+ # stride
245
+ stride_q_b,
246
+ stride_q_h,
247
+ stride_q_d,
248
+ stride_sink_h,
249
+ stride_sink_d,
250
+ stride_k_s,
251
+ stride_k_h,
252
+ stride_k_d,
253
+ stride_v_s,
254
+ stride_v_h,
255
+ stride_v_d,
256
+ stride_r2t_b,
257
+ stride_o_c,
258
+ stride_o_b,
259
+ stride_o_h,
260
+ stride_o_d,
261
+ stride_l_c,
262
+ stride_l_b,
263
+ stride_l_h,
264
+ stride_s_h,
265
+ stride_s_b,
266
+ stride_s_n,
267
+ # META parameters
268
+ BLOCK_SIZE_H: tl.constexpr,
269
+ BLOCK_SIZE_N: tl.constexpr,
270
+ BLOCK_SIZE_D: tl.constexpr,
271
+ NUM_KV_CHUNKS: tl.constexpr,
272
+ HAS_SINK: tl.constexpr,
273
+ SCORE_TYPE: tl.constexpr,
274
+ SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
275
+ ):
276
+ tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
277
+ sm_scale_log2e = sm_scale * 1.4426950409
278
+ tl.static_assert(BLOCK_SIZE_N >= block_size)
279
+ BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
280
+ # get batch id and head id
281
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
282
+ pid_b = pid_bc % batch_size
283
+ pid_c = pid_bc // batch_size
284
+ pid_h = pid_kh * gqa_group_size
285
+ # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
286
+ seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
287
+ num_blocks = (seq_len + block_size - 1) // block_size
288
+ chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
289
+ chunk_start_block = pid_c * chunk_size_blocks
290
+ chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
291
+ chunk_start = chunk_start_block * block_size
292
+ chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
293
+ if chunk_start_block >= chunk_end_block:
294
+ return
295
+ sid = (
296
+ tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
297
+ ) % max_slots # to avoid bugs when slot_ids is negative
298
+ # init qkv pointer
299
+ q_ptrs = tl.make_block_ptr(
300
+ base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
301
+ shape=(gqa_group_size, head_dim),
302
+ strides=(stride_q_h, stride_q_d),
303
+ offsets=(0, 0),
304
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
305
+ order=(1, 0),
306
+ )
307
+ s_ptrs = tl.make_block_ptr(
308
+ base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
309
+ shape=(gqa_group_size, chunk_end_block),
310
+ strides=(stride_s_h, stride_s_n),
311
+ offsets=(0, chunk_start_block),
312
+ block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
313
+ order=(1, 0),
314
+ )
315
+ # load q
316
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
317
+ # init statistics
318
+ off_n = tl.arange(0, BLOCK_SIZE_N)
319
+ off_d = tl.arange(0, BLOCK_SIZE_D)
320
+ off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
321
+ dim_mask = off_d < head_dim
322
+ if HAS_SINK:
323
+ if pid_c == 0:
324
+ sink_ptrs = tl.make_block_ptr(
325
+ base=sink_ptr + pid_h * stride_sink_h,
326
+ shape=(gqa_group_size, head_dim),
327
+ strides=(stride_sink_h, stride_sink_d),
328
+ offsets=(0, 0),
329
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
330
+ order=(1, 0),
331
+ )
332
+ sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
333
+ tl.float32
334
+ )
335
+ qsink = (
336
+ tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e
337
+ ) # (BLOCK_SIZE_H,)
338
+ m_i = qsink
339
+ l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32)
340
+ else:
341
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
342
+ l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
343
+ else:
344
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
345
+ l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
346
+ acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
347
+ # pre-compute local_start outside the loop (constant for entire batch)
348
+ local_start = tl.maximum(0, num_blocks - local_blocks)
349
+ # full attention or causal attention
350
+ for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
351
+ pos = i + off_n
352
+ pos_mask = pos < seq_len
353
+ # resolve slot per position via req_to_token
354
+ slots = tl.load(
355
+ req_to_token_ptr + sid * stride_r2t_b + pos,
356
+ mask=pos_mask,
357
+ other=0,
358
+ ).to(tl.int64)
359
+ slots = (slots + max_slots) % max_slots # safety against negative
360
+ # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
361
+ k_off = (
362
+ slots[None, :] * stride_k_s
363
+ + pid_kh * stride_k_h
364
+ + off_d[:, None] * stride_k_d
365
+ )
366
+ k = tl.load(
367
+ k_cache_ptr + k_off,
368
+ mask=dim_mask[:, None] & pos_mask[None, :],
369
+ other=0.0,
370
+ )
371
+ # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
372
+ v_off = (
373
+ slots[:, None] * stride_v_s
374
+ + pid_kh * stride_v_h
375
+ + off_d[None, :] * stride_v_d
376
+ )
377
+ v = tl.load(
378
+ v_cache_ptr + v_off,
379
+ mask=pos_mask[:, None] & dim_mask[None, :],
380
+ other=0.0,
381
+ )
382
+ # compute qk
383
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
384
+ qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
385
+ # [H, D], [D, N] -> [H, N]
386
+ qk += tl.dot(q, k) * sm_scale_log2e
387
+ # save qk to score
388
+ score = tl.reshape(
389
+ qk,
390
+ (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
391
+ can_reorder=False,
392
+ )
393
+ sub_max = tl.max(score, axis=2)
394
+ if SCORE_TYPE == "max":
395
+ score = sub_max
396
+ else: # "lse"
397
+ score = sub_max + tl.log2(
398
+ tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
399
+ )
400
+ score = tl.where(score != score, float("-inf"), score)
401
+ # apply init_blocks and local_blocks efficiently
402
+ # current block indices: [BLOCKS_PER_K_BLOCK]
403
+ curr_block_idx = i // block_size + off_bpk
404
+ # Combined condition with single nested tl.where:
405
+ # - local_blocks (1e29) takes priority over init_blocks (1e30)
406
+ # - When init_blocks=0: curr_block_idx < 0 is always False
407
+ # - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False
408
+ is_init = curr_block_idx < init_blocks
409
+ is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
410
+ # Apply in one fused operation: local > init > original
411
+ score = tl.where(
412
+ is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
413
+ )
414
+ if SKIP_TRIVIAL_TOPK_SCORE:
415
+ if num_blocks > topk:
416
+ tl.store(
417
+ s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
418
+ )
419
+ else:
420
+ tl.store(
421
+ s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
422
+ )
423
+ # max-of-max == max(qk), avoids re-scanning qk
424
+ m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1))
425
+ p = tl.exp2(qk - m_ij[:, None])
426
+ l_ij = tl.sum(p, axis=1)
427
+ acc_o_scale = tl.exp2(m_i - m_ij)
428
+ acc_o = acc_o * acc_o_scale[:, None]
429
+ # [H, N], [N, D] -> [H, D]
430
+ acc_o += tl.dot(p.to(v.dtype), v)
431
+ m_i = m_ij
432
+ l_i = l_i * acc_o_scale + l_ij
433
+ # update ptrs
434
+ s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
435
+ # final scale
436
+ acc_o = acc_o / l_i[:, None]
437
+ # compute lse in log2 scale for merge kernel
438
+ lse_i = m_i + tl.log2(l_i)
439
+ # save output
440
+ o_ptrs = tl.make_block_ptr(
441
+ base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
442
+ shape=(gqa_group_size, head_dim),
443
+ strides=(stride_o_h, stride_o_d),
444
+ offsets=(0, 0),
445
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
446
+ order=(1, 0),
447
+ )
448
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
449
+ # save lse (in log2 scale, will be used directly in merge kernel)
450
+ lse_ptrs = tl.make_block_ptr(
451
+ base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
452
+ shape=(gqa_group_size,),
453
+ strides=(stride_l_h,),
454
+ offsets=(0,),
455
+ block_shape=(BLOCK_SIZE_H,),
456
+ order=(0,),
457
+ )
458
+ tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
459
+
460
+
461
+ @triton.heuristics(
462
+ {
463
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
464
+ }
465
+ )
466
+ @triton.autotune(
467
+ configs=[
468
+ triton.Config({}, num_warps=nw, num_stages=ns)
469
+ for nw in [4, 8]
470
+ for ns in [2, 3, 4]
471
+ ],
472
+ key=["BLOCK_SIZE_D"],
473
+ restore_value=["o_ptr"],
474
+ )
475
+ @triton.jit
476
+ def _merge_attn_out_kernel(
477
+ o_ptr,
478
+ lse_ptr,
479
+ # shape
480
+ seq_lens,
481
+ head_dim,
482
+ block_size: tl.constexpr,
483
+ # stride
484
+ stride_o_c,
485
+ stride_o_b,
486
+ stride_o_h,
487
+ stride_o_d,
488
+ stride_l_c,
489
+ stride_l_b,
490
+ stride_l_h,
491
+ NUM_KV_CHUNKS: tl.constexpr,
492
+ BLOCK_SIZE_D: tl.constexpr,
493
+ ):
494
+ pid_b, pid_h = tl.program_id(0), tl.program_id(1)
495
+ # recompute chunk layout (must match _decode_score_attn_kernel)
496
+ seq_len = tl.load(seq_lens + pid_b)
497
+ num_blocks = (seq_len + block_size - 1) // block_size
498
+ chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
499
+ valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks)
500
+ # ptrs
501
+ off_c = tl.arange(0, NUM_KV_CHUNKS)
502
+ off_d = tl.arange(0, BLOCK_SIZE_D)
503
+ o_ptrs = tl.make_block_ptr(
504
+ base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
505
+ shape=(valid_chunks, head_dim),
506
+ strides=(stride_o_c, stride_o_d),
507
+ offsets=(0, 0),
508
+ block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D),
509
+ order=(1, 0),
510
+ )
511
+ lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
512
+ # load o and lse
513
+ o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
514
+ lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf"))
515
+ # merge o (lse is in log2 scale, use exp2/log2 for efficiency)
516
+ lse_shifted = lse - tl.max(lse, axis=0)
517
+ log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0))
518
+ scale = tl.exp2(lse_shifted - log_sum)
519
+ o = tl.sum(o * scale[:, None], axis=0)
520
+ # save o
521
+ o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
522
+ tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
523
+
524
+
525
+ @triton.jit
526
+ def _compare_and_swap(
527
+ x,
528
+ ids,
529
+ flip,
530
+ i: tl.constexpr,
531
+ n_dims: tl.constexpr,
532
+ ):
533
+ n_outer: tl.constexpr = x.numel >> n_dims
534
+ shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
535
+ y = tl.reshape(x, shape)
536
+ # slice left/right with 'stride' 2**(n_dims - i - 1)
537
+ mask = tl.arange(0, 2)[None, :, None]
538
+ left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
539
+ right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
540
+ left = tl.reshape(left, x.shape)
541
+ right = tl.reshape(right, x.shape)
542
+ # idx
543
+ y_idx = tl.reshape(ids, shape)
544
+ left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
545
+ right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
546
+ left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
547
+ right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
548
+ # actual compare-and-swap
549
+ idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
550
+ ileft = left.to(idtype, bitcast=True)
551
+ iright = right.to(idtype, bitcast=True)
552
+ ix = x.to(idtype, bitcast=True)
553
+
554
+ cond = (left > right) != flip
555
+ ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
556
+ new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
557
+ return ret.to(x.dtype, bitcast=True), new_ids
558
+
559
+
560
+ @triton.jit
561
+ def _bitonic_merge(
562
+ x,
563
+ ids,
564
+ stage: tl.constexpr,
565
+ order: tl.constexpr,
566
+ n_dims: tl.constexpr,
567
+ ):
568
+ n_outer: tl.constexpr = x.numel >> n_dims
569
+ tl.static_assert(stage <= n_dims)
570
+ # flip denotes whether to re-arrange sub-sequences of elements in ascending or
571
+ # descending order.
572
+ # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage
573
+ # if flip = 00110011... then all the elements will be re-arranged alternatingly (with
574
+ # a stride of 2) at this stage
575
+ if order == 2:
576
+ shape: tl.constexpr = [
577
+ n_outer * 2 ** (n_dims - 1 - stage),
578
+ 2,
579
+ 2**stage,
580
+ ]
581
+ flip = tl.reshape(
582
+ tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
583
+ )
584
+ else:
585
+ flip = order
586
+ # perform `stage` rounds of `compare-and-swap`
587
+ for i in tl.static_range(stage):
588
+ x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
589
+ return x, ids
590
+
591
+
592
+ @triton.heuristics(
593
+ {
594
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
595
+ }
596
+ )
597
+ @triton.autotune(
598
+ configs=[
599
+ triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
600
+ triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2),
601
+ triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
602
+ triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3),
603
+ triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
604
+ ],
605
+ key=["topk"],
606
+ )
607
+ @triton.jit
608
+ def _topk_index_partial_kernel(
609
+ s_ptr, # Score: [num_q_heads, BS, max_seqblock]
610
+ ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
611
+ ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape
612
+ seq_lens,
613
+ block_size: tl.constexpr,
614
+ topk: tl.constexpr,
615
+ chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns
616
+ # strides
617
+ stride_s_h,
618
+ stride_s_b,
619
+ stride_s_k,
620
+ stride_ts_c,
621
+ stride_ts_h,
622
+ stride_ts_b,
623
+ stride_ts_t,
624
+ stride_ti_c,
625
+ stride_ti_h,
626
+ stride_ti_b,
627
+ stride_ti_t,
628
+ # META
629
+ BLOCK_SIZE_K: tl.constexpr,
630
+ BLOCK_SIZE_T: tl.constexpr,
631
+ ):
632
+ tl.static_assert(BLOCK_SIZE_K > topk)
633
+ pid_b = tl.program_id(0)
634
+ pid_h = tl.program_id(1)
635
+ pid_chunk = tl.program_id(2)
636
+
637
+ seq_len = tl.load(seq_lens + pid_b)
638
+ num_blocks = (seq_len + block_size - 1) // block_size
639
+
640
+ # Slice this chunk owns within [0, num_blocks).
641
+ chunk_start = pid_chunk * chunk_blocks
642
+ chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
643
+ chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
644
+
645
+ off_k = tl.arange(0, BLOCK_SIZE_K)
646
+ off_t = tl.arange(0, BLOCK_SIZE_T)
647
+
648
+ # Pointer starts at chunk_start.
649
+ s_ptrs = (
650
+ s_ptr
651
+ + pid_b * stride_s_b
652
+ + pid_h * stride_s_h
653
+ + (chunk_start + off_k) * stride_s_k
654
+ )
655
+
656
+ topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
657
+ topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
658
+ left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
659
+
660
+ # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
661
+ # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
662
+ for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
663
+ mask = off_k < chunk_actual - i
664
+ score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
665
+ score = tl.where(score != score, -1e30, score)
666
+ s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
667
+ topk_score, last_topk_score = score, topk_score
668
+ topk_idx, last_topk_idx = (
669
+ tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
670
+ topk_idx,
671
+ )
672
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
673
+ for j in tl.static_range(1, n_dims):
674
+ topk_score, topk_idx = _bitonic_merge(
675
+ topk_score, topk_idx.to(tl.int32), j, 2, n_dims
676
+ )
677
+ if i != 0:
678
+ topk_score, topk_idx = _bitonic_merge(
679
+ topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
680
+ )
681
+ topk_score_new = last_topk_score * left_half_mask + topk_score * (
682
+ 1 - left_half_mask
683
+ )
684
+ topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
685
+ 1 - left_half_mask
686
+ )
687
+ topk_score, topk_idx = _bitonic_merge(
688
+ topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
689
+ )
690
+ else:
691
+ topk_score, topk_idx = _bitonic_merge(
692
+ topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
693
+ )
694
+
695
+ # Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort).
696
+ topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
697
+ final_score = tl.sum(
698
+ topk_mask_extract[:, None]
699
+ * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
700
+ axis=0,
701
+ )
702
+ final_idx = tl.sum(
703
+ topk_mask_extract[:, None]
704
+ * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
705
+ axis=0,
706
+ )
707
+
708
+ # Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry
709
+ # -1e30 / 0 sentinels and lose to real scores in the merge stage.
710
+ ts_ptrs = (
711
+ ts_partial_ptr
712
+ + pid_chunk * stride_ts_c
713
+ + pid_b * stride_ts_b
714
+ + pid_h * stride_ts_h
715
+ + off_t * stride_ts_t
716
+ )
717
+ ti_ptrs = (
718
+ ti_partial_ptr
719
+ + pid_chunk * stride_ti_c
720
+ + pid_b * stride_ti_b
721
+ + pid_h * stride_ti_h
722
+ + off_t * stride_ti_t
723
+ )
724
+ tl.store(ts_ptrs, final_score)
725
+ tl.store(ti_ptrs, final_idx)
726
+
727
+
728
+ @triton.heuristics(
729
+ {
730
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
731
+ "BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
732
+ args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"])
733
+ ),
734
+ }
735
+ )
736
+ @triton.jit
737
+ def _topk_index_merge_kernel(
738
+ ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
739
+ ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape
740
+ ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk]
741
+ seq_lens,
742
+ block_size: tl.constexpr,
743
+ topk: tl.constexpr,
744
+ # strides
745
+ stride_ts_c,
746
+ stride_ts_h,
747
+ stride_ts_b,
748
+ stride_ts_t,
749
+ stride_ti_c,
750
+ stride_ti_h,
751
+ stride_ti_b,
752
+ stride_ti_t,
753
+ stride_tif_h,
754
+ stride_tif_b,
755
+ stride_tif_t,
756
+ # META
757
+ NUM_TOPK_CHUNKS: tl.constexpr,
758
+ BLOCK_SIZE_K: tl.constexpr,
759
+ BLOCK_SIZE_T: tl.constexpr,
760
+ ):
761
+ pid_b = tl.program_id(0)
762
+ pid_h = tl.program_id(1)
763
+
764
+ seq_len = tl.load(seq_lens + pid_b)
765
+ num_blocks = (seq_len + block_size - 1) // block_size
766
+
767
+ # Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K.
768
+ # candidate at flat position p comes from chunk = p // BLOCK_SIZE_T,
769
+ # in_chunk = p % BLOCK_SIZE_T.
770
+ off = tl.arange(0, BLOCK_SIZE_K)
771
+ chunk_idx = off // BLOCK_SIZE_T
772
+ in_chunk_idx = off % BLOCK_SIZE_T
773
+ valid = chunk_idx < NUM_TOPK_CHUNKS
774
+
775
+ score_offset = (
776
+ chunk_idx * stride_ts_c
777
+ + pid_h * stride_ts_h
778
+ + pid_b * stride_ts_b
779
+ + in_chunk_idx * stride_ts_t
780
+ )
781
+ idx_offset = (
782
+ chunk_idx * stride_ti_c
783
+ + pid_h * stride_ti_h
784
+ + pid_b * stride_ti_b
785
+ + in_chunk_idx * stride_ti_t
786
+ )
787
+
788
+ score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
789
+ tl.float32
790
+ )
791
+ score = tl.where(score != score, -1e30, score)
792
+ idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
793
+
794
+ # Full bitonic descending sort of BLOCK_SIZE_K items.
795
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
796
+ for j in tl.static_range(1, n_dims):
797
+ score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
798
+ score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
799
+
800
+ # Extract first BLOCK_SIZE_T positions — these are the global top-K.
801
+ extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
802
+ topk_idx_final = tl.sum(
803
+ extract_mask[:, None]
804
+ * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
805
+ axis=0,
806
+ )
807
+
808
+ off_t = tl.arange(0, BLOCK_SIZE_T)
809
+ tif_ptrs = (
810
+ ti_final_ptr
811
+ + pid_h * stride_tif_h
812
+ + pid_b * stride_tif_b
813
+ + off_t * stride_tif_t
814
+ )
815
+ topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
816
+ tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty))
817
+
818
+
819
+ @torch.no_grad()
820
+ def flash_decode_with_topk_idx(
821
+ q: torch.Tensor, # [batch_size, num_heads, head_dim]
822
+ sink: Optional[torch.Tensor],
823
+ k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
824
+ v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True
825
+ req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
826
+ seq_lens: torch.Tensor, # [batch_size, ]
827
+ max_seqlen: int,
828
+ slot_ids: torch.Tensor, # [batch_size, ]
829
+ block_size: int,
830
+ topk: int,
831
+ init_blocks: int,
832
+ local_blocks: int,
833
+ sm_scale: Optional[float] = None,
834
+ use_tma: bool = True,
835
+ score_type: str = "max",
836
+ disable_index_value: bool = False,
837
+ use_dense_main_attn: bool = False, # NOTE: need transform idx in this case
838
+ page_size: int = 1,
839
+ ) -> torch.Tensor:
840
+ assert score_type in (
841
+ "max",
842
+ "lse",
843
+ ), f"score_type must be 'max' or 'lse', got {score_type!r}"
844
+ triton.set_allocator(robust_allocator)
845
+ # dtype check
846
+ assert (
847
+ q.dtype == torch.bfloat16
848
+ or q.dtype == torch.float16
849
+ and k_cache.dtype == q.dtype
850
+ )
851
+ if not disable_index_value:
852
+ assert v_cache is not None
853
+ # shape
854
+ batch_size, num_q_heads, head_dim = q.shape
855
+ max_slots, num_kv_heads, _ = k_cache.shape
856
+ max_kv_len = req_to_token.shape[1]
857
+ assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
858
+ # gqa
859
+ assert num_q_heads % num_kv_heads == 0
860
+ gqa_group_size = num_q_heads // num_kv_heads
861
+ # sm scale
862
+ if sm_scale is None:
863
+ sm_scale = head_dim**-0.5
864
+ # NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets.
865
+ # Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads.
866
+ # TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by:
867
+ # NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS)
868
+ # then rounded down to the nearest power of 2 (merge kernel requires it).
869
+ # Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len.
870
+ # Empty chunks early-return cheaply, so over-chunking is nearly free.
871
+ # E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096.
872
+ TARGET_GRID = 4096
873
+ MAX_NUM_KV_CHUNKS = 256
874
+ target = max(
875
+ 1,
876
+ min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)),
877
+ )
878
+ NUM_KV_CHUNKS = 1 << (target.bit_length() - 1)
879
+ score_kv_len = min(max_seqlen, max_kv_len)
880
+ # The score producers below write every valid block column
881
+ # [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers
882
+ # clamp their scan to the same per-row valid block count, so columns beyond
883
+ # seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph
884
+ # capture the static score shape can be much larger than the live context.
885
+ score = torch.empty(
886
+ (num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)),
887
+ dtype=torch.float32,
888
+ device=q.device,
889
+ )
890
+ use_jit_topk = (
891
+ envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get()
892
+ and score.shape[2] <= 4096
893
+ and topk <= 32
894
+ )
895
+ # If the live context has <= topk sparse blocks, the downstream dense
896
+ # page-table/JIT top-k kernels select every block from seq_lens directly
897
+ # without reading score. Keep this gate in sync with the consumers below:
898
+ # _skip_block_topk/use_dense_main_attn and use_jit_topk special-case
899
+ # num_blocks <= topk, while the Triton fallback reads score and must not
900
+ # skip these writes.
901
+ skip_trivial_topk_score = use_dense_main_attn or use_jit_topk
902
+
903
+ grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads)
904
+ if disable_index_value:
905
+ _decode_score_kernel[grid](
906
+ q,
907
+ k_cache,
908
+ req_to_token,
909
+ score,
910
+ seq_lens,
911
+ slot_ids,
912
+ max_slots,
913
+ batch_size,
914
+ gqa_group_size,
915
+ head_dim,
916
+ block_size,
917
+ topk,
918
+ sm_scale,
919
+ init_blocks,
920
+ local_blocks,
921
+ q.stride(0),
922
+ q.stride(1),
923
+ q.stride(2),
924
+ k_cache.stride(0),
925
+ k_cache.stride(1),
926
+ k_cache.stride(2),
927
+ req_to_token.stride(0),
928
+ score.stride(0),
929
+ score.stride(1),
930
+ score.stride(2),
931
+ NUM_KV_CHUNKS=NUM_KV_CHUNKS,
932
+ SCORE_TYPE=score_type,
933
+ SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
934
+ )
935
+ else:
936
+ assert v_cache is not None
937
+ o = torch.empty(
938
+ NUM_KV_CHUNKS,
939
+ batch_size,
940
+ num_q_heads,
941
+ head_dim,
942
+ dtype=q.dtype,
943
+ device=q.device,
944
+ )
945
+ lse = torch.empty(
946
+ NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device
947
+ )
948
+ _decode_score_attn_kernel[grid](
949
+ q,
950
+ sink,
951
+ k_cache,
952
+ v_cache,
953
+ req_to_token,
954
+ o,
955
+ lse,
956
+ score,
957
+ seq_lens,
958
+ slot_ids,
959
+ max_slots,
960
+ batch_size,
961
+ gqa_group_size,
962
+ head_dim,
963
+ block_size,
964
+ topk,
965
+ sm_scale,
966
+ init_blocks,
967
+ local_blocks,
968
+ q.stride(0),
969
+ q.stride(1),
970
+ q.stride(2),
971
+ sink.stride(0) if sink is not None else 0,
972
+ sink.stride(1) if sink is not None else 0,
973
+ k_cache.stride(0),
974
+ k_cache.stride(1),
975
+ k_cache.stride(2),
976
+ v_cache.stride(0),
977
+ v_cache.stride(1),
978
+ v_cache.stride(2),
979
+ req_to_token.stride(0),
980
+ o.stride(0),
981
+ o.stride(1),
982
+ o.stride(2),
983
+ o.stride(3),
984
+ lse.stride(0),
985
+ lse.stride(1),
986
+ lse.stride(2),
987
+ score.stride(0),
988
+ score.stride(1),
989
+ score.stride(2),
990
+ NUM_KV_CHUNKS=NUM_KV_CHUNKS,
991
+ SCORE_TYPE=score_type,
992
+ SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
993
+ )
994
+ # Fused top-k + page-table transform: emit the dense backend's page table
995
+ # directly (page-size-aware) instead of block ids, skipping a separate gather.
996
+ # The page table + per-query effective KV length are allocated and returned.
997
+ real_seq_lens = None
998
+ if use_dense_main_attn:
999
+ # VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel
1000
+ # `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The
1001
+ # standalone vendor only ships the pure-Triton path, so this is gated off
1002
+ # by default (use_dense_main_attn=False). If you opt in, vendor that .cuh
1003
+ # too. Original line: `from sglang.jit_kernel.minimax_decode_topk import (
1004
+ # minimax_decode_topk_page_table)`.
1005
+ raise NotImplementedError(
1006
+ "use_dense_main_attn=True requires the un-vendored CUDA JIT kernel "
1007
+ "sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; "
1008
+ "the standalone Triton vendor only supports use_dense_main_attn=False."
1009
+ )
1010
+ # num_q_heads (= num index/kv heads) may be > 1 for DP attention; the
1011
+ # kernel emits a flattened [batch*num_q_heads] page table + seq_lens with
1012
+ # the kv head head-encoded (head-minor) into the page index.
1013
+ topk_idx, real_seq_lens = minimax_decode_topk_page_table(
1014
+ score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size
1015
+ )
1016
+ if disable_index_value:
1017
+ return None, topk_idx, real_seq_lens
1018
+ # fall through to the idx_o (index value) merge, then return
1019
+ _skip_block_topk = True
1020
+ else:
1021
+ _skip_block_topk = False
1022
+
1023
+ # get topk index
1024
+ if not _skip_block_topk:
1025
+ topk_idx = torch.empty(
1026
+ (num_q_heads, batch_size, topk),
1027
+ device=score.device,
1028
+ dtype=torch.int32,
1029
+ )
1030
+
1031
+ if _skip_block_topk:
1032
+ pass
1033
+ elif use_jit_topk:
1034
+ # Single-stage JIT radix-select: one kernel, no intermediate buffers.
1035
+ # Equivalent output to the 2-stage path (set of block ids, front-packed,
1036
+ # -1 padded); ~2-16x faster for long context.
1037
+ # VENDOR NOTE: this needs the un-vendored CUDA JIT kernel
1038
+ # `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the
1039
+ # `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so
1040
+ # this branch is never taken in the standalone vendor (the 2-stage Triton
1041
+ # `else` fallback below runs instead). Original line:
1042
+ # `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`.
1043
+ raise NotImplementedError(
1044
+ "use_jit_topk requires the un-vendored CUDA JIT kernel "
1045
+ "sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor "
1046
+ "uses the 2-stage Triton fallback (keep "
1047
+ "SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)."
1048
+ )
1049
+ elif has_native_ops() and topk <= 64:
1050
+ topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk)
1051
+ else:
1052
+ # 2-stage split-K Triton fallback.
1053
+ # Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim.
1054
+ # Same constraints as flash_decode: must be deterministic from values
1055
+ # constant within a cuda graph (BS, num_q_heads), pow2, capped so the
1056
+ # merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk))
1057
+ # stays reasonable. With topk=32 and cap 16, merge sorts 512 items.
1058
+ TOPK_TARGET_GRID = 64
1059
+ MAX_NUM_TOPK_CHUNKS = 16
1060
+ topk_target = max(
1061
+ 1,
1062
+ min(
1063
+ MAX_NUM_TOPK_CHUNKS,
1064
+ TOPK_TARGET_GRID // max(1, batch_size * num_q_heads),
1065
+ ),
1066
+ )
1067
+ NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1)
1068
+ BLOCK_SIZE_T = triton.next_power_of_2(topk)
1069
+ max_seqblock = score.shape[2]
1070
+ chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
1071
+ topk_score_partial = torch.empty(
1072
+ NUM_TOPK_CHUNKS,
1073
+ num_q_heads,
1074
+ batch_size,
1075
+ BLOCK_SIZE_T,
1076
+ dtype=torch.float32,
1077
+ device=score.device,
1078
+ )
1079
+ topk_idx_partial = torch.empty(
1080
+ NUM_TOPK_CHUNKS,
1081
+ num_q_heads,
1082
+ batch_size,
1083
+ BLOCK_SIZE_T,
1084
+ dtype=torch.int32,
1085
+ device=score.device,
1086
+ )
1087
+ # stream 0 (default): topk partial → topk merge
1088
+ grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS)
1089
+ _topk_index_partial_kernel[grid](
1090
+ score,
1091
+ topk_score_partial,
1092
+ topk_idx_partial,
1093
+ seq_lens,
1094
+ block_size,
1095
+ topk,
1096
+ chunk_blocks,
1097
+ score.stride(0),
1098
+ score.stride(1),
1099
+ score.stride(2),
1100
+ topk_score_partial.stride(0),
1101
+ topk_score_partial.stride(1),
1102
+ topk_score_partial.stride(2),
1103
+ topk_score_partial.stride(3),
1104
+ topk_idx_partial.stride(0),
1105
+ topk_idx_partial.stride(1),
1106
+ topk_idx_partial.stride(2),
1107
+ topk_idx_partial.stride(3),
1108
+ )
1109
+ grid = (batch_size, num_q_heads)
1110
+ _topk_index_merge_kernel[grid](
1111
+ topk_score_partial,
1112
+ topk_idx_partial,
1113
+ topk_idx,
1114
+ seq_lens,
1115
+ block_size,
1116
+ topk,
1117
+ topk_score_partial.stride(0),
1118
+ topk_score_partial.stride(1),
1119
+ topk_score_partial.stride(2),
1120
+ topk_score_partial.stride(3),
1121
+ topk_idx_partial.stride(0),
1122
+ topk_idx_partial.stride(1),
1123
+ topk_idx_partial.stride(2),
1124
+ topk_idx_partial.stride(3),
1125
+ topk_idx.stride(0),
1126
+ topk_idx.stride(1),
1127
+ topk_idx.stride(2),
1128
+ NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
1129
+ )
1130
+ if disable_index_value:
1131
+ return None, topk_idx, real_seq_lens
1132
+ # attn output merge (default stream)
1133
+ grid = (batch_size, num_q_heads)
1134
+ _merge_attn_out_kernel[grid](
1135
+ o,
1136
+ lse,
1137
+ seq_lens,
1138
+ head_dim,
1139
+ block_size,
1140
+ o.stride(0),
1141
+ o.stride(1),
1142
+ o.stride(2),
1143
+ o.stride(3),
1144
+ lse.stride(0),
1145
+ lse.stride(1),
1146
+ lse.stride(2),
1147
+ NUM_KV_CHUNKS=NUM_KV_CHUNKS,
1148
+ )
1149
+ o = o[0].contiguous()
1150
+ return o, topk_idx, real_seq_lens
build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py ADDED
@@ -0,0 +1,452 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
10
+
11
+ from .._native import (
12
+ native_decode_mma_supported,
13
+ native_decode_sparse_attn_mma_paged,
14
+ )
15
+ from ..common.utils import robust_allocator
16
+
17
+ _is_hip = is_hip()
18
+
19
+
20
+ @triton.heuristics(
21
+ {
22
+ "BLOCK_SIZE_H": lambda args: max(
23
+ 16, triton.next_power_of_2(args["gqa_group_size"])
24
+ ),
25
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
26
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
27
+ "HAS_SINK": lambda args: args["sink_ptr"] is not None,
28
+ "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
29
+ }
30
+ )
31
+ @triton.autotune(
32
+ configs=[
33
+ triton.Config({}, num_warps=nw, num_stages=ns)
34
+ for nw in [4, 8]
35
+ for ns in [2, 3, 4, 5]
36
+ ],
37
+ key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"],
38
+ )
39
+ @triton.jit
40
+ def _gqa_share_sparse_decode_kernel(
41
+ q_ptr, # Q: b x qh x d
42
+ sink_ptr, # Sink: qh x d
43
+ k_cache_ptr, # K paged: max_slots x kh x d
44
+ v_cache_ptr, # V paged: max_slots x kh x d
45
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
46
+ idx_ptr, # topk index: qh x b x topk
47
+ o_ptr, # O partial: c x b x qh x d
48
+ lse_ptr, # lse partial: c x b x qh
49
+ seq_lens,
50
+ slot_ids,
51
+ # shape
52
+ max_slots,
53
+ batch_size,
54
+ gqa_group_size,
55
+ head_dim,
56
+ max_topk,
57
+ max_kv_len,
58
+ # sm_scale
59
+ sm_scale,
60
+ # stride
61
+ stride_q_b,
62
+ stride_q_h,
63
+ stride_q_d,
64
+ stride_sink_h,
65
+ stride_sink_d,
66
+ stride_k_s,
67
+ stride_k_h,
68
+ stride_k_d,
69
+ stride_v_s,
70
+ stride_v_h,
71
+ stride_v_d,
72
+ stride_r2t_b,
73
+ stride_ti_h,
74
+ stride_ti_b,
75
+ stride_ti_t,
76
+ stride_o_c,
77
+ stride_o_b,
78
+ stride_o_h,
79
+ stride_o_d,
80
+ stride_l_c,
81
+ stride_l_b,
82
+ stride_l_h,
83
+ # META parameters
84
+ BATCH_SIZE_BUCKET: tl.constexpr,
85
+ BLOCK_SIZE_H: tl.constexpr,
86
+ BLOCK_SIZE_N: tl.constexpr,
87
+ BLOCK_SIZE_D: tl.constexpr,
88
+ BLOCK_SIZE_T: tl.constexpr,
89
+ NUM_TOPK_CHUNKS: tl.constexpr,
90
+ HAS_SINK: tl.constexpr,
91
+ IS_FP8: tl.constexpr,
92
+ ):
93
+ # decode program ids: split-K over the topk dimension to give every SM
94
+ # something to do at small batch. pid(0) folds (batch, chunk) together so
95
+ # the grid size = batch_size * NUM_TOPK_CHUNKS.
96
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
97
+ pid_b = pid_bc % batch_size
98
+ pid_c = pid_bc // batch_size
99
+ pid_h = pid_kh * gqa_group_size
100
+ # per-chunk topk range. chunk_size is *runtime* (depends on max_topk which
101
+ # is a runtime arg, not constexpr), so don't annotate as tl.constexpr —
102
+ # doing so produces undefined behavior in Triton.
103
+ chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
104
+ chunk_start_topk = pid_c * chunk_size_topk
105
+ chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk
106
+ # get q k start and len after rmpad
107
+ seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len)
108
+ sid = (
109
+ tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
110
+ ) % max_slots # to avoid bugs when slot_ids is negative
111
+ # get real topk
112
+ off_t = tl.arange(0, BLOCK_SIZE_T)
113
+ idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b
114
+ topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1)
115
+ valid_idx = tl.where(topk_idx >= 0, off_t, -1)
116
+ real_topk = tl.sum(valid_idx != -1, axis=0)
117
+ chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk)
118
+ # init pointer
119
+ off_n = tl.arange(0, BLOCK_SIZE_N)
120
+ off_d = tl.arange(0, BLOCK_SIZE_D)
121
+ dim_mask = off_d < head_dim
122
+ # init statistics — kept at -inf so empty chunks (chunk_start >= real_topk)
123
+ # naturally fall out as weight=0 in the merge step.
124
+ # q is always needed; load it unconditionally.
125
+ q_ptrs = tl.make_block_ptr(
126
+ base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
127
+ shape=(gqa_group_size, head_dim),
128
+ strides=(stride_q_h, stride_q_d),
129
+ offsets=(0, 0),
130
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
131
+ order=(1, 0),
132
+ )
133
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
134
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
135
+ lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
136
+ # Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code-
137
+ # eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is
138
+ # then None). The original compound `if HAS_SINK and pid_c == 0` traced
139
+ # the body even when False -> NoneType make_block_ptr compile error on
140
+ # sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always
141
+ # False for us, but this keeps the sink path correct too.
142
+ if HAS_SINK:
143
+ if pid_c == 0:
144
+ sink_ptrs = tl.make_block_ptr(
145
+ base=sink_ptr + pid_h * stride_sink_h,
146
+ shape=(gqa_group_size, head_dim),
147
+ strides=(stride_sink_h, stride_sink_d),
148
+ offsets=(0, 0),
149
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
150
+ order=(1, 0),
151
+ )
152
+ sink = tl.load(
153
+ sink_ptrs, boundary_check=(0, 1), padding_option="zero"
154
+ ).to(tl.float32)
155
+ qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale
156
+ m_i = qsink
157
+ lse_i = qsink
158
+ acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
159
+ # only iterate over this chunk's topk slice. the load must respect the
160
+ # per-chunk start offset.
161
+ cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t
162
+ for _ in tl.range(chunk_start_topk, chunk_end_topk):
163
+ # load index
164
+ c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N
165
+ cur_idx_ptr = cur_idx_ptr + stride_ti_t
166
+ # resolve slots for this block via req_to_token
167
+ pos = c + off_n
168
+ pos_mask = pos < seq_len
169
+ slots = tl.load(
170
+ req_to_token_ptr + sid * stride_r2t_b + pos,
171
+ mask=pos_mask,
172
+ other=0,
173
+ ).to(tl.int64)
174
+ slots = (slots + max_slots) % max_slots # safety against negative
175
+ # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
176
+ k_off = (
177
+ slots[None, :] * stride_k_s
178
+ + pid_kh * stride_k_h
179
+ + off_d[:, None] * stride_k_d
180
+ )
181
+ k = tl.load(
182
+ k_cache_ptr + k_off,
183
+ mask=dim_mask[:, None] & pos_mask[None, :],
184
+ other=0.0,
185
+ )
186
+ if IS_FP8:
187
+ # fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no
188
+ # scale), so dequant is just a widening cast to the Q compute dtype
189
+ # before the tl.dot. Matches the bf16 path bit-for-bit when the cache
190
+ # is bf16 (IS_FP8 False -> this branch is compiled out).
191
+ k = k.to(q.dtype)
192
+ # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
193
+ v_off = (
194
+ slots[:, None] * stride_v_s
195
+ + pid_kh * stride_v_h
196
+ + off_d[None, :] * stride_v_d
197
+ )
198
+ v = tl.load(
199
+ v_cache_ptr + v_off,
200
+ mask=pos_mask[:, None] & dim_mask[None, :],
201
+ other=0.0,
202
+ )
203
+ if IS_FP8:
204
+ # Widen V before the P@V dot. This also makes the `p.to(v.dtype)`
205
+ # below cast P to the compute dtype (not to fp8, which would be
206
+ # catastrophic precision loss on the attention weights).
207
+ v = v.to(q.dtype)
208
+ # compute qk
209
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
210
+ qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf"))
211
+ # [H, D], [D, N] -> [H, N]
212
+ qk += tl.dot(q, k) * sm_scale
213
+ # compute m_ij and l_ij
214
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
215
+ p = tl.exp(qk - m_ij[:, None])
216
+ l_ij = tl.sum(p, axis=1)
217
+ # scale acc_o
218
+ acc_o_scale = tl.exp(m_i - m_ij)
219
+ acc_o = acc_o * acc_o_scale[:, None]
220
+ # load v and update acc_o
221
+ p = p.to(v.dtype)
222
+ # [H, N], [N, D] -> [H, D]
223
+ acc_o += tl.dot(p.to(v.dtype), v)
224
+ # update statistics
225
+ m_i = m_ij
226
+ lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij)
227
+ # final scale (matches the old non-split kernel for chunks where lse_i>-inf).
228
+ # For empty chunks (chunk_start_topk >= real_topk) the inner loop never
229
+ # runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute
230
+ # exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial
231
+ # and the merge result. Gate the scale with tl.where so empty chunks emit a
232
+ # clean zero (lse_i stays -inf which the merge correctly turns into weight=0).
233
+ scale = tl.where(
234
+ lse_i > float("-inf"),
235
+ tl.exp(m_i - lse_i),
236
+ tl.zeros_like(lse_i),
237
+ )
238
+ acc_o = acc_o * scale[:, None]
239
+ # save partial output and lse for the merge step
240
+ o_ptrs = tl.make_block_ptr(
241
+ base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
242
+ shape=(gqa_group_size, head_dim),
243
+ strides=(stride_o_h, stride_o_d),
244
+ offsets=(0, 0),
245
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
246
+ order=(1, 0),
247
+ )
248
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
249
+ lse_ptrs = tl.make_block_ptr(
250
+ base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
251
+ shape=(gqa_group_size,),
252
+ strides=(stride_l_h,),
253
+ offsets=(0,),
254
+ block_shape=(BLOCK_SIZE_H,),
255
+ order=(0,),
256
+ )
257
+ tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
258
+
259
+
260
+ @triton.heuristics(
261
+ {
262
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
263
+ }
264
+ )
265
+ @triton.jit
266
+ def _merge_topk_attn_out_kernel(
267
+ o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0
268
+ lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH]
269
+ head_dim,
270
+ stride_o_c,
271
+ stride_o_b,
272
+ stride_o_h,
273
+ stride_o_d,
274
+ stride_l_c,
275
+ stride_l_b,
276
+ stride_l_h,
277
+ NUM_TOPK_CHUNKS: tl.constexpr,
278
+ BLOCK_SIZE_D: tl.constexpr,
279
+ ):
280
+ pid_b, pid_h = tl.program_id(0), tl.program_id(1)
281
+ off_c = tl.arange(0, NUM_TOPK_CHUNKS)
282
+ off_d = tl.arange(0, BLOCK_SIZE_D)
283
+ o_ptrs = tl.make_block_ptr(
284
+ base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
285
+ shape=(NUM_TOPK_CHUNKS, head_dim),
286
+ strides=(stride_o_c, stride_o_d),
287
+ offsets=(0, 0),
288
+ block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
289
+ order=(1, 0),
290
+ )
291
+ lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
292
+ o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
293
+ lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
294
+ # standard flash-decoding merge in linear (not log2) space, matching the
295
+ # decode kernel which uses tl.exp / tl.log.
296
+ lse_max = tl.max(lse, axis=0)
297
+ weights = tl.exp(lse - lse_max)
298
+ weights = weights / tl.sum(weights, axis=0)
299
+ o_merged = tl.sum(o * weights[:, None], axis=0)
300
+ o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
301
+ tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
302
+
303
+
304
+ @torch.no_grad()
305
+ def flash_decode_with_gqa_share_sparse(
306
+ q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
307
+ sink: Optional[torch.Tensor],
308
+ k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
309
+ v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
310
+ req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
311
+ seq_lens: torch.Tensor, # [batch_size, ]
312
+ slot_ids: torch.Tensor, # [batch_size, ]
313
+ block_size: int,
314
+ topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
315
+ sm_scale: Optional[float] = None,
316
+ use_tma: bool = True,
317
+ ) -> torch.Tensor:
318
+ triton.set_allocator(robust_allocator)
319
+ # dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may
320
+ # be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel
321
+ # widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing
322
+ # sparse decode dtype contract unchanged by accepting fp8 only on HIP.
323
+ assert q.dtype in (torch.bfloat16, torch.float16)
324
+ _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
325
+ is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
326
+ assert k_cache.dtype == q.dtype or is_fp8, (
327
+ f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) "
328
+ f"or fp8 on HIP, got {k_cache.dtype}"
329
+ )
330
+ assert v_cache.dtype == k_cache.dtype
331
+ # shape
332
+ batch_size, num_q_heads, head_dim = q.shape
333
+ max_slots, num_kv_heads, _ = k_cache.shape
334
+ assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
335
+ assert topk_idx.shape[0] == num_kv_heads
336
+ assert (
337
+ triton.next_power_of_2(block_size) == block_size
338
+ ), f"block_size must be a power of 2, but got {block_size}"
339
+ # assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}"
340
+ max_kv_len = req_to_token.shape[1]
341
+ # gqa
342
+ assert num_q_heads % num_kv_heads == 0
343
+ gqa_group_size = num_q_heads // num_kv_heads
344
+ max_topk = topk_idx.shape[2]
345
+ # sm scale
346
+ if sm_scale is None:
347
+ sm_scale = head_dim**-0.5
348
+ # Fast path: native tensor-core decode for the M3 MSA shape. Falls through
349
+ # to the Triton kernel for other shapes, sink, or fp8 caches.
350
+ if sink is None and not is_fp8 and native_decode_mma_supported(
351
+ q, k_cache, block_size=block_size
352
+ ):
353
+ return native_decode_sparse_attn_mma_paged(
354
+ q,
355
+ k_cache,
356
+ v_cache,
357
+ req_to_token,
358
+ seq_lens,
359
+ slot_ids,
360
+ block_size,
361
+ topk_idx,
362
+ sm_scale,
363
+ )
364
+ # Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as
365
+ # flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must
366
+ # only depend on shape constants (so grid is fixed within a cuda graph).
367
+ # Capped by max_topk because chunks beyond real_topk early-fall-through to
368
+ # the merge-as-zero path; capping avoids wasting blocks at tiny topk.
369
+ TARGET_GRID = 256
370
+ target = max(
371
+ 1,
372
+ min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)),
373
+ )
374
+ NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1)
375
+ # output tensor: split-K partials, merged into chunk 0 by the merge kernel
376
+ o_partial = torch.empty(
377
+ NUM_TOPK_CHUNKS,
378
+ batch_size,
379
+ num_q_heads,
380
+ head_dim,
381
+ dtype=q.dtype,
382
+ device=q.device,
383
+ )
384
+ lse_partial = torch.empty(
385
+ NUM_TOPK_CHUNKS,
386
+ batch_size,
387
+ num_q_heads,
388
+ dtype=torch.float32,
389
+ device=q.device,
390
+ )
391
+ # launch attention kernel
392
+ grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads)
393
+ _gqa_share_sparse_decode_kernel[grid](
394
+ q,
395
+ sink,
396
+ k_cache,
397
+ v_cache,
398
+ req_to_token,
399
+ topk_idx,
400
+ o_partial,
401
+ lse_partial,
402
+ seq_lens,
403
+ slot_ids,
404
+ max_slots,
405
+ batch_size,
406
+ gqa_group_size,
407
+ head_dim,
408
+ max_topk,
409
+ max_kv_len,
410
+ sm_scale,
411
+ q.stride(0),
412
+ q.stride(1),
413
+ q.stride(2),
414
+ sink.stride(0) if sink is not None else 0,
415
+ sink.stride(1) if sink is not None else 0,
416
+ k_cache.stride(0),
417
+ k_cache.stride(1),
418
+ k_cache.stride(2),
419
+ v_cache.stride(0),
420
+ v_cache.stride(1),
421
+ v_cache.stride(2),
422
+ req_to_token.stride(0),
423
+ topk_idx.stride(0),
424
+ topk_idx.stride(1),
425
+ topk_idx.stride(2),
426
+ o_partial.stride(0),
427
+ o_partial.stride(1),
428
+ o_partial.stride(2),
429
+ o_partial.stride(3),
430
+ lse_partial.stride(0),
431
+ lse_partial.stride(1),
432
+ lse_partial.stride(2),
433
+ BLOCK_SIZE_N=block_size,
434
+ NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
435
+ IS_FP8=is_fp8,
436
+ )
437
+ # merge partials into chunk 0
438
+ merge_grid = (batch_size, num_q_heads)
439
+ _merge_topk_attn_out_kernel[merge_grid](
440
+ o_partial,
441
+ lse_partial,
442
+ head_dim,
443
+ o_partial.stride(0),
444
+ o_partial.stride(1),
445
+ o_partial.stride(2),
446
+ o_partial.stride(3),
447
+ lse_partial.stride(0),
448
+ lse_partial.stride(1),
449
+ lse_partial.stride(2),
450
+ NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
451
+ )
452
+ return o_partial[0].contiguous()
build/torch211-cxx11-cu130-x86_64-linux/interface.py ADDED
@@ -0,0 +1,679 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from ._native import (
11
+ has_native_ops,
12
+ native_indexer_block_scores,
13
+ native_nvfp4_dequant_swizzled_to_bf16,
14
+ )
15
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
16
+ from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse
17
+ from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16
18
+
19
+
20
+ def _page_table_to_req_to_token(
21
+ page_table: torch.Tensor,
22
+ seqused_k: torch.Tensor,
23
+ max_seqlen_k: int,
24
+ blk_kv: int,
25
+ ) -> torch.Tensor:
26
+ batch = int(page_table.shape[0])
27
+ max_seqlen_k = int(max_seqlen_k)
28
+ blk_kv = int(blk_kv)
29
+ logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
30
+ page = logical_pos // blk_kv
31
+ offset = logical_pos % blk_kv
32
+ page = page.clamp(max=int(page_table.shape[1]) - 1)
33
+ physical_page = page_table[:, page].to(torch.int64)
34
+ physical_token = physical_page * blk_kv + offset[None, :]
35
+ valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
36
+ return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
37
+
38
+
39
+ def _dense_q2k_indices(
40
+ *,
41
+ head_kv: int,
42
+ batch: int,
43
+ topk: int,
44
+ seq_lens: torch.Tensor,
45
+ blk_kv: int,
46
+ ) -> torch.Tensor:
47
+ idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
48
+ blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
49
+ valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
50
+ for b in range(batch):
51
+ valid = blocks < valid_blocks[b]
52
+ idx[:, b, valid] = blocks[valid]
53
+ return idx
54
+
55
+
56
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]:
57
+ seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist()
58
+ return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens]
59
+
60
+
61
+ def _packed_row_to_batch_block(
62
+ cu_seqlens_k: torch.Tensor,
63
+ blk_kv: int,
64
+ ) -> list[tuple[int, int]]:
65
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv))
66
+ max_rows = max(rows_per_batch, default=0)
67
+ row_map: list[tuple[int, int]] = []
68
+ for block_idx in range(max_rows):
69
+ for batch_idx, rows in enumerate(rows_per_batch):
70
+ if block_idx < rows:
71
+ row_map.append((batch_idx, block_idx))
72
+ return row_map
73
+
74
+
75
+ def _csr_to_q2k_indices(
76
+ k2q_row_ptr: torch.Tensor,
77
+ k2q_q_indices: torch.Tensor,
78
+ cu_seqlens_q: torch.Tensor,
79
+ cu_seqlens_k: torch.Tensor,
80
+ blk_kv: int,
81
+ topk: int,
82
+ ) -> torch.Tensor:
83
+ if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32:
84
+ raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32")
85
+ if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2:
86
+ raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2")
87
+ head_kv = int(k2q_row_ptr.shape[0])
88
+ total_q = int(cu_seqlens_q[-1].item())
89
+ topk = int(topk)
90
+ if topk <= 0:
91
+ raise ValueError(f"topK must be positive, got {topk}")
92
+
93
+ row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv))
94
+ if int(k2q_row_ptr.shape[1]) != len(row_map) + 1:
95
+ raise ValueError(
96
+ "k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: "
97
+ f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}"
98
+ )
99
+
100
+ cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
101
+ row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False)
102
+ q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False)
103
+ q2k = torch.full(
104
+ (head_kv, total_q, topk),
105
+ -1,
106
+ dtype=torch.int32,
107
+ device=k2q_row_ptr.device,
108
+ )
109
+
110
+ next_slot: dict[tuple[int, int], int] = {}
111
+ seen: set[tuple[int, int, int]] = set()
112
+ for h in range(head_kv):
113
+ for row, (batch_idx, kv_block_idx) in enumerate(row_map):
114
+ start = int(row_ptr_cpu[h, row].item())
115
+ end = int(row_ptr_cpu[h, row + 1].item())
116
+ q_start = int(cu_q_cpu[batch_idx])
117
+ q_end = int(cu_q_cpu[batch_idx + 1])
118
+ q_len = q_end - q_start
119
+ for pos in range(start, end):
120
+ q_val = int(q_idx_cpu[h, pos].item())
121
+ if q_val < 0:
122
+ continue
123
+ if q_start <= q_val < q_end:
124
+ q_global = q_val
125
+ elif q_val < q_len:
126
+ q_global = q_start + q_val
127
+ else:
128
+ continue
129
+ key = (h, q_global, kv_block_idx)
130
+ if key in seen:
131
+ continue
132
+ seen.add(key)
133
+ slot_key = (h, q_global)
134
+ slot = next_slot.get(slot_key, 0)
135
+ if slot >= topk:
136
+ continue
137
+ q2k[h, q_global, slot] = int(kv_block_idx)
138
+ next_slot[slot_key] = slot + 1
139
+ return q2k.contiguous()
140
+
141
+
142
+ def _make_prefill_cache_and_metadata(
143
+ q: torch.Tensor,
144
+ k: torch.Tensor,
145
+ v: torch.Tensor,
146
+ cu_seqlens_q: torch.Tensor,
147
+ cu_seqlens_k: torch.Tensor,
148
+ max_seqlen_k: int,
149
+ blk_kv: int,
150
+ *,
151
+ page_table: Optional[torch.Tensor],
152
+ seqused_k: Optional[torch.Tensor],
153
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
154
+ if page_table is None:
155
+ if k.ndim != 3 or v.ndim != 3:
156
+ raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]")
157
+ if k.shape != v.shape:
158
+ raise ValueError("k and v must have the same shape")
159
+ k_cache = k.contiguous()
160
+ v_cache = v.contiguous()
161
+ batch = int(cu_seqlens_q.numel()) - 1
162
+ max_seqlen_k = int(max_seqlen_k)
163
+ logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64)
164
+ cu_k = cu_seqlens_k.to(torch.int64)
165
+ seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous()
166
+ valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
167
+ req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32)
168
+ req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
169
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
170
+ else:
171
+ if k.ndim != 4 or v.ndim != 4:
172
+ raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]")
173
+ if k.shape != v.shape:
174
+ raise ValueError("k and v must have the same shape")
175
+ if int(k.shape[2]) != int(blk_kv):
176
+ raise ValueError("paged k/v page dimension must equal blk_kv")
177
+ if page_table.dtype != torch.int32:
178
+ raise TypeError("page_table must be torch.int32")
179
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
180
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
181
+ batch = int(page_table.shape[0])
182
+ if seqused_k is None:
183
+ seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous()
184
+ else:
185
+ seq_lens = seqused_k.to(torch.int32).contiguous()
186
+ logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64)
187
+ page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1)
188
+ offset = logical % int(blk_kv)
189
+ physical_page = page_table[:, page].to(torch.int64)
190
+ req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32)
191
+ valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
192
+ req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
193
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
194
+
195
+ q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32)
196
+ prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous()
197
+ return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens
198
+
199
+
200
+ def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int:
201
+ if kv_heads <= 0 or q_heads % kv_heads != 0:
202
+ raise ValueError("number of query heads must be divisible by KV heads")
203
+ gqa_group = q_heads // kv_heads
204
+ limit = max(1, 128 // max(1, gqa_group))
205
+ for candidate in (64, 32, 16, 8, 4, 2, 1):
206
+ if candidate <= limit:
207
+ return candidate
208
+ return 1
209
+
210
+
211
+ def _normalize_decode_q2k(
212
+ q2k_indices: Optional[torch.Tensor],
213
+ *,
214
+ batch: int,
215
+ seqlen_q: int,
216
+ head_kv: int,
217
+ seqused_k: torch.Tensor,
218
+ blk_kv: int,
219
+ ) -> torch.Tensor:
220
+ if q2k_indices is None:
221
+ topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
222
+ return _dense_q2k_indices(
223
+ head_kv=head_kv,
224
+ batch=batch,
225
+ topk=topk,
226
+ seq_lens=seqused_k,
227
+ blk_kv=blk_kv,
228
+ )
229
+ if q2k_indices.dtype != torch.int32:
230
+ raise TypeError("q2k_indices must be torch.int32")
231
+ if q2k_indices.ndim != 3:
232
+ raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
233
+ if q2k_indices.shape[0] != head_kv:
234
+ raise ValueError("q2k_indices first dimension must match Hkv")
235
+ if int(q2k_indices.shape[1]) != batch * seqlen_q:
236
+ raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
237
+ if seqlen_q != 1:
238
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
239
+ return q2k_indices[:, ::seqlen_q, :].contiguous()
240
+
241
+
242
+ def sparse_decode_atten_func(
243
+ q: torch.Tensor,
244
+ k: torch.Tensor,
245
+ v: torch.Tensor,
246
+ q2k_indices: Optional[torch.Tensor] = None,
247
+ *,
248
+ page_table: torch.Tensor,
249
+ seqused_k: torch.Tensor,
250
+ seqlen_q: int,
251
+ max_seqlen_k: int,
252
+ blk_kv: int = 128,
253
+ causal: bool = True,
254
+ softmax_scale: Optional[float] = None,
255
+ return_softmax_lse: bool = False,
256
+ schedule: Optional[object] = None,
257
+ O_partial: Optional[torch.Tensor] = None,
258
+ LSE_partial: Optional[torch.Tensor] = None,
259
+ ):
260
+ """Run MiniMax paged sparse decode on Blackwell.
261
+
262
+ This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
263
+ but accepts the Blackwell package's validated BF16/FP16 decode path. It
264
+ supports single-token decode (`seqlen_q=1`) with paged K/V
265
+ `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
266
+ """
267
+
268
+ if schedule is not None or O_partial is not None or LSE_partial is not None:
269
+ raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
270
+ if return_softmax_lse:
271
+ raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
272
+ if not causal:
273
+ raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
274
+ if int(seqlen_q) != 1:
275
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
276
+ if q.ndim != 3:
277
+ raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
278
+ if k.ndim != 4 or v.ndim != 4:
279
+ raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
280
+ if k.shape != v.shape:
281
+ raise ValueError("k and v must have the same shape")
282
+ if q.device != k.device or q.device != v.device or q.device != page_table.device:
283
+ raise ValueError("q, k, v, and page_table must be on the same device")
284
+ if q.dtype not in (torch.bfloat16, torch.float16):
285
+ raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
286
+ if k.dtype != q.dtype or v.dtype != q.dtype:
287
+ raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
288
+ if q.shape[-1] != 128 or k.shape[-1] != 128:
289
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
290
+ if int(k.shape[2]) != int(blk_kv):
291
+ raise ValueError("k.shape[2] must equal blk_kv")
292
+ if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
293
+ raise TypeError("page_table and seqused_k must be torch.int32")
294
+ if page_table.ndim != 2:
295
+ raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
296
+ batch = int(page_table.shape[0])
297
+ if int(q.shape[0]) != batch:
298
+ raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
299
+
300
+ head_kv = int(k.shape[1])
301
+ q2k = _normalize_decode_q2k(
302
+ q2k_indices,
303
+ batch=batch,
304
+ seqlen_q=int(seqlen_q),
305
+ head_kv=head_kv,
306
+ seqused_k=seqused_k,
307
+ blk_kv=int(blk_kv),
308
+ )
309
+ req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
310
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
311
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
312
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
313
+ return flash_decode_with_gqa_share_sparse(
314
+ q.contiguous(),
315
+ None,
316
+ k_cache,
317
+ v_cache,
318
+ req_to_token,
319
+ seqused_k.contiguous(),
320
+ slot_ids,
321
+ int(blk_kv),
322
+ q2k,
323
+ sm_scale=softmax_scale,
324
+ )
325
+
326
+
327
+ class SparseDecodePagedAttentionWrapper:
328
+ def __init__(self, *, blk_kv: int = 128, causal: bool = True):
329
+ self.blk_kv = int(blk_kv)
330
+ self.causal = bool(causal)
331
+ self.page_table: Optional[torch.Tensor] = None
332
+ self.seqused_k: Optional[torch.Tensor] = None
333
+ self.seqlen_q: Optional[int] = None
334
+ self.max_seqlen_k: Optional[int] = None
335
+ self.q2k_indices: Optional[torch.Tensor] = None
336
+
337
+ def plan(
338
+ self,
339
+ *,
340
+ page_table: torch.Tensor,
341
+ seqused_k: torch.Tensor,
342
+ seqlen_q: int,
343
+ max_seqlen_k: int,
344
+ q2k_indices: Optional[torch.Tensor] = None,
345
+ num_qo_heads: Optional[int] = None,
346
+ num_kv_heads: Optional[int] = None,
347
+ head_dim: Optional[int] = 128,
348
+ enable_cuda_graph: bool = False,
349
+ max_grid_size: Optional[int] = None,
350
+ fixed_split_size: Optional[int] = None,
351
+ disable_split_kv: bool = False,
352
+ ) -> "SparseDecodePagedAttentionWrapper":
353
+ if head_dim is not None and int(head_dim) != 128:
354
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
355
+ self.page_table = page_table.contiguous()
356
+ self.seqused_k = seqused_k.contiguous()
357
+ self.seqlen_q = int(seqlen_q)
358
+ self.max_seqlen_k = int(max_seqlen_k)
359
+ self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
360
+ return self
361
+
362
+ def run(
363
+ self,
364
+ q: torch.Tensor,
365
+ k: torch.Tensor,
366
+ v: torch.Tensor,
367
+ *,
368
+ softmax_scale: Optional[float] = None,
369
+ return_softmax_lse: bool = False,
370
+ out: Optional[torch.Tensor] = None,
371
+ lse: Optional[torch.Tensor] = None,
372
+ ):
373
+ if self.page_table is None or self.seqused_k is None:
374
+ raise RuntimeError("decode wrapper must be planned before run")
375
+ result = sparse_decode_atten_func(
376
+ q,
377
+ k,
378
+ v,
379
+ self.q2k_indices,
380
+ page_table=self.page_table,
381
+ seqused_k=self.seqused_k,
382
+ seqlen_q=self.seqlen_q,
383
+ max_seqlen_k=self.max_seqlen_k,
384
+ blk_kv=self.blk_kv,
385
+ causal=self.causal,
386
+ softmax_scale=softmax_scale,
387
+ return_softmax_lse=return_softmax_lse,
388
+ )
389
+ if out is not None:
390
+ out.copy_(result)
391
+ return out
392
+ return result
393
+
394
+
395
+ def sparse_atten_func(
396
+ q: torch.Tensor,
397
+ k: torch.Tensor,
398
+ v: torch.Tensor,
399
+ k2q_row_ptr: torch.Tensor,
400
+ k2q_q_indices: torch.Tensor,
401
+ topK: int,
402
+ *,
403
+ cu_seqlens_q: torch.Tensor,
404
+ cu_seqlens_k: torch.Tensor,
405
+ max_seqlen_q: int,
406
+ max_seqlen_k: int,
407
+ blk_kv: int = 128,
408
+ causal: bool = True,
409
+ softmax_scale: Optional[float] = None,
410
+ lse_temperature_scale: float = 1.0,
411
+ return_temperature_lse: bool = False,
412
+ partial_dtype: torch.dtype = torch.bfloat16,
413
+ return_softmax_lse: bool = False,
414
+ page_table: Optional[torch.Tensor] = None,
415
+ seqused_k: Optional[torch.Tensor] = None,
416
+ schedule: Optional[object] = None,
417
+ usable_SM_count: int = -1,
418
+ qk_dtype: Optional[torch.dtype] = None,
419
+ pv_dtype: Optional[torch.dtype] = None,
420
+ ):
421
+ """Run the official MiniMax CSR sparse prefill API on Blackwell.
422
+
423
+ The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list
424
+ consumed by the Blackwell Triton sparse prefill kernel. The default output
425
+ path is supported for BF16/FP16 tensors with head_dim=128.
426
+ """
427
+
428
+ del partial_dtype, usable_SM_count, qk_dtype, pv_dtype
429
+ if schedule is not None:
430
+ raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper")
431
+ if return_temperature_lse or return_softmax_lse:
432
+ raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper")
433
+ if float(lse_temperature_scale) != 1.0:
434
+ raise ValueError("lse_temperature_scale values other than 1.0 are not supported")
435
+ if not causal:
436
+ raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True")
437
+ if q.ndim != 3:
438
+ raise ValueError("q must have shape [total_q, Hq, D]")
439
+ if q.dtype not in (torch.bfloat16, torch.float16):
440
+ raise TypeError("q must be BF16 or FP16")
441
+ if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128:
442
+ raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128")
443
+ if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32:
444
+ raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32")
445
+ if q.device != k.device or q.device != v.device:
446
+ raise ValueError("q, k, and v must be on the same device")
447
+
448
+ if k.dtype != q.dtype or v.dtype != q.dtype:
449
+ raise TypeError("q, k, and v must have matching BF16/FP16 dtype")
450
+ kv_heads = int(k.shape[1])
451
+ q2k_indices = _csr_to_q2k_indices(
452
+ k2q_row_ptr.contiguous(),
453
+ k2q_q_indices.contiguous(),
454
+ cu_seqlens_q.contiguous(),
455
+ cu_seqlens_k.contiguous(),
456
+ int(blk_kv),
457
+ int(topK),
458
+ )
459
+ k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata(
460
+ q,
461
+ k,
462
+ v,
463
+ cu_seqlens_q.contiguous(),
464
+ cu_seqlens_k.contiguous(),
465
+ int(max_seqlen_k),
466
+ int(blk_kv),
467
+ page_table=None if page_table is None else page_table.contiguous(),
468
+ seqused_k=None if seqused_k is None else seqused_k.contiguous(),
469
+ )
470
+ block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads)
471
+ return flash_prefill_with_gqa_share_sparse(
472
+ q.contiguous(),
473
+ k_cache,
474
+ v_cache,
475
+ None,
476
+ req_to_token,
477
+ slot_ids,
478
+ q2k_indices,
479
+ block_size_q=block_size_q,
480
+ block_size_k=int(blk_kv),
481
+ cu_seqlens=cu_seqlens_q.contiguous(),
482
+ seq_lens=seq_lens,
483
+ prefix_lens=prefix_lens,
484
+ max_seqlen_q=int(max_seqlen_q),
485
+ sm_scale=softmax_scale,
486
+ )
487
+
488
+
489
+ def _nvfp4_dequant_kv(
490
+ packed: torch.Tensor,
491
+ scale_128x4: torch.Tensor,
492
+ global_scale: torch.Tensor,
493
+ ) -> torch.Tensor:
494
+ if packed.dtype is not torch.uint8:
495
+ return packed
496
+ if has_native_ops() and packed.is_cuda:
497
+ return native_nvfp4_dequant_swizzled_to_bf16(
498
+ packed,
499
+ scale_128x4,
500
+ global_scale,
501
+ )
502
+ original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
503
+ rows = 1
504
+ for dim in original_shape[:-1]:
505
+ rows *= int(dim)
506
+ qx = Nvfp4QuantizedTensor(
507
+ data=packed,
508
+ scale_128x4=scale_128x4,
509
+ global_scale=global_scale,
510
+ logical_scale_shape=(rows, int(original_shape[-1]) // 16),
511
+ original_shape=original_shape,
512
+ )
513
+ return dequantize_nvfp4_128x4_to_bf16(qx)
514
+
515
+
516
+ def sparse_atten_nvfp4_kv_func(
517
+ q: torch.Tensor,
518
+ k: torch.Tensor,
519
+ v: torch.Tensor,
520
+ k_scale_128x4: torch.Tensor,
521
+ v_scale_128x4: torch.Tensor,
522
+ k_global_scale: torch.Tensor,
523
+ v_global_scale: torch.Tensor,
524
+ k2q_row_ptr: torch.Tensor,
525
+ k2q_q_indices: torch.Tensor,
526
+ topK: int,
527
+ **kwargs,
528
+ ):
529
+ """NVFP4-KV official prefill API wrapper.
530
+
531
+ The Blackwell compatibility path dequantizes NVFP4 K/V with the package's
532
+ 128x4 metadata helper, then dispatches to `sparse_atten_func`.
533
+ """
534
+
535
+ k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale)
536
+ v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale)
537
+ return sparse_atten_func(
538
+ q,
539
+ k_bf16,
540
+ v_bf16,
541
+ k2q_row_ptr,
542
+ k2q_q_indices,
543
+ topK,
544
+ **kwargs,
545
+ )
546
+
547
+
548
+ def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor:
549
+ data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8)
550
+ lut = torch.tensor(
551
+ [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
552
+ dtype=torch.float32,
553
+ device=data.device,
554
+ )
555
+ out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device)
556
+ out[..., 0::2] = lut[(data & 0x0F).long()]
557
+ out[..., 1::2] = lut[(data >> 4).long()]
558
+ return out
559
+
560
+
561
+ def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
562
+ if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]):
563
+ return values * scale.to(torch.float32).repeat_interleave(16, dim=-1)
564
+ if scale.ndim == values.ndim - 1:
565
+ return values * scale.to(torch.float32).unsqueeze(-1)
566
+ return values
567
+
568
+
569
+ def fp4_indexer_block_scores(
570
+ q_fp4: torch.Tensor,
571
+ k_fp4: torch.Tensor,
572
+ q_scale: torch.Tensor,
573
+ k_scale: torch.Tensor,
574
+ cu_seqlens_q: torch.Tensor,
575
+ cu_seqlens_k: torch.Tensor,
576
+ cu_page_offsets: torch.Tensor,
577
+ *,
578
+ max_seqlen_q: int,
579
+ max_seqlen_k: int,
580
+ kv_indices: torch.Tensor,
581
+ fp4_format: str,
582
+ causal: bool = False,
583
+ qo_offset: Optional[torch.Tensor] = None,
584
+ scale_layout: str = "preordered_mma",
585
+ ) -> torch.Tensor:
586
+ """FP4 block-score API for Blackwell.
587
+
588
+ Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`.
589
+ Built artifacts route CUDA tensors through the native Blackwell block-score
590
+ kernel. Source-tree mode uses the reference loop below so the API remains
591
+ testable before the extension is built.
592
+ """
593
+
594
+ del qo_offset
595
+ if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}:
596
+ raise ValueError(f"unsupported fp4_format: {fp4_format}")
597
+ if scale_layout not in {"public", "preordered_mma"}:
598
+ raise ValueError(f"unsupported scale_layout: {scale_layout}")
599
+ if q_fp4.ndim != 3:
600
+ raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]")
601
+ if k_fp4.ndim != 4:
602
+ raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]")
603
+ total_q, hq, _ = q_fp4.shape
604
+ num_pages, hkv, page_size, _ = k_fp4.shape
605
+ if int(page_size) != 128:
606
+ raise ValueError("fp4_indexer_block_scores currently expects 128-token pages")
607
+ if hq % hkv != 0:
608
+ raise ValueError("Hq must be divisible by Hkv")
609
+ max_blocks = (int(max_seqlen_k) + 127) // 128
610
+ scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device)
611
+
612
+ q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale)
613
+ k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale)
614
+
615
+ # Native block-max QK kernel for the cubic scoring loop when available.
616
+ # Same dequant numerics as the reference (bf16 compute); falls back to the
617
+ # Python reference below in source-tree mode.
618
+ if has_native_ops() and q_fp4.is_cuda:
619
+ batch = int(cu_seqlens_q.numel()) - 1
620
+ q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64)
621
+ batch_of_q = torch.repeat_interleave(
622
+ torch.arange(batch, device=q_fp4.device, dtype=torch.int32),
623
+ q_lens,
624
+ )
625
+ return native_indexer_block_scores(
626
+ q.to(torch.bfloat16),
627
+ k.to(torch.bfloat16),
628
+ batch_of_q,
629
+ cu_seqlens_q.to(torch.int32),
630
+ cu_seqlens_k.to(torch.int32),
631
+ cu_page_offsets.to(torch.int32),
632
+ kv_indices.to(torch.int32),
633
+ max_blocks=max_blocks,
634
+ causal=causal,
635
+ )
636
+
637
+ gqa = hq // hkv
638
+ batch = int(cu_seqlens_q.numel()) - 1
639
+ cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
640
+ cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist()
641
+ cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist()
642
+ kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist()
643
+ for b in range(batch):
644
+ q_start, q_end = int(cu_q[b]), int(cu_q[b + 1])
645
+ k_len = int(cu_k[b + 1]) - int(cu_k[b])
646
+ page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1])
647
+ for logical_block, page_pos in enumerate(range(page_start, page_end)):
648
+ if logical_block >= max_blocks:
649
+ break
650
+ physical_page = int(kv_idx_cpu[page_pos])
651
+ if physical_page < 0 or physical_page >= num_pages:
652
+ continue
653
+ valid_tokens = min(128, max(0, k_len - logical_block * 128))
654
+ if valid_tokens <= 0:
655
+ continue
656
+ for qi in range(q_start, q_end):
657
+ local_q = qi - q_start
658
+ if causal and logical_block * 128 > local_q:
659
+ continue
660
+ for qh in range(hq):
661
+ kh = qh // gqa
662
+ key_block = k[physical_page, kh, :valid_tokens]
663
+ if causal:
664
+ visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1))
665
+ if visible <= 0:
666
+ continue
667
+ key_block = key_block[:visible]
668
+ block_scores = torch.matmul(key_block, q[qi, qh])
669
+ scores[qh, logical_block, qi] = block_scores.max()
670
+ return scores
671
+
672
+
673
+ __all__ = [
674
+ "sparse_atten_func",
675
+ "sparse_atten_nvfp4_kv_func",
676
+ "sparse_decode_atten_func",
677
+ "SparseDecodePagedAttentionWrapper",
678
+ "fp4_indexer_block_scores",
679
+ ]
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,39 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "minimaxai-msa-blackwell",
3
+ "id": "_minimaxai_msa_blackwell_cuda_25b0067",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "12.0",
11
+ "12.1"
12
+ ]
13
+ },
14
+ "digest": {
15
+ "algorithm": "sha256",
16
+ "files": {
17
+ "__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=",
18
+ "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
+ "_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "oFPB/WnE3z+U5WanxNiLqUP41c7OcUreUnifaBmpYaw=",
20
+ "_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=",
21
+ "_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=",
22
+ "api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=",
23
+ "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
24
+ "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
25
+ "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
26
+ "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
27
+ "decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=",
28
+ "interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=",
29
+ "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
30
+ "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
31
+ "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
32
+ "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
33
+ "prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=",
34
+ "prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=",
35
+ "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
36
+ "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
37
+ }
38
+ }
39
+ }
build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+ #
3
+ # VENDOR NOTE: rewritten to drop the `einops` dependency (the original used
4
+ # einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream
5
+ # `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced
6
+ # with native torch.einsum / torch.reshape so the standalone vendor needs only
7
+ # torch (+ triton for the kernels). Original einops expressions are kept in
8
+ # comments next to each replacement for auditability.
9
+
10
+ from typing import Optional
11
+
12
+ import torch
13
+
14
+
15
+ def naive_flash_decode_with_topk_idx(
16
+ q: torch.Tensor, # [batch_size, num_heads, head_dim]
17
+ sink: Optional[torch.Tensor], # [num_heads, head_dim]
18
+ kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim]
19
+ seq_lens: torch.Tensor, # [batch_size, ]
20
+ max_seqlen: int,
21
+ slot_ids: torch.Tensor, # [batch_size, ]
22
+ block_size: int,
23
+ topk: int,
24
+ sm_scale: Optional[float] = None,
25
+ init_blocks: int = 0,
26
+ local_blocks: int = 0,
27
+ ):
28
+ """Joint reference: lightning-indexer block top-k + sparse attention.
29
+
30
+ This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense
31
+ 5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by
32
+ `slot_ids`. It both (a) selects the top-k key blocks via per-block max score
33
+ (with optional forced init/local blocks) and (b) computes the resulting
34
+ sparse attention, returning (o, topk_idx).
35
+ """
36
+ assert (
37
+ kv_cache.shape[2] % block_size == 0
38
+ ), "max cache len must be divisible by block size"
39
+ if sm_scale is None:
40
+ sm_scale = q.shape[-1] ** -0.5
41
+ original_dtype = q.dtype
42
+ batch_size = q.shape[0]
43
+ num_q_heads = q.shape[1]
44
+ num_kv_heads = kv_cache.shape[3]
45
+ gqa_group_size = num_q_heads // num_kv_heads
46
+ head_dim = q.shape[-1]
47
+ # rearrange q: "b (h g) d -> b h g d"
48
+ q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim)
49
+ kv_cache_float = kv_cache.float()
50
+ k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d]
51
+ v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d]
52
+ # einsum(q, k, "b h g d, b n h d -> b h g n")
53
+ qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale
54
+ mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None]
55
+ qk = qk.masked_fill(~mask[:, None, None, :], float("-inf"))
56
+ # get score
57
+ score = qk.clone().to(torch.float32)
58
+ # rearrange "b h g (n s) -> b (h g) n s", s=block_size
59
+ n_blocks = score.shape[-1] // block_size
60
+ score = score.reshape(
61
+ batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size
62
+ ).reshape(batch_size, num_q_heads, n_blocks, block_size)
63
+ score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks]
64
+ # post-process score for init_blocks and local_blocks
65
+ INIT_SCORE = 1e30
66
+ LOCAL_SCORE = 1e29
67
+ if init_blocks > 0:
68
+ score[:, :, :init_blocks] = INIT_SCORE
69
+ if local_blocks > 0:
70
+ num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
71
+ for b in range(batch_size):
72
+ num_blks = num_blocks_per_batch[b].item()
73
+ local_start = max(0, num_blks - local_blocks)
74
+ score[b, :, local_start:num_blks] = LOCAL_SCORE
75
+ # compute topk indices per (batch, head)
76
+ # score shape: [batch_size, num_q_heads, num_blocks]
77
+ topk_idx = torch.full(
78
+ (num_q_heads, batch_size, topk),
79
+ fill_value=-1,
80
+ device=score.device,
81
+ dtype=torch.int32,
82
+ )
83
+ num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
84
+ for b in range(batch_size):
85
+ num_blks = num_blocks_per_batch[b].item()
86
+ actual_topk = min(topk, num_blks)
87
+ for h in range(num_q_heads):
88
+ # get topk indices for this (batch, head)
89
+ _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1)
90
+ topk_idx[h, b, :actual_topk] = indices.to(torch.int32)
91
+ # compute attention output with sink
92
+ if sink is not None:
93
+ # sink: [num_q_heads, head_dim] -> [h, g, d]
94
+ sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim)
95
+ # einsum(q, sink, "b h g d, h g d -> b h g")
96
+ qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale
97
+ qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1]
98
+ attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32)
99
+ attn = attn[..., 1:] # remove sink score
100
+ else:
101
+ attn = qk.softmax(dim=-1, dtype=torch.float32)
102
+ # einsum(attn, v, "b h g n, b n h d -> b h g d")
103
+ o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel)
104
+ # rearrange "b h g d -> b (h g) d"
105
+ o = o.reshape(batch_size, num_q_heads, head_dim)
106
+ return o.to(original_dtype), topk_idx
build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+
7
+
8
+ def naive_flash_decode_with_gqa_share_sparse(
9
+ q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
10
+ sink: Optional[torch.Tensor], # [num_q_heads, head_dim]
11
+ kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim]
12
+ seq_lens: torch.Tensor, # [batch_size, ]
13
+ slot_ids: torch.Tensor, # [batch_size, ]
14
+ block_size: int,
15
+ topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
16
+ sm_scale: Optional[float] = None,
17
+ ) -> torch.Tensor:
18
+ """
19
+ Naive implementation of sparse attention with GQA group sharing.
20
+ Each GQA group (all q heads sharing the same kv head) uses the same topk_idx.
21
+ """
22
+ if sm_scale is None:
23
+ sm_scale = q.shape[-1] ** -0.5
24
+
25
+ original_dtype = q.dtype
26
+ batch_size, num_q_heads, head_dim = q.shape
27
+ max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape
28
+ gqa_group_size = num_q_heads // num_kv_heads
29
+ # Output tensor
30
+ o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device)
31
+
32
+ for b in range(batch_size):
33
+ seq_len = seq_lens[b].item()
34
+ sid = slot_ids[b].item() % max_slots
35
+
36
+ for kh in range(num_kv_heads):
37
+ # Get topk indices for this (kv_head, batch)
38
+ block_indices = topk_idx[kh, b, :].tolist()
39
+
40
+ # Collect selected K and V blocks
41
+ selected_k_blocks = []
42
+ selected_v_blocks = []
43
+ for block_idx in block_indices:
44
+ if block_idx < 0:
45
+ continue # Invalid index
46
+ start = block_idx * block_size
47
+ end = min(start + block_size, seq_len)
48
+ if start >= seq_len:
49
+ continue
50
+ # K: [block_len, head_dim]
51
+ k_block = kv_cache[sid, 0, start:end, kh, :]
52
+ # V: [block_len, head_dim]
53
+ v_block = kv_cache[sid, 1, start:end, kh, :]
54
+ selected_k_blocks.append(k_block)
55
+ selected_v_blocks.append(v_block)
56
+
57
+ if len(selected_k_blocks) == 0:
58
+ continue
59
+
60
+ # Concatenate selected blocks: [total_selected_len, head_dim]
61
+ k_selected = torch.cat(selected_k_blocks, dim=0)
62
+ v_selected = torch.cat(selected_v_blocks, dim=0)
63
+
64
+ # Compute attention for all q heads in this GQA group
65
+ for g in range(gqa_group_size):
66
+ qh = kh * gqa_group_size + g
67
+ # q_vec: [head_dim]
68
+ q_vec = q[b, qh, :].float()
69
+
70
+ # Compute attention scores: [total_selected_len]
71
+ scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale
72
+
73
+ # Add sink to softmax normalization if present
74
+ if sink is not None:
75
+ sink_vec = sink[qh, :].float()
76
+ qsink = torch.dot(q_vec, sink_vec) * sm_scale
77
+ # Concatenate sink score with regular scores for softmax
78
+ scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0)
79
+ attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1)
80
+ # Remove sink weight (only used for normalization)
81
+ attn_weights = attn_weights_with_sink[1:]
82
+ else:
83
+ attn_weights = torch.softmax(scores, dim=-1)
84
+
85
+ # Compute output: [head_dim]
86
+ o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected)
87
+
88
+ return o
build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ from .topk_sparse import flash_prefill_with_gqa_share_sparse
3
+
4
+ __all__ = ["flash_prefill_with_gqa_share_sparse"]
build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py ADDED
@@ -0,0 +1,371 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2025 XunhaoLai. All rights reserved.
2
+
3
+ from typing import Optional
4
+
5
+ import torch
6
+ import triton
7
+ import triton.language as tl
8
+
9
+ from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
10
+
11
+ from ..common.utils import get_cu_seqblocks, robust_allocator
12
+
13
+ _is_hip = is_hip()
14
+
15
+
16
+ @triton.heuristics(
17
+ {
18
+ "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]),
19
+ "BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]),
20
+ "BLOCK_SIZE_H": lambda args: triton.next_power_of_2(
21
+ max(
22
+ 16 // args["BLOCK_SIZE_Q"],
23
+ triton.next_power_of_2(args["gqa_group_size"]),
24
+ )
25
+ ),
26
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
27
+ "BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"],
28
+ "HAS_SINK": lambda args: args["sink_ptr"] is not None,
29
+ }
30
+ )
31
+ @triton.autotune(
32
+ # Configs that fail to compile on the target arch are skipped, so widening
33
+ # the num_warps x num_stages grid only adds candidates, never a bad kernel.
34
+ configs=[
35
+ triton.Config({}, num_warps=nw, num_stages=ns)
36
+ for nw in (2, 4, 8)
37
+ for ns in (2, 3, 4)
38
+ ],
39
+ key=[
40
+ "BLOCK_SIZE_Q",
41
+ "BLOCK_SIZE_K",
42
+ "qk_head_dim",
43
+ "v_head_dim",
44
+ "gqa_group_size",
45
+ ],
46
+ )
47
+ @triton.jit
48
+ def _gqa_share_sparse_fwd_kernel(
49
+ q_ptr, # Q: n x h x d
50
+ k_cache_ptr, # K paged: max_slots x kh x d
51
+ v_cache_ptr, # V paged: max_slots x kh x d
52
+ sink_ptr, # Sink: h x d
53
+ t_ptr, # topk_idx: kh x n x k
54
+ o_ptr, # O: n x h x d
55
+ req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
56
+ # seqlens
57
+ cu_seqlens_q,
58
+ cu_seqblocks_q,
59
+ seq_lens,
60
+ prefix_lens,
61
+ slot_ids,
62
+ # shape
63
+ max_slots,
64
+ num_kv_heads,
65
+ gqa_group_size,
66
+ qk_head_dim,
67
+ v_head_dim,
68
+ max_topk,
69
+ # q loop num
70
+ num_q_loop,
71
+ # sm_scale
72
+ sm_scale,
73
+ # stride
74
+ stride_qn,
75
+ stride_qh,
76
+ stride_qd,
77
+ stride_ks,
78
+ stride_kh,
79
+ stride_kd,
80
+ stride_vs,
81
+ stride_vh,
82
+ stride_vd,
83
+ stride_sh,
84
+ stride_sd,
85
+ stride_th,
86
+ stride_tn,
87
+ stride_tk,
88
+ stride_on,
89
+ stride_oh,
90
+ stride_od,
91
+ stride_r2t_b,
92
+ # META parameters
93
+ BLOCK_SIZE_Q: tl.constexpr, # q block size
94
+ BLOCK_SIZE_K: tl.constexpr, # k block size
95
+ BLOCK_SIZE_KD: tl.constexpr,
96
+ BLOCK_SIZE_VD: tl.constexpr,
97
+ BLOCK_SIZE_H: tl.constexpr,
98
+ BLOCK_SIZE_T: tl.constexpr,
99
+ BLOCK_SIZE_QH: tl.constexpr,
100
+ # has sink
101
+ HAS_SINK: tl.constexpr,
102
+ USE_TMA: tl.constexpr,
103
+ IS_FP8: tl.constexpr,
104
+ ):
105
+ sm_scale_log2e = sm_scale * 1.4426950409
106
+ # get batch id and head id
107
+ pid_q = tl.program_id(0)
108
+ pid_kh = tl.program_id(1)
109
+ pid_b = tl.program_id(2)
110
+ pid_h = pid_kh * gqa_group_size
111
+ # get q k start and len after rmpad
112
+ q_start = tl.load(cu_seqlens_q + pid_b)
113
+ q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
114
+ q_block_start = tl.load(cu_seqblocks_q + pid_b)
115
+ q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start
116
+ seq_len = tl.load(seq_lens + pid_b)
117
+ prefix_len = tl.load(prefix_lens + pid_b)
118
+ sid = (
119
+ tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
120
+ ) % max_slots # safety against negative
121
+ if pid_q * num_q_loop >= q_block_len:
122
+ return
123
+ real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop)
124
+ if HAS_SINK:
125
+ sink_ptrs = tl.make_block_ptr(
126
+ base=sink_ptr + pid_h * stride_sh,
127
+ shape=(gqa_group_size, qk_head_dim),
128
+ strides=(stride_sh, stride_sd),
129
+ offsets=(0, 0),
130
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD),
131
+ order=(1, 0),
132
+ )
133
+ sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
134
+ tl.float32
135
+ )
136
+ # offsets for paged K/V load
137
+ off_n = tl.arange(0, BLOCK_SIZE_K)
138
+ off_kd = tl.arange(0, BLOCK_SIZE_KD)
139
+ off_vd = tl.arange(0, BLOCK_SIZE_VD)
140
+ kd_mask = off_kd < qk_head_dim
141
+ vd_mask = off_vd < v_head_dim
142
+ for j in range(real_q_loop):
143
+ pid_q_j = pid_q * num_q_loop + j
144
+ # init topk idx pointer
145
+ t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
146
+ # we assume that the topk_idx is right padded with -1
147
+ off_t = tl.arange(0, BLOCK_SIZE_T)
148
+ topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
149
+ valid_idx = tl.where(topk_idx >= 0, off_t, -1)
150
+ real_topk = tl.sum(valid_idx != -1, axis=0)
151
+ # init qkv pointer
152
+ q_ptrs = tl.make_block_ptr(
153
+ base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
154
+ shape=(q_len, gqa_group_size, qk_head_dim),
155
+ strides=(stride_qn, stride_qh, stride_qd),
156
+ offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
157
+ block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD),
158
+ order=(2, 1, 0),
159
+ )
160
+ # load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D]
161
+ q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero")
162
+ # init statistics
163
+ off_q_k = (
164
+ tl.arange(0, BLOCK_SIZE_Q)[:, None]
165
+ + pid_q_j * BLOCK_SIZE_Q
166
+ + prefix_len
167
+ - tl.arange(0, BLOCK_SIZE_K)[None, :]
168
+ )
169
+ if HAS_SINK:
170
+ m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
171
+ lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
172
+ qsink = (
173
+ tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e
174
+ ) # (BLOCK_SIZE_Q, BLOCK_SIZE_H)
175
+ m_i += qsink
176
+ lse_i += qsink
177
+ m_i = tl.reshape(m_i, BLOCK_SIZE_QH)
178
+ lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH)
179
+ else:
180
+ m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
181
+ lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
182
+ acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32)
183
+ q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD)
184
+ # sparse attention
185
+ for i in range(real_topk):
186
+ # get current block start index (absolute K position)
187
+ c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
188
+ t_ptr_j = t_ptr_j + stride_tk
189
+ # paged load K via req_to_token: pos -> slot -> k_cache
190
+ pos = c + off_n
191
+ pos_mask = pos < seq_len
192
+ slots = tl.load(
193
+ req_to_token_ptr + sid * stride_r2t_b + pos,
194
+ mask=pos_mask,
195
+ other=0,
196
+ ).to(tl.int64)
197
+ slots = (slots + max_slots) % max_slots # safety against negative
198
+ # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
199
+ k = tl.load(
200
+ k_cache_ptr
201
+ + slots[None, :] * stride_ks
202
+ + pid_kh * stride_kh
203
+ + off_kd[:, None] * stride_kd,
204
+ mask=kd_mask[:, None] & pos_mask[None, :],
205
+ other=0.0,
206
+ )
207
+ if IS_FP8:
208
+ # fp8 main K cache is unit-scaled; widen to the Q compute dtype
209
+ # before the tl.dot (compiled out when the cache is bf16).
210
+ k = k.to(q.dtype)
211
+ # compute qk
212
+ qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
213
+ # causal mask
214
+ qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
215
+ qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
216
+ # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
217
+ # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
218
+ qk += tl.dot(q, k) * sm_scale_log2e
219
+ # K boundary mask: positions beyond seq_len contribute -inf
220
+ qk += tl.where(pos_mask[None, :], 0, float("-inf"))
221
+ # compute m_ij and l_ij
222
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
223
+ p = tl.exp2(qk - m_ij[:, None])
224
+ l_ij = tl.sum(p, axis=1)
225
+ # scale acc_o
226
+ acc_o_scale = tl.exp2(m_i - m_ij)
227
+ acc_o = acc_o * acc_o_scale[:, None]
228
+ # paged load V
229
+ v = tl.load(
230
+ v_cache_ptr
231
+ + slots[:, None] * stride_vs
232
+ + pid_kh * stride_vh
233
+ + off_vd[None, :] * stride_vd,
234
+ mask=pos_mask[:, None] & vd_mask[None, :],
235
+ other=0.0,
236
+ )
237
+ if IS_FP8:
238
+ # Widen V so `p.to(v.dtype)` casts P to the compute dtype rather
239
+ # than to fp8 (which would wreck attention-weight precision).
240
+ v = v.to(q.dtype)
241
+ p = p.to(v.dtype)
242
+ acc_o += tl.dot(p, v)
243
+ # update statistics
244
+ m_i = m_ij
245
+ lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
246
+ # final scale
247
+ acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
248
+ # save output
249
+ acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD)
250
+ o_ptrs = tl.make_block_ptr(
251
+ base=o_ptr + q_start * stride_on + pid_h * stride_oh,
252
+ shape=(q_len, gqa_group_size, v_head_dim),
253
+ strides=(stride_on, stride_oh, stride_od),
254
+ offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
255
+ block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD),
256
+ order=(2, 1, 0),
257
+ )
258
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2))
259
+
260
+
261
+ @torch.no_grad()
262
+ def flash_prefill_with_gqa_share_sparse(
263
+ q: torch.Tensor,
264
+ k_cache: torch.Tensor,
265
+ v_cache: torch.Tensor,
266
+ sink: Optional[torch.Tensor],
267
+ req_to_token: torch.Tensor,
268
+ slot_ids: torch.Tensor,
269
+ topk_idx: torch.Tensor,
270
+ block_size_q: int,
271
+ block_size_k: int,
272
+ cu_seqlens: torch.Tensor,
273
+ seq_lens: torch.Tensor,
274
+ prefix_lens: torch.Tensor,
275
+ max_seqlen_q: int,
276
+ sm_scale: Optional[float] = None,
277
+ use_tma: bool = True,
278
+ cu_seqblocks_q: Optional[torch.Tensor] = None,
279
+ max_seqblock_q: Optional[int] = None,
280
+ ) -> torch.Tensor:
281
+ triton.set_allocator(robust_allocator)
282
+ # dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled)
283
+ # under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load.
284
+ # Keep CUDA's pre-existing sparse prefill dtype contract unchanged by
285
+ # accepting fp8 only on HIP.
286
+ assert q.dtype in (torch.bfloat16, torch.float16)
287
+ _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
288
+ is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
289
+ assert k_cache.dtype == q.dtype or is_fp8, (
290
+ f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) "
291
+ f"or fp8 on HIP, got {k_cache.dtype}"
292
+ )
293
+ assert v_cache.dtype == k_cache.dtype
294
+ assert block_size_q in {1, 2, 4, 8, 16, 32, 64}
295
+ assert block_size_k in {16, 32, 64, 128}
296
+ # shape
297
+ total_q, num_q_heads, qk_head_dim = q.shape
298
+ max_slots, num_k_heads, _ = k_cache.shape
299
+ _, num_v_heads, v_head_dim = v_cache.shape
300
+ batch_size = cu_seqlens.shape[0] - 1
301
+ topk = topk_idx.shape[-1]
302
+ assert topk_idx.shape[0] == num_k_heads
303
+ # gqa
304
+ assert num_k_heads == num_v_heads
305
+ assert num_q_heads % num_k_heads == 0
306
+ gqa_group_size = num_q_heads // num_k_heads
307
+ assert gqa_group_size * block_size_q <= 128
308
+ if sm_scale is None:
309
+ sm_scale = qk_head_dim**-0.5
310
+ if cu_seqblocks_q is None or max_seqblock_q is None:
311
+ cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks(
312
+ cu_seqlens, max_seqlen_q, block_size_q, block_size_k
313
+ )
314
+ # output tensor
315
+ o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype)
316
+ # launch kernel
317
+ num_q_loop = (
318
+ max_seqblock_q // 131072 + 1
319
+ ) # calculate multiple queries in one kernel if seqlence length is too long
320
+ BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q)
321
+ BLOCK_SIZE_K = triton.next_power_of_2(block_size_k)
322
+ grid = (
323
+ triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop),
324
+ num_k_heads,
325
+ batch_size,
326
+ )
327
+ _gqa_share_sparse_fwd_kernel[grid](
328
+ q,
329
+ k_cache,
330
+ v_cache,
331
+ sink,
332
+ topk_idx,
333
+ o,
334
+ req_to_token,
335
+ cu_seqlens,
336
+ cu_seqblocks_q,
337
+ seq_lens,
338
+ prefix_lens,
339
+ slot_ids,
340
+ max_slots,
341
+ num_k_heads,
342
+ gqa_group_size,
343
+ qk_head_dim,
344
+ v_head_dim,
345
+ topk,
346
+ num_q_loop,
347
+ sm_scale,
348
+ q.stride(0),
349
+ q.stride(1),
350
+ q.stride(2),
351
+ k_cache.stride(0),
352
+ k_cache.stride(1),
353
+ k_cache.stride(2),
354
+ v_cache.stride(0),
355
+ v_cache.stride(1),
356
+ v_cache.stride(2),
357
+ sink.stride(0) if sink is not None else 0,
358
+ sink.stride(1) if sink is not None else 0,
359
+ topk_idx.stride(0),
360
+ topk_idx.stride(1),
361
+ topk_idx.stride(2),
362
+ o.stride(0),
363
+ o.stride(1),
364
+ o.stride(2),
365
+ req_to_token.stride(0),
366
+ BLOCK_SIZE_Q=BLOCK_SIZE_Q,
367
+ BLOCK_SIZE_K=BLOCK_SIZE_K,
368
+ USE_TMA=use_tma,
369
+ IS_FP8=is_fp8,
370
+ )
371
+ return o
build/torch211-cxx11-cu130-x86_64-linux/quantize.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """NVFP4 helper API compatible with MiniMaxAI/msa.
3
+
4
+ The quantization entrypoints use Transformer Engine when it is installed. The
5
+ layout helpers and reference dequantizer are pure PyTorch and are included so
6
+ callers can prepare and validate NVFP4 metadata with the same public API names
7
+ as the upstream MiniMaxAI package.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Tuple
14
+
15
+ import torch
16
+
17
+
18
+ NVFP4_BLOCK_SIZE = 16
19
+ NVFP4_FP4_MAX = 6.0
20
+ NVFP4_FP8_E4M3_MAX = 448.0
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Nvfp4QuantizedTensor:
25
+ data: torch.Tensor
26
+ scale_128x4: torch.Tensor
27
+ global_scale: torch.Tensor
28
+ logical_scale_shape: Tuple[int, int]
29
+ original_shape: Tuple[int, ...]
30
+
31
+
32
+ def _round_up(x: int, multiple: int) -> int:
33
+ return ((int(x) + multiple - 1) // multiple) * multiple
34
+
35
+
36
+ def nvfp4_scale_128x4_offset(
37
+ row: torch.Tensor,
38
+ col: torch.Tensor,
39
+ scale_cols: int,
40
+ ) -> torch.Tensor:
41
+ tiles_n = _round_up(scale_cols, 4) // 4
42
+ tile_m = row // 128
43
+ tile_n = col // 4
44
+ outer = row % 128
45
+ inner = col % 4
46
+ return (
47
+ (tile_m * tiles_n + tile_n) * 512
48
+ + (outer % 32) * 16
49
+ + (outer // 32) * 4
50
+ + inner
51
+ )
52
+
53
+
54
+ def swizzle_nvfp4_scale_to_128x4(
55
+ scale: torch.Tensor,
56
+ *,
57
+ rows: int,
58
+ cols: int,
59
+ ) -> torch.Tensor:
60
+ if scale.ndim != 2:
61
+ raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
62
+
63
+ rows = int(rows)
64
+ cols = int(cols)
65
+ padded_rows = _round_up(rows, 128)
66
+ padded_cols = _round_up(cols, 4)
67
+ if scale.shape[0] < rows or scale.shape[1] < cols:
68
+ raise ValueError(
69
+ "scale is smaller than the requested logical shape: "
70
+ f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
71
+ )
72
+
73
+ logical = scale[:rows, :cols].contiguous()
74
+ if logical.shape != (padded_rows, padded_cols):
75
+ logical = torch.nn.functional.pad(
76
+ logical.to(torch.float32),
77
+ (0, padded_cols - cols, 0, padded_rows - rows),
78
+ ).to(scale.dtype)
79
+ swizzled = torch.empty_like(logical)
80
+
81
+ row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
82
+ col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
83
+ offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
84
+ swizzled.reshape(-1)[offset] = logical.reshape(-1)
85
+ return swizzled
86
+
87
+
88
+ def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
89
+ return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
90
+
91
+
92
+ def _import_te_nvfp4_quantizer():
93
+ try:
94
+ from transformer_engine.pytorch.tensor import NVFP4Quantizer
95
+ except Exception as exc:
96
+ raise RuntimeError(
97
+ "Transformer Engine NVFP4 quantization is unavailable. Install a "
98
+ "Transformer Engine build with NVFP4 PyTorch support to use "
99
+ "quantize_bf16_to_nvfp4_128x4."
100
+ ) from exc
101
+ return NVFP4Quantizer
102
+
103
+
104
+ def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
105
+ if not x.is_cuda:
106
+ raise ValueError("NVFP4 quantization requires a CUDA tensor")
107
+ if x.dtype not in (torch.bfloat16, torch.float16):
108
+ raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
109
+ if x.ndim < 2:
110
+ raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
111
+ if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
112
+ raise ValueError(
113
+ f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
114
+ )
115
+
116
+ rows = 1
117
+ for dim in x.shape[:-1]:
118
+ rows *= int(dim)
119
+ if rows % NVFP4_BLOCK_SIZE != 0:
120
+ raise ValueError(
121
+ "flattened row dimension must be divisible by "
122
+ f"{NVFP4_BLOCK_SIZE}, got {rows}"
123
+ )
124
+
125
+ NVFP4Quantizer = _import_te_nvfp4_quantizer()
126
+ quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
127
+ qx = quantizer.quantize(x.contiguous())
128
+ meta = qx.get_metadata()
129
+
130
+ data = meta["rowwise_data"]
131
+ if data.dtype is not torch.uint8:
132
+ data = data.view(torch.uint8)
133
+ logical_scale = meta["rowwise_scale_inv"]
134
+ amax = meta["amax_rowwise"]
135
+ scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
136
+ scale_128x4 = swizzle_nvfp4_scale_to_128x4(
137
+ logical_scale,
138
+ rows=rows,
139
+ cols=scale_cols,
140
+ )
141
+ global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
142
+
143
+ return Nvfp4QuantizedTensor(
144
+ data=data,
145
+ scale_128x4=scale_128x4,
146
+ global_scale=global_scale,
147
+ logical_scale_shape=(rows, scale_cols),
148
+ original_shape=tuple(int(v) for v in x.shape),
149
+ )
150
+
151
+
152
+ def quantize_kv_bf16_to_nvfp4_128x4(
153
+ k: torch.Tensor,
154
+ v: torch.Tensor,
155
+ ) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
156
+ return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
157
+
158
+
159
+ def dequantize_nvfp4_128x4_to_bf16(
160
+ qx: Nvfp4QuantizedTensor,
161
+ *,
162
+ include_global_scale: bool = True,
163
+ ) -> torch.Tensor:
164
+ data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
165
+ if data.shape[-1] * 2 != qx.original_shape[-1]:
166
+ raise ValueError(
167
+ "packed data last dimension does not match original shape: "
168
+ f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
169
+ )
170
+
171
+ rows, scale_cols = qx.logical_scale_shape
172
+ logical_dim = int(qx.original_shape[-1])
173
+ if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
174
+ raise ValueError(
175
+ "logical scale columns do not match original last dimension: "
176
+ f"{scale_cols} scale cols vs dim {logical_dim}"
177
+ )
178
+
179
+ fp4_lut = torch.tensor(
180
+ [
181
+ 0.0,
182
+ 0.5,
183
+ 1.0,
184
+ 1.5,
185
+ 2.0,
186
+ 3.0,
187
+ 4.0,
188
+ 6.0,
189
+ -0.0,
190
+ -0.5,
191
+ -1.0,
192
+ -1.5,
193
+ -2.0,
194
+ -3.0,
195
+ -4.0,
196
+ -6.0,
197
+ ],
198
+ dtype=torch.float32,
199
+ device=data.device,
200
+ )
201
+ packed = data.reshape(rows, logical_dim // 2)
202
+ lo = packed & 0x0F
203
+ hi = packed >> 4
204
+ values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
205
+ values[:, 0::2] = fp4_lut[lo.long()]
206
+ values[:, 1::2] = fp4_lut[hi.long()]
207
+
208
+ row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
209
+ col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
210
+ offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
211
+ scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
212
+ scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
213
+ scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
214
+ out = values * scale
215
+ if include_global_scale:
216
+ out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
217
+ return out.reshape(qx.original_shape).to(torch.bfloat16)
218
+
219
+
220
+ __all__ = [
221
+ "Nvfp4QuantizedTensor",
222
+ "quantize_bf16_to_nvfp4_128x4",
223
+ "quantize_kv_bf16_to_nvfp4_128x4",
224
+ "dequantize_nvfp4_128x4_to_bf16",
225
+ "swizzle_nvfp4_scale_to_128x4",
226
+ "nvfp4_global_scale_from_amax",
227
+ ]
build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Sparse-index helpers compatible with MiniMaxAI/msa public names."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+
11
+ def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
12
+ if cu_seqlens.dtype != torch.int32:
13
+ raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
14
+ if cu_seqlens.ndim != 1:
15
+ raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
16
+ if cu_seqlens.numel() < 1:
17
+ raise ValueError(f"{name} must have at least one element")
18
+ if not cu_seqlens.is_contiguous():
19
+ raise ValueError(f"{name} must be contiguous")
20
+
21
+
22
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
23
+ seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
24
+ return (seqlens_k + kv_block_size - 1) // kv_block_size
25
+
26
+
27
+ def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
28
+ rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
29
+ batch = len(rows_per_batch_cpu)
30
+ max_rows = max(rows_per_batch_cpu, default=0)
31
+ row_dtype = (
32
+ torch.int32
33
+ if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
34
+ else torch.int64
35
+ )
36
+ row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
37
+ row_linear = 0
38
+ for kv_block_idx in range(max_rows):
39
+ for batch_idx, row_count in enumerate(rows_per_batch_cpu):
40
+ if kv_block_idx < row_count:
41
+ row_map_cpu[batch_idx, kv_block_idx] = row_linear
42
+ row_linear += 1
43
+ return row_map_cpu.to(rows_per_batch.device), row_linear
44
+
45
+
46
+ def build_k2q_csr_torch_reference(
47
+ q2k_indices: torch.Tensor,
48
+ cu_seqlens_q: torch.Tensor,
49
+ cu_seqlens_k: torch.Tensor,
50
+ kv_block_size: int,
51
+ ) -> tuple[torch.Tensor, torch.Tensor]:
52
+ if kv_block_size <= 0:
53
+ raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
54
+ if q2k_indices.dtype != torch.int32:
55
+ raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
56
+ if q2k_indices.ndim != 3:
57
+ raise ValueError(
58
+ "q2k_indices must have shape [head_kv, total_q, topK], "
59
+ f"got {tuple(q2k_indices.shape)}"
60
+ )
61
+ _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
62
+ _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
63
+ if cu_seqlens_q.shape != cu_seqlens_k.shape:
64
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
65
+ if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
66
+ raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
67
+
68
+ head_kv, total_q, topk = q2k_indices.shape
69
+ if total_q != int(cu_seqlens_q[-1].item()):
70
+ raise ValueError(
71
+ f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
72
+ f"({int(cu_seqlens_q[-1].item())})"
73
+ )
74
+
75
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
76
+ row_map, total_rows = _build_packed_row_map(rows_per_batch)
77
+ nnz_upper_bound = total_q * topk
78
+
79
+ k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
80
+ k2q_q_indices = torch.full(
81
+ (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
82
+ )
83
+ if total_rows == 0 or total_q == 0 or topk == 0:
84
+ return k2q_row_ptr, k2q_q_indices
85
+
86
+ counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
87
+ total_entries = total_q * topk
88
+ row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
89
+ row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
90
+ q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
91
+ valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
92
+
93
+ for batch_idx in range(cu_seqlens_q.numel() - 1):
94
+ q_start = int(cu_seqlens_q[batch_idx].item())
95
+ q_end = int(cu_seqlens_q[batch_idx + 1].item())
96
+ if q_end <= q_start:
97
+ continue
98
+ q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
99
+ q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
100
+ kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
101
+ valid = kv_local >= 0
102
+ safe_kv = kv_local.clamp_min(0).long()
103
+ rows = row_map[batch_idx, safe_kv].to(row_dtype)
104
+ start = q_start * topk
105
+ end = q_end * topk
106
+ row_all[:, start:end] = rows
107
+ q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
108
+ valid_all[:, start:end] = valid & (rows >= 0)
109
+ counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
110
+
111
+ k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
112
+ offsets = torch.zeros_like(counts)
113
+ offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
114
+ cursor = torch.zeros_like(counts)
115
+ for h in range(head_kv):
116
+ for entry in range(total_entries):
117
+ if not bool(valid_all[h, entry]):
118
+ continue
119
+ row = int(row_all[h, entry].item())
120
+ pos = int(offsets[h, row].item() + cursor[h, row].item())
121
+ k2q_q_indices[h, pos] = q_all[h, entry]
122
+ cursor[h, row] += 1
123
+
124
+ return k2q_row_ptr, k2q_q_indices
125
+
126
+
127
+ def build_k2q_csr(
128
+ q2k_indices: torch.Tensor,
129
+ cu_seqlens_q: torch.Tensor,
130
+ cu_seqlens_k: torch.Tensor,
131
+ kv_block_size: int,
132
+ *,
133
+ total_k: Optional[int] = None,
134
+ max_seqlen_k: Optional[int] = None,
135
+ max_seqlen_q: Optional[int] = None,
136
+ total_rows: Optional[int] = None,
137
+ qhead_per_kv: int = 1,
138
+ return_schedule: bool = False,
139
+ ):
140
+ if return_schedule:
141
+ raise NotImplementedError(
142
+ "Blackwell build_k2q_csr currently returns CSR tensors only; "
143
+ "SM100 schedule objects are not part of this package."
144
+ )
145
+ if total_k is None:
146
+ raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
147
+ return build_k2q_csr_torch_reference(
148
+ q2k_indices.contiguous(),
149
+ cu_seqlens_q.contiguous(),
150
+ cu_seqlens_k.contiguous(),
151
+ int(kv_block_size),
152
+ )
153
+
154
+
155
+ class SparseK2qCsrBuilderSm100:
156
+ """Compatibility placeholder for the upstream SM100 class name.
157
+
158
+ The name is exported so code can feature-detect it, but schedule-producing
159
+ SM100 behavior is intentionally not claimed for this Blackwell package.
160
+ """
161
+
162
+ def __init__(self, *args, **kwargs):
163
+ self.args = args
164
+ self.kwargs = kwargs
165
+
166
+ def build(self, *args, **kwargs):
167
+ return build_k2q_csr(*args, **kwargs)
168
+
169
+
170
+ __all__ = [
171
+ "build_k2q_csr",
172
+ "build_k2q_csr_torch_reference",
173
+ "SparseK2qCsrBuilderSm100",
174
+ ]
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """FlashRT Blackwell extension of MiniMax MSA decode sparse attention.
3
+
4
+ This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on
5
+ DGX Spark / GB10 / SM121. It exposes the
6
+ MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime.
7
+
8
+ Implementation status:
9
+ * native CUDA helper: score -> top-k sparse block ids;
10
+ * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
11
+ * native CUDA FP4 block-score indexer;
12
+ * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path;
13
+ * Blackwell-validated sparse prefill path;
14
+ * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers,
15
+ and FP4 block-score helpers.
16
+
17
+ The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell
18
+ extension path. The public API is Tensor-oriented and independent from
19
+ FlashRT's serving runtime.
20
+
21
+ Public API
22
+ ----------
23
+ Lightning indexer (q.k blockmax score -> top-k block select), paged KV:
24
+ flash_decode_with_topk_idx (single-token decode, split-K)
25
+
26
+ Block-sparse GQA attention (consumes block_indices from the indexer):
27
+ flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks)
28
+ sparse_atten_func (official CSR prefill wrapper)
29
+ sparse_decode_atten_func (official paged decode wrapper)
30
+
31
+ PyTorch naive references (for correctness checks):
32
+ naive_flash_decode_with_gqa_share_sparse (attention given topk_idx)
33
+ naive_flash_decode_with_topk_idx (joint indexer + attention)
34
+
35
+ All Triton entrypoints expect a paged KV cache:
36
+ k_cache / v_cache : [max_slots, num_kv_heads, head_dim]
37
+ req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot)
38
+ slot_ids : [batch] (per-request row in req_to_token)
39
+ topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed,
40
+ -1 right-padding (the M3 indexer contract).
41
+ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
42
+ """
43
+
44
+ from .common.utils import get_cu_seqblocks, robust_allocator
45
+ from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
46
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
47
+ from .interface import (
48
+ SparseDecodePagedAttentionWrapper,
49
+ fp4_indexer_block_scores,
50
+ sparse_atten_func,
51
+ sparse_atten_nvfp4_kv_func,
52
+ sparse_decode_atten_func,
53
+ )
54
+ from ._native import (
55
+ has_native_ops,
56
+ native_nvfp4_dequant_swizzled_to_bf16,
57
+ native_topk_from_scores,
58
+ )
59
+ from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
60
+ from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
61
+ from .quantize import (
62
+ Nvfp4QuantizedTensor,
63
+ dequantize_nvfp4_128x4_to_bf16,
64
+ nvfp4_global_scale_from_amax,
65
+ quantize_bf16_to_nvfp4_128x4,
66
+ quantize_kv_bf16_to_nvfp4_128x4,
67
+ swizzle_nvfp4_scale_to_128x4,
68
+ )
69
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
70
+ from .api_status import (
71
+ OFFICIAL_API_STATUS,
72
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
73
+ V1_AVAILABLE_FUNCTIONS,
74
+ available_functions,
75
+ official_api_status,
76
+ official_minimax_msa_functions,
77
+ unsupported_official_functions,
78
+ )
79
+
80
+ __all__ = [
81
+ "sparse_atten_func",
82
+ "sparse_atten_nvfp4_kv_func",
83
+ "sparse_decode_atten_func",
84
+ "SparseDecodePagedAttentionWrapper",
85
+ "fp4_indexer_block_scores",
86
+ "build_k2q_csr",
87
+ "SparseK2qCsrBuilderSm100",
88
+ "Nvfp4QuantizedTensor",
89
+ "quantize_bf16_to_nvfp4_128x4",
90
+ "quantize_kv_bf16_to_nvfp4_128x4",
91
+ "dequantize_nvfp4_128x4_to_bf16",
92
+ "swizzle_nvfp4_scale_to_128x4",
93
+ "nvfp4_global_scale_from_amax",
94
+ "flash_decode_with_topk_idx",
95
+ "flash_decode_with_gqa_share_sparse",
96
+ "has_native_ops",
97
+ "native_topk_from_scores",
98
+ "native_nvfp4_dequant_swizzled_to_bf16",
99
+ "naive_flash_decode_with_gqa_share_sparse",
100
+ "naive_flash_decode_with_topk_idx",
101
+ "get_cu_seqblocks",
102
+ "robust_allocator",
103
+ "OFFICIAL_API_STATUS",
104
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
105
+ "V1_AVAILABLE_FUNCTIONS",
106
+ "available_functions",
107
+ "official_api_status",
108
+ "official_minimax_msa_functions",
109
+ "unsupported_official_functions",
110
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_compat.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton
3
+ # kernels. The upstream SGLang modules pull two tiny things from the framework:
4
+ #
5
+ # from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path)
6
+ # from sglang.srt.environ import envs -> a feature flag for an optional
7
+ # JIT-CUDA radix top-k fast path
8
+ #
9
+ # Both are replaced here so the kernels import with only torch + triton. The
10
+ # defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel
11
+ # radix select), which is what we want on sm_121 (GB10).
12
+
13
+
14
+ def is_hip() -> bool:
15
+ """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121).
16
+
17
+ Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV
18
+ widening branch (HIP-only). With this False, the CUDA dtype contract
19
+ (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA.
20
+ """
21
+ try:
22
+ import torch
23
+
24
+ return bool(getattr(torch.version, "hip", None))
25
+ except Exception:
26
+ return False
27
+
28
+
29
+ class _Flag:
30
+ """Minimal stand-in for an sglang.srt.environ boolean flag (always off)."""
31
+
32
+ def __init__(self, default: bool = False):
33
+ self._default = default
34
+
35
+ def get(self) -> bool:
36
+ return self._default
37
+
38
+
39
+ class _Envs:
40
+ """Stub for `sglang.srt.environ.envs`.
41
+
42
+ Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode
43
+ indexer. Keeping it False routes top-k selection through the 2-stage Triton
44
+ fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the
45
+ decode indexer stays pure-Triton.
46
+ """
47
+
48
+ SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False)
49
+
50
+
51
+ envs = _Envs()
build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ea862050a19dcc4b0c7cbfd3b519112f1b4e054cfc182a14f92876b4ae5b9f7
3
+ size 735888