Instructions to use flashrt/MiniMaxAI-msa-blackwell with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Kernels
How to use flashrt/MiniMaxAI-msa-blackwell with Kernels:
# !pip install kernels from kernels import get_kernel # a version (or an explicit revision) is required; see the "Files and versions" tab for the available ones kernel = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1) - Notebooks
- Google Colab
- Kaggle
Promote latest kernel artifacts to main
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +3 -35
- CARD.md +223 -0
- README.md +220 -6
- VALIDATION.md +206 -0
- benchmarks/benchmark_decode.py +76 -0
- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +110 -0
- build/torch211-cxx11-cu128-x86_64-linux/_compat.py +51 -0
- build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so +3 -0
- build/torch211-cxx11-cu128-x86_64-linux/_native.py +259 -0
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu128-x86_64-linux/api_status.py +146 -0
- build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py +1 -0
- build/torch211-cxx11-cu128-x86_64-linux/common/utils.py +144 -0
- build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py +1 -0
- build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py +1150 -0
- build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py +452 -0
- build/torch211-cxx11-cu128-x86_64-linux/interface.py +679 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +38 -0
- build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py +26 -0
- build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py +1 -0
- build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py +106 -0
- build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py +88 -0
- build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py +4 -0
- build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py +371 -0
- build/torch211-cxx11-cu128-x86_64-linux/quantize.py +227 -0
- build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py +174 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +110 -0
- build/torch211-cxx11-cu130-x86_64-linux/_compat.py +51 -0
- build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so +3 -0
- build/torch211-cxx11-cu130-x86_64-linux/_native.py +259 -0
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu130-x86_64-linux/api_status.py +146 -0
- build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py +1 -0
- build/torch211-cxx11-cu130-x86_64-linux/common/utils.py +144 -0
- build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py +1 -0
- build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py +1150 -0
- build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py +452 -0
- build/torch211-cxx11-cu130-x86_64-linux/interface.py +679 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +39 -0
- build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py +26 -0
- build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py +1 -0
- build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py +106 -0
- build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py +88 -0
- build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py +4 -0
- build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py +371 -0
- build/torch211-cxx11-cu130-x86_64-linux/quantize.py +227 -0
- build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py +174 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +110 -0
- build/torch212-cxx11-cu130-x86_64-linux/_compat.py +51 -0
- build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so +3 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.so filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.pyd filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.dylib filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
CARD.md
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: kernels
|
| 3 |
+
license: apache-2.0
|
| 4 |
+
tags:
|
| 5 |
+
- cuda
|
| 6 |
+
- triton
|
| 7 |
+
- native-cuda
|
| 8 |
+
- minimax
|
| 9 |
+
- sparse-attention
|
| 10 |
+
- blackwell
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
# MiniMaxAI MSA Blackwell
|
| 14 |
+
|
| 15 |
+
Blackwell-family package for MiniMax MSA decode sparse attention, maintained by
|
| 16 |
+
FlashRT. The upstream package is
|
| 17 |
+
[`MiniMaxAI/msa`](https://huggingface.co/kernels/MiniMaxAI/msa), which targets
|
| 18 |
+
SM100. This package extends the decode-sparse path to NVIDIA Blackwell
|
| 19 |
+
compute capability 12.x and has been validated in the FlashRT MiniMax-Spark
|
| 20 |
+
runtime on DGX Spark / GB10 / SM121.
|
| 21 |
+
|
| 22 |
+
## Load
|
| 23 |
+
|
| 24 |
+
```python
|
| 25 |
+
from kernels import get_kernel
|
| 26 |
+
|
| 27 |
+
msa = get_kernel(
|
| 28 |
+
"flashrt/MiniMaxAI-msa-blackwell",
|
| 29 |
+
version=1,
|
| 30 |
+
trust_remote_code=True,
|
| 31 |
+
)
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## What You Can Call
|
| 35 |
+
|
| 36 |
+
### Official MiniMaxAI/msa names
|
| 37 |
+
|
| 38 |
+
| Function/class | Status in this Blackwell package |
|
| 39 |
+
|---|---|
|
| 40 |
+
| `sparse_decode_atten_func` | Available. Blackwell paged BF16/FP16 single-token decode wrapper. |
|
| 41 |
+
| `SparseDecodePagedAttentionWrapper` | Available. `plan(...).run(...)` wrapper for the same decode path. |
|
| 42 |
+
| `build_k2q_csr` | Available. CSR construction helper for the official prefill API. |
|
| 43 |
+
| `SparseK2qCsrBuilderSm100` | Available compatibility class; `build()` delegates to `build_k2q_csr`. |
|
| 44 |
+
| `Nvfp4QuantizedTensor` | Available metadata dataclass. |
|
| 45 |
+
| `quantize_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
|
| 46 |
+
| `quantize_kv_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
|
| 47 |
+
| `dequantize_nvfp4_128x4_to_bf16` | Available reference dequantizer. |
|
| 48 |
+
| `swizzle_nvfp4_scale_to_128x4` | Available scale-layout helper. |
|
| 49 |
+
| `nvfp4_global_scale_from_amax` | Available scale helper. |
|
| 50 |
+
| `sparse_atten_func` | Available. Official CSR sparse prefill API backed by the Blackwell Triton BF16/FP16 prefill kernel. |
|
| 51 |
+
| `sparse_atten_nvfp4_kv_func` | Available. Built artifacts use native CUDA swizzled NVFP4 -> BF16 dequantization, then call Blackwell sparse prefill. |
|
| 52 |
+
| `fp4_indexer_block_scores` | Available. Built artifacts use the native CUDA Blackwell block-score kernel and return the official `[Hq, ceil(max_seqlen_k/128), total_q]` score layout. |
|
| 53 |
+
|
| 54 |
+
### FlashRT Blackwell helper names
|
| 55 |
+
|
| 56 |
+
These are the direct low-level APIs used by the FlashRT MiniMax-Spark decode
|
| 57 |
+
path:
|
| 58 |
+
|
| 59 |
+
- `flash_decode_with_topk_idx`
|
| 60 |
+
- `flash_decode_with_gqa_share_sparse`
|
| 61 |
+
- `native_topk_from_scores`
|
| 62 |
+
- `native_nvfp4_dequant_swizzled_to_bf16`
|
| 63 |
+
- `has_native_ops`
|
| 64 |
+
- `naive_flash_decode_with_topk_idx`
|
| 65 |
+
- `naive_flash_decode_with_gqa_share_sparse`
|
| 66 |
+
- `get_cu_seqblocks`
|
| 67 |
+
- `robust_allocator`
|
| 68 |
+
|
| 69 |
+
## Decode Example
|
| 70 |
+
|
| 71 |
+
This example uses the official MiniMax decode-facing name
|
| 72 |
+
`sparse_decode_atten_func`.
|
| 73 |
+
|
| 74 |
+
```python
|
| 75 |
+
import torch
|
| 76 |
+
from kernels import get_kernel
|
| 77 |
+
|
| 78 |
+
msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
|
| 79 |
+
|
| 80 |
+
B, Hq, Hkv, D = 1, 64, 4, 128
|
| 81 |
+
page_size = 128
|
| 82 |
+
num_pages = 32
|
| 83 |
+
topk = 16
|
| 84 |
+
device = "cuda"
|
| 85 |
+
dtype = torch.bfloat16
|
| 86 |
+
|
| 87 |
+
q = torch.randn(B, Hq, D, device=device, dtype=dtype)
|
| 88 |
+
k = torch.randn(num_pages, Hkv, page_size, D, device=device, dtype=dtype)
|
| 89 |
+
v = torch.randn_like(k)
|
| 90 |
+
page_table = torch.arange(num_pages, device=device, dtype=torch.int32).view(B, -1)
|
| 91 |
+
seqused_k = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
|
| 92 |
+
q2k_indices = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
|
| 93 |
+
q2k_indices = q2k_indices.expand(Hkv, B, topk).contiguous()
|
| 94 |
+
|
| 95 |
+
out = msa.sparse_decode_atten_func(
|
| 96 |
+
q,
|
| 97 |
+
k,
|
| 98 |
+
v,
|
| 99 |
+
q2k_indices,
|
| 100 |
+
page_table=page_table,
|
| 101 |
+
seqused_k=seqused_k,
|
| 102 |
+
seqlen_q=1,
|
| 103 |
+
max_seqlen_k=num_pages * page_size,
|
| 104 |
+
blk_kv=page_size,
|
| 105 |
+
)
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
## Wrapper Example
|
| 109 |
+
|
| 110 |
+
```python
|
| 111 |
+
wrapper = msa.SparseDecodePagedAttentionWrapper(blk_kv=128)
|
| 112 |
+
wrapper.plan(
|
| 113 |
+
page_table=page_table,
|
| 114 |
+
seqused_k=seqused_k,
|
| 115 |
+
seqlen_q=1,
|
| 116 |
+
max_seqlen_k=num_pages * page_size,
|
| 117 |
+
q2k_indices=q2k_indices,
|
| 118 |
+
num_qo_heads=Hq,
|
| 119 |
+
num_kv_heads=Hkv,
|
| 120 |
+
head_dim=D,
|
| 121 |
+
)
|
| 122 |
+
out = wrapper.run(q, k, v)
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
## Prefill Example
|
| 126 |
+
|
| 127 |
+
This example uses the official MiniMax CSR prefill-facing name
|
| 128 |
+
`sparse_atten_func`.
|
| 129 |
+
|
| 130 |
+
```python
|
| 131 |
+
import torch
|
| 132 |
+
from kernels import get_kernel
|
| 133 |
+
|
| 134 |
+
msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
|
| 135 |
+
|
| 136 |
+
T, Hq, Hkv, D = 512, 64, 4, 128
|
| 137 |
+
page_size = 128
|
| 138 |
+
topk = 16
|
| 139 |
+
device = "cuda"
|
| 140 |
+
dtype = torch.bfloat16
|
| 141 |
+
|
| 142 |
+
q = torch.randn(T, Hq, D, device=device, dtype=dtype)
|
| 143 |
+
k = torch.randn(T, Hkv, D, device=device, dtype=dtype)
|
| 144 |
+
v = torch.randn_like(k)
|
| 145 |
+
cu = torch.tensor([0, T], device=device, dtype=torch.int32)
|
| 146 |
+
|
| 147 |
+
q2k = torch.full((Hkv, T, topk), -1, device=device, dtype=torch.int32)
|
| 148 |
+
for qi in range(T):
|
| 149 |
+
blocks = torch.arange(qi // page_size + 1, device=device, dtype=torch.int32)
|
| 150 |
+
q2k[:, qi, : min(topk, blocks.numel())] = blocks[:topk]
|
| 151 |
+
|
| 152 |
+
k2q_row_ptr, k2q_q_indices = msa.build_k2q_csr(
|
| 153 |
+
q2k, cu, cu, page_size, total_k=T
|
| 154 |
+
)
|
| 155 |
+
out = msa.sparse_atten_func(
|
| 156 |
+
q,
|
| 157 |
+
k,
|
| 158 |
+
v,
|
| 159 |
+
k2q_row_ptr,
|
| 160 |
+
k2q_q_indices,
|
| 161 |
+
topk,
|
| 162 |
+
cu_seqlens_q=cu,
|
| 163 |
+
cu_seqlens_k=cu,
|
| 164 |
+
max_seqlen_q=T,
|
| 165 |
+
max_seqlen_k=T,
|
| 166 |
+
blk_kv=page_size,
|
| 167 |
+
)
|
| 168 |
+
```
|
| 169 |
+
|
| 170 |
+
## Direct FlashRT Decode Path
|
| 171 |
+
|
| 172 |
+
Use this lower-level path if you already have `topk_idx` in FlashRT's paged KV
|
| 173 |
+
layout.
|
| 174 |
+
|
| 175 |
+
```python
|
| 176 |
+
q = torch.randn(B, Hq, D, device=device, dtype=dtype)
|
| 177 |
+
k_cache = torch.randn(num_pages * page_size, Hkv, D, device=device, dtype=dtype)
|
| 178 |
+
v_cache = torch.randn_like(k_cache)
|
| 179 |
+
req_to_token = torch.arange(num_pages * page_size, device=device, dtype=torch.int32).view(B, -1)
|
| 180 |
+
seq_lens = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
|
| 181 |
+
slot_ids = torch.zeros(B, device=device, dtype=torch.int64)
|
| 182 |
+
topk_idx = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
|
| 183 |
+
topk_idx = topk_idx.expand(Hkv, B, topk).contiguous()
|
| 184 |
+
|
| 185 |
+
out = msa.flash_decode_with_gqa_share_sparse(
|
| 186 |
+
q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, page_size, topk_idx
|
| 187 |
+
)
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
## Scope
|
| 191 |
+
|
| 192 |
+
- Target family: NVIDIA Blackwell CUDA compute capability 12.x.
|
| 193 |
+
- Builder target: CUDA 12.8+ with `cuda-capabilities = ["12.0", "12.1"]`.
|
| 194 |
+
- Validated hardware: DGX Spark / GB10 / SM121.
|
| 195 |
+
- Validated MiniMax shape: query heads `64`, KV heads `4`, head dim `128`,
|
| 196 |
+
sparse block/page size `128`, top-k blocks `16`.
|
| 197 |
+
- End-to-end model validation: MiniMax-Spark runtime on GB10 through `32768`
|
| 198 |
+
context length.
|
| 199 |
+
- Standalone kernel long-context validation: `128`, `2048`, `4096`, `32768`,
|
| 200 |
+
`65536`, `131072`.
|
| 201 |
+
- Correctness gate: cosine similarity `>= 0.999` against paged FP32 PyTorch
|
| 202 |
+
references; official decode wrapper output matches the direct Blackwell
|
| 203 |
+
decode kernel.
|
| 204 |
+
|
| 205 |
+
## Implementation Notes
|
| 206 |
+
|
| 207 |
+
This package contains:
|
| 208 |
+
|
| 209 |
+
- native CUDA score-to-top-k helper;
|
| 210 |
+
- native CUDA tensor-core sparse decode route for the MiniMax-M3 Blackwell shape;
|
| 211 |
+
- native CUDA FP4 block-score indexer;
|
| 212 |
+
- native CUDA swizzled NVFP4 -> BF16 dequantization for the W4A16 quality path;
|
| 213 |
+
- Blackwell-validated sparse prefill attention wrapper;
|
| 214 |
+
- MiniMaxAI/msa-compatible Python API layer for decode, prefill, CSR, NVFP4,
|
| 215 |
+
and FP4 block-score helpers.
|
| 216 |
+
|
| 217 |
+
When loaded from Hub built artifacts, the decode, FP4 indexer, and NVFP4
|
| 218 |
+
dequant hot paths use compiled CUDA ops. The source-tree mode keeps reference
|
| 219 |
+
paths so the API and correctness tests remain runnable before a wheel/shared
|
| 220 |
+
object has been built.
|
| 221 |
+
|
| 222 |
+
Source provenance and validation details are documented in `SYNC.md` and
|
| 223 |
+
`VALIDATION.md`.
|
README.md
CHANGED
|
@@ -1,9 +1,223 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
-
|
| 4 |
-
that resolve repositories through the default Hugging Face model repo API.
|
| 5 |
|
| 6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
-
|
| 9 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: kernels
|
| 3 |
+
license: apache-2.0
|
| 4 |
+
tags:
|
| 5 |
+
- cuda
|
| 6 |
+
- triton
|
| 7 |
+
- native-cuda
|
| 8 |
+
- minimax
|
| 9 |
+
- sparse-attention
|
| 10 |
+
- blackwell
|
| 11 |
+
---
|
| 12 |
|
| 13 |
+
# MiniMaxAI MSA Blackwell
|
|
|
|
| 14 |
|
| 15 |
+
Blackwell-family package for MiniMax MSA decode sparse attention, maintained by
|
| 16 |
+
FlashRT. The upstream package is
|
| 17 |
+
[`MiniMaxAI/msa`](https://huggingface.co/kernels/MiniMaxAI/msa), which targets
|
| 18 |
+
SM100. This package extends the decode-sparse path to NVIDIA Blackwell
|
| 19 |
+
compute capability 12.x and has been validated in the FlashRT MiniMax-Spark
|
| 20 |
+
runtime on DGX Spark / GB10 / SM121.
|
| 21 |
|
| 22 |
+
## Load
|
| 23 |
+
|
| 24 |
+
```python
|
| 25 |
+
from kernels import get_kernel
|
| 26 |
+
|
| 27 |
+
msa = get_kernel(
|
| 28 |
+
"flashrt/MiniMaxAI-msa-blackwell",
|
| 29 |
+
version=1,
|
| 30 |
+
trust_remote_code=True,
|
| 31 |
+
)
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
## What You Can Call
|
| 35 |
+
|
| 36 |
+
### Official MiniMaxAI/msa names
|
| 37 |
+
|
| 38 |
+
| Function/class | Status in this Blackwell package |
|
| 39 |
+
|---|---|
|
| 40 |
+
| `sparse_decode_atten_func` | Available. Blackwell paged BF16/FP16 single-token decode wrapper. |
|
| 41 |
+
| `SparseDecodePagedAttentionWrapper` | Available. `plan(...).run(...)` wrapper for the same decode path. |
|
| 42 |
+
| `build_k2q_csr` | Available. CSR construction helper for the official prefill API. |
|
| 43 |
+
| `SparseK2qCsrBuilderSm100` | Available compatibility class; `build()` delegates to `build_k2q_csr`. |
|
| 44 |
+
| `Nvfp4QuantizedTensor` | Available metadata dataclass. |
|
| 45 |
+
| `quantize_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
|
| 46 |
+
| `quantize_kv_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. |
|
| 47 |
+
| `dequantize_nvfp4_128x4_to_bf16` | Available reference dequantizer. |
|
| 48 |
+
| `swizzle_nvfp4_scale_to_128x4` | Available scale-layout helper. |
|
| 49 |
+
| `nvfp4_global_scale_from_amax` | Available scale helper. |
|
| 50 |
+
| `sparse_atten_func` | Available. Official CSR sparse prefill API backed by the Blackwell Triton BF16/FP16 prefill kernel. |
|
| 51 |
+
| `sparse_atten_nvfp4_kv_func` | Available. Built artifacts use native CUDA swizzled NVFP4 -> BF16 dequantization, then call Blackwell sparse prefill. |
|
| 52 |
+
| `fp4_indexer_block_scores` | Available. Built artifacts use the native CUDA Blackwell block-score kernel and return the official `[Hq, ceil(max_seqlen_k/128), total_q]` score layout. |
|
| 53 |
+
|
| 54 |
+
### FlashRT Blackwell helper names
|
| 55 |
+
|
| 56 |
+
These are the direct low-level APIs used by the FlashRT MiniMax-Spark decode
|
| 57 |
+
path:
|
| 58 |
+
|
| 59 |
+
- `flash_decode_with_topk_idx`
|
| 60 |
+
- `flash_decode_with_gqa_share_sparse`
|
| 61 |
+
- `native_topk_from_scores`
|
| 62 |
+
- `native_nvfp4_dequant_swizzled_to_bf16`
|
| 63 |
+
- `has_native_ops`
|
| 64 |
+
- `naive_flash_decode_with_topk_idx`
|
| 65 |
+
- `naive_flash_decode_with_gqa_share_sparse`
|
| 66 |
+
- `get_cu_seqblocks`
|
| 67 |
+
- `robust_allocator`
|
| 68 |
+
|
| 69 |
+
## Decode Example
|
| 70 |
+
|
| 71 |
+
This example uses the official MiniMax decode-facing name
|
| 72 |
+
`sparse_decode_atten_func`.
|
| 73 |
+
|
| 74 |
+
```python
|
| 75 |
+
import torch
|
| 76 |
+
from kernels import get_kernel
|
| 77 |
+
|
| 78 |
+
msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
|
| 79 |
+
|
| 80 |
+
B, Hq, Hkv, D = 1, 64, 4, 128
|
| 81 |
+
page_size = 128
|
| 82 |
+
num_pages = 32
|
| 83 |
+
topk = 16
|
| 84 |
+
device = "cuda"
|
| 85 |
+
dtype = torch.bfloat16
|
| 86 |
+
|
| 87 |
+
q = torch.randn(B, Hq, D, device=device, dtype=dtype)
|
| 88 |
+
k = torch.randn(num_pages, Hkv, page_size, D, device=device, dtype=dtype)
|
| 89 |
+
v = torch.randn_like(k)
|
| 90 |
+
page_table = torch.arange(num_pages, device=device, dtype=torch.int32).view(B, -1)
|
| 91 |
+
seqused_k = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
|
| 92 |
+
q2k_indices = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
|
| 93 |
+
q2k_indices = q2k_indices.expand(Hkv, B, topk).contiguous()
|
| 94 |
+
|
| 95 |
+
out = msa.sparse_decode_atten_func(
|
| 96 |
+
q,
|
| 97 |
+
k,
|
| 98 |
+
v,
|
| 99 |
+
q2k_indices,
|
| 100 |
+
page_table=page_table,
|
| 101 |
+
seqused_k=seqused_k,
|
| 102 |
+
seqlen_q=1,
|
| 103 |
+
max_seqlen_k=num_pages * page_size,
|
| 104 |
+
blk_kv=page_size,
|
| 105 |
+
)
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
## Wrapper Example
|
| 109 |
+
|
| 110 |
+
```python
|
| 111 |
+
wrapper = msa.SparseDecodePagedAttentionWrapper(blk_kv=128)
|
| 112 |
+
wrapper.plan(
|
| 113 |
+
page_table=page_table,
|
| 114 |
+
seqused_k=seqused_k,
|
| 115 |
+
seqlen_q=1,
|
| 116 |
+
max_seqlen_k=num_pages * page_size,
|
| 117 |
+
q2k_indices=q2k_indices,
|
| 118 |
+
num_qo_heads=Hq,
|
| 119 |
+
num_kv_heads=Hkv,
|
| 120 |
+
head_dim=D,
|
| 121 |
+
)
|
| 122 |
+
out = wrapper.run(q, k, v)
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
+
## Prefill Example
|
| 126 |
+
|
| 127 |
+
This example uses the official MiniMax CSR prefill-facing name
|
| 128 |
+
`sparse_atten_func`.
|
| 129 |
+
|
| 130 |
+
```python
|
| 131 |
+
import torch
|
| 132 |
+
from kernels import get_kernel
|
| 133 |
+
|
| 134 |
+
msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True)
|
| 135 |
+
|
| 136 |
+
T, Hq, Hkv, D = 512, 64, 4, 128
|
| 137 |
+
page_size = 128
|
| 138 |
+
topk = 16
|
| 139 |
+
device = "cuda"
|
| 140 |
+
dtype = torch.bfloat16
|
| 141 |
+
|
| 142 |
+
q = torch.randn(T, Hq, D, device=device, dtype=dtype)
|
| 143 |
+
k = torch.randn(T, Hkv, D, device=device, dtype=dtype)
|
| 144 |
+
v = torch.randn_like(k)
|
| 145 |
+
cu = torch.tensor([0, T], device=device, dtype=torch.int32)
|
| 146 |
+
|
| 147 |
+
q2k = torch.full((Hkv, T, topk), -1, device=device, dtype=torch.int32)
|
| 148 |
+
for qi in range(T):
|
| 149 |
+
blocks = torch.arange(qi // page_size + 1, device=device, dtype=torch.int32)
|
| 150 |
+
q2k[:, qi, : min(topk, blocks.numel())] = blocks[:topk]
|
| 151 |
+
|
| 152 |
+
k2q_row_ptr, k2q_q_indices = msa.build_k2q_csr(
|
| 153 |
+
q2k, cu, cu, page_size, total_k=T
|
| 154 |
+
)
|
| 155 |
+
out = msa.sparse_atten_func(
|
| 156 |
+
q,
|
| 157 |
+
k,
|
| 158 |
+
v,
|
| 159 |
+
k2q_row_ptr,
|
| 160 |
+
k2q_q_indices,
|
| 161 |
+
topk,
|
| 162 |
+
cu_seqlens_q=cu,
|
| 163 |
+
cu_seqlens_k=cu,
|
| 164 |
+
max_seqlen_q=T,
|
| 165 |
+
max_seqlen_k=T,
|
| 166 |
+
blk_kv=page_size,
|
| 167 |
+
)
|
| 168 |
+
```
|
| 169 |
+
|
| 170 |
+
## Direct FlashRT Decode Path
|
| 171 |
+
|
| 172 |
+
Use this lower-level path if you already have `topk_idx` in FlashRT's paged KV
|
| 173 |
+
layout.
|
| 174 |
+
|
| 175 |
+
```python
|
| 176 |
+
q = torch.randn(B, Hq, D, device=device, dtype=dtype)
|
| 177 |
+
k_cache = torch.randn(num_pages * page_size, Hkv, D, device=device, dtype=dtype)
|
| 178 |
+
v_cache = torch.randn_like(k_cache)
|
| 179 |
+
req_to_token = torch.arange(num_pages * page_size, device=device, dtype=torch.int32).view(B, -1)
|
| 180 |
+
seq_lens = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32)
|
| 181 |
+
slot_ids = torch.zeros(B, device=device, dtype=torch.int64)
|
| 182 |
+
topk_idx = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk)
|
| 183 |
+
topk_idx = topk_idx.expand(Hkv, B, topk).contiguous()
|
| 184 |
+
|
| 185 |
+
out = msa.flash_decode_with_gqa_share_sparse(
|
| 186 |
+
q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, page_size, topk_idx
|
| 187 |
+
)
|
| 188 |
+
```
|
| 189 |
+
|
| 190 |
+
## Scope
|
| 191 |
+
|
| 192 |
+
- Target family: NVIDIA Blackwell CUDA compute capability 12.x.
|
| 193 |
+
- Builder target: CUDA 12.8+ with `cuda-capabilities = ["12.0", "12.1"]`.
|
| 194 |
+
- Validated hardware: DGX Spark / GB10 / SM121.
|
| 195 |
+
- Validated MiniMax shape: query heads `64`, KV heads `4`, head dim `128`,
|
| 196 |
+
sparse block/page size `128`, top-k blocks `16`.
|
| 197 |
+
- End-to-end model validation: MiniMax-Spark runtime on GB10 through `32768`
|
| 198 |
+
context length.
|
| 199 |
+
- Standalone kernel long-context validation: `128`, `2048`, `4096`, `32768`,
|
| 200 |
+
`65536`, `131072`.
|
| 201 |
+
- Correctness gate: cosine similarity `>= 0.999` against paged FP32 PyTorch
|
| 202 |
+
references; official decode wrapper output matches the direct Blackwell
|
| 203 |
+
decode kernel.
|
| 204 |
+
|
| 205 |
+
## Implementation Notes
|
| 206 |
+
|
| 207 |
+
This package contains:
|
| 208 |
+
|
| 209 |
+
- native CUDA score-to-top-k helper;
|
| 210 |
+
- native CUDA tensor-core sparse decode route for the MiniMax-M3 Blackwell shape;
|
| 211 |
+
- native CUDA FP4 block-score indexer;
|
| 212 |
+
- native CUDA swizzled NVFP4 -> BF16 dequantization for the W4A16 quality path;
|
| 213 |
+
- Blackwell-validated sparse prefill attention wrapper;
|
| 214 |
+
- MiniMaxAI/msa-compatible Python API layer for decode, prefill, CSR, NVFP4,
|
| 215 |
+
and FP4 block-score helpers.
|
| 216 |
+
|
| 217 |
+
When loaded from Hub built artifacts, the decode, FP4 indexer, and NVFP4
|
| 218 |
+
dequant hot paths use compiled CUDA ops. The source-tree mode keeps reference
|
| 219 |
+
paths so the API and correctness tests remain runnable before a wheel/shared
|
| 220 |
+
object has been built.
|
| 221 |
+
|
| 222 |
+
Source provenance and validation details are documented in `SYNC.md` and
|
| 223 |
+
`VALIDATION.md`.
|
VALIDATION.md
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Validation
|
| 2 |
+
|
| 3 |
+
## Target
|
| 4 |
+
|
| 5 |
+
- Kernel family: MiniMax M3 sparse attention (MSA)
|
| 6 |
+
- Package: `flashrt/MiniMaxAI-msa-blackwell`
|
| 7 |
+
- HF Jobs package selector: `MiniMaxAI-msa-blackwell`
|
| 8 |
+
- Package version: v1 Blackwell native-helper package
|
| 9 |
+
- Target GPU family: Blackwell CUDA compute capability 12.x
|
| 10 |
+
- Validated GPU: SM121 / GB10 / DGX Spark
|
| 11 |
+
- Dtype: BF16 inputs with FP32 accumulation references
|
| 12 |
+
- Layout: paged KV cache
|
| 13 |
+
- Model path: FlashRT MiniMax-Spark runtime on DGX Spark / GB10
|
| 14 |
+
|
| 15 |
+
## Correctness Gate
|
| 16 |
+
|
| 17 |
+
Run quick validation:
|
| 18 |
+
|
| 19 |
+
```bash
|
| 20 |
+
PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
|
| 21 |
+
python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py --quick
|
| 22 |
+
```
|
| 23 |
+
|
| 24 |
+
Run full validation:
|
| 25 |
+
|
| 26 |
+
```bash
|
| 27 |
+
PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
|
| 28 |
+
python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py
|
| 29 |
+
```
|
| 30 |
+
|
| 31 |
+
Run standalone long-context validation:
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
|
| 35 |
+
python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py --long-context
|
| 36 |
+
```
|
| 37 |
+
|
| 38 |
+
Expected full coverage:
|
| 39 |
+
|
| 40 |
+
| Area | Shapes | Reference | Required |
|
| 41 |
+
|---|---:|---|---|
|
| 42 |
+
| API surface | official `MiniMaxAI/msa` public names | `api_status.py` | all official root names exported; no unsupported public root API entries |
|
| 43 |
+
| Native CUDA top-k helper | heads 64, batch 1-2, blocks 1-256 | PyTorch top-k over valid blocks | exact set match |
|
| 44 |
+
| Decode sparse GQA attention | ctx 128, 2048, 4096, 32768 | paged FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 |
|
| 45 |
+
| Prefill sparse GQA attention | ctx 512, 4096 | paged causal FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 |
|
| 46 |
+
| Decode sparse GQA attention with sink | ctx 2048, 32768 | paged FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 |
|
| 47 |
+
| Official decode API wrapper | ctx 2048, 4096 | direct Blackwell decode kernel | cos = 1.0, max_abs = 0 |
|
| 48 |
+
| Official CSR prefill API wrapper | ctx 512, 2048 | direct Blackwell prefill kernel | cos = 1.0, max_abs = 0 under CSR-preserved block order |
|
| 49 |
+
| Official NVFP4 prefill API wrapper | ctx 512 BF16 dispatch path | `sparse_atten_func` | cos = 1.0, max_abs = 0 |
|
| 50 |
+
| Native CUDA NVFP4 dequant | rows/cols `(1,128)`, `(257,128)`, `(64,4096)` | Python NVFP4 reference | exact BF16 match |
|
| 51 |
+
| Official FP4 indexer API | tiny FP4 packed tensors; native artifact path when built | PyTorch block-score reference | returns official score layout |
|
| 52 |
+
| Decode lightning indexer | ctx 2048, 4096, 32768 | PyTorch blockmax top-k set | overlap >= 0.99 |
|
| 53 |
+
| Standalone long-context decode | ctx 65536, 131072 | paged FP32 PyTorch / direct kernel | cos >= 0.999; wrapper max_abs = 0 |
|
| 54 |
+
| Installed-artifact native long top-k | blocks 512, 1024 | PyTorch top-k over valid blocks | exact set match |
|
| 55 |
+
|
| 56 |
+
API surface validation:
|
| 57 |
+
|
| 58 |
+
```bash
|
| 59 |
+
PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
|
| 60 |
+
python -m pytest MiniMaxAI-msa-blackwell/tests/test_api_surface.py -q
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
The test tracks every official `MiniMaxAI/msa` public API name:
|
| 64 |
+
|
| 65 |
+
- `sparse_atten_func`
|
| 66 |
+
- `sparse_atten_nvfp4_kv_func`
|
| 67 |
+
- `sparse_decode_atten_func`
|
| 68 |
+
- `SparseDecodePagedAttentionWrapper`
|
| 69 |
+
- `fp4_indexer_block_scores`
|
| 70 |
+
- `build_k2q_csr`
|
| 71 |
+
- `SparseK2qCsrBuilderSm100`
|
| 72 |
+
- `Nvfp4QuantizedTensor`
|
| 73 |
+
- `quantize_bf16_to_nvfp4_128x4`
|
| 74 |
+
- `quantize_kv_bf16_to_nvfp4_128x4`
|
| 75 |
+
- `dequantize_nvfp4_128x4_to_bf16`
|
| 76 |
+
- `swizzle_nvfp4_scale_to_128x4`
|
| 77 |
+
- `nvfp4_global_scale_from_amax`
|
| 78 |
+
|
| 79 |
+
The root module exports every official public name. Decode, CSR prefill, NVFP4
|
| 80 |
+
prefill compatibility, FP4 block scoring, CSR, and NVFP4 helper names are all
|
| 81 |
+
callable. Hub built artifacts use compiled CUDA ops for the MiniMax-M3
|
| 82 |
+
Blackwell decode route, FP4 block-score indexer, and swizzled NVFP4 -> BF16
|
| 83 |
+
dequantization path. Source-tree mode keeps reference paths so the API remains
|
| 84 |
+
testable before the extension is built.
|
| 85 |
+
|
| 86 |
+
## FlashRT Integration Note
|
| 87 |
+
|
| 88 |
+
FlashRT has validated the decode sparse path on SM121 over context lengths
|
| 89 |
+
128 to 32768 with cosine similarity >= 0.999. The 32768 context length has
|
| 90 |
+
also been exercised in the FlashRT MiniMax-Spark model runtime on DGX Spark /
|
| 91 |
+
GB10, so it is the current end-to-end model validation boundary.
|
| 92 |
+
|
| 93 |
+
The standalone package kernel tests additionally cover 65536 and 131072
|
| 94 |
+
context lengths. These long-context rows validate the kernel and API wrapper
|
| 95 |
+
contract outside the full model runtime; they should not be described as
|
| 96 |
+
MiniMax-Spark end-to-end model validation until the full runtime path is rerun
|
| 97 |
+
at those lengths.
|
| 98 |
+
|
| 99 |
+
The same decode sparse path has also been exercised in FlashRT's MiniMax-Spark
|
| 100 |
+
model runtime on DGX Spark / GB10. That end-to-end validation is intentionally
|
| 101 |
+
kept as a FlashRT runtime validation item, while this Hub package exposes the
|
| 102 |
+
standalone kernel API for community use.
|
| 103 |
+
|
| 104 |
+
## Native Helper Compile Smoke
|
| 105 |
+
|
| 106 |
+
Before HF Jobs publish, the native helper was compiled locally as a PyTorch
|
| 107 |
+
extension using the same source files:
|
| 108 |
+
|
| 109 |
+
- `torch-ext/torch_binding.cpp`
|
| 110 |
+
- `csrc/msa_topk_from_scores.cu`
|
| 111 |
+
- `csrc/msa_decode_attn.cu`
|
| 112 |
+
- `csrc/msa_decode_attn_mma.cu`
|
| 113 |
+
- `csrc/msa_indexer_block_scores.cu`
|
| 114 |
+
- `csrc/msa_nvfp4_dequant.cu`
|
| 115 |
+
|
| 116 |
+
Environment:
|
| 117 |
+
|
| 118 |
+
| Field | Value |
|
| 119 |
+
|---|---|
|
| 120 |
+
| GPU | NVIDIA GeForce RTX 5090 |
|
| 121 |
+
| PyTorch | 2.9.1+cu128 |
|
| 122 |
+
| nvcc | CUDA 13.0 |
|
| 123 |
+
| Target arch | sm_120 |
|
| 124 |
+
|
| 125 |
+
Result:
|
| 126 |
+
|
| 127 |
+
| Check | Shape | Reference | Verdict |
|
| 128 |
+
|---|---:|---|---|
|
| 129 |
+
| Native score -> top-k | heads 64, batch 1, blocks 256, topk 16 | PyTorch top-k set | PASS |
|
| 130 |
+
| Native FP4 block-score indexer | official `[Hq, blocks, total_q]` score layout | PyTorch block-score reference | PASS |
|
| 131 |
+
| Native NVFP4 swizzled -> BF16 dequant | rows/cols `(1,128)`, `(257,128)`, `(64,4096)` | Python NVFP4 reference | PASS |
|
| 132 |
+
|
| 133 |
+
## Blackwell Package Validation
|
| 134 |
+
|
| 135 |
+
Remote Blackwell validation environment:
|
| 136 |
+
|
| 137 |
+
| Field | Value |
|
| 138 |
+
|---|---|
|
| 139 |
+
| Host | `spark-f517` |
|
| 140 |
+
| GPU | NVIDIA GB10 |
|
| 141 |
+
| Compute capability | 12.1 |
|
| 142 |
+
| Driver | 580.159.03 |
|
| 143 |
+
| Python | 3.12.3 |
|
| 144 |
+
| PyTorch | 2.12.0+cu130 |
|
| 145 |
+
| Triton | 3.7.0 |
|
| 146 |
+
|
| 147 |
+
Command:
|
| 148 |
+
|
| 149 |
+
```bash
|
| 150 |
+
PY=/home/leadtek/jax/bin/python
|
| 151 |
+
PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \
|
| 152 |
+
$PY MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
Result:
|
| 156 |
+
|
| 157 |
+
| Check | Shape | Cosine | Max abs / overlap | Verdict |
|
| 158 |
+
|---|---|---:|---:|---|
|
| 159 |
+
| Decode sparse GQA | ctx128_b1 | 0.999998 | 1.6032e-03 | PASS |
|
| 160 |
+
| Decode sparse GQA | ctx2048_b1 | 0.999996 | 4.9090e-04 | PASS |
|
| 161 |
+
| Decode sparse GQA | ctx2048_b2_sink | 0.999996 | 6.8302e-04 | PASS |
|
| 162 |
+
| Decode sparse GQA | ctx4096_b1 | 0.999996 | 4.5899e-04 | PASS |
|
| 163 |
+
| Decode sparse GQA | ctx4096_b2_mixed | 0.999996 | 7.3129e-04 | PASS |
|
| 164 |
+
| Decode sparse GQA | ctx32768_b1 | 0.999996 | 6.9451e-04 | PASS |
|
| 165 |
+
| Decode sparse GQA | ctx32768_b1_sink | 0.999996 | 5.6115e-04 | PASS |
|
| 166 |
+
| Decode sparse GQA | ctx65536_b1 | 0.999996 | 4.3470e-04 | PASS |
|
| 167 |
+
| Decode sparse GQA | ctx131072_b1 | 0.999996 | 7.1825e-04 | PASS |
|
| 168 |
+
| Decode top-k indexer | ctx2048 | n/a | overlap 1.000 | PASS |
|
| 169 |
+
| Decode top-k indexer | ctx4096 | n/a | overlap 1.000 | PASS |
|
| 170 |
+
| Decode top-k indexer | ctx32768 | n/a | overlap 1.000 | PASS |
|
| 171 |
+
| Decode top-k indexer | ctx65536 | n/a | overlap 1.000 | PASS |
|
| 172 |
+
| Decode top-k indexer | ctx131072 | n/a | overlap 1.000 | PASS |
|
| 173 |
+
| Official decode wrapper | ctx2048 | 1.000000 | 0.0000e+00 | PASS |
|
| 174 |
+
| Official decode wrapper | ctx4096 | 1.000000 | 0.0000e+00 | PASS |
|
| 175 |
+
| Official decode wrapper | ctx65536 | 1.000000 | 0.0000e+00 | PASS |
|
| 176 |
+
| Official decode wrapper | ctx131072 | 1.000000 | 0.0000e+00 | PASS |
|
| 177 |
+
| Native CUDA NVFP4 dequant | rows1_cols128 | 1.000000 | 0.0000e+00 | PASS |
|
| 178 |
+
| Native CUDA NVFP4 dequant | rows257_cols128 | 1.000000 | 0.0000e+00 | PASS |
|
| 179 |
+
| Native CUDA NVFP4 dequant | rows64_cols4096 | 1.000000 | 0.0000e+00 | PASS |
|
| 180 |
+
|
| 181 |
+
Installed-artifact native top-k validation on RTX 5090 / torch 2.11 / CUDA
|
| 182 |
+
12.8:
|
| 183 |
+
|
| 184 |
+
| Context | Blocks | Overlap | Verdict |
|
| 185 |
+
|---:|---:|---:|---|
|
| 186 |
+
| 32768 | 256 | 1.000 | PASS |
|
| 187 |
+
| 65536 | 512 | 1.000 | PASS |
|
| 188 |
+
| 131072 | 1024 | 1.000 | PASS |
|
| 189 |
+
|
| 190 |
+
The warning `tl.make_block_ptr is deprecated` appears with Triton 3.7.0. It is
|
| 191 |
+
a deprecation warning, not a correctness failure.
|
| 192 |
+
|
| 193 |
+
## Native Alignment Status
|
| 194 |
+
|
| 195 |
+
The upstream `MiniMaxAI/msa` package targets SM100. This Blackwell package
|
| 196 |
+
keeps the same public API surface where practical and provides native CUDA
|
| 197 |
+
implementations for the hot paths needed by the FlashRT MiniMax-Spark runtime:
|
| 198 |
+
|
| 199 |
+
- score-to-top-k sparse block selection;
|
| 200 |
+
- tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
|
| 201 |
+
- FP4 block-score indexing;
|
| 202 |
+
- swizzled NVFP4 -> BF16 dequantization for the W4A16 path.
|
| 203 |
+
|
| 204 |
+
The CSR prefill wrapper remains part of the public compatibility surface and is
|
| 205 |
+
validated against the package reference path. Shape and parameter restrictions
|
| 206 |
+
are explicit errors rather than silent wrong results.
|
benchmarks/benchmark_decode.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import argparse
|
| 2 |
+
import time
|
| 3 |
+
|
| 4 |
+
import torch
|
| 5 |
+
|
| 6 |
+
from minimaxai_msa_blackwell import (
|
| 7 |
+
flash_decode_with_gqa_share_sparse,
|
| 8 |
+
has_native_ops,
|
| 9 |
+
native_topk_from_scores,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def make_case(ctx: int, seed: int = 0):
|
| 14 |
+
g = torch.Generator(device="cuda").manual_seed(seed)
|
| 15 |
+
batch, hq, hkv, d = 1, 64, 4, 128
|
| 16 |
+
block, topk = 128, 16
|
| 17 |
+
q = torch.randn(batch, hq, d, device="cuda", dtype=torch.bfloat16, generator=g)
|
| 18 |
+
k_cache = torch.randn(ctx, hkv, d, device="cuda", dtype=torch.bfloat16, generator=g)
|
| 19 |
+
v_cache = torch.randn(ctx, hkv, d, device="cuda", dtype=torch.bfloat16, generator=g)
|
| 20 |
+
req_to_token = torch.arange(ctx, device="cuda", dtype=torch.int32).view(1, -1)
|
| 21 |
+
seq_lens = torch.tensor([ctx], device="cuda", dtype=torch.int32)
|
| 22 |
+
slot_ids = torch.zeros(batch, device="cuda", dtype=torch.int64)
|
| 23 |
+
nb = (ctx + block - 1) // block
|
| 24 |
+
n = min(topk, nb)
|
| 25 |
+
topk_idx = torch.full((hkv, batch, topk), -1, device="cuda", dtype=torch.int32)
|
| 26 |
+
topk_idx[:, :, :n] = torch.arange(n, device="cuda", dtype=torch.int32).view(1, 1, n)
|
| 27 |
+
return q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, block, topk_idx
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def bench(ctx: int, warmup: int, iters: int) -> float:
|
| 31 |
+
args = make_case(ctx)
|
| 32 |
+
for _ in range(warmup):
|
| 33 |
+
flash_decode_with_gqa_share_sparse(*args)
|
| 34 |
+
torch.cuda.synchronize()
|
| 35 |
+
start = time.perf_counter()
|
| 36 |
+
for _ in range(iters):
|
| 37 |
+
flash_decode_with_gqa_share_sparse(*args)
|
| 38 |
+
torch.cuda.synchronize()
|
| 39 |
+
return (time.perf_counter() - start) * 1e6 / iters
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def bench_native_topk(ctx: int, warmup: int, iters: int) -> float | None:
|
| 43 |
+
if not has_native_ops():
|
| 44 |
+
return None
|
| 45 |
+
heads, batch, block, topk = 64, 1, 128, 16
|
| 46 |
+
blocks = (ctx + block - 1) // block
|
| 47 |
+
score = torch.randn(heads, batch, blocks, device="cuda", dtype=torch.float32)
|
| 48 |
+
seq_lens = torch.tensor([ctx], device="cuda", dtype=torch.int32)
|
| 49 |
+
for _ in range(warmup):
|
| 50 |
+
native_topk_from_scores(score, seq_lens, block, topk)
|
| 51 |
+
torch.cuda.synchronize()
|
| 52 |
+
start = time.perf_counter()
|
| 53 |
+
for _ in range(iters):
|
| 54 |
+
native_topk_from_scores(score, seq_lens, block, topk)
|
| 55 |
+
torch.cuda.synchronize()
|
| 56 |
+
return (time.perf_counter() - start) * 1e6 / iters
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def main() -> None:
|
| 60 |
+
ap = argparse.ArgumentParser()
|
| 61 |
+
ap.add_argument("--ctx", type=int, nargs="+", default=[2048, 4096, 32768, 65536, 131072])
|
| 62 |
+
ap.add_argument("--warmup", type=int, default=10)
|
| 63 |
+
ap.add_argument("--iters", type=int, default=50)
|
| 64 |
+
args = ap.parse_args()
|
| 65 |
+
|
| 66 |
+
print("gpu:", torch.cuda.get_device_name())
|
| 67 |
+
print("ctx,attention_mean_us,native_topk_mean_us")
|
| 68 |
+
for ctx in args.ctx:
|
| 69 |
+
attn_us = bench(ctx, args.warmup, args.iters)
|
| 70 |
+
topk_us = bench_native_topk(ctx, args.warmup, args.iters)
|
| 71 |
+
topk_text = "NA" if topk_us is None else f"{topk_us:.3f}"
|
| 72 |
+
print(f"{ctx},{attn_us:.3f},{topk_text}")
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
if __name__ == "__main__":
|
| 76 |
+
main()
|
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention.
|
| 3 |
+
|
| 4 |
+
This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on
|
| 5 |
+
DGX Spark / GB10 / SM121. It exposes the
|
| 6 |
+
MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime.
|
| 7 |
+
|
| 8 |
+
Implementation status:
|
| 9 |
+
* native CUDA helper: score -> top-k sparse block ids;
|
| 10 |
+
* native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
|
| 11 |
+
* native CUDA FP4 block-score indexer;
|
| 12 |
+
* native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path;
|
| 13 |
+
* Blackwell-validated sparse prefill path;
|
| 14 |
+
* MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers,
|
| 15 |
+
and FP4 block-score helpers.
|
| 16 |
+
|
| 17 |
+
The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell
|
| 18 |
+
extension path. The public API is Tensor-oriented and independent from
|
| 19 |
+
FlashRT's serving runtime.
|
| 20 |
+
|
| 21 |
+
Public API
|
| 22 |
+
----------
|
| 23 |
+
Lightning indexer (q.k blockmax score -> top-k block select), paged KV:
|
| 24 |
+
flash_decode_with_topk_idx (single-token decode, split-K)
|
| 25 |
+
|
| 26 |
+
Block-sparse GQA attention (consumes block_indices from the indexer):
|
| 27 |
+
flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks)
|
| 28 |
+
sparse_atten_func (official CSR prefill wrapper)
|
| 29 |
+
sparse_decode_atten_func (official paged decode wrapper)
|
| 30 |
+
|
| 31 |
+
PyTorch naive references (for correctness checks):
|
| 32 |
+
naive_flash_decode_with_gqa_share_sparse (attention given topk_idx)
|
| 33 |
+
naive_flash_decode_with_topk_idx (joint indexer + attention)
|
| 34 |
+
|
| 35 |
+
All Triton entrypoints expect a paged KV cache:
|
| 36 |
+
k_cache / v_cache : [max_slots, num_kv_heads, head_dim]
|
| 37 |
+
req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot)
|
| 38 |
+
slot_ids : [batch] (per-request row in req_to_token)
|
| 39 |
+
topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed,
|
| 40 |
+
-1 right-padding (the M3 indexer contract).
|
| 41 |
+
See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 45 |
+
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 46 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 47 |
+
from .interface import (
|
| 48 |
+
SparseDecodePagedAttentionWrapper,
|
| 49 |
+
fp4_indexer_block_scores,
|
| 50 |
+
sparse_atten_func,
|
| 51 |
+
sparse_atten_nvfp4_kv_func,
|
| 52 |
+
sparse_decode_atten_func,
|
| 53 |
+
)
|
| 54 |
+
from ._native import (
|
| 55 |
+
has_native_ops,
|
| 56 |
+
native_nvfp4_dequant_swizzled_to_bf16,
|
| 57 |
+
native_topk_from_scores,
|
| 58 |
+
)
|
| 59 |
+
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 60 |
+
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 61 |
+
from .quantize import (
|
| 62 |
+
Nvfp4QuantizedTensor,
|
| 63 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 64 |
+
nvfp4_global_scale_from_amax,
|
| 65 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 66 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 67 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 68 |
+
)
|
| 69 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 70 |
+
from .api_status import (
|
| 71 |
+
OFFICIAL_API_STATUS,
|
| 72 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 73 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 74 |
+
available_functions,
|
| 75 |
+
official_api_status,
|
| 76 |
+
official_minimax_msa_functions,
|
| 77 |
+
unsupported_official_functions,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
__all__ = [
|
| 81 |
+
"sparse_atten_func",
|
| 82 |
+
"sparse_atten_nvfp4_kv_func",
|
| 83 |
+
"sparse_decode_atten_func",
|
| 84 |
+
"SparseDecodePagedAttentionWrapper",
|
| 85 |
+
"fp4_indexer_block_scores",
|
| 86 |
+
"build_k2q_csr",
|
| 87 |
+
"SparseK2qCsrBuilderSm100",
|
| 88 |
+
"Nvfp4QuantizedTensor",
|
| 89 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 90 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 91 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 92 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 93 |
+
"nvfp4_global_scale_from_amax",
|
| 94 |
+
"flash_decode_with_topk_idx",
|
| 95 |
+
"flash_decode_with_gqa_share_sparse",
|
| 96 |
+
"has_native_ops",
|
| 97 |
+
"native_topk_from_scores",
|
| 98 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 99 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 100 |
+
"naive_flash_decode_with_topk_idx",
|
| 101 |
+
"get_cu_seqblocks",
|
| 102 |
+
"robust_allocator",
|
| 103 |
+
"OFFICIAL_API_STATUS",
|
| 104 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 105 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 106 |
+
"available_functions",
|
| 107 |
+
"official_api_status",
|
| 108 |
+
"official_minimax_msa_functions",
|
| 109 |
+
"unsupported_official_functions",
|
| 110 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/_compat.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton
|
| 3 |
+
# kernels. The upstream SGLang modules pull two tiny things from the framework:
|
| 4 |
+
#
|
| 5 |
+
# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path)
|
| 6 |
+
# from sglang.srt.environ import envs -> a feature flag for an optional
|
| 7 |
+
# JIT-CUDA radix top-k fast path
|
| 8 |
+
#
|
| 9 |
+
# Both are replaced here so the kernels import with only torch + triton. The
|
| 10 |
+
# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel
|
| 11 |
+
# radix select), which is what we want on sm_121 (GB10).
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def is_hip() -> bool:
|
| 15 |
+
"""Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121).
|
| 16 |
+
|
| 17 |
+
Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV
|
| 18 |
+
widening branch (HIP-only). With this False, the CUDA dtype contract
|
| 19 |
+
(K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA.
|
| 20 |
+
"""
|
| 21 |
+
try:
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
return bool(getattr(torch.version, "hip", None))
|
| 25 |
+
except Exception:
|
| 26 |
+
return False
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class _Flag:
|
| 30 |
+
"""Minimal stand-in for an sglang.srt.environ boolean flag (always off)."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, default: bool = False):
|
| 33 |
+
self._default = default
|
| 34 |
+
|
| 35 |
+
def get(self) -> bool:
|
| 36 |
+
return self._default
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class _Envs:
|
| 40 |
+
"""Stub for `sglang.srt.environ.envs`.
|
| 41 |
+
|
| 42 |
+
Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode
|
| 43 |
+
indexer. Keeping it False routes top-k selection through the 2-stage Triton
|
| 44 |
+
fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the
|
| 45 |
+
decode indexer stays pure-Triton.
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
envs = _Envs()
|
build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:528883873873d9a482cbe93eb0ea0ec5be7f0eac009dd8b103a0609841c032d5
|
| 3 |
+
size 446928
|
build/torch211-cxx11-cu128-x86_64-linux/_native.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Optional native CUDA helpers for the FlashRT Blackwell MSA package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
try:
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
_HAS_NATIVE_OPS = True
|
| 12 |
+
except Exception:
|
| 13 |
+
add_op_namespace_prefix = None
|
| 14 |
+
ops = None
|
| 15 |
+
_HAS_NATIVE_OPS = False
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def has_native_ops() -> bool:
|
| 19 |
+
return _HAS_NATIVE_OPS
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
if _HAS_NATIVE_OPS:
|
| 23 |
+
|
| 24 |
+
@torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores"))
|
| 25 |
+
def _msa_topk_from_scores_fake(
|
| 26 |
+
score: torch.Tensor,
|
| 27 |
+
seq_lens: torch.Tensor,
|
| 28 |
+
block_size: int,
|
| 29 |
+
topk: int,
|
| 30 |
+
topk_idx: torch.Tensor,
|
| 31 |
+
) -> None:
|
| 32 |
+
if score.dim() != 3:
|
| 33 |
+
raise RuntimeError("score must have shape (heads, batch, max_blocks)")
|
| 34 |
+
heads, batch, _ = score.shape
|
| 35 |
+
if seq_lens.shape != (batch,):
|
| 36 |
+
raise RuntimeError("seq_lens must have shape (batch,)")
|
| 37 |
+
if topk_idx.shape != (heads, batch, topk):
|
| 38 |
+
raise RuntimeError("topk_idx must have shape (heads, batch, topk)")
|
| 39 |
+
return None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def native_topk_from_scores(
|
| 43 |
+
score: torch.Tensor,
|
| 44 |
+
seq_lens: torch.Tensor,
|
| 45 |
+
block_size: int,
|
| 46 |
+
topk: int,
|
| 47 |
+
) -> torch.Tensor:
|
| 48 |
+
"""Return top-k sparse block ids from a score tensor."""
|
| 49 |
+
|
| 50 |
+
if not _HAS_NATIVE_OPS:
|
| 51 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 52 |
+
if not score.is_contiguous():
|
| 53 |
+
score = score.contiguous()
|
| 54 |
+
if not seq_lens.is_contiguous():
|
| 55 |
+
seq_lens = seq_lens.contiguous()
|
| 56 |
+
out = torch.empty(
|
| 57 |
+
(score.shape[0], score.shape[1], int(topk)),
|
| 58 |
+
device=score.device,
|
| 59 |
+
dtype=torch.int32,
|
| 60 |
+
)
|
| 61 |
+
ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out)
|
| 62 |
+
return out
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
if _HAS_NATIVE_OPS:
|
| 66 |
+
|
| 67 |
+
@torch.library.register_fake(
|
| 68 |
+
add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged")
|
| 69 |
+
)
|
| 70 |
+
def _msa_decode_sparse_attn_mma_paged_fake(
|
| 71 |
+
q: torch.Tensor,
|
| 72 |
+
k_cache: torch.Tensor,
|
| 73 |
+
v_cache: torch.Tensor,
|
| 74 |
+
req_to_token: torch.Tensor,
|
| 75 |
+
seq_lens: torch.Tensor,
|
| 76 |
+
slot_ids: torch.Tensor,
|
| 77 |
+
topk_idx: torch.Tensor,
|
| 78 |
+
block_size: int,
|
| 79 |
+
sm_scale: float,
|
| 80 |
+
out: torch.Tensor,
|
| 81 |
+
) -> None:
|
| 82 |
+
return None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def native_decode_mma_supported(
|
| 86 |
+
q: torch.Tensor,
|
| 87 |
+
k_cache: torch.Tensor,
|
| 88 |
+
*,
|
| 89 |
+
block_size: int,
|
| 90 |
+
) -> bool:
|
| 91 |
+
"""Whether the tensor-core decode kernel applies to these inputs.
|
| 92 |
+
|
| 93 |
+
The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA
|
| 94 |
+
group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64.
|
| 95 |
+
"""
|
| 96 |
+
|
| 97 |
+
if not _HAS_NATIVE_OPS:
|
| 98 |
+
return False
|
| 99 |
+
if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16:
|
| 100 |
+
return False
|
| 101 |
+
if q.dim() != 3 or k_cache.dim() != 3:
|
| 102 |
+
return False
|
| 103 |
+
head_dim = int(q.shape[-1])
|
| 104 |
+
num_q_heads = int(q.shape[1])
|
| 105 |
+
num_kv_heads = int(k_cache.shape[1])
|
| 106 |
+
if head_dim != 128 or num_kv_heads <= 0:
|
| 107 |
+
return False
|
| 108 |
+
if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16:
|
| 109 |
+
return False
|
| 110 |
+
if int(block_size) % 64 != 0:
|
| 111 |
+
return False
|
| 112 |
+
return True
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def native_decode_sparse_attn_mma_paged(
|
| 116 |
+
q: torch.Tensor,
|
| 117 |
+
k_cache: torch.Tensor,
|
| 118 |
+
v_cache: torch.Tensor,
|
| 119 |
+
req_to_token: torch.Tensor,
|
| 120 |
+
seq_lens: torch.Tensor,
|
| 121 |
+
slot_ids: torch.Tensor,
|
| 122 |
+
block_size: int,
|
| 123 |
+
topk_idx: torch.Tensor,
|
| 124 |
+
sm_scale: float,
|
| 125 |
+
) -> torch.Tensor:
|
| 126 |
+
"""Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16."""
|
| 127 |
+
|
| 128 |
+
if not _HAS_NATIVE_OPS:
|
| 129 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 130 |
+
out = torch.empty(
|
| 131 |
+
(int(q.shape[0]), int(q.shape[1]), int(q.shape[2])),
|
| 132 |
+
device=q.device,
|
| 133 |
+
dtype=q.dtype,
|
| 134 |
+
)
|
| 135 |
+
ops.msa_decode_sparse_attn_mma_paged(
|
| 136 |
+
q.contiguous(),
|
| 137 |
+
k_cache.contiguous(),
|
| 138 |
+
v_cache.contiguous(),
|
| 139 |
+
req_to_token.contiguous(),
|
| 140 |
+
seq_lens.contiguous(),
|
| 141 |
+
slot_ids.contiguous(),
|
| 142 |
+
topk_idx.contiguous(),
|
| 143 |
+
int(block_size),
|
| 144 |
+
float(sm_scale),
|
| 145 |
+
out,
|
| 146 |
+
)
|
| 147 |
+
return out
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
if _HAS_NATIVE_OPS:
|
| 151 |
+
|
| 152 |
+
@torch.library.register_fake(
|
| 153 |
+
add_op_namespace_prefix("msa_indexer_block_scores")
|
| 154 |
+
)
|
| 155 |
+
def _msa_indexer_block_scores_fake(
|
| 156 |
+
q: torch.Tensor,
|
| 157 |
+
k_pages: torch.Tensor,
|
| 158 |
+
batch_of_q: torch.Tensor,
|
| 159 |
+
cu_q: torch.Tensor,
|
| 160 |
+
cu_k: torch.Tensor,
|
| 161 |
+
cu_pages: torch.Tensor,
|
| 162 |
+
kv_indices: torch.Tensor,
|
| 163 |
+
causal: int,
|
| 164 |
+
scores: torch.Tensor,
|
| 165 |
+
) -> None:
|
| 166 |
+
return None
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def native_indexer_block_scores(
|
| 170 |
+
q: torch.Tensor,
|
| 171 |
+
k_pages: torch.Tensor,
|
| 172 |
+
batch_of_q: torch.Tensor,
|
| 173 |
+
cu_q: torch.Tensor,
|
| 174 |
+
cu_k: torch.Tensor,
|
| 175 |
+
cu_pages: torch.Tensor,
|
| 176 |
+
kv_indices: torch.Tensor,
|
| 177 |
+
*,
|
| 178 |
+
max_blocks: int,
|
| 179 |
+
causal: bool,
|
| 180 |
+
) -> torch.Tensor:
|
| 181 |
+
"""Block-max QK indexer scores. q/k_pages are bf16 (already dequantized).
|
| 182 |
+
|
| 183 |
+
Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible).
|
| 184 |
+
"""
|
| 185 |
+
|
| 186 |
+
if not _HAS_NATIVE_OPS:
|
| 187 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 188 |
+
total_q = int(q.shape[0])
|
| 189 |
+
hq = int(q.shape[1])
|
| 190 |
+
scores = torch.full(
|
| 191 |
+
(hq, int(max_blocks), total_q),
|
| 192 |
+
float("-inf"),
|
| 193 |
+
dtype=torch.float32,
|
| 194 |
+
device=q.device,
|
| 195 |
+
)
|
| 196 |
+
ops.msa_indexer_block_scores(
|
| 197 |
+
q.contiguous(),
|
| 198 |
+
k_pages.contiguous(),
|
| 199 |
+
batch_of_q.contiguous(),
|
| 200 |
+
cu_q.contiguous(),
|
| 201 |
+
cu_k.contiguous(),
|
| 202 |
+
cu_pages.contiguous(),
|
| 203 |
+
kv_indices.contiguous(),
|
| 204 |
+
1 if causal else 0,
|
| 205 |
+
scores,
|
| 206 |
+
)
|
| 207 |
+
return scores
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
if _HAS_NATIVE_OPS:
|
| 211 |
+
|
| 212 |
+
@torch.library.register_fake(
|
| 213 |
+
add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16")
|
| 214 |
+
)
|
| 215 |
+
def _msa_nvfp4_dequant_swizzled_to_bf16_fake(
|
| 216 |
+
packed: torch.Tensor,
|
| 217 |
+
scale_128x4: torch.Tensor,
|
| 218 |
+
global_scale: float,
|
| 219 |
+
out: torch.Tensor,
|
| 220 |
+
) -> None:
|
| 221 |
+
return None
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def native_nvfp4_dequant_swizzled_to_bf16(
|
| 225 |
+
packed: torch.Tensor,
|
| 226 |
+
scale_128x4: torch.Tensor,
|
| 227 |
+
global_scale: torch.Tensor | float,
|
| 228 |
+
) -> torch.Tensor:
|
| 229 |
+
"""Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path."""
|
| 230 |
+
|
| 231 |
+
if not _HAS_NATIVE_OPS:
|
| 232 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 233 |
+
if packed.dtype is not torch.uint8:
|
| 234 |
+
raise TypeError("packed must be torch.uint8")
|
| 235 |
+
if scale_128x4.dtype is not torch.uint8:
|
| 236 |
+
raise TypeError("scale_128x4 must be torch.uint8")
|
| 237 |
+
original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
|
| 238 |
+
out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16)
|
| 239 |
+
if isinstance(global_scale, torch.Tensor):
|
| 240 |
+
global_scale_value = float(global_scale.reshape(-1)[0].item())
|
| 241 |
+
else:
|
| 242 |
+
global_scale_value = float(global_scale)
|
| 243 |
+
ops.msa_nvfp4_dequant_swizzled_to_bf16(
|
| 244 |
+
packed.contiguous(),
|
| 245 |
+
scale_128x4.contiguous(),
|
| 246 |
+
global_scale_value,
|
| 247 |
+
out,
|
| 248 |
+
)
|
| 249 |
+
return out
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
__all__ = [
|
| 253 |
+
"has_native_ops",
|
| 254 |
+
"native_topk_from_scores",
|
| 255 |
+
"native_decode_mma_supported",
|
| 256 |
+
"native_decode_sparse_attn_mma_paged",
|
| 257 |
+
"native_indexer_block_scores",
|
| 258 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 259 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _minimaxai_msa_blackwell_cuda_25b0067
|
| 3 |
+
ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/api_status.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Public API status for the FlashRT MiniMax MSA Blackwell package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from types import MappingProxyType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
V1_AVAILABLE_FUNCTIONS = (
|
| 10 |
+
"sparse_atten_func",
|
| 11 |
+
"sparse_atten_nvfp4_kv_func",
|
| 12 |
+
"sparse_decode_atten_func",
|
| 13 |
+
"SparseDecodePagedAttentionWrapper",
|
| 14 |
+
"fp4_indexer_block_scores",
|
| 15 |
+
"build_k2q_csr",
|
| 16 |
+
"SparseK2qCsrBuilderSm100",
|
| 17 |
+
"Nvfp4QuantizedTensor",
|
| 18 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 19 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 20 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 21 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 22 |
+
"nvfp4_global_scale_from_amax",
|
| 23 |
+
"has_native_ops",
|
| 24 |
+
"native_topk_from_scores",
|
| 25 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 26 |
+
"flash_decode_with_topk_idx",
|
| 27 |
+
"flash_decode_with_gqa_share_sparse",
|
| 28 |
+
"naive_flash_decode_with_topk_idx",
|
| 29 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 30 |
+
"get_cu_seqblocks",
|
| 31 |
+
"robust_allocator",
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
|
| 35 |
+
"sparse_atten_func",
|
| 36 |
+
"sparse_atten_nvfp4_kv_func",
|
| 37 |
+
"sparse_decode_atten_func",
|
| 38 |
+
"SparseDecodePagedAttentionWrapper",
|
| 39 |
+
"fp4_indexer_block_scores",
|
| 40 |
+
"build_k2q_csr",
|
| 41 |
+
"SparseK2qCsrBuilderSm100",
|
| 42 |
+
"Nvfp4QuantizedTensor",
|
| 43 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 44 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 45 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 46 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 47 |
+
"nvfp4_global_scale_from_amax",
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
_OFFICIAL_API_STATUS = {
|
| 51 |
+
"sparse_atten_func": {
|
| 52 |
+
"status": "available",
|
| 53 |
+
"target": "blackwell-prefill",
|
| 54 |
+
"reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.",
|
| 55 |
+
},
|
| 56 |
+
"sparse_atten_nvfp4_kv_func": {
|
| 57 |
+
"status": "available",
|
| 58 |
+
"target": "blackwell-prefill-nvfp4",
|
| 59 |
+
"reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.",
|
| 60 |
+
},
|
| 61 |
+
"sparse_decode_atten_func": {
|
| 62 |
+
"status": "available",
|
| 63 |
+
"target": "v1",
|
| 64 |
+
"reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
|
| 65 |
+
},
|
| 66 |
+
"SparseDecodePagedAttentionWrapper": {
|
| 67 |
+
"status": "available",
|
| 68 |
+
"target": "v1",
|
| 69 |
+
"reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
|
| 70 |
+
},
|
| 71 |
+
"fp4_indexer_block_scores": {
|
| 72 |
+
"status": "available",
|
| 73 |
+
"target": "blackwell-fp4-indexer",
|
| 74 |
+
"reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.",
|
| 75 |
+
},
|
| 76 |
+
"build_k2q_csr": {
|
| 77 |
+
"status": "available",
|
| 78 |
+
"target": "v1",
|
| 79 |
+
"reason": "CSR construction helper is available for the official prefill API.",
|
| 80 |
+
},
|
| 81 |
+
"SparseK2qCsrBuilderSm100": {
|
| 82 |
+
"status": "available",
|
| 83 |
+
"target": "v1",
|
| 84 |
+
"reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
|
| 85 |
+
},
|
| 86 |
+
"Nvfp4QuantizedTensor": {
|
| 87 |
+
"status": "available",
|
| 88 |
+
"target": "v1",
|
| 89 |
+
"reason": "NVFP4 metadata dataclass is available.",
|
| 90 |
+
},
|
| 91 |
+
"quantize_bf16_to_nvfp4_128x4": {
|
| 92 |
+
"status": "available_optional_te",
|
| 93 |
+
"target": "v1",
|
| 94 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 95 |
+
},
|
| 96 |
+
"quantize_kv_bf16_to_nvfp4_128x4": {
|
| 97 |
+
"status": "available_optional_te",
|
| 98 |
+
"target": "v1",
|
| 99 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 100 |
+
},
|
| 101 |
+
"dequantize_nvfp4_128x4_to_bf16": {
|
| 102 |
+
"status": "available",
|
| 103 |
+
"target": "v1",
|
| 104 |
+
"reason": "Reference dequant helper is available.",
|
| 105 |
+
},
|
| 106 |
+
"swizzle_nvfp4_scale_to_128x4": {
|
| 107 |
+
"status": "available",
|
| 108 |
+
"target": "v1",
|
| 109 |
+
"reason": "Scale swizzle helper is available.",
|
| 110 |
+
},
|
| 111 |
+
"nvfp4_global_scale_from_amax": {
|
| 112 |
+
"status": "available",
|
| 113 |
+
"target": "v1",
|
| 114 |
+
"reason": "Global scale helper is available.",
|
| 115 |
+
},
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def available_functions() -> tuple[str, ...]:
|
| 122 |
+
"""Return the functions/classes intentionally available in v1."""
|
| 123 |
+
|
| 124 |
+
return V1_AVAILABLE_FUNCTIONS
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def official_minimax_msa_functions() -> tuple[str, ...]:
|
| 128 |
+
"""Return the upstream MiniMaxAI/msa public function/class names tracked here."""
|
| 129 |
+
|
| 130 |
+
return OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def official_api_status() -> dict[str, dict[str, str]]:
|
| 134 |
+
"""Return a copy of the official API compatibility status table."""
|
| 135 |
+
|
| 136 |
+
return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def unsupported_official_functions() -> tuple[str, ...]:
|
| 140 |
+
"""Return official MiniMaxAI/msa names that are not exported by this v1 package."""
|
| 141 |
+
|
| 142 |
+
return tuple(
|
| 143 |
+
name
|
| 144 |
+
for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 145 |
+
if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
|
| 146 |
+
)
|
build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
build/torch211-cxx11-cu128-x86_64-linux/common/utils.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
import functools
|
| 4 |
+
from collections import deque
|
| 5 |
+
from typing import Any, Callable, Tuple
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
import triton.language as tl
|
| 9 |
+
|
| 10 |
+
_tma_keep_alive_buf = deque(maxlen=200)
|
| 11 |
+
|
| 12 |
+
try:
|
| 13 |
+
make_tensor_descriptor = tl.make_tensor_descriptor
|
| 14 |
+
except Exception:
|
| 15 |
+
make_tensor_descriptor = tl._experimental_make_tensor_descriptor
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def robust_allocator(size: int, alignment: int, stream: int = None):
|
| 19 |
+
"""Allocator for Triton TMA descriptors.
|
| 20 |
+
|
| 21 |
+
We keep reference in deque to prevent GC from collecting the buffer.
|
| 22 |
+
"""
|
| 23 |
+
tensor = torch.empty(size, device="cuda", dtype=torch.uint8)
|
| 24 |
+
_tma_keep_alive_buf.append(tensor)
|
| 25 |
+
return tensor
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def tensor_cache(maxsize: int = 8):
|
| 29 |
+
"""
|
| 30 |
+
Cache function results using identity comparison.
|
| 31 |
+
Zero-overhead cache hit: no hash, no DtoH, just pointer comparison.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
maxsize: Maximum number of cached entries. Supports multi-GPU scenarios
|
| 35 |
+
where different devices have different tensor arguments.
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 39 |
+
# LRU-style cache: list of (args, kwargs, result) tuples
|
| 40 |
+
# Most recently used at the end
|
| 41 |
+
_cache: list = []
|
| 42 |
+
|
| 43 |
+
def _args_match(args: tuple, cached_args: tuple) -> bool:
|
| 44 |
+
if len(args) != len(cached_args):
|
| 45 |
+
return False
|
| 46 |
+
for i in range(len(args)):
|
| 47 |
+
if args[i] is not cached_args[i]:
|
| 48 |
+
return False
|
| 49 |
+
return True
|
| 50 |
+
|
| 51 |
+
def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool:
|
| 52 |
+
if not kwargs and not cached_kwargs:
|
| 53 |
+
return True
|
| 54 |
+
if kwargs.keys() != cached_kwargs.keys():
|
| 55 |
+
return False
|
| 56 |
+
for k, v in kwargs.items():
|
| 57 |
+
if v is not cached_kwargs[k]:
|
| 58 |
+
return False
|
| 59 |
+
return True
|
| 60 |
+
|
| 61 |
+
@functools.wraps(fn)
|
| 62 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 63 |
+
# Search cache (most recent first for better hit rate)
|
| 64 |
+
for i in range(len(_cache) - 1, -1, -1):
|
| 65 |
+
cached_args, cached_kwargs, cached_result = _cache[i]
|
| 66 |
+
if _args_match(args, cached_args) and _kwargs_match(
|
| 67 |
+
kwargs, cached_kwargs
|
| 68 |
+
):
|
| 69 |
+
# Move to end (most recently used)
|
| 70 |
+
if i != len(_cache) - 1:
|
| 71 |
+
_cache.append(_cache.pop(i))
|
| 72 |
+
return cached_result
|
| 73 |
+
|
| 74 |
+
# Cache miss
|
| 75 |
+
result = fn(*args, **kwargs)
|
| 76 |
+
|
| 77 |
+
# Add to cache
|
| 78 |
+
if len(_cache) >= maxsize:
|
| 79 |
+
_cache.pop(0) # Remove oldest
|
| 80 |
+
_cache.append((args, kwargs, result))
|
| 81 |
+
return result
|
| 82 |
+
|
| 83 |
+
# Expose cache for manual clearing if needed
|
| 84 |
+
wrapper.cache_clear = lambda: _cache.clear()
|
| 85 |
+
wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize}
|
| 86 |
+
|
| 87 |
+
return wrapper
|
| 88 |
+
|
| 89 |
+
return decorator
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@tensor_cache(maxsize=8)
|
| 93 |
+
def get_cu_seqblocks(
|
| 94 |
+
cu_seqlens: torch.Tensor,
|
| 95 |
+
max_seqlen: int,
|
| 96 |
+
block_size_q: int,
|
| 97 |
+
block_size_k: int,
|
| 98 |
+
) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]:
|
| 99 |
+
"""Compute cumulative sequence block indices for blocked sparse attention.
|
| 100 |
+
|
| 101 |
+
Converts token-level cumulative sequence lengths to block-level indices,
|
| 102 |
+
which are needed for block-sparse attention kernels.
|
| 103 |
+
|
| 104 |
+
Note:
|
| 105 |
+
Results are cached (maxsize=8) based on input arguments. Repeated calls
|
| 106 |
+
with the same cu_seqlens, max_seqlen, and block sizes will return cached
|
| 107 |
+
results without recomputation.
|
| 108 |
+
|
| 109 |
+
Args:
|
| 110 |
+
cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32.
|
| 111 |
+
max_seqlen: Maximum sequence length in the batch.
|
| 112 |
+
block_size_q: Query block size.
|
| 113 |
+
block_size_k: Key-value block size.
|
| 114 |
+
|
| 115 |
+
Returns:
|
| 116 |
+
A tuple of 6 values:
|
| 117 |
+
- cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1]
|
| 118 |
+
- max_seqblock_q: Maximum number of query blocks per sequence.
|
| 119 |
+
- all_seqblock_q: Total number of query blocks across all sequences.
|
| 120 |
+
- cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1]
|
| 121 |
+
- max_seqblock_k: Maximum number of key blocks per sequence.
|
| 122 |
+
- all_seqblock_k: Total number of key blocks across all sequences.
|
| 123 |
+
"""
|
| 124 |
+
cu_seqblocks_q = torch.zeros_like(cu_seqlens)
|
| 125 |
+
cu_seqblocks_k = torch.zeros_like(cu_seqlens)
|
| 126 |
+
seq_lens = torch.diff(cu_seqlens)
|
| 127 |
+
seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q
|
| 128 |
+
seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k
|
| 129 |
+
max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q
|
| 130 |
+
max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k
|
| 131 |
+
cu_seqblocks_q[1:] = seqblocks_q
|
| 132 |
+
cu_seqblocks_k[1:] = seqblocks_k
|
| 133 |
+
cu_seqblocks_q.cumsum_(0)
|
| 134 |
+
cu_seqblocks_k.cumsum_(0)
|
| 135 |
+
all_seqblock_q = seqblocks_q.sum().item()
|
| 136 |
+
all_seqblock_k = seqblocks_k.sum().item()
|
| 137 |
+
return (
|
| 138 |
+
cu_seqblocks_q,
|
| 139 |
+
max_seqblock_q,
|
| 140 |
+
all_seqblock_q,
|
| 141 |
+
cu_seqblocks_k,
|
| 142 |
+
max_seqblock_k,
|
| 143 |
+
all_seqblock_k,
|
| 144 |
+
)
|
build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py
ADDED
|
@@ -0,0 +1,1150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .._compat import envs # vendored: was `from sglang.srt.environ import envs`
|
| 10 |
+
|
| 11 |
+
from ..common.utils import robust_allocator
|
| 12 |
+
from .._native import has_native_ops, native_topk_from_scores
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@triton.heuristics(
|
| 16 |
+
{
|
| 17 |
+
"BLOCK_SIZE_H": lambda args: max(
|
| 18 |
+
16, triton.next_power_of_2(args["gqa_group_size"])
|
| 19 |
+
),
|
| 20 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 21 |
+
"BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
|
| 22 |
+
}
|
| 23 |
+
)
|
| 24 |
+
@triton.autotune(
|
| 25 |
+
configs=[
|
| 26 |
+
triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
|
| 27 |
+
for BN in [64, 128, 256, 512]
|
| 28 |
+
for nw in [4, 8, 16]
|
| 29 |
+
for ns in [1, 2, 3]
|
| 30 |
+
],
|
| 31 |
+
key=[
|
| 32 |
+
"BATCH_SIZE_BUCKET",
|
| 33 |
+
"gqa_group_size",
|
| 34 |
+
"head_dim",
|
| 35 |
+
"block_size",
|
| 36 |
+
"SCORE_TYPE",
|
| 37 |
+
],
|
| 38 |
+
)
|
| 39 |
+
@triton.jit
|
| 40 |
+
def _decode_score_kernel(
|
| 41 |
+
q_ptr, # Q: b x qh x d
|
| 42 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 43 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 44 |
+
score_ptr, # Score: qh x b x max_seqblock
|
| 45 |
+
seq_lens,
|
| 46 |
+
slot_ids,
|
| 47 |
+
# shape
|
| 48 |
+
max_slots,
|
| 49 |
+
batch_size,
|
| 50 |
+
gqa_group_size,
|
| 51 |
+
head_dim,
|
| 52 |
+
# block size
|
| 53 |
+
block_size: tl.constexpr,
|
| 54 |
+
topk: tl.constexpr,
|
| 55 |
+
# sm_scale
|
| 56 |
+
sm_scale,
|
| 57 |
+
# init and local blocks
|
| 58 |
+
init_blocks,
|
| 59 |
+
local_blocks,
|
| 60 |
+
# stride
|
| 61 |
+
stride_q_b,
|
| 62 |
+
stride_q_h,
|
| 63 |
+
stride_q_d,
|
| 64 |
+
stride_k_s,
|
| 65 |
+
stride_k_h,
|
| 66 |
+
stride_k_d,
|
| 67 |
+
stride_r2t_b,
|
| 68 |
+
stride_s_h,
|
| 69 |
+
stride_s_b,
|
| 70 |
+
stride_s_n,
|
| 71 |
+
# META parameters
|
| 72 |
+
BATCH_SIZE_BUCKET: tl.constexpr,
|
| 73 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 74 |
+
BLOCK_SIZE_N: tl.constexpr,
|
| 75 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 76 |
+
NUM_KV_CHUNKS: tl.constexpr,
|
| 77 |
+
SCORE_TYPE: tl.constexpr,
|
| 78 |
+
SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
|
| 79 |
+
):
|
| 80 |
+
tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
|
| 81 |
+
sm_scale_log2e = sm_scale * 1.4426950409
|
| 82 |
+
tl.static_assert(BLOCK_SIZE_N >= block_size)
|
| 83 |
+
BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
|
| 84 |
+
# get batch id and head id
|
| 85 |
+
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
| 86 |
+
pid_b = pid_bc % batch_size
|
| 87 |
+
pid_c = pid_bc // batch_size
|
| 88 |
+
pid_h = pid_kh * gqa_group_size
|
| 89 |
+
# block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
|
| 90 |
+
seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
|
| 91 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 92 |
+
if SKIP_TRIVIAL_TOPK_SCORE:
|
| 93 |
+
if num_blocks <= topk:
|
| 94 |
+
return
|
| 95 |
+
chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
|
| 96 |
+
chunk_start_block = pid_c * chunk_size_blocks
|
| 97 |
+
chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
|
| 98 |
+
chunk_start = chunk_start_block * block_size
|
| 99 |
+
chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
|
| 100 |
+
if chunk_start_block >= chunk_end_block:
|
| 101 |
+
return
|
| 102 |
+
sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots
|
| 103 |
+
# init qkv pointer
|
| 104 |
+
q_ptrs = tl.make_block_ptr(
|
| 105 |
+
base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
|
| 106 |
+
shape=(gqa_group_size, head_dim),
|
| 107 |
+
strides=(stride_q_h, stride_q_d),
|
| 108 |
+
offsets=(0, 0),
|
| 109 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 110 |
+
order=(1, 0),
|
| 111 |
+
)
|
| 112 |
+
s_ptrs = tl.make_block_ptr(
|
| 113 |
+
base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
|
| 114 |
+
shape=(gqa_group_size, chunk_end_block),
|
| 115 |
+
strides=(stride_s_h, stride_s_n),
|
| 116 |
+
offsets=(0, chunk_start_block),
|
| 117 |
+
block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
|
| 118 |
+
order=(1, 0),
|
| 119 |
+
)
|
| 120 |
+
# load q
|
| 121 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 122 |
+
# init statistics
|
| 123 |
+
off_n = tl.arange(0, BLOCK_SIZE_N)
|
| 124 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 125 |
+
off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
|
| 126 |
+
dim_mask = off_d < head_dim
|
| 127 |
+
# pre-compute local_start outside the loop (constant for entire batch)
|
| 128 |
+
local_start = tl.maximum(0, num_blocks - local_blocks)
|
| 129 |
+
# prefetch first iteration's slots
|
| 130 |
+
r2t_base = req_to_token_ptr + sid * stride_r2t_b
|
| 131 |
+
prefetch_pos = chunk_start + off_n
|
| 132 |
+
prefetch_mask = prefetch_pos < seq_len
|
| 133 |
+
prefetched_slots = tl.load(
|
| 134 |
+
r2t_base + prefetch_pos,
|
| 135 |
+
mask=prefetch_mask,
|
| 136 |
+
other=0,
|
| 137 |
+
).to(tl.int64)
|
| 138 |
+
# score-only: compute block scores without loading V
|
| 139 |
+
for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
|
| 140 |
+
pos_mask = prefetch_mask
|
| 141 |
+
slots = prefetched_slots
|
| 142 |
+
# prefetch next iteration's slots
|
| 143 |
+
next_i = i + BLOCK_SIZE_N
|
| 144 |
+
if next_i < chunk_end:
|
| 145 |
+
next_pos = next_i + off_n
|
| 146 |
+
prefetch_mask = next_pos < seq_len
|
| 147 |
+
prefetched_slots = tl.load(
|
| 148 |
+
r2t_base + next_pos,
|
| 149 |
+
mask=prefetch_mask,
|
| 150 |
+
other=0,
|
| 151 |
+
).to(tl.int64)
|
| 152 |
+
slots = (slots + max_slots) % max_slots
|
| 153 |
+
# load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
|
| 154 |
+
k_off = (
|
| 155 |
+
slots[None, :] * stride_k_s
|
| 156 |
+
+ pid_kh * stride_k_h
|
| 157 |
+
+ off_d[:, None] * stride_k_d
|
| 158 |
+
)
|
| 159 |
+
k = tl.load(
|
| 160 |
+
k_cache_ptr + k_off,
|
| 161 |
+
mask=dim_mask[:, None] & pos_mask[None, :],
|
| 162 |
+
other=0.0,
|
| 163 |
+
)
|
| 164 |
+
# compute qk
|
| 165 |
+
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
|
| 166 |
+
qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
|
| 167 |
+
# [H, D], [D, N] -> [H, N]
|
| 168 |
+
qk += tl.dot(q, k) * sm_scale_log2e
|
| 169 |
+
# save qk to score
|
| 170 |
+
score = tl.reshape(
|
| 171 |
+
qk,
|
| 172 |
+
(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
|
| 173 |
+
can_reorder=False,
|
| 174 |
+
)
|
| 175 |
+
sub_max = tl.max(score, axis=2)
|
| 176 |
+
if SCORE_TYPE == "max":
|
| 177 |
+
score = sub_max
|
| 178 |
+
else: # "lse"
|
| 179 |
+
score = sub_max + tl.log2(
|
| 180 |
+
tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
|
| 181 |
+
)
|
| 182 |
+
score = tl.where(score != score, float("-inf"), score)
|
| 183 |
+
# apply init_blocks and local_blocks efficiently
|
| 184 |
+
curr_block_idx = i // block_size + off_bpk
|
| 185 |
+
is_init = curr_block_idx < init_blocks
|
| 186 |
+
is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
|
| 187 |
+
score = tl.where(
|
| 188 |
+
is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
|
| 189 |
+
)
|
| 190 |
+
tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1))
|
| 191 |
+
# update ptrs
|
| 192 |
+
s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
@triton.heuristics(
|
| 196 |
+
{
|
| 197 |
+
"BLOCK_SIZE_H": lambda args: max(
|
| 198 |
+
16, triton.next_power_of_2(args["gqa_group_size"])
|
| 199 |
+
),
|
| 200 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 201 |
+
"HAS_SINK": lambda args: args["sink_ptr"] is not None,
|
| 202 |
+
}
|
| 203 |
+
)
|
| 204 |
+
@triton.autotune(
|
| 205 |
+
configs=[
|
| 206 |
+
triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
|
| 207 |
+
for BN in [64, 128, 256, 512]
|
| 208 |
+
for nw in [4, 8, 16]
|
| 209 |
+
for ns in [1, 2, 3]
|
| 210 |
+
],
|
| 211 |
+
key=[
|
| 212 |
+
"gqa_group_size",
|
| 213 |
+
"head_dim",
|
| 214 |
+
"block_size",
|
| 215 |
+
"HAS_SINK",
|
| 216 |
+
"SCORE_TYPE",
|
| 217 |
+
],
|
| 218 |
+
)
|
| 219 |
+
@triton.jit
|
| 220 |
+
def _decode_score_attn_kernel(
|
| 221 |
+
q_ptr, # Q: b x qh x d
|
| 222 |
+
sink_ptr, # Sink: qh x d
|
| 223 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 224 |
+
v_cache_ptr, # V paged: max_slots x kh x d
|
| 225 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 226 |
+
o_ptr, # O: c x b x qh x d
|
| 227 |
+
lse_ptr, # lse: c x b x qh
|
| 228 |
+
score_ptr, # Score: qh x b x max_seqblock
|
| 229 |
+
seq_lens,
|
| 230 |
+
slot_ids,
|
| 231 |
+
# shape
|
| 232 |
+
max_slots,
|
| 233 |
+
batch_size,
|
| 234 |
+
gqa_group_size,
|
| 235 |
+
head_dim,
|
| 236 |
+
# block size
|
| 237 |
+
block_size: tl.constexpr,
|
| 238 |
+
topk: tl.constexpr,
|
| 239 |
+
# sm_scale
|
| 240 |
+
sm_scale,
|
| 241 |
+
# init and local blocks
|
| 242 |
+
init_blocks,
|
| 243 |
+
local_blocks,
|
| 244 |
+
# stride
|
| 245 |
+
stride_q_b,
|
| 246 |
+
stride_q_h,
|
| 247 |
+
stride_q_d,
|
| 248 |
+
stride_sink_h,
|
| 249 |
+
stride_sink_d,
|
| 250 |
+
stride_k_s,
|
| 251 |
+
stride_k_h,
|
| 252 |
+
stride_k_d,
|
| 253 |
+
stride_v_s,
|
| 254 |
+
stride_v_h,
|
| 255 |
+
stride_v_d,
|
| 256 |
+
stride_r2t_b,
|
| 257 |
+
stride_o_c,
|
| 258 |
+
stride_o_b,
|
| 259 |
+
stride_o_h,
|
| 260 |
+
stride_o_d,
|
| 261 |
+
stride_l_c,
|
| 262 |
+
stride_l_b,
|
| 263 |
+
stride_l_h,
|
| 264 |
+
stride_s_h,
|
| 265 |
+
stride_s_b,
|
| 266 |
+
stride_s_n,
|
| 267 |
+
# META parameters
|
| 268 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 269 |
+
BLOCK_SIZE_N: tl.constexpr,
|
| 270 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 271 |
+
NUM_KV_CHUNKS: tl.constexpr,
|
| 272 |
+
HAS_SINK: tl.constexpr,
|
| 273 |
+
SCORE_TYPE: tl.constexpr,
|
| 274 |
+
SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
|
| 275 |
+
):
|
| 276 |
+
tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
|
| 277 |
+
sm_scale_log2e = sm_scale * 1.4426950409
|
| 278 |
+
tl.static_assert(BLOCK_SIZE_N >= block_size)
|
| 279 |
+
BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
|
| 280 |
+
# get batch id and head id
|
| 281 |
+
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
| 282 |
+
pid_b = pid_bc % batch_size
|
| 283 |
+
pid_c = pid_bc // batch_size
|
| 284 |
+
pid_h = pid_kh * gqa_group_size
|
| 285 |
+
# block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
|
| 286 |
+
seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
|
| 287 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 288 |
+
chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
|
| 289 |
+
chunk_start_block = pid_c * chunk_size_blocks
|
| 290 |
+
chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
|
| 291 |
+
chunk_start = chunk_start_block * block_size
|
| 292 |
+
chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
|
| 293 |
+
if chunk_start_block >= chunk_end_block:
|
| 294 |
+
return
|
| 295 |
+
sid = (
|
| 296 |
+
tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
|
| 297 |
+
) % max_slots # to avoid bugs when slot_ids is negative
|
| 298 |
+
# init qkv pointer
|
| 299 |
+
q_ptrs = tl.make_block_ptr(
|
| 300 |
+
base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
|
| 301 |
+
shape=(gqa_group_size, head_dim),
|
| 302 |
+
strides=(stride_q_h, stride_q_d),
|
| 303 |
+
offsets=(0, 0),
|
| 304 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 305 |
+
order=(1, 0),
|
| 306 |
+
)
|
| 307 |
+
s_ptrs = tl.make_block_ptr(
|
| 308 |
+
base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
|
| 309 |
+
shape=(gqa_group_size, chunk_end_block),
|
| 310 |
+
strides=(stride_s_h, stride_s_n),
|
| 311 |
+
offsets=(0, chunk_start_block),
|
| 312 |
+
block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
|
| 313 |
+
order=(1, 0),
|
| 314 |
+
)
|
| 315 |
+
# load q
|
| 316 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 317 |
+
# init statistics
|
| 318 |
+
off_n = tl.arange(0, BLOCK_SIZE_N)
|
| 319 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 320 |
+
off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
|
| 321 |
+
dim_mask = off_d < head_dim
|
| 322 |
+
if HAS_SINK:
|
| 323 |
+
if pid_c == 0:
|
| 324 |
+
sink_ptrs = tl.make_block_ptr(
|
| 325 |
+
base=sink_ptr + pid_h * stride_sink_h,
|
| 326 |
+
shape=(gqa_group_size, head_dim),
|
| 327 |
+
strides=(stride_sink_h, stride_sink_d),
|
| 328 |
+
offsets=(0, 0),
|
| 329 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 330 |
+
order=(1, 0),
|
| 331 |
+
)
|
| 332 |
+
sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
|
| 333 |
+
tl.float32
|
| 334 |
+
)
|
| 335 |
+
qsink = (
|
| 336 |
+
tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e
|
| 337 |
+
) # (BLOCK_SIZE_H,)
|
| 338 |
+
m_i = qsink
|
| 339 |
+
l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32)
|
| 340 |
+
else:
|
| 341 |
+
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 342 |
+
l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
|
| 343 |
+
else:
|
| 344 |
+
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 345 |
+
l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
|
| 346 |
+
acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
|
| 347 |
+
# pre-compute local_start outside the loop (constant for entire batch)
|
| 348 |
+
local_start = tl.maximum(0, num_blocks - local_blocks)
|
| 349 |
+
# full attention or causal attention
|
| 350 |
+
for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
|
| 351 |
+
pos = i + off_n
|
| 352 |
+
pos_mask = pos < seq_len
|
| 353 |
+
# resolve slot per position via req_to_token
|
| 354 |
+
slots = tl.load(
|
| 355 |
+
req_to_token_ptr + sid * stride_r2t_b + pos,
|
| 356 |
+
mask=pos_mask,
|
| 357 |
+
other=0,
|
| 358 |
+
).to(tl.int64)
|
| 359 |
+
slots = (slots + max_slots) % max_slots # safety against negative
|
| 360 |
+
# load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
|
| 361 |
+
k_off = (
|
| 362 |
+
slots[None, :] * stride_k_s
|
| 363 |
+
+ pid_kh * stride_k_h
|
| 364 |
+
+ off_d[:, None] * stride_k_d
|
| 365 |
+
)
|
| 366 |
+
k = tl.load(
|
| 367 |
+
k_cache_ptr + k_off,
|
| 368 |
+
mask=dim_mask[:, None] & pos_mask[None, :],
|
| 369 |
+
other=0.0,
|
| 370 |
+
)
|
| 371 |
+
# load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
|
| 372 |
+
v_off = (
|
| 373 |
+
slots[:, None] * stride_v_s
|
| 374 |
+
+ pid_kh * stride_v_h
|
| 375 |
+
+ off_d[None, :] * stride_v_d
|
| 376 |
+
)
|
| 377 |
+
v = tl.load(
|
| 378 |
+
v_cache_ptr + v_off,
|
| 379 |
+
mask=pos_mask[:, None] & dim_mask[None, :],
|
| 380 |
+
other=0.0,
|
| 381 |
+
)
|
| 382 |
+
# compute qk
|
| 383 |
+
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
|
| 384 |
+
qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
|
| 385 |
+
# [H, D], [D, N] -> [H, N]
|
| 386 |
+
qk += tl.dot(q, k) * sm_scale_log2e
|
| 387 |
+
# save qk to score
|
| 388 |
+
score = tl.reshape(
|
| 389 |
+
qk,
|
| 390 |
+
(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
|
| 391 |
+
can_reorder=False,
|
| 392 |
+
)
|
| 393 |
+
sub_max = tl.max(score, axis=2)
|
| 394 |
+
if SCORE_TYPE == "max":
|
| 395 |
+
score = sub_max
|
| 396 |
+
else: # "lse"
|
| 397 |
+
score = sub_max + tl.log2(
|
| 398 |
+
tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
|
| 399 |
+
)
|
| 400 |
+
score = tl.where(score != score, float("-inf"), score)
|
| 401 |
+
# apply init_blocks and local_blocks efficiently
|
| 402 |
+
# current block indices: [BLOCKS_PER_K_BLOCK]
|
| 403 |
+
curr_block_idx = i // block_size + off_bpk
|
| 404 |
+
# Combined condition with single nested tl.where:
|
| 405 |
+
# - local_blocks (1e29) takes priority over init_blocks (1e30)
|
| 406 |
+
# - When init_blocks=0: curr_block_idx < 0 is always False
|
| 407 |
+
# - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False
|
| 408 |
+
is_init = curr_block_idx < init_blocks
|
| 409 |
+
is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
|
| 410 |
+
# Apply in one fused operation: local > init > original
|
| 411 |
+
score = tl.where(
|
| 412 |
+
is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
|
| 413 |
+
)
|
| 414 |
+
if SKIP_TRIVIAL_TOPK_SCORE:
|
| 415 |
+
if num_blocks > topk:
|
| 416 |
+
tl.store(
|
| 417 |
+
s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
|
| 418 |
+
)
|
| 419 |
+
else:
|
| 420 |
+
tl.store(
|
| 421 |
+
s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
|
| 422 |
+
)
|
| 423 |
+
# max-of-max == max(qk), avoids re-scanning qk
|
| 424 |
+
m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1))
|
| 425 |
+
p = tl.exp2(qk - m_ij[:, None])
|
| 426 |
+
l_ij = tl.sum(p, axis=1)
|
| 427 |
+
acc_o_scale = tl.exp2(m_i - m_ij)
|
| 428 |
+
acc_o = acc_o * acc_o_scale[:, None]
|
| 429 |
+
# [H, N], [N, D] -> [H, D]
|
| 430 |
+
acc_o += tl.dot(p.to(v.dtype), v)
|
| 431 |
+
m_i = m_ij
|
| 432 |
+
l_i = l_i * acc_o_scale + l_ij
|
| 433 |
+
# update ptrs
|
| 434 |
+
s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
|
| 435 |
+
# final scale
|
| 436 |
+
acc_o = acc_o / l_i[:, None]
|
| 437 |
+
# compute lse in log2 scale for merge kernel
|
| 438 |
+
lse_i = m_i + tl.log2(l_i)
|
| 439 |
+
# save output
|
| 440 |
+
o_ptrs = tl.make_block_ptr(
|
| 441 |
+
base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 442 |
+
shape=(gqa_group_size, head_dim),
|
| 443 |
+
strides=(stride_o_h, stride_o_d),
|
| 444 |
+
offsets=(0, 0),
|
| 445 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 446 |
+
order=(1, 0),
|
| 447 |
+
)
|
| 448 |
+
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
|
| 449 |
+
# save lse (in log2 scale, will be used directly in merge kernel)
|
| 450 |
+
lse_ptrs = tl.make_block_ptr(
|
| 451 |
+
base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
|
| 452 |
+
shape=(gqa_group_size,),
|
| 453 |
+
strides=(stride_l_h,),
|
| 454 |
+
offsets=(0,),
|
| 455 |
+
block_shape=(BLOCK_SIZE_H,),
|
| 456 |
+
order=(0,),
|
| 457 |
+
)
|
| 458 |
+
tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
@triton.heuristics(
|
| 462 |
+
{
|
| 463 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 464 |
+
}
|
| 465 |
+
)
|
| 466 |
+
@triton.autotune(
|
| 467 |
+
configs=[
|
| 468 |
+
triton.Config({}, num_warps=nw, num_stages=ns)
|
| 469 |
+
for nw in [4, 8]
|
| 470 |
+
for ns in [2, 3, 4]
|
| 471 |
+
],
|
| 472 |
+
key=["BLOCK_SIZE_D"],
|
| 473 |
+
restore_value=["o_ptr"],
|
| 474 |
+
)
|
| 475 |
+
@triton.jit
|
| 476 |
+
def _merge_attn_out_kernel(
|
| 477 |
+
o_ptr,
|
| 478 |
+
lse_ptr,
|
| 479 |
+
# shape
|
| 480 |
+
seq_lens,
|
| 481 |
+
head_dim,
|
| 482 |
+
block_size: tl.constexpr,
|
| 483 |
+
# stride
|
| 484 |
+
stride_o_c,
|
| 485 |
+
stride_o_b,
|
| 486 |
+
stride_o_h,
|
| 487 |
+
stride_o_d,
|
| 488 |
+
stride_l_c,
|
| 489 |
+
stride_l_b,
|
| 490 |
+
stride_l_h,
|
| 491 |
+
NUM_KV_CHUNKS: tl.constexpr,
|
| 492 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 493 |
+
):
|
| 494 |
+
pid_b, pid_h = tl.program_id(0), tl.program_id(1)
|
| 495 |
+
# recompute chunk layout (must match _decode_score_attn_kernel)
|
| 496 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 497 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 498 |
+
chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
|
| 499 |
+
valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks)
|
| 500 |
+
# ptrs
|
| 501 |
+
off_c = tl.arange(0, NUM_KV_CHUNKS)
|
| 502 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 503 |
+
o_ptrs = tl.make_block_ptr(
|
| 504 |
+
base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 505 |
+
shape=(valid_chunks, head_dim),
|
| 506 |
+
strides=(stride_o_c, stride_o_d),
|
| 507 |
+
offsets=(0, 0),
|
| 508 |
+
block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D),
|
| 509 |
+
order=(1, 0),
|
| 510 |
+
)
|
| 511 |
+
lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
|
| 512 |
+
# load o and lse
|
| 513 |
+
o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 514 |
+
lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf"))
|
| 515 |
+
# merge o (lse is in log2 scale, use exp2/log2 for efficiency)
|
| 516 |
+
lse_shifted = lse - tl.max(lse, axis=0)
|
| 517 |
+
log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0))
|
| 518 |
+
scale = tl.exp2(lse_shifted - log_sum)
|
| 519 |
+
o = tl.sum(o * scale[:, None], axis=0)
|
| 520 |
+
# save o
|
| 521 |
+
o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
|
| 522 |
+
tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
@triton.jit
|
| 526 |
+
def _compare_and_swap(
|
| 527 |
+
x,
|
| 528 |
+
ids,
|
| 529 |
+
flip,
|
| 530 |
+
i: tl.constexpr,
|
| 531 |
+
n_dims: tl.constexpr,
|
| 532 |
+
):
|
| 533 |
+
n_outer: tl.constexpr = x.numel >> n_dims
|
| 534 |
+
shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
|
| 535 |
+
y = tl.reshape(x, shape)
|
| 536 |
+
# slice left/right with 'stride' 2**(n_dims - i - 1)
|
| 537 |
+
mask = tl.arange(0, 2)[None, :, None]
|
| 538 |
+
left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
|
| 539 |
+
right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
|
| 540 |
+
left = tl.reshape(left, x.shape)
|
| 541 |
+
right = tl.reshape(right, x.shape)
|
| 542 |
+
# idx
|
| 543 |
+
y_idx = tl.reshape(ids, shape)
|
| 544 |
+
left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
|
| 545 |
+
right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
|
| 546 |
+
left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
|
| 547 |
+
right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
|
| 548 |
+
# actual compare-and-swap
|
| 549 |
+
idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
|
| 550 |
+
ileft = left.to(idtype, bitcast=True)
|
| 551 |
+
iright = right.to(idtype, bitcast=True)
|
| 552 |
+
ix = x.to(idtype, bitcast=True)
|
| 553 |
+
|
| 554 |
+
cond = (left > right) != flip
|
| 555 |
+
ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
|
| 556 |
+
new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
|
| 557 |
+
return ret.to(x.dtype, bitcast=True), new_ids
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
@triton.jit
|
| 561 |
+
def _bitonic_merge(
|
| 562 |
+
x,
|
| 563 |
+
ids,
|
| 564 |
+
stage: tl.constexpr,
|
| 565 |
+
order: tl.constexpr,
|
| 566 |
+
n_dims: tl.constexpr,
|
| 567 |
+
):
|
| 568 |
+
n_outer: tl.constexpr = x.numel >> n_dims
|
| 569 |
+
tl.static_assert(stage <= n_dims)
|
| 570 |
+
# flip denotes whether to re-arrange sub-sequences of elements in ascending or
|
| 571 |
+
# descending order.
|
| 572 |
+
# if flip = 00000000... then all elements will be re-arranged ascendingly at this stage
|
| 573 |
+
# if flip = 00110011... then all the elements will be re-arranged alternatingly (with
|
| 574 |
+
# a stride of 2) at this stage
|
| 575 |
+
if order == 2:
|
| 576 |
+
shape: tl.constexpr = [
|
| 577 |
+
n_outer * 2 ** (n_dims - 1 - stage),
|
| 578 |
+
2,
|
| 579 |
+
2**stage,
|
| 580 |
+
]
|
| 581 |
+
flip = tl.reshape(
|
| 582 |
+
tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
|
| 583 |
+
)
|
| 584 |
+
else:
|
| 585 |
+
flip = order
|
| 586 |
+
# perform `stage` rounds of `compare-and-swap`
|
| 587 |
+
for i in tl.static_range(stage):
|
| 588 |
+
x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
|
| 589 |
+
return x, ids
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
@triton.heuristics(
|
| 593 |
+
{
|
| 594 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
|
| 595 |
+
}
|
| 596 |
+
)
|
| 597 |
+
@triton.autotune(
|
| 598 |
+
configs=[
|
| 599 |
+
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
|
| 600 |
+
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2),
|
| 601 |
+
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
|
| 602 |
+
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3),
|
| 603 |
+
triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
|
| 604 |
+
],
|
| 605 |
+
key=["topk"],
|
| 606 |
+
)
|
| 607 |
+
@triton.jit
|
| 608 |
+
def _topk_index_partial_kernel(
|
| 609 |
+
s_ptr, # Score: [num_q_heads, BS, max_seqblock]
|
| 610 |
+
ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
|
| 611 |
+
ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape
|
| 612 |
+
seq_lens,
|
| 613 |
+
block_size: tl.constexpr,
|
| 614 |
+
topk: tl.constexpr,
|
| 615 |
+
chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns
|
| 616 |
+
# strides
|
| 617 |
+
stride_s_h,
|
| 618 |
+
stride_s_b,
|
| 619 |
+
stride_s_k,
|
| 620 |
+
stride_ts_c,
|
| 621 |
+
stride_ts_h,
|
| 622 |
+
stride_ts_b,
|
| 623 |
+
stride_ts_t,
|
| 624 |
+
stride_ti_c,
|
| 625 |
+
stride_ti_h,
|
| 626 |
+
stride_ti_b,
|
| 627 |
+
stride_ti_t,
|
| 628 |
+
# META
|
| 629 |
+
BLOCK_SIZE_K: tl.constexpr,
|
| 630 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 631 |
+
):
|
| 632 |
+
tl.static_assert(BLOCK_SIZE_K > topk)
|
| 633 |
+
pid_b = tl.program_id(0)
|
| 634 |
+
pid_h = tl.program_id(1)
|
| 635 |
+
pid_chunk = tl.program_id(2)
|
| 636 |
+
|
| 637 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 638 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 639 |
+
|
| 640 |
+
# Slice this chunk owns within [0, num_blocks).
|
| 641 |
+
chunk_start = pid_chunk * chunk_blocks
|
| 642 |
+
chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
|
| 643 |
+
chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
|
| 644 |
+
|
| 645 |
+
off_k = tl.arange(0, BLOCK_SIZE_K)
|
| 646 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 647 |
+
|
| 648 |
+
# Pointer starts at chunk_start.
|
| 649 |
+
s_ptrs = (
|
| 650 |
+
s_ptr
|
| 651 |
+
+ pid_b * stride_s_b
|
| 652 |
+
+ pid_h * stride_s_h
|
| 653 |
+
+ (chunk_start + off_k) * stride_s_k
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
|
| 657 |
+
topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
|
| 658 |
+
left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
|
| 659 |
+
|
| 660 |
+
# Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
|
| 661 |
+
# chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
|
| 662 |
+
for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
|
| 663 |
+
mask = off_k < chunk_actual - i
|
| 664 |
+
score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
|
| 665 |
+
score = tl.where(score != score, -1e30, score)
|
| 666 |
+
s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
|
| 667 |
+
topk_score, last_topk_score = score, topk_score
|
| 668 |
+
topk_idx, last_topk_idx = (
|
| 669 |
+
tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
|
| 670 |
+
topk_idx,
|
| 671 |
+
)
|
| 672 |
+
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
| 673 |
+
for j in tl.static_range(1, n_dims):
|
| 674 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 675 |
+
topk_score, topk_idx.to(tl.int32), j, 2, n_dims
|
| 676 |
+
)
|
| 677 |
+
if i != 0:
|
| 678 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 679 |
+
topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
|
| 680 |
+
)
|
| 681 |
+
topk_score_new = last_topk_score * left_half_mask + topk_score * (
|
| 682 |
+
1 - left_half_mask
|
| 683 |
+
)
|
| 684 |
+
topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
|
| 685 |
+
1 - left_half_mask
|
| 686 |
+
)
|
| 687 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 688 |
+
topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
|
| 689 |
+
)
|
| 690 |
+
else:
|
| 691 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 692 |
+
topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
|
| 693 |
+
)
|
| 694 |
+
|
| 695 |
+
# Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort).
|
| 696 |
+
topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
| 697 |
+
final_score = tl.sum(
|
| 698 |
+
topk_mask_extract[:, None]
|
| 699 |
+
* tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
| 700 |
+
axis=0,
|
| 701 |
+
)
|
| 702 |
+
final_idx = tl.sum(
|
| 703 |
+
topk_mask_extract[:, None]
|
| 704 |
+
* tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
| 705 |
+
axis=0,
|
| 706 |
+
)
|
| 707 |
+
|
| 708 |
+
# Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry
|
| 709 |
+
# -1e30 / 0 sentinels and lose to real scores in the merge stage.
|
| 710 |
+
ts_ptrs = (
|
| 711 |
+
ts_partial_ptr
|
| 712 |
+
+ pid_chunk * stride_ts_c
|
| 713 |
+
+ pid_b * stride_ts_b
|
| 714 |
+
+ pid_h * stride_ts_h
|
| 715 |
+
+ off_t * stride_ts_t
|
| 716 |
+
)
|
| 717 |
+
ti_ptrs = (
|
| 718 |
+
ti_partial_ptr
|
| 719 |
+
+ pid_chunk * stride_ti_c
|
| 720 |
+
+ pid_b * stride_ti_b
|
| 721 |
+
+ pid_h * stride_ti_h
|
| 722 |
+
+ off_t * stride_ti_t
|
| 723 |
+
)
|
| 724 |
+
tl.store(ts_ptrs, final_score)
|
| 725 |
+
tl.store(ti_ptrs, final_idx)
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
@triton.heuristics(
|
| 729 |
+
{
|
| 730 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
|
| 731 |
+
"BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
|
| 732 |
+
args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"])
|
| 733 |
+
),
|
| 734 |
+
}
|
| 735 |
+
)
|
| 736 |
+
@triton.jit
|
| 737 |
+
def _topk_index_merge_kernel(
|
| 738 |
+
ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
|
| 739 |
+
ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape
|
| 740 |
+
ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk]
|
| 741 |
+
seq_lens,
|
| 742 |
+
block_size: tl.constexpr,
|
| 743 |
+
topk: tl.constexpr,
|
| 744 |
+
# strides
|
| 745 |
+
stride_ts_c,
|
| 746 |
+
stride_ts_h,
|
| 747 |
+
stride_ts_b,
|
| 748 |
+
stride_ts_t,
|
| 749 |
+
stride_ti_c,
|
| 750 |
+
stride_ti_h,
|
| 751 |
+
stride_ti_b,
|
| 752 |
+
stride_ti_t,
|
| 753 |
+
stride_tif_h,
|
| 754 |
+
stride_tif_b,
|
| 755 |
+
stride_tif_t,
|
| 756 |
+
# META
|
| 757 |
+
NUM_TOPK_CHUNKS: tl.constexpr,
|
| 758 |
+
BLOCK_SIZE_K: tl.constexpr,
|
| 759 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 760 |
+
):
|
| 761 |
+
pid_b = tl.program_id(0)
|
| 762 |
+
pid_h = tl.program_id(1)
|
| 763 |
+
|
| 764 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 765 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 766 |
+
|
| 767 |
+
# Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K.
|
| 768 |
+
# candidate at flat position p comes from chunk = p // BLOCK_SIZE_T,
|
| 769 |
+
# in_chunk = p % BLOCK_SIZE_T.
|
| 770 |
+
off = tl.arange(0, BLOCK_SIZE_K)
|
| 771 |
+
chunk_idx = off // BLOCK_SIZE_T
|
| 772 |
+
in_chunk_idx = off % BLOCK_SIZE_T
|
| 773 |
+
valid = chunk_idx < NUM_TOPK_CHUNKS
|
| 774 |
+
|
| 775 |
+
score_offset = (
|
| 776 |
+
chunk_idx * stride_ts_c
|
| 777 |
+
+ pid_h * stride_ts_h
|
| 778 |
+
+ pid_b * stride_ts_b
|
| 779 |
+
+ in_chunk_idx * stride_ts_t
|
| 780 |
+
)
|
| 781 |
+
idx_offset = (
|
| 782 |
+
chunk_idx * stride_ti_c
|
| 783 |
+
+ pid_h * stride_ti_h
|
| 784 |
+
+ pid_b * stride_ti_b
|
| 785 |
+
+ in_chunk_idx * stride_ti_t
|
| 786 |
+
)
|
| 787 |
+
|
| 788 |
+
score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
|
| 789 |
+
tl.float32
|
| 790 |
+
)
|
| 791 |
+
score = tl.where(score != score, -1e30, score)
|
| 792 |
+
idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
|
| 793 |
+
|
| 794 |
+
# Full bitonic descending sort of BLOCK_SIZE_K items.
|
| 795 |
+
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
| 796 |
+
for j in tl.static_range(1, n_dims):
|
| 797 |
+
score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
|
| 798 |
+
score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
|
| 799 |
+
|
| 800 |
+
# Extract first BLOCK_SIZE_T positions — these are the global top-K.
|
| 801 |
+
extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
| 802 |
+
topk_idx_final = tl.sum(
|
| 803 |
+
extract_mask[:, None]
|
| 804 |
+
* tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
| 805 |
+
axis=0,
|
| 806 |
+
)
|
| 807 |
+
|
| 808 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 809 |
+
tif_ptrs = (
|
| 810 |
+
ti_final_ptr
|
| 811 |
+
+ pid_h * stride_tif_h
|
| 812 |
+
+ pid_b * stride_tif_b
|
| 813 |
+
+ off_t * stride_tif_t
|
| 814 |
+
)
|
| 815 |
+
topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
|
| 816 |
+
tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty))
|
| 817 |
+
|
| 818 |
+
|
| 819 |
+
@torch.no_grad()
|
| 820 |
+
def flash_decode_with_topk_idx(
|
| 821 |
+
q: torch.Tensor, # [batch_size, num_heads, head_dim]
|
| 822 |
+
sink: Optional[torch.Tensor],
|
| 823 |
+
k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
|
| 824 |
+
v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True
|
| 825 |
+
req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
|
| 826 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 827 |
+
max_seqlen: int,
|
| 828 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 829 |
+
block_size: int,
|
| 830 |
+
topk: int,
|
| 831 |
+
init_blocks: int,
|
| 832 |
+
local_blocks: int,
|
| 833 |
+
sm_scale: Optional[float] = None,
|
| 834 |
+
use_tma: bool = True,
|
| 835 |
+
score_type: str = "max",
|
| 836 |
+
disable_index_value: bool = False,
|
| 837 |
+
use_dense_main_attn: bool = False, # NOTE: need transform idx in this case
|
| 838 |
+
page_size: int = 1,
|
| 839 |
+
) -> torch.Tensor:
|
| 840 |
+
assert score_type in (
|
| 841 |
+
"max",
|
| 842 |
+
"lse",
|
| 843 |
+
), f"score_type must be 'max' or 'lse', got {score_type!r}"
|
| 844 |
+
triton.set_allocator(robust_allocator)
|
| 845 |
+
# dtype check
|
| 846 |
+
assert (
|
| 847 |
+
q.dtype == torch.bfloat16
|
| 848 |
+
or q.dtype == torch.float16
|
| 849 |
+
and k_cache.dtype == q.dtype
|
| 850 |
+
)
|
| 851 |
+
if not disable_index_value:
|
| 852 |
+
assert v_cache is not None
|
| 853 |
+
# shape
|
| 854 |
+
batch_size, num_q_heads, head_dim = q.shape
|
| 855 |
+
max_slots, num_kv_heads, _ = k_cache.shape
|
| 856 |
+
max_kv_len = req_to_token.shape[1]
|
| 857 |
+
assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
|
| 858 |
+
# gqa
|
| 859 |
+
assert num_q_heads % num_kv_heads == 0
|
| 860 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 861 |
+
# sm scale
|
| 862 |
+
if sm_scale is None:
|
| 863 |
+
sm_scale = head_dim**-0.5
|
| 864 |
+
# NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets.
|
| 865 |
+
# Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads.
|
| 866 |
+
# TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by:
|
| 867 |
+
# NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS)
|
| 868 |
+
# then rounded down to the nearest power of 2 (merge kernel requires it).
|
| 869 |
+
# Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len.
|
| 870 |
+
# Empty chunks early-return cheaply, so over-chunking is nearly free.
|
| 871 |
+
# E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096.
|
| 872 |
+
TARGET_GRID = 4096
|
| 873 |
+
MAX_NUM_KV_CHUNKS = 256
|
| 874 |
+
target = max(
|
| 875 |
+
1,
|
| 876 |
+
min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)),
|
| 877 |
+
)
|
| 878 |
+
NUM_KV_CHUNKS = 1 << (target.bit_length() - 1)
|
| 879 |
+
score_kv_len = min(max_seqlen, max_kv_len)
|
| 880 |
+
# The score producers below write every valid block column
|
| 881 |
+
# [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers
|
| 882 |
+
# clamp their scan to the same per-row valid block count, so columns beyond
|
| 883 |
+
# seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph
|
| 884 |
+
# capture the static score shape can be much larger than the live context.
|
| 885 |
+
score = torch.empty(
|
| 886 |
+
(num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)),
|
| 887 |
+
dtype=torch.float32,
|
| 888 |
+
device=q.device,
|
| 889 |
+
)
|
| 890 |
+
use_jit_topk = (
|
| 891 |
+
envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get()
|
| 892 |
+
and score.shape[2] <= 4096
|
| 893 |
+
and topk <= 32
|
| 894 |
+
)
|
| 895 |
+
# If the live context has <= topk sparse blocks, the downstream dense
|
| 896 |
+
# page-table/JIT top-k kernels select every block from seq_lens directly
|
| 897 |
+
# without reading score. Keep this gate in sync with the consumers below:
|
| 898 |
+
# _skip_block_topk/use_dense_main_attn and use_jit_topk special-case
|
| 899 |
+
# num_blocks <= topk, while the Triton fallback reads score and must not
|
| 900 |
+
# skip these writes.
|
| 901 |
+
skip_trivial_topk_score = use_dense_main_attn or use_jit_topk
|
| 902 |
+
|
| 903 |
+
grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads)
|
| 904 |
+
if disable_index_value:
|
| 905 |
+
_decode_score_kernel[grid](
|
| 906 |
+
q,
|
| 907 |
+
k_cache,
|
| 908 |
+
req_to_token,
|
| 909 |
+
score,
|
| 910 |
+
seq_lens,
|
| 911 |
+
slot_ids,
|
| 912 |
+
max_slots,
|
| 913 |
+
batch_size,
|
| 914 |
+
gqa_group_size,
|
| 915 |
+
head_dim,
|
| 916 |
+
block_size,
|
| 917 |
+
topk,
|
| 918 |
+
sm_scale,
|
| 919 |
+
init_blocks,
|
| 920 |
+
local_blocks,
|
| 921 |
+
q.stride(0),
|
| 922 |
+
q.stride(1),
|
| 923 |
+
q.stride(2),
|
| 924 |
+
k_cache.stride(0),
|
| 925 |
+
k_cache.stride(1),
|
| 926 |
+
k_cache.stride(2),
|
| 927 |
+
req_to_token.stride(0),
|
| 928 |
+
score.stride(0),
|
| 929 |
+
score.stride(1),
|
| 930 |
+
score.stride(2),
|
| 931 |
+
NUM_KV_CHUNKS=NUM_KV_CHUNKS,
|
| 932 |
+
SCORE_TYPE=score_type,
|
| 933 |
+
SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
|
| 934 |
+
)
|
| 935 |
+
else:
|
| 936 |
+
assert v_cache is not None
|
| 937 |
+
o = torch.empty(
|
| 938 |
+
NUM_KV_CHUNKS,
|
| 939 |
+
batch_size,
|
| 940 |
+
num_q_heads,
|
| 941 |
+
head_dim,
|
| 942 |
+
dtype=q.dtype,
|
| 943 |
+
device=q.device,
|
| 944 |
+
)
|
| 945 |
+
lse = torch.empty(
|
| 946 |
+
NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device
|
| 947 |
+
)
|
| 948 |
+
_decode_score_attn_kernel[grid](
|
| 949 |
+
q,
|
| 950 |
+
sink,
|
| 951 |
+
k_cache,
|
| 952 |
+
v_cache,
|
| 953 |
+
req_to_token,
|
| 954 |
+
o,
|
| 955 |
+
lse,
|
| 956 |
+
score,
|
| 957 |
+
seq_lens,
|
| 958 |
+
slot_ids,
|
| 959 |
+
max_slots,
|
| 960 |
+
batch_size,
|
| 961 |
+
gqa_group_size,
|
| 962 |
+
head_dim,
|
| 963 |
+
block_size,
|
| 964 |
+
topk,
|
| 965 |
+
sm_scale,
|
| 966 |
+
init_blocks,
|
| 967 |
+
local_blocks,
|
| 968 |
+
q.stride(0),
|
| 969 |
+
q.stride(1),
|
| 970 |
+
q.stride(2),
|
| 971 |
+
sink.stride(0) if sink is not None else 0,
|
| 972 |
+
sink.stride(1) if sink is not None else 0,
|
| 973 |
+
k_cache.stride(0),
|
| 974 |
+
k_cache.stride(1),
|
| 975 |
+
k_cache.stride(2),
|
| 976 |
+
v_cache.stride(0),
|
| 977 |
+
v_cache.stride(1),
|
| 978 |
+
v_cache.stride(2),
|
| 979 |
+
req_to_token.stride(0),
|
| 980 |
+
o.stride(0),
|
| 981 |
+
o.stride(1),
|
| 982 |
+
o.stride(2),
|
| 983 |
+
o.stride(3),
|
| 984 |
+
lse.stride(0),
|
| 985 |
+
lse.stride(1),
|
| 986 |
+
lse.stride(2),
|
| 987 |
+
score.stride(0),
|
| 988 |
+
score.stride(1),
|
| 989 |
+
score.stride(2),
|
| 990 |
+
NUM_KV_CHUNKS=NUM_KV_CHUNKS,
|
| 991 |
+
SCORE_TYPE=score_type,
|
| 992 |
+
SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
|
| 993 |
+
)
|
| 994 |
+
# Fused top-k + page-table transform: emit the dense backend's page table
|
| 995 |
+
# directly (page-size-aware) instead of block ids, skipping a separate gather.
|
| 996 |
+
# The page table + per-query effective KV length are allocated and returned.
|
| 997 |
+
real_seq_lens = None
|
| 998 |
+
if use_dense_main_attn:
|
| 999 |
+
# VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel
|
| 1000 |
+
# `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The
|
| 1001 |
+
# standalone vendor only ships the pure-Triton path, so this is gated off
|
| 1002 |
+
# by default (use_dense_main_attn=False). If you opt in, vendor that .cuh
|
| 1003 |
+
# too. Original line: `from sglang.jit_kernel.minimax_decode_topk import (
|
| 1004 |
+
# minimax_decode_topk_page_table)`.
|
| 1005 |
+
raise NotImplementedError(
|
| 1006 |
+
"use_dense_main_attn=True requires the un-vendored CUDA JIT kernel "
|
| 1007 |
+
"sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; "
|
| 1008 |
+
"the standalone Triton vendor only supports use_dense_main_attn=False."
|
| 1009 |
+
)
|
| 1010 |
+
# num_q_heads (= num index/kv heads) may be > 1 for DP attention; the
|
| 1011 |
+
# kernel emits a flattened [batch*num_q_heads] page table + seq_lens with
|
| 1012 |
+
# the kv head head-encoded (head-minor) into the page index.
|
| 1013 |
+
topk_idx, real_seq_lens = minimax_decode_topk_page_table(
|
| 1014 |
+
score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size
|
| 1015 |
+
)
|
| 1016 |
+
if disable_index_value:
|
| 1017 |
+
return None, topk_idx, real_seq_lens
|
| 1018 |
+
# fall through to the idx_o (index value) merge, then return
|
| 1019 |
+
_skip_block_topk = True
|
| 1020 |
+
else:
|
| 1021 |
+
_skip_block_topk = False
|
| 1022 |
+
|
| 1023 |
+
# get topk index
|
| 1024 |
+
if not _skip_block_topk:
|
| 1025 |
+
topk_idx = torch.empty(
|
| 1026 |
+
(num_q_heads, batch_size, topk),
|
| 1027 |
+
device=score.device,
|
| 1028 |
+
dtype=torch.int32,
|
| 1029 |
+
)
|
| 1030 |
+
|
| 1031 |
+
if _skip_block_topk:
|
| 1032 |
+
pass
|
| 1033 |
+
elif use_jit_topk:
|
| 1034 |
+
# Single-stage JIT radix-select: one kernel, no intermediate buffers.
|
| 1035 |
+
# Equivalent output to the 2-stage path (set of block ids, front-packed,
|
| 1036 |
+
# -1 padded); ~2-16x faster for long context.
|
| 1037 |
+
# VENDOR NOTE: this needs the un-vendored CUDA JIT kernel
|
| 1038 |
+
# `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the
|
| 1039 |
+
# `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so
|
| 1040 |
+
# this branch is never taken in the standalone vendor (the 2-stage Triton
|
| 1041 |
+
# `else` fallback below runs instead). Original line:
|
| 1042 |
+
# `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`.
|
| 1043 |
+
raise NotImplementedError(
|
| 1044 |
+
"use_jit_topk requires the un-vendored CUDA JIT kernel "
|
| 1045 |
+
"sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor "
|
| 1046 |
+
"uses the 2-stage Triton fallback (keep "
|
| 1047 |
+
"SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)."
|
| 1048 |
+
)
|
| 1049 |
+
elif has_native_ops() and topk <= 64:
|
| 1050 |
+
topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk)
|
| 1051 |
+
else:
|
| 1052 |
+
# 2-stage split-K Triton fallback.
|
| 1053 |
+
# Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim.
|
| 1054 |
+
# Same constraints as flash_decode: must be deterministic from values
|
| 1055 |
+
# constant within a cuda graph (BS, num_q_heads), pow2, capped so the
|
| 1056 |
+
# merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk))
|
| 1057 |
+
# stays reasonable. With topk=32 and cap 16, merge sorts 512 items.
|
| 1058 |
+
TOPK_TARGET_GRID = 64
|
| 1059 |
+
MAX_NUM_TOPK_CHUNKS = 16
|
| 1060 |
+
topk_target = max(
|
| 1061 |
+
1,
|
| 1062 |
+
min(
|
| 1063 |
+
MAX_NUM_TOPK_CHUNKS,
|
| 1064 |
+
TOPK_TARGET_GRID // max(1, batch_size * num_q_heads),
|
| 1065 |
+
),
|
| 1066 |
+
)
|
| 1067 |
+
NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1)
|
| 1068 |
+
BLOCK_SIZE_T = triton.next_power_of_2(topk)
|
| 1069 |
+
max_seqblock = score.shape[2]
|
| 1070 |
+
chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
|
| 1071 |
+
topk_score_partial = torch.empty(
|
| 1072 |
+
NUM_TOPK_CHUNKS,
|
| 1073 |
+
num_q_heads,
|
| 1074 |
+
batch_size,
|
| 1075 |
+
BLOCK_SIZE_T,
|
| 1076 |
+
dtype=torch.float32,
|
| 1077 |
+
device=score.device,
|
| 1078 |
+
)
|
| 1079 |
+
topk_idx_partial = torch.empty(
|
| 1080 |
+
NUM_TOPK_CHUNKS,
|
| 1081 |
+
num_q_heads,
|
| 1082 |
+
batch_size,
|
| 1083 |
+
BLOCK_SIZE_T,
|
| 1084 |
+
dtype=torch.int32,
|
| 1085 |
+
device=score.device,
|
| 1086 |
+
)
|
| 1087 |
+
# stream 0 (default): topk partial → topk merge
|
| 1088 |
+
grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS)
|
| 1089 |
+
_topk_index_partial_kernel[grid](
|
| 1090 |
+
score,
|
| 1091 |
+
topk_score_partial,
|
| 1092 |
+
topk_idx_partial,
|
| 1093 |
+
seq_lens,
|
| 1094 |
+
block_size,
|
| 1095 |
+
topk,
|
| 1096 |
+
chunk_blocks,
|
| 1097 |
+
score.stride(0),
|
| 1098 |
+
score.stride(1),
|
| 1099 |
+
score.stride(2),
|
| 1100 |
+
topk_score_partial.stride(0),
|
| 1101 |
+
topk_score_partial.stride(1),
|
| 1102 |
+
topk_score_partial.stride(2),
|
| 1103 |
+
topk_score_partial.stride(3),
|
| 1104 |
+
topk_idx_partial.stride(0),
|
| 1105 |
+
topk_idx_partial.stride(1),
|
| 1106 |
+
topk_idx_partial.stride(2),
|
| 1107 |
+
topk_idx_partial.stride(3),
|
| 1108 |
+
)
|
| 1109 |
+
grid = (batch_size, num_q_heads)
|
| 1110 |
+
_topk_index_merge_kernel[grid](
|
| 1111 |
+
topk_score_partial,
|
| 1112 |
+
topk_idx_partial,
|
| 1113 |
+
topk_idx,
|
| 1114 |
+
seq_lens,
|
| 1115 |
+
block_size,
|
| 1116 |
+
topk,
|
| 1117 |
+
topk_score_partial.stride(0),
|
| 1118 |
+
topk_score_partial.stride(1),
|
| 1119 |
+
topk_score_partial.stride(2),
|
| 1120 |
+
topk_score_partial.stride(3),
|
| 1121 |
+
topk_idx_partial.stride(0),
|
| 1122 |
+
topk_idx_partial.stride(1),
|
| 1123 |
+
topk_idx_partial.stride(2),
|
| 1124 |
+
topk_idx_partial.stride(3),
|
| 1125 |
+
topk_idx.stride(0),
|
| 1126 |
+
topk_idx.stride(1),
|
| 1127 |
+
topk_idx.stride(2),
|
| 1128 |
+
NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
|
| 1129 |
+
)
|
| 1130 |
+
if disable_index_value:
|
| 1131 |
+
return None, topk_idx, real_seq_lens
|
| 1132 |
+
# attn output merge (default stream)
|
| 1133 |
+
grid = (batch_size, num_q_heads)
|
| 1134 |
+
_merge_attn_out_kernel[grid](
|
| 1135 |
+
o,
|
| 1136 |
+
lse,
|
| 1137 |
+
seq_lens,
|
| 1138 |
+
head_dim,
|
| 1139 |
+
block_size,
|
| 1140 |
+
o.stride(0),
|
| 1141 |
+
o.stride(1),
|
| 1142 |
+
o.stride(2),
|
| 1143 |
+
o.stride(3),
|
| 1144 |
+
lse.stride(0),
|
| 1145 |
+
lse.stride(1),
|
| 1146 |
+
lse.stride(2),
|
| 1147 |
+
NUM_KV_CHUNKS=NUM_KV_CHUNKS,
|
| 1148 |
+
)
|
| 1149 |
+
o = o[0].contiguous()
|
| 1150 |
+
return o, topk_idx, real_seq_lens
|
build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
|
| 10 |
+
|
| 11 |
+
from .._native import (
|
| 12 |
+
native_decode_mma_supported,
|
| 13 |
+
native_decode_sparse_attn_mma_paged,
|
| 14 |
+
)
|
| 15 |
+
from ..common.utils import robust_allocator
|
| 16 |
+
|
| 17 |
+
_is_hip = is_hip()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@triton.heuristics(
|
| 21 |
+
{
|
| 22 |
+
"BLOCK_SIZE_H": lambda args: max(
|
| 23 |
+
16, triton.next_power_of_2(args["gqa_group_size"])
|
| 24 |
+
),
|
| 25 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 26 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
| 27 |
+
"HAS_SINK": lambda args: args["sink_ptr"] is not None,
|
| 28 |
+
"BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
@triton.autotune(
|
| 32 |
+
configs=[
|
| 33 |
+
triton.Config({}, num_warps=nw, num_stages=ns)
|
| 34 |
+
for nw in [4, 8]
|
| 35 |
+
for ns in [2, 3, 4, 5]
|
| 36 |
+
],
|
| 37 |
+
key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"],
|
| 38 |
+
)
|
| 39 |
+
@triton.jit
|
| 40 |
+
def _gqa_share_sparse_decode_kernel(
|
| 41 |
+
q_ptr, # Q: b x qh x d
|
| 42 |
+
sink_ptr, # Sink: qh x d
|
| 43 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 44 |
+
v_cache_ptr, # V paged: max_slots x kh x d
|
| 45 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 46 |
+
idx_ptr, # topk index: qh x b x topk
|
| 47 |
+
o_ptr, # O partial: c x b x qh x d
|
| 48 |
+
lse_ptr, # lse partial: c x b x qh
|
| 49 |
+
seq_lens,
|
| 50 |
+
slot_ids,
|
| 51 |
+
# shape
|
| 52 |
+
max_slots,
|
| 53 |
+
batch_size,
|
| 54 |
+
gqa_group_size,
|
| 55 |
+
head_dim,
|
| 56 |
+
max_topk,
|
| 57 |
+
max_kv_len,
|
| 58 |
+
# sm_scale
|
| 59 |
+
sm_scale,
|
| 60 |
+
# stride
|
| 61 |
+
stride_q_b,
|
| 62 |
+
stride_q_h,
|
| 63 |
+
stride_q_d,
|
| 64 |
+
stride_sink_h,
|
| 65 |
+
stride_sink_d,
|
| 66 |
+
stride_k_s,
|
| 67 |
+
stride_k_h,
|
| 68 |
+
stride_k_d,
|
| 69 |
+
stride_v_s,
|
| 70 |
+
stride_v_h,
|
| 71 |
+
stride_v_d,
|
| 72 |
+
stride_r2t_b,
|
| 73 |
+
stride_ti_h,
|
| 74 |
+
stride_ti_b,
|
| 75 |
+
stride_ti_t,
|
| 76 |
+
stride_o_c,
|
| 77 |
+
stride_o_b,
|
| 78 |
+
stride_o_h,
|
| 79 |
+
stride_o_d,
|
| 80 |
+
stride_l_c,
|
| 81 |
+
stride_l_b,
|
| 82 |
+
stride_l_h,
|
| 83 |
+
# META parameters
|
| 84 |
+
BATCH_SIZE_BUCKET: tl.constexpr,
|
| 85 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 86 |
+
BLOCK_SIZE_N: tl.constexpr,
|
| 87 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 88 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 89 |
+
NUM_TOPK_CHUNKS: tl.constexpr,
|
| 90 |
+
HAS_SINK: tl.constexpr,
|
| 91 |
+
IS_FP8: tl.constexpr,
|
| 92 |
+
):
|
| 93 |
+
# decode program ids: split-K over the topk dimension to give every SM
|
| 94 |
+
# something to do at small batch. pid(0) folds (batch, chunk) together so
|
| 95 |
+
# the grid size = batch_size * NUM_TOPK_CHUNKS.
|
| 96 |
+
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
| 97 |
+
pid_b = pid_bc % batch_size
|
| 98 |
+
pid_c = pid_bc // batch_size
|
| 99 |
+
pid_h = pid_kh * gqa_group_size
|
| 100 |
+
# per-chunk topk range. chunk_size is *runtime* (depends on max_topk which
|
| 101 |
+
# is a runtime arg, not constexpr), so don't annotate as tl.constexpr —
|
| 102 |
+
# doing so produces undefined behavior in Triton.
|
| 103 |
+
chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
|
| 104 |
+
chunk_start_topk = pid_c * chunk_size_topk
|
| 105 |
+
chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk
|
| 106 |
+
# get q k start and len after rmpad
|
| 107 |
+
seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len)
|
| 108 |
+
sid = (
|
| 109 |
+
tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
|
| 110 |
+
) % max_slots # to avoid bugs when slot_ids is negative
|
| 111 |
+
# get real topk
|
| 112 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 113 |
+
idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b
|
| 114 |
+
topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1)
|
| 115 |
+
valid_idx = tl.where(topk_idx >= 0, off_t, -1)
|
| 116 |
+
real_topk = tl.sum(valid_idx != -1, axis=0)
|
| 117 |
+
chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk)
|
| 118 |
+
# init pointer
|
| 119 |
+
off_n = tl.arange(0, BLOCK_SIZE_N)
|
| 120 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 121 |
+
dim_mask = off_d < head_dim
|
| 122 |
+
# init statistics — kept at -inf so empty chunks (chunk_start >= real_topk)
|
| 123 |
+
# naturally fall out as weight=0 in the merge step.
|
| 124 |
+
# q is always needed; load it unconditionally.
|
| 125 |
+
q_ptrs = tl.make_block_ptr(
|
| 126 |
+
base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
|
| 127 |
+
shape=(gqa_group_size, head_dim),
|
| 128 |
+
strides=(stride_q_h, stride_q_d),
|
| 129 |
+
offsets=(0, 0),
|
| 130 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 131 |
+
order=(1, 0),
|
| 132 |
+
)
|
| 133 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 134 |
+
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 135 |
+
lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 136 |
+
# Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code-
|
| 137 |
+
# eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is
|
| 138 |
+
# then None). The original compound `if HAS_SINK and pid_c == 0` traced
|
| 139 |
+
# the body even when False -> NoneType make_block_ptr compile error on
|
| 140 |
+
# sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always
|
| 141 |
+
# False for us, but this keeps the sink path correct too.
|
| 142 |
+
if HAS_SINK:
|
| 143 |
+
if pid_c == 0:
|
| 144 |
+
sink_ptrs = tl.make_block_ptr(
|
| 145 |
+
base=sink_ptr + pid_h * stride_sink_h,
|
| 146 |
+
shape=(gqa_group_size, head_dim),
|
| 147 |
+
strides=(stride_sink_h, stride_sink_d),
|
| 148 |
+
offsets=(0, 0),
|
| 149 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 150 |
+
order=(1, 0),
|
| 151 |
+
)
|
| 152 |
+
sink = tl.load(
|
| 153 |
+
sink_ptrs, boundary_check=(0, 1), padding_option="zero"
|
| 154 |
+
).to(tl.float32)
|
| 155 |
+
qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale
|
| 156 |
+
m_i = qsink
|
| 157 |
+
lse_i = qsink
|
| 158 |
+
acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
|
| 159 |
+
# only iterate over this chunk's topk slice. the load must respect the
|
| 160 |
+
# per-chunk start offset.
|
| 161 |
+
cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t
|
| 162 |
+
for _ in tl.range(chunk_start_topk, chunk_end_topk):
|
| 163 |
+
# load index
|
| 164 |
+
c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N
|
| 165 |
+
cur_idx_ptr = cur_idx_ptr + stride_ti_t
|
| 166 |
+
# resolve slots for this block via req_to_token
|
| 167 |
+
pos = c + off_n
|
| 168 |
+
pos_mask = pos < seq_len
|
| 169 |
+
slots = tl.load(
|
| 170 |
+
req_to_token_ptr + sid * stride_r2t_b + pos,
|
| 171 |
+
mask=pos_mask,
|
| 172 |
+
other=0,
|
| 173 |
+
).to(tl.int64)
|
| 174 |
+
slots = (slots + max_slots) % max_slots # safety against negative
|
| 175 |
+
# load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
|
| 176 |
+
k_off = (
|
| 177 |
+
slots[None, :] * stride_k_s
|
| 178 |
+
+ pid_kh * stride_k_h
|
| 179 |
+
+ off_d[:, None] * stride_k_d
|
| 180 |
+
)
|
| 181 |
+
k = tl.load(
|
| 182 |
+
k_cache_ptr + k_off,
|
| 183 |
+
mask=dim_mask[:, None] & pos_mask[None, :],
|
| 184 |
+
other=0.0,
|
| 185 |
+
)
|
| 186 |
+
if IS_FP8:
|
| 187 |
+
# fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no
|
| 188 |
+
# scale), so dequant is just a widening cast to the Q compute dtype
|
| 189 |
+
# before the tl.dot. Matches the bf16 path bit-for-bit when the cache
|
| 190 |
+
# is bf16 (IS_FP8 False -> this branch is compiled out).
|
| 191 |
+
k = k.to(q.dtype)
|
| 192 |
+
# load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
|
| 193 |
+
v_off = (
|
| 194 |
+
slots[:, None] * stride_v_s
|
| 195 |
+
+ pid_kh * stride_v_h
|
| 196 |
+
+ off_d[None, :] * stride_v_d
|
| 197 |
+
)
|
| 198 |
+
v = tl.load(
|
| 199 |
+
v_cache_ptr + v_off,
|
| 200 |
+
mask=pos_mask[:, None] & dim_mask[None, :],
|
| 201 |
+
other=0.0,
|
| 202 |
+
)
|
| 203 |
+
if IS_FP8:
|
| 204 |
+
# Widen V before the P@V dot. This also makes the `p.to(v.dtype)`
|
| 205 |
+
# below cast P to the compute dtype (not to fp8, which would be
|
| 206 |
+
# catastrophic precision loss on the attention weights).
|
| 207 |
+
v = v.to(q.dtype)
|
| 208 |
+
# compute qk
|
| 209 |
+
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
|
| 210 |
+
qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf"))
|
| 211 |
+
# [H, D], [D, N] -> [H, N]
|
| 212 |
+
qk += tl.dot(q, k) * sm_scale
|
| 213 |
+
# compute m_ij and l_ij
|
| 214 |
+
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
| 215 |
+
p = tl.exp(qk - m_ij[:, None])
|
| 216 |
+
l_ij = tl.sum(p, axis=1)
|
| 217 |
+
# scale acc_o
|
| 218 |
+
acc_o_scale = tl.exp(m_i - m_ij)
|
| 219 |
+
acc_o = acc_o * acc_o_scale[:, None]
|
| 220 |
+
# load v and update acc_o
|
| 221 |
+
p = p.to(v.dtype)
|
| 222 |
+
# [H, N], [N, D] -> [H, D]
|
| 223 |
+
acc_o += tl.dot(p.to(v.dtype), v)
|
| 224 |
+
# update statistics
|
| 225 |
+
m_i = m_ij
|
| 226 |
+
lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij)
|
| 227 |
+
# final scale (matches the old non-split kernel for chunks where lse_i>-inf).
|
| 228 |
+
# For empty chunks (chunk_start_topk >= real_topk) the inner loop never
|
| 229 |
+
# runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute
|
| 230 |
+
# exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial
|
| 231 |
+
# and the merge result. Gate the scale with tl.where so empty chunks emit a
|
| 232 |
+
# clean zero (lse_i stays -inf which the merge correctly turns into weight=0).
|
| 233 |
+
scale = tl.where(
|
| 234 |
+
lse_i > float("-inf"),
|
| 235 |
+
tl.exp(m_i - lse_i),
|
| 236 |
+
tl.zeros_like(lse_i),
|
| 237 |
+
)
|
| 238 |
+
acc_o = acc_o * scale[:, None]
|
| 239 |
+
# save partial output and lse for the merge step
|
| 240 |
+
o_ptrs = tl.make_block_ptr(
|
| 241 |
+
base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 242 |
+
shape=(gqa_group_size, head_dim),
|
| 243 |
+
strides=(stride_o_h, stride_o_d),
|
| 244 |
+
offsets=(0, 0),
|
| 245 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 246 |
+
order=(1, 0),
|
| 247 |
+
)
|
| 248 |
+
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
|
| 249 |
+
lse_ptrs = tl.make_block_ptr(
|
| 250 |
+
base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
|
| 251 |
+
shape=(gqa_group_size,),
|
| 252 |
+
strides=(stride_l_h,),
|
| 253 |
+
offsets=(0,),
|
| 254 |
+
block_shape=(BLOCK_SIZE_H,),
|
| 255 |
+
order=(0,),
|
| 256 |
+
)
|
| 257 |
+
tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
@triton.heuristics(
|
| 261 |
+
{
|
| 262 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 263 |
+
}
|
| 264 |
+
)
|
| 265 |
+
@triton.jit
|
| 266 |
+
def _merge_topk_attn_out_kernel(
|
| 267 |
+
o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0
|
| 268 |
+
lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH]
|
| 269 |
+
head_dim,
|
| 270 |
+
stride_o_c,
|
| 271 |
+
stride_o_b,
|
| 272 |
+
stride_o_h,
|
| 273 |
+
stride_o_d,
|
| 274 |
+
stride_l_c,
|
| 275 |
+
stride_l_b,
|
| 276 |
+
stride_l_h,
|
| 277 |
+
NUM_TOPK_CHUNKS: tl.constexpr,
|
| 278 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 279 |
+
):
|
| 280 |
+
pid_b, pid_h = tl.program_id(0), tl.program_id(1)
|
| 281 |
+
off_c = tl.arange(0, NUM_TOPK_CHUNKS)
|
| 282 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 283 |
+
o_ptrs = tl.make_block_ptr(
|
| 284 |
+
base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 285 |
+
shape=(NUM_TOPK_CHUNKS, head_dim),
|
| 286 |
+
strides=(stride_o_c, stride_o_d),
|
| 287 |
+
offsets=(0, 0),
|
| 288 |
+
block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
|
| 289 |
+
order=(1, 0),
|
| 290 |
+
)
|
| 291 |
+
lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
|
| 292 |
+
o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 293 |
+
lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
|
| 294 |
+
# standard flash-decoding merge in linear (not log2) space, matching the
|
| 295 |
+
# decode kernel which uses tl.exp / tl.log.
|
| 296 |
+
lse_max = tl.max(lse, axis=0)
|
| 297 |
+
weights = tl.exp(lse - lse_max)
|
| 298 |
+
weights = weights / tl.sum(weights, axis=0)
|
| 299 |
+
o_merged = tl.sum(o * weights[:, None], axis=0)
|
| 300 |
+
o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
|
| 301 |
+
tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
@torch.no_grad()
|
| 305 |
+
def flash_decode_with_gqa_share_sparse(
|
| 306 |
+
q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
|
| 307 |
+
sink: Optional[torch.Tensor],
|
| 308 |
+
k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
|
| 309 |
+
v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
|
| 310 |
+
req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
|
| 311 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 312 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 313 |
+
block_size: int,
|
| 314 |
+
topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
|
| 315 |
+
sm_scale: Optional[float] = None,
|
| 316 |
+
use_tma: bool = True,
|
| 317 |
+
) -> torch.Tensor:
|
| 318 |
+
triton.set_allocator(robust_allocator)
|
| 319 |
+
# dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may
|
| 320 |
+
# be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel
|
| 321 |
+
# widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing
|
| 322 |
+
# sparse decode dtype contract unchanged by accepting fp8 only on HIP.
|
| 323 |
+
assert q.dtype in (torch.bfloat16, torch.float16)
|
| 324 |
+
_FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
|
| 325 |
+
is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
|
| 326 |
+
assert k_cache.dtype == q.dtype or is_fp8, (
|
| 327 |
+
f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) "
|
| 328 |
+
f"or fp8 on HIP, got {k_cache.dtype}"
|
| 329 |
+
)
|
| 330 |
+
assert v_cache.dtype == k_cache.dtype
|
| 331 |
+
# shape
|
| 332 |
+
batch_size, num_q_heads, head_dim = q.shape
|
| 333 |
+
max_slots, num_kv_heads, _ = k_cache.shape
|
| 334 |
+
assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
|
| 335 |
+
assert topk_idx.shape[0] == num_kv_heads
|
| 336 |
+
assert (
|
| 337 |
+
triton.next_power_of_2(block_size) == block_size
|
| 338 |
+
), f"block_size must be a power of 2, but got {block_size}"
|
| 339 |
+
# assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}"
|
| 340 |
+
max_kv_len = req_to_token.shape[1]
|
| 341 |
+
# gqa
|
| 342 |
+
assert num_q_heads % num_kv_heads == 0
|
| 343 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 344 |
+
max_topk = topk_idx.shape[2]
|
| 345 |
+
# sm scale
|
| 346 |
+
if sm_scale is None:
|
| 347 |
+
sm_scale = head_dim**-0.5
|
| 348 |
+
# Fast path: native tensor-core decode for the M3 MSA shape. Falls through
|
| 349 |
+
# to the Triton kernel for other shapes, sink, or fp8 caches.
|
| 350 |
+
if sink is None and not is_fp8 and native_decode_mma_supported(
|
| 351 |
+
q, k_cache, block_size=block_size
|
| 352 |
+
):
|
| 353 |
+
return native_decode_sparse_attn_mma_paged(
|
| 354 |
+
q,
|
| 355 |
+
k_cache,
|
| 356 |
+
v_cache,
|
| 357 |
+
req_to_token,
|
| 358 |
+
seq_lens,
|
| 359 |
+
slot_ids,
|
| 360 |
+
block_size,
|
| 361 |
+
topk_idx,
|
| 362 |
+
sm_scale,
|
| 363 |
+
)
|
| 364 |
+
# Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as
|
| 365 |
+
# flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must
|
| 366 |
+
# only depend on shape constants (so grid is fixed within a cuda graph).
|
| 367 |
+
# Capped by max_topk because chunks beyond real_topk early-fall-through to
|
| 368 |
+
# the merge-as-zero path; capping avoids wasting blocks at tiny topk.
|
| 369 |
+
TARGET_GRID = 256
|
| 370 |
+
target = max(
|
| 371 |
+
1,
|
| 372 |
+
min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)),
|
| 373 |
+
)
|
| 374 |
+
NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1)
|
| 375 |
+
# output tensor: split-K partials, merged into chunk 0 by the merge kernel
|
| 376 |
+
o_partial = torch.empty(
|
| 377 |
+
NUM_TOPK_CHUNKS,
|
| 378 |
+
batch_size,
|
| 379 |
+
num_q_heads,
|
| 380 |
+
head_dim,
|
| 381 |
+
dtype=q.dtype,
|
| 382 |
+
device=q.device,
|
| 383 |
+
)
|
| 384 |
+
lse_partial = torch.empty(
|
| 385 |
+
NUM_TOPK_CHUNKS,
|
| 386 |
+
batch_size,
|
| 387 |
+
num_q_heads,
|
| 388 |
+
dtype=torch.float32,
|
| 389 |
+
device=q.device,
|
| 390 |
+
)
|
| 391 |
+
# launch attention kernel
|
| 392 |
+
grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads)
|
| 393 |
+
_gqa_share_sparse_decode_kernel[grid](
|
| 394 |
+
q,
|
| 395 |
+
sink,
|
| 396 |
+
k_cache,
|
| 397 |
+
v_cache,
|
| 398 |
+
req_to_token,
|
| 399 |
+
topk_idx,
|
| 400 |
+
o_partial,
|
| 401 |
+
lse_partial,
|
| 402 |
+
seq_lens,
|
| 403 |
+
slot_ids,
|
| 404 |
+
max_slots,
|
| 405 |
+
batch_size,
|
| 406 |
+
gqa_group_size,
|
| 407 |
+
head_dim,
|
| 408 |
+
max_topk,
|
| 409 |
+
max_kv_len,
|
| 410 |
+
sm_scale,
|
| 411 |
+
q.stride(0),
|
| 412 |
+
q.stride(1),
|
| 413 |
+
q.stride(2),
|
| 414 |
+
sink.stride(0) if sink is not None else 0,
|
| 415 |
+
sink.stride(1) if sink is not None else 0,
|
| 416 |
+
k_cache.stride(0),
|
| 417 |
+
k_cache.stride(1),
|
| 418 |
+
k_cache.stride(2),
|
| 419 |
+
v_cache.stride(0),
|
| 420 |
+
v_cache.stride(1),
|
| 421 |
+
v_cache.stride(2),
|
| 422 |
+
req_to_token.stride(0),
|
| 423 |
+
topk_idx.stride(0),
|
| 424 |
+
topk_idx.stride(1),
|
| 425 |
+
topk_idx.stride(2),
|
| 426 |
+
o_partial.stride(0),
|
| 427 |
+
o_partial.stride(1),
|
| 428 |
+
o_partial.stride(2),
|
| 429 |
+
o_partial.stride(3),
|
| 430 |
+
lse_partial.stride(0),
|
| 431 |
+
lse_partial.stride(1),
|
| 432 |
+
lse_partial.stride(2),
|
| 433 |
+
BLOCK_SIZE_N=block_size,
|
| 434 |
+
NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
|
| 435 |
+
IS_FP8=is_fp8,
|
| 436 |
+
)
|
| 437 |
+
# merge partials into chunk 0
|
| 438 |
+
merge_grid = (batch_size, num_q_heads)
|
| 439 |
+
_merge_topk_attn_out_kernel[merge_grid](
|
| 440 |
+
o_partial,
|
| 441 |
+
lse_partial,
|
| 442 |
+
head_dim,
|
| 443 |
+
o_partial.stride(0),
|
| 444 |
+
o_partial.stride(1),
|
| 445 |
+
o_partial.stride(2),
|
| 446 |
+
o_partial.stride(3),
|
| 447 |
+
lse_partial.stride(0),
|
| 448 |
+
lse_partial.stride(1),
|
| 449 |
+
lse_partial.stride(2),
|
| 450 |
+
NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
|
| 451 |
+
)
|
| 452 |
+
return o_partial[0].contiguous()
|
build/torch211-cxx11-cu128-x86_64-linux/interface.py
ADDED
|
@@ -0,0 +1,679 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from ._native import (
|
| 11 |
+
has_native_ops,
|
| 12 |
+
native_indexer_block_scores,
|
| 13 |
+
native_nvfp4_dequant_swizzled_to_bf16,
|
| 14 |
+
)
|
| 15 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 16 |
+
from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse
|
| 17 |
+
from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _page_table_to_req_to_token(
|
| 21 |
+
page_table: torch.Tensor,
|
| 22 |
+
seqused_k: torch.Tensor,
|
| 23 |
+
max_seqlen_k: int,
|
| 24 |
+
blk_kv: int,
|
| 25 |
+
) -> torch.Tensor:
|
| 26 |
+
batch = int(page_table.shape[0])
|
| 27 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 28 |
+
blk_kv = int(blk_kv)
|
| 29 |
+
logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
|
| 30 |
+
page = logical_pos // blk_kv
|
| 31 |
+
offset = logical_pos % blk_kv
|
| 32 |
+
page = page.clamp(max=int(page_table.shape[1]) - 1)
|
| 33 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 34 |
+
physical_token = physical_page * blk_kv + offset[None, :]
|
| 35 |
+
valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
|
| 36 |
+
return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _dense_q2k_indices(
|
| 40 |
+
*,
|
| 41 |
+
head_kv: int,
|
| 42 |
+
batch: int,
|
| 43 |
+
topk: int,
|
| 44 |
+
seq_lens: torch.Tensor,
|
| 45 |
+
blk_kv: int,
|
| 46 |
+
) -> torch.Tensor:
|
| 47 |
+
idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
|
| 48 |
+
blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
|
| 49 |
+
valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
|
| 50 |
+
for b in range(batch):
|
| 51 |
+
valid = blocks < valid_blocks[b]
|
| 52 |
+
idx[:, b, valid] = blocks[valid]
|
| 53 |
+
return idx
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]:
|
| 57 |
+
seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist()
|
| 58 |
+
return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _packed_row_to_batch_block(
|
| 62 |
+
cu_seqlens_k: torch.Tensor,
|
| 63 |
+
blk_kv: int,
|
| 64 |
+
) -> list[tuple[int, int]]:
|
| 65 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv))
|
| 66 |
+
max_rows = max(rows_per_batch, default=0)
|
| 67 |
+
row_map: list[tuple[int, int]] = []
|
| 68 |
+
for block_idx in range(max_rows):
|
| 69 |
+
for batch_idx, rows in enumerate(rows_per_batch):
|
| 70 |
+
if block_idx < rows:
|
| 71 |
+
row_map.append((batch_idx, block_idx))
|
| 72 |
+
return row_map
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _csr_to_q2k_indices(
|
| 76 |
+
k2q_row_ptr: torch.Tensor,
|
| 77 |
+
k2q_q_indices: torch.Tensor,
|
| 78 |
+
cu_seqlens_q: torch.Tensor,
|
| 79 |
+
cu_seqlens_k: torch.Tensor,
|
| 80 |
+
blk_kv: int,
|
| 81 |
+
topk: int,
|
| 82 |
+
) -> torch.Tensor:
|
| 83 |
+
if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32:
|
| 84 |
+
raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32")
|
| 85 |
+
if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2:
|
| 86 |
+
raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2")
|
| 87 |
+
head_kv = int(k2q_row_ptr.shape[0])
|
| 88 |
+
total_q = int(cu_seqlens_q[-1].item())
|
| 89 |
+
topk = int(topk)
|
| 90 |
+
if topk <= 0:
|
| 91 |
+
raise ValueError(f"topK must be positive, got {topk}")
|
| 92 |
+
|
| 93 |
+
row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv))
|
| 94 |
+
if int(k2q_row_ptr.shape[1]) != len(row_map) + 1:
|
| 95 |
+
raise ValueError(
|
| 96 |
+
"k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: "
|
| 97 |
+
f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}"
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
|
| 101 |
+
row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False)
|
| 102 |
+
q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False)
|
| 103 |
+
q2k = torch.full(
|
| 104 |
+
(head_kv, total_q, topk),
|
| 105 |
+
-1,
|
| 106 |
+
dtype=torch.int32,
|
| 107 |
+
device=k2q_row_ptr.device,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
next_slot: dict[tuple[int, int], int] = {}
|
| 111 |
+
seen: set[tuple[int, int, int]] = set()
|
| 112 |
+
for h in range(head_kv):
|
| 113 |
+
for row, (batch_idx, kv_block_idx) in enumerate(row_map):
|
| 114 |
+
start = int(row_ptr_cpu[h, row].item())
|
| 115 |
+
end = int(row_ptr_cpu[h, row + 1].item())
|
| 116 |
+
q_start = int(cu_q_cpu[batch_idx])
|
| 117 |
+
q_end = int(cu_q_cpu[batch_idx + 1])
|
| 118 |
+
q_len = q_end - q_start
|
| 119 |
+
for pos in range(start, end):
|
| 120 |
+
q_val = int(q_idx_cpu[h, pos].item())
|
| 121 |
+
if q_val < 0:
|
| 122 |
+
continue
|
| 123 |
+
if q_start <= q_val < q_end:
|
| 124 |
+
q_global = q_val
|
| 125 |
+
elif q_val < q_len:
|
| 126 |
+
q_global = q_start + q_val
|
| 127 |
+
else:
|
| 128 |
+
continue
|
| 129 |
+
key = (h, q_global, kv_block_idx)
|
| 130 |
+
if key in seen:
|
| 131 |
+
continue
|
| 132 |
+
seen.add(key)
|
| 133 |
+
slot_key = (h, q_global)
|
| 134 |
+
slot = next_slot.get(slot_key, 0)
|
| 135 |
+
if slot >= topk:
|
| 136 |
+
continue
|
| 137 |
+
q2k[h, q_global, slot] = int(kv_block_idx)
|
| 138 |
+
next_slot[slot_key] = slot + 1
|
| 139 |
+
return q2k.contiguous()
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _make_prefill_cache_and_metadata(
|
| 143 |
+
q: torch.Tensor,
|
| 144 |
+
k: torch.Tensor,
|
| 145 |
+
v: torch.Tensor,
|
| 146 |
+
cu_seqlens_q: torch.Tensor,
|
| 147 |
+
cu_seqlens_k: torch.Tensor,
|
| 148 |
+
max_seqlen_k: int,
|
| 149 |
+
blk_kv: int,
|
| 150 |
+
*,
|
| 151 |
+
page_table: Optional[torch.Tensor],
|
| 152 |
+
seqused_k: Optional[torch.Tensor],
|
| 153 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 154 |
+
if page_table is None:
|
| 155 |
+
if k.ndim != 3 or v.ndim != 3:
|
| 156 |
+
raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]")
|
| 157 |
+
if k.shape != v.shape:
|
| 158 |
+
raise ValueError("k and v must have the same shape")
|
| 159 |
+
k_cache = k.contiguous()
|
| 160 |
+
v_cache = v.contiguous()
|
| 161 |
+
batch = int(cu_seqlens_q.numel()) - 1
|
| 162 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 163 |
+
logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64)
|
| 164 |
+
cu_k = cu_seqlens_k.to(torch.int64)
|
| 165 |
+
seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous()
|
| 166 |
+
valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
|
| 167 |
+
req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32)
|
| 168 |
+
req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
|
| 169 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 170 |
+
else:
|
| 171 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 172 |
+
raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 173 |
+
if k.shape != v.shape:
|
| 174 |
+
raise ValueError("k and v must have the same shape")
|
| 175 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 176 |
+
raise ValueError("paged k/v page dimension must equal blk_kv")
|
| 177 |
+
if page_table.dtype != torch.int32:
|
| 178 |
+
raise TypeError("page_table must be torch.int32")
|
| 179 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 180 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 181 |
+
batch = int(page_table.shape[0])
|
| 182 |
+
if seqused_k is None:
|
| 183 |
+
seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous()
|
| 184 |
+
else:
|
| 185 |
+
seq_lens = seqused_k.to(torch.int32).contiguous()
|
| 186 |
+
logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64)
|
| 187 |
+
page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1)
|
| 188 |
+
offset = logical % int(blk_kv)
|
| 189 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 190 |
+
req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32)
|
| 191 |
+
valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
|
| 192 |
+
req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
|
| 193 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 194 |
+
|
| 195 |
+
q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32)
|
| 196 |
+
prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous()
|
| 197 |
+
return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int:
|
| 201 |
+
if kv_heads <= 0 or q_heads % kv_heads != 0:
|
| 202 |
+
raise ValueError("number of query heads must be divisible by KV heads")
|
| 203 |
+
gqa_group = q_heads // kv_heads
|
| 204 |
+
limit = max(1, 128 // max(1, gqa_group))
|
| 205 |
+
for candidate in (64, 32, 16, 8, 4, 2, 1):
|
| 206 |
+
if candidate <= limit:
|
| 207 |
+
return candidate
|
| 208 |
+
return 1
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _normalize_decode_q2k(
|
| 212 |
+
q2k_indices: Optional[torch.Tensor],
|
| 213 |
+
*,
|
| 214 |
+
batch: int,
|
| 215 |
+
seqlen_q: int,
|
| 216 |
+
head_kv: int,
|
| 217 |
+
seqused_k: torch.Tensor,
|
| 218 |
+
blk_kv: int,
|
| 219 |
+
) -> torch.Tensor:
|
| 220 |
+
if q2k_indices is None:
|
| 221 |
+
topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
|
| 222 |
+
return _dense_q2k_indices(
|
| 223 |
+
head_kv=head_kv,
|
| 224 |
+
batch=batch,
|
| 225 |
+
topk=topk,
|
| 226 |
+
seq_lens=seqused_k,
|
| 227 |
+
blk_kv=blk_kv,
|
| 228 |
+
)
|
| 229 |
+
if q2k_indices.dtype != torch.int32:
|
| 230 |
+
raise TypeError("q2k_indices must be torch.int32")
|
| 231 |
+
if q2k_indices.ndim != 3:
|
| 232 |
+
raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
|
| 233 |
+
if q2k_indices.shape[0] != head_kv:
|
| 234 |
+
raise ValueError("q2k_indices first dimension must match Hkv")
|
| 235 |
+
if int(q2k_indices.shape[1]) != batch * seqlen_q:
|
| 236 |
+
raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
|
| 237 |
+
if seqlen_q != 1:
|
| 238 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 239 |
+
return q2k_indices[:, ::seqlen_q, :].contiguous()
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def sparse_decode_atten_func(
|
| 243 |
+
q: torch.Tensor,
|
| 244 |
+
k: torch.Tensor,
|
| 245 |
+
v: torch.Tensor,
|
| 246 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 247 |
+
*,
|
| 248 |
+
page_table: torch.Tensor,
|
| 249 |
+
seqused_k: torch.Tensor,
|
| 250 |
+
seqlen_q: int,
|
| 251 |
+
max_seqlen_k: int,
|
| 252 |
+
blk_kv: int = 128,
|
| 253 |
+
causal: bool = True,
|
| 254 |
+
softmax_scale: Optional[float] = None,
|
| 255 |
+
return_softmax_lse: bool = False,
|
| 256 |
+
schedule: Optional[object] = None,
|
| 257 |
+
O_partial: Optional[torch.Tensor] = None,
|
| 258 |
+
LSE_partial: Optional[torch.Tensor] = None,
|
| 259 |
+
):
|
| 260 |
+
"""Run MiniMax paged sparse decode on Blackwell.
|
| 261 |
+
|
| 262 |
+
This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
|
| 263 |
+
but accepts the Blackwell package's validated BF16/FP16 decode path. It
|
| 264 |
+
supports single-token decode (`seqlen_q=1`) with paged K/V
|
| 265 |
+
`[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
|
| 266 |
+
"""
|
| 267 |
+
|
| 268 |
+
if schedule is not None or O_partial is not None or LSE_partial is not None:
|
| 269 |
+
raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
|
| 270 |
+
if return_softmax_lse:
|
| 271 |
+
raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
|
| 272 |
+
if not causal:
|
| 273 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
|
| 274 |
+
if int(seqlen_q) != 1:
|
| 275 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 276 |
+
if q.ndim != 3:
|
| 277 |
+
raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
|
| 278 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 279 |
+
raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 280 |
+
if k.shape != v.shape:
|
| 281 |
+
raise ValueError("k and v must have the same shape")
|
| 282 |
+
if q.device != k.device or q.device != v.device or q.device != page_table.device:
|
| 283 |
+
raise ValueError("q, k, v, and page_table must be on the same device")
|
| 284 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 285 |
+
raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
|
| 286 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 287 |
+
raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
|
| 288 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128:
|
| 289 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 290 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 291 |
+
raise ValueError("k.shape[2] must equal blk_kv")
|
| 292 |
+
if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
|
| 293 |
+
raise TypeError("page_table and seqused_k must be torch.int32")
|
| 294 |
+
if page_table.ndim != 2:
|
| 295 |
+
raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
|
| 296 |
+
batch = int(page_table.shape[0])
|
| 297 |
+
if int(q.shape[0]) != batch:
|
| 298 |
+
raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
|
| 299 |
+
|
| 300 |
+
head_kv = int(k.shape[1])
|
| 301 |
+
q2k = _normalize_decode_q2k(
|
| 302 |
+
q2k_indices,
|
| 303 |
+
batch=batch,
|
| 304 |
+
seqlen_q=int(seqlen_q),
|
| 305 |
+
head_kv=head_kv,
|
| 306 |
+
seqused_k=seqused_k,
|
| 307 |
+
blk_kv=int(blk_kv),
|
| 308 |
+
)
|
| 309 |
+
req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
|
| 310 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 311 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 312 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 313 |
+
return flash_decode_with_gqa_share_sparse(
|
| 314 |
+
q.contiguous(),
|
| 315 |
+
None,
|
| 316 |
+
k_cache,
|
| 317 |
+
v_cache,
|
| 318 |
+
req_to_token,
|
| 319 |
+
seqused_k.contiguous(),
|
| 320 |
+
slot_ids,
|
| 321 |
+
int(blk_kv),
|
| 322 |
+
q2k,
|
| 323 |
+
sm_scale=softmax_scale,
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
class SparseDecodePagedAttentionWrapper:
|
| 328 |
+
def __init__(self, *, blk_kv: int = 128, causal: bool = True):
|
| 329 |
+
self.blk_kv = int(blk_kv)
|
| 330 |
+
self.causal = bool(causal)
|
| 331 |
+
self.page_table: Optional[torch.Tensor] = None
|
| 332 |
+
self.seqused_k: Optional[torch.Tensor] = None
|
| 333 |
+
self.seqlen_q: Optional[int] = None
|
| 334 |
+
self.max_seqlen_k: Optional[int] = None
|
| 335 |
+
self.q2k_indices: Optional[torch.Tensor] = None
|
| 336 |
+
|
| 337 |
+
def plan(
|
| 338 |
+
self,
|
| 339 |
+
*,
|
| 340 |
+
page_table: torch.Tensor,
|
| 341 |
+
seqused_k: torch.Tensor,
|
| 342 |
+
seqlen_q: int,
|
| 343 |
+
max_seqlen_k: int,
|
| 344 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 345 |
+
num_qo_heads: Optional[int] = None,
|
| 346 |
+
num_kv_heads: Optional[int] = None,
|
| 347 |
+
head_dim: Optional[int] = 128,
|
| 348 |
+
enable_cuda_graph: bool = False,
|
| 349 |
+
max_grid_size: Optional[int] = None,
|
| 350 |
+
fixed_split_size: Optional[int] = None,
|
| 351 |
+
disable_split_kv: bool = False,
|
| 352 |
+
) -> "SparseDecodePagedAttentionWrapper":
|
| 353 |
+
if head_dim is not None and int(head_dim) != 128:
|
| 354 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 355 |
+
self.page_table = page_table.contiguous()
|
| 356 |
+
self.seqused_k = seqused_k.contiguous()
|
| 357 |
+
self.seqlen_q = int(seqlen_q)
|
| 358 |
+
self.max_seqlen_k = int(max_seqlen_k)
|
| 359 |
+
self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
|
| 360 |
+
return self
|
| 361 |
+
|
| 362 |
+
def run(
|
| 363 |
+
self,
|
| 364 |
+
q: torch.Tensor,
|
| 365 |
+
k: torch.Tensor,
|
| 366 |
+
v: torch.Tensor,
|
| 367 |
+
*,
|
| 368 |
+
softmax_scale: Optional[float] = None,
|
| 369 |
+
return_softmax_lse: bool = False,
|
| 370 |
+
out: Optional[torch.Tensor] = None,
|
| 371 |
+
lse: Optional[torch.Tensor] = None,
|
| 372 |
+
):
|
| 373 |
+
if self.page_table is None or self.seqused_k is None:
|
| 374 |
+
raise RuntimeError("decode wrapper must be planned before run")
|
| 375 |
+
result = sparse_decode_atten_func(
|
| 376 |
+
q,
|
| 377 |
+
k,
|
| 378 |
+
v,
|
| 379 |
+
self.q2k_indices,
|
| 380 |
+
page_table=self.page_table,
|
| 381 |
+
seqused_k=self.seqused_k,
|
| 382 |
+
seqlen_q=self.seqlen_q,
|
| 383 |
+
max_seqlen_k=self.max_seqlen_k,
|
| 384 |
+
blk_kv=self.blk_kv,
|
| 385 |
+
causal=self.causal,
|
| 386 |
+
softmax_scale=softmax_scale,
|
| 387 |
+
return_softmax_lse=return_softmax_lse,
|
| 388 |
+
)
|
| 389 |
+
if out is not None:
|
| 390 |
+
out.copy_(result)
|
| 391 |
+
return out
|
| 392 |
+
return result
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def sparse_atten_func(
|
| 396 |
+
q: torch.Tensor,
|
| 397 |
+
k: torch.Tensor,
|
| 398 |
+
v: torch.Tensor,
|
| 399 |
+
k2q_row_ptr: torch.Tensor,
|
| 400 |
+
k2q_q_indices: torch.Tensor,
|
| 401 |
+
topK: int,
|
| 402 |
+
*,
|
| 403 |
+
cu_seqlens_q: torch.Tensor,
|
| 404 |
+
cu_seqlens_k: torch.Tensor,
|
| 405 |
+
max_seqlen_q: int,
|
| 406 |
+
max_seqlen_k: int,
|
| 407 |
+
blk_kv: int = 128,
|
| 408 |
+
causal: bool = True,
|
| 409 |
+
softmax_scale: Optional[float] = None,
|
| 410 |
+
lse_temperature_scale: float = 1.0,
|
| 411 |
+
return_temperature_lse: bool = False,
|
| 412 |
+
partial_dtype: torch.dtype = torch.bfloat16,
|
| 413 |
+
return_softmax_lse: bool = False,
|
| 414 |
+
page_table: Optional[torch.Tensor] = None,
|
| 415 |
+
seqused_k: Optional[torch.Tensor] = None,
|
| 416 |
+
schedule: Optional[object] = None,
|
| 417 |
+
usable_SM_count: int = -1,
|
| 418 |
+
qk_dtype: Optional[torch.dtype] = None,
|
| 419 |
+
pv_dtype: Optional[torch.dtype] = None,
|
| 420 |
+
):
|
| 421 |
+
"""Run the official MiniMax CSR sparse prefill API on Blackwell.
|
| 422 |
+
|
| 423 |
+
The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list
|
| 424 |
+
consumed by the Blackwell Triton sparse prefill kernel. The default output
|
| 425 |
+
path is supported for BF16/FP16 tensors with head_dim=128.
|
| 426 |
+
"""
|
| 427 |
+
|
| 428 |
+
del partial_dtype, usable_SM_count, qk_dtype, pv_dtype
|
| 429 |
+
if schedule is not None:
|
| 430 |
+
raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper")
|
| 431 |
+
if return_temperature_lse or return_softmax_lse:
|
| 432 |
+
raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper")
|
| 433 |
+
if float(lse_temperature_scale) != 1.0:
|
| 434 |
+
raise ValueError("lse_temperature_scale values other than 1.0 are not supported")
|
| 435 |
+
if not causal:
|
| 436 |
+
raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True")
|
| 437 |
+
if q.ndim != 3:
|
| 438 |
+
raise ValueError("q must have shape [total_q, Hq, D]")
|
| 439 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 440 |
+
raise TypeError("q must be BF16 or FP16")
|
| 441 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128:
|
| 442 |
+
raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128")
|
| 443 |
+
if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32:
|
| 444 |
+
raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32")
|
| 445 |
+
if q.device != k.device or q.device != v.device:
|
| 446 |
+
raise ValueError("q, k, and v must be on the same device")
|
| 447 |
+
|
| 448 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 449 |
+
raise TypeError("q, k, and v must have matching BF16/FP16 dtype")
|
| 450 |
+
kv_heads = int(k.shape[1])
|
| 451 |
+
q2k_indices = _csr_to_q2k_indices(
|
| 452 |
+
k2q_row_ptr.contiguous(),
|
| 453 |
+
k2q_q_indices.contiguous(),
|
| 454 |
+
cu_seqlens_q.contiguous(),
|
| 455 |
+
cu_seqlens_k.contiguous(),
|
| 456 |
+
int(blk_kv),
|
| 457 |
+
int(topK),
|
| 458 |
+
)
|
| 459 |
+
k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata(
|
| 460 |
+
q,
|
| 461 |
+
k,
|
| 462 |
+
v,
|
| 463 |
+
cu_seqlens_q.contiguous(),
|
| 464 |
+
cu_seqlens_k.contiguous(),
|
| 465 |
+
int(max_seqlen_k),
|
| 466 |
+
int(blk_kv),
|
| 467 |
+
page_table=None if page_table is None else page_table.contiguous(),
|
| 468 |
+
seqused_k=None if seqused_k is None else seqused_k.contiguous(),
|
| 469 |
+
)
|
| 470 |
+
block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads)
|
| 471 |
+
return flash_prefill_with_gqa_share_sparse(
|
| 472 |
+
q.contiguous(),
|
| 473 |
+
k_cache,
|
| 474 |
+
v_cache,
|
| 475 |
+
None,
|
| 476 |
+
req_to_token,
|
| 477 |
+
slot_ids,
|
| 478 |
+
q2k_indices,
|
| 479 |
+
block_size_q=block_size_q,
|
| 480 |
+
block_size_k=int(blk_kv),
|
| 481 |
+
cu_seqlens=cu_seqlens_q.contiguous(),
|
| 482 |
+
seq_lens=seq_lens,
|
| 483 |
+
prefix_lens=prefix_lens,
|
| 484 |
+
max_seqlen_q=int(max_seqlen_q),
|
| 485 |
+
sm_scale=softmax_scale,
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def _nvfp4_dequant_kv(
|
| 490 |
+
packed: torch.Tensor,
|
| 491 |
+
scale_128x4: torch.Tensor,
|
| 492 |
+
global_scale: torch.Tensor,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
if packed.dtype is not torch.uint8:
|
| 495 |
+
return packed
|
| 496 |
+
if has_native_ops() and packed.is_cuda:
|
| 497 |
+
return native_nvfp4_dequant_swizzled_to_bf16(
|
| 498 |
+
packed,
|
| 499 |
+
scale_128x4,
|
| 500 |
+
global_scale,
|
| 501 |
+
)
|
| 502 |
+
original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
|
| 503 |
+
rows = 1
|
| 504 |
+
for dim in original_shape[:-1]:
|
| 505 |
+
rows *= int(dim)
|
| 506 |
+
qx = Nvfp4QuantizedTensor(
|
| 507 |
+
data=packed,
|
| 508 |
+
scale_128x4=scale_128x4,
|
| 509 |
+
global_scale=global_scale,
|
| 510 |
+
logical_scale_shape=(rows, int(original_shape[-1]) // 16),
|
| 511 |
+
original_shape=original_shape,
|
| 512 |
+
)
|
| 513 |
+
return dequantize_nvfp4_128x4_to_bf16(qx)
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def sparse_atten_nvfp4_kv_func(
|
| 517 |
+
q: torch.Tensor,
|
| 518 |
+
k: torch.Tensor,
|
| 519 |
+
v: torch.Tensor,
|
| 520 |
+
k_scale_128x4: torch.Tensor,
|
| 521 |
+
v_scale_128x4: torch.Tensor,
|
| 522 |
+
k_global_scale: torch.Tensor,
|
| 523 |
+
v_global_scale: torch.Tensor,
|
| 524 |
+
k2q_row_ptr: torch.Tensor,
|
| 525 |
+
k2q_q_indices: torch.Tensor,
|
| 526 |
+
topK: int,
|
| 527 |
+
**kwargs,
|
| 528 |
+
):
|
| 529 |
+
"""NVFP4-KV official prefill API wrapper.
|
| 530 |
+
|
| 531 |
+
The Blackwell compatibility path dequantizes NVFP4 K/V with the package's
|
| 532 |
+
128x4 metadata helper, then dispatches to `sparse_atten_func`.
|
| 533 |
+
"""
|
| 534 |
+
|
| 535 |
+
k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale)
|
| 536 |
+
v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale)
|
| 537 |
+
return sparse_atten_func(
|
| 538 |
+
q,
|
| 539 |
+
k_bf16,
|
| 540 |
+
v_bf16,
|
| 541 |
+
k2q_row_ptr,
|
| 542 |
+
k2q_q_indices,
|
| 543 |
+
topK,
|
| 544 |
+
**kwargs,
|
| 545 |
+
)
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor:
|
| 549 |
+
data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8)
|
| 550 |
+
lut = torch.tensor(
|
| 551 |
+
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
|
| 552 |
+
dtype=torch.float32,
|
| 553 |
+
device=data.device,
|
| 554 |
+
)
|
| 555 |
+
out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device)
|
| 556 |
+
out[..., 0::2] = lut[(data & 0x0F).long()]
|
| 557 |
+
out[..., 1::2] = lut[(data >> 4).long()]
|
| 558 |
+
return out
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
| 562 |
+
if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]):
|
| 563 |
+
return values * scale.to(torch.float32).repeat_interleave(16, dim=-1)
|
| 564 |
+
if scale.ndim == values.ndim - 1:
|
| 565 |
+
return values * scale.to(torch.float32).unsqueeze(-1)
|
| 566 |
+
return values
|
| 567 |
+
|
| 568 |
+
|
| 569 |
+
def fp4_indexer_block_scores(
|
| 570 |
+
q_fp4: torch.Tensor,
|
| 571 |
+
k_fp4: torch.Tensor,
|
| 572 |
+
q_scale: torch.Tensor,
|
| 573 |
+
k_scale: torch.Tensor,
|
| 574 |
+
cu_seqlens_q: torch.Tensor,
|
| 575 |
+
cu_seqlens_k: torch.Tensor,
|
| 576 |
+
cu_page_offsets: torch.Tensor,
|
| 577 |
+
*,
|
| 578 |
+
max_seqlen_q: int,
|
| 579 |
+
max_seqlen_k: int,
|
| 580 |
+
kv_indices: torch.Tensor,
|
| 581 |
+
fp4_format: str,
|
| 582 |
+
causal: bool = False,
|
| 583 |
+
qo_offset: Optional[torch.Tensor] = None,
|
| 584 |
+
scale_layout: str = "preordered_mma",
|
| 585 |
+
) -> torch.Tensor:
|
| 586 |
+
"""FP4 block-score API for Blackwell.
|
| 587 |
+
|
| 588 |
+
Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`.
|
| 589 |
+
Built artifacts route CUDA tensors through the native Blackwell block-score
|
| 590 |
+
kernel. Source-tree mode uses the reference loop below so the API remains
|
| 591 |
+
testable before the extension is built.
|
| 592 |
+
"""
|
| 593 |
+
|
| 594 |
+
del qo_offset
|
| 595 |
+
if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}:
|
| 596 |
+
raise ValueError(f"unsupported fp4_format: {fp4_format}")
|
| 597 |
+
if scale_layout not in {"public", "preordered_mma"}:
|
| 598 |
+
raise ValueError(f"unsupported scale_layout: {scale_layout}")
|
| 599 |
+
if q_fp4.ndim != 3:
|
| 600 |
+
raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]")
|
| 601 |
+
if k_fp4.ndim != 4:
|
| 602 |
+
raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]")
|
| 603 |
+
total_q, hq, _ = q_fp4.shape
|
| 604 |
+
num_pages, hkv, page_size, _ = k_fp4.shape
|
| 605 |
+
if int(page_size) != 128:
|
| 606 |
+
raise ValueError("fp4_indexer_block_scores currently expects 128-token pages")
|
| 607 |
+
if hq % hkv != 0:
|
| 608 |
+
raise ValueError("Hq must be divisible by Hkv")
|
| 609 |
+
max_blocks = (int(max_seqlen_k) + 127) // 128
|
| 610 |
+
scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device)
|
| 611 |
+
|
| 612 |
+
q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale)
|
| 613 |
+
k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale)
|
| 614 |
+
|
| 615 |
+
# Native block-max QK kernel for the cubic scoring loop when available.
|
| 616 |
+
# Same dequant numerics as the reference (bf16 compute); falls back to the
|
| 617 |
+
# Python reference below in source-tree mode.
|
| 618 |
+
if has_native_ops() and q_fp4.is_cuda:
|
| 619 |
+
batch = int(cu_seqlens_q.numel()) - 1
|
| 620 |
+
q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64)
|
| 621 |
+
batch_of_q = torch.repeat_interleave(
|
| 622 |
+
torch.arange(batch, device=q_fp4.device, dtype=torch.int32),
|
| 623 |
+
q_lens,
|
| 624 |
+
)
|
| 625 |
+
return native_indexer_block_scores(
|
| 626 |
+
q.to(torch.bfloat16),
|
| 627 |
+
k.to(torch.bfloat16),
|
| 628 |
+
batch_of_q,
|
| 629 |
+
cu_seqlens_q.to(torch.int32),
|
| 630 |
+
cu_seqlens_k.to(torch.int32),
|
| 631 |
+
cu_page_offsets.to(torch.int32),
|
| 632 |
+
kv_indices.to(torch.int32),
|
| 633 |
+
max_blocks=max_blocks,
|
| 634 |
+
causal=causal,
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
gqa = hq // hkv
|
| 638 |
+
batch = int(cu_seqlens_q.numel()) - 1
|
| 639 |
+
cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
|
| 640 |
+
cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist()
|
| 641 |
+
cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist()
|
| 642 |
+
kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist()
|
| 643 |
+
for b in range(batch):
|
| 644 |
+
q_start, q_end = int(cu_q[b]), int(cu_q[b + 1])
|
| 645 |
+
k_len = int(cu_k[b + 1]) - int(cu_k[b])
|
| 646 |
+
page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1])
|
| 647 |
+
for logical_block, page_pos in enumerate(range(page_start, page_end)):
|
| 648 |
+
if logical_block >= max_blocks:
|
| 649 |
+
break
|
| 650 |
+
physical_page = int(kv_idx_cpu[page_pos])
|
| 651 |
+
if physical_page < 0 or physical_page >= num_pages:
|
| 652 |
+
continue
|
| 653 |
+
valid_tokens = min(128, max(0, k_len - logical_block * 128))
|
| 654 |
+
if valid_tokens <= 0:
|
| 655 |
+
continue
|
| 656 |
+
for qi in range(q_start, q_end):
|
| 657 |
+
local_q = qi - q_start
|
| 658 |
+
if causal and logical_block * 128 > local_q:
|
| 659 |
+
continue
|
| 660 |
+
for qh in range(hq):
|
| 661 |
+
kh = qh // gqa
|
| 662 |
+
key_block = k[physical_page, kh, :valid_tokens]
|
| 663 |
+
if causal:
|
| 664 |
+
visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1))
|
| 665 |
+
if visible <= 0:
|
| 666 |
+
continue
|
| 667 |
+
key_block = key_block[:visible]
|
| 668 |
+
block_scores = torch.matmul(key_block, q[qi, qh])
|
| 669 |
+
scores[qh, logical_block, qi] = block_scores.max()
|
| 670 |
+
return scores
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
__all__ = [
|
| 674 |
+
"sparse_atten_func",
|
| 675 |
+
"sparse_atten_nvfp4_kv_func",
|
| 676 |
+
"sparse_decode_atten_func",
|
| 677 |
+
"SparseDecodePagedAttentionWrapper",
|
| 678 |
+
"fp4_indexer_block_scores",
|
| 679 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "minimaxai-msa-blackwell",
|
| 3 |
+
"id": "_minimaxai_msa_blackwell_cuda_25b0067",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"12.0"
|
| 11 |
+
]
|
| 12 |
+
},
|
| 13 |
+
"digest": {
|
| 14 |
+
"algorithm": "sha256",
|
| 15 |
+
"files": {
|
| 16 |
+
"__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=",
|
| 17 |
+
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 18 |
+
"_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "UoiDhzhz2aSCy+k+sOoOxb5/DqwAndixA6BgmEHAMtU=",
|
| 19 |
+
"_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=",
|
| 20 |
+
"_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=",
|
| 21 |
+
"api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=",
|
| 22 |
+
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 23 |
+
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 24 |
+
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 25 |
+
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 26 |
+
"decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=",
|
| 27 |
+
"interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=",
|
| 28 |
+
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 29 |
+
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 30 |
+
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 31 |
+
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
|
| 32 |
+
"prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=",
|
| 33 |
+
"prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=",
|
| 34 |
+
"quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
|
| 35 |
+
"sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
}
|
build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# VENDOR NOTE: rewritten to drop the `einops` dependency (the original used
|
| 4 |
+
# einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream
|
| 5 |
+
# `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced
|
| 6 |
+
# with native torch.einsum / torch.reshape so the standalone vendor needs only
|
| 7 |
+
# torch (+ triton for the kernels). Original einops expressions are kept in
|
| 8 |
+
# comments next to each replacement for auditability.
|
| 9 |
+
|
| 10 |
+
from typing import Optional
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def naive_flash_decode_with_topk_idx(
|
| 16 |
+
q: torch.Tensor, # [batch_size, num_heads, head_dim]
|
| 17 |
+
sink: Optional[torch.Tensor], # [num_heads, head_dim]
|
| 18 |
+
kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim]
|
| 19 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 20 |
+
max_seqlen: int,
|
| 21 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 22 |
+
block_size: int,
|
| 23 |
+
topk: int,
|
| 24 |
+
sm_scale: Optional[float] = None,
|
| 25 |
+
init_blocks: int = 0,
|
| 26 |
+
local_blocks: int = 0,
|
| 27 |
+
):
|
| 28 |
+
"""Joint reference: lightning-indexer block top-k + sparse attention.
|
| 29 |
+
|
| 30 |
+
This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense
|
| 31 |
+
5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by
|
| 32 |
+
`slot_ids`. It both (a) selects the top-k key blocks via per-block max score
|
| 33 |
+
(with optional forced init/local blocks) and (b) computes the resulting
|
| 34 |
+
sparse attention, returning (o, topk_idx).
|
| 35 |
+
"""
|
| 36 |
+
assert (
|
| 37 |
+
kv_cache.shape[2] % block_size == 0
|
| 38 |
+
), "max cache len must be divisible by block size"
|
| 39 |
+
if sm_scale is None:
|
| 40 |
+
sm_scale = q.shape[-1] ** -0.5
|
| 41 |
+
original_dtype = q.dtype
|
| 42 |
+
batch_size = q.shape[0]
|
| 43 |
+
num_q_heads = q.shape[1]
|
| 44 |
+
num_kv_heads = kv_cache.shape[3]
|
| 45 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 46 |
+
head_dim = q.shape[-1]
|
| 47 |
+
# rearrange q: "b (h g) d -> b h g d"
|
| 48 |
+
q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim)
|
| 49 |
+
kv_cache_float = kv_cache.float()
|
| 50 |
+
k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d]
|
| 51 |
+
v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d]
|
| 52 |
+
# einsum(q, k, "b h g d, b n h d -> b h g n")
|
| 53 |
+
qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale
|
| 54 |
+
mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None]
|
| 55 |
+
qk = qk.masked_fill(~mask[:, None, None, :], float("-inf"))
|
| 56 |
+
# get score
|
| 57 |
+
score = qk.clone().to(torch.float32)
|
| 58 |
+
# rearrange "b h g (n s) -> b (h g) n s", s=block_size
|
| 59 |
+
n_blocks = score.shape[-1] // block_size
|
| 60 |
+
score = score.reshape(
|
| 61 |
+
batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size
|
| 62 |
+
).reshape(batch_size, num_q_heads, n_blocks, block_size)
|
| 63 |
+
score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks]
|
| 64 |
+
# post-process score for init_blocks and local_blocks
|
| 65 |
+
INIT_SCORE = 1e30
|
| 66 |
+
LOCAL_SCORE = 1e29
|
| 67 |
+
if init_blocks > 0:
|
| 68 |
+
score[:, :, :init_blocks] = INIT_SCORE
|
| 69 |
+
if local_blocks > 0:
|
| 70 |
+
num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
|
| 71 |
+
for b in range(batch_size):
|
| 72 |
+
num_blks = num_blocks_per_batch[b].item()
|
| 73 |
+
local_start = max(0, num_blks - local_blocks)
|
| 74 |
+
score[b, :, local_start:num_blks] = LOCAL_SCORE
|
| 75 |
+
# compute topk indices per (batch, head)
|
| 76 |
+
# score shape: [batch_size, num_q_heads, num_blocks]
|
| 77 |
+
topk_idx = torch.full(
|
| 78 |
+
(num_q_heads, batch_size, topk),
|
| 79 |
+
fill_value=-1,
|
| 80 |
+
device=score.device,
|
| 81 |
+
dtype=torch.int32,
|
| 82 |
+
)
|
| 83 |
+
num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
|
| 84 |
+
for b in range(batch_size):
|
| 85 |
+
num_blks = num_blocks_per_batch[b].item()
|
| 86 |
+
actual_topk = min(topk, num_blks)
|
| 87 |
+
for h in range(num_q_heads):
|
| 88 |
+
# get topk indices for this (batch, head)
|
| 89 |
+
_, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1)
|
| 90 |
+
topk_idx[h, b, :actual_topk] = indices.to(torch.int32)
|
| 91 |
+
# compute attention output with sink
|
| 92 |
+
if sink is not None:
|
| 93 |
+
# sink: [num_q_heads, head_dim] -> [h, g, d]
|
| 94 |
+
sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim)
|
| 95 |
+
# einsum(q, sink, "b h g d, h g d -> b h g")
|
| 96 |
+
qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale
|
| 97 |
+
qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1]
|
| 98 |
+
attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32)
|
| 99 |
+
attn = attn[..., 1:] # remove sink score
|
| 100 |
+
else:
|
| 101 |
+
attn = qk.softmax(dim=-1, dtype=torch.float32)
|
| 102 |
+
# einsum(attn, v, "b h g n, b n h d -> b h g d")
|
| 103 |
+
o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel)
|
| 104 |
+
# rearrange "b h g d -> b (h g) d"
|
| 105 |
+
o = o.reshape(batch_size, num_q_heads, head_dim)
|
| 106 |
+
return o.to(original_dtype), topk_idx
|
build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def naive_flash_decode_with_gqa_share_sparse(
|
| 9 |
+
q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
|
| 10 |
+
sink: Optional[torch.Tensor], # [num_q_heads, head_dim]
|
| 11 |
+
kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim]
|
| 12 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 13 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 14 |
+
block_size: int,
|
| 15 |
+
topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
|
| 16 |
+
sm_scale: Optional[float] = None,
|
| 17 |
+
) -> torch.Tensor:
|
| 18 |
+
"""
|
| 19 |
+
Naive implementation of sparse attention with GQA group sharing.
|
| 20 |
+
Each GQA group (all q heads sharing the same kv head) uses the same topk_idx.
|
| 21 |
+
"""
|
| 22 |
+
if sm_scale is None:
|
| 23 |
+
sm_scale = q.shape[-1] ** -0.5
|
| 24 |
+
|
| 25 |
+
original_dtype = q.dtype
|
| 26 |
+
batch_size, num_q_heads, head_dim = q.shape
|
| 27 |
+
max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape
|
| 28 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 29 |
+
# Output tensor
|
| 30 |
+
o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device)
|
| 31 |
+
|
| 32 |
+
for b in range(batch_size):
|
| 33 |
+
seq_len = seq_lens[b].item()
|
| 34 |
+
sid = slot_ids[b].item() % max_slots
|
| 35 |
+
|
| 36 |
+
for kh in range(num_kv_heads):
|
| 37 |
+
# Get topk indices for this (kv_head, batch)
|
| 38 |
+
block_indices = topk_idx[kh, b, :].tolist()
|
| 39 |
+
|
| 40 |
+
# Collect selected K and V blocks
|
| 41 |
+
selected_k_blocks = []
|
| 42 |
+
selected_v_blocks = []
|
| 43 |
+
for block_idx in block_indices:
|
| 44 |
+
if block_idx < 0:
|
| 45 |
+
continue # Invalid index
|
| 46 |
+
start = block_idx * block_size
|
| 47 |
+
end = min(start + block_size, seq_len)
|
| 48 |
+
if start >= seq_len:
|
| 49 |
+
continue
|
| 50 |
+
# K: [block_len, head_dim]
|
| 51 |
+
k_block = kv_cache[sid, 0, start:end, kh, :]
|
| 52 |
+
# V: [block_len, head_dim]
|
| 53 |
+
v_block = kv_cache[sid, 1, start:end, kh, :]
|
| 54 |
+
selected_k_blocks.append(k_block)
|
| 55 |
+
selected_v_blocks.append(v_block)
|
| 56 |
+
|
| 57 |
+
if len(selected_k_blocks) == 0:
|
| 58 |
+
continue
|
| 59 |
+
|
| 60 |
+
# Concatenate selected blocks: [total_selected_len, head_dim]
|
| 61 |
+
k_selected = torch.cat(selected_k_blocks, dim=0)
|
| 62 |
+
v_selected = torch.cat(selected_v_blocks, dim=0)
|
| 63 |
+
|
| 64 |
+
# Compute attention for all q heads in this GQA group
|
| 65 |
+
for g in range(gqa_group_size):
|
| 66 |
+
qh = kh * gqa_group_size + g
|
| 67 |
+
# q_vec: [head_dim]
|
| 68 |
+
q_vec = q[b, qh, :].float()
|
| 69 |
+
|
| 70 |
+
# Compute attention scores: [total_selected_len]
|
| 71 |
+
scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale
|
| 72 |
+
|
| 73 |
+
# Add sink to softmax normalization if present
|
| 74 |
+
if sink is not None:
|
| 75 |
+
sink_vec = sink[qh, :].float()
|
| 76 |
+
qsink = torch.dot(q_vec, sink_vec) * sm_scale
|
| 77 |
+
# Concatenate sink score with regular scores for softmax
|
| 78 |
+
scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0)
|
| 79 |
+
attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1)
|
| 80 |
+
# Remove sink weight (only used for normalization)
|
| 81 |
+
attn_weights = attn_weights_with_sink[1:]
|
| 82 |
+
else:
|
| 83 |
+
attn_weights = torch.softmax(scores, dim=-1)
|
| 84 |
+
|
| 85 |
+
# Compute output: [head_dim]
|
| 86 |
+
o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected)
|
| 87 |
+
|
| 88 |
+
return o
|
build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
from .topk_sparse import flash_prefill_with_gqa_share_sparse
|
| 3 |
+
|
| 4 |
+
__all__ = ["flash_prefill_with_gqa_share_sparse"]
|
build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
|
| 10 |
+
|
| 11 |
+
from ..common.utils import get_cu_seqblocks, robust_allocator
|
| 12 |
+
|
| 13 |
+
_is_hip = is_hip()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@triton.heuristics(
|
| 17 |
+
{
|
| 18 |
+
"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]),
|
| 19 |
+
"BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]),
|
| 20 |
+
"BLOCK_SIZE_H": lambda args: triton.next_power_of_2(
|
| 21 |
+
max(
|
| 22 |
+
16 // args["BLOCK_SIZE_Q"],
|
| 23 |
+
triton.next_power_of_2(args["gqa_group_size"]),
|
| 24 |
+
)
|
| 25 |
+
),
|
| 26 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
| 27 |
+
"BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"],
|
| 28 |
+
"HAS_SINK": lambda args: args["sink_ptr"] is not None,
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
@triton.autotune(
|
| 32 |
+
# Configs that fail to compile on the target arch are skipped, so widening
|
| 33 |
+
# the num_warps x num_stages grid only adds candidates, never a bad kernel.
|
| 34 |
+
configs=[
|
| 35 |
+
triton.Config({}, num_warps=nw, num_stages=ns)
|
| 36 |
+
for nw in (2, 4, 8)
|
| 37 |
+
for ns in (2, 3, 4)
|
| 38 |
+
],
|
| 39 |
+
key=[
|
| 40 |
+
"BLOCK_SIZE_Q",
|
| 41 |
+
"BLOCK_SIZE_K",
|
| 42 |
+
"qk_head_dim",
|
| 43 |
+
"v_head_dim",
|
| 44 |
+
"gqa_group_size",
|
| 45 |
+
],
|
| 46 |
+
)
|
| 47 |
+
@triton.jit
|
| 48 |
+
def _gqa_share_sparse_fwd_kernel(
|
| 49 |
+
q_ptr, # Q: n x h x d
|
| 50 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 51 |
+
v_cache_ptr, # V paged: max_slots x kh x d
|
| 52 |
+
sink_ptr, # Sink: h x d
|
| 53 |
+
t_ptr, # topk_idx: kh x n x k
|
| 54 |
+
o_ptr, # O: n x h x d
|
| 55 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 56 |
+
# seqlens
|
| 57 |
+
cu_seqlens_q,
|
| 58 |
+
cu_seqblocks_q,
|
| 59 |
+
seq_lens,
|
| 60 |
+
prefix_lens,
|
| 61 |
+
slot_ids,
|
| 62 |
+
# shape
|
| 63 |
+
max_slots,
|
| 64 |
+
num_kv_heads,
|
| 65 |
+
gqa_group_size,
|
| 66 |
+
qk_head_dim,
|
| 67 |
+
v_head_dim,
|
| 68 |
+
max_topk,
|
| 69 |
+
# q loop num
|
| 70 |
+
num_q_loop,
|
| 71 |
+
# sm_scale
|
| 72 |
+
sm_scale,
|
| 73 |
+
# stride
|
| 74 |
+
stride_qn,
|
| 75 |
+
stride_qh,
|
| 76 |
+
stride_qd,
|
| 77 |
+
stride_ks,
|
| 78 |
+
stride_kh,
|
| 79 |
+
stride_kd,
|
| 80 |
+
stride_vs,
|
| 81 |
+
stride_vh,
|
| 82 |
+
stride_vd,
|
| 83 |
+
stride_sh,
|
| 84 |
+
stride_sd,
|
| 85 |
+
stride_th,
|
| 86 |
+
stride_tn,
|
| 87 |
+
stride_tk,
|
| 88 |
+
stride_on,
|
| 89 |
+
stride_oh,
|
| 90 |
+
stride_od,
|
| 91 |
+
stride_r2t_b,
|
| 92 |
+
# META parameters
|
| 93 |
+
BLOCK_SIZE_Q: tl.constexpr, # q block size
|
| 94 |
+
BLOCK_SIZE_K: tl.constexpr, # k block size
|
| 95 |
+
BLOCK_SIZE_KD: tl.constexpr,
|
| 96 |
+
BLOCK_SIZE_VD: tl.constexpr,
|
| 97 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 98 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 99 |
+
BLOCK_SIZE_QH: tl.constexpr,
|
| 100 |
+
# has sink
|
| 101 |
+
HAS_SINK: tl.constexpr,
|
| 102 |
+
USE_TMA: tl.constexpr,
|
| 103 |
+
IS_FP8: tl.constexpr,
|
| 104 |
+
):
|
| 105 |
+
sm_scale_log2e = sm_scale * 1.4426950409
|
| 106 |
+
# get batch id and head id
|
| 107 |
+
pid_q = tl.program_id(0)
|
| 108 |
+
pid_kh = tl.program_id(1)
|
| 109 |
+
pid_b = tl.program_id(2)
|
| 110 |
+
pid_h = pid_kh * gqa_group_size
|
| 111 |
+
# get q k start and len after rmpad
|
| 112 |
+
q_start = tl.load(cu_seqlens_q + pid_b)
|
| 113 |
+
q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
|
| 114 |
+
q_block_start = tl.load(cu_seqblocks_q + pid_b)
|
| 115 |
+
q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start
|
| 116 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 117 |
+
prefix_len = tl.load(prefix_lens + pid_b)
|
| 118 |
+
sid = (
|
| 119 |
+
tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
|
| 120 |
+
) % max_slots # safety against negative
|
| 121 |
+
if pid_q * num_q_loop >= q_block_len:
|
| 122 |
+
return
|
| 123 |
+
real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop)
|
| 124 |
+
if HAS_SINK:
|
| 125 |
+
sink_ptrs = tl.make_block_ptr(
|
| 126 |
+
base=sink_ptr + pid_h * stride_sh,
|
| 127 |
+
shape=(gqa_group_size, qk_head_dim),
|
| 128 |
+
strides=(stride_sh, stride_sd),
|
| 129 |
+
offsets=(0, 0),
|
| 130 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD),
|
| 131 |
+
order=(1, 0),
|
| 132 |
+
)
|
| 133 |
+
sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
|
| 134 |
+
tl.float32
|
| 135 |
+
)
|
| 136 |
+
# offsets for paged K/V load
|
| 137 |
+
off_n = tl.arange(0, BLOCK_SIZE_K)
|
| 138 |
+
off_kd = tl.arange(0, BLOCK_SIZE_KD)
|
| 139 |
+
off_vd = tl.arange(0, BLOCK_SIZE_VD)
|
| 140 |
+
kd_mask = off_kd < qk_head_dim
|
| 141 |
+
vd_mask = off_vd < v_head_dim
|
| 142 |
+
for j in range(real_q_loop):
|
| 143 |
+
pid_q_j = pid_q * num_q_loop + j
|
| 144 |
+
# init topk idx pointer
|
| 145 |
+
t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
|
| 146 |
+
# we assume that the topk_idx is right padded with -1
|
| 147 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 148 |
+
topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
|
| 149 |
+
valid_idx = tl.where(topk_idx >= 0, off_t, -1)
|
| 150 |
+
real_topk = tl.sum(valid_idx != -1, axis=0)
|
| 151 |
+
# init qkv pointer
|
| 152 |
+
q_ptrs = tl.make_block_ptr(
|
| 153 |
+
base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
|
| 154 |
+
shape=(q_len, gqa_group_size, qk_head_dim),
|
| 155 |
+
strides=(stride_qn, stride_qh, stride_qd),
|
| 156 |
+
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
|
| 157 |
+
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD),
|
| 158 |
+
order=(2, 1, 0),
|
| 159 |
+
)
|
| 160 |
+
# load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D]
|
| 161 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero")
|
| 162 |
+
# init statistics
|
| 163 |
+
off_q_k = (
|
| 164 |
+
tl.arange(0, BLOCK_SIZE_Q)[:, None]
|
| 165 |
+
+ pid_q_j * BLOCK_SIZE_Q
|
| 166 |
+
+ prefix_len
|
| 167 |
+
- tl.arange(0, BLOCK_SIZE_K)[None, :]
|
| 168 |
+
)
|
| 169 |
+
if HAS_SINK:
|
| 170 |
+
m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
|
| 171 |
+
lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
|
| 172 |
+
qsink = (
|
| 173 |
+
tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e
|
| 174 |
+
) # (BLOCK_SIZE_Q, BLOCK_SIZE_H)
|
| 175 |
+
m_i += qsink
|
| 176 |
+
lse_i += qsink
|
| 177 |
+
m_i = tl.reshape(m_i, BLOCK_SIZE_QH)
|
| 178 |
+
lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH)
|
| 179 |
+
else:
|
| 180 |
+
m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
|
| 181 |
+
lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
|
| 182 |
+
acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32)
|
| 183 |
+
q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD)
|
| 184 |
+
# sparse attention
|
| 185 |
+
for i in range(real_topk):
|
| 186 |
+
# get current block start index (absolute K position)
|
| 187 |
+
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
|
| 188 |
+
t_ptr_j = t_ptr_j + stride_tk
|
| 189 |
+
# paged load K via req_to_token: pos -> slot -> k_cache
|
| 190 |
+
pos = c + off_n
|
| 191 |
+
pos_mask = pos < seq_len
|
| 192 |
+
slots = tl.load(
|
| 193 |
+
req_to_token_ptr + sid * stride_r2t_b + pos,
|
| 194 |
+
mask=pos_mask,
|
| 195 |
+
other=0,
|
| 196 |
+
).to(tl.int64)
|
| 197 |
+
slots = (slots + max_slots) % max_slots # safety against negative
|
| 198 |
+
# k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
|
| 199 |
+
k = tl.load(
|
| 200 |
+
k_cache_ptr
|
| 201 |
+
+ slots[None, :] * stride_ks
|
| 202 |
+
+ pid_kh * stride_kh
|
| 203 |
+
+ off_kd[:, None] * stride_kd,
|
| 204 |
+
mask=kd_mask[:, None] & pos_mask[None, :],
|
| 205 |
+
other=0.0,
|
| 206 |
+
)
|
| 207 |
+
if IS_FP8:
|
| 208 |
+
# fp8 main K cache is unit-scaled; widen to the Q compute dtype
|
| 209 |
+
# before the tl.dot (compiled out when the cache is bf16).
|
| 210 |
+
k = k.to(q.dtype)
|
| 211 |
+
# compute qk
|
| 212 |
+
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
|
| 213 |
+
# causal mask
|
| 214 |
+
qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
|
| 215 |
+
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
|
| 216 |
+
# [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
|
| 217 |
+
# -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
|
| 218 |
+
qk += tl.dot(q, k) * sm_scale_log2e
|
| 219 |
+
# K boundary mask: positions beyond seq_len contribute -inf
|
| 220 |
+
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
| 221 |
+
# compute m_ij and l_ij
|
| 222 |
+
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
| 223 |
+
p = tl.exp2(qk - m_ij[:, None])
|
| 224 |
+
l_ij = tl.sum(p, axis=1)
|
| 225 |
+
# scale acc_o
|
| 226 |
+
acc_o_scale = tl.exp2(m_i - m_ij)
|
| 227 |
+
acc_o = acc_o * acc_o_scale[:, None]
|
| 228 |
+
# paged load V
|
| 229 |
+
v = tl.load(
|
| 230 |
+
v_cache_ptr
|
| 231 |
+
+ slots[:, None] * stride_vs
|
| 232 |
+
+ pid_kh * stride_vh
|
| 233 |
+
+ off_vd[None, :] * stride_vd,
|
| 234 |
+
mask=pos_mask[:, None] & vd_mask[None, :],
|
| 235 |
+
other=0.0,
|
| 236 |
+
)
|
| 237 |
+
if IS_FP8:
|
| 238 |
+
# Widen V so `p.to(v.dtype)` casts P to the compute dtype rather
|
| 239 |
+
# than to fp8 (which would wreck attention-weight precision).
|
| 240 |
+
v = v.to(q.dtype)
|
| 241 |
+
p = p.to(v.dtype)
|
| 242 |
+
acc_o += tl.dot(p, v)
|
| 243 |
+
# update statistics
|
| 244 |
+
m_i = m_ij
|
| 245 |
+
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
| 246 |
+
# final scale
|
| 247 |
+
acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
|
| 248 |
+
# save output
|
| 249 |
+
acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD)
|
| 250 |
+
o_ptrs = tl.make_block_ptr(
|
| 251 |
+
base=o_ptr + q_start * stride_on + pid_h * stride_oh,
|
| 252 |
+
shape=(q_len, gqa_group_size, v_head_dim),
|
| 253 |
+
strides=(stride_on, stride_oh, stride_od),
|
| 254 |
+
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
|
| 255 |
+
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD),
|
| 256 |
+
order=(2, 1, 0),
|
| 257 |
+
)
|
| 258 |
+
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2))
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
@torch.no_grad()
|
| 262 |
+
def flash_prefill_with_gqa_share_sparse(
|
| 263 |
+
q: torch.Tensor,
|
| 264 |
+
k_cache: torch.Tensor,
|
| 265 |
+
v_cache: torch.Tensor,
|
| 266 |
+
sink: Optional[torch.Tensor],
|
| 267 |
+
req_to_token: torch.Tensor,
|
| 268 |
+
slot_ids: torch.Tensor,
|
| 269 |
+
topk_idx: torch.Tensor,
|
| 270 |
+
block_size_q: int,
|
| 271 |
+
block_size_k: int,
|
| 272 |
+
cu_seqlens: torch.Tensor,
|
| 273 |
+
seq_lens: torch.Tensor,
|
| 274 |
+
prefix_lens: torch.Tensor,
|
| 275 |
+
max_seqlen_q: int,
|
| 276 |
+
sm_scale: Optional[float] = None,
|
| 277 |
+
use_tma: bool = True,
|
| 278 |
+
cu_seqblocks_q: Optional[torch.Tensor] = None,
|
| 279 |
+
max_seqblock_q: Optional[int] = None,
|
| 280 |
+
) -> torch.Tensor:
|
| 281 |
+
triton.set_allocator(robust_allocator)
|
| 282 |
+
# dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled)
|
| 283 |
+
# under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load.
|
| 284 |
+
# Keep CUDA's pre-existing sparse prefill dtype contract unchanged by
|
| 285 |
+
# accepting fp8 only on HIP.
|
| 286 |
+
assert q.dtype in (torch.bfloat16, torch.float16)
|
| 287 |
+
_FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
|
| 288 |
+
is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
|
| 289 |
+
assert k_cache.dtype == q.dtype or is_fp8, (
|
| 290 |
+
f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) "
|
| 291 |
+
f"or fp8 on HIP, got {k_cache.dtype}"
|
| 292 |
+
)
|
| 293 |
+
assert v_cache.dtype == k_cache.dtype
|
| 294 |
+
assert block_size_q in {1, 2, 4, 8, 16, 32, 64}
|
| 295 |
+
assert block_size_k in {16, 32, 64, 128}
|
| 296 |
+
# shape
|
| 297 |
+
total_q, num_q_heads, qk_head_dim = q.shape
|
| 298 |
+
max_slots, num_k_heads, _ = k_cache.shape
|
| 299 |
+
_, num_v_heads, v_head_dim = v_cache.shape
|
| 300 |
+
batch_size = cu_seqlens.shape[0] - 1
|
| 301 |
+
topk = topk_idx.shape[-1]
|
| 302 |
+
assert topk_idx.shape[0] == num_k_heads
|
| 303 |
+
# gqa
|
| 304 |
+
assert num_k_heads == num_v_heads
|
| 305 |
+
assert num_q_heads % num_k_heads == 0
|
| 306 |
+
gqa_group_size = num_q_heads // num_k_heads
|
| 307 |
+
assert gqa_group_size * block_size_q <= 128
|
| 308 |
+
if sm_scale is None:
|
| 309 |
+
sm_scale = qk_head_dim**-0.5
|
| 310 |
+
if cu_seqblocks_q is None or max_seqblock_q is None:
|
| 311 |
+
cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks(
|
| 312 |
+
cu_seqlens, max_seqlen_q, block_size_q, block_size_k
|
| 313 |
+
)
|
| 314 |
+
# output tensor
|
| 315 |
+
o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype)
|
| 316 |
+
# launch kernel
|
| 317 |
+
num_q_loop = (
|
| 318 |
+
max_seqblock_q // 131072 + 1
|
| 319 |
+
) # calculate multiple queries in one kernel if seqlence length is too long
|
| 320 |
+
BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q)
|
| 321 |
+
BLOCK_SIZE_K = triton.next_power_of_2(block_size_k)
|
| 322 |
+
grid = (
|
| 323 |
+
triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop),
|
| 324 |
+
num_k_heads,
|
| 325 |
+
batch_size,
|
| 326 |
+
)
|
| 327 |
+
_gqa_share_sparse_fwd_kernel[grid](
|
| 328 |
+
q,
|
| 329 |
+
k_cache,
|
| 330 |
+
v_cache,
|
| 331 |
+
sink,
|
| 332 |
+
topk_idx,
|
| 333 |
+
o,
|
| 334 |
+
req_to_token,
|
| 335 |
+
cu_seqlens,
|
| 336 |
+
cu_seqblocks_q,
|
| 337 |
+
seq_lens,
|
| 338 |
+
prefix_lens,
|
| 339 |
+
slot_ids,
|
| 340 |
+
max_slots,
|
| 341 |
+
num_k_heads,
|
| 342 |
+
gqa_group_size,
|
| 343 |
+
qk_head_dim,
|
| 344 |
+
v_head_dim,
|
| 345 |
+
topk,
|
| 346 |
+
num_q_loop,
|
| 347 |
+
sm_scale,
|
| 348 |
+
q.stride(0),
|
| 349 |
+
q.stride(1),
|
| 350 |
+
q.stride(2),
|
| 351 |
+
k_cache.stride(0),
|
| 352 |
+
k_cache.stride(1),
|
| 353 |
+
k_cache.stride(2),
|
| 354 |
+
v_cache.stride(0),
|
| 355 |
+
v_cache.stride(1),
|
| 356 |
+
v_cache.stride(2),
|
| 357 |
+
sink.stride(0) if sink is not None else 0,
|
| 358 |
+
sink.stride(1) if sink is not None else 0,
|
| 359 |
+
topk_idx.stride(0),
|
| 360 |
+
topk_idx.stride(1),
|
| 361 |
+
topk_idx.stride(2),
|
| 362 |
+
o.stride(0),
|
| 363 |
+
o.stride(1),
|
| 364 |
+
o.stride(2),
|
| 365 |
+
req_to_token.stride(0),
|
| 366 |
+
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
|
| 367 |
+
BLOCK_SIZE_K=BLOCK_SIZE_K,
|
| 368 |
+
USE_TMA=use_tma,
|
| 369 |
+
IS_FP8=is_fp8,
|
| 370 |
+
)
|
| 371 |
+
return o
|
build/torch211-cxx11-cu128-x86_64-linux/quantize.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""NVFP4 helper API compatible with MiniMaxAI/msa.
|
| 3 |
+
|
| 4 |
+
The quantization entrypoints use Transformer Engine when it is installed. The
|
| 5 |
+
layout helpers and reference dequantizer are pure PyTorch and are included so
|
| 6 |
+
callers can prepare and validate NVFP4 metadata with the same public API names
|
| 7 |
+
as the upstream MiniMaxAI package.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Tuple
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
NVFP4_BLOCK_SIZE = 16
|
| 19 |
+
NVFP4_FP4_MAX = 6.0
|
| 20 |
+
NVFP4_FP8_E4M3_MAX = 448.0
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class Nvfp4QuantizedTensor:
|
| 25 |
+
data: torch.Tensor
|
| 26 |
+
scale_128x4: torch.Tensor
|
| 27 |
+
global_scale: torch.Tensor
|
| 28 |
+
logical_scale_shape: Tuple[int, int]
|
| 29 |
+
original_shape: Tuple[int, ...]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _round_up(x: int, multiple: int) -> int:
|
| 33 |
+
return ((int(x) + multiple - 1) // multiple) * multiple
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def nvfp4_scale_128x4_offset(
|
| 37 |
+
row: torch.Tensor,
|
| 38 |
+
col: torch.Tensor,
|
| 39 |
+
scale_cols: int,
|
| 40 |
+
) -> torch.Tensor:
|
| 41 |
+
tiles_n = _round_up(scale_cols, 4) // 4
|
| 42 |
+
tile_m = row // 128
|
| 43 |
+
tile_n = col // 4
|
| 44 |
+
outer = row % 128
|
| 45 |
+
inner = col % 4
|
| 46 |
+
return (
|
| 47 |
+
(tile_m * tiles_n + tile_n) * 512
|
| 48 |
+
+ (outer % 32) * 16
|
| 49 |
+
+ (outer // 32) * 4
|
| 50 |
+
+ inner
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def swizzle_nvfp4_scale_to_128x4(
|
| 55 |
+
scale: torch.Tensor,
|
| 56 |
+
*,
|
| 57 |
+
rows: int,
|
| 58 |
+
cols: int,
|
| 59 |
+
) -> torch.Tensor:
|
| 60 |
+
if scale.ndim != 2:
|
| 61 |
+
raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
|
| 62 |
+
|
| 63 |
+
rows = int(rows)
|
| 64 |
+
cols = int(cols)
|
| 65 |
+
padded_rows = _round_up(rows, 128)
|
| 66 |
+
padded_cols = _round_up(cols, 4)
|
| 67 |
+
if scale.shape[0] < rows or scale.shape[1] < cols:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
"scale is smaller than the requested logical shape: "
|
| 70 |
+
f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
logical = scale[:rows, :cols].contiguous()
|
| 74 |
+
if logical.shape != (padded_rows, padded_cols):
|
| 75 |
+
logical = torch.nn.functional.pad(
|
| 76 |
+
logical.to(torch.float32),
|
| 77 |
+
(0, padded_cols - cols, 0, padded_rows - rows),
|
| 78 |
+
).to(scale.dtype)
|
| 79 |
+
swizzled = torch.empty_like(logical)
|
| 80 |
+
|
| 81 |
+
row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
|
| 82 |
+
col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
|
| 83 |
+
offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
|
| 84 |
+
swizzled.reshape(-1)[offset] = logical.reshape(-1)
|
| 85 |
+
return swizzled
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _import_te_nvfp4_quantizer():
|
| 93 |
+
try:
|
| 94 |
+
from transformer_engine.pytorch.tensor import NVFP4Quantizer
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
"Transformer Engine NVFP4 quantization is unavailable. Install a "
|
| 98 |
+
"Transformer Engine build with NVFP4 PyTorch support to use "
|
| 99 |
+
"quantize_bf16_to_nvfp4_128x4."
|
| 100 |
+
) from exc
|
| 101 |
+
return NVFP4Quantizer
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
|
| 105 |
+
if not x.is_cuda:
|
| 106 |
+
raise ValueError("NVFP4 quantization requires a CUDA tensor")
|
| 107 |
+
if x.dtype not in (torch.bfloat16, torch.float16):
|
| 108 |
+
raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
|
| 109 |
+
if x.ndim < 2:
|
| 110 |
+
raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
|
| 111 |
+
if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
rows = 1
|
| 117 |
+
for dim in x.shape[:-1]:
|
| 118 |
+
rows *= int(dim)
|
| 119 |
+
if rows % NVFP4_BLOCK_SIZE != 0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"flattened row dimension must be divisible by "
|
| 122 |
+
f"{NVFP4_BLOCK_SIZE}, got {rows}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
NVFP4Quantizer = _import_te_nvfp4_quantizer()
|
| 126 |
+
quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
|
| 127 |
+
qx = quantizer.quantize(x.contiguous())
|
| 128 |
+
meta = qx.get_metadata()
|
| 129 |
+
|
| 130 |
+
data = meta["rowwise_data"]
|
| 131 |
+
if data.dtype is not torch.uint8:
|
| 132 |
+
data = data.view(torch.uint8)
|
| 133 |
+
logical_scale = meta["rowwise_scale_inv"]
|
| 134 |
+
amax = meta["amax_rowwise"]
|
| 135 |
+
scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
|
| 136 |
+
scale_128x4 = swizzle_nvfp4_scale_to_128x4(
|
| 137 |
+
logical_scale,
|
| 138 |
+
rows=rows,
|
| 139 |
+
cols=scale_cols,
|
| 140 |
+
)
|
| 141 |
+
global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
|
| 142 |
+
|
| 143 |
+
return Nvfp4QuantizedTensor(
|
| 144 |
+
data=data,
|
| 145 |
+
scale_128x4=scale_128x4,
|
| 146 |
+
global_scale=global_scale,
|
| 147 |
+
logical_scale_shape=(rows, scale_cols),
|
| 148 |
+
original_shape=tuple(int(v) for v in x.shape),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def quantize_kv_bf16_to_nvfp4_128x4(
|
| 153 |
+
k: torch.Tensor,
|
| 154 |
+
v: torch.Tensor,
|
| 155 |
+
) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
|
| 156 |
+
return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def dequantize_nvfp4_128x4_to_bf16(
|
| 160 |
+
qx: Nvfp4QuantizedTensor,
|
| 161 |
+
*,
|
| 162 |
+
include_global_scale: bool = True,
|
| 163 |
+
) -> torch.Tensor:
|
| 164 |
+
data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
|
| 165 |
+
if data.shape[-1] * 2 != qx.original_shape[-1]:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"packed data last dimension does not match original shape: "
|
| 168 |
+
f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
rows, scale_cols = qx.logical_scale_shape
|
| 172 |
+
logical_dim = int(qx.original_shape[-1])
|
| 173 |
+
if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
"logical scale columns do not match original last dimension: "
|
| 176 |
+
f"{scale_cols} scale cols vs dim {logical_dim}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
fp4_lut = torch.tensor(
|
| 180 |
+
[
|
| 181 |
+
0.0,
|
| 182 |
+
0.5,
|
| 183 |
+
1.0,
|
| 184 |
+
1.5,
|
| 185 |
+
2.0,
|
| 186 |
+
3.0,
|
| 187 |
+
4.0,
|
| 188 |
+
6.0,
|
| 189 |
+
-0.0,
|
| 190 |
+
-0.5,
|
| 191 |
+
-1.0,
|
| 192 |
+
-1.5,
|
| 193 |
+
-2.0,
|
| 194 |
+
-3.0,
|
| 195 |
+
-4.0,
|
| 196 |
+
-6.0,
|
| 197 |
+
],
|
| 198 |
+
dtype=torch.float32,
|
| 199 |
+
device=data.device,
|
| 200 |
+
)
|
| 201 |
+
packed = data.reshape(rows, logical_dim // 2)
|
| 202 |
+
lo = packed & 0x0F
|
| 203 |
+
hi = packed >> 4
|
| 204 |
+
values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
|
| 205 |
+
values[:, 0::2] = fp4_lut[lo.long()]
|
| 206 |
+
values[:, 1::2] = fp4_lut[hi.long()]
|
| 207 |
+
|
| 208 |
+
row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
|
| 209 |
+
col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
|
| 210 |
+
offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
|
| 211 |
+
scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
|
| 212 |
+
scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
|
| 213 |
+
scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
|
| 214 |
+
out = values * scale
|
| 215 |
+
if include_global_scale:
|
| 216 |
+
out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
|
| 217 |
+
return out.reshape(qx.original_shape).to(torch.bfloat16)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
__all__ = [
|
| 221 |
+
"Nvfp4QuantizedTensor",
|
| 222 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 223 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 224 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 225 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 226 |
+
"nvfp4_global_scale_from_amax",
|
| 227 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Sparse-index helpers compatible with MiniMaxAI/msa public names."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
|
| 12 |
+
if cu_seqlens.dtype != torch.int32:
|
| 13 |
+
raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
|
| 14 |
+
if cu_seqlens.ndim != 1:
|
| 15 |
+
raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
|
| 16 |
+
if cu_seqlens.numel() < 1:
|
| 17 |
+
raise ValueError(f"{name} must have at least one element")
|
| 18 |
+
if not cu_seqlens.is_contiguous():
|
| 19 |
+
raise ValueError(f"{name} must be contiguous")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
|
| 23 |
+
seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
|
| 24 |
+
return (seqlens_k + kv_block_size - 1) // kv_block_size
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
|
| 28 |
+
rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
|
| 29 |
+
batch = len(rows_per_batch_cpu)
|
| 30 |
+
max_rows = max(rows_per_batch_cpu, default=0)
|
| 31 |
+
row_dtype = (
|
| 32 |
+
torch.int32
|
| 33 |
+
if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
|
| 34 |
+
else torch.int64
|
| 35 |
+
)
|
| 36 |
+
row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
|
| 37 |
+
row_linear = 0
|
| 38 |
+
for kv_block_idx in range(max_rows):
|
| 39 |
+
for batch_idx, row_count in enumerate(rows_per_batch_cpu):
|
| 40 |
+
if kv_block_idx < row_count:
|
| 41 |
+
row_map_cpu[batch_idx, kv_block_idx] = row_linear
|
| 42 |
+
row_linear += 1
|
| 43 |
+
return row_map_cpu.to(rows_per_batch.device), row_linear
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_k2q_csr_torch_reference(
|
| 47 |
+
q2k_indices: torch.Tensor,
|
| 48 |
+
cu_seqlens_q: torch.Tensor,
|
| 49 |
+
cu_seqlens_k: torch.Tensor,
|
| 50 |
+
kv_block_size: int,
|
| 51 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 52 |
+
if kv_block_size <= 0:
|
| 53 |
+
raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
|
| 54 |
+
if q2k_indices.dtype != torch.int32:
|
| 55 |
+
raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
|
| 56 |
+
if q2k_indices.ndim != 3:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
"q2k_indices must have shape [head_kv, total_q, topK], "
|
| 59 |
+
f"got {tuple(q2k_indices.shape)}"
|
| 60 |
+
)
|
| 61 |
+
_validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
|
| 62 |
+
_validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
|
| 63 |
+
if cu_seqlens_q.shape != cu_seqlens_k.shape:
|
| 64 |
+
raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
|
| 65 |
+
if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
|
| 66 |
+
raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
|
| 67 |
+
|
| 68 |
+
head_kv, total_q, topk = q2k_indices.shape
|
| 69 |
+
if total_q != int(cu_seqlens_q[-1].item()):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
|
| 72 |
+
f"({int(cu_seqlens_q[-1].item())})"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
|
| 76 |
+
row_map, total_rows = _build_packed_row_map(rows_per_batch)
|
| 77 |
+
nnz_upper_bound = total_q * topk
|
| 78 |
+
|
| 79 |
+
k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
|
| 80 |
+
k2q_q_indices = torch.full(
|
| 81 |
+
(head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
|
| 82 |
+
)
|
| 83 |
+
if total_rows == 0 or total_q == 0 or topk == 0:
|
| 84 |
+
return k2q_row_ptr, k2q_q_indices
|
| 85 |
+
|
| 86 |
+
counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
|
| 87 |
+
total_entries = total_q * topk
|
| 88 |
+
row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
|
| 89 |
+
row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
|
| 90 |
+
q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
|
| 91 |
+
valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
|
| 92 |
+
|
| 93 |
+
for batch_idx in range(cu_seqlens_q.numel() - 1):
|
| 94 |
+
q_start = int(cu_seqlens_q[batch_idx].item())
|
| 95 |
+
q_end = int(cu_seqlens_q[batch_idx + 1].item())
|
| 96 |
+
if q_end <= q_start:
|
| 97 |
+
continue
|
| 98 |
+
q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
|
| 99 |
+
q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
|
| 100 |
+
kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
|
| 101 |
+
valid = kv_local >= 0
|
| 102 |
+
safe_kv = kv_local.clamp_min(0).long()
|
| 103 |
+
rows = row_map[batch_idx, safe_kv].to(row_dtype)
|
| 104 |
+
start = q_start * topk
|
| 105 |
+
end = q_end * topk
|
| 106 |
+
row_all[:, start:end] = rows
|
| 107 |
+
q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
|
| 108 |
+
valid_all[:, start:end] = valid & (rows >= 0)
|
| 109 |
+
counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
|
| 110 |
+
|
| 111 |
+
k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
|
| 112 |
+
offsets = torch.zeros_like(counts)
|
| 113 |
+
offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
|
| 114 |
+
cursor = torch.zeros_like(counts)
|
| 115 |
+
for h in range(head_kv):
|
| 116 |
+
for entry in range(total_entries):
|
| 117 |
+
if not bool(valid_all[h, entry]):
|
| 118 |
+
continue
|
| 119 |
+
row = int(row_all[h, entry].item())
|
| 120 |
+
pos = int(offsets[h, row].item() + cursor[h, row].item())
|
| 121 |
+
k2q_q_indices[h, pos] = q_all[h, entry]
|
| 122 |
+
cursor[h, row] += 1
|
| 123 |
+
|
| 124 |
+
return k2q_row_ptr, k2q_q_indices
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def build_k2q_csr(
|
| 128 |
+
q2k_indices: torch.Tensor,
|
| 129 |
+
cu_seqlens_q: torch.Tensor,
|
| 130 |
+
cu_seqlens_k: torch.Tensor,
|
| 131 |
+
kv_block_size: int,
|
| 132 |
+
*,
|
| 133 |
+
total_k: Optional[int] = None,
|
| 134 |
+
max_seqlen_k: Optional[int] = None,
|
| 135 |
+
max_seqlen_q: Optional[int] = None,
|
| 136 |
+
total_rows: Optional[int] = None,
|
| 137 |
+
qhead_per_kv: int = 1,
|
| 138 |
+
return_schedule: bool = False,
|
| 139 |
+
):
|
| 140 |
+
if return_schedule:
|
| 141 |
+
raise NotImplementedError(
|
| 142 |
+
"Blackwell build_k2q_csr currently returns CSR tensors only; "
|
| 143 |
+
"SM100 schedule objects are not part of this package."
|
| 144 |
+
)
|
| 145 |
+
if total_k is None:
|
| 146 |
+
raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
|
| 147 |
+
return build_k2q_csr_torch_reference(
|
| 148 |
+
q2k_indices.contiguous(),
|
| 149 |
+
cu_seqlens_q.contiguous(),
|
| 150 |
+
cu_seqlens_k.contiguous(),
|
| 151 |
+
int(kv_block_size),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SparseK2qCsrBuilderSm100:
|
| 156 |
+
"""Compatibility placeholder for the upstream SM100 class name.
|
| 157 |
+
|
| 158 |
+
The name is exported so code can feature-detect it, but schedule-producing
|
| 159 |
+
SM100 behavior is intentionally not claimed for this Blackwell package.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(self, *args, **kwargs):
|
| 163 |
+
self.args = args
|
| 164 |
+
self.kwargs = kwargs
|
| 165 |
+
|
| 166 |
+
def build(self, *args, **kwargs):
|
| 167 |
+
return build_k2q_csr(*args, **kwargs)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
__all__ = [
|
| 171 |
+
"build_k2q_csr",
|
| 172 |
+
"build_k2q_csr_torch_reference",
|
| 173 |
+
"SparseK2qCsrBuilderSm100",
|
| 174 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention.
|
| 3 |
+
|
| 4 |
+
This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on
|
| 5 |
+
DGX Spark / GB10 / SM121. It exposes the
|
| 6 |
+
MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime.
|
| 7 |
+
|
| 8 |
+
Implementation status:
|
| 9 |
+
* native CUDA helper: score -> top-k sparse block ids;
|
| 10 |
+
* native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
|
| 11 |
+
* native CUDA FP4 block-score indexer;
|
| 12 |
+
* native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path;
|
| 13 |
+
* Blackwell-validated sparse prefill path;
|
| 14 |
+
* MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers,
|
| 15 |
+
and FP4 block-score helpers.
|
| 16 |
+
|
| 17 |
+
The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell
|
| 18 |
+
extension path. The public API is Tensor-oriented and independent from
|
| 19 |
+
FlashRT's serving runtime.
|
| 20 |
+
|
| 21 |
+
Public API
|
| 22 |
+
----------
|
| 23 |
+
Lightning indexer (q.k blockmax score -> top-k block select), paged KV:
|
| 24 |
+
flash_decode_with_topk_idx (single-token decode, split-K)
|
| 25 |
+
|
| 26 |
+
Block-sparse GQA attention (consumes block_indices from the indexer):
|
| 27 |
+
flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks)
|
| 28 |
+
sparse_atten_func (official CSR prefill wrapper)
|
| 29 |
+
sparse_decode_atten_func (official paged decode wrapper)
|
| 30 |
+
|
| 31 |
+
PyTorch naive references (for correctness checks):
|
| 32 |
+
naive_flash_decode_with_gqa_share_sparse (attention given topk_idx)
|
| 33 |
+
naive_flash_decode_with_topk_idx (joint indexer + attention)
|
| 34 |
+
|
| 35 |
+
All Triton entrypoints expect a paged KV cache:
|
| 36 |
+
k_cache / v_cache : [max_slots, num_kv_heads, head_dim]
|
| 37 |
+
req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot)
|
| 38 |
+
slot_ids : [batch] (per-request row in req_to_token)
|
| 39 |
+
topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed,
|
| 40 |
+
-1 right-padding (the M3 indexer contract).
|
| 41 |
+
See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 45 |
+
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 46 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 47 |
+
from .interface import (
|
| 48 |
+
SparseDecodePagedAttentionWrapper,
|
| 49 |
+
fp4_indexer_block_scores,
|
| 50 |
+
sparse_atten_func,
|
| 51 |
+
sparse_atten_nvfp4_kv_func,
|
| 52 |
+
sparse_decode_atten_func,
|
| 53 |
+
)
|
| 54 |
+
from ._native import (
|
| 55 |
+
has_native_ops,
|
| 56 |
+
native_nvfp4_dequant_swizzled_to_bf16,
|
| 57 |
+
native_topk_from_scores,
|
| 58 |
+
)
|
| 59 |
+
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 60 |
+
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 61 |
+
from .quantize import (
|
| 62 |
+
Nvfp4QuantizedTensor,
|
| 63 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 64 |
+
nvfp4_global_scale_from_amax,
|
| 65 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 66 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 67 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 68 |
+
)
|
| 69 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 70 |
+
from .api_status import (
|
| 71 |
+
OFFICIAL_API_STATUS,
|
| 72 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 73 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 74 |
+
available_functions,
|
| 75 |
+
official_api_status,
|
| 76 |
+
official_minimax_msa_functions,
|
| 77 |
+
unsupported_official_functions,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
__all__ = [
|
| 81 |
+
"sparse_atten_func",
|
| 82 |
+
"sparse_atten_nvfp4_kv_func",
|
| 83 |
+
"sparse_decode_atten_func",
|
| 84 |
+
"SparseDecodePagedAttentionWrapper",
|
| 85 |
+
"fp4_indexer_block_scores",
|
| 86 |
+
"build_k2q_csr",
|
| 87 |
+
"SparseK2qCsrBuilderSm100",
|
| 88 |
+
"Nvfp4QuantizedTensor",
|
| 89 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 90 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 91 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 92 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 93 |
+
"nvfp4_global_scale_from_amax",
|
| 94 |
+
"flash_decode_with_topk_idx",
|
| 95 |
+
"flash_decode_with_gqa_share_sparse",
|
| 96 |
+
"has_native_ops",
|
| 97 |
+
"native_topk_from_scores",
|
| 98 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 99 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 100 |
+
"naive_flash_decode_with_topk_idx",
|
| 101 |
+
"get_cu_seqblocks",
|
| 102 |
+
"robust_allocator",
|
| 103 |
+
"OFFICIAL_API_STATUS",
|
| 104 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 105 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 106 |
+
"available_functions",
|
| 107 |
+
"official_api_status",
|
| 108 |
+
"official_minimax_msa_functions",
|
| 109 |
+
"unsupported_official_functions",
|
| 110 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/_compat.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton
|
| 3 |
+
# kernels. The upstream SGLang modules pull two tiny things from the framework:
|
| 4 |
+
#
|
| 5 |
+
# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path)
|
| 6 |
+
# from sglang.srt.environ import envs -> a feature flag for an optional
|
| 7 |
+
# JIT-CUDA radix top-k fast path
|
| 8 |
+
#
|
| 9 |
+
# Both are replaced here so the kernels import with only torch + triton. The
|
| 10 |
+
# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel
|
| 11 |
+
# radix select), which is what we want on sm_121 (GB10).
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def is_hip() -> bool:
|
| 15 |
+
"""Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121).
|
| 16 |
+
|
| 17 |
+
Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV
|
| 18 |
+
widening branch (HIP-only). With this False, the CUDA dtype contract
|
| 19 |
+
(K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA.
|
| 20 |
+
"""
|
| 21 |
+
try:
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
return bool(getattr(torch.version, "hip", None))
|
| 25 |
+
except Exception:
|
| 26 |
+
return False
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class _Flag:
|
| 30 |
+
"""Minimal stand-in for an sglang.srt.environ boolean flag (always off)."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, default: bool = False):
|
| 33 |
+
self._default = default
|
| 34 |
+
|
| 35 |
+
def get(self) -> bool:
|
| 36 |
+
return self._default
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class _Envs:
|
| 40 |
+
"""Stub for `sglang.srt.environ.envs`.
|
| 41 |
+
|
| 42 |
+
Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode
|
| 43 |
+
indexer. Keeping it False routes top-k selection through the 2-stage Triton
|
| 44 |
+
fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the
|
| 45 |
+
decode indexer stays pure-Triton.
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
envs = _Envs()
|
build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a053c1fd69c4df3f94e566a7c4d88ba943f8d5cece714ade52789f6819a961ac
|
| 3 |
+
size 729824
|
build/torch211-cxx11-cu130-x86_64-linux/_native.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Optional native CUDA helpers for the FlashRT Blackwell MSA package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
try:
|
| 9 |
+
from ._ops import add_op_namespace_prefix, ops
|
| 10 |
+
|
| 11 |
+
_HAS_NATIVE_OPS = True
|
| 12 |
+
except Exception:
|
| 13 |
+
add_op_namespace_prefix = None
|
| 14 |
+
ops = None
|
| 15 |
+
_HAS_NATIVE_OPS = False
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def has_native_ops() -> bool:
|
| 19 |
+
return _HAS_NATIVE_OPS
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
if _HAS_NATIVE_OPS:
|
| 23 |
+
|
| 24 |
+
@torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores"))
|
| 25 |
+
def _msa_topk_from_scores_fake(
|
| 26 |
+
score: torch.Tensor,
|
| 27 |
+
seq_lens: torch.Tensor,
|
| 28 |
+
block_size: int,
|
| 29 |
+
topk: int,
|
| 30 |
+
topk_idx: torch.Tensor,
|
| 31 |
+
) -> None:
|
| 32 |
+
if score.dim() != 3:
|
| 33 |
+
raise RuntimeError("score must have shape (heads, batch, max_blocks)")
|
| 34 |
+
heads, batch, _ = score.shape
|
| 35 |
+
if seq_lens.shape != (batch,):
|
| 36 |
+
raise RuntimeError("seq_lens must have shape (batch,)")
|
| 37 |
+
if topk_idx.shape != (heads, batch, topk):
|
| 38 |
+
raise RuntimeError("topk_idx must have shape (heads, batch, topk)")
|
| 39 |
+
return None
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def native_topk_from_scores(
|
| 43 |
+
score: torch.Tensor,
|
| 44 |
+
seq_lens: torch.Tensor,
|
| 45 |
+
block_size: int,
|
| 46 |
+
topk: int,
|
| 47 |
+
) -> torch.Tensor:
|
| 48 |
+
"""Return top-k sparse block ids from a score tensor."""
|
| 49 |
+
|
| 50 |
+
if not _HAS_NATIVE_OPS:
|
| 51 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 52 |
+
if not score.is_contiguous():
|
| 53 |
+
score = score.contiguous()
|
| 54 |
+
if not seq_lens.is_contiguous():
|
| 55 |
+
seq_lens = seq_lens.contiguous()
|
| 56 |
+
out = torch.empty(
|
| 57 |
+
(score.shape[0], score.shape[1], int(topk)),
|
| 58 |
+
device=score.device,
|
| 59 |
+
dtype=torch.int32,
|
| 60 |
+
)
|
| 61 |
+
ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out)
|
| 62 |
+
return out
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
if _HAS_NATIVE_OPS:
|
| 66 |
+
|
| 67 |
+
@torch.library.register_fake(
|
| 68 |
+
add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged")
|
| 69 |
+
)
|
| 70 |
+
def _msa_decode_sparse_attn_mma_paged_fake(
|
| 71 |
+
q: torch.Tensor,
|
| 72 |
+
k_cache: torch.Tensor,
|
| 73 |
+
v_cache: torch.Tensor,
|
| 74 |
+
req_to_token: torch.Tensor,
|
| 75 |
+
seq_lens: torch.Tensor,
|
| 76 |
+
slot_ids: torch.Tensor,
|
| 77 |
+
topk_idx: torch.Tensor,
|
| 78 |
+
block_size: int,
|
| 79 |
+
sm_scale: float,
|
| 80 |
+
out: torch.Tensor,
|
| 81 |
+
) -> None:
|
| 82 |
+
return None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def native_decode_mma_supported(
|
| 86 |
+
q: torch.Tensor,
|
| 87 |
+
k_cache: torch.Tensor,
|
| 88 |
+
*,
|
| 89 |
+
block_size: int,
|
| 90 |
+
) -> bool:
|
| 91 |
+
"""Whether the tensor-core decode kernel applies to these inputs.
|
| 92 |
+
|
| 93 |
+
The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA
|
| 94 |
+
group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64.
|
| 95 |
+
"""
|
| 96 |
+
|
| 97 |
+
if not _HAS_NATIVE_OPS:
|
| 98 |
+
return False
|
| 99 |
+
if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16:
|
| 100 |
+
return False
|
| 101 |
+
if q.dim() != 3 or k_cache.dim() != 3:
|
| 102 |
+
return False
|
| 103 |
+
head_dim = int(q.shape[-1])
|
| 104 |
+
num_q_heads = int(q.shape[1])
|
| 105 |
+
num_kv_heads = int(k_cache.shape[1])
|
| 106 |
+
if head_dim != 128 or num_kv_heads <= 0:
|
| 107 |
+
return False
|
| 108 |
+
if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16:
|
| 109 |
+
return False
|
| 110 |
+
if int(block_size) % 64 != 0:
|
| 111 |
+
return False
|
| 112 |
+
return True
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def native_decode_sparse_attn_mma_paged(
|
| 116 |
+
q: torch.Tensor,
|
| 117 |
+
k_cache: torch.Tensor,
|
| 118 |
+
v_cache: torch.Tensor,
|
| 119 |
+
req_to_token: torch.Tensor,
|
| 120 |
+
seq_lens: torch.Tensor,
|
| 121 |
+
slot_ids: torch.Tensor,
|
| 122 |
+
block_size: int,
|
| 123 |
+
topk_idx: torch.Tensor,
|
| 124 |
+
sm_scale: float,
|
| 125 |
+
) -> torch.Tensor:
|
| 126 |
+
"""Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16."""
|
| 127 |
+
|
| 128 |
+
if not _HAS_NATIVE_OPS:
|
| 129 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 130 |
+
out = torch.empty(
|
| 131 |
+
(int(q.shape[0]), int(q.shape[1]), int(q.shape[2])),
|
| 132 |
+
device=q.device,
|
| 133 |
+
dtype=q.dtype,
|
| 134 |
+
)
|
| 135 |
+
ops.msa_decode_sparse_attn_mma_paged(
|
| 136 |
+
q.contiguous(),
|
| 137 |
+
k_cache.contiguous(),
|
| 138 |
+
v_cache.contiguous(),
|
| 139 |
+
req_to_token.contiguous(),
|
| 140 |
+
seq_lens.contiguous(),
|
| 141 |
+
slot_ids.contiguous(),
|
| 142 |
+
topk_idx.contiguous(),
|
| 143 |
+
int(block_size),
|
| 144 |
+
float(sm_scale),
|
| 145 |
+
out,
|
| 146 |
+
)
|
| 147 |
+
return out
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
if _HAS_NATIVE_OPS:
|
| 151 |
+
|
| 152 |
+
@torch.library.register_fake(
|
| 153 |
+
add_op_namespace_prefix("msa_indexer_block_scores")
|
| 154 |
+
)
|
| 155 |
+
def _msa_indexer_block_scores_fake(
|
| 156 |
+
q: torch.Tensor,
|
| 157 |
+
k_pages: torch.Tensor,
|
| 158 |
+
batch_of_q: torch.Tensor,
|
| 159 |
+
cu_q: torch.Tensor,
|
| 160 |
+
cu_k: torch.Tensor,
|
| 161 |
+
cu_pages: torch.Tensor,
|
| 162 |
+
kv_indices: torch.Tensor,
|
| 163 |
+
causal: int,
|
| 164 |
+
scores: torch.Tensor,
|
| 165 |
+
) -> None:
|
| 166 |
+
return None
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def native_indexer_block_scores(
|
| 170 |
+
q: torch.Tensor,
|
| 171 |
+
k_pages: torch.Tensor,
|
| 172 |
+
batch_of_q: torch.Tensor,
|
| 173 |
+
cu_q: torch.Tensor,
|
| 174 |
+
cu_k: torch.Tensor,
|
| 175 |
+
cu_pages: torch.Tensor,
|
| 176 |
+
kv_indices: torch.Tensor,
|
| 177 |
+
*,
|
| 178 |
+
max_blocks: int,
|
| 179 |
+
causal: bool,
|
| 180 |
+
) -> torch.Tensor:
|
| 181 |
+
"""Block-max QK indexer scores. q/k_pages are bf16 (already dequantized).
|
| 182 |
+
|
| 183 |
+
Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible).
|
| 184 |
+
"""
|
| 185 |
+
|
| 186 |
+
if not _HAS_NATIVE_OPS:
|
| 187 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 188 |
+
total_q = int(q.shape[0])
|
| 189 |
+
hq = int(q.shape[1])
|
| 190 |
+
scores = torch.full(
|
| 191 |
+
(hq, int(max_blocks), total_q),
|
| 192 |
+
float("-inf"),
|
| 193 |
+
dtype=torch.float32,
|
| 194 |
+
device=q.device,
|
| 195 |
+
)
|
| 196 |
+
ops.msa_indexer_block_scores(
|
| 197 |
+
q.contiguous(),
|
| 198 |
+
k_pages.contiguous(),
|
| 199 |
+
batch_of_q.contiguous(),
|
| 200 |
+
cu_q.contiguous(),
|
| 201 |
+
cu_k.contiguous(),
|
| 202 |
+
cu_pages.contiguous(),
|
| 203 |
+
kv_indices.contiguous(),
|
| 204 |
+
1 if causal else 0,
|
| 205 |
+
scores,
|
| 206 |
+
)
|
| 207 |
+
return scores
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
if _HAS_NATIVE_OPS:
|
| 211 |
+
|
| 212 |
+
@torch.library.register_fake(
|
| 213 |
+
add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16")
|
| 214 |
+
)
|
| 215 |
+
def _msa_nvfp4_dequant_swizzled_to_bf16_fake(
|
| 216 |
+
packed: torch.Tensor,
|
| 217 |
+
scale_128x4: torch.Tensor,
|
| 218 |
+
global_scale: float,
|
| 219 |
+
out: torch.Tensor,
|
| 220 |
+
) -> None:
|
| 221 |
+
return None
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def native_nvfp4_dequant_swizzled_to_bf16(
|
| 225 |
+
packed: torch.Tensor,
|
| 226 |
+
scale_128x4: torch.Tensor,
|
| 227 |
+
global_scale: torch.Tensor | float,
|
| 228 |
+
) -> torch.Tensor:
|
| 229 |
+
"""Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path."""
|
| 230 |
+
|
| 231 |
+
if not _HAS_NATIVE_OPS:
|
| 232 |
+
raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode")
|
| 233 |
+
if packed.dtype is not torch.uint8:
|
| 234 |
+
raise TypeError("packed must be torch.uint8")
|
| 235 |
+
if scale_128x4.dtype is not torch.uint8:
|
| 236 |
+
raise TypeError("scale_128x4 must be torch.uint8")
|
| 237 |
+
original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
|
| 238 |
+
out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16)
|
| 239 |
+
if isinstance(global_scale, torch.Tensor):
|
| 240 |
+
global_scale_value = float(global_scale.reshape(-1)[0].item())
|
| 241 |
+
else:
|
| 242 |
+
global_scale_value = float(global_scale)
|
| 243 |
+
ops.msa_nvfp4_dequant_swizzled_to_bf16(
|
| 244 |
+
packed.contiguous(),
|
| 245 |
+
scale_128x4.contiguous(),
|
| 246 |
+
global_scale_value,
|
| 247 |
+
out,
|
| 248 |
+
)
|
| 249 |
+
return out
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
__all__ = [
|
| 253 |
+
"has_native_ops",
|
| 254 |
+
"native_topk_from_scores",
|
| 255 |
+
"native_decode_mma_supported",
|
| 256 |
+
"native_decode_sparse_attn_mma_paged",
|
| 257 |
+
"native_indexer_block_scores",
|
| 258 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 259 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _minimaxai_msa_blackwell_cuda_25b0067
|
| 3 |
+
ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/api_status.py
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Public API status for the FlashRT MiniMax MSA Blackwell package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from types import MappingProxyType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
V1_AVAILABLE_FUNCTIONS = (
|
| 10 |
+
"sparse_atten_func",
|
| 11 |
+
"sparse_atten_nvfp4_kv_func",
|
| 12 |
+
"sparse_decode_atten_func",
|
| 13 |
+
"SparseDecodePagedAttentionWrapper",
|
| 14 |
+
"fp4_indexer_block_scores",
|
| 15 |
+
"build_k2q_csr",
|
| 16 |
+
"SparseK2qCsrBuilderSm100",
|
| 17 |
+
"Nvfp4QuantizedTensor",
|
| 18 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 19 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 20 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 21 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 22 |
+
"nvfp4_global_scale_from_amax",
|
| 23 |
+
"has_native_ops",
|
| 24 |
+
"native_topk_from_scores",
|
| 25 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 26 |
+
"flash_decode_with_topk_idx",
|
| 27 |
+
"flash_decode_with_gqa_share_sparse",
|
| 28 |
+
"naive_flash_decode_with_topk_idx",
|
| 29 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 30 |
+
"get_cu_seqblocks",
|
| 31 |
+
"robust_allocator",
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
|
| 35 |
+
"sparse_atten_func",
|
| 36 |
+
"sparse_atten_nvfp4_kv_func",
|
| 37 |
+
"sparse_decode_atten_func",
|
| 38 |
+
"SparseDecodePagedAttentionWrapper",
|
| 39 |
+
"fp4_indexer_block_scores",
|
| 40 |
+
"build_k2q_csr",
|
| 41 |
+
"SparseK2qCsrBuilderSm100",
|
| 42 |
+
"Nvfp4QuantizedTensor",
|
| 43 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 44 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 45 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 46 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 47 |
+
"nvfp4_global_scale_from_amax",
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
_OFFICIAL_API_STATUS = {
|
| 51 |
+
"sparse_atten_func": {
|
| 52 |
+
"status": "available",
|
| 53 |
+
"target": "blackwell-prefill",
|
| 54 |
+
"reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.",
|
| 55 |
+
},
|
| 56 |
+
"sparse_atten_nvfp4_kv_func": {
|
| 57 |
+
"status": "available",
|
| 58 |
+
"target": "blackwell-prefill-nvfp4",
|
| 59 |
+
"reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.",
|
| 60 |
+
},
|
| 61 |
+
"sparse_decode_atten_func": {
|
| 62 |
+
"status": "available",
|
| 63 |
+
"target": "v1",
|
| 64 |
+
"reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
|
| 65 |
+
},
|
| 66 |
+
"SparseDecodePagedAttentionWrapper": {
|
| 67 |
+
"status": "available",
|
| 68 |
+
"target": "v1",
|
| 69 |
+
"reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
|
| 70 |
+
},
|
| 71 |
+
"fp4_indexer_block_scores": {
|
| 72 |
+
"status": "available",
|
| 73 |
+
"target": "blackwell-fp4-indexer",
|
| 74 |
+
"reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.",
|
| 75 |
+
},
|
| 76 |
+
"build_k2q_csr": {
|
| 77 |
+
"status": "available",
|
| 78 |
+
"target": "v1",
|
| 79 |
+
"reason": "CSR construction helper is available for the official prefill API.",
|
| 80 |
+
},
|
| 81 |
+
"SparseK2qCsrBuilderSm100": {
|
| 82 |
+
"status": "available",
|
| 83 |
+
"target": "v1",
|
| 84 |
+
"reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
|
| 85 |
+
},
|
| 86 |
+
"Nvfp4QuantizedTensor": {
|
| 87 |
+
"status": "available",
|
| 88 |
+
"target": "v1",
|
| 89 |
+
"reason": "NVFP4 metadata dataclass is available.",
|
| 90 |
+
},
|
| 91 |
+
"quantize_bf16_to_nvfp4_128x4": {
|
| 92 |
+
"status": "available_optional_te",
|
| 93 |
+
"target": "v1",
|
| 94 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 95 |
+
},
|
| 96 |
+
"quantize_kv_bf16_to_nvfp4_128x4": {
|
| 97 |
+
"status": "available_optional_te",
|
| 98 |
+
"target": "v1",
|
| 99 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 100 |
+
},
|
| 101 |
+
"dequantize_nvfp4_128x4_to_bf16": {
|
| 102 |
+
"status": "available",
|
| 103 |
+
"target": "v1",
|
| 104 |
+
"reason": "Reference dequant helper is available.",
|
| 105 |
+
},
|
| 106 |
+
"swizzle_nvfp4_scale_to_128x4": {
|
| 107 |
+
"status": "available",
|
| 108 |
+
"target": "v1",
|
| 109 |
+
"reason": "Scale swizzle helper is available.",
|
| 110 |
+
},
|
| 111 |
+
"nvfp4_global_scale_from_amax": {
|
| 112 |
+
"status": "available",
|
| 113 |
+
"target": "v1",
|
| 114 |
+
"reason": "Global scale helper is available.",
|
| 115 |
+
},
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def available_functions() -> tuple[str, ...]:
|
| 122 |
+
"""Return the functions/classes intentionally available in v1."""
|
| 123 |
+
|
| 124 |
+
return V1_AVAILABLE_FUNCTIONS
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def official_minimax_msa_functions() -> tuple[str, ...]:
|
| 128 |
+
"""Return the upstream MiniMaxAI/msa public function/class names tracked here."""
|
| 129 |
+
|
| 130 |
+
return OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def official_api_status() -> dict[str, dict[str, str]]:
|
| 134 |
+
"""Return a copy of the official API compatibility status table."""
|
| 135 |
+
|
| 136 |
+
return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def unsupported_official_functions() -> tuple[str, ...]:
|
| 140 |
+
"""Return official MiniMaxAI/msa names that are not exported by this v1 package."""
|
| 141 |
+
|
| 142 |
+
return tuple(
|
| 143 |
+
name
|
| 144 |
+
for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 145 |
+
if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
|
| 146 |
+
)
|
build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
build/torch211-cxx11-cu130-x86_64-linux/common/utils.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
import functools
|
| 4 |
+
from collections import deque
|
| 5 |
+
from typing import Any, Callable, Tuple
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
import triton.language as tl
|
| 9 |
+
|
| 10 |
+
_tma_keep_alive_buf = deque(maxlen=200)
|
| 11 |
+
|
| 12 |
+
try:
|
| 13 |
+
make_tensor_descriptor = tl.make_tensor_descriptor
|
| 14 |
+
except Exception:
|
| 15 |
+
make_tensor_descriptor = tl._experimental_make_tensor_descriptor
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def robust_allocator(size: int, alignment: int, stream: int = None):
|
| 19 |
+
"""Allocator for Triton TMA descriptors.
|
| 20 |
+
|
| 21 |
+
We keep reference in deque to prevent GC from collecting the buffer.
|
| 22 |
+
"""
|
| 23 |
+
tensor = torch.empty(size, device="cuda", dtype=torch.uint8)
|
| 24 |
+
_tma_keep_alive_buf.append(tensor)
|
| 25 |
+
return tensor
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def tensor_cache(maxsize: int = 8):
|
| 29 |
+
"""
|
| 30 |
+
Cache function results using identity comparison.
|
| 31 |
+
Zero-overhead cache hit: no hash, no DtoH, just pointer comparison.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
maxsize: Maximum number of cached entries. Supports multi-GPU scenarios
|
| 35 |
+
where different devices have different tensor arguments.
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
|
| 39 |
+
# LRU-style cache: list of (args, kwargs, result) tuples
|
| 40 |
+
# Most recently used at the end
|
| 41 |
+
_cache: list = []
|
| 42 |
+
|
| 43 |
+
def _args_match(args: tuple, cached_args: tuple) -> bool:
|
| 44 |
+
if len(args) != len(cached_args):
|
| 45 |
+
return False
|
| 46 |
+
for i in range(len(args)):
|
| 47 |
+
if args[i] is not cached_args[i]:
|
| 48 |
+
return False
|
| 49 |
+
return True
|
| 50 |
+
|
| 51 |
+
def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool:
|
| 52 |
+
if not kwargs and not cached_kwargs:
|
| 53 |
+
return True
|
| 54 |
+
if kwargs.keys() != cached_kwargs.keys():
|
| 55 |
+
return False
|
| 56 |
+
for k, v in kwargs.items():
|
| 57 |
+
if v is not cached_kwargs[k]:
|
| 58 |
+
return False
|
| 59 |
+
return True
|
| 60 |
+
|
| 61 |
+
@functools.wraps(fn)
|
| 62 |
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
| 63 |
+
# Search cache (most recent first for better hit rate)
|
| 64 |
+
for i in range(len(_cache) - 1, -1, -1):
|
| 65 |
+
cached_args, cached_kwargs, cached_result = _cache[i]
|
| 66 |
+
if _args_match(args, cached_args) and _kwargs_match(
|
| 67 |
+
kwargs, cached_kwargs
|
| 68 |
+
):
|
| 69 |
+
# Move to end (most recently used)
|
| 70 |
+
if i != len(_cache) - 1:
|
| 71 |
+
_cache.append(_cache.pop(i))
|
| 72 |
+
return cached_result
|
| 73 |
+
|
| 74 |
+
# Cache miss
|
| 75 |
+
result = fn(*args, **kwargs)
|
| 76 |
+
|
| 77 |
+
# Add to cache
|
| 78 |
+
if len(_cache) >= maxsize:
|
| 79 |
+
_cache.pop(0) # Remove oldest
|
| 80 |
+
_cache.append((args, kwargs, result))
|
| 81 |
+
return result
|
| 82 |
+
|
| 83 |
+
# Expose cache for manual clearing if needed
|
| 84 |
+
wrapper.cache_clear = lambda: _cache.clear()
|
| 85 |
+
wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize}
|
| 86 |
+
|
| 87 |
+
return wrapper
|
| 88 |
+
|
| 89 |
+
return decorator
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@tensor_cache(maxsize=8)
|
| 93 |
+
def get_cu_seqblocks(
|
| 94 |
+
cu_seqlens: torch.Tensor,
|
| 95 |
+
max_seqlen: int,
|
| 96 |
+
block_size_q: int,
|
| 97 |
+
block_size_k: int,
|
| 98 |
+
) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]:
|
| 99 |
+
"""Compute cumulative sequence block indices for blocked sparse attention.
|
| 100 |
+
|
| 101 |
+
Converts token-level cumulative sequence lengths to block-level indices,
|
| 102 |
+
which are needed for block-sparse attention kernels.
|
| 103 |
+
|
| 104 |
+
Note:
|
| 105 |
+
Results are cached (maxsize=8) based on input arguments. Repeated calls
|
| 106 |
+
with the same cu_seqlens, max_seqlen, and block sizes will return cached
|
| 107 |
+
results without recomputation.
|
| 108 |
+
|
| 109 |
+
Args:
|
| 110 |
+
cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32.
|
| 111 |
+
max_seqlen: Maximum sequence length in the batch.
|
| 112 |
+
block_size_q: Query block size.
|
| 113 |
+
block_size_k: Key-value block size.
|
| 114 |
+
|
| 115 |
+
Returns:
|
| 116 |
+
A tuple of 6 values:
|
| 117 |
+
- cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1]
|
| 118 |
+
- max_seqblock_q: Maximum number of query blocks per sequence.
|
| 119 |
+
- all_seqblock_q: Total number of query blocks across all sequences.
|
| 120 |
+
- cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1]
|
| 121 |
+
- max_seqblock_k: Maximum number of key blocks per sequence.
|
| 122 |
+
- all_seqblock_k: Total number of key blocks across all sequences.
|
| 123 |
+
"""
|
| 124 |
+
cu_seqblocks_q = torch.zeros_like(cu_seqlens)
|
| 125 |
+
cu_seqblocks_k = torch.zeros_like(cu_seqlens)
|
| 126 |
+
seq_lens = torch.diff(cu_seqlens)
|
| 127 |
+
seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q
|
| 128 |
+
seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k
|
| 129 |
+
max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q
|
| 130 |
+
max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k
|
| 131 |
+
cu_seqblocks_q[1:] = seqblocks_q
|
| 132 |
+
cu_seqblocks_k[1:] = seqblocks_k
|
| 133 |
+
cu_seqblocks_q.cumsum_(0)
|
| 134 |
+
cu_seqblocks_k.cumsum_(0)
|
| 135 |
+
all_seqblock_q = seqblocks_q.sum().item()
|
| 136 |
+
all_seqblock_k = seqblocks_k.sum().item()
|
| 137 |
+
return (
|
| 138 |
+
cu_seqblocks_q,
|
| 139 |
+
max_seqblock_q,
|
| 140 |
+
all_seqblock_q,
|
| 141 |
+
cu_seqblocks_k,
|
| 142 |
+
max_seqblock_k,
|
| 143 |
+
all_seqblock_k,
|
| 144 |
+
)
|
build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py
ADDED
|
@@ -0,0 +1,1150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .._compat import envs # vendored: was `from sglang.srt.environ import envs`
|
| 10 |
+
|
| 11 |
+
from ..common.utils import robust_allocator
|
| 12 |
+
from .._native import has_native_ops, native_topk_from_scores
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@triton.heuristics(
|
| 16 |
+
{
|
| 17 |
+
"BLOCK_SIZE_H": lambda args: max(
|
| 18 |
+
16, triton.next_power_of_2(args["gqa_group_size"])
|
| 19 |
+
),
|
| 20 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 21 |
+
"BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
|
| 22 |
+
}
|
| 23 |
+
)
|
| 24 |
+
@triton.autotune(
|
| 25 |
+
configs=[
|
| 26 |
+
triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
|
| 27 |
+
for BN in [64, 128, 256, 512]
|
| 28 |
+
for nw in [4, 8, 16]
|
| 29 |
+
for ns in [1, 2, 3]
|
| 30 |
+
],
|
| 31 |
+
key=[
|
| 32 |
+
"BATCH_SIZE_BUCKET",
|
| 33 |
+
"gqa_group_size",
|
| 34 |
+
"head_dim",
|
| 35 |
+
"block_size",
|
| 36 |
+
"SCORE_TYPE",
|
| 37 |
+
],
|
| 38 |
+
)
|
| 39 |
+
@triton.jit
|
| 40 |
+
def _decode_score_kernel(
|
| 41 |
+
q_ptr, # Q: b x qh x d
|
| 42 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 43 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 44 |
+
score_ptr, # Score: qh x b x max_seqblock
|
| 45 |
+
seq_lens,
|
| 46 |
+
slot_ids,
|
| 47 |
+
# shape
|
| 48 |
+
max_slots,
|
| 49 |
+
batch_size,
|
| 50 |
+
gqa_group_size,
|
| 51 |
+
head_dim,
|
| 52 |
+
# block size
|
| 53 |
+
block_size: tl.constexpr,
|
| 54 |
+
topk: tl.constexpr,
|
| 55 |
+
# sm_scale
|
| 56 |
+
sm_scale,
|
| 57 |
+
# init and local blocks
|
| 58 |
+
init_blocks,
|
| 59 |
+
local_blocks,
|
| 60 |
+
# stride
|
| 61 |
+
stride_q_b,
|
| 62 |
+
stride_q_h,
|
| 63 |
+
stride_q_d,
|
| 64 |
+
stride_k_s,
|
| 65 |
+
stride_k_h,
|
| 66 |
+
stride_k_d,
|
| 67 |
+
stride_r2t_b,
|
| 68 |
+
stride_s_h,
|
| 69 |
+
stride_s_b,
|
| 70 |
+
stride_s_n,
|
| 71 |
+
# META parameters
|
| 72 |
+
BATCH_SIZE_BUCKET: tl.constexpr,
|
| 73 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 74 |
+
BLOCK_SIZE_N: tl.constexpr,
|
| 75 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 76 |
+
NUM_KV_CHUNKS: tl.constexpr,
|
| 77 |
+
SCORE_TYPE: tl.constexpr,
|
| 78 |
+
SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
|
| 79 |
+
):
|
| 80 |
+
tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
|
| 81 |
+
sm_scale_log2e = sm_scale * 1.4426950409
|
| 82 |
+
tl.static_assert(BLOCK_SIZE_N >= block_size)
|
| 83 |
+
BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
|
| 84 |
+
# get batch id and head id
|
| 85 |
+
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
| 86 |
+
pid_b = pid_bc % batch_size
|
| 87 |
+
pid_c = pid_bc // batch_size
|
| 88 |
+
pid_h = pid_kh * gqa_group_size
|
| 89 |
+
# block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
|
| 90 |
+
seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
|
| 91 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 92 |
+
if SKIP_TRIVIAL_TOPK_SCORE:
|
| 93 |
+
if num_blocks <= topk:
|
| 94 |
+
return
|
| 95 |
+
chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
|
| 96 |
+
chunk_start_block = pid_c * chunk_size_blocks
|
| 97 |
+
chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
|
| 98 |
+
chunk_start = chunk_start_block * block_size
|
| 99 |
+
chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
|
| 100 |
+
if chunk_start_block >= chunk_end_block:
|
| 101 |
+
return
|
| 102 |
+
sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots
|
| 103 |
+
# init qkv pointer
|
| 104 |
+
q_ptrs = tl.make_block_ptr(
|
| 105 |
+
base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
|
| 106 |
+
shape=(gqa_group_size, head_dim),
|
| 107 |
+
strides=(stride_q_h, stride_q_d),
|
| 108 |
+
offsets=(0, 0),
|
| 109 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 110 |
+
order=(1, 0),
|
| 111 |
+
)
|
| 112 |
+
s_ptrs = tl.make_block_ptr(
|
| 113 |
+
base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
|
| 114 |
+
shape=(gqa_group_size, chunk_end_block),
|
| 115 |
+
strides=(stride_s_h, stride_s_n),
|
| 116 |
+
offsets=(0, chunk_start_block),
|
| 117 |
+
block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
|
| 118 |
+
order=(1, 0),
|
| 119 |
+
)
|
| 120 |
+
# load q
|
| 121 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 122 |
+
# init statistics
|
| 123 |
+
off_n = tl.arange(0, BLOCK_SIZE_N)
|
| 124 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 125 |
+
off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
|
| 126 |
+
dim_mask = off_d < head_dim
|
| 127 |
+
# pre-compute local_start outside the loop (constant for entire batch)
|
| 128 |
+
local_start = tl.maximum(0, num_blocks - local_blocks)
|
| 129 |
+
# prefetch first iteration's slots
|
| 130 |
+
r2t_base = req_to_token_ptr + sid * stride_r2t_b
|
| 131 |
+
prefetch_pos = chunk_start + off_n
|
| 132 |
+
prefetch_mask = prefetch_pos < seq_len
|
| 133 |
+
prefetched_slots = tl.load(
|
| 134 |
+
r2t_base + prefetch_pos,
|
| 135 |
+
mask=prefetch_mask,
|
| 136 |
+
other=0,
|
| 137 |
+
).to(tl.int64)
|
| 138 |
+
# score-only: compute block scores without loading V
|
| 139 |
+
for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
|
| 140 |
+
pos_mask = prefetch_mask
|
| 141 |
+
slots = prefetched_slots
|
| 142 |
+
# prefetch next iteration's slots
|
| 143 |
+
next_i = i + BLOCK_SIZE_N
|
| 144 |
+
if next_i < chunk_end:
|
| 145 |
+
next_pos = next_i + off_n
|
| 146 |
+
prefetch_mask = next_pos < seq_len
|
| 147 |
+
prefetched_slots = tl.load(
|
| 148 |
+
r2t_base + next_pos,
|
| 149 |
+
mask=prefetch_mask,
|
| 150 |
+
other=0,
|
| 151 |
+
).to(tl.int64)
|
| 152 |
+
slots = (slots + max_slots) % max_slots
|
| 153 |
+
# load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
|
| 154 |
+
k_off = (
|
| 155 |
+
slots[None, :] * stride_k_s
|
| 156 |
+
+ pid_kh * stride_k_h
|
| 157 |
+
+ off_d[:, None] * stride_k_d
|
| 158 |
+
)
|
| 159 |
+
k = tl.load(
|
| 160 |
+
k_cache_ptr + k_off,
|
| 161 |
+
mask=dim_mask[:, None] & pos_mask[None, :],
|
| 162 |
+
other=0.0,
|
| 163 |
+
)
|
| 164 |
+
# compute qk
|
| 165 |
+
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
|
| 166 |
+
qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
|
| 167 |
+
# [H, D], [D, N] -> [H, N]
|
| 168 |
+
qk += tl.dot(q, k) * sm_scale_log2e
|
| 169 |
+
# save qk to score
|
| 170 |
+
score = tl.reshape(
|
| 171 |
+
qk,
|
| 172 |
+
(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
|
| 173 |
+
can_reorder=False,
|
| 174 |
+
)
|
| 175 |
+
sub_max = tl.max(score, axis=2)
|
| 176 |
+
if SCORE_TYPE == "max":
|
| 177 |
+
score = sub_max
|
| 178 |
+
else: # "lse"
|
| 179 |
+
score = sub_max + tl.log2(
|
| 180 |
+
tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
|
| 181 |
+
)
|
| 182 |
+
score = tl.where(score != score, float("-inf"), score)
|
| 183 |
+
# apply init_blocks and local_blocks efficiently
|
| 184 |
+
curr_block_idx = i // block_size + off_bpk
|
| 185 |
+
is_init = curr_block_idx < init_blocks
|
| 186 |
+
is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
|
| 187 |
+
score = tl.where(
|
| 188 |
+
is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
|
| 189 |
+
)
|
| 190 |
+
tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1))
|
| 191 |
+
# update ptrs
|
| 192 |
+
s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
@triton.heuristics(
|
| 196 |
+
{
|
| 197 |
+
"BLOCK_SIZE_H": lambda args: max(
|
| 198 |
+
16, triton.next_power_of_2(args["gqa_group_size"])
|
| 199 |
+
),
|
| 200 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 201 |
+
"HAS_SINK": lambda args: args["sink_ptr"] is not None,
|
| 202 |
+
}
|
| 203 |
+
)
|
| 204 |
+
@triton.autotune(
|
| 205 |
+
configs=[
|
| 206 |
+
triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns)
|
| 207 |
+
for BN in [64, 128, 256, 512]
|
| 208 |
+
for nw in [4, 8, 16]
|
| 209 |
+
for ns in [1, 2, 3]
|
| 210 |
+
],
|
| 211 |
+
key=[
|
| 212 |
+
"gqa_group_size",
|
| 213 |
+
"head_dim",
|
| 214 |
+
"block_size",
|
| 215 |
+
"HAS_SINK",
|
| 216 |
+
"SCORE_TYPE",
|
| 217 |
+
],
|
| 218 |
+
)
|
| 219 |
+
@triton.jit
|
| 220 |
+
def _decode_score_attn_kernel(
|
| 221 |
+
q_ptr, # Q: b x qh x d
|
| 222 |
+
sink_ptr, # Sink: qh x d
|
| 223 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 224 |
+
v_cache_ptr, # V paged: max_slots x kh x d
|
| 225 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 226 |
+
o_ptr, # O: c x b x qh x d
|
| 227 |
+
lse_ptr, # lse: c x b x qh
|
| 228 |
+
score_ptr, # Score: qh x b x max_seqblock
|
| 229 |
+
seq_lens,
|
| 230 |
+
slot_ids,
|
| 231 |
+
# shape
|
| 232 |
+
max_slots,
|
| 233 |
+
batch_size,
|
| 234 |
+
gqa_group_size,
|
| 235 |
+
head_dim,
|
| 236 |
+
# block size
|
| 237 |
+
block_size: tl.constexpr,
|
| 238 |
+
topk: tl.constexpr,
|
| 239 |
+
# sm_scale
|
| 240 |
+
sm_scale,
|
| 241 |
+
# init and local blocks
|
| 242 |
+
init_blocks,
|
| 243 |
+
local_blocks,
|
| 244 |
+
# stride
|
| 245 |
+
stride_q_b,
|
| 246 |
+
stride_q_h,
|
| 247 |
+
stride_q_d,
|
| 248 |
+
stride_sink_h,
|
| 249 |
+
stride_sink_d,
|
| 250 |
+
stride_k_s,
|
| 251 |
+
stride_k_h,
|
| 252 |
+
stride_k_d,
|
| 253 |
+
stride_v_s,
|
| 254 |
+
stride_v_h,
|
| 255 |
+
stride_v_d,
|
| 256 |
+
stride_r2t_b,
|
| 257 |
+
stride_o_c,
|
| 258 |
+
stride_o_b,
|
| 259 |
+
stride_o_h,
|
| 260 |
+
stride_o_d,
|
| 261 |
+
stride_l_c,
|
| 262 |
+
stride_l_b,
|
| 263 |
+
stride_l_h,
|
| 264 |
+
stride_s_h,
|
| 265 |
+
stride_s_b,
|
| 266 |
+
stride_s_n,
|
| 267 |
+
# META parameters
|
| 268 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 269 |
+
BLOCK_SIZE_N: tl.constexpr,
|
| 270 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 271 |
+
NUM_KV_CHUNKS: tl.constexpr,
|
| 272 |
+
HAS_SINK: tl.constexpr,
|
| 273 |
+
SCORE_TYPE: tl.constexpr,
|
| 274 |
+
SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr,
|
| 275 |
+
):
|
| 276 |
+
tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse")
|
| 277 |
+
sm_scale_log2e = sm_scale * 1.4426950409
|
| 278 |
+
tl.static_assert(BLOCK_SIZE_N >= block_size)
|
| 279 |
+
BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size
|
| 280 |
+
# get batch id and head id
|
| 281 |
+
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
| 282 |
+
pid_b = pid_bc % batch_size
|
| 283 |
+
pid_c = pid_bc // batch_size
|
| 284 |
+
pid_h = pid_kh * gqa_group_size
|
| 285 |
+
# block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph)
|
| 286 |
+
seq_len = tl.load(seq_lens + pid_b).to(tl.int32)
|
| 287 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 288 |
+
chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
|
| 289 |
+
chunk_start_block = pid_c * chunk_size_blocks
|
| 290 |
+
chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
|
| 291 |
+
chunk_start = chunk_start_block * block_size
|
| 292 |
+
chunk_end = tl.minimum(chunk_end_block * block_size, seq_len)
|
| 293 |
+
if chunk_start_block >= chunk_end_block:
|
| 294 |
+
return
|
| 295 |
+
sid = (
|
| 296 |
+
tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
|
| 297 |
+
) % max_slots # to avoid bugs when slot_ids is negative
|
| 298 |
+
# init qkv pointer
|
| 299 |
+
q_ptrs = tl.make_block_ptr(
|
| 300 |
+
base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
|
| 301 |
+
shape=(gqa_group_size, head_dim),
|
| 302 |
+
strides=(stride_q_h, stride_q_d),
|
| 303 |
+
offsets=(0, 0),
|
| 304 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 305 |
+
order=(1, 0),
|
| 306 |
+
)
|
| 307 |
+
s_ptrs = tl.make_block_ptr(
|
| 308 |
+
base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h,
|
| 309 |
+
shape=(gqa_group_size, chunk_end_block),
|
| 310 |
+
strides=(stride_s_h, stride_s_n),
|
| 311 |
+
offsets=(0, chunk_start_block),
|
| 312 |
+
block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK),
|
| 313 |
+
order=(1, 0),
|
| 314 |
+
)
|
| 315 |
+
# load q
|
| 316 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 317 |
+
# init statistics
|
| 318 |
+
off_n = tl.arange(0, BLOCK_SIZE_N)
|
| 319 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 320 |
+
off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK)
|
| 321 |
+
dim_mask = off_d < head_dim
|
| 322 |
+
if HAS_SINK:
|
| 323 |
+
if pid_c == 0:
|
| 324 |
+
sink_ptrs = tl.make_block_ptr(
|
| 325 |
+
base=sink_ptr + pid_h * stride_sink_h,
|
| 326 |
+
shape=(gqa_group_size, head_dim),
|
| 327 |
+
strides=(stride_sink_h, stride_sink_d),
|
| 328 |
+
offsets=(0, 0),
|
| 329 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 330 |
+
order=(1, 0),
|
| 331 |
+
)
|
| 332 |
+
sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
|
| 333 |
+
tl.float32
|
| 334 |
+
)
|
| 335 |
+
qsink = (
|
| 336 |
+
tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e
|
| 337 |
+
) # (BLOCK_SIZE_H,)
|
| 338 |
+
m_i = qsink
|
| 339 |
+
l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32)
|
| 340 |
+
else:
|
| 341 |
+
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 342 |
+
l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
|
| 343 |
+
else:
|
| 344 |
+
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 345 |
+
l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32)
|
| 346 |
+
acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
|
| 347 |
+
# pre-compute local_start outside the loop (constant for entire batch)
|
| 348 |
+
local_start = tl.maximum(0, num_blocks - local_blocks)
|
| 349 |
+
# full attention or causal attention
|
| 350 |
+
for i in range(chunk_start, chunk_end, BLOCK_SIZE_N):
|
| 351 |
+
pos = i + off_n
|
| 352 |
+
pos_mask = pos < seq_len
|
| 353 |
+
# resolve slot per position via req_to_token
|
| 354 |
+
slots = tl.load(
|
| 355 |
+
req_to_token_ptr + sid * stride_r2t_b + pos,
|
| 356 |
+
mask=pos_mask,
|
| 357 |
+
other=0,
|
| 358 |
+
).to(tl.int64)
|
| 359 |
+
slots = (slots + max_slots) % max_slots # safety against negative
|
| 360 |
+
# load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
|
| 361 |
+
k_off = (
|
| 362 |
+
slots[None, :] * stride_k_s
|
| 363 |
+
+ pid_kh * stride_k_h
|
| 364 |
+
+ off_d[:, None] * stride_k_d
|
| 365 |
+
)
|
| 366 |
+
k = tl.load(
|
| 367 |
+
k_cache_ptr + k_off,
|
| 368 |
+
mask=dim_mask[:, None] & pos_mask[None, :],
|
| 369 |
+
other=0.0,
|
| 370 |
+
)
|
| 371 |
+
# load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
|
| 372 |
+
v_off = (
|
| 373 |
+
slots[:, None] * stride_v_s
|
| 374 |
+
+ pid_kh * stride_v_h
|
| 375 |
+
+ off_d[None, :] * stride_v_d
|
| 376 |
+
)
|
| 377 |
+
v = tl.load(
|
| 378 |
+
v_cache_ptr + v_off,
|
| 379 |
+
mask=pos_mask[:, None] & dim_mask[None, :],
|
| 380 |
+
other=0.0,
|
| 381 |
+
)
|
| 382 |
+
# compute qk
|
| 383 |
+
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
|
| 384 |
+
qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf"))
|
| 385 |
+
# [H, D], [D, N] -> [H, N]
|
| 386 |
+
qk += tl.dot(q, k) * sm_scale_log2e
|
| 387 |
+
# save qk to score
|
| 388 |
+
score = tl.reshape(
|
| 389 |
+
qk,
|
| 390 |
+
(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size),
|
| 391 |
+
can_reorder=False,
|
| 392 |
+
)
|
| 393 |
+
sub_max = tl.max(score, axis=2)
|
| 394 |
+
if SCORE_TYPE == "max":
|
| 395 |
+
score = sub_max
|
| 396 |
+
else: # "lse"
|
| 397 |
+
score = sub_max + tl.log2(
|
| 398 |
+
tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2)
|
| 399 |
+
)
|
| 400 |
+
score = tl.where(score != score, float("-inf"), score)
|
| 401 |
+
# apply init_blocks and local_blocks efficiently
|
| 402 |
+
# current block indices: [BLOCKS_PER_K_BLOCK]
|
| 403 |
+
curr_block_idx = i // block_size + off_bpk
|
| 404 |
+
# Combined condition with single nested tl.where:
|
| 405 |
+
# - local_blocks (1e29) takes priority over init_blocks (1e30)
|
| 406 |
+
# - When init_blocks=0: curr_block_idx < 0 is always False
|
| 407 |
+
# - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False
|
| 408 |
+
is_init = curr_block_idx < init_blocks
|
| 409 |
+
is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks)
|
| 410 |
+
# Apply in one fused operation: local > init > original
|
| 411 |
+
score = tl.where(
|
| 412 |
+
is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score)
|
| 413 |
+
)
|
| 414 |
+
if SKIP_TRIVIAL_TOPK_SCORE:
|
| 415 |
+
if num_blocks > topk:
|
| 416 |
+
tl.store(
|
| 417 |
+
s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
|
| 418 |
+
)
|
| 419 |
+
else:
|
| 420 |
+
tl.store(
|
| 421 |
+
s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)
|
| 422 |
+
)
|
| 423 |
+
# max-of-max == max(qk), avoids re-scanning qk
|
| 424 |
+
m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1))
|
| 425 |
+
p = tl.exp2(qk - m_ij[:, None])
|
| 426 |
+
l_ij = tl.sum(p, axis=1)
|
| 427 |
+
acc_o_scale = tl.exp2(m_i - m_ij)
|
| 428 |
+
acc_o = acc_o * acc_o_scale[:, None]
|
| 429 |
+
# [H, N], [N, D] -> [H, D]
|
| 430 |
+
acc_o += tl.dot(p.to(v.dtype), v)
|
| 431 |
+
m_i = m_ij
|
| 432 |
+
l_i = l_i * acc_o_scale + l_ij
|
| 433 |
+
# update ptrs
|
| 434 |
+
s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK))
|
| 435 |
+
# final scale
|
| 436 |
+
acc_o = acc_o / l_i[:, None]
|
| 437 |
+
# compute lse in log2 scale for merge kernel
|
| 438 |
+
lse_i = m_i + tl.log2(l_i)
|
| 439 |
+
# save output
|
| 440 |
+
o_ptrs = tl.make_block_ptr(
|
| 441 |
+
base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 442 |
+
shape=(gqa_group_size, head_dim),
|
| 443 |
+
strides=(stride_o_h, stride_o_d),
|
| 444 |
+
offsets=(0, 0),
|
| 445 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 446 |
+
order=(1, 0),
|
| 447 |
+
)
|
| 448 |
+
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
|
| 449 |
+
# save lse (in log2 scale, will be used directly in merge kernel)
|
| 450 |
+
lse_ptrs = tl.make_block_ptr(
|
| 451 |
+
base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
|
| 452 |
+
shape=(gqa_group_size,),
|
| 453 |
+
strides=(stride_l_h,),
|
| 454 |
+
offsets=(0,),
|
| 455 |
+
block_shape=(BLOCK_SIZE_H,),
|
| 456 |
+
order=(0,),
|
| 457 |
+
)
|
| 458 |
+
tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
@triton.heuristics(
|
| 462 |
+
{
|
| 463 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 464 |
+
}
|
| 465 |
+
)
|
| 466 |
+
@triton.autotune(
|
| 467 |
+
configs=[
|
| 468 |
+
triton.Config({}, num_warps=nw, num_stages=ns)
|
| 469 |
+
for nw in [4, 8]
|
| 470 |
+
for ns in [2, 3, 4]
|
| 471 |
+
],
|
| 472 |
+
key=["BLOCK_SIZE_D"],
|
| 473 |
+
restore_value=["o_ptr"],
|
| 474 |
+
)
|
| 475 |
+
@triton.jit
|
| 476 |
+
def _merge_attn_out_kernel(
|
| 477 |
+
o_ptr,
|
| 478 |
+
lse_ptr,
|
| 479 |
+
# shape
|
| 480 |
+
seq_lens,
|
| 481 |
+
head_dim,
|
| 482 |
+
block_size: tl.constexpr,
|
| 483 |
+
# stride
|
| 484 |
+
stride_o_c,
|
| 485 |
+
stride_o_b,
|
| 486 |
+
stride_o_h,
|
| 487 |
+
stride_o_d,
|
| 488 |
+
stride_l_c,
|
| 489 |
+
stride_l_b,
|
| 490 |
+
stride_l_h,
|
| 491 |
+
NUM_KV_CHUNKS: tl.constexpr,
|
| 492 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 493 |
+
):
|
| 494 |
+
pid_b, pid_h = tl.program_id(0), tl.program_id(1)
|
| 495 |
+
# recompute chunk layout (must match _decode_score_attn_kernel)
|
| 496 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 497 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 498 |
+
chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS)
|
| 499 |
+
valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks)
|
| 500 |
+
# ptrs
|
| 501 |
+
off_c = tl.arange(0, NUM_KV_CHUNKS)
|
| 502 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 503 |
+
o_ptrs = tl.make_block_ptr(
|
| 504 |
+
base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 505 |
+
shape=(valid_chunks, head_dim),
|
| 506 |
+
strides=(stride_o_c, stride_o_d),
|
| 507 |
+
offsets=(0, 0),
|
| 508 |
+
block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D),
|
| 509 |
+
order=(1, 0),
|
| 510 |
+
)
|
| 511 |
+
lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
|
| 512 |
+
# load o and lse
|
| 513 |
+
o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 514 |
+
lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf"))
|
| 515 |
+
# merge o (lse is in log2 scale, use exp2/log2 for efficiency)
|
| 516 |
+
lse_shifted = lse - tl.max(lse, axis=0)
|
| 517 |
+
log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0))
|
| 518 |
+
scale = tl.exp2(lse_shifted - log_sum)
|
| 519 |
+
o = tl.sum(o * scale[:, None], axis=0)
|
| 520 |
+
# save o
|
| 521 |
+
o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
|
| 522 |
+
tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
|
| 523 |
+
|
| 524 |
+
|
| 525 |
+
@triton.jit
|
| 526 |
+
def _compare_and_swap(
|
| 527 |
+
x,
|
| 528 |
+
ids,
|
| 529 |
+
flip,
|
| 530 |
+
i: tl.constexpr,
|
| 531 |
+
n_dims: tl.constexpr,
|
| 532 |
+
):
|
| 533 |
+
n_outer: tl.constexpr = x.numel >> n_dims
|
| 534 |
+
shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
|
| 535 |
+
y = tl.reshape(x, shape)
|
| 536 |
+
# slice left/right with 'stride' 2**(n_dims - i - 1)
|
| 537 |
+
mask = tl.arange(0, 2)[None, :, None]
|
| 538 |
+
left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
|
| 539 |
+
right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
|
| 540 |
+
left = tl.reshape(left, x.shape)
|
| 541 |
+
right = tl.reshape(right, x.shape)
|
| 542 |
+
# idx
|
| 543 |
+
y_idx = tl.reshape(ids, shape)
|
| 544 |
+
left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
|
| 545 |
+
right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
|
| 546 |
+
left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
|
| 547 |
+
right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
|
| 548 |
+
# actual compare-and-swap
|
| 549 |
+
idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
|
| 550 |
+
ileft = left.to(idtype, bitcast=True)
|
| 551 |
+
iright = right.to(idtype, bitcast=True)
|
| 552 |
+
ix = x.to(idtype, bitcast=True)
|
| 553 |
+
|
| 554 |
+
cond = (left > right) != flip
|
| 555 |
+
ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
|
| 556 |
+
new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
|
| 557 |
+
return ret.to(x.dtype, bitcast=True), new_ids
|
| 558 |
+
|
| 559 |
+
|
| 560 |
+
@triton.jit
|
| 561 |
+
def _bitonic_merge(
|
| 562 |
+
x,
|
| 563 |
+
ids,
|
| 564 |
+
stage: tl.constexpr,
|
| 565 |
+
order: tl.constexpr,
|
| 566 |
+
n_dims: tl.constexpr,
|
| 567 |
+
):
|
| 568 |
+
n_outer: tl.constexpr = x.numel >> n_dims
|
| 569 |
+
tl.static_assert(stage <= n_dims)
|
| 570 |
+
# flip denotes whether to re-arrange sub-sequences of elements in ascending or
|
| 571 |
+
# descending order.
|
| 572 |
+
# if flip = 00000000... then all elements will be re-arranged ascendingly at this stage
|
| 573 |
+
# if flip = 00110011... then all the elements will be re-arranged alternatingly (with
|
| 574 |
+
# a stride of 2) at this stage
|
| 575 |
+
if order == 2:
|
| 576 |
+
shape: tl.constexpr = [
|
| 577 |
+
n_outer * 2 ** (n_dims - 1 - stage),
|
| 578 |
+
2,
|
| 579 |
+
2**stage,
|
| 580 |
+
]
|
| 581 |
+
flip = tl.reshape(
|
| 582 |
+
tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
|
| 583 |
+
)
|
| 584 |
+
else:
|
| 585 |
+
flip = order
|
| 586 |
+
# perform `stage` rounds of `compare-and-swap`
|
| 587 |
+
for i in tl.static_range(stage):
|
| 588 |
+
x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
|
| 589 |
+
return x, ids
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
@triton.heuristics(
|
| 593 |
+
{
|
| 594 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
|
| 595 |
+
}
|
| 596 |
+
)
|
| 597 |
+
@triton.autotune(
|
| 598 |
+
configs=[
|
| 599 |
+
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
|
| 600 |
+
triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2),
|
| 601 |
+
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
|
| 602 |
+
triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3),
|
| 603 |
+
triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
|
| 604 |
+
],
|
| 605 |
+
key=["topk"],
|
| 606 |
+
)
|
| 607 |
+
@triton.jit
|
| 608 |
+
def _topk_index_partial_kernel(
|
| 609 |
+
s_ptr, # Score: [num_q_heads, BS, max_seqblock]
|
| 610 |
+
ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
|
| 611 |
+
ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape
|
| 612 |
+
seq_lens,
|
| 613 |
+
block_size: tl.constexpr,
|
| 614 |
+
topk: tl.constexpr,
|
| 615 |
+
chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns
|
| 616 |
+
# strides
|
| 617 |
+
stride_s_h,
|
| 618 |
+
stride_s_b,
|
| 619 |
+
stride_s_k,
|
| 620 |
+
stride_ts_c,
|
| 621 |
+
stride_ts_h,
|
| 622 |
+
stride_ts_b,
|
| 623 |
+
stride_ts_t,
|
| 624 |
+
stride_ti_c,
|
| 625 |
+
stride_ti_h,
|
| 626 |
+
stride_ti_b,
|
| 627 |
+
stride_ti_t,
|
| 628 |
+
# META
|
| 629 |
+
BLOCK_SIZE_K: tl.constexpr,
|
| 630 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 631 |
+
):
|
| 632 |
+
tl.static_assert(BLOCK_SIZE_K > topk)
|
| 633 |
+
pid_b = tl.program_id(0)
|
| 634 |
+
pid_h = tl.program_id(1)
|
| 635 |
+
pid_chunk = tl.program_id(2)
|
| 636 |
+
|
| 637 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 638 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 639 |
+
|
| 640 |
+
# Slice this chunk owns within [0, num_blocks).
|
| 641 |
+
chunk_start = pid_chunk * chunk_blocks
|
| 642 |
+
chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
|
| 643 |
+
chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
|
| 644 |
+
|
| 645 |
+
off_k = tl.arange(0, BLOCK_SIZE_K)
|
| 646 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 647 |
+
|
| 648 |
+
# Pointer starts at chunk_start.
|
| 649 |
+
s_ptrs = (
|
| 650 |
+
s_ptr
|
| 651 |
+
+ pid_b * stride_s_b
|
| 652 |
+
+ pid_h * stride_s_h
|
| 653 |
+
+ (chunk_start + off_k) * stride_s_k
|
| 654 |
+
)
|
| 655 |
+
|
| 656 |
+
topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
|
| 657 |
+
topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
|
| 658 |
+
left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
|
| 659 |
+
|
| 660 |
+
# Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
|
| 661 |
+
# chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
|
| 662 |
+
for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
|
| 663 |
+
mask = off_k < chunk_actual - i
|
| 664 |
+
score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
|
| 665 |
+
score = tl.where(score != score, -1e30, score)
|
| 666 |
+
s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
|
| 667 |
+
topk_score, last_topk_score = score, topk_score
|
| 668 |
+
topk_idx, last_topk_idx = (
|
| 669 |
+
tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
|
| 670 |
+
topk_idx,
|
| 671 |
+
)
|
| 672 |
+
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
| 673 |
+
for j in tl.static_range(1, n_dims):
|
| 674 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 675 |
+
topk_score, topk_idx.to(tl.int32), j, 2, n_dims
|
| 676 |
+
)
|
| 677 |
+
if i != 0:
|
| 678 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 679 |
+
topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
|
| 680 |
+
)
|
| 681 |
+
topk_score_new = last_topk_score * left_half_mask + topk_score * (
|
| 682 |
+
1 - left_half_mask
|
| 683 |
+
)
|
| 684 |
+
topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
|
| 685 |
+
1 - left_half_mask
|
| 686 |
+
)
|
| 687 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 688 |
+
topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
|
| 689 |
+
)
|
| 690 |
+
else:
|
| 691 |
+
topk_score, topk_idx = _bitonic_merge(
|
| 692 |
+
topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
|
| 693 |
+
)
|
| 694 |
+
|
| 695 |
+
# Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort).
|
| 696 |
+
topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
| 697 |
+
final_score = tl.sum(
|
| 698 |
+
topk_mask_extract[:, None]
|
| 699 |
+
* tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
| 700 |
+
axis=0,
|
| 701 |
+
)
|
| 702 |
+
final_idx = tl.sum(
|
| 703 |
+
topk_mask_extract[:, None]
|
| 704 |
+
* tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
| 705 |
+
axis=0,
|
| 706 |
+
)
|
| 707 |
+
|
| 708 |
+
# Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry
|
| 709 |
+
# -1e30 / 0 sentinels and lose to real scores in the merge stage.
|
| 710 |
+
ts_ptrs = (
|
| 711 |
+
ts_partial_ptr
|
| 712 |
+
+ pid_chunk * stride_ts_c
|
| 713 |
+
+ pid_b * stride_ts_b
|
| 714 |
+
+ pid_h * stride_ts_h
|
| 715 |
+
+ off_t * stride_ts_t
|
| 716 |
+
)
|
| 717 |
+
ti_ptrs = (
|
| 718 |
+
ti_partial_ptr
|
| 719 |
+
+ pid_chunk * stride_ti_c
|
| 720 |
+
+ pid_b * stride_ti_b
|
| 721 |
+
+ pid_h * stride_ti_h
|
| 722 |
+
+ off_t * stride_ti_t
|
| 723 |
+
)
|
| 724 |
+
tl.store(ts_ptrs, final_score)
|
| 725 |
+
tl.store(ti_ptrs, final_idx)
|
| 726 |
+
|
| 727 |
+
|
| 728 |
+
@triton.heuristics(
|
| 729 |
+
{
|
| 730 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
|
| 731 |
+
"BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
|
| 732 |
+
args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"])
|
| 733 |
+
),
|
| 734 |
+
}
|
| 735 |
+
)
|
| 736 |
+
@triton.jit
|
| 737 |
+
def _topk_index_merge_kernel(
|
| 738 |
+
ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T]
|
| 739 |
+
ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape
|
| 740 |
+
ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk]
|
| 741 |
+
seq_lens,
|
| 742 |
+
block_size: tl.constexpr,
|
| 743 |
+
topk: tl.constexpr,
|
| 744 |
+
# strides
|
| 745 |
+
stride_ts_c,
|
| 746 |
+
stride_ts_h,
|
| 747 |
+
stride_ts_b,
|
| 748 |
+
stride_ts_t,
|
| 749 |
+
stride_ti_c,
|
| 750 |
+
stride_ti_h,
|
| 751 |
+
stride_ti_b,
|
| 752 |
+
stride_ti_t,
|
| 753 |
+
stride_tif_h,
|
| 754 |
+
stride_tif_b,
|
| 755 |
+
stride_tif_t,
|
| 756 |
+
# META
|
| 757 |
+
NUM_TOPK_CHUNKS: tl.constexpr,
|
| 758 |
+
BLOCK_SIZE_K: tl.constexpr,
|
| 759 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 760 |
+
):
|
| 761 |
+
pid_b = tl.program_id(0)
|
| 762 |
+
pid_h = tl.program_id(1)
|
| 763 |
+
|
| 764 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 765 |
+
num_blocks = (seq_len + block_size - 1) // block_size
|
| 766 |
+
|
| 767 |
+
# Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K.
|
| 768 |
+
# candidate at flat position p comes from chunk = p // BLOCK_SIZE_T,
|
| 769 |
+
# in_chunk = p % BLOCK_SIZE_T.
|
| 770 |
+
off = tl.arange(0, BLOCK_SIZE_K)
|
| 771 |
+
chunk_idx = off // BLOCK_SIZE_T
|
| 772 |
+
in_chunk_idx = off % BLOCK_SIZE_T
|
| 773 |
+
valid = chunk_idx < NUM_TOPK_CHUNKS
|
| 774 |
+
|
| 775 |
+
score_offset = (
|
| 776 |
+
chunk_idx * stride_ts_c
|
| 777 |
+
+ pid_h * stride_ts_h
|
| 778 |
+
+ pid_b * stride_ts_b
|
| 779 |
+
+ in_chunk_idx * stride_ts_t
|
| 780 |
+
)
|
| 781 |
+
idx_offset = (
|
| 782 |
+
chunk_idx * stride_ti_c
|
| 783 |
+
+ pid_h * stride_ti_h
|
| 784 |
+
+ pid_b * stride_ti_b
|
| 785 |
+
+ in_chunk_idx * stride_ti_t
|
| 786 |
+
)
|
| 787 |
+
|
| 788 |
+
score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
|
| 789 |
+
tl.float32
|
| 790 |
+
)
|
| 791 |
+
score = tl.where(score != score, -1e30, score)
|
| 792 |
+
idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
|
| 793 |
+
|
| 794 |
+
# Full bitonic descending sort of BLOCK_SIZE_K items.
|
| 795 |
+
n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
|
| 796 |
+
for j in tl.static_range(1, n_dims):
|
| 797 |
+
score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
|
| 798 |
+
score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
|
| 799 |
+
|
| 800 |
+
# Extract first BLOCK_SIZE_T positions — these are the global top-K.
|
| 801 |
+
extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
|
| 802 |
+
topk_idx_final = tl.sum(
|
| 803 |
+
extract_mask[:, None]
|
| 804 |
+
* tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
|
| 805 |
+
axis=0,
|
| 806 |
+
)
|
| 807 |
+
|
| 808 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 809 |
+
tif_ptrs = (
|
| 810 |
+
ti_final_ptr
|
| 811 |
+
+ pid_h * stride_tif_h
|
| 812 |
+
+ pid_b * stride_tif_b
|
| 813 |
+
+ off_t * stride_tif_t
|
| 814 |
+
)
|
| 815 |
+
topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
|
| 816 |
+
tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty))
|
| 817 |
+
|
| 818 |
+
|
| 819 |
+
@torch.no_grad()
|
| 820 |
+
def flash_decode_with_topk_idx(
|
| 821 |
+
q: torch.Tensor, # [batch_size, num_heads, head_dim]
|
| 822 |
+
sink: Optional[torch.Tensor],
|
| 823 |
+
k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
|
| 824 |
+
v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True
|
| 825 |
+
req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
|
| 826 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 827 |
+
max_seqlen: int,
|
| 828 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 829 |
+
block_size: int,
|
| 830 |
+
topk: int,
|
| 831 |
+
init_blocks: int,
|
| 832 |
+
local_blocks: int,
|
| 833 |
+
sm_scale: Optional[float] = None,
|
| 834 |
+
use_tma: bool = True,
|
| 835 |
+
score_type: str = "max",
|
| 836 |
+
disable_index_value: bool = False,
|
| 837 |
+
use_dense_main_attn: bool = False, # NOTE: need transform idx in this case
|
| 838 |
+
page_size: int = 1,
|
| 839 |
+
) -> torch.Tensor:
|
| 840 |
+
assert score_type in (
|
| 841 |
+
"max",
|
| 842 |
+
"lse",
|
| 843 |
+
), f"score_type must be 'max' or 'lse', got {score_type!r}"
|
| 844 |
+
triton.set_allocator(robust_allocator)
|
| 845 |
+
# dtype check
|
| 846 |
+
assert (
|
| 847 |
+
q.dtype == torch.bfloat16
|
| 848 |
+
or q.dtype == torch.float16
|
| 849 |
+
and k_cache.dtype == q.dtype
|
| 850 |
+
)
|
| 851 |
+
if not disable_index_value:
|
| 852 |
+
assert v_cache is not None
|
| 853 |
+
# shape
|
| 854 |
+
batch_size, num_q_heads, head_dim = q.shape
|
| 855 |
+
max_slots, num_kv_heads, _ = k_cache.shape
|
| 856 |
+
max_kv_len = req_to_token.shape[1]
|
| 857 |
+
assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
|
| 858 |
+
# gqa
|
| 859 |
+
assert num_q_heads % num_kv_heads == 0
|
| 860 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 861 |
+
# sm scale
|
| 862 |
+
if sm_scale is None:
|
| 863 |
+
sm_scale = head_dim**-0.5
|
| 864 |
+
# NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets.
|
| 865 |
+
# Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads.
|
| 866 |
+
# TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by:
|
| 867 |
+
# NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS)
|
| 868 |
+
# then rounded down to the nearest power of 2 (merge kernel requires it).
|
| 869 |
+
# Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len.
|
| 870 |
+
# Empty chunks early-return cheaply, so over-chunking is nearly free.
|
| 871 |
+
# E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096.
|
| 872 |
+
TARGET_GRID = 4096
|
| 873 |
+
MAX_NUM_KV_CHUNKS = 256
|
| 874 |
+
target = max(
|
| 875 |
+
1,
|
| 876 |
+
min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)),
|
| 877 |
+
)
|
| 878 |
+
NUM_KV_CHUNKS = 1 << (target.bit_length() - 1)
|
| 879 |
+
score_kv_len = min(max_seqlen, max_kv_len)
|
| 880 |
+
# The score producers below write every valid block column
|
| 881 |
+
# [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers
|
| 882 |
+
# clamp their scan to the same per-row valid block count, so columns beyond
|
| 883 |
+
# seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph
|
| 884 |
+
# capture the static score shape can be much larger than the live context.
|
| 885 |
+
score = torch.empty(
|
| 886 |
+
(num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)),
|
| 887 |
+
dtype=torch.float32,
|
| 888 |
+
device=q.device,
|
| 889 |
+
)
|
| 890 |
+
use_jit_topk = (
|
| 891 |
+
envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get()
|
| 892 |
+
and score.shape[2] <= 4096
|
| 893 |
+
and topk <= 32
|
| 894 |
+
)
|
| 895 |
+
# If the live context has <= topk sparse blocks, the downstream dense
|
| 896 |
+
# page-table/JIT top-k kernels select every block from seq_lens directly
|
| 897 |
+
# without reading score. Keep this gate in sync with the consumers below:
|
| 898 |
+
# _skip_block_topk/use_dense_main_attn and use_jit_topk special-case
|
| 899 |
+
# num_blocks <= topk, while the Triton fallback reads score and must not
|
| 900 |
+
# skip these writes.
|
| 901 |
+
skip_trivial_topk_score = use_dense_main_attn or use_jit_topk
|
| 902 |
+
|
| 903 |
+
grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads)
|
| 904 |
+
if disable_index_value:
|
| 905 |
+
_decode_score_kernel[grid](
|
| 906 |
+
q,
|
| 907 |
+
k_cache,
|
| 908 |
+
req_to_token,
|
| 909 |
+
score,
|
| 910 |
+
seq_lens,
|
| 911 |
+
slot_ids,
|
| 912 |
+
max_slots,
|
| 913 |
+
batch_size,
|
| 914 |
+
gqa_group_size,
|
| 915 |
+
head_dim,
|
| 916 |
+
block_size,
|
| 917 |
+
topk,
|
| 918 |
+
sm_scale,
|
| 919 |
+
init_blocks,
|
| 920 |
+
local_blocks,
|
| 921 |
+
q.stride(0),
|
| 922 |
+
q.stride(1),
|
| 923 |
+
q.stride(2),
|
| 924 |
+
k_cache.stride(0),
|
| 925 |
+
k_cache.stride(1),
|
| 926 |
+
k_cache.stride(2),
|
| 927 |
+
req_to_token.stride(0),
|
| 928 |
+
score.stride(0),
|
| 929 |
+
score.stride(1),
|
| 930 |
+
score.stride(2),
|
| 931 |
+
NUM_KV_CHUNKS=NUM_KV_CHUNKS,
|
| 932 |
+
SCORE_TYPE=score_type,
|
| 933 |
+
SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
|
| 934 |
+
)
|
| 935 |
+
else:
|
| 936 |
+
assert v_cache is not None
|
| 937 |
+
o = torch.empty(
|
| 938 |
+
NUM_KV_CHUNKS,
|
| 939 |
+
batch_size,
|
| 940 |
+
num_q_heads,
|
| 941 |
+
head_dim,
|
| 942 |
+
dtype=q.dtype,
|
| 943 |
+
device=q.device,
|
| 944 |
+
)
|
| 945 |
+
lse = torch.empty(
|
| 946 |
+
NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device
|
| 947 |
+
)
|
| 948 |
+
_decode_score_attn_kernel[grid](
|
| 949 |
+
q,
|
| 950 |
+
sink,
|
| 951 |
+
k_cache,
|
| 952 |
+
v_cache,
|
| 953 |
+
req_to_token,
|
| 954 |
+
o,
|
| 955 |
+
lse,
|
| 956 |
+
score,
|
| 957 |
+
seq_lens,
|
| 958 |
+
slot_ids,
|
| 959 |
+
max_slots,
|
| 960 |
+
batch_size,
|
| 961 |
+
gqa_group_size,
|
| 962 |
+
head_dim,
|
| 963 |
+
block_size,
|
| 964 |
+
topk,
|
| 965 |
+
sm_scale,
|
| 966 |
+
init_blocks,
|
| 967 |
+
local_blocks,
|
| 968 |
+
q.stride(0),
|
| 969 |
+
q.stride(1),
|
| 970 |
+
q.stride(2),
|
| 971 |
+
sink.stride(0) if sink is not None else 0,
|
| 972 |
+
sink.stride(1) if sink is not None else 0,
|
| 973 |
+
k_cache.stride(0),
|
| 974 |
+
k_cache.stride(1),
|
| 975 |
+
k_cache.stride(2),
|
| 976 |
+
v_cache.stride(0),
|
| 977 |
+
v_cache.stride(1),
|
| 978 |
+
v_cache.stride(2),
|
| 979 |
+
req_to_token.stride(0),
|
| 980 |
+
o.stride(0),
|
| 981 |
+
o.stride(1),
|
| 982 |
+
o.stride(2),
|
| 983 |
+
o.stride(3),
|
| 984 |
+
lse.stride(0),
|
| 985 |
+
lse.stride(1),
|
| 986 |
+
lse.stride(2),
|
| 987 |
+
score.stride(0),
|
| 988 |
+
score.stride(1),
|
| 989 |
+
score.stride(2),
|
| 990 |
+
NUM_KV_CHUNKS=NUM_KV_CHUNKS,
|
| 991 |
+
SCORE_TYPE=score_type,
|
| 992 |
+
SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score,
|
| 993 |
+
)
|
| 994 |
+
# Fused top-k + page-table transform: emit the dense backend's page table
|
| 995 |
+
# directly (page-size-aware) instead of block ids, skipping a separate gather.
|
| 996 |
+
# The page table + per-query effective KV length are allocated and returned.
|
| 997 |
+
real_seq_lens = None
|
| 998 |
+
if use_dense_main_attn:
|
| 999 |
+
# VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel
|
| 1000 |
+
# `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The
|
| 1001 |
+
# standalone vendor only ships the pure-Triton path, so this is gated off
|
| 1002 |
+
# by default (use_dense_main_attn=False). If you opt in, vendor that .cuh
|
| 1003 |
+
# too. Original line: `from sglang.jit_kernel.minimax_decode_topk import (
|
| 1004 |
+
# minimax_decode_topk_page_table)`.
|
| 1005 |
+
raise NotImplementedError(
|
| 1006 |
+
"use_dense_main_attn=True requires the un-vendored CUDA JIT kernel "
|
| 1007 |
+
"sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; "
|
| 1008 |
+
"the standalone Triton vendor only supports use_dense_main_attn=False."
|
| 1009 |
+
)
|
| 1010 |
+
# num_q_heads (= num index/kv heads) may be > 1 for DP attention; the
|
| 1011 |
+
# kernel emits a flattened [batch*num_q_heads] page table + seq_lens with
|
| 1012 |
+
# the kv head head-encoded (head-minor) into the page index.
|
| 1013 |
+
topk_idx, real_seq_lens = minimax_decode_topk_page_table(
|
| 1014 |
+
score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size
|
| 1015 |
+
)
|
| 1016 |
+
if disable_index_value:
|
| 1017 |
+
return None, topk_idx, real_seq_lens
|
| 1018 |
+
# fall through to the idx_o (index value) merge, then return
|
| 1019 |
+
_skip_block_topk = True
|
| 1020 |
+
else:
|
| 1021 |
+
_skip_block_topk = False
|
| 1022 |
+
|
| 1023 |
+
# get topk index
|
| 1024 |
+
if not _skip_block_topk:
|
| 1025 |
+
topk_idx = torch.empty(
|
| 1026 |
+
(num_q_heads, batch_size, topk),
|
| 1027 |
+
device=score.device,
|
| 1028 |
+
dtype=torch.int32,
|
| 1029 |
+
)
|
| 1030 |
+
|
| 1031 |
+
if _skip_block_topk:
|
| 1032 |
+
pass
|
| 1033 |
+
elif use_jit_topk:
|
| 1034 |
+
# Single-stage JIT radix-select: one kernel, no intermediate buffers.
|
| 1035 |
+
# Equivalent output to the 2-stage path (set of block ids, front-packed,
|
| 1036 |
+
# -1 padded); ~2-16x faster for long context.
|
| 1037 |
+
# VENDOR NOTE: this needs the un-vendored CUDA JIT kernel
|
| 1038 |
+
# `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the
|
| 1039 |
+
# `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so
|
| 1040 |
+
# this branch is never taken in the standalone vendor (the 2-stage Triton
|
| 1041 |
+
# `else` fallback below runs instead). Original line:
|
| 1042 |
+
# `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`.
|
| 1043 |
+
raise NotImplementedError(
|
| 1044 |
+
"use_jit_topk requires the un-vendored CUDA JIT kernel "
|
| 1045 |
+
"sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor "
|
| 1046 |
+
"uses the 2-stage Triton fallback (keep "
|
| 1047 |
+
"SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)."
|
| 1048 |
+
)
|
| 1049 |
+
elif has_native_ops() and topk <= 64:
|
| 1050 |
+
topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk)
|
| 1051 |
+
else:
|
| 1052 |
+
# 2-stage split-K Triton fallback.
|
| 1053 |
+
# Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim.
|
| 1054 |
+
# Same constraints as flash_decode: must be deterministic from values
|
| 1055 |
+
# constant within a cuda graph (BS, num_q_heads), pow2, capped so the
|
| 1056 |
+
# merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk))
|
| 1057 |
+
# stays reasonable. With topk=32 and cap 16, merge sorts 512 items.
|
| 1058 |
+
TOPK_TARGET_GRID = 64
|
| 1059 |
+
MAX_NUM_TOPK_CHUNKS = 16
|
| 1060 |
+
topk_target = max(
|
| 1061 |
+
1,
|
| 1062 |
+
min(
|
| 1063 |
+
MAX_NUM_TOPK_CHUNKS,
|
| 1064 |
+
TOPK_TARGET_GRID // max(1, batch_size * num_q_heads),
|
| 1065 |
+
),
|
| 1066 |
+
)
|
| 1067 |
+
NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1)
|
| 1068 |
+
BLOCK_SIZE_T = triton.next_power_of_2(topk)
|
| 1069 |
+
max_seqblock = score.shape[2]
|
| 1070 |
+
chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
|
| 1071 |
+
topk_score_partial = torch.empty(
|
| 1072 |
+
NUM_TOPK_CHUNKS,
|
| 1073 |
+
num_q_heads,
|
| 1074 |
+
batch_size,
|
| 1075 |
+
BLOCK_SIZE_T,
|
| 1076 |
+
dtype=torch.float32,
|
| 1077 |
+
device=score.device,
|
| 1078 |
+
)
|
| 1079 |
+
topk_idx_partial = torch.empty(
|
| 1080 |
+
NUM_TOPK_CHUNKS,
|
| 1081 |
+
num_q_heads,
|
| 1082 |
+
batch_size,
|
| 1083 |
+
BLOCK_SIZE_T,
|
| 1084 |
+
dtype=torch.int32,
|
| 1085 |
+
device=score.device,
|
| 1086 |
+
)
|
| 1087 |
+
# stream 0 (default): topk partial → topk merge
|
| 1088 |
+
grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS)
|
| 1089 |
+
_topk_index_partial_kernel[grid](
|
| 1090 |
+
score,
|
| 1091 |
+
topk_score_partial,
|
| 1092 |
+
topk_idx_partial,
|
| 1093 |
+
seq_lens,
|
| 1094 |
+
block_size,
|
| 1095 |
+
topk,
|
| 1096 |
+
chunk_blocks,
|
| 1097 |
+
score.stride(0),
|
| 1098 |
+
score.stride(1),
|
| 1099 |
+
score.stride(2),
|
| 1100 |
+
topk_score_partial.stride(0),
|
| 1101 |
+
topk_score_partial.stride(1),
|
| 1102 |
+
topk_score_partial.stride(2),
|
| 1103 |
+
topk_score_partial.stride(3),
|
| 1104 |
+
topk_idx_partial.stride(0),
|
| 1105 |
+
topk_idx_partial.stride(1),
|
| 1106 |
+
topk_idx_partial.stride(2),
|
| 1107 |
+
topk_idx_partial.stride(3),
|
| 1108 |
+
)
|
| 1109 |
+
grid = (batch_size, num_q_heads)
|
| 1110 |
+
_topk_index_merge_kernel[grid](
|
| 1111 |
+
topk_score_partial,
|
| 1112 |
+
topk_idx_partial,
|
| 1113 |
+
topk_idx,
|
| 1114 |
+
seq_lens,
|
| 1115 |
+
block_size,
|
| 1116 |
+
topk,
|
| 1117 |
+
topk_score_partial.stride(0),
|
| 1118 |
+
topk_score_partial.stride(1),
|
| 1119 |
+
topk_score_partial.stride(2),
|
| 1120 |
+
topk_score_partial.stride(3),
|
| 1121 |
+
topk_idx_partial.stride(0),
|
| 1122 |
+
topk_idx_partial.stride(1),
|
| 1123 |
+
topk_idx_partial.stride(2),
|
| 1124 |
+
topk_idx_partial.stride(3),
|
| 1125 |
+
topk_idx.stride(0),
|
| 1126 |
+
topk_idx.stride(1),
|
| 1127 |
+
topk_idx.stride(2),
|
| 1128 |
+
NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
|
| 1129 |
+
)
|
| 1130 |
+
if disable_index_value:
|
| 1131 |
+
return None, topk_idx, real_seq_lens
|
| 1132 |
+
# attn output merge (default stream)
|
| 1133 |
+
grid = (batch_size, num_q_heads)
|
| 1134 |
+
_merge_attn_out_kernel[grid](
|
| 1135 |
+
o,
|
| 1136 |
+
lse,
|
| 1137 |
+
seq_lens,
|
| 1138 |
+
head_dim,
|
| 1139 |
+
block_size,
|
| 1140 |
+
o.stride(0),
|
| 1141 |
+
o.stride(1),
|
| 1142 |
+
o.stride(2),
|
| 1143 |
+
o.stride(3),
|
| 1144 |
+
lse.stride(0),
|
| 1145 |
+
lse.stride(1),
|
| 1146 |
+
lse.stride(2),
|
| 1147 |
+
NUM_KV_CHUNKS=NUM_KV_CHUNKS,
|
| 1148 |
+
)
|
| 1149 |
+
o = o[0].contiguous()
|
| 1150 |
+
return o, topk_idx, real_seq_lens
|
build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py
ADDED
|
@@ -0,0 +1,452 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
|
| 10 |
+
|
| 11 |
+
from .._native import (
|
| 12 |
+
native_decode_mma_supported,
|
| 13 |
+
native_decode_sparse_attn_mma_paged,
|
| 14 |
+
)
|
| 15 |
+
from ..common.utils import robust_allocator
|
| 16 |
+
|
| 17 |
+
_is_hip = is_hip()
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@triton.heuristics(
|
| 21 |
+
{
|
| 22 |
+
"BLOCK_SIZE_H": lambda args: max(
|
| 23 |
+
16, triton.next_power_of_2(args["gqa_group_size"])
|
| 24 |
+
),
|
| 25 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 26 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
| 27 |
+
"HAS_SINK": lambda args: args["sink_ptr"] is not None,
|
| 28 |
+
"BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]),
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
@triton.autotune(
|
| 32 |
+
configs=[
|
| 33 |
+
triton.Config({}, num_warps=nw, num_stages=ns)
|
| 34 |
+
for nw in [4, 8]
|
| 35 |
+
for ns in [2, 3, 4, 5]
|
| 36 |
+
],
|
| 37 |
+
key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"],
|
| 38 |
+
)
|
| 39 |
+
@triton.jit
|
| 40 |
+
def _gqa_share_sparse_decode_kernel(
|
| 41 |
+
q_ptr, # Q: b x qh x d
|
| 42 |
+
sink_ptr, # Sink: qh x d
|
| 43 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 44 |
+
v_cache_ptr, # V paged: max_slots x kh x d
|
| 45 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 46 |
+
idx_ptr, # topk index: qh x b x topk
|
| 47 |
+
o_ptr, # O partial: c x b x qh x d
|
| 48 |
+
lse_ptr, # lse partial: c x b x qh
|
| 49 |
+
seq_lens,
|
| 50 |
+
slot_ids,
|
| 51 |
+
# shape
|
| 52 |
+
max_slots,
|
| 53 |
+
batch_size,
|
| 54 |
+
gqa_group_size,
|
| 55 |
+
head_dim,
|
| 56 |
+
max_topk,
|
| 57 |
+
max_kv_len,
|
| 58 |
+
# sm_scale
|
| 59 |
+
sm_scale,
|
| 60 |
+
# stride
|
| 61 |
+
stride_q_b,
|
| 62 |
+
stride_q_h,
|
| 63 |
+
stride_q_d,
|
| 64 |
+
stride_sink_h,
|
| 65 |
+
stride_sink_d,
|
| 66 |
+
stride_k_s,
|
| 67 |
+
stride_k_h,
|
| 68 |
+
stride_k_d,
|
| 69 |
+
stride_v_s,
|
| 70 |
+
stride_v_h,
|
| 71 |
+
stride_v_d,
|
| 72 |
+
stride_r2t_b,
|
| 73 |
+
stride_ti_h,
|
| 74 |
+
stride_ti_b,
|
| 75 |
+
stride_ti_t,
|
| 76 |
+
stride_o_c,
|
| 77 |
+
stride_o_b,
|
| 78 |
+
stride_o_h,
|
| 79 |
+
stride_o_d,
|
| 80 |
+
stride_l_c,
|
| 81 |
+
stride_l_b,
|
| 82 |
+
stride_l_h,
|
| 83 |
+
# META parameters
|
| 84 |
+
BATCH_SIZE_BUCKET: tl.constexpr,
|
| 85 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 86 |
+
BLOCK_SIZE_N: tl.constexpr,
|
| 87 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 88 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 89 |
+
NUM_TOPK_CHUNKS: tl.constexpr,
|
| 90 |
+
HAS_SINK: tl.constexpr,
|
| 91 |
+
IS_FP8: tl.constexpr,
|
| 92 |
+
):
|
| 93 |
+
# decode program ids: split-K over the topk dimension to give every SM
|
| 94 |
+
# something to do at small batch. pid(0) folds (batch, chunk) together so
|
| 95 |
+
# the grid size = batch_size * NUM_TOPK_CHUNKS.
|
| 96 |
+
pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
|
| 97 |
+
pid_b = pid_bc % batch_size
|
| 98 |
+
pid_c = pid_bc // batch_size
|
| 99 |
+
pid_h = pid_kh * gqa_group_size
|
| 100 |
+
# per-chunk topk range. chunk_size is *runtime* (depends on max_topk which
|
| 101 |
+
# is a runtime arg, not constexpr), so don't annotate as tl.constexpr —
|
| 102 |
+
# doing so produces undefined behavior in Triton.
|
| 103 |
+
chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
|
| 104 |
+
chunk_start_topk = pid_c * chunk_size_topk
|
| 105 |
+
chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk
|
| 106 |
+
# get q k start and len after rmpad
|
| 107 |
+
seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len)
|
| 108 |
+
sid = (
|
| 109 |
+
tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
|
| 110 |
+
) % max_slots # to avoid bugs when slot_ids is negative
|
| 111 |
+
# get real topk
|
| 112 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 113 |
+
idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b
|
| 114 |
+
topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1)
|
| 115 |
+
valid_idx = tl.where(topk_idx >= 0, off_t, -1)
|
| 116 |
+
real_topk = tl.sum(valid_idx != -1, axis=0)
|
| 117 |
+
chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk)
|
| 118 |
+
# init pointer
|
| 119 |
+
off_n = tl.arange(0, BLOCK_SIZE_N)
|
| 120 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 121 |
+
dim_mask = off_d < head_dim
|
| 122 |
+
# init statistics — kept at -inf so empty chunks (chunk_start >= real_topk)
|
| 123 |
+
# naturally fall out as weight=0 in the merge step.
|
| 124 |
+
# q is always needed; load it unconditionally.
|
| 125 |
+
q_ptrs = tl.make_block_ptr(
|
| 126 |
+
base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h,
|
| 127 |
+
shape=(gqa_group_size, head_dim),
|
| 128 |
+
strides=(stride_q_h, stride_q_d),
|
| 129 |
+
offsets=(0, 0),
|
| 130 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 131 |
+
order=(1, 0),
|
| 132 |
+
)
|
| 133 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 134 |
+
m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 135 |
+
lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
|
| 136 |
+
# Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code-
|
| 137 |
+
# eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is
|
| 138 |
+
# then None). The original compound `if HAS_SINK and pid_c == 0` traced
|
| 139 |
+
# the body even when False -> NoneType make_block_ptr compile error on
|
| 140 |
+
# sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always
|
| 141 |
+
# False for us, but this keeps the sink path correct too.
|
| 142 |
+
if HAS_SINK:
|
| 143 |
+
if pid_c == 0:
|
| 144 |
+
sink_ptrs = tl.make_block_ptr(
|
| 145 |
+
base=sink_ptr + pid_h * stride_sink_h,
|
| 146 |
+
shape=(gqa_group_size, head_dim),
|
| 147 |
+
strides=(stride_sink_h, stride_sink_d),
|
| 148 |
+
offsets=(0, 0),
|
| 149 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 150 |
+
order=(1, 0),
|
| 151 |
+
)
|
| 152 |
+
sink = tl.load(
|
| 153 |
+
sink_ptrs, boundary_check=(0, 1), padding_option="zero"
|
| 154 |
+
).to(tl.float32)
|
| 155 |
+
qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale
|
| 156 |
+
m_i = qsink
|
| 157 |
+
lse_i = qsink
|
| 158 |
+
acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32)
|
| 159 |
+
# only iterate over this chunk's topk slice. the load must respect the
|
| 160 |
+
# per-chunk start offset.
|
| 161 |
+
cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t
|
| 162 |
+
for _ in tl.range(chunk_start_topk, chunk_end_topk):
|
| 163 |
+
# load index
|
| 164 |
+
c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N
|
| 165 |
+
cur_idx_ptr = cur_idx_ptr + stride_ti_t
|
| 166 |
+
# resolve slots for this block via req_to_token
|
| 167 |
+
pos = c + off_n
|
| 168 |
+
pos_mask = pos < seq_len
|
| 169 |
+
slots = tl.load(
|
| 170 |
+
req_to_token_ptr + sid * stride_r2t_b + pos,
|
| 171 |
+
mask=pos_mask,
|
| 172 |
+
other=0,
|
| 173 |
+
).to(tl.int64)
|
| 174 |
+
slots = (slots + max_slots) % max_slots # safety against negative
|
| 175 |
+
# load K as (head_dim, BLOCK_SIZE_N) via indirect addressing
|
| 176 |
+
k_off = (
|
| 177 |
+
slots[None, :] * stride_k_s
|
| 178 |
+
+ pid_kh * stride_k_h
|
| 179 |
+
+ off_d[:, None] * stride_k_d
|
| 180 |
+
)
|
| 181 |
+
k = tl.load(
|
| 182 |
+
k_cache_ptr + k_off,
|
| 183 |
+
mask=dim_mask[:, None] & pos_mask[None, :],
|
| 184 |
+
other=0.0,
|
| 185 |
+
)
|
| 186 |
+
if IS_FP8:
|
| 187 |
+
# fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no
|
| 188 |
+
# scale), so dequant is just a widening cast to the Q compute dtype
|
| 189 |
+
# before the tl.dot. Matches the bf16 path bit-for-bit when the cache
|
| 190 |
+
# is bf16 (IS_FP8 False -> this branch is compiled out).
|
| 191 |
+
k = k.to(q.dtype)
|
| 192 |
+
# load V as (BLOCK_SIZE_N, head_dim) via indirect addressing
|
| 193 |
+
v_off = (
|
| 194 |
+
slots[:, None] * stride_v_s
|
| 195 |
+
+ pid_kh * stride_v_h
|
| 196 |
+
+ off_d[None, :] * stride_v_d
|
| 197 |
+
)
|
| 198 |
+
v = tl.load(
|
| 199 |
+
v_cache_ptr + v_off,
|
| 200 |
+
mask=pos_mask[:, None] & dim_mask[None, :],
|
| 201 |
+
other=0.0,
|
| 202 |
+
)
|
| 203 |
+
if IS_FP8:
|
| 204 |
+
# Widen V before the P@V dot. This also makes the `p.to(v.dtype)`
|
| 205 |
+
# below cast P to the compute dtype (not to fp8, which would be
|
| 206 |
+
# catastrophic precision loss on the attention weights).
|
| 207 |
+
v = v.to(q.dtype)
|
| 208 |
+
# compute qk
|
| 209 |
+
qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32)
|
| 210 |
+
qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf"))
|
| 211 |
+
# [H, D], [D, N] -> [H, N]
|
| 212 |
+
qk += tl.dot(q, k) * sm_scale
|
| 213 |
+
# compute m_ij and l_ij
|
| 214 |
+
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
| 215 |
+
p = tl.exp(qk - m_ij[:, None])
|
| 216 |
+
l_ij = tl.sum(p, axis=1)
|
| 217 |
+
# scale acc_o
|
| 218 |
+
acc_o_scale = tl.exp(m_i - m_ij)
|
| 219 |
+
acc_o = acc_o * acc_o_scale[:, None]
|
| 220 |
+
# load v and update acc_o
|
| 221 |
+
p = p.to(v.dtype)
|
| 222 |
+
# [H, N], [N, D] -> [H, D]
|
| 223 |
+
acc_o += tl.dot(p.to(v.dtype), v)
|
| 224 |
+
# update statistics
|
| 225 |
+
m_i = m_ij
|
| 226 |
+
lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij)
|
| 227 |
+
# final scale (matches the old non-split kernel for chunks where lse_i>-inf).
|
| 228 |
+
# For empty chunks (chunk_start_topk >= real_topk) the inner loop never
|
| 229 |
+
# runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute
|
| 230 |
+
# exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial
|
| 231 |
+
# and the merge result. Gate the scale with tl.where so empty chunks emit a
|
| 232 |
+
# clean zero (lse_i stays -inf which the merge correctly turns into weight=0).
|
| 233 |
+
scale = tl.where(
|
| 234 |
+
lse_i > float("-inf"),
|
| 235 |
+
tl.exp(m_i - lse_i),
|
| 236 |
+
tl.zeros_like(lse_i),
|
| 237 |
+
)
|
| 238 |
+
acc_o = acc_o * scale[:, None]
|
| 239 |
+
# save partial output and lse for the merge step
|
| 240 |
+
o_ptrs = tl.make_block_ptr(
|
| 241 |
+
base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 242 |
+
shape=(gqa_group_size, head_dim),
|
| 243 |
+
strides=(stride_o_h, stride_o_d),
|
| 244 |
+
offsets=(0, 0),
|
| 245 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
|
| 246 |
+
order=(1, 0),
|
| 247 |
+
)
|
| 248 |
+
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
|
| 249 |
+
lse_ptrs = tl.make_block_ptr(
|
| 250 |
+
base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
|
| 251 |
+
shape=(gqa_group_size,),
|
| 252 |
+
strides=(stride_l_h,),
|
| 253 |
+
offsets=(0,),
|
| 254 |
+
block_shape=(BLOCK_SIZE_H,),
|
| 255 |
+
order=(0,),
|
| 256 |
+
)
|
| 257 |
+
tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
@triton.heuristics(
|
| 261 |
+
{
|
| 262 |
+
"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
|
| 263 |
+
}
|
| 264 |
+
)
|
| 265 |
+
@triton.jit
|
| 266 |
+
def _merge_topk_attn_out_kernel(
|
| 267 |
+
o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0
|
| 268 |
+
lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH]
|
| 269 |
+
head_dim,
|
| 270 |
+
stride_o_c,
|
| 271 |
+
stride_o_b,
|
| 272 |
+
stride_o_h,
|
| 273 |
+
stride_o_d,
|
| 274 |
+
stride_l_c,
|
| 275 |
+
stride_l_b,
|
| 276 |
+
stride_l_h,
|
| 277 |
+
NUM_TOPK_CHUNKS: tl.constexpr,
|
| 278 |
+
BLOCK_SIZE_D: tl.constexpr,
|
| 279 |
+
):
|
| 280 |
+
pid_b, pid_h = tl.program_id(0), tl.program_id(1)
|
| 281 |
+
off_c = tl.arange(0, NUM_TOPK_CHUNKS)
|
| 282 |
+
off_d = tl.arange(0, BLOCK_SIZE_D)
|
| 283 |
+
o_ptrs = tl.make_block_ptr(
|
| 284 |
+
base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
|
| 285 |
+
shape=(NUM_TOPK_CHUNKS, head_dim),
|
| 286 |
+
strides=(stride_o_c, stride_o_d),
|
| 287 |
+
offsets=(0, 0),
|
| 288 |
+
block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
|
| 289 |
+
order=(1, 0),
|
| 290 |
+
)
|
| 291 |
+
lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
|
| 292 |
+
o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
|
| 293 |
+
lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
|
| 294 |
+
# standard flash-decoding merge in linear (not log2) space, matching the
|
| 295 |
+
# decode kernel which uses tl.exp / tl.log.
|
| 296 |
+
lse_max = tl.max(lse, axis=0)
|
| 297 |
+
weights = tl.exp(lse - lse_max)
|
| 298 |
+
weights = weights / tl.sum(weights, axis=0)
|
| 299 |
+
o_merged = tl.sum(o * weights[:, None], axis=0)
|
| 300 |
+
o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d
|
| 301 |
+
tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim)
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
@torch.no_grad()
|
| 305 |
+
def flash_decode_with_gqa_share_sparse(
|
| 306 |
+
q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
|
| 307 |
+
sink: Optional[torch.Tensor],
|
| 308 |
+
k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
|
| 309 |
+
v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged)
|
| 310 |
+
req_to_token: torch.Tensor, # [max_reqs, max_kv_len]
|
| 311 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 312 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 313 |
+
block_size: int,
|
| 314 |
+
topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
|
| 315 |
+
sm_scale: Optional[float] = None,
|
| 316 |
+
use_tma: bool = True,
|
| 317 |
+
) -> torch.Tensor:
|
| 318 |
+
triton.set_allocator(robust_allocator)
|
| 319 |
+
# dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may
|
| 320 |
+
# be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel
|
| 321 |
+
# widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing
|
| 322 |
+
# sparse decode dtype contract unchanged by accepting fp8 only on HIP.
|
| 323 |
+
assert q.dtype in (torch.bfloat16, torch.float16)
|
| 324 |
+
_FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
|
| 325 |
+
is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
|
| 326 |
+
assert k_cache.dtype == q.dtype or is_fp8, (
|
| 327 |
+
f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) "
|
| 328 |
+
f"or fp8 on HIP, got {k_cache.dtype}"
|
| 329 |
+
)
|
| 330 |
+
assert v_cache.dtype == k_cache.dtype
|
| 331 |
+
# shape
|
| 332 |
+
batch_size, num_q_heads, head_dim = q.shape
|
| 333 |
+
max_slots, num_kv_heads, _ = k_cache.shape
|
| 334 |
+
assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size
|
| 335 |
+
assert topk_idx.shape[0] == num_kv_heads
|
| 336 |
+
assert (
|
| 337 |
+
triton.next_power_of_2(block_size) == block_size
|
| 338 |
+
), f"block_size must be a power of 2, but got {block_size}"
|
| 339 |
+
# assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}"
|
| 340 |
+
max_kv_len = req_to_token.shape[1]
|
| 341 |
+
# gqa
|
| 342 |
+
assert num_q_heads % num_kv_heads == 0
|
| 343 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 344 |
+
max_topk = topk_idx.shape[2]
|
| 345 |
+
# sm scale
|
| 346 |
+
if sm_scale is None:
|
| 347 |
+
sm_scale = head_dim**-0.5
|
| 348 |
+
# Fast path: native tensor-core decode for the M3 MSA shape. Falls through
|
| 349 |
+
# to the Triton kernel for other shapes, sink, or fp8 caches.
|
| 350 |
+
if sink is None and not is_fp8 and native_decode_mma_supported(
|
| 351 |
+
q, k_cache, block_size=block_size
|
| 352 |
+
):
|
| 353 |
+
return native_decode_sparse_attn_mma_paged(
|
| 354 |
+
q,
|
| 355 |
+
k_cache,
|
| 356 |
+
v_cache,
|
| 357 |
+
req_to_token,
|
| 358 |
+
seq_lens,
|
| 359 |
+
slot_ids,
|
| 360 |
+
block_size,
|
| 361 |
+
topk_idx,
|
| 362 |
+
sm_scale,
|
| 363 |
+
)
|
| 364 |
+
# Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as
|
| 365 |
+
# flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must
|
| 366 |
+
# only depend on shape constants (so grid is fixed within a cuda graph).
|
| 367 |
+
# Capped by max_topk because chunks beyond real_topk early-fall-through to
|
| 368 |
+
# the merge-as-zero path; capping avoids wasting blocks at tiny topk.
|
| 369 |
+
TARGET_GRID = 256
|
| 370 |
+
target = max(
|
| 371 |
+
1,
|
| 372 |
+
min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)),
|
| 373 |
+
)
|
| 374 |
+
NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1)
|
| 375 |
+
# output tensor: split-K partials, merged into chunk 0 by the merge kernel
|
| 376 |
+
o_partial = torch.empty(
|
| 377 |
+
NUM_TOPK_CHUNKS,
|
| 378 |
+
batch_size,
|
| 379 |
+
num_q_heads,
|
| 380 |
+
head_dim,
|
| 381 |
+
dtype=q.dtype,
|
| 382 |
+
device=q.device,
|
| 383 |
+
)
|
| 384 |
+
lse_partial = torch.empty(
|
| 385 |
+
NUM_TOPK_CHUNKS,
|
| 386 |
+
batch_size,
|
| 387 |
+
num_q_heads,
|
| 388 |
+
dtype=torch.float32,
|
| 389 |
+
device=q.device,
|
| 390 |
+
)
|
| 391 |
+
# launch attention kernel
|
| 392 |
+
grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads)
|
| 393 |
+
_gqa_share_sparse_decode_kernel[grid](
|
| 394 |
+
q,
|
| 395 |
+
sink,
|
| 396 |
+
k_cache,
|
| 397 |
+
v_cache,
|
| 398 |
+
req_to_token,
|
| 399 |
+
topk_idx,
|
| 400 |
+
o_partial,
|
| 401 |
+
lse_partial,
|
| 402 |
+
seq_lens,
|
| 403 |
+
slot_ids,
|
| 404 |
+
max_slots,
|
| 405 |
+
batch_size,
|
| 406 |
+
gqa_group_size,
|
| 407 |
+
head_dim,
|
| 408 |
+
max_topk,
|
| 409 |
+
max_kv_len,
|
| 410 |
+
sm_scale,
|
| 411 |
+
q.stride(0),
|
| 412 |
+
q.stride(1),
|
| 413 |
+
q.stride(2),
|
| 414 |
+
sink.stride(0) if sink is not None else 0,
|
| 415 |
+
sink.stride(1) if sink is not None else 0,
|
| 416 |
+
k_cache.stride(0),
|
| 417 |
+
k_cache.stride(1),
|
| 418 |
+
k_cache.stride(2),
|
| 419 |
+
v_cache.stride(0),
|
| 420 |
+
v_cache.stride(1),
|
| 421 |
+
v_cache.stride(2),
|
| 422 |
+
req_to_token.stride(0),
|
| 423 |
+
topk_idx.stride(0),
|
| 424 |
+
topk_idx.stride(1),
|
| 425 |
+
topk_idx.stride(2),
|
| 426 |
+
o_partial.stride(0),
|
| 427 |
+
o_partial.stride(1),
|
| 428 |
+
o_partial.stride(2),
|
| 429 |
+
o_partial.stride(3),
|
| 430 |
+
lse_partial.stride(0),
|
| 431 |
+
lse_partial.stride(1),
|
| 432 |
+
lse_partial.stride(2),
|
| 433 |
+
BLOCK_SIZE_N=block_size,
|
| 434 |
+
NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
|
| 435 |
+
IS_FP8=is_fp8,
|
| 436 |
+
)
|
| 437 |
+
# merge partials into chunk 0
|
| 438 |
+
merge_grid = (batch_size, num_q_heads)
|
| 439 |
+
_merge_topk_attn_out_kernel[merge_grid](
|
| 440 |
+
o_partial,
|
| 441 |
+
lse_partial,
|
| 442 |
+
head_dim,
|
| 443 |
+
o_partial.stride(0),
|
| 444 |
+
o_partial.stride(1),
|
| 445 |
+
o_partial.stride(2),
|
| 446 |
+
o_partial.stride(3),
|
| 447 |
+
lse_partial.stride(0),
|
| 448 |
+
lse_partial.stride(1),
|
| 449 |
+
lse_partial.stride(2),
|
| 450 |
+
NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS,
|
| 451 |
+
)
|
| 452 |
+
return o_partial[0].contiguous()
|
build/torch211-cxx11-cu130-x86_64-linux/interface.py
ADDED
|
@@ -0,0 +1,679 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from ._native import (
|
| 11 |
+
has_native_ops,
|
| 12 |
+
native_indexer_block_scores,
|
| 13 |
+
native_nvfp4_dequant_swizzled_to_bf16,
|
| 14 |
+
)
|
| 15 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 16 |
+
from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse
|
| 17 |
+
from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _page_table_to_req_to_token(
|
| 21 |
+
page_table: torch.Tensor,
|
| 22 |
+
seqused_k: torch.Tensor,
|
| 23 |
+
max_seqlen_k: int,
|
| 24 |
+
blk_kv: int,
|
| 25 |
+
) -> torch.Tensor:
|
| 26 |
+
batch = int(page_table.shape[0])
|
| 27 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 28 |
+
blk_kv = int(blk_kv)
|
| 29 |
+
logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
|
| 30 |
+
page = logical_pos // blk_kv
|
| 31 |
+
offset = logical_pos % blk_kv
|
| 32 |
+
page = page.clamp(max=int(page_table.shape[1]) - 1)
|
| 33 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 34 |
+
physical_token = physical_page * blk_kv + offset[None, :]
|
| 35 |
+
valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
|
| 36 |
+
return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _dense_q2k_indices(
|
| 40 |
+
*,
|
| 41 |
+
head_kv: int,
|
| 42 |
+
batch: int,
|
| 43 |
+
topk: int,
|
| 44 |
+
seq_lens: torch.Tensor,
|
| 45 |
+
blk_kv: int,
|
| 46 |
+
) -> torch.Tensor:
|
| 47 |
+
idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
|
| 48 |
+
blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
|
| 49 |
+
valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
|
| 50 |
+
for b in range(batch):
|
| 51 |
+
valid = blocks < valid_blocks[b]
|
| 52 |
+
idx[:, b, valid] = blocks[valid]
|
| 53 |
+
return idx
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]:
|
| 57 |
+
seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist()
|
| 58 |
+
return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def _packed_row_to_batch_block(
|
| 62 |
+
cu_seqlens_k: torch.Tensor,
|
| 63 |
+
blk_kv: int,
|
| 64 |
+
) -> list[tuple[int, int]]:
|
| 65 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv))
|
| 66 |
+
max_rows = max(rows_per_batch, default=0)
|
| 67 |
+
row_map: list[tuple[int, int]] = []
|
| 68 |
+
for block_idx in range(max_rows):
|
| 69 |
+
for batch_idx, rows in enumerate(rows_per_batch):
|
| 70 |
+
if block_idx < rows:
|
| 71 |
+
row_map.append((batch_idx, block_idx))
|
| 72 |
+
return row_map
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _csr_to_q2k_indices(
|
| 76 |
+
k2q_row_ptr: torch.Tensor,
|
| 77 |
+
k2q_q_indices: torch.Tensor,
|
| 78 |
+
cu_seqlens_q: torch.Tensor,
|
| 79 |
+
cu_seqlens_k: torch.Tensor,
|
| 80 |
+
blk_kv: int,
|
| 81 |
+
topk: int,
|
| 82 |
+
) -> torch.Tensor:
|
| 83 |
+
if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32:
|
| 84 |
+
raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32")
|
| 85 |
+
if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2:
|
| 86 |
+
raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2")
|
| 87 |
+
head_kv = int(k2q_row_ptr.shape[0])
|
| 88 |
+
total_q = int(cu_seqlens_q[-1].item())
|
| 89 |
+
topk = int(topk)
|
| 90 |
+
if topk <= 0:
|
| 91 |
+
raise ValueError(f"topK must be positive, got {topk}")
|
| 92 |
+
|
| 93 |
+
row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv))
|
| 94 |
+
if int(k2q_row_ptr.shape[1]) != len(row_map) + 1:
|
| 95 |
+
raise ValueError(
|
| 96 |
+
"k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: "
|
| 97 |
+
f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}"
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
|
| 101 |
+
row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False)
|
| 102 |
+
q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False)
|
| 103 |
+
q2k = torch.full(
|
| 104 |
+
(head_kv, total_q, topk),
|
| 105 |
+
-1,
|
| 106 |
+
dtype=torch.int32,
|
| 107 |
+
device=k2q_row_ptr.device,
|
| 108 |
+
)
|
| 109 |
+
|
| 110 |
+
next_slot: dict[tuple[int, int], int] = {}
|
| 111 |
+
seen: set[tuple[int, int, int]] = set()
|
| 112 |
+
for h in range(head_kv):
|
| 113 |
+
for row, (batch_idx, kv_block_idx) in enumerate(row_map):
|
| 114 |
+
start = int(row_ptr_cpu[h, row].item())
|
| 115 |
+
end = int(row_ptr_cpu[h, row + 1].item())
|
| 116 |
+
q_start = int(cu_q_cpu[batch_idx])
|
| 117 |
+
q_end = int(cu_q_cpu[batch_idx + 1])
|
| 118 |
+
q_len = q_end - q_start
|
| 119 |
+
for pos in range(start, end):
|
| 120 |
+
q_val = int(q_idx_cpu[h, pos].item())
|
| 121 |
+
if q_val < 0:
|
| 122 |
+
continue
|
| 123 |
+
if q_start <= q_val < q_end:
|
| 124 |
+
q_global = q_val
|
| 125 |
+
elif q_val < q_len:
|
| 126 |
+
q_global = q_start + q_val
|
| 127 |
+
else:
|
| 128 |
+
continue
|
| 129 |
+
key = (h, q_global, kv_block_idx)
|
| 130 |
+
if key in seen:
|
| 131 |
+
continue
|
| 132 |
+
seen.add(key)
|
| 133 |
+
slot_key = (h, q_global)
|
| 134 |
+
slot = next_slot.get(slot_key, 0)
|
| 135 |
+
if slot >= topk:
|
| 136 |
+
continue
|
| 137 |
+
q2k[h, q_global, slot] = int(kv_block_idx)
|
| 138 |
+
next_slot[slot_key] = slot + 1
|
| 139 |
+
return q2k.contiguous()
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def _make_prefill_cache_and_metadata(
|
| 143 |
+
q: torch.Tensor,
|
| 144 |
+
k: torch.Tensor,
|
| 145 |
+
v: torch.Tensor,
|
| 146 |
+
cu_seqlens_q: torch.Tensor,
|
| 147 |
+
cu_seqlens_k: torch.Tensor,
|
| 148 |
+
max_seqlen_k: int,
|
| 149 |
+
blk_kv: int,
|
| 150 |
+
*,
|
| 151 |
+
page_table: Optional[torch.Tensor],
|
| 152 |
+
seqused_k: Optional[torch.Tensor],
|
| 153 |
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 154 |
+
if page_table is None:
|
| 155 |
+
if k.ndim != 3 or v.ndim != 3:
|
| 156 |
+
raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]")
|
| 157 |
+
if k.shape != v.shape:
|
| 158 |
+
raise ValueError("k and v must have the same shape")
|
| 159 |
+
k_cache = k.contiguous()
|
| 160 |
+
v_cache = v.contiguous()
|
| 161 |
+
batch = int(cu_seqlens_q.numel()) - 1
|
| 162 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 163 |
+
logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64)
|
| 164 |
+
cu_k = cu_seqlens_k.to(torch.int64)
|
| 165 |
+
seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous()
|
| 166 |
+
valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
|
| 167 |
+
req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32)
|
| 168 |
+
req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
|
| 169 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 170 |
+
else:
|
| 171 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 172 |
+
raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 173 |
+
if k.shape != v.shape:
|
| 174 |
+
raise ValueError("k and v must have the same shape")
|
| 175 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 176 |
+
raise ValueError("paged k/v page dimension must equal blk_kv")
|
| 177 |
+
if page_table.dtype != torch.int32:
|
| 178 |
+
raise TypeError("page_table must be torch.int32")
|
| 179 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 180 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 181 |
+
batch = int(page_table.shape[0])
|
| 182 |
+
if seqused_k is None:
|
| 183 |
+
seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous()
|
| 184 |
+
else:
|
| 185 |
+
seq_lens = seqused_k.to(torch.int32).contiguous()
|
| 186 |
+
logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64)
|
| 187 |
+
page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1)
|
| 188 |
+
offset = logical % int(blk_kv)
|
| 189 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 190 |
+
req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32)
|
| 191 |
+
valid = logical[None, :] < seq_lens.to(torch.int64)[:, None]
|
| 192 |
+
req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous()
|
| 193 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 194 |
+
|
| 195 |
+
q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32)
|
| 196 |
+
prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous()
|
| 197 |
+
return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int:
|
| 201 |
+
if kv_heads <= 0 or q_heads % kv_heads != 0:
|
| 202 |
+
raise ValueError("number of query heads must be divisible by KV heads")
|
| 203 |
+
gqa_group = q_heads // kv_heads
|
| 204 |
+
limit = max(1, 128 // max(1, gqa_group))
|
| 205 |
+
for candidate in (64, 32, 16, 8, 4, 2, 1):
|
| 206 |
+
if candidate <= limit:
|
| 207 |
+
return candidate
|
| 208 |
+
return 1
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _normalize_decode_q2k(
|
| 212 |
+
q2k_indices: Optional[torch.Tensor],
|
| 213 |
+
*,
|
| 214 |
+
batch: int,
|
| 215 |
+
seqlen_q: int,
|
| 216 |
+
head_kv: int,
|
| 217 |
+
seqused_k: torch.Tensor,
|
| 218 |
+
blk_kv: int,
|
| 219 |
+
) -> torch.Tensor:
|
| 220 |
+
if q2k_indices is None:
|
| 221 |
+
topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
|
| 222 |
+
return _dense_q2k_indices(
|
| 223 |
+
head_kv=head_kv,
|
| 224 |
+
batch=batch,
|
| 225 |
+
topk=topk,
|
| 226 |
+
seq_lens=seqused_k,
|
| 227 |
+
blk_kv=blk_kv,
|
| 228 |
+
)
|
| 229 |
+
if q2k_indices.dtype != torch.int32:
|
| 230 |
+
raise TypeError("q2k_indices must be torch.int32")
|
| 231 |
+
if q2k_indices.ndim != 3:
|
| 232 |
+
raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
|
| 233 |
+
if q2k_indices.shape[0] != head_kv:
|
| 234 |
+
raise ValueError("q2k_indices first dimension must match Hkv")
|
| 235 |
+
if int(q2k_indices.shape[1]) != batch * seqlen_q:
|
| 236 |
+
raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
|
| 237 |
+
if seqlen_q != 1:
|
| 238 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 239 |
+
return q2k_indices[:, ::seqlen_q, :].contiguous()
|
| 240 |
+
|
| 241 |
+
|
| 242 |
+
def sparse_decode_atten_func(
|
| 243 |
+
q: torch.Tensor,
|
| 244 |
+
k: torch.Tensor,
|
| 245 |
+
v: torch.Tensor,
|
| 246 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 247 |
+
*,
|
| 248 |
+
page_table: torch.Tensor,
|
| 249 |
+
seqused_k: torch.Tensor,
|
| 250 |
+
seqlen_q: int,
|
| 251 |
+
max_seqlen_k: int,
|
| 252 |
+
blk_kv: int = 128,
|
| 253 |
+
causal: bool = True,
|
| 254 |
+
softmax_scale: Optional[float] = None,
|
| 255 |
+
return_softmax_lse: bool = False,
|
| 256 |
+
schedule: Optional[object] = None,
|
| 257 |
+
O_partial: Optional[torch.Tensor] = None,
|
| 258 |
+
LSE_partial: Optional[torch.Tensor] = None,
|
| 259 |
+
):
|
| 260 |
+
"""Run MiniMax paged sparse decode on Blackwell.
|
| 261 |
+
|
| 262 |
+
This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
|
| 263 |
+
but accepts the Blackwell package's validated BF16/FP16 decode path. It
|
| 264 |
+
supports single-token decode (`seqlen_q=1`) with paged K/V
|
| 265 |
+
`[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
|
| 266 |
+
"""
|
| 267 |
+
|
| 268 |
+
if schedule is not None or O_partial is not None or LSE_partial is not None:
|
| 269 |
+
raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
|
| 270 |
+
if return_softmax_lse:
|
| 271 |
+
raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
|
| 272 |
+
if not causal:
|
| 273 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
|
| 274 |
+
if int(seqlen_q) != 1:
|
| 275 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 276 |
+
if q.ndim != 3:
|
| 277 |
+
raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
|
| 278 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 279 |
+
raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 280 |
+
if k.shape != v.shape:
|
| 281 |
+
raise ValueError("k and v must have the same shape")
|
| 282 |
+
if q.device != k.device or q.device != v.device or q.device != page_table.device:
|
| 283 |
+
raise ValueError("q, k, v, and page_table must be on the same device")
|
| 284 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 285 |
+
raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
|
| 286 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 287 |
+
raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
|
| 288 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128:
|
| 289 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 290 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 291 |
+
raise ValueError("k.shape[2] must equal blk_kv")
|
| 292 |
+
if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
|
| 293 |
+
raise TypeError("page_table and seqused_k must be torch.int32")
|
| 294 |
+
if page_table.ndim != 2:
|
| 295 |
+
raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
|
| 296 |
+
batch = int(page_table.shape[0])
|
| 297 |
+
if int(q.shape[0]) != batch:
|
| 298 |
+
raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
|
| 299 |
+
|
| 300 |
+
head_kv = int(k.shape[1])
|
| 301 |
+
q2k = _normalize_decode_q2k(
|
| 302 |
+
q2k_indices,
|
| 303 |
+
batch=batch,
|
| 304 |
+
seqlen_q=int(seqlen_q),
|
| 305 |
+
head_kv=head_kv,
|
| 306 |
+
seqused_k=seqused_k,
|
| 307 |
+
blk_kv=int(blk_kv),
|
| 308 |
+
)
|
| 309 |
+
req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
|
| 310 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 311 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 312 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 313 |
+
return flash_decode_with_gqa_share_sparse(
|
| 314 |
+
q.contiguous(),
|
| 315 |
+
None,
|
| 316 |
+
k_cache,
|
| 317 |
+
v_cache,
|
| 318 |
+
req_to_token,
|
| 319 |
+
seqused_k.contiguous(),
|
| 320 |
+
slot_ids,
|
| 321 |
+
int(blk_kv),
|
| 322 |
+
q2k,
|
| 323 |
+
sm_scale=softmax_scale,
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
class SparseDecodePagedAttentionWrapper:
|
| 328 |
+
def __init__(self, *, blk_kv: int = 128, causal: bool = True):
|
| 329 |
+
self.blk_kv = int(blk_kv)
|
| 330 |
+
self.causal = bool(causal)
|
| 331 |
+
self.page_table: Optional[torch.Tensor] = None
|
| 332 |
+
self.seqused_k: Optional[torch.Tensor] = None
|
| 333 |
+
self.seqlen_q: Optional[int] = None
|
| 334 |
+
self.max_seqlen_k: Optional[int] = None
|
| 335 |
+
self.q2k_indices: Optional[torch.Tensor] = None
|
| 336 |
+
|
| 337 |
+
def plan(
|
| 338 |
+
self,
|
| 339 |
+
*,
|
| 340 |
+
page_table: torch.Tensor,
|
| 341 |
+
seqused_k: torch.Tensor,
|
| 342 |
+
seqlen_q: int,
|
| 343 |
+
max_seqlen_k: int,
|
| 344 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 345 |
+
num_qo_heads: Optional[int] = None,
|
| 346 |
+
num_kv_heads: Optional[int] = None,
|
| 347 |
+
head_dim: Optional[int] = 128,
|
| 348 |
+
enable_cuda_graph: bool = False,
|
| 349 |
+
max_grid_size: Optional[int] = None,
|
| 350 |
+
fixed_split_size: Optional[int] = None,
|
| 351 |
+
disable_split_kv: bool = False,
|
| 352 |
+
) -> "SparseDecodePagedAttentionWrapper":
|
| 353 |
+
if head_dim is not None and int(head_dim) != 128:
|
| 354 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 355 |
+
self.page_table = page_table.contiguous()
|
| 356 |
+
self.seqused_k = seqused_k.contiguous()
|
| 357 |
+
self.seqlen_q = int(seqlen_q)
|
| 358 |
+
self.max_seqlen_k = int(max_seqlen_k)
|
| 359 |
+
self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
|
| 360 |
+
return self
|
| 361 |
+
|
| 362 |
+
def run(
|
| 363 |
+
self,
|
| 364 |
+
q: torch.Tensor,
|
| 365 |
+
k: torch.Tensor,
|
| 366 |
+
v: torch.Tensor,
|
| 367 |
+
*,
|
| 368 |
+
softmax_scale: Optional[float] = None,
|
| 369 |
+
return_softmax_lse: bool = False,
|
| 370 |
+
out: Optional[torch.Tensor] = None,
|
| 371 |
+
lse: Optional[torch.Tensor] = None,
|
| 372 |
+
):
|
| 373 |
+
if self.page_table is None or self.seqused_k is None:
|
| 374 |
+
raise RuntimeError("decode wrapper must be planned before run")
|
| 375 |
+
result = sparse_decode_atten_func(
|
| 376 |
+
q,
|
| 377 |
+
k,
|
| 378 |
+
v,
|
| 379 |
+
self.q2k_indices,
|
| 380 |
+
page_table=self.page_table,
|
| 381 |
+
seqused_k=self.seqused_k,
|
| 382 |
+
seqlen_q=self.seqlen_q,
|
| 383 |
+
max_seqlen_k=self.max_seqlen_k,
|
| 384 |
+
blk_kv=self.blk_kv,
|
| 385 |
+
causal=self.causal,
|
| 386 |
+
softmax_scale=softmax_scale,
|
| 387 |
+
return_softmax_lse=return_softmax_lse,
|
| 388 |
+
)
|
| 389 |
+
if out is not None:
|
| 390 |
+
out.copy_(result)
|
| 391 |
+
return out
|
| 392 |
+
return result
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
def sparse_atten_func(
|
| 396 |
+
q: torch.Tensor,
|
| 397 |
+
k: torch.Tensor,
|
| 398 |
+
v: torch.Tensor,
|
| 399 |
+
k2q_row_ptr: torch.Tensor,
|
| 400 |
+
k2q_q_indices: torch.Tensor,
|
| 401 |
+
topK: int,
|
| 402 |
+
*,
|
| 403 |
+
cu_seqlens_q: torch.Tensor,
|
| 404 |
+
cu_seqlens_k: torch.Tensor,
|
| 405 |
+
max_seqlen_q: int,
|
| 406 |
+
max_seqlen_k: int,
|
| 407 |
+
blk_kv: int = 128,
|
| 408 |
+
causal: bool = True,
|
| 409 |
+
softmax_scale: Optional[float] = None,
|
| 410 |
+
lse_temperature_scale: float = 1.0,
|
| 411 |
+
return_temperature_lse: bool = False,
|
| 412 |
+
partial_dtype: torch.dtype = torch.bfloat16,
|
| 413 |
+
return_softmax_lse: bool = False,
|
| 414 |
+
page_table: Optional[torch.Tensor] = None,
|
| 415 |
+
seqused_k: Optional[torch.Tensor] = None,
|
| 416 |
+
schedule: Optional[object] = None,
|
| 417 |
+
usable_SM_count: int = -1,
|
| 418 |
+
qk_dtype: Optional[torch.dtype] = None,
|
| 419 |
+
pv_dtype: Optional[torch.dtype] = None,
|
| 420 |
+
):
|
| 421 |
+
"""Run the official MiniMax CSR sparse prefill API on Blackwell.
|
| 422 |
+
|
| 423 |
+
The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list
|
| 424 |
+
consumed by the Blackwell Triton sparse prefill kernel. The default output
|
| 425 |
+
path is supported for BF16/FP16 tensors with head_dim=128.
|
| 426 |
+
"""
|
| 427 |
+
|
| 428 |
+
del partial_dtype, usable_SM_count, qk_dtype, pv_dtype
|
| 429 |
+
if schedule is not None:
|
| 430 |
+
raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper")
|
| 431 |
+
if return_temperature_lse or return_softmax_lse:
|
| 432 |
+
raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper")
|
| 433 |
+
if float(lse_temperature_scale) != 1.0:
|
| 434 |
+
raise ValueError("lse_temperature_scale values other than 1.0 are not supported")
|
| 435 |
+
if not causal:
|
| 436 |
+
raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True")
|
| 437 |
+
if q.ndim != 3:
|
| 438 |
+
raise ValueError("q must have shape [total_q, Hq, D]")
|
| 439 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 440 |
+
raise TypeError("q must be BF16 or FP16")
|
| 441 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128:
|
| 442 |
+
raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128")
|
| 443 |
+
if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32:
|
| 444 |
+
raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32")
|
| 445 |
+
if q.device != k.device or q.device != v.device:
|
| 446 |
+
raise ValueError("q, k, and v must be on the same device")
|
| 447 |
+
|
| 448 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 449 |
+
raise TypeError("q, k, and v must have matching BF16/FP16 dtype")
|
| 450 |
+
kv_heads = int(k.shape[1])
|
| 451 |
+
q2k_indices = _csr_to_q2k_indices(
|
| 452 |
+
k2q_row_ptr.contiguous(),
|
| 453 |
+
k2q_q_indices.contiguous(),
|
| 454 |
+
cu_seqlens_q.contiguous(),
|
| 455 |
+
cu_seqlens_k.contiguous(),
|
| 456 |
+
int(blk_kv),
|
| 457 |
+
int(topK),
|
| 458 |
+
)
|
| 459 |
+
k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata(
|
| 460 |
+
q,
|
| 461 |
+
k,
|
| 462 |
+
v,
|
| 463 |
+
cu_seqlens_q.contiguous(),
|
| 464 |
+
cu_seqlens_k.contiguous(),
|
| 465 |
+
int(max_seqlen_k),
|
| 466 |
+
int(blk_kv),
|
| 467 |
+
page_table=None if page_table is None else page_table.contiguous(),
|
| 468 |
+
seqused_k=None if seqused_k is None else seqused_k.contiguous(),
|
| 469 |
+
)
|
| 470 |
+
block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads)
|
| 471 |
+
return flash_prefill_with_gqa_share_sparse(
|
| 472 |
+
q.contiguous(),
|
| 473 |
+
k_cache,
|
| 474 |
+
v_cache,
|
| 475 |
+
None,
|
| 476 |
+
req_to_token,
|
| 477 |
+
slot_ids,
|
| 478 |
+
q2k_indices,
|
| 479 |
+
block_size_q=block_size_q,
|
| 480 |
+
block_size_k=int(blk_kv),
|
| 481 |
+
cu_seqlens=cu_seqlens_q.contiguous(),
|
| 482 |
+
seq_lens=seq_lens,
|
| 483 |
+
prefix_lens=prefix_lens,
|
| 484 |
+
max_seqlen_q=int(max_seqlen_q),
|
| 485 |
+
sm_scale=softmax_scale,
|
| 486 |
+
)
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def _nvfp4_dequant_kv(
|
| 490 |
+
packed: torch.Tensor,
|
| 491 |
+
scale_128x4: torch.Tensor,
|
| 492 |
+
global_scale: torch.Tensor,
|
| 493 |
+
) -> torch.Tensor:
|
| 494 |
+
if packed.dtype is not torch.uint8:
|
| 495 |
+
return packed
|
| 496 |
+
if has_native_ops() and packed.is_cuda:
|
| 497 |
+
return native_nvfp4_dequant_swizzled_to_bf16(
|
| 498 |
+
packed,
|
| 499 |
+
scale_128x4,
|
| 500 |
+
global_scale,
|
| 501 |
+
)
|
| 502 |
+
original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,)
|
| 503 |
+
rows = 1
|
| 504 |
+
for dim in original_shape[:-1]:
|
| 505 |
+
rows *= int(dim)
|
| 506 |
+
qx = Nvfp4QuantizedTensor(
|
| 507 |
+
data=packed,
|
| 508 |
+
scale_128x4=scale_128x4,
|
| 509 |
+
global_scale=global_scale,
|
| 510 |
+
logical_scale_shape=(rows, int(original_shape[-1]) // 16),
|
| 511 |
+
original_shape=original_shape,
|
| 512 |
+
)
|
| 513 |
+
return dequantize_nvfp4_128x4_to_bf16(qx)
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
def sparse_atten_nvfp4_kv_func(
|
| 517 |
+
q: torch.Tensor,
|
| 518 |
+
k: torch.Tensor,
|
| 519 |
+
v: torch.Tensor,
|
| 520 |
+
k_scale_128x4: torch.Tensor,
|
| 521 |
+
v_scale_128x4: torch.Tensor,
|
| 522 |
+
k_global_scale: torch.Tensor,
|
| 523 |
+
v_global_scale: torch.Tensor,
|
| 524 |
+
k2q_row_ptr: torch.Tensor,
|
| 525 |
+
k2q_q_indices: torch.Tensor,
|
| 526 |
+
topK: int,
|
| 527 |
+
**kwargs,
|
| 528 |
+
):
|
| 529 |
+
"""NVFP4-KV official prefill API wrapper.
|
| 530 |
+
|
| 531 |
+
The Blackwell compatibility path dequantizes NVFP4 K/V with the package's
|
| 532 |
+
128x4 metadata helper, then dispatches to `sparse_atten_func`.
|
| 533 |
+
"""
|
| 534 |
+
|
| 535 |
+
k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale)
|
| 536 |
+
v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale)
|
| 537 |
+
return sparse_atten_func(
|
| 538 |
+
q,
|
| 539 |
+
k_bf16,
|
| 540 |
+
v_bf16,
|
| 541 |
+
k2q_row_ptr,
|
| 542 |
+
k2q_q_indices,
|
| 543 |
+
topK,
|
| 544 |
+
**kwargs,
|
| 545 |
+
)
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor:
|
| 549 |
+
data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8)
|
| 550 |
+
lut = torch.tensor(
|
| 551 |
+
[0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
|
| 552 |
+
dtype=torch.float32,
|
| 553 |
+
device=data.device,
|
| 554 |
+
)
|
| 555 |
+
out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device)
|
| 556 |
+
out[..., 0::2] = lut[(data & 0x0F).long()]
|
| 557 |
+
out[..., 1::2] = lut[(data >> 4).long()]
|
| 558 |
+
return out
|
| 559 |
+
|
| 560 |
+
|
| 561 |
+
def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
| 562 |
+
if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]):
|
| 563 |
+
return values * scale.to(torch.float32).repeat_interleave(16, dim=-1)
|
| 564 |
+
if scale.ndim == values.ndim - 1:
|
| 565 |
+
return values * scale.to(torch.float32).unsqueeze(-1)
|
| 566 |
+
return values
|
| 567 |
+
|
| 568 |
+
|
| 569 |
+
def fp4_indexer_block_scores(
|
| 570 |
+
q_fp4: torch.Tensor,
|
| 571 |
+
k_fp4: torch.Tensor,
|
| 572 |
+
q_scale: torch.Tensor,
|
| 573 |
+
k_scale: torch.Tensor,
|
| 574 |
+
cu_seqlens_q: torch.Tensor,
|
| 575 |
+
cu_seqlens_k: torch.Tensor,
|
| 576 |
+
cu_page_offsets: torch.Tensor,
|
| 577 |
+
*,
|
| 578 |
+
max_seqlen_q: int,
|
| 579 |
+
max_seqlen_k: int,
|
| 580 |
+
kv_indices: torch.Tensor,
|
| 581 |
+
fp4_format: str,
|
| 582 |
+
causal: bool = False,
|
| 583 |
+
qo_offset: Optional[torch.Tensor] = None,
|
| 584 |
+
scale_layout: str = "preordered_mma",
|
| 585 |
+
) -> torch.Tensor:
|
| 586 |
+
"""FP4 block-score API for Blackwell.
|
| 587 |
+
|
| 588 |
+
Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`.
|
| 589 |
+
Built artifacts route CUDA tensors through the native Blackwell block-score
|
| 590 |
+
kernel. Source-tree mode uses the reference loop below so the API remains
|
| 591 |
+
testable before the extension is built.
|
| 592 |
+
"""
|
| 593 |
+
|
| 594 |
+
del qo_offset
|
| 595 |
+
if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}:
|
| 596 |
+
raise ValueError(f"unsupported fp4_format: {fp4_format}")
|
| 597 |
+
if scale_layout not in {"public", "preordered_mma"}:
|
| 598 |
+
raise ValueError(f"unsupported scale_layout: {scale_layout}")
|
| 599 |
+
if q_fp4.ndim != 3:
|
| 600 |
+
raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]")
|
| 601 |
+
if k_fp4.ndim != 4:
|
| 602 |
+
raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]")
|
| 603 |
+
total_q, hq, _ = q_fp4.shape
|
| 604 |
+
num_pages, hkv, page_size, _ = k_fp4.shape
|
| 605 |
+
if int(page_size) != 128:
|
| 606 |
+
raise ValueError("fp4_indexer_block_scores currently expects 128-token pages")
|
| 607 |
+
if hq % hkv != 0:
|
| 608 |
+
raise ValueError("Hq must be divisible by Hkv")
|
| 609 |
+
max_blocks = (int(max_seqlen_k) + 127) // 128
|
| 610 |
+
scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device)
|
| 611 |
+
|
| 612 |
+
q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale)
|
| 613 |
+
k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale)
|
| 614 |
+
|
| 615 |
+
# Native block-max QK kernel for the cubic scoring loop when available.
|
| 616 |
+
# Same dequant numerics as the reference (bf16 compute); falls back to the
|
| 617 |
+
# Python reference below in source-tree mode.
|
| 618 |
+
if has_native_ops() and q_fp4.is_cuda:
|
| 619 |
+
batch = int(cu_seqlens_q.numel()) - 1
|
| 620 |
+
q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64)
|
| 621 |
+
batch_of_q = torch.repeat_interleave(
|
| 622 |
+
torch.arange(batch, device=q_fp4.device, dtype=torch.int32),
|
| 623 |
+
q_lens,
|
| 624 |
+
)
|
| 625 |
+
return native_indexer_block_scores(
|
| 626 |
+
q.to(torch.bfloat16),
|
| 627 |
+
k.to(torch.bfloat16),
|
| 628 |
+
batch_of_q,
|
| 629 |
+
cu_seqlens_q.to(torch.int32),
|
| 630 |
+
cu_seqlens_k.to(torch.int32),
|
| 631 |
+
cu_page_offsets.to(torch.int32),
|
| 632 |
+
kv_indices.to(torch.int32),
|
| 633 |
+
max_blocks=max_blocks,
|
| 634 |
+
causal=causal,
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
gqa = hq // hkv
|
| 638 |
+
batch = int(cu_seqlens_q.numel()) - 1
|
| 639 |
+
cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist()
|
| 640 |
+
cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist()
|
| 641 |
+
cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist()
|
| 642 |
+
kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist()
|
| 643 |
+
for b in range(batch):
|
| 644 |
+
q_start, q_end = int(cu_q[b]), int(cu_q[b + 1])
|
| 645 |
+
k_len = int(cu_k[b + 1]) - int(cu_k[b])
|
| 646 |
+
page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1])
|
| 647 |
+
for logical_block, page_pos in enumerate(range(page_start, page_end)):
|
| 648 |
+
if logical_block >= max_blocks:
|
| 649 |
+
break
|
| 650 |
+
physical_page = int(kv_idx_cpu[page_pos])
|
| 651 |
+
if physical_page < 0 or physical_page >= num_pages:
|
| 652 |
+
continue
|
| 653 |
+
valid_tokens = min(128, max(0, k_len - logical_block * 128))
|
| 654 |
+
if valid_tokens <= 0:
|
| 655 |
+
continue
|
| 656 |
+
for qi in range(q_start, q_end):
|
| 657 |
+
local_q = qi - q_start
|
| 658 |
+
if causal and logical_block * 128 > local_q:
|
| 659 |
+
continue
|
| 660 |
+
for qh in range(hq):
|
| 661 |
+
kh = qh // gqa
|
| 662 |
+
key_block = k[physical_page, kh, :valid_tokens]
|
| 663 |
+
if causal:
|
| 664 |
+
visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1))
|
| 665 |
+
if visible <= 0:
|
| 666 |
+
continue
|
| 667 |
+
key_block = key_block[:visible]
|
| 668 |
+
block_scores = torch.matmul(key_block, q[qi, qh])
|
| 669 |
+
scores[qh, logical_block, qi] = block_scores.max()
|
| 670 |
+
return scores
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
__all__ = [
|
| 674 |
+
"sparse_atten_func",
|
| 675 |
+
"sparse_atten_nvfp4_kv_func",
|
| 676 |
+
"sparse_decode_atten_func",
|
| 677 |
+
"SparseDecodePagedAttentionWrapper",
|
| 678 |
+
"fp4_indexer_block_scores",
|
| 679 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "minimaxai-msa-blackwell",
|
| 3 |
+
"id": "_minimaxai_msa_blackwell_cuda_25b0067",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"12.0",
|
| 11 |
+
"12.1"
|
| 12 |
+
]
|
| 13 |
+
},
|
| 14 |
+
"digest": {
|
| 15 |
+
"algorithm": "sha256",
|
| 16 |
+
"files": {
|
| 17 |
+
"__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=",
|
| 18 |
+
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
+
"_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "oFPB/WnE3z+U5WanxNiLqUP41c7OcUreUnifaBmpYaw=",
|
| 20 |
+
"_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=",
|
| 21 |
+
"_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=",
|
| 22 |
+
"api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=",
|
| 23 |
+
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 24 |
+
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 25 |
+
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 26 |
+
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 27 |
+
"decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=",
|
| 28 |
+
"interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=",
|
| 29 |
+
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 30 |
+
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 31 |
+
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 32 |
+
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
|
| 33 |
+
"prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=",
|
| 34 |
+
"prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=",
|
| 35 |
+
"quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
|
| 36 |
+
"sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
|
| 37 |
+
}
|
| 38 |
+
}
|
| 39 |
+
}
|
build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# VENDOR NOTE: rewritten to drop the `einops` dependency (the original used
|
| 4 |
+
# einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream
|
| 5 |
+
# `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced
|
| 6 |
+
# with native torch.einsum / torch.reshape so the standalone vendor needs only
|
| 7 |
+
# torch (+ triton for the kernels). Original einops expressions are kept in
|
| 8 |
+
# comments next to each replacement for auditability.
|
| 9 |
+
|
| 10 |
+
from typing import Optional
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def naive_flash_decode_with_topk_idx(
|
| 16 |
+
q: torch.Tensor, # [batch_size, num_heads, head_dim]
|
| 17 |
+
sink: Optional[torch.Tensor], # [num_heads, head_dim]
|
| 18 |
+
kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim]
|
| 19 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 20 |
+
max_seqlen: int,
|
| 21 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 22 |
+
block_size: int,
|
| 23 |
+
topk: int,
|
| 24 |
+
sm_scale: Optional[float] = None,
|
| 25 |
+
init_blocks: int = 0,
|
| 26 |
+
local_blocks: int = 0,
|
| 27 |
+
):
|
| 28 |
+
"""Joint reference: lightning-indexer block top-k + sparse attention.
|
| 29 |
+
|
| 30 |
+
This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense
|
| 31 |
+
5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by
|
| 32 |
+
`slot_ids`. It both (a) selects the top-k key blocks via per-block max score
|
| 33 |
+
(with optional forced init/local blocks) and (b) computes the resulting
|
| 34 |
+
sparse attention, returning (o, topk_idx).
|
| 35 |
+
"""
|
| 36 |
+
assert (
|
| 37 |
+
kv_cache.shape[2] % block_size == 0
|
| 38 |
+
), "max cache len must be divisible by block size"
|
| 39 |
+
if sm_scale is None:
|
| 40 |
+
sm_scale = q.shape[-1] ** -0.5
|
| 41 |
+
original_dtype = q.dtype
|
| 42 |
+
batch_size = q.shape[0]
|
| 43 |
+
num_q_heads = q.shape[1]
|
| 44 |
+
num_kv_heads = kv_cache.shape[3]
|
| 45 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 46 |
+
head_dim = q.shape[-1]
|
| 47 |
+
# rearrange q: "b (h g) d -> b h g d"
|
| 48 |
+
q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim)
|
| 49 |
+
kv_cache_float = kv_cache.float()
|
| 50 |
+
k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d]
|
| 51 |
+
v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d]
|
| 52 |
+
# einsum(q, k, "b h g d, b n h d -> b h g n")
|
| 53 |
+
qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale
|
| 54 |
+
mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None]
|
| 55 |
+
qk = qk.masked_fill(~mask[:, None, None, :], float("-inf"))
|
| 56 |
+
# get score
|
| 57 |
+
score = qk.clone().to(torch.float32)
|
| 58 |
+
# rearrange "b h g (n s) -> b (h g) n s", s=block_size
|
| 59 |
+
n_blocks = score.shape[-1] // block_size
|
| 60 |
+
score = score.reshape(
|
| 61 |
+
batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size
|
| 62 |
+
).reshape(batch_size, num_q_heads, n_blocks, block_size)
|
| 63 |
+
score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks]
|
| 64 |
+
# post-process score for init_blocks and local_blocks
|
| 65 |
+
INIT_SCORE = 1e30
|
| 66 |
+
LOCAL_SCORE = 1e29
|
| 67 |
+
if init_blocks > 0:
|
| 68 |
+
score[:, :, :init_blocks] = INIT_SCORE
|
| 69 |
+
if local_blocks > 0:
|
| 70 |
+
num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
|
| 71 |
+
for b in range(batch_size):
|
| 72 |
+
num_blks = num_blocks_per_batch[b].item()
|
| 73 |
+
local_start = max(0, num_blks - local_blocks)
|
| 74 |
+
score[b, :, local_start:num_blks] = LOCAL_SCORE
|
| 75 |
+
# compute topk indices per (batch, head)
|
| 76 |
+
# score shape: [batch_size, num_q_heads, num_blocks]
|
| 77 |
+
topk_idx = torch.full(
|
| 78 |
+
(num_q_heads, batch_size, topk),
|
| 79 |
+
fill_value=-1,
|
| 80 |
+
device=score.device,
|
| 81 |
+
dtype=torch.int32,
|
| 82 |
+
)
|
| 83 |
+
num_blocks_per_batch = (seq_lens + block_size - 1) // block_size
|
| 84 |
+
for b in range(batch_size):
|
| 85 |
+
num_blks = num_blocks_per_batch[b].item()
|
| 86 |
+
actual_topk = min(topk, num_blks)
|
| 87 |
+
for h in range(num_q_heads):
|
| 88 |
+
# get topk indices for this (batch, head)
|
| 89 |
+
_, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1)
|
| 90 |
+
topk_idx[h, b, :actual_topk] = indices.to(torch.int32)
|
| 91 |
+
# compute attention output with sink
|
| 92 |
+
if sink is not None:
|
| 93 |
+
# sink: [num_q_heads, head_dim] -> [h, g, d]
|
| 94 |
+
sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim)
|
| 95 |
+
# einsum(q, sink, "b h g d, h g d -> b h g")
|
| 96 |
+
qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale
|
| 97 |
+
qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1]
|
| 98 |
+
attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32)
|
| 99 |
+
attn = attn[..., 1:] # remove sink score
|
| 100 |
+
else:
|
| 101 |
+
attn = qk.softmax(dim=-1, dtype=torch.float32)
|
| 102 |
+
# einsum(attn, v, "b h g n, b n h d -> b h g d")
|
| 103 |
+
o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel)
|
| 104 |
+
# rearrange "b h g d -> b (h g) d"
|
| 105 |
+
o = o.reshape(batch_size, num_q_heads, head_dim)
|
| 106 |
+
return o.to(original_dtype), topk_idx
|
build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def naive_flash_decode_with_gqa_share_sparse(
|
| 9 |
+
q: torch.Tensor, # [batch_size, num_q_heads, head_dim]
|
| 10 |
+
sink: Optional[torch.Tensor], # [num_q_heads, head_dim]
|
| 11 |
+
kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim]
|
| 12 |
+
seq_lens: torch.Tensor, # [batch_size, ]
|
| 13 |
+
slot_ids: torch.Tensor, # [batch_size, ]
|
| 14 |
+
block_size: int,
|
| 15 |
+
topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk]
|
| 16 |
+
sm_scale: Optional[float] = None,
|
| 17 |
+
) -> torch.Tensor:
|
| 18 |
+
"""
|
| 19 |
+
Naive implementation of sparse attention with GQA group sharing.
|
| 20 |
+
Each GQA group (all q heads sharing the same kv head) uses the same topk_idx.
|
| 21 |
+
"""
|
| 22 |
+
if sm_scale is None:
|
| 23 |
+
sm_scale = q.shape[-1] ** -0.5
|
| 24 |
+
|
| 25 |
+
original_dtype = q.dtype
|
| 26 |
+
batch_size, num_q_heads, head_dim = q.shape
|
| 27 |
+
max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape
|
| 28 |
+
gqa_group_size = num_q_heads // num_kv_heads
|
| 29 |
+
# Output tensor
|
| 30 |
+
o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device)
|
| 31 |
+
|
| 32 |
+
for b in range(batch_size):
|
| 33 |
+
seq_len = seq_lens[b].item()
|
| 34 |
+
sid = slot_ids[b].item() % max_slots
|
| 35 |
+
|
| 36 |
+
for kh in range(num_kv_heads):
|
| 37 |
+
# Get topk indices for this (kv_head, batch)
|
| 38 |
+
block_indices = topk_idx[kh, b, :].tolist()
|
| 39 |
+
|
| 40 |
+
# Collect selected K and V blocks
|
| 41 |
+
selected_k_blocks = []
|
| 42 |
+
selected_v_blocks = []
|
| 43 |
+
for block_idx in block_indices:
|
| 44 |
+
if block_idx < 0:
|
| 45 |
+
continue # Invalid index
|
| 46 |
+
start = block_idx * block_size
|
| 47 |
+
end = min(start + block_size, seq_len)
|
| 48 |
+
if start >= seq_len:
|
| 49 |
+
continue
|
| 50 |
+
# K: [block_len, head_dim]
|
| 51 |
+
k_block = kv_cache[sid, 0, start:end, kh, :]
|
| 52 |
+
# V: [block_len, head_dim]
|
| 53 |
+
v_block = kv_cache[sid, 1, start:end, kh, :]
|
| 54 |
+
selected_k_blocks.append(k_block)
|
| 55 |
+
selected_v_blocks.append(v_block)
|
| 56 |
+
|
| 57 |
+
if len(selected_k_blocks) == 0:
|
| 58 |
+
continue
|
| 59 |
+
|
| 60 |
+
# Concatenate selected blocks: [total_selected_len, head_dim]
|
| 61 |
+
k_selected = torch.cat(selected_k_blocks, dim=0)
|
| 62 |
+
v_selected = torch.cat(selected_v_blocks, dim=0)
|
| 63 |
+
|
| 64 |
+
# Compute attention for all q heads in this GQA group
|
| 65 |
+
for g in range(gqa_group_size):
|
| 66 |
+
qh = kh * gqa_group_size + g
|
| 67 |
+
# q_vec: [head_dim]
|
| 68 |
+
q_vec = q[b, qh, :].float()
|
| 69 |
+
|
| 70 |
+
# Compute attention scores: [total_selected_len]
|
| 71 |
+
scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale
|
| 72 |
+
|
| 73 |
+
# Add sink to softmax normalization if present
|
| 74 |
+
if sink is not None:
|
| 75 |
+
sink_vec = sink[qh, :].float()
|
| 76 |
+
qsink = torch.dot(q_vec, sink_vec) * sm_scale
|
| 77 |
+
# Concatenate sink score with regular scores for softmax
|
| 78 |
+
scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0)
|
| 79 |
+
attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1)
|
| 80 |
+
# Remove sink weight (only used for normalization)
|
| 81 |
+
attn_weights = attn_weights_with_sink[1:]
|
| 82 |
+
else:
|
| 83 |
+
attn_weights = torch.softmax(scores, dim=-1)
|
| 84 |
+
|
| 85 |
+
# Compute output: [head_dim]
|
| 86 |
+
o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected)
|
| 87 |
+
|
| 88 |
+
return o
|
build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
from .topk_sparse import flash_prefill_with_gqa_share_sparse
|
| 3 |
+
|
| 4 |
+
__all__ = ["flash_prefill_with_gqa_share_sparse"]
|
build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2025 XunhaoLai. All rights reserved.
|
| 2 |
+
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
import triton
|
| 7 |
+
import triton.language as tl
|
| 8 |
+
|
| 9 |
+
from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip`
|
| 10 |
+
|
| 11 |
+
from ..common.utils import get_cu_seqblocks, robust_allocator
|
| 12 |
+
|
| 13 |
+
_is_hip = is_hip()
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
@triton.heuristics(
|
| 17 |
+
{
|
| 18 |
+
"BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]),
|
| 19 |
+
"BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]),
|
| 20 |
+
"BLOCK_SIZE_H": lambda args: triton.next_power_of_2(
|
| 21 |
+
max(
|
| 22 |
+
16 // args["BLOCK_SIZE_Q"],
|
| 23 |
+
triton.next_power_of_2(args["gqa_group_size"]),
|
| 24 |
+
)
|
| 25 |
+
),
|
| 26 |
+
"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]),
|
| 27 |
+
"BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"],
|
| 28 |
+
"HAS_SINK": lambda args: args["sink_ptr"] is not None,
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
@triton.autotune(
|
| 32 |
+
# Configs that fail to compile on the target arch are skipped, so widening
|
| 33 |
+
# the num_warps x num_stages grid only adds candidates, never a bad kernel.
|
| 34 |
+
configs=[
|
| 35 |
+
triton.Config({}, num_warps=nw, num_stages=ns)
|
| 36 |
+
for nw in (2, 4, 8)
|
| 37 |
+
for ns in (2, 3, 4)
|
| 38 |
+
],
|
| 39 |
+
key=[
|
| 40 |
+
"BLOCK_SIZE_Q",
|
| 41 |
+
"BLOCK_SIZE_K",
|
| 42 |
+
"qk_head_dim",
|
| 43 |
+
"v_head_dim",
|
| 44 |
+
"gqa_group_size",
|
| 45 |
+
],
|
| 46 |
+
)
|
| 47 |
+
@triton.jit
|
| 48 |
+
def _gqa_share_sparse_fwd_kernel(
|
| 49 |
+
q_ptr, # Q: n x h x d
|
| 50 |
+
k_cache_ptr, # K paged: max_slots x kh x d
|
| 51 |
+
v_cache_ptr, # V paged: max_slots x kh x d
|
| 52 |
+
sink_ptr, # Sink: h x d
|
| 53 |
+
t_ptr, # topk_idx: kh x n x k
|
| 54 |
+
o_ptr, # O: n x h x d
|
| 55 |
+
req_to_token_ptr, # req_to_token: max_reqs x max_kv_len
|
| 56 |
+
# seqlens
|
| 57 |
+
cu_seqlens_q,
|
| 58 |
+
cu_seqblocks_q,
|
| 59 |
+
seq_lens,
|
| 60 |
+
prefix_lens,
|
| 61 |
+
slot_ids,
|
| 62 |
+
# shape
|
| 63 |
+
max_slots,
|
| 64 |
+
num_kv_heads,
|
| 65 |
+
gqa_group_size,
|
| 66 |
+
qk_head_dim,
|
| 67 |
+
v_head_dim,
|
| 68 |
+
max_topk,
|
| 69 |
+
# q loop num
|
| 70 |
+
num_q_loop,
|
| 71 |
+
# sm_scale
|
| 72 |
+
sm_scale,
|
| 73 |
+
# stride
|
| 74 |
+
stride_qn,
|
| 75 |
+
stride_qh,
|
| 76 |
+
stride_qd,
|
| 77 |
+
stride_ks,
|
| 78 |
+
stride_kh,
|
| 79 |
+
stride_kd,
|
| 80 |
+
stride_vs,
|
| 81 |
+
stride_vh,
|
| 82 |
+
stride_vd,
|
| 83 |
+
stride_sh,
|
| 84 |
+
stride_sd,
|
| 85 |
+
stride_th,
|
| 86 |
+
stride_tn,
|
| 87 |
+
stride_tk,
|
| 88 |
+
stride_on,
|
| 89 |
+
stride_oh,
|
| 90 |
+
stride_od,
|
| 91 |
+
stride_r2t_b,
|
| 92 |
+
# META parameters
|
| 93 |
+
BLOCK_SIZE_Q: tl.constexpr, # q block size
|
| 94 |
+
BLOCK_SIZE_K: tl.constexpr, # k block size
|
| 95 |
+
BLOCK_SIZE_KD: tl.constexpr,
|
| 96 |
+
BLOCK_SIZE_VD: tl.constexpr,
|
| 97 |
+
BLOCK_SIZE_H: tl.constexpr,
|
| 98 |
+
BLOCK_SIZE_T: tl.constexpr,
|
| 99 |
+
BLOCK_SIZE_QH: tl.constexpr,
|
| 100 |
+
# has sink
|
| 101 |
+
HAS_SINK: tl.constexpr,
|
| 102 |
+
USE_TMA: tl.constexpr,
|
| 103 |
+
IS_FP8: tl.constexpr,
|
| 104 |
+
):
|
| 105 |
+
sm_scale_log2e = sm_scale * 1.4426950409
|
| 106 |
+
# get batch id and head id
|
| 107 |
+
pid_q = tl.program_id(0)
|
| 108 |
+
pid_kh = tl.program_id(1)
|
| 109 |
+
pid_b = tl.program_id(2)
|
| 110 |
+
pid_h = pid_kh * gqa_group_size
|
| 111 |
+
# get q k start and len after rmpad
|
| 112 |
+
q_start = tl.load(cu_seqlens_q + pid_b)
|
| 113 |
+
q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
|
| 114 |
+
q_block_start = tl.load(cu_seqblocks_q + pid_b)
|
| 115 |
+
q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start
|
| 116 |
+
seq_len = tl.load(seq_lens + pid_b)
|
| 117 |
+
prefix_len = tl.load(prefix_lens + pid_b)
|
| 118 |
+
sid = (
|
| 119 |
+
tl.load(slot_ids + pid_b).to(tl.int64) + max_slots
|
| 120 |
+
) % max_slots # safety against negative
|
| 121 |
+
if pid_q * num_q_loop >= q_block_len:
|
| 122 |
+
return
|
| 123 |
+
real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop)
|
| 124 |
+
if HAS_SINK:
|
| 125 |
+
sink_ptrs = tl.make_block_ptr(
|
| 126 |
+
base=sink_ptr + pid_h * stride_sh,
|
| 127 |
+
shape=(gqa_group_size, qk_head_dim),
|
| 128 |
+
strides=(stride_sh, stride_sd),
|
| 129 |
+
offsets=(0, 0),
|
| 130 |
+
block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD),
|
| 131 |
+
order=(1, 0),
|
| 132 |
+
)
|
| 133 |
+
sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to(
|
| 134 |
+
tl.float32
|
| 135 |
+
)
|
| 136 |
+
# offsets for paged K/V load
|
| 137 |
+
off_n = tl.arange(0, BLOCK_SIZE_K)
|
| 138 |
+
off_kd = tl.arange(0, BLOCK_SIZE_KD)
|
| 139 |
+
off_vd = tl.arange(0, BLOCK_SIZE_VD)
|
| 140 |
+
kd_mask = off_kd < qk_head_dim
|
| 141 |
+
vd_mask = off_vd < v_head_dim
|
| 142 |
+
for j in range(real_q_loop):
|
| 143 |
+
pid_q_j = pid_q * num_q_loop + j
|
| 144 |
+
# init topk idx pointer
|
| 145 |
+
t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th
|
| 146 |
+
# we assume that the topk_idx is right padded with -1
|
| 147 |
+
off_t = tl.arange(0, BLOCK_SIZE_T)
|
| 148 |
+
topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1)
|
| 149 |
+
valid_idx = tl.where(topk_idx >= 0, off_t, -1)
|
| 150 |
+
real_topk = tl.sum(valid_idx != -1, axis=0)
|
| 151 |
+
# init qkv pointer
|
| 152 |
+
q_ptrs = tl.make_block_ptr(
|
| 153 |
+
base=q_ptr + q_start * stride_qn + pid_h * stride_qh,
|
| 154 |
+
shape=(q_len, gqa_group_size, qk_head_dim),
|
| 155 |
+
strides=(stride_qn, stride_qh, stride_qd),
|
| 156 |
+
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
|
| 157 |
+
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD),
|
| 158 |
+
order=(2, 1, 0),
|
| 159 |
+
)
|
| 160 |
+
# load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D]
|
| 161 |
+
q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero")
|
| 162 |
+
# init statistics
|
| 163 |
+
off_q_k = (
|
| 164 |
+
tl.arange(0, BLOCK_SIZE_Q)[:, None]
|
| 165 |
+
+ pid_q_j * BLOCK_SIZE_Q
|
| 166 |
+
+ prefix_len
|
| 167 |
+
- tl.arange(0, BLOCK_SIZE_K)[None, :]
|
| 168 |
+
)
|
| 169 |
+
if HAS_SINK:
|
| 170 |
+
m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
|
| 171 |
+
lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32)
|
| 172 |
+
qsink = (
|
| 173 |
+
tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e
|
| 174 |
+
) # (BLOCK_SIZE_Q, BLOCK_SIZE_H)
|
| 175 |
+
m_i += qsink
|
| 176 |
+
lse_i += qsink
|
| 177 |
+
m_i = tl.reshape(m_i, BLOCK_SIZE_QH)
|
| 178 |
+
lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH)
|
| 179 |
+
else:
|
| 180 |
+
m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
|
| 181 |
+
lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32)
|
| 182 |
+
acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32)
|
| 183 |
+
q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD)
|
| 184 |
+
# sparse attention
|
| 185 |
+
for i in range(real_topk):
|
| 186 |
+
# get current block start index (absolute K position)
|
| 187 |
+
c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K
|
| 188 |
+
t_ptr_j = t_ptr_j + stride_tk
|
| 189 |
+
# paged load K via req_to_token: pos -> slot -> k_cache
|
| 190 |
+
pos = c + off_n
|
| 191 |
+
pos_mask = pos < seq_len
|
| 192 |
+
slots = tl.load(
|
| 193 |
+
req_to_token_ptr + sid * stride_r2t_b + pos,
|
| 194 |
+
mask=pos_mask,
|
| 195 |
+
other=0,
|
| 196 |
+
).to(tl.int64)
|
| 197 |
+
slots = (slots + max_slots) % max_slots # safety against negative
|
| 198 |
+
# k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot)
|
| 199 |
+
k = tl.load(
|
| 200 |
+
k_cache_ptr
|
| 201 |
+
+ slots[None, :] * stride_ks
|
| 202 |
+
+ pid_kh * stride_kh
|
| 203 |
+
+ off_kd[:, None] * stride_kd,
|
| 204 |
+
mask=kd_mask[:, None] & pos_mask[None, :],
|
| 205 |
+
other=0.0,
|
| 206 |
+
)
|
| 207 |
+
if IS_FP8:
|
| 208 |
+
# fp8 main K cache is unit-scaled; widen to the Q compute dtype
|
| 209 |
+
# before the tl.dot (compiled out when the cache is bf16).
|
| 210 |
+
k = k.to(q.dtype)
|
| 211 |
+
# compute qk
|
| 212 |
+
qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
|
| 213 |
+
# causal mask
|
| 214 |
+
qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf"))
|
| 215 |
+
qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K)
|
| 216 |
+
# [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K]
|
| 217 |
+
# -> [BLOCK_SIZE_QH, BLOCK_SIZE_K]
|
| 218 |
+
qk += tl.dot(q, k) * sm_scale_log2e
|
| 219 |
+
# K boundary mask: positions beyond seq_len contribute -inf
|
| 220 |
+
qk += tl.where(pos_mask[None, :], 0, float("-inf"))
|
| 221 |
+
# compute m_ij and l_ij
|
| 222 |
+
m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
|
| 223 |
+
p = tl.exp2(qk - m_ij[:, None])
|
| 224 |
+
l_ij = tl.sum(p, axis=1)
|
| 225 |
+
# scale acc_o
|
| 226 |
+
acc_o_scale = tl.exp2(m_i - m_ij)
|
| 227 |
+
acc_o = acc_o * acc_o_scale[:, None]
|
| 228 |
+
# paged load V
|
| 229 |
+
v = tl.load(
|
| 230 |
+
v_cache_ptr
|
| 231 |
+
+ slots[:, None] * stride_vs
|
| 232 |
+
+ pid_kh * stride_vh
|
| 233 |
+
+ off_vd[None, :] * stride_vd,
|
| 234 |
+
mask=pos_mask[:, None] & vd_mask[None, :],
|
| 235 |
+
other=0.0,
|
| 236 |
+
)
|
| 237 |
+
if IS_FP8:
|
| 238 |
+
# Widen V so `p.to(v.dtype)` casts P to the compute dtype rather
|
| 239 |
+
# than to fp8 (which would wreck attention-weight precision).
|
| 240 |
+
v = v.to(q.dtype)
|
| 241 |
+
p = p.to(v.dtype)
|
| 242 |
+
acc_o += tl.dot(p, v)
|
| 243 |
+
# update statistics
|
| 244 |
+
m_i = m_ij
|
| 245 |
+
lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
|
| 246 |
+
# final scale
|
| 247 |
+
acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
|
| 248 |
+
# save output
|
| 249 |
+
acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD)
|
| 250 |
+
o_ptrs = tl.make_block_ptr(
|
| 251 |
+
base=o_ptr + q_start * stride_on + pid_h * stride_oh,
|
| 252 |
+
shape=(q_len, gqa_group_size, v_head_dim),
|
| 253 |
+
strides=(stride_on, stride_oh, stride_od),
|
| 254 |
+
offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0),
|
| 255 |
+
block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD),
|
| 256 |
+
order=(2, 1, 0),
|
| 257 |
+
)
|
| 258 |
+
tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2))
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
@torch.no_grad()
|
| 262 |
+
def flash_prefill_with_gqa_share_sparse(
|
| 263 |
+
q: torch.Tensor,
|
| 264 |
+
k_cache: torch.Tensor,
|
| 265 |
+
v_cache: torch.Tensor,
|
| 266 |
+
sink: Optional[torch.Tensor],
|
| 267 |
+
req_to_token: torch.Tensor,
|
| 268 |
+
slot_ids: torch.Tensor,
|
| 269 |
+
topk_idx: torch.Tensor,
|
| 270 |
+
block_size_q: int,
|
| 271 |
+
block_size_k: int,
|
| 272 |
+
cu_seqlens: torch.Tensor,
|
| 273 |
+
seq_lens: torch.Tensor,
|
| 274 |
+
prefix_lens: torch.Tensor,
|
| 275 |
+
max_seqlen_q: int,
|
| 276 |
+
sm_scale: Optional[float] = None,
|
| 277 |
+
use_tma: bool = True,
|
| 278 |
+
cu_seqblocks_q: Optional[torch.Tensor] = None,
|
| 279 |
+
max_seqblock_q: Optional[int] = None,
|
| 280 |
+
) -> torch.Tensor:
|
| 281 |
+
triton.set_allocator(robust_allocator)
|
| 282 |
+
# dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled)
|
| 283 |
+
# under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load.
|
| 284 |
+
# Keep CUDA's pre-existing sparse prefill dtype contract unchanged by
|
| 285 |
+
# accepting fp8 only on HIP.
|
| 286 |
+
assert q.dtype in (torch.bfloat16, torch.float16)
|
| 287 |
+
_FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz)
|
| 288 |
+
is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES
|
| 289 |
+
assert k_cache.dtype == q.dtype or is_fp8, (
|
| 290 |
+
f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) "
|
| 291 |
+
f"or fp8 on HIP, got {k_cache.dtype}"
|
| 292 |
+
)
|
| 293 |
+
assert v_cache.dtype == k_cache.dtype
|
| 294 |
+
assert block_size_q in {1, 2, 4, 8, 16, 32, 64}
|
| 295 |
+
assert block_size_k in {16, 32, 64, 128}
|
| 296 |
+
# shape
|
| 297 |
+
total_q, num_q_heads, qk_head_dim = q.shape
|
| 298 |
+
max_slots, num_k_heads, _ = k_cache.shape
|
| 299 |
+
_, num_v_heads, v_head_dim = v_cache.shape
|
| 300 |
+
batch_size = cu_seqlens.shape[0] - 1
|
| 301 |
+
topk = topk_idx.shape[-1]
|
| 302 |
+
assert topk_idx.shape[0] == num_k_heads
|
| 303 |
+
# gqa
|
| 304 |
+
assert num_k_heads == num_v_heads
|
| 305 |
+
assert num_q_heads % num_k_heads == 0
|
| 306 |
+
gqa_group_size = num_q_heads // num_k_heads
|
| 307 |
+
assert gqa_group_size * block_size_q <= 128
|
| 308 |
+
if sm_scale is None:
|
| 309 |
+
sm_scale = qk_head_dim**-0.5
|
| 310 |
+
if cu_seqblocks_q is None or max_seqblock_q is None:
|
| 311 |
+
cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks(
|
| 312 |
+
cu_seqlens, max_seqlen_q, block_size_q, block_size_k
|
| 313 |
+
)
|
| 314 |
+
# output tensor
|
| 315 |
+
o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype)
|
| 316 |
+
# launch kernel
|
| 317 |
+
num_q_loop = (
|
| 318 |
+
max_seqblock_q // 131072 + 1
|
| 319 |
+
) # calculate multiple queries in one kernel if seqlence length is too long
|
| 320 |
+
BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q)
|
| 321 |
+
BLOCK_SIZE_K = triton.next_power_of_2(block_size_k)
|
| 322 |
+
grid = (
|
| 323 |
+
triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop),
|
| 324 |
+
num_k_heads,
|
| 325 |
+
batch_size,
|
| 326 |
+
)
|
| 327 |
+
_gqa_share_sparse_fwd_kernel[grid](
|
| 328 |
+
q,
|
| 329 |
+
k_cache,
|
| 330 |
+
v_cache,
|
| 331 |
+
sink,
|
| 332 |
+
topk_idx,
|
| 333 |
+
o,
|
| 334 |
+
req_to_token,
|
| 335 |
+
cu_seqlens,
|
| 336 |
+
cu_seqblocks_q,
|
| 337 |
+
seq_lens,
|
| 338 |
+
prefix_lens,
|
| 339 |
+
slot_ids,
|
| 340 |
+
max_slots,
|
| 341 |
+
num_k_heads,
|
| 342 |
+
gqa_group_size,
|
| 343 |
+
qk_head_dim,
|
| 344 |
+
v_head_dim,
|
| 345 |
+
topk,
|
| 346 |
+
num_q_loop,
|
| 347 |
+
sm_scale,
|
| 348 |
+
q.stride(0),
|
| 349 |
+
q.stride(1),
|
| 350 |
+
q.stride(2),
|
| 351 |
+
k_cache.stride(0),
|
| 352 |
+
k_cache.stride(1),
|
| 353 |
+
k_cache.stride(2),
|
| 354 |
+
v_cache.stride(0),
|
| 355 |
+
v_cache.stride(1),
|
| 356 |
+
v_cache.stride(2),
|
| 357 |
+
sink.stride(0) if sink is not None else 0,
|
| 358 |
+
sink.stride(1) if sink is not None else 0,
|
| 359 |
+
topk_idx.stride(0),
|
| 360 |
+
topk_idx.stride(1),
|
| 361 |
+
topk_idx.stride(2),
|
| 362 |
+
o.stride(0),
|
| 363 |
+
o.stride(1),
|
| 364 |
+
o.stride(2),
|
| 365 |
+
req_to_token.stride(0),
|
| 366 |
+
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
|
| 367 |
+
BLOCK_SIZE_K=BLOCK_SIZE_K,
|
| 368 |
+
USE_TMA=use_tma,
|
| 369 |
+
IS_FP8=is_fp8,
|
| 370 |
+
)
|
| 371 |
+
return o
|
build/torch211-cxx11-cu130-x86_64-linux/quantize.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""NVFP4 helper API compatible with MiniMaxAI/msa.
|
| 3 |
+
|
| 4 |
+
The quantization entrypoints use Transformer Engine when it is installed. The
|
| 5 |
+
layout helpers and reference dequantizer are pure PyTorch and are included so
|
| 6 |
+
callers can prepare and validate NVFP4 metadata with the same public API names
|
| 7 |
+
as the upstream MiniMaxAI package.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Tuple
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
NVFP4_BLOCK_SIZE = 16
|
| 19 |
+
NVFP4_FP4_MAX = 6.0
|
| 20 |
+
NVFP4_FP8_E4M3_MAX = 448.0
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class Nvfp4QuantizedTensor:
|
| 25 |
+
data: torch.Tensor
|
| 26 |
+
scale_128x4: torch.Tensor
|
| 27 |
+
global_scale: torch.Tensor
|
| 28 |
+
logical_scale_shape: Tuple[int, int]
|
| 29 |
+
original_shape: Tuple[int, ...]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _round_up(x: int, multiple: int) -> int:
|
| 33 |
+
return ((int(x) + multiple - 1) // multiple) * multiple
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def nvfp4_scale_128x4_offset(
|
| 37 |
+
row: torch.Tensor,
|
| 38 |
+
col: torch.Tensor,
|
| 39 |
+
scale_cols: int,
|
| 40 |
+
) -> torch.Tensor:
|
| 41 |
+
tiles_n = _round_up(scale_cols, 4) // 4
|
| 42 |
+
tile_m = row // 128
|
| 43 |
+
tile_n = col // 4
|
| 44 |
+
outer = row % 128
|
| 45 |
+
inner = col % 4
|
| 46 |
+
return (
|
| 47 |
+
(tile_m * tiles_n + tile_n) * 512
|
| 48 |
+
+ (outer % 32) * 16
|
| 49 |
+
+ (outer // 32) * 4
|
| 50 |
+
+ inner
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def swizzle_nvfp4_scale_to_128x4(
|
| 55 |
+
scale: torch.Tensor,
|
| 56 |
+
*,
|
| 57 |
+
rows: int,
|
| 58 |
+
cols: int,
|
| 59 |
+
) -> torch.Tensor:
|
| 60 |
+
if scale.ndim != 2:
|
| 61 |
+
raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
|
| 62 |
+
|
| 63 |
+
rows = int(rows)
|
| 64 |
+
cols = int(cols)
|
| 65 |
+
padded_rows = _round_up(rows, 128)
|
| 66 |
+
padded_cols = _round_up(cols, 4)
|
| 67 |
+
if scale.shape[0] < rows or scale.shape[1] < cols:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
"scale is smaller than the requested logical shape: "
|
| 70 |
+
f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
logical = scale[:rows, :cols].contiguous()
|
| 74 |
+
if logical.shape != (padded_rows, padded_cols):
|
| 75 |
+
logical = torch.nn.functional.pad(
|
| 76 |
+
logical.to(torch.float32),
|
| 77 |
+
(0, padded_cols - cols, 0, padded_rows - rows),
|
| 78 |
+
).to(scale.dtype)
|
| 79 |
+
swizzled = torch.empty_like(logical)
|
| 80 |
+
|
| 81 |
+
row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
|
| 82 |
+
col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
|
| 83 |
+
offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
|
| 84 |
+
swizzled.reshape(-1)[offset] = logical.reshape(-1)
|
| 85 |
+
return swizzled
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _import_te_nvfp4_quantizer():
|
| 93 |
+
try:
|
| 94 |
+
from transformer_engine.pytorch.tensor import NVFP4Quantizer
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
"Transformer Engine NVFP4 quantization is unavailable. Install a "
|
| 98 |
+
"Transformer Engine build with NVFP4 PyTorch support to use "
|
| 99 |
+
"quantize_bf16_to_nvfp4_128x4."
|
| 100 |
+
) from exc
|
| 101 |
+
return NVFP4Quantizer
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
|
| 105 |
+
if not x.is_cuda:
|
| 106 |
+
raise ValueError("NVFP4 quantization requires a CUDA tensor")
|
| 107 |
+
if x.dtype not in (torch.bfloat16, torch.float16):
|
| 108 |
+
raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
|
| 109 |
+
if x.ndim < 2:
|
| 110 |
+
raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
|
| 111 |
+
if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
rows = 1
|
| 117 |
+
for dim in x.shape[:-1]:
|
| 118 |
+
rows *= int(dim)
|
| 119 |
+
if rows % NVFP4_BLOCK_SIZE != 0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"flattened row dimension must be divisible by "
|
| 122 |
+
f"{NVFP4_BLOCK_SIZE}, got {rows}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
NVFP4Quantizer = _import_te_nvfp4_quantizer()
|
| 126 |
+
quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
|
| 127 |
+
qx = quantizer.quantize(x.contiguous())
|
| 128 |
+
meta = qx.get_metadata()
|
| 129 |
+
|
| 130 |
+
data = meta["rowwise_data"]
|
| 131 |
+
if data.dtype is not torch.uint8:
|
| 132 |
+
data = data.view(torch.uint8)
|
| 133 |
+
logical_scale = meta["rowwise_scale_inv"]
|
| 134 |
+
amax = meta["amax_rowwise"]
|
| 135 |
+
scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
|
| 136 |
+
scale_128x4 = swizzle_nvfp4_scale_to_128x4(
|
| 137 |
+
logical_scale,
|
| 138 |
+
rows=rows,
|
| 139 |
+
cols=scale_cols,
|
| 140 |
+
)
|
| 141 |
+
global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
|
| 142 |
+
|
| 143 |
+
return Nvfp4QuantizedTensor(
|
| 144 |
+
data=data,
|
| 145 |
+
scale_128x4=scale_128x4,
|
| 146 |
+
global_scale=global_scale,
|
| 147 |
+
logical_scale_shape=(rows, scale_cols),
|
| 148 |
+
original_shape=tuple(int(v) for v in x.shape),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def quantize_kv_bf16_to_nvfp4_128x4(
|
| 153 |
+
k: torch.Tensor,
|
| 154 |
+
v: torch.Tensor,
|
| 155 |
+
) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
|
| 156 |
+
return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def dequantize_nvfp4_128x4_to_bf16(
|
| 160 |
+
qx: Nvfp4QuantizedTensor,
|
| 161 |
+
*,
|
| 162 |
+
include_global_scale: bool = True,
|
| 163 |
+
) -> torch.Tensor:
|
| 164 |
+
data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
|
| 165 |
+
if data.shape[-1] * 2 != qx.original_shape[-1]:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"packed data last dimension does not match original shape: "
|
| 168 |
+
f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
rows, scale_cols = qx.logical_scale_shape
|
| 172 |
+
logical_dim = int(qx.original_shape[-1])
|
| 173 |
+
if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
"logical scale columns do not match original last dimension: "
|
| 176 |
+
f"{scale_cols} scale cols vs dim {logical_dim}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
fp4_lut = torch.tensor(
|
| 180 |
+
[
|
| 181 |
+
0.0,
|
| 182 |
+
0.5,
|
| 183 |
+
1.0,
|
| 184 |
+
1.5,
|
| 185 |
+
2.0,
|
| 186 |
+
3.0,
|
| 187 |
+
4.0,
|
| 188 |
+
6.0,
|
| 189 |
+
-0.0,
|
| 190 |
+
-0.5,
|
| 191 |
+
-1.0,
|
| 192 |
+
-1.5,
|
| 193 |
+
-2.0,
|
| 194 |
+
-3.0,
|
| 195 |
+
-4.0,
|
| 196 |
+
-6.0,
|
| 197 |
+
],
|
| 198 |
+
dtype=torch.float32,
|
| 199 |
+
device=data.device,
|
| 200 |
+
)
|
| 201 |
+
packed = data.reshape(rows, logical_dim // 2)
|
| 202 |
+
lo = packed & 0x0F
|
| 203 |
+
hi = packed >> 4
|
| 204 |
+
values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
|
| 205 |
+
values[:, 0::2] = fp4_lut[lo.long()]
|
| 206 |
+
values[:, 1::2] = fp4_lut[hi.long()]
|
| 207 |
+
|
| 208 |
+
row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
|
| 209 |
+
col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
|
| 210 |
+
offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
|
| 211 |
+
scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
|
| 212 |
+
scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
|
| 213 |
+
scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
|
| 214 |
+
out = values * scale
|
| 215 |
+
if include_global_scale:
|
| 216 |
+
out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
|
| 217 |
+
return out.reshape(qx.original_shape).to(torch.bfloat16)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
__all__ = [
|
| 221 |
+
"Nvfp4QuantizedTensor",
|
| 222 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 223 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 224 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 225 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 226 |
+
"nvfp4_global_scale_from_amax",
|
| 227 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Sparse-index helpers compatible with MiniMaxAI/msa public names."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
|
| 12 |
+
if cu_seqlens.dtype != torch.int32:
|
| 13 |
+
raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
|
| 14 |
+
if cu_seqlens.ndim != 1:
|
| 15 |
+
raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
|
| 16 |
+
if cu_seqlens.numel() < 1:
|
| 17 |
+
raise ValueError(f"{name} must have at least one element")
|
| 18 |
+
if not cu_seqlens.is_contiguous():
|
| 19 |
+
raise ValueError(f"{name} must be contiguous")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
|
| 23 |
+
seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
|
| 24 |
+
return (seqlens_k + kv_block_size - 1) // kv_block_size
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
|
| 28 |
+
rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
|
| 29 |
+
batch = len(rows_per_batch_cpu)
|
| 30 |
+
max_rows = max(rows_per_batch_cpu, default=0)
|
| 31 |
+
row_dtype = (
|
| 32 |
+
torch.int32
|
| 33 |
+
if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
|
| 34 |
+
else torch.int64
|
| 35 |
+
)
|
| 36 |
+
row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
|
| 37 |
+
row_linear = 0
|
| 38 |
+
for kv_block_idx in range(max_rows):
|
| 39 |
+
for batch_idx, row_count in enumerate(rows_per_batch_cpu):
|
| 40 |
+
if kv_block_idx < row_count:
|
| 41 |
+
row_map_cpu[batch_idx, kv_block_idx] = row_linear
|
| 42 |
+
row_linear += 1
|
| 43 |
+
return row_map_cpu.to(rows_per_batch.device), row_linear
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_k2q_csr_torch_reference(
|
| 47 |
+
q2k_indices: torch.Tensor,
|
| 48 |
+
cu_seqlens_q: torch.Tensor,
|
| 49 |
+
cu_seqlens_k: torch.Tensor,
|
| 50 |
+
kv_block_size: int,
|
| 51 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 52 |
+
if kv_block_size <= 0:
|
| 53 |
+
raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
|
| 54 |
+
if q2k_indices.dtype != torch.int32:
|
| 55 |
+
raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
|
| 56 |
+
if q2k_indices.ndim != 3:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
"q2k_indices must have shape [head_kv, total_q, topK], "
|
| 59 |
+
f"got {tuple(q2k_indices.shape)}"
|
| 60 |
+
)
|
| 61 |
+
_validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
|
| 62 |
+
_validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
|
| 63 |
+
if cu_seqlens_q.shape != cu_seqlens_k.shape:
|
| 64 |
+
raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
|
| 65 |
+
if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
|
| 66 |
+
raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
|
| 67 |
+
|
| 68 |
+
head_kv, total_q, topk = q2k_indices.shape
|
| 69 |
+
if total_q != int(cu_seqlens_q[-1].item()):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
|
| 72 |
+
f"({int(cu_seqlens_q[-1].item())})"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
|
| 76 |
+
row_map, total_rows = _build_packed_row_map(rows_per_batch)
|
| 77 |
+
nnz_upper_bound = total_q * topk
|
| 78 |
+
|
| 79 |
+
k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
|
| 80 |
+
k2q_q_indices = torch.full(
|
| 81 |
+
(head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
|
| 82 |
+
)
|
| 83 |
+
if total_rows == 0 or total_q == 0 or topk == 0:
|
| 84 |
+
return k2q_row_ptr, k2q_q_indices
|
| 85 |
+
|
| 86 |
+
counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
|
| 87 |
+
total_entries = total_q * topk
|
| 88 |
+
row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
|
| 89 |
+
row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
|
| 90 |
+
q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
|
| 91 |
+
valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
|
| 92 |
+
|
| 93 |
+
for batch_idx in range(cu_seqlens_q.numel() - 1):
|
| 94 |
+
q_start = int(cu_seqlens_q[batch_idx].item())
|
| 95 |
+
q_end = int(cu_seqlens_q[batch_idx + 1].item())
|
| 96 |
+
if q_end <= q_start:
|
| 97 |
+
continue
|
| 98 |
+
q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
|
| 99 |
+
q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
|
| 100 |
+
kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
|
| 101 |
+
valid = kv_local >= 0
|
| 102 |
+
safe_kv = kv_local.clamp_min(0).long()
|
| 103 |
+
rows = row_map[batch_idx, safe_kv].to(row_dtype)
|
| 104 |
+
start = q_start * topk
|
| 105 |
+
end = q_end * topk
|
| 106 |
+
row_all[:, start:end] = rows
|
| 107 |
+
q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
|
| 108 |
+
valid_all[:, start:end] = valid & (rows >= 0)
|
| 109 |
+
counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
|
| 110 |
+
|
| 111 |
+
k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
|
| 112 |
+
offsets = torch.zeros_like(counts)
|
| 113 |
+
offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
|
| 114 |
+
cursor = torch.zeros_like(counts)
|
| 115 |
+
for h in range(head_kv):
|
| 116 |
+
for entry in range(total_entries):
|
| 117 |
+
if not bool(valid_all[h, entry]):
|
| 118 |
+
continue
|
| 119 |
+
row = int(row_all[h, entry].item())
|
| 120 |
+
pos = int(offsets[h, row].item() + cursor[h, row].item())
|
| 121 |
+
k2q_q_indices[h, pos] = q_all[h, entry]
|
| 122 |
+
cursor[h, row] += 1
|
| 123 |
+
|
| 124 |
+
return k2q_row_ptr, k2q_q_indices
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def build_k2q_csr(
|
| 128 |
+
q2k_indices: torch.Tensor,
|
| 129 |
+
cu_seqlens_q: torch.Tensor,
|
| 130 |
+
cu_seqlens_k: torch.Tensor,
|
| 131 |
+
kv_block_size: int,
|
| 132 |
+
*,
|
| 133 |
+
total_k: Optional[int] = None,
|
| 134 |
+
max_seqlen_k: Optional[int] = None,
|
| 135 |
+
max_seqlen_q: Optional[int] = None,
|
| 136 |
+
total_rows: Optional[int] = None,
|
| 137 |
+
qhead_per_kv: int = 1,
|
| 138 |
+
return_schedule: bool = False,
|
| 139 |
+
):
|
| 140 |
+
if return_schedule:
|
| 141 |
+
raise NotImplementedError(
|
| 142 |
+
"Blackwell build_k2q_csr currently returns CSR tensors only; "
|
| 143 |
+
"SM100 schedule objects are not part of this package."
|
| 144 |
+
)
|
| 145 |
+
if total_k is None:
|
| 146 |
+
raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
|
| 147 |
+
return build_k2q_csr_torch_reference(
|
| 148 |
+
q2k_indices.contiguous(),
|
| 149 |
+
cu_seqlens_q.contiguous(),
|
| 150 |
+
cu_seqlens_k.contiguous(),
|
| 151 |
+
int(kv_block_size),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SparseK2qCsrBuilderSm100:
|
| 156 |
+
"""Compatibility placeholder for the upstream SM100 class name.
|
| 157 |
+
|
| 158 |
+
The name is exported so code can feature-detect it, but schedule-producing
|
| 159 |
+
SM100 behavior is intentionally not claimed for this Blackwell package.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(self, *args, **kwargs):
|
| 163 |
+
self.args = args
|
| 164 |
+
self.kwargs = kwargs
|
| 165 |
+
|
| 166 |
+
def build(self, *args, **kwargs):
|
| 167 |
+
return build_k2q_csr(*args, **kwargs)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
__all__ = [
|
| 171 |
+
"build_k2q_csr",
|
| 172 |
+
"build_k2q_csr_torch_reference",
|
| 173 |
+
"SparseK2qCsrBuilderSm100",
|
| 174 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention.
|
| 3 |
+
|
| 4 |
+
This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on
|
| 5 |
+
DGX Spark / GB10 / SM121. It exposes the
|
| 6 |
+
MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime.
|
| 7 |
+
|
| 8 |
+
Implementation status:
|
| 9 |
+
* native CUDA helper: score -> top-k sparse block ids;
|
| 10 |
+
* native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape;
|
| 11 |
+
* native CUDA FP4 block-score indexer;
|
| 12 |
+
* native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path;
|
| 13 |
+
* Blackwell-validated sparse prefill path;
|
| 14 |
+
* MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers,
|
| 15 |
+
and FP4 block-score helpers.
|
| 16 |
+
|
| 17 |
+
The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell
|
| 18 |
+
extension path. The public API is Tensor-oriented and independent from
|
| 19 |
+
FlashRT's serving runtime.
|
| 20 |
+
|
| 21 |
+
Public API
|
| 22 |
+
----------
|
| 23 |
+
Lightning indexer (q.k blockmax score -> top-k block select), paged KV:
|
| 24 |
+
flash_decode_with_topk_idx (single-token decode, split-K)
|
| 25 |
+
|
| 26 |
+
Block-sparse GQA attention (consumes block_indices from the indexer):
|
| 27 |
+
flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks)
|
| 28 |
+
sparse_atten_func (official CSR prefill wrapper)
|
| 29 |
+
sparse_decode_atten_func (official paged decode wrapper)
|
| 30 |
+
|
| 31 |
+
PyTorch naive references (for correctness checks):
|
| 32 |
+
naive_flash_decode_with_gqa_share_sparse (attention given topk_idx)
|
| 33 |
+
naive_flash_decode_with_topk_idx (joint indexer + attention)
|
| 34 |
+
|
| 35 |
+
All Triton entrypoints expect a paged KV cache:
|
| 36 |
+
k_cache / v_cache : [max_slots, num_kv_heads, head_dim]
|
| 37 |
+
req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot)
|
| 38 |
+
slot_ids : [batch] (per-request row in req_to_token)
|
| 39 |
+
topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed,
|
| 40 |
+
-1 right-padding (the M3 indexer contract).
|
| 41 |
+
See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 45 |
+
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 46 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 47 |
+
from .interface import (
|
| 48 |
+
SparseDecodePagedAttentionWrapper,
|
| 49 |
+
fp4_indexer_block_scores,
|
| 50 |
+
sparse_atten_func,
|
| 51 |
+
sparse_atten_nvfp4_kv_func,
|
| 52 |
+
sparse_decode_atten_func,
|
| 53 |
+
)
|
| 54 |
+
from ._native import (
|
| 55 |
+
has_native_ops,
|
| 56 |
+
native_nvfp4_dequant_swizzled_to_bf16,
|
| 57 |
+
native_topk_from_scores,
|
| 58 |
+
)
|
| 59 |
+
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 60 |
+
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 61 |
+
from .quantize import (
|
| 62 |
+
Nvfp4QuantizedTensor,
|
| 63 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 64 |
+
nvfp4_global_scale_from_amax,
|
| 65 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 66 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 67 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 68 |
+
)
|
| 69 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 70 |
+
from .api_status import (
|
| 71 |
+
OFFICIAL_API_STATUS,
|
| 72 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 73 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 74 |
+
available_functions,
|
| 75 |
+
official_api_status,
|
| 76 |
+
official_minimax_msa_functions,
|
| 77 |
+
unsupported_official_functions,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
__all__ = [
|
| 81 |
+
"sparse_atten_func",
|
| 82 |
+
"sparse_atten_nvfp4_kv_func",
|
| 83 |
+
"sparse_decode_atten_func",
|
| 84 |
+
"SparseDecodePagedAttentionWrapper",
|
| 85 |
+
"fp4_indexer_block_scores",
|
| 86 |
+
"build_k2q_csr",
|
| 87 |
+
"SparseK2qCsrBuilderSm100",
|
| 88 |
+
"Nvfp4QuantizedTensor",
|
| 89 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 90 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 91 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 92 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 93 |
+
"nvfp4_global_scale_from_amax",
|
| 94 |
+
"flash_decode_with_topk_idx",
|
| 95 |
+
"flash_decode_with_gqa_share_sparse",
|
| 96 |
+
"has_native_ops",
|
| 97 |
+
"native_topk_from_scores",
|
| 98 |
+
"native_nvfp4_dequant_swizzled_to_bf16",
|
| 99 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 100 |
+
"naive_flash_decode_with_topk_idx",
|
| 101 |
+
"get_cu_seqblocks",
|
| 102 |
+
"robust_allocator",
|
| 103 |
+
"OFFICIAL_API_STATUS",
|
| 104 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 105 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 106 |
+
"available_functions",
|
| 107 |
+
"official_api_status",
|
| 108 |
+
"official_minimax_msa_functions",
|
| 109 |
+
"unsupported_official_functions",
|
| 110 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/_compat.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton
|
| 3 |
+
# kernels. The upstream SGLang modules pull two tiny things from the framework:
|
| 4 |
+
#
|
| 5 |
+
# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path)
|
| 6 |
+
# from sglang.srt.environ import envs -> a feature flag for an optional
|
| 7 |
+
# JIT-CUDA radix top-k fast path
|
| 8 |
+
#
|
| 9 |
+
# Both are replaced here so the kernels import with only torch + triton. The
|
| 10 |
+
# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel
|
| 11 |
+
# radix select), which is what we want on sm_121 (GB10).
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def is_hip() -> bool:
|
| 15 |
+
"""Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121).
|
| 16 |
+
|
| 17 |
+
Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV
|
| 18 |
+
widening branch (HIP-only). With this False, the CUDA dtype contract
|
| 19 |
+
(K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA.
|
| 20 |
+
"""
|
| 21 |
+
try:
|
| 22 |
+
import torch
|
| 23 |
+
|
| 24 |
+
return bool(getattr(torch.version, "hip", None))
|
| 25 |
+
except Exception:
|
| 26 |
+
return False
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class _Flag:
|
| 30 |
+
"""Minimal stand-in for an sglang.srt.environ boolean flag (always off)."""
|
| 31 |
+
|
| 32 |
+
def __init__(self, default: bool = False):
|
| 33 |
+
self._default = default
|
| 34 |
+
|
| 35 |
+
def get(self) -> bool:
|
| 36 |
+
return self._default
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class _Envs:
|
| 40 |
+
"""Stub for `sglang.srt.environ.envs`.
|
| 41 |
+
|
| 42 |
+
Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode
|
| 43 |
+
indexer. Keeping it False routes top-k selection through the 2-stage Triton
|
| 44 |
+
fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the
|
| 45 |
+
decode indexer stays pure-Triton.
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
envs = _Envs()
|
build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2ea862050a19dcc4b0c7cbfd3b519112f1b4e054cfc182a14f92876b4ae5b9f7
|
| 3 |
+
size 735888
|