diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..6b7a283f6a9d189d65bccdb859c6f80016f52228 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,35 +1,3 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text +*.so filter=lfs diff=lfs merge=lfs -text +*.pyd filter=lfs diff=lfs merge=lfs -text +*.dylib filter=lfs diff=lfs merge=lfs -text diff --git a/CARD.md b/CARD.md new file mode 100644 index 0000000000000000000000000000000000000000..a64cddd9b6bb4aaccea2910e09b5ef174d88a346 --- /dev/null +++ b/CARD.md @@ -0,0 +1,223 @@ +--- +library_name: kernels +license: apache-2.0 +tags: + - cuda + - triton + - native-cuda + - minimax + - sparse-attention + - blackwell +--- + +# MiniMaxAI MSA Blackwell + +Blackwell-family package for MiniMax MSA decode sparse attention, maintained by +FlashRT. The upstream package is +[`MiniMaxAI/msa`](https://huggingface.co/kernels/MiniMaxAI/msa), which targets +SM100. This package extends the decode-sparse path to NVIDIA Blackwell +compute capability 12.x and has been validated in the FlashRT MiniMax-Spark +runtime on DGX Spark / GB10 / SM121. + +## Load + +```python +from kernels import get_kernel + +msa = get_kernel( + "flashrt/MiniMaxAI-msa-blackwell", + version=1, + trust_remote_code=True, +) +``` + +## What You Can Call + +### Official MiniMaxAI/msa names + +| Function/class | Status in this Blackwell package | +|---|---| +| `sparse_decode_atten_func` | Available. Blackwell paged BF16/FP16 single-token decode wrapper. | +| `SparseDecodePagedAttentionWrapper` | Available. `plan(...).run(...)` wrapper for the same decode path. | +| `build_k2q_csr` | Available. CSR construction helper for the official prefill API. | +| `SparseK2qCsrBuilderSm100` | Available compatibility class; `build()` delegates to `build_k2q_csr`. | +| `Nvfp4QuantizedTensor` | Available metadata dataclass. | +| `quantize_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. | +| `quantize_kv_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. | +| `dequantize_nvfp4_128x4_to_bf16` | Available reference dequantizer. | +| `swizzle_nvfp4_scale_to_128x4` | Available scale-layout helper. | +| `nvfp4_global_scale_from_amax` | Available scale helper. | +| `sparse_atten_func` | Available. Official CSR sparse prefill API backed by the Blackwell Triton BF16/FP16 prefill kernel. | +| `sparse_atten_nvfp4_kv_func` | Available. Built artifacts use native CUDA swizzled NVFP4 -> BF16 dequantization, then call Blackwell sparse prefill. | +| `fp4_indexer_block_scores` | Available. Built artifacts use the native CUDA Blackwell block-score kernel and return the official `[Hq, ceil(max_seqlen_k/128), total_q]` score layout. | + +### FlashRT Blackwell helper names + +These are the direct low-level APIs used by the FlashRT MiniMax-Spark decode +path: + +- `flash_decode_with_topk_idx` +- `flash_decode_with_gqa_share_sparse` +- `native_topk_from_scores` +- `native_nvfp4_dequant_swizzled_to_bf16` +- `has_native_ops` +- `naive_flash_decode_with_topk_idx` +- `naive_flash_decode_with_gqa_share_sparse` +- `get_cu_seqblocks` +- `robust_allocator` + +## Decode Example + +This example uses the official MiniMax decode-facing name +`sparse_decode_atten_func`. + +```python +import torch +from kernels import get_kernel + +msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True) + +B, Hq, Hkv, D = 1, 64, 4, 128 +page_size = 128 +num_pages = 32 +topk = 16 +device = "cuda" +dtype = torch.bfloat16 + +q = torch.randn(B, Hq, D, device=device, dtype=dtype) +k = torch.randn(num_pages, Hkv, page_size, D, device=device, dtype=dtype) +v = torch.randn_like(k) +page_table = torch.arange(num_pages, device=device, dtype=torch.int32).view(B, -1) +seqused_k = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32) +q2k_indices = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk) +q2k_indices = q2k_indices.expand(Hkv, B, topk).contiguous() + +out = msa.sparse_decode_atten_func( + q, + k, + v, + q2k_indices, + page_table=page_table, + seqused_k=seqused_k, + seqlen_q=1, + max_seqlen_k=num_pages * page_size, + blk_kv=page_size, +) +``` + +## Wrapper Example + +```python +wrapper = msa.SparseDecodePagedAttentionWrapper(blk_kv=128) +wrapper.plan( + page_table=page_table, + seqused_k=seqused_k, + seqlen_q=1, + max_seqlen_k=num_pages * page_size, + q2k_indices=q2k_indices, + num_qo_heads=Hq, + num_kv_heads=Hkv, + head_dim=D, +) +out = wrapper.run(q, k, v) +``` + +## Prefill Example + +This example uses the official MiniMax CSR prefill-facing name +`sparse_atten_func`. + +```python +import torch +from kernels import get_kernel + +msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True) + +T, Hq, Hkv, D = 512, 64, 4, 128 +page_size = 128 +topk = 16 +device = "cuda" +dtype = torch.bfloat16 + +q = torch.randn(T, Hq, D, device=device, dtype=dtype) +k = torch.randn(T, Hkv, D, device=device, dtype=dtype) +v = torch.randn_like(k) +cu = torch.tensor([0, T], device=device, dtype=torch.int32) + +q2k = torch.full((Hkv, T, topk), -1, device=device, dtype=torch.int32) +for qi in range(T): + blocks = torch.arange(qi // page_size + 1, device=device, dtype=torch.int32) + q2k[:, qi, : min(topk, blocks.numel())] = blocks[:topk] + +k2q_row_ptr, k2q_q_indices = msa.build_k2q_csr( + q2k, cu, cu, page_size, total_k=T +) +out = msa.sparse_atten_func( + q, + k, + v, + k2q_row_ptr, + k2q_q_indices, + topk, + cu_seqlens_q=cu, + cu_seqlens_k=cu, + max_seqlen_q=T, + max_seqlen_k=T, + blk_kv=page_size, +) +``` + +## Direct FlashRT Decode Path + +Use this lower-level path if you already have `topk_idx` in FlashRT's paged KV +layout. + +```python +q = torch.randn(B, Hq, D, device=device, dtype=dtype) +k_cache = torch.randn(num_pages * page_size, Hkv, D, device=device, dtype=dtype) +v_cache = torch.randn_like(k_cache) +req_to_token = torch.arange(num_pages * page_size, device=device, dtype=torch.int32).view(B, -1) +seq_lens = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32) +slot_ids = torch.zeros(B, device=device, dtype=torch.int64) +topk_idx = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk) +topk_idx = topk_idx.expand(Hkv, B, topk).contiguous() + +out = msa.flash_decode_with_gqa_share_sparse( + q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, page_size, topk_idx +) +``` + +## Scope + +- Target family: NVIDIA Blackwell CUDA compute capability 12.x. +- Builder target: CUDA 12.8+ with `cuda-capabilities = ["12.0", "12.1"]`. +- Validated hardware: DGX Spark / GB10 / SM121. +- Validated MiniMax shape: query heads `64`, KV heads `4`, head dim `128`, + sparse block/page size `128`, top-k blocks `16`. +- End-to-end model validation: MiniMax-Spark runtime on GB10 through `32768` + context length. +- Standalone kernel long-context validation: `128`, `2048`, `4096`, `32768`, + `65536`, `131072`. +- Correctness gate: cosine similarity `>= 0.999` against paged FP32 PyTorch + references; official decode wrapper output matches the direct Blackwell + decode kernel. + +## Implementation Notes + +This package contains: + +- native CUDA score-to-top-k helper; +- native CUDA tensor-core sparse decode route for the MiniMax-M3 Blackwell shape; +- native CUDA FP4 block-score indexer; +- native CUDA swizzled NVFP4 -> BF16 dequantization for the W4A16 quality path; +- Blackwell-validated sparse prefill attention wrapper; +- MiniMaxAI/msa-compatible Python API layer for decode, prefill, CSR, NVFP4, + and FP4 block-score helpers. + +When loaded from Hub built artifacts, the decode, FP4 indexer, and NVFP4 +dequant hot paths use compiled CUDA ops. The source-tree mode keeps reference +paths so the API and correctness tests remain runnable before a wheel/shared +object has been built. + +Source provenance and validation details are documented in `SYNC.md` and +`VALIDATION.md`. diff --git a/README.md b/README.md index c87ac38692716cf6ca5d460efff170611c1cd7c3..a64cddd9b6bb4aaccea2910e09b5ef174d88a346 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,223 @@ -# flashrt/MiniMaxAI-msa-blackwell +--- +library_name: kernels +license: apache-2.0 +tags: + - cuda + - triton + - native-cuda + - minimax + - sparse-attention + - blackwell +--- -This repository is a compatibility mirror for older `kernels` clients -that resolve repositories through the default Hugging Face model repo API. +# MiniMaxAI MSA Blackwell -Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/MiniMaxAI-msa-blackwell +Blackwell-family package for MiniMax MSA decode sparse attention, maintained by +FlashRT. The upstream package is +[`MiniMaxAI/msa`](https://huggingface.co/kernels/MiniMaxAI/msa), which targets +SM100. This package extends the decode-sparse path to NVIDIA Blackwell +compute capability 12.x and has been validated in the FlashRT MiniMax-Spark +runtime on DGX Spark / GB10 / SM121. -Do not edit this mirror by hand. It is generated from the Kernel Hub -`vN` branches and contains the same `build/**` artifacts. +## Load + +```python +from kernels import get_kernel + +msa = get_kernel( + "flashrt/MiniMaxAI-msa-blackwell", + version=1, + trust_remote_code=True, +) +``` + +## What You Can Call + +### Official MiniMaxAI/msa names + +| Function/class | Status in this Blackwell package | +|---|---| +| `sparse_decode_atten_func` | Available. Blackwell paged BF16/FP16 single-token decode wrapper. | +| `SparseDecodePagedAttentionWrapper` | Available. `plan(...).run(...)` wrapper for the same decode path. | +| `build_k2q_csr` | Available. CSR construction helper for the official prefill API. | +| `SparseK2qCsrBuilderSm100` | Available compatibility class; `build()` delegates to `build_k2q_csr`. | +| `Nvfp4QuantizedTensor` | Available metadata dataclass. | +| `quantize_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. | +| `quantize_kv_bf16_to_nvfp4_128x4` | Available when Transformer Engine NVFP4 support is installed. | +| `dequantize_nvfp4_128x4_to_bf16` | Available reference dequantizer. | +| `swizzle_nvfp4_scale_to_128x4` | Available scale-layout helper. | +| `nvfp4_global_scale_from_amax` | Available scale helper. | +| `sparse_atten_func` | Available. Official CSR sparse prefill API backed by the Blackwell Triton BF16/FP16 prefill kernel. | +| `sparse_atten_nvfp4_kv_func` | Available. Built artifacts use native CUDA swizzled NVFP4 -> BF16 dequantization, then call Blackwell sparse prefill. | +| `fp4_indexer_block_scores` | Available. Built artifacts use the native CUDA Blackwell block-score kernel and return the official `[Hq, ceil(max_seqlen_k/128), total_q]` score layout. | + +### FlashRT Blackwell helper names + +These are the direct low-level APIs used by the FlashRT MiniMax-Spark decode +path: + +- `flash_decode_with_topk_idx` +- `flash_decode_with_gqa_share_sparse` +- `native_topk_from_scores` +- `native_nvfp4_dequant_swizzled_to_bf16` +- `has_native_ops` +- `naive_flash_decode_with_topk_idx` +- `naive_flash_decode_with_gqa_share_sparse` +- `get_cu_seqblocks` +- `robust_allocator` + +## Decode Example + +This example uses the official MiniMax decode-facing name +`sparse_decode_atten_func`. + +```python +import torch +from kernels import get_kernel + +msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True) + +B, Hq, Hkv, D = 1, 64, 4, 128 +page_size = 128 +num_pages = 32 +topk = 16 +device = "cuda" +dtype = torch.bfloat16 + +q = torch.randn(B, Hq, D, device=device, dtype=dtype) +k = torch.randn(num_pages, Hkv, page_size, D, device=device, dtype=dtype) +v = torch.randn_like(k) +page_table = torch.arange(num_pages, device=device, dtype=torch.int32).view(B, -1) +seqused_k = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32) +q2k_indices = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk) +q2k_indices = q2k_indices.expand(Hkv, B, topk).contiguous() + +out = msa.sparse_decode_atten_func( + q, + k, + v, + q2k_indices, + page_table=page_table, + seqused_k=seqused_k, + seqlen_q=1, + max_seqlen_k=num_pages * page_size, + blk_kv=page_size, +) +``` + +## Wrapper Example + +```python +wrapper = msa.SparseDecodePagedAttentionWrapper(blk_kv=128) +wrapper.plan( + page_table=page_table, + seqused_k=seqused_k, + seqlen_q=1, + max_seqlen_k=num_pages * page_size, + q2k_indices=q2k_indices, + num_qo_heads=Hq, + num_kv_heads=Hkv, + head_dim=D, +) +out = wrapper.run(q, k, v) +``` + +## Prefill Example + +This example uses the official MiniMax CSR prefill-facing name +`sparse_atten_func`. + +```python +import torch +from kernels import get_kernel + +msa = get_kernel("flashrt/MiniMaxAI-msa-blackwell", version=1, trust_remote_code=True) + +T, Hq, Hkv, D = 512, 64, 4, 128 +page_size = 128 +topk = 16 +device = "cuda" +dtype = torch.bfloat16 + +q = torch.randn(T, Hq, D, device=device, dtype=dtype) +k = torch.randn(T, Hkv, D, device=device, dtype=dtype) +v = torch.randn_like(k) +cu = torch.tensor([0, T], device=device, dtype=torch.int32) + +q2k = torch.full((Hkv, T, topk), -1, device=device, dtype=torch.int32) +for qi in range(T): + blocks = torch.arange(qi // page_size + 1, device=device, dtype=torch.int32) + q2k[:, qi, : min(topk, blocks.numel())] = blocks[:topk] + +k2q_row_ptr, k2q_q_indices = msa.build_k2q_csr( + q2k, cu, cu, page_size, total_k=T +) +out = msa.sparse_atten_func( + q, + k, + v, + k2q_row_ptr, + k2q_q_indices, + topk, + cu_seqlens_q=cu, + cu_seqlens_k=cu, + max_seqlen_q=T, + max_seqlen_k=T, + blk_kv=page_size, +) +``` + +## Direct FlashRT Decode Path + +Use this lower-level path if you already have `topk_idx` in FlashRT's paged KV +layout. + +```python +q = torch.randn(B, Hq, D, device=device, dtype=dtype) +k_cache = torch.randn(num_pages * page_size, Hkv, D, device=device, dtype=dtype) +v_cache = torch.randn_like(k_cache) +req_to_token = torch.arange(num_pages * page_size, device=device, dtype=torch.int32).view(B, -1) +seq_lens = torch.tensor([num_pages * page_size], device=device, dtype=torch.int32) +slot_ids = torch.zeros(B, device=device, dtype=torch.int64) +topk_idx = torch.arange(topk, device=device, dtype=torch.int32).view(1, 1, topk) +topk_idx = topk_idx.expand(Hkv, B, topk).contiguous() + +out = msa.flash_decode_with_gqa_share_sparse( + q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, page_size, topk_idx +) +``` + +## Scope + +- Target family: NVIDIA Blackwell CUDA compute capability 12.x. +- Builder target: CUDA 12.8+ with `cuda-capabilities = ["12.0", "12.1"]`. +- Validated hardware: DGX Spark / GB10 / SM121. +- Validated MiniMax shape: query heads `64`, KV heads `4`, head dim `128`, + sparse block/page size `128`, top-k blocks `16`. +- End-to-end model validation: MiniMax-Spark runtime on GB10 through `32768` + context length. +- Standalone kernel long-context validation: `128`, `2048`, `4096`, `32768`, + `65536`, `131072`. +- Correctness gate: cosine similarity `>= 0.999` against paged FP32 PyTorch + references; official decode wrapper output matches the direct Blackwell + decode kernel. + +## Implementation Notes + +This package contains: + +- native CUDA score-to-top-k helper; +- native CUDA tensor-core sparse decode route for the MiniMax-M3 Blackwell shape; +- native CUDA FP4 block-score indexer; +- native CUDA swizzled NVFP4 -> BF16 dequantization for the W4A16 quality path; +- Blackwell-validated sparse prefill attention wrapper; +- MiniMaxAI/msa-compatible Python API layer for decode, prefill, CSR, NVFP4, + and FP4 block-score helpers. + +When loaded from Hub built artifacts, the decode, FP4 indexer, and NVFP4 +dequant hot paths use compiled CUDA ops. The source-tree mode keeps reference +paths so the API and correctness tests remain runnable before a wheel/shared +object has been built. + +Source provenance and validation details are documented in `SYNC.md` and +`VALIDATION.md`. diff --git a/VALIDATION.md b/VALIDATION.md new file mode 100644 index 0000000000000000000000000000000000000000..a2f8133a8a867929d7e83e0bd7bd8cd961cfad12 --- /dev/null +++ b/VALIDATION.md @@ -0,0 +1,206 @@ +# Validation + +## Target + +- Kernel family: MiniMax M3 sparse attention (MSA) +- Package: `flashrt/MiniMaxAI-msa-blackwell` +- HF Jobs package selector: `MiniMaxAI-msa-blackwell` +- Package version: v1 Blackwell native-helper package +- Target GPU family: Blackwell CUDA compute capability 12.x +- Validated GPU: SM121 / GB10 / DGX Spark +- Dtype: BF16 inputs with FP32 accumulation references +- Layout: paged KV cache +- Model path: FlashRT MiniMax-Spark runtime on DGX Spark / GB10 + +## Correctness Gate + +Run quick validation: + +```bash +PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \ + python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py --quick +``` + +Run full validation: + +```bash +PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \ + python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py +``` + +Run standalone long-context validation: + +```bash +PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \ + python MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py --long-context +``` + +Expected full coverage: + +| Area | Shapes | Reference | Required | +|---|---:|---|---| +| API surface | official `MiniMaxAI/msa` public names | `api_status.py` | all official root names exported; no unsupported public root API entries | +| Native CUDA top-k helper | heads 64, batch 1-2, blocks 1-256 | PyTorch top-k over valid blocks | exact set match | +| Decode sparse GQA attention | ctx 128, 2048, 4096, 32768 | paged FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 | +| Prefill sparse GQA attention | ctx 512, 4096 | paged causal FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 | +| Decode sparse GQA attention with sink | ctx 2048, 32768 | paged FP32 PyTorch | cos >= 0.999, max_abs <= 5e-2 | +| Official decode API wrapper | ctx 2048, 4096 | direct Blackwell decode kernel | cos = 1.0, max_abs = 0 | +| Official CSR prefill API wrapper | ctx 512, 2048 | direct Blackwell prefill kernel | cos = 1.0, max_abs = 0 under CSR-preserved block order | +| Official NVFP4 prefill API wrapper | ctx 512 BF16 dispatch path | `sparse_atten_func` | cos = 1.0, max_abs = 0 | +| Native CUDA NVFP4 dequant | rows/cols `(1,128)`, `(257,128)`, `(64,4096)` | Python NVFP4 reference | exact BF16 match | +| Official FP4 indexer API | tiny FP4 packed tensors; native artifact path when built | PyTorch block-score reference | returns official score layout | +| Decode lightning indexer | ctx 2048, 4096, 32768 | PyTorch blockmax top-k set | overlap >= 0.99 | +| Standalone long-context decode | ctx 65536, 131072 | paged FP32 PyTorch / direct kernel | cos >= 0.999; wrapper max_abs = 0 | +| Installed-artifact native long top-k | blocks 512, 1024 | PyTorch top-k over valid blocks | exact set match | + +API surface validation: + +```bash +PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \ + python -m pytest MiniMaxAI-msa-blackwell/tests/test_api_surface.py -q +``` + +The test tracks every official `MiniMaxAI/msa` public API name: + +- `sparse_atten_func` +- `sparse_atten_nvfp4_kv_func` +- `sparse_decode_atten_func` +- `SparseDecodePagedAttentionWrapper` +- `fp4_indexer_block_scores` +- `build_k2q_csr` +- `SparseK2qCsrBuilderSm100` +- `Nvfp4QuantizedTensor` +- `quantize_bf16_to_nvfp4_128x4` +- `quantize_kv_bf16_to_nvfp4_128x4` +- `dequantize_nvfp4_128x4_to_bf16` +- `swizzle_nvfp4_scale_to_128x4` +- `nvfp4_global_scale_from_amax` + +The root module exports every official public name. Decode, CSR prefill, NVFP4 +prefill compatibility, FP4 block scoring, CSR, and NVFP4 helper names are all +callable. Hub built artifacts use compiled CUDA ops for the MiniMax-M3 +Blackwell decode route, FP4 block-score indexer, and swizzled NVFP4 -> BF16 +dequantization path. Source-tree mode keeps reference paths so the API remains +testable before the extension is built. + +## FlashRT Integration Note + +FlashRT has validated the decode sparse path on SM121 over context lengths +128 to 32768 with cosine similarity >= 0.999. The 32768 context length has +also been exercised in the FlashRT MiniMax-Spark model runtime on DGX Spark / +GB10, so it is the current end-to-end model validation boundary. + +The standalone package kernel tests additionally cover 65536 and 131072 +context lengths. These long-context rows validate the kernel and API wrapper +contract outside the full model runtime; they should not be described as +MiniMax-Spark end-to-end model validation until the full runtime path is rerun +at those lengths. + +The same decode sparse path has also been exercised in FlashRT's MiniMax-Spark +model runtime on DGX Spark / GB10. That end-to-end validation is intentionally +kept as a FlashRT runtime validation item, while this Hub package exposes the +standalone kernel API for community use. + +## Native Helper Compile Smoke + +Before HF Jobs publish, the native helper was compiled locally as a PyTorch +extension using the same source files: + +- `torch-ext/torch_binding.cpp` +- `csrc/msa_topk_from_scores.cu` +- `csrc/msa_decode_attn.cu` +- `csrc/msa_decode_attn_mma.cu` +- `csrc/msa_indexer_block_scores.cu` +- `csrc/msa_nvfp4_dequant.cu` + +Environment: + +| Field | Value | +|---|---| +| GPU | NVIDIA GeForce RTX 5090 | +| PyTorch | 2.9.1+cu128 | +| nvcc | CUDA 13.0 | +| Target arch | sm_120 | + +Result: + +| Check | Shape | Reference | Verdict | +|---|---:|---|---| +| Native score -> top-k | heads 64, batch 1, blocks 256, topk 16 | PyTorch top-k set | PASS | +| Native FP4 block-score indexer | official `[Hq, blocks, total_q]` score layout | PyTorch block-score reference | PASS | +| Native NVFP4 swizzled -> BF16 dequant | rows/cols `(1,128)`, `(257,128)`, `(64,4096)` | Python NVFP4 reference | PASS | + +## Blackwell Package Validation + +Remote Blackwell validation environment: + +| Field | Value | +|---|---| +| Host | `spark-f517` | +| GPU | NVIDIA GB10 | +| Compute capability | 12.1 | +| Driver | 580.159.03 | +| Python | 3.12.3 | +| PyTorch | 2.12.0+cu130 | +| Triton | 3.7.0 | + +Command: + +```bash +PY=/home/leadtek/jax/bin/python +PYTHONPATH=MiniMaxAI-msa-blackwell/torch-ext \ + $PY MiniMaxAI-msa-blackwell/tests/test_msa_blackwell.py +``` + +Result: + +| Check | Shape | Cosine | Max abs / overlap | Verdict | +|---|---|---:|---:|---| +| Decode sparse GQA | ctx128_b1 | 0.999998 | 1.6032e-03 | PASS | +| Decode sparse GQA | ctx2048_b1 | 0.999996 | 4.9090e-04 | PASS | +| Decode sparse GQA | ctx2048_b2_sink | 0.999996 | 6.8302e-04 | PASS | +| Decode sparse GQA | ctx4096_b1 | 0.999996 | 4.5899e-04 | PASS | +| Decode sparse GQA | ctx4096_b2_mixed | 0.999996 | 7.3129e-04 | PASS | +| Decode sparse GQA | ctx32768_b1 | 0.999996 | 6.9451e-04 | PASS | +| Decode sparse GQA | ctx32768_b1_sink | 0.999996 | 5.6115e-04 | PASS | +| Decode sparse GQA | ctx65536_b1 | 0.999996 | 4.3470e-04 | PASS | +| Decode sparse GQA | ctx131072_b1 | 0.999996 | 7.1825e-04 | PASS | +| Decode top-k indexer | ctx2048 | n/a | overlap 1.000 | PASS | +| Decode top-k indexer | ctx4096 | n/a | overlap 1.000 | PASS | +| Decode top-k indexer | ctx32768 | n/a | overlap 1.000 | PASS | +| Decode top-k indexer | ctx65536 | n/a | overlap 1.000 | PASS | +| Decode top-k indexer | ctx131072 | n/a | overlap 1.000 | PASS | +| Official decode wrapper | ctx2048 | 1.000000 | 0.0000e+00 | PASS | +| Official decode wrapper | ctx4096 | 1.000000 | 0.0000e+00 | PASS | +| Official decode wrapper | ctx65536 | 1.000000 | 0.0000e+00 | PASS | +| Official decode wrapper | ctx131072 | 1.000000 | 0.0000e+00 | PASS | +| Native CUDA NVFP4 dequant | rows1_cols128 | 1.000000 | 0.0000e+00 | PASS | +| Native CUDA NVFP4 dequant | rows257_cols128 | 1.000000 | 0.0000e+00 | PASS | +| Native CUDA NVFP4 dequant | rows64_cols4096 | 1.000000 | 0.0000e+00 | PASS | + +Installed-artifact native top-k validation on RTX 5090 / torch 2.11 / CUDA +12.8: + +| Context | Blocks | Overlap | Verdict | +|---:|---:|---:|---| +| 32768 | 256 | 1.000 | PASS | +| 65536 | 512 | 1.000 | PASS | +| 131072 | 1024 | 1.000 | PASS | + +The warning `tl.make_block_ptr is deprecated` appears with Triton 3.7.0. It is +a deprecation warning, not a correctness failure. + +## Native Alignment Status + +The upstream `MiniMaxAI/msa` package targets SM100. This Blackwell package +keeps the same public API surface where practical and provides native CUDA +implementations for the hot paths needed by the FlashRT MiniMax-Spark runtime: + +- score-to-top-k sparse block selection; +- tensor-core sparse decode for the MiniMax-M3 Blackwell shape; +- FP4 block-score indexing; +- swizzled NVFP4 -> BF16 dequantization for the W4A16 path. + +The CSR prefill wrapper remains part of the public compatibility surface and is +validated against the package reference path. Shape and parameter restrictions +are explicit errors rather than silent wrong results. diff --git a/benchmarks/benchmark_decode.py b/benchmarks/benchmark_decode.py new file mode 100644 index 0000000000000000000000000000000000000000..e38c6a7d95bbbfafb178d3ee1eabddf00a989404 --- /dev/null +++ b/benchmarks/benchmark_decode.py @@ -0,0 +1,76 @@ +import argparse +import time + +import torch + +from minimaxai_msa_blackwell import ( + flash_decode_with_gqa_share_sparse, + has_native_ops, + native_topk_from_scores, +) + + +def make_case(ctx: int, seed: int = 0): + g = torch.Generator(device="cuda").manual_seed(seed) + batch, hq, hkv, d = 1, 64, 4, 128 + block, topk = 128, 16 + q = torch.randn(batch, hq, d, device="cuda", dtype=torch.bfloat16, generator=g) + k_cache = torch.randn(ctx, hkv, d, device="cuda", dtype=torch.bfloat16, generator=g) + v_cache = torch.randn(ctx, hkv, d, device="cuda", dtype=torch.bfloat16, generator=g) + req_to_token = torch.arange(ctx, device="cuda", dtype=torch.int32).view(1, -1) + seq_lens = torch.tensor([ctx], device="cuda", dtype=torch.int32) + slot_ids = torch.zeros(batch, device="cuda", dtype=torch.int64) + nb = (ctx + block - 1) // block + n = min(topk, nb) + topk_idx = torch.full((hkv, batch, topk), -1, device="cuda", dtype=torch.int32) + topk_idx[:, :, :n] = torch.arange(n, device="cuda", dtype=torch.int32).view(1, 1, n) + return q, None, k_cache, v_cache, req_to_token, seq_lens, slot_ids, block, topk_idx + + +def bench(ctx: int, warmup: int, iters: int) -> float: + args = make_case(ctx) + for _ in range(warmup): + flash_decode_with_gqa_share_sparse(*args) + torch.cuda.synchronize() + start = time.perf_counter() + for _ in range(iters): + flash_decode_with_gqa_share_sparse(*args) + torch.cuda.synchronize() + return (time.perf_counter() - start) * 1e6 / iters + + +def bench_native_topk(ctx: int, warmup: int, iters: int) -> float | None: + if not has_native_ops(): + return None + heads, batch, block, topk = 64, 1, 128, 16 + blocks = (ctx + block - 1) // block + score = torch.randn(heads, batch, blocks, device="cuda", dtype=torch.float32) + seq_lens = torch.tensor([ctx], device="cuda", dtype=torch.int32) + for _ in range(warmup): + native_topk_from_scores(score, seq_lens, block, topk) + torch.cuda.synchronize() + start = time.perf_counter() + for _ in range(iters): + native_topk_from_scores(score, seq_lens, block, topk) + torch.cuda.synchronize() + return (time.perf_counter() - start) * 1e6 / iters + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--ctx", type=int, nargs="+", default=[2048, 4096, 32768, 65536, 131072]) + ap.add_argument("--warmup", type=int, default=10) + ap.add_argument("--iters", type=int, default=50) + args = ap.parse_args() + + print("gpu:", torch.cuda.get_device_name()) + print("ctx,attention_mean_us,native_topk_mean_us") + for ctx in args.ctx: + attn_us = bench(ctx, args.warmup, args.iters) + topk_us = bench_native_topk(ctx, args.warmup, args.iters) + topk_text = "NA" if topk_us is None else f"{topk_us:.3f}" + print(f"{ctx},{attn_us:.3f},{topk_text}") + + +if __name__ == "__main__": + main() diff --git a/build/torch211-cxx11-cu128-x86_64-linux/__init__.py b/build/torch211-cxx11-cu128-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a666afa7b37c9d5448b9bb04d573dcfe4e45750 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/__init__.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention. + +This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on +DGX Spark / GB10 / SM121. It exposes the +MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime. + +Implementation status: + * native CUDA helper: score -> top-k sparse block ids; + * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape; + * native CUDA FP4 block-score indexer; + * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path; + * Blackwell-validated sparse prefill path; + * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers, + and FP4 block-score helpers. + +The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell +extension path. The public API is Tensor-oriented and independent from +FlashRT's serving runtime. + +Public API +---------- +Lightning indexer (q.k blockmax score -> top-k block select), paged KV: + flash_decode_with_topk_idx (single-token decode, split-K) + +Block-sparse GQA attention (consumes block_indices from the indexer): + flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks) + sparse_atten_func (official CSR prefill wrapper) + sparse_decode_atten_func (official paged decode wrapper) + +PyTorch naive references (for correctness checks): + naive_flash_decode_with_gqa_share_sparse (attention given topk_idx) + naive_flash_decode_with_topk_idx (joint indexer + attention) + +All Triton entrypoints expect a paged KV cache: + k_cache / v_cache : [max_slots, num_kv_heads, head_dim] + req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot) + slot_ids : [batch] (per-request row in req_to_token) + topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed, + -1 right-padding (the M3 indexer contract). +See VENDOR_NOTES.md for full shape/dtype/layout contracts. +""" + +from .common.utils import get_cu_seqblocks, robust_allocator +from .decode.flash_with_topk_idx import flash_decode_with_topk_idx +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .interface import ( + SparseDecodePagedAttentionWrapper, + fp4_indexer_block_scores, + sparse_atten_func, + sparse_atten_nvfp4_kv_func, + sparse_decode_atten_func, +) +from ._native import ( + has_native_ops, + native_nvfp4_dequant_swizzled_to_bf16, + native_topk_from_scores, +) +from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx +from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse +from .quantize import ( + Nvfp4QuantizedTensor, + dequantize_nvfp4_128x4_to_bf16, + nvfp4_global_scale_from_amax, + quantize_bf16_to_nvfp4_128x4, + quantize_kv_bf16_to_nvfp4_128x4, + swizzle_nvfp4_scale_to_128x4, +) +from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr +from .api_status import ( + OFFICIAL_API_STATUS, + OFFICIAL_MINIMAX_MSA_FUNCTIONS, + V1_AVAILABLE_FUNCTIONS, + available_functions, + official_api_status, + official_minimax_msa_functions, + unsupported_official_functions, +) + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "naive_flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "get_cu_seqblocks", + "robust_allocator", + "OFFICIAL_API_STATUS", + "OFFICIAL_MINIMAX_MSA_FUNCTIONS", + "V1_AVAILABLE_FUNCTIONS", + "available_functions", + "official_api_status", + "official_minimax_msa_functions", + "unsupported_official_functions", +] diff --git a/build/torch211-cxx11-cu128-x86_64-linux/_compat.py b/build/torch211-cxx11-cu128-x86_64-linux/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..da445342268947ab3b1c5bfd16e95dc4705c7abe --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/_compat.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton +# kernels. The upstream SGLang modules pull two tiny things from the framework: +# +# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path) +# from sglang.srt.environ import envs -> a feature flag for an optional +# JIT-CUDA radix top-k fast path +# +# Both are replaced here so the kernels import with only torch + triton. The +# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel +# radix select), which is what we want on sm_121 (GB10). + + +def is_hip() -> bool: + """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121). + + Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV + widening branch (HIP-only). With this False, the CUDA dtype contract + (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA. + """ + try: + import torch + + return bool(getattr(torch.version, "hip", None)) + except Exception: + return False + + +class _Flag: + """Minimal stand-in for an sglang.srt.environ boolean flag (always off).""" + + def __init__(self, default: bool = False): + self._default = default + + def get(self) -> bool: + return self._default + + +class _Envs: + """Stub for `sglang.srt.environ.envs`. + + Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode + indexer. Keeping it False routes top-k selection through the 2-stage Triton + fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the + decode indexer stays pure-Triton. + """ + + SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False) + + +envs = _Envs() diff --git a/build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so b/build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..9b16ac9b8805c138a2e2851f4514509935e51895 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:528883873873d9a482cbe93eb0ea0ec5be7f0eac009dd8b103a0609841c032d5 +size 446928 diff --git a/build/torch211-cxx11-cu128-x86_64-linux/_native.py b/build/torch211-cxx11-cu128-x86_64-linux/_native.py new file mode 100644 index 0000000000000000000000000000000000000000..dd54c53374469c8424d0ca6e3f25ea534005c8d5 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/_native.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional native CUDA helpers for the FlashRT Blackwell MSA package.""" + +from __future__ import annotations + +import torch + +try: + from ._ops import add_op_namespace_prefix, ops + + _HAS_NATIVE_OPS = True +except Exception: + add_op_namespace_prefix = None + ops = None + _HAS_NATIVE_OPS = False + + +def has_native_ops() -> bool: + return _HAS_NATIVE_OPS + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores")) + def _msa_topk_from_scores_fake( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, + topk_idx: torch.Tensor, + ) -> None: + if score.dim() != 3: + raise RuntimeError("score must have shape (heads, batch, max_blocks)") + heads, batch, _ = score.shape + if seq_lens.shape != (batch,): + raise RuntimeError("seq_lens must have shape (batch,)") + if topk_idx.shape != (heads, batch, topk): + raise RuntimeError("topk_idx must have shape (heads, batch, topk)") + return None + + +def native_topk_from_scores( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, +) -> torch.Tensor: + """Return top-k sparse block ids from a score tensor.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if not score.is_contiguous(): + score = score.contiguous() + if not seq_lens.is_contiguous(): + seq_lens = seq_lens.contiguous() + out = torch.empty( + (score.shape[0], score.shape[1], int(topk)), + device=score.device, + dtype=torch.int32, + ) + ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged") + ) + def _msa_decode_sparse_attn_mma_paged_fake( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size: int, + sm_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_decode_mma_supported( + q: torch.Tensor, + k_cache: torch.Tensor, + *, + block_size: int, +) -> bool: + """Whether the tensor-core decode kernel applies to these inputs. + + The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA + group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64. + """ + + if not _HAS_NATIVE_OPS: + return False + if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16: + return False + if q.dim() != 3 or k_cache.dim() != 3: + return False + head_dim = int(q.shape[-1]) + num_q_heads = int(q.shape[1]) + num_kv_heads = int(k_cache.shape[1]) + if head_dim != 128 or num_kv_heads <= 0: + return False + if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16: + return False + if int(block_size) % 64 != 0: + return False + return True + + +def native_decode_sparse_attn_mma_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + block_size: int, + topk_idx: torch.Tensor, + sm_scale: float, +) -> torch.Tensor: + """Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + out = torch.empty( + (int(q.shape[0]), int(q.shape[1]), int(q.shape[2])), + device=q.device, + dtype=q.dtype, + ) + ops.msa_decode_sparse_attn_mma_paged( + q.contiguous(), + k_cache.contiguous(), + v_cache.contiguous(), + req_to_token.contiguous(), + seq_lens.contiguous(), + slot_ids.contiguous(), + topk_idx.contiguous(), + int(block_size), + float(sm_scale), + out, + ) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_indexer_block_scores") + ) + def _msa_indexer_block_scores_fake( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + causal: int, + scores: torch.Tensor, + ) -> None: + return None + + +def native_indexer_block_scores( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + *, + max_blocks: int, + causal: bool, +) -> torch.Tensor: + """Block-max QK indexer scores. q/k_pages are bf16 (already dequantized). + + Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible). + """ + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + total_q = int(q.shape[0]) + hq = int(q.shape[1]) + scores = torch.full( + (hq, int(max_blocks), total_q), + float("-inf"), + dtype=torch.float32, + device=q.device, + ) + ops.msa_indexer_block_scores( + q.contiguous(), + k_pages.contiguous(), + batch_of_q.contiguous(), + cu_q.contiguous(), + cu_k.contiguous(), + cu_pages.contiguous(), + kv_indices.contiguous(), + 1 if causal else 0, + scores, + ) + return scores + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16") + ) + def _msa_nvfp4_dequant_swizzled_to_bf16_fake( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_nvfp4_dequant_swizzled_to_bf16( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor | float, +) -> torch.Tensor: + """Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if packed.dtype is not torch.uint8: + raise TypeError("packed must be torch.uint8") + if scale_128x4.dtype is not torch.uint8: + raise TypeError("scale_128x4 must be torch.uint8") + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16) + if isinstance(global_scale, torch.Tensor): + global_scale_value = float(global_scale.reshape(-1)[0].item()) + else: + global_scale_value = float(global_scale) + ops.msa_nvfp4_dequant_swizzled_to_bf16( + packed.contiguous(), + scale_128x4.contiguous(), + global_scale_value, + out, + ) + return out + + +__all__ = [ + "has_native_ops", + "native_topk_from_scores", + "native_decode_mma_supported", + "native_decode_sparse_attn_mma_paged", + "native_indexer_block_scores", + "native_nvfp4_dequant_swizzled_to_bf16", +] diff --git a/build/torch211-cxx11-cu128-x86_64-linux/_ops.py b/build/torch211-cxx11-cu128-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..41e39102ea6d028564a7e10c3b7004e7c40dae7b --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _minimaxai_msa_blackwell_cuda_25b0067 +ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}" diff --git a/build/torch211-cxx11-cu128-x86_64-linux/api_status.py b/build/torch211-cxx11-cu128-x86_64-linux/api_status.py new file mode 100644 index 0000000000000000000000000000000000000000..3dd4d744b49f1bc7acd6b3ba2a2e2c5f7b591252 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/api_status.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Public API status for the FlashRT MiniMax MSA Blackwell package.""" + +from __future__ import annotations + +from types import MappingProxyType + + +V1_AVAILABLE_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "naive_flash_decode_with_gqa_share_sparse", + "get_cu_seqblocks", + "robust_allocator", +) + +OFFICIAL_MINIMAX_MSA_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +) + +_OFFICIAL_API_STATUS = { + "sparse_atten_func": { + "status": "available", + "target": "blackwell-prefill", + "reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.", + }, + "sparse_atten_nvfp4_kv_func": { + "status": "available", + "target": "blackwell-prefill-nvfp4", + "reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.", + }, + "sparse_decode_atten_func": { + "status": "available", + "target": "v1", + "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.", + }, + "SparseDecodePagedAttentionWrapper": { + "status": "available", + "target": "v1", + "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.", + }, + "fp4_indexer_block_scores": { + "status": "available", + "target": "blackwell-fp4-indexer", + "reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.", + }, + "build_k2q_csr": { + "status": "available", + "target": "v1", + "reason": "CSR construction helper is available for the official prefill API.", + }, + "SparseK2qCsrBuilderSm100": { + "status": "available", + "target": "v1", + "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.", + }, + "Nvfp4QuantizedTensor": { + "status": "available", + "target": "v1", + "reason": "NVFP4 metadata dataclass is available.", + }, + "quantize_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "quantize_kv_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "dequantize_nvfp4_128x4_to_bf16": { + "status": "available", + "target": "v1", + "reason": "Reference dequant helper is available.", + }, + "swizzle_nvfp4_scale_to_128x4": { + "status": "available", + "target": "v1", + "reason": "Scale swizzle helper is available.", + }, + "nvfp4_global_scale_from_amax": { + "status": "available", + "target": "v1", + "reason": "Global scale helper is available.", + }, +} + +OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS) + + +def available_functions() -> tuple[str, ...]: + """Return the functions/classes intentionally available in v1.""" + + return V1_AVAILABLE_FUNCTIONS + + +def official_minimax_msa_functions() -> tuple[str, ...]: + """Return the upstream MiniMaxAI/msa public function/class names tracked here.""" + + return OFFICIAL_MINIMAX_MSA_FUNCTIONS + + +def official_api_status() -> dict[str, dict[str, str]]: + """Return a copy of the official API compatibility status table.""" + + return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()} + + +def unsupported_official_functions() -> tuple[str, ...]: + """Return official MiniMaxAI/msa names that are not exported by this v1 package.""" + + return tuple( + name + for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS + if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"} + ) diff --git a/build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py b/build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/common/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch211-cxx11-cu128-x86_64-linux/common/utils.py b/build/torch211-cxx11-cu128-x86_64-linux/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..81b7816b719973729312c0d84b36f9f50579f105 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/common/utils.py @@ -0,0 +1,144 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +import functools +from collections import deque +from typing import Any, Callable, Tuple + +import torch +import triton.language as tl + +_tma_keep_alive_buf = deque(maxlen=200) + +try: + make_tensor_descriptor = tl.make_tensor_descriptor +except Exception: + make_tensor_descriptor = tl._experimental_make_tensor_descriptor + + +def robust_allocator(size: int, alignment: int, stream: int = None): + """Allocator for Triton TMA descriptors. + + We keep reference in deque to prevent GC from collecting the buffer. + """ + tensor = torch.empty(size, device="cuda", dtype=torch.uint8) + _tma_keep_alive_buf.append(tensor) + return tensor + + +def tensor_cache(maxsize: int = 8): + """ + Cache function results using identity comparison. + Zero-overhead cache hit: no hash, no DtoH, just pointer comparison. + + Args: + maxsize: Maximum number of cached entries. Supports multi-GPU scenarios + where different devices have different tensor arguments. + """ + + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + # LRU-style cache: list of (args, kwargs, result) tuples + # Most recently used at the end + _cache: list = [] + + def _args_match(args: tuple, cached_args: tuple) -> bool: + if len(args) != len(cached_args): + return False + for i in range(len(args)): + if args[i] is not cached_args[i]: + return False + return True + + def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool: + if not kwargs and not cached_kwargs: + return True + if kwargs.keys() != cached_kwargs.keys(): + return False + for k, v in kwargs.items(): + if v is not cached_kwargs[k]: + return False + return True + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + # Search cache (most recent first for better hit rate) + for i in range(len(_cache) - 1, -1, -1): + cached_args, cached_kwargs, cached_result = _cache[i] + if _args_match(args, cached_args) and _kwargs_match( + kwargs, cached_kwargs + ): + # Move to end (most recently used) + if i != len(_cache) - 1: + _cache.append(_cache.pop(i)) + return cached_result + + # Cache miss + result = fn(*args, **kwargs) + + # Add to cache + if len(_cache) >= maxsize: + _cache.pop(0) # Remove oldest + _cache.append((args, kwargs, result)) + return result + + # Expose cache for manual clearing if needed + wrapper.cache_clear = lambda: _cache.clear() + wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize} + + return wrapper + + return decorator + + +@tensor_cache(maxsize=8) +def get_cu_seqblocks( + cu_seqlens: torch.Tensor, + max_seqlen: int, + block_size_q: int, + block_size_k: int, +) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]: + """Compute cumulative sequence block indices for blocked sparse attention. + + Converts token-level cumulative sequence lengths to block-level indices, + which are needed for block-sparse attention kernels. + + Note: + Results are cached (maxsize=8) based on input arguments. Repeated calls + with the same cu_seqlens, max_seqlen, and block sizes will return cached + results without recomputation. + + Args: + cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32. + max_seqlen: Maximum sequence length in the batch. + block_size_q: Query block size. + block_size_k: Key-value block size. + + Returns: + A tuple of 6 values: + - cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1] + - max_seqblock_q: Maximum number of query blocks per sequence. + - all_seqblock_q: Total number of query blocks across all sequences. + - cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1] + - max_seqblock_k: Maximum number of key blocks per sequence. + - all_seqblock_k: Total number of key blocks across all sequences. + """ + cu_seqblocks_q = torch.zeros_like(cu_seqlens) + cu_seqblocks_k = torch.zeros_like(cu_seqlens) + seq_lens = torch.diff(cu_seqlens) + seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q + seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k + max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q + max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k + cu_seqblocks_q[1:] = seqblocks_q + cu_seqblocks_k[1:] = seqblocks_k + cu_seqblocks_q.cumsum_(0) + cu_seqblocks_k.cumsum_(0) + all_seqblock_q = seqblocks_q.sum().item() + all_seqblock_k = seqblocks_k.sum().item() + return ( + cu_seqblocks_q, + max_seqblock_q, + all_seqblock_q, + cu_seqblocks_k, + max_seqblock_k, + all_seqblock_k, + ) diff --git a/build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py b/build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/decode/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py b/build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..8f56bdba2908e8af036785eae158338758555ee5 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/decode/flash_with_topk_idx.py @@ -0,0 +1,1150 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import envs # vendored: was `from sglang.srt.environ import envs` + +from ..common.utils import robust_allocator +from .._native import has_native_ops, native_topk_from_scores + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "BATCH_SIZE_BUCKET", + "gqa_group_size", + "head_dim", + "block_size", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_kernel( + q_ptr, # Q: b x qh x d + k_cache_ptr, # K paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_r2t_b, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks <= topk: + return + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # prefetch first iteration's slots + r2t_base = req_to_token_ptr + sid * stride_r2t_b + prefetch_pos = chunk_start + off_n + prefetch_mask = prefetch_pos < seq_len + prefetched_slots = tl.load( + r2t_base + prefetch_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + # score-only: compute block scores without loading V + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos_mask = prefetch_mask + slots = prefetched_slots + # prefetch next iteration's slots + next_i = i + BLOCK_SIZE_N + if next_i < chunk_end: + next_pos = next_i + off_n + prefetch_mask = next_pos < seq_len + prefetched_slots = tl.load( + r2t_base + next_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + curr_block_idx = i // block_size + off_bpk + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)) + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "gqa_group_size", + "head_dim", + "block_size", + "HAS_SINK", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_attn_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + o_ptr, # O: c x b x qh x d + lse_ptr, # lse: c x b x qh + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + qsink = ( + tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e + ) # (BLOCK_SIZE_H,) + m_i = qsink + l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # full attention or causal attention + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos = i + off_n + pos_mask = pos < seq_len + # resolve slot per position via req_to_token + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + # current block indices: [BLOCKS_PER_K_BLOCK] + curr_block_idx = i // block_size + off_bpk + # Combined condition with single nested tl.where: + # - local_blocks (1e29) takes priority over init_blocks (1e30) + # - When init_blocks=0: curr_block_idx < 0 is always False + # - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + # Apply in one fused operation: local > init > original + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks > topk: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + else: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + # max-of-max == max(qk), avoids re-scanning qk + m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + m_i = m_ij + l_i = l_i * acc_o_scale + l_ij + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + # final scale + acc_o = acc_o / l_i[:, None] + # compute lse in log2 scale for merge kernel + lse_i = m_i + tl.log2(l_i) + # save output + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + # save lse (in log2 scale, will be used directly in merge kernel) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4] + ], + key=["BLOCK_SIZE_D"], + restore_value=["o_ptr"], +) +@triton.jit +def _merge_attn_out_kernel( + o_ptr, + lse_ptr, + # shape + seq_lens, + head_dim, + block_size: tl.constexpr, + # stride + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_KV_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + # recompute chunk layout (must match _decode_score_attn_kernel) + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks) + # ptrs + off_c = tl.arange(0, NUM_KV_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(valid_chunks, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + # load o and lse + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf")) + # merge o (lse is in log2 scale, use exp2/log2 for efficiency) + lse_shifted = lse - tl.max(lse, axis=0) + log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0)) + scale = tl.exp2(lse_shifted - log_sum) + o = tl.sum(o * scale[:, None], axis=0) + # save o + o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@triton.jit +def _compare_and_swap( + x, + ids, + flip, + i: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)] + y = tl.reshape(x, shape) + # slice left/right with 'stride' 2**(n_dims - i - 1) + mask = tl.arange(0, 2)[None, :, None] + left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype) + right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype) + left = tl.reshape(left, x.shape) + right = tl.reshape(right, x.shape) + # idx + y_idx = tl.reshape(ids, shape) + left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape) + right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape) + left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype) + right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype) + # actual compare-and-swap + idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True) + ileft = left.to(idtype, bitcast=True) + iright = right.to(idtype, bitcast=True) + ix = x.to(idtype, bitcast=True) + + cond = (left > right) != flip + ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix)) + new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids)) + return ret.to(x.dtype, bitcast=True), new_ids + + +@triton.jit +def _bitonic_merge( + x, + ids, + stage: tl.constexpr, + order: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + tl.static_assert(stage <= n_dims) + # flip denotes whether to re-arrange sub-sequences of elements in ascending or + # descending order. + # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage + # if flip = 00110011... then all the elements will be re-arranged alternatingly (with + # a stride of 2) at this stage + if order == 2: + shape: tl.constexpr = [ + n_outer * 2 ** (n_dims - 1 - stage), + 2, + 2**stage, + ] + flip = tl.reshape( + tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape + ) + else: + flip = order + # perform `stage` rounds of `compare-and-swap` + for i in tl.static_range(stage): + x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims) + return x, ids + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2), + ], + key=["topk"], +) +@triton.jit +def _topk_index_partial_kernel( + s_ptr, # Score: [num_q_heads, BS, max_seqblock] + ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns + # strides + stride_s_h, + stride_s_b, + stride_s_k, + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + # META + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + tl.static_assert(BLOCK_SIZE_K > topk) + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + pid_chunk = tl.program_id(2) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Slice this chunk owns within [0, num_blocks). + chunk_start = pid_chunk * chunk_blocks + chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks) + chunk_actual = tl.maximum(chunk_end - chunk_start, 0) + + off_k = tl.arange(0, BLOCK_SIZE_K) + off_t = tl.arange(0, BLOCK_SIZE_T) + + # Pointer starts at chunk_start. + s_ptrs = ( + s_ptr + + pid_b * stride_s_b + + pid_h * stride_s_h + + (chunk_start + off_k) * stride_s_k + ) + + topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32) + topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32) + left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2 + + # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty + # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0. + for i in tl.range(0, chunk_actual, BLOCK_SIZE_K): + mask = off_k < chunk_actual - i + score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32) + score = tl.where(score != score, -1e30, score) + s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K + topk_score, last_topk_score = score, topk_score + topk_idx, last_topk_idx = ( + tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global + topk_idx, + ) + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), j, 2, n_dims + ) + if i != 0: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims + ) + topk_score_new = last_topk_score * left_half_mask + topk_score * ( + 1 - left_half_mask + ) + topk_idx_new = last_topk_idx * left_half_mask + topk_idx * ( + 1 - left_half_mask + ) + topk_score, topk_idx = _bitonic_merge( + topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims + ) + else: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims + ) + + # Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort). + topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + final_score = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + final_idx = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + # Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry + # -1e30 / 0 sentinels and lose to real scores in the merge stage. + ts_ptrs = ( + ts_partial_ptr + + pid_chunk * stride_ts_c + + pid_b * stride_ts_b + + pid_h * stride_ts_h + + off_t * stride_ts_t + ) + ti_ptrs = ( + ti_partial_ptr + + pid_chunk * stride_ti_c + + pid_b * stride_ti_b + + pid_h * stride_ti_h + + off_t * stride_ti_t + ) + tl.store(ts_ptrs, final_score) + tl.store(ti_ptrs, final_idx) + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + "BLOCK_SIZE_K": lambda args: triton.next_power_of_2( + args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"]) + ), + } +) +@triton.jit +def _topk_index_merge_kernel( + ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape + ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk] + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + # strides + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_tif_h, + stride_tif_b, + stride_tif_t, + # META + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K. + # candidate at flat position p comes from chunk = p // BLOCK_SIZE_T, + # in_chunk = p % BLOCK_SIZE_T. + off = tl.arange(0, BLOCK_SIZE_K) + chunk_idx = off // BLOCK_SIZE_T + in_chunk_idx = off % BLOCK_SIZE_T + valid = chunk_idx < NUM_TOPK_CHUNKS + + score_offset = ( + chunk_idx * stride_ts_c + + pid_h * stride_ts_h + + pid_b * stride_ts_b + + in_chunk_idx * stride_ts_t + ) + idx_offset = ( + chunk_idx * stride_ti_c + + pid_h * stride_ti_h + + pid_b * stride_ti_b + + in_chunk_idx * stride_ti_t + ) + + score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to( + tl.float32 + ) + score = tl.where(score != score, -1e30, score) + idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32) + + # Full bitonic descending sort of BLOCK_SIZE_K items. + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims) + score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims) + + # Extract first BLOCK_SIZE_T positions — these are the global top-K. + extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + topk_idx_final = tl.sum( + extract_mask[:, None] + * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + off_t = tl.arange(0, BLOCK_SIZE_T) + tif_ptrs = ( + ti_final_ptr + + pid_h * stride_tif_h + + pid_b * stride_tif_b + + off_t * stride_tif_t + ) + topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1) + tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty)) + + +@torch.no_grad() +def flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + init_blocks: int, + local_blocks: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + score_type: str = "max", + disable_index_value: bool = False, + use_dense_main_attn: bool = False, # NOTE: need transform idx in this case + page_size: int = 1, +) -> torch.Tensor: + assert score_type in ( + "max", + "lse", + ), f"score_type must be 'max' or 'lse', got {score_type!r}" + triton.set_allocator(robust_allocator) + # dtype check + assert ( + q.dtype == torch.bfloat16 + or q.dtype == torch.float16 + and k_cache.dtype == q.dtype + ) + if not disable_index_value: + assert v_cache is not None + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + max_kv_len = req_to_token.shape[1] + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets. + # Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads. + # TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by: + # NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS) + # then rounded down to the nearest power of 2 (merge kernel requires it). + # Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len. + # Empty chunks early-return cheaply, so over-chunking is nearly free. + # E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096. + TARGET_GRID = 4096 + MAX_NUM_KV_CHUNKS = 256 + target = max( + 1, + min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_KV_CHUNKS = 1 << (target.bit_length() - 1) + score_kv_len = min(max_seqlen, max_kv_len) + # The score producers below write every valid block column + # [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers + # clamp their scan to the same per-row valid block count, so columns beyond + # seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph + # capture the static score shape can be much larger than the live context. + score = torch.empty( + (num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)), + dtype=torch.float32, + device=q.device, + ) + use_jit_topk = ( + envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get() + and score.shape[2] <= 4096 + and topk <= 32 + ) + # If the live context has <= topk sparse blocks, the downstream dense + # page-table/JIT top-k kernels select every block from seq_lens directly + # without reading score. Keep this gate in sync with the consumers below: + # _skip_block_topk/use_dense_main_attn and use_jit_topk special-case + # num_blocks <= topk, while the Triton fallback reads score and must not + # skip these writes. + skip_trivial_topk_score = use_dense_main_attn or use_jit_topk + + grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads) + if disable_index_value: + _decode_score_kernel[grid]( + q, + k_cache, + req_to_token, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + req_to_token.stride(0), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + else: + assert v_cache is not None + o = torch.empty( + NUM_KV_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse = torch.empty( + NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device + ) + _decode_score_attn_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + o, + lse, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + # Fused top-k + page-table transform: emit the dense backend's page table + # directly (page-size-aware) instead of block ids, skipping a separate gather. + # The page table + per-query effective KV length are allocated and returned. + real_seq_lens = None + if use_dense_main_attn: + # VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The + # standalone vendor only ships the pure-Triton path, so this is gated off + # by default (use_dense_main_attn=False). If you opt in, vendor that .cuh + # too. Original line: `from sglang.jit_kernel.minimax_decode_topk import ( + # minimax_decode_topk_page_table)`. + raise NotImplementedError( + "use_dense_main_attn=True requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; " + "the standalone Triton vendor only supports use_dense_main_attn=False." + ) + # num_q_heads (= num index/kv heads) may be > 1 for DP attention; the + # kernel emits a flattened [batch*num_q_heads] page table + seq_lens with + # the kv head head-encoded (head-minor) into the page index. + topk_idx, real_seq_lens = minimax_decode_topk_page_table( + score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # fall through to the idx_o (index value) merge, then return + _skip_block_topk = True + else: + _skip_block_topk = False + + # get topk index + if not _skip_block_topk: + topk_idx = torch.empty( + (num_q_heads, batch_size, topk), + device=score.device, + dtype=torch.int32, + ) + + if _skip_block_topk: + pass + elif use_jit_topk: + # Single-stage JIT radix-select: one kernel, no intermediate buffers. + # Equivalent output to the 2-stage path (set of block ids, front-packed, + # -1 padded); ~2-16x faster for long context. + # VENDOR NOTE: this needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the + # `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so + # this branch is never taken in the standalone vendor (the 2-stage Triton + # `else` fallback below runs instead). Original line: + # `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`. + raise NotImplementedError( + "use_jit_topk requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor " + "uses the 2-stage Triton fallback (keep " + "SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)." + ) + elif has_native_ops() and topk <= 64: + topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk) + else: + # 2-stage split-K Triton fallback. + # Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim. + # Same constraints as flash_decode: must be deterministic from values + # constant within a cuda graph (BS, num_q_heads), pow2, capped so the + # merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk)) + # stays reasonable. With topk=32 and cap 16, merge sorts 512 items. + TOPK_TARGET_GRID = 64 + MAX_NUM_TOPK_CHUNKS = 16 + topk_target = max( + 1, + min( + MAX_NUM_TOPK_CHUNKS, + TOPK_TARGET_GRID // max(1, batch_size * num_q_heads), + ), + ) + NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1) + BLOCK_SIZE_T = triton.next_power_of_2(topk) + max_seqblock = score.shape[2] + chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + topk_score_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.float32, + device=score.device, + ) + topk_idx_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.int32, + device=score.device, + ) + # stream 0 (default): topk partial → topk merge + grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS) + _topk_index_partial_kernel[grid]( + score, + topk_score_partial, + topk_idx_partial, + seq_lens, + block_size, + topk, + chunk_blocks, + score.stride(0), + score.stride(1), + score.stride(2), + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + ) + grid = (batch_size, num_q_heads) + _topk_index_merge_kernel[grid]( + topk_score_partial, + topk_idx_partial, + topk_idx, + seq_lens, + block_size, + topk, + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # attn output merge (default stream) + grid = (batch_size, num_q_heads) + _merge_attn_out_kernel[grid]( + o, + lse, + seq_lens, + head_dim, + block_size, + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + ) + o = o[0].contiguous() + return o, topk_idx, real_seq_lens diff --git a/build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py b/build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..8fe24cce53337c4f44e6c462fc3c90a258a34fc7 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/decode/topk_sparse.py @@ -0,0 +1,452 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from .._native import ( + native_decode_mma_supported, + native_decode_sparse_attn_mma_paged, +) +from ..common.utils import robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4, 5] + ], + key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"], +) +@triton.jit +def _gqa_share_sparse_decode_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + idx_ptr, # topk index: qh x b x topk + o_ptr, # O partial: c x b x qh x d + lse_ptr, # lse partial: c x b x qh + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + # sm_scale + sm_scale, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + NUM_TOPK_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + IS_FP8: tl.constexpr, +): + # decode program ids: split-K over the topk dimension to give every SM + # something to do at small batch. pid(0) folds (batch, chunk) together so + # the grid size = batch_size * NUM_TOPK_CHUNKS. + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # per-chunk topk range. chunk_size is *runtime* (depends on max_topk which + # is a runtime arg, not constexpr), so don't annotate as tl.constexpr — + # doing so produces undefined behavior in Triton. + chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + chunk_start_topk = pid_c * chunk_size_topk + chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk + # get q k start and len after rmpad + seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # get real topk + off_t = tl.arange(0, BLOCK_SIZE_T) + idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b + topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk) + # init pointer + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + dim_mask = off_d < head_dim + # init statistics — kept at -inf so empty chunks (chunk_start >= real_topk) + # naturally fall out as weight=0 in the merge step. + # q is always needed; load it unconditionally. + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + # Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code- + # eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is + # then None). The original compound `if HAS_SINK and pid_c == 0` traced + # the body even when False -> NoneType make_block_ptr compile error on + # sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always + # False for us, but this keeps the sink path correct too. + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load( + sink_ptrs, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale + m_i = qsink + lse_i = qsink + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # only iterate over this chunk's topk slice. the load must respect the + # per-chunk start offset. + cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t + for _ in tl.range(chunk_start_topk, chunk_end_topk): + # load index + c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N + cur_idx_ptr = cur_idx_ptr + stride_ti_t + # resolve slots for this block via req_to_token + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no + # scale), so dequant is just a widening cast to the Q compute dtype + # before the tl.dot. Matches the bf16 path bit-for-bit when the cache + # is bf16 (IS_FP8 False -> this branch is compiled out). + k = k.to(q.dtype) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V before the P@V dot. This also makes the `p.to(v.dtype)` + # below cast P to the compute dtype (not to fp8, which would be + # catastrophic precision loss on the attention weights). + v = v.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # load v and update acc_o + p = p.to(v.dtype) + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij) + # final scale (matches the old non-split kernel for chunks where lse_i>-inf). + # For empty chunks (chunk_start_topk >= real_topk) the inner loop never + # runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute + # exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial + # and the merge result. Gate the scale with tl.where so empty chunks emit a + # clean zero (lse_i stays -inf which the merge correctly turns into weight=0). + scale = tl.where( + lse_i > float("-inf"), + tl.exp(m_i - lse_i), + tl.zeros_like(lse_i), + ) + acc_o = acc_o * scale[:, None] + # save partial output and lse for the merge step + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.jit +def _merge_topk_attn_out_kernel( + o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0 + lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH] + head_dim, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + off_c = tl.arange(0, NUM_TOPK_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(NUM_TOPK_CHUNKS, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0 + # standard flash-decoding merge in linear (not log2) space, matching the + # decode kernel which uses tl.exp / tl.log. + lse_max = tl.max(lse, axis=0) + weights = tl.exp(lse - lse_max) + weights = weights / tl.sum(weights, axis=0) + o_merged = tl.sum(o * weights[:, None], axis=0) + o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@torch.no_grad() +def flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, + use_tma: bool = True, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may + # be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel + # widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing + # sparse decode dtype contract unchanged by accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + assert topk_idx.shape[0] == num_kv_heads + assert ( + triton.next_power_of_2(block_size) == block_size + ), f"block_size must be a power of 2, but got {block_size}" + # assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}" + max_kv_len = req_to_token.shape[1] + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + max_topk = topk_idx.shape[2] + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # Fast path: native tensor-core decode for the M3 MSA shape. Falls through + # to the Triton kernel for other shapes, sink, or fp8 caches. + if sink is None and not is_fp8 and native_decode_mma_supported( + q, k_cache, block_size=block_size + ): + return native_decode_sparse_attn_mma_paged( + q, + k_cache, + v_cache, + req_to_token, + seq_lens, + slot_ids, + block_size, + topk_idx, + sm_scale, + ) + # Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as + # flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must + # only depend on shape constants (so grid is fixed within a cuda graph). + # Capped by max_topk because chunks beyond real_topk early-fall-through to + # the merge-as-zero path; capping avoids wasting blocks at tiny topk. + TARGET_GRID = 256 + target = max( + 1, + min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1) + # output tensor: split-K partials, merged into chunk 0 by the merge kernel + o_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + dtype=torch.float32, + device=q.device, + ) + # launch attention kernel + grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads) + _gqa_share_sparse_decode_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + topk_idx, + o_partial, + lse_partial, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + BLOCK_SIZE_N=block_size, + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + IS_FP8=is_fp8, + ) + # merge partials into chunk 0 + merge_grid = (batch_size, num_q_heads) + _merge_topk_attn_out_kernel[merge_grid]( + o_partial, + lse_partial, + head_dim, + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + return o_partial[0].contiguous() diff --git a/build/torch211-cxx11-cu128-x86_64-linux/interface.py b/build/torch211-cxx11-cu128-x86_64-linux/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..7939722609218f2bb41eadcdb2f965213d459d1c --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/interface.py @@ -0,0 +1,679 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from ._native import ( + has_native_ops, + native_indexer_block_scores, + native_nvfp4_dequant_swizzled_to_bf16, +) +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse +from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16 + + +def _page_table_to_req_to_token( + page_table: torch.Tensor, + seqused_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, +) -> torch.Tensor: + batch = int(page_table.shape[0]) + max_seqlen_k = int(max_seqlen_k) + blk_kv = int(blk_kv) + logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64) + page = logical_pos // blk_kv + offset = logical_pos % blk_kv + page = page.clamp(max=int(page_table.shape[1]) - 1) + physical_page = page_table[:, page].to(torch.int64) + physical_token = physical_page * blk_kv + offset[None, :] + valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None] + return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous() + + +def _dense_q2k_indices( + *, + head_kv: int, + batch: int, + topk: int, + seq_lens: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device) + blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device) + valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv) + for b in range(batch): + valid = blocks < valid_blocks[b] + idx[:, b, valid] = blocks[valid] + return idx + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]: + seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist() + return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens] + + +def _packed_row_to_batch_block( + cu_seqlens_k: torch.Tensor, + blk_kv: int, +) -> list[tuple[int, int]]: + rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv)) + max_rows = max(rows_per_batch, default=0) + row_map: list[tuple[int, int]] = [] + for block_idx in range(max_rows): + for batch_idx, rows in enumerate(rows_per_batch): + if block_idx < rows: + row_map.append((batch_idx, block_idx)) + return row_map + + +def _csr_to_q2k_indices( + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + blk_kv: int, + topk: int, +) -> torch.Tensor: + if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32: + raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32") + if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2: + raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2") + head_kv = int(k2q_row_ptr.shape[0]) + total_q = int(cu_seqlens_q[-1].item()) + topk = int(topk) + if topk <= 0: + raise ValueError(f"topK must be positive, got {topk}") + + row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv)) + if int(k2q_row_ptr.shape[1]) != len(row_map) + 1: + raise ValueError( + "k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: " + f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}" + ) + + cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False) + q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False) + q2k = torch.full( + (head_kv, total_q, topk), + -1, + dtype=torch.int32, + device=k2q_row_ptr.device, + ) + + next_slot: dict[tuple[int, int], int] = {} + seen: set[tuple[int, int, int]] = set() + for h in range(head_kv): + for row, (batch_idx, kv_block_idx) in enumerate(row_map): + start = int(row_ptr_cpu[h, row].item()) + end = int(row_ptr_cpu[h, row + 1].item()) + q_start = int(cu_q_cpu[batch_idx]) + q_end = int(cu_q_cpu[batch_idx + 1]) + q_len = q_end - q_start + for pos in range(start, end): + q_val = int(q_idx_cpu[h, pos].item()) + if q_val < 0: + continue + if q_start <= q_val < q_end: + q_global = q_val + elif q_val < q_len: + q_global = q_start + q_val + else: + continue + key = (h, q_global, kv_block_idx) + if key in seen: + continue + seen.add(key) + slot_key = (h, q_global) + slot = next_slot.get(slot_key, 0) + if slot >= topk: + continue + q2k[h, q_global, slot] = int(kv_block_idx) + next_slot[slot_key] = slot + 1 + return q2k.contiguous() + + +def _make_prefill_cache_and_metadata( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, + *, + page_table: Optional[torch.Tensor], + seqused_k: Optional[torch.Tensor], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if page_table is None: + if k.ndim != 3 or v.ndim != 3: + raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + k_cache = k.contiguous() + v_cache = v.contiguous() + batch = int(cu_seqlens_q.numel()) - 1 + max_seqlen_k = int(max_seqlen_k) + logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64) + cu_k = cu_seqlens_k.to(torch.int64) + seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous() + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32) + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + else: + if k.ndim != 4 or v.ndim != 4: + raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("paged k/v page dimension must equal blk_kv") + if page_table.dtype != torch.int32: + raise TypeError("page_table must be torch.int32") + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + batch = int(page_table.shape[0]) + if seqused_k is None: + seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous() + else: + seq_lens = seqused_k.to(torch.int32).contiguous() + logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64) + page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1) + offset = logical % int(blk_kv) + physical_page = page_table[:, page].to(torch.int64) + req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32) + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32) + prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous() + return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens + + +def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int: + if kv_heads <= 0 or q_heads % kv_heads != 0: + raise ValueError("number of query heads must be divisible by KV heads") + gqa_group = q_heads // kv_heads + limit = max(1, 128 // max(1, gqa_group)) + for candidate in (64, 32, 16, 8, 4, 2, 1): + if candidate <= limit: + return candidate + return 1 + + +def _normalize_decode_q2k( + q2k_indices: Optional[torch.Tensor], + *, + batch: int, + seqlen_q: int, + head_kv: int, + seqused_k: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + if q2k_indices is None: + topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv)) + return _dense_q2k_indices( + head_kv=head_kv, + batch=batch, + topk=topk, + seq_lens=seqused_k, + blk_kv=blk_kv, + ) + if q2k_indices.dtype != torch.int32: + raise TypeError("q2k_indices must be torch.int32") + if q2k_indices.ndim != 3: + raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]") + if q2k_indices.shape[0] != head_kv: + raise ValueError("q2k_indices first dimension must match Hkv") + if int(q2k_indices.shape[1]) != batch * seqlen_q: + raise ValueError("q2k_indices second dimension must equal batch * seqlen_q") + if seqlen_q != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + return q2k_indices[:, ::seqlen_q, :].contiguous() + + +def sparse_decode_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q2k_indices: Optional[torch.Tensor] = None, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + schedule: Optional[object] = None, + O_partial: Optional[torch.Tensor] = None, + LSE_partial: Optional[torch.Tensor] = None, +): + """Run MiniMax paged sparse decode on Blackwell. + + This is compatible with the upstream MiniMaxAI/msa decode-facing API name, + but accepts the Blackwell package's validated BF16/FP16 decode path. It + supports single-token decode (`seqlen_q=1`) with paged K/V + `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`. + """ + + if schedule is not None or O_partial is not None or LSE_partial is not None: + raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper") + if return_softmax_lse: + raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper") + if not causal: + raise NotImplementedError("Blackwell decode wrapper currently supports causal=True") + if int(seqlen_q) != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + if q.ndim != 3: + raise ValueError("q must have shape [batch * seqlen_q, Hq, D]") + if k.ndim != 4 or v.ndim != 4: + raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if q.device != k.device or q.device != v.device or q.device != page_table.device: + raise ValueError("q, k, v, and page_table must be on the same device") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("Blackwell decode wrapper supports BF16/FP16 q") + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper") + if q.shape[-1] != 128 or k.shape[-1] != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("k.shape[2] must equal blk_kv") + if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32: + raise TypeError("page_table and seqused_k must be torch.int32") + if page_table.ndim != 2: + raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]") + batch = int(page_table.shape[0]) + if int(q.shape[0]) != batch: + raise ValueError("for seqlen_q=1, q.shape[0] must equal batch") + + head_kv = int(k.shape[1]) + q2k = _normalize_decode_q2k( + q2k_indices, + batch=batch, + seqlen_q=int(seqlen_q), + head_kv=head_kv, + seqused_k=seqused_k, + blk_kv=int(blk_kv), + ) + req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv)) + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + return flash_decode_with_gqa_share_sparse( + q.contiguous(), + None, + k_cache, + v_cache, + req_to_token, + seqused_k.contiguous(), + slot_ids, + int(blk_kv), + q2k, + sm_scale=softmax_scale, + ) + + +class SparseDecodePagedAttentionWrapper: + def __init__(self, *, blk_kv: int = 128, causal: bool = True): + self.blk_kv = int(blk_kv) + self.causal = bool(causal) + self.page_table: Optional[torch.Tensor] = None + self.seqused_k: Optional[torch.Tensor] = None + self.seqlen_q: Optional[int] = None + self.max_seqlen_k: Optional[int] = None + self.q2k_indices: Optional[torch.Tensor] = None + + def plan( + self, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + q2k_indices: Optional[torch.Tensor] = None, + num_qo_heads: Optional[int] = None, + num_kv_heads: Optional[int] = None, + head_dim: Optional[int] = 128, + enable_cuda_graph: bool = False, + max_grid_size: Optional[int] = None, + fixed_split_size: Optional[int] = None, + disable_split_kv: bool = False, + ) -> "SparseDecodePagedAttentionWrapper": + if head_dim is not None and int(head_dim) != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + self.page_table = page_table.contiguous() + self.seqused_k = seqused_k.contiguous() + self.seqlen_q = int(seqlen_q) + self.max_seqlen_k = int(max_seqlen_k) + self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous() + return self + + def run( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + out: Optional[torch.Tensor] = None, + lse: Optional[torch.Tensor] = None, + ): + if self.page_table is None or self.seqused_k is None: + raise RuntimeError("decode wrapper must be planned before run") + result = sparse_decode_atten_func( + q, + k, + v, + self.q2k_indices, + page_table=self.page_table, + seqused_k=self.seqused_k, + seqlen_q=self.seqlen_q, + max_seqlen_k=self.max_seqlen_k, + blk_kv=self.blk_kv, + causal=self.causal, + softmax_scale=softmax_scale, + return_softmax_lse=return_softmax_lse, + ) + if out is not None: + out.copy_(result) + return out + return result + + +def sparse_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + *, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + lse_temperature_scale: float = 1.0, + return_temperature_lse: bool = False, + partial_dtype: torch.dtype = torch.bfloat16, + return_softmax_lse: bool = False, + page_table: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + schedule: Optional[object] = None, + usable_SM_count: int = -1, + qk_dtype: Optional[torch.dtype] = None, + pv_dtype: Optional[torch.dtype] = None, +): + """Run the official MiniMax CSR sparse prefill API on Blackwell. + + The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list + consumed by the Blackwell Triton sparse prefill kernel. The default output + path is supported for BF16/FP16 tensors with head_dim=128. + """ + + del partial_dtype, usable_SM_count, qk_dtype, pv_dtype + if schedule is not None: + raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper") + if return_temperature_lse or return_softmax_lse: + raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper") + if float(lse_temperature_scale) != 1.0: + raise ValueError("lse_temperature_scale values other than 1.0 are not supported") + if not causal: + raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True") + if q.ndim != 3: + raise ValueError("q must have shape [total_q, Hq, D]") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("q must be BF16 or FP16") + if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128: + raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128") + if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32: + raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32") + if q.device != k.device or q.device != v.device: + raise ValueError("q, k, and v must be on the same device") + + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have matching BF16/FP16 dtype") + kv_heads = int(k.shape[1]) + q2k_indices = _csr_to_q2k_indices( + k2q_row_ptr.contiguous(), + k2q_q_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(blk_kv), + int(topK), + ) + k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata( + q, + k, + v, + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(max_seqlen_k), + int(blk_kv), + page_table=None if page_table is None else page_table.contiguous(), + seqused_k=None if seqused_k is None else seqused_k.contiguous(), + ) + block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads) + return flash_prefill_with_gqa_share_sparse( + q.contiguous(), + k_cache, + v_cache, + None, + req_to_token, + slot_ids, + q2k_indices, + block_size_q=block_size_q, + block_size_k=int(blk_kv), + cu_seqlens=cu_seqlens_q.contiguous(), + seq_lens=seq_lens, + prefix_lens=prefix_lens, + max_seqlen_q=int(max_seqlen_q), + sm_scale=softmax_scale, + ) + + +def _nvfp4_dequant_kv( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor, +) -> torch.Tensor: + if packed.dtype is not torch.uint8: + return packed + if has_native_ops() and packed.is_cuda: + return native_nvfp4_dequant_swizzled_to_bf16( + packed, + scale_128x4, + global_scale, + ) + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + rows = 1 + for dim in original_shape[:-1]: + rows *= int(dim) + qx = Nvfp4QuantizedTensor( + data=packed, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, int(original_shape[-1]) // 16), + original_shape=original_shape, + ) + return dequantize_nvfp4_128x4_to_bf16(qx) + + +def sparse_atten_nvfp4_kv_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k_scale_128x4: torch.Tensor, + v_scale_128x4: torch.Tensor, + k_global_scale: torch.Tensor, + v_global_scale: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + **kwargs, +): + """NVFP4-KV official prefill API wrapper. + + The Blackwell compatibility path dequantizes NVFP4 K/V with the package's + 128x4 metadata helper, then dispatches to `sparse_atten_func`. + """ + + k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale) + v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale) + return sparse_atten_func( + q, + k_bf16, + v_bf16, + k2q_row_ptr, + k2q_q_indices, + topK, + **kwargs, + ) + + +def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor: + data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8) + lut = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0], + dtype=torch.float32, + device=data.device, + ) + out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device) + out[..., 0::2] = lut[(data & 0x0F).long()] + out[..., 1::2] = lut[(data >> 4).long()] + return out + + +def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]): + return values * scale.to(torch.float32).repeat_interleave(16, dim=-1) + if scale.ndim == values.ndim - 1: + return values * scale.to(torch.float32).unsqueeze(-1) + return values + + +def fp4_indexer_block_scores( + q_fp4: torch.Tensor, + k_fp4: torch.Tensor, + q_scale: torch.Tensor, + k_scale: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + cu_page_offsets: torch.Tensor, + *, + max_seqlen_q: int, + max_seqlen_k: int, + kv_indices: torch.Tensor, + fp4_format: str, + causal: bool = False, + qo_offset: Optional[torch.Tensor] = None, + scale_layout: str = "preordered_mma", +) -> torch.Tensor: + """FP4 block-score API for Blackwell. + + Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`. + Built artifacts route CUDA tensors through the native Blackwell block-score + kernel. Source-tree mode uses the reference loop below so the API remains + testable before the extension is built. + """ + + del qo_offset + if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}: + raise ValueError(f"unsupported fp4_format: {fp4_format}") + if scale_layout not in {"public", "preordered_mma"}: + raise ValueError(f"unsupported scale_layout: {scale_layout}") + if q_fp4.ndim != 3: + raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]") + if k_fp4.ndim != 4: + raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]") + total_q, hq, _ = q_fp4.shape + num_pages, hkv, page_size, _ = k_fp4.shape + if int(page_size) != 128: + raise ValueError("fp4_indexer_block_scores currently expects 128-token pages") + if hq % hkv != 0: + raise ValueError("Hq must be divisible by Hkv") + max_blocks = (int(max_seqlen_k) + 127) // 128 + scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device) + + q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale) + k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale) + + # Native block-max QK kernel for the cubic scoring loop when available. + # Same dequant numerics as the reference (bf16 compute); falls back to the + # Python reference below in source-tree mode. + if has_native_ops() and q_fp4.is_cuda: + batch = int(cu_seqlens_q.numel()) - 1 + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64) + batch_of_q = torch.repeat_interleave( + torch.arange(batch, device=q_fp4.device, dtype=torch.int32), + q_lens, + ) + return native_indexer_block_scores( + q.to(torch.bfloat16), + k.to(torch.bfloat16), + batch_of_q, + cu_seqlens_q.to(torch.int32), + cu_seqlens_k.to(torch.int32), + cu_page_offsets.to(torch.int32), + kv_indices.to(torch.int32), + max_blocks=max_blocks, + causal=causal, + ) + + gqa = hq // hkv + batch = int(cu_seqlens_q.numel()) - 1 + cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist() + cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist() + kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist() + for b in range(batch): + q_start, q_end = int(cu_q[b]), int(cu_q[b + 1]) + k_len = int(cu_k[b + 1]) - int(cu_k[b]) + page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1]) + for logical_block, page_pos in enumerate(range(page_start, page_end)): + if logical_block >= max_blocks: + break + physical_page = int(kv_idx_cpu[page_pos]) + if physical_page < 0 or physical_page >= num_pages: + continue + valid_tokens = min(128, max(0, k_len - logical_block * 128)) + if valid_tokens <= 0: + continue + for qi in range(q_start, q_end): + local_q = qi - q_start + if causal and logical_block * 128 > local_q: + continue + for qh in range(hq): + kh = qh // gqa + key_block = k[physical_page, kh, :valid_tokens] + if causal: + visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1)) + if visible <= 0: + continue + key_block = key_block[:visible] + block_scores = torch.matmul(key_block, q[qi, qh]) + scores[qh, logical_block, qi] = block_scores.max() + return scores + + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", +] diff --git a/build/torch211-cxx11-cu128-x86_64-linux/metadata.json b/build/torch211-cxx11-cu128-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a386aa646839bbd39fce10965d3420d770877366 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/metadata.json @@ -0,0 +1,38 @@ +{ + "name": "minimaxai-msa-blackwell", + "id": "_minimaxai_msa_blackwell_cuda_25b0067", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": { + "type": "cuda", + "archs": [ + "12.0" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=", + "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=", + "_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "UoiDhzhz2aSCy+k+sOoOxb5/DqwAndixA6BgmEHAMtU=", + "_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=", + "_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=", + "api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=", + "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=", + "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=", + "decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=", + "interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=", + "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=", + "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=", + "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=", + "prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=", + "prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=", + "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=", + "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w=" + } + } +} \ No newline at end of file diff --git a/build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py b/build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b2672c1cd85b74c1b3ded0fc0b2100e1aeac23 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/minimaxai_msa_blackwell/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py b/build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/naive/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py b/build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..db7caceb6faa5d8d1669d3c57f1e4ca9578a4804 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/naive/flash_with_topk_idx.py @@ -0,0 +1,106 @@ +# Copyright 2025 XunhaoLai. All rights reserved. +# +# VENDOR NOTE: rewritten to drop the `einops` dependency (the original used +# einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream +# `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced +# with native torch.einsum / torch.reshape so the standalone vendor needs only +# torch (+ triton for the kernels). Original einops expressions are kept in +# comments next to each replacement for auditability. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], # [num_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + sm_scale: Optional[float] = None, + init_blocks: int = 0, + local_blocks: int = 0, +): + """Joint reference: lightning-indexer block top-k + sparse attention. + + This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense + 5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by + `slot_ids`. It both (a) selects the top-k key blocks via per-block max score + (with optional forced init/local blocks) and (b) computes the resulting + sparse attention, returning (o, topk_idx). + """ + assert ( + kv_cache.shape[2] % block_size == 0 + ), "max cache len must be divisible by block size" + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + original_dtype = q.dtype + batch_size = q.shape[0] + num_q_heads = q.shape[1] + num_kv_heads = kv_cache.shape[3] + gqa_group_size = num_q_heads // num_kv_heads + head_dim = q.shape[-1] + # rearrange q: "b (h g) d -> b h g d" + q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim) + kv_cache_float = kv_cache.float() + k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d] + v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d] + # einsum(q, k, "b h g d, b n h d -> b h g n") + qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale + mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None] + qk = qk.masked_fill(~mask[:, None, None, :], float("-inf")) + # get score + score = qk.clone().to(torch.float32) + # rearrange "b h g (n s) -> b (h g) n s", s=block_size + n_blocks = score.shape[-1] // block_size + score = score.reshape( + batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size + ).reshape(batch_size, num_q_heads, n_blocks, block_size) + score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks] + # post-process score for init_blocks and local_blocks + INIT_SCORE = 1e30 + LOCAL_SCORE = 1e29 + if init_blocks > 0: + score[:, :, :init_blocks] = INIT_SCORE + if local_blocks > 0: + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + local_start = max(0, num_blks - local_blocks) + score[b, :, local_start:num_blks] = LOCAL_SCORE + # compute topk indices per (batch, head) + # score shape: [batch_size, num_q_heads, num_blocks] + topk_idx = torch.full( + (num_q_heads, batch_size, topk), + fill_value=-1, + device=score.device, + dtype=torch.int32, + ) + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + actual_topk = min(topk, num_blks) + for h in range(num_q_heads): + # get topk indices for this (batch, head) + _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1) + topk_idx[h, b, :actual_topk] = indices.to(torch.int32) + # compute attention output with sink + if sink is not None: + # sink: [num_q_heads, head_dim] -> [h, g, d] + sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim) + # einsum(q, sink, "b h g d, h g d -> b h g") + qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale + qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1] + attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32) + attn = attn[..., 1:] # remove sink score + else: + attn = qk.softmax(dim=-1, dtype=torch.float32) + # einsum(attn, v, "b h g n, b n h d -> b h g d") + o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel) + # rearrange "b h g d -> b (h g) d" + o = o.reshape(batch_size, num_q_heads, head_dim) + return o.to(original_dtype), topk_idx diff --git a/build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py b/build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..6fefac38e0d131d2b37dcb392e405d71a93ace61 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/naive/topk_sparse.py @@ -0,0 +1,88 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], # [num_q_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, +) -> torch.Tensor: + """ + Naive implementation of sparse attention with GQA group sharing. + Each GQA group (all q heads sharing the same kv head) uses the same topk_idx. + """ + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + + original_dtype = q.dtype + batch_size, num_q_heads, head_dim = q.shape + max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape + gqa_group_size = num_q_heads // num_kv_heads + # Output tensor + o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device) + + for b in range(batch_size): + seq_len = seq_lens[b].item() + sid = slot_ids[b].item() % max_slots + + for kh in range(num_kv_heads): + # Get topk indices for this (kv_head, batch) + block_indices = topk_idx[kh, b, :].tolist() + + # Collect selected K and V blocks + selected_k_blocks = [] + selected_v_blocks = [] + for block_idx in block_indices: + if block_idx < 0: + continue # Invalid index + start = block_idx * block_size + end = min(start + block_size, seq_len) + if start >= seq_len: + continue + # K: [block_len, head_dim] + k_block = kv_cache[sid, 0, start:end, kh, :] + # V: [block_len, head_dim] + v_block = kv_cache[sid, 1, start:end, kh, :] + selected_k_blocks.append(k_block) + selected_v_blocks.append(v_block) + + if len(selected_k_blocks) == 0: + continue + + # Concatenate selected blocks: [total_selected_len, head_dim] + k_selected = torch.cat(selected_k_blocks, dim=0) + v_selected = torch.cat(selected_v_blocks, dim=0) + + # Compute attention for all q heads in this GQA group + for g in range(gqa_group_size): + qh = kh * gqa_group_size + g + # q_vec: [head_dim] + q_vec = q[b, qh, :].float() + + # Compute attention scores: [total_selected_len] + scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale + + # Add sink to softmax normalization if present + if sink is not None: + sink_vec = sink[qh, :].float() + qsink = torch.dot(q_vec, sink_vec) * sm_scale + # Concatenate sink score with regular scores for softmax + scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0) + attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1) + # Remove sink weight (only used for normalization) + attn_weights = attn_weights_with_sink[1:] + else: + attn_weights = torch.softmax(scores, dim=-1) + + # Compute output: [head_dim] + o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected) + + return o diff --git a/build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py b/build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b6b69345bf3ffe07f6ccac8c570ee17e3b5e6c69 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/prefill/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +from .topk_sparse import flash_prefill_with_gqa_share_sparse + +__all__ = ["flash_prefill_with_gqa_share_sparse"] diff --git a/build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py b/build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..9bf83d65eac260f8e71180ea6f28970e8db582d7 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/prefill/topk_sparse.py @@ -0,0 +1,371 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from ..common.utils import get_cu_seqblocks, robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]), + "BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]), + "BLOCK_SIZE_H": lambda args: triton.next_power_of_2( + max( + 16 // args["BLOCK_SIZE_Q"], + triton.next_power_of_2(args["gqa_group_size"]), + ) + ), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"], + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + # Configs that fail to compile on the target arch are skipped, so widening + # the num_warps x num_stages grid only adds candidates, never a bad kernel. + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in (2, 4, 8) + for ns in (2, 3, 4) + ], + key=[ + "BLOCK_SIZE_Q", + "BLOCK_SIZE_K", + "qk_head_dim", + "v_head_dim", + "gqa_group_size", + ], +) +@triton.jit +def _gqa_share_sparse_fwd_kernel( + q_ptr, # Q: n x h x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + sink_ptr, # Sink: h x d + t_ptr, # topk_idx: kh x n x k + o_ptr, # O: n x h x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + # seqlens + cu_seqlens_q, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + # shape + max_slots, + num_kv_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + max_topk, + # q loop num + num_q_loop, + # sm_scale + sm_scale, + # stride + stride_qn, + stride_qh, + stride_qd, + stride_ks, + stride_kh, + stride_kd, + stride_vs, + stride_vh, + stride_vd, + stride_sh, + stride_sd, + stride_th, + stride_tn, + stride_tk, + stride_on, + stride_oh, + stride_od, + stride_r2t_b, + # META parameters + BLOCK_SIZE_Q: tl.constexpr, # q block size + BLOCK_SIZE_K: tl.constexpr, # k block size + BLOCK_SIZE_KD: tl.constexpr, + BLOCK_SIZE_VD: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + BLOCK_SIZE_QH: tl.constexpr, + # has sink + HAS_SINK: tl.constexpr, + USE_TMA: tl.constexpr, + IS_FP8: tl.constexpr, +): + sm_scale_log2e = sm_scale * 1.4426950409 + # get batch id and head id + pid_q = tl.program_id(0) + pid_kh = tl.program_id(1) + pid_b = tl.program_id(2) + pid_h = pid_kh * gqa_group_size + # get q k start and len after rmpad + q_start = tl.load(cu_seqlens_q + pid_b) + q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start + q_block_start = tl.load(cu_seqblocks_q + pid_b) + q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start + seq_len = tl.load(seq_lens + pid_b) + prefix_len = tl.load(prefix_lens + pid_b) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # safety against negative + if pid_q * num_q_loop >= q_block_len: + return + real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop) + if HAS_SINK: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sh, + shape=(gqa_group_size, qk_head_dim), + strides=(stride_sh, stride_sd), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + # offsets for paged K/V load + off_n = tl.arange(0, BLOCK_SIZE_K) + off_kd = tl.arange(0, BLOCK_SIZE_KD) + off_vd = tl.arange(0, BLOCK_SIZE_VD) + kd_mask = off_kd < qk_head_dim + vd_mask = off_vd < v_head_dim + for j in range(real_q_loop): + pid_q_j = pid_q * num_q_loop + j + # init topk idx pointer + t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th + # we assume that the topk_idx is right padded with -1 + off_t = tl.arange(0, BLOCK_SIZE_T) + topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + q_start * stride_qn + pid_h * stride_qh, + shape=(q_len, gqa_group_size, qk_head_dim), + strides=(stride_qn, stride_qh, stride_qd), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(2, 1, 0), + ) + # load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D] + q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero") + # init statistics + off_q_k = ( + tl.arange(0, BLOCK_SIZE_Q)[:, None] + + pid_q_j * BLOCK_SIZE_Q + + prefix_len + - tl.arange(0, BLOCK_SIZE_K)[None, :] + ) + if HAS_SINK: + m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + qsink = ( + tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e + ) # (BLOCK_SIZE_Q, BLOCK_SIZE_H) + m_i += qsink + lse_i += qsink + m_i = tl.reshape(m_i, BLOCK_SIZE_QH) + lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH) + else: + m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32) + q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD) + # sparse attention + for i in range(real_topk): + # get current block start index (absolute K position) + c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K + t_ptr_j = t_ptr_j + stride_tk + # paged load K via req_to_token: pos -> slot -> k_cache + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot) + k = tl.load( + k_cache_ptr + + slots[None, :] * stride_ks + + pid_kh * stride_kh + + off_kd[:, None] * stride_kd, + mask=kd_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 main K cache is unit-scaled; widen to the Q compute dtype + # before the tl.dot (compiled out when the cache is bf16). + k = k.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32) + # causal mask + qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf")) + qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K) + # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K] + # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K] + qk += tl.dot(q, k) * sm_scale_log2e + # K boundary mask: positions beyond seq_len contribute -inf + qk += tl.where(pos_mask[None, :], 0, float("-inf")) + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # paged load V + v = tl.load( + v_cache_ptr + + slots[:, None] * stride_vs + + pid_kh * stride_vh + + off_vd[None, :] * stride_vd, + mask=pos_mask[:, None] & vd_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V so `p.to(v.dtype)` casts P to the compute dtype rather + # than to fp8 (which would wreck attention-weight precision). + v = v.to(q.dtype) + p = p.to(v.dtype) + acc_o += tl.dot(p, v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + # final scale + acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None] + # save output + acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD) + o_ptrs = tl.make_block_ptr( + base=o_ptr + q_start * stride_on + pid_h * stride_oh, + shape=(q_len, gqa_group_size, v_head_dim), + strides=(stride_on, stride_oh, stride_od), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD), + order=(2, 1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2)) + + +@torch.no_grad() +def flash_prefill_with_gqa_share_sparse( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + sink: Optional[torch.Tensor], + req_to_token: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size_q: int, + block_size_k: int, + cu_seqlens: torch.Tensor, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + max_seqlen_q: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + cu_seqblocks_q: Optional[torch.Tensor] = None, + max_seqblock_q: Optional[int] = None, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled) + # under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load. + # Keep CUDA's pre-existing sparse prefill dtype contract unchanged by + # accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + assert block_size_q in {1, 2, 4, 8, 16, 32, 64} + assert block_size_k in {16, 32, 64, 128} + # shape + total_q, num_q_heads, qk_head_dim = q.shape + max_slots, num_k_heads, _ = k_cache.shape + _, num_v_heads, v_head_dim = v_cache.shape + batch_size = cu_seqlens.shape[0] - 1 + topk = topk_idx.shape[-1] + assert topk_idx.shape[0] == num_k_heads + # gqa + assert num_k_heads == num_v_heads + assert num_q_heads % num_k_heads == 0 + gqa_group_size = num_q_heads // num_k_heads + assert gqa_group_size * block_size_q <= 128 + if sm_scale is None: + sm_scale = qk_head_dim**-0.5 + if cu_seqblocks_q is None or max_seqblock_q is None: + cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks( + cu_seqlens, max_seqlen_q, block_size_q, block_size_k + ) + # output tensor + o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype) + # launch kernel + num_q_loop = ( + max_seqblock_q // 131072 + 1 + ) # calculate multiple queries in one kernel if seqlence length is too long + BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q) + BLOCK_SIZE_K = triton.next_power_of_2(block_size_k) + grid = ( + triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop), + num_k_heads, + batch_size, + ) + _gqa_share_sparse_fwd_kernel[grid]( + q, + k_cache, + v_cache, + sink, + topk_idx, + o, + req_to_token, + cu_seqlens, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_k_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + topk, + num_q_loop, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + req_to_token.stride(0), + BLOCK_SIZE_Q=BLOCK_SIZE_Q, + BLOCK_SIZE_K=BLOCK_SIZE_K, + USE_TMA=use_tma, + IS_FP8=is_fp8, + ) + return o diff --git a/build/torch211-cxx11-cu128-x86_64-linux/quantize.py b/build/torch211-cxx11-cu128-x86_64-linux/quantize.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2675ac49d03e5599e9559187efecdf63bb6387 --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/quantize.py @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: Apache-2.0 +"""NVFP4 helper API compatible with MiniMaxAI/msa. + +The quantization entrypoints use Transformer Engine when it is installed. The +layout helpers and reference dequantizer are pure PyTorch and are included so +callers can prepare and validate NVFP4 metadata with the same public API names +as the upstream MiniMaxAI package. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Tuple + +import torch + + +NVFP4_BLOCK_SIZE = 16 +NVFP4_FP4_MAX = 6.0 +NVFP4_FP8_E4M3_MAX = 448.0 + + +@dataclass(frozen=True) +class Nvfp4QuantizedTensor: + data: torch.Tensor + scale_128x4: torch.Tensor + global_scale: torch.Tensor + logical_scale_shape: Tuple[int, int] + original_shape: Tuple[int, ...] + + +def _round_up(x: int, multiple: int) -> int: + return ((int(x) + multiple - 1) // multiple) * multiple + + +def nvfp4_scale_128x4_offset( + row: torch.Tensor, + col: torch.Tensor, + scale_cols: int, +) -> torch.Tensor: + tiles_n = _round_up(scale_cols, 4) // 4 + tile_m = row // 128 + tile_n = col // 4 + outer = row % 128 + inner = col % 4 + return ( + (tile_m * tiles_n + tile_n) * 512 + + (outer % 32) * 16 + + (outer // 32) * 4 + + inner + ) + + +def swizzle_nvfp4_scale_to_128x4( + scale: torch.Tensor, + *, + rows: int, + cols: int, +) -> torch.Tensor: + if scale.ndim != 2: + raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}") + + rows = int(rows) + cols = int(cols) + padded_rows = _round_up(rows, 128) + padded_cols = _round_up(cols, 4) + if scale.shape[0] < rows or scale.shape[1] < cols: + raise ValueError( + "scale is smaller than the requested logical shape: " + f"got {tuple(scale.shape)}, need at least {(rows, cols)}" + ) + + logical = scale[:rows, :cols].contiguous() + if logical.shape != (padded_rows, padded_cols): + logical = torch.nn.functional.pad( + logical.to(torch.float32), + (0, padded_cols - cols, 0, padded_rows - rows), + ).to(scale.dtype) + swizzled = torch.empty_like(logical) + + row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None] + col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1) + swizzled.reshape(-1)[offset] = logical.reshape(-1) + return swizzled + + +def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor: + return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX) + + +def _import_te_nvfp4_quantizer(): + try: + from transformer_engine.pytorch.tensor import NVFP4Quantizer + except Exception as exc: + raise RuntimeError( + "Transformer Engine NVFP4 quantization is unavailable. Install a " + "Transformer Engine build with NVFP4 PyTorch support to use " + "quantize_bf16_to_nvfp4_128x4." + ) from exc + return NVFP4Quantizer + + +def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor: + if not x.is_cuda: + raise ValueError("NVFP4 quantization requires a CUDA tensor") + if x.dtype not in (torch.bfloat16, torch.float16): + raise TypeError(f"x must be bf16 or fp16, got {x.dtype}") + if x.ndim < 2: + raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}") + if x.shape[-1] % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}" + ) + + rows = 1 + for dim in x.shape[:-1]: + rows *= int(dim) + if rows % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + "flattened row dimension must be divisible by " + f"{NVFP4_BLOCK_SIZE}, got {rows}" + ) + + NVFP4Quantizer = _import_te_nvfp4_quantizer() + quantizer = NVFP4Quantizer(rowwise=True, columnwise=False) + qx = quantizer.quantize(x.contiguous()) + meta = qx.get_metadata() + + data = meta["rowwise_data"] + if data.dtype is not torch.uint8: + data = data.view(torch.uint8) + logical_scale = meta["rowwise_scale_inv"] + amax = meta["amax_rowwise"] + scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE + scale_128x4 = swizzle_nvfp4_scale_to_128x4( + logical_scale, + rows=rows, + cols=scale_cols, + ) + global_scale = nvfp4_global_scale_from_amax(amax).contiguous() + + return Nvfp4QuantizedTensor( + data=data, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, scale_cols), + original_shape=tuple(int(v) for v in x.shape), + ) + + +def quantize_kv_bf16_to_nvfp4_128x4( + k: torch.Tensor, + v: torch.Tensor, +) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]: + return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v) + + +def dequantize_nvfp4_128x4_to_bf16( + qx: Nvfp4QuantizedTensor, + *, + include_global_scale: bool = True, +) -> torch.Tensor: + data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8) + if data.shape[-1] * 2 != qx.original_shape[-1]: + raise ValueError( + "packed data last dimension does not match original shape: " + f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical" + ) + + rows, scale_cols = qx.logical_scale_shape + logical_dim = int(qx.original_shape[-1]) + if scale_cols * NVFP4_BLOCK_SIZE != logical_dim: + raise ValueError( + "logical scale columns do not match original last dimension: " + f"{scale_cols} scale cols vs dim {logical_dim}" + ) + + fp4_lut = torch.tensor( + [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + -0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, + ], + dtype=torch.float32, + device=data.device, + ) + packed = data.reshape(rows, logical_dim // 2) + lo = packed & 0x0F + hi = packed >> 4 + values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device) + values[:, 0::2] = fp4_lut[lo.long()] + values[:, 1::2] = fp4_lut[hi.long()] + + row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None] + col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, scale_cols) + scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols) + scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32) + scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1) + out = values * scale + if include_global_scale: + out = out * qx.global_scale.reshape(-1)[0].to(torch.float32) + return out.reshape(qx.original_shape).to(torch.bfloat16) + + +__all__ = [ + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +] diff --git a/build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py b/build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..177f864da993328cc460b8de391c6091f3be0f9d --- /dev/null +++ b/build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Sparse-index helpers compatible with MiniMaxAI/msa public names.""" + +from __future__ import annotations + +from typing import Optional + +import torch + + +def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None: + if cu_seqlens.dtype != torch.int32: + raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}") + if cu_seqlens.ndim != 1: + raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}") + if cu_seqlens.numel() < 1: + raise ValueError(f"{name} must have at least one element") + if not cu_seqlens.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor: + seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1] + return (seqlens_k + kv_block_size - 1) // kv_block_size + + +def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]: + rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist() + batch = len(rows_per_batch_cpu) + max_rows = max(rows_per_batch_cpu, default=0) + row_dtype = ( + torch.int32 + if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max + else torch.int64 + ) + row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype) + row_linear = 0 + for kv_block_idx in range(max_rows): + for batch_idx, row_count in enumerate(rows_per_batch_cpu): + if kv_block_idx < row_count: + row_map_cpu[batch_idx, kv_block_idx] = row_linear + row_linear += 1 + return row_map_cpu.to(rows_per_batch.device), row_linear + + +def build_k2q_csr_torch_reference( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if kv_block_size <= 0: + raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}") + if q2k_indices.dtype != torch.int32: + raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}") + if q2k_indices.ndim != 3: + raise ValueError( + "q2k_indices must have shape [head_kv, total_q, topK], " + f"got {tuple(q2k_indices.shape)}" + ) + _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q") + _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k") + if cu_seqlens_q.shape != cu_seqlens_k.shape: + raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]") + if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device: + raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device") + + head_kv, total_q, topk = q2k_indices.shape + if total_q != int(cu_seqlens_q[-1].item()): + raise ValueError( + f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] " + f"({int(cu_seqlens_q[-1].item())})" + ) + + rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size) + row_map, total_rows = _build_packed_row_map(rows_per_batch) + nnz_upper_bound = total_q * topk + + k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device) + k2q_q_indices = torch.full( + (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device + ) + if total_rows == 0 or total_q == 0 or topk == 0: + return k2q_row_ptr, k2q_q_indices + + counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device) + total_entries = total_q * topk + row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64 + row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device) + q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device) + valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device) + + for batch_idx in range(cu_seqlens_q.numel() - 1): + q_start = int(cu_seqlens_q[batch_idx].item()) + q_end = int(cu_seqlens_q[batch_idx + 1].item()) + if q_end <= q_start: + continue + q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device) + q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1) + kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1) + valid = kv_local >= 0 + safe_kv = kv_local.clamp_min(0).long() + rows = row_map[batch_idx, safe_kv].to(row_dtype) + start = q_start * topk + end = q_end * topk + row_all[:, start:end] = rows + q_all[:, start:end] = q_local[None, :].expand(head_kv, -1) + valid_all[:, start:end] = valid & (rows >= 0) + counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32)) + + k2q_row_ptr[:, 1:] = counts.cumsum(dim=1) + offsets = torch.zeros_like(counts) + offsets[:, 1:] = counts[:, :-1].cumsum(dim=1) + cursor = torch.zeros_like(counts) + for h in range(head_kv): + for entry in range(total_entries): + if not bool(valid_all[h, entry]): + continue + row = int(row_all[h, entry].item()) + pos = int(offsets[h, row].item() + cursor[h, row].item()) + k2q_q_indices[h, pos] = q_all[h, entry] + cursor[h, row] += 1 + + return k2q_row_ptr, k2q_q_indices + + +def build_k2q_csr( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, + *, + total_k: Optional[int] = None, + max_seqlen_k: Optional[int] = None, + max_seqlen_q: Optional[int] = None, + total_rows: Optional[int] = None, + qhead_per_kv: int = 1, + return_schedule: bool = False, +): + if return_schedule: + raise NotImplementedError( + "Blackwell build_k2q_csr currently returns CSR tensors only; " + "SM100 schedule objects are not part of this package." + ) + if total_k is None: + raise ValueError("build_k2q_csr requires total_k from k.shape[0]") + return build_k2q_csr_torch_reference( + q2k_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(kv_block_size), + ) + + +class SparseK2qCsrBuilderSm100: + """Compatibility placeholder for the upstream SM100 class name. + + The name is exported so code can feature-detect it, but schedule-producing + SM100 behavior is intentionally not claimed for this Blackwell package. + """ + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def build(self, *args, **kwargs): + return build_k2q_csr(*args, **kwargs) + + +__all__ = [ + "build_k2q_csr", + "build_k2q_csr_torch_reference", + "SparseK2qCsrBuilderSm100", +] diff --git a/build/torch211-cxx11-cu130-x86_64-linux/__init__.py b/build/torch211-cxx11-cu130-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a666afa7b37c9d5448b9bb04d573dcfe4e45750 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/__init__.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention. + +This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on +DGX Spark / GB10 / SM121. It exposes the +MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime. + +Implementation status: + * native CUDA helper: score -> top-k sparse block ids; + * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape; + * native CUDA FP4 block-score indexer; + * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path; + * Blackwell-validated sparse prefill path; + * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers, + and FP4 block-score helpers. + +The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell +extension path. The public API is Tensor-oriented and independent from +FlashRT's serving runtime. + +Public API +---------- +Lightning indexer (q.k blockmax score -> top-k block select), paged KV: + flash_decode_with_topk_idx (single-token decode, split-K) + +Block-sparse GQA attention (consumes block_indices from the indexer): + flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks) + sparse_atten_func (official CSR prefill wrapper) + sparse_decode_atten_func (official paged decode wrapper) + +PyTorch naive references (for correctness checks): + naive_flash_decode_with_gqa_share_sparse (attention given topk_idx) + naive_flash_decode_with_topk_idx (joint indexer + attention) + +All Triton entrypoints expect a paged KV cache: + k_cache / v_cache : [max_slots, num_kv_heads, head_dim] + req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot) + slot_ids : [batch] (per-request row in req_to_token) + topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed, + -1 right-padding (the M3 indexer contract). +See VENDOR_NOTES.md for full shape/dtype/layout contracts. +""" + +from .common.utils import get_cu_seqblocks, robust_allocator +from .decode.flash_with_topk_idx import flash_decode_with_topk_idx +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .interface import ( + SparseDecodePagedAttentionWrapper, + fp4_indexer_block_scores, + sparse_atten_func, + sparse_atten_nvfp4_kv_func, + sparse_decode_atten_func, +) +from ._native import ( + has_native_ops, + native_nvfp4_dequant_swizzled_to_bf16, + native_topk_from_scores, +) +from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx +from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse +from .quantize import ( + Nvfp4QuantizedTensor, + dequantize_nvfp4_128x4_to_bf16, + nvfp4_global_scale_from_amax, + quantize_bf16_to_nvfp4_128x4, + quantize_kv_bf16_to_nvfp4_128x4, + swizzle_nvfp4_scale_to_128x4, +) +from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr +from .api_status import ( + OFFICIAL_API_STATUS, + OFFICIAL_MINIMAX_MSA_FUNCTIONS, + V1_AVAILABLE_FUNCTIONS, + available_functions, + official_api_status, + official_minimax_msa_functions, + unsupported_official_functions, +) + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "naive_flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "get_cu_seqblocks", + "robust_allocator", + "OFFICIAL_API_STATUS", + "OFFICIAL_MINIMAX_MSA_FUNCTIONS", + "V1_AVAILABLE_FUNCTIONS", + "available_functions", + "official_api_status", + "official_minimax_msa_functions", + "unsupported_official_functions", +] diff --git a/build/torch211-cxx11-cu130-x86_64-linux/_compat.py b/build/torch211-cxx11-cu130-x86_64-linux/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..da445342268947ab3b1c5bfd16e95dc4705c7abe --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/_compat.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton +# kernels. The upstream SGLang modules pull two tiny things from the framework: +# +# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path) +# from sglang.srt.environ import envs -> a feature flag for an optional +# JIT-CUDA radix top-k fast path +# +# Both are replaced here so the kernels import with only torch + triton. The +# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel +# radix select), which is what we want on sm_121 (GB10). + + +def is_hip() -> bool: + """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121). + + Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV + widening branch (HIP-only). With this False, the CUDA dtype contract + (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA. + """ + try: + import torch + + return bool(getattr(torch.version, "hip", None)) + except Exception: + return False + + +class _Flag: + """Minimal stand-in for an sglang.srt.environ boolean flag (always off).""" + + def __init__(self, default: bool = False): + self._default = default + + def get(self) -> bool: + return self._default + + +class _Envs: + """Stub for `sglang.srt.environ.envs`. + + Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode + indexer. Keeping it False routes top-k selection through the 2-stage Triton + fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the + decode indexer stays pure-Triton. + """ + + SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False) + + +envs = _Envs() diff --git a/build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so b/build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..b9207483faf48a32e6dd97ee287913e2c5ee67a0 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a053c1fd69c4df3f94e566a7c4d88ba943f8d5cece714ade52789f6819a961ac +size 729824 diff --git a/build/torch211-cxx11-cu130-x86_64-linux/_native.py b/build/torch211-cxx11-cu130-x86_64-linux/_native.py new file mode 100644 index 0000000000000000000000000000000000000000..dd54c53374469c8424d0ca6e3f25ea534005c8d5 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/_native.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional native CUDA helpers for the FlashRT Blackwell MSA package.""" + +from __future__ import annotations + +import torch + +try: + from ._ops import add_op_namespace_prefix, ops + + _HAS_NATIVE_OPS = True +except Exception: + add_op_namespace_prefix = None + ops = None + _HAS_NATIVE_OPS = False + + +def has_native_ops() -> bool: + return _HAS_NATIVE_OPS + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores")) + def _msa_topk_from_scores_fake( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, + topk_idx: torch.Tensor, + ) -> None: + if score.dim() != 3: + raise RuntimeError("score must have shape (heads, batch, max_blocks)") + heads, batch, _ = score.shape + if seq_lens.shape != (batch,): + raise RuntimeError("seq_lens must have shape (batch,)") + if topk_idx.shape != (heads, batch, topk): + raise RuntimeError("topk_idx must have shape (heads, batch, topk)") + return None + + +def native_topk_from_scores( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, +) -> torch.Tensor: + """Return top-k sparse block ids from a score tensor.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if not score.is_contiguous(): + score = score.contiguous() + if not seq_lens.is_contiguous(): + seq_lens = seq_lens.contiguous() + out = torch.empty( + (score.shape[0], score.shape[1], int(topk)), + device=score.device, + dtype=torch.int32, + ) + ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged") + ) + def _msa_decode_sparse_attn_mma_paged_fake( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size: int, + sm_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_decode_mma_supported( + q: torch.Tensor, + k_cache: torch.Tensor, + *, + block_size: int, +) -> bool: + """Whether the tensor-core decode kernel applies to these inputs. + + The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA + group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64. + """ + + if not _HAS_NATIVE_OPS: + return False + if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16: + return False + if q.dim() != 3 or k_cache.dim() != 3: + return False + head_dim = int(q.shape[-1]) + num_q_heads = int(q.shape[1]) + num_kv_heads = int(k_cache.shape[1]) + if head_dim != 128 or num_kv_heads <= 0: + return False + if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16: + return False + if int(block_size) % 64 != 0: + return False + return True + + +def native_decode_sparse_attn_mma_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + block_size: int, + topk_idx: torch.Tensor, + sm_scale: float, +) -> torch.Tensor: + """Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + out = torch.empty( + (int(q.shape[0]), int(q.shape[1]), int(q.shape[2])), + device=q.device, + dtype=q.dtype, + ) + ops.msa_decode_sparse_attn_mma_paged( + q.contiguous(), + k_cache.contiguous(), + v_cache.contiguous(), + req_to_token.contiguous(), + seq_lens.contiguous(), + slot_ids.contiguous(), + topk_idx.contiguous(), + int(block_size), + float(sm_scale), + out, + ) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_indexer_block_scores") + ) + def _msa_indexer_block_scores_fake( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + causal: int, + scores: torch.Tensor, + ) -> None: + return None + + +def native_indexer_block_scores( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + *, + max_blocks: int, + causal: bool, +) -> torch.Tensor: + """Block-max QK indexer scores. q/k_pages are bf16 (already dequantized). + + Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible). + """ + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + total_q = int(q.shape[0]) + hq = int(q.shape[1]) + scores = torch.full( + (hq, int(max_blocks), total_q), + float("-inf"), + dtype=torch.float32, + device=q.device, + ) + ops.msa_indexer_block_scores( + q.contiguous(), + k_pages.contiguous(), + batch_of_q.contiguous(), + cu_q.contiguous(), + cu_k.contiguous(), + cu_pages.contiguous(), + kv_indices.contiguous(), + 1 if causal else 0, + scores, + ) + return scores + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16") + ) + def _msa_nvfp4_dequant_swizzled_to_bf16_fake( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_nvfp4_dequant_swizzled_to_bf16( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor | float, +) -> torch.Tensor: + """Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if packed.dtype is not torch.uint8: + raise TypeError("packed must be torch.uint8") + if scale_128x4.dtype is not torch.uint8: + raise TypeError("scale_128x4 must be torch.uint8") + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16) + if isinstance(global_scale, torch.Tensor): + global_scale_value = float(global_scale.reshape(-1)[0].item()) + else: + global_scale_value = float(global_scale) + ops.msa_nvfp4_dequant_swizzled_to_bf16( + packed.contiguous(), + scale_128x4.contiguous(), + global_scale_value, + out, + ) + return out + + +__all__ = [ + "has_native_ops", + "native_topk_from_scores", + "native_decode_mma_supported", + "native_decode_sparse_attn_mma_paged", + "native_indexer_block_scores", + "native_nvfp4_dequant_swizzled_to_bf16", +] diff --git a/build/torch211-cxx11-cu130-x86_64-linux/_ops.py b/build/torch211-cxx11-cu130-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..41e39102ea6d028564a7e10c3b7004e7c40dae7b --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _minimaxai_msa_blackwell_cuda_25b0067 +ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}" diff --git a/build/torch211-cxx11-cu130-x86_64-linux/api_status.py b/build/torch211-cxx11-cu130-x86_64-linux/api_status.py new file mode 100644 index 0000000000000000000000000000000000000000..3dd4d744b49f1bc7acd6b3ba2a2e2c5f7b591252 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/api_status.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Public API status for the FlashRT MiniMax MSA Blackwell package.""" + +from __future__ import annotations + +from types import MappingProxyType + + +V1_AVAILABLE_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "naive_flash_decode_with_gqa_share_sparse", + "get_cu_seqblocks", + "robust_allocator", +) + +OFFICIAL_MINIMAX_MSA_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +) + +_OFFICIAL_API_STATUS = { + "sparse_atten_func": { + "status": "available", + "target": "blackwell-prefill", + "reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.", + }, + "sparse_atten_nvfp4_kv_func": { + "status": "available", + "target": "blackwell-prefill-nvfp4", + "reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.", + }, + "sparse_decode_atten_func": { + "status": "available", + "target": "v1", + "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.", + }, + "SparseDecodePagedAttentionWrapper": { + "status": "available", + "target": "v1", + "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.", + }, + "fp4_indexer_block_scores": { + "status": "available", + "target": "blackwell-fp4-indexer", + "reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.", + }, + "build_k2q_csr": { + "status": "available", + "target": "v1", + "reason": "CSR construction helper is available for the official prefill API.", + }, + "SparseK2qCsrBuilderSm100": { + "status": "available", + "target": "v1", + "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.", + }, + "Nvfp4QuantizedTensor": { + "status": "available", + "target": "v1", + "reason": "NVFP4 metadata dataclass is available.", + }, + "quantize_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "quantize_kv_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "dequantize_nvfp4_128x4_to_bf16": { + "status": "available", + "target": "v1", + "reason": "Reference dequant helper is available.", + }, + "swizzle_nvfp4_scale_to_128x4": { + "status": "available", + "target": "v1", + "reason": "Scale swizzle helper is available.", + }, + "nvfp4_global_scale_from_amax": { + "status": "available", + "target": "v1", + "reason": "Global scale helper is available.", + }, +} + +OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS) + + +def available_functions() -> tuple[str, ...]: + """Return the functions/classes intentionally available in v1.""" + + return V1_AVAILABLE_FUNCTIONS + + +def official_minimax_msa_functions() -> tuple[str, ...]: + """Return the upstream MiniMaxAI/msa public function/class names tracked here.""" + + return OFFICIAL_MINIMAX_MSA_FUNCTIONS + + +def official_api_status() -> dict[str, dict[str, str]]: + """Return a copy of the official API compatibility status table.""" + + return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()} + + +def unsupported_official_functions() -> tuple[str, ...]: + """Return official MiniMaxAI/msa names that are not exported by this v1 package.""" + + return tuple( + name + for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS + if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"} + ) diff --git a/build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py b/build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/common/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch211-cxx11-cu130-x86_64-linux/common/utils.py b/build/torch211-cxx11-cu130-x86_64-linux/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..81b7816b719973729312c0d84b36f9f50579f105 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/common/utils.py @@ -0,0 +1,144 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +import functools +from collections import deque +from typing import Any, Callable, Tuple + +import torch +import triton.language as tl + +_tma_keep_alive_buf = deque(maxlen=200) + +try: + make_tensor_descriptor = tl.make_tensor_descriptor +except Exception: + make_tensor_descriptor = tl._experimental_make_tensor_descriptor + + +def robust_allocator(size: int, alignment: int, stream: int = None): + """Allocator for Triton TMA descriptors. + + We keep reference in deque to prevent GC from collecting the buffer. + """ + tensor = torch.empty(size, device="cuda", dtype=torch.uint8) + _tma_keep_alive_buf.append(tensor) + return tensor + + +def tensor_cache(maxsize: int = 8): + """ + Cache function results using identity comparison. + Zero-overhead cache hit: no hash, no DtoH, just pointer comparison. + + Args: + maxsize: Maximum number of cached entries. Supports multi-GPU scenarios + where different devices have different tensor arguments. + """ + + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + # LRU-style cache: list of (args, kwargs, result) tuples + # Most recently used at the end + _cache: list = [] + + def _args_match(args: tuple, cached_args: tuple) -> bool: + if len(args) != len(cached_args): + return False + for i in range(len(args)): + if args[i] is not cached_args[i]: + return False + return True + + def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool: + if not kwargs and not cached_kwargs: + return True + if kwargs.keys() != cached_kwargs.keys(): + return False + for k, v in kwargs.items(): + if v is not cached_kwargs[k]: + return False + return True + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + # Search cache (most recent first for better hit rate) + for i in range(len(_cache) - 1, -1, -1): + cached_args, cached_kwargs, cached_result = _cache[i] + if _args_match(args, cached_args) and _kwargs_match( + kwargs, cached_kwargs + ): + # Move to end (most recently used) + if i != len(_cache) - 1: + _cache.append(_cache.pop(i)) + return cached_result + + # Cache miss + result = fn(*args, **kwargs) + + # Add to cache + if len(_cache) >= maxsize: + _cache.pop(0) # Remove oldest + _cache.append((args, kwargs, result)) + return result + + # Expose cache for manual clearing if needed + wrapper.cache_clear = lambda: _cache.clear() + wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize} + + return wrapper + + return decorator + + +@tensor_cache(maxsize=8) +def get_cu_seqblocks( + cu_seqlens: torch.Tensor, + max_seqlen: int, + block_size_q: int, + block_size_k: int, +) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]: + """Compute cumulative sequence block indices for blocked sparse attention. + + Converts token-level cumulative sequence lengths to block-level indices, + which are needed for block-sparse attention kernels. + + Note: + Results are cached (maxsize=8) based on input arguments. Repeated calls + with the same cu_seqlens, max_seqlen, and block sizes will return cached + results without recomputation. + + Args: + cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32. + max_seqlen: Maximum sequence length in the batch. + block_size_q: Query block size. + block_size_k: Key-value block size. + + Returns: + A tuple of 6 values: + - cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1] + - max_seqblock_q: Maximum number of query blocks per sequence. + - all_seqblock_q: Total number of query blocks across all sequences. + - cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1] + - max_seqblock_k: Maximum number of key blocks per sequence. + - all_seqblock_k: Total number of key blocks across all sequences. + """ + cu_seqblocks_q = torch.zeros_like(cu_seqlens) + cu_seqblocks_k = torch.zeros_like(cu_seqlens) + seq_lens = torch.diff(cu_seqlens) + seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q + seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k + max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q + max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k + cu_seqblocks_q[1:] = seqblocks_q + cu_seqblocks_k[1:] = seqblocks_k + cu_seqblocks_q.cumsum_(0) + cu_seqblocks_k.cumsum_(0) + all_seqblock_q = seqblocks_q.sum().item() + all_seqblock_k = seqblocks_k.sum().item() + return ( + cu_seqblocks_q, + max_seqblock_q, + all_seqblock_q, + cu_seqblocks_k, + max_seqblock_k, + all_seqblock_k, + ) diff --git a/build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py b/build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/decode/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py b/build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..8f56bdba2908e8af036785eae158338758555ee5 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py @@ -0,0 +1,1150 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import envs # vendored: was `from sglang.srt.environ import envs` + +from ..common.utils import robust_allocator +from .._native import has_native_ops, native_topk_from_scores + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "BATCH_SIZE_BUCKET", + "gqa_group_size", + "head_dim", + "block_size", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_kernel( + q_ptr, # Q: b x qh x d + k_cache_ptr, # K paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_r2t_b, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks <= topk: + return + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # prefetch first iteration's slots + r2t_base = req_to_token_ptr + sid * stride_r2t_b + prefetch_pos = chunk_start + off_n + prefetch_mask = prefetch_pos < seq_len + prefetched_slots = tl.load( + r2t_base + prefetch_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + # score-only: compute block scores without loading V + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos_mask = prefetch_mask + slots = prefetched_slots + # prefetch next iteration's slots + next_i = i + BLOCK_SIZE_N + if next_i < chunk_end: + next_pos = next_i + off_n + prefetch_mask = next_pos < seq_len + prefetched_slots = tl.load( + r2t_base + next_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + curr_block_idx = i // block_size + off_bpk + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)) + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "gqa_group_size", + "head_dim", + "block_size", + "HAS_SINK", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_attn_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + o_ptr, # O: c x b x qh x d + lse_ptr, # lse: c x b x qh + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + qsink = ( + tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e + ) # (BLOCK_SIZE_H,) + m_i = qsink + l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # full attention or causal attention + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos = i + off_n + pos_mask = pos < seq_len + # resolve slot per position via req_to_token + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + # current block indices: [BLOCKS_PER_K_BLOCK] + curr_block_idx = i // block_size + off_bpk + # Combined condition with single nested tl.where: + # - local_blocks (1e29) takes priority over init_blocks (1e30) + # - When init_blocks=0: curr_block_idx < 0 is always False + # - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + # Apply in one fused operation: local > init > original + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks > topk: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + else: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + # max-of-max == max(qk), avoids re-scanning qk + m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + m_i = m_ij + l_i = l_i * acc_o_scale + l_ij + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + # final scale + acc_o = acc_o / l_i[:, None] + # compute lse in log2 scale for merge kernel + lse_i = m_i + tl.log2(l_i) + # save output + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + # save lse (in log2 scale, will be used directly in merge kernel) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4] + ], + key=["BLOCK_SIZE_D"], + restore_value=["o_ptr"], +) +@triton.jit +def _merge_attn_out_kernel( + o_ptr, + lse_ptr, + # shape + seq_lens, + head_dim, + block_size: tl.constexpr, + # stride + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_KV_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + # recompute chunk layout (must match _decode_score_attn_kernel) + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks) + # ptrs + off_c = tl.arange(0, NUM_KV_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(valid_chunks, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + # load o and lse + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf")) + # merge o (lse is in log2 scale, use exp2/log2 for efficiency) + lse_shifted = lse - tl.max(lse, axis=0) + log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0)) + scale = tl.exp2(lse_shifted - log_sum) + o = tl.sum(o * scale[:, None], axis=0) + # save o + o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@triton.jit +def _compare_and_swap( + x, + ids, + flip, + i: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)] + y = tl.reshape(x, shape) + # slice left/right with 'stride' 2**(n_dims - i - 1) + mask = tl.arange(0, 2)[None, :, None] + left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype) + right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype) + left = tl.reshape(left, x.shape) + right = tl.reshape(right, x.shape) + # idx + y_idx = tl.reshape(ids, shape) + left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape) + right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape) + left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype) + right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype) + # actual compare-and-swap + idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True) + ileft = left.to(idtype, bitcast=True) + iright = right.to(idtype, bitcast=True) + ix = x.to(idtype, bitcast=True) + + cond = (left > right) != flip + ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix)) + new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids)) + return ret.to(x.dtype, bitcast=True), new_ids + + +@triton.jit +def _bitonic_merge( + x, + ids, + stage: tl.constexpr, + order: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + tl.static_assert(stage <= n_dims) + # flip denotes whether to re-arrange sub-sequences of elements in ascending or + # descending order. + # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage + # if flip = 00110011... then all the elements will be re-arranged alternatingly (with + # a stride of 2) at this stage + if order == 2: + shape: tl.constexpr = [ + n_outer * 2 ** (n_dims - 1 - stage), + 2, + 2**stage, + ] + flip = tl.reshape( + tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape + ) + else: + flip = order + # perform `stage` rounds of `compare-and-swap` + for i in tl.static_range(stage): + x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims) + return x, ids + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2), + ], + key=["topk"], +) +@triton.jit +def _topk_index_partial_kernel( + s_ptr, # Score: [num_q_heads, BS, max_seqblock] + ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns + # strides + stride_s_h, + stride_s_b, + stride_s_k, + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + # META + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + tl.static_assert(BLOCK_SIZE_K > topk) + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + pid_chunk = tl.program_id(2) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Slice this chunk owns within [0, num_blocks). + chunk_start = pid_chunk * chunk_blocks + chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks) + chunk_actual = tl.maximum(chunk_end - chunk_start, 0) + + off_k = tl.arange(0, BLOCK_SIZE_K) + off_t = tl.arange(0, BLOCK_SIZE_T) + + # Pointer starts at chunk_start. + s_ptrs = ( + s_ptr + + pid_b * stride_s_b + + pid_h * stride_s_h + + (chunk_start + off_k) * stride_s_k + ) + + topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32) + topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32) + left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2 + + # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty + # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0. + for i in tl.range(0, chunk_actual, BLOCK_SIZE_K): + mask = off_k < chunk_actual - i + score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32) + score = tl.where(score != score, -1e30, score) + s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K + topk_score, last_topk_score = score, topk_score + topk_idx, last_topk_idx = ( + tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global + topk_idx, + ) + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), j, 2, n_dims + ) + if i != 0: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims + ) + topk_score_new = last_topk_score * left_half_mask + topk_score * ( + 1 - left_half_mask + ) + topk_idx_new = last_topk_idx * left_half_mask + topk_idx * ( + 1 - left_half_mask + ) + topk_score, topk_idx = _bitonic_merge( + topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims + ) + else: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims + ) + + # Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort). + topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + final_score = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + final_idx = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + # Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry + # -1e30 / 0 sentinels and lose to real scores in the merge stage. + ts_ptrs = ( + ts_partial_ptr + + pid_chunk * stride_ts_c + + pid_b * stride_ts_b + + pid_h * stride_ts_h + + off_t * stride_ts_t + ) + ti_ptrs = ( + ti_partial_ptr + + pid_chunk * stride_ti_c + + pid_b * stride_ti_b + + pid_h * stride_ti_h + + off_t * stride_ti_t + ) + tl.store(ts_ptrs, final_score) + tl.store(ti_ptrs, final_idx) + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + "BLOCK_SIZE_K": lambda args: triton.next_power_of_2( + args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"]) + ), + } +) +@triton.jit +def _topk_index_merge_kernel( + ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape + ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk] + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + # strides + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_tif_h, + stride_tif_b, + stride_tif_t, + # META + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K. + # candidate at flat position p comes from chunk = p // BLOCK_SIZE_T, + # in_chunk = p % BLOCK_SIZE_T. + off = tl.arange(0, BLOCK_SIZE_K) + chunk_idx = off // BLOCK_SIZE_T + in_chunk_idx = off % BLOCK_SIZE_T + valid = chunk_idx < NUM_TOPK_CHUNKS + + score_offset = ( + chunk_idx * stride_ts_c + + pid_h * stride_ts_h + + pid_b * stride_ts_b + + in_chunk_idx * stride_ts_t + ) + idx_offset = ( + chunk_idx * stride_ti_c + + pid_h * stride_ti_h + + pid_b * stride_ti_b + + in_chunk_idx * stride_ti_t + ) + + score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to( + tl.float32 + ) + score = tl.where(score != score, -1e30, score) + idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32) + + # Full bitonic descending sort of BLOCK_SIZE_K items. + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims) + score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims) + + # Extract first BLOCK_SIZE_T positions — these are the global top-K. + extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + topk_idx_final = tl.sum( + extract_mask[:, None] + * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + off_t = tl.arange(0, BLOCK_SIZE_T) + tif_ptrs = ( + ti_final_ptr + + pid_h * stride_tif_h + + pid_b * stride_tif_b + + off_t * stride_tif_t + ) + topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1) + tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty)) + + +@torch.no_grad() +def flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + init_blocks: int, + local_blocks: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + score_type: str = "max", + disable_index_value: bool = False, + use_dense_main_attn: bool = False, # NOTE: need transform idx in this case + page_size: int = 1, +) -> torch.Tensor: + assert score_type in ( + "max", + "lse", + ), f"score_type must be 'max' or 'lse', got {score_type!r}" + triton.set_allocator(robust_allocator) + # dtype check + assert ( + q.dtype == torch.bfloat16 + or q.dtype == torch.float16 + and k_cache.dtype == q.dtype + ) + if not disable_index_value: + assert v_cache is not None + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + max_kv_len = req_to_token.shape[1] + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets. + # Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads. + # TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by: + # NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS) + # then rounded down to the nearest power of 2 (merge kernel requires it). + # Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len. + # Empty chunks early-return cheaply, so over-chunking is nearly free. + # E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096. + TARGET_GRID = 4096 + MAX_NUM_KV_CHUNKS = 256 + target = max( + 1, + min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_KV_CHUNKS = 1 << (target.bit_length() - 1) + score_kv_len = min(max_seqlen, max_kv_len) + # The score producers below write every valid block column + # [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers + # clamp their scan to the same per-row valid block count, so columns beyond + # seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph + # capture the static score shape can be much larger than the live context. + score = torch.empty( + (num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)), + dtype=torch.float32, + device=q.device, + ) + use_jit_topk = ( + envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get() + and score.shape[2] <= 4096 + and topk <= 32 + ) + # If the live context has <= topk sparse blocks, the downstream dense + # page-table/JIT top-k kernels select every block from seq_lens directly + # without reading score. Keep this gate in sync with the consumers below: + # _skip_block_topk/use_dense_main_attn and use_jit_topk special-case + # num_blocks <= topk, while the Triton fallback reads score and must not + # skip these writes. + skip_trivial_topk_score = use_dense_main_attn or use_jit_topk + + grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads) + if disable_index_value: + _decode_score_kernel[grid]( + q, + k_cache, + req_to_token, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + req_to_token.stride(0), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + else: + assert v_cache is not None + o = torch.empty( + NUM_KV_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse = torch.empty( + NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device + ) + _decode_score_attn_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + o, + lse, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + # Fused top-k + page-table transform: emit the dense backend's page table + # directly (page-size-aware) instead of block ids, skipping a separate gather. + # The page table + per-query effective KV length are allocated and returned. + real_seq_lens = None + if use_dense_main_attn: + # VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The + # standalone vendor only ships the pure-Triton path, so this is gated off + # by default (use_dense_main_attn=False). If you opt in, vendor that .cuh + # too. Original line: `from sglang.jit_kernel.minimax_decode_topk import ( + # minimax_decode_topk_page_table)`. + raise NotImplementedError( + "use_dense_main_attn=True requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; " + "the standalone Triton vendor only supports use_dense_main_attn=False." + ) + # num_q_heads (= num index/kv heads) may be > 1 for DP attention; the + # kernel emits a flattened [batch*num_q_heads] page table + seq_lens with + # the kv head head-encoded (head-minor) into the page index. + topk_idx, real_seq_lens = minimax_decode_topk_page_table( + score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # fall through to the idx_o (index value) merge, then return + _skip_block_topk = True + else: + _skip_block_topk = False + + # get topk index + if not _skip_block_topk: + topk_idx = torch.empty( + (num_q_heads, batch_size, topk), + device=score.device, + dtype=torch.int32, + ) + + if _skip_block_topk: + pass + elif use_jit_topk: + # Single-stage JIT radix-select: one kernel, no intermediate buffers. + # Equivalent output to the 2-stage path (set of block ids, front-packed, + # -1 padded); ~2-16x faster for long context. + # VENDOR NOTE: this needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the + # `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so + # this branch is never taken in the standalone vendor (the 2-stage Triton + # `else` fallback below runs instead). Original line: + # `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`. + raise NotImplementedError( + "use_jit_topk requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor " + "uses the 2-stage Triton fallback (keep " + "SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)." + ) + elif has_native_ops() and topk <= 64: + topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk) + else: + # 2-stage split-K Triton fallback. + # Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim. + # Same constraints as flash_decode: must be deterministic from values + # constant within a cuda graph (BS, num_q_heads), pow2, capped so the + # merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk)) + # stays reasonable. With topk=32 and cap 16, merge sorts 512 items. + TOPK_TARGET_GRID = 64 + MAX_NUM_TOPK_CHUNKS = 16 + topk_target = max( + 1, + min( + MAX_NUM_TOPK_CHUNKS, + TOPK_TARGET_GRID // max(1, batch_size * num_q_heads), + ), + ) + NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1) + BLOCK_SIZE_T = triton.next_power_of_2(topk) + max_seqblock = score.shape[2] + chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + topk_score_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.float32, + device=score.device, + ) + topk_idx_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.int32, + device=score.device, + ) + # stream 0 (default): topk partial → topk merge + grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS) + _topk_index_partial_kernel[grid]( + score, + topk_score_partial, + topk_idx_partial, + seq_lens, + block_size, + topk, + chunk_blocks, + score.stride(0), + score.stride(1), + score.stride(2), + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + ) + grid = (batch_size, num_q_heads) + _topk_index_merge_kernel[grid]( + topk_score_partial, + topk_idx_partial, + topk_idx, + seq_lens, + block_size, + topk, + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # attn output merge (default stream) + grid = (batch_size, num_q_heads) + _merge_attn_out_kernel[grid]( + o, + lse, + seq_lens, + head_dim, + block_size, + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + ) + o = o[0].contiguous() + return o, topk_idx, real_seq_lens diff --git a/build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py b/build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..8fe24cce53337c4f44e6c462fc3c90a258a34fc7 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/decode/topk_sparse.py @@ -0,0 +1,452 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from .._native import ( + native_decode_mma_supported, + native_decode_sparse_attn_mma_paged, +) +from ..common.utils import robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4, 5] + ], + key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"], +) +@triton.jit +def _gqa_share_sparse_decode_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + idx_ptr, # topk index: qh x b x topk + o_ptr, # O partial: c x b x qh x d + lse_ptr, # lse partial: c x b x qh + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + # sm_scale + sm_scale, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + NUM_TOPK_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + IS_FP8: tl.constexpr, +): + # decode program ids: split-K over the topk dimension to give every SM + # something to do at small batch. pid(0) folds (batch, chunk) together so + # the grid size = batch_size * NUM_TOPK_CHUNKS. + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # per-chunk topk range. chunk_size is *runtime* (depends on max_topk which + # is a runtime arg, not constexpr), so don't annotate as tl.constexpr — + # doing so produces undefined behavior in Triton. + chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + chunk_start_topk = pid_c * chunk_size_topk + chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk + # get q k start and len after rmpad + seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # get real topk + off_t = tl.arange(0, BLOCK_SIZE_T) + idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b + topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk) + # init pointer + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + dim_mask = off_d < head_dim + # init statistics — kept at -inf so empty chunks (chunk_start >= real_topk) + # naturally fall out as weight=0 in the merge step. + # q is always needed; load it unconditionally. + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + # Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code- + # eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is + # then None). The original compound `if HAS_SINK and pid_c == 0` traced + # the body even when False -> NoneType make_block_ptr compile error on + # sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always + # False for us, but this keeps the sink path correct too. + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load( + sink_ptrs, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale + m_i = qsink + lse_i = qsink + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # only iterate over this chunk's topk slice. the load must respect the + # per-chunk start offset. + cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t + for _ in tl.range(chunk_start_topk, chunk_end_topk): + # load index + c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N + cur_idx_ptr = cur_idx_ptr + stride_ti_t + # resolve slots for this block via req_to_token + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no + # scale), so dequant is just a widening cast to the Q compute dtype + # before the tl.dot. Matches the bf16 path bit-for-bit when the cache + # is bf16 (IS_FP8 False -> this branch is compiled out). + k = k.to(q.dtype) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V before the P@V dot. This also makes the `p.to(v.dtype)` + # below cast P to the compute dtype (not to fp8, which would be + # catastrophic precision loss on the attention weights). + v = v.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # load v and update acc_o + p = p.to(v.dtype) + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij) + # final scale (matches the old non-split kernel for chunks where lse_i>-inf). + # For empty chunks (chunk_start_topk >= real_topk) the inner loop never + # runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute + # exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial + # and the merge result. Gate the scale with tl.where so empty chunks emit a + # clean zero (lse_i stays -inf which the merge correctly turns into weight=0). + scale = tl.where( + lse_i > float("-inf"), + tl.exp(m_i - lse_i), + tl.zeros_like(lse_i), + ) + acc_o = acc_o * scale[:, None] + # save partial output and lse for the merge step + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.jit +def _merge_topk_attn_out_kernel( + o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0 + lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH] + head_dim, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + off_c = tl.arange(0, NUM_TOPK_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(NUM_TOPK_CHUNKS, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0 + # standard flash-decoding merge in linear (not log2) space, matching the + # decode kernel which uses tl.exp / tl.log. + lse_max = tl.max(lse, axis=0) + weights = tl.exp(lse - lse_max) + weights = weights / tl.sum(weights, axis=0) + o_merged = tl.sum(o * weights[:, None], axis=0) + o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@torch.no_grad() +def flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, + use_tma: bool = True, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may + # be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel + # widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing + # sparse decode dtype contract unchanged by accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + assert topk_idx.shape[0] == num_kv_heads + assert ( + triton.next_power_of_2(block_size) == block_size + ), f"block_size must be a power of 2, but got {block_size}" + # assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}" + max_kv_len = req_to_token.shape[1] + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + max_topk = topk_idx.shape[2] + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # Fast path: native tensor-core decode for the M3 MSA shape. Falls through + # to the Triton kernel for other shapes, sink, or fp8 caches. + if sink is None and not is_fp8 and native_decode_mma_supported( + q, k_cache, block_size=block_size + ): + return native_decode_sparse_attn_mma_paged( + q, + k_cache, + v_cache, + req_to_token, + seq_lens, + slot_ids, + block_size, + topk_idx, + sm_scale, + ) + # Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as + # flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must + # only depend on shape constants (so grid is fixed within a cuda graph). + # Capped by max_topk because chunks beyond real_topk early-fall-through to + # the merge-as-zero path; capping avoids wasting blocks at tiny topk. + TARGET_GRID = 256 + target = max( + 1, + min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1) + # output tensor: split-K partials, merged into chunk 0 by the merge kernel + o_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + dtype=torch.float32, + device=q.device, + ) + # launch attention kernel + grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads) + _gqa_share_sparse_decode_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + topk_idx, + o_partial, + lse_partial, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + BLOCK_SIZE_N=block_size, + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + IS_FP8=is_fp8, + ) + # merge partials into chunk 0 + merge_grid = (batch_size, num_q_heads) + _merge_topk_attn_out_kernel[merge_grid]( + o_partial, + lse_partial, + head_dim, + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + return o_partial[0].contiguous() diff --git a/build/torch211-cxx11-cu130-x86_64-linux/interface.py b/build/torch211-cxx11-cu130-x86_64-linux/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..7939722609218f2bb41eadcdb2f965213d459d1c --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/interface.py @@ -0,0 +1,679 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from ._native import ( + has_native_ops, + native_indexer_block_scores, + native_nvfp4_dequant_swizzled_to_bf16, +) +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse +from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16 + + +def _page_table_to_req_to_token( + page_table: torch.Tensor, + seqused_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, +) -> torch.Tensor: + batch = int(page_table.shape[0]) + max_seqlen_k = int(max_seqlen_k) + blk_kv = int(blk_kv) + logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64) + page = logical_pos // blk_kv + offset = logical_pos % blk_kv + page = page.clamp(max=int(page_table.shape[1]) - 1) + physical_page = page_table[:, page].to(torch.int64) + physical_token = physical_page * blk_kv + offset[None, :] + valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None] + return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous() + + +def _dense_q2k_indices( + *, + head_kv: int, + batch: int, + topk: int, + seq_lens: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device) + blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device) + valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv) + for b in range(batch): + valid = blocks < valid_blocks[b] + idx[:, b, valid] = blocks[valid] + return idx + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]: + seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist() + return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens] + + +def _packed_row_to_batch_block( + cu_seqlens_k: torch.Tensor, + blk_kv: int, +) -> list[tuple[int, int]]: + rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv)) + max_rows = max(rows_per_batch, default=0) + row_map: list[tuple[int, int]] = [] + for block_idx in range(max_rows): + for batch_idx, rows in enumerate(rows_per_batch): + if block_idx < rows: + row_map.append((batch_idx, block_idx)) + return row_map + + +def _csr_to_q2k_indices( + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + blk_kv: int, + topk: int, +) -> torch.Tensor: + if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32: + raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32") + if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2: + raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2") + head_kv = int(k2q_row_ptr.shape[0]) + total_q = int(cu_seqlens_q[-1].item()) + topk = int(topk) + if topk <= 0: + raise ValueError(f"topK must be positive, got {topk}") + + row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv)) + if int(k2q_row_ptr.shape[1]) != len(row_map) + 1: + raise ValueError( + "k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: " + f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}" + ) + + cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False) + q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False) + q2k = torch.full( + (head_kv, total_q, topk), + -1, + dtype=torch.int32, + device=k2q_row_ptr.device, + ) + + next_slot: dict[tuple[int, int], int] = {} + seen: set[tuple[int, int, int]] = set() + for h in range(head_kv): + for row, (batch_idx, kv_block_idx) in enumerate(row_map): + start = int(row_ptr_cpu[h, row].item()) + end = int(row_ptr_cpu[h, row + 1].item()) + q_start = int(cu_q_cpu[batch_idx]) + q_end = int(cu_q_cpu[batch_idx + 1]) + q_len = q_end - q_start + for pos in range(start, end): + q_val = int(q_idx_cpu[h, pos].item()) + if q_val < 0: + continue + if q_start <= q_val < q_end: + q_global = q_val + elif q_val < q_len: + q_global = q_start + q_val + else: + continue + key = (h, q_global, kv_block_idx) + if key in seen: + continue + seen.add(key) + slot_key = (h, q_global) + slot = next_slot.get(slot_key, 0) + if slot >= topk: + continue + q2k[h, q_global, slot] = int(kv_block_idx) + next_slot[slot_key] = slot + 1 + return q2k.contiguous() + + +def _make_prefill_cache_and_metadata( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, + *, + page_table: Optional[torch.Tensor], + seqused_k: Optional[torch.Tensor], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if page_table is None: + if k.ndim != 3 or v.ndim != 3: + raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + k_cache = k.contiguous() + v_cache = v.contiguous() + batch = int(cu_seqlens_q.numel()) - 1 + max_seqlen_k = int(max_seqlen_k) + logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64) + cu_k = cu_seqlens_k.to(torch.int64) + seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous() + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32) + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + else: + if k.ndim != 4 or v.ndim != 4: + raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("paged k/v page dimension must equal blk_kv") + if page_table.dtype != torch.int32: + raise TypeError("page_table must be torch.int32") + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + batch = int(page_table.shape[0]) + if seqused_k is None: + seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous() + else: + seq_lens = seqused_k.to(torch.int32).contiguous() + logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64) + page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1) + offset = logical % int(blk_kv) + physical_page = page_table[:, page].to(torch.int64) + req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32) + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32) + prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous() + return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens + + +def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int: + if kv_heads <= 0 or q_heads % kv_heads != 0: + raise ValueError("number of query heads must be divisible by KV heads") + gqa_group = q_heads // kv_heads + limit = max(1, 128 // max(1, gqa_group)) + for candidate in (64, 32, 16, 8, 4, 2, 1): + if candidate <= limit: + return candidate + return 1 + + +def _normalize_decode_q2k( + q2k_indices: Optional[torch.Tensor], + *, + batch: int, + seqlen_q: int, + head_kv: int, + seqused_k: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + if q2k_indices is None: + topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv)) + return _dense_q2k_indices( + head_kv=head_kv, + batch=batch, + topk=topk, + seq_lens=seqused_k, + blk_kv=blk_kv, + ) + if q2k_indices.dtype != torch.int32: + raise TypeError("q2k_indices must be torch.int32") + if q2k_indices.ndim != 3: + raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]") + if q2k_indices.shape[0] != head_kv: + raise ValueError("q2k_indices first dimension must match Hkv") + if int(q2k_indices.shape[1]) != batch * seqlen_q: + raise ValueError("q2k_indices second dimension must equal batch * seqlen_q") + if seqlen_q != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + return q2k_indices[:, ::seqlen_q, :].contiguous() + + +def sparse_decode_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q2k_indices: Optional[torch.Tensor] = None, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + schedule: Optional[object] = None, + O_partial: Optional[torch.Tensor] = None, + LSE_partial: Optional[torch.Tensor] = None, +): + """Run MiniMax paged sparse decode on Blackwell. + + This is compatible with the upstream MiniMaxAI/msa decode-facing API name, + but accepts the Blackwell package's validated BF16/FP16 decode path. It + supports single-token decode (`seqlen_q=1`) with paged K/V + `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`. + """ + + if schedule is not None or O_partial is not None or LSE_partial is not None: + raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper") + if return_softmax_lse: + raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper") + if not causal: + raise NotImplementedError("Blackwell decode wrapper currently supports causal=True") + if int(seqlen_q) != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + if q.ndim != 3: + raise ValueError("q must have shape [batch * seqlen_q, Hq, D]") + if k.ndim != 4 or v.ndim != 4: + raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if q.device != k.device or q.device != v.device or q.device != page_table.device: + raise ValueError("q, k, v, and page_table must be on the same device") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("Blackwell decode wrapper supports BF16/FP16 q") + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper") + if q.shape[-1] != 128 or k.shape[-1] != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("k.shape[2] must equal blk_kv") + if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32: + raise TypeError("page_table and seqused_k must be torch.int32") + if page_table.ndim != 2: + raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]") + batch = int(page_table.shape[0]) + if int(q.shape[0]) != batch: + raise ValueError("for seqlen_q=1, q.shape[0] must equal batch") + + head_kv = int(k.shape[1]) + q2k = _normalize_decode_q2k( + q2k_indices, + batch=batch, + seqlen_q=int(seqlen_q), + head_kv=head_kv, + seqused_k=seqused_k, + blk_kv=int(blk_kv), + ) + req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv)) + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + return flash_decode_with_gqa_share_sparse( + q.contiguous(), + None, + k_cache, + v_cache, + req_to_token, + seqused_k.contiguous(), + slot_ids, + int(blk_kv), + q2k, + sm_scale=softmax_scale, + ) + + +class SparseDecodePagedAttentionWrapper: + def __init__(self, *, blk_kv: int = 128, causal: bool = True): + self.blk_kv = int(blk_kv) + self.causal = bool(causal) + self.page_table: Optional[torch.Tensor] = None + self.seqused_k: Optional[torch.Tensor] = None + self.seqlen_q: Optional[int] = None + self.max_seqlen_k: Optional[int] = None + self.q2k_indices: Optional[torch.Tensor] = None + + def plan( + self, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + q2k_indices: Optional[torch.Tensor] = None, + num_qo_heads: Optional[int] = None, + num_kv_heads: Optional[int] = None, + head_dim: Optional[int] = 128, + enable_cuda_graph: bool = False, + max_grid_size: Optional[int] = None, + fixed_split_size: Optional[int] = None, + disable_split_kv: bool = False, + ) -> "SparseDecodePagedAttentionWrapper": + if head_dim is not None and int(head_dim) != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + self.page_table = page_table.contiguous() + self.seqused_k = seqused_k.contiguous() + self.seqlen_q = int(seqlen_q) + self.max_seqlen_k = int(max_seqlen_k) + self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous() + return self + + def run( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + out: Optional[torch.Tensor] = None, + lse: Optional[torch.Tensor] = None, + ): + if self.page_table is None or self.seqused_k is None: + raise RuntimeError("decode wrapper must be planned before run") + result = sparse_decode_atten_func( + q, + k, + v, + self.q2k_indices, + page_table=self.page_table, + seqused_k=self.seqused_k, + seqlen_q=self.seqlen_q, + max_seqlen_k=self.max_seqlen_k, + blk_kv=self.blk_kv, + causal=self.causal, + softmax_scale=softmax_scale, + return_softmax_lse=return_softmax_lse, + ) + if out is not None: + out.copy_(result) + return out + return result + + +def sparse_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + *, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + lse_temperature_scale: float = 1.0, + return_temperature_lse: bool = False, + partial_dtype: torch.dtype = torch.bfloat16, + return_softmax_lse: bool = False, + page_table: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + schedule: Optional[object] = None, + usable_SM_count: int = -1, + qk_dtype: Optional[torch.dtype] = None, + pv_dtype: Optional[torch.dtype] = None, +): + """Run the official MiniMax CSR sparse prefill API on Blackwell. + + The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list + consumed by the Blackwell Triton sparse prefill kernel. The default output + path is supported for BF16/FP16 tensors with head_dim=128. + """ + + del partial_dtype, usable_SM_count, qk_dtype, pv_dtype + if schedule is not None: + raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper") + if return_temperature_lse or return_softmax_lse: + raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper") + if float(lse_temperature_scale) != 1.0: + raise ValueError("lse_temperature_scale values other than 1.0 are not supported") + if not causal: + raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True") + if q.ndim != 3: + raise ValueError("q must have shape [total_q, Hq, D]") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("q must be BF16 or FP16") + if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128: + raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128") + if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32: + raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32") + if q.device != k.device or q.device != v.device: + raise ValueError("q, k, and v must be on the same device") + + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have matching BF16/FP16 dtype") + kv_heads = int(k.shape[1]) + q2k_indices = _csr_to_q2k_indices( + k2q_row_ptr.contiguous(), + k2q_q_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(blk_kv), + int(topK), + ) + k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata( + q, + k, + v, + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(max_seqlen_k), + int(blk_kv), + page_table=None if page_table is None else page_table.contiguous(), + seqused_k=None if seqused_k is None else seqused_k.contiguous(), + ) + block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads) + return flash_prefill_with_gqa_share_sparse( + q.contiguous(), + k_cache, + v_cache, + None, + req_to_token, + slot_ids, + q2k_indices, + block_size_q=block_size_q, + block_size_k=int(blk_kv), + cu_seqlens=cu_seqlens_q.contiguous(), + seq_lens=seq_lens, + prefix_lens=prefix_lens, + max_seqlen_q=int(max_seqlen_q), + sm_scale=softmax_scale, + ) + + +def _nvfp4_dequant_kv( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor, +) -> torch.Tensor: + if packed.dtype is not torch.uint8: + return packed + if has_native_ops() and packed.is_cuda: + return native_nvfp4_dequant_swizzled_to_bf16( + packed, + scale_128x4, + global_scale, + ) + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + rows = 1 + for dim in original_shape[:-1]: + rows *= int(dim) + qx = Nvfp4QuantizedTensor( + data=packed, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, int(original_shape[-1]) // 16), + original_shape=original_shape, + ) + return dequantize_nvfp4_128x4_to_bf16(qx) + + +def sparse_atten_nvfp4_kv_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k_scale_128x4: torch.Tensor, + v_scale_128x4: torch.Tensor, + k_global_scale: torch.Tensor, + v_global_scale: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + **kwargs, +): + """NVFP4-KV official prefill API wrapper. + + The Blackwell compatibility path dequantizes NVFP4 K/V with the package's + 128x4 metadata helper, then dispatches to `sparse_atten_func`. + """ + + k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale) + v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale) + return sparse_atten_func( + q, + k_bf16, + v_bf16, + k2q_row_ptr, + k2q_q_indices, + topK, + **kwargs, + ) + + +def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor: + data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8) + lut = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0], + dtype=torch.float32, + device=data.device, + ) + out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device) + out[..., 0::2] = lut[(data & 0x0F).long()] + out[..., 1::2] = lut[(data >> 4).long()] + return out + + +def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]): + return values * scale.to(torch.float32).repeat_interleave(16, dim=-1) + if scale.ndim == values.ndim - 1: + return values * scale.to(torch.float32).unsqueeze(-1) + return values + + +def fp4_indexer_block_scores( + q_fp4: torch.Tensor, + k_fp4: torch.Tensor, + q_scale: torch.Tensor, + k_scale: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + cu_page_offsets: torch.Tensor, + *, + max_seqlen_q: int, + max_seqlen_k: int, + kv_indices: torch.Tensor, + fp4_format: str, + causal: bool = False, + qo_offset: Optional[torch.Tensor] = None, + scale_layout: str = "preordered_mma", +) -> torch.Tensor: + """FP4 block-score API for Blackwell. + + Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`. + Built artifacts route CUDA tensors through the native Blackwell block-score + kernel. Source-tree mode uses the reference loop below so the API remains + testable before the extension is built. + """ + + del qo_offset + if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}: + raise ValueError(f"unsupported fp4_format: {fp4_format}") + if scale_layout not in {"public", "preordered_mma"}: + raise ValueError(f"unsupported scale_layout: {scale_layout}") + if q_fp4.ndim != 3: + raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]") + if k_fp4.ndim != 4: + raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]") + total_q, hq, _ = q_fp4.shape + num_pages, hkv, page_size, _ = k_fp4.shape + if int(page_size) != 128: + raise ValueError("fp4_indexer_block_scores currently expects 128-token pages") + if hq % hkv != 0: + raise ValueError("Hq must be divisible by Hkv") + max_blocks = (int(max_seqlen_k) + 127) // 128 + scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device) + + q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale) + k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale) + + # Native block-max QK kernel for the cubic scoring loop when available. + # Same dequant numerics as the reference (bf16 compute); falls back to the + # Python reference below in source-tree mode. + if has_native_ops() and q_fp4.is_cuda: + batch = int(cu_seqlens_q.numel()) - 1 + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64) + batch_of_q = torch.repeat_interleave( + torch.arange(batch, device=q_fp4.device, dtype=torch.int32), + q_lens, + ) + return native_indexer_block_scores( + q.to(torch.bfloat16), + k.to(torch.bfloat16), + batch_of_q, + cu_seqlens_q.to(torch.int32), + cu_seqlens_k.to(torch.int32), + cu_page_offsets.to(torch.int32), + kv_indices.to(torch.int32), + max_blocks=max_blocks, + causal=causal, + ) + + gqa = hq // hkv + batch = int(cu_seqlens_q.numel()) - 1 + cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist() + cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist() + kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist() + for b in range(batch): + q_start, q_end = int(cu_q[b]), int(cu_q[b + 1]) + k_len = int(cu_k[b + 1]) - int(cu_k[b]) + page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1]) + for logical_block, page_pos in enumerate(range(page_start, page_end)): + if logical_block >= max_blocks: + break + physical_page = int(kv_idx_cpu[page_pos]) + if physical_page < 0 or physical_page >= num_pages: + continue + valid_tokens = min(128, max(0, k_len - logical_block * 128)) + if valid_tokens <= 0: + continue + for qi in range(q_start, q_end): + local_q = qi - q_start + if causal and logical_block * 128 > local_q: + continue + for qh in range(hq): + kh = qh // gqa + key_block = k[physical_page, kh, :valid_tokens] + if causal: + visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1)) + if visible <= 0: + continue + key_block = key_block[:visible] + block_scores = torch.matmul(key_block, q[qi, qh]) + scores[qh, logical_block, qi] = block_scores.max() + return scores + + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", +] diff --git a/build/torch211-cxx11-cu130-x86_64-linux/metadata.json b/build/torch211-cxx11-cu130-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fbeeab99c6a63a0ec5fb8d407d016a0cf9061ca4 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/metadata.json @@ -0,0 +1,39 @@ +{ + "name": "minimaxai-msa-blackwell", + "id": "_minimaxai_msa_blackwell_cuda_25b0067", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": { + "type": "cuda", + "archs": [ + "12.0", + "12.1" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=", + "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=", + "_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "oFPB/WnE3z+U5WanxNiLqUP41c7OcUreUnifaBmpYaw=", + "_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=", + "_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=", + "api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=", + "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=", + "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=", + "decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=", + "interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=", + "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=", + "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=", + "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=", + "prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=", + "prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=", + "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=", + "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w=" + } + } +} \ No newline at end of file diff --git a/build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py b/build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b2672c1cd85b74c1b3ded0fc0b2100e1aeac23 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py b/build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/naive/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py b/build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..db7caceb6faa5d8d1669d3c57f1e4ca9578a4804 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py @@ -0,0 +1,106 @@ +# Copyright 2025 XunhaoLai. All rights reserved. +# +# VENDOR NOTE: rewritten to drop the `einops` dependency (the original used +# einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream +# `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced +# with native torch.einsum / torch.reshape so the standalone vendor needs only +# torch (+ triton for the kernels). Original einops expressions are kept in +# comments next to each replacement for auditability. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], # [num_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + sm_scale: Optional[float] = None, + init_blocks: int = 0, + local_blocks: int = 0, +): + """Joint reference: lightning-indexer block top-k + sparse attention. + + This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense + 5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by + `slot_ids`. It both (a) selects the top-k key blocks via per-block max score + (with optional forced init/local blocks) and (b) computes the resulting + sparse attention, returning (o, topk_idx). + """ + assert ( + kv_cache.shape[2] % block_size == 0 + ), "max cache len must be divisible by block size" + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + original_dtype = q.dtype + batch_size = q.shape[0] + num_q_heads = q.shape[1] + num_kv_heads = kv_cache.shape[3] + gqa_group_size = num_q_heads // num_kv_heads + head_dim = q.shape[-1] + # rearrange q: "b (h g) d -> b h g d" + q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim) + kv_cache_float = kv_cache.float() + k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d] + v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d] + # einsum(q, k, "b h g d, b n h d -> b h g n") + qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale + mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None] + qk = qk.masked_fill(~mask[:, None, None, :], float("-inf")) + # get score + score = qk.clone().to(torch.float32) + # rearrange "b h g (n s) -> b (h g) n s", s=block_size + n_blocks = score.shape[-1] // block_size + score = score.reshape( + batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size + ).reshape(batch_size, num_q_heads, n_blocks, block_size) + score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks] + # post-process score for init_blocks and local_blocks + INIT_SCORE = 1e30 + LOCAL_SCORE = 1e29 + if init_blocks > 0: + score[:, :, :init_blocks] = INIT_SCORE + if local_blocks > 0: + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + local_start = max(0, num_blks - local_blocks) + score[b, :, local_start:num_blks] = LOCAL_SCORE + # compute topk indices per (batch, head) + # score shape: [batch_size, num_q_heads, num_blocks] + topk_idx = torch.full( + (num_q_heads, batch_size, topk), + fill_value=-1, + device=score.device, + dtype=torch.int32, + ) + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + actual_topk = min(topk, num_blks) + for h in range(num_q_heads): + # get topk indices for this (batch, head) + _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1) + topk_idx[h, b, :actual_topk] = indices.to(torch.int32) + # compute attention output with sink + if sink is not None: + # sink: [num_q_heads, head_dim] -> [h, g, d] + sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim) + # einsum(q, sink, "b h g d, h g d -> b h g") + qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale + qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1] + attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32) + attn = attn[..., 1:] # remove sink score + else: + attn = qk.softmax(dim=-1, dtype=torch.float32) + # einsum(attn, v, "b h g n, b n h d -> b h g d") + o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel) + # rearrange "b h g d -> b (h g) d" + o = o.reshape(batch_size, num_q_heads, head_dim) + return o.to(original_dtype), topk_idx diff --git a/build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py b/build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..6fefac38e0d131d2b37dcb392e405d71a93ace61 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/naive/topk_sparse.py @@ -0,0 +1,88 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], # [num_q_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, +) -> torch.Tensor: + """ + Naive implementation of sparse attention with GQA group sharing. + Each GQA group (all q heads sharing the same kv head) uses the same topk_idx. + """ + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + + original_dtype = q.dtype + batch_size, num_q_heads, head_dim = q.shape + max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape + gqa_group_size = num_q_heads // num_kv_heads + # Output tensor + o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device) + + for b in range(batch_size): + seq_len = seq_lens[b].item() + sid = slot_ids[b].item() % max_slots + + for kh in range(num_kv_heads): + # Get topk indices for this (kv_head, batch) + block_indices = topk_idx[kh, b, :].tolist() + + # Collect selected K and V blocks + selected_k_blocks = [] + selected_v_blocks = [] + for block_idx in block_indices: + if block_idx < 0: + continue # Invalid index + start = block_idx * block_size + end = min(start + block_size, seq_len) + if start >= seq_len: + continue + # K: [block_len, head_dim] + k_block = kv_cache[sid, 0, start:end, kh, :] + # V: [block_len, head_dim] + v_block = kv_cache[sid, 1, start:end, kh, :] + selected_k_blocks.append(k_block) + selected_v_blocks.append(v_block) + + if len(selected_k_blocks) == 0: + continue + + # Concatenate selected blocks: [total_selected_len, head_dim] + k_selected = torch.cat(selected_k_blocks, dim=0) + v_selected = torch.cat(selected_v_blocks, dim=0) + + # Compute attention for all q heads in this GQA group + for g in range(gqa_group_size): + qh = kh * gqa_group_size + g + # q_vec: [head_dim] + q_vec = q[b, qh, :].float() + + # Compute attention scores: [total_selected_len] + scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale + + # Add sink to softmax normalization if present + if sink is not None: + sink_vec = sink[qh, :].float() + qsink = torch.dot(q_vec, sink_vec) * sm_scale + # Concatenate sink score with regular scores for softmax + scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0) + attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1) + # Remove sink weight (only used for normalization) + attn_weights = attn_weights_with_sink[1:] + else: + attn_weights = torch.softmax(scores, dim=-1) + + # Compute output: [head_dim] + o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected) + + return o diff --git a/build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py b/build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b6b69345bf3ffe07f6ccac8c570ee17e3b5e6c69 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/prefill/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +from .topk_sparse import flash_prefill_with_gqa_share_sparse + +__all__ = ["flash_prefill_with_gqa_share_sparse"] diff --git a/build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py b/build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..9bf83d65eac260f8e71180ea6f28970e8db582d7 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py @@ -0,0 +1,371 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from ..common.utils import get_cu_seqblocks, robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]), + "BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]), + "BLOCK_SIZE_H": lambda args: triton.next_power_of_2( + max( + 16 // args["BLOCK_SIZE_Q"], + triton.next_power_of_2(args["gqa_group_size"]), + ) + ), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"], + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + # Configs that fail to compile on the target arch are skipped, so widening + # the num_warps x num_stages grid only adds candidates, never a bad kernel. + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in (2, 4, 8) + for ns in (2, 3, 4) + ], + key=[ + "BLOCK_SIZE_Q", + "BLOCK_SIZE_K", + "qk_head_dim", + "v_head_dim", + "gqa_group_size", + ], +) +@triton.jit +def _gqa_share_sparse_fwd_kernel( + q_ptr, # Q: n x h x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + sink_ptr, # Sink: h x d + t_ptr, # topk_idx: kh x n x k + o_ptr, # O: n x h x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + # seqlens + cu_seqlens_q, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + # shape + max_slots, + num_kv_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + max_topk, + # q loop num + num_q_loop, + # sm_scale + sm_scale, + # stride + stride_qn, + stride_qh, + stride_qd, + stride_ks, + stride_kh, + stride_kd, + stride_vs, + stride_vh, + stride_vd, + stride_sh, + stride_sd, + stride_th, + stride_tn, + stride_tk, + stride_on, + stride_oh, + stride_od, + stride_r2t_b, + # META parameters + BLOCK_SIZE_Q: tl.constexpr, # q block size + BLOCK_SIZE_K: tl.constexpr, # k block size + BLOCK_SIZE_KD: tl.constexpr, + BLOCK_SIZE_VD: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + BLOCK_SIZE_QH: tl.constexpr, + # has sink + HAS_SINK: tl.constexpr, + USE_TMA: tl.constexpr, + IS_FP8: tl.constexpr, +): + sm_scale_log2e = sm_scale * 1.4426950409 + # get batch id and head id + pid_q = tl.program_id(0) + pid_kh = tl.program_id(1) + pid_b = tl.program_id(2) + pid_h = pid_kh * gqa_group_size + # get q k start and len after rmpad + q_start = tl.load(cu_seqlens_q + pid_b) + q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start + q_block_start = tl.load(cu_seqblocks_q + pid_b) + q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start + seq_len = tl.load(seq_lens + pid_b) + prefix_len = tl.load(prefix_lens + pid_b) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # safety against negative + if pid_q * num_q_loop >= q_block_len: + return + real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop) + if HAS_SINK: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sh, + shape=(gqa_group_size, qk_head_dim), + strides=(stride_sh, stride_sd), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + # offsets for paged K/V load + off_n = tl.arange(0, BLOCK_SIZE_K) + off_kd = tl.arange(0, BLOCK_SIZE_KD) + off_vd = tl.arange(0, BLOCK_SIZE_VD) + kd_mask = off_kd < qk_head_dim + vd_mask = off_vd < v_head_dim + for j in range(real_q_loop): + pid_q_j = pid_q * num_q_loop + j + # init topk idx pointer + t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th + # we assume that the topk_idx is right padded with -1 + off_t = tl.arange(0, BLOCK_SIZE_T) + topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + q_start * stride_qn + pid_h * stride_qh, + shape=(q_len, gqa_group_size, qk_head_dim), + strides=(stride_qn, stride_qh, stride_qd), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(2, 1, 0), + ) + # load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D] + q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero") + # init statistics + off_q_k = ( + tl.arange(0, BLOCK_SIZE_Q)[:, None] + + pid_q_j * BLOCK_SIZE_Q + + prefix_len + - tl.arange(0, BLOCK_SIZE_K)[None, :] + ) + if HAS_SINK: + m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + qsink = ( + tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e + ) # (BLOCK_SIZE_Q, BLOCK_SIZE_H) + m_i += qsink + lse_i += qsink + m_i = tl.reshape(m_i, BLOCK_SIZE_QH) + lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH) + else: + m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32) + q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD) + # sparse attention + for i in range(real_topk): + # get current block start index (absolute K position) + c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K + t_ptr_j = t_ptr_j + stride_tk + # paged load K via req_to_token: pos -> slot -> k_cache + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot) + k = tl.load( + k_cache_ptr + + slots[None, :] * stride_ks + + pid_kh * stride_kh + + off_kd[:, None] * stride_kd, + mask=kd_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 main K cache is unit-scaled; widen to the Q compute dtype + # before the tl.dot (compiled out when the cache is bf16). + k = k.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32) + # causal mask + qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf")) + qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K) + # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K] + # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K] + qk += tl.dot(q, k) * sm_scale_log2e + # K boundary mask: positions beyond seq_len contribute -inf + qk += tl.where(pos_mask[None, :], 0, float("-inf")) + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # paged load V + v = tl.load( + v_cache_ptr + + slots[:, None] * stride_vs + + pid_kh * stride_vh + + off_vd[None, :] * stride_vd, + mask=pos_mask[:, None] & vd_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V so `p.to(v.dtype)` casts P to the compute dtype rather + # than to fp8 (which would wreck attention-weight precision). + v = v.to(q.dtype) + p = p.to(v.dtype) + acc_o += tl.dot(p, v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + # final scale + acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None] + # save output + acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD) + o_ptrs = tl.make_block_ptr( + base=o_ptr + q_start * stride_on + pid_h * stride_oh, + shape=(q_len, gqa_group_size, v_head_dim), + strides=(stride_on, stride_oh, stride_od), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD), + order=(2, 1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2)) + + +@torch.no_grad() +def flash_prefill_with_gqa_share_sparse( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + sink: Optional[torch.Tensor], + req_to_token: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size_q: int, + block_size_k: int, + cu_seqlens: torch.Tensor, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + max_seqlen_q: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + cu_seqblocks_q: Optional[torch.Tensor] = None, + max_seqblock_q: Optional[int] = None, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled) + # under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load. + # Keep CUDA's pre-existing sparse prefill dtype contract unchanged by + # accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + assert block_size_q in {1, 2, 4, 8, 16, 32, 64} + assert block_size_k in {16, 32, 64, 128} + # shape + total_q, num_q_heads, qk_head_dim = q.shape + max_slots, num_k_heads, _ = k_cache.shape + _, num_v_heads, v_head_dim = v_cache.shape + batch_size = cu_seqlens.shape[0] - 1 + topk = topk_idx.shape[-1] + assert topk_idx.shape[0] == num_k_heads + # gqa + assert num_k_heads == num_v_heads + assert num_q_heads % num_k_heads == 0 + gqa_group_size = num_q_heads // num_k_heads + assert gqa_group_size * block_size_q <= 128 + if sm_scale is None: + sm_scale = qk_head_dim**-0.5 + if cu_seqblocks_q is None or max_seqblock_q is None: + cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks( + cu_seqlens, max_seqlen_q, block_size_q, block_size_k + ) + # output tensor + o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype) + # launch kernel + num_q_loop = ( + max_seqblock_q // 131072 + 1 + ) # calculate multiple queries in one kernel if seqlence length is too long + BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q) + BLOCK_SIZE_K = triton.next_power_of_2(block_size_k) + grid = ( + triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop), + num_k_heads, + batch_size, + ) + _gqa_share_sparse_fwd_kernel[grid]( + q, + k_cache, + v_cache, + sink, + topk_idx, + o, + req_to_token, + cu_seqlens, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_k_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + topk, + num_q_loop, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + req_to_token.stride(0), + BLOCK_SIZE_Q=BLOCK_SIZE_Q, + BLOCK_SIZE_K=BLOCK_SIZE_K, + USE_TMA=use_tma, + IS_FP8=is_fp8, + ) + return o diff --git a/build/torch211-cxx11-cu130-x86_64-linux/quantize.py b/build/torch211-cxx11-cu130-x86_64-linux/quantize.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2675ac49d03e5599e9559187efecdf63bb6387 --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/quantize.py @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: Apache-2.0 +"""NVFP4 helper API compatible with MiniMaxAI/msa. + +The quantization entrypoints use Transformer Engine when it is installed. The +layout helpers and reference dequantizer are pure PyTorch and are included so +callers can prepare and validate NVFP4 metadata with the same public API names +as the upstream MiniMaxAI package. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Tuple + +import torch + + +NVFP4_BLOCK_SIZE = 16 +NVFP4_FP4_MAX = 6.0 +NVFP4_FP8_E4M3_MAX = 448.0 + + +@dataclass(frozen=True) +class Nvfp4QuantizedTensor: + data: torch.Tensor + scale_128x4: torch.Tensor + global_scale: torch.Tensor + logical_scale_shape: Tuple[int, int] + original_shape: Tuple[int, ...] + + +def _round_up(x: int, multiple: int) -> int: + return ((int(x) + multiple - 1) // multiple) * multiple + + +def nvfp4_scale_128x4_offset( + row: torch.Tensor, + col: torch.Tensor, + scale_cols: int, +) -> torch.Tensor: + tiles_n = _round_up(scale_cols, 4) // 4 + tile_m = row // 128 + tile_n = col // 4 + outer = row % 128 + inner = col % 4 + return ( + (tile_m * tiles_n + tile_n) * 512 + + (outer % 32) * 16 + + (outer // 32) * 4 + + inner + ) + + +def swizzle_nvfp4_scale_to_128x4( + scale: torch.Tensor, + *, + rows: int, + cols: int, +) -> torch.Tensor: + if scale.ndim != 2: + raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}") + + rows = int(rows) + cols = int(cols) + padded_rows = _round_up(rows, 128) + padded_cols = _round_up(cols, 4) + if scale.shape[0] < rows or scale.shape[1] < cols: + raise ValueError( + "scale is smaller than the requested logical shape: " + f"got {tuple(scale.shape)}, need at least {(rows, cols)}" + ) + + logical = scale[:rows, :cols].contiguous() + if logical.shape != (padded_rows, padded_cols): + logical = torch.nn.functional.pad( + logical.to(torch.float32), + (0, padded_cols - cols, 0, padded_rows - rows), + ).to(scale.dtype) + swizzled = torch.empty_like(logical) + + row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None] + col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1) + swizzled.reshape(-1)[offset] = logical.reshape(-1) + return swizzled + + +def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor: + return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX) + + +def _import_te_nvfp4_quantizer(): + try: + from transformer_engine.pytorch.tensor import NVFP4Quantizer + except Exception as exc: + raise RuntimeError( + "Transformer Engine NVFP4 quantization is unavailable. Install a " + "Transformer Engine build with NVFP4 PyTorch support to use " + "quantize_bf16_to_nvfp4_128x4." + ) from exc + return NVFP4Quantizer + + +def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor: + if not x.is_cuda: + raise ValueError("NVFP4 quantization requires a CUDA tensor") + if x.dtype not in (torch.bfloat16, torch.float16): + raise TypeError(f"x must be bf16 or fp16, got {x.dtype}") + if x.ndim < 2: + raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}") + if x.shape[-1] % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}" + ) + + rows = 1 + for dim in x.shape[:-1]: + rows *= int(dim) + if rows % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + "flattened row dimension must be divisible by " + f"{NVFP4_BLOCK_SIZE}, got {rows}" + ) + + NVFP4Quantizer = _import_te_nvfp4_quantizer() + quantizer = NVFP4Quantizer(rowwise=True, columnwise=False) + qx = quantizer.quantize(x.contiguous()) + meta = qx.get_metadata() + + data = meta["rowwise_data"] + if data.dtype is not torch.uint8: + data = data.view(torch.uint8) + logical_scale = meta["rowwise_scale_inv"] + amax = meta["amax_rowwise"] + scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE + scale_128x4 = swizzle_nvfp4_scale_to_128x4( + logical_scale, + rows=rows, + cols=scale_cols, + ) + global_scale = nvfp4_global_scale_from_amax(amax).contiguous() + + return Nvfp4QuantizedTensor( + data=data, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, scale_cols), + original_shape=tuple(int(v) for v in x.shape), + ) + + +def quantize_kv_bf16_to_nvfp4_128x4( + k: torch.Tensor, + v: torch.Tensor, +) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]: + return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v) + + +def dequantize_nvfp4_128x4_to_bf16( + qx: Nvfp4QuantizedTensor, + *, + include_global_scale: bool = True, +) -> torch.Tensor: + data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8) + if data.shape[-1] * 2 != qx.original_shape[-1]: + raise ValueError( + "packed data last dimension does not match original shape: " + f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical" + ) + + rows, scale_cols = qx.logical_scale_shape + logical_dim = int(qx.original_shape[-1]) + if scale_cols * NVFP4_BLOCK_SIZE != logical_dim: + raise ValueError( + "logical scale columns do not match original last dimension: " + f"{scale_cols} scale cols vs dim {logical_dim}" + ) + + fp4_lut = torch.tensor( + [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + -0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, + ], + dtype=torch.float32, + device=data.device, + ) + packed = data.reshape(rows, logical_dim // 2) + lo = packed & 0x0F + hi = packed >> 4 + values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device) + values[:, 0::2] = fp4_lut[lo.long()] + values[:, 1::2] = fp4_lut[hi.long()] + + row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None] + col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, scale_cols) + scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols) + scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32) + scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1) + out = values * scale + if include_global_scale: + out = out * qx.global_scale.reshape(-1)[0].to(torch.float32) + return out.reshape(qx.original_shape).to(torch.bfloat16) + + +__all__ = [ + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +] diff --git a/build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py b/build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..177f864da993328cc460b8de391c6091f3be0f9d --- /dev/null +++ b/build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Sparse-index helpers compatible with MiniMaxAI/msa public names.""" + +from __future__ import annotations + +from typing import Optional + +import torch + + +def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None: + if cu_seqlens.dtype != torch.int32: + raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}") + if cu_seqlens.ndim != 1: + raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}") + if cu_seqlens.numel() < 1: + raise ValueError(f"{name} must have at least one element") + if not cu_seqlens.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor: + seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1] + return (seqlens_k + kv_block_size - 1) // kv_block_size + + +def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]: + rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist() + batch = len(rows_per_batch_cpu) + max_rows = max(rows_per_batch_cpu, default=0) + row_dtype = ( + torch.int32 + if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max + else torch.int64 + ) + row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype) + row_linear = 0 + for kv_block_idx in range(max_rows): + for batch_idx, row_count in enumerate(rows_per_batch_cpu): + if kv_block_idx < row_count: + row_map_cpu[batch_idx, kv_block_idx] = row_linear + row_linear += 1 + return row_map_cpu.to(rows_per_batch.device), row_linear + + +def build_k2q_csr_torch_reference( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if kv_block_size <= 0: + raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}") + if q2k_indices.dtype != torch.int32: + raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}") + if q2k_indices.ndim != 3: + raise ValueError( + "q2k_indices must have shape [head_kv, total_q, topK], " + f"got {tuple(q2k_indices.shape)}" + ) + _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q") + _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k") + if cu_seqlens_q.shape != cu_seqlens_k.shape: + raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]") + if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device: + raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device") + + head_kv, total_q, topk = q2k_indices.shape + if total_q != int(cu_seqlens_q[-1].item()): + raise ValueError( + f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] " + f"({int(cu_seqlens_q[-1].item())})" + ) + + rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size) + row_map, total_rows = _build_packed_row_map(rows_per_batch) + nnz_upper_bound = total_q * topk + + k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device) + k2q_q_indices = torch.full( + (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device + ) + if total_rows == 0 or total_q == 0 or topk == 0: + return k2q_row_ptr, k2q_q_indices + + counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device) + total_entries = total_q * topk + row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64 + row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device) + q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device) + valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device) + + for batch_idx in range(cu_seqlens_q.numel() - 1): + q_start = int(cu_seqlens_q[batch_idx].item()) + q_end = int(cu_seqlens_q[batch_idx + 1].item()) + if q_end <= q_start: + continue + q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device) + q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1) + kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1) + valid = kv_local >= 0 + safe_kv = kv_local.clamp_min(0).long() + rows = row_map[batch_idx, safe_kv].to(row_dtype) + start = q_start * topk + end = q_end * topk + row_all[:, start:end] = rows + q_all[:, start:end] = q_local[None, :].expand(head_kv, -1) + valid_all[:, start:end] = valid & (rows >= 0) + counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32)) + + k2q_row_ptr[:, 1:] = counts.cumsum(dim=1) + offsets = torch.zeros_like(counts) + offsets[:, 1:] = counts[:, :-1].cumsum(dim=1) + cursor = torch.zeros_like(counts) + for h in range(head_kv): + for entry in range(total_entries): + if not bool(valid_all[h, entry]): + continue + row = int(row_all[h, entry].item()) + pos = int(offsets[h, row].item() + cursor[h, row].item()) + k2q_q_indices[h, pos] = q_all[h, entry] + cursor[h, row] += 1 + + return k2q_row_ptr, k2q_q_indices + + +def build_k2q_csr( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, + *, + total_k: Optional[int] = None, + max_seqlen_k: Optional[int] = None, + max_seqlen_q: Optional[int] = None, + total_rows: Optional[int] = None, + qhead_per_kv: int = 1, + return_schedule: bool = False, +): + if return_schedule: + raise NotImplementedError( + "Blackwell build_k2q_csr currently returns CSR tensors only; " + "SM100 schedule objects are not part of this package." + ) + if total_k is None: + raise ValueError("build_k2q_csr requires total_k from k.shape[0]") + return build_k2q_csr_torch_reference( + q2k_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(kv_block_size), + ) + + +class SparseK2qCsrBuilderSm100: + """Compatibility placeholder for the upstream SM100 class name. + + The name is exported so code can feature-detect it, but schedule-producing + SM100 behavior is intentionally not claimed for this Blackwell package. + """ + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def build(self, *args, **kwargs): + return build_k2q_csr(*args, **kwargs) + + +__all__ = [ + "build_k2q_csr", + "build_k2q_csr_torch_reference", + "SparseK2qCsrBuilderSm100", +] diff --git a/build/torch212-cxx11-cu130-x86_64-linux/__init__.py b/build/torch212-cxx11-cu130-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a666afa7b37c9d5448b9bb04d573dcfe4e45750 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/__init__.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention. + +This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on +DGX Spark / GB10 / SM121. It exposes the +MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime. + +Implementation status: + * native CUDA helper: score -> top-k sparse block ids; + * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape; + * native CUDA FP4 block-score indexer; + * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path; + * Blackwell-validated sparse prefill path; + * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers, + and FP4 block-score helpers. + +The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell +extension path. The public API is Tensor-oriented and independent from +FlashRT's serving runtime. + +Public API +---------- +Lightning indexer (q.k blockmax score -> top-k block select), paged KV: + flash_decode_with_topk_idx (single-token decode, split-K) + +Block-sparse GQA attention (consumes block_indices from the indexer): + flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks) + sparse_atten_func (official CSR prefill wrapper) + sparse_decode_atten_func (official paged decode wrapper) + +PyTorch naive references (for correctness checks): + naive_flash_decode_with_gqa_share_sparse (attention given topk_idx) + naive_flash_decode_with_topk_idx (joint indexer + attention) + +All Triton entrypoints expect a paged KV cache: + k_cache / v_cache : [max_slots, num_kv_heads, head_dim] + req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot) + slot_ids : [batch] (per-request row in req_to_token) + topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed, + -1 right-padding (the M3 indexer contract). +See VENDOR_NOTES.md for full shape/dtype/layout contracts. +""" + +from .common.utils import get_cu_seqblocks, robust_allocator +from .decode.flash_with_topk_idx import flash_decode_with_topk_idx +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .interface import ( + SparseDecodePagedAttentionWrapper, + fp4_indexer_block_scores, + sparse_atten_func, + sparse_atten_nvfp4_kv_func, + sparse_decode_atten_func, +) +from ._native import ( + has_native_ops, + native_nvfp4_dequant_swizzled_to_bf16, + native_topk_from_scores, +) +from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx +from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse +from .quantize import ( + Nvfp4QuantizedTensor, + dequantize_nvfp4_128x4_to_bf16, + nvfp4_global_scale_from_amax, + quantize_bf16_to_nvfp4_128x4, + quantize_kv_bf16_to_nvfp4_128x4, + swizzle_nvfp4_scale_to_128x4, +) +from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr +from .api_status import ( + OFFICIAL_API_STATUS, + OFFICIAL_MINIMAX_MSA_FUNCTIONS, + V1_AVAILABLE_FUNCTIONS, + available_functions, + official_api_status, + official_minimax_msa_functions, + unsupported_official_functions, +) + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "naive_flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "get_cu_seqblocks", + "robust_allocator", + "OFFICIAL_API_STATUS", + "OFFICIAL_MINIMAX_MSA_FUNCTIONS", + "V1_AVAILABLE_FUNCTIONS", + "available_functions", + "official_api_status", + "official_minimax_msa_functions", + "unsupported_official_functions", +] diff --git a/build/torch212-cxx11-cu130-x86_64-linux/_compat.py b/build/torch212-cxx11-cu130-x86_64-linux/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..da445342268947ab3b1c5bfd16e95dc4705c7abe --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/_compat.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton +# kernels. The upstream SGLang modules pull two tiny things from the framework: +# +# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path) +# from sglang.srt.environ import envs -> a feature flag for an optional +# JIT-CUDA radix top-k fast path +# +# Both are replaced here so the kernels import with only torch + triton. The +# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel +# radix select), which is what we want on sm_121 (GB10). + + +def is_hip() -> bool: + """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121). + + Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV + widening branch (HIP-only). With this False, the CUDA dtype contract + (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA. + """ + try: + import torch + + return bool(getattr(torch.version, "hip", None)) + except Exception: + return False + + +class _Flag: + """Minimal stand-in for an sglang.srt.environ boolean flag (always off).""" + + def __init__(self, default: bool = False): + self._default = default + + def get(self) -> bool: + return self._default + + +class _Envs: + """Stub for `sglang.srt.environ.envs`. + + Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode + indexer. Keeping it False routes top-k selection through the 2-stage Triton + fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the + decode indexer stays pure-Triton. + """ + + SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False) + + +envs = _Envs() diff --git a/build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so b/build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..260614efc95bdb36100017b12a582e61336afc00 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ea862050a19dcc4b0c7cbfd3b519112f1b4e054cfc182a14f92876b4ae5b9f7 +size 735888 diff --git a/build/torch212-cxx11-cu130-x86_64-linux/_native.py b/build/torch212-cxx11-cu130-x86_64-linux/_native.py new file mode 100644 index 0000000000000000000000000000000000000000..dd54c53374469c8424d0ca6e3f25ea534005c8d5 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/_native.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional native CUDA helpers for the FlashRT Blackwell MSA package.""" + +from __future__ import annotations + +import torch + +try: + from ._ops import add_op_namespace_prefix, ops + + _HAS_NATIVE_OPS = True +except Exception: + add_op_namespace_prefix = None + ops = None + _HAS_NATIVE_OPS = False + + +def has_native_ops() -> bool: + return _HAS_NATIVE_OPS + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores")) + def _msa_topk_from_scores_fake( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, + topk_idx: torch.Tensor, + ) -> None: + if score.dim() != 3: + raise RuntimeError("score must have shape (heads, batch, max_blocks)") + heads, batch, _ = score.shape + if seq_lens.shape != (batch,): + raise RuntimeError("seq_lens must have shape (batch,)") + if topk_idx.shape != (heads, batch, topk): + raise RuntimeError("topk_idx must have shape (heads, batch, topk)") + return None + + +def native_topk_from_scores( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, +) -> torch.Tensor: + """Return top-k sparse block ids from a score tensor.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if not score.is_contiguous(): + score = score.contiguous() + if not seq_lens.is_contiguous(): + seq_lens = seq_lens.contiguous() + out = torch.empty( + (score.shape[0], score.shape[1], int(topk)), + device=score.device, + dtype=torch.int32, + ) + ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged") + ) + def _msa_decode_sparse_attn_mma_paged_fake( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size: int, + sm_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_decode_mma_supported( + q: torch.Tensor, + k_cache: torch.Tensor, + *, + block_size: int, +) -> bool: + """Whether the tensor-core decode kernel applies to these inputs. + + The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA + group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64. + """ + + if not _HAS_NATIVE_OPS: + return False + if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16: + return False + if q.dim() != 3 or k_cache.dim() != 3: + return False + head_dim = int(q.shape[-1]) + num_q_heads = int(q.shape[1]) + num_kv_heads = int(k_cache.shape[1]) + if head_dim != 128 or num_kv_heads <= 0: + return False + if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16: + return False + if int(block_size) % 64 != 0: + return False + return True + + +def native_decode_sparse_attn_mma_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + block_size: int, + topk_idx: torch.Tensor, + sm_scale: float, +) -> torch.Tensor: + """Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + out = torch.empty( + (int(q.shape[0]), int(q.shape[1]), int(q.shape[2])), + device=q.device, + dtype=q.dtype, + ) + ops.msa_decode_sparse_attn_mma_paged( + q.contiguous(), + k_cache.contiguous(), + v_cache.contiguous(), + req_to_token.contiguous(), + seq_lens.contiguous(), + slot_ids.contiguous(), + topk_idx.contiguous(), + int(block_size), + float(sm_scale), + out, + ) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_indexer_block_scores") + ) + def _msa_indexer_block_scores_fake( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + causal: int, + scores: torch.Tensor, + ) -> None: + return None + + +def native_indexer_block_scores( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + *, + max_blocks: int, + causal: bool, +) -> torch.Tensor: + """Block-max QK indexer scores. q/k_pages are bf16 (already dequantized). + + Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible). + """ + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + total_q = int(q.shape[0]) + hq = int(q.shape[1]) + scores = torch.full( + (hq, int(max_blocks), total_q), + float("-inf"), + dtype=torch.float32, + device=q.device, + ) + ops.msa_indexer_block_scores( + q.contiguous(), + k_pages.contiguous(), + batch_of_q.contiguous(), + cu_q.contiguous(), + cu_k.contiguous(), + cu_pages.contiguous(), + kv_indices.contiguous(), + 1 if causal else 0, + scores, + ) + return scores + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16") + ) + def _msa_nvfp4_dequant_swizzled_to_bf16_fake( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_nvfp4_dequant_swizzled_to_bf16( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor | float, +) -> torch.Tensor: + """Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if packed.dtype is not torch.uint8: + raise TypeError("packed must be torch.uint8") + if scale_128x4.dtype is not torch.uint8: + raise TypeError("scale_128x4 must be torch.uint8") + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16) + if isinstance(global_scale, torch.Tensor): + global_scale_value = float(global_scale.reshape(-1)[0].item()) + else: + global_scale_value = float(global_scale) + ops.msa_nvfp4_dequant_swizzled_to_bf16( + packed.contiguous(), + scale_128x4.contiguous(), + global_scale_value, + out, + ) + return out + + +__all__ = [ + "has_native_ops", + "native_topk_from_scores", + "native_decode_mma_supported", + "native_decode_sparse_attn_mma_paged", + "native_indexer_block_scores", + "native_nvfp4_dequant_swizzled_to_bf16", +] diff --git a/build/torch212-cxx11-cu130-x86_64-linux/_ops.py b/build/torch212-cxx11-cu130-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..41e39102ea6d028564a7e10c3b7004e7c40dae7b --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _minimaxai_msa_blackwell_cuda_25b0067 +ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}" diff --git a/build/torch212-cxx11-cu130-x86_64-linux/api_status.py b/build/torch212-cxx11-cu130-x86_64-linux/api_status.py new file mode 100644 index 0000000000000000000000000000000000000000..3dd4d744b49f1bc7acd6b3ba2a2e2c5f7b591252 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/api_status.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Public API status for the FlashRT MiniMax MSA Blackwell package.""" + +from __future__ import annotations + +from types import MappingProxyType + + +V1_AVAILABLE_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "naive_flash_decode_with_gqa_share_sparse", + "get_cu_seqblocks", + "robust_allocator", +) + +OFFICIAL_MINIMAX_MSA_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +) + +_OFFICIAL_API_STATUS = { + "sparse_atten_func": { + "status": "available", + "target": "blackwell-prefill", + "reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.", + }, + "sparse_atten_nvfp4_kv_func": { + "status": "available", + "target": "blackwell-prefill-nvfp4", + "reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.", + }, + "sparse_decode_atten_func": { + "status": "available", + "target": "v1", + "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.", + }, + "SparseDecodePagedAttentionWrapper": { + "status": "available", + "target": "v1", + "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.", + }, + "fp4_indexer_block_scores": { + "status": "available", + "target": "blackwell-fp4-indexer", + "reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.", + }, + "build_k2q_csr": { + "status": "available", + "target": "v1", + "reason": "CSR construction helper is available for the official prefill API.", + }, + "SparseK2qCsrBuilderSm100": { + "status": "available", + "target": "v1", + "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.", + }, + "Nvfp4QuantizedTensor": { + "status": "available", + "target": "v1", + "reason": "NVFP4 metadata dataclass is available.", + }, + "quantize_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "quantize_kv_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "dequantize_nvfp4_128x4_to_bf16": { + "status": "available", + "target": "v1", + "reason": "Reference dequant helper is available.", + }, + "swizzle_nvfp4_scale_to_128x4": { + "status": "available", + "target": "v1", + "reason": "Scale swizzle helper is available.", + }, + "nvfp4_global_scale_from_amax": { + "status": "available", + "target": "v1", + "reason": "Global scale helper is available.", + }, +} + +OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS) + + +def available_functions() -> tuple[str, ...]: + """Return the functions/classes intentionally available in v1.""" + + return V1_AVAILABLE_FUNCTIONS + + +def official_minimax_msa_functions() -> tuple[str, ...]: + """Return the upstream MiniMaxAI/msa public function/class names tracked here.""" + + return OFFICIAL_MINIMAX_MSA_FUNCTIONS + + +def official_api_status() -> dict[str, dict[str, str]]: + """Return a copy of the official API compatibility status table.""" + + return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()} + + +def unsupported_official_functions() -> tuple[str, ...]: + """Return official MiniMaxAI/msa names that are not exported by this v1 package.""" + + return tuple( + name + for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS + if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"} + ) diff --git a/build/torch212-cxx11-cu130-x86_64-linux/common/__init__.py b/build/torch212-cxx11-cu130-x86_64-linux/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/common/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch212-cxx11-cu130-x86_64-linux/common/utils.py b/build/torch212-cxx11-cu130-x86_64-linux/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..81b7816b719973729312c0d84b36f9f50579f105 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/common/utils.py @@ -0,0 +1,144 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +import functools +from collections import deque +from typing import Any, Callable, Tuple + +import torch +import triton.language as tl + +_tma_keep_alive_buf = deque(maxlen=200) + +try: + make_tensor_descriptor = tl.make_tensor_descriptor +except Exception: + make_tensor_descriptor = tl._experimental_make_tensor_descriptor + + +def robust_allocator(size: int, alignment: int, stream: int = None): + """Allocator for Triton TMA descriptors. + + We keep reference in deque to prevent GC from collecting the buffer. + """ + tensor = torch.empty(size, device="cuda", dtype=torch.uint8) + _tma_keep_alive_buf.append(tensor) + return tensor + + +def tensor_cache(maxsize: int = 8): + """ + Cache function results using identity comparison. + Zero-overhead cache hit: no hash, no DtoH, just pointer comparison. + + Args: + maxsize: Maximum number of cached entries. Supports multi-GPU scenarios + where different devices have different tensor arguments. + """ + + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + # LRU-style cache: list of (args, kwargs, result) tuples + # Most recently used at the end + _cache: list = [] + + def _args_match(args: tuple, cached_args: tuple) -> bool: + if len(args) != len(cached_args): + return False + for i in range(len(args)): + if args[i] is not cached_args[i]: + return False + return True + + def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool: + if not kwargs and not cached_kwargs: + return True + if kwargs.keys() != cached_kwargs.keys(): + return False + for k, v in kwargs.items(): + if v is not cached_kwargs[k]: + return False + return True + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + # Search cache (most recent first for better hit rate) + for i in range(len(_cache) - 1, -1, -1): + cached_args, cached_kwargs, cached_result = _cache[i] + if _args_match(args, cached_args) and _kwargs_match( + kwargs, cached_kwargs + ): + # Move to end (most recently used) + if i != len(_cache) - 1: + _cache.append(_cache.pop(i)) + return cached_result + + # Cache miss + result = fn(*args, **kwargs) + + # Add to cache + if len(_cache) >= maxsize: + _cache.pop(0) # Remove oldest + _cache.append((args, kwargs, result)) + return result + + # Expose cache for manual clearing if needed + wrapper.cache_clear = lambda: _cache.clear() + wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize} + + return wrapper + + return decorator + + +@tensor_cache(maxsize=8) +def get_cu_seqblocks( + cu_seqlens: torch.Tensor, + max_seqlen: int, + block_size_q: int, + block_size_k: int, +) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]: + """Compute cumulative sequence block indices for blocked sparse attention. + + Converts token-level cumulative sequence lengths to block-level indices, + which are needed for block-sparse attention kernels. + + Note: + Results are cached (maxsize=8) based on input arguments. Repeated calls + with the same cu_seqlens, max_seqlen, and block sizes will return cached + results without recomputation. + + Args: + cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32. + max_seqlen: Maximum sequence length in the batch. + block_size_q: Query block size. + block_size_k: Key-value block size. + + Returns: + A tuple of 6 values: + - cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1] + - max_seqblock_q: Maximum number of query blocks per sequence. + - all_seqblock_q: Total number of query blocks across all sequences. + - cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1] + - max_seqblock_k: Maximum number of key blocks per sequence. + - all_seqblock_k: Total number of key blocks across all sequences. + """ + cu_seqblocks_q = torch.zeros_like(cu_seqlens) + cu_seqblocks_k = torch.zeros_like(cu_seqlens) + seq_lens = torch.diff(cu_seqlens) + seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q + seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k + max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q + max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k + cu_seqblocks_q[1:] = seqblocks_q + cu_seqblocks_k[1:] = seqblocks_k + cu_seqblocks_q.cumsum_(0) + cu_seqblocks_k.cumsum_(0) + all_seqblock_q = seqblocks_q.sum().item() + all_seqblock_k = seqblocks_k.sum().item() + return ( + cu_seqblocks_q, + max_seqblock_q, + all_seqblock_q, + cu_seqblocks_k, + max_seqblock_k, + all_seqblock_k, + ) diff --git a/build/torch212-cxx11-cu130-x86_64-linux/decode/__init__.py b/build/torch212-cxx11-cu130-x86_64-linux/decode/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/decode/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch212-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py b/build/torch212-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..8f56bdba2908e8af036785eae158338758555ee5 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/decode/flash_with_topk_idx.py @@ -0,0 +1,1150 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import envs # vendored: was `from sglang.srt.environ import envs` + +from ..common.utils import robust_allocator +from .._native import has_native_ops, native_topk_from_scores + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "BATCH_SIZE_BUCKET", + "gqa_group_size", + "head_dim", + "block_size", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_kernel( + q_ptr, # Q: b x qh x d + k_cache_ptr, # K paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_r2t_b, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks <= topk: + return + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # prefetch first iteration's slots + r2t_base = req_to_token_ptr + sid * stride_r2t_b + prefetch_pos = chunk_start + off_n + prefetch_mask = prefetch_pos < seq_len + prefetched_slots = tl.load( + r2t_base + prefetch_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + # score-only: compute block scores without loading V + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos_mask = prefetch_mask + slots = prefetched_slots + # prefetch next iteration's slots + next_i = i + BLOCK_SIZE_N + if next_i < chunk_end: + next_pos = next_i + off_n + prefetch_mask = next_pos < seq_len + prefetched_slots = tl.load( + r2t_base + next_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + curr_block_idx = i // block_size + off_bpk + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)) + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "gqa_group_size", + "head_dim", + "block_size", + "HAS_SINK", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_attn_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + o_ptr, # O: c x b x qh x d + lse_ptr, # lse: c x b x qh + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + qsink = ( + tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e + ) # (BLOCK_SIZE_H,) + m_i = qsink + l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # full attention or causal attention + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos = i + off_n + pos_mask = pos < seq_len + # resolve slot per position via req_to_token + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + # current block indices: [BLOCKS_PER_K_BLOCK] + curr_block_idx = i // block_size + off_bpk + # Combined condition with single nested tl.where: + # - local_blocks (1e29) takes priority over init_blocks (1e30) + # - When init_blocks=0: curr_block_idx < 0 is always False + # - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + # Apply in one fused operation: local > init > original + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks > topk: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + else: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + # max-of-max == max(qk), avoids re-scanning qk + m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + m_i = m_ij + l_i = l_i * acc_o_scale + l_ij + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + # final scale + acc_o = acc_o / l_i[:, None] + # compute lse in log2 scale for merge kernel + lse_i = m_i + tl.log2(l_i) + # save output + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + # save lse (in log2 scale, will be used directly in merge kernel) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4] + ], + key=["BLOCK_SIZE_D"], + restore_value=["o_ptr"], +) +@triton.jit +def _merge_attn_out_kernel( + o_ptr, + lse_ptr, + # shape + seq_lens, + head_dim, + block_size: tl.constexpr, + # stride + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_KV_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + # recompute chunk layout (must match _decode_score_attn_kernel) + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks) + # ptrs + off_c = tl.arange(0, NUM_KV_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(valid_chunks, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + # load o and lse + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf")) + # merge o (lse is in log2 scale, use exp2/log2 for efficiency) + lse_shifted = lse - tl.max(lse, axis=0) + log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0)) + scale = tl.exp2(lse_shifted - log_sum) + o = tl.sum(o * scale[:, None], axis=0) + # save o + o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@triton.jit +def _compare_and_swap( + x, + ids, + flip, + i: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)] + y = tl.reshape(x, shape) + # slice left/right with 'stride' 2**(n_dims - i - 1) + mask = tl.arange(0, 2)[None, :, None] + left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype) + right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype) + left = tl.reshape(left, x.shape) + right = tl.reshape(right, x.shape) + # idx + y_idx = tl.reshape(ids, shape) + left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape) + right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape) + left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype) + right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype) + # actual compare-and-swap + idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True) + ileft = left.to(idtype, bitcast=True) + iright = right.to(idtype, bitcast=True) + ix = x.to(idtype, bitcast=True) + + cond = (left > right) != flip + ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix)) + new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids)) + return ret.to(x.dtype, bitcast=True), new_ids + + +@triton.jit +def _bitonic_merge( + x, + ids, + stage: tl.constexpr, + order: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + tl.static_assert(stage <= n_dims) + # flip denotes whether to re-arrange sub-sequences of elements in ascending or + # descending order. + # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage + # if flip = 00110011... then all the elements will be re-arranged alternatingly (with + # a stride of 2) at this stage + if order == 2: + shape: tl.constexpr = [ + n_outer * 2 ** (n_dims - 1 - stage), + 2, + 2**stage, + ] + flip = tl.reshape( + tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape + ) + else: + flip = order + # perform `stage` rounds of `compare-and-swap` + for i in tl.static_range(stage): + x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims) + return x, ids + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2), + ], + key=["topk"], +) +@triton.jit +def _topk_index_partial_kernel( + s_ptr, # Score: [num_q_heads, BS, max_seqblock] + ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns + # strides + stride_s_h, + stride_s_b, + stride_s_k, + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + # META + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + tl.static_assert(BLOCK_SIZE_K > topk) + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + pid_chunk = tl.program_id(2) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Slice this chunk owns within [0, num_blocks). + chunk_start = pid_chunk * chunk_blocks + chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks) + chunk_actual = tl.maximum(chunk_end - chunk_start, 0) + + off_k = tl.arange(0, BLOCK_SIZE_K) + off_t = tl.arange(0, BLOCK_SIZE_T) + + # Pointer starts at chunk_start. + s_ptrs = ( + s_ptr + + pid_b * stride_s_b + + pid_h * stride_s_h + + (chunk_start + off_k) * stride_s_k + ) + + topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32) + topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32) + left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2 + + # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty + # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0. + for i in tl.range(0, chunk_actual, BLOCK_SIZE_K): + mask = off_k < chunk_actual - i + score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32) + score = tl.where(score != score, -1e30, score) + s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K + topk_score, last_topk_score = score, topk_score + topk_idx, last_topk_idx = ( + tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global + topk_idx, + ) + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), j, 2, n_dims + ) + if i != 0: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims + ) + topk_score_new = last_topk_score * left_half_mask + topk_score * ( + 1 - left_half_mask + ) + topk_idx_new = last_topk_idx * left_half_mask + topk_idx * ( + 1 - left_half_mask + ) + topk_score, topk_idx = _bitonic_merge( + topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims + ) + else: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims + ) + + # Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort). + topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + final_score = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + final_idx = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + # Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry + # -1e30 / 0 sentinels and lose to real scores in the merge stage. + ts_ptrs = ( + ts_partial_ptr + + pid_chunk * stride_ts_c + + pid_b * stride_ts_b + + pid_h * stride_ts_h + + off_t * stride_ts_t + ) + ti_ptrs = ( + ti_partial_ptr + + pid_chunk * stride_ti_c + + pid_b * stride_ti_b + + pid_h * stride_ti_h + + off_t * stride_ti_t + ) + tl.store(ts_ptrs, final_score) + tl.store(ti_ptrs, final_idx) + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + "BLOCK_SIZE_K": lambda args: triton.next_power_of_2( + args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"]) + ), + } +) +@triton.jit +def _topk_index_merge_kernel( + ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape + ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk] + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + # strides + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_tif_h, + stride_tif_b, + stride_tif_t, + # META + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K. + # candidate at flat position p comes from chunk = p // BLOCK_SIZE_T, + # in_chunk = p % BLOCK_SIZE_T. + off = tl.arange(0, BLOCK_SIZE_K) + chunk_idx = off // BLOCK_SIZE_T + in_chunk_idx = off % BLOCK_SIZE_T + valid = chunk_idx < NUM_TOPK_CHUNKS + + score_offset = ( + chunk_idx * stride_ts_c + + pid_h * stride_ts_h + + pid_b * stride_ts_b + + in_chunk_idx * stride_ts_t + ) + idx_offset = ( + chunk_idx * stride_ti_c + + pid_h * stride_ti_h + + pid_b * stride_ti_b + + in_chunk_idx * stride_ti_t + ) + + score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to( + tl.float32 + ) + score = tl.where(score != score, -1e30, score) + idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32) + + # Full bitonic descending sort of BLOCK_SIZE_K items. + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims) + score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims) + + # Extract first BLOCK_SIZE_T positions — these are the global top-K. + extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + topk_idx_final = tl.sum( + extract_mask[:, None] + * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + off_t = tl.arange(0, BLOCK_SIZE_T) + tif_ptrs = ( + ti_final_ptr + + pid_h * stride_tif_h + + pid_b * stride_tif_b + + off_t * stride_tif_t + ) + topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1) + tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty)) + + +@torch.no_grad() +def flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + init_blocks: int, + local_blocks: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + score_type: str = "max", + disable_index_value: bool = False, + use_dense_main_attn: bool = False, # NOTE: need transform idx in this case + page_size: int = 1, +) -> torch.Tensor: + assert score_type in ( + "max", + "lse", + ), f"score_type must be 'max' or 'lse', got {score_type!r}" + triton.set_allocator(robust_allocator) + # dtype check + assert ( + q.dtype == torch.bfloat16 + or q.dtype == torch.float16 + and k_cache.dtype == q.dtype + ) + if not disable_index_value: + assert v_cache is not None + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + max_kv_len = req_to_token.shape[1] + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets. + # Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads. + # TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by: + # NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS) + # then rounded down to the nearest power of 2 (merge kernel requires it). + # Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len. + # Empty chunks early-return cheaply, so over-chunking is nearly free. + # E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096. + TARGET_GRID = 4096 + MAX_NUM_KV_CHUNKS = 256 + target = max( + 1, + min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_KV_CHUNKS = 1 << (target.bit_length() - 1) + score_kv_len = min(max_seqlen, max_kv_len) + # The score producers below write every valid block column + # [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers + # clamp their scan to the same per-row valid block count, so columns beyond + # seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph + # capture the static score shape can be much larger than the live context. + score = torch.empty( + (num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)), + dtype=torch.float32, + device=q.device, + ) + use_jit_topk = ( + envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get() + and score.shape[2] <= 4096 + and topk <= 32 + ) + # If the live context has <= topk sparse blocks, the downstream dense + # page-table/JIT top-k kernels select every block from seq_lens directly + # without reading score. Keep this gate in sync with the consumers below: + # _skip_block_topk/use_dense_main_attn and use_jit_topk special-case + # num_blocks <= topk, while the Triton fallback reads score and must not + # skip these writes. + skip_trivial_topk_score = use_dense_main_attn or use_jit_topk + + grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads) + if disable_index_value: + _decode_score_kernel[grid]( + q, + k_cache, + req_to_token, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + req_to_token.stride(0), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + else: + assert v_cache is not None + o = torch.empty( + NUM_KV_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse = torch.empty( + NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device + ) + _decode_score_attn_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + o, + lse, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + # Fused top-k + page-table transform: emit the dense backend's page table + # directly (page-size-aware) instead of block ids, skipping a separate gather. + # The page table + per-query effective KV length are allocated and returned. + real_seq_lens = None + if use_dense_main_attn: + # VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The + # standalone vendor only ships the pure-Triton path, so this is gated off + # by default (use_dense_main_attn=False). If you opt in, vendor that .cuh + # too. Original line: `from sglang.jit_kernel.minimax_decode_topk import ( + # minimax_decode_topk_page_table)`. + raise NotImplementedError( + "use_dense_main_attn=True requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; " + "the standalone Triton vendor only supports use_dense_main_attn=False." + ) + # num_q_heads (= num index/kv heads) may be > 1 for DP attention; the + # kernel emits a flattened [batch*num_q_heads] page table + seq_lens with + # the kv head head-encoded (head-minor) into the page index. + topk_idx, real_seq_lens = minimax_decode_topk_page_table( + score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # fall through to the idx_o (index value) merge, then return + _skip_block_topk = True + else: + _skip_block_topk = False + + # get topk index + if not _skip_block_topk: + topk_idx = torch.empty( + (num_q_heads, batch_size, topk), + device=score.device, + dtype=torch.int32, + ) + + if _skip_block_topk: + pass + elif use_jit_topk: + # Single-stage JIT radix-select: one kernel, no intermediate buffers. + # Equivalent output to the 2-stage path (set of block ids, front-packed, + # -1 padded); ~2-16x faster for long context. + # VENDOR NOTE: this needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the + # `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so + # this branch is never taken in the standalone vendor (the 2-stage Triton + # `else` fallback below runs instead). Original line: + # `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`. + raise NotImplementedError( + "use_jit_topk requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor " + "uses the 2-stage Triton fallback (keep " + "SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)." + ) + elif has_native_ops() and topk <= 64: + topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk) + else: + # 2-stage split-K Triton fallback. + # Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim. + # Same constraints as flash_decode: must be deterministic from values + # constant within a cuda graph (BS, num_q_heads), pow2, capped so the + # merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk)) + # stays reasonable. With topk=32 and cap 16, merge sorts 512 items. + TOPK_TARGET_GRID = 64 + MAX_NUM_TOPK_CHUNKS = 16 + topk_target = max( + 1, + min( + MAX_NUM_TOPK_CHUNKS, + TOPK_TARGET_GRID // max(1, batch_size * num_q_heads), + ), + ) + NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1) + BLOCK_SIZE_T = triton.next_power_of_2(topk) + max_seqblock = score.shape[2] + chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + topk_score_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.float32, + device=score.device, + ) + topk_idx_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.int32, + device=score.device, + ) + # stream 0 (default): topk partial → topk merge + grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS) + _topk_index_partial_kernel[grid]( + score, + topk_score_partial, + topk_idx_partial, + seq_lens, + block_size, + topk, + chunk_blocks, + score.stride(0), + score.stride(1), + score.stride(2), + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + ) + grid = (batch_size, num_q_heads) + _topk_index_merge_kernel[grid]( + topk_score_partial, + topk_idx_partial, + topk_idx, + seq_lens, + block_size, + topk, + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # attn output merge (default stream) + grid = (batch_size, num_q_heads) + _merge_attn_out_kernel[grid]( + o, + lse, + seq_lens, + head_dim, + block_size, + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + ) + o = o[0].contiguous() + return o, topk_idx, real_seq_lens diff --git a/build/torch212-cxx11-cu130-x86_64-linux/decode/topk_sparse.py b/build/torch212-cxx11-cu130-x86_64-linux/decode/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..8fe24cce53337c4f44e6c462fc3c90a258a34fc7 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/decode/topk_sparse.py @@ -0,0 +1,452 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from .._native import ( + native_decode_mma_supported, + native_decode_sparse_attn_mma_paged, +) +from ..common.utils import robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4, 5] + ], + key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"], +) +@triton.jit +def _gqa_share_sparse_decode_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + idx_ptr, # topk index: qh x b x topk + o_ptr, # O partial: c x b x qh x d + lse_ptr, # lse partial: c x b x qh + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + # sm_scale + sm_scale, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + NUM_TOPK_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + IS_FP8: tl.constexpr, +): + # decode program ids: split-K over the topk dimension to give every SM + # something to do at small batch. pid(0) folds (batch, chunk) together so + # the grid size = batch_size * NUM_TOPK_CHUNKS. + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # per-chunk topk range. chunk_size is *runtime* (depends on max_topk which + # is a runtime arg, not constexpr), so don't annotate as tl.constexpr — + # doing so produces undefined behavior in Triton. + chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + chunk_start_topk = pid_c * chunk_size_topk + chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk + # get q k start and len after rmpad + seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # get real topk + off_t = tl.arange(0, BLOCK_SIZE_T) + idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b + topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk) + # init pointer + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + dim_mask = off_d < head_dim + # init statistics — kept at -inf so empty chunks (chunk_start >= real_topk) + # naturally fall out as weight=0 in the merge step. + # q is always needed; load it unconditionally. + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + # Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code- + # eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is + # then None). The original compound `if HAS_SINK and pid_c == 0` traced + # the body even when False -> NoneType make_block_ptr compile error on + # sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always + # False for us, but this keeps the sink path correct too. + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load( + sink_ptrs, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale + m_i = qsink + lse_i = qsink + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # only iterate over this chunk's topk slice. the load must respect the + # per-chunk start offset. + cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t + for _ in tl.range(chunk_start_topk, chunk_end_topk): + # load index + c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N + cur_idx_ptr = cur_idx_ptr + stride_ti_t + # resolve slots for this block via req_to_token + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no + # scale), so dequant is just a widening cast to the Q compute dtype + # before the tl.dot. Matches the bf16 path bit-for-bit when the cache + # is bf16 (IS_FP8 False -> this branch is compiled out). + k = k.to(q.dtype) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V before the P@V dot. This also makes the `p.to(v.dtype)` + # below cast P to the compute dtype (not to fp8, which would be + # catastrophic precision loss on the attention weights). + v = v.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # load v and update acc_o + p = p.to(v.dtype) + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij) + # final scale (matches the old non-split kernel for chunks where lse_i>-inf). + # For empty chunks (chunk_start_topk >= real_topk) the inner loop never + # runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute + # exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial + # and the merge result. Gate the scale with tl.where so empty chunks emit a + # clean zero (lse_i stays -inf which the merge correctly turns into weight=0). + scale = tl.where( + lse_i > float("-inf"), + tl.exp(m_i - lse_i), + tl.zeros_like(lse_i), + ) + acc_o = acc_o * scale[:, None] + # save partial output and lse for the merge step + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.jit +def _merge_topk_attn_out_kernel( + o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0 + lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH] + head_dim, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + off_c = tl.arange(0, NUM_TOPK_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(NUM_TOPK_CHUNKS, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0 + # standard flash-decoding merge in linear (not log2) space, matching the + # decode kernel which uses tl.exp / tl.log. + lse_max = tl.max(lse, axis=0) + weights = tl.exp(lse - lse_max) + weights = weights / tl.sum(weights, axis=0) + o_merged = tl.sum(o * weights[:, None], axis=0) + o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@torch.no_grad() +def flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, + use_tma: bool = True, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may + # be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel + # widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing + # sparse decode dtype contract unchanged by accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + assert topk_idx.shape[0] == num_kv_heads + assert ( + triton.next_power_of_2(block_size) == block_size + ), f"block_size must be a power of 2, but got {block_size}" + # assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}" + max_kv_len = req_to_token.shape[1] + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + max_topk = topk_idx.shape[2] + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # Fast path: native tensor-core decode for the M3 MSA shape. Falls through + # to the Triton kernel for other shapes, sink, or fp8 caches. + if sink is None and not is_fp8 and native_decode_mma_supported( + q, k_cache, block_size=block_size + ): + return native_decode_sparse_attn_mma_paged( + q, + k_cache, + v_cache, + req_to_token, + seq_lens, + slot_ids, + block_size, + topk_idx, + sm_scale, + ) + # Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as + # flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must + # only depend on shape constants (so grid is fixed within a cuda graph). + # Capped by max_topk because chunks beyond real_topk early-fall-through to + # the merge-as-zero path; capping avoids wasting blocks at tiny topk. + TARGET_GRID = 256 + target = max( + 1, + min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1) + # output tensor: split-K partials, merged into chunk 0 by the merge kernel + o_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + dtype=torch.float32, + device=q.device, + ) + # launch attention kernel + grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads) + _gqa_share_sparse_decode_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + topk_idx, + o_partial, + lse_partial, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + BLOCK_SIZE_N=block_size, + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + IS_FP8=is_fp8, + ) + # merge partials into chunk 0 + merge_grid = (batch_size, num_q_heads) + _merge_topk_attn_out_kernel[merge_grid]( + o_partial, + lse_partial, + head_dim, + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + return o_partial[0].contiguous() diff --git a/build/torch212-cxx11-cu130-x86_64-linux/interface.py b/build/torch212-cxx11-cu130-x86_64-linux/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..7939722609218f2bb41eadcdb2f965213d459d1c --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/interface.py @@ -0,0 +1,679 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from ._native import ( + has_native_ops, + native_indexer_block_scores, + native_nvfp4_dequant_swizzled_to_bf16, +) +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse +from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16 + + +def _page_table_to_req_to_token( + page_table: torch.Tensor, + seqused_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, +) -> torch.Tensor: + batch = int(page_table.shape[0]) + max_seqlen_k = int(max_seqlen_k) + blk_kv = int(blk_kv) + logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64) + page = logical_pos // blk_kv + offset = logical_pos % blk_kv + page = page.clamp(max=int(page_table.shape[1]) - 1) + physical_page = page_table[:, page].to(torch.int64) + physical_token = physical_page * blk_kv + offset[None, :] + valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None] + return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous() + + +def _dense_q2k_indices( + *, + head_kv: int, + batch: int, + topk: int, + seq_lens: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device) + blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device) + valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv) + for b in range(batch): + valid = blocks < valid_blocks[b] + idx[:, b, valid] = blocks[valid] + return idx + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]: + seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist() + return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens] + + +def _packed_row_to_batch_block( + cu_seqlens_k: torch.Tensor, + blk_kv: int, +) -> list[tuple[int, int]]: + rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv)) + max_rows = max(rows_per_batch, default=0) + row_map: list[tuple[int, int]] = [] + for block_idx in range(max_rows): + for batch_idx, rows in enumerate(rows_per_batch): + if block_idx < rows: + row_map.append((batch_idx, block_idx)) + return row_map + + +def _csr_to_q2k_indices( + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + blk_kv: int, + topk: int, +) -> torch.Tensor: + if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32: + raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32") + if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2: + raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2") + head_kv = int(k2q_row_ptr.shape[0]) + total_q = int(cu_seqlens_q[-1].item()) + topk = int(topk) + if topk <= 0: + raise ValueError(f"topK must be positive, got {topk}") + + row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv)) + if int(k2q_row_ptr.shape[1]) != len(row_map) + 1: + raise ValueError( + "k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: " + f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}" + ) + + cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False) + q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False) + q2k = torch.full( + (head_kv, total_q, topk), + -1, + dtype=torch.int32, + device=k2q_row_ptr.device, + ) + + next_slot: dict[tuple[int, int], int] = {} + seen: set[tuple[int, int, int]] = set() + for h in range(head_kv): + for row, (batch_idx, kv_block_idx) in enumerate(row_map): + start = int(row_ptr_cpu[h, row].item()) + end = int(row_ptr_cpu[h, row + 1].item()) + q_start = int(cu_q_cpu[batch_idx]) + q_end = int(cu_q_cpu[batch_idx + 1]) + q_len = q_end - q_start + for pos in range(start, end): + q_val = int(q_idx_cpu[h, pos].item()) + if q_val < 0: + continue + if q_start <= q_val < q_end: + q_global = q_val + elif q_val < q_len: + q_global = q_start + q_val + else: + continue + key = (h, q_global, kv_block_idx) + if key in seen: + continue + seen.add(key) + slot_key = (h, q_global) + slot = next_slot.get(slot_key, 0) + if slot >= topk: + continue + q2k[h, q_global, slot] = int(kv_block_idx) + next_slot[slot_key] = slot + 1 + return q2k.contiguous() + + +def _make_prefill_cache_and_metadata( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, + *, + page_table: Optional[torch.Tensor], + seqused_k: Optional[torch.Tensor], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if page_table is None: + if k.ndim != 3 or v.ndim != 3: + raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + k_cache = k.contiguous() + v_cache = v.contiguous() + batch = int(cu_seqlens_q.numel()) - 1 + max_seqlen_k = int(max_seqlen_k) + logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64) + cu_k = cu_seqlens_k.to(torch.int64) + seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous() + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32) + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + else: + if k.ndim != 4 or v.ndim != 4: + raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("paged k/v page dimension must equal blk_kv") + if page_table.dtype != torch.int32: + raise TypeError("page_table must be torch.int32") + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + batch = int(page_table.shape[0]) + if seqused_k is None: + seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous() + else: + seq_lens = seqused_k.to(torch.int32).contiguous() + logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64) + page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1) + offset = logical % int(blk_kv) + physical_page = page_table[:, page].to(torch.int64) + req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32) + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32) + prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous() + return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens + + +def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int: + if kv_heads <= 0 or q_heads % kv_heads != 0: + raise ValueError("number of query heads must be divisible by KV heads") + gqa_group = q_heads // kv_heads + limit = max(1, 128 // max(1, gqa_group)) + for candidate in (64, 32, 16, 8, 4, 2, 1): + if candidate <= limit: + return candidate + return 1 + + +def _normalize_decode_q2k( + q2k_indices: Optional[torch.Tensor], + *, + batch: int, + seqlen_q: int, + head_kv: int, + seqused_k: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + if q2k_indices is None: + topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv)) + return _dense_q2k_indices( + head_kv=head_kv, + batch=batch, + topk=topk, + seq_lens=seqused_k, + blk_kv=blk_kv, + ) + if q2k_indices.dtype != torch.int32: + raise TypeError("q2k_indices must be torch.int32") + if q2k_indices.ndim != 3: + raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]") + if q2k_indices.shape[0] != head_kv: + raise ValueError("q2k_indices first dimension must match Hkv") + if int(q2k_indices.shape[1]) != batch * seqlen_q: + raise ValueError("q2k_indices second dimension must equal batch * seqlen_q") + if seqlen_q != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + return q2k_indices[:, ::seqlen_q, :].contiguous() + + +def sparse_decode_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q2k_indices: Optional[torch.Tensor] = None, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + schedule: Optional[object] = None, + O_partial: Optional[torch.Tensor] = None, + LSE_partial: Optional[torch.Tensor] = None, +): + """Run MiniMax paged sparse decode on Blackwell. + + This is compatible with the upstream MiniMaxAI/msa decode-facing API name, + but accepts the Blackwell package's validated BF16/FP16 decode path. It + supports single-token decode (`seqlen_q=1`) with paged K/V + `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`. + """ + + if schedule is not None or O_partial is not None or LSE_partial is not None: + raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper") + if return_softmax_lse: + raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper") + if not causal: + raise NotImplementedError("Blackwell decode wrapper currently supports causal=True") + if int(seqlen_q) != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + if q.ndim != 3: + raise ValueError("q must have shape [batch * seqlen_q, Hq, D]") + if k.ndim != 4 or v.ndim != 4: + raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if q.device != k.device or q.device != v.device or q.device != page_table.device: + raise ValueError("q, k, v, and page_table must be on the same device") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("Blackwell decode wrapper supports BF16/FP16 q") + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper") + if q.shape[-1] != 128 or k.shape[-1] != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("k.shape[2] must equal blk_kv") + if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32: + raise TypeError("page_table and seqused_k must be torch.int32") + if page_table.ndim != 2: + raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]") + batch = int(page_table.shape[0]) + if int(q.shape[0]) != batch: + raise ValueError("for seqlen_q=1, q.shape[0] must equal batch") + + head_kv = int(k.shape[1]) + q2k = _normalize_decode_q2k( + q2k_indices, + batch=batch, + seqlen_q=int(seqlen_q), + head_kv=head_kv, + seqused_k=seqused_k, + blk_kv=int(blk_kv), + ) + req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv)) + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + return flash_decode_with_gqa_share_sparse( + q.contiguous(), + None, + k_cache, + v_cache, + req_to_token, + seqused_k.contiguous(), + slot_ids, + int(blk_kv), + q2k, + sm_scale=softmax_scale, + ) + + +class SparseDecodePagedAttentionWrapper: + def __init__(self, *, blk_kv: int = 128, causal: bool = True): + self.blk_kv = int(blk_kv) + self.causal = bool(causal) + self.page_table: Optional[torch.Tensor] = None + self.seqused_k: Optional[torch.Tensor] = None + self.seqlen_q: Optional[int] = None + self.max_seqlen_k: Optional[int] = None + self.q2k_indices: Optional[torch.Tensor] = None + + def plan( + self, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + q2k_indices: Optional[torch.Tensor] = None, + num_qo_heads: Optional[int] = None, + num_kv_heads: Optional[int] = None, + head_dim: Optional[int] = 128, + enable_cuda_graph: bool = False, + max_grid_size: Optional[int] = None, + fixed_split_size: Optional[int] = None, + disable_split_kv: bool = False, + ) -> "SparseDecodePagedAttentionWrapper": + if head_dim is not None and int(head_dim) != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + self.page_table = page_table.contiguous() + self.seqused_k = seqused_k.contiguous() + self.seqlen_q = int(seqlen_q) + self.max_seqlen_k = int(max_seqlen_k) + self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous() + return self + + def run( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + out: Optional[torch.Tensor] = None, + lse: Optional[torch.Tensor] = None, + ): + if self.page_table is None or self.seqused_k is None: + raise RuntimeError("decode wrapper must be planned before run") + result = sparse_decode_atten_func( + q, + k, + v, + self.q2k_indices, + page_table=self.page_table, + seqused_k=self.seqused_k, + seqlen_q=self.seqlen_q, + max_seqlen_k=self.max_seqlen_k, + blk_kv=self.blk_kv, + causal=self.causal, + softmax_scale=softmax_scale, + return_softmax_lse=return_softmax_lse, + ) + if out is not None: + out.copy_(result) + return out + return result + + +def sparse_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + *, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + lse_temperature_scale: float = 1.0, + return_temperature_lse: bool = False, + partial_dtype: torch.dtype = torch.bfloat16, + return_softmax_lse: bool = False, + page_table: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + schedule: Optional[object] = None, + usable_SM_count: int = -1, + qk_dtype: Optional[torch.dtype] = None, + pv_dtype: Optional[torch.dtype] = None, +): + """Run the official MiniMax CSR sparse prefill API on Blackwell. + + The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list + consumed by the Blackwell Triton sparse prefill kernel. The default output + path is supported for BF16/FP16 tensors with head_dim=128. + """ + + del partial_dtype, usable_SM_count, qk_dtype, pv_dtype + if schedule is not None: + raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper") + if return_temperature_lse or return_softmax_lse: + raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper") + if float(lse_temperature_scale) != 1.0: + raise ValueError("lse_temperature_scale values other than 1.0 are not supported") + if not causal: + raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True") + if q.ndim != 3: + raise ValueError("q must have shape [total_q, Hq, D]") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("q must be BF16 or FP16") + if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128: + raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128") + if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32: + raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32") + if q.device != k.device or q.device != v.device: + raise ValueError("q, k, and v must be on the same device") + + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have matching BF16/FP16 dtype") + kv_heads = int(k.shape[1]) + q2k_indices = _csr_to_q2k_indices( + k2q_row_ptr.contiguous(), + k2q_q_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(blk_kv), + int(topK), + ) + k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata( + q, + k, + v, + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(max_seqlen_k), + int(blk_kv), + page_table=None if page_table is None else page_table.contiguous(), + seqused_k=None if seqused_k is None else seqused_k.contiguous(), + ) + block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads) + return flash_prefill_with_gqa_share_sparse( + q.contiguous(), + k_cache, + v_cache, + None, + req_to_token, + slot_ids, + q2k_indices, + block_size_q=block_size_q, + block_size_k=int(blk_kv), + cu_seqlens=cu_seqlens_q.contiguous(), + seq_lens=seq_lens, + prefix_lens=prefix_lens, + max_seqlen_q=int(max_seqlen_q), + sm_scale=softmax_scale, + ) + + +def _nvfp4_dequant_kv( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor, +) -> torch.Tensor: + if packed.dtype is not torch.uint8: + return packed + if has_native_ops() and packed.is_cuda: + return native_nvfp4_dequant_swizzled_to_bf16( + packed, + scale_128x4, + global_scale, + ) + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + rows = 1 + for dim in original_shape[:-1]: + rows *= int(dim) + qx = Nvfp4QuantizedTensor( + data=packed, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, int(original_shape[-1]) // 16), + original_shape=original_shape, + ) + return dequantize_nvfp4_128x4_to_bf16(qx) + + +def sparse_atten_nvfp4_kv_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k_scale_128x4: torch.Tensor, + v_scale_128x4: torch.Tensor, + k_global_scale: torch.Tensor, + v_global_scale: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + **kwargs, +): + """NVFP4-KV official prefill API wrapper. + + The Blackwell compatibility path dequantizes NVFP4 K/V with the package's + 128x4 metadata helper, then dispatches to `sparse_atten_func`. + """ + + k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale) + v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale) + return sparse_atten_func( + q, + k_bf16, + v_bf16, + k2q_row_ptr, + k2q_q_indices, + topK, + **kwargs, + ) + + +def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor: + data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8) + lut = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0], + dtype=torch.float32, + device=data.device, + ) + out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device) + out[..., 0::2] = lut[(data & 0x0F).long()] + out[..., 1::2] = lut[(data >> 4).long()] + return out + + +def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]): + return values * scale.to(torch.float32).repeat_interleave(16, dim=-1) + if scale.ndim == values.ndim - 1: + return values * scale.to(torch.float32).unsqueeze(-1) + return values + + +def fp4_indexer_block_scores( + q_fp4: torch.Tensor, + k_fp4: torch.Tensor, + q_scale: torch.Tensor, + k_scale: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + cu_page_offsets: torch.Tensor, + *, + max_seqlen_q: int, + max_seqlen_k: int, + kv_indices: torch.Tensor, + fp4_format: str, + causal: bool = False, + qo_offset: Optional[torch.Tensor] = None, + scale_layout: str = "preordered_mma", +) -> torch.Tensor: + """FP4 block-score API for Blackwell. + + Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`. + Built artifacts route CUDA tensors through the native Blackwell block-score + kernel. Source-tree mode uses the reference loop below so the API remains + testable before the extension is built. + """ + + del qo_offset + if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}: + raise ValueError(f"unsupported fp4_format: {fp4_format}") + if scale_layout not in {"public", "preordered_mma"}: + raise ValueError(f"unsupported scale_layout: {scale_layout}") + if q_fp4.ndim != 3: + raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]") + if k_fp4.ndim != 4: + raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]") + total_q, hq, _ = q_fp4.shape + num_pages, hkv, page_size, _ = k_fp4.shape + if int(page_size) != 128: + raise ValueError("fp4_indexer_block_scores currently expects 128-token pages") + if hq % hkv != 0: + raise ValueError("Hq must be divisible by Hkv") + max_blocks = (int(max_seqlen_k) + 127) // 128 + scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device) + + q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale) + k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale) + + # Native block-max QK kernel for the cubic scoring loop when available. + # Same dequant numerics as the reference (bf16 compute); falls back to the + # Python reference below in source-tree mode. + if has_native_ops() and q_fp4.is_cuda: + batch = int(cu_seqlens_q.numel()) - 1 + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64) + batch_of_q = torch.repeat_interleave( + torch.arange(batch, device=q_fp4.device, dtype=torch.int32), + q_lens, + ) + return native_indexer_block_scores( + q.to(torch.bfloat16), + k.to(torch.bfloat16), + batch_of_q, + cu_seqlens_q.to(torch.int32), + cu_seqlens_k.to(torch.int32), + cu_page_offsets.to(torch.int32), + kv_indices.to(torch.int32), + max_blocks=max_blocks, + causal=causal, + ) + + gqa = hq // hkv + batch = int(cu_seqlens_q.numel()) - 1 + cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist() + cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist() + kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist() + for b in range(batch): + q_start, q_end = int(cu_q[b]), int(cu_q[b + 1]) + k_len = int(cu_k[b + 1]) - int(cu_k[b]) + page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1]) + for logical_block, page_pos in enumerate(range(page_start, page_end)): + if logical_block >= max_blocks: + break + physical_page = int(kv_idx_cpu[page_pos]) + if physical_page < 0 or physical_page >= num_pages: + continue + valid_tokens = min(128, max(0, k_len - logical_block * 128)) + if valid_tokens <= 0: + continue + for qi in range(q_start, q_end): + local_q = qi - q_start + if causal and logical_block * 128 > local_q: + continue + for qh in range(hq): + kh = qh // gqa + key_block = k[physical_page, kh, :valid_tokens] + if causal: + visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1)) + if visible <= 0: + continue + key_block = key_block[:visible] + block_scores = torch.matmul(key_block, q[qi, qh]) + scores[qh, logical_block, qi] = block_scores.max() + return scores + + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", +] diff --git a/build/torch212-cxx11-cu130-x86_64-linux/metadata.json b/build/torch212-cxx11-cu130-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cf3cb63cb2bffa2ace3741b53698334348b13ad7 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/metadata.json @@ -0,0 +1,39 @@ +{ + "name": "minimaxai-msa-blackwell", + "id": "_minimaxai_msa_blackwell_cuda_25b0067", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": { + "type": "cuda", + "archs": [ + "12.0", + "12.1" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=", + "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=", + "_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "LqhiBQoZ3MSwx8v9O1GREvG04FTPwYKhT5KHa0rlufc=", + "_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=", + "_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=", + "api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=", + "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=", + "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=", + "decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=", + "interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=", + "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=", + "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=", + "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=", + "prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=", + "prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=", + "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=", + "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w=" + } + } +} \ No newline at end of file diff --git a/build/torch212-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py b/build/torch212-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b2672c1cd85b74c1b3ded0fc0b2100e1aeac23 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/minimaxai_msa_blackwell/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch212-cxx11-cu130-x86_64-linux/naive/__init__.py b/build/torch212-cxx11-cu130-x86_64-linux/naive/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/naive/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch212-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py b/build/torch212-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..db7caceb6faa5d8d1669d3c57f1e4ca9578a4804 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/naive/flash_with_topk_idx.py @@ -0,0 +1,106 @@ +# Copyright 2025 XunhaoLai. All rights reserved. +# +# VENDOR NOTE: rewritten to drop the `einops` dependency (the original used +# einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream +# `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced +# with native torch.einsum / torch.reshape so the standalone vendor needs only +# torch (+ triton for the kernels). Original einops expressions are kept in +# comments next to each replacement for auditability. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], # [num_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + sm_scale: Optional[float] = None, + init_blocks: int = 0, + local_blocks: int = 0, +): + """Joint reference: lightning-indexer block top-k + sparse attention. + + This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense + 5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by + `slot_ids`. It both (a) selects the top-k key blocks via per-block max score + (with optional forced init/local blocks) and (b) computes the resulting + sparse attention, returning (o, topk_idx). + """ + assert ( + kv_cache.shape[2] % block_size == 0 + ), "max cache len must be divisible by block size" + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + original_dtype = q.dtype + batch_size = q.shape[0] + num_q_heads = q.shape[1] + num_kv_heads = kv_cache.shape[3] + gqa_group_size = num_q_heads // num_kv_heads + head_dim = q.shape[-1] + # rearrange q: "b (h g) d -> b h g d" + q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim) + kv_cache_float = kv_cache.float() + k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d] + v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d] + # einsum(q, k, "b h g d, b n h d -> b h g n") + qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale + mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None] + qk = qk.masked_fill(~mask[:, None, None, :], float("-inf")) + # get score + score = qk.clone().to(torch.float32) + # rearrange "b h g (n s) -> b (h g) n s", s=block_size + n_blocks = score.shape[-1] // block_size + score = score.reshape( + batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size + ).reshape(batch_size, num_q_heads, n_blocks, block_size) + score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks] + # post-process score for init_blocks and local_blocks + INIT_SCORE = 1e30 + LOCAL_SCORE = 1e29 + if init_blocks > 0: + score[:, :, :init_blocks] = INIT_SCORE + if local_blocks > 0: + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + local_start = max(0, num_blks - local_blocks) + score[b, :, local_start:num_blks] = LOCAL_SCORE + # compute topk indices per (batch, head) + # score shape: [batch_size, num_q_heads, num_blocks] + topk_idx = torch.full( + (num_q_heads, batch_size, topk), + fill_value=-1, + device=score.device, + dtype=torch.int32, + ) + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + actual_topk = min(topk, num_blks) + for h in range(num_q_heads): + # get topk indices for this (batch, head) + _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1) + topk_idx[h, b, :actual_topk] = indices.to(torch.int32) + # compute attention output with sink + if sink is not None: + # sink: [num_q_heads, head_dim] -> [h, g, d] + sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim) + # einsum(q, sink, "b h g d, h g d -> b h g") + qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale + qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1] + attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32) + attn = attn[..., 1:] # remove sink score + else: + attn = qk.softmax(dim=-1, dtype=torch.float32) + # einsum(attn, v, "b h g n, b n h d -> b h g d") + o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel) + # rearrange "b h g d -> b (h g) d" + o = o.reshape(batch_size, num_q_heads, head_dim) + return o.to(original_dtype), topk_idx diff --git a/build/torch212-cxx11-cu130-x86_64-linux/naive/topk_sparse.py b/build/torch212-cxx11-cu130-x86_64-linux/naive/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..6fefac38e0d131d2b37dcb392e405d71a93ace61 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/naive/topk_sparse.py @@ -0,0 +1,88 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], # [num_q_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, +) -> torch.Tensor: + """ + Naive implementation of sparse attention with GQA group sharing. + Each GQA group (all q heads sharing the same kv head) uses the same topk_idx. + """ + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + + original_dtype = q.dtype + batch_size, num_q_heads, head_dim = q.shape + max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape + gqa_group_size = num_q_heads // num_kv_heads + # Output tensor + o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device) + + for b in range(batch_size): + seq_len = seq_lens[b].item() + sid = slot_ids[b].item() % max_slots + + for kh in range(num_kv_heads): + # Get topk indices for this (kv_head, batch) + block_indices = topk_idx[kh, b, :].tolist() + + # Collect selected K and V blocks + selected_k_blocks = [] + selected_v_blocks = [] + for block_idx in block_indices: + if block_idx < 0: + continue # Invalid index + start = block_idx * block_size + end = min(start + block_size, seq_len) + if start >= seq_len: + continue + # K: [block_len, head_dim] + k_block = kv_cache[sid, 0, start:end, kh, :] + # V: [block_len, head_dim] + v_block = kv_cache[sid, 1, start:end, kh, :] + selected_k_blocks.append(k_block) + selected_v_blocks.append(v_block) + + if len(selected_k_blocks) == 0: + continue + + # Concatenate selected blocks: [total_selected_len, head_dim] + k_selected = torch.cat(selected_k_blocks, dim=0) + v_selected = torch.cat(selected_v_blocks, dim=0) + + # Compute attention for all q heads in this GQA group + for g in range(gqa_group_size): + qh = kh * gqa_group_size + g + # q_vec: [head_dim] + q_vec = q[b, qh, :].float() + + # Compute attention scores: [total_selected_len] + scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale + + # Add sink to softmax normalization if present + if sink is not None: + sink_vec = sink[qh, :].float() + qsink = torch.dot(q_vec, sink_vec) * sm_scale + # Concatenate sink score with regular scores for softmax + scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0) + attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1) + # Remove sink weight (only used for normalization) + attn_weights = attn_weights_with_sink[1:] + else: + attn_weights = torch.softmax(scores, dim=-1) + + # Compute output: [head_dim] + o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected) + + return o diff --git a/build/torch212-cxx11-cu130-x86_64-linux/prefill/__init__.py b/build/torch212-cxx11-cu130-x86_64-linux/prefill/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b6b69345bf3ffe07f6ccac8c570ee17e3b5e6c69 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/prefill/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +from .topk_sparse import flash_prefill_with_gqa_share_sparse + +__all__ = ["flash_prefill_with_gqa_share_sparse"] diff --git a/build/torch212-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py b/build/torch212-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..9bf83d65eac260f8e71180ea6f28970e8db582d7 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/prefill/topk_sparse.py @@ -0,0 +1,371 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from ..common.utils import get_cu_seqblocks, robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]), + "BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]), + "BLOCK_SIZE_H": lambda args: triton.next_power_of_2( + max( + 16 // args["BLOCK_SIZE_Q"], + triton.next_power_of_2(args["gqa_group_size"]), + ) + ), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"], + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + # Configs that fail to compile on the target arch are skipped, so widening + # the num_warps x num_stages grid only adds candidates, never a bad kernel. + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in (2, 4, 8) + for ns in (2, 3, 4) + ], + key=[ + "BLOCK_SIZE_Q", + "BLOCK_SIZE_K", + "qk_head_dim", + "v_head_dim", + "gqa_group_size", + ], +) +@triton.jit +def _gqa_share_sparse_fwd_kernel( + q_ptr, # Q: n x h x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + sink_ptr, # Sink: h x d + t_ptr, # topk_idx: kh x n x k + o_ptr, # O: n x h x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + # seqlens + cu_seqlens_q, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + # shape + max_slots, + num_kv_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + max_topk, + # q loop num + num_q_loop, + # sm_scale + sm_scale, + # stride + stride_qn, + stride_qh, + stride_qd, + stride_ks, + stride_kh, + stride_kd, + stride_vs, + stride_vh, + stride_vd, + stride_sh, + stride_sd, + stride_th, + stride_tn, + stride_tk, + stride_on, + stride_oh, + stride_od, + stride_r2t_b, + # META parameters + BLOCK_SIZE_Q: tl.constexpr, # q block size + BLOCK_SIZE_K: tl.constexpr, # k block size + BLOCK_SIZE_KD: tl.constexpr, + BLOCK_SIZE_VD: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + BLOCK_SIZE_QH: tl.constexpr, + # has sink + HAS_SINK: tl.constexpr, + USE_TMA: tl.constexpr, + IS_FP8: tl.constexpr, +): + sm_scale_log2e = sm_scale * 1.4426950409 + # get batch id and head id + pid_q = tl.program_id(0) + pid_kh = tl.program_id(1) + pid_b = tl.program_id(2) + pid_h = pid_kh * gqa_group_size + # get q k start and len after rmpad + q_start = tl.load(cu_seqlens_q + pid_b) + q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start + q_block_start = tl.load(cu_seqblocks_q + pid_b) + q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start + seq_len = tl.load(seq_lens + pid_b) + prefix_len = tl.load(prefix_lens + pid_b) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # safety against negative + if pid_q * num_q_loop >= q_block_len: + return + real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop) + if HAS_SINK: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sh, + shape=(gqa_group_size, qk_head_dim), + strides=(stride_sh, stride_sd), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + # offsets for paged K/V load + off_n = tl.arange(0, BLOCK_SIZE_K) + off_kd = tl.arange(0, BLOCK_SIZE_KD) + off_vd = tl.arange(0, BLOCK_SIZE_VD) + kd_mask = off_kd < qk_head_dim + vd_mask = off_vd < v_head_dim + for j in range(real_q_loop): + pid_q_j = pid_q * num_q_loop + j + # init topk idx pointer + t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th + # we assume that the topk_idx is right padded with -1 + off_t = tl.arange(0, BLOCK_SIZE_T) + topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + q_start * stride_qn + pid_h * stride_qh, + shape=(q_len, gqa_group_size, qk_head_dim), + strides=(stride_qn, stride_qh, stride_qd), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(2, 1, 0), + ) + # load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D] + q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero") + # init statistics + off_q_k = ( + tl.arange(0, BLOCK_SIZE_Q)[:, None] + + pid_q_j * BLOCK_SIZE_Q + + prefix_len + - tl.arange(0, BLOCK_SIZE_K)[None, :] + ) + if HAS_SINK: + m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + qsink = ( + tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e + ) # (BLOCK_SIZE_Q, BLOCK_SIZE_H) + m_i += qsink + lse_i += qsink + m_i = tl.reshape(m_i, BLOCK_SIZE_QH) + lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH) + else: + m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32) + q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD) + # sparse attention + for i in range(real_topk): + # get current block start index (absolute K position) + c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K + t_ptr_j = t_ptr_j + stride_tk + # paged load K via req_to_token: pos -> slot -> k_cache + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot) + k = tl.load( + k_cache_ptr + + slots[None, :] * stride_ks + + pid_kh * stride_kh + + off_kd[:, None] * stride_kd, + mask=kd_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 main K cache is unit-scaled; widen to the Q compute dtype + # before the tl.dot (compiled out when the cache is bf16). + k = k.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32) + # causal mask + qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf")) + qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K) + # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K] + # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K] + qk += tl.dot(q, k) * sm_scale_log2e + # K boundary mask: positions beyond seq_len contribute -inf + qk += tl.where(pos_mask[None, :], 0, float("-inf")) + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # paged load V + v = tl.load( + v_cache_ptr + + slots[:, None] * stride_vs + + pid_kh * stride_vh + + off_vd[None, :] * stride_vd, + mask=pos_mask[:, None] & vd_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V so `p.to(v.dtype)` casts P to the compute dtype rather + # than to fp8 (which would wreck attention-weight precision). + v = v.to(q.dtype) + p = p.to(v.dtype) + acc_o += tl.dot(p, v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + # final scale + acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None] + # save output + acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD) + o_ptrs = tl.make_block_ptr( + base=o_ptr + q_start * stride_on + pid_h * stride_oh, + shape=(q_len, gqa_group_size, v_head_dim), + strides=(stride_on, stride_oh, stride_od), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD), + order=(2, 1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2)) + + +@torch.no_grad() +def flash_prefill_with_gqa_share_sparse( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + sink: Optional[torch.Tensor], + req_to_token: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size_q: int, + block_size_k: int, + cu_seqlens: torch.Tensor, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + max_seqlen_q: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + cu_seqblocks_q: Optional[torch.Tensor] = None, + max_seqblock_q: Optional[int] = None, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled) + # under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load. + # Keep CUDA's pre-existing sparse prefill dtype contract unchanged by + # accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + assert block_size_q in {1, 2, 4, 8, 16, 32, 64} + assert block_size_k in {16, 32, 64, 128} + # shape + total_q, num_q_heads, qk_head_dim = q.shape + max_slots, num_k_heads, _ = k_cache.shape + _, num_v_heads, v_head_dim = v_cache.shape + batch_size = cu_seqlens.shape[0] - 1 + topk = topk_idx.shape[-1] + assert topk_idx.shape[0] == num_k_heads + # gqa + assert num_k_heads == num_v_heads + assert num_q_heads % num_k_heads == 0 + gqa_group_size = num_q_heads // num_k_heads + assert gqa_group_size * block_size_q <= 128 + if sm_scale is None: + sm_scale = qk_head_dim**-0.5 + if cu_seqblocks_q is None or max_seqblock_q is None: + cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks( + cu_seqlens, max_seqlen_q, block_size_q, block_size_k + ) + # output tensor + o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype) + # launch kernel + num_q_loop = ( + max_seqblock_q // 131072 + 1 + ) # calculate multiple queries in one kernel if seqlence length is too long + BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q) + BLOCK_SIZE_K = triton.next_power_of_2(block_size_k) + grid = ( + triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop), + num_k_heads, + batch_size, + ) + _gqa_share_sparse_fwd_kernel[grid]( + q, + k_cache, + v_cache, + sink, + topk_idx, + o, + req_to_token, + cu_seqlens, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_k_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + topk, + num_q_loop, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + req_to_token.stride(0), + BLOCK_SIZE_Q=BLOCK_SIZE_Q, + BLOCK_SIZE_K=BLOCK_SIZE_K, + USE_TMA=use_tma, + IS_FP8=is_fp8, + ) + return o diff --git a/build/torch212-cxx11-cu130-x86_64-linux/quantize.py b/build/torch212-cxx11-cu130-x86_64-linux/quantize.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2675ac49d03e5599e9559187efecdf63bb6387 --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/quantize.py @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: Apache-2.0 +"""NVFP4 helper API compatible with MiniMaxAI/msa. + +The quantization entrypoints use Transformer Engine when it is installed. The +layout helpers and reference dequantizer are pure PyTorch and are included so +callers can prepare and validate NVFP4 metadata with the same public API names +as the upstream MiniMaxAI package. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Tuple + +import torch + + +NVFP4_BLOCK_SIZE = 16 +NVFP4_FP4_MAX = 6.0 +NVFP4_FP8_E4M3_MAX = 448.0 + + +@dataclass(frozen=True) +class Nvfp4QuantizedTensor: + data: torch.Tensor + scale_128x4: torch.Tensor + global_scale: torch.Tensor + logical_scale_shape: Tuple[int, int] + original_shape: Tuple[int, ...] + + +def _round_up(x: int, multiple: int) -> int: + return ((int(x) + multiple - 1) // multiple) * multiple + + +def nvfp4_scale_128x4_offset( + row: torch.Tensor, + col: torch.Tensor, + scale_cols: int, +) -> torch.Tensor: + tiles_n = _round_up(scale_cols, 4) // 4 + tile_m = row // 128 + tile_n = col // 4 + outer = row % 128 + inner = col % 4 + return ( + (tile_m * tiles_n + tile_n) * 512 + + (outer % 32) * 16 + + (outer // 32) * 4 + + inner + ) + + +def swizzle_nvfp4_scale_to_128x4( + scale: torch.Tensor, + *, + rows: int, + cols: int, +) -> torch.Tensor: + if scale.ndim != 2: + raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}") + + rows = int(rows) + cols = int(cols) + padded_rows = _round_up(rows, 128) + padded_cols = _round_up(cols, 4) + if scale.shape[0] < rows or scale.shape[1] < cols: + raise ValueError( + "scale is smaller than the requested logical shape: " + f"got {tuple(scale.shape)}, need at least {(rows, cols)}" + ) + + logical = scale[:rows, :cols].contiguous() + if logical.shape != (padded_rows, padded_cols): + logical = torch.nn.functional.pad( + logical.to(torch.float32), + (0, padded_cols - cols, 0, padded_rows - rows), + ).to(scale.dtype) + swizzled = torch.empty_like(logical) + + row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None] + col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1) + swizzled.reshape(-1)[offset] = logical.reshape(-1) + return swizzled + + +def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor: + return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX) + + +def _import_te_nvfp4_quantizer(): + try: + from transformer_engine.pytorch.tensor import NVFP4Quantizer + except Exception as exc: + raise RuntimeError( + "Transformer Engine NVFP4 quantization is unavailable. Install a " + "Transformer Engine build with NVFP4 PyTorch support to use " + "quantize_bf16_to_nvfp4_128x4." + ) from exc + return NVFP4Quantizer + + +def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor: + if not x.is_cuda: + raise ValueError("NVFP4 quantization requires a CUDA tensor") + if x.dtype not in (torch.bfloat16, torch.float16): + raise TypeError(f"x must be bf16 or fp16, got {x.dtype}") + if x.ndim < 2: + raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}") + if x.shape[-1] % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}" + ) + + rows = 1 + for dim in x.shape[:-1]: + rows *= int(dim) + if rows % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + "flattened row dimension must be divisible by " + f"{NVFP4_BLOCK_SIZE}, got {rows}" + ) + + NVFP4Quantizer = _import_te_nvfp4_quantizer() + quantizer = NVFP4Quantizer(rowwise=True, columnwise=False) + qx = quantizer.quantize(x.contiguous()) + meta = qx.get_metadata() + + data = meta["rowwise_data"] + if data.dtype is not torch.uint8: + data = data.view(torch.uint8) + logical_scale = meta["rowwise_scale_inv"] + amax = meta["amax_rowwise"] + scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE + scale_128x4 = swizzle_nvfp4_scale_to_128x4( + logical_scale, + rows=rows, + cols=scale_cols, + ) + global_scale = nvfp4_global_scale_from_amax(amax).contiguous() + + return Nvfp4QuantizedTensor( + data=data, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, scale_cols), + original_shape=tuple(int(v) for v in x.shape), + ) + + +def quantize_kv_bf16_to_nvfp4_128x4( + k: torch.Tensor, + v: torch.Tensor, +) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]: + return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v) + + +def dequantize_nvfp4_128x4_to_bf16( + qx: Nvfp4QuantizedTensor, + *, + include_global_scale: bool = True, +) -> torch.Tensor: + data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8) + if data.shape[-1] * 2 != qx.original_shape[-1]: + raise ValueError( + "packed data last dimension does not match original shape: " + f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical" + ) + + rows, scale_cols = qx.logical_scale_shape + logical_dim = int(qx.original_shape[-1]) + if scale_cols * NVFP4_BLOCK_SIZE != logical_dim: + raise ValueError( + "logical scale columns do not match original last dimension: " + f"{scale_cols} scale cols vs dim {logical_dim}" + ) + + fp4_lut = torch.tensor( + [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + -0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, + ], + dtype=torch.float32, + device=data.device, + ) + packed = data.reshape(rows, logical_dim // 2) + lo = packed & 0x0F + hi = packed >> 4 + values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device) + values[:, 0::2] = fp4_lut[lo.long()] + values[:, 1::2] = fp4_lut[hi.long()] + + row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None] + col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, scale_cols) + scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols) + scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32) + scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1) + out = values * scale + if include_global_scale: + out = out * qx.global_scale.reshape(-1)[0].to(torch.float32) + return out.reshape(qx.original_shape).to(torch.bfloat16) + + +__all__ = [ + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +] diff --git a/build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py b/build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..177f864da993328cc460b8de391c6091f3be0f9d --- /dev/null +++ b/build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Sparse-index helpers compatible with MiniMaxAI/msa public names.""" + +from __future__ import annotations + +from typing import Optional + +import torch + + +def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None: + if cu_seqlens.dtype != torch.int32: + raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}") + if cu_seqlens.ndim != 1: + raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}") + if cu_seqlens.numel() < 1: + raise ValueError(f"{name} must have at least one element") + if not cu_seqlens.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor: + seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1] + return (seqlens_k + kv_block_size - 1) // kv_block_size + + +def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]: + rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist() + batch = len(rows_per_batch_cpu) + max_rows = max(rows_per_batch_cpu, default=0) + row_dtype = ( + torch.int32 + if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max + else torch.int64 + ) + row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype) + row_linear = 0 + for kv_block_idx in range(max_rows): + for batch_idx, row_count in enumerate(rows_per_batch_cpu): + if kv_block_idx < row_count: + row_map_cpu[batch_idx, kv_block_idx] = row_linear + row_linear += 1 + return row_map_cpu.to(rows_per_batch.device), row_linear + + +def build_k2q_csr_torch_reference( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if kv_block_size <= 0: + raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}") + if q2k_indices.dtype != torch.int32: + raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}") + if q2k_indices.ndim != 3: + raise ValueError( + "q2k_indices must have shape [head_kv, total_q, topK], " + f"got {tuple(q2k_indices.shape)}" + ) + _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q") + _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k") + if cu_seqlens_q.shape != cu_seqlens_k.shape: + raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]") + if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device: + raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device") + + head_kv, total_q, topk = q2k_indices.shape + if total_q != int(cu_seqlens_q[-1].item()): + raise ValueError( + f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] " + f"({int(cu_seqlens_q[-1].item())})" + ) + + rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size) + row_map, total_rows = _build_packed_row_map(rows_per_batch) + nnz_upper_bound = total_q * topk + + k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device) + k2q_q_indices = torch.full( + (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device + ) + if total_rows == 0 or total_q == 0 or topk == 0: + return k2q_row_ptr, k2q_q_indices + + counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device) + total_entries = total_q * topk + row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64 + row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device) + q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device) + valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device) + + for batch_idx in range(cu_seqlens_q.numel() - 1): + q_start = int(cu_seqlens_q[batch_idx].item()) + q_end = int(cu_seqlens_q[batch_idx + 1].item()) + if q_end <= q_start: + continue + q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device) + q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1) + kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1) + valid = kv_local >= 0 + safe_kv = kv_local.clamp_min(0).long() + rows = row_map[batch_idx, safe_kv].to(row_dtype) + start = q_start * topk + end = q_end * topk + row_all[:, start:end] = rows + q_all[:, start:end] = q_local[None, :].expand(head_kv, -1) + valid_all[:, start:end] = valid & (rows >= 0) + counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32)) + + k2q_row_ptr[:, 1:] = counts.cumsum(dim=1) + offsets = torch.zeros_like(counts) + offsets[:, 1:] = counts[:, :-1].cumsum(dim=1) + cursor = torch.zeros_like(counts) + for h in range(head_kv): + for entry in range(total_entries): + if not bool(valid_all[h, entry]): + continue + row = int(row_all[h, entry].item()) + pos = int(offsets[h, row].item() + cursor[h, row].item()) + k2q_q_indices[h, pos] = q_all[h, entry] + cursor[h, row] += 1 + + return k2q_row_ptr, k2q_q_indices + + +def build_k2q_csr( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, + *, + total_k: Optional[int] = None, + max_seqlen_k: Optional[int] = None, + max_seqlen_q: Optional[int] = None, + total_rows: Optional[int] = None, + qhead_per_kv: int = 1, + return_schedule: bool = False, +): + if return_schedule: + raise NotImplementedError( + "Blackwell build_k2q_csr currently returns CSR tensors only; " + "SM100 schedule objects are not part of this package." + ) + if total_k is None: + raise ValueError("build_k2q_csr requires total_k from k.shape[0]") + return build_k2q_csr_torch_reference( + q2k_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(kv_block_size), + ) + + +class SparseK2qCsrBuilderSm100: + """Compatibility placeholder for the upstream SM100 class name. + + The name is exported so code can feature-detect it, but schedule-producing + SM100 behavior is intentionally not claimed for this Blackwell package. + """ + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def build(self, *args, **kwargs): + return build_k2q_csr(*args, **kwargs) + + +__all__ = [ + "build_k2q_csr", + "build_k2q_csr_torch_reference", + "SparseK2qCsrBuilderSm100", +] diff --git a/build/torch212-cxx11-cu132-x86_64-linux/__init__.py b/build/torch212-cxx11-cu132-x86_64-linux/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6a666afa7b37c9d5448b9bb04d573dcfe4e45750 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/__init__.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +"""FlashRT Blackwell extension of MiniMax MSA decode sparse attention. + +This package targets NVIDIA Blackwell-family CUDA 12.x GPUs and is validated on +DGX Spark / GB10 / SM121. It exposes the +MiniMax-M3 decode-sparse attention path used by FlashRT's MiniMax Spark runtime. + +Implementation status: + * native CUDA helper: score -> top-k sparse block ids; + * native CUDA tensor-core sparse decode for the MiniMax-M3 Blackwell shape; + * native CUDA FP4 block-score indexer; + * native CUDA helper: swizzled NVFP4 -> BF16 dequant for the W4A16 quality path; + * Blackwell-validated sparse prefill path; + * MiniMaxAI/msa compatibility wrappers for CSR prefill, decode, NVFP4 helpers, + and FP4 block-score helpers. + +The upstream MiniMaxAI/msa package is SM100-only; this package is the Blackwell +extension path. The public API is Tensor-oriented and independent from +FlashRT's serving runtime. + +Public API +---------- +Lightning indexer (q.k blockmax score -> top-k block select), paged KV: + flash_decode_with_topk_idx (single-token decode, split-K) + +Block-sparse GQA attention (consumes block_indices from the indexer): + flash_decode_with_gqa_share_sparse (decode, split-K over top-k blocks) + sparse_atten_func (official CSR prefill wrapper) + sparse_decode_atten_func (official paged decode wrapper) + +PyTorch naive references (for correctness checks): + naive_flash_decode_with_gqa_share_sparse (attention given topk_idx) + naive_flash_decode_with_topk_idx (joint indexer + attention) + +All Triton entrypoints expect a paged KV cache: + k_cache / v_cache : [max_slots, num_kv_heads, head_dim] + req_to_token : [max_reqs, max_kv_len] (logical pos -> physical slot) + slot_ids : [batch] (per-request row in req_to_token) + topk_idx : [num_kv_heads, n, topk] int32, valid ids left-packed, + -1 right-padding (the M3 indexer contract). +See VENDOR_NOTES.md for full shape/dtype/layout contracts. +""" + +from .common.utils import get_cu_seqblocks, robust_allocator +from .decode.flash_with_topk_idx import flash_decode_with_topk_idx +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .interface import ( + SparseDecodePagedAttentionWrapper, + fp4_indexer_block_scores, + sparse_atten_func, + sparse_atten_nvfp4_kv_func, + sparse_decode_atten_func, +) +from ._native import ( + has_native_ops, + native_nvfp4_dequant_swizzled_to_bf16, + native_topk_from_scores, +) +from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx +from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse +from .quantize import ( + Nvfp4QuantizedTensor, + dequantize_nvfp4_128x4_to_bf16, + nvfp4_global_scale_from_amax, + quantize_bf16_to_nvfp4_128x4, + quantize_kv_bf16_to_nvfp4_128x4, + swizzle_nvfp4_scale_to_128x4, +) +from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr +from .api_status import ( + OFFICIAL_API_STATUS, + OFFICIAL_MINIMAX_MSA_FUNCTIONS, + V1_AVAILABLE_FUNCTIONS, + available_functions, + official_api_status, + official_minimax_msa_functions, + unsupported_official_functions, +) + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "naive_flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "get_cu_seqblocks", + "robust_allocator", + "OFFICIAL_API_STATUS", + "OFFICIAL_MINIMAX_MSA_FUNCTIONS", + "V1_AVAILABLE_FUNCTIONS", + "available_functions", + "official_api_status", + "official_minimax_msa_functions", + "unsupported_official_functions", +] diff --git a/build/torch212-cxx11-cu132-x86_64-linux/_compat.py b/build/torch212-cxx11-cu132-x86_64-linux/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..da445342268947ab3b1c5bfd16e95dc4705c7abe --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/_compat.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# Framework-decoupling shim for the vendored MiniMax-M3 sparse-attention Triton +# kernels. The upstream SGLang modules pull two tiny things from the framework: +# +# from sglang.srt.utils import is_hip -> HIP/ROCm detection (fp8 KV path) +# from sglang.srt.environ import envs -> a feature flag for an optional +# JIT-CUDA radix top-k fast path +# +# Both are replaced here so the kernels import with only torch + triton. The +# defaults select the pure-Triton CUDA path (no HIP fp8 widening, no jit_kernel +# radix select), which is what we want on sm_121 (GB10). + + +def is_hip() -> bool: + """Return True on AMD/ROCm builds. Always False here (we target CUDA sm_121). + + Used by the decode topk_sparse kernels to gate the `IS_FP8` paged-KV + widening branch (HIP-only). With this False, the CUDA dtype contract + (K/V cache dtype == Q dtype) is enforced exactly as upstream on CUDA. + """ + try: + import torch + + return bool(getattr(torch.version, "hip", None)) + except Exception: + return False + + +class _Flag: + """Minimal stand-in for an sglang.srt.environ boolean flag (always off).""" + + def __init__(self, default: bool = False): + self._default = default + + def get(self) -> bool: + return self._default + + +class _Envs: + """Stub for `sglang.srt.environ.envs`. + + Only `SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX` is referenced by the decode + indexer. Keeping it False routes top-k selection through the 2-stage Triton + fallback (no `sglang.jit_kernel.minimax_decode_topk` CUDA import), so the + decode indexer stays pure-Triton. + """ + + SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX = _Flag(False) + + +envs = _Envs() diff --git a/build/torch212-cxx11-cu132-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so b/build/torch212-cxx11-cu132-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..853be9a48ace4f26fe441be86be3220e9cf6a863 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/_minimaxai_msa_blackwell_cuda_25b0067.abi3.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb01c1a96c3e640474b3a52f2d38da4be2af0290c4b945c50f90e867d9e907f +size 744080 diff --git a/build/torch212-cxx11-cu132-x86_64-linux/_native.py b/build/torch212-cxx11-cu132-x86_64-linux/_native.py new file mode 100644 index 0000000000000000000000000000000000000000..dd54c53374469c8424d0ca6e3f25ea534005c8d5 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/_native.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Optional native CUDA helpers for the FlashRT Blackwell MSA package.""" + +from __future__ import annotations + +import torch + +try: + from ._ops import add_op_namespace_prefix, ops + + _HAS_NATIVE_OPS = True +except Exception: + add_op_namespace_prefix = None + ops = None + _HAS_NATIVE_OPS = False + + +def has_native_ops() -> bool: + return _HAS_NATIVE_OPS + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake(add_op_namespace_prefix("msa_topk_from_scores")) + def _msa_topk_from_scores_fake( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, + topk_idx: torch.Tensor, + ) -> None: + if score.dim() != 3: + raise RuntimeError("score must have shape (heads, batch, max_blocks)") + heads, batch, _ = score.shape + if seq_lens.shape != (batch,): + raise RuntimeError("seq_lens must have shape (batch,)") + if topk_idx.shape != (heads, batch, topk): + raise RuntimeError("topk_idx must have shape (heads, batch, topk)") + return None + + +def native_topk_from_scores( + score: torch.Tensor, + seq_lens: torch.Tensor, + block_size: int, + topk: int, +) -> torch.Tensor: + """Return top-k sparse block ids from a score tensor.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if not score.is_contiguous(): + score = score.contiguous() + if not seq_lens.is_contiguous(): + seq_lens = seq_lens.contiguous() + out = torch.empty( + (score.shape[0], score.shape[1], int(topk)), + device=score.device, + dtype=torch.int32, + ) + ops.msa_topk_from_scores(score, seq_lens, int(block_size), int(topk), out) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_decode_sparse_attn_mma_paged") + ) + def _msa_decode_sparse_attn_mma_paged_fake( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size: int, + sm_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_decode_mma_supported( + q: torch.Tensor, + k_cache: torch.Tensor, + *, + block_size: int, +) -> bool: + """Whether the tensor-core decode kernel applies to these inputs. + + The mma kernel is specialized for the M3 MSA shape: head_dim 128, GQA + group (Hq/Hkv) 16, bf16, and a block size that is a multiple of 64. + """ + + if not _HAS_NATIVE_OPS: + return False + if q.dtype is not torch.bfloat16 or k_cache.dtype is not torch.bfloat16: + return False + if q.dim() != 3 or k_cache.dim() != 3: + return False + head_dim = int(q.shape[-1]) + num_q_heads = int(q.shape[1]) + num_kv_heads = int(k_cache.shape[1]) + if head_dim != 128 or num_kv_heads <= 0: + return False + if num_q_heads % num_kv_heads != 0 or num_q_heads // num_kv_heads != 16: + return False + if int(block_size) % 64 != 0: + return False + return True + + +def native_decode_sparse_attn_mma_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + seq_lens: torch.Tensor, + slot_ids: torch.Tensor, + block_size: int, + topk_idx: torch.Tensor, + sm_scale: float, +) -> torch.Tensor: + """Paged tensor-core block-sparse GQA decode. Returns [B, Hq, D] bf16.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + out = torch.empty( + (int(q.shape[0]), int(q.shape[1]), int(q.shape[2])), + device=q.device, + dtype=q.dtype, + ) + ops.msa_decode_sparse_attn_mma_paged( + q.contiguous(), + k_cache.contiguous(), + v_cache.contiguous(), + req_to_token.contiguous(), + seq_lens.contiguous(), + slot_ids.contiguous(), + topk_idx.contiguous(), + int(block_size), + float(sm_scale), + out, + ) + return out + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_indexer_block_scores") + ) + def _msa_indexer_block_scores_fake( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + causal: int, + scores: torch.Tensor, + ) -> None: + return None + + +def native_indexer_block_scores( + q: torch.Tensor, + k_pages: torch.Tensor, + batch_of_q: torch.Tensor, + cu_q: torch.Tensor, + cu_k: torch.Tensor, + cu_pages: torch.Tensor, + kv_indices: torch.Tensor, + *, + max_blocks: int, + causal: bool, +) -> torch.Tensor: + """Block-max QK indexer scores. q/k_pages are bf16 (already dequantized). + + Returns [Hq, max_blocks, total_q] f32 (-inf where no key is visible). + """ + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + total_q = int(q.shape[0]) + hq = int(q.shape[1]) + scores = torch.full( + (hq, int(max_blocks), total_q), + float("-inf"), + dtype=torch.float32, + device=q.device, + ) + ops.msa_indexer_block_scores( + q.contiguous(), + k_pages.contiguous(), + batch_of_q.contiguous(), + cu_q.contiguous(), + cu_k.contiguous(), + cu_pages.contiguous(), + kv_indices.contiguous(), + 1 if causal else 0, + scores, + ) + return scores + + +if _HAS_NATIVE_OPS: + + @torch.library.register_fake( + add_op_namespace_prefix("msa_nvfp4_dequant_swizzled_to_bf16") + ) + def _msa_nvfp4_dequant_swizzled_to_bf16_fake( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: float, + out: torch.Tensor, + ) -> None: + return None + + +def native_nvfp4_dequant_swizzled_to_bf16( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor | float, +) -> torch.Tensor: + """Expand swizzled NVFP4 weights/KV to dense BF16 with the native CUDA path.""" + + if not _HAS_NATIVE_OPS: + raise RuntimeError("native MiniMax MSA ops are not available in source-tree mode") + if packed.dtype is not torch.uint8: + raise TypeError("packed must be torch.uint8") + if scale_128x4.dtype is not torch.uint8: + raise TypeError("scale_128x4 must be torch.uint8") + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + out = torch.empty(original_shape, device=packed.device, dtype=torch.bfloat16) + if isinstance(global_scale, torch.Tensor): + global_scale_value = float(global_scale.reshape(-1)[0].item()) + else: + global_scale_value = float(global_scale) + ops.msa_nvfp4_dequant_swizzled_to_bf16( + packed.contiguous(), + scale_128x4.contiguous(), + global_scale_value, + out, + ) + return out + + +__all__ = [ + "has_native_ops", + "native_topk_from_scores", + "native_decode_mma_supported", + "native_decode_sparse_attn_mma_paged", + "native_indexer_block_scores", + "native_nvfp4_dequant_swizzled_to_bf16", +] diff --git a/build/torch212-cxx11-cu132-x86_64-linux/_ops.py b/build/torch212-cxx11-cu132-x86_64-linux/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..41e39102ea6d028564a7e10c3b7004e7c40dae7b --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/_ops.py @@ -0,0 +1,9 @@ +import torch +from . import _minimaxai_msa_blackwell_cuda_25b0067 +ops = torch.ops._minimaxai_msa_blackwell_cuda_25b0067 + +def add_op_namespace_prefix(op_name: str): + """ + Prefix op by namespace. + """ + return f"_minimaxai_msa_blackwell_cuda_25b0067::{op_name}" diff --git a/build/torch212-cxx11-cu132-x86_64-linux/api_status.py b/build/torch212-cxx11-cu132-x86_64-linux/api_status.py new file mode 100644 index 0000000000000000000000000000000000000000..3dd4d744b49f1bc7acd6b3ba2a2e2c5f7b591252 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/api_status.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Public API status for the FlashRT MiniMax MSA Blackwell package.""" + +from __future__ import annotations + +from types import MappingProxyType + + +V1_AVAILABLE_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", + "has_native_ops", + "native_topk_from_scores", + "native_nvfp4_dequant_swizzled_to_bf16", + "flash_decode_with_topk_idx", + "flash_decode_with_gqa_share_sparse", + "naive_flash_decode_with_topk_idx", + "naive_flash_decode_with_gqa_share_sparse", + "get_cu_seqblocks", + "robust_allocator", +) + +OFFICIAL_MINIMAX_MSA_FUNCTIONS = ( + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", + "build_k2q_csr", + "SparseK2qCsrBuilderSm100", + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +) + +_OFFICIAL_API_STATUS = { + "sparse_atten_func": { + "status": "available", + "target": "blackwell-prefill", + "reason": "Official CSR sparse prefill API maps to the Blackwell Triton BF16/FP16 prefill wrapper.", + }, + "sparse_atten_nvfp4_kv_func": { + "status": "available", + "target": "blackwell-prefill-nvfp4", + "reason": "Official NVFP4 KV prefill API is available via native swizzled NVFP4 dequantization followed by the Blackwell prefill wrapper.", + }, + "sparse_decode_atten_func": { + "status": "available", + "target": "v1", + "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.", + }, + "SparseDecodePagedAttentionWrapper": { + "status": "available", + "target": "v1", + "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.", + }, + "fp4_indexer_block_scores": { + "status": "available", + "target": "blackwell-fp4-indexer", + "reason": "Official FP4 block-score API uses the native Blackwell block-score kernel in built artifacts, with a reference fallback in source-tree mode.", + }, + "build_k2q_csr": { + "status": "available", + "target": "v1", + "reason": "CSR construction helper is available for the official prefill API.", + }, + "SparseK2qCsrBuilderSm100": { + "status": "available", + "target": "v1", + "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.", + }, + "Nvfp4QuantizedTensor": { + "status": "available", + "target": "v1", + "reason": "NVFP4 metadata dataclass is available.", + }, + "quantize_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "quantize_kv_bf16_to_nvfp4_128x4": { + "status": "available_optional_te", + "target": "v1", + "reason": "Function is available and uses Transformer Engine when installed.", + }, + "dequantize_nvfp4_128x4_to_bf16": { + "status": "available", + "target": "v1", + "reason": "Reference dequant helper is available.", + }, + "swizzle_nvfp4_scale_to_128x4": { + "status": "available", + "target": "v1", + "reason": "Scale swizzle helper is available.", + }, + "nvfp4_global_scale_from_amax": { + "status": "available", + "target": "v1", + "reason": "Global scale helper is available.", + }, +} + +OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS) + + +def available_functions() -> tuple[str, ...]: + """Return the functions/classes intentionally available in v1.""" + + return V1_AVAILABLE_FUNCTIONS + + +def official_minimax_msa_functions() -> tuple[str, ...]: + """Return the upstream MiniMaxAI/msa public function/class names tracked here.""" + + return OFFICIAL_MINIMAX_MSA_FUNCTIONS + + +def official_api_status() -> dict[str, dict[str, str]]: + """Return a copy of the official API compatibility status table.""" + + return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()} + + +def unsupported_official_functions() -> tuple[str, ...]: + """Return official MiniMaxAI/msa names that are not exported by this v1 package.""" + + return tuple( + name + for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS + if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"} + ) diff --git a/build/torch212-cxx11-cu132-x86_64-linux/common/__init__.py b/build/torch212-cxx11-cu132-x86_64-linux/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/common/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch212-cxx11-cu132-x86_64-linux/common/utils.py b/build/torch212-cxx11-cu132-x86_64-linux/common/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..81b7816b719973729312c0d84b36f9f50579f105 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/common/utils.py @@ -0,0 +1,144 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +import functools +from collections import deque +from typing import Any, Callable, Tuple + +import torch +import triton.language as tl + +_tma_keep_alive_buf = deque(maxlen=200) + +try: + make_tensor_descriptor = tl.make_tensor_descriptor +except Exception: + make_tensor_descriptor = tl._experimental_make_tensor_descriptor + + +def robust_allocator(size: int, alignment: int, stream: int = None): + """Allocator for Triton TMA descriptors. + + We keep reference in deque to prevent GC from collecting the buffer. + """ + tensor = torch.empty(size, device="cuda", dtype=torch.uint8) + _tma_keep_alive_buf.append(tensor) + return tensor + + +def tensor_cache(maxsize: int = 8): + """ + Cache function results using identity comparison. + Zero-overhead cache hit: no hash, no DtoH, just pointer comparison. + + Args: + maxsize: Maximum number of cached entries. Supports multi-GPU scenarios + where different devices have different tensor arguments. + """ + + def decorator(fn: Callable[..., Any]) -> Callable[..., Any]: + # LRU-style cache: list of (args, kwargs, result) tuples + # Most recently used at the end + _cache: list = [] + + def _args_match(args: tuple, cached_args: tuple) -> bool: + if len(args) != len(cached_args): + return False + for i in range(len(args)): + if args[i] is not cached_args[i]: + return False + return True + + def _kwargs_match(kwargs: dict, cached_kwargs: dict) -> bool: + if not kwargs and not cached_kwargs: + return True + if kwargs.keys() != cached_kwargs.keys(): + return False + for k, v in kwargs.items(): + if v is not cached_kwargs[k]: + return False + return True + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + # Search cache (most recent first for better hit rate) + for i in range(len(_cache) - 1, -1, -1): + cached_args, cached_kwargs, cached_result = _cache[i] + if _args_match(args, cached_args) and _kwargs_match( + kwargs, cached_kwargs + ): + # Move to end (most recently used) + if i != len(_cache) - 1: + _cache.append(_cache.pop(i)) + return cached_result + + # Cache miss + result = fn(*args, **kwargs) + + # Add to cache + if len(_cache) >= maxsize: + _cache.pop(0) # Remove oldest + _cache.append((args, kwargs, result)) + return result + + # Expose cache for manual clearing if needed + wrapper.cache_clear = lambda: _cache.clear() + wrapper.cache_info = lambda: {"size": len(_cache), "maxsize": maxsize} + + return wrapper + + return decorator + + +@tensor_cache(maxsize=8) +def get_cu_seqblocks( + cu_seqlens: torch.Tensor, + max_seqlen: int, + block_size_q: int, + block_size_k: int, +) -> Tuple[torch.Tensor, int, int, torch.Tensor, int, int]: + """Compute cumulative sequence block indices for blocked sparse attention. + + Converts token-level cumulative sequence lengths to block-level indices, + which are needed for block-sparse attention kernels. + + Note: + Results are cached (maxsize=8) based on input arguments. Repeated calls + with the same cu_seqlens, max_seqlen, and block sizes will return cached + results without recomputation. + + Args: + cu_seqlens: Cumulative sequence lengths. Shape: [batch_size + 1], dtype: int32. + max_seqlen: Maximum sequence length in the batch. + block_size_q: Query block size. + block_size_k: Key-value block size. + + Returns: + A tuple of 6 values: + - cu_seqblocks_q: Cumulative query block indices. Shape: [batch_size + 1] + - max_seqblock_q: Maximum number of query blocks per sequence. + - all_seqblock_q: Total number of query blocks across all sequences. + - cu_seqblocks_k: Cumulative key block indices. Shape: [batch_size + 1] + - max_seqblock_k: Maximum number of key blocks per sequence. + - all_seqblock_k: Total number of key blocks across all sequences. + """ + cu_seqblocks_q = torch.zeros_like(cu_seqlens) + cu_seqblocks_k = torch.zeros_like(cu_seqlens) + seq_lens = torch.diff(cu_seqlens) + seqblocks_q = (seq_lens + block_size_q - 1) // block_size_q + seqblocks_k = (seq_lens + block_size_k - 1) // block_size_k + max_seqblock_q = (max_seqlen + block_size_q - 1) // block_size_q + max_seqblock_k = (max_seqlen + block_size_k - 1) // block_size_k + cu_seqblocks_q[1:] = seqblocks_q + cu_seqblocks_k[1:] = seqblocks_k + cu_seqblocks_q.cumsum_(0) + cu_seqblocks_k.cumsum_(0) + all_seqblock_q = seqblocks_q.sum().item() + all_seqblock_k = seqblocks_k.sum().item() + return ( + cu_seqblocks_q, + max_seqblock_q, + all_seqblock_q, + cu_seqblocks_k, + max_seqblock_k, + all_seqblock_k, + ) diff --git a/build/torch212-cxx11-cu132-x86_64-linux/decode/__init__.py b/build/torch212-cxx11-cu132-x86_64-linux/decode/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/decode/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch212-cxx11-cu132-x86_64-linux/decode/flash_with_topk_idx.py b/build/torch212-cxx11-cu132-x86_64-linux/decode/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..8f56bdba2908e8af036785eae158338758555ee5 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/decode/flash_with_topk_idx.py @@ -0,0 +1,1150 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import envs # vendored: was `from sglang.srt.environ import envs` + +from ..common.utils import robust_allocator +from .._native import has_native_ops, native_topk_from_scores + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "BATCH_SIZE_BUCKET", + "gqa_group_size", + "head_dim", + "block_size", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_kernel( + q_ptr, # Q: b x qh x d + k_cache_ptr, # K paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_r2t_b, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks <= topk: + return + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = (tl.load(slot_ids + pid_b).to(tl.int64) + max_slots) % max_slots + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # prefetch first iteration's slots + r2t_base = req_to_token_ptr + sid * stride_r2t_b + prefetch_pos = chunk_start + off_n + prefetch_mask = prefetch_pos < seq_len + prefetched_slots = tl.load( + r2t_base + prefetch_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + # score-only: compute block scores without loading V + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos_mask = prefetch_mask + slots = prefetched_slots + # prefetch next iteration's slots + next_i = i + BLOCK_SIZE_N + if next_i < chunk_end: + next_pos = next_i + off_n + prefetch_mask = next_pos < seq_len + prefetched_slots = tl.load( + r2t_base + next_pos, + mask=prefetch_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + curr_block_idx = i // block_size + off_bpk + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + tl.store(s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1)) + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_N": BN}, num_warps=nw, num_stages=ns) + for BN in [64, 128, 256, 512] + for nw in [4, 8, 16] + for ns in [1, 2, 3] + ], + key=[ + "gqa_group_size", + "head_dim", + "block_size", + "HAS_SINK", + "SCORE_TYPE", + ], +) +@triton.jit +def _decode_score_attn_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + o_ptr, # O: c x b x qh x d + lse_ptr, # lse: c x b x qh + score_ptr, # Score: qh x b x max_seqblock + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + # block size + block_size: tl.constexpr, + topk: tl.constexpr, + # sm_scale + sm_scale, + # init and local blocks + init_blocks, + local_blocks, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + stride_s_h, + stride_s_b, + stride_s_n, + # META parameters + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + NUM_KV_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + SCORE_TYPE: tl.constexpr, + SKIP_TRIVIAL_TOPK_SCORE: tl.constexpr, +): + tl.static_assert(SCORE_TYPE == "max" or SCORE_TYPE == "lse") + sm_scale_log2e = sm_scale * 1.4426950409 + tl.static_assert(BLOCK_SIZE_N >= block_size) + BLOCKS_PER_K_BLOCK: tl.constexpr = BLOCK_SIZE_N // block_size + # get batch id and head id + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # block-aligned fixed-count chunked decode (grid independent of seq_len for cuda graph) + seq_len = tl.load(seq_lens + pid_b).to(tl.int32) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + chunk_start_block = pid_c * chunk_size_blocks + chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks) + chunk_start = chunk_start_block * block_size + chunk_end = tl.minimum(chunk_end_block * block_size, seq_len) + if chunk_start_block >= chunk_end_block: + return + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + s_ptrs = tl.make_block_ptr( + base=score_ptr + pid_b * stride_s_b + pid_h * stride_s_h, + shape=(gqa_group_size, chunk_end_block), + strides=(stride_s_h, stride_s_n), + offsets=(0, chunk_start_block), + block_shape=(BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK), + order=(1, 0), + ) + # load q + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + # init statistics + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + off_bpk = tl.arange(0, BLOCKS_PER_K_BLOCK) + dim_mask = off_d < head_dim + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + qsink = ( + tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale_log2e + ) # (BLOCK_SIZE_H,) + m_i = qsink + l_i = tl.full((BLOCK_SIZE_H,), 1.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + else: + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + l_i = tl.full((BLOCK_SIZE_H,), 0.0, dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # pre-compute local_start outside the loop (constant for entire batch) + local_start = tl.maximum(0, num_blocks - local_blocks) + # full attention or causal attention + for i in range(chunk_start, chunk_end, BLOCK_SIZE_N): + pos = i + off_n + pos_mask = pos < seq_len + # resolve slot per position via req_to_token + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < chunk_end - i, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale_log2e + # save qk to score + score = tl.reshape( + qk, + (BLOCK_SIZE_H, BLOCKS_PER_K_BLOCK, block_size), + can_reorder=False, + ) + sub_max = tl.max(score, axis=2) + if SCORE_TYPE == "max": + score = sub_max + else: # "lse" + score = sub_max + tl.log2( + tl.sum(tl.exp2(score - sub_max[:, :, None]), axis=2) + ) + score = tl.where(score != score, float("-inf"), score) + # apply init_blocks and local_blocks efficiently + # current block indices: [BLOCKS_PER_K_BLOCK] + curr_block_idx = i // block_size + off_bpk + # Combined condition with single nested tl.where: + # - local_blocks (1e29) takes priority over init_blocks (1e30) + # - When init_blocks=0: curr_block_idx < 0 is always False + # - When local_blocks=0: local_start=num_blocks, so curr_block_idx >= num_blocks is always False + is_init = curr_block_idx < init_blocks + is_local = (curr_block_idx >= local_start) & (curr_block_idx < num_blocks) + # Apply in one fused operation: local > init > original + score = tl.where( + is_local[None, :], 1e29, tl.where(is_init[None, :], 1e30, score) + ) + if SKIP_TRIVIAL_TOPK_SCORE: + if num_blocks > topk: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + else: + tl.store( + s_ptrs, score.to(score_ptr.dtype.element_ty), boundary_check=(0, 1) + ) + # max-of-max == max(qk), avoids re-scanning qk + m_ij = tl.maximum(m_i, tl.max(sub_max, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + m_i = m_ij + l_i = l_i * acc_o_scale + l_ij + # update ptrs + s_ptrs = tl.advance(s_ptrs, (0, BLOCKS_PER_K_BLOCK)) + # final scale + acc_o = acc_o / l_i[:, None] + # compute lse in log2 scale for merge kernel + lse_i = m_i + tl.log2(l_i) + # save output + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + # save lse (in log2 scale, will be used directly in merge kernel) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4] + ], + key=["BLOCK_SIZE_D"], + restore_value=["o_ptr"], +) +@triton.jit +def _merge_attn_out_kernel( + o_ptr, + lse_ptr, + # shape + seq_lens, + head_dim, + block_size: tl.constexpr, + # stride + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_KV_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + # recompute chunk layout (must match _decode_score_attn_kernel) + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + chunk_size_blocks = tl.cdiv(num_blocks, NUM_KV_CHUNKS) + valid_chunks = tl.cdiv(num_blocks, chunk_size_blocks) + # ptrs + off_c = tl.arange(0, NUM_KV_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(valid_chunks, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_KV_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + # load o and lse + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs, mask=off_c < valid_chunks, other=float("-inf")) + # merge o (lse is in log2 scale, use exp2/log2 for efficiency) + lse_shifted = lse - tl.max(lse, axis=0) + log_sum = tl.log2(tl.sum(tl.exp2(lse_shifted), axis=0)) + scale = tl.exp2(lse_shifted - log_sum) + o = tl.sum(o * scale[:, None], axis=0) + # save o + o_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_ptrs, o.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@triton.jit +def _compare_and_swap( + x, + ids, + flip, + i: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)] + y = tl.reshape(x, shape) + # slice left/right with 'stride' 2**(n_dims - i - 1) + mask = tl.arange(0, 2)[None, :, None] + left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype) + right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype) + left = tl.reshape(left, x.shape) + right = tl.reshape(right, x.shape) + # idx + y_idx = tl.reshape(ids, shape) + left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape) + right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape) + left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype) + right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype) + # actual compare-and-swap + idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True) + ileft = left.to(idtype, bitcast=True) + iright = right.to(idtype, bitcast=True) + ix = x.to(idtype, bitcast=True) + + cond = (left > right) != flip + ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix)) + new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids)) + return ret.to(x.dtype, bitcast=True), new_ids + + +@triton.jit +def _bitonic_merge( + x, + ids, + stage: tl.constexpr, + order: tl.constexpr, + n_dims: tl.constexpr, +): + n_outer: tl.constexpr = x.numel >> n_dims + tl.static_assert(stage <= n_dims) + # flip denotes whether to re-arrange sub-sequences of elements in ascending or + # descending order. + # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage + # if flip = 00110011... then all the elements will be re-arranged alternatingly (with + # a stride of 2) at this stage + if order == 2: + shape: tl.constexpr = [ + n_outer * 2 ** (n_dims - 1 - stage), + 2, + 2**stage, + ] + flip = tl.reshape( + tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape + ) + else: + flip = order + # perform `stage` rounds of `compare-and-swap` + for i in tl.static_range(stage): + x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims) + return x, ids + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + } +) +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 256}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=3), + triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2), + ], + key=["topk"], +) +@triton.jit +def _topk_index_partial_kernel( + s_ptr, # Score: [num_q_heads, BS, max_seqblock] + ts_partial_ptr, # Partial scores out: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx out (1-indexed global, 0=invalid): same shape + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + chunk_blocks: tl.constexpr, # how many score-blocks each chunk owns + # strides + stride_s_h, + stride_s_b, + stride_s_k, + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + # META + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + tl.static_assert(BLOCK_SIZE_K > topk) + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + pid_chunk = tl.program_id(2) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Slice this chunk owns within [0, num_blocks). + chunk_start = pid_chunk * chunk_blocks + chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks) + chunk_actual = tl.maximum(chunk_end - chunk_start, 0) + + off_k = tl.arange(0, BLOCK_SIZE_K) + off_t = tl.arange(0, BLOCK_SIZE_T) + + # Pointer starts at chunk_start. + s_ptrs = ( + s_ptr + + pid_b * stride_s_b + + pid_h * stride_s_h + + (chunk_start + off_k) * stride_s_k + ) + + topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32) + topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32) + left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2 + + # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty + # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0. + for i in tl.range(0, chunk_actual, BLOCK_SIZE_K): + mask = off_k < chunk_actual - i + score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32) + score = tl.where(score != score, -1e30, score) + s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K + topk_score, last_topk_score = score, topk_score + topk_idx, last_topk_idx = ( + tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global + topk_idx, + ) + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), j, 2, n_dims + ) + if i != 0: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims + ) + topk_score_new = last_topk_score * left_half_mask + topk_score * ( + 1 - left_half_mask + ) + topk_idx_new = last_topk_idx * left_half_mask + topk_idx * ( + 1 - left_half_mask + ) + topk_score, topk_idx = _bitonic_merge( + topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims + ) + else: + topk_score, topk_idx = _bitonic_merge( + topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims + ) + + # Extract first BLOCK_SIZE_T entries (top-K of this chunk after descending sort). + topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + final_score = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + final_idx = tl.sum( + topk_mask_extract[:, None] + * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + # Store partial. Always write all BLOCK_SIZE_T slots — invalid slots carry + # -1e30 / 0 sentinels and lose to real scores in the merge stage. + ts_ptrs = ( + ts_partial_ptr + + pid_chunk * stride_ts_c + + pid_b * stride_ts_b + + pid_h * stride_ts_h + + off_t * stride_ts_t + ) + ti_ptrs = ( + ti_partial_ptr + + pid_chunk * stride_ti_c + + pid_b * stride_ti_b + + pid_h * stride_ti_h + + off_t * stride_ti_t + ) + tl.store(ts_ptrs, final_score) + tl.store(ti_ptrs, final_idx) + + +@triton.heuristics( + { + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]), + "BLOCK_SIZE_K": lambda args: triton.next_power_of_2( + args["NUM_TOPK_CHUNKS"] * triton.next_power_of_2(args["topk"]) + ), + } +) +@triton.jit +def _topk_index_merge_kernel( + ts_partial_ptr, # Partial scores: [NUM_TOPK_CHUNKS, num_q_heads, BS, BLOCK_SIZE_T] + ti_partial_ptr, # Partial idx (1-indexed global, 0=invalid): same shape + ti_final_ptr, # Final idx (0-indexed global, -1=invalid): [num_q_heads, BS, topk] + seq_lens, + block_size: tl.constexpr, + topk: tl.constexpr, + # strides + stride_ts_c, + stride_ts_h, + stride_ts_b, + stride_ts_t, + stride_ti_c, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_tif_h, + stride_tif_b, + stride_tif_t, + # META + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + pid_b = tl.program_id(0) + pid_h = tl.program_id(1) + + seq_len = tl.load(seq_lens + pid_b) + num_blocks = (seq_len + block_size - 1) // block_size + + # Layout: load NUM_TOPK_CHUNKS * BLOCK_SIZE_T candidates, padded to BLOCK_SIZE_K. + # candidate at flat position p comes from chunk = p // BLOCK_SIZE_T, + # in_chunk = p % BLOCK_SIZE_T. + off = tl.arange(0, BLOCK_SIZE_K) + chunk_idx = off // BLOCK_SIZE_T + in_chunk_idx = off % BLOCK_SIZE_T + valid = chunk_idx < NUM_TOPK_CHUNKS + + score_offset = ( + chunk_idx * stride_ts_c + + pid_h * stride_ts_h + + pid_b * stride_ts_b + + in_chunk_idx * stride_ts_t + ) + idx_offset = ( + chunk_idx * stride_ti_c + + pid_h * stride_ti_h + + pid_b * stride_ti_b + + in_chunk_idx * stride_ti_t + ) + + score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to( + tl.float32 + ) + score = tl.where(score != score, -1e30, score) + idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32) + + # Full bitonic descending sort of BLOCK_SIZE_K items. + n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K) + for j in tl.static_range(1, n_dims): + score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims) + score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims) + + # Extract first BLOCK_SIZE_T positions — these are the global top-K. + extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0 + topk_idx_final = tl.sum( + extract_mask[:, None] + * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]), + axis=0, + ) + + off_t = tl.arange(0, BLOCK_SIZE_T) + tif_ptrs = ( + ti_final_ptr + + pid_h * stride_tif_h + + pid_b * stride_tif_b + + off_t * stride_tif_t + ) + topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1) + tl.store(tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty)) + + +@torch.no_grad() +def flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: Optional[torch.Tensor], # paged; ignored when disable_index_value=True + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + init_blocks: int, + local_blocks: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + score_type: str = "max", + disable_index_value: bool = False, + use_dense_main_attn: bool = False, # NOTE: need transform idx in this case + page_size: int = 1, +) -> torch.Tensor: + assert score_type in ( + "max", + "lse", + ), f"score_type must be 'max' or 'lse', got {score_type!r}" + triton.set_allocator(robust_allocator) + # dtype check + assert ( + q.dtype == torch.bfloat16 + or q.dtype == torch.float16 + and k_cache.dtype == q.dtype + ) + if not disable_index_value: + assert v_cache is not None + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + max_kv_len = req_to_token.shape[1] + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # NUM_KV_CHUNKS controls how many parallel chunks each (batch, kv_head) gets. + # Total CTAs = batch_size * NUM_KV_CHUNKS * num_kv_heads. + # TARGET_GRID is the desired total CTA count; NUM_KV_CHUNKS is derived by: + # NUM_KV_CHUNKS = clamp(TARGET_GRID // (BS * num_kv_heads), 1, MAX_NUM_KV_CHUNKS) + # then rounded down to the nearest power of 2 (merge kernel requires it). + # Must only depend on cuda-graph-constant quantities (BS, num_kv_heads), not seq_len. + # Empty chunks early-return cheaply, so over-chunking is nearly free. + # E.g. with num_kv_heads=1: BS=1→NKC=256,CTAs=256; BS=32→NKC=128,CTAs=4096. + TARGET_GRID = 4096 + MAX_NUM_KV_CHUNKS = 256 + target = max( + 1, + min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_KV_CHUNKS = 1 << (target.bit_length() - 1) + score_kv_len = min(max_seqlen, max_kv_len) + # The score producers below write every valid block column + # [0, ceil(seq_len / block_size)) for each (head, batch) row. All consumers + # clamp their scan to the same per-row valid block count, so columns beyond + # seq_len are never read. Avoid a full-tensor -inf memset here; on CUDA graph + # capture the static score shape can be much larger than the live context. + score = torch.empty( + (num_q_heads, batch_size, triton.cdiv(score_kv_len, block_size)), + dtype=torch.float32, + device=q.device, + ) + use_jit_topk = ( + envs.SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX.get() + and score.shape[2] <= 4096 + and topk <= 32 + ) + # If the live context has <= topk sparse blocks, the downstream dense + # page-table/JIT top-k kernels select every block from seq_lens directly + # without reading score. Keep this gate in sync with the consumers below: + # _skip_block_topk/use_dense_main_attn and use_jit_topk special-case + # num_blocks <= topk, while the Triton fallback reads score and must not + # skip these writes. + skip_trivial_topk_score = use_dense_main_attn or use_jit_topk + + grid = (batch_size * NUM_KV_CHUNKS, num_kv_heads) + if disable_index_value: + _decode_score_kernel[grid]( + q, + k_cache, + req_to_token, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + req_to_token.stride(0), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + else: + assert v_cache is not None + o = torch.empty( + NUM_KV_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse = torch.empty( + NUM_KV_CHUNKS, batch_size, num_q_heads, dtype=torch.float32, device=q.device + ) + _decode_score_attn_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + o, + lse, + score, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + block_size, + topk, + sm_scale, + init_blocks, + local_blocks, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + score.stride(0), + score.stride(1), + score.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + SCORE_TYPE=score_type, + SKIP_TRIVIAL_TOPK_SCORE=skip_trivial_topk_score, + ) + # Fused top-k + page-table transform: emit the dense backend's page table + # directly (page-size-aware) instead of block ids, skipping a separate gather. + # The page table + per-query effective KV length are allocated and returned. + real_seq_lens = None + if use_dense_main_attn: + # VENDOR NOTE: this fast path needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk` (minimax_decode_topk.cuh). The + # standalone vendor only ships the pure-Triton path, so this is gated off + # by default (use_dense_main_attn=False). If you opt in, vendor that .cuh + # too. Original line: `from sglang.jit_kernel.minimax_decode_topk import ( + # minimax_decode_topk_page_table)`. + raise NotImplementedError( + "use_dense_main_attn=True requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk.minimax_decode_topk_page_table; " + "the standalone Triton vendor only supports use_dense_main_attn=False." + ) + # num_q_heads (= num index/kv heads) may be > 1 for DP attention; the + # kernel emits a flattened [batch*num_q_heads] page table + seq_lens with + # the kv head head-encoded (head-minor) into the page index. + topk_idx, real_seq_lens = minimax_decode_topk_page_table( + score, seq_lens, req_to_token, slot_ids, block_size, topk, page_size + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # fall through to the idx_o (index value) merge, then return + _skip_block_topk = True + else: + _skip_block_topk = False + + # get topk index + if not _skip_block_topk: + topk_idx = torch.empty( + (num_q_heads, batch_size, topk), + device=score.device, + dtype=torch.int32, + ) + + if _skip_block_topk: + pass + elif use_jit_topk: + # Single-stage JIT radix-select: one kernel, no intermediate buffers. + # Equivalent output to the 2-stage path (set of block ids, front-packed, + # -1 padded); ~2-16x faster for long context. + # VENDOR NOTE: this needs the un-vendored CUDA JIT kernel + # `sglang.jit_kernel.minimax_decode_topk`. It is gated off by the + # `_compat.envs` stub (SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX=False), so + # this branch is never taken in the standalone vendor (the 2-stage Triton + # `else` fallback below runs instead). Original line: + # `from sglang.jit_kernel.minimax_decode_topk import minimax_decode_topk`. + raise NotImplementedError( + "use_jit_topk requires the un-vendored CUDA JIT kernel " + "sglang.jit_kernel.minimax_decode_topk; the standalone Triton vendor " + "uses the 2-stage Triton fallback (keep " + "SGLANG_OPT_USE_MINIMAX_DECODE_TOPK_RADIX off)." + ) + elif has_native_ops() and topk <= 64: + topk_idx = native_topk_from_scores(score, seq_lens, block_size, topk) + else: + # 2-stage split-K Triton fallback. + # Choose NUM_TOPK_CHUNKS to add parallelism over the seqblock dim. + # Same constraints as flash_decode: must be deterministic from values + # constant within a cuda graph (BS, num_q_heads), pow2, capped so the + # merge kernel's BLOCK_SIZE_K = pow2(NUM_TOPK_CHUNKS * pow2(topk)) + # stays reasonable. With topk=32 and cap 16, merge sorts 512 items. + TOPK_TARGET_GRID = 64 + MAX_NUM_TOPK_CHUNKS = 16 + topk_target = max( + 1, + min( + MAX_NUM_TOPK_CHUNKS, + TOPK_TARGET_GRID // max(1, batch_size * num_q_heads), + ), + ) + NUM_TOPK_CHUNKS = 1 << (topk_target.bit_length() - 1) + BLOCK_SIZE_T = triton.next_power_of_2(topk) + max_seqblock = score.shape[2] + chunk_blocks = (max_seqblock + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + topk_score_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.float32, + device=score.device, + ) + topk_idx_partial = torch.empty( + NUM_TOPK_CHUNKS, + num_q_heads, + batch_size, + BLOCK_SIZE_T, + dtype=torch.int32, + device=score.device, + ) + # stream 0 (default): topk partial → topk merge + grid = (batch_size, num_q_heads, NUM_TOPK_CHUNKS) + _topk_index_partial_kernel[grid]( + score, + topk_score_partial, + topk_idx_partial, + seq_lens, + block_size, + topk, + chunk_blocks, + score.stride(0), + score.stride(1), + score.stride(2), + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + ) + grid = (batch_size, num_q_heads) + _topk_index_merge_kernel[grid]( + topk_score_partial, + topk_idx_partial, + topk_idx, + seq_lens, + block_size, + topk, + topk_score_partial.stride(0), + topk_score_partial.stride(1), + topk_score_partial.stride(2), + topk_score_partial.stride(3), + topk_idx_partial.stride(0), + topk_idx_partial.stride(1), + topk_idx_partial.stride(2), + topk_idx_partial.stride(3), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + if disable_index_value: + return None, topk_idx, real_seq_lens + # attn output merge (default stream) + grid = (batch_size, num_q_heads) + _merge_attn_out_kernel[grid]( + o, + lse, + seq_lens, + head_dim, + block_size, + o.stride(0), + o.stride(1), + o.stride(2), + o.stride(3), + lse.stride(0), + lse.stride(1), + lse.stride(2), + NUM_KV_CHUNKS=NUM_KV_CHUNKS, + ) + o = o[0].contiguous() + return o, topk_idx, real_seq_lens diff --git a/build/torch212-cxx11-cu132-x86_64-linux/decode/topk_sparse.py b/build/torch212-cxx11-cu132-x86_64-linux/decode/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..8fe24cce53337c4f44e6c462fc3c90a258a34fc7 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/decode/topk_sparse.py @@ -0,0 +1,452 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from .._native import ( + native_decode_mma_supported, + native_decode_sparse_attn_mma_paged, +) +from ..common.utils import robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_H": lambda args: max( + 16, triton.next_power_of_2(args["gqa_group_size"]) + ), + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + "BATCH_SIZE_BUCKET": lambda args: triton.next_power_of_2(args["batch_size"]), + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in [4, 8] + for ns in [2, 3, 4, 5] + ], + key=["BATCH_SIZE_BUCKET", "gqa_group_size", "head_dim", "block_size", "HAS_SINK"], +) +@triton.jit +def _gqa_share_sparse_decode_kernel( + q_ptr, # Q: b x qh x d + sink_ptr, # Sink: qh x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + idx_ptr, # topk index: qh x b x topk + o_ptr, # O partial: c x b x qh x d + lse_ptr, # lse partial: c x b x qh + seq_lens, + slot_ids, + # shape + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + # sm_scale + sm_scale, + # stride + stride_q_b, + stride_q_h, + stride_q_d, + stride_sink_h, + stride_sink_d, + stride_k_s, + stride_k_h, + stride_k_d, + stride_v_s, + stride_v_h, + stride_v_d, + stride_r2t_b, + stride_ti_h, + stride_ti_b, + stride_ti_t, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + # META parameters + BATCH_SIZE_BUCKET: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + NUM_TOPK_CHUNKS: tl.constexpr, + HAS_SINK: tl.constexpr, + IS_FP8: tl.constexpr, +): + # decode program ids: split-K over the topk dimension to give every SM + # something to do at small batch. pid(0) folds (batch, chunk) together so + # the grid size = batch_size * NUM_TOPK_CHUNKS. + pid_bc, pid_kh = tl.program_id(0), tl.program_id(1) + pid_b = pid_bc % batch_size + pid_c = pid_bc // batch_size + pid_h = pid_kh * gqa_group_size + # per-chunk topk range. chunk_size is *runtime* (depends on max_topk which + # is a runtime arg, not constexpr), so don't annotate as tl.constexpr — + # doing so produces undefined behavior in Triton. + chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS + chunk_start_topk = pid_c * chunk_size_topk + chunk_end_topk_compiletime = chunk_start_topk + chunk_size_topk + # get q k start and len after rmpad + seq_len = tl.minimum(tl.load(seq_lens + pid_b), max_kv_len) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # to avoid bugs when slot_ids is negative + # get real topk + off_t = tl.arange(0, BLOCK_SIZE_T) + idx_base = idx_ptr + pid_kh * stride_ti_h + pid_b * stride_ti_b + topk_idx = tl.load(idx_base + off_t * stride_ti_t, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + chunk_end_topk = tl.minimum(chunk_end_topk_compiletime, real_topk) + # init pointer + off_n = tl.arange(0, BLOCK_SIZE_N) + off_d = tl.arange(0, BLOCK_SIZE_D) + dim_mask = off_d < head_dim + # init statistics — kept at -inf so empty chunks (chunk_start >= real_topk) + # naturally fall out as weight=0 in the merge step. + # q is always needed; load it unconditionally. + q_ptrs = tl.make_block_ptr( + base=q_ptr + pid_b * stride_q_b + pid_h * stride_q_h, + shape=(gqa_group_size, head_dim), + strides=(stride_q_h, stride_q_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero") + m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32) + # Sink init under a PURE-constexpr `if HAS_SINK:` so Triton dead-code- + # eliminates the sink_ptr reference when HAS_SINK is False (sink_ptr is + # then None). The original compound `if HAS_SINK and pid_c == 0` traced + # the body even when False -> NoneType make_block_ptr compile error on + # sm_121. M3 has no sink (sparse_init_block=0), so HAS_SINK is always + # False for us, but this keeps the sink path correct too. + if HAS_SINK: + if pid_c == 0: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sink_h, + shape=(gqa_group_size, head_dim), + strides=(stride_sink_h, stride_sink_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + sink = tl.load( + sink_ptrs, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + qsink = tl.sum(q.to(tl.float32) * sink, axis=1) * sm_scale + m_i = qsink + lse_i = qsink + acc_o = tl.full((BLOCK_SIZE_H, BLOCK_SIZE_D), 0, dtype=tl.float32) + # only iterate over this chunk's topk slice. the load must respect the + # per-chunk start offset. + cur_idx_ptr = idx_base + chunk_start_topk * stride_ti_t + for _ in tl.range(chunk_start_topk, chunk_end_topk): + # load index + c = tl.load(cur_idx_ptr).to(tl.int32) * BLOCK_SIZE_N + cur_idx_ptr = cur_idx_ptr + stride_ti_t + # resolve slots for this block via req_to_token + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # load K as (head_dim, BLOCK_SIZE_N) via indirect addressing + k_off = ( + slots[None, :] * stride_k_s + + pid_kh * stride_k_h + + off_d[:, None] * stride_k_d + ) + k = tl.load( + k_cache_ptr + k_off, + mask=dim_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 KV cache is unit-scaled (set_kv_buffer casts bf16->fp8 with no + # scale), so dequant is just a widening cast to the Q compute dtype + # before the tl.dot. Matches the bf16 path bit-for-bit when the cache + # is bf16 (IS_FP8 False -> this branch is compiled out). + k = k.to(q.dtype) + # load V as (BLOCK_SIZE_N, head_dim) via indirect addressing + v_off = ( + slots[:, None] * stride_v_s + + pid_kh * stride_v_h + + off_d[None, :] * stride_v_d + ) + v = tl.load( + v_cache_ptr + v_off, + mask=pos_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V before the P@V dot. This also makes the `p.to(v.dtype)` + # below cast P to the compute dtype (not to fp8, which would be + # catastrophic precision loss on the attention weights). + v = v.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_N), dtype=tl.float32) + qk += tl.where(off_n[None, :] < seq_len - c, 0, float("-inf")) + # [H, D], [D, N] -> [H, N] + qk += tl.dot(q, k) * sm_scale + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # load v and update acc_o + p = p.to(v.dtype) + # [H, N], [N, D] -> [H, D] + acc_o += tl.dot(p.to(v.dtype), v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log(tl.exp(lse_i - m_ij) + l_ij) + # final scale (matches the old non-split kernel for chunks where lse_i>-inf). + # For empty chunks (chunk_start_topk >= real_topk) the inner loop never + # runs, so m_i = lse_i = -inf and naive `tl.exp(m_i - lse_i)` would compute + # exp(-inf - (-inf)) = exp(NaN) = NaN, then 0 * NaN = NaN poisons o_partial + # and the merge result. Gate the scale with tl.where so empty chunks emit a + # clean zero (lse_i stays -inf which the merge correctly turns into weight=0). + scale = tl.where( + lse_i > float("-inf"), + tl.exp(m_i - lse_i), + tl.zeros_like(lse_i), + ) + acc_o = acc_o * scale[:, None] + # save partial output and lse for the merge step + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(gqa_group_size, head_dim), + strides=(stride_o_h, stride_o_d), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D), + order=(1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1)) + lse_ptrs = tl.make_block_ptr( + base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h, + shape=(gqa_group_size,), + strides=(stride_l_h,), + offsets=(0,), + block_shape=(BLOCK_SIZE_H,), + order=(0,), + ) + tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]), + } +) +@triton.jit +def _merge_topk_attn_out_kernel( + o_ptr, # [NUM_TOPK_CHUNKS, BS, NQH, D] — partials in, merged out at chunk 0 + lse_ptr, # [NUM_TOPK_CHUNKS, BS, NQH] + head_dim, + stride_o_c, + stride_o_b, + stride_o_h, + stride_o_d, + stride_l_c, + stride_l_b, + stride_l_h, + NUM_TOPK_CHUNKS: tl.constexpr, + BLOCK_SIZE_D: tl.constexpr, +): + pid_b, pid_h = tl.program_id(0), tl.program_id(1) + off_c = tl.arange(0, NUM_TOPK_CHUNKS) + off_d = tl.arange(0, BLOCK_SIZE_D) + o_ptrs = tl.make_block_ptr( + base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h, + shape=(NUM_TOPK_CHUNKS, head_dim), + strides=(stride_o_c, stride_o_d), + offsets=(0, 0), + block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D), + order=(1, 0), + ) + lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c + o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero") + lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0 + # standard flash-decoding merge in linear (not log2) space, matching the + # decode kernel which uses tl.exp / tl.log. + lse_max = tl.max(lse, axis=0) + weights = tl.exp(lse - lse_max) + weights = weights / tl.sum(weights, axis=0) + o_merged = tl.sum(o * weights[:, None], axis=0) + o_out_ptrs = o_ptr + pid_b * stride_o_b + pid_h * stride_o_h + off_d * stride_o_d + tl.store(o_out_ptrs, o_merged.to(o_ptr.dtype.element_ty), mask=off_d < head_dim) + + +@torch.no_grad() +def flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], + k_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + v_cache: torch.Tensor, # [max_slots, num_kv_heads, head_dim] (paged) + req_to_token: torch.Tensor, # [max_reqs, max_kv_len] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, + use_tma: bool = True, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. Q is always bf16/fp16. On HIP, the paged main K/V cache may + # be fp8 (unit-scaled) when running with --kv-cache-dtype fp8_*; the kernel + # widens it to the Q dtype on load (IS_FP8 branch). Keep CUDA's pre-existing + # sparse decode dtype contract unchanged by accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse decode expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + # shape + batch_size, num_q_heads, head_dim = q.shape + max_slots, num_kv_heads, _ = k_cache.shape + assert slot_ids.shape[0] == batch_size and seq_lens.shape[0] == batch_size + assert topk_idx.shape[0] == num_kv_heads + assert ( + triton.next_power_of_2(block_size) == block_size + ), f"block_size must be a power of 2, but got {block_size}" + # assert slot_ids.max() < max_slots, f"get slot_ids {slot_ids}, but kv_cache shape is {kv_cache.shape}" + max_kv_len = req_to_token.shape[1] + # gqa + assert num_q_heads % num_kv_heads == 0 + gqa_group_size = num_q_heads // num_kv_heads + max_topk = topk_idx.shape[2] + # sm scale + if sm_scale is None: + sm_scale = head_dim**-0.5 + # Fast path: native tensor-core decode for the M3 MSA shape. Falls through + # to the Triton kernel for other shapes, sink, or fp8 caches. + if sink is None and not is_fp8 and native_decode_mma_supported( + q, k_cache, block_size=block_size + ): + return native_decode_sparse_attn_mma_paged( + q, + k_cache, + v_cache, + req_to_token, + seq_lens, + slot_ids, + block_size, + topk_idx, + sm_scale, + ) + # Pick NUM_TOPK_CHUNKS so total grid ≈ TARGET_GRID. Same constraints as + # flash_decode_with_topk_idx: must be power of 2 (Triton arange) and must + # only depend on shape constants (so grid is fixed within a cuda graph). + # Capped by max_topk because chunks beyond real_topk early-fall-through to + # the merge-as-zero path; capping avoids wasting blocks at tiny topk. + TARGET_GRID = 256 + target = max( + 1, + min(max_topk, TARGET_GRID // max(1, batch_size * num_kv_heads)), + ) + NUM_TOPK_CHUNKS = 1 << (target.bit_length() - 1) + # output tensor: split-K partials, merged into chunk 0 by the merge kernel + o_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + head_dim, + dtype=q.dtype, + device=q.device, + ) + lse_partial = torch.empty( + NUM_TOPK_CHUNKS, + batch_size, + num_q_heads, + dtype=torch.float32, + device=q.device, + ) + # launch attention kernel + grid = (batch_size * NUM_TOPK_CHUNKS, num_kv_heads) + _gqa_share_sparse_decode_kernel[grid]( + q, + sink, + k_cache, + v_cache, + req_to_token, + topk_idx, + o_partial, + lse_partial, + seq_lens, + slot_ids, + max_slots, + batch_size, + gqa_group_size, + head_dim, + max_topk, + max_kv_len, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + req_to_token.stride(0), + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + BLOCK_SIZE_N=block_size, + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + IS_FP8=is_fp8, + ) + # merge partials into chunk 0 + merge_grid = (batch_size, num_q_heads) + _merge_topk_attn_out_kernel[merge_grid]( + o_partial, + lse_partial, + head_dim, + o_partial.stride(0), + o_partial.stride(1), + o_partial.stride(2), + o_partial.stride(3), + lse_partial.stride(0), + lse_partial.stride(1), + lse_partial.stride(2), + NUM_TOPK_CHUNKS=NUM_TOPK_CHUNKS, + ) + return o_partial[0].contiguous() diff --git a/build/torch212-cxx11-cu132-x86_64-linux/interface.py b/build/torch212-cxx11-cu132-x86_64-linux/interface.py new file mode 100644 index 0000000000000000000000000000000000000000..7939722609218f2bb41eadcdb2f965213d459d1c --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/interface.py @@ -0,0 +1,679 @@ +# SPDX-License-Identifier: Apache-2.0 +"""MiniMaxAI/msa-compatible public interface for Blackwell sparse MSA paths.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from ._native import ( + has_native_ops, + native_indexer_block_scores, + native_nvfp4_dequant_swizzled_to_bf16, +) +from .decode.topk_sparse import flash_decode_with_gqa_share_sparse +from .prefill.topk_sparse import flash_prefill_with_gqa_share_sparse +from .quantize import Nvfp4QuantizedTensor, dequantize_nvfp4_128x4_to_bf16 + + +def _page_table_to_req_to_token( + page_table: torch.Tensor, + seqused_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, +) -> torch.Tensor: + batch = int(page_table.shape[0]) + max_seqlen_k = int(max_seqlen_k) + blk_kv = int(blk_kv) + logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64) + page = logical_pos // blk_kv + offset = logical_pos % blk_kv + page = page.clamp(max=int(page_table.shape[1]) - 1) + physical_page = page_table[:, page].to(torch.int64) + physical_token = physical_page * blk_kv + offset[None, :] + valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None] + return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous() + + +def _dense_q2k_indices( + *, + head_kv: int, + batch: int, + topk: int, + seq_lens: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device) + blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device) + valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv) + for b in range(batch): + valid = blocks < valid_blocks[b] + idx[:, b, valid] = blocks[valid] + return idx + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, blk_kv: int) -> list[int]: + seqlens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to("cpu").tolist() + return [(int(v) + int(blk_kv) - 1) // int(blk_kv) for v in seqlens] + + +def _packed_row_to_batch_block( + cu_seqlens_k: torch.Tensor, + blk_kv: int, +) -> list[tuple[int, int]]: + rows_per_batch = _rows_per_batch(cu_seqlens_k, int(blk_kv)) + max_rows = max(rows_per_batch, default=0) + row_map: list[tuple[int, int]] = [] + for block_idx in range(max_rows): + for batch_idx, rows in enumerate(rows_per_batch): + if block_idx < rows: + row_map.append((batch_idx, block_idx)) + return row_map + + +def _csr_to_q2k_indices( + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + blk_kv: int, + topk: int, +) -> torch.Tensor: + if k2q_row_ptr.dtype != torch.int32 or k2q_q_indices.dtype != torch.int32: + raise TypeError("k2q_row_ptr and k2q_q_indices must be torch.int32") + if k2q_row_ptr.ndim != 2 or k2q_q_indices.ndim != 2: + raise ValueError("k2q_row_ptr and k2q_q_indices must be rank-2") + head_kv = int(k2q_row_ptr.shape[0]) + total_q = int(cu_seqlens_q[-1].item()) + topk = int(topk) + if topk <= 0: + raise ValueError(f"topK must be positive, got {topk}") + + row_map = _packed_row_to_batch_block(cu_seqlens_k, int(blk_kv)) + if int(k2q_row_ptr.shape[1]) != len(row_map) + 1: + raise ValueError( + "k2q_row_ptr shape does not match cu_seqlens_k/blk_kv packed rows: " + f"got {tuple(k2q_row_ptr.shape)}, expected second dim {len(row_map) + 1}" + ) + + cu_q_cpu = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + row_ptr_cpu = k2q_row_ptr.to("cpu", non_blocking=False) + q_idx_cpu = k2q_q_indices.to("cpu", non_blocking=False) + q2k = torch.full( + (head_kv, total_q, topk), + -1, + dtype=torch.int32, + device=k2q_row_ptr.device, + ) + + next_slot: dict[tuple[int, int], int] = {} + seen: set[tuple[int, int, int]] = set() + for h in range(head_kv): + for row, (batch_idx, kv_block_idx) in enumerate(row_map): + start = int(row_ptr_cpu[h, row].item()) + end = int(row_ptr_cpu[h, row + 1].item()) + q_start = int(cu_q_cpu[batch_idx]) + q_end = int(cu_q_cpu[batch_idx + 1]) + q_len = q_end - q_start + for pos in range(start, end): + q_val = int(q_idx_cpu[h, pos].item()) + if q_val < 0: + continue + if q_start <= q_val < q_end: + q_global = q_val + elif q_val < q_len: + q_global = q_start + q_val + else: + continue + key = (h, q_global, kv_block_idx) + if key in seen: + continue + seen.add(key) + slot_key = (h, q_global) + slot = next_slot.get(slot_key, 0) + if slot >= topk: + continue + q2k[h, q_global, slot] = int(kv_block_idx) + next_slot[slot_key] = slot + 1 + return q2k.contiguous() + + +def _make_prefill_cache_and_metadata( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_k: int, + blk_kv: int, + *, + page_table: Optional[torch.Tensor], + seqused_k: Optional[torch.Tensor], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if page_table is None: + if k.ndim != 3 or v.ndim != 3: + raise ValueError("dense prefill k/v must have shape [total_k, Hkv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + k_cache = k.contiguous() + v_cache = v.contiguous() + batch = int(cu_seqlens_q.numel()) - 1 + max_seqlen_k = int(max_seqlen_k) + logical = torch.arange(max_seqlen_k, device=q.device, dtype=torch.int64) + cu_k = cu_seqlens_k.to(torch.int64) + seq_lens = (cu_k[1:] - cu_k[:-1]).to(torch.int32).contiguous() + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = (cu_k[:-1, None] + logical[None, :]).to(torch.int32) + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + else: + if k.ndim != 4 or v.ndim != 4: + raise ValueError("paged prefill k/v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("paged k/v page dimension must equal blk_kv") + if page_table.dtype != torch.int32: + raise TypeError("page_table must be torch.int32") + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + batch = int(page_table.shape[0]) + if seqused_k is None: + seq_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).to(torch.int32).contiguous() + else: + seq_lens = seqused_k.to(torch.int32).contiguous() + logical = torch.arange(int(max_seqlen_k), device=q.device, dtype=torch.int64) + page = (logical // int(blk_kv)).clamp(max=int(page_table.shape[1]) - 1) + offset = logical % int(blk_kv) + physical_page = page_table[:, page].to(torch.int64) + req_to_token = (physical_page * int(blk_kv) + offset[None, :]).to(torch.int32) + valid = logical[None, :] < seq_lens.to(torch.int64)[:, None] + req_to_token = torch.where(valid, req_to_token, torch.zeros_like(req_to_token)).contiguous() + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int32) + prefix_lens = (seq_lens - q_lens).clamp_min(0).to(torch.int32).contiguous() + return k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens + + +def _select_prefill_block_size_q(q_heads: int, kv_heads: int) -> int: + if kv_heads <= 0 or q_heads % kv_heads != 0: + raise ValueError("number of query heads must be divisible by KV heads") + gqa_group = q_heads // kv_heads + limit = max(1, 128 // max(1, gqa_group)) + for candidate in (64, 32, 16, 8, 4, 2, 1): + if candidate <= limit: + return candidate + return 1 + + +def _normalize_decode_q2k( + q2k_indices: Optional[torch.Tensor], + *, + batch: int, + seqlen_q: int, + head_kv: int, + seqused_k: torch.Tensor, + blk_kv: int, +) -> torch.Tensor: + if q2k_indices is None: + topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv)) + return _dense_q2k_indices( + head_kv=head_kv, + batch=batch, + topk=topk, + seq_lens=seqused_k, + blk_kv=blk_kv, + ) + if q2k_indices.dtype != torch.int32: + raise TypeError("q2k_indices must be torch.int32") + if q2k_indices.ndim != 3: + raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]") + if q2k_indices.shape[0] != head_kv: + raise ValueError("q2k_indices first dimension must match Hkv") + if int(q2k_indices.shape[1]) != batch * seqlen_q: + raise ValueError("q2k_indices second dimension must equal batch * seqlen_q") + if seqlen_q != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + return q2k_indices[:, ::seqlen_q, :].contiguous() + + +def sparse_decode_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + q2k_indices: Optional[torch.Tensor] = None, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + schedule: Optional[object] = None, + O_partial: Optional[torch.Tensor] = None, + LSE_partial: Optional[torch.Tensor] = None, +): + """Run MiniMax paged sparse decode on Blackwell. + + This is compatible with the upstream MiniMaxAI/msa decode-facing API name, + but accepts the Blackwell package's validated BF16/FP16 decode path. It + supports single-token decode (`seqlen_q=1`) with paged K/V + `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`. + """ + + if schedule is not None or O_partial is not None or LSE_partial is not None: + raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper") + if return_softmax_lse: + raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper") + if not causal: + raise NotImplementedError("Blackwell decode wrapper currently supports causal=True") + if int(seqlen_q) != 1: + raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1") + if q.ndim != 3: + raise ValueError("q must have shape [batch * seqlen_q, Hq, D]") + if k.ndim != 4 or v.ndim != 4: + raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]") + if k.shape != v.shape: + raise ValueError("k and v must have the same shape") + if q.device != k.device or q.device != v.device or q.device != page_table.device: + raise ValueError("q, k, v, and page_table must be on the same device") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("Blackwell decode wrapper supports BF16/FP16 q") + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper") + if q.shape[-1] != 128 or k.shape[-1] != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + if int(k.shape[2]) != int(blk_kv): + raise ValueError("k.shape[2] must equal blk_kv") + if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32: + raise TypeError("page_table and seqused_k must be torch.int32") + if page_table.ndim != 2: + raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]") + batch = int(page_table.shape[0]) + if int(q.shape[0]) != batch: + raise ValueError("for seqlen_q=1, q.shape[0] must equal batch") + + head_kv = int(k.shape[1]) + q2k = _normalize_decode_q2k( + q2k_indices, + batch=batch, + seqlen_q=int(seqlen_q), + head_kv=head_kv, + seqused_k=seqused_k, + blk_kv=int(blk_kv), + ) + req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv)) + slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64) + k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous() + v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous() + return flash_decode_with_gqa_share_sparse( + q.contiguous(), + None, + k_cache, + v_cache, + req_to_token, + seqused_k.contiguous(), + slot_ids, + int(blk_kv), + q2k, + sm_scale=softmax_scale, + ) + + +class SparseDecodePagedAttentionWrapper: + def __init__(self, *, blk_kv: int = 128, causal: bool = True): + self.blk_kv = int(blk_kv) + self.causal = bool(causal) + self.page_table: Optional[torch.Tensor] = None + self.seqused_k: Optional[torch.Tensor] = None + self.seqlen_q: Optional[int] = None + self.max_seqlen_k: Optional[int] = None + self.q2k_indices: Optional[torch.Tensor] = None + + def plan( + self, + *, + page_table: torch.Tensor, + seqused_k: torch.Tensor, + seqlen_q: int, + max_seqlen_k: int, + q2k_indices: Optional[torch.Tensor] = None, + num_qo_heads: Optional[int] = None, + num_kv_heads: Optional[int] = None, + head_dim: Optional[int] = 128, + enable_cuda_graph: bool = False, + max_grid_size: Optional[int] = None, + fixed_split_size: Optional[int] = None, + disable_split_kv: bool = False, + ) -> "SparseDecodePagedAttentionWrapper": + if head_dim is not None and int(head_dim) != 128: + raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128") + self.page_table = page_table.contiguous() + self.seqused_k = seqused_k.contiguous() + self.seqlen_q = int(seqlen_q) + self.max_seqlen_k = int(max_seqlen_k) + self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous() + return self + + def run( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + *, + softmax_scale: Optional[float] = None, + return_softmax_lse: bool = False, + out: Optional[torch.Tensor] = None, + lse: Optional[torch.Tensor] = None, + ): + if self.page_table is None or self.seqused_k is None: + raise RuntimeError("decode wrapper must be planned before run") + result = sparse_decode_atten_func( + q, + k, + v, + self.q2k_indices, + page_table=self.page_table, + seqused_k=self.seqused_k, + seqlen_q=self.seqlen_q, + max_seqlen_k=self.max_seqlen_k, + blk_kv=self.blk_kv, + causal=self.causal, + softmax_scale=softmax_scale, + return_softmax_lse=return_softmax_lse, + ) + if out is not None: + out.copy_(result) + return out + return result + + +def sparse_atten_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + *, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + blk_kv: int = 128, + causal: bool = True, + softmax_scale: Optional[float] = None, + lse_temperature_scale: float = 1.0, + return_temperature_lse: bool = False, + partial_dtype: torch.dtype = torch.bfloat16, + return_softmax_lse: bool = False, + page_table: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + schedule: Optional[object] = None, + usable_SM_count: int = -1, + qk_dtype: Optional[torch.dtype] = None, + pv_dtype: Optional[torch.dtype] = None, +): + """Run the official MiniMax CSR sparse prefill API on Blackwell. + + The wrapper converts MiniMax's K-to-Q CSR selection to the Q-to-K block list + consumed by the Blackwell Triton sparse prefill kernel. The default output + path is supported for BF16/FP16 tensors with head_dim=128. + """ + + del partial_dtype, usable_SM_count, qk_dtype, pv_dtype + if schedule is not None: + raise ValueError("schedule objects are not consumed by the Blackwell prefill wrapper") + if return_temperature_lse or return_softmax_lse: + raise ValueError("LSE return paths are not exposed by the Blackwell prefill wrapper") + if float(lse_temperature_scale) != 1.0: + raise ValueError("lse_temperature_scale values other than 1.0 are not supported") + if not causal: + raise ValueError("Blackwell sparse prefill wrapper currently supports causal=True") + if q.ndim != 3: + raise ValueError("q must have shape [total_q, Hq, D]") + if q.dtype not in (torch.bfloat16, torch.float16): + raise TypeError("q must be BF16 or FP16") + if q.shape[-1] != 128 or k.shape[-1] != 128 or v.shape[-1] != 128: + raise ValueError("Blackwell sparse prefill wrapper currently supports head_dim=128") + if cu_seqlens_q.dtype != torch.int32 or cu_seqlens_k.dtype != torch.int32: + raise TypeError("cu_seqlens_q and cu_seqlens_k must be torch.int32") + if q.device != k.device or q.device != v.device: + raise ValueError("q, k, and v must be on the same device") + + if k.dtype != q.dtype or v.dtype != q.dtype: + raise TypeError("q, k, and v must have matching BF16/FP16 dtype") + kv_heads = int(k.shape[1]) + q2k_indices = _csr_to_q2k_indices( + k2q_row_ptr.contiguous(), + k2q_q_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(blk_kv), + int(topK), + ) + k_cache, v_cache, req_to_token, slot_ids, seq_lens, prefix_lens = _make_prefill_cache_and_metadata( + q, + k, + v, + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(max_seqlen_k), + int(blk_kv), + page_table=None if page_table is None else page_table.contiguous(), + seqused_k=None if seqused_k is None else seqused_k.contiguous(), + ) + block_size_q = _select_prefill_block_size_q(int(q.shape[1]), kv_heads) + return flash_prefill_with_gqa_share_sparse( + q.contiguous(), + k_cache, + v_cache, + None, + req_to_token, + slot_ids, + q2k_indices, + block_size_q=block_size_q, + block_size_k=int(blk_kv), + cu_seqlens=cu_seqlens_q.contiguous(), + seq_lens=seq_lens, + prefix_lens=prefix_lens, + max_seqlen_q=int(max_seqlen_q), + sm_scale=softmax_scale, + ) + + +def _nvfp4_dequant_kv( + packed: torch.Tensor, + scale_128x4: torch.Tensor, + global_scale: torch.Tensor, +) -> torch.Tensor: + if packed.dtype is not torch.uint8: + return packed + if has_native_ops() and packed.is_cuda: + return native_nvfp4_dequant_swizzled_to_bf16( + packed, + scale_128x4, + global_scale, + ) + original_shape = tuple(int(v) for v in packed.shape[:-1]) + (int(packed.shape[-1]) * 2,) + rows = 1 + for dim in original_shape[:-1]: + rows *= int(dim) + qx = Nvfp4QuantizedTensor( + data=packed, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, int(original_shape[-1]) // 16), + original_shape=original_shape, + ) + return dequantize_nvfp4_128x4_to_bf16(qx) + + +def sparse_atten_nvfp4_kv_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + k_scale_128x4: torch.Tensor, + v_scale_128x4: torch.Tensor, + k_global_scale: torch.Tensor, + v_global_scale: torch.Tensor, + k2q_row_ptr: torch.Tensor, + k2q_q_indices: torch.Tensor, + topK: int, + **kwargs, +): + """NVFP4-KV official prefill API wrapper. + + The Blackwell compatibility path dequantizes NVFP4 K/V with the package's + 128x4 metadata helper, then dispatches to `sparse_atten_func`. + """ + + k_bf16 = _nvfp4_dequant_kv(k, k_scale_128x4, k_global_scale) + v_bf16 = _nvfp4_dequant_kv(v, v_scale_128x4, v_global_scale) + return sparse_atten_func( + q, + k_bf16, + v_bf16, + k2q_row_ptr, + k2q_q_indices, + topK, + **kwargs, + ) + + +def _unpack_e2m1_fp4(packed: torch.Tensor) -> torch.Tensor: + data = packed if packed.dtype is torch.uint8 else packed.view(torch.uint8) + lut = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0], + dtype=torch.float32, + device=data.device, + ) + out = torch.empty(tuple(data.shape[:-1]) + (int(data.shape[-1]) * 2,), dtype=torch.float32, device=data.device) + out[..., 0::2] = lut[(data & 0x0F).long()] + out[..., 1::2] = lut[(data >> 4).long()] + return out + + +def _apply_public_fp4_scale(values: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + if scale.ndim == values.ndim and int(scale.shape[-1]) * 16 == int(values.shape[-1]): + return values * scale.to(torch.float32).repeat_interleave(16, dim=-1) + if scale.ndim == values.ndim - 1: + return values * scale.to(torch.float32).unsqueeze(-1) + return values + + +def fp4_indexer_block_scores( + q_fp4: torch.Tensor, + k_fp4: torch.Tensor, + q_scale: torch.Tensor, + k_scale: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + cu_page_offsets: torch.Tensor, + *, + max_seqlen_q: int, + max_seqlen_k: int, + kv_indices: torch.Tensor, + fp4_format: str, + causal: bool = False, + qo_offset: Optional[torch.Tensor] = None, + scale_layout: str = "preordered_mma", +) -> torch.Tensor: + """FP4 block-score API for Blackwell. + + Returns the official shape `[Hq, ceil(max_seqlen_k / 128), total_q]`. + Built artifacts route CUDA tensors through the native Blackwell block-score + kernel. Source-tree mode uses the reference loop below so the API remains + testable before the extension is built. + """ + + del qo_offset + if fp4_format.lower() not in {"nvfp4", "e2m1", "mxfp4"}: + raise ValueError(f"unsupported fp4_format: {fp4_format}") + if scale_layout not in {"public", "preordered_mma"}: + raise ValueError(f"unsupported scale_layout: {scale_layout}") + if q_fp4.ndim != 3: + raise ValueError("q_fp4 must have shape [total_q, Hq, packed_D]") + if k_fp4.ndim != 4: + raise ValueError("k_fp4 must have shape [num_pages, Hkv, 128, packed_D]") + total_q, hq, _ = q_fp4.shape + num_pages, hkv, page_size, _ = k_fp4.shape + if int(page_size) != 128: + raise ValueError("fp4_indexer_block_scores currently expects 128-token pages") + if hq % hkv != 0: + raise ValueError("Hq must be divisible by Hkv") + max_blocks = (int(max_seqlen_k) + 127) // 128 + scores = torch.full((hq, max_blocks, total_q), float("-inf"), dtype=torch.float32, device=q_fp4.device) + + q = _apply_public_fp4_scale(_unpack_e2m1_fp4(q_fp4), q_scale) + k = _apply_public_fp4_scale(_unpack_e2m1_fp4(k_fp4), k_scale) + + # Native block-max QK kernel for the cubic scoring loop when available. + # Same dequant numerics as the reference (bf16 compute); falls back to the + # Python reference below in source-tree mode. + if has_native_ops() and q_fp4.is_cuda: + batch = int(cu_seqlens_q.numel()) - 1 + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).to(torch.int64) + batch_of_q = torch.repeat_interleave( + torch.arange(batch, device=q_fp4.device, dtype=torch.int32), + q_lens, + ) + return native_indexer_block_scores( + q.to(torch.bfloat16), + k.to(torch.bfloat16), + batch_of_q, + cu_seqlens_q.to(torch.int32), + cu_seqlens_k.to(torch.int32), + cu_page_offsets.to(torch.int32), + kv_indices.to(torch.int32), + max_blocks=max_blocks, + causal=causal, + ) + + gqa = hq // hkv + batch = int(cu_seqlens_q.numel()) - 1 + cu_q = cu_seqlens_q.to("cpu", non_blocking=False).tolist() + cu_k = cu_seqlens_k.to("cpu", non_blocking=False).tolist() + cu_pages = cu_page_offsets.to("cpu", non_blocking=False).tolist() + kv_idx_cpu = kv_indices.to("cpu", non_blocking=False).tolist() + for b in range(batch): + q_start, q_end = int(cu_q[b]), int(cu_q[b + 1]) + k_len = int(cu_k[b + 1]) - int(cu_k[b]) + page_start, page_end = int(cu_pages[b]), int(cu_pages[b + 1]) + for logical_block, page_pos in enumerate(range(page_start, page_end)): + if logical_block >= max_blocks: + break + physical_page = int(kv_idx_cpu[page_pos]) + if physical_page < 0 or physical_page >= num_pages: + continue + valid_tokens = min(128, max(0, k_len - logical_block * 128)) + if valid_tokens <= 0: + continue + for qi in range(q_start, q_end): + local_q = qi - q_start + if causal and logical_block * 128 > local_q: + continue + for qh in range(hq): + kh = qh // gqa + key_block = k[physical_page, kh, :valid_tokens] + if causal: + visible = min(valid_tokens, max(0, local_q - logical_block * 128 + 1)) + if visible <= 0: + continue + key_block = key_block[:visible] + block_scores = torch.matmul(key_block, q[qi, qh]) + scores[qh, logical_block, qi] = block_scores.max() + return scores + + +__all__ = [ + "sparse_atten_func", + "sparse_atten_nvfp4_kv_func", + "sparse_decode_atten_func", + "SparseDecodePagedAttentionWrapper", + "fp4_indexer_block_scores", +] diff --git a/build/torch212-cxx11-cu132-x86_64-linux/metadata.json b/build/torch212-cxx11-cu132-x86_64-linux/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bf836127ba7607762e765557c9885dc81b5d14fd --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/metadata.json @@ -0,0 +1,39 @@ +{ + "name": "minimaxai-msa-blackwell", + "id": "_minimaxai_msa_blackwell_cuda_25b0067", + "version": 1, + "license": "Apache-2.0", + "python-depends": [], + "backend": { + "type": "cuda", + "archs": [ + "12.0", + "12.1" + ] + }, + "digest": { + "algorithm": "sha256", + "files": { + "__init__.py": "oPB7xQhjHd9+aLVJkEmJkZOXOwMEarmCdj3SBbhH2bM=", + "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=", + "_minimaxai_msa_blackwell_cuda_25b0067.abi3.so": "TrAcGpbD5kBHSzpS8tONpL4q8CkMS5RcUPkOhn2ekH8=", + "_native.py": "wTh66DiIKxsJ5m8fkAXKSCBYD3CVFVx4PFxceQiwvPY=", + "_ops.py": "ikU3Lg+kgswKktYjNSq5f4DmgyTR8dnxV8lH3rmOWvk=", + "api_status.py": "7oZzIkb4fuwkAz3r4Q1slHbOfaP3jwVB647Ok223ATk=", + "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=", + "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=", + "decode/topk_sparse.py": "uINcV9SP5sAf9jYPsPvyQ9lon8sajZ8Qn/yJ+2LA8NU=", + "interface.py": "jfhBHGyunFF+OdFycJYyptNNIAngKrXY67buTXAN66M=", + "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=", + "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=", + "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=", + "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=", + "prefill/__init__.py": "9S6fOQTMZUvHe5pC7nUFi45xH/k+443sdYjs2T6t6Lo=", + "prefill/topk_sparse.py": "pO+mgNVPqKzd//s6pTh4ox8J79B0q0wPRcMh08Bw88E=", + "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=", + "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w=" + } + } +} \ No newline at end of file diff --git a/build/torch212-cxx11-cu132-x86_64-linux/minimaxai_msa_blackwell/__init__.py b/build/torch212-cxx11-cu132-x86_64-linux/minimaxai_msa_blackwell/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b2672c1cd85b74c1b3ded0fc0b2100e1aeac23 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/minimaxai_msa_blackwell/__init__.py @@ -0,0 +1,26 @@ +import ctypes +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + + +def _import_from_path(file_path: Path) -> ModuleType: + # We cannot use the module name as-is, after adding it to `sys.modules`, + # it would also be used for other imports. So, we make a module name that + # depends on the path for it to be unique using the hex-encoded hash of + # the path. + path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value) + module_name = path_hash + spec = importlib.util.spec_from_file_location(module_name, file_path) + if spec is None: + raise ImportError(f"Cannot load spec for {module_name} from {file_path}") + module = importlib.util.module_from_spec(spec) + if module is None: + raise ImportError(f"Cannot load module {module_name} from spec") + sys.modules[module_name] = module + spec.loader.exec_module(module) # type: ignore + return module + + +globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py"))) diff --git a/build/torch212-cxx11-cu132-x86_64-linux/naive/__init__.py b/build/torch212-cxx11-cu132-x86_64-linux/naive/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9881313609aae2c16a19dae665af4268bd54431d --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/naive/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/build/torch212-cxx11-cu132-x86_64-linux/naive/flash_with_topk_idx.py b/build/torch212-cxx11-cu132-x86_64-linux/naive/flash_with_topk_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..db7caceb6faa5d8d1669d3c57f1e4ca9578a4804 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/naive/flash_with_topk_idx.py @@ -0,0 +1,106 @@ +# Copyright 2025 XunhaoLai. All rights reserved. +# +# VENDOR NOTE: rewritten to drop the `einops` dependency (the original used +# einops.einsum / einops.rearrange). Logic is equivalent to the SGLang upstream +# `naive/flash_with_topk_idx.py`; only the einsum/rearrange calls were replaced +# with native torch.einsum / torch.reshape so the standalone vendor needs only +# torch (+ triton for the kernels). Original einops expressions are kept in +# comments next to each replacement for auditability. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_topk_idx( + q: torch.Tensor, # [batch_size, num_heads, head_dim] + sink: Optional[torch.Tensor], # [num_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + max_seqlen: int, + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk: int, + sm_scale: Optional[float] = None, + init_blocks: int = 0, + local_blocks: int = 0, +): + """Joint reference: lightning-indexer block top-k + sparse attention. + + This is the *contiguous* (non-paged) reference. `kv_cache` here is the dense + 5-D buffer [max_slots, 2(k/v), max_len, num_kv_heads, head_dim], indexed by + `slot_ids`. It both (a) selects the top-k key blocks via per-block max score + (with optional forced init/local blocks) and (b) computes the resulting + sparse attention, returning (o, topk_idx). + """ + assert ( + kv_cache.shape[2] % block_size == 0 + ), "max cache len must be divisible by block size" + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + original_dtype = q.dtype + batch_size = q.shape[0] + num_q_heads = q.shape[1] + num_kv_heads = kv_cache.shape[3] + gqa_group_size = num_q_heads // num_kv_heads + head_dim = q.shape[-1] + # rearrange q: "b (h g) d -> b h g d" + q = q.float().reshape(batch_size, num_kv_heads, gqa_group_size, head_dim) + kv_cache_float = kv_cache.float() + k_sel = kv_cache_float[slot_ids, 0, ...] # [b, n, h, d] + v_sel = kv_cache_float[slot_ids, 1, ...] # [b, n, h, d] + # einsum(q, k, "b h g d, b n h d -> b h g n") + qk = torch.einsum("bhgd,bnhd->bhgn", q, k_sel) * sm_scale + mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None] + qk = qk.masked_fill(~mask[:, None, None, :], float("-inf")) + # get score + score = qk.clone().to(torch.float32) + # rearrange "b h g (n s) -> b (h g) n s", s=block_size + n_blocks = score.shape[-1] // block_size + score = score.reshape( + batch_size, num_kv_heads, gqa_group_size, n_blocks, block_size + ).reshape(batch_size, num_q_heads, n_blocks, block_size) + score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks] + # post-process score for init_blocks and local_blocks + INIT_SCORE = 1e30 + LOCAL_SCORE = 1e29 + if init_blocks > 0: + score[:, :, :init_blocks] = INIT_SCORE + if local_blocks > 0: + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + local_start = max(0, num_blks - local_blocks) + score[b, :, local_start:num_blks] = LOCAL_SCORE + # compute topk indices per (batch, head) + # score shape: [batch_size, num_q_heads, num_blocks] + topk_idx = torch.full( + (num_q_heads, batch_size, topk), + fill_value=-1, + device=score.device, + dtype=torch.int32, + ) + num_blocks_per_batch = (seq_lens + block_size - 1) // block_size + for b in range(batch_size): + num_blks = num_blocks_per_batch[b].item() + actual_topk = min(topk, num_blks) + for h in range(num_q_heads): + # get topk indices for this (batch, head) + _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1) + topk_idx[h, b, :actual_topk] = indices.to(torch.int32) + # compute attention output with sink + if sink is not None: + # sink: [num_q_heads, head_dim] -> [h, g, d] + sink_reshaped = sink.float().reshape(num_kv_heads, gqa_group_size, head_dim) + # einsum(q, sink, "b h g d, h g d -> b h g") + qsink = torch.einsum("bhgd,hgd->bhg", q, sink_reshaped) * sm_scale + qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1] + attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32) + attn = attn[..., 1:] # remove sink score + else: + attn = qk.softmax(dim=-1, dtype=torch.float32) + # einsum(attn, v, "b h g n, b n h d -> b h g d") + o = torch.einsum("bhgn,bnhd->bhgd", attn, v_sel) + # rearrange "b h g d -> b (h g) d" + o = o.reshape(batch_size, num_q_heads, head_dim) + return o.to(original_dtype), topk_idx diff --git a/build/torch212-cxx11-cu132-x86_64-linux/naive/topk_sparse.py b/build/torch212-cxx11-cu132-x86_64-linux/naive/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..6fefac38e0d131d2b37dcb392e405d71a93ace61 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/naive/topk_sparse.py @@ -0,0 +1,88 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch + + +def naive_flash_decode_with_gqa_share_sparse( + q: torch.Tensor, # [batch_size, num_q_heads, head_dim] + sink: Optional[torch.Tensor], # [num_q_heads, head_dim] + kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim] + seq_lens: torch.Tensor, # [batch_size, ] + slot_ids: torch.Tensor, # [batch_size, ] + block_size: int, + topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] + sm_scale: Optional[float] = None, +) -> torch.Tensor: + """ + Naive implementation of sparse attention with GQA group sharing. + Each GQA group (all q heads sharing the same kv head) uses the same topk_idx. + """ + if sm_scale is None: + sm_scale = q.shape[-1] ** -0.5 + + original_dtype = q.dtype + batch_size, num_q_heads, head_dim = q.shape + max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape + gqa_group_size = num_q_heads // num_kv_heads + # Output tensor + o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device) + + for b in range(batch_size): + seq_len = seq_lens[b].item() + sid = slot_ids[b].item() % max_slots + + for kh in range(num_kv_heads): + # Get topk indices for this (kv_head, batch) + block_indices = topk_idx[kh, b, :].tolist() + + # Collect selected K and V blocks + selected_k_blocks = [] + selected_v_blocks = [] + for block_idx in block_indices: + if block_idx < 0: + continue # Invalid index + start = block_idx * block_size + end = min(start + block_size, seq_len) + if start >= seq_len: + continue + # K: [block_len, head_dim] + k_block = kv_cache[sid, 0, start:end, kh, :] + # V: [block_len, head_dim] + v_block = kv_cache[sid, 1, start:end, kh, :] + selected_k_blocks.append(k_block) + selected_v_blocks.append(v_block) + + if len(selected_k_blocks) == 0: + continue + + # Concatenate selected blocks: [total_selected_len, head_dim] + k_selected = torch.cat(selected_k_blocks, dim=0) + v_selected = torch.cat(selected_v_blocks, dim=0) + + # Compute attention for all q heads in this GQA group + for g in range(gqa_group_size): + qh = kh * gqa_group_size + g + # q_vec: [head_dim] + q_vec = q[b, qh, :].float() + + # Compute attention scores: [total_selected_len] + scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale + + # Add sink to softmax normalization if present + if sink is not None: + sink_vec = sink[qh, :].float() + qsink = torch.dot(q_vec, sink_vec) * sm_scale + # Concatenate sink score with regular scores for softmax + scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0) + attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1) + # Remove sink weight (only used for normalization) + attn_weights = attn_weights_with_sink[1:] + else: + attn_weights = torch.softmax(scores, dim=-1) + + # Compute output: [head_dim] + o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected) + + return o diff --git a/build/torch212-cxx11-cu132-x86_64-linux/prefill/__init__.py b/build/torch212-cxx11-cu132-x86_64-linux/prefill/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b6b69345bf3ffe07f6ccac8c570ee17e3b5e6c69 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/prefill/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +from .topk_sparse import flash_prefill_with_gqa_share_sparse + +__all__ = ["flash_prefill_with_gqa_share_sparse"] diff --git a/build/torch212-cxx11-cu132-x86_64-linux/prefill/topk_sparse.py b/build/torch212-cxx11-cu132-x86_64-linux/prefill/topk_sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..9bf83d65eac260f8e71180ea6f28970e8db582d7 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/prefill/topk_sparse.py @@ -0,0 +1,371 @@ +# Copyright 2025 XunhaoLai. All rights reserved. + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from .._compat import is_hip # vendored: was `from sglang.srt.utils import is_hip` + +from ..common.utils import get_cu_seqblocks, robust_allocator + +_is_hip = is_hip() + + +@triton.heuristics( + { + "BLOCK_SIZE_KD": lambda args: triton.next_power_of_2(args["qk_head_dim"]), + "BLOCK_SIZE_VD": lambda args: triton.next_power_of_2(args["v_head_dim"]), + "BLOCK_SIZE_H": lambda args: triton.next_power_of_2( + max( + 16 // args["BLOCK_SIZE_Q"], + triton.next_power_of_2(args["gqa_group_size"]), + ) + ), + "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["max_topk"]), + "BLOCK_SIZE_QH": lambda args: args["BLOCK_SIZE_Q"] * args["BLOCK_SIZE_H"], + "HAS_SINK": lambda args: args["sink_ptr"] is not None, + } +) +@triton.autotune( + # Configs that fail to compile on the target arch are skipped, so widening + # the num_warps x num_stages grid only adds candidates, never a bad kernel. + configs=[ + triton.Config({}, num_warps=nw, num_stages=ns) + for nw in (2, 4, 8) + for ns in (2, 3, 4) + ], + key=[ + "BLOCK_SIZE_Q", + "BLOCK_SIZE_K", + "qk_head_dim", + "v_head_dim", + "gqa_group_size", + ], +) +@triton.jit +def _gqa_share_sparse_fwd_kernel( + q_ptr, # Q: n x h x d + k_cache_ptr, # K paged: max_slots x kh x d + v_cache_ptr, # V paged: max_slots x kh x d + sink_ptr, # Sink: h x d + t_ptr, # topk_idx: kh x n x k + o_ptr, # O: n x h x d + req_to_token_ptr, # req_to_token: max_reqs x max_kv_len + # seqlens + cu_seqlens_q, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + # shape + max_slots, + num_kv_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + max_topk, + # q loop num + num_q_loop, + # sm_scale + sm_scale, + # stride + stride_qn, + stride_qh, + stride_qd, + stride_ks, + stride_kh, + stride_kd, + stride_vs, + stride_vh, + stride_vd, + stride_sh, + stride_sd, + stride_th, + stride_tn, + stride_tk, + stride_on, + stride_oh, + stride_od, + stride_r2t_b, + # META parameters + BLOCK_SIZE_Q: tl.constexpr, # q block size + BLOCK_SIZE_K: tl.constexpr, # k block size + BLOCK_SIZE_KD: tl.constexpr, + BLOCK_SIZE_VD: tl.constexpr, + BLOCK_SIZE_H: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, + BLOCK_SIZE_QH: tl.constexpr, + # has sink + HAS_SINK: tl.constexpr, + USE_TMA: tl.constexpr, + IS_FP8: tl.constexpr, +): + sm_scale_log2e = sm_scale * 1.4426950409 + # get batch id and head id + pid_q = tl.program_id(0) + pid_kh = tl.program_id(1) + pid_b = tl.program_id(2) + pid_h = pid_kh * gqa_group_size + # get q k start and len after rmpad + q_start = tl.load(cu_seqlens_q + pid_b) + q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start + q_block_start = tl.load(cu_seqblocks_q + pid_b) + q_block_len = tl.load(cu_seqblocks_q + pid_b + 1) - q_block_start + seq_len = tl.load(seq_lens + pid_b) + prefix_len = tl.load(prefix_lens + pid_b) + sid = ( + tl.load(slot_ids + pid_b).to(tl.int64) + max_slots + ) % max_slots # safety against negative + if pid_q * num_q_loop >= q_block_len: + return + real_q_loop = min(num_q_loop, q_block_len - pid_q * num_q_loop) + if HAS_SINK: + sink_ptrs = tl.make_block_ptr( + base=sink_ptr + pid_h * stride_sh, + shape=(gqa_group_size, qk_head_dim), + strides=(stride_sh, stride_sd), + offsets=(0, 0), + block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(1, 0), + ) + sink = tl.load(sink_ptrs, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + # offsets for paged K/V load + off_n = tl.arange(0, BLOCK_SIZE_K) + off_kd = tl.arange(0, BLOCK_SIZE_KD) + off_vd = tl.arange(0, BLOCK_SIZE_VD) + kd_mask = off_kd < qk_head_dim + vd_mask = off_vd < v_head_dim + for j in range(real_q_loop): + pid_q_j = pid_q * num_q_loop + j + # init topk idx pointer + t_ptr_j = t_ptr + (q_block_start + pid_q_j) * stride_tn + pid_kh * stride_th + # we assume that the topk_idx is right padded with -1 + off_t = tl.arange(0, BLOCK_SIZE_T) + topk_idx = tl.load(t_ptr_j + off_t * stride_tk, mask=off_t < max_topk, other=-1) + valid_idx = tl.where(topk_idx >= 0, off_t, -1) + real_topk = tl.sum(valid_idx != -1, axis=0) + # init qkv pointer + q_ptrs = tl.make_block_ptr( + base=q_ptr + q_start * stride_qn + pid_h * stride_qh, + shape=(q_len, gqa_group_size, qk_head_dim), + strides=(stride_qn, stride_qh, stride_qd), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_KD), + order=(2, 1, 0), + ) + # load q, shape: [BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_D] -> [BLOCK_SIZE_QH, BLOCK_SIZE_D] + q = tl.load(q_ptrs, boundary_check=(0, 1, 2), padding_option="zero") + # init statistics + off_q_k = ( + tl.arange(0, BLOCK_SIZE_Q)[:, None] + + pid_q_j * BLOCK_SIZE_Q + + prefix_len + - tl.arange(0, BLOCK_SIZE_K)[None, :] + ) + if HAS_SINK: + m_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + lse_i = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H), dtype=tl.float32) + qsink = ( + tl.sum(q.to(tl.float32) * sink[None, :, :], axis=2) * sm_scale_log2e + ) # (BLOCK_SIZE_Q, BLOCK_SIZE_H) + m_i += qsink + lse_i += qsink + m_i = tl.reshape(m_i, BLOCK_SIZE_QH) + lse_i = tl.reshape(lse_i, BLOCK_SIZE_QH) + else: + m_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + lse_i = tl.full((BLOCK_SIZE_QH,), float("-inf"), dtype=tl.float32) + acc_o = tl.full((BLOCK_SIZE_QH, BLOCK_SIZE_VD), 0, dtype=tl.float32) + q = tl.reshape(q, BLOCK_SIZE_QH, BLOCK_SIZE_KD) + # sparse attention + for i in range(real_topk): + # get current block start index (absolute K position) + c = tl.load(t_ptr_j).to(tl.int32) * BLOCK_SIZE_K + t_ptr_j = t_ptr_j + stride_tk + # paged load K via req_to_token: pos -> slot -> k_cache + pos = c + off_n + pos_mask = pos < seq_len + slots = tl.load( + req_to_token_ptr + sid * stride_r2t_b + pos, + mask=pos_mask, + other=0, + ).to(tl.int64) + slots = (slots + max_slots) % max_slots # safety against negative + # k shape: [BLOCK_SIZE_KD, BLOCK_SIZE_K] (transposed for tl.dot) + k = tl.load( + k_cache_ptr + + slots[None, :] * stride_ks + + pid_kh * stride_kh + + off_kd[:, None] * stride_kd, + mask=kd_mask[:, None] & pos_mask[None, :], + other=0.0, + ) + if IS_FP8: + # fp8 main K cache is unit-scaled; widen to the Q compute dtype + # before the tl.dot (compiled out when the cache is bf16). + k = k.to(q.dtype) + # compute qk + qk = tl.zeros((BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32) + # causal mask + qk += tl.where(off_q_k[:, None, :] >= c, 0, float("-inf")) + qk = tl.reshape(qk, BLOCK_SIZE_QH, BLOCK_SIZE_K) + # [BLOCK_SIZE_QH, qk_head_dim] @ [qk_head_dim, BLOCK_SIZE_K] + # -> [BLOCK_SIZE_QH, BLOCK_SIZE_K] + qk += tl.dot(q, k) * sm_scale_log2e + # K boundary mask: positions beyond seq_len contribute -inf + qk += tl.where(pos_mask[None, :], 0, float("-inf")) + # compute m_ij and l_ij + m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) + p = tl.exp2(qk - m_ij[:, None]) + l_ij = tl.sum(p, axis=1) + # scale acc_o + acc_o_scale = tl.exp2(m_i - m_ij) + acc_o = acc_o * acc_o_scale[:, None] + # paged load V + v = tl.load( + v_cache_ptr + + slots[:, None] * stride_vs + + pid_kh * stride_vh + + off_vd[None, :] * stride_vd, + mask=pos_mask[:, None] & vd_mask[None, :], + other=0.0, + ) + if IS_FP8: + # Widen V so `p.to(v.dtype)` casts P to the compute dtype rather + # than to fp8 (which would wreck attention-weight precision). + v = v.to(q.dtype) + p = p.to(v.dtype) + acc_o += tl.dot(p, v) + # update statistics + m_i = m_ij + lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij) + # final scale + acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None] + # save output + acc_o = tl.reshape(acc_o, BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD) + o_ptrs = tl.make_block_ptr( + base=o_ptr + q_start * stride_on + pid_h * stride_oh, + shape=(q_len, gqa_group_size, v_head_dim), + strides=(stride_on, stride_oh, stride_od), + offsets=(pid_q_j * BLOCK_SIZE_Q, 0, 0), + block_shape=(BLOCK_SIZE_Q, BLOCK_SIZE_H, BLOCK_SIZE_VD), + order=(2, 1, 0), + ) + tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1, 2)) + + +@torch.no_grad() +def flash_prefill_with_gqa_share_sparse( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + sink: Optional[torch.Tensor], + req_to_token: torch.Tensor, + slot_ids: torch.Tensor, + topk_idx: torch.Tensor, + block_size_q: int, + block_size_k: int, + cu_seqlens: torch.Tensor, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + max_seqlen_q: int, + sm_scale: Optional[float] = None, + use_tma: bool = True, + cu_seqblocks_q: Optional[torch.Tensor] = None, + max_seqblock_q: Optional[int] = None, +) -> torch.Tensor: + triton.set_allocator(robust_allocator) + # dtype check. On HIP, the paged main K/V cache may be fp8 (unit-scaled) + # under --kv-cache-dtype fp8_*; the kernel widens it to the Q dtype on load. + # Keep CUDA's pre-existing sparse prefill dtype contract unchanged by + # accepting fp8 only on HIP. + assert q.dtype in (torch.bfloat16, torch.float16) + _FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz) + is_fp8 = _is_hip and k_cache.dtype in _FP8_DTYPES + assert k_cache.dtype == q.dtype or is_fp8, ( + f"sparse prefill expects K cache dtype == Q dtype ({q.dtype}) " + f"or fp8 on HIP, got {k_cache.dtype}" + ) + assert v_cache.dtype == k_cache.dtype + assert block_size_q in {1, 2, 4, 8, 16, 32, 64} + assert block_size_k in {16, 32, 64, 128} + # shape + total_q, num_q_heads, qk_head_dim = q.shape + max_slots, num_k_heads, _ = k_cache.shape + _, num_v_heads, v_head_dim = v_cache.shape + batch_size = cu_seqlens.shape[0] - 1 + topk = topk_idx.shape[-1] + assert topk_idx.shape[0] == num_k_heads + # gqa + assert num_k_heads == num_v_heads + assert num_q_heads % num_k_heads == 0 + gqa_group_size = num_q_heads // num_k_heads + assert gqa_group_size * block_size_q <= 128 + if sm_scale is None: + sm_scale = qk_head_dim**-0.5 + if cu_seqblocks_q is None or max_seqblock_q is None: + cu_seqblocks_q, max_seqblock_q, _, _, _, _ = get_cu_seqblocks( + cu_seqlens, max_seqlen_q, block_size_q, block_size_k + ) + # output tensor + o = torch.empty(total_q, num_q_heads, v_head_dim, device=q.device, dtype=q.dtype) + # launch kernel + num_q_loop = ( + max_seqblock_q // 131072 + 1 + ) # calculate multiple queries in one kernel if seqlence length is too long + BLOCK_SIZE_Q = triton.next_power_of_2(block_size_q) + BLOCK_SIZE_K = triton.next_power_of_2(block_size_k) + grid = ( + triton.cdiv(triton.cdiv(max_seqlen_q, block_size_q), num_q_loop), + num_k_heads, + batch_size, + ) + _gqa_share_sparse_fwd_kernel[grid]( + q, + k_cache, + v_cache, + sink, + topk_idx, + o, + req_to_token, + cu_seqlens, + cu_seqblocks_q, + seq_lens, + prefix_lens, + slot_ids, + max_slots, + num_k_heads, + gqa_group_size, + qk_head_dim, + v_head_dim, + topk, + num_q_loop, + sm_scale, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + sink.stride(0) if sink is not None else 0, + sink.stride(1) if sink is not None else 0, + topk_idx.stride(0), + topk_idx.stride(1), + topk_idx.stride(2), + o.stride(0), + o.stride(1), + o.stride(2), + req_to_token.stride(0), + BLOCK_SIZE_Q=BLOCK_SIZE_Q, + BLOCK_SIZE_K=BLOCK_SIZE_K, + USE_TMA=use_tma, + IS_FP8=is_fp8, + ) + return o diff --git a/build/torch212-cxx11-cu132-x86_64-linux/quantize.py b/build/torch212-cxx11-cu132-x86_64-linux/quantize.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2675ac49d03e5599e9559187efecdf63bb6387 --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/quantize.py @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: Apache-2.0 +"""NVFP4 helper API compatible with MiniMaxAI/msa. + +The quantization entrypoints use Transformer Engine when it is installed. The +layout helpers and reference dequantizer are pure PyTorch and are included so +callers can prepare and validate NVFP4 metadata with the same public API names +as the upstream MiniMaxAI package. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Tuple + +import torch + + +NVFP4_BLOCK_SIZE = 16 +NVFP4_FP4_MAX = 6.0 +NVFP4_FP8_E4M3_MAX = 448.0 + + +@dataclass(frozen=True) +class Nvfp4QuantizedTensor: + data: torch.Tensor + scale_128x4: torch.Tensor + global_scale: torch.Tensor + logical_scale_shape: Tuple[int, int] + original_shape: Tuple[int, ...] + + +def _round_up(x: int, multiple: int) -> int: + return ((int(x) + multiple - 1) // multiple) * multiple + + +def nvfp4_scale_128x4_offset( + row: torch.Tensor, + col: torch.Tensor, + scale_cols: int, +) -> torch.Tensor: + tiles_n = _round_up(scale_cols, 4) // 4 + tile_m = row // 128 + tile_n = col // 4 + outer = row % 128 + inner = col % 4 + return ( + (tile_m * tiles_n + tile_n) * 512 + + (outer % 32) * 16 + + (outer // 32) * 4 + + inner + ) + + +def swizzle_nvfp4_scale_to_128x4( + scale: torch.Tensor, + *, + rows: int, + cols: int, +) -> torch.Tensor: + if scale.ndim != 2: + raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}") + + rows = int(rows) + cols = int(cols) + padded_rows = _round_up(rows, 128) + padded_cols = _round_up(cols, 4) + if scale.shape[0] < rows or scale.shape[1] < cols: + raise ValueError( + "scale is smaller than the requested logical shape: " + f"got {tuple(scale.shape)}, need at least {(rows, cols)}" + ) + + logical = scale[:rows, :cols].contiguous() + if logical.shape != (padded_rows, padded_cols): + logical = torch.nn.functional.pad( + logical.to(torch.float32), + (0, padded_cols - cols, 0, padded_rows - rows), + ).to(scale.dtype) + swizzled = torch.empty_like(logical) + + row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None] + col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1) + swizzled.reshape(-1)[offset] = logical.reshape(-1) + return swizzled + + +def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor: + return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX) + + +def _import_te_nvfp4_quantizer(): + try: + from transformer_engine.pytorch.tensor import NVFP4Quantizer + except Exception as exc: + raise RuntimeError( + "Transformer Engine NVFP4 quantization is unavailable. Install a " + "Transformer Engine build with NVFP4 PyTorch support to use " + "quantize_bf16_to_nvfp4_128x4." + ) from exc + return NVFP4Quantizer + + +def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor: + if not x.is_cuda: + raise ValueError("NVFP4 quantization requires a CUDA tensor") + if x.dtype not in (torch.bfloat16, torch.float16): + raise TypeError(f"x must be bf16 or fp16, got {x.dtype}") + if x.ndim < 2: + raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}") + if x.shape[-1] % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}" + ) + + rows = 1 + for dim in x.shape[:-1]: + rows *= int(dim) + if rows % NVFP4_BLOCK_SIZE != 0: + raise ValueError( + "flattened row dimension must be divisible by " + f"{NVFP4_BLOCK_SIZE}, got {rows}" + ) + + NVFP4Quantizer = _import_te_nvfp4_quantizer() + quantizer = NVFP4Quantizer(rowwise=True, columnwise=False) + qx = quantizer.quantize(x.contiguous()) + meta = qx.get_metadata() + + data = meta["rowwise_data"] + if data.dtype is not torch.uint8: + data = data.view(torch.uint8) + logical_scale = meta["rowwise_scale_inv"] + amax = meta["amax_rowwise"] + scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE + scale_128x4 = swizzle_nvfp4_scale_to_128x4( + logical_scale, + rows=rows, + cols=scale_cols, + ) + global_scale = nvfp4_global_scale_from_amax(amax).contiguous() + + return Nvfp4QuantizedTensor( + data=data, + scale_128x4=scale_128x4, + global_scale=global_scale, + logical_scale_shape=(rows, scale_cols), + original_shape=tuple(int(v) for v in x.shape), + ) + + +def quantize_kv_bf16_to_nvfp4_128x4( + k: torch.Tensor, + v: torch.Tensor, +) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]: + return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v) + + +def dequantize_nvfp4_128x4_to_bf16( + qx: Nvfp4QuantizedTensor, + *, + include_global_scale: bool = True, +) -> torch.Tensor: + data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8) + if data.shape[-1] * 2 != qx.original_shape[-1]: + raise ValueError( + "packed data last dimension does not match original shape: " + f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical" + ) + + rows, scale_cols = qx.logical_scale_shape + logical_dim = int(qx.original_shape[-1]) + if scale_cols * NVFP4_BLOCK_SIZE != logical_dim: + raise ValueError( + "logical scale columns do not match original last dimension: " + f"{scale_cols} scale cols vs dim {logical_dim}" + ) + + fp4_lut = torch.tensor( + [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + -0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, + ], + dtype=torch.float32, + device=data.device, + ) + packed = data.reshape(rows, logical_dim // 2) + lo = packed & 0x0F + hi = packed >> 4 + values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device) + values[:, 0::2] = fp4_lut[lo.long()] + values[:, 1::2] = fp4_lut[hi.long()] + + row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None] + col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :] + offset = nvfp4_scale_128x4_offset(row, col, scale_cols) + scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols) + scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32) + scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1) + out = values * scale + if include_global_scale: + out = out * qx.global_scale.reshape(-1)[0].to(torch.float32) + return out.reshape(qx.original_shape).to(torch.bfloat16) + + +__all__ = [ + "Nvfp4QuantizedTensor", + "quantize_bf16_to_nvfp4_128x4", + "quantize_kv_bf16_to_nvfp4_128x4", + "dequantize_nvfp4_128x4_to_bf16", + "swizzle_nvfp4_scale_to_128x4", + "nvfp4_global_scale_from_amax", +] diff --git a/build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py b/build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..177f864da993328cc460b8de391c6091f3be0f9d --- /dev/null +++ b/build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Sparse-index helpers compatible with MiniMaxAI/msa public names.""" + +from __future__ import annotations + +from typing import Optional + +import torch + + +def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None: + if cu_seqlens.dtype != torch.int32: + raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}") + if cu_seqlens.ndim != 1: + raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}") + if cu_seqlens.numel() < 1: + raise ValueError(f"{name} must have at least one element") + if not cu_seqlens.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + + +def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor: + seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1] + return (seqlens_k + kv_block_size - 1) // kv_block_size + + +def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]: + rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist() + batch = len(rows_per_batch_cpu) + max_rows = max(rows_per_batch_cpu, default=0) + row_dtype = ( + torch.int32 + if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max + else torch.int64 + ) + row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype) + row_linear = 0 + for kv_block_idx in range(max_rows): + for batch_idx, row_count in enumerate(rows_per_batch_cpu): + if kv_block_idx < row_count: + row_map_cpu[batch_idx, kv_block_idx] = row_linear + row_linear += 1 + return row_map_cpu.to(rows_per_batch.device), row_linear + + +def build_k2q_csr_torch_reference( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, +) -> tuple[torch.Tensor, torch.Tensor]: + if kv_block_size <= 0: + raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}") + if q2k_indices.dtype != torch.int32: + raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}") + if q2k_indices.ndim != 3: + raise ValueError( + "q2k_indices must have shape [head_kv, total_q, topK], " + f"got {tuple(q2k_indices.shape)}" + ) + _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q") + _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k") + if cu_seqlens_q.shape != cu_seqlens_k.shape: + raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]") + if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device: + raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device") + + head_kv, total_q, topk = q2k_indices.shape + if total_q != int(cu_seqlens_q[-1].item()): + raise ValueError( + f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] " + f"({int(cu_seqlens_q[-1].item())})" + ) + + rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size) + row_map, total_rows = _build_packed_row_map(rows_per_batch) + nnz_upper_bound = total_q * topk + + k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device) + k2q_q_indices = torch.full( + (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device + ) + if total_rows == 0 or total_q == 0 or topk == 0: + return k2q_row_ptr, k2q_q_indices + + counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device) + total_entries = total_q * topk + row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64 + row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device) + q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device) + valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device) + + for batch_idx in range(cu_seqlens_q.numel() - 1): + q_start = int(cu_seqlens_q[batch_idx].item()) + q_end = int(cu_seqlens_q[batch_idx + 1].item()) + if q_end <= q_start: + continue + q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device) + q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1) + kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1) + valid = kv_local >= 0 + safe_kv = kv_local.clamp_min(0).long() + rows = row_map[batch_idx, safe_kv].to(row_dtype) + start = q_start * topk + end = q_end * topk + row_all[:, start:end] = rows + q_all[:, start:end] = q_local[None, :].expand(head_kv, -1) + valid_all[:, start:end] = valid & (rows >= 0) + counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32)) + + k2q_row_ptr[:, 1:] = counts.cumsum(dim=1) + offsets = torch.zeros_like(counts) + offsets[:, 1:] = counts[:, :-1].cumsum(dim=1) + cursor = torch.zeros_like(counts) + for h in range(head_kv): + for entry in range(total_entries): + if not bool(valid_all[h, entry]): + continue + row = int(row_all[h, entry].item()) + pos = int(offsets[h, row].item() + cursor[h, row].item()) + k2q_q_indices[h, pos] = q_all[h, entry] + cursor[h, row] += 1 + + return k2q_row_ptr, k2q_q_indices + + +def build_k2q_csr( + q2k_indices: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + kv_block_size: int, + *, + total_k: Optional[int] = None, + max_seqlen_k: Optional[int] = None, + max_seqlen_q: Optional[int] = None, + total_rows: Optional[int] = None, + qhead_per_kv: int = 1, + return_schedule: bool = False, +): + if return_schedule: + raise NotImplementedError( + "Blackwell build_k2q_csr currently returns CSR tensors only; " + "SM100 schedule objects are not part of this package." + ) + if total_k is None: + raise ValueError("build_k2q_csr requires total_k from k.shape[0]") + return build_k2q_csr_torch_reference( + q2k_indices.contiguous(), + cu_seqlens_q.contiguous(), + cu_seqlens_k.contiguous(), + int(kv_block_size), + ) + + +class SparseK2qCsrBuilderSm100: + """Compatibility placeholder for the upstream SM100 class name. + + The name is exported so code can feature-detect it, but schedule-producing + SM100 behavior is intentionally not claimed for this Blackwell package. + """ + + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def build(self, *args, **kwargs): + return build_k2q_csr(*args, **kwargs) + + +__all__ = [ + "build_k2q_csr", + "build_k2q_csr_torch_reference", + "SparseK2qCsrBuilderSm100", +]