liangsu9988 commited on
Commit
2f841ca
·
verified ·
1 Parent(s): 03cf969

Uploaded using `kernel-builder`.

Browse files
Files changed (32) hide show
  1. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +45 -0
  2. build/torch211-cxx11-cu128-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
  3. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  4. build/torch211-cxx11-cu128-x86_64-linux/api_status.py +145 -0
  5. build/torch211-cxx11-cu128-x86_64-linux/interface.py +257 -0
  6. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +9 -5
  7. build/torch211-cxx11-cu128-x86_64-linux/quantize.py +227 -0
  8. build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py +174 -0
  9. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +45 -0
  10. build/torch211-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
  11. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  12. build/torch211-cxx11-cu130-x86_64-linux/api_status.py +145 -0
  13. build/torch211-cxx11-cu130-x86_64-linux/interface.py +257 -0
  14. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +9 -5
  15. build/torch211-cxx11-cu130-x86_64-linux/quantize.py +227 -0
  16. build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py +174 -0
  17. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +45 -0
  18. build/torch212-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
  19. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  20. build/torch212-cxx11-cu130-x86_64-linux/api_status.py +145 -0
  21. build/torch212-cxx11-cu130-x86_64-linux/interface.py +257 -0
  22. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +9 -5
  23. build/torch212-cxx11-cu130-x86_64-linux/quantize.py +227 -0
  24. build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py +174 -0
  25. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +45 -0
  26. build/torch212-cxx11-cu132-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
  27. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  28. build/torch212-cxx11-cu132-x86_64-linux/api_status.py +145 -0
  29. build/torch212-cxx11-cu132-x86_64-linux/interface.py +257 -0
  30. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +9 -5
  31. build/torch212-cxx11-cu132-x86_64-linux/quantize.py +227 -0
  32. build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py +174 -0
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
40
  from ._native import has_native_ops, native_topk_from_scores
41
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
42
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
 
44
  __all__ = [
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  "flash_decode_with_topk_idx",
46
  "flash_decode_with_gqa_share_sparse",
47
  "has_native_ops",
@@ -50,4 +88,11 @@ __all__ = [
50
  "naive_flash_decode_with_topk_idx",
51
  "get_cu_seqblocks",
52
  "robust_allocator",
 
 
 
 
 
 
 
53
  ]
 
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
40
+ from .interface import (
41
+ SparseDecodePagedAttentionWrapper,
42
+ fp4_indexer_block_scores,
43
+ sparse_atten_func,
44
+ sparse_atten_nvfp4_kv_func,
45
+ sparse_decode_atten_func,
46
+ )
47
  from ._native import has_native_ops, native_topk_from_scores
48
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
49
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
50
+ from .quantize import (
51
+ Nvfp4QuantizedTensor,
52
+ dequantize_nvfp4_128x4_to_bf16,
53
+ nvfp4_global_scale_from_amax,
54
+ quantize_bf16_to_nvfp4_128x4,
55
+ quantize_kv_bf16_to_nvfp4_128x4,
56
+ swizzle_nvfp4_scale_to_128x4,
57
+ )
58
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
59
+ from .api_status import (
60
+ OFFICIAL_API_STATUS,
61
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
62
+ V1_AVAILABLE_FUNCTIONS,
63
+ available_functions,
64
+ official_api_status,
65
+ official_minimax_msa_functions,
66
+ unsupported_official_functions,
67
+ )
68
 
69
  __all__ = [
70
+ "sparse_atten_func",
71
+ "sparse_atten_nvfp4_kv_func",
72
+ "sparse_decode_atten_func",
73
+ "SparseDecodePagedAttentionWrapper",
74
+ "fp4_indexer_block_scores",
75
+ "build_k2q_csr",
76
+ "SparseK2qCsrBuilderSm100",
77
+ "Nvfp4QuantizedTensor",
78
+ "quantize_bf16_to_nvfp4_128x4",
79
+ "quantize_kv_bf16_to_nvfp4_128x4",
80
+ "dequantize_nvfp4_128x4_to_bf16",
81
+ "swizzle_nvfp4_scale_to_128x4",
82
+ "nvfp4_global_scale_from_amax",
83
  "flash_decode_with_topk_idx",
84
  "flash_decode_with_gqa_share_sparse",
85
  "has_native_ops",
 
88
  "naive_flash_decode_with_topk_idx",
89
  "get_cu_seqblocks",
90
  "robust_allocator",
91
+ "OFFICIAL_API_STATUS",
92
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
93
+ "V1_AVAILABLE_FUNCTIONS",
94
+ "available_functions",
95
+ "official_api_status",
96
+ "official_minimax_msa_functions",
97
+ "unsupported_official_functions",
98
  ]
build/torch211-cxx11-cu128-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0739017863acc2b1c48173e2480ae8e37301058af72ce28534dd82125a0ab62d
3
  size 119536
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0cc5542463661012370d32bb20bc9740c100711bb93eac37cab3c7f09d841781
3
  size 119536
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _minimaxai_msa_blackwell_cuda_6347cca
3
- ops = torch.ops._minimaxai_msa_blackwell_cuda_6347cca
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_minimaxai_msa_blackwell_cuda_6347cca::{op_name}"
 
1
  import torch
2
+ from . import _minimaxai_msa_blackwell_cuda_3dca94b
3
+ ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/api_status.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Public API status for the FlashRT MiniMax MSA Blackwell package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from types import MappingProxyType
7
+
8
+
9
+ V1_AVAILABLE_FUNCTIONS = (
10
+ "sparse_atten_func",
11
+ "sparse_atten_nvfp4_kv_func",
12
+ "sparse_decode_atten_func",
13
+ "SparseDecodePagedAttentionWrapper",
14
+ "fp4_indexer_block_scores",
15
+ "build_k2q_csr",
16
+ "SparseK2qCsrBuilderSm100",
17
+ "Nvfp4QuantizedTensor",
18
+ "quantize_bf16_to_nvfp4_128x4",
19
+ "quantize_kv_bf16_to_nvfp4_128x4",
20
+ "dequantize_nvfp4_128x4_to_bf16",
21
+ "swizzle_nvfp4_scale_to_128x4",
22
+ "nvfp4_global_scale_from_amax",
23
+ "has_native_ops",
24
+ "native_topk_from_scores",
25
+ "flash_decode_with_topk_idx",
26
+ "flash_decode_with_gqa_share_sparse",
27
+ "naive_flash_decode_with_topk_idx",
28
+ "naive_flash_decode_with_gqa_share_sparse",
29
+ "get_cu_seqblocks",
30
+ "robust_allocator",
31
+ )
32
+
33
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
34
+ "sparse_atten_func",
35
+ "sparse_atten_nvfp4_kv_func",
36
+ "sparse_decode_atten_func",
37
+ "SparseDecodePagedAttentionWrapper",
38
+ "fp4_indexer_block_scores",
39
+ "build_k2q_csr",
40
+ "SparseK2qCsrBuilderSm100",
41
+ "Nvfp4QuantizedTensor",
42
+ "quantize_bf16_to_nvfp4_128x4",
43
+ "quantize_kv_bf16_to_nvfp4_128x4",
44
+ "dequantize_nvfp4_128x4_to_bf16",
45
+ "swizzle_nvfp4_scale_to_128x4",
46
+ "nvfp4_global_scale_from_amax",
47
+ )
48
+
49
+ _OFFICIAL_API_STATUS = {
50
+ "sparse_atten_func": {
51
+ "status": "unsupported_callable",
52
+ "target": "blackwell-prefill",
53
+ "reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
54
+ },
55
+ "sparse_atten_nvfp4_kv_func": {
56
+ "status": "unsupported_callable",
57
+ "target": "blackwell-prefill-nvfp4",
58
+ "reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
59
+ },
60
+ "sparse_decode_atten_func": {
61
+ "status": "available",
62
+ "target": "v1",
63
+ "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
64
+ },
65
+ "SparseDecodePagedAttentionWrapper": {
66
+ "status": "available",
67
+ "target": "v1",
68
+ "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
69
+ },
70
+ "fp4_indexer_block_scores": {
71
+ "status": "unsupported_callable",
72
+ "target": "blackwell-fp4-indexer",
73
+ "reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
74
+ },
75
+ "build_k2q_csr": {
76
+ "status": "available",
77
+ "target": "v1",
78
+ "reason": "Torch CSR construction fallback is available for compatibility.",
79
+ },
80
+ "SparseK2qCsrBuilderSm100": {
81
+ "status": "available",
82
+ "target": "v1",
83
+ "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
84
+ },
85
+ "Nvfp4QuantizedTensor": {
86
+ "status": "available",
87
+ "target": "v1",
88
+ "reason": "NVFP4 metadata dataclass is available.",
89
+ },
90
+ "quantize_bf16_to_nvfp4_128x4": {
91
+ "status": "available_optional_te",
92
+ "target": "v1",
93
+ "reason": "Function is available and uses Transformer Engine when installed.",
94
+ },
95
+ "quantize_kv_bf16_to_nvfp4_128x4": {
96
+ "status": "available_optional_te",
97
+ "target": "v1",
98
+ "reason": "Function is available and uses Transformer Engine when installed.",
99
+ },
100
+ "dequantize_nvfp4_128x4_to_bf16": {
101
+ "status": "available",
102
+ "target": "v1",
103
+ "reason": "Reference dequant helper is available.",
104
+ },
105
+ "swizzle_nvfp4_scale_to_128x4": {
106
+ "status": "available",
107
+ "target": "v1",
108
+ "reason": "Scale swizzle helper is available.",
109
+ },
110
+ "nvfp4_global_scale_from_amax": {
111
+ "status": "available",
112
+ "target": "v1",
113
+ "reason": "Global scale helper is available.",
114
+ },
115
+ }
116
+
117
+ OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
118
+
119
+
120
+ def available_functions() -> tuple[str, ...]:
121
+ """Return the functions/classes intentionally available in v1."""
122
+
123
+ return V1_AVAILABLE_FUNCTIONS
124
+
125
+
126
+ def official_minimax_msa_functions() -> tuple[str, ...]:
127
+ """Return the upstream MiniMaxAI/msa public function/class names tracked here."""
128
+
129
+ return OFFICIAL_MINIMAX_MSA_FUNCTIONS
130
+
131
+
132
+ def official_api_status() -> dict[str, dict[str, str]]:
133
+ """Return a copy of the official API compatibility status table."""
134
+
135
+ return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
136
+
137
+
138
+ def unsupported_official_functions() -> tuple[str, ...]:
139
+ """Return official MiniMaxAI/msa names that are not exported by this v1 package."""
140
+
141
+ return tuple(
142
+ name
143
+ for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
144
+ if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
145
+ )
build/torch211-cxx11-cu128-x86_64-linux/interface.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
11
+
12
+
13
+ def _unsupported(name: str, reason: str):
14
+ raise NotImplementedError(
15
+ f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
16
+ )
17
+
18
+
19
+ def _page_table_to_req_to_token(
20
+ page_table: torch.Tensor,
21
+ seqused_k: torch.Tensor,
22
+ max_seqlen_k: int,
23
+ blk_kv: int,
24
+ ) -> torch.Tensor:
25
+ batch = int(page_table.shape[0])
26
+ max_seqlen_k = int(max_seqlen_k)
27
+ blk_kv = int(blk_kv)
28
+ logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
29
+ page = logical_pos // blk_kv
30
+ offset = logical_pos % blk_kv
31
+ page = page.clamp(max=int(page_table.shape[1]) - 1)
32
+ physical_page = page_table[:, page].to(torch.int64)
33
+ physical_token = physical_page * blk_kv + offset[None, :]
34
+ valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
35
+ return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
36
+
37
+
38
+ def _dense_q2k_indices(
39
+ *,
40
+ head_kv: int,
41
+ batch: int,
42
+ topk: int,
43
+ seq_lens: torch.Tensor,
44
+ blk_kv: int,
45
+ ) -> torch.Tensor:
46
+ idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
47
+ blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
48
+ valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
49
+ for b in range(batch):
50
+ valid = blocks < valid_blocks[b]
51
+ idx[:, b, valid] = blocks[valid]
52
+ return idx
53
+
54
+
55
+ def _normalize_decode_q2k(
56
+ q2k_indices: Optional[torch.Tensor],
57
+ *,
58
+ batch: int,
59
+ seqlen_q: int,
60
+ head_kv: int,
61
+ seqused_k: torch.Tensor,
62
+ blk_kv: int,
63
+ ) -> torch.Tensor:
64
+ if q2k_indices is None:
65
+ topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
66
+ return _dense_q2k_indices(
67
+ head_kv=head_kv,
68
+ batch=batch,
69
+ topk=topk,
70
+ seq_lens=seqused_k,
71
+ blk_kv=blk_kv,
72
+ )
73
+ if q2k_indices.dtype != torch.int32:
74
+ raise TypeError("q2k_indices must be torch.int32")
75
+ if q2k_indices.ndim != 3:
76
+ raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
77
+ if q2k_indices.shape[0] != head_kv:
78
+ raise ValueError("q2k_indices first dimension must match Hkv")
79
+ if int(q2k_indices.shape[1]) != batch * seqlen_q:
80
+ raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
81
+ if seqlen_q != 1:
82
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
83
+ return q2k_indices[:, ::seqlen_q, :].contiguous()
84
+
85
+
86
+ def sparse_decode_atten_func(
87
+ q: torch.Tensor,
88
+ k: torch.Tensor,
89
+ v: torch.Tensor,
90
+ q2k_indices: Optional[torch.Tensor] = None,
91
+ *,
92
+ page_table: torch.Tensor,
93
+ seqused_k: torch.Tensor,
94
+ seqlen_q: int,
95
+ max_seqlen_k: int,
96
+ blk_kv: int = 128,
97
+ causal: bool = True,
98
+ softmax_scale: Optional[float] = None,
99
+ return_softmax_lse: bool = False,
100
+ schedule: Optional[object] = None,
101
+ O_partial: Optional[torch.Tensor] = None,
102
+ LSE_partial: Optional[torch.Tensor] = None,
103
+ ):
104
+ """Run MiniMax paged sparse decode on Blackwell.
105
+
106
+ This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
107
+ but accepts the Blackwell package's validated BF16/FP16 decode path. It
108
+ supports single-token decode (`seqlen_q=1`) with paged K/V
109
+ `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
110
+ """
111
+
112
+ if schedule is not None or O_partial is not None or LSE_partial is not None:
113
+ raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
114
+ if return_softmax_lse:
115
+ raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
116
+ if not causal:
117
+ raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
118
+ if int(seqlen_q) != 1:
119
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
120
+ if q.ndim != 3:
121
+ raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
122
+ if k.ndim != 4 or v.ndim != 4:
123
+ raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
124
+ if k.shape != v.shape:
125
+ raise ValueError("k and v must have the same shape")
126
+ if q.device != k.device or q.device != v.device or q.device != page_table.device:
127
+ raise ValueError("q, k, v, and page_table must be on the same device")
128
+ if q.dtype not in (torch.bfloat16, torch.float16):
129
+ raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
130
+ if k.dtype != q.dtype or v.dtype != q.dtype:
131
+ raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
132
+ if q.shape[-1] != 128 or k.shape[-1] != 128:
133
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
134
+ if int(k.shape[2]) != int(blk_kv):
135
+ raise ValueError("k.shape[2] must equal blk_kv")
136
+ if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
137
+ raise TypeError("page_table and seqused_k must be torch.int32")
138
+ if page_table.ndim != 2:
139
+ raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
140
+ batch = int(page_table.shape[0])
141
+ if int(q.shape[0]) != batch:
142
+ raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
143
+
144
+ head_kv = int(k.shape[1])
145
+ q2k = _normalize_decode_q2k(
146
+ q2k_indices,
147
+ batch=batch,
148
+ seqlen_q=int(seqlen_q),
149
+ head_kv=head_kv,
150
+ seqused_k=seqused_k,
151
+ blk_kv=int(blk_kv),
152
+ )
153
+ req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
154
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
155
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
156
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
157
+ return flash_decode_with_gqa_share_sparse(
158
+ q.contiguous(),
159
+ None,
160
+ k_cache,
161
+ v_cache,
162
+ req_to_token,
163
+ seqused_k.contiguous(),
164
+ slot_ids,
165
+ int(blk_kv),
166
+ q2k,
167
+ sm_scale=softmax_scale,
168
+ )
169
+
170
+
171
+ class SparseDecodePagedAttentionWrapper:
172
+ def __init__(self, *, blk_kv: int = 128, causal: bool = True):
173
+ self.blk_kv = int(blk_kv)
174
+ self.causal = bool(causal)
175
+ self.page_table: Optional[torch.Tensor] = None
176
+ self.seqused_k: Optional[torch.Tensor] = None
177
+ self.seqlen_q: Optional[int] = None
178
+ self.max_seqlen_k: Optional[int] = None
179
+ self.q2k_indices: Optional[torch.Tensor] = None
180
+
181
+ def plan(
182
+ self,
183
+ *,
184
+ page_table: torch.Tensor,
185
+ seqused_k: torch.Tensor,
186
+ seqlen_q: int,
187
+ max_seqlen_k: int,
188
+ q2k_indices: Optional[torch.Tensor] = None,
189
+ num_qo_heads: Optional[int] = None,
190
+ num_kv_heads: Optional[int] = None,
191
+ head_dim: Optional[int] = 128,
192
+ enable_cuda_graph: bool = False,
193
+ max_grid_size: Optional[int] = None,
194
+ fixed_split_size: Optional[int] = None,
195
+ disable_split_kv: bool = False,
196
+ ) -> "SparseDecodePagedAttentionWrapper":
197
+ if head_dim is not None and int(head_dim) != 128:
198
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
199
+ self.page_table = page_table.contiguous()
200
+ self.seqused_k = seqused_k.contiguous()
201
+ self.seqlen_q = int(seqlen_q)
202
+ self.max_seqlen_k = int(max_seqlen_k)
203
+ self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
204
+ return self
205
+
206
+ def run(
207
+ self,
208
+ q: torch.Tensor,
209
+ k: torch.Tensor,
210
+ v: torch.Tensor,
211
+ *,
212
+ softmax_scale: Optional[float] = None,
213
+ return_softmax_lse: bool = False,
214
+ out: Optional[torch.Tensor] = None,
215
+ lse: Optional[torch.Tensor] = None,
216
+ ):
217
+ if self.page_table is None or self.seqused_k is None:
218
+ raise RuntimeError("decode wrapper must be planned before run")
219
+ result = sparse_decode_atten_func(
220
+ q,
221
+ k,
222
+ v,
223
+ self.q2k_indices,
224
+ page_table=self.page_table,
225
+ seqused_k=self.seqused_k,
226
+ seqlen_q=self.seqlen_q,
227
+ max_seqlen_k=self.max_seqlen_k,
228
+ blk_kv=self.blk_kv,
229
+ causal=self.causal,
230
+ softmax_scale=softmax_scale,
231
+ return_softmax_lse=return_softmax_lse,
232
+ )
233
+ if out is not None:
234
+ out.copy_(result)
235
+ return out
236
+ return result
237
+
238
+
239
+ def sparse_atten_func(*args, **kwargs):
240
+ _unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
241
+
242
+
243
+ def sparse_atten_nvfp4_kv_func(*args, **kwargs):
244
+ _unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
245
+
246
+
247
+ def fp4_indexer_block_scores(*args, **kwargs):
248
+ _unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
249
+
250
+
251
+ __all__ = [
252
+ "sparse_atten_func",
253
+ "sparse_atten_nvfp4_kv_func",
254
+ "sparse_decode_atten_func",
255
+ "SparseDecodePagedAttentionWrapper",
256
+ "fp4_indexer_block_scores",
257
+ ]
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
- "id": "_minimaxai_msa_blackwell_cuda_6347cca",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,20 +13,24 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "g6O7Aw9I1Db01ScTRhjP7xcQsx1f33SAdQjOpMRMgX0=",
17
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
18
- "_minimaxai_msa_blackwell_cuda_6347cca.abi3.so": "BzkBeGOswrHEgXPiSAro43MBBYr3LOKFNN2CEloKti0=",
19
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
20
- "_ops.py": "JaMCArFhNkHXPIKm1OiPosUE0JMLdsdDAtMk1gGh2FQ=",
 
21
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
22
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
23
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
24
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
25
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
 
26
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
27
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
28
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
29
- "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
 
 
30
  }
31
  }
32
  }
 
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
+ "id": "_minimaxai_msa_blackwell_cuda_3dca94b",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
17
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
18
+ "_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "DMVUJGNmEBI3DTK7ILyXQMEAcRu5Pqw3yrPH8J2EF4E=",
19
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
20
+ "_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
21
+ "api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
22
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
23
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
24
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
25
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
26
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
27
+ "interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
28
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
29
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
30
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
31
+ "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
32
+ "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
33
+ "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
34
  }
35
  }
36
  }
build/torch211-cxx11-cu128-x86_64-linux/quantize.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """NVFP4 helper API compatible with MiniMaxAI/msa.
3
+
4
+ The quantization entrypoints use Transformer Engine when it is installed. The
5
+ layout helpers and reference dequantizer are pure PyTorch and are included so
6
+ callers can prepare and validate NVFP4 metadata with the same public API names
7
+ as the upstream MiniMaxAI package.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Tuple
14
+
15
+ import torch
16
+
17
+
18
+ NVFP4_BLOCK_SIZE = 16
19
+ NVFP4_FP4_MAX = 6.0
20
+ NVFP4_FP8_E4M3_MAX = 448.0
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Nvfp4QuantizedTensor:
25
+ data: torch.Tensor
26
+ scale_128x4: torch.Tensor
27
+ global_scale: torch.Tensor
28
+ logical_scale_shape: Tuple[int, int]
29
+ original_shape: Tuple[int, ...]
30
+
31
+
32
+ def _round_up(x: int, multiple: int) -> int:
33
+ return ((int(x) + multiple - 1) // multiple) * multiple
34
+
35
+
36
+ def nvfp4_scale_128x4_offset(
37
+ row: torch.Tensor,
38
+ col: torch.Tensor,
39
+ scale_cols: int,
40
+ ) -> torch.Tensor:
41
+ tiles_n = _round_up(scale_cols, 4) // 4
42
+ tile_m = row // 128
43
+ tile_n = col // 4
44
+ outer = row % 128
45
+ inner = col % 4
46
+ return (
47
+ (tile_m * tiles_n + tile_n) * 512
48
+ + (outer % 32) * 16
49
+ + (outer // 32) * 4
50
+ + inner
51
+ )
52
+
53
+
54
+ def swizzle_nvfp4_scale_to_128x4(
55
+ scale: torch.Tensor,
56
+ *,
57
+ rows: int,
58
+ cols: int,
59
+ ) -> torch.Tensor:
60
+ if scale.ndim != 2:
61
+ raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
62
+
63
+ rows = int(rows)
64
+ cols = int(cols)
65
+ padded_rows = _round_up(rows, 128)
66
+ padded_cols = _round_up(cols, 4)
67
+ if scale.shape[0] < rows or scale.shape[1] < cols:
68
+ raise ValueError(
69
+ "scale is smaller than the requested logical shape: "
70
+ f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
71
+ )
72
+
73
+ logical = scale[:rows, :cols].contiguous()
74
+ if logical.shape != (padded_rows, padded_cols):
75
+ logical = torch.nn.functional.pad(
76
+ logical.to(torch.float32),
77
+ (0, padded_cols - cols, 0, padded_rows - rows),
78
+ ).to(scale.dtype)
79
+ swizzled = torch.empty_like(logical)
80
+
81
+ row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
82
+ col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
83
+ offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
84
+ swizzled.reshape(-1)[offset] = logical.reshape(-1)
85
+ return swizzled
86
+
87
+
88
+ def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
89
+ return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
90
+
91
+
92
+ def _import_te_nvfp4_quantizer():
93
+ try:
94
+ from transformer_engine.pytorch.tensor import NVFP4Quantizer
95
+ except Exception as exc:
96
+ raise RuntimeError(
97
+ "Transformer Engine NVFP4 quantization is unavailable. Install a "
98
+ "Transformer Engine build with NVFP4 PyTorch support to use "
99
+ "quantize_bf16_to_nvfp4_128x4."
100
+ ) from exc
101
+ return NVFP4Quantizer
102
+
103
+
104
+ def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
105
+ if not x.is_cuda:
106
+ raise ValueError("NVFP4 quantization requires a CUDA tensor")
107
+ if x.dtype not in (torch.bfloat16, torch.float16):
108
+ raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
109
+ if x.ndim < 2:
110
+ raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
111
+ if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
112
+ raise ValueError(
113
+ f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
114
+ )
115
+
116
+ rows = 1
117
+ for dim in x.shape[:-1]:
118
+ rows *= int(dim)
119
+ if rows % NVFP4_BLOCK_SIZE != 0:
120
+ raise ValueError(
121
+ "flattened row dimension must be divisible by "
122
+ f"{NVFP4_BLOCK_SIZE}, got {rows}"
123
+ )
124
+
125
+ NVFP4Quantizer = _import_te_nvfp4_quantizer()
126
+ quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
127
+ qx = quantizer.quantize(x.contiguous())
128
+ meta = qx.get_metadata()
129
+
130
+ data = meta["rowwise_data"]
131
+ if data.dtype is not torch.uint8:
132
+ data = data.view(torch.uint8)
133
+ logical_scale = meta["rowwise_scale_inv"]
134
+ amax = meta["amax_rowwise"]
135
+ scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
136
+ scale_128x4 = swizzle_nvfp4_scale_to_128x4(
137
+ logical_scale,
138
+ rows=rows,
139
+ cols=scale_cols,
140
+ )
141
+ global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
142
+
143
+ return Nvfp4QuantizedTensor(
144
+ data=data,
145
+ scale_128x4=scale_128x4,
146
+ global_scale=global_scale,
147
+ logical_scale_shape=(rows, scale_cols),
148
+ original_shape=tuple(int(v) for v in x.shape),
149
+ )
150
+
151
+
152
+ def quantize_kv_bf16_to_nvfp4_128x4(
153
+ k: torch.Tensor,
154
+ v: torch.Tensor,
155
+ ) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
156
+ return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
157
+
158
+
159
+ def dequantize_nvfp4_128x4_to_bf16(
160
+ qx: Nvfp4QuantizedTensor,
161
+ *,
162
+ include_global_scale: bool = True,
163
+ ) -> torch.Tensor:
164
+ data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
165
+ if data.shape[-1] * 2 != qx.original_shape[-1]:
166
+ raise ValueError(
167
+ "packed data last dimension does not match original shape: "
168
+ f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
169
+ )
170
+
171
+ rows, scale_cols = qx.logical_scale_shape
172
+ logical_dim = int(qx.original_shape[-1])
173
+ if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
174
+ raise ValueError(
175
+ "logical scale columns do not match original last dimension: "
176
+ f"{scale_cols} scale cols vs dim {logical_dim}"
177
+ )
178
+
179
+ fp4_lut = torch.tensor(
180
+ [
181
+ 0.0,
182
+ 0.5,
183
+ 1.0,
184
+ 1.5,
185
+ 2.0,
186
+ 3.0,
187
+ 4.0,
188
+ 6.0,
189
+ -0.0,
190
+ -0.5,
191
+ -1.0,
192
+ -1.5,
193
+ -2.0,
194
+ -3.0,
195
+ -4.0,
196
+ -6.0,
197
+ ],
198
+ dtype=torch.float32,
199
+ device=data.device,
200
+ )
201
+ packed = data.reshape(rows, logical_dim // 2)
202
+ lo = packed & 0x0F
203
+ hi = packed >> 4
204
+ values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
205
+ values[:, 0::2] = fp4_lut[lo.long()]
206
+ values[:, 1::2] = fp4_lut[hi.long()]
207
+
208
+ row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
209
+ col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
210
+ offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
211
+ scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
212
+ scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
213
+ scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
214
+ out = values * scale
215
+ if include_global_scale:
216
+ out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
217
+ return out.reshape(qx.original_shape).to(torch.bfloat16)
218
+
219
+
220
+ __all__ = [
221
+ "Nvfp4QuantizedTensor",
222
+ "quantize_bf16_to_nvfp4_128x4",
223
+ "quantize_kv_bf16_to_nvfp4_128x4",
224
+ "dequantize_nvfp4_128x4_to_bf16",
225
+ "swizzle_nvfp4_scale_to_128x4",
226
+ "nvfp4_global_scale_from_amax",
227
+ ]
build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Sparse-index helpers compatible with MiniMaxAI/msa public names."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+
11
+ def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
12
+ if cu_seqlens.dtype != torch.int32:
13
+ raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
14
+ if cu_seqlens.ndim != 1:
15
+ raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
16
+ if cu_seqlens.numel() < 1:
17
+ raise ValueError(f"{name} must have at least one element")
18
+ if not cu_seqlens.is_contiguous():
19
+ raise ValueError(f"{name} must be contiguous")
20
+
21
+
22
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
23
+ seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
24
+ return (seqlens_k + kv_block_size - 1) // kv_block_size
25
+
26
+
27
+ def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
28
+ rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
29
+ batch = len(rows_per_batch_cpu)
30
+ max_rows = max(rows_per_batch_cpu, default=0)
31
+ row_dtype = (
32
+ torch.int32
33
+ if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
34
+ else torch.int64
35
+ )
36
+ row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
37
+ row_linear = 0
38
+ for kv_block_idx in range(max_rows):
39
+ for batch_idx, row_count in enumerate(rows_per_batch_cpu):
40
+ if kv_block_idx < row_count:
41
+ row_map_cpu[batch_idx, kv_block_idx] = row_linear
42
+ row_linear += 1
43
+ return row_map_cpu.to(rows_per_batch.device), row_linear
44
+
45
+
46
+ def build_k2q_csr_torch_reference(
47
+ q2k_indices: torch.Tensor,
48
+ cu_seqlens_q: torch.Tensor,
49
+ cu_seqlens_k: torch.Tensor,
50
+ kv_block_size: int,
51
+ ) -> tuple[torch.Tensor, torch.Tensor]:
52
+ if kv_block_size <= 0:
53
+ raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
54
+ if q2k_indices.dtype != torch.int32:
55
+ raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
56
+ if q2k_indices.ndim != 3:
57
+ raise ValueError(
58
+ "q2k_indices must have shape [head_kv, total_q, topK], "
59
+ f"got {tuple(q2k_indices.shape)}"
60
+ )
61
+ _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
62
+ _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
63
+ if cu_seqlens_q.shape != cu_seqlens_k.shape:
64
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
65
+ if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
66
+ raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
67
+
68
+ head_kv, total_q, topk = q2k_indices.shape
69
+ if total_q != int(cu_seqlens_q[-1].item()):
70
+ raise ValueError(
71
+ f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
72
+ f"({int(cu_seqlens_q[-1].item())})"
73
+ )
74
+
75
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
76
+ row_map, total_rows = _build_packed_row_map(rows_per_batch)
77
+ nnz_upper_bound = total_q * topk
78
+
79
+ k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
80
+ k2q_q_indices = torch.full(
81
+ (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
82
+ )
83
+ if total_rows == 0 or total_q == 0 or topk == 0:
84
+ return k2q_row_ptr, k2q_q_indices
85
+
86
+ counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
87
+ total_entries = total_q * topk
88
+ row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
89
+ row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
90
+ q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
91
+ valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
92
+
93
+ for batch_idx in range(cu_seqlens_q.numel() - 1):
94
+ q_start = int(cu_seqlens_q[batch_idx].item())
95
+ q_end = int(cu_seqlens_q[batch_idx + 1].item())
96
+ if q_end <= q_start:
97
+ continue
98
+ q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
99
+ q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
100
+ kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
101
+ valid = kv_local >= 0
102
+ safe_kv = kv_local.clamp_min(0).long()
103
+ rows = row_map[batch_idx, safe_kv].to(row_dtype)
104
+ start = q_start * topk
105
+ end = q_end * topk
106
+ row_all[:, start:end] = rows
107
+ q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
108
+ valid_all[:, start:end] = valid & (rows >= 0)
109
+ counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
110
+
111
+ k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
112
+ offsets = torch.zeros_like(counts)
113
+ offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
114
+ cursor = torch.zeros_like(counts)
115
+ for h in range(head_kv):
116
+ for entry in range(total_entries):
117
+ if not bool(valid_all[h, entry]):
118
+ continue
119
+ row = int(row_all[h, entry].item())
120
+ pos = int(offsets[h, row].item() + cursor[h, row].item())
121
+ k2q_q_indices[h, pos] = q_all[h, entry]
122
+ cursor[h, row] += 1
123
+
124
+ return k2q_row_ptr, k2q_q_indices
125
+
126
+
127
+ def build_k2q_csr(
128
+ q2k_indices: torch.Tensor,
129
+ cu_seqlens_q: torch.Tensor,
130
+ cu_seqlens_k: torch.Tensor,
131
+ kv_block_size: int,
132
+ *,
133
+ total_k: Optional[int] = None,
134
+ max_seqlen_k: Optional[int] = None,
135
+ max_seqlen_q: Optional[int] = None,
136
+ total_rows: Optional[int] = None,
137
+ qhead_per_kv: int = 1,
138
+ return_schedule: bool = False,
139
+ ):
140
+ if return_schedule:
141
+ raise NotImplementedError(
142
+ "Blackwell build_k2q_csr currently returns CSR tensors only; "
143
+ "SM100 schedule objects are not part of this package."
144
+ )
145
+ if total_k is None:
146
+ raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
147
+ return build_k2q_csr_torch_reference(
148
+ q2k_indices.contiguous(),
149
+ cu_seqlens_q.contiguous(),
150
+ cu_seqlens_k.contiguous(),
151
+ int(kv_block_size),
152
+ )
153
+
154
+
155
+ class SparseK2qCsrBuilderSm100:
156
+ """Compatibility placeholder for the upstream SM100 class name.
157
+
158
+ The name is exported so code can feature-detect it, but schedule-producing
159
+ SM100 behavior is intentionally not claimed for this Blackwell package.
160
+ """
161
+
162
+ def __init__(self, *args, **kwargs):
163
+ self.args = args
164
+ self.kwargs = kwargs
165
+
166
+ def build(self, *args, **kwargs):
167
+ return build_k2q_csr(*args, **kwargs)
168
+
169
+
170
+ __all__ = [
171
+ "build_k2q_csr",
172
+ "build_k2q_csr_torch_reference",
173
+ "SparseK2qCsrBuilderSm100",
174
+ ]
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
40
  from ._native import has_native_ops, native_topk_from_scores
41
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
42
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
 
44
  __all__ = [
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  "flash_decode_with_topk_idx",
46
  "flash_decode_with_gqa_share_sparse",
47
  "has_native_ops",
@@ -50,4 +88,11 @@ __all__ = [
50
  "naive_flash_decode_with_topk_idx",
51
  "get_cu_seqblocks",
52
  "robust_allocator",
 
 
 
 
 
 
 
53
  ]
 
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
40
+ from .interface import (
41
+ SparseDecodePagedAttentionWrapper,
42
+ fp4_indexer_block_scores,
43
+ sparse_atten_func,
44
+ sparse_atten_nvfp4_kv_func,
45
+ sparse_decode_atten_func,
46
+ )
47
  from ._native import has_native_ops, native_topk_from_scores
48
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
49
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
50
+ from .quantize import (
51
+ Nvfp4QuantizedTensor,
52
+ dequantize_nvfp4_128x4_to_bf16,
53
+ nvfp4_global_scale_from_amax,
54
+ quantize_bf16_to_nvfp4_128x4,
55
+ quantize_kv_bf16_to_nvfp4_128x4,
56
+ swizzle_nvfp4_scale_to_128x4,
57
+ )
58
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
59
+ from .api_status import (
60
+ OFFICIAL_API_STATUS,
61
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
62
+ V1_AVAILABLE_FUNCTIONS,
63
+ available_functions,
64
+ official_api_status,
65
+ official_minimax_msa_functions,
66
+ unsupported_official_functions,
67
+ )
68
 
69
  __all__ = [
70
+ "sparse_atten_func",
71
+ "sparse_atten_nvfp4_kv_func",
72
+ "sparse_decode_atten_func",
73
+ "SparseDecodePagedAttentionWrapper",
74
+ "fp4_indexer_block_scores",
75
+ "build_k2q_csr",
76
+ "SparseK2qCsrBuilderSm100",
77
+ "Nvfp4QuantizedTensor",
78
+ "quantize_bf16_to_nvfp4_128x4",
79
+ "quantize_kv_bf16_to_nvfp4_128x4",
80
+ "dequantize_nvfp4_128x4_to_bf16",
81
+ "swizzle_nvfp4_scale_to_128x4",
82
+ "nvfp4_global_scale_from_amax",
83
  "flash_decode_with_topk_idx",
84
  "flash_decode_with_gqa_share_sparse",
85
  "has_native_ops",
 
88
  "naive_flash_decode_with_topk_idx",
89
  "get_cu_seqblocks",
90
  "robust_allocator",
91
+ "OFFICIAL_API_STATUS",
92
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
93
+ "V1_AVAILABLE_FUNCTIONS",
94
+ "available_functions",
95
+ "official_api_status",
96
+ "official_minimax_msa_functions",
97
+ "unsupported_official_functions",
98
  ]
build/torch211-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d6fed1fc5beb513468758713a57a7d6cf352ad4b6ce455d1b422d4cd66c51b05
3
  size 145216
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98433d5e5a693cb4c91a4a16c73ec96cc8852ca1c49111f7399a28c7a292b589
3
  size 145216
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _minimaxai_msa_blackwell_cuda_6347cca
3
- ops = torch.ops._minimaxai_msa_blackwell_cuda_6347cca
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_minimaxai_msa_blackwell_cuda_6347cca::{op_name}"
 
1
  import torch
2
+ from . import _minimaxai_msa_blackwell_cuda_3dca94b
3
+ ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/api_status.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Public API status for the FlashRT MiniMax MSA Blackwell package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from types import MappingProxyType
7
+
8
+
9
+ V1_AVAILABLE_FUNCTIONS = (
10
+ "sparse_atten_func",
11
+ "sparse_atten_nvfp4_kv_func",
12
+ "sparse_decode_atten_func",
13
+ "SparseDecodePagedAttentionWrapper",
14
+ "fp4_indexer_block_scores",
15
+ "build_k2q_csr",
16
+ "SparseK2qCsrBuilderSm100",
17
+ "Nvfp4QuantizedTensor",
18
+ "quantize_bf16_to_nvfp4_128x4",
19
+ "quantize_kv_bf16_to_nvfp4_128x4",
20
+ "dequantize_nvfp4_128x4_to_bf16",
21
+ "swizzle_nvfp4_scale_to_128x4",
22
+ "nvfp4_global_scale_from_amax",
23
+ "has_native_ops",
24
+ "native_topk_from_scores",
25
+ "flash_decode_with_topk_idx",
26
+ "flash_decode_with_gqa_share_sparse",
27
+ "naive_flash_decode_with_topk_idx",
28
+ "naive_flash_decode_with_gqa_share_sparse",
29
+ "get_cu_seqblocks",
30
+ "robust_allocator",
31
+ )
32
+
33
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
34
+ "sparse_atten_func",
35
+ "sparse_atten_nvfp4_kv_func",
36
+ "sparse_decode_atten_func",
37
+ "SparseDecodePagedAttentionWrapper",
38
+ "fp4_indexer_block_scores",
39
+ "build_k2q_csr",
40
+ "SparseK2qCsrBuilderSm100",
41
+ "Nvfp4QuantizedTensor",
42
+ "quantize_bf16_to_nvfp4_128x4",
43
+ "quantize_kv_bf16_to_nvfp4_128x4",
44
+ "dequantize_nvfp4_128x4_to_bf16",
45
+ "swizzle_nvfp4_scale_to_128x4",
46
+ "nvfp4_global_scale_from_amax",
47
+ )
48
+
49
+ _OFFICIAL_API_STATUS = {
50
+ "sparse_atten_func": {
51
+ "status": "unsupported_callable",
52
+ "target": "blackwell-prefill",
53
+ "reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
54
+ },
55
+ "sparse_atten_nvfp4_kv_func": {
56
+ "status": "unsupported_callable",
57
+ "target": "blackwell-prefill-nvfp4",
58
+ "reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
59
+ },
60
+ "sparse_decode_atten_func": {
61
+ "status": "available",
62
+ "target": "v1",
63
+ "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
64
+ },
65
+ "SparseDecodePagedAttentionWrapper": {
66
+ "status": "available",
67
+ "target": "v1",
68
+ "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
69
+ },
70
+ "fp4_indexer_block_scores": {
71
+ "status": "unsupported_callable",
72
+ "target": "blackwell-fp4-indexer",
73
+ "reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
74
+ },
75
+ "build_k2q_csr": {
76
+ "status": "available",
77
+ "target": "v1",
78
+ "reason": "Torch CSR construction fallback is available for compatibility.",
79
+ },
80
+ "SparseK2qCsrBuilderSm100": {
81
+ "status": "available",
82
+ "target": "v1",
83
+ "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
84
+ },
85
+ "Nvfp4QuantizedTensor": {
86
+ "status": "available",
87
+ "target": "v1",
88
+ "reason": "NVFP4 metadata dataclass is available.",
89
+ },
90
+ "quantize_bf16_to_nvfp4_128x4": {
91
+ "status": "available_optional_te",
92
+ "target": "v1",
93
+ "reason": "Function is available and uses Transformer Engine when installed.",
94
+ },
95
+ "quantize_kv_bf16_to_nvfp4_128x4": {
96
+ "status": "available_optional_te",
97
+ "target": "v1",
98
+ "reason": "Function is available and uses Transformer Engine when installed.",
99
+ },
100
+ "dequantize_nvfp4_128x4_to_bf16": {
101
+ "status": "available",
102
+ "target": "v1",
103
+ "reason": "Reference dequant helper is available.",
104
+ },
105
+ "swizzle_nvfp4_scale_to_128x4": {
106
+ "status": "available",
107
+ "target": "v1",
108
+ "reason": "Scale swizzle helper is available.",
109
+ },
110
+ "nvfp4_global_scale_from_amax": {
111
+ "status": "available",
112
+ "target": "v1",
113
+ "reason": "Global scale helper is available.",
114
+ },
115
+ }
116
+
117
+ OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
118
+
119
+
120
+ def available_functions() -> tuple[str, ...]:
121
+ """Return the functions/classes intentionally available in v1."""
122
+
123
+ return V1_AVAILABLE_FUNCTIONS
124
+
125
+
126
+ def official_minimax_msa_functions() -> tuple[str, ...]:
127
+ """Return the upstream MiniMaxAI/msa public function/class names tracked here."""
128
+
129
+ return OFFICIAL_MINIMAX_MSA_FUNCTIONS
130
+
131
+
132
+ def official_api_status() -> dict[str, dict[str, str]]:
133
+ """Return a copy of the official API compatibility status table."""
134
+
135
+ return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
136
+
137
+
138
+ def unsupported_official_functions() -> tuple[str, ...]:
139
+ """Return official MiniMaxAI/msa names that are not exported by this v1 package."""
140
+
141
+ return tuple(
142
+ name
143
+ for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
144
+ if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
145
+ )
build/torch211-cxx11-cu130-x86_64-linux/interface.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
11
+
12
+
13
+ def _unsupported(name: str, reason: str):
14
+ raise NotImplementedError(
15
+ f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
16
+ )
17
+
18
+
19
+ def _page_table_to_req_to_token(
20
+ page_table: torch.Tensor,
21
+ seqused_k: torch.Tensor,
22
+ max_seqlen_k: int,
23
+ blk_kv: int,
24
+ ) -> torch.Tensor:
25
+ batch = int(page_table.shape[0])
26
+ max_seqlen_k = int(max_seqlen_k)
27
+ blk_kv = int(blk_kv)
28
+ logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
29
+ page = logical_pos // blk_kv
30
+ offset = logical_pos % blk_kv
31
+ page = page.clamp(max=int(page_table.shape[1]) - 1)
32
+ physical_page = page_table[:, page].to(torch.int64)
33
+ physical_token = physical_page * blk_kv + offset[None, :]
34
+ valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
35
+ return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
36
+
37
+
38
+ def _dense_q2k_indices(
39
+ *,
40
+ head_kv: int,
41
+ batch: int,
42
+ topk: int,
43
+ seq_lens: torch.Tensor,
44
+ blk_kv: int,
45
+ ) -> torch.Tensor:
46
+ idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
47
+ blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
48
+ valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
49
+ for b in range(batch):
50
+ valid = blocks < valid_blocks[b]
51
+ idx[:, b, valid] = blocks[valid]
52
+ return idx
53
+
54
+
55
+ def _normalize_decode_q2k(
56
+ q2k_indices: Optional[torch.Tensor],
57
+ *,
58
+ batch: int,
59
+ seqlen_q: int,
60
+ head_kv: int,
61
+ seqused_k: torch.Tensor,
62
+ blk_kv: int,
63
+ ) -> torch.Tensor:
64
+ if q2k_indices is None:
65
+ topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
66
+ return _dense_q2k_indices(
67
+ head_kv=head_kv,
68
+ batch=batch,
69
+ topk=topk,
70
+ seq_lens=seqused_k,
71
+ blk_kv=blk_kv,
72
+ )
73
+ if q2k_indices.dtype != torch.int32:
74
+ raise TypeError("q2k_indices must be torch.int32")
75
+ if q2k_indices.ndim != 3:
76
+ raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
77
+ if q2k_indices.shape[0] != head_kv:
78
+ raise ValueError("q2k_indices first dimension must match Hkv")
79
+ if int(q2k_indices.shape[1]) != batch * seqlen_q:
80
+ raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
81
+ if seqlen_q != 1:
82
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
83
+ return q2k_indices[:, ::seqlen_q, :].contiguous()
84
+
85
+
86
+ def sparse_decode_atten_func(
87
+ q: torch.Tensor,
88
+ k: torch.Tensor,
89
+ v: torch.Tensor,
90
+ q2k_indices: Optional[torch.Tensor] = None,
91
+ *,
92
+ page_table: torch.Tensor,
93
+ seqused_k: torch.Tensor,
94
+ seqlen_q: int,
95
+ max_seqlen_k: int,
96
+ blk_kv: int = 128,
97
+ causal: bool = True,
98
+ softmax_scale: Optional[float] = None,
99
+ return_softmax_lse: bool = False,
100
+ schedule: Optional[object] = None,
101
+ O_partial: Optional[torch.Tensor] = None,
102
+ LSE_partial: Optional[torch.Tensor] = None,
103
+ ):
104
+ """Run MiniMax paged sparse decode on Blackwell.
105
+
106
+ This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
107
+ but accepts the Blackwell package's validated BF16/FP16 decode path. It
108
+ supports single-token decode (`seqlen_q=1`) with paged K/V
109
+ `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
110
+ """
111
+
112
+ if schedule is not None or O_partial is not None or LSE_partial is not None:
113
+ raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
114
+ if return_softmax_lse:
115
+ raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
116
+ if not causal:
117
+ raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
118
+ if int(seqlen_q) != 1:
119
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
120
+ if q.ndim != 3:
121
+ raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
122
+ if k.ndim != 4 or v.ndim != 4:
123
+ raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
124
+ if k.shape != v.shape:
125
+ raise ValueError("k and v must have the same shape")
126
+ if q.device != k.device or q.device != v.device or q.device != page_table.device:
127
+ raise ValueError("q, k, v, and page_table must be on the same device")
128
+ if q.dtype not in (torch.bfloat16, torch.float16):
129
+ raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
130
+ if k.dtype != q.dtype or v.dtype != q.dtype:
131
+ raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
132
+ if q.shape[-1] != 128 or k.shape[-1] != 128:
133
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
134
+ if int(k.shape[2]) != int(blk_kv):
135
+ raise ValueError("k.shape[2] must equal blk_kv")
136
+ if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
137
+ raise TypeError("page_table and seqused_k must be torch.int32")
138
+ if page_table.ndim != 2:
139
+ raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
140
+ batch = int(page_table.shape[0])
141
+ if int(q.shape[0]) != batch:
142
+ raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
143
+
144
+ head_kv = int(k.shape[1])
145
+ q2k = _normalize_decode_q2k(
146
+ q2k_indices,
147
+ batch=batch,
148
+ seqlen_q=int(seqlen_q),
149
+ head_kv=head_kv,
150
+ seqused_k=seqused_k,
151
+ blk_kv=int(blk_kv),
152
+ )
153
+ req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
154
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
155
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
156
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
157
+ return flash_decode_with_gqa_share_sparse(
158
+ q.contiguous(),
159
+ None,
160
+ k_cache,
161
+ v_cache,
162
+ req_to_token,
163
+ seqused_k.contiguous(),
164
+ slot_ids,
165
+ int(blk_kv),
166
+ q2k,
167
+ sm_scale=softmax_scale,
168
+ )
169
+
170
+
171
+ class SparseDecodePagedAttentionWrapper:
172
+ def __init__(self, *, blk_kv: int = 128, causal: bool = True):
173
+ self.blk_kv = int(blk_kv)
174
+ self.causal = bool(causal)
175
+ self.page_table: Optional[torch.Tensor] = None
176
+ self.seqused_k: Optional[torch.Tensor] = None
177
+ self.seqlen_q: Optional[int] = None
178
+ self.max_seqlen_k: Optional[int] = None
179
+ self.q2k_indices: Optional[torch.Tensor] = None
180
+
181
+ def plan(
182
+ self,
183
+ *,
184
+ page_table: torch.Tensor,
185
+ seqused_k: torch.Tensor,
186
+ seqlen_q: int,
187
+ max_seqlen_k: int,
188
+ q2k_indices: Optional[torch.Tensor] = None,
189
+ num_qo_heads: Optional[int] = None,
190
+ num_kv_heads: Optional[int] = None,
191
+ head_dim: Optional[int] = 128,
192
+ enable_cuda_graph: bool = False,
193
+ max_grid_size: Optional[int] = None,
194
+ fixed_split_size: Optional[int] = None,
195
+ disable_split_kv: bool = False,
196
+ ) -> "SparseDecodePagedAttentionWrapper":
197
+ if head_dim is not None and int(head_dim) != 128:
198
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
199
+ self.page_table = page_table.contiguous()
200
+ self.seqused_k = seqused_k.contiguous()
201
+ self.seqlen_q = int(seqlen_q)
202
+ self.max_seqlen_k = int(max_seqlen_k)
203
+ self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
204
+ return self
205
+
206
+ def run(
207
+ self,
208
+ q: torch.Tensor,
209
+ k: torch.Tensor,
210
+ v: torch.Tensor,
211
+ *,
212
+ softmax_scale: Optional[float] = None,
213
+ return_softmax_lse: bool = False,
214
+ out: Optional[torch.Tensor] = None,
215
+ lse: Optional[torch.Tensor] = None,
216
+ ):
217
+ if self.page_table is None or self.seqused_k is None:
218
+ raise RuntimeError("decode wrapper must be planned before run")
219
+ result = sparse_decode_atten_func(
220
+ q,
221
+ k,
222
+ v,
223
+ self.q2k_indices,
224
+ page_table=self.page_table,
225
+ seqused_k=self.seqused_k,
226
+ seqlen_q=self.seqlen_q,
227
+ max_seqlen_k=self.max_seqlen_k,
228
+ blk_kv=self.blk_kv,
229
+ causal=self.causal,
230
+ softmax_scale=softmax_scale,
231
+ return_softmax_lse=return_softmax_lse,
232
+ )
233
+ if out is not None:
234
+ out.copy_(result)
235
+ return out
236
+ return result
237
+
238
+
239
+ def sparse_atten_func(*args, **kwargs):
240
+ _unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
241
+
242
+
243
+ def sparse_atten_nvfp4_kv_func(*args, **kwargs):
244
+ _unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
245
+
246
+
247
+ def fp4_indexer_block_scores(*args, **kwargs):
248
+ _unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
249
+
250
+
251
+ __all__ = [
252
+ "sparse_atten_func",
253
+ "sparse_atten_nvfp4_kv_func",
254
+ "sparse_decode_atten_func",
255
+ "SparseDecodePagedAttentionWrapper",
256
+ "fp4_indexer_block_scores",
257
+ ]
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
- "id": "_minimaxai_msa_blackwell_cuda_6347cca",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,20 +14,24 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "g6O7Aw9I1Db01ScTRhjP7xcQsx1f33SAdQjOpMRMgX0=",
18
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
- "_minimaxai_msa_blackwell_cuda_6347cca.abi3.so": "1v7R/FvrUTRodYcTpXp9bPNSrUts5FXRtCLUzWbFGwU=",
20
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
21
- "_ops.py": "JaMCArFhNkHXPIKm1OiPosUE0JMLdsdDAtMk1gGh2FQ=",
 
22
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
23
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
24
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
25
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
26
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
 
27
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
28
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
29
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
30
- "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
 
 
31
  }
32
  }
33
  }
 
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
+ "id": "_minimaxai_msa_blackwell_cuda_3dca94b",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
18
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
+ "_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "mEM9XlppPLTJGkoWxz7JbMiFLKHEkRH3OZoox6KStYk=",
20
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
21
+ "_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
22
+ "api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
23
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
24
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
25
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
26
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
27
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
28
+ "interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
29
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
30
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
31
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
32
+ "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
33
+ "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
34
+ "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
35
  }
36
  }
37
  }
build/torch211-cxx11-cu130-x86_64-linux/quantize.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """NVFP4 helper API compatible with MiniMaxAI/msa.
3
+
4
+ The quantization entrypoints use Transformer Engine when it is installed. The
5
+ layout helpers and reference dequantizer are pure PyTorch and are included so
6
+ callers can prepare and validate NVFP4 metadata with the same public API names
7
+ as the upstream MiniMaxAI package.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Tuple
14
+
15
+ import torch
16
+
17
+
18
+ NVFP4_BLOCK_SIZE = 16
19
+ NVFP4_FP4_MAX = 6.0
20
+ NVFP4_FP8_E4M3_MAX = 448.0
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Nvfp4QuantizedTensor:
25
+ data: torch.Tensor
26
+ scale_128x4: torch.Tensor
27
+ global_scale: torch.Tensor
28
+ logical_scale_shape: Tuple[int, int]
29
+ original_shape: Tuple[int, ...]
30
+
31
+
32
+ def _round_up(x: int, multiple: int) -> int:
33
+ return ((int(x) + multiple - 1) // multiple) * multiple
34
+
35
+
36
+ def nvfp4_scale_128x4_offset(
37
+ row: torch.Tensor,
38
+ col: torch.Tensor,
39
+ scale_cols: int,
40
+ ) -> torch.Tensor:
41
+ tiles_n = _round_up(scale_cols, 4) // 4
42
+ tile_m = row // 128
43
+ tile_n = col // 4
44
+ outer = row % 128
45
+ inner = col % 4
46
+ return (
47
+ (tile_m * tiles_n + tile_n) * 512
48
+ + (outer % 32) * 16
49
+ + (outer // 32) * 4
50
+ + inner
51
+ )
52
+
53
+
54
+ def swizzle_nvfp4_scale_to_128x4(
55
+ scale: torch.Tensor,
56
+ *,
57
+ rows: int,
58
+ cols: int,
59
+ ) -> torch.Tensor:
60
+ if scale.ndim != 2:
61
+ raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
62
+
63
+ rows = int(rows)
64
+ cols = int(cols)
65
+ padded_rows = _round_up(rows, 128)
66
+ padded_cols = _round_up(cols, 4)
67
+ if scale.shape[0] < rows or scale.shape[1] < cols:
68
+ raise ValueError(
69
+ "scale is smaller than the requested logical shape: "
70
+ f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
71
+ )
72
+
73
+ logical = scale[:rows, :cols].contiguous()
74
+ if logical.shape != (padded_rows, padded_cols):
75
+ logical = torch.nn.functional.pad(
76
+ logical.to(torch.float32),
77
+ (0, padded_cols - cols, 0, padded_rows - rows),
78
+ ).to(scale.dtype)
79
+ swizzled = torch.empty_like(logical)
80
+
81
+ row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
82
+ col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
83
+ offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
84
+ swizzled.reshape(-1)[offset] = logical.reshape(-1)
85
+ return swizzled
86
+
87
+
88
+ def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
89
+ return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
90
+
91
+
92
+ def _import_te_nvfp4_quantizer():
93
+ try:
94
+ from transformer_engine.pytorch.tensor import NVFP4Quantizer
95
+ except Exception as exc:
96
+ raise RuntimeError(
97
+ "Transformer Engine NVFP4 quantization is unavailable. Install a "
98
+ "Transformer Engine build with NVFP4 PyTorch support to use "
99
+ "quantize_bf16_to_nvfp4_128x4."
100
+ ) from exc
101
+ return NVFP4Quantizer
102
+
103
+
104
+ def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
105
+ if not x.is_cuda:
106
+ raise ValueError("NVFP4 quantization requires a CUDA tensor")
107
+ if x.dtype not in (torch.bfloat16, torch.float16):
108
+ raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
109
+ if x.ndim < 2:
110
+ raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
111
+ if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
112
+ raise ValueError(
113
+ f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
114
+ )
115
+
116
+ rows = 1
117
+ for dim in x.shape[:-1]:
118
+ rows *= int(dim)
119
+ if rows % NVFP4_BLOCK_SIZE != 0:
120
+ raise ValueError(
121
+ "flattened row dimension must be divisible by "
122
+ f"{NVFP4_BLOCK_SIZE}, got {rows}"
123
+ )
124
+
125
+ NVFP4Quantizer = _import_te_nvfp4_quantizer()
126
+ quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
127
+ qx = quantizer.quantize(x.contiguous())
128
+ meta = qx.get_metadata()
129
+
130
+ data = meta["rowwise_data"]
131
+ if data.dtype is not torch.uint8:
132
+ data = data.view(torch.uint8)
133
+ logical_scale = meta["rowwise_scale_inv"]
134
+ amax = meta["amax_rowwise"]
135
+ scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
136
+ scale_128x4 = swizzle_nvfp4_scale_to_128x4(
137
+ logical_scale,
138
+ rows=rows,
139
+ cols=scale_cols,
140
+ )
141
+ global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
142
+
143
+ return Nvfp4QuantizedTensor(
144
+ data=data,
145
+ scale_128x4=scale_128x4,
146
+ global_scale=global_scale,
147
+ logical_scale_shape=(rows, scale_cols),
148
+ original_shape=tuple(int(v) for v in x.shape),
149
+ )
150
+
151
+
152
+ def quantize_kv_bf16_to_nvfp4_128x4(
153
+ k: torch.Tensor,
154
+ v: torch.Tensor,
155
+ ) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
156
+ return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
157
+
158
+
159
+ def dequantize_nvfp4_128x4_to_bf16(
160
+ qx: Nvfp4QuantizedTensor,
161
+ *,
162
+ include_global_scale: bool = True,
163
+ ) -> torch.Tensor:
164
+ data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
165
+ if data.shape[-1] * 2 != qx.original_shape[-1]:
166
+ raise ValueError(
167
+ "packed data last dimension does not match original shape: "
168
+ f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
169
+ )
170
+
171
+ rows, scale_cols = qx.logical_scale_shape
172
+ logical_dim = int(qx.original_shape[-1])
173
+ if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
174
+ raise ValueError(
175
+ "logical scale columns do not match original last dimension: "
176
+ f"{scale_cols} scale cols vs dim {logical_dim}"
177
+ )
178
+
179
+ fp4_lut = torch.tensor(
180
+ [
181
+ 0.0,
182
+ 0.5,
183
+ 1.0,
184
+ 1.5,
185
+ 2.0,
186
+ 3.0,
187
+ 4.0,
188
+ 6.0,
189
+ -0.0,
190
+ -0.5,
191
+ -1.0,
192
+ -1.5,
193
+ -2.0,
194
+ -3.0,
195
+ -4.0,
196
+ -6.0,
197
+ ],
198
+ dtype=torch.float32,
199
+ device=data.device,
200
+ )
201
+ packed = data.reshape(rows, logical_dim // 2)
202
+ lo = packed & 0x0F
203
+ hi = packed >> 4
204
+ values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
205
+ values[:, 0::2] = fp4_lut[lo.long()]
206
+ values[:, 1::2] = fp4_lut[hi.long()]
207
+
208
+ row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
209
+ col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
210
+ offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
211
+ scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
212
+ scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
213
+ scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
214
+ out = values * scale
215
+ if include_global_scale:
216
+ out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
217
+ return out.reshape(qx.original_shape).to(torch.bfloat16)
218
+
219
+
220
+ __all__ = [
221
+ "Nvfp4QuantizedTensor",
222
+ "quantize_bf16_to_nvfp4_128x4",
223
+ "quantize_kv_bf16_to_nvfp4_128x4",
224
+ "dequantize_nvfp4_128x4_to_bf16",
225
+ "swizzle_nvfp4_scale_to_128x4",
226
+ "nvfp4_global_scale_from_amax",
227
+ ]
build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Sparse-index helpers compatible with MiniMaxAI/msa public names."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+
11
+ def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
12
+ if cu_seqlens.dtype != torch.int32:
13
+ raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
14
+ if cu_seqlens.ndim != 1:
15
+ raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
16
+ if cu_seqlens.numel() < 1:
17
+ raise ValueError(f"{name} must have at least one element")
18
+ if not cu_seqlens.is_contiguous():
19
+ raise ValueError(f"{name} must be contiguous")
20
+
21
+
22
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
23
+ seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
24
+ return (seqlens_k + kv_block_size - 1) // kv_block_size
25
+
26
+
27
+ def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
28
+ rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
29
+ batch = len(rows_per_batch_cpu)
30
+ max_rows = max(rows_per_batch_cpu, default=0)
31
+ row_dtype = (
32
+ torch.int32
33
+ if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
34
+ else torch.int64
35
+ )
36
+ row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
37
+ row_linear = 0
38
+ for kv_block_idx in range(max_rows):
39
+ for batch_idx, row_count in enumerate(rows_per_batch_cpu):
40
+ if kv_block_idx < row_count:
41
+ row_map_cpu[batch_idx, kv_block_idx] = row_linear
42
+ row_linear += 1
43
+ return row_map_cpu.to(rows_per_batch.device), row_linear
44
+
45
+
46
+ def build_k2q_csr_torch_reference(
47
+ q2k_indices: torch.Tensor,
48
+ cu_seqlens_q: torch.Tensor,
49
+ cu_seqlens_k: torch.Tensor,
50
+ kv_block_size: int,
51
+ ) -> tuple[torch.Tensor, torch.Tensor]:
52
+ if kv_block_size <= 0:
53
+ raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
54
+ if q2k_indices.dtype != torch.int32:
55
+ raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
56
+ if q2k_indices.ndim != 3:
57
+ raise ValueError(
58
+ "q2k_indices must have shape [head_kv, total_q, topK], "
59
+ f"got {tuple(q2k_indices.shape)}"
60
+ )
61
+ _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
62
+ _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
63
+ if cu_seqlens_q.shape != cu_seqlens_k.shape:
64
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
65
+ if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
66
+ raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
67
+
68
+ head_kv, total_q, topk = q2k_indices.shape
69
+ if total_q != int(cu_seqlens_q[-1].item()):
70
+ raise ValueError(
71
+ f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
72
+ f"({int(cu_seqlens_q[-1].item())})"
73
+ )
74
+
75
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
76
+ row_map, total_rows = _build_packed_row_map(rows_per_batch)
77
+ nnz_upper_bound = total_q * topk
78
+
79
+ k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
80
+ k2q_q_indices = torch.full(
81
+ (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
82
+ )
83
+ if total_rows == 0 or total_q == 0 or topk == 0:
84
+ return k2q_row_ptr, k2q_q_indices
85
+
86
+ counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
87
+ total_entries = total_q * topk
88
+ row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
89
+ row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
90
+ q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
91
+ valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
92
+
93
+ for batch_idx in range(cu_seqlens_q.numel() - 1):
94
+ q_start = int(cu_seqlens_q[batch_idx].item())
95
+ q_end = int(cu_seqlens_q[batch_idx + 1].item())
96
+ if q_end <= q_start:
97
+ continue
98
+ q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
99
+ q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
100
+ kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
101
+ valid = kv_local >= 0
102
+ safe_kv = kv_local.clamp_min(0).long()
103
+ rows = row_map[batch_idx, safe_kv].to(row_dtype)
104
+ start = q_start * topk
105
+ end = q_end * topk
106
+ row_all[:, start:end] = rows
107
+ q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
108
+ valid_all[:, start:end] = valid & (rows >= 0)
109
+ counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
110
+
111
+ k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
112
+ offsets = torch.zeros_like(counts)
113
+ offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
114
+ cursor = torch.zeros_like(counts)
115
+ for h in range(head_kv):
116
+ for entry in range(total_entries):
117
+ if not bool(valid_all[h, entry]):
118
+ continue
119
+ row = int(row_all[h, entry].item())
120
+ pos = int(offsets[h, row].item() + cursor[h, row].item())
121
+ k2q_q_indices[h, pos] = q_all[h, entry]
122
+ cursor[h, row] += 1
123
+
124
+ return k2q_row_ptr, k2q_q_indices
125
+
126
+
127
+ def build_k2q_csr(
128
+ q2k_indices: torch.Tensor,
129
+ cu_seqlens_q: torch.Tensor,
130
+ cu_seqlens_k: torch.Tensor,
131
+ kv_block_size: int,
132
+ *,
133
+ total_k: Optional[int] = None,
134
+ max_seqlen_k: Optional[int] = None,
135
+ max_seqlen_q: Optional[int] = None,
136
+ total_rows: Optional[int] = None,
137
+ qhead_per_kv: int = 1,
138
+ return_schedule: bool = False,
139
+ ):
140
+ if return_schedule:
141
+ raise NotImplementedError(
142
+ "Blackwell build_k2q_csr currently returns CSR tensors only; "
143
+ "SM100 schedule objects are not part of this package."
144
+ )
145
+ if total_k is None:
146
+ raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
147
+ return build_k2q_csr_torch_reference(
148
+ q2k_indices.contiguous(),
149
+ cu_seqlens_q.contiguous(),
150
+ cu_seqlens_k.contiguous(),
151
+ int(kv_block_size),
152
+ )
153
+
154
+
155
+ class SparseK2qCsrBuilderSm100:
156
+ """Compatibility placeholder for the upstream SM100 class name.
157
+
158
+ The name is exported so code can feature-detect it, but schedule-producing
159
+ SM100 behavior is intentionally not claimed for this Blackwell package.
160
+ """
161
+
162
+ def __init__(self, *args, **kwargs):
163
+ self.args = args
164
+ self.kwargs = kwargs
165
+
166
+ def build(self, *args, **kwargs):
167
+ return build_k2q_csr(*args, **kwargs)
168
+
169
+
170
+ __all__ = [
171
+ "build_k2q_csr",
172
+ "build_k2q_csr_torch_reference",
173
+ "SparseK2qCsrBuilderSm100",
174
+ ]
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
40
  from ._native import has_native_ops, native_topk_from_scores
41
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
42
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
 
44
  __all__ = [
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  "flash_decode_with_topk_idx",
46
  "flash_decode_with_gqa_share_sparse",
47
  "has_native_ops",
@@ -50,4 +88,11 @@ __all__ = [
50
  "naive_flash_decode_with_topk_idx",
51
  "get_cu_seqblocks",
52
  "robust_allocator",
 
 
 
 
 
 
 
53
  ]
 
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
40
+ from .interface import (
41
+ SparseDecodePagedAttentionWrapper,
42
+ fp4_indexer_block_scores,
43
+ sparse_atten_func,
44
+ sparse_atten_nvfp4_kv_func,
45
+ sparse_decode_atten_func,
46
+ )
47
  from ._native import has_native_ops, native_topk_from_scores
48
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
49
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
50
+ from .quantize import (
51
+ Nvfp4QuantizedTensor,
52
+ dequantize_nvfp4_128x4_to_bf16,
53
+ nvfp4_global_scale_from_amax,
54
+ quantize_bf16_to_nvfp4_128x4,
55
+ quantize_kv_bf16_to_nvfp4_128x4,
56
+ swizzle_nvfp4_scale_to_128x4,
57
+ )
58
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
59
+ from .api_status import (
60
+ OFFICIAL_API_STATUS,
61
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
62
+ V1_AVAILABLE_FUNCTIONS,
63
+ available_functions,
64
+ official_api_status,
65
+ official_minimax_msa_functions,
66
+ unsupported_official_functions,
67
+ )
68
 
69
  __all__ = [
70
+ "sparse_atten_func",
71
+ "sparse_atten_nvfp4_kv_func",
72
+ "sparse_decode_atten_func",
73
+ "SparseDecodePagedAttentionWrapper",
74
+ "fp4_indexer_block_scores",
75
+ "build_k2q_csr",
76
+ "SparseK2qCsrBuilderSm100",
77
+ "Nvfp4QuantizedTensor",
78
+ "quantize_bf16_to_nvfp4_128x4",
79
+ "quantize_kv_bf16_to_nvfp4_128x4",
80
+ "dequantize_nvfp4_128x4_to_bf16",
81
+ "swizzle_nvfp4_scale_to_128x4",
82
+ "nvfp4_global_scale_from_amax",
83
  "flash_decode_with_topk_idx",
84
  "flash_decode_with_gqa_share_sparse",
85
  "has_native_ops",
 
88
  "naive_flash_decode_with_topk_idx",
89
  "get_cu_seqblocks",
90
  "robust_allocator",
91
+ "OFFICIAL_API_STATUS",
92
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
93
+ "V1_AVAILABLE_FUNCTIONS",
94
+ "available_functions",
95
+ "official_api_status",
96
+ "official_minimax_msa_functions",
97
+ "unsupported_official_functions",
98
  ]
build/torch212-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d0c5c0b8614bd4c0fbde1a650688e21517cce92221147e170bf527582e7e47b6
3
  size 151920
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de9c51b30c5d31511676be36604ab66dee1fdf8ccac21ea8a32cc7ba321bb2e2
3
  size 151920
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _minimaxai_msa_blackwell_cuda_6347cca
3
- ops = torch.ops._minimaxai_msa_blackwell_cuda_6347cca
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_minimaxai_msa_blackwell_cuda_6347cca::{op_name}"
 
1
  import torch
2
+ from . import _minimaxai_msa_blackwell_cuda_3dca94b
3
+ ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/api_status.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Public API status for the FlashRT MiniMax MSA Blackwell package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from types import MappingProxyType
7
+
8
+
9
+ V1_AVAILABLE_FUNCTIONS = (
10
+ "sparse_atten_func",
11
+ "sparse_atten_nvfp4_kv_func",
12
+ "sparse_decode_atten_func",
13
+ "SparseDecodePagedAttentionWrapper",
14
+ "fp4_indexer_block_scores",
15
+ "build_k2q_csr",
16
+ "SparseK2qCsrBuilderSm100",
17
+ "Nvfp4QuantizedTensor",
18
+ "quantize_bf16_to_nvfp4_128x4",
19
+ "quantize_kv_bf16_to_nvfp4_128x4",
20
+ "dequantize_nvfp4_128x4_to_bf16",
21
+ "swizzle_nvfp4_scale_to_128x4",
22
+ "nvfp4_global_scale_from_amax",
23
+ "has_native_ops",
24
+ "native_topk_from_scores",
25
+ "flash_decode_with_topk_idx",
26
+ "flash_decode_with_gqa_share_sparse",
27
+ "naive_flash_decode_with_topk_idx",
28
+ "naive_flash_decode_with_gqa_share_sparse",
29
+ "get_cu_seqblocks",
30
+ "robust_allocator",
31
+ )
32
+
33
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
34
+ "sparse_atten_func",
35
+ "sparse_atten_nvfp4_kv_func",
36
+ "sparse_decode_atten_func",
37
+ "SparseDecodePagedAttentionWrapper",
38
+ "fp4_indexer_block_scores",
39
+ "build_k2q_csr",
40
+ "SparseK2qCsrBuilderSm100",
41
+ "Nvfp4QuantizedTensor",
42
+ "quantize_bf16_to_nvfp4_128x4",
43
+ "quantize_kv_bf16_to_nvfp4_128x4",
44
+ "dequantize_nvfp4_128x4_to_bf16",
45
+ "swizzle_nvfp4_scale_to_128x4",
46
+ "nvfp4_global_scale_from_amax",
47
+ )
48
+
49
+ _OFFICIAL_API_STATUS = {
50
+ "sparse_atten_func": {
51
+ "status": "unsupported_callable",
52
+ "target": "blackwell-prefill",
53
+ "reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
54
+ },
55
+ "sparse_atten_nvfp4_kv_func": {
56
+ "status": "unsupported_callable",
57
+ "target": "blackwell-prefill-nvfp4",
58
+ "reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
59
+ },
60
+ "sparse_decode_atten_func": {
61
+ "status": "available",
62
+ "target": "v1",
63
+ "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
64
+ },
65
+ "SparseDecodePagedAttentionWrapper": {
66
+ "status": "available",
67
+ "target": "v1",
68
+ "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
69
+ },
70
+ "fp4_indexer_block_scores": {
71
+ "status": "unsupported_callable",
72
+ "target": "blackwell-fp4-indexer",
73
+ "reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
74
+ },
75
+ "build_k2q_csr": {
76
+ "status": "available",
77
+ "target": "v1",
78
+ "reason": "Torch CSR construction fallback is available for compatibility.",
79
+ },
80
+ "SparseK2qCsrBuilderSm100": {
81
+ "status": "available",
82
+ "target": "v1",
83
+ "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
84
+ },
85
+ "Nvfp4QuantizedTensor": {
86
+ "status": "available",
87
+ "target": "v1",
88
+ "reason": "NVFP4 metadata dataclass is available.",
89
+ },
90
+ "quantize_bf16_to_nvfp4_128x4": {
91
+ "status": "available_optional_te",
92
+ "target": "v1",
93
+ "reason": "Function is available and uses Transformer Engine when installed.",
94
+ },
95
+ "quantize_kv_bf16_to_nvfp4_128x4": {
96
+ "status": "available_optional_te",
97
+ "target": "v1",
98
+ "reason": "Function is available and uses Transformer Engine when installed.",
99
+ },
100
+ "dequantize_nvfp4_128x4_to_bf16": {
101
+ "status": "available",
102
+ "target": "v1",
103
+ "reason": "Reference dequant helper is available.",
104
+ },
105
+ "swizzle_nvfp4_scale_to_128x4": {
106
+ "status": "available",
107
+ "target": "v1",
108
+ "reason": "Scale swizzle helper is available.",
109
+ },
110
+ "nvfp4_global_scale_from_amax": {
111
+ "status": "available",
112
+ "target": "v1",
113
+ "reason": "Global scale helper is available.",
114
+ },
115
+ }
116
+
117
+ OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
118
+
119
+
120
+ def available_functions() -> tuple[str, ...]:
121
+ """Return the functions/classes intentionally available in v1."""
122
+
123
+ return V1_AVAILABLE_FUNCTIONS
124
+
125
+
126
+ def official_minimax_msa_functions() -> tuple[str, ...]:
127
+ """Return the upstream MiniMaxAI/msa public function/class names tracked here."""
128
+
129
+ return OFFICIAL_MINIMAX_MSA_FUNCTIONS
130
+
131
+
132
+ def official_api_status() -> dict[str, dict[str, str]]:
133
+ """Return a copy of the official API compatibility status table."""
134
+
135
+ return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
136
+
137
+
138
+ def unsupported_official_functions() -> tuple[str, ...]:
139
+ """Return official MiniMaxAI/msa names that are not exported by this v1 package."""
140
+
141
+ return tuple(
142
+ name
143
+ for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
144
+ if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
145
+ )
build/torch212-cxx11-cu130-x86_64-linux/interface.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
11
+
12
+
13
+ def _unsupported(name: str, reason: str):
14
+ raise NotImplementedError(
15
+ f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
16
+ )
17
+
18
+
19
+ def _page_table_to_req_to_token(
20
+ page_table: torch.Tensor,
21
+ seqused_k: torch.Tensor,
22
+ max_seqlen_k: int,
23
+ blk_kv: int,
24
+ ) -> torch.Tensor:
25
+ batch = int(page_table.shape[0])
26
+ max_seqlen_k = int(max_seqlen_k)
27
+ blk_kv = int(blk_kv)
28
+ logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
29
+ page = logical_pos // blk_kv
30
+ offset = logical_pos % blk_kv
31
+ page = page.clamp(max=int(page_table.shape[1]) - 1)
32
+ physical_page = page_table[:, page].to(torch.int64)
33
+ physical_token = physical_page * blk_kv + offset[None, :]
34
+ valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
35
+ return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
36
+
37
+
38
+ def _dense_q2k_indices(
39
+ *,
40
+ head_kv: int,
41
+ batch: int,
42
+ topk: int,
43
+ seq_lens: torch.Tensor,
44
+ blk_kv: int,
45
+ ) -> torch.Tensor:
46
+ idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
47
+ blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
48
+ valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
49
+ for b in range(batch):
50
+ valid = blocks < valid_blocks[b]
51
+ idx[:, b, valid] = blocks[valid]
52
+ return idx
53
+
54
+
55
+ def _normalize_decode_q2k(
56
+ q2k_indices: Optional[torch.Tensor],
57
+ *,
58
+ batch: int,
59
+ seqlen_q: int,
60
+ head_kv: int,
61
+ seqused_k: torch.Tensor,
62
+ blk_kv: int,
63
+ ) -> torch.Tensor:
64
+ if q2k_indices is None:
65
+ topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
66
+ return _dense_q2k_indices(
67
+ head_kv=head_kv,
68
+ batch=batch,
69
+ topk=topk,
70
+ seq_lens=seqused_k,
71
+ blk_kv=blk_kv,
72
+ )
73
+ if q2k_indices.dtype != torch.int32:
74
+ raise TypeError("q2k_indices must be torch.int32")
75
+ if q2k_indices.ndim != 3:
76
+ raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
77
+ if q2k_indices.shape[0] != head_kv:
78
+ raise ValueError("q2k_indices first dimension must match Hkv")
79
+ if int(q2k_indices.shape[1]) != batch * seqlen_q:
80
+ raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
81
+ if seqlen_q != 1:
82
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
83
+ return q2k_indices[:, ::seqlen_q, :].contiguous()
84
+
85
+
86
+ def sparse_decode_atten_func(
87
+ q: torch.Tensor,
88
+ k: torch.Tensor,
89
+ v: torch.Tensor,
90
+ q2k_indices: Optional[torch.Tensor] = None,
91
+ *,
92
+ page_table: torch.Tensor,
93
+ seqused_k: torch.Tensor,
94
+ seqlen_q: int,
95
+ max_seqlen_k: int,
96
+ blk_kv: int = 128,
97
+ causal: bool = True,
98
+ softmax_scale: Optional[float] = None,
99
+ return_softmax_lse: bool = False,
100
+ schedule: Optional[object] = None,
101
+ O_partial: Optional[torch.Tensor] = None,
102
+ LSE_partial: Optional[torch.Tensor] = None,
103
+ ):
104
+ """Run MiniMax paged sparse decode on Blackwell.
105
+
106
+ This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
107
+ but accepts the Blackwell package's validated BF16/FP16 decode path. It
108
+ supports single-token decode (`seqlen_q=1`) with paged K/V
109
+ `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
110
+ """
111
+
112
+ if schedule is not None or O_partial is not None or LSE_partial is not None:
113
+ raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
114
+ if return_softmax_lse:
115
+ raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
116
+ if not causal:
117
+ raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
118
+ if int(seqlen_q) != 1:
119
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
120
+ if q.ndim != 3:
121
+ raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
122
+ if k.ndim != 4 or v.ndim != 4:
123
+ raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
124
+ if k.shape != v.shape:
125
+ raise ValueError("k and v must have the same shape")
126
+ if q.device != k.device or q.device != v.device or q.device != page_table.device:
127
+ raise ValueError("q, k, v, and page_table must be on the same device")
128
+ if q.dtype not in (torch.bfloat16, torch.float16):
129
+ raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
130
+ if k.dtype != q.dtype or v.dtype != q.dtype:
131
+ raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
132
+ if q.shape[-1] != 128 or k.shape[-1] != 128:
133
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
134
+ if int(k.shape[2]) != int(blk_kv):
135
+ raise ValueError("k.shape[2] must equal blk_kv")
136
+ if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
137
+ raise TypeError("page_table and seqused_k must be torch.int32")
138
+ if page_table.ndim != 2:
139
+ raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
140
+ batch = int(page_table.shape[0])
141
+ if int(q.shape[0]) != batch:
142
+ raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
143
+
144
+ head_kv = int(k.shape[1])
145
+ q2k = _normalize_decode_q2k(
146
+ q2k_indices,
147
+ batch=batch,
148
+ seqlen_q=int(seqlen_q),
149
+ head_kv=head_kv,
150
+ seqused_k=seqused_k,
151
+ blk_kv=int(blk_kv),
152
+ )
153
+ req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
154
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
155
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
156
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
157
+ return flash_decode_with_gqa_share_sparse(
158
+ q.contiguous(),
159
+ None,
160
+ k_cache,
161
+ v_cache,
162
+ req_to_token,
163
+ seqused_k.contiguous(),
164
+ slot_ids,
165
+ int(blk_kv),
166
+ q2k,
167
+ sm_scale=softmax_scale,
168
+ )
169
+
170
+
171
+ class SparseDecodePagedAttentionWrapper:
172
+ def __init__(self, *, blk_kv: int = 128, causal: bool = True):
173
+ self.blk_kv = int(blk_kv)
174
+ self.causal = bool(causal)
175
+ self.page_table: Optional[torch.Tensor] = None
176
+ self.seqused_k: Optional[torch.Tensor] = None
177
+ self.seqlen_q: Optional[int] = None
178
+ self.max_seqlen_k: Optional[int] = None
179
+ self.q2k_indices: Optional[torch.Tensor] = None
180
+
181
+ def plan(
182
+ self,
183
+ *,
184
+ page_table: torch.Tensor,
185
+ seqused_k: torch.Tensor,
186
+ seqlen_q: int,
187
+ max_seqlen_k: int,
188
+ q2k_indices: Optional[torch.Tensor] = None,
189
+ num_qo_heads: Optional[int] = None,
190
+ num_kv_heads: Optional[int] = None,
191
+ head_dim: Optional[int] = 128,
192
+ enable_cuda_graph: bool = False,
193
+ max_grid_size: Optional[int] = None,
194
+ fixed_split_size: Optional[int] = None,
195
+ disable_split_kv: bool = False,
196
+ ) -> "SparseDecodePagedAttentionWrapper":
197
+ if head_dim is not None and int(head_dim) != 128:
198
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
199
+ self.page_table = page_table.contiguous()
200
+ self.seqused_k = seqused_k.contiguous()
201
+ self.seqlen_q = int(seqlen_q)
202
+ self.max_seqlen_k = int(max_seqlen_k)
203
+ self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
204
+ return self
205
+
206
+ def run(
207
+ self,
208
+ q: torch.Tensor,
209
+ k: torch.Tensor,
210
+ v: torch.Tensor,
211
+ *,
212
+ softmax_scale: Optional[float] = None,
213
+ return_softmax_lse: bool = False,
214
+ out: Optional[torch.Tensor] = None,
215
+ lse: Optional[torch.Tensor] = None,
216
+ ):
217
+ if self.page_table is None or self.seqused_k is None:
218
+ raise RuntimeError("decode wrapper must be planned before run")
219
+ result = sparse_decode_atten_func(
220
+ q,
221
+ k,
222
+ v,
223
+ self.q2k_indices,
224
+ page_table=self.page_table,
225
+ seqused_k=self.seqused_k,
226
+ seqlen_q=self.seqlen_q,
227
+ max_seqlen_k=self.max_seqlen_k,
228
+ blk_kv=self.blk_kv,
229
+ causal=self.causal,
230
+ softmax_scale=softmax_scale,
231
+ return_softmax_lse=return_softmax_lse,
232
+ )
233
+ if out is not None:
234
+ out.copy_(result)
235
+ return out
236
+ return result
237
+
238
+
239
+ def sparse_atten_func(*args, **kwargs):
240
+ _unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
241
+
242
+
243
+ def sparse_atten_nvfp4_kv_func(*args, **kwargs):
244
+ _unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
245
+
246
+
247
+ def fp4_indexer_block_scores(*args, **kwargs):
248
+ _unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
249
+
250
+
251
+ __all__ = [
252
+ "sparse_atten_func",
253
+ "sparse_atten_nvfp4_kv_func",
254
+ "sparse_decode_atten_func",
255
+ "SparseDecodePagedAttentionWrapper",
256
+ "fp4_indexer_block_scores",
257
+ ]
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
- "id": "_minimaxai_msa_blackwell_cuda_6347cca",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,20 +14,24 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "g6O7Aw9I1Db01ScTRhjP7xcQsx1f33SAdQjOpMRMgX0=",
18
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
- "_minimaxai_msa_blackwell_cuda_6347cca.abi3.so": "0MXAuGFL1MD73hplBojiFRfM6SIhFH4XC/UnWC5+R7Y=",
20
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
21
- "_ops.py": "JaMCArFhNkHXPIKm1OiPosUE0JMLdsdDAtMk1gGh2FQ=",
 
22
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
23
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
24
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
25
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
26
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
 
27
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
28
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
29
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
30
- "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
 
 
31
  }
32
  }
33
  }
 
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
+ "id": "_minimaxai_msa_blackwell_cuda_3dca94b",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
18
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
+ "_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "3pxRswxdMVEWdr42YEq2be4f34zKwh6ooyzHujIbsuI=",
20
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
21
+ "_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
22
+ "api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
23
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
24
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
25
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
26
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
27
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
28
+ "interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
29
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
30
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
31
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
32
+ "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
33
+ "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
34
+ "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
35
  }
36
  }
37
  }
build/torch212-cxx11-cu130-x86_64-linux/quantize.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """NVFP4 helper API compatible with MiniMaxAI/msa.
3
+
4
+ The quantization entrypoints use Transformer Engine when it is installed. The
5
+ layout helpers and reference dequantizer are pure PyTorch and are included so
6
+ callers can prepare and validate NVFP4 metadata with the same public API names
7
+ as the upstream MiniMaxAI package.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Tuple
14
+
15
+ import torch
16
+
17
+
18
+ NVFP4_BLOCK_SIZE = 16
19
+ NVFP4_FP4_MAX = 6.0
20
+ NVFP4_FP8_E4M3_MAX = 448.0
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Nvfp4QuantizedTensor:
25
+ data: torch.Tensor
26
+ scale_128x4: torch.Tensor
27
+ global_scale: torch.Tensor
28
+ logical_scale_shape: Tuple[int, int]
29
+ original_shape: Tuple[int, ...]
30
+
31
+
32
+ def _round_up(x: int, multiple: int) -> int:
33
+ return ((int(x) + multiple - 1) // multiple) * multiple
34
+
35
+
36
+ def nvfp4_scale_128x4_offset(
37
+ row: torch.Tensor,
38
+ col: torch.Tensor,
39
+ scale_cols: int,
40
+ ) -> torch.Tensor:
41
+ tiles_n = _round_up(scale_cols, 4) // 4
42
+ tile_m = row // 128
43
+ tile_n = col // 4
44
+ outer = row % 128
45
+ inner = col % 4
46
+ return (
47
+ (tile_m * tiles_n + tile_n) * 512
48
+ + (outer % 32) * 16
49
+ + (outer // 32) * 4
50
+ + inner
51
+ )
52
+
53
+
54
+ def swizzle_nvfp4_scale_to_128x4(
55
+ scale: torch.Tensor,
56
+ *,
57
+ rows: int,
58
+ cols: int,
59
+ ) -> torch.Tensor:
60
+ if scale.ndim != 2:
61
+ raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
62
+
63
+ rows = int(rows)
64
+ cols = int(cols)
65
+ padded_rows = _round_up(rows, 128)
66
+ padded_cols = _round_up(cols, 4)
67
+ if scale.shape[0] < rows or scale.shape[1] < cols:
68
+ raise ValueError(
69
+ "scale is smaller than the requested logical shape: "
70
+ f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
71
+ )
72
+
73
+ logical = scale[:rows, :cols].contiguous()
74
+ if logical.shape != (padded_rows, padded_cols):
75
+ logical = torch.nn.functional.pad(
76
+ logical.to(torch.float32),
77
+ (0, padded_cols - cols, 0, padded_rows - rows),
78
+ ).to(scale.dtype)
79
+ swizzled = torch.empty_like(logical)
80
+
81
+ row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
82
+ col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
83
+ offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
84
+ swizzled.reshape(-1)[offset] = logical.reshape(-1)
85
+ return swizzled
86
+
87
+
88
+ def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
89
+ return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
90
+
91
+
92
+ def _import_te_nvfp4_quantizer():
93
+ try:
94
+ from transformer_engine.pytorch.tensor import NVFP4Quantizer
95
+ except Exception as exc:
96
+ raise RuntimeError(
97
+ "Transformer Engine NVFP4 quantization is unavailable. Install a "
98
+ "Transformer Engine build with NVFP4 PyTorch support to use "
99
+ "quantize_bf16_to_nvfp4_128x4."
100
+ ) from exc
101
+ return NVFP4Quantizer
102
+
103
+
104
+ def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
105
+ if not x.is_cuda:
106
+ raise ValueError("NVFP4 quantization requires a CUDA tensor")
107
+ if x.dtype not in (torch.bfloat16, torch.float16):
108
+ raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
109
+ if x.ndim < 2:
110
+ raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
111
+ if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
112
+ raise ValueError(
113
+ f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
114
+ )
115
+
116
+ rows = 1
117
+ for dim in x.shape[:-1]:
118
+ rows *= int(dim)
119
+ if rows % NVFP4_BLOCK_SIZE != 0:
120
+ raise ValueError(
121
+ "flattened row dimension must be divisible by "
122
+ f"{NVFP4_BLOCK_SIZE}, got {rows}"
123
+ )
124
+
125
+ NVFP4Quantizer = _import_te_nvfp4_quantizer()
126
+ quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
127
+ qx = quantizer.quantize(x.contiguous())
128
+ meta = qx.get_metadata()
129
+
130
+ data = meta["rowwise_data"]
131
+ if data.dtype is not torch.uint8:
132
+ data = data.view(torch.uint8)
133
+ logical_scale = meta["rowwise_scale_inv"]
134
+ amax = meta["amax_rowwise"]
135
+ scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
136
+ scale_128x4 = swizzle_nvfp4_scale_to_128x4(
137
+ logical_scale,
138
+ rows=rows,
139
+ cols=scale_cols,
140
+ )
141
+ global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
142
+
143
+ return Nvfp4QuantizedTensor(
144
+ data=data,
145
+ scale_128x4=scale_128x4,
146
+ global_scale=global_scale,
147
+ logical_scale_shape=(rows, scale_cols),
148
+ original_shape=tuple(int(v) for v in x.shape),
149
+ )
150
+
151
+
152
+ def quantize_kv_bf16_to_nvfp4_128x4(
153
+ k: torch.Tensor,
154
+ v: torch.Tensor,
155
+ ) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
156
+ return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
157
+
158
+
159
+ def dequantize_nvfp4_128x4_to_bf16(
160
+ qx: Nvfp4QuantizedTensor,
161
+ *,
162
+ include_global_scale: bool = True,
163
+ ) -> torch.Tensor:
164
+ data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
165
+ if data.shape[-1] * 2 != qx.original_shape[-1]:
166
+ raise ValueError(
167
+ "packed data last dimension does not match original shape: "
168
+ f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
169
+ )
170
+
171
+ rows, scale_cols = qx.logical_scale_shape
172
+ logical_dim = int(qx.original_shape[-1])
173
+ if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
174
+ raise ValueError(
175
+ "logical scale columns do not match original last dimension: "
176
+ f"{scale_cols} scale cols vs dim {logical_dim}"
177
+ )
178
+
179
+ fp4_lut = torch.tensor(
180
+ [
181
+ 0.0,
182
+ 0.5,
183
+ 1.0,
184
+ 1.5,
185
+ 2.0,
186
+ 3.0,
187
+ 4.0,
188
+ 6.0,
189
+ -0.0,
190
+ -0.5,
191
+ -1.0,
192
+ -1.5,
193
+ -2.0,
194
+ -3.0,
195
+ -4.0,
196
+ -6.0,
197
+ ],
198
+ dtype=torch.float32,
199
+ device=data.device,
200
+ )
201
+ packed = data.reshape(rows, logical_dim // 2)
202
+ lo = packed & 0x0F
203
+ hi = packed >> 4
204
+ values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
205
+ values[:, 0::2] = fp4_lut[lo.long()]
206
+ values[:, 1::2] = fp4_lut[hi.long()]
207
+
208
+ row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
209
+ col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
210
+ offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
211
+ scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
212
+ scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
213
+ scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
214
+ out = values * scale
215
+ if include_global_scale:
216
+ out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
217
+ return out.reshape(qx.original_shape).to(torch.bfloat16)
218
+
219
+
220
+ __all__ = [
221
+ "Nvfp4QuantizedTensor",
222
+ "quantize_bf16_to_nvfp4_128x4",
223
+ "quantize_kv_bf16_to_nvfp4_128x4",
224
+ "dequantize_nvfp4_128x4_to_bf16",
225
+ "swizzle_nvfp4_scale_to_128x4",
226
+ "nvfp4_global_scale_from_amax",
227
+ ]
build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Sparse-index helpers compatible with MiniMaxAI/msa public names."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+
11
+ def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
12
+ if cu_seqlens.dtype != torch.int32:
13
+ raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
14
+ if cu_seqlens.ndim != 1:
15
+ raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
16
+ if cu_seqlens.numel() < 1:
17
+ raise ValueError(f"{name} must have at least one element")
18
+ if not cu_seqlens.is_contiguous():
19
+ raise ValueError(f"{name} must be contiguous")
20
+
21
+
22
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
23
+ seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
24
+ return (seqlens_k + kv_block_size - 1) // kv_block_size
25
+
26
+
27
+ def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
28
+ rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
29
+ batch = len(rows_per_batch_cpu)
30
+ max_rows = max(rows_per_batch_cpu, default=0)
31
+ row_dtype = (
32
+ torch.int32
33
+ if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
34
+ else torch.int64
35
+ )
36
+ row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
37
+ row_linear = 0
38
+ for kv_block_idx in range(max_rows):
39
+ for batch_idx, row_count in enumerate(rows_per_batch_cpu):
40
+ if kv_block_idx < row_count:
41
+ row_map_cpu[batch_idx, kv_block_idx] = row_linear
42
+ row_linear += 1
43
+ return row_map_cpu.to(rows_per_batch.device), row_linear
44
+
45
+
46
+ def build_k2q_csr_torch_reference(
47
+ q2k_indices: torch.Tensor,
48
+ cu_seqlens_q: torch.Tensor,
49
+ cu_seqlens_k: torch.Tensor,
50
+ kv_block_size: int,
51
+ ) -> tuple[torch.Tensor, torch.Tensor]:
52
+ if kv_block_size <= 0:
53
+ raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
54
+ if q2k_indices.dtype != torch.int32:
55
+ raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
56
+ if q2k_indices.ndim != 3:
57
+ raise ValueError(
58
+ "q2k_indices must have shape [head_kv, total_q, topK], "
59
+ f"got {tuple(q2k_indices.shape)}"
60
+ )
61
+ _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
62
+ _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
63
+ if cu_seqlens_q.shape != cu_seqlens_k.shape:
64
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
65
+ if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
66
+ raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
67
+
68
+ head_kv, total_q, topk = q2k_indices.shape
69
+ if total_q != int(cu_seqlens_q[-1].item()):
70
+ raise ValueError(
71
+ f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
72
+ f"({int(cu_seqlens_q[-1].item())})"
73
+ )
74
+
75
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
76
+ row_map, total_rows = _build_packed_row_map(rows_per_batch)
77
+ nnz_upper_bound = total_q * topk
78
+
79
+ k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
80
+ k2q_q_indices = torch.full(
81
+ (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
82
+ )
83
+ if total_rows == 0 or total_q == 0 or topk == 0:
84
+ return k2q_row_ptr, k2q_q_indices
85
+
86
+ counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
87
+ total_entries = total_q * topk
88
+ row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
89
+ row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
90
+ q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
91
+ valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
92
+
93
+ for batch_idx in range(cu_seqlens_q.numel() - 1):
94
+ q_start = int(cu_seqlens_q[batch_idx].item())
95
+ q_end = int(cu_seqlens_q[batch_idx + 1].item())
96
+ if q_end <= q_start:
97
+ continue
98
+ q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
99
+ q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
100
+ kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
101
+ valid = kv_local >= 0
102
+ safe_kv = kv_local.clamp_min(0).long()
103
+ rows = row_map[batch_idx, safe_kv].to(row_dtype)
104
+ start = q_start * topk
105
+ end = q_end * topk
106
+ row_all[:, start:end] = rows
107
+ q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
108
+ valid_all[:, start:end] = valid & (rows >= 0)
109
+ counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
110
+
111
+ k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
112
+ offsets = torch.zeros_like(counts)
113
+ offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
114
+ cursor = torch.zeros_like(counts)
115
+ for h in range(head_kv):
116
+ for entry in range(total_entries):
117
+ if not bool(valid_all[h, entry]):
118
+ continue
119
+ row = int(row_all[h, entry].item())
120
+ pos = int(offsets[h, row].item() + cursor[h, row].item())
121
+ k2q_q_indices[h, pos] = q_all[h, entry]
122
+ cursor[h, row] += 1
123
+
124
+ return k2q_row_ptr, k2q_q_indices
125
+
126
+
127
+ def build_k2q_csr(
128
+ q2k_indices: torch.Tensor,
129
+ cu_seqlens_q: torch.Tensor,
130
+ cu_seqlens_k: torch.Tensor,
131
+ kv_block_size: int,
132
+ *,
133
+ total_k: Optional[int] = None,
134
+ max_seqlen_k: Optional[int] = None,
135
+ max_seqlen_q: Optional[int] = None,
136
+ total_rows: Optional[int] = None,
137
+ qhead_per_kv: int = 1,
138
+ return_schedule: bool = False,
139
+ ):
140
+ if return_schedule:
141
+ raise NotImplementedError(
142
+ "Blackwell build_k2q_csr currently returns CSR tensors only; "
143
+ "SM100 schedule objects are not part of this package."
144
+ )
145
+ if total_k is None:
146
+ raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
147
+ return build_k2q_csr_torch_reference(
148
+ q2k_indices.contiguous(),
149
+ cu_seqlens_q.contiguous(),
150
+ cu_seqlens_k.contiguous(),
151
+ int(kv_block_size),
152
+ )
153
+
154
+
155
+ class SparseK2qCsrBuilderSm100:
156
+ """Compatibility placeholder for the upstream SM100 class name.
157
+
158
+ The name is exported so code can feature-detect it, but schedule-producing
159
+ SM100 behavior is intentionally not claimed for this Blackwell package.
160
+ """
161
+
162
+ def __init__(self, *args, **kwargs):
163
+ self.args = args
164
+ self.kwargs = kwargs
165
+
166
+ def build(self, *args, **kwargs):
167
+ return build_k2q_csr(*args, **kwargs)
168
+
169
+
170
+ __all__ = [
171
+ "build_k2q_csr",
172
+ "build_k2q_csr_torch_reference",
173
+ "SparseK2qCsrBuilderSm100",
174
+ ]
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
40
  from ._native import has_native_ops, native_topk_from_scores
41
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
42
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
 
44
  __all__ = [
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  "flash_decode_with_topk_idx",
46
  "flash_decode_with_gqa_share_sparse",
47
  "has_native_ops",
@@ -50,4 +88,11 @@ __all__ = [
50
  "naive_flash_decode_with_topk_idx",
51
  "get_cu_seqblocks",
52
  "robust_allocator",
 
 
 
 
 
 
 
53
  ]
 
37
  from .common.utils import get_cu_seqblocks, robust_allocator
38
  from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
39
  from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
40
+ from .interface import (
41
+ SparseDecodePagedAttentionWrapper,
42
+ fp4_indexer_block_scores,
43
+ sparse_atten_func,
44
+ sparse_atten_nvfp4_kv_func,
45
+ sparse_decode_atten_func,
46
+ )
47
  from ._native import has_native_ops, native_topk_from_scores
48
  from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
49
  from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
50
+ from .quantize import (
51
+ Nvfp4QuantizedTensor,
52
+ dequantize_nvfp4_128x4_to_bf16,
53
+ nvfp4_global_scale_from_amax,
54
+ quantize_bf16_to_nvfp4_128x4,
55
+ quantize_kv_bf16_to_nvfp4_128x4,
56
+ swizzle_nvfp4_scale_to_128x4,
57
+ )
58
+ from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
59
+ from .api_status import (
60
+ OFFICIAL_API_STATUS,
61
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS,
62
+ V1_AVAILABLE_FUNCTIONS,
63
+ available_functions,
64
+ official_api_status,
65
+ official_minimax_msa_functions,
66
+ unsupported_official_functions,
67
+ )
68
 
69
  __all__ = [
70
+ "sparse_atten_func",
71
+ "sparse_atten_nvfp4_kv_func",
72
+ "sparse_decode_atten_func",
73
+ "SparseDecodePagedAttentionWrapper",
74
+ "fp4_indexer_block_scores",
75
+ "build_k2q_csr",
76
+ "SparseK2qCsrBuilderSm100",
77
+ "Nvfp4QuantizedTensor",
78
+ "quantize_bf16_to_nvfp4_128x4",
79
+ "quantize_kv_bf16_to_nvfp4_128x4",
80
+ "dequantize_nvfp4_128x4_to_bf16",
81
+ "swizzle_nvfp4_scale_to_128x4",
82
+ "nvfp4_global_scale_from_amax",
83
  "flash_decode_with_topk_idx",
84
  "flash_decode_with_gqa_share_sparse",
85
  "has_native_ops",
 
88
  "naive_flash_decode_with_topk_idx",
89
  "get_cu_seqblocks",
90
  "robust_allocator",
91
+ "OFFICIAL_API_STATUS",
92
+ "OFFICIAL_MINIMAX_MSA_FUNCTIONS",
93
+ "V1_AVAILABLE_FUNCTIONS",
94
+ "available_functions",
95
+ "official_api_status",
96
+ "official_minimax_msa_functions",
97
+ "unsupported_official_functions",
98
  ]
build/torch212-cxx11-cu132-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2b61eca24c60526ed6f86c7bdc1c3642f61d2ec3f8858c31ab10614b5ccb56f6
3
  size 164208
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e201843e3833b5493ab946e23f740140fdae18953189dfddcc3485102e233327
3
  size 164208
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _minimaxai_msa_blackwell_cuda_6347cca
3
- ops = torch.ops._minimaxai_msa_blackwell_cuda_6347cca
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_minimaxai_msa_blackwell_cuda_6347cca::{op_name}"
 
1
  import torch
2
+ from . import _minimaxai_msa_blackwell_cuda_3dca94b
3
+ ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/api_status.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Public API status for the FlashRT MiniMax MSA Blackwell package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from types import MappingProxyType
7
+
8
+
9
+ V1_AVAILABLE_FUNCTIONS = (
10
+ "sparse_atten_func",
11
+ "sparse_atten_nvfp4_kv_func",
12
+ "sparse_decode_atten_func",
13
+ "SparseDecodePagedAttentionWrapper",
14
+ "fp4_indexer_block_scores",
15
+ "build_k2q_csr",
16
+ "SparseK2qCsrBuilderSm100",
17
+ "Nvfp4QuantizedTensor",
18
+ "quantize_bf16_to_nvfp4_128x4",
19
+ "quantize_kv_bf16_to_nvfp4_128x4",
20
+ "dequantize_nvfp4_128x4_to_bf16",
21
+ "swizzle_nvfp4_scale_to_128x4",
22
+ "nvfp4_global_scale_from_amax",
23
+ "has_native_ops",
24
+ "native_topk_from_scores",
25
+ "flash_decode_with_topk_idx",
26
+ "flash_decode_with_gqa_share_sparse",
27
+ "naive_flash_decode_with_topk_idx",
28
+ "naive_flash_decode_with_gqa_share_sparse",
29
+ "get_cu_seqblocks",
30
+ "robust_allocator",
31
+ )
32
+
33
+ OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
34
+ "sparse_atten_func",
35
+ "sparse_atten_nvfp4_kv_func",
36
+ "sparse_decode_atten_func",
37
+ "SparseDecodePagedAttentionWrapper",
38
+ "fp4_indexer_block_scores",
39
+ "build_k2q_csr",
40
+ "SparseK2qCsrBuilderSm100",
41
+ "Nvfp4QuantizedTensor",
42
+ "quantize_bf16_to_nvfp4_128x4",
43
+ "quantize_kv_bf16_to_nvfp4_128x4",
44
+ "dequantize_nvfp4_128x4_to_bf16",
45
+ "swizzle_nvfp4_scale_to_128x4",
46
+ "nvfp4_global_scale_from_amax",
47
+ )
48
+
49
+ _OFFICIAL_API_STATUS = {
50
+ "sparse_atten_func": {
51
+ "status": "unsupported_callable",
52
+ "target": "blackwell-prefill",
53
+ "reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
54
+ },
55
+ "sparse_atten_nvfp4_kv_func": {
56
+ "status": "unsupported_callable",
57
+ "target": "blackwell-prefill-nvfp4",
58
+ "reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
59
+ },
60
+ "sparse_decode_atten_func": {
61
+ "status": "available",
62
+ "target": "v1",
63
+ "reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
64
+ },
65
+ "SparseDecodePagedAttentionWrapper": {
66
+ "status": "available",
67
+ "target": "v1",
68
+ "reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
69
+ },
70
+ "fp4_indexer_block_scores": {
71
+ "status": "unsupported_callable",
72
+ "target": "blackwell-fp4-indexer",
73
+ "reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
74
+ },
75
+ "build_k2q_csr": {
76
+ "status": "available",
77
+ "target": "v1",
78
+ "reason": "Torch CSR construction fallback is available for compatibility.",
79
+ },
80
+ "SparseK2qCsrBuilderSm100": {
81
+ "status": "available",
82
+ "target": "v1",
83
+ "reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
84
+ },
85
+ "Nvfp4QuantizedTensor": {
86
+ "status": "available",
87
+ "target": "v1",
88
+ "reason": "NVFP4 metadata dataclass is available.",
89
+ },
90
+ "quantize_bf16_to_nvfp4_128x4": {
91
+ "status": "available_optional_te",
92
+ "target": "v1",
93
+ "reason": "Function is available and uses Transformer Engine when installed.",
94
+ },
95
+ "quantize_kv_bf16_to_nvfp4_128x4": {
96
+ "status": "available_optional_te",
97
+ "target": "v1",
98
+ "reason": "Function is available and uses Transformer Engine when installed.",
99
+ },
100
+ "dequantize_nvfp4_128x4_to_bf16": {
101
+ "status": "available",
102
+ "target": "v1",
103
+ "reason": "Reference dequant helper is available.",
104
+ },
105
+ "swizzle_nvfp4_scale_to_128x4": {
106
+ "status": "available",
107
+ "target": "v1",
108
+ "reason": "Scale swizzle helper is available.",
109
+ },
110
+ "nvfp4_global_scale_from_amax": {
111
+ "status": "available",
112
+ "target": "v1",
113
+ "reason": "Global scale helper is available.",
114
+ },
115
+ }
116
+
117
+ OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
118
+
119
+
120
+ def available_functions() -> tuple[str, ...]:
121
+ """Return the functions/classes intentionally available in v1."""
122
+
123
+ return V1_AVAILABLE_FUNCTIONS
124
+
125
+
126
+ def official_minimax_msa_functions() -> tuple[str, ...]:
127
+ """Return the upstream MiniMaxAI/msa public function/class names tracked here."""
128
+
129
+ return OFFICIAL_MINIMAX_MSA_FUNCTIONS
130
+
131
+
132
+ def official_api_status() -> dict[str, dict[str, str]]:
133
+ """Return a copy of the official API compatibility status table."""
134
+
135
+ return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
136
+
137
+
138
+ def unsupported_official_functions() -> tuple[str, ...]:
139
+ """Return official MiniMaxAI/msa names that are not exported by this v1 package."""
140
+
141
+ return tuple(
142
+ name
143
+ for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
144
+ if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
145
+ )
build/torch212-cxx11-cu132-x86_64-linux/interface.py ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+ from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
11
+
12
+
13
+ def _unsupported(name: str, reason: str):
14
+ raise NotImplementedError(
15
+ f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
16
+ )
17
+
18
+
19
+ def _page_table_to_req_to_token(
20
+ page_table: torch.Tensor,
21
+ seqused_k: torch.Tensor,
22
+ max_seqlen_k: int,
23
+ blk_kv: int,
24
+ ) -> torch.Tensor:
25
+ batch = int(page_table.shape[0])
26
+ max_seqlen_k = int(max_seqlen_k)
27
+ blk_kv = int(blk_kv)
28
+ logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
29
+ page = logical_pos // blk_kv
30
+ offset = logical_pos % blk_kv
31
+ page = page.clamp(max=int(page_table.shape[1]) - 1)
32
+ physical_page = page_table[:, page].to(torch.int64)
33
+ physical_token = physical_page * blk_kv + offset[None, :]
34
+ valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
35
+ return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
36
+
37
+
38
+ def _dense_q2k_indices(
39
+ *,
40
+ head_kv: int,
41
+ batch: int,
42
+ topk: int,
43
+ seq_lens: torch.Tensor,
44
+ blk_kv: int,
45
+ ) -> torch.Tensor:
46
+ idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
47
+ blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
48
+ valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
49
+ for b in range(batch):
50
+ valid = blocks < valid_blocks[b]
51
+ idx[:, b, valid] = blocks[valid]
52
+ return idx
53
+
54
+
55
+ def _normalize_decode_q2k(
56
+ q2k_indices: Optional[torch.Tensor],
57
+ *,
58
+ batch: int,
59
+ seqlen_q: int,
60
+ head_kv: int,
61
+ seqused_k: torch.Tensor,
62
+ blk_kv: int,
63
+ ) -> torch.Tensor:
64
+ if q2k_indices is None:
65
+ topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
66
+ return _dense_q2k_indices(
67
+ head_kv=head_kv,
68
+ batch=batch,
69
+ topk=topk,
70
+ seq_lens=seqused_k,
71
+ blk_kv=blk_kv,
72
+ )
73
+ if q2k_indices.dtype != torch.int32:
74
+ raise TypeError("q2k_indices must be torch.int32")
75
+ if q2k_indices.ndim != 3:
76
+ raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
77
+ if q2k_indices.shape[0] != head_kv:
78
+ raise ValueError("q2k_indices first dimension must match Hkv")
79
+ if int(q2k_indices.shape[1]) != batch * seqlen_q:
80
+ raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
81
+ if seqlen_q != 1:
82
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
83
+ return q2k_indices[:, ::seqlen_q, :].contiguous()
84
+
85
+
86
+ def sparse_decode_atten_func(
87
+ q: torch.Tensor,
88
+ k: torch.Tensor,
89
+ v: torch.Tensor,
90
+ q2k_indices: Optional[torch.Tensor] = None,
91
+ *,
92
+ page_table: torch.Tensor,
93
+ seqused_k: torch.Tensor,
94
+ seqlen_q: int,
95
+ max_seqlen_k: int,
96
+ blk_kv: int = 128,
97
+ causal: bool = True,
98
+ softmax_scale: Optional[float] = None,
99
+ return_softmax_lse: bool = False,
100
+ schedule: Optional[object] = None,
101
+ O_partial: Optional[torch.Tensor] = None,
102
+ LSE_partial: Optional[torch.Tensor] = None,
103
+ ):
104
+ """Run MiniMax paged sparse decode on Blackwell.
105
+
106
+ This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
107
+ but accepts the Blackwell package's validated BF16/FP16 decode path. It
108
+ supports single-token decode (`seqlen_q=1`) with paged K/V
109
+ `[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
110
+ """
111
+
112
+ if schedule is not None or O_partial is not None or LSE_partial is not None:
113
+ raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
114
+ if return_softmax_lse:
115
+ raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
116
+ if not causal:
117
+ raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
118
+ if int(seqlen_q) != 1:
119
+ raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
120
+ if q.ndim != 3:
121
+ raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
122
+ if k.ndim != 4 or v.ndim != 4:
123
+ raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
124
+ if k.shape != v.shape:
125
+ raise ValueError("k and v must have the same shape")
126
+ if q.device != k.device or q.device != v.device or q.device != page_table.device:
127
+ raise ValueError("q, k, v, and page_table must be on the same device")
128
+ if q.dtype not in (torch.bfloat16, torch.float16):
129
+ raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
130
+ if k.dtype != q.dtype or v.dtype != q.dtype:
131
+ raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
132
+ if q.shape[-1] != 128 or k.shape[-1] != 128:
133
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
134
+ if int(k.shape[2]) != int(blk_kv):
135
+ raise ValueError("k.shape[2] must equal blk_kv")
136
+ if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
137
+ raise TypeError("page_table and seqused_k must be torch.int32")
138
+ if page_table.ndim != 2:
139
+ raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
140
+ batch = int(page_table.shape[0])
141
+ if int(q.shape[0]) != batch:
142
+ raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
143
+
144
+ head_kv = int(k.shape[1])
145
+ q2k = _normalize_decode_q2k(
146
+ q2k_indices,
147
+ batch=batch,
148
+ seqlen_q=int(seqlen_q),
149
+ head_kv=head_kv,
150
+ seqused_k=seqused_k,
151
+ blk_kv=int(blk_kv),
152
+ )
153
+ req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
154
+ slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
155
+ k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
156
+ v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
157
+ return flash_decode_with_gqa_share_sparse(
158
+ q.contiguous(),
159
+ None,
160
+ k_cache,
161
+ v_cache,
162
+ req_to_token,
163
+ seqused_k.contiguous(),
164
+ slot_ids,
165
+ int(blk_kv),
166
+ q2k,
167
+ sm_scale=softmax_scale,
168
+ )
169
+
170
+
171
+ class SparseDecodePagedAttentionWrapper:
172
+ def __init__(self, *, blk_kv: int = 128, causal: bool = True):
173
+ self.blk_kv = int(blk_kv)
174
+ self.causal = bool(causal)
175
+ self.page_table: Optional[torch.Tensor] = None
176
+ self.seqused_k: Optional[torch.Tensor] = None
177
+ self.seqlen_q: Optional[int] = None
178
+ self.max_seqlen_k: Optional[int] = None
179
+ self.q2k_indices: Optional[torch.Tensor] = None
180
+
181
+ def plan(
182
+ self,
183
+ *,
184
+ page_table: torch.Tensor,
185
+ seqused_k: torch.Tensor,
186
+ seqlen_q: int,
187
+ max_seqlen_k: int,
188
+ q2k_indices: Optional[torch.Tensor] = None,
189
+ num_qo_heads: Optional[int] = None,
190
+ num_kv_heads: Optional[int] = None,
191
+ head_dim: Optional[int] = 128,
192
+ enable_cuda_graph: bool = False,
193
+ max_grid_size: Optional[int] = None,
194
+ fixed_split_size: Optional[int] = None,
195
+ disable_split_kv: bool = False,
196
+ ) -> "SparseDecodePagedAttentionWrapper":
197
+ if head_dim is not None and int(head_dim) != 128:
198
+ raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
199
+ self.page_table = page_table.contiguous()
200
+ self.seqused_k = seqused_k.contiguous()
201
+ self.seqlen_q = int(seqlen_q)
202
+ self.max_seqlen_k = int(max_seqlen_k)
203
+ self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
204
+ return self
205
+
206
+ def run(
207
+ self,
208
+ q: torch.Tensor,
209
+ k: torch.Tensor,
210
+ v: torch.Tensor,
211
+ *,
212
+ softmax_scale: Optional[float] = None,
213
+ return_softmax_lse: bool = False,
214
+ out: Optional[torch.Tensor] = None,
215
+ lse: Optional[torch.Tensor] = None,
216
+ ):
217
+ if self.page_table is None or self.seqused_k is None:
218
+ raise RuntimeError("decode wrapper must be planned before run")
219
+ result = sparse_decode_atten_func(
220
+ q,
221
+ k,
222
+ v,
223
+ self.q2k_indices,
224
+ page_table=self.page_table,
225
+ seqused_k=self.seqused_k,
226
+ seqlen_q=self.seqlen_q,
227
+ max_seqlen_k=self.max_seqlen_k,
228
+ blk_kv=self.blk_kv,
229
+ causal=self.causal,
230
+ softmax_scale=softmax_scale,
231
+ return_softmax_lse=return_softmax_lse,
232
+ )
233
+ if out is not None:
234
+ out.copy_(result)
235
+ return out
236
+ return result
237
+
238
+
239
+ def sparse_atten_func(*args, **kwargs):
240
+ _unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
241
+
242
+
243
+ def sparse_atten_nvfp4_kv_func(*args, **kwargs):
244
+ _unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
245
+
246
+
247
+ def fp4_indexer_block_scores(*args, **kwargs):
248
+ _unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
249
+
250
+
251
+ __all__ = [
252
+ "sparse_atten_func",
253
+ "sparse_atten_nvfp4_kv_func",
254
+ "sparse_decode_atten_func",
255
+ "SparseDecodePagedAttentionWrapper",
256
+ "fp4_indexer_block_scores",
257
+ ]
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
- "id": "_minimaxai_msa_blackwell_cuda_6347cca",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,20 +14,24 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "g6O7Aw9I1Db01ScTRhjP7xcQsx1f33SAdQjOpMRMgX0=",
18
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
- "_minimaxai_msa_blackwell_cuda_6347cca.abi3.so": "K2HsokxgUm7W+Gx73Bw2QvYdLsP4hYwxqxBhS1zLVvY=",
20
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
21
- "_ops.py": "JaMCArFhNkHXPIKm1OiPosUE0JMLdsdDAtMk1gGh2FQ=",
 
22
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
23
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
24
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
25
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
26
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
 
27
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
28
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
29
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
30
- "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
 
 
31
  }
32
  }
33
  }
 
1
  {
2
  "name": "minimaxai-msa-blackwell",
3
+ "id": "_minimaxai_msa_blackwell_cuda_3dca94b",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
18
  "_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
19
+ "_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "4gGEPjgztUk6uUbiP3QBQP2uGJUxid/dzDSFEC4jMyc=",
20
  "_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
21
+ "_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
22
+ "api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
23
  "common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
24
  "common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
25
  "decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
26
  "decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
27
  "decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
28
+ "interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
29
  "minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
30
  "naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
31
  "naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
32
+ "naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
33
+ "quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
34
+ "sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
35
  }
36
  }
37
  }
build/torch212-cxx11-cu132-x86_64-linux/quantize.py ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """NVFP4 helper API compatible with MiniMaxAI/msa.
3
+
4
+ The quantization entrypoints use Transformer Engine when it is installed. The
5
+ layout helpers and reference dequantizer are pure PyTorch and are included so
6
+ callers can prepare and validate NVFP4 metadata with the same public API names
7
+ as the upstream MiniMaxAI package.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass
13
+ from typing import Tuple
14
+
15
+ import torch
16
+
17
+
18
+ NVFP4_BLOCK_SIZE = 16
19
+ NVFP4_FP4_MAX = 6.0
20
+ NVFP4_FP8_E4M3_MAX = 448.0
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Nvfp4QuantizedTensor:
25
+ data: torch.Tensor
26
+ scale_128x4: torch.Tensor
27
+ global_scale: torch.Tensor
28
+ logical_scale_shape: Tuple[int, int]
29
+ original_shape: Tuple[int, ...]
30
+
31
+
32
+ def _round_up(x: int, multiple: int) -> int:
33
+ return ((int(x) + multiple - 1) // multiple) * multiple
34
+
35
+
36
+ def nvfp4_scale_128x4_offset(
37
+ row: torch.Tensor,
38
+ col: torch.Tensor,
39
+ scale_cols: int,
40
+ ) -> torch.Tensor:
41
+ tiles_n = _round_up(scale_cols, 4) // 4
42
+ tile_m = row // 128
43
+ tile_n = col // 4
44
+ outer = row % 128
45
+ inner = col % 4
46
+ return (
47
+ (tile_m * tiles_n + tile_n) * 512
48
+ + (outer % 32) * 16
49
+ + (outer // 32) * 4
50
+ + inner
51
+ )
52
+
53
+
54
+ def swizzle_nvfp4_scale_to_128x4(
55
+ scale: torch.Tensor,
56
+ *,
57
+ rows: int,
58
+ cols: int,
59
+ ) -> torch.Tensor:
60
+ if scale.ndim != 2:
61
+ raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
62
+
63
+ rows = int(rows)
64
+ cols = int(cols)
65
+ padded_rows = _round_up(rows, 128)
66
+ padded_cols = _round_up(cols, 4)
67
+ if scale.shape[0] < rows or scale.shape[1] < cols:
68
+ raise ValueError(
69
+ "scale is smaller than the requested logical shape: "
70
+ f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
71
+ )
72
+
73
+ logical = scale[:rows, :cols].contiguous()
74
+ if logical.shape != (padded_rows, padded_cols):
75
+ logical = torch.nn.functional.pad(
76
+ logical.to(torch.float32),
77
+ (0, padded_cols - cols, 0, padded_rows - rows),
78
+ ).to(scale.dtype)
79
+ swizzled = torch.empty_like(logical)
80
+
81
+ row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
82
+ col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
83
+ offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
84
+ swizzled.reshape(-1)[offset] = logical.reshape(-1)
85
+ return swizzled
86
+
87
+
88
+ def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
89
+ return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
90
+
91
+
92
+ def _import_te_nvfp4_quantizer():
93
+ try:
94
+ from transformer_engine.pytorch.tensor import NVFP4Quantizer
95
+ except Exception as exc:
96
+ raise RuntimeError(
97
+ "Transformer Engine NVFP4 quantization is unavailable. Install a "
98
+ "Transformer Engine build with NVFP4 PyTorch support to use "
99
+ "quantize_bf16_to_nvfp4_128x4."
100
+ ) from exc
101
+ return NVFP4Quantizer
102
+
103
+
104
+ def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
105
+ if not x.is_cuda:
106
+ raise ValueError("NVFP4 quantization requires a CUDA tensor")
107
+ if x.dtype not in (torch.bfloat16, torch.float16):
108
+ raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
109
+ if x.ndim < 2:
110
+ raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
111
+ if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
112
+ raise ValueError(
113
+ f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
114
+ )
115
+
116
+ rows = 1
117
+ for dim in x.shape[:-1]:
118
+ rows *= int(dim)
119
+ if rows % NVFP4_BLOCK_SIZE != 0:
120
+ raise ValueError(
121
+ "flattened row dimension must be divisible by "
122
+ f"{NVFP4_BLOCK_SIZE}, got {rows}"
123
+ )
124
+
125
+ NVFP4Quantizer = _import_te_nvfp4_quantizer()
126
+ quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
127
+ qx = quantizer.quantize(x.contiguous())
128
+ meta = qx.get_metadata()
129
+
130
+ data = meta["rowwise_data"]
131
+ if data.dtype is not torch.uint8:
132
+ data = data.view(torch.uint8)
133
+ logical_scale = meta["rowwise_scale_inv"]
134
+ amax = meta["amax_rowwise"]
135
+ scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
136
+ scale_128x4 = swizzle_nvfp4_scale_to_128x4(
137
+ logical_scale,
138
+ rows=rows,
139
+ cols=scale_cols,
140
+ )
141
+ global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
142
+
143
+ return Nvfp4QuantizedTensor(
144
+ data=data,
145
+ scale_128x4=scale_128x4,
146
+ global_scale=global_scale,
147
+ logical_scale_shape=(rows, scale_cols),
148
+ original_shape=tuple(int(v) for v in x.shape),
149
+ )
150
+
151
+
152
+ def quantize_kv_bf16_to_nvfp4_128x4(
153
+ k: torch.Tensor,
154
+ v: torch.Tensor,
155
+ ) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
156
+ return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
157
+
158
+
159
+ def dequantize_nvfp4_128x4_to_bf16(
160
+ qx: Nvfp4QuantizedTensor,
161
+ *,
162
+ include_global_scale: bool = True,
163
+ ) -> torch.Tensor:
164
+ data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
165
+ if data.shape[-1] * 2 != qx.original_shape[-1]:
166
+ raise ValueError(
167
+ "packed data last dimension does not match original shape: "
168
+ f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
169
+ )
170
+
171
+ rows, scale_cols = qx.logical_scale_shape
172
+ logical_dim = int(qx.original_shape[-1])
173
+ if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
174
+ raise ValueError(
175
+ "logical scale columns do not match original last dimension: "
176
+ f"{scale_cols} scale cols vs dim {logical_dim}"
177
+ )
178
+
179
+ fp4_lut = torch.tensor(
180
+ [
181
+ 0.0,
182
+ 0.5,
183
+ 1.0,
184
+ 1.5,
185
+ 2.0,
186
+ 3.0,
187
+ 4.0,
188
+ 6.0,
189
+ -0.0,
190
+ -0.5,
191
+ -1.0,
192
+ -1.5,
193
+ -2.0,
194
+ -3.0,
195
+ -4.0,
196
+ -6.0,
197
+ ],
198
+ dtype=torch.float32,
199
+ device=data.device,
200
+ )
201
+ packed = data.reshape(rows, logical_dim // 2)
202
+ lo = packed & 0x0F
203
+ hi = packed >> 4
204
+ values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
205
+ values[:, 0::2] = fp4_lut[lo.long()]
206
+ values[:, 1::2] = fp4_lut[hi.long()]
207
+
208
+ row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
209
+ col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
210
+ offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
211
+ scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
212
+ scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
213
+ scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
214
+ out = values * scale
215
+ if include_global_scale:
216
+ out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
217
+ return out.reshape(qx.original_shape).to(torch.bfloat16)
218
+
219
+
220
+ __all__ = [
221
+ "Nvfp4QuantizedTensor",
222
+ "quantize_bf16_to_nvfp4_128x4",
223
+ "quantize_kv_bf16_to_nvfp4_128x4",
224
+ "dequantize_nvfp4_128x4_to_bf16",
225
+ "swizzle_nvfp4_scale_to_128x4",
226
+ "nvfp4_global_scale_from_amax",
227
+ ]
build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Sparse-index helpers compatible with MiniMaxAI/msa public names."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import Optional
7
+
8
+ import torch
9
+
10
+
11
+ def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
12
+ if cu_seqlens.dtype != torch.int32:
13
+ raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
14
+ if cu_seqlens.ndim != 1:
15
+ raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
16
+ if cu_seqlens.numel() < 1:
17
+ raise ValueError(f"{name} must have at least one element")
18
+ if not cu_seqlens.is_contiguous():
19
+ raise ValueError(f"{name} must be contiguous")
20
+
21
+
22
+ def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
23
+ seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
24
+ return (seqlens_k + kv_block_size - 1) // kv_block_size
25
+
26
+
27
+ def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
28
+ rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
29
+ batch = len(rows_per_batch_cpu)
30
+ max_rows = max(rows_per_batch_cpu, default=0)
31
+ row_dtype = (
32
+ torch.int32
33
+ if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
34
+ else torch.int64
35
+ )
36
+ row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
37
+ row_linear = 0
38
+ for kv_block_idx in range(max_rows):
39
+ for batch_idx, row_count in enumerate(rows_per_batch_cpu):
40
+ if kv_block_idx < row_count:
41
+ row_map_cpu[batch_idx, kv_block_idx] = row_linear
42
+ row_linear += 1
43
+ return row_map_cpu.to(rows_per_batch.device), row_linear
44
+
45
+
46
+ def build_k2q_csr_torch_reference(
47
+ q2k_indices: torch.Tensor,
48
+ cu_seqlens_q: torch.Tensor,
49
+ cu_seqlens_k: torch.Tensor,
50
+ kv_block_size: int,
51
+ ) -> tuple[torch.Tensor, torch.Tensor]:
52
+ if kv_block_size <= 0:
53
+ raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
54
+ if q2k_indices.dtype != torch.int32:
55
+ raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
56
+ if q2k_indices.ndim != 3:
57
+ raise ValueError(
58
+ "q2k_indices must have shape [head_kv, total_q, topK], "
59
+ f"got {tuple(q2k_indices.shape)}"
60
+ )
61
+ _validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
62
+ _validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
63
+ if cu_seqlens_q.shape != cu_seqlens_k.shape:
64
+ raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
65
+ if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
66
+ raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
67
+
68
+ head_kv, total_q, topk = q2k_indices.shape
69
+ if total_q != int(cu_seqlens_q[-1].item()):
70
+ raise ValueError(
71
+ f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
72
+ f"({int(cu_seqlens_q[-1].item())})"
73
+ )
74
+
75
+ rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
76
+ row_map, total_rows = _build_packed_row_map(rows_per_batch)
77
+ nnz_upper_bound = total_q * topk
78
+
79
+ k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
80
+ k2q_q_indices = torch.full(
81
+ (head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
82
+ )
83
+ if total_rows == 0 or total_q == 0 or topk == 0:
84
+ return k2q_row_ptr, k2q_q_indices
85
+
86
+ counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
87
+ total_entries = total_q * topk
88
+ row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
89
+ row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
90
+ q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
91
+ valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
92
+
93
+ for batch_idx in range(cu_seqlens_q.numel() - 1):
94
+ q_start = int(cu_seqlens_q[batch_idx].item())
95
+ q_end = int(cu_seqlens_q[batch_idx + 1].item())
96
+ if q_end <= q_start:
97
+ continue
98
+ q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
99
+ q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
100
+ kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
101
+ valid = kv_local >= 0
102
+ safe_kv = kv_local.clamp_min(0).long()
103
+ rows = row_map[batch_idx, safe_kv].to(row_dtype)
104
+ start = q_start * topk
105
+ end = q_end * topk
106
+ row_all[:, start:end] = rows
107
+ q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
108
+ valid_all[:, start:end] = valid & (rows >= 0)
109
+ counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
110
+
111
+ k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
112
+ offsets = torch.zeros_like(counts)
113
+ offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
114
+ cursor = torch.zeros_like(counts)
115
+ for h in range(head_kv):
116
+ for entry in range(total_entries):
117
+ if not bool(valid_all[h, entry]):
118
+ continue
119
+ row = int(row_all[h, entry].item())
120
+ pos = int(offsets[h, row].item() + cursor[h, row].item())
121
+ k2q_q_indices[h, pos] = q_all[h, entry]
122
+ cursor[h, row] += 1
123
+
124
+ return k2q_row_ptr, k2q_q_indices
125
+
126
+
127
+ def build_k2q_csr(
128
+ q2k_indices: torch.Tensor,
129
+ cu_seqlens_q: torch.Tensor,
130
+ cu_seqlens_k: torch.Tensor,
131
+ kv_block_size: int,
132
+ *,
133
+ total_k: Optional[int] = None,
134
+ max_seqlen_k: Optional[int] = None,
135
+ max_seqlen_q: Optional[int] = None,
136
+ total_rows: Optional[int] = None,
137
+ qhead_per_kv: int = 1,
138
+ return_schedule: bool = False,
139
+ ):
140
+ if return_schedule:
141
+ raise NotImplementedError(
142
+ "Blackwell build_k2q_csr currently returns CSR tensors only; "
143
+ "SM100 schedule objects are not part of this package."
144
+ )
145
+ if total_k is None:
146
+ raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
147
+ return build_k2q_csr_torch_reference(
148
+ q2k_indices.contiguous(),
149
+ cu_seqlens_q.contiguous(),
150
+ cu_seqlens_k.contiguous(),
151
+ int(kv_block_size),
152
+ )
153
+
154
+
155
+ class SparseK2qCsrBuilderSm100:
156
+ """Compatibility placeholder for the upstream SM100 class name.
157
+
158
+ The name is exported so code can feature-detect it, but schedule-producing
159
+ SM100 behavior is intentionally not claimed for this Blackwell package.
160
+ """
161
+
162
+ def __init__(self, *args, **kwargs):
163
+ self.args = args
164
+ self.kwargs = kwargs
165
+
166
+ def build(self, *args, **kwargs):
167
+ return build_k2q_csr(*args, **kwargs)
168
+
169
+
170
+ __all__ = [
171
+ "build_k2q_csr",
172
+ "build_k2q_csr_torch_reference",
173
+ "SparseK2qCsrBuilderSm100",
174
+ ]