Uploaded using `kernel-builder`.
Browse files- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +45 -0
- build/torch211-cxx11-cu128-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
- build/torch211-cxx11-cu128-x86_64-linux/api_status.py +145 -0
- build/torch211-cxx11-cu128-x86_64-linux/interface.py +257 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +9 -5
- build/torch211-cxx11-cu128-x86_64-linux/quantize.py +227 -0
- build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py +174 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +45 -0
- build/torch211-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
- build/torch211-cxx11-cu130-x86_64-linux/api_status.py +145 -0
- build/torch211-cxx11-cu130-x86_64-linux/interface.py +257 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +9 -5
- build/torch211-cxx11-cu130-x86_64-linux/quantize.py +227 -0
- build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py +174 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +45 -0
- build/torch212-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-cu130-x86_64-linux/api_status.py +145 -0
- build/torch212-cxx11-cu130-x86_64-linux/interface.py +257 -0
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +9 -5
- build/torch212-cxx11-cu130-x86_64-linux/quantize.py +227 -0
- build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py +174 -0
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +45 -0
- build/torch212-cxx11-cu132-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so} +1 -1
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-cu132-x86_64-linux/api_status.py +145 -0
- build/torch212-cxx11-cu132-x86_64-linux/interface.py +257 -0
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +9 -5
- build/torch212-cxx11-cu132-x86_64-linux/quantize.py +227 -0
- build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py +174 -0
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
CHANGED
|
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
from ._native import has_native_ops, native_topk_from_scores
|
| 41 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 42 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
__all__ = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"flash_decode_with_topk_idx",
|
| 46 |
"flash_decode_with_gqa_share_sparse",
|
| 47 |
"has_native_ops",
|
|
@@ -50,4 +88,11 @@ __all__ = [
|
|
| 50 |
"naive_flash_decode_with_topk_idx",
|
| 51 |
"get_cu_seqblocks",
|
| 52 |
"robust_allocator",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
]
|
|
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 40 |
+
from .interface import (
|
| 41 |
+
SparseDecodePagedAttentionWrapper,
|
| 42 |
+
fp4_indexer_block_scores,
|
| 43 |
+
sparse_atten_func,
|
| 44 |
+
sparse_atten_nvfp4_kv_func,
|
| 45 |
+
sparse_decode_atten_func,
|
| 46 |
+
)
|
| 47 |
from ._native import has_native_ops, native_topk_from_scores
|
| 48 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 49 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 50 |
+
from .quantize import (
|
| 51 |
+
Nvfp4QuantizedTensor,
|
| 52 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 53 |
+
nvfp4_global_scale_from_amax,
|
| 54 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 55 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 56 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 57 |
+
)
|
| 58 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 59 |
+
from .api_status import (
|
| 60 |
+
OFFICIAL_API_STATUS,
|
| 61 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 62 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 63 |
+
available_functions,
|
| 64 |
+
official_api_status,
|
| 65 |
+
official_minimax_msa_functions,
|
| 66 |
+
unsupported_official_functions,
|
| 67 |
+
)
|
| 68 |
|
| 69 |
__all__ = [
|
| 70 |
+
"sparse_atten_func",
|
| 71 |
+
"sparse_atten_nvfp4_kv_func",
|
| 72 |
+
"sparse_decode_atten_func",
|
| 73 |
+
"SparseDecodePagedAttentionWrapper",
|
| 74 |
+
"fp4_indexer_block_scores",
|
| 75 |
+
"build_k2q_csr",
|
| 76 |
+
"SparseK2qCsrBuilderSm100",
|
| 77 |
+
"Nvfp4QuantizedTensor",
|
| 78 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 79 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 80 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 81 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 82 |
+
"nvfp4_global_scale_from_amax",
|
| 83 |
"flash_decode_with_topk_idx",
|
| 84 |
"flash_decode_with_gqa_share_sparse",
|
| 85 |
"has_native_ops",
|
|
|
|
| 88 |
"naive_flash_decode_with_topk_idx",
|
| 89 |
"get_cu_seqblocks",
|
| 90 |
"robust_allocator",
|
| 91 |
+
"OFFICIAL_API_STATUS",
|
| 92 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 93 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 94 |
+
"available_functions",
|
| 95 |
+
"official_api_status",
|
| 96 |
+
"official_minimax_msa_functions",
|
| 97 |
+
"unsupported_official_functions",
|
| 98 |
]
|
build/torch211-cxx11-cu128-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 119536
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:0cc5542463661012370d32bb20bc9740c100711bb93eac37cab3c7f09d841781
|
| 3 |
size 119536
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _minimaxai_msa_blackwell_cuda_3dca94b
|
| 3 |
+
ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/api_status.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Public API status for the FlashRT MiniMax MSA Blackwell package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from types import MappingProxyType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
V1_AVAILABLE_FUNCTIONS = (
|
| 10 |
+
"sparse_atten_func",
|
| 11 |
+
"sparse_atten_nvfp4_kv_func",
|
| 12 |
+
"sparse_decode_atten_func",
|
| 13 |
+
"SparseDecodePagedAttentionWrapper",
|
| 14 |
+
"fp4_indexer_block_scores",
|
| 15 |
+
"build_k2q_csr",
|
| 16 |
+
"SparseK2qCsrBuilderSm100",
|
| 17 |
+
"Nvfp4QuantizedTensor",
|
| 18 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 19 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 20 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 21 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 22 |
+
"nvfp4_global_scale_from_amax",
|
| 23 |
+
"has_native_ops",
|
| 24 |
+
"native_topk_from_scores",
|
| 25 |
+
"flash_decode_with_topk_idx",
|
| 26 |
+
"flash_decode_with_gqa_share_sparse",
|
| 27 |
+
"naive_flash_decode_with_topk_idx",
|
| 28 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 29 |
+
"get_cu_seqblocks",
|
| 30 |
+
"robust_allocator",
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
|
| 34 |
+
"sparse_atten_func",
|
| 35 |
+
"sparse_atten_nvfp4_kv_func",
|
| 36 |
+
"sparse_decode_atten_func",
|
| 37 |
+
"SparseDecodePagedAttentionWrapper",
|
| 38 |
+
"fp4_indexer_block_scores",
|
| 39 |
+
"build_k2q_csr",
|
| 40 |
+
"SparseK2qCsrBuilderSm100",
|
| 41 |
+
"Nvfp4QuantizedTensor",
|
| 42 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 43 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 44 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 45 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 46 |
+
"nvfp4_global_scale_from_amax",
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
_OFFICIAL_API_STATUS = {
|
| 50 |
+
"sparse_atten_func": {
|
| 51 |
+
"status": "unsupported_callable",
|
| 52 |
+
"target": "blackwell-prefill",
|
| 53 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 54 |
+
},
|
| 55 |
+
"sparse_atten_nvfp4_kv_func": {
|
| 56 |
+
"status": "unsupported_callable",
|
| 57 |
+
"target": "blackwell-prefill-nvfp4",
|
| 58 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 59 |
+
},
|
| 60 |
+
"sparse_decode_atten_func": {
|
| 61 |
+
"status": "available",
|
| 62 |
+
"target": "v1",
|
| 63 |
+
"reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
|
| 64 |
+
},
|
| 65 |
+
"SparseDecodePagedAttentionWrapper": {
|
| 66 |
+
"status": "available",
|
| 67 |
+
"target": "v1",
|
| 68 |
+
"reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
|
| 69 |
+
},
|
| 70 |
+
"fp4_indexer_block_scores": {
|
| 71 |
+
"status": "unsupported_callable",
|
| 72 |
+
"target": "blackwell-fp4-indexer",
|
| 73 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
|
| 74 |
+
},
|
| 75 |
+
"build_k2q_csr": {
|
| 76 |
+
"status": "available",
|
| 77 |
+
"target": "v1",
|
| 78 |
+
"reason": "Torch CSR construction fallback is available for compatibility.",
|
| 79 |
+
},
|
| 80 |
+
"SparseK2qCsrBuilderSm100": {
|
| 81 |
+
"status": "available",
|
| 82 |
+
"target": "v1",
|
| 83 |
+
"reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
|
| 84 |
+
},
|
| 85 |
+
"Nvfp4QuantizedTensor": {
|
| 86 |
+
"status": "available",
|
| 87 |
+
"target": "v1",
|
| 88 |
+
"reason": "NVFP4 metadata dataclass is available.",
|
| 89 |
+
},
|
| 90 |
+
"quantize_bf16_to_nvfp4_128x4": {
|
| 91 |
+
"status": "available_optional_te",
|
| 92 |
+
"target": "v1",
|
| 93 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 94 |
+
},
|
| 95 |
+
"quantize_kv_bf16_to_nvfp4_128x4": {
|
| 96 |
+
"status": "available_optional_te",
|
| 97 |
+
"target": "v1",
|
| 98 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 99 |
+
},
|
| 100 |
+
"dequantize_nvfp4_128x4_to_bf16": {
|
| 101 |
+
"status": "available",
|
| 102 |
+
"target": "v1",
|
| 103 |
+
"reason": "Reference dequant helper is available.",
|
| 104 |
+
},
|
| 105 |
+
"swizzle_nvfp4_scale_to_128x4": {
|
| 106 |
+
"status": "available",
|
| 107 |
+
"target": "v1",
|
| 108 |
+
"reason": "Scale swizzle helper is available.",
|
| 109 |
+
},
|
| 110 |
+
"nvfp4_global_scale_from_amax": {
|
| 111 |
+
"status": "available",
|
| 112 |
+
"target": "v1",
|
| 113 |
+
"reason": "Global scale helper is available.",
|
| 114 |
+
},
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def available_functions() -> tuple[str, ...]:
|
| 121 |
+
"""Return the functions/classes intentionally available in v1."""
|
| 122 |
+
|
| 123 |
+
return V1_AVAILABLE_FUNCTIONS
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def official_minimax_msa_functions() -> tuple[str, ...]:
|
| 127 |
+
"""Return the upstream MiniMaxAI/msa public function/class names tracked here."""
|
| 128 |
+
|
| 129 |
+
return OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def official_api_status() -> dict[str, dict[str, str]]:
|
| 133 |
+
"""Return a copy of the official API compatibility status table."""
|
| 134 |
+
|
| 135 |
+
return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def unsupported_official_functions() -> tuple[str, ...]:
|
| 139 |
+
"""Return official MiniMaxAI/msa names that are not exported by this v1 package."""
|
| 140 |
+
|
| 141 |
+
return tuple(
|
| 142 |
+
name
|
| 143 |
+
for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 144 |
+
if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
|
| 145 |
+
)
|
build/torch211-cxx11-cu128-x86_64-linux/interface.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _unsupported(name: str, reason: str):
|
| 14 |
+
raise NotImplementedError(
|
| 15 |
+
f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _page_table_to_req_to_token(
|
| 20 |
+
page_table: torch.Tensor,
|
| 21 |
+
seqused_k: torch.Tensor,
|
| 22 |
+
max_seqlen_k: int,
|
| 23 |
+
blk_kv: int,
|
| 24 |
+
) -> torch.Tensor:
|
| 25 |
+
batch = int(page_table.shape[0])
|
| 26 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 27 |
+
blk_kv = int(blk_kv)
|
| 28 |
+
logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
|
| 29 |
+
page = logical_pos // blk_kv
|
| 30 |
+
offset = logical_pos % blk_kv
|
| 31 |
+
page = page.clamp(max=int(page_table.shape[1]) - 1)
|
| 32 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 33 |
+
physical_token = physical_page * blk_kv + offset[None, :]
|
| 34 |
+
valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
|
| 35 |
+
return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _dense_q2k_indices(
|
| 39 |
+
*,
|
| 40 |
+
head_kv: int,
|
| 41 |
+
batch: int,
|
| 42 |
+
topk: int,
|
| 43 |
+
seq_lens: torch.Tensor,
|
| 44 |
+
blk_kv: int,
|
| 45 |
+
) -> torch.Tensor:
|
| 46 |
+
idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
|
| 47 |
+
blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
|
| 48 |
+
valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
|
| 49 |
+
for b in range(batch):
|
| 50 |
+
valid = blocks < valid_blocks[b]
|
| 51 |
+
idx[:, b, valid] = blocks[valid]
|
| 52 |
+
return idx
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _normalize_decode_q2k(
|
| 56 |
+
q2k_indices: Optional[torch.Tensor],
|
| 57 |
+
*,
|
| 58 |
+
batch: int,
|
| 59 |
+
seqlen_q: int,
|
| 60 |
+
head_kv: int,
|
| 61 |
+
seqused_k: torch.Tensor,
|
| 62 |
+
blk_kv: int,
|
| 63 |
+
) -> torch.Tensor:
|
| 64 |
+
if q2k_indices is None:
|
| 65 |
+
topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
|
| 66 |
+
return _dense_q2k_indices(
|
| 67 |
+
head_kv=head_kv,
|
| 68 |
+
batch=batch,
|
| 69 |
+
topk=topk,
|
| 70 |
+
seq_lens=seqused_k,
|
| 71 |
+
blk_kv=blk_kv,
|
| 72 |
+
)
|
| 73 |
+
if q2k_indices.dtype != torch.int32:
|
| 74 |
+
raise TypeError("q2k_indices must be torch.int32")
|
| 75 |
+
if q2k_indices.ndim != 3:
|
| 76 |
+
raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
|
| 77 |
+
if q2k_indices.shape[0] != head_kv:
|
| 78 |
+
raise ValueError("q2k_indices first dimension must match Hkv")
|
| 79 |
+
if int(q2k_indices.shape[1]) != batch * seqlen_q:
|
| 80 |
+
raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
|
| 81 |
+
if seqlen_q != 1:
|
| 82 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 83 |
+
return q2k_indices[:, ::seqlen_q, :].contiguous()
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def sparse_decode_atten_func(
|
| 87 |
+
q: torch.Tensor,
|
| 88 |
+
k: torch.Tensor,
|
| 89 |
+
v: torch.Tensor,
|
| 90 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 91 |
+
*,
|
| 92 |
+
page_table: torch.Tensor,
|
| 93 |
+
seqused_k: torch.Tensor,
|
| 94 |
+
seqlen_q: int,
|
| 95 |
+
max_seqlen_k: int,
|
| 96 |
+
blk_kv: int = 128,
|
| 97 |
+
causal: bool = True,
|
| 98 |
+
softmax_scale: Optional[float] = None,
|
| 99 |
+
return_softmax_lse: bool = False,
|
| 100 |
+
schedule: Optional[object] = None,
|
| 101 |
+
O_partial: Optional[torch.Tensor] = None,
|
| 102 |
+
LSE_partial: Optional[torch.Tensor] = None,
|
| 103 |
+
):
|
| 104 |
+
"""Run MiniMax paged sparse decode on Blackwell.
|
| 105 |
+
|
| 106 |
+
This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
|
| 107 |
+
but accepts the Blackwell package's validated BF16/FP16 decode path. It
|
| 108 |
+
supports single-token decode (`seqlen_q=1`) with paged K/V
|
| 109 |
+
`[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
|
| 110 |
+
"""
|
| 111 |
+
|
| 112 |
+
if schedule is not None or O_partial is not None or LSE_partial is not None:
|
| 113 |
+
raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
|
| 114 |
+
if return_softmax_lse:
|
| 115 |
+
raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
|
| 116 |
+
if not causal:
|
| 117 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
|
| 118 |
+
if int(seqlen_q) != 1:
|
| 119 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 120 |
+
if q.ndim != 3:
|
| 121 |
+
raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
|
| 122 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 123 |
+
raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 124 |
+
if k.shape != v.shape:
|
| 125 |
+
raise ValueError("k and v must have the same shape")
|
| 126 |
+
if q.device != k.device or q.device != v.device or q.device != page_table.device:
|
| 127 |
+
raise ValueError("q, k, v, and page_table must be on the same device")
|
| 128 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 129 |
+
raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
|
| 130 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 131 |
+
raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
|
| 132 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128:
|
| 133 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 134 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 135 |
+
raise ValueError("k.shape[2] must equal blk_kv")
|
| 136 |
+
if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
|
| 137 |
+
raise TypeError("page_table and seqused_k must be torch.int32")
|
| 138 |
+
if page_table.ndim != 2:
|
| 139 |
+
raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
|
| 140 |
+
batch = int(page_table.shape[0])
|
| 141 |
+
if int(q.shape[0]) != batch:
|
| 142 |
+
raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
|
| 143 |
+
|
| 144 |
+
head_kv = int(k.shape[1])
|
| 145 |
+
q2k = _normalize_decode_q2k(
|
| 146 |
+
q2k_indices,
|
| 147 |
+
batch=batch,
|
| 148 |
+
seqlen_q=int(seqlen_q),
|
| 149 |
+
head_kv=head_kv,
|
| 150 |
+
seqused_k=seqused_k,
|
| 151 |
+
blk_kv=int(blk_kv),
|
| 152 |
+
)
|
| 153 |
+
req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
|
| 154 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 155 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 156 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 157 |
+
return flash_decode_with_gqa_share_sparse(
|
| 158 |
+
q.contiguous(),
|
| 159 |
+
None,
|
| 160 |
+
k_cache,
|
| 161 |
+
v_cache,
|
| 162 |
+
req_to_token,
|
| 163 |
+
seqused_k.contiguous(),
|
| 164 |
+
slot_ids,
|
| 165 |
+
int(blk_kv),
|
| 166 |
+
q2k,
|
| 167 |
+
sm_scale=softmax_scale,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class SparseDecodePagedAttentionWrapper:
|
| 172 |
+
def __init__(self, *, blk_kv: int = 128, causal: bool = True):
|
| 173 |
+
self.blk_kv = int(blk_kv)
|
| 174 |
+
self.causal = bool(causal)
|
| 175 |
+
self.page_table: Optional[torch.Tensor] = None
|
| 176 |
+
self.seqused_k: Optional[torch.Tensor] = None
|
| 177 |
+
self.seqlen_q: Optional[int] = None
|
| 178 |
+
self.max_seqlen_k: Optional[int] = None
|
| 179 |
+
self.q2k_indices: Optional[torch.Tensor] = None
|
| 180 |
+
|
| 181 |
+
def plan(
|
| 182 |
+
self,
|
| 183 |
+
*,
|
| 184 |
+
page_table: torch.Tensor,
|
| 185 |
+
seqused_k: torch.Tensor,
|
| 186 |
+
seqlen_q: int,
|
| 187 |
+
max_seqlen_k: int,
|
| 188 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 189 |
+
num_qo_heads: Optional[int] = None,
|
| 190 |
+
num_kv_heads: Optional[int] = None,
|
| 191 |
+
head_dim: Optional[int] = 128,
|
| 192 |
+
enable_cuda_graph: bool = False,
|
| 193 |
+
max_grid_size: Optional[int] = None,
|
| 194 |
+
fixed_split_size: Optional[int] = None,
|
| 195 |
+
disable_split_kv: bool = False,
|
| 196 |
+
) -> "SparseDecodePagedAttentionWrapper":
|
| 197 |
+
if head_dim is not None and int(head_dim) != 128:
|
| 198 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 199 |
+
self.page_table = page_table.contiguous()
|
| 200 |
+
self.seqused_k = seqused_k.contiguous()
|
| 201 |
+
self.seqlen_q = int(seqlen_q)
|
| 202 |
+
self.max_seqlen_k = int(max_seqlen_k)
|
| 203 |
+
self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
|
| 204 |
+
return self
|
| 205 |
+
|
| 206 |
+
def run(
|
| 207 |
+
self,
|
| 208 |
+
q: torch.Tensor,
|
| 209 |
+
k: torch.Tensor,
|
| 210 |
+
v: torch.Tensor,
|
| 211 |
+
*,
|
| 212 |
+
softmax_scale: Optional[float] = None,
|
| 213 |
+
return_softmax_lse: bool = False,
|
| 214 |
+
out: Optional[torch.Tensor] = None,
|
| 215 |
+
lse: Optional[torch.Tensor] = None,
|
| 216 |
+
):
|
| 217 |
+
if self.page_table is None or self.seqused_k is None:
|
| 218 |
+
raise RuntimeError("decode wrapper must be planned before run")
|
| 219 |
+
result = sparse_decode_atten_func(
|
| 220 |
+
q,
|
| 221 |
+
k,
|
| 222 |
+
v,
|
| 223 |
+
self.q2k_indices,
|
| 224 |
+
page_table=self.page_table,
|
| 225 |
+
seqused_k=self.seqused_k,
|
| 226 |
+
seqlen_q=self.seqlen_q,
|
| 227 |
+
max_seqlen_k=self.max_seqlen_k,
|
| 228 |
+
blk_kv=self.blk_kv,
|
| 229 |
+
causal=self.causal,
|
| 230 |
+
softmax_scale=softmax_scale,
|
| 231 |
+
return_softmax_lse=return_softmax_lse,
|
| 232 |
+
)
|
| 233 |
+
if out is not None:
|
| 234 |
+
out.copy_(result)
|
| 235 |
+
return out
|
| 236 |
+
return result
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def sparse_atten_func(*args, **kwargs):
|
| 240 |
+
_unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def sparse_atten_nvfp4_kv_func(*args, **kwargs):
|
| 244 |
+
_unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def fp4_indexer_block_scores(*args, **kwargs):
|
| 248 |
+
_unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
__all__ = [
|
| 252 |
+
"sparse_atten_func",
|
| 253 |
+
"sparse_atten_nvfp4_kv_func",
|
| 254 |
+
"sparse_decode_atten_func",
|
| 255 |
+
"SparseDecodePagedAttentionWrapper",
|
| 256 |
+
"fp4_indexer_block_scores",
|
| 257 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -13,20 +13,24 @@
|
|
| 13 |
"digest": {
|
| 14 |
"algorithm": "sha256",
|
| 15 |
"files": {
|
| 16 |
-
"__init__.py": "
|
| 17 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 18 |
-
"
|
| 19 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 20 |
-
"_ops.py": "
|
|
|
|
| 21 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 22 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 23 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 24 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 25 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
|
|
|
| 26 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 27 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 28 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 29 |
-
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
|
|
|
|
|
|
|
| 30 |
}
|
| 31 |
}
|
| 32 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
+
"id": "_minimaxai_msa_blackwell_cuda_3dca94b",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 13 |
"digest": {
|
| 14 |
"algorithm": "sha256",
|
| 15 |
"files": {
|
| 16 |
+
"__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
|
| 17 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 18 |
+
"_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "DMVUJGNmEBI3DTK7ILyXQMEAcRu5Pqw3yrPH8J2EF4E=",
|
| 19 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 20 |
+
"_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
|
| 21 |
+
"api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
|
| 22 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 23 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 24 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 25 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 26 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
| 27 |
+
"interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
|
| 28 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 29 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 30 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 31 |
+
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
|
| 32 |
+
"quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
|
| 33 |
+
"sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
|
| 34 |
}
|
| 35 |
}
|
| 36 |
}
|
build/torch211-cxx11-cu128-x86_64-linux/quantize.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""NVFP4 helper API compatible with MiniMaxAI/msa.
|
| 3 |
+
|
| 4 |
+
The quantization entrypoints use Transformer Engine when it is installed. The
|
| 5 |
+
layout helpers and reference dequantizer are pure PyTorch and are included so
|
| 6 |
+
callers can prepare and validate NVFP4 metadata with the same public API names
|
| 7 |
+
as the upstream MiniMaxAI package.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Tuple
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
NVFP4_BLOCK_SIZE = 16
|
| 19 |
+
NVFP4_FP4_MAX = 6.0
|
| 20 |
+
NVFP4_FP8_E4M3_MAX = 448.0
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class Nvfp4QuantizedTensor:
|
| 25 |
+
data: torch.Tensor
|
| 26 |
+
scale_128x4: torch.Tensor
|
| 27 |
+
global_scale: torch.Tensor
|
| 28 |
+
logical_scale_shape: Tuple[int, int]
|
| 29 |
+
original_shape: Tuple[int, ...]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _round_up(x: int, multiple: int) -> int:
|
| 33 |
+
return ((int(x) + multiple - 1) // multiple) * multiple
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def nvfp4_scale_128x4_offset(
|
| 37 |
+
row: torch.Tensor,
|
| 38 |
+
col: torch.Tensor,
|
| 39 |
+
scale_cols: int,
|
| 40 |
+
) -> torch.Tensor:
|
| 41 |
+
tiles_n = _round_up(scale_cols, 4) // 4
|
| 42 |
+
tile_m = row // 128
|
| 43 |
+
tile_n = col // 4
|
| 44 |
+
outer = row % 128
|
| 45 |
+
inner = col % 4
|
| 46 |
+
return (
|
| 47 |
+
(tile_m * tiles_n + tile_n) * 512
|
| 48 |
+
+ (outer % 32) * 16
|
| 49 |
+
+ (outer // 32) * 4
|
| 50 |
+
+ inner
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def swizzle_nvfp4_scale_to_128x4(
|
| 55 |
+
scale: torch.Tensor,
|
| 56 |
+
*,
|
| 57 |
+
rows: int,
|
| 58 |
+
cols: int,
|
| 59 |
+
) -> torch.Tensor:
|
| 60 |
+
if scale.ndim != 2:
|
| 61 |
+
raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
|
| 62 |
+
|
| 63 |
+
rows = int(rows)
|
| 64 |
+
cols = int(cols)
|
| 65 |
+
padded_rows = _round_up(rows, 128)
|
| 66 |
+
padded_cols = _round_up(cols, 4)
|
| 67 |
+
if scale.shape[0] < rows or scale.shape[1] < cols:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
"scale is smaller than the requested logical shape: "
|
| 70 |
+
f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
logical = scale[:rows, :cols].contiguous()
|
| 74 |
+
if logical.shape != (padded_rows, padded_cols):
|
| 75 |
+
logical = torch.nn.functional.pad(
|
| 76 |
+
logical.to(torch.float32),
|
| 77 |
+
(0, padded_cols - cols, 0, padded_rows - rows),
|
| 78 |
+
).to(scale.dtype)
|
| 79 |
+
swizzled = torch.empty_like(logical)
|
| 80 |
+
|
| 81 |
+
row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
|
| 82 |
+
col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
|
| 83 |
+
offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
|
| 84 |
+
swizzled.reshape(-1)[offset] = logical.reshape(-1)
|
| 85 |
+
return swizzled
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _import_te_nvfp4_quantizer():
|
| 93 |
+
try:
|
| 94 |
+
from transformer_engine.pytorch.tensor import NVFP4Quantizer
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
"Transformer Engine NVFP4 quantization is unavailable. Install a "
|
| 98 |
+
"Transformer Engine build with NVFP4 PyTorch support to use "
|
| 99 |
+
"quantize_bf16_to_nvfp4_128x4."
|
| 100 |
+
) from exc
|
| 101 |
+
return NVFP4Quantizer
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
|
| 105 |
+
if not x.is_cuda:
|
| 106 |
+
raise ValueError("NVFP4 quantization requires a CUDA tensor")
|
| 107 |
+
if x.dtype not in (torch.bfloat16, torch.float16):
|
| 108 |
+
raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
|
| 109 |
+
if x.ndim < 2:
|
| 110 |
+
raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
|
| 111 |
+
if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
rows = 1
|
| 117 |
+
for dim in x.shape[:-1]:
|
| 118 |
+
rows *= int(dim)
|
| 119 |
+
if rows % NVFP4_BLOCK_SIZE != 0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"flattened row dimension must be divisible by "
|
| 122 |
+
f"{NVFP4_BLOCK_SIZE}, got {rows}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
NVFP4Quantizer = _import_te_nvfp4_quantizer()
|
| 126 |
+
quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
|
| 127 |
+
qx = quantizer.quantize(x.contiguous())
|
| 128 |
+
meta = qx.get_metadata()
|
| 129 |
+
|
| 130 |
+
data = meta["rowwise_data"]
|
| 131 |
+
if data.dtype is not torch.uint8:
|
| 132 |
+
data = data.view(torch.uint8)
|
| 133 |
+
logical_scale = meta["rowwise_scale_inv"]
|
| 134 |
+
amax = meta["amax_rowwise"]
|
| 135 |
+
scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
|
| 136 |
+
scale_128x4 = swizzle_nvfp4_scale_to_128x4(
|
| 137 |
+
logical_scale,
|
| 138 |
+
rows=rows,
|
| 139 |
+
cols=scale_cols,
|
| 140 |
+
)
|
| 141 |
+
global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
|
| 142 |
+
|
| 143 |
+
return Nvfp4QuantizedTensor(
|
| 144 |
+
data=data,
|
| 145 |
+
scale_128x4=scale_128x4,
|
| 146 |
+
global_scale=global_scale,
|
| 147 |
+
logical_scale_shape=(rows, scale_cols),
|
| 148 |
+
original_shape=tuple(int(v) for v in x.shape),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def quantize_kv_bf16_to_nvfp4_128x4(
|
| 153 |
+
k: torch.Tensor,
|
| 154 |
+
v: torch.Tensor,
|
| 155 |
+
) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
|
| 156 |
+
return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def dequantize_nvfp4_128x4_to_bf16(
|
| 160 |
+
qx: Nvfp4QuantizedTensor,
|
| 161 |
+
*,
|
| 162 |
+
include_global_scale: bool = True,
|
| 163 |
+
) -> torch.Tensor:
|
| 164 |
+
data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
|
| 165 |
+
if data.shape[-1] * 2 != qx.original_shape[-1]:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"packed data last dimension does not match original shape: "
|
| 168 |
+
f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
rows, scale_cols = qx.logical_scale_shape
|
| 172 |
+
logical_dim = int(qx.original_shape[-1])
|
| 173 |
+
if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
"logical scale columns do not match original last dimension: "
|
| 176 |
+
f"{scale_cols} scale cols vs dim {logical_dim}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
fp4_lut = torch.tensor(
|
| 180 |
+
[
|
| 181 |
+
0.0,
|
| 182 |
+
0.5,
|
| 183 |
+
1.0,
|
| 184 |
+
1.5,
|
| 185 |
+
2.0,
|
| 186 |
+
3.0,
|
| 187 |
+
4.0,
|
| 188 |
+
6.0,
|
| 189 |
+
-0.0,
|
| 190 |
+
-0.5,
|
| 191 |
+
-1.0,
|
| 192 |
+
-1.5,
|
| 193 |
+
-2.0,
|
| 194 |
+
-3.0,
|
| 195 |
+
-4.0,
|
| 196 |
+
-6.0,
|
| 197 |
+
],
|
| 198 |
+
dtype=torch.float32,
|
| 199 |
+
device=data.device,
|
| 200 |
+
)
|
| 201 |
+
packed = data.reshape(rows, logical_dim // 2)
|
| 202 |
+
lo = packed & 0x0F
|
| 203 |
+
hi = packed >> 4
|
| 204 |
+
values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
|
| 205 |
+
values[:, 0::2] = fp4_lut[lo.long()]
|
| 206 |
+
values[:, 1::2] = fp4_lut[hi.long()]
|
| 207 |
+
|
| 208 |
+
row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
|
| 209 |
+
col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
|
| 210 |
+
offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
|
| 211 |
+
scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
|
| 212 |
+
scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
|
| 213 |
+
scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
|
| 214 |
+
out = values * scale
|
| 215 |
+
if include_global_scale:
|
| 216 |
+
out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
|
| 217 |
+
return out.reshape(qx.original_shape).to(torch.bfloat16)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
__all__ = [
|
| 221 |
+
"Nvfp4QuantizedTensor",
|
| 222 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 223 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 224 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 225 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 226 |
+
"nvfp4_global_scale_from_amax",
|
| 227 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/sparse_index_utils.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Sparse-index helpers compatible with MiniMaxAI/msa public names."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
|
| 12 |
+
if cu_seqlens.dtype != torch.int32:
|
| 13 |
+
raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
|
| 14 |
+
if cu_seqlens.ndim != 1:
|
| 15 |
+
raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
|
| 16 |
+
if cu_seqlens.numel() < 1:
|
| 17 |
+
raise ValueError(f"{name} must have at least one element")
|
| 18 |
+
if not cu_seqlens.is_contiguous():
|
| 19 |
+
raise ValueError(f"{name} must be contiguous")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
|
| 23 |
+
seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
|
| 24 |
+
return (seqlens_k + kv_block_size - 1) // kv_block_size
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
|
| 28 |
+
rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
|
| 29 |
+
batch = len(rows_per_batch_cpu)
|
| 30 |
+
max_rows = max(rows_per_batch_cpu, default=0)
|
| 31 |
+
row_dtype = (
|
| 32 |
+
torch.int32
|
| 33 |
+
if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
|
| 34 |
+
else torch.int64
|
| 35 |
+
)
|
| 36 |
+
row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
|
| 37 |
+
row_linear = 0
|
| 38 |
+
for kv_block_idx in range(max_rows):
|
| 39 |
+
for batch_idx, row_count in enumerate(rows_per_batch_cpu):
|
| 40 |
+
if kv_block_idx < row_count:
|
| 41 |
+
row_map_cpu[batch_idx, kv_block_idx] = row_linear
|
| 42 |
+
row_linear += 1
|
| 43 |
+
return row_map_cpu.to(rows_per_batch.device), row_linear
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_k2q_csr_torch_reference(
|
| 47 |
+
q2k_indices: torch.Tensor,
|
| 48 |
+
cu_seqlens_q: torch.Tensor,
|
| 49 |
+
cu_seqlens_k: torch.Tensor,
|
| 50 |
+
kv_block_size: int,
|
| 51 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 52 |
+
if kv_block_size <= 0:
|
| 53 |
+
raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
|
| 54 |
+
if q2k_indices.dtype != torch.int32:
|
| 55 |
+
raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
|
| 56 |
+
if q2k_indices.ndim != 3:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
"q2k_indices must have shape [head_kv, total_q, topK], "
|
| 59 |
+
f"got {tuple(q2k_indices.shape)}"
|
| 60 |
+
)
|
| 61 |
+
_validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
|
| 62 |
+
_validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
|
| 63 |
+
if cu_seqlens_q.shape != cu_seqlens_k.shape:
|
| 64 |
+
raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
|
| 65 |
+
if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
|
| 66 |
+
raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
|
| 67 |
+
|
| 68 |
+
head_kv, total_q, topk = q2k_indices.shape
|
| 69 |
+
if total_q != int(cu_seqlens_q[-1].item()):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
|
| 72 |
+
f"({int(cu_seqlens_q[-1].item())})"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
|
| 76 |
+
row_map, total_rows = _build_packed_row_map(rows_per_batch)
|
| 77 |
+
nnz_upper_bound = total_q * topk
|
| 78 |
+
|
| 79 |
+
k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
|
| 80 |
+
k2q_q_indices = torch.full(
|
| 81 |
+
(head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
|
| 82 |
+
)
|
| 83 |
+
if total_rows == 0 or total_q == 0 or topk == 0:
|
| 84 |
+
return k2q_row_ptr, k2q_q_indices
|
| 85 |
+
|
| 86 |
+
counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
|
| 87 |
+
total_entries = total_q * topk
|
| 88 |
+
row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
|
| 89 |
+
row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
|
| 90 |
+
q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
|
| 91 |
+
valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
|
| 92 |
+
|
| 93 |
+
for batch_idx in range(cu_seqlens_q.numel() - 1):
|
| 94 |
+
q_start = int(cu_seqlens_q[batch_idx].item())
|
| 95 |
+
q_end = int(cu_seqlens_q[batch_idx + 1].item())
|
| 96 |
+
if q_end <= q_start:
|
| 97 |
+
continue
|
| 98 |
+
q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
|
| 99 |
+
q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
|
| 100 |
+
kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
|
| 101 |
+
valid = kv_local >= 0
|
| 102 |
+
safe_kv = kv_local.clamp_min(0).long()
|
| 103 |
+
rows = row_map[batch_idx, safe_kv].to(row_dtype)
|
| 104 |
+
start = q_start * topk
|
| 105 |
+
end = q_end * topk
|
| 106 |
+
row_all[:, start:end] = rows
|
| 107 |
+
q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
|
| 108 |
+
valid_all[:, start:end] = valid & (rows >= 0)
|
| 109 |
+
counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
|
| 110 |
+
|
| 111 |
+
k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
|
| 112 |
+
offsets = torch.zeros_like(counts)
|
| 113 |
+
offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
|
| 114 |
+
cursor = torch.zeros_like(counts)
|
| 115 |
+
for h in range(head_kv):
|
| 116 |
+
for entry in range(total_entries):
|
| 117 |
+
if not bool(valid_all[h, entry]):
|
| 118 |
+
continue
|
| 119 |
+
row = int(row_all[h, entry].item())
|
| 120 |
+
pos = int(offsets[h, row].item() + cursor[h, row].item())
|
| 121 |
+
k2q_q_indices[h, pos] = q_all[h, entry]
|
| 122 |
+
cursor[h, row] += 1
|
| 123 |
+
|
| 124 |
+
return k2q_row_ptr, k2q_q_indices
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def build_k2q_csr(
|
| 128 |
+
q2k_indices: torch.Tensor,
|
| 129 |
+
cu_seqlens_q: torch.Tensor,
|
| 130 |
+
cu_seqlens_k: torch.Tensor,
|
| 131 |
+
kv_block_size: int,
|
| 132 |
+
*,
|
| 133 |
+
total_k: Optional[int] = None,
|
| 134 |
+
max_seqlen_k: Optional[int] = None,
|
| 135 |
+
max_seqlen_q: Optional[int] = None,
|
| 136 |
+
total_rows: Optional[int] = None,
|
| 137 |
+
qhead_per_kv: int = 1,
|
| 138 |
+
return_schedule: bool = False,
|
| 139 |
+
):
|
| 140 |
+
if return_schedule:
|
| 141 |
+
raise NotImplementedError(
|
| 142 |
+
"Blackwell build_k2q_csr currently returns CSR tensors only; "
|
| 143 |
+
"SM100 schedule objects are not part of this package."
|
| 144 |
+
)
|
| 145 |
+
if total_k is None:
|
| 146 |
+
raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
|
| 147 |
+
return build_k2q_csr_torch_reference(
|
| 148 |
+
q2k_indices.contiguous(),
|
| 149 |
+
cu_seqlens_q.contiguous(),
|
| 150 |
+
cu_seqlens_k.contiguous(),
|
| 151 |
+
int(kv_block_size),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SparseK2qCsrBuilderSm100:
|
| 156 |
+
"""Compatibility placeholder for the upstream SM100 class name.
|
| 157 |
+
|
| 158 |
+
The name is exported so code can feature-detect it, but schedule-producing
|
| 159 |
+
SM100 behavior is intentionally not claimed for this Blackwell package.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(self, *args, **kwargs):
|
| 163 |
+
self.args = args
|
| 164 |
+
self.kwargs = kwargs
|
| 165 |
+
|
| 166 |
+
def build(self, *args, **kwargs):
|
| 167 |
+
return build_k2q_csr(*args, **kwargs)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
__all__ = [
|
| 171 |
+
"build_k2q_csr",
|
| 172 |
+
"build_k2q_csr_torch_reference",
|
| 173 |
+
"SparseK2qCsrBuilderSm100",
|
| 174 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
CHANGED
|
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
from ._native import has_native_ops, native_topk_from_scores
|
| 41 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 42 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
__all__ = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"flash_decode_with_topk_idx",
|
| 46 |
"flash_decode_with_gqa_share_sparse",
|
| 47 |
"has_native_ops",
|
|
@@ -50,4 +88,11 @@ __all__ = [
|
|
| 50 |
"naive_flash_decode_with_topk_idx",
|
| 51 |
"get_cu_seqblocks",
|
| 52 |
"robust_allocator",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
]
|
|
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 40 |
+
from .interface import (
|
| 41 |
+
SparseDecodePagedAttentionWrapper,
|
| 42 |
+
fp4_indexer_block_scores,
|
| 43 |
+
sparse_atten_func,
|
| 44 |
+
sparse_atten_nvfp4_kv_func,
|
| 45 |
+
sparse_decode_atten_func,
|
| 46 |
+
)
|
| 47 |
from ._native import has_native_ops, native_topk_from_scores
|
| 48 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 49 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 50 |
+
from .quantize import (
|
| 51 |
+
Nvfp4QuantizedTensor,
|
| 52 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 53 |
+
nvfp4_global_scale_from_amax,
|
| 54 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 55 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 56 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 57 |
+
)
|
| 58 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 59 |
+
from .api_status import (
|
| 60 |
+
OFFICIAL_API_STATUS,
|
| 61 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 62 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 63 |
+
available_functions,
|
| 64 |
+
official_api_status,
|
| 65 |
+
official_minimax_msa_functions,
|
| 66 |
+
unsupported_official_functions,
|
| 67 |
+
)
|
| 68 |
|
| 69 |
__all__ = [
|
| 70 |
+
"sparse_atten_func",
|
| 71 |
+
"sparse_atten_nvfp4_kv_func",
|
| 72 |
+
"sparse_decode_atten_func",
|
| 73 |
+
"SparseDecodePagedAttentionWrapper",
|
| 74 |
+
"fp4_indexer_block_scores",
|
| 75 |
+
"build_k2q_csr",
|
| 76 |
+
"SparseK2qCsrBuilderSm100",
|
| 77 |
+
"Nvfp4QuantizedTensor",
|
| 78 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 79 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 80 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 81 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 82 |
+
"nvfp4_global_scale_from_amax",
|
| 83 |
"flash_decode_with_topk_idx",
|
| 84 |
"flash_decode_with_gqa_share_sparse",
|
| 85 |
"has_native_ops",
|
|
|
|
| 88 |
"naive_flash_decode_with_topk_idx",
|
| 89 |
"get_cu_seqblocks",
|
| 90 |
"robust_allocator",
|
| 91 |
+
"OFFICIAL_API_STATUS",
|
| 92 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 93 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 94 |
+
"available_functions",
|
| 95 |
+
"official_api_status",
|
| 96 |
+
"official_minimax_msa_functions",
|
| 97 |
+
"unsupported_official_functions",
|
| 98 |
]
|
build/torch211-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 145216
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:98433d5e5a693cb4c91a4a16c73ec96cc8852ca1c49111f7399a28c7a292b589
|
| 3 |
size 145216
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _minimaxai_msa_blackwell_cuda_3dca94b
|
| 3 |
+
ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/api_status.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Public API status for the FlashRT MiniMax MSA Blackwell package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from types import MappingProxyType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
V1_AVAILABLE_FUNCTIONS = (
|
| 10 |
+
"sparse_atten_func",
|
| 11 |
+
"sparse_atten_nvfp4_kv_func",
|
| 12 |
+
"sparse_decode_atten_func",
|
| 13 |
+
"SparseDecodePagedAttentionWrapper",
|
| 14 |
+
"fp4_indexer_block_scores",
|
| 15 |
+
"build_k2q_csr",
|
| 16 |
+
"SparseK2qCsrBuilderSm100",
|
| 17 |
+
"Nvfp4QuantizedTensor",
|
| 18 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 19 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 20 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 21 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 22 |
+
"nvfp4_global_scale_from_amax",
|
| 23 |
+
"has_native_ops",
|
| 24 |
+
"native_topk_from_scores",
|
| 25 |
+
"flash_decode_with_topk_idx",
|
| 26 |
+
"flash_decode_with_gqa_share_sparse",
|
| 27 |
+
"naive_flash_decode_with_topk_idx",
|
| 28 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 29 |
+
"get_cu_seqblocks",
|
| 30 |
+
"robust_allocator",
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
|
| 34 |
+
"sparse_atten_func",
|
| 35 |
+
"sparse_atten_nvfp4_kv_func",
|
| 36 |
+
"sparse_decode_atten_func",
|
| 37 |
+
"SparseDecodePagedAttentionWrapper",
|
| 38 |
+
"fp4_indexer_block_scores",
|
| 39 |
+
"build_k2q_csr",
|
| 40 |
+
"SparseK2qCsrBuilderSm100",
|
| 41 |
+
"Nvfp4QuantizedTensor",
|
| 42 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 43 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 44 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 45 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 46 |
+
"nvfp4_global_scale_from_amax",
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
_OFFICIAL_API_STATUS = {
|
| 50 |
+
"sparse_atten_func": {
|
| 51 |
+
"status": "unsupported_callable",
|
| 52 |
+
"target": "blackwell-prefill",
|
| 53 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 54 |
+
},
|
| 55 |
+
"sparse_atten_nvfp4_kv_func": {
|
| 56 |
+
"status": "unsupported_callable",
|
| 57 |
+
"target": "blackwell-prefill-nvfp4",
|
| 58 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 59 |
+
},
|
| 60 |
+
"sparse_decode_atten_func": {
|
| 61 |
+
"status": "available",
|
| 62 |
+
"target": "v1",
|
| 63 |
+
"reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
|
| 64 |
+
},
|
| 65 |
+
"SparseDecodePagedAttentionWrapper": {
|
| 66 |
+
"status": "available",
|
| 67 |
+
"target": "v1",
|
| 68 |
+
"reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
|
| 69 |
+
},
|
| 70 |
+
"fp4_indexer_block_scores": {
|
| 71 |
+
"status": "unsupported_callable",
|
| 72 |
+
"target": "blackwell-fp4-indexer",
|
| 73 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
|
| 74 |
+
},
|
| 75 |
+
"build_k2q_csr": {
|
| 76 |
+
"status": "available",
|
| 77 |
+
"target": "v1",
|
| 78 |
+
"reason": "Torch CSR construction fallback is available for compatibility.",
|
| 79 |
+
},
|
| 80 |
+
"SparseK2qCsrBuilderSm100": {
|
| 81 |
+
"status": "available",
|
| 82 |
+
"target": "v1",
|
| 83 |
+
"reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
|
| 84 |
+
},
|
| 85 |
+
"Nvfp4QuantizedTensor": {
|
| 86 |
+
"status": "available",
|
| 87 |
+
"target": "v1",
|
| 88 |
+
"reason": "NVFP4 metadata dataclass is available.",
|
| 89 |
+
},
|
| 90 |
+
"quantize_bf16_to_nvfp4_128x4": {
|
| 91 |
+
"status": "available_optional_te",
|
| 92 |
+
"target": "v1",
|
| 93 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 94 |
+
},
|
| 95 |
+
"quantize_kv_bf16_to_nvfp4_128x4": {
|
| 96 |
+
"status": "available_optional_te",
|
| 97 |
+
"target": "v1",
|
| 98 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 99 |
+
},
|
| 100 |
+
"dequantize_nvfp4_128x4_to_bf16": {
|
| 101 |
+
"status": "available",
|
| 102 |
+
"target": "v1",
|
| 103 |
+
"reason": "Reference dequant helper is available.",
|
| 104 |
+
},
|
| 105 |
+
"swizzle_nvfp4_scale_to_128x4": {
|
| 106 |
+
"status": "available",
|
| 107 |
+
"target": "v1",
|
| 108 |
+
"reason": "Scale swizzle helper is available.",
|
| 109 |
+
},
|
| 110 |
+
"nvfp4_global_scale_from_amax": {
|
| 111 |
+
"status": "available",
|
| 112 |
+
"target": "v1",
|
| 113 |
+
"reason": "Global scale helper is available.",
|
| 114 |
+
},
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def available_functions() -> tuple[str, ...]:
|
| 121 |
+
"""Return the functions/classes intentionally available in v1."""
|
| 122 |
+
|
| 123 |
+
return V1_AVAILABLE_FUNCTIONS
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def official_minimax_msa_functions() -> tuple[str, ...]:
|
| 127 |
+
"""Return the upstream MiniMaxAI/msa public function/class names tracked here."""
|
| 128 |
+
|
| 129 |
+
return OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def official_api_status() -> dict[str, dict[str, str]]:
|
| 133 |
+
"""Return a copy of the official API compatibility status table."""
|
| 134 |
+
|
| 135 |
+
return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def unsupported_official_functions() -> tuple[str, ...]:
|
| 139 |
+
"""Return official MiniMaxAI/msa names that are not exported by this v1 package."""
|
| 140 |
+
|
| 141 |
+
return tuple(
|
| 142 |
+
name
|
| 143 |
+
for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 144 |
+
if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
|
| 145 |
+
)
|
build/torch211-cxx11-cu130-x86_64-linux/interface.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _unsupported(name: str, reason: str):
|
| 14 |
+
raise NotImplementedError(
|
| 15 |
+
f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _page_table_to_req_to_token(
|
| 20 |
+
page_table: torch.Tensor,
|
| 21 |
+
seqused_k: torch.Tensor,
|
| 22 |
+
max_seqlen_k: int,
|
| 23 |
+
blk_kv: int,
|
| 24 |
+
) -> torch.Tensor:
|
| 25 |
+
batch = int(page_table.shape[0])
|
| 26 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 27 |
+
blk_kv = int(blk_kv)
|
| 28 |
+
logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
|
| 29 |
+
page = logical_pos // blk_kv
|
| 30 |
+
offset = logical_pos % blk_kv
|
| 31 |
+
page = page.clamp(max=int(page_table.shape[1]) - 1)
|
| 32 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 33 |
+
physical_token = physical_page * blk_kv + offset[None, :]
|
| 34 |
+
valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
|
| 35 |
+
return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _dense_q2k_indices(
|
| 39 |
+
*,
|
| 40 |
+
head_kv: int,
|
| 41 |
+
batch: int,
|
| 42 |
+
topk: int,
|
| 43 |
+
seq_lens: torch.Tensor,
|
| 44 |
+
blk_kv: int,
|
| 45 |
+
) -> torch.Tensor:
|
| 46 |
+
idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
|
| 47 |
+
blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
|
| 48 |
+
valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
|
| 49 |
+
for b in range(batch):
|
| 50 |
+
valid = blocks < valid_blocks[b]
|
| 51 |
+
idx[:, b, valid] = blocks[valid]
|
| 52 |
+
return idx
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _normalize_decode_q2k(
|
| 56 |
+
q2k_indices: Optional[torch.Tensor],
|
| 57 |
+
*,
|
| 58 |
+
batch: int,
|
| 59 |
+
seqlen_q: int,
|
| 60 |
+
head_kv: int,
|
| 61 |
+
seqused_k: torch.Tensor,
|
| 62 |
+
blk_kv: int,
|
| 63 |
+
) -> torch.Tensor:
|
| 64 |
+
if q2k_indices is None:
|
| 65 |
+
topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
|
| 66 |
+
return _dense_q2k_indices(
|
| 67 |
+
head_kv=head_kv,
|
| 68 |
+
batch=batch,
|
| 69 |
+
topk=topk,
|
| 70 |
+
seq_lens=seqused_k,
|
| 71 |
+
blk_kv=blk_kv,
|
| 72 |
+
)
|
| 73 |
+
if q2k_indices.dtype != torch.int32:
|
| 74 |
+
raise TypeError("q2k_indices must be torch.int32")
|
| 75 |
+
if q2k_indices.ndim != 3:
|
| 76 |
+
raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
|
| 77 |
+
if q2k_indices.shape[0] != head_kv:
|
| 78 |
+
raise ValueError("q2k_indices first dimension must match Hkv")
|
| 79 |
+
if int(q2k_indices.shape[1]) != batch * seqlen_q:
|
| 80 |
+
raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
|
| 81 |
+
if seqlen_q != 1:
|
| 82 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 83 |
+
return q2k_indices[:, ::seqlen_q, :].contiguous()
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def sparse_decode_atten_func(
|
| 87 |
+
q: torch.Tensor,
|
| 88 |
+
k: torch.Tensor,
|
| 89 |
+
v: torch.Tensor,
|
| 90 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 91 |
+
*,
|
| 92 |
+
page_table: torch.Tensor,
|
| 93 |
+
seqused_k: torch.Tensor,
|
| 94 |
+
seqlen_q: int,
|
| 95 |
+
max_seqlen_k: int,
|
| 96 |
+
blk_kv: int = 128,
|
| 97 |
+
causal: bool = True,
|
| 98 |
+
softmax_scale: Optional[float] = None,
|
| 99 |
+
return_softmax_lse: bool = False,
|
| 100 |
+
schedule: Optional[object] = None,
|
| 101 |
+
O_partial: Optional[torch.Tensor] = None,
|
| 102 |
+
LSE_partial: Optional[torch.Tensor] = None,
|
| 103 |
+
):
|
| 104 |
+
"""Run MiniMax paged sparse decode on Blackwell.
|
| 105 |
+
|
| 106 |
+
This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
|
| 107 |
+
but accepts the Blackwell package's validated BF16/FP16 decode path. It
|
| 108 |
+
supports single-token decode (`seqlen_q=1`) with paged K/V
|
| 109 |
+
`[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
|
| 110 |
+
"""
|
| 111 |
+
|
| 112 |
+
if schedule is not None or O_partial is not None or LSE_partial is not None:
|
| 113 |
+
raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
|
| 114 |
+
if return_softmax_lse:
|
| 115 |
+
raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
|
| 116 |
+
if not causal:
|
| 117 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
|
| 118 |
+
if int(seqlen_q) != 1:
|
| 119 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 120 |
+
if q.ndim != 3:
|
| 121 |
+
raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
|
| 122 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 123 |
+
raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 124 |
+
if k.shape != v.shape:
|
| 125 |
+
raise ValueError("k and v must have the same shape")
|
| 126 |
+
if q.device != k.device or q.device != v.device or q.device != page_table.device:
|
| 127 |
+
raise ValueError("q, k, v, and page_table must be on the same device")
|
| 128 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 129 |
+
raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
|
| 130 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 131 |
+
raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
|
| 132 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128:
|
| 133 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 134 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 135 |
+
raise ValueError("k.shape[2] must equal blk_kv")
|
| 136 |
+
if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
|
| 137 |
+
raise TypeError("page_table and seqused_k must be torch.int32")
|
| 138 |
+
if page_table.ndim != 2:
|
| 139 |
+
raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
|
| 140 |
+
batch = int(page_table.shape[0])
|
| 141 |
+
if int(q.shape[0]) != batch:
|
| 142 |
+
raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
|
| 143 |
+
|
| 144 |
+
head_kv = int(k.shape[1])
|
| 145 |
+
q2k = _normalize_decode_q2k(
|
| 146 |
+
q2k_indices,
|
| 147 |
+
batch=batch,
|
| 148 |
+
seqlen_q=int(seqlen_q),
|
| 149 |
+
head_kv=head_kv,
|
| 150 |
+
seqused_k=seqused_k,
|
| 151 |
+
blk_kv=int(blk_kv),
|
| 152 |
+
)
|
| 153 |
+
req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
|
| 154 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 155 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 156 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 157 |
+
return flash_decode_with_gqa_share_sparse(
|
| 158 |
+
q.contiguous(),
|
| 159 |
+
None,
|
| 160 |
+
k_cache,
|
| 161 |
+
v_cache,
|
| 162 |
+
req_to_token,
|
| 163 |
+
seqused_k.contiguous(),
|
| 164 |
+
slot_ids,
|
| 165 |
+
int(blk_kv),
|
| 166 |
+
q2k,
|
| 167 |
+
sm_scale=softmax_scale,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class SparseDecodePagedAttentionWrapper:
|
| 172 |
+
def __init__(self, *, blk_kv: int = 128, causal: bool = True):
|
| 173 |
+
self.blk_kv = int(blk_kv)
|
| 174 |
+
self.causal = bool(causal)
|
| 175 |
+
self.page_table: Optional[torch.Tensor] = None
|
| 176 |
+
self.seqused_k: Optional[torch.Tensor] = None
|
| 177 |
+
self.seqlen_q: Optional[int] = None
|
| 178 |
+
self.max_seqlen_k: Optional[int] = None
|
| 179 |
+
self.q2k_indices: Optional[torch.Tensor] = None
|
| 180 |
+
|
| 181 |
+
def plan(
|
| 182 |
+
self,
|
| 183 |
+
*,
|
| 184 |
+
page_table: torch.Tensor,
|
| 185 |
+
seqused_k: torch.Tensor,
|
| 186 |
+
seqlen_q: int,
|
| 187 |
+
max_seqlen_k: int,
|
| 188 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 189 |
+
num_qo_heads: Optional[int] = None,
|
| 190 |
+
num_kv_heads: Optional[int] = None,
|
| 191 |
+
head_dim: Optional[int] = 128,
|
| 192 |
+
enable_cuda_graph: bool = False,
|
| 193 |
+
max_grid_size: Optional[int] = None,
|
| 194 |
+
fixed_split_size: Optional[int] = None,
|
| 195 |
+
disable_split_kv: bool = False,
|
| 196 |
+
) -> "SparseDecodePagedAttentionWrapper":
|
| 197 |
+
if head_dim is not None and int(head_dim) != 128:
|
| 198 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 199 |
+
self.page_table = page_table.contiguous()
|
| 200 |
+
self.seqused_k = seqused_k.contiguous()
|
| 201 |
+
self.seqlen_q = int(seqlen_q)
|
| 202 |
+
self.max_seqlen_k = int(max_seqlen_k)
|
| 203 |
+
self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
|
| 204 |
+
return self
|
| 205 |
+
|
| 206 |
+
def run(
|
| 207 |
+
self,
|
| 208 |
+
q: torch.Tensor,
|
| 209 |
+
k: torch.Tensor,
|
| 210 |
+
v: torch.Tensor,
|
| 211 |
+
*,
|
| 212 |
+
softmax_scale: Optional[float] = None,
|
| 213 |
+
return_softmax_lse: bool = False,
|
| 214 |
+
out: Optional[torch.Tensor] = None,
|
| 215 |
+
lse: Optional[torch.Tensor] = None,
|
| 216 |
+
):
|
| 217 |
+
if self.page_table is None or self.seqused_k is None:
|
| 218 |
+
raise RuntimeError("decode wrapper must be planned before run")
|
| 219 |
+
result = sparse_decode_atten_func(
|
| 220 |
+
q,
|
| 221 |
+
k,
|
| 222 |
+
v,
|
| 223 |
+
self.q2k_indices,
|
| 224 |
+
page_table=self.page_table,
|
| 225 |
+
seqused_k=self.seqused_k,
|
| 226 |
+
seqlen_q=self.seqlen_q,
|
| 227 |
+
max_seqlen_k=self.max_seqlen_k,
|
| 228 |
+
blk_kv=self.blk_kv,
|
| 229 |
+
causal=self.causal,
|
| 230 |
+
softmax_scale=softmax_scale,
|
| 231 |
+
return_softmax_lse=return_softmax_lse,
|
| 232 |
+
)
|
| 233 |
+
if out is not None:
|
| 234 |
+
out.copy_(result)
|
| 235 |
+
return out
|
| 236 |
+
return result
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def sparse_atten_func(*args, **kwargs):
|
| 240 |
+
_unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def sparse_atten_nvfp4_kv_func(*args, **kwargs):
|
| 244 |
+
_unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def fp4_indexer_block_scores(*args, **kwargs):
|
| 248 |
+
_unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
__all__ = [
|
| 252 |
+
"sparse_atten_func",
|
| 253 |
+
"sparse_atten_nvfp4_kv_func",
|
| 254 |
+
"sparse_decode_atten_func",
|
| 255 |
+
"SparseDecodePagedAttentionWrapper",
|
| 256 |
+
"fp4_indexer_block_scores",
|
| 257 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,20 +14,24 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
-
"
|
| 20 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 21 |
-
"_ops.py": "
|
|
|
|
| 22 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 23 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 24 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 25 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 26 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
|
|
|
| 27 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 28 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 29 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 30 |
-
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
|
|
|
|
|
|
|
| 31 |
}
|
| 32 |
}
|
| 33 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
+
"id": "_minimaxai_msa_blackwell_cuda_3dca94b",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
|
| 18 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
+
"_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "mEM9XlppPLTJGkoWxz7JbMiFLKHEkRH3OZoox6KStYk=",
|
| 20 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 21 |
+
"_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
|
| 22 |
+
"api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
|
| 23 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 24 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 25 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 26 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 27 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
| 28 |
+
"interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
|
| 29 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 30 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 31 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 32 |
+
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
|
| 33 |
+
"quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
|
| 34 |
+
"sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
|
| 35 |
}
|
| 36 |
}
|
| 37 |
}
|
build/torch211-cxx11-cu130-x86_64-linux/quantize.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""NVFP4 helper API compatible with MiniMaxAI/msa.
|
| 3 |
+
|
| 4 |
+
The quantization entrypoints use Transformer Engine when it is installed. The
|
| 5 |
+
layout helpers and reference dequantizer are pure PyTorch and are included so
|
| 6 |
+
callers can prepare and validate NVFP4 metadata with the same public API names
|
| 7 |
+
as the upstream MiniMaxAI package.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Tuple
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
NVFP4_BLOCK_SIZE = 16
|
| 19 |
+
NVFP4_FP4_MAX = 6.0
|
| 20 |
+
NVFP4_FP8_E4M3_MAX = 448.0
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class Nvfp4QuantizedTensor:
|
| 25 |
+
data: torch.Tensor
|
| 26 |
+
scale_128x4: torch.Tensor
|
| 27 |
+
global_scale: torch.Tensor
|
| 28 |
+
logical_scale_shape: Tuple[int, int]
|
| 29 |
+
original_shape: Tuple[int, ...]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _round_up(x: int, multiple: int) -> int:
|
| 33 |
+
return ((int(x) + multiple - 1) // multiple) * multiple
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def nvfp4_scale_128x4_offset(
|
| 37 |
+
row: torch.Tensor,
|
| 38 |
+
col: torch.Tensor,
|
| 39 |
+
scale_cols: int,
|
| 40 |
+
) -> torch.Tensor:
|
| 41 |
+
tiles_n = _round_up(scale_cols, 4) // 4
|
| 42 |
+
tile_m = row // 128
|
| 43 |
+
tile_n = col // 4
|
| 44 |
+
outer = row % 128
|
| 45 |
+
inner = col % 4
|
| 46 |
+
return (
|
| 47 |
+
(tile_m * tiles_n + tile_n) * 512
|
| 48 |
+
+ (outer % 32) * 16
|
| 49 |
+
+ (outer // 32) * 4
|
| 50 |
+
+ inner
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def swizzle_nvfp4_scale_to_128x4(
|
| 55 |
+
scale: torch.Tensor,
|
| 56 |
+
*,
|
| 57 |
+
rows: int,
|
| 58 |
+
cols: int,
|
| 59 |
+
) -> torch.Tensor:
|
| 60 |
+
if scale.ndim != 2:
|
| 61 |
+
raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
|
| 62 |
+
|
| 63 |
+
rows = int(rows)
|
| 64 |
+
cols = int(cols)
|
| 65 |
+
padded_rows = _round_up(rows, 128)
|
| 66 |
+
padded_cols = _round_up(cols, 4)
|
| 67 |
+
if scale.shape[0] < rows or scale.shape[1] < cols:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
"scale is smaller than the requested logical shape: "
|
| 70 |
+
f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
logical = scale[:rows, :cols].contiguous()
|
| 74 |
+
if logical.shape != (padded_rows, padded_cols):
|
| 75 |
+
logical = torch.nn.functional.pad(
|
| 76 |
+
logical.to(torch.float32),
|
| 77 |
+
(0, padded_cols - cols, 0, padded_rows - rows),
|
| 78 |
+
).to(scale.dtype)
|
| 79 |
+
swizzled = torch.empty_like(logical)
|
| 80 |
+
|
| 81 |
+
row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
|
| 82 |
+
col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
|
| 83 |
+
offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
|
| 84 |
+
swizzled.reshape(-1)[offset] = logical.reshape(-1)
|
| 85 |
+
return swizzled
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _import_te_nvfp4_quantizer():
|
| 93 |
+
try:
|
| 94 |
+
from transformer_engine.pytorch.tensor import NVFP4Quantizer
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
"Transformer Engine NVFP4 quantization is unavailable. Install a "
|
| 98 |
+
"Transformer Engine build with NVFP4 PyTorch support to use "
|
| 99 |
+
"quantize_bf16_to_nvfp4_128x4."
|
| 100 |
+
) from exc
|
| 101 |
+
return NVFP4Quantizer
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
|
| 105 |
+
if not x.is_cuda:
|
| 106 |
+
raise ValueError("NVFP4 quantization requires a CUDA tensor")
|
| 107 |
+
if x.dtype not in (torch.bfloat16, torch.float16):
|
| 108 |
+
raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
|
| 109 |
+
if x.ndim < 2:
|
| 110 |
+
raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
|
| 111 |
+
if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
rows = 1
|
| 117 |
+
for dim in x.shape[:-1]:
|
| 118 |
+
rows *= int(dim)
|
| 119 |
+
if rows % NVFP4_BLOCK_SIZE != 0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"flattened row dimension must be divisible by "
|
| 122 |
+
f"{NVFP4_BLOCK_SIZE}, got {rows}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
NVFP4Quantizer = _import_te_nvfp4_quantizer()
|
| 126 |
+
quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
|
| 127 |
+
qx = quantizer.quantize(x.contiguous())
|
| 128 |
+
meta = qx.get_metadata()
|
| 129 |
+
|
| 130 |
+
data = meta["rowwise_data"]
|
| 131 |
+
if data.dtype is not torch.uint8:
|
| 132 |
+
data = data.view(torch.uint8)
|
| 133 |
+
logical_scale = meta["rowwise_scale_inv"]
|
| 134 |
+
amax = meta["amax_rowwise"]
|
| 135 |
+
scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
|
| 136 |
+
scale_128x4 = swizzle_nvfp4_scale_to_128x4(
|
| 137 |
+
logical_scale,
|
| 138 |
+
rows=rows,
|
| 139 |
+
cols=scale_cols,
|
| 140 |
+
)
|
| 141 |
+
global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
|
| 142 |
+
|
| 143 |
+
return Nvfp4QuantizedTensor(
|
| 144 |
+
data=data,
|
| 145 |
+
scale_128x4=scale_128x4,
|
| 146 |
+
global_scale=global_scale,
|
| 147 |
+
logical_scale_shape=(rows, scale_cols),
|
| 148 |
+
original_shape=tuple(int(v) for v in x.shape),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def quantize_kv_bf16_to_nvfp4_128x4(
|
| 153 |
+
k: torch.Tensor,
|
| 154 |
+
v: torch.Tensor,
|
| 155 |
+
) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
|
| 156 |
+
return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def dequantize_nvfp4_128x4_to_bf16(
|
| 160 |
+
qx: Nvfp4QuantizedTensor,
|
| 161 |
+
*,
|
| 162 |
+
include_global_scale: bool = True,
|
| 163 |
+
) -> torch.Tensor:
|
| 164 |
+
data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
|
| 165 |
+
if data.shape[-1] * 2 != qx.original_shape[-1]:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"packed data last dimension does not match original shape: "
|
| 168 |
+
f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
rows, scale_cols = qx.logical_scale_shape
|
| 172 |
+
logical_dim = int(qx.original_shape[-1])
|
| 173 |
+
if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
"logical scale columns do not match original last dimension: "
|
| 176 |
+
f"{scale_cols} scale cols vs dim {logical_dim}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
fp4_lut = torch.tensor(
|
| 180 |
+
[
|
| 181 |
+
0.0,
|
| 182 |
+
0.5,
|
| 183 |
+
1.0,
|
| 184 |
+
1.5,
|
| 185 |
+
2.0,
|
| 186 |
+
3.0,
|
| 187 |
+
4.0,
|
| 188 |
+
6.0,
|
| 189 |
+
-0.0,
|
| 190 |
+
-0.5,
|
| 191 |
+
-1.0,
|
| 192 |
+
-1.5,
|
| 193 |
+
-2.0,
|
| 194 |
+
-3.0,
|
| 195 |
+
-4.0,
|
| 196 |
+
-6.0,
|
| 197 |
+
],
|
| 198 |
+
dtype=torch.float32,
|
| 199 |
+
device=data.device,
|
| 200 |
+
)
|
| 201 |
+
packed = data.reshape(rows, logical_dim // 2)
|
| 202 |
+
lo = packed & 0x0F
|
| 203 |
+
hi = packed >> 4
|
| 204 |
+
values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
|
| 205 |
+
values[:, 0::2] = fp4_lut[lo.long()]
|
| 206 |
+
values[:, 1::2] = fp4_lut[hi.long()]
|
| 207 |
+
|
| 208 |
+
row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
|
| 209 |
+
col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
|
| 210 |
+
offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
|
| 211 |
+
scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
|
| 212 |
+
scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
|
| 213 |
+
scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
|
| 214 |
+
out = values * scale
|
| 215 |
+
if include_global_scale:
|
| 216 |
+
out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
|
| 217 |
+
return out.reshape(qx.original_shape).to(torch.bfloat16)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
__all__ = [
|
| 221 |
+
"Nvfp4QuantizedTensor",
|
| 222 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 223 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 224 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 225 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 226 |
+
"nvfp4_global_scale_from_amax",
|
| 227 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/sparse_index_utils.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Sparse-index helpers compatible with MiniMaxAI/msa public names."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
|
| 12 |
+
if cu_seqlens.dtype != torch.int32:
|
| 13 |
+
raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
|
| 14 |
+
if cu_seqlens.ndim != 1:
|
| 15 |
+
raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
|
| 16 |
+
if cu_seqlens.numel() < 1:
|
| 17 |
+
raise ValueError(f"{name} must have at least one element")
|
| 18 |
+
if not cu_seqlens.is_contiguous():
|
| 19 |
+
raise ValueError(f"{name} must be contiguous")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
|
| 23 |
+
seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
|
| 24 |
+
return (seqlens_k + kv_block_size - 1) // kv_block_size
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
|
| 28 |
+
rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
|
| 29 |
+
batch = len(rows_per_batch_cpu)
|
| 30 |
+
max_rows = max(rows_per_batch_cpu, default=0)
|
| 31 |
+
row_dtype = (
|
| 32 |
+
torch.int32
|
| 33 |
+
if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
|
| 34 |
+
else torch.int64
|
| 35 |
+
)
|
| 36 |
+
row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
|
| 37 |
+
row_linear = 0
|
| 38 |
+
for kv_block_idx in range(max_rows):
|
| 39 |
+
for batch_idx, row_count in enumerate(rows_per_batch_cpu):
|
| 40 |
+
if kv_block_idx < row_count:
|
| 41 |
+
row_map_cpu[batch_idx, kv_block_idx] = row_linear
|
| 42 |
+
row_linear += 1
|
| 43 |
+
return row_map_cpu.to(rows_per_batch.device), row_linear
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_k2q_csr_torch_reference(
|
| 47 |
+
q2k_indices: torch.Tensor,
|
| 48 |
+
cu_seqlens_q: torch.Tensor,
|
| 49 |
+
cu_seqlens_k: torch.Tensor,
|
| 50 |
+
kv_block_size: int,
|
| 51 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 52 |
+
if kv_block_size <= 0:
|
| 53 |
+
raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
|
| 54 |
+
if q2k_indices.dtype != torch.int32:
|
| 55 |
+
raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
|
| 56 |
+
if q2k_indices.ndim != 3:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
"q2k_indices must have shape [head_kv, total_q, topK], "
|
| 59 |
+
f"got {tuple(q2k_indices.shape)}"
|
| 60 |
+
)
|
| 61 |
+
_validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
|
| 62 |
+
_validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
|
| 63 |
+
if cu_seqlens_q.shape != cu_seqlens_k.shape:
|
| 64 |
+
raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
|
| 65 |
+
if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
|
| 66 |
+
raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
|
| 67 |
+
|
| 68 |
+
head_kv, total_q, topk = q2k_indices.shape
|
| 69 |
+
if total_q != int(cu_seqlens_q[-1].item()):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
|
| 72 |
+
f"({int(cu_seqlens_q[-1].item())})"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
|
| 76 |
+
row_map, total_rows = _build_packed_row_map(rows_per_batch)
|
| 77 |
+
nnz_upper_bound = total_q * topk
|
| 78 |
+
|
| 79 |
+
k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
|
| 80 |
+
k2q_q_indices = torch.full(
|
| 81 |
+
(head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
|
| 82 |
+
)
|
| 83 |
+
if total_rows == 0 or total_q == 0 or topk == 0:
|
| 84 |
+
return k2q_row_ptr, k2q_q_indices
|
| 85 |
+
|
| 86 |
+
counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
|
| 87 |
+
total_entries = total_q * topk
|
| 88 |
+
row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
|
| 89 |
+
row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
|
| 90 |
+
q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
|
| 91 |
+
valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
|
| 92 |
+
|
| 93 |
+
for batch_idx in range(cu_seqlens_q.numel() - 1):
|
| 94 |
+
q_start = int(cu_seqlens_q[batch_idx].item())
|
| 95 |
+
q_end = int(cu_seqlens_q[batch_idx + 1].item())
|
| 96 |
+
if q_end <= q_start:
|
| 97 |
+
continue
|
| 98 |
+
q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
|
| 99 |
+
q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
|
| 100 |
+
kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
|
| 101 |
+
valid = kv_local >= 0
|
| 102 |
+
safe_kv = kv_local.clamp_min(0).long()
|
| 103 |
+
rows = row_map[batch_idx, safe_kv].to(row_dtype)
|
| 104 |
+
start = q_start * topk
|
| 105 |
+
end = q_end * topk
|
| 106 |
+
row_all[:, start:end] = rows
|
| 107 |
+
q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
|
| 108 |
+
valid_all[:, start:end] = valid & (rows >= 0)
|
| 109 |
+
counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
|
| 110 |
+
|
| 111 |
+
k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
|
| 112 |
+
offsets = torch.zeros_like(counts)
|
| 113 |
+
offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
|
| 114 |
+
cursor = torch.zeros_like(counts)
|
| 115 |
+
for h in range(head_kv):
|
| 116 |
+
for entry in range(total_entries):
|
| 117 |
+
if not bool(valid_all[h, entry]):
|
| 118 |
+
continue
|
| 119 |
+
row = int(row_all[h, entry].item())
|
| 120 |
+
pos = int(offsets[h, row].item() + cursor[h, row].item())
|
| 121 |
+
k2q_q_indices[h, pos] = q_all[h, entry]
|
| 122 |
+
cursor[h, row] += 1
|
| 123 |
+
|
| 124 |
+
return k2q_row_ptr, k2q_q_indices
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def build_k2q_csr(
|
| 128 |
+
q2k_indices: torch.Tensor,
|
| 129 |
+
cu_seqlens_q: torch.Tensor,
|
| 130 |
+
cu_seqlens_k: torch.Tensor,
|
| 131 |
+
kv_block_size: int,
|
| 132 |
+
*,
|
| 133 |
+
total_k: Optional[int] = None,
|
| 134 |
+
max_seqlen_k: Optional[int] = None,
|
| 135 |
+
max_seqlen_q: Optional[int] = None,
|
| 136 |
+
total_rows: Optional[int] = None,
|
| 137 |
+
qhead_per_kv: int = 1,
|
| 138 |
+
return_schedule: bool = False,
|
| 139 |
+
):
|
| 140 |
+
if return_schedule:
|
| 141 |
+
raise NotImplementedError(
|
| 142 |
+
"Blackwell build_k2q_csr currently returns CSR tensors only; "
|
| 143 |
+
"SM100 schedule objects are not part of this package."
|
| 144 |
+
)
|
| 145 |
+
if total_k is None:
|
| 146 |
+
raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
|
| 147 |
+
return build_k2q_csr_torch_reference(
|
| 148 |
+
q2k_indices.contiguous(),
|
| 149 |
+
cu_seqlens_q.contiguous(),
|
| 150 |
+
cu_seqlens_k.contiguous(),
|
| 151 |
+
int(kv_block_size),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SparseK2qCsrBuilderSm100:
|
| 156 |
+
"""Compatibility placeholder for the upstream SM100 class name.
|
| 157 |
+
|
| 158 |
+
The name is exported so code can feature-detect it, but schedule-producing
|
| 159 |
+
SM100 behavior is intentionally not claimed for this Blackwell package.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(self, *args, **kwargs):
|
| 163 |
+
self.args = args
|
| 164 |
+
self.kwargs = kwargs
|
| 165 |
+
|
| 166 |
+
def build(self, *args, **kwargs):
|
| 167 |
+
return build_k2q_csr(*args, **kwargs)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
__all__ = [
|
| 171 |
+
"build_k2q_csr",
|
| 172 |
+
"build_k2q_csr_torch_reference",
|
| 173 |
+
"SparseK2qCsrBuilderSm100",
|
| 174 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
CHANGED
|
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
from ._native import has_native_ops, native_topk_from_scores
|
| 41 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 42 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
__all__ = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"flash_decode_with_topk_idx",
|
| 46 |
"flash_decode_with_gqa_share_sparse",
|
| 47 |
"has_native_ops",
|
|
@@ -50,4 +88,11 @@ __all__ = [
|
|
| 50 |
"naive_flash_decode_with_topk_idx",
|
| 51 |
"get_cu_seqblocks",
|
| 52 |
"robust_allocator",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
]
|
|
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 40 |
+
from .interface import (
|
| 41 |
+
SparseDecodePagedAttentionWrapper,
|
| 42 |
+
fp4_indexer_block_scores,
|
| 43 |
+
sparse_atten_func,
|
| 44 |
+
sparse_atten_nvfp4_kv_func,
|
| 45 |
+
sparse_decode_atten_func,
|
| 46 |
+
)
|
| 47 |
from ._native import has_native_ops, native_topk_from_scores
|
| 48 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 49 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 50 |
+
from .quantize import (
|
| 51 |
+
Nvfp4QuantizedTensor,
|
| 52 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 53 |
+
nvfp4_global_scale_from_amax,
|
| 54 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 55 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 56 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 57 |
+
)
|
| 58 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 59 |
+
from .api_status import (
|
| 60 |
+
OFFICIAL_API_STATUS,
|
| 61 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 62 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 63 |
+
available_functions,
|
| 64 |
+
official_api_status,
|
| 65 |
+
official_minimax_msa_functions,
|
| 66 |
+
unsupported_official_functions,
|
| 67 |
+
)
|
| 68 |
|
| 69 |
__all__ = [
|
| 70 |
+
"sparse_atten_func",
|
| 71 |
+
"sparse_atten_nvfp4_kv_func",
|
| 72 |
+
"sparse_decode_atten_func",
|
| 73 |
+
"SparseDecodePagedAttentionWrapper",
|
| 74 |
+
"fp4_indexer_block_scores",
|
| 75 |
+
"build_k2q_csr",
|
| 76 |
+
"SparseK2qCsrBuilderSm100",
|
| 77 |
+
"Nvfp4QuantizedTensor",
|
| 78 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 79 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 80 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 81 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 82 |
+
"nvfp4_global_scale_from_amax",
|
| 83 |
"flash_decode_with_topk_idx",
|
| 84 |
"flash_decode_with_gqa_share_sparse",
|
| 85 |
"has_native_ops",
|
|
|
|
| 88 |
"naive_flash_decode_with_topk_idx",
|
| 89 |
"get_cu_seqblocks",
|
| 90 |
"robust_allocator",
|
| 91 |
+
"OFFICIAL_API_STATUS",
|
| 92 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 93 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 94 |
+
"available_functions",
|
| 95 |
+
"official_api_status",
|
| 96 |
+
"official_minimax_msa_functions",
|
| 97 |
+
"unsupported_official_functions",
|
| 98 |
]
|
build/torch212-cxx11-cu130-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 151920
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:de9c51b30c5d31511676be36604ab66dee1fdf8ccac21ea8a32cc7ba321bb2e2
|
| 3 |
size 151920
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _minimaxai_msa_blackwell_cuda_3dca94b
|
| 3 |
+
ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/api_status.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Public API status for the FlashRT MiniMax MSA Blackwell package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from types import MappingProxyType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
V1_AVAILABLE_FUNCTIONS = (
|
| 10 |
+
"sparse_atten_func",
|
| 11 |
+
"sparse_atten_nvfp4_kv_func",
|
| 12 |
+
"sparse_decode_atten_func",
|
| 13 |
+
"SparseDecodePagedAttentionWrapper",
|
| 14 |
+
"fp4_indexer_block_scores",
|
| 15 |
+
"build_k2q_csr",
|
| 16 |
+
"SparseK2qCsrBuilderSm100",
|
| 17 |
+
"Nvfp4QuantizedTensor",
|
| 18 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 19 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 20 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 21 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 22 |
+
"nvfp4_global_scale_from_amax",
|
| 23 |
+
"has_native_ops",
|
| 24 |
+
"native_topk_from_scores",
|
| 25 |
+
"flash_decode_with_topk_idx",
|
| 26 |
+
"flash_decode_with_gqa_share_sparse",
|
| 27 |
+
"naive_flash_decode_with_topk_idx",
|
| 28 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 29 |
+
"get_cu_seqblocks",
|
| 30 |
+
"robust_allocator",
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
|
| 34 |
+
"sparse_atten_func",
|
| 35 |
+
"sparse_atten_nvfp4_kv_func",
|
| 36 |
+
"sparse_decode_atten_func",
|
| 37 |
+
"SparseDecodePagedAttentionWrapper",
|
| 38 |
+
"fp4_indexer_block_scores",
|
| 39 |
+
"build_k2q_csr",
|
| 40 |
+
"SparseK2qCsrBuilderSm100",
|
| 41 |
+
"Nvfp4QuantizedTensor",
|
| 42 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 43 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 44 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 45 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 46 |
+
"nvfp4_global_scale_from_amax",
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
_OFFICIAL_API_STATUS = {
|
| 50 |
+
"sparse_atten_func": {
|
| 51 |
+
"status": "unsupported_callable",
|
| 52 |
+
"target": "blackwell-prefill",
|
| 53 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 54 |
+
},
|
| 55 |
+
"sparse_atten_nvfp4_kv_func": {
|
| 56 |
+
"status": "unsupported_callable",
|
| 57 |
+
"target": "blackwell-prefill-nvfp4",
|
| 58 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 59 |
+
},
|
| 60 |
+
"sparse_decode_atten_func": {
|
| 61 |
+
"status": "available",
|
| 62 |
+
"target": "v1",
|
| 63 |
+
"reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
|
| 64 |
+
},
|
| 65 |
+
"SparseDecodePagedAttentionWrapper": {
|
| 66 |
+
"status": "available",
|
| 67 |
+
"target": "v1",
|
| 68 |
+
"reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
|
| 69 |
+
},
|
| 70 |
+
"fp4_indexer_block_scores": {
|
| 71 |
+
"status": "unsupported_callable",
|
| 72 |
+
"target": "blackwell-fp4-indexer",
|
| 73 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
|
| 74 |
+
},
|
| 75 |
+
"build_k2q_csr": {
|
| 76 |
+
"status": "available",
|
| 77 |
+
"target": "v1",
|
| 78 |
+
"reason": "Torch CSR construction fallback is available for compatibility.",
|
| 79 |
+
},
|
| 80 |
+
"SparseK2qCsrBuilderSm100": {
|
| 81 |
+
"status": "available",
|
| 82 |
+
"target": "v1",
|
| 83 |
+
"reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
|
| 84 |
+
},
|
| 85 |
+
"Nvfp4QuantizedTensor": {
|
| 86 |
+
"status": "available",
|
| 87 |
+
"target": "v1",
|
| 88 |
+
"reason": "NVFP4 metadata dataclass is available.",
|
| 89 |
+
},
|
| 90 |
+
"quantize_bf16_to_nvfp4_128x4": {
|
| 91 |
+
"status": "available_optional_te",
|
| 92 |
+
"target": "v1",
|
| 93 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 94 |
+
},
|
| 95 |
+
"quantize_kv_bf16_to_nvfp4_128x4": {
|
| 96 |
+
"status": "available_optional_te",
|
| 97 |
+
"target": "v1",
|
| 98 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 99 |
+
},
|
| 100 |
+
"dequantize_nvfp4_128x4_to_bf16": {
|
| 101 |
+
"status": "available",
|
| 102 |
+
"target": "v1",
|
| 103 |
+
"reason": "Reference dequant helper is available.",
|
| 104 |
+
},
|
| 105 |
+
"swizzle_nvfp4_scale_to_128x4": {
|
| 106 |
+
"status": "available",
|
| 107 |
+
"target": "v1",
|
| 108 |
+
"reason": "Scale swizzle helper is available.",
|
| 109 |
+
},
|
| 110 |
+
"nvfp4_global_scale_from_amax": {
|
| 111 |
+
"status": "available",
|
| 112 |
+
"target": "v1",
|
| 113 |
+
"reason": "Global scale helper is available.",
|
| 114 |
+
},
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def available_functions() -> tuple[str, ...]:
|
| 121 |
+
"""Return the functions/classes intentionally available in v1."""
|
| 122 |
+
|
| 123 |
+
return V1_AVAILABLE_FUNCTIONS
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def official_minimax_msa_functions() -> tuple[str, ...]:
|
| 127 |
+
"""Return the upstream MiniMaxAI/msa public function/class names tracked here."""
|
| 128 |
+
|
| 129 |
+
return OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def official_api_status() -> dict[str, dict[str, str]]:
|
| 133 |
+
"""Return a copy of the official API compatibility status table."""
|
| 134 |
+
|
| 135 |
+
return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def unsupported_official_functions() -> tuple[str, ...]:
|
| 139 |
+
"""Return official MiniMaxAI/msa names that are not exported by this v1 package."""
|
| 140 |
+
|
| 141 |
+
return tuple(
|
| 142 |
+
name
|
| 143 |
+
for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 144 |
+
if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
|
| 145 |
+
)
|
build/torch212-cxx11-cu130-x86_64-linux/interface.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _unsupported(name: str, reason: str):
|
| 14 |
+
raise NotImplementedError(
|
| 15 |
+
f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _page_table_to_req_to_token(
|
| 20 |
+
page_table: torch.Tensor,
|
| 21 |
+
seqused_k: torch.Tensor,
|
| 22 |
+
max_seqlen_k: int,
|
| 23 |
+
blk_kv: int,
|
| 24 |
+
) -> torch.Tensor:
|
| 25 |
+
batch = int(page_table.shape[0])
|
| 26 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 27 |
+
blk_kv = int(blk_kv)
|
| 28 |
+
logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
|
| 29 |
+
page = logical_pos // blk_kv
|
| 30 |
+
offset = logical_pos % blk_kv
|
| 31 |
+
page = page.clamp(max=int(page_table.shape[1]) - 1)
|
| 32 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 33 |
+
physical_token = physical_page * blk_kv + offset[None, :]
|
| 34 |
+
valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
|
| 35 |
+
return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _dense_q2k_indices(
|
| 39 |
+
*,
|
| 40 |
+
head_kv: int,
|
| 41 |
+
batch: int,
|
| 42 |
+
topk: int,
|
| 43 |
+
seq_lens: torch.Tensor,
|
| 44 |
+
blk_kv: int,
|
| 45 |
+
) -> torch.Tensor:
|
| 46 |
+
idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
|
| 47 |
+
blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
|
| 48 |
+
valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
|
| 49 |
+
for b in range(batch):
|
| 50 |
+
valid = blocks < valid_blocks[b]
|
| 51 |
+
idx[:, b, valid] = blocks[valid]
|
| 52 |
+
return idx
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _normalize_decode_q2k(
|
| 56 |
+
q2k_indices: Optional[torch.Tensor],
|
| 57 |
+
*,
|
| 58 |
+
batch: int,
|
| 59 |
+
seqlen_q: int,
|
| 60 |
+
head_kv: int,
|
| 61 |
+
seqused_k: torch.Tensor,
|
| 62 |
+
blk_kv: int,
|
| 63 |
+
) -> torch.Tensor:
|
| 64 |
+
if q2k_indices is None:
|
| 65 |
+
topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
|
| 66 |
+
return _dense_q2k_indices(
|
| 67 |
+
head_kv=head_kv,
|
| 68 |
+
batch=batch,
|
| 69 |
+
topk=topk,
|
| 70 |
+
seq_lens=seqused_k,
|
| 71 |
+
blk_kv=blk_kv,
|
| 72 |
+
)
|
| 73 |
+
if q2k_indices.dtype != torch.int32:
|
| 74 |
+
raise TypeError("q2k_indices must be torch.int32")
|
| 75 |
+
if q2k_indices.ndim != 3:
|
| 76 |
+
raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
|
| 77 |
+
if q2k_indices.shape[0] != head_kv:
|
| 78 |
+
raise ValueError("q2k_indices first dimension must match Hkv")
|
| 79 |
+
if int(q2k_indices.shape[1]) != batch * seqlen_q:
|
| 80 |
+
raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
|
| 81 |
+
if seqlen_q != 1:
|
| 82 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 83 |
+
return q2k_indices[:, ::seqlen_q, :].contiguous()
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def sparse_decode_atten_func(
|
| 87 |
+
q: torch.Tensor,
|
| 88 |
+
k: torch.Tensor,
|
| 89 |
+
v: torch.Tensor,
|
| 90 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 91 |
+
*,
|
| 92 |
+
page_table: torch.Tensor,
|
| 93 |
+
seqused_k: torch.Tensor,
|
| 94 |
+
seqlen_q: int,
|
| 95 |
+
max_seqlen_k: int,
|
| 96 |
+
blk_kv: int = 128,
|
| 97 |
+
causal: bool = True,
|
| 98 |
+
softmax_scale: Optional[float] = None,
|
| 99 |
+
return_softmax_lse: bool = False,
|
| 100 |
+
schedule: Optional[object] = None,
|
| 101 |
+
O_partial: Optional[torch.Tensor] = None,
|
| 102 |
+
LSE_partial: Optional[torch.Tensor] = None,
|
| 103 |
+
):
|
| 104 |
+
"""Run MiniMax paged sparse decode on Blackwell.
|
| 105 |
+
|
| 106 |
+
This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
|
| 107 |
+
but accepts the Blackwell package's validated BF16/FP16 decode path. It
|
| 108 |
+
supports single-token decode (`seqlen_q=1`) with paged K/V
|
| 109 |
+
`[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
|
| 110 |
+
"""
|
| 111 |
+
|
| 112 |
+
if schedule is not None or O_partial is not None or LSE_partial is not None:
|
| 113 |
+
raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
|
| 114 |
+
if return_softmax_lse:
|
| 115 |
+
raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
|
| 116 |
+
if not causal:
|
| 117 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
|
| 118 |
+
if int(seqlen_q) != 1:
|
| 119 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 120 |
+
if q.ndim != 3:
|
| 121 |
+
raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
|
| 122 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 123 |
+
raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 124 |
+
if k.shape != v.shape:
|
| 125 |
+
raise ValueError("k and v must have the same shape")
|
| 126 |
+
if q.device != k.device or q.device != v.device or q.device != page_table.device:
|
| 127 |
+
raise ValueError("q, k, v, and page_table must be on the same device")
|
| 128 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 129 |
+
raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
|
| 130 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 131 |
+
raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
|
| 132 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128:
|
| 133 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 134 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 135 |
+
raise ValueError("k.shape[2] must equal blk_kv")
|
| 136 |
+
if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
|
| 137 |
+
raise TypeError("page_table and seqused_k must be torch.int32")
|
| 138 |
+
if page_table.ndim != 2:
|
| 139 |
+
raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
|
| 140 |
+
batch = int(page_table.shape[0])
|
| 141 |
+
if int(q.shape[0]) != batch:
|
| 142 |
+
raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
|
| 143 |
+
|
| 144 |
+
head_kv = int(k.shape[1])
|
| 145 |
+
q2k = _normalize_decode_q2k(
|
| 146 |
+
q2k_indices,
|
| 147 |
+
batch=batch,
|
| 148 |
+
seqlen_q=int(seqlen_q),
|
| 149 |
+
head_kv=head_kv,
|
| 150 |
+
seqused_k=seqused_k,
|
| 151 |
+
blk_kv=int(blk_kv),
|
| 152 |
+
)
|
| 153 |
+
req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
|
| 154 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 155 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 156 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 157 |
+
return flash_decode_with_gqa_share_sparse(
|
| 158 |
+
q.contiguous(),
|
| 159 |
+
None,
|
| 160 |
+
k_cache,
|
| 161 |
+
v_cache,
|
| 162 |
+
req_to_token,
|
| 163 |
+
seqused_k.contiguous(),
|
| 164 |
+
slot_ids,
|
| 165 |
+
int(blk_kv),
|
| 166 |
+
q2k,
|
| 167 |
+
sm_scale=softmax_scale,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class SparseDecodePagedAttentionWrapper:
|
| 172 |
+
def __init__(self, *, blk_kv: int = 128, causal: bool = True):
|
| 173 |
+
self.blk_kv = int(blk_kv)
|
| 174 |
+
self.causal = bool(causal)
|
| 175 |
+
self.page_table: Optional[torch.Tensor] = None
|
| 176 |
+
self.seqused_k: Optional[torch.Tensor] = None
|
| 177 |
+
self.seqlen_q: Optional[int] = None
|
| 178 |
+
self.max_seqlen_k: Optional[int] = None
|
| 179 |
+
self.q2k_indices: Optional[torch.Tensor] = None
|
| 180 |
+
|
| 181 |
+
def plan(
|
| 182 |
+
self,
|
| 183 |
+
*,
|
| 184 |
+
page_table: torch.Tensor,
|
| 185 |
+
seqused_k: torch.Tensor,
|
| 186 |
+
seqlen_q: int,
|
| 187 |
+
max_seqlen_k: int,
|
| 188 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 189 |
+
num_qo_heads: Optional[int] = None,
|
| 190 |
+
num_kv_heads: Optional[int] = None,
|
| 191 |
+
head_dim: Optional[int] = 128,
|
| 192 |
+
enable_cuda_graph: bool = False,
|
| 193 |
+
max_grid_size: Optional[int] = None,
|
| 194 |
+
fixed_split_size: Optional[int] = None,
|
| 195 |
+
disable_split_kv: bool = False,
|
| 196 |
+
) -> "SparseDecodePagedAttentionWrapper":
|
| 197 |
+
if head_dim is not None and int(head_dim) != 128:
|
| 198 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 199 |
+
self.page_table = page_table.contiguous()
|
| 200 |
+
self.seqused_k = seqused_k.contiguous()
|
| 201 |
+
self.seqlen_q = int(seqlen_q)
|
| 202 |
+
self.max_seqlen_k = int(max_seqlen_k)
|
| 203 |
+
self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
|
| 204 |
+
return self
|
| 205 |
+
|
| 206 |
+
def run(
|
| 207 |
+
self,
|
| 208 |
+
q: torch.Tensor,
|
| 209 |
+
k: torch.Tensor,
|
| 210 |
+
v: torch.Tensor,
|
| 211 |
+
*,
|
| 212 |
+
softmax_scale: Optional[float] = None,
|
| 213 |
+
return_softmax_lse: bool = False,
|
| 214 |
+
out: Optional[torch.Tensor] = None,
|
| 215 |
+
lse: Optional[torch.Tensor] = None,
|
| 216 |
+
):
|
| 217 |
+
if self.page_table is None or self.seqused_k is None:
|
| 218 |
+
raise RuntimeError("decode wrapper must be planned before run")
|
| 219 |
+
result = sparse_decode_atten_func(
|
| 220 |
+
q,
|
| 221 |
+
k,
|
| 222 |
+
v,
|
| 223 |
+
self.q2k_indices,
|
| 224 |
+
page_table=self.page_table,
|
| 225 |
+
seqused_k=self.seqused_k,
|
| 226 |
+
seqlen_q=self.seqlen_q,
|
| 227 |
+
max_seqlen_k=self.max_seqlen_k,
|
| 228 |
+
blk_kv=self.blk_kv,
|
| 229 |
+
causal=self.causal,
|
| 230 |
+
softmax_scale=softmax_scale,
|
| 231 |
+
return_softmax_lse=return_softmax_lse,
|
| 232 |
+
)
|
| 233 |
+
if out is not None:
|
| 234 |
+
out.copy_(result)
|
| 235 |
+
return out
|
| 236 |
+
return result
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def sparse_atten_func(*args, **kwargs):
|
| 240 |
+
_unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def sparse_atten_nvfp4_kv_func(*args, **kwargs):
|
| 244 |
+
_unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def fp4_indexer_block_scores(*args, **kwargs):
|
| 248 |
+
_unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
__all__ = [
|
| 252 |
+
"sparse_atten_func",
|
| 253 |
+
"sparse_atten_nvfp4_kv_func",
|
| 254 |
+
"sparse_decode_atten_func",
|
| 255 |
+
"SparseDecodePagedAttentionWrapper",
|
| 256 |
+
"fp4_indexer_block_scores",
|
| 257 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,20 +14,24 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
-
"
|
| 20 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 21 |
-
"_ops.py": "
|
|
|
|
| 22 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 23 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 24 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 25 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 26 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
|
|
|
| 27 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 28 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 29 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 30 |
-
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
|
|
|
|
|
|
|
| 31 |
}
|
| 32 |
}
|
| 33 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
+
"id": "_minimaxai_msa_blackwell_cuda_3dca94b",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
|
| 18 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
+
"_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "3pxRswxdMVEWdr42YEq2be4f34zKwh6ooyzHujIbsuI=",
|
| 20 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 21 |
+
"_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
|
| 22 |
+
"api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
|
| 23 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 24 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 25 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 26 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 27 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
| 28 |
+
"interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
|
| 29 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 30 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 31 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 32 |
+
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
|
| 33 |
+
"quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
|
| 34 |
+
"sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
|
| 35 |
}
|
| 36 |
}
|
| 37 |
}
|
build/torch212-cxx11-cu130-x86_64-linux/quantize.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""NVFP4 helper API compatible with MiniMaxAI/msa.
|
| 3 |
+
|
| 4 |
+
The quantization entrypoints use Transformer Engine when it is installed. The
|
| 5 |
+
layout helpers and reference dequantizer are pure PyTorch and are included so
|
| 6 |
+
callers can prepare and validate NVFP4 metadata with the same public API names
|
| 7 |
+
as the upstream MiniMaxAI package.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Tuple
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
NVFP4_BLOCK_SIZE = 16
|
| 19 |
+
NVFP4_FP4_MAX = 6.0
|
| 20 |
+
NVFP4_FP8_E4M3_MAX = 448.0
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class Nvfp4QuantizedTensor:
|
| 25 |
+
data: torch.Tensor
|
| 26 |
+
scale_128x4: torch.Tensor
|
| 27 |
+
global_scale: torch.Tensor
|
| 28 |
+
logical_scale_shape: Tuple[int, int]
|
| 29 |
+
original_shape: Tuple[int, ...]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _round_up(x: int, multiple: int) -> int:
|
| 33 |
+
return ((int(x) + multiple - 1) // multiple) * multiple
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def nvfp4_scale_128x4_offset(
|
| 37 |
+
row: torch.Tensor,
|
| 38 |
+
col: torch.Tensor,
|
| 39 |
+
scale_cols: int,
|
| 40 |
+
) -> torch.Tensor:
|
| 41 |
+
tiles_n = _round_up(scale_cols, 4) // 4
|
| 42 |
+
tile_m = row // 128
|
| 43 |
+
tile_n = col // 4
|
| 44 |
+
outer = row % 128
|
| 45 |
+
inner = col % 4
|
| 46 |
+
return (
|
| 47 |
+
(tile_m * tiles_n + tile_n) * 512
|
| 48 |
+
+ (outer % 32) * 16
|
| 49 |
+
+ (outer // 32) * 4
|
| 50 |
+
+ inner
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def swizzle_nvfp4_scale_to_128x4(
|
| 55 |
+
scale: torch.Tensor,
|
| 56 |
+
*,
|
| 57 |
+
rows: int,
|
| 58 |
+
cols: int,
|
| 59 |
+
) -> torch.Tensor:
|
| 60 |
+
if scale.ndim != 2:
|
| 61 |
+
raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
|
| 62 |
+
|
| 63 |
+
rows = int(rows)
|
| 64 |
+
cols = int(cols)
|
| 65 |
+
padded_rows = _round_up(rows, 128)
|
| 66 |
+
padded_cols = _round_up(cols, 4)
|
| 67 |
+
if scale.shape[0] < rows or scale.shape[1] < cols:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
"scale is smaller than the requested logical shape: "
|
| 70 |
+
f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
logical = scale[:rows, :cols].contiguous()
|
| 74 |
+
if logical.shape != (padded_rows, padded_cols):
|
| 75 |
+
logical = torch.nn.functional.pad(
|
| 76 |
+
logical.to(torch.float32),
|
| 77 |
+
(0, padded_cols - cols, 0, padded_rows - rows),
|
| 78 |
+
).to(scale.dtype)
|
| 79 |
+
swizzled = torch.empty_like(logical)
|
| 80 |
+
|
| 81 |
+
row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
|
| 82 |
+
col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
|
| 83 |
+
offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
|
| 84 |
+
swizzled.reshape(-1)[offset] = logical.reshape(-1)
|
| 85 |
+
return swizzled
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _import_te_nvfp4_quantizer():
|
| 93 |
+
try:
|
| 94 |
+
from transformer_engine.pytorch.tensor import NVFP4Quantizer
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
"Transformer Engine NVFP4 quantization is unavailable. Install a "
|
| 98 |
+
"Transformer Engine build with NVFP4 PyTorch support to use "
|
| 99 |
+
"quantize_bf16_to_nvfp4_128x4."
|
| 100 |
+
) from exc
|
| 101 |
+
return NVFP4Quantizer
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
|
| 105 |
+
if not x.is_cuda:
|
| 106 |
+
raise ValueError("NVFP4 quantization requires a CUDA tensor")
|
| 107 |
+
if x.dtype not in (torch.bfloat16, torch.float16):
|
| 108 |
+
raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
|
| 109 |
+
if x.ndim < 2:
|
| 110 |
+
raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
|
| 111 |
+
if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
rows = 1
|
| 117 |
+
for dim in x.shape[:-1]:
|
| 118 |
+
rows *= int(dim)
|
| 119 |
+
if rows % NVFP4_BLOCK_SIZE != 0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"flattened row dimension must be divisible by "
|
| 122 |
+
f"{NVFP4_BLOCK_SIZE}, got {rows}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
NVFP4Quantizer = _import_te_nvfp4_quantizer()
|
| 126 |
+
quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
|
| 127 |
+
qx = quantizer.quantize(x.contiguous())
|
| 128 |
+
meta = qx.get_metadata()
|
| 129 |
+
|
| 130 |
+
data = meta["rowwise_data"]
|
| 131 |
+
if data.dtype is not torch.uint8:
|
| 132 |
+
data = data.view(torch.uint8)
|
| 133 |
+
logical_scale = meta["rowwise_scale_inv"]
|
| 134 |
+
amax = meta["amax_rowwise"]
|
| 135 |
+
scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
|
| 136 |
+
scale_128x4 = swizzle_nvfp4_scale_to_128x4(
|
| 137 |
+
logical_scale,
|
| 138 |
+
rows=rows,
|
| 139 |
+
cols=scale_cols,
|
| 140 |
+
)
|
| 141 |
+
global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
|
| 142 |
+
|
| 143 |
+
return Nvfp4QuantizedTensor(
|
| 144 |
+
data=data,
|
| 145 |
+
scale_128x4=scale_128x4,
|
| 146 |
+
global_scale=global_scale,
|
| 147 |
+
logical_scale_shape=(rows, scale_cols),
|
| 148 |
+
original_shape=tuple(int(v) for v in x.shape),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def quantize_kv_bf16_to_nvfp4_128x4(
|
| 153 |
+
k: torch.Tensor,
|
| 154 |
+
v: torch.Tensor,
|
| 155 |
+
) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
|
| 156 |
+
return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def dequantize_nvfp4_128x4_to_bf16(
|
| 160 |
+
qx: Nvfp4QuantizedTensor,
|
| 161 |
+
*,
|
| 162 |
+
include_global_scale: bool = True,
|
| 163 |
+
) -> torch.Tensor:
|
| 164 |
+
data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
|
| 165 |
+
if data.shape[-1] * 2 != qx.original_shape[-1]:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"packed data last dimension does not match original shape: "
|
| 168 |
+
f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
rows, scale_cols = qx.logical_scale_shape
|
| 172 |
+
logical_dim = int(qx.original_shape[-1])
|
| 173 |
+
if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
"logical scale columns do not match original last dimension: "
|
| 176 |
+
f"{scale_cols} scale cols vs dim {logical_dim}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
fp4_lut = torch.tensor(
|
| 180 |
+
[
|
| 181 |
+
0.0,
|
| 182 |
+
0.5,
|
| 183 |
+
1.0,
|
| 184 |
+
1.5,
|
| 185 |
+
2.0,
|
| 186 |
+
3.0,
|
| 187 |
+
4.0,
|
| 188 |
+
6.0,
|
| 189 |
+
-0.0,
|
| 190 |
+
-0.5,
|
| 191 |
+
-1.0,
|
| 192 |
+
-1.5,
|
| 193 |
+
-2.0,
|
| 194 |
+
-3.0,
|
| 195 |
+
-4.0,
|
| 196 |
+
-6.0,
|
| 197 |
+
],
|
| 198 |
+
dtype=torch.float32,
|
| 199 |
+
device=data.device,
|
| 200 |
+
)
|
| 201 |
+
packed = data.reshape(rows, logical_dim // 2)
|
| 202 |
+
lo = packed & 0x0F
|
| 203 |
+
hi = packed >> 4
|
| 204 |
+
values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
|
| 205 |
+
values[:, 0::2] = fp4_lut[lo.long()]
|
| 206 |
+
values[:, 1::2] = fp4_lut[hi.long()]
|
| 207 |
+
|
| 208 |
+
row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
|
| 209 |
+
col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
|
| 210 |
+
offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
|
| 211 |
+
scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
|
| 212 |
+
scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
|
| 213 |
+
scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
|
| 214 |
+
out = values * scale
|
| 215 |
+
if include_global_scale:
|
| 216 |
+
out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
|
| 217 |
+
return out.reshape(qx.original_shape).to(torch.bfloat16)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
__all__ = [
|
| 221 |
+
"Nvfp4QuantizedTensor",
|
| 222 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 223 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 224 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 225 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 226 |
+
"nvfp4_global_scale_from_amax",
|
| 227 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/sparse_index_utils.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Sparse-index helpers compatible with MiniMaxAI/msa public names."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
|
| 12 |
+
if cu_seqlens.dtype != torch.int32:
|
| 13 |
+
raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
|
| 14 |
+
if cu_seqlens.ndim != 1:
|
| 15 |
+
raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
|
| 16 |
+
if cu_seqlens.numel() < 1:
|
| 17 |
+
raise ValueError(f"{name} must have at least one element")
|
| 18 |
+
if not cu_seqlens.is_contiguous():
|
| 19 |
+
raise ValueError(f"{name} must be contiguous")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
|
| 23 |
+
seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
|
| 24 |
+
return (seqlens_k + kv_block_size - 1) // kv_block_size
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
|
| 28 |
+
rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
|
| 29 |
+
batch = len(rows_per_batch_cpu)
|
| 30 |
+
max_rows = max(rows_per_batch_cpu, default=0)
|
| 31 |
+
row_dtype = (
|
| 32 |
+
torch.int32
|
| 33 |
+
if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
|
| 34 |
+
else torch.int64
|
| 35 |
+
)
|
| 36 |
+
row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
|
| 37 |
+
row_linear = 0
|
| 38 |
+
for kv_block_idx in range(max_rows):
|
| 39 |
+
for batch_idx, row_count in enumerate(rows_per_batch_cpu):
|
| 40 |
+
if kv_block_idx < row_count:
|
| 41 |
+
row_map_cpu[batch_idx, kv_block_idx] = row_linear
|
| 42 |
+
row_linear += 1
|
| 43 |
+
return row_map_cpu.to(rows_per_batch.device), row_linear
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_k2q_csr_torch_reference(
|
| 47 |
+
q2k_indices: torch.Tensor,
|
| 48 |
+
cu_seqlens_q: torch.Tensor,
|
| 49 |
+
cu_seqlens_k: torch.Tensor,
|
| 50 |
+
kv_block_size: int,
|
| 51 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 52 |
+
if kv_block_size <= 0:
|
| 53 |
+
raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
|
| 54 |
+
if q2k_indices.dtype != torch.int32:
|
| 55 |
+
raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
|
| 56 |
+
if q2k_indices.ndim != 3:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
"q2k_indices must have shape [head_kv, total_q, topK], "
|
| 59 |
+
f"got {tuple(q2k_indices.shape)}"
|
| 60 |
+
)
|
| 61 |
+
_validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
|
| 62 |
+
_validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
|
| 63 |
+
if cu_seqlens_q.shape != cu_seqlens_k.shape:
|
| 64 |
+
raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
|
| 65 |
+
if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
|
| 66 |
+
raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
|
| 67 |
+
|
| 68 |
+
head_kv, total_q, topk = q2k_indices.shape
|
| 69 |
+
if total_q != int(cu_seqlens_q[-1].item()):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
|
| 72 |
+
f"({int(cu_seqlens_q[-1].item())})"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
|
| 76 |
+
row_map, total_rows = _build_packed_row_map(rows_per_batch)
|
| 77 |
+
nnz_upper_bound = total_q * topk
|
| 78 |
+
|
| 79 |
+
k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
|
| 80 |
+
k2q_q_indices = torch.full(
|
| 81 |
+
(head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
|
| 82 |
+
)
|
| 83 |
+
if total_rows == 0 or total_q == 0 or topk == 0:
|
| 84 |
+
return k2q_row_ptr, k2q_q_indices
|
| 85 |
+
|
| 86 |
+
counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
|
| 87 |
+
total_entries = total_q * topk
|
| 88 |
+
row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
|
| 89 |
+
row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
|
| 90 |
+
q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
|
| 91 |
+
valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
|
| 92 |
+
|
| 93 |
+
for batch_idx in range(cu_seqlens_q.numel() - 1):
|
| 94 |
+
q_start = int(cu_seqlens_q[batch_idx].item())
|
| 95 |
+
q_end = int(cu_seqlens_q[batch_idx + 1].item())
|
| 96 |
+
if q_end <= q_start:
|
| 97 |
+
continue
|
| 98 |
+
q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
|
| 99 |
+
q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
|
| 100 |
+
kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
|
| 101 |
+
valid = kv_local >= 0
|
| 102 |
+
safe_kv = kv_local.clamp_min(0).long()
|
| 103 |
+
rows = row_map[batch_idx, safe_kv].to(row_dtype)
|
| 104 |
+
start = q_start * topk
|
| 105 |
+
end = q_end * topk
|
| 106 |
+
row_all[:, start:end] = rows
|
| 107 |
+
q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
|
| 108 |
+
valid_all[:, start:end] = valid & (rows >= 0)
|
| 109 |
+
counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
|
| 110 |
+
|
| 111 |
+
k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
|
| 112 |
+
offsets = torch.zeros_like(counts)
|
| 113 |
+
offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
|
| 114 |
+
cursor = torch.zeros_like(counts)
|
| 115 |
+
for h in range(head_kv):
|
| 116 |
+
for entry in range(total_entries):
|
| 117 |
+
if not bool(valid_all[h, entry]):
|
| 118 |
+
continue
|
| 119 |
+
row = int(row_all[h, entry].item())
|
| 120 |
+
pos = int(offsets[h, row].item() + cursor[h, row].item())
|
| 121 |
+
k2q_q_indices[h, pos] = q_all[h, entry]
|
| 122 |
+
cursor[h, row] += 1
|
| 123 |
+
|
| 124 |
+
return k2q_row_ptr, k2q_q_indices
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def build_k2q_csr(
|
| 128 |
+
q2k_indices: torch.Tensor,
|
| 129 |
+
cu_seqlens_q: torch.Tensor,
|
| 130 |
+
cu_seqlens_k: torch.Tensor,
|
| 131 |
+
kv_block_size: int,
|
| 132 |
+
*,
|
| 133 |
+
total_k: Optional[int] = None,
|
| 134 |
+
max_seqlen_k: Optional[int] = None,
|
| 135 |
+
max_seqlen_q: Optional[int] = None,
|
| 136 |
+
total_rows: Optional[int] = None,
|
| 137 |
+
qhead_per_kv: int = 1,
|
| 138 |
+
return_schedule: bool = False,
|
| 139 |
+
):
|
| 140 |
+
if return_schedule:
|
| 141 |
+
raise NotImplementedError(
|
| 142 |
+
"Blackwell build_k2q_csr currently returns CSR tensors only; "
|
| 143 |
+
"SM100 schedule objects are not part of this package."
|
| 144 |
+
)
|
| 145 |
+
if total_k is None:
|
| 146 |
+
raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
|
| 147 |
+
return build_k2q_csr_torch_reference(
|
| 148 |
+
q2k_indices.contiguous(),
|
| 149 |
+
cu_seqlens_q.contiguous(),
|
| 150 |
+
cu_seqlens_k.contiguous(),
|
| 151 |
+
int(kv_block_size),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SparseK2qCsrBuilderSm100:
|
| 156 |
+
"""Compatibility placeholder for the upstream SM100 class name.
|
| 157 |
+
|
| 158 |
+
The name is exported so code can feature-detect it, but schedule-producing
|
| 159 |
+
SM100 behavior is intentionally not claimed for this Blackwell package.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(self, *args, **kwargs):
|
| 163 |
+
self.args = args
|
| 164 |
+
self.kwargs = kwargs
|
| 165 |
+
|
| 166 |
+
def build(self, *args, **kwargs):
|
| 167 |
+
return build_k2q_csr(*args, **kwargs)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
__all__ = [
|
| 171 |
+
"build_k2q_csr",
|
| 172 |
+
"build_k2q_csr_torch_reference",
|
| 173 |
+
"SparseK2qCsrBuilderSm100",
|
| 174 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
CHANGED
|
@@ -37,11 +37,49 @@ See VENDOR_NOTES.md for full shape/dtype/layout contracts.
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
from ._native import has_native_ops, native_topk_from_scores
|
| 41 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 42 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
__all__ = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
"flash_decode_with_topk_idx",
|
| 46 |
"flash_decode_with_gqa_share_sparse",
|
| 47 |
"has_native_ops",
|
|
@@ -50,4 +88,11 @@ __all__ = [
|
|
| 50 |
"naive_flash_decode_with_topk_idx",
|
| 51 |
"get_cu_seqblocks",
|
| 52 |
"robust_allocator",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
]
|
|
|
|
| 37 |
from .common.utils import get_cu_seqblocks, robust_allocator
|
| 38 |
from .decode.flash_with_topk_idx import flash_decode_with_topk_idx
|
| 39 |
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 40 |
+
from .interface import (
|
| 41 |
+
SparseDecodePagedAttentionWrapper,
|
| 42 |
+
fp4_indexer_block_scores,
|
| 43 |
+
sparse_atten_func,
|
| 44 |
+
sparse_atten_nvfp4_kv_func,
|
| 45 |
+
sparse_decode_atten_func,
|
| 46 |
+
)
|
| 47 |
from ._native import has_native_ops, native_topk_from_scores
|
| 48 |
from .naive.flash_with_topk_idx import naive_flash_decode_with_topk_idx
|
| 49 |
from .naive.topk_sparse import naive_flash_decode_with_gqa_share_sparse
|
| 50 |
+
from .quantize import (
|
| 51 |
+
Nvfp4QuantizedTensor,
|
| 52 |
+
dequantize_nvfp4_128x4_to_bf16,
|
| 53 |
+
nvfp4_global_scale_from_amax,
|
| 54 |
+
quantize_bf16_to_nvfp4_128x4,
|
| 55 |
+
quantize_kv_bf16_to_nvfp4_128x4,
|
| 56 |
+
swizzle_nvfp4_scale_to_128x4,
|
| 57 |
+
)
|
| 58 |
+
from .sparse_index_utils import SparseK2qCsrBuilderSm100, build_k2q_csr
|
| 59 |
+
from .api_status import (
|
| 60 |
+
OFFICIAL_API_STATUS,
|
| 61 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS,
|
| 62 |
+
V1_AVAILABLE_FUNCTIONS,
|
| 63 |
+
available_functions,
|
| 64 |
+
official_api_status,
|
| 65 |
+
official_minimax_msa_functions,
|
| 66 |
+
unsupported_official_functions,
|
| 67 |
+
)
|
| 68 |
|
| 69 |
__all__ = [
|
| 70 |
+
"sparse_atten_func",
|
| 71 |
+
"sparse_atten_nvfp4_kv_func",
|
| 72 |
+
"sparse_decode_atten_func",
|
| 73 |
+
"SparseDecodePagedAttentionWrapper",
|
| 74 |
+
"fp4_indexer_block_scores",
|
| 75 |
+
"build_k2q_csr",
|
| 76 |
+
"SparseK2qCsrBuilderSm100",
|
| 77 |
+
"Nvfp4QuantizedTensor",
|
| 78 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 79 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 80 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 81 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 82 |
+
"nvfp4_global_scale_from_amax",
|
| 83 |
"flash_decode_with_topk_idx",
|
| 84 |
"flash_decode_with_gqa_share_sparse",
|
| 85 |
"has_native_ops",
|
|
|
|
| 88 |
"naive_flash_decode_with_topk_idx",
|
| 89 |
"get_cu_seqblocks",
|
| 90 |
"robust_allocator",
|
| 91 |
+
"OFFICIAL_API_STATUS",
|
| 92 |
+
"OFFICIAL_MINIMAX_MSA_FUNCTIONS",
|
| 93 |
+
"V1_AVAILABLE_FUNCTIONS",
|
| 94 |
+
"available_functions",
|
| 95 |
+
"official_api_status",
|
| 96 |
+
"official_minimax_msa_functions",
|
| 97 |
+
"unsupported_official_functions",
|
| 98 |
]
|
build/torch212-cxx11-cu132-x86_64-linux/{_minimaxai_msa_blackwell_cuda_6347cca.abi3.so → _minimaxai_msa_blackwell_cuda_3dca94b.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
size 164208
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e201843e3833b5493ab946e23f740140fdae18953189dfddcc3485102e233327
|
| 3 |
size 164208
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _minimaxai_msa_blackwell_cuda_3dca94b
|
| 3 |
+
ops = torch.ops._minimaxai_msa_blackwell_cuda_3dca94b
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_minimaxai_msa_blackwell_cuda_3dca94b::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/api_status.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Public API status for the FlashRT MiniMax MSA Blackwell package."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from types import MappingProxyType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
V1_AVAILABLE_FUNCTIONS = (
|
| 10 |
+
"sparse_atten_func",
|
| 11 |
+
"sparse_atten_nvfp4_kv_func",
|
| 12 |
+
"sparse_decode_atten_func",
|
| 13 |
+
"SparseDecodePagedAttentionWrapper",
|
| 14 |
+
"fp4_indexer_block_scores",
|
| 15 |
+
"build_k2q_csr",
|
| 16 |
+
"SparseK2qCsrBuilderSm100",
|
| 17 |
+
"Nvfp4QuantizedTensor",
|
| 18 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 19 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 20 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 21 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 22 |
+
"nvfp4_global_scale_from_amax",
|
| 23 |
+
"has_native_ops",
|
| 24 |
+
"native_topk_from_scores",
|
| 25 |
+
"flash_decode_with_topk_idx",
|
| 26 |
+
"flash_decode_with_gqa_share_sparse",
|
| 27 |
+
"naive_flash_decode_with_topk_idx",
|
| 28 |
+
"naive_flash_decode_with_gqa_share_sparse",
|
| 29 |
+
"get_cu_seqblocks",
|
| 30 |
+
"robust_allocator",
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
OFFICIAL_MINIMAX_MSA_FUNCTIONS = (
|
| 34 |
+
"sparse_atten_func",
|
| 35 |
+
"sparse_atten_nvfp4_kv_func",
|
| 36 |
+
"sparse_decode_atten_func",
|
| 37 |
+
"SparseDecodePagedAttentionWrapper",
|
| 38 |
+
"fp4_indexer_block_scores",
|
| 39 |
+
"build_k2q_csr",
|
| 40 |
+
"SparseK2qCsrBuilderSm100",
|
| 41 |
+
"Nvfp4QuantizedTensor",
|
| 42 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 43 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 44 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 45 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 46 |
+
"nvfp4_global_scale_from_amax",
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
_OFFICIAL_API_STATUS = {
|
| 50 |
+
"sparse_atten_func": {
|
| 51 |
+
"status": "unsupported_callable",
|
| 52 |
+
"target": "blackwell-prefill",
|
| 53 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 54 |
+
},
|
| 55 |
+
"sparse_atten_nvfp4_kv_func": {
|
| 56 |
+
"status": "unsupported_callable",
|
| 57 |
+
"target": "blackwell-prefill-nvfp4",
|
| 58 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 NVFP4 CSR/CUTE prefill is not ported to Blackwell yet.",
|
| 59 |
+
},
|
| 60 |
+
"sparse_decode_atten_func": {
|
| 61 |
+
"status": "available",
|
| 62 |
+
"target": "v1",
|
| 63 |
+
"reason": "Official decode API name maps to the Blackwell validated BF16/FP16 paged sparse decode wrapper.",
|
| 64 |
+
},
|
| 65 |
+
"SparseDecodePagedAttentionWrapper": {
|
| 66 |
+
"status": "available",
|
| 67 |
+
"target": "v1",
|
| 68 |
+
"reason": "Official wrapper name is available for the Blackwell single-token paged sparse decode path.",
|
| 69 |
+
},
|
| 70 |
+
"fp4_indexer_block_scores": {
|
| 71 |
+
"status": "unsupported_callable",
|
| 72 |
+
"target": "blackwell-fp4-indexer",
|
| 73 |
+
"reason": "Root API is present and raises NotImplementedError; SM100 FP4 CUTE indexer is not ported to Blackwell yet.",
|
| 74 |
+
},
|
| 75 |
+
"build_k2q_csr": {
|
| 76 |
+
"status": "available",
|
| 77 |
+
"target": "v1",
|
| 78 |
+
"reason": "Torch CSR construction fallback is available for compatibility.",
|
| 79 |
+
},
|
| 80 |
+
"SparseK2qCsrBuilderSm100": {
|
| 81 |
+
"status": "available",
|
| 82 |
+
"target": "v1",
|
| 83 |
+
"reason": "Compatibility class name is exported and delegates build() to build_k2q_csr.",
|
| 84 |
+
},
|
| 85 |
+
"Nvfp4QuantizedTensor": {
|
| 86 |
+
"status": "available",
|
| 87 |
+
"target": "v1",
|
| 88 |
+
"reason": "NVFP4 metadata dataclass is available.",
|
| 89 |
+
},
|
| 90 |
+
"quantize_bf16_to_nvfp4_128x4": {
|
| 91 |
+
"status": "available_optional_te",
|
| 92 |
+
"target": "v1",
|
| 93 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 94 |
+
},
|
| 95 |
+
"quantize_kv_bf16_to_nvfp4_128x4": {
|
| 96 |
+
"status": "available_optional_te",
|
| 97 |
+
"target": "v1",
|
| 98 |
+
"reason": "Function is available and uses Transformer Engine when installed.",
|
| 99 |
+
},
|
| 100 |
+
"dequantize_nvfp4_128x4_to_bf16": {
|
| 101 |
+
"status": "available",
|
| 102 |
+
"target": "v1",
|
| 103 |
+
"reason": "Reference dequant helper is available.",
|
| 104 |
+
},
|
| 105 |
+
"swizzle_nvfp4_scale_to_128x4": {
|
| 106 |
+
"status": "available",
|
| 107 |
+
"target": "v1",
|
| 108 |
+
"reason": "Scale swizzle helper is available.",
|
| 109 |
+
},
|
| 110 |
+
"nvfp4_global_scale_from_amax": {
|
| 111 |
+
"status": "available",
|
| 112 |
+
"target": "v1",
|
| 113 |
+
"reason": "Global scale helper is available.",
|
| 114 |
+
},
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
OFFICIAL_API_STATUS = MappingProxyType(_OFFICIAL_API_STATUS)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def available_functions() -> tuple[str, ...]:
|
| 121 |
+
"""Return the functions/classes intentionally available in v1."""
|
| 122 |
+
|
| 123 |
+
return V1_AVAILABLE_FUNCTIONS
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def official_minimax_msa_functions() -> tuple[str, ...]:
|
| 127 |
+
"""Return the upstream MiniMaxAI/msa public function/class names tracked here."""
|
| 128 |
+
|
| 129 |
+
return OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def official_api_status() -> dict[str, dict[str, str]]:
|
| 133 |
+
"""Return a copy of the official API compatibility status table."""
|
| 134 |
+
|
| 135 |
+
return {name: dict(status) for name, status in OFFICIAL_API_STATUS.items()}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def unsupported_official_functions() -> tuple[str, ...]:
|
| 139 |
+
"""Return official MiniMaxAI/msa names that are not exported by this v1 package."""
|
| 140 |
+
|
| 141 |
+
return tuple(
|
| 142 |
+
name
|
| 143 |
+
for name in OFFICIAL_MINIMAX_MSA_FUNCTIONS
|
| 144 |
+
if OFFICIAL_API_STATUS[name]["status"] not in {"available", "available_optional_te"}
|
| 145 |
+
)
|
build/torch212-cxx11-cu132-x86_64-linux/interface.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""MiniMaxAI/msa-compatible public interface for Blackwell decode sparse path."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
from .decode.topk_sparse import flash_decode_with_gqa_share_sparse
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def _unsupported(name: str, reason: str):
|
| 14 |
+
raise NotImplementedError(
|
| 15 |
+
f"{name} is not implemented in flashrt/MiniMaxAI-msa-blackwell yet: {reason}"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _page_table_to_req_to_token(
|
| 20 |
+
page_table: torch.Tensor,
|
| 21 |
+
seqused_k: torch.Tensor,
|
| 22 |
+
max_seqlen_k: int,
|
| 23 |
+
blk_kv: int,
|
| 24 |
+
) -> torch.Tensor:
|
| 25 |
+
batch = int(page_table.shape[0])
|
| 26 |
+
max_seqlen_k = int(max_seqlen_k)
|
| 27 |
+
blk_kv = int(blk_kv)
|
| 28 |
+
logical_pos = torch.arange(max_seqlen_k, device=page_table.device, dtype=torch.int64)
|
| 29 |
+
page = logical_pos // blk_kv
|
| 30 |
+
offset = logical_pos % blk_kv
|
| 31 |
+
page = page.clamp(max=int(page_table.shape[1]) - 1)
|
| 32 |
+
physical_page = page_table[:, page].to(torch.int64)
|
| 33 |
+
physical_token = physical_page * blk_kv + offset[None, :]
|
| 34 |
+
valid = logical_pos[None, :] < seqused_k.to(torch.int64)[:, None]
|
| 35 |
+
return torch.where(valid, physical_token, torch.zeros_like(physical_token)).to(torch.int32).contiguous()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _dense_q2k_indices(
|
| 39 |
+
*,
|
| 40 |
+
head_kv: int,
|
| 41 |
+
batch: int,
|
| 42 |
+
topk: int,
|
| 43 |
+
seq_lens: torch.Tensor,
|
| 44 |
+
blk_kv: int,
|
| 45 |
+
) -> torch.Tensor:
|
| 46 |
+
idx = torch.full((head_kv, batch, topk), -1, dtype=torch.int32, device=seq_lens.device)
|
| 47 |
+
blocks = torch.arange(topk, dtype=torch.int32, device=seq_lens.device)
|
| 48 |
+
valid_blocks = (seq_lens.to(torch.int64) + int(blk_kv) - 1) // int(blk_kv)
|
| 49 |
+
for b in range(batch):
|
| 50 |
+
valid = blocks < valid_blocks[b]
|
| 51 |
+
idx[:, b, valid] = blocks[valid]
|
| 52 |
+
return idx
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _normalize_decode_q2k(
|
| 56 |
+
q2k_indices: Optional[torch.Tensor],
|
| 57 |
+
*,
|
| 58 |
+
batch: int,
|
| 59 |
+
seqlen_q: int,
|
| 60 |
+
head_kv: int,
|
| 61 |
+
seqused_k: torch.Tensor,
|
| 62 |
+
blk_kv: int,
|
| 63 |
+
) -> torch.Tensor:
|
| 64 |
+
if q2k_indices is None:
|
| 65 |
+
topk = int((int(seqused_k.max().item()) + int(blk_kv) - 1) // int(blk_kv))
|
| 66 |
+
return _dense_q2k_indices(
|
| 67 |
+
head_kv=head_kv,
|
| 68 |
+
batch=batch,
|
| 69 |
+
topk=topk,
|
| 70 |
+
seq_lens=seqused_k,
|
| 71 |
+
blk_kv=blk_kv,
|
| 72 |
+
)
|
| 73 |
+
if q2k_indices.dtype != torch.int32:
|
| 74 |
+
raise TypeError("q2k_indices must be torch.int32")
|
| 75 |
+
if q2k_indices.ndim != 3:
|
| 76 |
+
raise ValueError("q2k_indices must have shape [Hkv, total_q, topK]")
|
| 77 |
+
if q2k_indices.shape[0] != head_kv:
|
| 78 |
+
raise ValueError("q2k_indices first dimension must match Hkv")
|
| 79 |
+
if int(q2k_indices.shape[1]) != batch * seqlen_q:
|
| 80 |
+
raise ValueError("q2k_indices second dimension must equal batch * seqlen_q")
|
| 81 |
+
if seqlen_q != 1:
|
| 82 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 83 |
+
return q2k_indices[:, ::seqlen_q, :].contiguous()
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def sparse_decode_atten_func(
|
| 87 |
+
q: torch.Tensor,
|
| 88 |
+
k: torch.Tensor,
|
| 89 |
+
v: torch.Tensor,
|
| 90 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 91 |
+
*,
|
| 92 |
+
page_table: torch.Tensor,
|
| 93 |
+
seqused_k: torch.Tensor,
|
| 94 |
+
seqlen_q: int,
|
| 95 |
+
max_seqlen_k: int,
|
| 96 |
+
blk_kv: int = 128,
|
| 97 |
+
causal: bool = True,
|
| 98 |
+
softmax_scale: Optional[float] = None,
|
| 99 |
+
return_softmax_lse: bool = False,
|
| 100 |
+
schedule: Optional[object] = None,
|
| 101 |
+
O_partial: Optional[torch.Tensor] = None,
|
| 102 |
+
LSE_partial: Optional[torch.Tensor] = None,
|
| 103 |
+
):
|
| 104 |
+
"""Run MiniMax paged sparse decode on Blackwell.
|
| 105 |
+
|
| 106 |
+
This is compatible with the upstream MiniMaxAI/msa decode-facing API name,
|
| 107 |
+
but accepts the Blackwell package's validated BF16/FP16 decode path. It
|
| 108 |
+
supports single-token decode (`seqlen_q=1`) with paged K/V
|
| 109 |
+
`[num_pages, Hkv, blk_kv, 128]` and optional sparse `q2k_indices`.
|
| 110 |
+
"""
|
| 111 |
+
|
| 112 |
+
if schedule is not None or O_partial is not None or LSE_partial is not None:
|
| 113 |
+
raise NotImplementedError("explicit SM100 decode schedules/partials are not used by the Blackwell wrapper")
|
| 114 |
+
if return_softmax_lse:
|
| 115 |
+
raise NotImplementedError("return_softmax_lse is not implemented for the Blackwell wrapper")
|
| 116 |
+
if not causal:
|
| 117 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports causal=True")
|
| 118 |
+
if int(seqlen_q) != 1:
|
| 119 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports seqlen_q=1")
|
| 120 |
+
if q.ndim != 3:
|
| 121 |
+
raise ValueError("q must have shape [batch * seqlen_q, Hq, D]")
|
| 122 |
+
if k.ndim != 4 or v.ndim != 4:
|
| 123 |
+
raise ValueError("k and v must have shape [num_pages, Hkv, blk_kv, D]")
|
| 124 |
+
if k.shape != v.shape:
|
| 125 |
+
raise ValueError("k and v must have the same shape")
|
| 126 |
+
if q.device != k.device or q.device != v.device or q.device != page_table.device:
|
| 127 |
+
raise ValueError("q, k, v, and page_table must be on the same device")
|
| 128 |
+
if q.dtype not in (torch.bfloat16, torch.float16):
|
| 129 |
+
raise TypeError("Blackwell decode wrapper supports BF16/FP16 q")
|
| 130 |
+
if k.dtype != q.dtype or v.dtype != q.dtype:
|
| 131 |
+
raise TypeError("q, k, and v must have the same dtype for the Blackwell decode wrapper")
|
| 132 |
+
if q.shape[-1] != 128 or k.shape[-1] != 128:
|
| 133 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 134 |
+
if int(k.shape[2]) != int(blk_kv):
|
| 135 |
+
raise ValueError("k.shape[2] must equal blk_kv")
|
| 136 |
+
if page_table.dtype != torch.int32 or seqused_k.dtype != torch.int32:
|
| 137 |
+
raise TypeError("page_table and seqused_k must be torch.int32")
|
| 138 |
+
if page_table.ndim != 2:
|
| 139 |
+
raise ValueError("page_table must have shape [batch, max_num_pages_per_seq]")
|
| 140 |
+
batch = int(page_table.shape[0])
|
| 141 |
+
if int(q.shape[0]) != batch:
|
| 142 |
+
raise ValueError("for seqlen_q=1, q.shape[0] must equal batch")
|
| 143 |
+
|
| 144 |
+
head_kv = int(k.shape[1])
|
| 145 |
+
q2k = _normalize_decode_q2k(
|
| 146 |
+
q2k_indices,
|
| 147 |
+
batch=batch,
|
| 148 |
+
seqlen_q=int(seqlen_q),
|
| 149 |
+
head_kv=head_kv,
|
| 150 |
+
seqused_k=seqused_k,
|
| 151 |
+
blk_kv=int(blk_kv),
|
| 152 |
+
)
|
| 153 |
+
req_to_token = _page_table_to_req_to_token(page_table, seqused_k, int(max_seqlen_k), int(blk_kv))
|
| 154 |
+
slot_ids = torch.arange(batch, device=q.device, dtype=torch.int64)
|
| 155 |
+
k_cache = k.transpose(1, 2).reshape(k.shape[0] * k.shape[2], k.shape[1], k.shape[3]).contiguous()
|
| 156 |
+
v_cache = v.transpose(1, 2).reshape(v.shape[0] * v.shape[2], v.shape[1], v.shape[3]).contiguous()
|
| 157 |
+
return flash_decode_with_gqa_share_sparse(
|
| 158 |
+
q.contiguous(),
|
| 159 |
+
None,
|
| 160 |
+
k_cache,
|
| 161 |
+
v_cache,
|
| 162 |
+
req_to_token,
|
| 163 |
+
seqused_k.contiguous(),
|
| 164 |
+
slot_ids,
|
| 165 |
+
int(blk_kv),
|
| 166 |
+
q2k,
|
| 167 |
+
sm_scale=softmax_scale,
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
class SparseDecodePagedAttentionWrapper:
|
| 172 |
+
def __init__(self, *, blk_kv: int = 128, causal: bool = True):
|
| 173 |
+
self.blk_kv = int(blk_kv)
|
| 174 |
+
self.causal = bool(causal)
|
| 175 |
+
self.page_table: Optional[torch.Tensor] = None
|
| 176 |
+
self.seqused_k: Optional[torch.Tensor] = None
|
| 177 |
+
self.seqlen_q: Optional[int] = None
|
| 178 |
+
self.max_seqlen_k: Optional[int] = None
|
| 179 |
+
self.q2k_indices: Optional[torch.Tensor] = None
|
| 180 |
+
|
| 181 |
+
def plan(
|
| 182 |
+
self,
|
| 183 |
+
*,
|
| 184 |
+
page_table: torch.Tensor,
|
| 185 |
+
seqused_k: torch.Tensor,
|
| 186 |
+
seqlen_q: int,
|
| 187 |
+
max_seqlen_k: int,
|
| 188 |
+
q2k_indices: Optional[torch.Tensor] = None,
|
| 189 |
+
num_qo_heads: Optional[int] = None,
|
| 190 |
+
num_kv_heads: Optional[int] = None,
|
| 191 |
+
head_dim: Optional[int] = 128,
|
| 192 |
+
enable_cuda_graph: bool = False,
|
| 193 |
+
max_grid_size: Optional[int] = None,
|
| 194 |
+
fixed_split_size: Optional[int] = None,
|
| 195 |
+
disable_split_kv: bool = False,
|
| 196 |
+
) -> "SparseDecodePagedAttentionWrapper":
|
| 197 |
+
if head_dim is not None and int(head_dim) != 128:
|
| 198 |
+
raise NotImplementedError("Blackwell decode wrapper currently supports head_dim=128")
|
| 199 |
+
self.page_table = page_table.contiguous()
|
| 200 |
+
self.seqused_k = seqused_k.contiguous()
|
| 201 |
+
self.seqlen_q = int(seqlen_q)
|
| 202 |
+
self.max_seqlen_k = int(max_seqlen_k)
|
| 203 |
+
self.q2k_indices = None if q2k_indices is None else q2k_indices.contiguous()
|
| 204 |
+
return self
|
| 205 |
+
|
| 206 |
+
def run(
|
| 207 |
+
self,
|
| 208 |
+
q: torch.Tensor,
|
| 209 |
+
k: torch.Tensor,
|
| 210 |
+
v: torch.Tensor,
|
| 211 |
+
*,
|
| 212 |
+
softmax_scale: Optional[float] = None,
|
| 213 |
+
return_softmax_lse: bool = False,
|
| 214 |
+
out: Optional[torch.Tensor] = None,
|
| 215 |
+
lse: Optional[torch.Tensor] = None,
|
| 216 |
+
):
|
| 217 |
+
if self.page_table is None or self.seqused_k is None:
|
| 218 |
+
raise RuntimeError("decode wrapper must be planned before run")
|
| 219 |
+
result = sparse_decode_atten_func(
|
| 220 |
+
q,
|
| 221 |
+
k,
|
| 222 |
+
v,
|
| 223 |
+
self.q2k_indices,
|
| 224 |
+
page_table=self.page_table,
|
| 225 |
+
seqused_k=self.seqused_k,
|
| 226 |
+
seqlen_q=self.seqlen_q,
|
| 227 |
+
max_seqlen_k=self.max_seqlen_k,
|
| 228 |
+
blk_kv=self.blk_kv,
|
| 229 |
+
causal=self.causal,
|
| 230 |
+
softmax_scale=softmax_scale,
|
| 231 |
+
return_softmax_lse=return_softmax_lse,
|
| 232 |
+
)
|
| 233 |
+
if out is not None:
|
| 234 |
+
out.copy_(result)
|
| 235 |
+
return out
|
| 236 |
+
return result
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def sparse_atten_func(*args, **kwargs):
|
| 240 |
+
_unsupported("sparse_atten_func", "SM100 CSR prefill attention is not ported to Blackwell in this package")
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def sparse_atten_nvfp4_kv_func(*args, **kwargs):
|
| 244 |
+
_unsupported("sparse_atten_nvfp4_kv_func", "SM100 NVFP4 CSR prefill attention is not ported to Blackwell in this package")
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def fp4_indexer_block_scores(*args, **kwargs):
|
| 248 |
+
_unsupported("fp4_indexer_block_scores", "SM100 FP4 CUTE indexer is not ported to Blackwell in this package")
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
__all__ = [
|
| 252 |
+
"sparse_atten_func",
|
| 253 |
+
"sparse_atten_nvfp4_kv_func",
|
| 254 |
+
"sparse_decode_atten_func",
|
| 255 |
+
"SparseDecodePagedAttentionWrapper",
|
| 256 |
+
"fp4_indexer_block_scores",
|
| 257 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,20 +14,24 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
-
"
|
| 20 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 21 |
-
"_ops.py": "
|
|
|
|
| 22 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 23 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 24 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 25 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 26 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
|
|
|
| 27 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 28 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 29 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 30 |
-
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U="
|
|
|
|
|
|
|
| 31 |
}
|
| 32 |
}
|
| 33 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "minimaxai-msa-blackwell",
|
| 3 |
+
"id": "_minimaxai_msa_blackwell_cuda_3dca94b",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "dm+w4zHgJ/hd6BLT1LZUvDFHOrH2MEGL2Lnw5dXRrpI=",
|
| 18 |
"_compat.py": "5Wy4a96HVMD8omRNsGg8rQJ+0qCCeXiDANk0CuyyP0I=",
|
| 19 |
+
"_minimaxai_msa_blackwell_cuda_3dca94b.abi3.so": "4gGEPjgztUk6uUbiP3QBQP2uGJUxid/dzDSFEC4jMyc=",
|
| 20 |
"_native.py": "mIDvfkcQVd3H9plDsodJelrIFD2XHiEQgk4uqiiRLLE=",
|
| 21 |
+
"_ops.py": "bFy+1smFDZc8RG5qeF4Mwnds/YeB/jC9+Z2/Kj2hXI4=",
|
| 22 |
+
"api_status.py": "BIekw2nohUdtjyMs+JsQV1bIh8rPlFDd1wyTI/nFSiA=",
|
| 23 |
"common/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 24 |
"common/utils.py": "qtwh+wZoySlsWWALvWGVsmcIHKkdZUwePxUZ42tfWxc=",
|
| 25 |
"decode/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 26 |
"decode/flash_with_topk_idx.py": "Pk70+HJ60IKuy4KJC6hPdMtD2BjwPfSMabFiIBODWiQ=",
|
| 27 |
"decode/topk_sparse.py": "PLwQnOyiZT44pCXcZXzXdlZSixfBeanTZsfbpMOjlpQ=",
|
| 28 |
+
"interface.py": "1QbQug26ZVaAXT0SrKToezphajBM7Px880ZzrN2AefI=",
|
| 29 |
"minimaxai_msa_blackwell/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY=",
|
| 30 |
"naive/__init__.py": "bG7Ffen4LvQtgnYPFEpFccsWs81t4zqqeqn9ZeirH6E=",
|
| 31 |
"naive/flash_with_topk_idx.py": "K7WQsRdO9+H2sz7pZ6axnJwE1OmnF0egdKtftHuChig=",
|
| 32 |
+
"naive/topk_sparse.py": "P5kKo1RP9jHVI++oL6rhyoNtDBFW3DAoxzpx7vB1M5U=",
|
| 33 |
+
"quantize.py": "YA5mY3fxTB95QoJRdf1nmJeNosRiG0vfCMO/cp0qIEE=",
|
| 34 |
+
"sparse_index_utils.py": "I0Z6Un43fPhobKeJhopFalkqDCu62iY0u6D7/uE+Y6w="
|
| 35 |
}
|
| 36 |
}
|
| 37 |
}
|
build/torch212-cxx11-cu132-x86_64-linux/quantize.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""NVFP4 helper API compatible with MiniMaxAI/msa.
|
| 3 |
+
|
| 4 |
+
The quantization entrypoints use Transformer Engine when it is installed. The
|
| 5 |
+
layout helpers and reference dequantizer are pure PyTorch and are included so
|
| 6 |
+
callers can prepare and validate NVFP4 metadata with the same public API names
|
| 7 |
+
as the upstream MiniMaxAI package.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from dataclasses import dataclass
|
| 13 |
+
from typing import Tuple
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
NVFP4_BLOCK_SIZE = 16
|
| 19 |
+
NVFP4_FP4_MAX = 6.0
|
| 20 |
+
NVFP4_FP8_E4M3_MAX = 448.0
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@dataclass(frozen=True)
|
| 24 |
+
class Nvfp4QuantizedTensor:
|
| 25 |
+
data: torch.Tensor
|
| 26 |
+
scale_128x4: torch.Tensor
|
| 27 |
+
global_scale: torch.Tensor
|
| 28 |
+
logical_scale_shape: Tuple[int, int]
|
| 29 |
+
original_shape: Tuple[int, ...]
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _round_up(x: int, multiple: int) -> int:
|
| 33 |
+
return ((int(x) + multiple - 1) // multiple) * multiple
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def nvfp4_scale_128x4_offset(
|
| 37 |
+
row: torch.Tensor,
|
| 38 |
+
col: torch.Tensor,
|
| 39 |
+
scale_cols: int,
|
| 40 |
+
) -> torch.Tensor:
|
| 41 |
+
tiles_n = _round_up(scale_cols, 4) // 4
|
| 42 |
+
tile_m = row // 128
|
| 43 |
+
tile_n = col // 4
|
| 44 |
+
outer = row % 128
|
| 45 |
+
inner = col % 4
|
| 46 |
+
return (
|
| 47 |
+
(tile_m * tiles_n + tile_n) * 512
|
| 48 |
+
+ (outer % 32) * 16
|
| 49 |
+
+ (outer // 32) * 4
|
| 50 |
+
+ inner
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def swizzle_nvfp4_scale_to_128x4(
|
| 55 |
+
scale: torch.Tensor,
|
| 56 |
+
*,
|
| 57 |
+
rows: int,
|
| 58 |
+
cols: int,
|
| 59 |
+
) -> torch.Tensor:
|
| 60 |
+
if scale.ndim != 2:
|
| 61 |
+
raise ValueError(f"scale must be 2D, got shape {tuple(scale.shape)}")
|
| 62 |
+
|
| 63 |
+
rows = int(rows)
|
| 64 |
+
cols = int(cols)
|
| 65 |
+
padded_rows = _round_up(rows, 128)
|
| 66 |
+
padded_cols = _round_up(cols, 4)
|
| 67 |
+
if scale.shape[0] < rows or scale.shape[1] < cols:
|
| 68 |
+
raise ValueError(
|
| 69 |
+
"scale is smaller than the requested logical shape: "
|
| 70 |
+
f"got {tuple(scale.shape)}, need at least {(rows, cols)}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
logical = scale[:rows, :cols].contiguous()
|
| 74 |
+
if logical.shape != (padded_rows, padded_cols):
|
| 75 |
+
logical = torch.nn.functional.pad(
|
| 76 |
+
logical.to(torch.float32),
|
| 77 |
+
(0, padded_cols - cols, 0, padded_rows - rows),
|
| 78 |
+
).to(scale.dtype)
|
| 79 |
+
swizzled = torch.empty_like(logical)
|
| 80 |
+
|
| 81 |
+
row = torch.arange(padded_rows, device=scale.device, dtype=torch.int64)[:, None]
|
| 82 |
+
col = torch.arange(padded_cols, device=scale.device, dtype=torch.int64)[None, :]
|
| 83 |
+
offset = nvfp4_scale_128x4_offset(row, col, padded_cols).reshape(-1)
|
| 84 |
+
swizzled.reshape(-1)[offset] = logical.reshape(-1)
|
| 85 |
+
return swizzled
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def nvfp4_global_scale_from_amax(amax: torch.Tensor) -> torch.Tensor:
|
| 89 |
+
return amax.to(torch.float32) / (NVFP4_FP8_E4M3_MAX * NVFP4_FP4_MAX)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def _import_te_nvfp4_quantizer():
|
| 93 |
+
try:
|
| 94 |
+
from transformer_engine.pytorch.tensor import NVFP4Quantizer
|
| 95 |
+
except Exception as exc:
|
| 96 |
+
raise RuntimeError(
|
| 97 |
+
"Transformer Engine NVFP4 quantization is unavailable. Install a "
|
| 98 |
+
"Transformer Engine build with NVFP4 PyTorch support to use "
|
| 99 |
+
"quantize_bf16_to_nvfp4_128x4."
|
| 100 |
+
) from exc
|
| 101 |
+
return NVFP4Quantizer
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def quantize_bf16_to_nvfp4_128x4(x: torch.Tensor) -> Nvfp4QuantizedTensor:
|
| 105 |
+
if not x.is_cuda:
|
| 106 |
+
raise ValueError("NVFP4 quantization requires a CUDA tensor")
|
| 107 |
+
if x.dtype not in (torch.bfloat16, torch.float16):
|
| 108 |
+
raise TypeError(f"x must be bf16 or fp16, got {x.dtype}")
|
| 109 |
+
if x.ndim < 2:
|
| 110 |
+
raise ValueError(f"x must have at least 2 dimensions, got {x.ndim}")
|
| 111 |
+
if x.shape[-1] % NVFP4_BLOCK_SIZE != 0:
|
| 112 |
+
raise ValueError(
|
| 113 |
+
f"last dimension must be divisible by {NVFP4_BLOCK_SIZE}, got {x.shape[-1]}"
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
rows = 1
|
| 117 |
+
for dim in x.shape[:-1]:
|
| 118 |
+
rows *= int(dim)
|
| 119 |
+
if rows % NVFP4_BLOCK_SIZE != 0:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
"flattened row dimension must be divisible by "
|
| 122 |
+
f"{NVFP4_BLOCK_SIZE}, got {rows}"
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
NVFP4Quantizer = _import_te_nvfp4_quantizer()
|
| 126 |
+
quantizer = NVFP4Quantizer(rowwise=True, columnwise=False)
|
| 127 |
+
qx = quantizer.quantize(x.contiguous())
|
| 128 |
+
meta = qx.get_metadata()
|
| 129 |
+
|
| 130 |
+
data = meta["rowwise_data"]
|
| 131 |
+
if data.dtype is not torch.uint8:
|
| 132 |
+
data = data.view(torch.uint8)
|
| 133 |
+
logical_scale = meta["rowwise_scale_inv"]
|
| 134 |
+
amax = meta["amax_rowwise"]
|
| 135 |
+
scale_cols = int(x.shape[-1]) // NVFP4_BLOCK_SIZE
|
| 136 |
+
scale_128x4 = swizzle_nvfp4_scale_to_128x4(
|
| 137 |
+
logical_scale,
|
| 138 |
+
rows=rows,
|
| 139 |
+
cols=scale_cols,
|
| 140 |
+
)
|
| 141 |
+
global_scale = nvfp4_global_scale_from_amax(amax).contiguous()
|
| 142 |
+
|
| 143 |
+
return Nvfp4QuantizedTensor(
|
| 144 |
+
data=data,
|
| 145 |
+
scale_128x4=scale_128x4,
|
| 146 |
+
global_scale=global_scale,
|
| 147 |
+
logical_scale_shape=(rows, scale_cols),
|
| 148 |
+
original_shape=tuple(int(v) for v in x.shape),
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def quantize_kv_bf16_to_nvfp4_128x4(
|
| 153 |
+
k: torch.Tensor,
|
| 154 |
+
v: torch.Tensor,
|
| 155 |
+
) -> tuple[Nvfp4QuantizedTensor, Nvfp4QuantizedTensor]:
|
| 156 |
+
return quantize_bf16_to_nvfp4_128x4(k), quantize_bf16_to_nvfp4_128x4(v)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def dequantize_nvfp4_128x4_to_bf16(
|
| 160 |
+
qx: Nvfp4QuantizedTensor,
|
| 161 |
+
*,
|
| 162 |
+
include_global_scale: bool = True,
|
| 163 |
+
) -> torch.Tensor:
|
| 164 |
+
data = qx.data if qx.data.dtype is torch.uint8 else qx.data.view(torch.uint8)
|
| 165 |
+
if data.shape[-1] * 2 != qx.original_shape[-1]:
|
| 166 |
+
raise ValueError(
|
| 167 |
+
"packed data last dimension does not match original shape: "
|
| 168 |
+
f"{data.shape[-1]} packed vs {qx.original_shape[-1]} logical"
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
rows, scale_cols = qx.logical_scale_shape
|
| 172 |
+
logical_dim = int(qx.original_shape[-1])
|
| 173 |
+
if scale_cols * NVFP4_BLOCK_SIZE != logical_dim:
|
| 174 |
+
raise ValueError(
|
| 175 |
+
"logical scale columns do not match original last dimension: "
|
| 176 |
+
f"{scale_cols} scale cols vs dim {logical_dim}"
|
| 177 |
+
)
|
| 178 |
+
|
| 179 |
+
fp4_lut = torch.tensor(
|
| 180 |
+
[
|
| 181 |
+
0.0,
|
| 182 |
+
0.5,
|
| 183 |
+
1.0,
|
| 184 |
+
1.5,
|
| 185 |
+
2.0,
|
| 186 |
+
3.0,
|
| 187 |
+
4.0,
|
| 188 |
+
6.0,
|
| 189 |
+
-0.0,
|
| 190 |
+
-0.5,
|
| 191 |
+
-1.0,
|
| 192 |
+
-1.5,
|
| 193 |
+
-2.0,
|
| 194 |
+
-3.0,
|
| 195 |
+
-4.0,
|
| 196 |
+
-6.0,
|
| 197 |
+
],
|
| 198 |
+
dtype=torch.float32,
|
| 199 |
+
device=data.device,
|
| 200 |
+
)
|
| 201 |
+
packed = data.reshape(rows, logical_dim // 2)
|
| 202 |
+
lo = packed & 0x0F
|
| 203 |
+
hi = packed >> 4
|
| 204 |
+
values = torch.empty((rows, logical_dim), dtype=torch.float32, device=data.device)
|
| 205 |
+
values[:, 0::2] = fp4_lut[lo.long()]
|
| 206 |
+
values[:, 1::2] = fp4_lut[hi.long()]
|
| 207 |
+
|
| 208 |
+
row = torch.arange(rows, device=data.device, dtype=torch.int64)[:, None]
|
| 209 |
+
col = torch.arange(scale_cols, device=data.device, dtype=torch.int64)[None, :]
|
| 210 |
+
offset = nvfp4_scale_128x4_offset(row, col, scale_cols)
|
| 211 |
+
scale_u8 = qx.scale_128x4.reshape(-1)[offset.reshape(-1)].reshape(rows, scale_cols)
|
| 212 |
+
scale = scale_u8.view(torch.float8_e4m3fn).to(torch.float32)
|
| 213 |
+
scale = scale.repeat_interleave(NVFP4_BLOCK_SIZE, dim=1)
|
| 214 |
+
out = values * scale
|
| 215 |
+
if include_global_scale:
|
| 216 |
+
out = out * qx.global_scale.reshape(-1)[0].to(torch.float32)
|
| 217 |
+
return out.reshape(qx.original_shape).to(torch.bfloat16)
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
__all__ = [
|
| 221 |
+
"Nvfp4QuantizedTensor",
|
| 222 |
+
"quantize_bf16_to_nvfp4_128x4",
|
| 223 |
+
"quantize_kv_bf16_to_nvfp4_128x4",
|
| 224 |
+
"dequantize_nvfp4_128x4_to_bf16",
|
| 225 |
+
"swizzle_nvfp4_scale_to_128x4",
|
| 226 |
+
"nvfp4_global_scale_from_amax",
|
| 227 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/sparse_index_utils.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# SPDX-License-Identifier: Apache-2.0
|
| 2 |
+
"""Sparse-index helpers compatible with MiniMaxAI/msa public names."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
from typing import Optional
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _validate_cu_seqlens(cu_seqlens: torch.Tensor, *, name: str) -> None:
|
| 12 |
+
if cu_seqlens.dtype != torch.int32:
|
| 13 |
+
raise TypeError(f"{name} must be torch.int32, got {cu_seqlens.dtype}")
|
| 14 |
+
if cu_seqlens.ndim != 1:
|
| 15 |
+
raise ValueError(f"{name} must be rank-1, got shape {tuple(cu_seqlens.shape)}")
|
| 16 |
+
if cu_seqlens.numel() < 1:
|
| 17 |
+
raise ValueError(f"{name} must have at least one element")
|
| 18 |
+
if not cu_seqlens.is_contiguous():
|
| 19 |
+
raise ValueError(f"{name} must be contiguous")
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _rows_per_batch(cu_seqlens_k: torch.Tensor, kv_block_size: int) -> torch.Tensor:
|
| 23 |
+
seqlens_k = cu_seqlens_k[1:] - cu_seqlens_k[:-1]
|
| 24 |
+
return (seqlens_k + kv_block_size - 1) // kv_block_size
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _build_packed_row_map(rows_per_batch: torch.Tensor) -> tuple[torch.Tensor, int]:
|
| 28 |
+
rows_per_batch_cpu = rows_per_batch.to("cpu", non_blocking=False).tolist()
|
| 29 |
+
batch = len(rows_per_batch_cpu)
|
| 30 |
+
max_rows = max(rows_per_batch_cpu, default=0)
|
| 31 |
+
row_dtype = (
|
| 32 |
+
torch.int32
|
| 33 |
+
if sum(rows_per_batch_cpu) < torch.iinfo(torch.int32).max
|
| 34 |
+
else torch.int64
|
| 35 |
+
)
|
| 36 |
+
row_map_cpu = torch.full((batch, max_rows), -1, dtype=row_dtype)
|
| 37 |
+
row_linear = 0
|
| 38 |
+
for kv_block_idx in range(max_rows):
|
| 39 |
+
for batch_idx, row_count in enumerate(rows_per_batch_cpu):
|
| 40 |
+
if kv_block_idx < row_count:
|
| 41 |
+
row_map_cpu[batch_idx, kv_block_idx] = row_linear
|
| 42 |
+
row_linear += 1
|
| 43 |
+
return row_map_cpu.to(rows_per_batch.device), row_linear
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_k2q_csr_torch_reference(
|
| 47 |
+
q2k_indices: torch.Tensor,
|
| 48 |
+
cu_seqlens_q: torch.Tensor,
|
| 49 |
+
cu_seqlens_k: torch.Tensor,
|
| 50 |
+
kv_block_size: int,
|
| 51 |
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
| 52 |
+
if kv_block_size <= 0:
|
| 53 |
+
raise ValueError(f"kv_block_size must be > 0, got {kv_block_size}")
|
| 54 |
+
if q2k_indices.dtype != torch.int32:
|
| 55 |
+
raise TypeError(f"q2k_indices must be torch.int32, got {q2k_indices.dtype}")
|
| 56 |
+
if q2k_indices.ndim != 3:
|
| 57 |
+
raise ValueError(
|
| 58 |
+
"q2k_indices must have shape [head_kv, total_q, topK], "
|
| 59 |
+
f"got {tuple(q2k_indices.shape)}"
|
| 60 |
+
)
|
| 61 |
+
_validate_cu_seqlens(cu_seqlens_q, name="cu_seqlens_q")
|
| 62 |
+
_validate_cu_seqlens(cu_seqlens_k, name="cu_seqlens_k")
|
| 63 |
+
if cu_seqlens_q.shape != cu_seqlens_k.shape:
|
| 64 |
+
raise ValueError("cu_seqlens_q and cu_seqlens_k must have the same shape [B + 1]")
|
| 65 |
+
if q2k_indices.device != cu_seqlens_q.device or q2k_indices.device != cu_seqlens_k.device:
|
| 66 |
+
raise ValueError("q2k_indices, cu_seqlens_q, and cu_seqlens_k must be on the same device")
|
| 67 |
+
|
| 68 |
+
head_kv, total_q, topk = q2k_indices.shape
|
| 69 |
+
if total_q != int(cu_seqlens_q[-1].item()):
|
| 70 |
+
raise ValueError(
|
| 71 |
+
f"q2k_indices.shape[1] ({total_q}) must equal cu_seqlens_q[-1] "
|
| 72 |
+
f"({int(cu_seqlens_q[-1].item())})"
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
rows_per_batch = _rows_per_batch(cu_seqlens_k, kv_block_size)
|
| 76 |
+
row_map, total_rows = _build_packed_row_map(rows_per_batch)
|
| 77 |
+
nnz_upper_bound = total_q * topk
|
| 78 |
+
|
| 79 |
+
k2q_row_ptr = torch.zeros((head_kv, total_rows + 1), dtype=torch.int32, device=q2k_indices.device)
|
| 80 |
+
k2q_q_indices = torch.full(
|
| 81 |
+
(head_kv, nnz_upper_bound), -1, dtype=torch.int32, device=q2k_indices.device
|
| 82 |
+
)
|
| 83 |
+
if total_rows == 0 or total_q == 0 or topk == 0:
|
| 84 |
+
return k2q_row_ptr, k2q_q_indices
|
| 85 |
+
|
| 86 |
+
counts = torch.zeros((head_kv, total_rows), dtype=torch.int32, device=q2k_indices.device)
|
| 87 |
+
total_entries = total_q * topk
|
| 88 |
+
row_dtype = torch.int32 if total_rows < torch.iinfo(torch.int32).max else torch.int64
|
| 89 |
+
row_all = torch.empty((head_kv, total_entries), dtype=row_dtype, device=q2k_indices.device)
|
| 90 |
+
q_all = torch.empty((head_kv, total_entries), dtype=torch.int32, device=q2k_indices.device)
|
| 91 |
+
valid_all = torch.empty((head_kv, total_entries), dtype=torch.bool, device=q2k_indices.device)
|
| 92 |
+
|
| 93 |
+
for batch_idx in range(cu_seqlens_q.numel() - 1):
|
| 94 |
+
q_start = int(cu_seqlens_q[batch_idx].item())
|
| 95 |
+
q_end = int(cu_seqlens_q[batch_idx + 1].item())
|
| 96 |
+
if q_end <= q_start:
|
| 97 |
+
continue
|
| 98 |
+
q_local = torch.arange(q_start, q_end, dtype=torch.int32, device=q2k_indices.device)
|
| 99 |
+
q_local = q_local[:, None].expand(q_end - q_start, topk).reshape(-1)
|
| 100 |
+
kv_local = q2k_indices[:, q_start:q_end, :].reshape(head_kv, -1)
|
| 101 |
+
valid = kv_local >= 0
|
| 102 |
+
safe_kv = kv_local.clamp_min(0).long()
|
| 103 |
+
rows = row_map[batch_idx, safe_kv].to(row_dtype)
|
| 104 |
+
start = q_start * topk
|
| 105 |
+
end = q_end * topk
|
| 106 |
+
row_all[:, start:end] = rows
|
| 107 |
+
q_all[:, start:end] = q_local[None, :].expand(head_kv, -1)
|
| 108 |
+
valid_all[:, start:end] = valid & (rows >= 0)
|
| 109 |
+
counts.scatter_add_(1, rows.clamp_min(0).long(), valid_all[:, start:end].to(torch.int32))
|
| 110 |
+
|
| 111 |
+
k2q_row_ptr[:, 1:] = counts.cumsum(dim=1)
|
| 112 |
+
offsets = torch.zeros_like(counts)
|
| 113 |
+
offsets[:, 1:] = counts[:, :-1].cumsum(dim=1)
|
| 114 |
+
cursor = torch.zeros_like(counts)
|
| 115 |
+
for h in range(head_kv):
|
| 116 |
+
for entry in range(total_entries):
|
| 117 |
+
if not bool(valid_all[h, entry]):
|
| 118 |
+
continue
|
| 119 |
+
row = int(row_all[h, entry].item())
|
| 120 |
+
pos = int(offsets[h, row].item() + cursor[h, row].item())
|
| 121 |
+
k2q_q_indices[h, pos] = q_all[h, entry]
|
| 122 |
+
cursor[h, row] += 1
|
| 123 |
+
|
| 124 |
+
return k2q_row_ptr, k2q_q_indices
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def build_k2q_csr(
|
| 128 |
+
q2k_indices: torch.Tensor,
|
| 129 |
+
cu_seqlens_q: torch.Tensor,
|
| 130 |
+
cu_seqlens_k: torch.Tensor,
|
| 131 |
+
kv_block_size: int,
|
| 132 |
+
*,
|
| 133 |
+
total_k: Optional[int] = None,
|
| 134 |
+
max_seqlen_k: Optional[int] = None,
|
| 135 |
+
max_seqlen_q: Optional[int] = None,
|
| 136 |
+
total_rows: Optional[int] = None,
|
| 137 |
+
qhead_per_kv: int = 1,
|
| 138 |
+
return_schedule: bool = False,
|
| 139 |
+
):
|
| 140 |
+
if return_schedule:
|
| 141 |
+
raise NotImplementedError(
|
| 142 |
+
"Blackwell build_k2q_csr currently returns CSR tensors only; "
|
| 143 |
+
"SM100 schedule objects are not part of this package."
|
| 144 |
+
)
|
| 145 |
+
if total_k is None:
|
| 146 |
+
raise ValueError("build_k2q_csr requires total_k from k.shape[0]")
|
| 147 |
+
return build_k2q_csr_torch_reference(
|
| 148 |
+
q2k_indices.contiguous(),
|
| 149 |
+
cu_seqlens_q.contiguous(),
|
| 150 |
+
cu_seqlens_k.contiguous(),
|
| 151 |
+
int(kv_block_size),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
class SparseK2qCsrBuilderSm100:
|
| 156 |
+
"""Compatibility placeholder for the upstream SM100 class name.
|
| 157 |
+
|
| 158 |
+
The name is exported so code can feature-detect it, but schedule-producing
|
| 159 |
+
SM100 behavior is intentionally not claimed for this Blackwell package.
|
| 160 |
+
"""
|
| 161 |
+
|
| 162 |
+
def __init__(self, *args, **kwargs):
|
| 163 |
+
self.args = args
|
| 164 |
+
self.kwargs = kwargs
|
| 165 |
+
|
| 166 |
+
def build(self, *args, **kwargs):
|
| 167 |
+
return build_k2q_csr(*args, **kwargs)
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
__all__ = [
|
| 171 |
+
"build_k2q_csr",
|
| 172 |
+
"build_k2q_csr_torch_reference",
|
| 173 |
+
"SparseK2qCsrBuilderSm100",
|
| 174 |
+
]
|