Uploaded using `kernel-builder`.
Browse files- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +34 -0
- build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} +2 -2
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +5 -5
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +34 -0
- build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} +2 -2
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +5 -5
- build/torch213-cxx11-cu130-x86_64-linux/__init__.py +34 -0
- build/torch213-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} +2 -2
- build/torch213-cxx11-cu130-x86_64-linux/_ops.py +3 -3
- build/torch213-cxx11-cu130-x86_64-linux/metadata.json +5 -5
- build/torch213-cxx11-cu132-x86_64-linux/__init__.py +34 -0
- build/torch213-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} +2 -2
- build/torch213-cxx11-cu132-x86_64-linux/_ops.py +3 -3
- build/torch213-cxx11-cu132-x86_64-linux/metadata.json +5 -5
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
CHANGED
|
@@ -36,6 +36,15 @@ def _linear_fake(
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 40 |
def _legacy_linear_fake(
|
| 41 |
a_packed: torch.Tensor,
|
|
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
|
|
| 125 |
return out
|
| 126 |
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
def fp4_w4a16_linear_bf16(
|
| 129 |
a_packed: torch.Tensor,
|
| 130 |
b_packed: torch.Tensor,
|
|
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
|
|
| 243 |
__all__ = [
|
| 244 |
"dequantize_fp4_sfa_fp16",
|
| 245 |
"fp4_w4a16_linear_bf16",
|
|
|
|
| 246 |
"nvfp4_gemm_bf16",
|
| 247 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 248 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
|
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
| 39 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
|
| 40 |
+
def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
|
| 41 |
+
if a_packed.shape[0] != 1:
|
| 42 |
+
raise RuntimeError("warp-split GEMV serves M=1 only")
|
| 43 |
+
if out.shape != (1, b_packed.shape[0]):
|
| 44 |
+
raise RuntimeError("out must have shape (1, N)")
|
| 45 |
+
return None
|
| 46 |
+
|
| 47 |
+
|
| 48 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 49 |
def _legacy_linear_fake(
|
| 50 |
a_packed: torch.Tensor,
|
|
|
|
| 134 |
return out
|
| 135 |
|
| 136 |
|
| 137 |
+
def fp4_w4a4_gemv_warpsplit_bf16(
|
| 138 |
+
a_packed: torch.Tensor,
|
| 139 |
+
b_packed: torch.Tensor,
|
| 140 |
+
sfa: torch.Tensor,
|
| 141 |
+
sfb: torch.Tensor,
|
| 142 |
+
*,
|
| 143 |
+
alpha: float = 1.0,
|
| 144 |
+
warps: int = 4,
|
| 145 |
+
stages: int = 4,
|
| 146 |
+
out: Optional[torch.Tensor] = None,
|
| 147 |
+
) -> torch.Tensor:
|
| 148 |
+
"""Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
|
| 149 |
+
|
| 150 |
+
Splits K across warps inside one block with a shared-memory reduce -
|
| 151 |
+
no cross-block intermediate, so it stays safe under CUDA-graph
|
| 152 |
+
replay - and fills the SMs the tiled GEMM underfills at long-K
|
| 153 |
+
small-M decode shapes. Same packed/scale layouts as the linear
|
| 154 |
+
entry points."""
|
| 155 |
+
if out is None:
|
| 156 |
+
out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
|
| 157 |
+
ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
|
| 158 |
+
return out
|
| 159 |
+
|
| 160 |
+
|
| 161 |
def fp4_w4a16_linear_bf16(
|
| 162 |
a_packed: torch.Tensor,
|
| 163 |
b_packed: torch.Tensor,
|
|
|
|
| 276 |
__all__ = [
|
| 277 |
"dequantize_fp4_sfa_fp16",
|
| 278 |
"fp4_w4a16_linear_bf16",
|
| 279 |
+
"fp4_w4a4_gemv_warpsplit_bf16",
|
| 280 |
"nvfp4_gemm_bf16",
|
| 281 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 282 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3c115b378d31629c86b81dcb460c62b661bad375035f89fbc291e5521a16561d
|
| 3 |
+
size 2392016
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _fp4_gemm_cuda_bc05275
|
| 3 |
+
ops = torch.ops._fp4_gemm_cuda_bc05275
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_fp4_gemm_cuda_bc05275::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,9 +14,9 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
-
"
|
| 19 |
-
"_ops.py": "
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
@@ -26,7 +26,7 @@
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
-
"sha": "
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
+
"id": "_fp4_gemm_cuda_bc05275",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
|
| 18 |
+
"_fp4_gemm_cuda_bc05275.abi3.so": "PBFbN40xYpyGuB3LRgxitmG603UDX4n7wpHlUhoWVh0=",
|
| 19 |
+
"_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
+
"sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
CHANGED
|
@@ -36,6 +36,15 @@ def _linear_fake(
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 40 |
def _legacy_linear_fake(
|
| 41 |
a_packed: torch.Tensor,
|
|
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
|
|
| 125 |
return out
|
| 126 |
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
def fp4_w4a16_linear_bf16(
|
| 129 |
a_packed: torch.Tensor,
|
| 130 |
b_packed: torch.Tensor,
|
|
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
|
|
| 243 |
__all__ = [
|
| 244 |
"dequantize_fp4_sfa_fp16",
|
| 245 |
"fp4_w4a16_linear_bf16",
|
|
|
|
| 246 |
"nvfp4_gemm_bf16",
|
| 247 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 248 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
|
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
| 39 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
|
| 40 |
+
def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
|
| 41 |
+
if a_packed.shape[0] != 1:
|
| 42 |
+
raise RuntimeError("warp-split GEMV serves M=1 only")
|
| 43 |
+
if out.shape != (1, b_packed.shape[0]):
|
| 44 |
+
raise RuntimeError("out must have shape (1, N)")
|
| 45 |
+
return None
|
| 46 |
+
|
| 47 |
+
|
| 48 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 49 |
def _legacy_linear_fake(
|
| 50 |
a_packed: torch.Tensor,
|
|
|
|
| 134 |
return out
|
| 135 |
|
| 136 |
|
| 137 |
+
def fp4_w4a4_gemv_warpsplit_bf16(
|
| 138 |
+
a_packed: torch.Tensor,
|
| 139 |
+
b_packed: torch.Tensor,
|
| 140 |
+
sfa: torch.Tensor,
|
| 141 |
+
sfb: torch.Tensor,
|
| 142 |
+
*,
|
| 143 |
+
alpha: float = 1.0,
|
| 144 |
+
warps: int = 4,
|
| 145 |
+
stages: int = 4,
|
| 146 |
+
out: Optional[torch.Tensor] = None,
|
| 147 |
+
) -> torch.Tensor:
|
| 148 |
+
"""Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
|
| 149 |
+
|
| 150 |
+
Splits K across warps inside one block with a shared-memory reduce -
|
| 151 |
+
no cross-block intermediate, so it stays safe under CUDA-graph
|
| 152 |
+
replay - and fills the SMs the tiled GEMM underfills at long-K
|
| 153 |
+
small-M decode shapes. Same packed/scale layouts as the linear
|
| 154 |
+
entry points."""
|
| 155 |
+
if out is None:
|
| 156 |
+
out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
|
| 157 |
+
ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
|
| 158 |
+
return out
|
| 159 |
+
|
| 160 |
+
|
| 161 |
def fp4_w4a16_linear_bf16(
|
| 162 |
a_packed: torch.Tensor,
|
| 163 |
b_packed: torch.Tensor,
|
|
|
|
| 276 |
__all__ = [
|
| 277 |
"dequantize_fp4_sfa_fp16",
|
| 278 |
"fp4_w4a16_linear_bf16",
|
| 279 |
+
"fp4_w4a4_gemv_warpsplit_bf16",
|
| 280 |
"nvfp4_gemm_bf16",
|
| 281 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 282 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1d32cd7155fc01598c8ada3587b4c776e0b349df6ce49f50e8d95268b4a52fc2
|
| 3 |
+
size 2387872
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _fp4_gemm_cuda_bc05275
|
| 3 |
+
ops = torch.ops._fp4_gemm_cuda_bc05275
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_fp4_gemm_cuda_bc05275::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,9 +14,9 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
-
"
|
| 19 |
-
"_ops.py": "
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
@@ -26,7 +26,7 @@
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
-
"sha": "
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
+
"id": "_fp4_gemm_cuda_bc05275",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
|
| 18 |
+
"_fp4_gemm_cuda_bc05275.abi3.so": "HTLNcVX8AVmMito1h7THduCzSd9s5J9Q6NlSaLSlL8I=",
|
| 19 |
+
"_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
+
"sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
build/torch213-cxx11-cu130-x86_64-linux/__init__.py
CHANGED
|
@@ -36,6 +36,15 @@ def _linear_fake(
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 40 |
def _legacy_linear_fake(
|
| 41 |
a_packed: torch.Tensor,
|
|
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
|
|
| 125 |
return out
|
| 126 |
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
def fp4_w4a16_linear_bf16(
|
| 129 |
a_packed: torch.Tensor,
|
| 130 |
b_packed: torch.Tensor,
|
|
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
|
|
| 243 |
__all__ = [
|
| 244 |
"dequantize_fp4_sfa_fp16",
|
| 245 |
"fp4_w4a16_linear_bf16",
|
|
|
|
| 246 |
"nvfp4_gemm_bf16",
|
| 247 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 248 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
|
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
| 39 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
|
| 40 |
+
def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
|
| 41 |
+
if a_packed.shape[0] != 1:
|
| 42 |
+
raise RuntimeError("warp-split GEMV serves M=1 only")
|
| 43 |
+
if out.shape != (1, b_packed.shape[0]):
|
| 44 |
+
raise RuntimeError("out must have shape (1, N)")
|
| 45 |
+
return None
|
| 46 |
+
|
| 47 |
+
|
| 48 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 49 |
def _legacy_linear_fake(
|
| 50 |
a_packed: torch.Tensor,
|
|
|
|
| 134 |
return out
|
| 135 |
|
| 136 |
|
| 137 |
+
def fp4_w4a4_gemv_warpsplit_bf16(
|
| 138 |
+
a_packed: torch.Tensor,
|
| 139 |
+
b_packed: torch.Tensor,
|
| 140 |
+
sfa: torch.Tensor,
|
| 141 |
+
sfb: torch.Tensor,
|
| 142 |
+
*,
|
| 143 |
+
alpha: float = 1.0,
|
| 144 |
+
warps: int = 4,
|
| 145 |
+
stages: int = 4,
|
| 146 |
+
out: Optional[torch.Tensor] = None,
|
| 147 |
+
) -> torch.Tensor:
|
| 148 |
+
"""Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
|
| 149 |
+
|
| 150 |
+
Splits K across warps inside one block with a shared-memory reduce -
|
| 151 |
+
no cross-block intermediate, so it stays safe under CUDA-graph
|
| 152 |
+
replay - and fills the SMs the tiled GEMM underfills at long-K
|
| 153 |
+
small-M decode shapes. Same packed/scale layouts as the linear
|
| 154 |
+
entry points."""
|
| 155 |
+
if out is None:
|
| 156 |
+
out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
|
| 157 |
+
ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
|
| 158 |
+
return out
|
| 159 |
+
|
| 160 |
+
|
| 161 |
def fp4_w4a16_linear_bf16(
|
| 162 |
a_packed: torch.Tensor,
|
| 163 |
b_packed: torch.Tensor,
|
|
|
|
| 276 |
__all__ = [
|
| 277 |
"dequantize_fp4_sfa_fp16",
|
| 278 |
"fp4_w4a16_linear_bf16",
|
| 279 |
+
"fp4_w4a4_gemv_warpsplit_bf16",
|
| 280 |
"nvfp4_gemm_bf16",
|
| 281 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 282 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
build/torch213-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f66e3a05532f5978a4fec614d526887cffff3c527f1ba622cf8472d3f2fabc7d
|
| 3 |
+
size 2391864
|
build/torch213-cxx11-cu130-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _fp4_gemm_cuda_bc05275
|
| 3 |
+
ops = torch.ops._fp4_gemm_cuda_bc05275
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_fp4_gemm_cuda_bc05275::{op_name}"
|
build/torch213-cxx11-cu130-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,9 +14,9 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
-
"
|
| 19 |
-
"_ops.py": "
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
@@ -26,7 +26,7 @@
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
-
"sha": "
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
+
"id": "_fp4_gemm_cuda_bc05275",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
|
| 18 |
+
"_fp4_gemm_cuda_bc05275.abi3.so": "9m46BVMvWXik/sYU1SaIfP//PFJ/G6Yiz4Ry0/L6vH0=",
|
| 19 |
+
"_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
+
"sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
build/torch213-cxx11-cu132-x86_64-linux/__init__.py
CHANGED
|
@@ -36,6 +36,15 @@ def _linear_fake(
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 40 |
def _legacy_linear_fake(
|
| 41 |
a_packed: torch.Tensor,
|
|
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
|
|
| 125 |
return out
|
| 126 |
|
| 127 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 128 |
def fp4_w4a16_linear_bf16(
|
| 129 |
a_packed: torch.Tensor,
|
| 130 |
b_packed: torch.Tensor,
|
|
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
|
|
| 243 |
__all__ = [
|
| 244 |
"dequantize_fp4_sfa_fp16",
|
| 245 |
"fp4_w4a16_linear_bf16",
|
|
|
|
| 246 |
"nvfp4_gemm_bf16",
|
| 247 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 248 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
|
|
|
| 36 |
return None
|
| 37 |
|
| 38 |
|
| 39 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
|
| 40 |
+
def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
|
| 41 |
+
if a_packed.shape[0] != 1:
|
| 42 |
+
raise RuntimeError("warp-split GEMV serves M=1 only")
|
| 43 |
+
if out.shape != (1, b_packed.shape[0]):
|
| 44 |
+
raise RuntimeError("out must have shape (1, N)")
|
| 45 |
+
return None
|
| 46 |
+
|
| 47 |
+
|
| 48 |
@torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
|
| 49 |
def _legacy_linear_fake(
|
| 50 |
a_packed: torch.Tensor,
|
|
|
|
| 134 |
return out
|
| 135 |
|
| 136 |
|
| 137 |
+
def fp4_w4a4_gemv_warpsplit_bf16(
|
| 138 |
+
a_packed: torch.Tensor,
|
| 139 |
+
b_packed: torch.Tensor,
|
| 140 |
+
sfa: torch.Tensor,
|
| 141 |
+
sfb: torch.Tensor,
|
| 142 |
+
*,
|
| 143 |
+
alpha: float = 1.0,
|
| 144 |
+
warps: int = 4,
|
| 145 |
+
stages: int = 4,
|
| 146 |
+
out: Optional[torch.Tensor] = None,
|
| 147 |
+
) -> torch.Tensor:
|
| 148 |
+
"""Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
|
| 149 |
+
|
| 150 |
+
Splits K across warps inside one block with a shared-memory reduce -
|
| 151 |
+
no cross-block intermediate, so it stays safe under CUDA-graph
|
| 152 |
+
replay - and fills the SMs the tiled GEMM underfills at long-K
|
| 153 |
+
small-M decode shapes. Same packed/scale layouts as the linear
|
| 154 |
+
entry points."""
|
| 155 |
+
if out is None:
|
| 156 |
+
out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
|
| 157 |
+
ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
|
| 158 |
+
return out
|
| 159 |
+
|
| 160 |
+
|
| 161 |
def fp4_w4a16_linear_bf16(
|
| 162 |
a_packed: torch.Tensor,
|
| 163 |
b_packed: torch.Tensor,
|
|
|
|
| 276 |
__all__ = [
|
| 277 |
"dequantize_fp4_sfa_fp16",
|
| 278 |
"fp4_w4a16_linear_bf16",
|
| 279 |
+
"fp4_w4a4_gemv_warpsplit_bf16",
|
| 280 |
"nvfp4_gemm_bf16",
|
| 281 |
"nvfp4_gemm_bias_gelu_bf16",
|
| 282 |
"nvfp4_gemm_bias_gelu_nvfp4",
|
build/torch213-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e389511593a98a8aec2b708ef40ae2dca175c36be86cf8dc7d6ccb31f0753542
|
| 3 |
+
size 2387712
|
build/torch213-cxx11-cu132-x86_64-linux/_ops.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _fp4_gemm_cuda_bc05275
|
| 3 |
+
ops = torch.ops._fp4_gemm_cuda_bc05275
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
"""
|
| 7 |
Prefix op by namespace.
|
| 8 |
"""
|
| 9 |
+
return f"_fp4_gemm_cuda_bc05275::{op_name}"
|
build/torch213-cxx11-cu132-x86_64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -14,9 +14,9 @@
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
-
"__init__.py": "
|
| 18 |
-
"
|
| 19 |
-
"_ops.py": "
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
@@ -26,7 +26,7 @@
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
-
"sha": "
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "fp4-gemm",
|
| 3 |
+
"id": "_fp4_gemm_cuda_bc05275",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 14 |
"digest": {
|
| 15 |
"algorithm": "sha256",
|
| 16 |
"files": {
|
| 17 |
+
"__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
|
| 18 |
+
"_fp4_gemm_cuda_bc05275.abi3.so": "44lRFZOpiorsK3CO9Ari3KF1w2vobPjcfWzLMfB1NUI=",
|
| 19 |
+
"_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
|
| 20 |
}
|
| 21 |
},
|
| 22 |
"provenance": {
|
|
|
|
| 26 |
"dirty": false
|
| 27 |
},
|
| 28 |
"kernel": {
|
| 29 |
+
"sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
|
| 30 |
"dirty": false
|
| 31 |
}
|
| 32 |
}
|