liangsu9988 commited on
Commit
1e81aaa
·
verified ·
1 Parent(s): fdf2b52

Uploaded using `kernel-builder`.

Browse files
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -36,6 +36,15 @@ def _linear_fake(
36
  return None
37
 
38
 
 
 
 
 
 
 
 
 
 
39
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
40
  def _legacy_linear_fake(
41
  a_packed: torch.Tensor,
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
125
  return out
126
 
127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  def fp4_w4a16_linear_bf16(
129
  a_packed: torch.Tensor,
130
  b_packed: torch.Tensor,
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
 
246
  "nvfp4_gemm_bf16",
247
  "nvfp4_gemm_bias_gelu_bf16",
248
  "nvfp4_gemm_bias_gelu_nvfp4",
 
36
  return None
37
 
38
 
39
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
40
+ def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
41
+ if a_packed.shape[0] != 1:
42
+ raise RuntimeError("warp-split GEMV serves M=1 only")
43
+ if out.shape != (1, b_packed.shape[0]):
44
+ raise RuntimeError("out must have shape (1, N)")
45
+ return None
46
+
47
+
48
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
49
  def _legacy_linear_fake(
50
  a_packed: torch.Tensor,
 
134
  return out
135
 
136
 
137
+ def fp4_w4a4_gemv_warpsplit_bf16(
138
+ a_packed: torch.Tensor,
139
+ b_packed: torch.Tensor,
140
+ sfa: torch.Tensor,
141
+ sfb: torch.Tensor,
142
+ *,
143
+ alpha: float = 1.0,
144
+ warps: int = 4,
145
+ stages: int = 4,
146
+ out: Optional[torch.Tensor] = None,
147
+ ) -> torch.Tensor:
148
+ """Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
149
+
150
+ Splits K across warps inside one block with a shared-memory reduce -
151
+ no cross-block intermediate, so it stays safe under CUDA-graph
152
+ replay - and fills the SMs the tiled GEMM underfills at long-K
153
+ small-M decode shapes. Same packed/scale layouts as the linear
154
+ entry points."""
155
+ if out is None:
156
+ out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
157
+ ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
158
+ return out
159
+
160
+
161
  def fp4_w4a16_linear_bf16(
162
  a_packed: torch.Tensor,
163
  b_packed: torch.Tensor,
 
276
  __all__ = [
277
  "dequantize_fp4_sfa_fp16",
278
  "fp4_w4a16_linear_bf16",
279
+ "fp4_w4a4_gemv_warpsplit_bf16",
280
  "nvfp4_gemm_bf16",
281
  "nvfp4_gemm_bias_gelu_bf16",
282
  "nvfp4_gemm_bias_gelu_nvfp4",
build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:090297ab2b647b6dc060ba10c77feb79668c92aafae15053b54d1c14625bf71d
3
- size 2213120
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3c115b378d31629c86b81dcb460c62b661bad375035f89fbc291e5521a16561d
3
+ size 2392016
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_4c95454
3
- ops = torch.ops._fp4_gemm_cuda_4c95454
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_4c95454::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_bc05275
3
+ ops = torch.ops._fp4_gemm_cuda_bc05275
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_bc05275::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_4c95454",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "EJ8HPhVwOKhifotSgdwCmB6MRbtnP1S0QfUGOVascy0=",
18
- "_fp4_gemm_cuda_4c95454.abi3.so": "CQKXqytke23AYLoQx3/reWaMkqr64VBTtU0cFGJb9x0=",
19
- "_ops.py": "zg1yY7ZypjpIvhFjVWntmzhlu2W/m2dDxA5HuBl+2Hw="
20
  }
21
  },
22
  "provenance": {
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "4c95454360b18c79bed75ec6346a7bac47ae2ca2",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_bc05275",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
18
+ "_fp4_gemm_cuda_bc05275.abi3.so": "PBFbN40xYpyGuB3LRgxitmG603UDX4n7wpHlUhoWVh0=",
19
+ "_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
20
  }
21
  },
22
  "provenance": {
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
30
  "dirty": false
31
  }
32
  }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -36,6 +36,15 @@ def _linear_fake(
36
  return None
37
 
38
 
 
 
 
 
 
 
 
 
 
39
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
40
  def _legacy_linear_fake(
41
  a_packed: torch.Tensor,
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
125
  return out
126
 
127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  def fp4_w4a16_linear_bf16(
129
  a_packed: torch.Tensor,
130
  b_packed: torch.Tensor,
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
 
246
  "nvfp4_gemm_bf16",
247
  "nvfp4_gemm_bias_gelu_bf16",
248
  "nvfp4_gemm_bias_gelu_nvfp4",
 
36
  return None
37
 
38
 
39
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
40
+ def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
41
+ if a_packed.shape[0] != 1:
42
+ raise RuntimeError("warp-split GEMV serves M=1 only")
43
+ if out.shape != (1, b_packed.shape[0]):
44
+ raise RuntimeError("out must have shape (1, N)")
45
+ return None
46
+
47
+
48
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
49
  def _legacy_linear_fake(
50
  a_packed: torch.Tensor,
 
134
  return out
135
 
136
 
137
+ def fp4_w4a4_gemv_warpsplit_bf16(
138
+ a_packed: torch.Tensor,
139
+ b_packed: torch.Tensor,
140
+ sfa: torch.Tensor,
141
+ sfb: torch.Tensor,
142
+ *,
143
+ alpha: float = 1.0,
144
+ warps: int = 4,
145
+ stages: int = 4,
146
+ out: Optional[torch.Tensor] = None,
147
+ ) -> torch.Tensor:
148
+ """Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
149
+
150
+ Splits K across warps inside one block with a shared-memory reduce -
151
+ no cross-block intermediate, so it stays safe under CUDA-graph
152
+ replay - and fills the SMs the tiled GEMM underfills at long-K
153
+ small-M decode shapes. Same packed/scale layouts as the linear
154
+ entry points."""
155
+ if out is None:
156
+ out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
157
+ ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
158
+ return out
159
+
160
+
161
  def fp4_w4a16_linear_bf16(
162
  a_packed: torch.Tensor,
163
  b_packed: torch.Tensor,
 
276
  __all__ = [
277
  "dequantize_fp4_sfa_fp16",
278
  "fp4_w4a16_linear_bf16",
279
+ "fp4_w4a4_gemv_warpsplit_bf16",
280
  "nvfp4_gemm_bf16",
281
  "nvfp4_gemm_bias_gelu_bf16",
282
  "nvfp4_gemm_bias_gelu_nvfp4",
build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:81d95427ab370e0a8d0a2327a2218b8e48ce8e8efb7b4e1b78a82e197e6bd1fe
3
- size 2217168
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d32cd7155fc01598c8ada3587b4c776e0b349df6ce49f50e8d95268b4a52fc2
3
+ size 2387872
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_4c95454
3
- ops = torch.ops._fp4_gemm_cuda_4c95454
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_4c95454::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_bc05275
3
+ ops = torch.ops._fp4_gemm_cuda_bc05275
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_bc05275::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_4c95454",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "EJ8HPhVwOKhifotSgdwCmB6MRbtnP1S0QfUGOVascy0=",
18
- "_fp4_gemm_cuda_4c95454.abi3.so": "gdlUJ6s3DgqNCiMnoiGLjkjOjo77e04beKguGX5r0f4=",
19
- "_ops.py": "zg1yY7ZypjpIvhFjVWntmzhlu2W/m2dDxA5HuBl+2Hw="
20
  }
21
  },
22
  "provenance": {
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "4c95454360b18c79bed75ec6346a7bac47ae2ca2",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_bc05275",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
18
+ "_fp4_gemm_cuda_bc05275.abi3.so": "HTLNcVX8AVmMito1h7THduCzSd9s5J9Q6NlSaLSlL8I=",
19
+ "_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
20
  }
21
  },
22
  "provenance": {
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
30
  "dirty": false
31
  }
32
  }
build/torch213-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -36,6 +36,15 @@ def _linear_fake(
36
  return None
37
 
38
 
 
 
 
 
 
 
 
 
 
39
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
40
  def _legacy_linear_fake(
41
  a_packed: torch.Tensor,
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
125
  return out
126
 
127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  def fp4_w4a16_linear_bf16(
129
  a_packed: torch.Tensor,
130
  b_packed: torch.Tensor,
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
 
246
  "nvfp4_gemm_bf16",
247
  "nvfp4_gemm_bias_gelu_bf16",
248
  "nvfp4_gemm_bias_gelu_nvfp4",
 
36
  return None
37
 
38
 
39
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
40
+ def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
41
+ if a_packed.shape[0] != 1:
42
+ raise RuntimeError("warp-split GEMV serves M=1 only")
43
+ if out.shape != (1, b_packed.shape[0]):
44
+ raise RuntimeError("out must have shape (1, N)")
45
+ return None
46
+
47
+
48
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
49
  def _legacy_linear_fake(
50
  a_packed: torch.Tensor,
 
134
  return out
135
 
136
 
137
+ def fp4_w4a4_gemv_warpsplit_bf16(
138
+ a_packed: torch.Tensor,
139
+ b_packed: torch.Tensor,
140
+ sfa: torch.Tensor,
141
+ sfb: torch.Tensor,
142
+ *,
143
+ alpha: float = 1.0,
144
+ warps: int = 4,
145
+ stages: int = 4,
146
+ out: Optional[torch.Tensor] = None,
147
+ ) -> torch.Tensor:
148
+ """Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
149
+
150
+ Splits K across warps inside one block with a shared-memory reduce -
151
+ no cross-block intermediate, so it stays safe under CUDA-graph
152
+ replay - and fills the SMs the tiled GEMM underfills at long-K
153
+ small-M decode shapes. Same packed/scale layouts as the linear
154
+ entry points."""
155
+ if out is None:
156
+ out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
157
+ ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
158
+ return out
159
+
160
+
161
  def fp4_w4a16_linear_bf16(
162
  a_packed: torch.Tensor,
163
  b_packed: torch.Tensor,
 
276
  __all__ = [
277
  "dequantize_fp4_sfa_fp16",
278
  "fp4_w4a16_linear_bf16",
279
+ "fp4_w4a4_gemv_warpsplit_bf16",
280
  "nvfp4_gemm_bf16",
281
  "nvfp4_gemm_bias_gelu_bf16",
282
  "nvfp4_gemm_bias_gelu_nvfp4",
build/torch213-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:07bd157549def1de48d1f23be0e1c365cfe9af099f15e99d0ec3f17f1acf2f1d
3
- size 2212960
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f66e3a05532f5978a4fec614d526887cffff3c527f1ba622cf8472d3f2fabc7d
3
+ size 2391864
build/torch213-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_4c95454
3
- ops = torch.ops._fp4_gemm_cuda_4c95454
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_4c95454::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_bc05275
3
+ ops = torch.ops._fp4_gemm_cuda_bc05275
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_bc05275::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_4c95454",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "EJ8HPhVwOKhifotSgdwCmB6MRbtnP1S0QfUGOVascy0=",
18
- "_fp4_gemm_cuda_4c95454.abi3.so": "B70VdUne8d5I0fI74OHDZc/prwmfFemdDsPxfxrPLx0=",
19
- "_ops.py": "zg1yY7ZypjpIvhFjVWntmzhlu2W/m2dDxA5HuBl+2Hw="
20
  }
21
  },
22
  "provenance": {
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "4c95454360b18c79bed75ec6346a7bac47ae2ca2",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_bc05275",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
18
+ "_fp4_gemm_cuda_bc05275.abi3.so": "9m46BVMvWXik/sYU1SaIfP//PFJ/G6Yiz4Ry0/L6vH0=",
19
+ "_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
20
  }
21
  },
22
  "provenance": {
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
30
  "dirty": false
31
  }
32
  }
build/torch213-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -36,6 +36,15 @@ def _linear_fake(
36
  return None
37
 
38
 
 
 
 
 
 
 
 
 
 
39
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
40
  def _legacy_linear_fake(
41
  a_packed: torch.Tensor,
@@ -125,6 +134,30 @@ def nvfp4_gemm_bf16(
125
  return out
126
 
127
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  def fp4_w4a16_linear_bf16(
129
  a_packed: torch.Tensor,
130
  b_packed: torch.Tensor,
@@ -243,6 +276,7 @@ def nvfp4_gemm_streamk_bias_bf16(
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
 
246
  "nvfp4_gemm_bf16",
247
  "nvfp4_gemm_bias_gelu_bf16",
248
  "nvfp4_gemm_bias_gelu_nvfp4",
 
36
  return None
37
 
38
 
39
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
40
+ def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
41
+ if a_packed.shape[0] != 1:
42
+ raise RuntimeError("warp-split GEMV serves M=1 only")
43
+ if out.shape != (1, b_packed.shape[0]):
44
+ raise RuntimeError("out must have shape (1, N)")
45
+ return None
46
+
47
+
48
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
49
  def _legacy_linear_fake(
50
  a_packed: torch.Tensor,
 
134
  return out
135
 
136
 
137
+ def fp4_w4a4_gemv_warpsplit_bf16(
138
+ a_packed: torch.Tensor,
139
+ b_packed: torch.Tensor,
140
+ sfa: torch.Tensor,
141
+ sfb: torch.Tensor,
142
+ *,
143
+ alpha: float = 1.0,
144
+ warps: int = 4,
145
+ stages: int = 4,
146
+ out: Optional[torch.Tensor] = None,
147
+ ) -> torch.Tensor:
148
+ """Warp-split-K NVFP4 W4A4 GEMV for the M=1 decode row (SM120).
149
+
150
+ Splits K across warps inside one block with a shared-memory reduce -
151
+ no cross-block intermediate, so it stays safe under CUDA-graph
152
+ replay - and fills the SMs the tiled GEMM underfills at long-K
153
+ small-M decode shapes. Same packed/scale layouts as the linear
154
+ entry points."""
155
+ if out is None:
156
+ out = torch.empty((1, b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
157
+ ops.fp4_w4a4_gemv_warpsplit_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
158
+ return out
159
+
160
+
161
  def fp4_w4a16_linear_bf16(
162
  a_packed: torch.Tensor,
163
  b_packed: torch.Tensor,
 
276
  __all__ = [
277
  "dequantize_fp4_sfa_fp16",
278
  "fp4_w4a16_linear_bf16",
279
+ "fp4_w4a4_gemv_warpsplit_bf16",
280
  "nvfp4_gemm_bf16",
281
  "nvfp4_gemm_bias_gelu_bf16",
282
  "nvfp4_gemm_bias_gelu_nvfp4",
build/torch213-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_4c95454.abi3.so → _fp4_gemm_cuda_bc05275.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:df29fbde46f73f2014d41c41360e8786bfdbd4a5585f2a0e8307f0adcd0f487e
3
- size 2217008
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e389511593a98a8aec2b708ef40ae2dca175c36be86cf8dc7d6ccb31f0753542
3
+ size 2387712
build/torch213-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_4c95454
3
- ops = torch.ops._fp4_gemm_cuda_4c95454
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_4c95454::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_bc05275
3
+ ops = torch.ops._fp4_gemm_cuda_bc05275
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_bc05275::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_4c95454",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "EJ8HPhVwOKhifotSgdwCmB6MRbtnP1S0QfUGOVascy0=",
18
- "_fp4_gemm_cuda_4c95454.abi3.so": "3yn73kb3PyAU1BxBNg6Hhr/b1KVYXyoOgwfwrc0PSH4=",
19
- "_ops.py": "zg1yY7ZypjpIvhFjVWntmzhlu2W/m2dDxA5HuBl+2Hw="
20
  }
21
  },
22
  "provenance": {
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "4c95454360b18c79bed75ec6346a7bac47ae2ca2",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_bc05275",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "RgO5rk/tnxl/Js0AdmwYVlHrJI/4znyBjoeomRSrQ+g=",
18
+ "_fp4_gemm_cuda_bc05275.abi3.so": "44lRFZOpiorsK3CO9Ari3KF1w2vobPjcfWzLMfB1NUI=",
19
+ "_ops.py": "5jLjoORk6CgQDTJohVml4gpqqUTXue40IIMbNn8LjP4="
20
  }
21
  },
22
  "provenance": {
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "bc052754b4beee9afe1008c0fb6bdd50300a3b27",
30
  "dirty": false
31
  }
32
  }