liangsu9988 commited on
Commit
0821e64
·
verified ·
1 Parent(s): bf1cd65

Uploaded using `kernel-builder`.

Browse files
Files changed (27) hide show
  1. benchmarks/benchmark.py +43 -2
  2. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +132 -0
  3. build/torch211-cxx11-cu128-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} +2 -2
  4. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  5. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +5 -5
  6. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +132 -0
  7. build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} +2 -2
  8. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  9. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  10. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +132 -0
  11. build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} +2 -2
  12. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  13. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  14. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +132 -0
  15. build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} +2 -2
  16. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  17. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +5 -5
  18. build/torch213-cxx11-cu130-x86_64-linux/__init__.py +132 -0
  19. build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_752e924.abi3.so +0 -3
  20. build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_7781728.abi3.so +3 -0
  21. build/torch213-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  22. build/torch213-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  23. build/torch213-cxx11-cu132-x86_64-linux/__init__.py +132 -0
  24. build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_752e924.abi3.so +0 -3
  25. build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_7781728.abi3.so +3 -0
  26. build/torch213-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  27. build/torch213-cxx11-cu132-x86_64-linux/metadata.json +5 -5
benchmarks/benchmark.py CHANGED
@@ -24,11 +24,13 @@ class BenchResult:
24
  N: int
25
  K: int
26
  variant: int
 
27
  flashrt_us: float
28
  torch_eager_us: float
29
  torch_compile_us: float
30
  speedup_vs_eager: float
31
  speedup_vs_compile: float
 
32
  max_abs: float
33
  mean_abs: float
34
  p99_abs: float
@@ -60,7 +62,7 @@ def measure(fn, warmup: int, iters: int) -> float:
60
  return float(start.elapsed_time(end) * 1000.0 / iters)
61
 
62
 
63
- def bench_case(helpers, ops, name: str, shape: tuple[int, int, int], warmup: int, iters: int) -> list[BenchResult]:
64
  m, n, k = shape
65
  a_packed, b_packed, sfa, sfb, expected = helpers.prepare_quantized(ops, m, n, k)
66
  a_deq = torch.empty((m, k), device="cuda", dtype=torch.float16)
@@ -75,6 +77,7 @@ def bench_case(helpers, ops, name: str, shape: tuple[int, int, int], warmup: int
75
  torch_eager_us = measure(torch_ref, warmup, iters)
76
  compiled_ref = torch.compile(torch_ref, mode="max-autotune-no-cudagraphs")
77
  torch_compile_us = measure(compiled_ref, warmup, iters)
 
78
  results: list[BenchResult] = []
79
  for variant in (0, 1, 2):
80
  out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
@@ -86,6 +89,29 @@ def bench_case(helpers, ops, name: str, shape: tuple[int, int, int], warmup: int
86
  warmup,
87
  iters,
88
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  results.append(
90
  BenchResult(
91
  shape=name,
@@ -93,11 +119,13 @@ def bench_case(helpers, ops, name: str, shape: tuple[int, int, int], warmup: int
93
  N=n,
94
  K=k,
95
  variant=variant,
 
96
  flashrt_us=flashrt_us,
97
  torch_eager_us=torch_eager_us,
98
  torch_compile_us=torch_compile_us,
99
  speedup_vs_eager=torch_eager_us / flashrt_us,
100
  speedup_vs_compile=torch_compile_us / flashrt_us,
 
101
  max_abs=max_abs,
102
  mean_abs=mean_abs,
103
  p99_abs=p99_abs,
@@ -119,6 +147,11 @@ def main() -> int:
119
  args = parser.parse_args()
120
 
121
  helpers = load_helpers()
 
 
 
 
 
122
  ops = (
123
  helpers.load_source_ops()
124
  if args.backend == "source"
@@ -128,12 +161,20 @@ def main() -> int:
128
  "small_m16_n128_k128": (16, 128, 128),
129
  "small_m32_n256_k256": (32, 256, 256),
130
  "mlp_tile_m64_n512_k512": (64, 512, 512),
 
 
 
 
131
  }
132
  if args.mode == "smoke":
133
  shapes = {"small_m16_n128_k128": shapes["small_m16_n128_k128"]}
134
  results: list[BenchResult] = []
135
  for name, shape in shapes.items():
136
- results.extend(bench_case(helpers, ops, name, shape, args.warmup, args.iterations))
 
 
 
 
137
  payload = {
138
  "mode": args.mode,
139
  "backend": args.backend,
 
24
  N: int
25
  K: int
26
  variant: int
27
+ native_us: float
28
  flashrt_us: float
29
  torch_eager_us: float
30
  torch_compile_us: float
31
  speedup_vs_eager: float
32
  speedup_vs_compile: float
33
+ wrapper_over_native: float
34
  max_abs: float
35
  mean_abs: float
36
  p99_abs: float
 
62
  return float(start.elapsed_time(end) * 1000.0 / iters)
63
 
64
 
65
+ def bench_case(helpers, ops, native, name: str, shape: tuple[int, int, int], warmup: int, iters: int) -> list[BenchResult]:
66
  m, n, k = shape
67
  a_packed, b_packed, sfa, sfb, expected = helpers.prepare_quantized(ops, m, n, k)
68
  a_deq = torch.empty((m, k), device="cuda", dtype=torch.float16)
 
77
  torch_eager_us = measure(torch_ref, warmup, iters)
78
  compiled_ref = torch.compile(torch_ref, mode="max-autotune-no-cudagraphs")
79
  torch_compile_us = measure(compiled_ref, warmup, iters)
80
+ stream = torch.cuda.current_stream().cuda_stream
81
  results: list[BenchResult] = []
82
  for variant in (0, 1, 2):
83
  out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
 
89
  warmup,
90
  iters,
91
  )
92
+ native_function = (
93
+ native.fp4_w4a16_gemm_sm120_bf16out
94
+ if variant == 0
95
+ else native.fp4_w4a16_gemm_sm120_bf16out_widen
96
+ if variant == 1
97
+ else native.fp4_w4a16_gemm_sm120_bf16out_pingpong
98
+ )
99
+ native_us = measure(
100
+ lambda: native_function(
101
+ a_packed.data_ptr(),
102
+ b_packed.data_ptr(),
103
+ out.data_ptr(),
104
+ m,
105
+ n,
106
+ k,
107
+ sfa.data_ptr(),
108
+ sfb.data_ptr(),
109
+ 1.0,
110
+ stream,
111
+ ),
112
+ warmup,
113
+ iters,
114
+ )
115
  results.append(
116
  BenchResult(
117
  shape=name,
 
119
  N=n,
120
  K=k,
121
  variant=variant,
122
+ native_us=native_us,
123
  flashrt_us=flashrt_us,
124
  torch_eager_us=torch_eager_us,
125
  torch_compile_us=torch_compile_us,
126
  speedup_vs_eager=torch_eager_us / flashrt_us,
127
  speedup_vs_compile=torch_compile_us / flashrt_us,
128
+ wrapper_over_native=flashrt_us / native_us,
129
  max_abs=max_abs,
130
  mean_abs=mean_abs,
131
  p99_abs=p99_abs,
 
147
  args = parser.parse_args()
148
 
149
  helpers = load_helpers()
150
+ sys.path.insert(0, str(ROOT.parent / "official" / "FlashRT"))
151
+ try:
152
+ import flash_rt.flash_rt_kernels as native
153
+ finally:
154
+ sys.path.pop(0)
155
  ops = (
156
  helpers.load_source_ops()
157
  if args.backend == "source"
 
161
  "small_m16_n128_k128": (16, 128, 128),
162
  "small_m32_n256_k256": (32, 256, 256),
163
  "mlp_tile_m64_n512_k512": (64, 512, 512),
164
+ "groot_dit_projection": (51, 1536, 1536),
165
+ "vla_projection": (105, 2048, 2048),
166
+ "motus_up": (360, 14336, 3072),
167
+ "motus_down": (360, 3072, 14336),
168
  }
169
  if args.mode == "smoke":
170
  shapes = {"small_m16_n128_k128": shapes["small_m16_n128_k128"]}
171
  results: list[BenchResult] = []
172
  for name, shape in shapes.items():
173
+ results.extend(
174
+ bench_case(
175
+ helpers, ops, native, name, shape, args.warmup, args.iterations
176
+ )
177
+ )
178
  payload = {
179
  "mode": args.mode,
180
  "backend": args.backend,
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -59,6 +59,33 @@ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is
59
  return None
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def quantize_fp4_sfa_fp16(
63
  x: torch.Tensor,
64
  packed: torch.Tensor | None = None,
@@ -113,10 +140,115 @@ def fp4_w4a16_linear_bf16(
113
  )
114
 
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  __all__ = [
117
  "dequantize_fp4_sfa_fp16",
118
  "fp4_w4a16_linear_bf16",
119
  "nvfp4_gemm_bf16",
 
 
 
 
 
120
  "quantize_fp4_sfa_fp16",
121
  "sfa_size_bytes",
122
  ]
 
59
  return None
60
 
61
 
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
  def quantize_fp4_sfa_fp16(
90
  x: torch.Tensor,
91
  packed: torch.Tensor | None = None,
 
140
  )
141
 
142
 
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
246
  "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
  "quantize_fp4_sfa_fp16",
253
  "sfa_size_bytes",
254
  ]
build/torch211-cxx11-cu128-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:2b2591080622003318c532d593c7f0738803d71d75953ee7331c3fab57e48e3a
3
- size 671784
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:43c2606ea93854faa4bd43f78835977beed5ddd1a005411ab41dc79988d99920
3
+ size 1633016
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_752e924
3
- ops = torch.ops._fp4_gemm_cuda_752e924
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_752e924::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_752e924",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "12G0sUObMOVtJ0cD3xF6s+xKU/n1ciO3m7nh1TZkS7w=",
17
- "_fp4_gemm_cuda_752e924.abi3.so": "KyWRCAYiADMYxTLVk8fwc4gD1x11lT7nMxw/q1fkjjo=",
18
- "_ops.py": "u0gVG4RRqNtjB3WknduXgbVTAnrWkWVGrWIfvCSaaXA=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "752e9241351caea5ae5c13bf8b51f1daab967bee",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "FD6tCd6u2NRYfLKq1Uz7OYi+tZV5o9GDAb+KN3GX2oU=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "Q8Jgbqk4VPqkvUP3iDWXe+7V3dGgBUEatB3HmYjZmSA=",
18
+ "_ops.py": "4wRBLd5HIJqgJskLFCRBmKJpvQfYeE9AmKTqIPQtPnw=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "77817286c522d8f7abf8d2cd873e1f84a79357b9",
30
  "dirty": false
31
  }
32
  }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -59,6 +59,33 @@ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is
59
  return None
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def quantize_fp4_sfa_fp16(
63
  x: torch.Tensor,
64
  packed: torch.Tensor | None = None,
@@ -113,10 +140,115 @@ def fp4_w4a16_linear_bf16(
113
  )
114
 
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  __all__ = [
117
  "dequantize_fp4_sfa_fp16",
118
  "fp4_w4a16_linear_bf16",
119
  "nvfp4_gemm_bf16",
 
 
 
 
 
120
  "quantize_fp4_sfa_fp16",
121
  "sfa_size_bytes",
122
  ]
 
59
  return None
60
 
61
 
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
  def quantize_fp4_sfa_fp16(
90
  x: torch.Tensor,
91
  packed: torch.Tensor | None = None,
 
140
  )
141
 
142
 
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
246
  "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
  "quantize_fp4_sfa_fp16",
253
  "sfa_size_bytes",
254
  ]
build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:97398c771ed76118703b7e99ed1f757deb2ea0e65512c6548455008b74dd955b
3
- size 714360
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c48b26279edf27c786fa4acba75e8b66e410e2f0fbb6fa1c318bfa84e23c392
3
+ size 1721016
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_752e924
3
- ops = torch.ops._fp4_gemm_cuda_752e924
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_752e924::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_752e924",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "12G0sUObMOVtJ0cD3xF6s+xKU/n1ciO3m7nh1TZkS7w=",
17
- "_fp4_gemm_cuda_752e924.abi3.so": "lzmMdx7XYRhwO36Z7R91fesuoOZVEsZUhFUAi3TdlVs=",
18
- "_ops.py": "u0gVG4RRqNtjB3WknduXgbVTAnrWkWVGrWIfvCSaaXA=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "752e9241351caea5ae5c13bf8b51f1daab967bee",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "FD6tCd6u2NRYfLKq1Uz7OYi+tZV5o9GDAb+KN3GX2oU=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "TEiyYnnt8nx4b6SsunXotm5BDi8Pu2+hwxi/qE4jw5I=",
18
+ "_ops.py": "4wRBLd5HIJqgJskLFCRBmKJpvQfYeE9AmKTqIPQtPnw=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "77817286c522d8f7abf8d2cd873e1f84a79357b9",
30
  "dirty": false
31
  }
32
  }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -59,6 +59,33 @@ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is
59
  return None
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def quantize_fp4_sfa_fp16(
63
  x: torch.Tensor,
64
  packed: torch.Tensor | None = None,
@@ -113,10 +140,115 @@ def fp4_w4a16_linear_bf16(
113
  )
114
 
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  __all__ = [
117
  "dequantize_fp4_sfa_fp16",
118
  "fp4_w4a16_linear_bf16",
119
  "nvfp4_gemm_bf16",
 
 
 
 
 
120
  "quantize_fp4_sfa_fp16",
121
  "sfa_size_bytes",
122
  ]
 
59
  return None
60
 
61
 
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
  def quantize_fp4_sfa_fp16(
90
  x: torch.Tensor,
91
  packed: torch.Tensor | None = None,
 
140
  )
141
 
142
 
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
246
  "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
  "quantize_fp4_sfa_fp16",
253
  "sfa_size_bytes",
254
  ]
build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4a5f761dd4108c0aad7cd2ef8875be4fb74ebff498e4b8f456a56c1f54e2689a
3
- size 724760
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:613e52c9f5c1c0e3bc1ab0baa45cefcb88af74b53393cbb280bbc1d0128ecebe
3
+ size 1731384
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_752e924
3
- ops = torch.ops._fp4_gemm_cuda_752e924
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_752e924::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_752e924",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "12G0sUObMOVtJ0cD3xF6s+xKU/n1ciO3m7nh1TZkS7w=",
17
- "_fp4_gemm_cuda_752e924.abi3.so": "Sl92HdQQjAqtfNLviHW+T7dOv/SY5Lj0VqVsH1TiaJo=",
18
- "_ops.py": "u0gVG4RRqNtjB3WknduXgbVTAnrWkWVGrWIfvCSaaXA=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "752e9241351caea5ae5c13bf8b51f1daab967bee",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "FD6tCd6u2NRYfLKq1Uz7OYi+tZV5o9GDAb+KN3GX2oU=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "YT5SyfXBwOO8GrC6pFzvy4ivdLUzk8uygLvB0BKOzr4=",
18
+ "_ops.py": "4wRBLd5HIJqgJskLFCRBmKJpvQfYeE9AmKTqIPQtPnw=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "77817286c522d8f7abf8d2cd873e1f84a79357b9",
30
  "dirty": false
31
  }
32
  }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -59,6 +59,33 @@ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is
59
  return None
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def quantize_fp4_sfa_fp16(
63
  x: torch.Tensor,
64
  packed: torch.Tensor | None = None,
@@ -113,10 +140,115 @@ def fp4_w4a16_linear_bf16(
113
  )
114
 
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  __all__ = [
117
  "dequantize_fp4_sfa_fp16",
118
  "fp4_w4a16_linear_bf16",
119
  "nvfp4_gemm_bf16",
 
 
 
 
 
120
  "quantize_fp4_sfa_fp16",
121
  "sfa_size_bytes",
122
  ]
 
59
  return None
60
 
61
 
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
  def quantize_fp4_sfa_fp16(
90
  x: torch.Tensor,
91
  packed: torch.Tensor | None = None,
 
140
  )
141
 
142
 
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
246
  "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
  "quantize_fp4_sfa_fp16",
253
  "sfa_size_bytes",
254
  ]
build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_752e924.abi3.so → _fp4_gemm_cuda_7781728.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:99fe652b5e4b685c15570441145e6aae1175055697a3ffa17430b85ea1ea0b87
3
- size 724816
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7ece4673cfce8c42fe58afc11e2c05d23e88408fa6eff7b03df8852797d4350
3
+ size 1731432
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_752e924
3
- ops = torch.ops._fp4_gemm_cuda_752e924
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_752e924::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_752e924",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "12G0sUObMOVtJ0cD3xF6s+xKU/n1ciO3m7nh1TZkS7w=",
17
- "_fp4_gemm_cuda_752e924.abi3.so": "mf5lK15LaFwVVwRBFF5qrhF1BVaXo/+hdDC4XqHqC4c=",
18
- "_ops.py": "u0gVG4RRqNtjB3WknduXgbVTAnrWkWVGrWIfvCSaaXA=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "752e9241351caea5ae5c13bf8b51f1daab967bee",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "FD6tCd6u2NRYfLKq1Uz7OYi+tZV5o9GDAb+KN3GX2oU=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "p+zkZzz86MQv5Yr8EeLAXSPohAj6bv97A9+IUnl9Q1A=",
18
+ "_ops.py": "4wRBLd5HIJqgJskLFCRBmKJpvQfYeE9AmKTqIPQtPnw=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "77817286c522d8f7abf8d2cd873e1f84a79357b9",
30
  "dirty": false
31
  }
32
  }
build/torch213-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -59,6 +59,33 @@ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is
59
  return None
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def quantize_fp4_sfa_fp16(
63
  x: torch.Tensor,
64
  packed: torch.Tensor | None = None,
@@ -113,10 +140,115 @@ def fp4_w4a16_linear_bf16(
113
  )
114
 
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  __all__ = [
117
  "dequantize_fp4_sfa_fp16",
118
  "fp4_w4a16_linear_bf16",
119
  "nvfp4_gemm_bf16",
 
 
 
 
 
120
  "quantize_fp4_sfa_fp16",
121
  "sfa_size_bytes",
122
  ]
 
59
  return None
60
 
61
 
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
  def quantize_fp4_sfa_fp16(
90
  x: torch.Tensor,
91
  packed: torch.Tensor | None = None,
 
140
  )
141
 
142
 
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
246
  "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
  "quantize_fp4_sfa_fp16",
253
  "sfa_size_bytes",
254
  ]
build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_752e924.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:5bc5b957118e7c4d272ea8eff37b23d864fbf0327d579bb7d79ad05c6e6facdf
3
- size 724608
 
 
 
 
build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_7781728.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8094f1a754b442779a3422dca4cb8626602bb38a5e5a3804cff765d0247da470
3
+ size 1727128
build/torch213-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_752e924
3
- ops = torch.ops._fp4_gemm_cuda_752e924
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_752e924::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_752e924",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "12G0sUObMOVtJ0cD3xF6s+xKU/n1ciO3m7nh1TZkS7w=",
17
- "_fp4_gemm_cuda_752e924.abi3.so": "W8W5VxGOfE0nLqjv83sj2GT78DJ9V5u315rQXG5vrN8=",
18
- "_ops.py": "u0gVG4RRqNtjB3WknduXgbVTAnrWkWVGrWIfvCSaaXA=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "752e9241351caea5ae5c13bf8b51f1daab967bee",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "FD6tCd6u2NRYfLKq1Uz7OYi+tZV5o9GDAb+KN3GX2oU=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "gJTxp1S0QneaNCLcpMuGJmArs4peWjgEz/dl0CR9pHA=",
18
+ "_ops.py": "4wRBLd5HIJqgJskLFCRBmKJpvQfYeE9AmKTqIPQtPnw=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "77817286c522d8f7abf8d2cd873e1f84a79357b9",
30
  "dirty": false
31
  }
32
  }
build/torch213-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -59,6 +59,33 @@ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is
59
  return None
60
 
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  def quantize_fp4_sfa_fp16(
63
  x: torch.Tensor,
64
  packed: torch.Tensor | None = None,
@@ -113,10 +140,115 @@ def fp4_w4a16_linear_bf16(
113
  )
114
 
115
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  __all__ = [
117
  "dequantize_fp4_sfa_fp16",
118
  "fp4_w4a16_linear_bf16",
119
  "nvfp4_gemm_bf16",
 
 
 
 
 
120
  "quantize_fp4_sfa_fp16",
121
  "sfa_size_bytes",
122
  ]
 
59
  return None
60
 
61
 
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
  def quantize_fp4_sfa_fp16(
90
  x: torch.Tensor,
91
  packed: torch.Tensor | None = None,
 
140
  )
141
 
142
 
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
  __all__ = [
244
  "dequantize_fp4_sfa_fp16",
245
  "fp4_w4a16_linear_bf16",
246
  "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
  "quantize_fp4_sfa_fp16",
253
  "sfa_size_bytes",
254
  ]
build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_752e924.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:660955d1fb41a89b8822862c7b20caba35f289f6e068f9e83eab13810f82b49b
3
- size 724656
 
 
 
 
build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_7781728.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c9a7111f4e0f3db4a1766b796c906f968cf3b7253b2e3ba2ef2228263d19f5fd
3
+ size 1731272
build/torch213-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_752e924
3
- ops = torch.ops._fp4_gemm_cuda_752e924
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_752e924::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_752e924",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "12G0sUObMOVtJ0cD3xF6s+xKU/n1ciO3m7nh1TZkS7w=",
17
- "_fp4_gemm_cuda_752e924.abi3.so": "ZglV0ftBqJuIIoYseyDKujXyifbgaPnoPqsTgQ+CtJs=",
18
- "_ops.py": "u0gVG4RRqNtjB3WknduXgbVTAnrWkWVGrWIfvCSaaXA=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "752e9241351caea5ae5c13bf8b51f1daab967bee",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "FD6tCd6u2NRYfLKq1Uz7OYi+tZV5o9GDAb+KN3GX2oU=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "yacRH04PPbShdmt5bJBvlozztyU7Ljui7yIoJj0Z9f0=",
18
+ "_ops.py": "4wRBLd5HIJqgJskLFCRBmKJpvQfYeE9AmKTqIPQtPnw=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "77817286c522d8f7abf8d2cd873e1f84a79357b9",
30
  "dirty": false
31
  }
32
  }