liangsu9988 commited on
Commit
4667778
·
verified ·
1 Parent(s): ab02a6f

Uploaded using `kernel-builder`.

Browse files
Files changed (27) hide show
  1. benchmarks/benchmark.py +93 -6
  2. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +122 -0
  3. build/torch211-cxx11-cu128-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} +2 -2
  4. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  5. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +5 -5
  6. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +122 -0
  7. build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} +2 -2
  8. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  9. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  10. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +122 -0
  11. build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} +2 -2
  12. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  13. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  14. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +122 -0
  15. build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} +2 -2
  16. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  17. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +5 -5
  18. build/torch213-cxx11-cu130-x86_64-linux/__init__.py +122 -0
  19. build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_6971f7a.abi3.so +3 -0
  20. build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_dc57d15.abi3.so +0 -3
  21. build/torch213-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  22. build/torch213-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  23. build/torch213-cxx11-cu132-x86_64-linux/__init__.py +122 -0
  24. build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_6971f7a.abi3.so +3 -0
  25. build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_dc57d15.abi3.so +0 -3
  26. build/torch213-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  27. build/torch213-cxx11-cu132-x86_64-linux/metadata.json +5 -5
benchmarks/benchmark.py CHANGED
@@ -185,12 +185,79 @@ def bench_bf16_producer(ops, native, k: int, warmup: int, iters: int):
185
  }
186
 
187
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
  def main() -> int:
189
  parser = argparse.ArgumentParser()
190
  parser.add_argument("--backend", choices=["source", "installed"], default="source")
191
  parser.add_argument("--artifact", default=None)
192
  parser.add_argument(
193
- "--mode", choices=["smoke", "headline", "thor-models"], default="headline"
 
 
194
  )
195
  parser.add_argument("--warmup", type=int, default=20)
196
  parser.add_argument("--iterations", type=int, default=100)
@@ -198,6 +265,31 @@ def main() -> int:
198
  args = parser.parse_args()
199
 
200
  helpers = load_helpers()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
201
  native_root = Path(
202
  os.environ.get("FLASHRT_NATIVE_ROOT", str(ROOT.parent / "official" / "FlashRT"))
203
  )
@@ -206,11 +298,6 @@ def main() -> int:
206
  import flash_rt.flash_rt_kernels as native
207
  finally:
208
  sys.path.pop(0)
209
- ops = (
210
- helpers.load_source_ops()
211
- if args.backend == "source"
212
- else helpers.load_installed_ops(args.artifact)
213
- )
214
  shapes = {
215
  "small_m16_n128_k128": (16, 128, 128),
216
  "small_m32_n256_k256": (32, 256, 256),
 
185
  }
186
 
187
 
188
+ def bench_w4a16_vocab(ops, warmup: int, iters: int):
189
+ n, k = 152064, 5120
190
+ gen = torch.Generator(device="cuda").manual_seed(20260818)
191
+ weight = torch.randint(
192
+ 0, 256, (n, k // 2), device="cuda", dtype=torch.uint8,
193
+ generator=gen,
194
+ )
195
+ weight_scale = (
196
+ torch.rand((n, k // 16), device="cuda", generator=gen) * 0.1 + 0.01
197
+ ).to(torch.float8_e4m3fn)
198
+ global_scale = torch.tensor([0.00390625], device="cuda", dtype=torch.float32)
199
+ weight_marlin, scales_marlin, global_marlin, workspace = (
200
+ ops.adopt_w4a16_marlin(weight, weight_scale, global_scale)
201
+ )
202
+
203
+ vllm_ops = None
204
+ scalar_types = None
205
+ try:
206
+ from vllm import _custom_ops as vllm_ops
207
+ from vllm.scalar_type import scalar_types
208
+ except ImportError:
209
+ pass
210
+
211
+ rows = []
212
+ for m in (1, 7, 8, 16):
213
+ x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16)
214
+ out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16)
215
+
216
+ def flashrt_call():
217
+ ops.w4a16_marlin(
218
+ x, weight_marlin, scales_marlin, global_marlin, workspace, out
219
+ )
220
+
221
+ flashrt_call()
222
+ flashrt_us = measure(flashrt_call, warmup, iters)
223
+ row = {
224
+ "M": m,
225
+ "N": n,
226
+ "K": k,
227
+ "flashrt_us": flashrt_us,
228
+ "target_us": 400.0 if m == 8 else None,
229
+ "target_passed": bool(m != 8 or flashrt_us <= 400.0),
230
+ }
231
+ if vllm_ops is not None and scalar_types is not None:
232
+ native_out = torch.empty_like(out)
233
+
234
+ def native_call():
235
+ vllm_ops.marlin_gemm(
236
+ x, native_out, weight_marlin, None, scales_marlin, None,
237
+ global_marlin, None, None, None, workspace,
238
+ scalar_types.float4_e2m1f, m, n, k,
239
+ True, False, False, False,
240
+ )
241
+
242
+ native_call()
243
+ native_us = measure(native_call, warmup, iters)
244
+ row.update({
245
+ "vllm_native_us": native_us,
246
+ "wrapper_over_native": flashrt_us / native_us,
247
+ "exact_vs_vllm": bool(torch.equal(out, native_out)),
248
+ })
249
+ rows.append(row)
250
+ return rows
251
+
252
+
253
  def main() -> int:
254
  parser = argparse.ArgumentParser()
255
  parser.add_argument("--backend", choices=["source", "installed"], default="source")
256
  parser.add_argument("--artifact", default=None)
257
  parser.add_argument(
258
+ "--mode",
259
+ choices=["smoke", "headline", "thor-models", "w4a16-vocab"],
260
+ default="headline",
261
  )
262
  parser.add_argument("--warmup", type=int, default=20)
263
  parser.add_argument("--iterations", type=int, default=100)
 
265
  args = parser.parse_args()
266
 
267
  helpers = load_helpers()
268
+ ops = (
269
+ helpers.load_source_ops()
270
+ if args.backend == "source"
271
+ else helpers.load_installed_ops(args.artifact)
272
+ )
273
+ if args.mode == "w4a16-vocab":
274
+ rows = bench_w4a16_vocab(ops, args.warmup, args.iterations)
275
+ payload = {
276
+ "mode": args.mode,
277
+ "backend": args.backend,
278
+ "device": torch.cuda.get_device_name(),
279
+ "torch": torch.__version__,
280
+ "results": rows,
281
+ "passed": all(
282
+ row["target_passed"] and row.get("exact_vs_vllm", True)
283
+ for row in rows
284
+ ),
285
+ }
286
+ print(json.dumps(payload, indent=2))
287
+ if args.json_out:
288
+ out = Path(args.json_out)
289
+ out.parent.mkdir(parents=True, exist_ok=True)
290
+ out.write_text(json.dumps(payload, indent=2) + "\n")
291
+ return 0 if payload["passed"] else 1
292
+
293
  native_root = Path(
294
  os.environ.get("FLASHRT_NATIVE_ROOT", str(ROOT.parent / "official" / "FlashRT"))
295
  )
 
298
  import flash_rt.flash_rt_kernels as native
299
  finally:
300
  sys.path.pop(0)
 
 
 
 
 
301
  shapes = {
302
  "small_m16_n128_k128": (16, 128, 128),
303
  "small_m32_n256_k256": (32, 256, 256),
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,11 @@ def capabilities() -> dict[str, object]:
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
 
 
 
 
 
40
  "errors": "exceptions",
41
  }
42
 
@@ -172,6 +177,28 @@ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0)
172
  return None
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
176
  def _legacy_linear_fake(
177
  a_packed: torch.Tensor,
@@ -771,6 +798,98 @@ def nvfp4_gemm_m256_bf16(
771
  return out
772
 
773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  def fp4_w4a16_linear_bf16(
775
  a_packed: torch.Tensor,
776
  b_packed: torch.Tensor,
@@ -932,6 +1051,8 @@ __all__ = [
932
  "SUPPORTED_CUDA_CAPABILITIES",
933
  "SUPPORTED_LAYOUTS",
934
  "capabilities",
 
 
935
  "aligned_fp4_dim",
936
  "cutlass_fp4_gemm_geglu_il_hw_v10",
937
  "dequantize_fp4_sfa_fp16",
@@ -947,6 +1068,7 @@ __all__ = [
947
  "nvfp4_gemm_nvfp4",
948
  "nvfp4_gemm_m256_bf16",
949
  "nvfp4_gemm_m256_workspace_size",
 
950
  "nvfp4_gemm_geglu_nvfp4_fp16",
951
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
952
  "nvfp4_gemm_bias_residual_fp16",
 
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
+ "sm120_w4a16_marlin_m": (1, 16),
41
+ "sm120_w4a16_marlin_n_alignment": 64,
42
+ "sm120_w4a16_marlin_k_alignment": 128,
43
+ "sm120_w4a16_marlin_weight_layout": "marlin-repacked-nvfp4",
44
+ "sm120_w4a16_marlin_scale_layout": "marlin-permuted-fp8-e4m3-block16",
45
  "errors": "exceptions",
46
  }
47
 
 
177
  return None
178
 
179
 
180
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_bf16"))
181
+ def _w4a16_marlin_fake(x, weight, weight_scale, global_scale, workspace, out) -> None:
182
+ del weight, weight_scale, global_scale, workspace
183
+ if x.ndim != 2 or out.ndim != 2 or out.shape[0] != x.shape[0]:
184
+ raise RuntimeError("x/out must have shapes (M, K)/(M, N)")
185
+ if not 1 <= x.shape[0] <= 16:
186
+ raise RuntimeError("the Marlin W4A16 tier serves M in [1, 16]")
187
+ if x.shape[1] % 128 or out.shape[1] % 64:
188
+ raise RuntimeError("K/N must be divisible by 128/64")
189
+ return None
190
+
191
+
192
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_repack"))
193
+ def _w4a16_marlin_repack_fake(qweight_kn, weight_marlin) -> None:
194
+ if qweight_kn.ndim != 2:
195
+ raise RuntimeError("qweight_kn must have shape (K / 8, N)")
196
+ k, n = qweight_kn.shape[0] * 8, qweight_kn.shape[1]
197
+ if weight_marlin.shape != (k // 16, n * 2):
198
+ raise RuntimeError("weight_marlin must have shape (K / 16, 2 * N)")
199
+ return None
200
+
201
+
202
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
203
  def _legacy_linear_fake(
204
  a_packed: torch.Tensor,
 
798
  return out
799
 
800
 
801
+ def allocate_w4a16_marlin_workspace(device: torch.device | str) -> torch.Tensor:
802
+ """Allocate the persistent Marlin lock workspace before graph capture."""
803
+ props = torch.cuda.get_device_properties(device)
804
+ return torch.zeros(props.multi_processor_count, device=device, dtype=torch.int32)
805
+
806
+
807
+ def adopt_nvfp4_w4a16_marlin(
808
+ weight_packed: torch.Tensor,
809
+ weight_scale: torch.Tensor,
810
+ weight_scale_2: torch.Tensor,
811
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
812
+ """Convert standard ModelOpt NVFP4 tensors once at model bind time.
813
+
814
+ Inputs use ``weight[N,K/2]``, ``weight_scale[N,K/16]`` and scalar
815
+ ``weight_scale_2``. Returned tensors are the three runtime operands plus a
816
+ zeroed lock workspace. This function is intentionally outside the graph
817
+ hot path.
818
+ """
819
+ if weight_packed.dtype != torch.uint8 or weight_packed.ndim != 2:
820
+ raise ValueError("weight_packed must be uint8 with shape (N, K / 2)")
821
+ n, k = weight_packed.shape[0], weight_packed.shape[1] * 2
822
+ if k % 128 or n % 64:
823
+ raise ValueError("K/N must be divisible by 128/64")
824
+ if weight_scale.shape != (n, k // NVFP4_BLOCK_SIZE):
825
+ raise ValueError("weight_scale must have shape (N, K / 16)")
826
+ if weight_scale_2.numel() != 1:
827
+ raise ValueError("weight_scale_2 must contain one global scale")
828
+
829
+ qweight_kn = weight_packed.view(torch.int32).T.contiguous()
830
+ weight_marlin = torch.empty(
831
+ (k // 16, n * 2), device=weight_packed.device, dtype=torch.int32
832
+ )
833
+ ops.nvfp4_w4a16_marlin_repack(qweight_kn, weight_marlin)
834
+
835
+ scales = weight_scale.T.contiguous().to(torch.bfloat16)
836
+ scale_perm = [i + 8 * j for i in range(8) for j in range(8)]
837
+ scales = scales.reshape(-1, 64)[:, scale_perm].reshape(-1, n).contiguous()
838
+
839
+ # Match vLLM's NVFP4 Marlin contract exactly: scale permutation uses the
840
+ # model parameter dtype, then S0E5M3 encoding starts from FP16.
841
+ scales = scales.to(torch.float16)
842
+ scales = scales.view(-1, 4)[:, [0, 2, 1, 3]].reshape(scales.shape)
843
+ scaled = scales.float() * (2**7)
844
+ nonzero = scaled > 0
845
+ scale_factor = 1.0
846
+ if bool(nonzero.any()):
847
+ max_value = scaled[nonzero].max()
848
+ if bool(max_value < 448 * (2**7)):
849
+ scale_factor = float((448 * (2**7) / max_value).log2().floor().exp2())
850
+ if scale_factor > 1.0:
851
+ scales = (scales.float() * scale_factor).to(torch.float16)
852
+ scales = scales * (2**7)
853
+ scales[scales < 2] = 0
854
+ scales = (scales.view(torch.int16) << 1).view(torch.float8_e4m3fn)
855
+ scales = scales[:, 1::2].contiguous()
856
+
857
+ # Marlin's BF16 dequantizer uses exponent bias 126 and removes the seven
858
+ # bits introduced while encoding the special unsigned FP8 scale format.
859
+ global_scale = (
860
+ weight_scale_2.to(device=weight_packed.device, dtype=torch.float32)
861
+ * (2.0 ** 119)
862
+ / scale_factor
863
+ ).contiguous()
864
+ workspace = allocate_w4a16_marlin_workspace(weight_packed.device)
865
+ return weight_marlin, scales, global_scale, workspace
866
+
867
+
868
+ def nvfp4_w4a16_marlin_bf16(
869
+ x: torch.Tensor,
870
+ weight_marlin: torch.Tensor,
871
+ weight_scale_marlin: torch.Tensor,
872
+ weight_global_scale: torch.Tensor,
873
+ *,
874
+ workspace: torch.Tensor,
875
+ out: torch.Tensor | None = None,
876
+ ) -> torch.Tensor:
877
+ """Small-M BF16 x NVFP4 linear using bind-time Marlin layouts.
878
+
879
+ ``weight_marlin``, ``weight_scale_marlin`` and
880
+ ``weight_global_scale`` are the tensors produced by a standard NVFP4
881
+ Marlin adoption step. The operation supports M in [1, 16] and never
882
+ allocates internally, making it suitable for CUDA Graph replay.
883
+ """
884
+ if out is None:
885
+ n = weight_scale_marlin.numel() // (x.shape[1] // NVFP4_BLOCK_SIZE)
886
+ out = torch.empty((x.shape[0], n), device=x.device, dtype=torch.bfloat16)
887
+ ops.nvfp4_w4a16_marlin_bf16(
888
+ x, weight_marlin, weight_scale_marlin, weight_global_scale, workspace, out
889
+ )
890
+ return out
891
+
892
+
893
  def fp4_w4a16_linear_bf16(
894
  a_packed: torch.Tensor,
895
  b_packed: torch.Tensor,
 
1051
  "SUPPORTED_CUDA_CAPABILITIES",
1052
  "SUPPORTED_LAYOUTS",
1053
  "capabilities",
1054
+ "allocate_w4a16_marlin_workspace",
1055
+ "adopt_nvfp4_w4a16_marlin",
1056
  "aligned_fp4_dim",
1057
  "cutlass_fp4_gemm_geglu_il_hw_v10",
1058
  "dequantize_fp4_sfa_fp16",
 
1068
  "nvfp4_gemm_nvfp4",
1069
  "nvfp4_gemm_m256_bf16",
1070
  "nvfp4_gemm_m256_workspace_size",
1071
+ "nvfp4_w4a16_marlin_bf16",
1072
  "nvfp4_gemm_geglu_nvfp4_fp16",
1073
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
1074
  "nvfp4_gemm_bias_residual_fp16",
build/torch211-cxx11-cu128-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d1da1c9c5de85d62d2a39e29178eb3fb6e1fc4d772b04195fbcfd9683b63fcb5
3
- size 2721416
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f01d9ffadff9a04439f0a83b6095305d1e20d3903fd275624cd4b2c0001ceaaf
3
+ size 4915160
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_dc57d15
3
- ops = torch.ops._fp4_gemm_cuda_dc57d15
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_dc57d15::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_6971f7a
3
+ ops = torch.ops._fp4_gemm_cuda_6971f7a
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_6971f7a::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_dc57d15",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "XycjAPC24qtK6IH+GtF/jrPmWnB6DyNgSny26ep3w1Y=",
17
- "_fp4_gemm_cuda_dc57d15.abi3.so": "0docnF3oXWLSo54pF46z+24fxNdysEGV+8/ZaDtj/LU=",
18
- "_ops.py": "TM5HQsh2RfinhrEZ4U0XyWF2hFyrPWpxR79FQ41D5jk=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "dc57d15b8bb1d4b52d82012f7f789ae5ae6db17f",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_6971f7a",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "0hDN2W6X3IMoWIi5BBufrnqrq2yj5N1/q9wBhCwBRxA=",
17
+ "_fp4_gemm_cuda_6971f7a.abi3.so": "8B2f+t/5oEQ58Kg7YJUwXR4g05A/0nViTNSywAAc6q8=",
18
+ "_ops.py": "D7InzLughnNL5JQowMPNYpSpNPRFeeD8mlgs38Xw03g=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "6971f7a37410ba0d6edae93b389dbcbe3ba924d5",
30
  "dirty": false
31
  }
32
  }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,11 @@ def capabilities() -> dict[str, object]:
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
 
 
 
 
 
40
  "errors": "exceptions",
41
  }
42
 
@@ -172,6 +177,28 @@ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0)
172
  return None
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
176
  def _legacy_linear_fake(
177
  a_packed: torch.Tensor,
@@ -771,6 +798,98 @@ def nvfp4_gemm_m256_bf16(
771
  return out
772
 
773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  def fp4_w4a16_linear_bf16(
775
  a_packed: torch.Tensor,
776
  b_packed: torch.Tensor,
@@ -932,6 +1051,8 @@ __all__ = [
932
  "SUPPORTED_CUDA_CAPABILITIES",
933
  "SUPPORTED_LAYOUTS",
934
  "capabilities",
 
 
935
  "aligned_fp4_dim",
936
  "cutlass_fp4_gemm_geglu_il_hw_v10",
937
  "dequantize_fp4_sfa_fp16",
@@ -947,6 +1068,7 @@ __all__ = [
947
  "nvfp4_gemm_nvfp4",
948
  "nvfp4_gemm_m256_bf16",
949
  "nvfp4_gemm_m256_workspace_size",
 
950
  "nvfp4_gemm_geglu_nvfp4_fp16",
951
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
952
  "nvfp4_gemm_bias_residual_fp16",
 
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
+ "sm120_w4a16_marlin_m": (1, 16),
41
+ "sm120_w4a16_marlin_n_alignment": 64,
42
+ "sm120_w4a16_marlin_k_alignment": 128,
43
+ "sm120_w4a16_marlin_weight_layout": "marlin-repacked-nvfp4",
44
+ "sm120_w4a16_marlin_scale_layout": "marlin-permuted-fp8-e4m3-block16",
45
  "errors": "exceptions",
46
  }
47
 
 
177
  return None
178
 
179
 
180
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_bf16"))
181
+ def _w4a16_marlin_fake(x, weight, weight_scale, global_scale, workspace, out) -> None:
182
+ del weight, weight_scale, global_scale, workspace
183
+ if x.ndim != 2 or out.ndim != 2 or out.shape[0] != x.shape[0]:
184
+ raise RuntimeError("x/out must have shapes (M, K)/(M, N)")
185
+ if not 1 <= x.shape[0] <= 16:
186
+ raise RuntimeError("the Marlin W4A16 tier serves M in [1, 16]")
187
+ if x.shape[1] % 128 or out.shape[1] % 64:
188
+ raise RuntimeError("K/N must be divisible by 128/64")
189
+ return None
190
+
191
+
192
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_repack"))
193
+ def _w4a16_marlin_repack_fake(qweight_kn, weight_marlin) -> None:
194
+ if qweight_kn.ndim != 2:
195
+ raise RuntimeError("qweight_kn must have shape (K / 8, N)")
196
+ k, n = qweight_kn.shape[0] * 8, qweight_kn.shape[1]
197
+ if weight_marlin.shape != (k // 16, n * 2):
198
+ raise RuntimeError("weight_marlin must have shape (K / 16, 2 * N)")
199
+ return None
200
+
201
+
202
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
203
  def _legacy_linear_fake(
204
  a_packed: torch.Tensor,
 
798
  return out
799
 
800
 
801
+ def allocate_w4a16_marlin_workspace(device: torch.device | str) -> torch.Tensor:
802
+ """Allocate the persistent Marlin lock workspace before graph capture."""
803
+ props = torch.cuda.get_device_properties(device)
804
+ return torch.zeros(props.multi_processor_count, device=device, dtype=torch.int32)
805
+
806
+
807
+ def adopt_nvfp4_w4a16_marlin(
808
+ weight_packed: torch.Tensor,
809
+ weight_scale: torch.Tensor,
810
+ weight_scale_2: torch.Tensor,
811
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
812
+ """Convert standard ModelOpt NVFP4 tensors once at model bind time.
813
+
814
+ Inputs use ``weight[N,K/2]``, ``weight_scale[N,K/16]`` and scalar
815
+ ``weight_scale_2``. Returned tensors are the three runtime operands plus a
816
+ zeroed lock workspace. This function is intentionally outside the graph
817
+ hot path.
818
+ """
819
+ if weight_packed.dtype != torch.uint8 or weight_packed.ndim != 2:
820
+ raise ValueError("weight_packed must be uint8 with shape (N, K / 2)")
821
+ n, k = weight_packed.shape[0], weight_packed.shape[1] * 2
822
+ if k % 128 or n % 64:
823
+ raise ValueError("K/N must be divisible by 128/64")
824
+ if weight_scale.shape != (n, k // NVFP4_BLOCK_SIZE):
825
+ raise ValueError("weight_scale must have shape (N, K / 16)")
826
+ if weight_scale_2.numel() != 1:
827
+ raise ValueError("weight_scale_2 must contain one global scale")
828
+
829
+ qweight_kn = weight_packed.view(torch.int32).T.contiguous()
830
+ weight_marlin = torch.empty(
831
+ (k // 16, n * 2), device=weight_packed.device, dtype=torch.int32
832
+ )
833
+ ops.nvfp4_w4a16_marlin_repack(qweight_kn, weight_marlin)
834
+
835
+ scales = weight_scale.T.contiguous().to(torch.bfloat16)
836
+ scale_perm = [i + 8 * j for i in range(8) for j in range(8)]
837
+ scales = scales.reshape(-1, 64)[:, scale_perm].reshape(-1, n).contiguous()
838
+
839
+ # Match vLLM's NVFP4 Marlin contract exactly: scale permutation uses the
840
+ # model parameter dtype, then S0E5M3 encoding starts from FP16.
841
+ scales = scales.to(torch.float16)
842
+ scales = scales.view(-1, 4)[:, [0, 2, 1, 3]].reshape(scales.shape)
843
+ scaled = scales.float() * (2**7)
844
+ nonzero = scaled > 0
845
+ scale_factor = 1.0
846
+ if bool(nonzero.any()):
847
+ max_value = scaled[nonzero].max()
848
+ if bool(max_value < 448 * (2**7)):
849
+ scale_factor = float((448 * (2**7) / max_value).log2().floor().exp2())
850
+ if scale_factor > 1.0:
851
+ scales = (scales.float() * scale_factor).to(torch.float16)
852
+ scales = scales * (2**7)
853
+ scales[scales < 2] = 0
854
+ scales = (scales.view(torch.int16) << 1).view(torch.float8_e4m3fn)
855
+ scales = scales[:, 1::2].contiguous()
856
+
857
+ # Marlin's BF16 dequantizer uses exponent bias 126 and removes the seven
858
+ # bits introduced while encoding the special unsigned FP8 scale format.
859
+ global_scale = (
860
+ weight_scale_2.to(device=weight_packed.device, dtype=torch.float32)
861
+ * (2.0 ** 119)
862
+ / scale_factor
863
+ ).contiguous()
864
+ workspace = allocate_w4a16_marlin_workspace(weight_packed.device)
865
+ return weight_marlin, scales, global_scale, workspace
866
+
867
+
868
+ def nvfp4_w4a16_marlin_bf16(
869
+ x: torch.Tensor,
870
+ weight_marlin: torch.Tensor,
871
+ weight_scale_marlin: torch.Tensor,
872
+ weight_global_scale: torch.Tensor,
873
+ *,
874
+ workspace: torch.Tensor,
875
+ out: torch.Tensor | None = None,
876
+ ) -> torch.Tensor:
877
+ """Small-M BF16 x NVFP4 linear using bind-time Marlin layouts.
878
+
879
+ ``weight_marlin``, ``weight_scale_marlin`` and
880
+ ``weight_global_scale`` are the tensors produced by a standard NVFP4
881
+ Marlin adoption step. The operation supports M in [1, 16] and never
882
+ allocates internally, making it suitable for CUDA Graph replay.
883
+ """
884
+ if out is None:
885
+ n = weight_scale_marlin.numel() // (x.shape[1] // NVFP4_BLOCK_SIZE)
886
+ out = torch.empty((x.shape[0], n), device=x.device, dtype=torch.bfloat16)
887
+ ops.nvfp4_w4a16_marlin_bf16(
888
+ x, weight_marlin, weight_scale_marlin, weight_global_scale, workspace, out
889
+ )
890
+ return out
891
+
892
+
893
  def fp4_w4a16_linear_bf16(
894
  a_packed: torch.Tensor,
895
  b_packed: torch.Tensor,
 
1051
  "SUPPORTED_CUDA_CAPABILITIES",
1052
  "SUPPORTED_LAYOUTS",
1053
  "capabilities",
1054
+ "allocate_w4a16_marlin_workspace",
1055
+ "adopt_nvfp4_w4a16_marlin",
1056
  "aligned_fp4_dim",
1057
  "cutlass_fp4_gemm_geglu_il_hw_v10",
1058
  "dequantize_fp4_sfa_fp16",
 
1068
  "nvfp4_gemm_nvfp4",
1069
  "nvfp4_gemm_m256_bf16",
1070
  "nvfp4_gemm_m256_workspace_size",
1071
+ "nvfp4_w4a16_marlin_bf16",
1072
  "nvfp4_gemm_geglu_nvfp4_fp16",
1073
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
1074
  "nvfp4_gemm_bias_residual_fp16",
build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:687ec5c797f1519fed83a51f82bc7ecf53059fbe35843981d86fefe4143c3b87
3
- size 8564792
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:555ecae796a12b3fed4ebc261381ef859509f47967e7fc84901fbbca05b31923
3
+ size 10756144
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_dc57d15
3
- ops = torch.ops._fp4_gemm_cuda_dc57d15
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_dc57d15::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_6971f7a
3
+ ops = torch.ops._fp4_gemm_cuda_6971f7a
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_6971f7a::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_dc57d15",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "XycjAPC24qtK6IH+GtF/jrPmWnB6DyNgSny26ep3w1Y=",
18
- "_fp4_gemm_cuda_dc57d15.abi3.so": "aH7Fx5fxUZ/tg6Ufgrx+z1MFn741hDmB2G/v5BQ8O4c=",
19
- "_ops.py": "TM5HQsh2RfinhrEZ4U0XyWF2hFyrPWpxR79FQ41D5jk=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
@@ -27,7 +27,7 @@
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "dc57d15b8bb1d4b52d82012f7f789ae5ae6db17f",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_6971f7a",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "0hDN2W6X3IMoWIi5BBufrnqrq2yj5N1/q9wBhCwBRxA=",
18
+ "_fp4_gemm_cuda_6971f7a.abi3.so": "VV7K55ahKz/tTrwmE4HvhZUJ9Hln5/yEkB+7ygWzGSM=",
19
+ "_ops.py": "D7InzLughnNL5JQowMPNYpSpNPRFeeD8mlgs38Xw03g=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
 
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "6971f7a37410ba0d6edae93b389dbcbe3ba924d5",
31
  "dirty": false
32
  }
33
  }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,11 @@ def capabilities() -> dict[str, object]:
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
 
 
 
 
 
40
  "errors": "exceptions",
41
  }
42
 
@@ -172,6 +177,28 @@ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0)
172
  return None
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
176
  def _legacy_linear_fake(
177
  a_packed: torch.Tensor,
@@ -771,6 +798,98 @@ def nvfp4_gemm_m256_bf16(
771
  return out
772
 
773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  def fp4_w4a16_linear_bf16(
775
  a_packed: torch.Tensor,
776
  b_packed: torch.Tensor,
@@ -932,6 +1051,8 @@ __all__ = [
932
  "SUPPORTED_CUDA_CAPABILITIES",
933
  "SUPPORTED_LAYOUTS",
934
  "capabilities",
 
 
935
  "aligned_fp4_dim",
936
  "cutlass_fp4_gemm_geglu_il_hw_v10",
937
  "dequantize_fp4_sfa_fp16",
@@ -947,6 +1068,7 @@ __all__ = [
947
  "nvfp4_gemm_nvfp4",
948
  "nvfp4_gemm_m256_bf16",
949
  "nvfp4_gemm_m256_workspace_size",
 
950
  "nvfp4_gemm_geglu_nvfp4_fp16",
951
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
952
  "nvfp4_gemm_bias_residual_fp16",
 
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
+ "sm120_w4a16_marlin_m": (1, 16),
41
+ "sm120_w4a16_marlin_n_alignment": 64,
42
+ "sm120_w4a16_marlin_k_alignment": 128,
43
+ "sm120_w4a16_marlin_weight_layout": "marlin-repacked-nvfp4",
44
+ "sm120_w4a16_marlin_scale_layout": "marlin-permuted-fp8-e4m3-block16",
45
  "errors": "exceptions",
46
  }
47
 
 
177
  return None
178
 
179
 
180
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_bf16"))
181
+ def _w4a16_marlin_fake(x, weight, weight_scale, global_scale, workspace, out) -> None:
182
+ del weight, weight_scale, global_scale, workspace
183
+ if x.ndim != 2 or out.ndim != 2 or out.shape[0] != x.shape[0]:
184
+ raise RuntimeError("x/out must have shapes (M, K)/(M, N)")
185
+ if not 1 <= x.shape[0] <= 16:
186
+ raise RuntimeError("the Marlin W4A16 tier serves M in [1, 16]")
187
+ if x.shape[1] % 128 or out.shape[1] % 64:
188
+ raise RuntimeError("K/N must be divisible by 128/64")
189
+ return None
190
+
191
+
192
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_repack"))
193
+ def _w4a16_marlin_repack_fake(qweight_kn, weight_marlin) -> None:
194
+ if qweight_kn.ndim != 2:
195
+ raise RuntimeError("qweight_kn must have shape (K / 8, N)")
196
+ k, n = qweight_kn.shape[0] * 8, qweight_kn.shape[1]
197
+ if weight_marlin.shape != (k // 16, n * 2):
198
+ raise RuntimeError("weight_marlin must have shape (K / 16, 2 * N)")
199
+ return None
200
+
201
+
202
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
203
  def _legacy_linear_fake(
204
  a_packed: torch.Tensor,
 
798
  return out
799
 
800
 
801
+ def allocate_w4a16_marlin_workspace(device: torch.device | str) -> torch.Tensor:
802
+ """Allocate the persistent Marlin lock workspace before graph capture."""
803
+ props = torch.cuda.get_device_properties(device)
804
+ return torch.zeros(props.multi_processor_count, device=device, dtype=torch.int32)
805
+
806
+
807
+ def adopt_nvfp4_w4a16_marlin(
808
+ weight_packed: torch.Tensor,
809
+ weight_scale: torch.Tensor,
810
+ weight_scale_2: torch.Tensor,
811
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
812
+ """Convert standard ModelOpt NVFP4 tensors once at model bind time.
813
+
814
+ Inputs use ``weight[N,K/2]``, ``weight_scale[N,K/16]`` and scalar
815
+ ``weight_scale_2``. Returned tensors are the three runtime operands plus a
816
+ zeroed lock workspace. This function is intentionally outside the graph
817
+ hot path.
818
+ """
819
+ if weight_packed.dtype != torch.uint8 or weight_packed.ndim != 2:
820
+ raise ValueError("weight_packed must be uint8 with shape (N, K / 2)")
821
+ n, k = weight_packed.shape[0], weight_packed.shape[1] * 2
822
+ if k % 128 or n % 64:
823
+ raise ValueError("K/N must be divisible by 128/64")
824
+ if weight_scale.shape != (n, k // NVFP4_BLOCK_SIZE):
825
+ raise ValueError("weight_scale must have shape (N, K / 16)")
826
+ if weight_scale_2.numel() != 1:
827
+ raise ValueError("weight_scale_2 must contain one global scale")
828
+
829
+ qweight_kn = weight_packed.view(torch.int32).T.contiguous()
830
+ weight_marlin = torch.empty(
831
+ (k // 16, n * 2), device=weight_packed.device, dtype=torch.int32
832
+ )
833
+ ops.nvfp4_w4a16_marlin_repack(qweight_kn, weight_marlin)
834
+
835
+ scales = weight_scale.T.contiguous().to(torch.bfloat16)
836
+ scale_perm = [i + 8 * j for i in range(8) for j in range(8)]
837
+ scales = scales.reshape(-1, 64)[:, scale_perm].reshape(-1, n).contiguous()
838
+
839
+ # Match vLLM's NVFP4 Marlin contract exactly: scale permutation uses the
840
+ # model parameter dtype, then S0E5M3 encoding starts from FP16.
841
+ scales = scales.to(torch.float16)
842
+ scales = scales.view(-1, 4)[:, [0, 2, 1, 3]].reshape(scales.shape)
843
+ scaled = scales.float() * (2**7)
844
+ nonzero = scaled > 0
845
+ scale_factor = 1.0
846
+ if bool(nonzero.any()):
847
+ max_value = scaled[nonzero].max()
848
+ if bool(max_value < 448 * (2**7)):
849
+ scale_factor = float((448 * (2**7) / max_value).log2().floor().exp2())
850
+ if scale_factor > 1.0:
851
+ scales = (scales.float() * scale_factor).to(torch.float16)
852
+ scales = scales * (2**7)
853
+ scales[scales < 2] = 0
854
+ scales = (scales.view(torch.int16) << 1).view(torch.float8_e4m3fn)
855
+ scales = scales[:, 1::2].contiguous()
856
+
857
+ # Marlin's BF16 dequantizer uses exponent bias 126 and removes the seven
858
+ # bits introduced while encoding the special unsigned FP8 scale format.
859
+ global_scale = (
860
+ weight_scale_2.to(device=weight_packed.device, dtype=torch.float32)
861
+ * (2.0 ** 119)
862
+ / scale_factor
863
+ ).contiguous()
864
+ workspace = allocate_w4a16_marlin_workspace(weight_packed.device)
865
+ return weight_marlin, scales, global_scale, workspace
866
+
867
+
868
+ def nvfp4_w4a16_marlin_bf16(
869
+ x: torch.Tensor,
870
+ weight_marlin: torch.Tensor,
871
+ weight_scale_marlin: torch.Tensor,
872
+ weight_global_scale: torch.Tensor,
873
+ *,
874
+ workspace: torch.Tensor,
875
+ out: torch.Tensor | None = None,
876
+ ) -> torch.Tensor:
877
+ """Small-M BF16 x NVFP4 linear using bind-time Marlin layouts.
878
+
879
+ ``weight_marlin``, ``weight_scale_marlin`` and
880
+ ``weight_global_scale`` are the tensors produced by a standard NVFP4
881
+ Marlin adoption step. The operation supports M in [1, 16] and never
882
+ allocates internally, making it suitable for CUDA Graph replay.
883
+ """
884
+ if out is None:
885
+ n = weight_scale_marlin.numel() // (x.shape[1] // NVFP4_BLOCK_SIZE)
886
+ out = torch.empty((x.shape[0], n), device=x.device, dtype=torch.bfloat16)
887
+ ops.nvfp4_w4a16_marlin_bf16(
888
+ x, weight_marlin, weight_scale_marlin, weight_global_scale, workspace, out
889
+ )
890
+ return out
891
+
892
+
893
  def fp4_w4a16_linear_bf16(
894
  a_packed: torch.Tensor,
895
  b_packed: torch.Tensor,
 
1051
  "SUPPORTED_CUDA_CAPABILITIES",
1052
  "SUPPORTED_LAYOUTS",
1053
  "capabilities",
1054
+ "allocate_w4a16_marlin_workspace",
1055
+ "adopt_nvfp4_w4a16_marlin",
1056
  "aligned_fp4_dim",
1057
  "cutlass_fp4_gemm_geglu_il_hw_v10",
1058
  "dequantize_fp4_sfa_fp16",
 
1068
  "nvfp4_gemm_nvfp4",
1069
  "nvfp4_gemm_m256_bf16",
1070
  "nvfp4_gemm_m256_workspace_size",
1071
+ "nvfp4_w4a16_marlin_bf16",
1072
  "nvfp4_gemm_geglu_nvfp4_fp16",
1073
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
1074
  "nvfp4_gemm_bias_residual_fp16",
build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f26894cd8000a3c464811b255f07732758d8d53521c2b84c305e9cc07d96619
3
- size 8566680
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ecd0fd14beafd0648b7353999f47e14168ca7a7e051d51527729b836e648548
3
+ size 10758024
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_dc57d15
3
- ops = torch.ops._fp4_gemm_cuda_dc57d15
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_dc57d15::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_6971f7a
3
+ ops = torch.ops._fp4_gemm_cuda_6971f7a
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_6971f7a::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_dc57d15",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "XycjAPC24qtK6IH+GtF/jrPmWnB6DyNgSny26ep3w1Y=",
18
- "_fp4_gemm_cuda_dc57d15.abi3.so": "DyaJTNgACjxGSBGyVfB3MnWNjVNSHCuEwwXpzAfZZhk=",
19
- "_ops.py": "TM5HQsh2RfinhrEZ4U0XyWF2hFyrPWpxR79FQ41D5jk=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
@@ -27,7 +27,7 @@
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "dc57d15b8bb1d4b52d82012f7f789ae5ae6db17f",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_6971f7a",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "0hDN2W6X3IMoWIi5BBufrnqrq2yj5N1/q9wBhCwBRxA=",
18
+ "_fp4_gemm_cuda_6971f7a.abi3.so": "Ts0P0Uvq/QZItzU5mfR+FBaMp6fgUdUVJ3Kbg25khUg=",
19
+ "_ops.py": "D7InzLughnNL5JQowMPNYpSpNPRFeeD8mlgs38Xw03g=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
 
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "6971f7a37410ba0d6edae93b389dbcbe3ba924d5",
31
  "dirty": false
32
  }
33
  }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,11 @@ def capabilities() -> dict[str, object]:
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
 
 
 
 
 
40
  "errors": "exceptions",
41
  }
42
 
@@ -172,6 +177,28 @@ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0)
172
  return None
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
176
  def _legacy_linear_fake(
177
  a_packed: torch.Tensor,
@@ -771,6 +798,98 @@ def nvfp4_gemm_m256_bf16(
771
  return out
772
 
773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  def fp4_w4a16_linear_bf16(
775
  a_packed: torch.Tensor,
776
  b_packed: torch.Tensor,
@@ -932,6 +1051,8 @@ __all__ = [
932
  "SUPPORTED_CUDA_CAPABILITIES",
933
  "SUPPORTED_LAYOUTS",
934
  "capabilities",
 
 
935
  "aligned_fp4_dim",
936
  "cutlass_fp4_gemm_geglu_il_hw_v10",
937
  "dequantize_fp4_sfa_fp16",
@@ -947,6 +1068,7 @@ __all__ = [
947
  "nvfp4_gemm_nvfp4",
948
  "nvfp4_gemm_m256_bf16",
949
  "nvfp4_gemm_m256_workspace_size",
 
950
  "nvfp4_gemm_geglu_nvfp4_fp16",
951
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
952
  "nvfp4_gemm_bias_residual_fp16",
 
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
+ "sm120_w4a16_marlin_m": (1, 16),
41
+ "sm120_w4a16_marlin_n_alignment": 64,
42
+ "sm120_w4a16_marlin_k_alignment": 128,
43
+ "sm120_w4a16_marlin_weight_layout": "marlin-repacked-nvfp4",
44
+ "sm120_w4a16_marlin_scale_layout": "marlin-permuted-fp8-e4m3-block16",
45
  "errors": "exceptions",
46
  }
47
 
 
177
  return None
178
 
179
 
180
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_bf16"))
181
+ def _w4a16_marlin_fake(x, weight, weight_scale, global_scale, workspace, out) -> None:
182
+ del weight, weight_scale, global_scale, workspace
183
+ if x.ndim != 2 or out.ndim != 2 or out.shape[0] != x.shape[0]:
184
+ raise RuntimeError("x/out must have shapes (M, K)/(M, N)")
185
+ if not 1 <= x.shape[0] <= 16:
186
+ raise RuntimeError("the Marlin W4A16 tier serves M in [1, 16]")
187
+ if x.shape[1] % 128 or out.shape[1] % 64:
188
+ raise RuntimeError("K/N must be divisible by 128/64")
189
+ return None
190
+
191
+
192
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_repack"))
193
+ def _w4a16_marlin_repack_fake(qweight_kn, weight_marlin) -> None:
194
+ if qweight_kn.ndim != 2:
195
+ raise RuntimeError("qweight_kn must have shape (K / 8, N)")
196
+ k, n = qweight_kn.shape[0] * 8, qweight_kn.shape[1]
197
+ if weight_marlin.shape != (k // 16, n * 2):
198
+ raise RuntimeError("weight_marlin must have shape (K / 16, 2 * N)")
199
+ return None
200
+
201
+
202
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
203
  def _legacy_linear_fake(
204
  a_packed: torch.Tensor,
 
798
  return out
799
 
800
 
801
+ def allocate_w4a16_marlin_workspace(device: torch.device | str) -> torch.Tensor:
802
+ """Allocate the persistent Marlin lock workspace before graph capture."""
803
+ props = torch.cuda.get_device_properties(device)
804
+ return torch.zeros(props.multi_processor_count, device=device, dtype=torch.int32)
805
+
806
+
807
+ def adopt_nvfp4_w4a16_marlin(
808
+ weight_packed: torch.Tensor,
809
+ weight_scale: torch.Tensor,
810
+ weight_scale_2: torch.Tensor,
811
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
812
+ """Convert standard ModelOpt NVFP4 tensors once at model bind time.
813
+
814
+ Inputs use ``weight[N,K/2]``, ``weight_scale[N,K/16]`` and scalar
815
+ ``weight_scale_2``. Returned tensors are the three runtime operands plus a
816
+ zeroed lock workspace. This function is intentionally outside the graph
817
+ hot path.
818
+ """
819
+ if weight_packed.dtype != torch.uint8 or weight_packed.ndim != 2:
820
+ raise ValueError("weight_packed must be uint8 with shape (N, K / 2)")
821
+ n, k = weight_packed.shape[0], weight_packed.shape[1] * 2
822
+ if k % 128 or n % 64:
823
+ raise ValueError("K/N must be divisible by 128/64")
824
+ if weight_scale.shape != (n, k // NVFP4_BLOCK_SIZE):
825
+ raise ValueError("weight_scale must have shape (N, K / 16)")
826
+ if weight_scale_2.numel() != 1:
827
+ raise ValueError("weight_scale_2 must contain one global scale")
828
+
829
+ qweight_kn = weight_packed.view(torch.int32).T.contiguous()
830
+ weight_marlin = torch.empty(
831
+ (k // 16, n * 2), device=weight_packed.device, dtype=torch.int32
832
+ )
833
+ ops.nvfp4_w4a16_marlin_repack(qweight_kn, weight_marlin)
834
+
835
+ scales = weight_scale.T.contiguous().to(torch.bfloat16)
836
+ scale_perm = [i + 8 * j for i in range(8) for j in range(8)]
837
+ scales = scales.reshape(-1, 64)[:, scale_perm].reshape(-1, n).contiguous()
838
+
839
+ # Match vLLM's NVFP4 Marlin contract exactly: scale permutation uses the
840
+ # model parameter dtype, then S0E5M3 encoding starts from FP16.
841
+ scales = scales.to(torch.float16)
842
+ scales = scales.view(-1, 4)[:, [0, 2, 1, 3]].reshape(scales.shape)
843
+ scaled = scales.float() * (2**7)
844
+ nonzero = scaled > 0
845
+ scale_factor = 1.0
846
+ if bool(nonzero.any()):
847
+ max_value = scaled[nonzero].max()
848
+ if bool(max_value < 448 * (2**7)):
849
+ scale_factor = float((448 * (2**7) / max_value).log2().floor().exp2())
850
+ if scale_factor > 1.0:
851
+ scales = (scales.float() * scale_factor).to(torch.float16)
852
+ scales = scales * (2**7)
853
+ scales[scales < 2] = 0
854
+ scales = (scales.view(torch.int16) << 1).view(torch.float8_e4m3fn)
855
+ scales = scales[:, 1::2].contiguous()
856
+
857
+ # Marlin's BF16 dequantizer uses exponent bias 126 and removes the seven
858
+ # bits introduced while encoding the special unsigned FP8 scale format.
859
+ global_scale = (
860
+ weight_scale_2.to(device=weight_packed.device, dtype=torch.float32)
861
+ * (2.0 ** 119)
862
+ / scale_factor
863
+ ).contiguous()
864
+ workspace = allocate_w4a16_marlin_workspace(weight_packed.device)
865
+ return weight_marlin, scales, global_scale, workspace
866
+
867
+
868
+ def nvfp4_w4a16_marlin_bf16(
869
+ x: torch.Tensor,
870
+ weight_marlin: torch.Tensor,
871
+ weight_scale_marlin: torch.Tensor,
872
+ weight_global_scale: torch.Tensor,
873
+ *,
874
+ workspace: torch.Tensor,
875
+ out: torch.Tensor | None = None,
876
+ ) -> torch.Tensor:
877
+ """Small-M BF16 x NVFP4 linear using bind-time Marlin layouts.
878
+
879
+ ``weight_marlin``, ``weight_scale_marlin`` and
880
+ ``weight_global_scale`` are the tensors produced by a standard NVFP4
881
+ Marlin adoption step. The operation supports M in [1, 16] and never
882
+ allocates internally, making it suitable for CUDA Graph replay.
883
+ """
884
+ if out is None:
885
+ n = weight_scale_marlin.numel() // (x.shape[1] // NVFP4_BLOCK_SIZE)
886
+ out = torch.empty((x.shape[0], n), device=x.device, dtype=torch.bfloat16)
887
+ ops.nvfp4_w4a16_marlin_bf16(
888
+ x, weight_marlin, weight_scale_marlin, weight_global_scale, workspace, out
889
+ )
890
+ return out
891
+
892
+
893
  def fp4_w4a16_linear_bf16(
894
  a_packed: torch.Tensor,
895
  b_packed: torch.Tensor,
 
1051
  "SUPPORTED_CUDA_CAPABILITIES",
1052
  "SUPPORTED_LAYOUTS",
1053
  "capabilities",
1054
+ "allocate_w4a16_marlin_workspace",
1055
+ "adopt_nvfp4_w4a16_marlin",
1056
  "aligned_fp4_dim",
1057
  "cutlass_fp4_gemm_geglu_il_hw_v10",
1058
  "dequantize_fp4_sfa_fp16",
 
1068
  "nvfp4_gemm_nvfp4",
1069
  "nvfp4_gemm_m256_bf16",
1070
  "nvfp4_gemm_m256_workspace_size",
1071
+ "nvfp4_w4a16_marlin_bf16",
1072
  "nvfp4_gemm_geglu_nvfp4_fp16",
1073
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
1074
  "nvfp4_gemm_bias_residual_fp16",
build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_dc57d15.abi3.so → _fp4_gemm_cuda_6971f7a.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3b1fc2155f02132014a4ed86a37eb5c945d705651eeda4cf4fc294c4e4d5a442
3
- size 8550344
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d2c4336526e99aec6d7644e1f9bbc3cc0175e0d9b63689e50d8639b58dd0daa
3
+ size 10663864
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_dc57d15
3
- ops = torch.ops._fp4_gemm_cuda_dc57d15
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_dc57d15::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_6971f7a
3
+ ops = torch.ops._fp4_gemm_cuda_6971f7a
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_6971f7a::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_dc57d15",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "XycjAPC24qtK6IH+GtF/jrPmWnB6DyNgSny26ep3w1Y=",
18
- "_fp4_gemm_cuda_dc57d15.abi3.so": "Ox/CFV8CEyAUpO2Go361yUXXBWUe7aTPT8KUxOTVpEI=",
19
- "_ops.py": "TM5HQsh2RfinhrEZ4U0XyWF2hFyrPWpxR79FQ41D5jk=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
@@ -27,7 +27,7 @@
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "dc57d15b8bb1d4b52d82012f7f789ae5ae6db17f",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_6971f7a",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "0hDN2W6X3IMoWIi5BBufrnqrq2yj5N1/q9wBhCwBRxA=",
18
+ "_fp4_gemm_cuda_6971f7a.abi3.so": "jSxDNlJuma7G12ROH5u8PMAXXg2bY2ieUNhjm1jdDao=",
19
+ "_ops.py": "D7InzLughnNL5JQowMPNYpSpNPRFeeD8mlgs38Xw03g=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
 
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "6971f7a37410ba0d6edae93b389dbcbe3ba924d5",
31
  "dirty": false
32
  }
33
  }
build/torch213-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,11 @@ def capabilities() -> dict[str, object]:
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
 
 
 
 
 
40
  "errors": "exceptions",
41
  }
42
 
@@ -172,6 +177,28 @@ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0)
172
  return None
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
176
  def _legacy_linear_fake(
177
  a_packed: torch.Tensor,
@@ -771,6 +798,98 @@ def nvfp4_gemm_m256_bf16(
771
  return out
772
 
773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  def fp4_w4a16_linear_bf16(
775
  a_packed: torch.Tensor,
776
  b_packed: torch.Tensor,
@@ -932,6 +1051,8 @@ __all__ = [
932
  "SUPPORTED_CUDA_CAPABILITIES",
933
  "SUPPORTED_LAYOUTS",
934
  "capabilities",
 
 
935
  "aligned_fp4_dim",
936
  "cutlass_fp4_gemm_geglu_il_hw_v10",
937
  "dequantize_fp4_sfa_fp16",
@@ -947,6 +1068,7 @@ __all__ = [
947
  "nvfp4_gemm_nvfp4",
948
  "nvfp4_gemm_m256_bf16",
949
  "nvfp4_gemm_m256_workspace_size",
 
950
  "nvfp4_gemm_geglu_nvfp4_fp16",
951
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
952
  "nvfp4_gemm_bias_residual_fp16",
 
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
+ "sm120_w4a16_marlin_m": (1, 16),
41
+ "sm120_w4a16_marlin_n_alignment": 64,
42
+ "sm120_w4a16_marlin_k_alignment": 128,
43
+ "sm120_w4a16_marlin_weight_layout": "marlin-repacked-nvfp4",
44
+ "sm120_w4a16_marlin_scale_layout": "marlin-permuted-fp8-e4m3-block16",
45
  "errors": "exceptions",
46
  }
47
 
 
177
  return None
178
 
179
 
180
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_bf16"))
181
+ def _w4a16_marlin_fake(x, weight, weight_scale, global_scale, workspace, out) -> None:
182
+ del weight, weight_scale, global_scale, workspace
183
+ if x.ndim != 2 or out.ndim != 2 or out.shape[0] != x.shape[0]:
184
+ raise RuntimeError("x/out must have shapes (M, K)/(M, N)")
185
+ if not 1 <= x.shape[0] <= 16:
186
+ raise RuntimeError("the Marlin W4A16 tier serves M in [1, 16]")
187
+ if x.shape[1] % 128 or out.shape[1] % 64:
188
+ raise RuntimeError("K/N must be divisible by 128/64")
189
+ return None
190
+
191
+
192
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_repack"))
193
+ def _w4a16_marlin_repack_fake(qweight_kn, weight_marlin) -> None:
194
+ if qweight_kn.ndim != 2:
195
+ raise RuntimeError("qweight_kn must have shape (K / 8, N)")
196
+ k, n = qweight_kn.shape[0] * 8, qweight_kn.shape[1]
197
+ if weight_marlin.shape != (k // 16, n * 2):
198
+ raise RuntimeError("weight_marlin must have shape (K / 16, 2 * N)")
199
+ return None
200
+
201
+
202
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
203
  def _legacy_linear_fake(
204
  a_packed: torch.Tensor,
 
798
  return out
799
 
800
 
801
+ def allocate_w4a16_marlin_workspace(device: torch.device | str) -> torch.Tensor:
802
+ """Allocate the persistent Marlin lock workspace before graph capture."""
803
+ props = torch.cuda.get_device_properties(device)
804
+ return torch.zeros(props.multi_processor_count, device=device, dtype=torch.int32)
805
+
806
+
807
+ def adopt_nvfp4_w4a16_marlin(
808
+ weight_packed: torch.Tensor,
809
+ weight_scale: torch.Tensor,
810
+ weight_scale_2: torch.Tensor,
811
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
812
+ """Convert standard ModelOpt NVFP4 tensors once at model bind time.
813
+
814
+ Inputs use ``weight[N,K/2]``, ``weight_scale[N,K/16]`` and scalar
815
+ ``weight_scale_2``. Returned tensors are the three runtime operands plus a
816
+ zeroed lock workspace. This function is intentionally outside the graph
817
+ hot path.
818
+ """
819
+ if weight_packed.dtype != torch.uint8 or weight_packed.ndim != 2:
820
+ raise ValueError("weight_packed must be uint8 with shape (N, K / 2)")
821
+ n, k = weight_packed.shape[0], weight_packed.shape[1] * 2
822
+ if k % 128 or n % 64:
823
+ raise ValueError("K/N must be divisible by 128/64")
824
+ if weight_scale.shape != (n, k // NVFP4_BLOCK_SIZE):
825
+ raise ValueError("weight_scale must have shape (N, K / 16)")
826
+ if weight_scale_2.numel() != 1:
827
+ raise ValueError("weight_scale_2 must contain one global scale")
828
+
829
+ qweight_kn = weight_packed.view(torch.int32).T.contiguous()
830
+ weight_marlin = torch.empty(
831
+ (k // 16, n * 2), device=weight_packed.device, dtype=torch.int32
832
+ )
833
+ ops.nvfp4_w4a16_marlin_repack(qweight_kn, weight_marlin)
834
+
835
+ scales = weight_scale.T.contiguous().to(torch.bfloat16)
836
+ scale_perm = [i + 8 * j for i in range(8) for j in range(8)]
837
+ scales = scales.reshape(-1, 64)[:, scale_perm].reshape(-1, n).contiguous()
838
+
839
+ # Match vLLM's NVFP4 Marlin contract exactly: scale permutation uses the
840
+ # model parameter dtype, then S0E5M3 encoding starts from FP16.
841
+ scales = scales.to(torch.float16)
842
+ scales = scales.view(-1, 4)[:, [0, 2, 1, 3]].reshape(scales.shape)
843
+ scaled = scales.float() * (2**7)
844
+ nonzero = scaled > 0
845
+ scale_factor = 1.0
846
+ if bool(nonzero.any()):
847
+ max_value = scaled[nonzero].max()
848
+ if bool(max_value < 448 * (2**7)):
849
+ scale_factor = float((448 * (2**7) / max_value).log2().floor().exp2())
850
+ if scale_factor > 1.0:
851
+ scales = (scales.float() * scale_factor).to(torch.float16)
852
+ scales = scales * (2**7)
853
+ scales[scales < 2] = 0
854
+ scales = (scales.view(torch.int16) << 1).view(torch.float8_e4m3fn)
855
+ scales = scales[:, 1::2].contiguous()
856
+
857
+ # Marlin's BF16 dequantizer uses exponent bias 126 and removes the seven
858
+ # bits introduced while encoding the special unsigned FP8 scale format.
859
+ global_scale = (
860
+ weight_scale_2.to(device=weight_packed.device, dtype=torch.float32)
861
+ * (2.0 ** 119)
862
+ / scale_factor
863
+ ).contiguous()
864
+ workspace = allocate_w4a16_marlin_workspace(weight_packed.device)
865
+ return weight_marlin, scales, global_scale, workspace
866
+
867
+
868
+ def nvfp4_w4a16_marlin_bf16(
869
+ x: torch.Tensor,
870
+ weight_marlin: torch.Tensor,
871
+ weight_scale_marlin: torch.Tensor,
872
+ weight_global_scale: torch.Tensor,
873
+ *,
874
+ workspace: torch.Tensor,
875
+ out: torch.Tensor | None = None,
876
+ ) -> torch.Tensor:
877
+ """Small-M BF16 x NVFP4 linear using bind-time Marlin layouts.
878
+
879
+ ``weight_marlin``, ``weight_scale_marlin`` and
880
+ ``weight_global_scale`` are the tensors produced by a standard NVFP4
881
+ Marlin adoption step. The operation supports M in [1, 16] and never
882
+ allocates internally, making it suitable for CUDA Graph replay.
883
+ """
884
+ if out is None:
885
+ n = weight_scale_marlin.numel() // (x.shape[1] // NVFP4_BLOCK_SIZE)
886
+ out = torch.empty((x.shape[0], n), device=x.device, dtype=torch.bfloat16)
887
+ ops.nvfp4_w4a16_marlin_bf16(
888
+ x, weight_marlin, weight_scale_marlin, weight_global_scale, workspace, out
889
+ )
890
+ return out
891
+
892
+
893
  def fp4_w4a16_linear_bf16(
894
  a_packed: torch.Tensor,
895
  b_packed: torch.Tensor,
 
1051
  "SUPPORTED_CUDA_CAPABILITIES",
1052
  "SUPPORTED_LAYOUTS",
1053
  "capabilities",
1054
+ "allocate_w4a16_marlin_workspace",
1055
+ "adopt_nvfp4_w4a16_marlin",
1056
  "aligned_fp4_dim",
1057
  "cutlass_fp4_gemm_geglu_il_hw_v10",
1058
  "dequantize_fp4_sfa_fp16",
 
1068
  "nvfp4_gemm_nvfp4",
1069
  "nvfp4_gemm_m256_bf16",
1070
  "nvfp4_gemm_m256_workspace_size",
1071
+ "nvfp4_w4a16_marlin_bf16",
1072
  "nvfp4_gemm_geglu_nvfp4_fp16",
1073
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
1074
  "nvfp4_gemm_bias_residual_fp16",
build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_6971f7a.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6075d51ae508759aacba4d16f7bf01c2f27c57de03afcb0745ffb073ee1c3320
3
+ size 10757864
build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_dc57d15.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:a2e973560e69449ee79707984b7703d19e8f8ef9bba3902513887c57b64682c0
3
- size 8566520
 
 
 
 
build/torch213-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_dc57d15
3
- ops = torch.ops._fp4_gemm_cuda_dc57d15
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_dc57d15::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_6971f7a
3
+ ops = torch.ops._fp4_gemm_cuda_6971f7a
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_6971f7a::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_dc57d15",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "XycjAPC24qtK6IH+GtF/jrPmWnB6DyNgSny26ep3w1Y=",
18
- "_fp4_gemm_cuda_dc57d15.abi3.so": "oulzVg5pRJ7nlweYS3cD0Z6Pjvm7o5AlE4h8V7ZGgsA=",
19
- "_ops.py": "TM5HQsh2RfinhrEZ4U0XyWF2hFyrPWpxR79FQ41D5jk=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
@@ -27,7 +27,7 @@
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "dc57d15b8bb1d4b52d82012f7f789ae5ae6db17f",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_6971f7a",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "0hDN2W6X3IMoWIi5BBufrnqrq2yj5N1/q9wBhCwBRxA=",
18
+ "_fp4_gemm_cuda_6971f7a.abi3.so": "YHXVGuUIdZqsuk0W978BwvJ8V94Dr8sHRf+wc+4cMyA=",
19
+ "_ops.py": "D7InzLughnNL5JQowMPNYpSpNPRFeeD8mlgs38Xw03g=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
 
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "6971f7a37410ba0d6edae93b389dbcbe3ba924d5",
31
  "dirty": false
32
  }
33
  }
build/torch213-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -37,6 +37,11 @@ def capabilities() -> dict[str, object]:
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
 
 
 
 
 
40
  "errors": "exceptions",
41
  }
42
 
@@ -172,6 +177,28 @@ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0)
172
  return None
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
176
  def _legacy_linear_fake(
177
  a_packed: torch.Tensor,
@@ -771,6 +798,98 @@ def nvfp4_gemm_m256_bf16(
771
  return out
772
 
773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
774
  def fp4_w4a16_linear_bf16(
775
  a_packed: torch.Tensor,
776
  b_packed: torch.Tensor,
@@ -932,6 +1051,8 @@ __all__ = [
932
  "SUPPORTED_CUDA_CAPABILITIES",
933
  "SUPPORTED_LAYOUTS",
934
  "capabilities",
 
 
935
  "aligned_fp4_dim",
936
  "cutlass_fp4_gemm_geglu_il_hw_v10",
937
  "dequantize_fp4_sfa_fp16",
@@ -947,6 +1068,7 @@ __all__ = [
947
  "nvfp4_gemm_nvfp4",
948
  "nvfp4_gemm_m256_bf16",
949
  "nvfp4_gemm_m256_workspace_size",
 
950
  "nvfp4_gemm_geglu_nvfp4_fp16",
951
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
952
  "nvfp4_gemm_bias_residual_fp16",
 
37
  "sm120_warpsplit_mrows": (1, 16),
38
  "sm120_warpsplit_mrows_n_alignment": 8,
39
  "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
+ "sm120_w4a16_marlin_m": (1, 16),
41
+ "sm120_w4a16_marlin_n_alignment": 64,
42
+ "sm120_w4a16_marlin_k_alignment": 128,
43
+ "sm120_w4a16_marlin_weight_layout": "marlin-repacked-nvfp4",
44
+ "sm120_w4a16_marlin_scale_layout": "marlin-permuted-fp8-e4m3-block16",
45
  "errors": "exceptions",
46
  }
47
 
 
177
  return None
178
 
179
 
180
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_bf16"))
181
+ def _w4a16_marlin_fake(x, weight, weight_scale, global_scale, workspace, out) -> None:
182
+ del weight, weight_scale, global_scale, workspace
183
+ if x.ndim != 2 or out.ndim != 2 or out.shape[0] != x.shape[0]:
184
+ raise RuntimeError("x/out must have shapes (M, K)/(M, N)")
185
+ if not 1 <= x.shape[0] <= 16:
186
+ raise RuntimeError("the Marlin W4A16 tier serves M in [1, 16]")
187
+ if x.shape[1] % 128 or out.shape[1] % 64:
188
+ raise RuntimeError("K/N must be divisible by 128/64")
189
+ return None
190
+
191
+
192
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_w4a16_marlin_repack"))
193
+ def _w4a16_marlin_repack_fake(qweight_kn, weight_marlin) -> None:
194
+ if qweight_kn.ndim != 2:
195
+ raise RuntimeError("qweight_kn must have shape (K / 8, N)")
196
+ k, n = qweight_kn.shape[0] * 8, qweight_kn.shape[1]
197
+ if weight_marlin.shape != (k // 16, n * 2):
198
+ raise RuntimeError("weight_marlin must have shape (K / 16, 2 * N)")
199
+ return None
200
+
201
+
202
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
203
  def _legacy_linear_fake(
204
  a_packed: torch.Tensor,
 
798
  return out
799
 
800
 
801
+ def allocate_w4a16_marlin_workspace(device: torch.device | str) -> torch.Tensor:
802
+ """Allocate the persistent Marlin lock workspace before graph capture."""
803
+ props = torch.cuda.get_device_properties(device)
804
+ return torch.zeros(props.multi_processor_count, device=device, dtype=torch.int32)
805
+
806
+
807
+ def adopt_nvfp4_w4a16_marlin(
808
+ weight_packed: torch.Tensor,
809
+ weight_scale: torch.Tensor,
810
+ weight_scale_2: torch.Tensor,
811
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
812
+ """Convert standard ModelOpt NVFP4 tensors once at model bind time.
813
+
814
+ Inputs use ``weight[N,K/2]``, ``weight_scale[N,K/16]`` and scalar
815
+ ``weight_scale_2``. Returned tensors are the three runtime operands plus a
816
+ zeroed lock workspace. This function is intentionally outside the graph
817
+ hot path.
818
+ """
819
+ if weight_packed.dtype != torch.uint8 or weight_packed.ndim != 2:
820
+ raise ValueError("weight_packed must be uint8 with shape (N, K / 2)")
821
+ n, k = weight_packed.shape[0], weight_packed.shape[1] * 2
822
+ if k % 128 or n % 64:
823
+ raise ValueError("K/N must be divisible by 128/64")
824
+ if weight_scale.shape != (n, k // NVFP4_BLOCK_SIZE):
825
+ raise ValueError("weight_scale must have shape (N, K / 16)")
826
+ if weight_scale_2.numel() != 1:
827
+ raise ValueError("weight_scale_2 must contain one global scale")
828
+
829
+ qweight_kn = weight_packed.view(torch.int32).T.contiguous()
830
+ weight_marlin = torch.empty(
831
+ (k // 16, n * 2), device=weight_packed.device, dtype=torch.int32
832
+ )
833
+ ops.nvfp4_w4a16_marlin_repack(qweight_kn, weight_marlin)
834
+
835
+ scales = weight_scale.T.contiguous().to(torch.bfloat16)
836
+ scale_perm = [i + 8 * j for i in range(8) for j in range(8)]
837
+ scales = scales.reshape(-1, 64)[:, scale_perm].reshape(-1, n).contiguous()
838
+
839
+ # Match vLLM's NVFP4 Marlin contract exactly: scale permutation uses the
840
+ # model parameter dtype, then S0E5M3 encoding starts from FP16.
841
+ scales = scales.to(torch.float16)
842
+ scales = scales.view(-1, 4)[:, [0, 2, 1, 3]].reshape(scales.shape)
843
+ scaled = scales.float() * (2**7)
844
+ nonzero = scaled > 0
845
+ scale_factor = 1.0
846
+ if bool(nonzero.any()):
847
+ max_value = scaled[nonzero].max()
848
+ if bool(max_value < 448 * (2**7)):
849
+ scale_factor = float((448 * (2**7) / max_value).log2().floor().exp2())
850
+ if scale_factor > 1.0:
851
+ scales = (scales.float() * scale_factor).to(torch.float16)
852
+ scales = scales * (2**7)
853
+ scales[scales < 2] = 0
854
+ scales = (scales.view(torch.int16) << 1).view(torch.float8_e4m3fn)
855
+ scales = scales[:, 1::2].contiguous()
856
+
857
+ # Marlin's BF16 dequantizer uses exponent bias 126 and removes the seven
858
+ # bits introduced while encoding the special unsigned FP8 scale format.
859
+ global_scale = (
860
+ weight_scale_2.to(device=weight_packed.device, dtype=torch.float32)
861
+ * (2.0 ** 119)
862
+ / scale_factor
863
+ ).contiguous()
864
+ workspace = allocate_w4a16_marlin_workspace(weight_packed.device)
865
+ return weight_marlin, scales, global_scale, workspace
866
+
867
+
868
+ def nvfp4_w4a16_marlin_bf16(
869
+ x: torch.Tensor,
870
+ weight_marlin: torch.Tensor,
871
+ weight_scale_marlin: torch.Tensor,
872
+ weight_global_scale: torch.Tensor,
873
+ *,
874
+ workspace: torch.Tensor,
875
+ out: torch.Tensor | None = None,
876
+ ) -> torch.Tensor:
877
+ """Small-M BF16 x NVFP4 linear using bind-time Marlin layouts.
878
+
879
+ ``weight_marlin``, ``weight_scale_marlin`` and
880
+ ``weight_global_scale`` are the tensors produced by a standard NVFP4
881
+ Marlin adoption step. The operation supports M in [1, 16] and never
882
+ allocates internally, making it suitable for CUDA Graph replay.
883
+ """
884
+ if out is None:
885
+ n = weight_scale_marlin.numel() // (x.shape[1] // NVFP4_BLOCK_SIZE)
886
+ out = torch.empty((x.shape[0], n), device=x.device, dtype=torch.bfloat16)
887
+ ops.nvfp4_w4a16_marlin_bf16(
888
+ x, weight_marlin, weight_scale_marlin, weight_global_scale, workspace, out
889
+ )
890
+ return out
891
+
892
+
893
  def fp4_w4a16_linear_bf16(
894
  a_packed: torch.Tensor,
895
  b_packed: torch.Tensor,
 
1051
  "SUPPORTED_CUDA_CAPABILITIES",
1052
  "SUPPORTED_LAYOUTS",
1053
  "capabilities",
1054
+ "allocate_w4a16_marlin_workspace",
1055
+ "adopt_nvfp4_w4a16_marlin",
1056
  "aligned_fp4_dim",
1057
  "cutlass_fp4_gemm_geglu_il_hw_v10",
1058
  "dequantize_fp4_sfa_fp16",
 
1068
  "nvfp4_gemm_nvfp4",
1069
  "nvfp4_gemm_m256_bf16",
1070
  "nvfp4_gemm_m256_workspace_size",
1071
+ "nvfp4_w4a16_marlin_bf16",
1072
  "nvfp4_gemm_geglu_nvfp4_fp16",
1073
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
1074
  "nvfp4_gemm_bias_residual_fp16",
build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_6971f7a.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d89dfa19d243671a415f0fdaeeb8a6261c417bf7d4f45e5d661745387e2ef1f7
3
+ size 10663712
build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_dc57d15.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3c5c466015e9fda36acc0d9426f2c9281bb9e8af175afb1686edd37348a4f975
3
- size 8550192
 
 
 
 
build/torch213-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_dc57d15
3
- ops = torch.ops._fp4_gemm_cuda_dc57d15
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_dc57d15::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_6971f7a
3
+ ops = torch.ops._fp4_gemm_cuda_6971f7a
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_6971f7a::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_dc57d15",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "XycjAPC24qtK6IH+GtF/jrPmWnB6DyNgSny26ep3w1Y=",
18
- "_fp4_gemm_cuda_dc57d15.abi3.so": "PFxGYBXp/aNqzA2UJvLJKBu56K8XWvsWhu3Tc0ik+XU=",
19
- "_ops.py": "TM5HQsh2RfinhrEZ4U0XyWF2hFyrPWpxR79FQ41D5jk=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
@@ -27,7 +27,7 @@
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "dc57d15b8bb1d4b52d82012f7f789ae5ae6db17f",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_6971f7a",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "0hDN2W6X3IMoWIi5BBufrnqrq2yj5N1/q9wBhCwBRxA=",
18
+ "_fp4_gemm_cuda_6971f7a.abi3.so": "2J36GdJDZxpBXw/a7rimJhxBe/fU9F5dZhdFOH4u8fc=",
19
+ "_ops.py": "D7InzLughnNL5JQowMPNYpSpNPRFeeD8mlgs38Xw03g=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
 
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "6971f7a37410ba0d6edae93b389dbcbe3ba924d5",
31
  "dirty": false
32
  }
33
  }