liangsu9988 commited on
Commit
e95a3f5
·
verified ·
1 Parent(s): efb780e

Promote latest kernel artifacts to main

Browse files
Files changed (31) hide show
  1. README.md +113 -6
  2. build/torch211-cxx11-cu128-x86_64-linux/__init__.py +50 -0
  3. build/torch211-cxx11-cu128-x86_64-linux/{_fp4_gemm_cuda_0a10e1f.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} +2 -2
  4. build/torch211-cxx11-cu128-x86_64-linux/_ops.py +3 -3
  5. build/torch211-cxx11-cu128-x86_64-linux/metadata.json +5 -5
  6. build/torch211-cxx11-cu130-x86_64-linux/__init__.py +50 -0
  7. build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_014dcce.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} +2 -2
  8. build/torch211-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  9. build/torch211-cxx11-cu130-x86_64-linux/metadata.json +5 -5
  10. build/torch212-cxx11-cu130-x86_64-linux/__init__.py +50 -0
  11. build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_e9b9a96.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} +2 -2
  12. build/torch212-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  13. build/torch212-cxx11-cu130-x86_64-linux/fp4_gemm/__init__.py +26 -0
  14. build/torch212-cxx11-cu130-x86_64-linux/metadata.json +7 -7
  15. build/torch212-cxx11-cu132-x86_64-linux/__init__.py +50 -0
  16. build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_e9b9a96.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} +2 -2
  17. build/torch212-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  18. build/torch212-cxx11-cu132-x86_64-linux/fp4_gemm/__init__.py +26 -0
  19. build/torch212-cxx11-cu132-x86_64-linux/metadata.json +7 -7
  20. build/torch213-cxx11-cu130-x86_64-linux/__init__.py +50 -0
  21. build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_e1f10b6.abi3.so +3 -0
  22. build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_e9b9a96.abi3.so +0 -3
  23. build/torch213-cxx11-cu130-x86_64-linux/_ops.py +3 -3
  24. build/torch213-cxx11-cu130-x86_64-linux/fp4_gemm/__init__.py +26 -0
  25. build/torch213-cxx11-cu130-x86_64-linux/metadata.json +7 -7
  26. build/torch213-cxx11-cu132-x86_64-linux/__init__.py +50 -0
  27. build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_e1f10b6.abi3.so +3 -0
  28. build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_e9b9a96.abi3.so +0 -3
  29. build/torch213-cxx11-cu132-x86_64-linux/_ops.py +3 -3
  30. build/torch213-cxx11-cu132-x86_64-linux/fp4_gemm/__init__.py +26 -0
  31. build/torch213-cxx11-cu132-x86_64-linux/metadata.json +7 -7
README.md CHANGED
@@ -1,9 +1,116 @@
1
- # flashrt/fp4-gemm
2
 
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
 
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/fp4-gemm
 
 
 
7
 
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # fp4-gemm
2
 
3
+ FlashRT native Blackwell NVFP4 A4W4 GEMM kernels.
 
4
 
5
+ This package consumes packed FP4 E2M1 tensors plus CUTLASS Sm1xx SFA/SFB scale
6
+ buffers and produces BF16 output. It is designed to pair with
7
+ `flashrt/fp4-fused-ops` and other static low-bit transformer/diffuser runtime
8
+ paths.
9
 
10
+ ## Available Functions
11
+
12
+ - `sfa_size_bytes(rows, dim)`
13
+ - `quantize_fp4_sfa_fp16(x, packed=None, sfa=None, is_sfb=False)`
14
+ - `quantize_fp4_sfa_bf16(x, packed=None, sfa=None, is_sfb=False)`
15
+ - `dequantize_fp4_sfa_fp16(packed, sfa, out=None, is_sfb=False)`
16
+ - `nvfp4_gemm_bf16(a_packed, b_packed, sfa, sfb, alpha=1.0, out=None, variant=-1)`
17
+ - `nvfp4_gemm_bias_bf16(a_packed, b_packed, sfa, sfb, bias, out=None)`
18
+ - `nvfp4_gemm_bias_residual_bf16(a_packed, b_packed, sfa, sfb, bias, residual, out=None)`
19
+ - `nvfp4_gemm_residual_bf16(a_packed, b_packed, sfa, sfb, residual, alpha=1.0, out=None)`
20
+ - `nvfp4_gemm_bias_gelu_bf16(a_packed, b_packed, sfa, sfb, bias, alpha=1.0, out=None)`
21
+ - `nvfp4_gemm_bias_gelu_nvfp4(a_packed, b_packed, sfa, sfb, bias, alpha=1.0, out_packed=None, out_sfa=None)`
22
+ - `nvfp4_gemm_streamk_bf16(a_packed, b_packed, sfa, sfb, alpha=1.0, out=None)`
23
+ - `nvfp4_gemm_streamk_bias_bf16(a_packed, b_packed, sfa, sfb, bias, alpha=1.0, out=None)`
24
+ - `fp4_w4a16_linear_bf16(...)` is retained as a compatibility alias
25
+
26
+ ## Tensor Contract
27
+
28
+ - `a_packed`: `torch.uint8`, shape `(M, K / 2)`.
29
+ - `b_packed`: `torch.uint8`, shape `(N, K / 2)`.
30
+ - `sfa`: `torch.uint8`, CUTLASS SFA layout for `(M, K)`.
31
+ - `sfb`: `torch.uint8`, CUTLASS SFB layout for `(N, K)`.
32
+ - output: `torch.bfloat16`, shape `(M, N)`.
33
+ - `K` must be divisible by 16.
34
+ - Targets: Blackwell `sm_110a` (Jetson AGX Thor, CUDA 13+) and `sm_120a`
35
+ (RTX Blackwell, CUDA 12.8+).
36
+
37
+ `variant` selects the CUTLASS schedule:
38
+
39
+ - `-1`: architecture-aware auto-dispatch (public default).
40
+ - `0`: default `<128,128,256>` cooperative schedule.
41
+ - `1`: widen `<128,256,128>` schedule, intended for very large `N`.
42
+ - `2`: pingpong schedule for A/B testing shape-specific wins.
43
+
44
+ The canonical linear API and FP4/SFA quantize/dequantize helpers are available
45
+ on both SM110 and SM120. SM110 additionally provides the GROOT N1.7 production
46
+ epilogues `nvfp4_gemm_bias_bf16`, `nvfp4_gemm_bias_residual_bf16`, and
47
+ `nvfp4_gemm_bias_gelu_nvfp4`. The latter emits packed FP4 plus CUTLASS SFA so
48
+ the following projection can consume it without a BF16 materialization and a
49
+ standalone quantization launch. Stream-K and the older BF16 GELU epilogue keep
50
+ their existing SM120 dispatch and reject unsupported architectures explicitly.
51
+
52
+ The SM110 release gate includes the production `(M,N,K)` shapes
53
+ `(41,4608,1536)`, `(41,6144,1536)`, and `(41,1536,6144)`, plus the legacy
54
+ `M=51` compatibility row. The kernels are the native sources used by FlashRT's
55
+ GROOT N1.7 Thor NVFP4 pipeline.
56
+
57
+ ## Minimal Usage
58
+
59
+ ```python
60
+ from kernels import get_kernel
61
+ import torch
62
+
63
+ ops = get_kernel("flashrt/fp4-gemm", version=1, trust_remote_code=True)
64
+
65
+ x = torch.randn((32, 256), device="cuda", dtype=torch.float16)
66
+ w = torch.randn((512, 256), device="cuda", dtype=torch.float16)
67
+
68
+ a_packed, sfa = ops.quantize_fp4_sfa_fp16(x, is_sfb=False)
69
+ b_packed, sfb = ops.quantize_fp4_sfa_fp16(w, is_sfb=True)
70
+
71
+ y = ops.nvfp4_gemm_bf16(a_packed, b_packed, sfa, sfb, alpha=1.0)
72
+ ```
73
+
74
+ For BF16 model activations, use the direct producer so the hot path does not
75
+ materialize an intermediate FP16 tensor:
76
+
77
+ ```python
78
+ x_bf16 = torch.randn((1, 5120), device="cuda", dtype=torch.bfloat16)
79
+ a_packed, sfa = ops.quantize_fp4_sfa_bf16(x_bf16)
80
+ ```
81
+
82
+ The BF16 entry writes the same E2M1 bytes and CUTLASS SFA/SFB layout as
83
+ `quantize_fp4_sfa_fp16(x_bf16.to(torch.float16))` for finite FP16-range
84
+ inputs. It is an additive API; the existing FP16 producer remains unchanged.
85
+
86
+ The quantize/dequantize helpers are included for examples and validation. A
87
+ production runtime should keep weights prepacked and should avoid quantizing in
88
+ the hot path unless that producer kernel is part of the intended low-bit block.
89
+
90
+ Use the bias/GELU and residual variants to avoid returning to BF16
91
+ elementwise code between low-bit GEMMs. Stream-K variants are selected only
92
+ for the validated large down-projection shapes; unsupported shapes reject
93
+ rather than silently selecting a losing schedule.
94
+
95
+ ## Validation
96
+
97
+ ```bash
98
+ python fp4-gemm/tests/test_fp4_gemm.py --backend source --mode full
99
+ python fp4-gemm/tests/test_fp4_gemm.py --backend installed --mode full \
100
+ --artifact fp4-gemm/build/torch211-cxx11-cu128-x86_64-linux
101
+ python fp4-gemm/benchmarks/benchmark.py --backend installed --mode headline \
102
+ --artifact fp4-gemm/build/torch211-cxx11-cu128-x86_64-linux
103
+
104
+ # Thor model-shape gate
105
+ python fp4-gemm/tests/test_fp4_gemm.py --backend installed \
106
+ --mode thor-models \
107
+ --artifact fp4-gemm/build/torch211-cxx11-cu130-aarch64-linux
108
+ ```
109
+
110
+ The correctness reference dequantizes the same FP4/SFA and FP4/SFB inputs used
111
+ by the kernel, then computes the PyTorch GEMM reference from those dequantized
112
+ low-bit values.
113
+
114
+ The producer gate also checks the BF16 direct entry byte-for-byte against the
115
+ established FP16 compatibility chain at decode widths 5120, 6144 and 17408,
116
+ plus multi-row activation and SFB layouts.
build/torch211-cxx11-cu128-x86_64-linux/__init__.py CHANGED
@@ -34,6 +34,9 @@ def capabilities() -> dict[str, object]:
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
 
 
 
37
  "errors": "exceptions",
38
  }
39
 
@@ -98,6 +101,27 @@ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
98
  return None
99
 
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
102
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
103
  if a_packed.shape[0] != 1:
@@ -594,6 +618,31 @@ def nvfp4_gemm_bias_residual_fp16(
594
  return out
595
 
596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
597
  def fp4_w4a4_gemv_warpsplit_bf16(
598
  a_packed: torch.Tensor,
599
  b_packed: torch.Tensor,
@@ -870,6 +919,7 @@ __all__ = [
870
  "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
 
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
 
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "sm120_warpsplit_mrows": (1, 16),
38
+ "sm120_warpsplit_mrows_n_alignment": 8,
39
+ "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
  "errors": "exceptions",
41
  }
42
 
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemm_warpsplit_mrows_bf16"))
105
+ def _gemm_warpsplit_mrows_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 2, stages: int = 6) -> None:
106
+ if a_packed.dim() != 2 or b_packed.dim() != 2:
107
+ raise RuntimeError("a_packed and b_packed must be rank-2")
108
+ if a_packed.shape[1] != b_packed.shape[1]:
109
+ raise RuntimeError("a_packed and b_packed must have the same K / 2")
110
+ if not 1 <= a_packed.shape[0] <= 16:
111
+ raise RuntimeError("the multi-row warp-split tier serves 1..16 rows")
112
+ if b_packed.shape[0] % 8:
113
+ raise RuntimeError("N must be a multiple of 8")
114
+ if warps not in (2, 4, 8):
115
+ raise RuntimeError("warps must be 2, 4 or 8")
116
+ if stages not in (3, 4, 6) or (warps == 8 and stages == 6):
117
+ raise RuntimeError("unsupported stages/warps combination")
118
+ if (a_packed.shape[1] * 2) % (64 * warps):
119
+ raise RuntimeError("K must be a multiple of 64*warps")
120
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
121
+ raise RuntimeError("out must have shape (M, N)")
122
+ return None
123
+
124
+
125
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
126
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
127
  if a_packed.shape[0] != 1:
 
618
  return out
619
 
620
 
621
+ def fp4_w4a4_gemm_warpsplit_mrows_bf16(
622
+ a_packed: torch.Tensor,
623
+ b_packed: torch.Tensor,
624
+ sfa: torch.Tensor,
625
+ sfb: torch.Tensor,
626
+ *,
627
+ alpha: float = 1.0,
628
+ warps: int = 2,
629
+ stages: int = 6,
630
+ out: Optional[torch.Tensor] = None,
631
+ ) -> torch.Tensor:
632
+ """Multi-row (M<=16) warp-split-K NVFP4 W4A4 GEMM (SM120).
633
+
634
+ The 16x8x64 block-scaled MMA atom computes a full 16-row output
635
+ tile, so up to sixteen rows ride one weight stream at near-GEMV
636
+ cost - the spec-verify block and its re-advance prefixes are the
637
+ customers. Same packed/scale layouts as the linear entry points;
638
+ deeper default stages hide the strided-B latency the extra A-row
639
+ loads expose."""
640
+ if out is None:
641
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
642
+ ops.fp4_w4a4_gemm_warpsplit_mrows_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
643
+ return out
644
+
645
+
646
  def fp4_w4a4_gemv_warpsplit_bf16(
647
  a_packed: torch.Tensor,
648
  b_packed: torch.Tensor,
 
919
  "fp4_repack_b_interleaved_sm120",
920
  "fp4_w4a4_gemv_warpsplit_bf16",
921
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
922
+ "fp4_w4a4_gemm_warpsplit_mrows_bf16",
923
  "nvfp4_gemm_bf16",
924
  "nvfp4_gemm_fp16",
925
  "nvfp4_gemm_variant_bf16",
build/torch211-cxx11-cu128-x86_64-linux/{_fp4_gemm_cuda_0a10e1f.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:7d86f743a39f009ff82abe0b2cef01f15eafc24ca2c024c450d651ba0e0447e3
3
- size 2461088
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5d384b35a1df0fce0389696709ac1d7ab6ca045a4e93b6ff389ea3a4666d118a
3
+ size 2721416
build/torch211-cxx11-cu128-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_0a10e1f
3
- ops = torch.ops._fp4_gemm_cuda_0a10e1f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_0a10e1f::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_e1f10b6
3
+ ops = torch.ops._fp4_gemm_cuda_e1f10b6
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_e1f10b6::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_0a10e1f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -13,9 +13,9 @@
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
- "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
17
- "_fp4_gemm_cuda_0a10e1f.abi3.so": "fYb3Q6OfAJ/4Kr4LLO8B8V6vwkyiwCTEUNZRug4ER+M=",
18
- "_ops.py": "dj/4ddT43Gl5g5YOtm2kvHHREJY3sqJsQFxHG3JEBdc=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
@@ -26,7 +26,7 @@
26
  "dirty": false
27
  },
28
  "kernel": {
29
- "sha": "0a10e1f3487fa4e673aa18173383d515de28ddd8",
30
  "dirty": false
31
  }
32
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_e1f10b6",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
13
  "digest": {
14
  "algorithm": "sha256",
15
  "files": {
16
+ "__init__.py": "x7M8jFG4z1Mx/zhAh30IsOFooDn28i8uyrkkNIjyG+c=",
17
+ "_fp4_gemm_cuda_e1f10b6.abi3.so": "XThLNaHfD84DiWlnCawderbKBFpOk7b/OJ6jpGZtEYo=",
18
+ "_ops.py": "FGdLd66XdfjhguPY8EEn/zd8T/lGPpQP+r8VJnZaCPE=",
19
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
20
  }
21
  },
 
26
  "dirty": false
27
  },
28
  "kernel": {
29
+ "sha": "e1f10b68fd3d9f68ecd85895128283aa6768ea6f",
30
  "dirty": false
31
  }
32
  }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -34,6 +34,9 @@ def capabilities() -> dict[str, object]:
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
 
 
 
37
  "errors": "exceptions",
38
  }
39
 
@@ -98,6 +101,27 @@ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
98
  return None
99
 
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
102
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
103
  if a_packed.shape[0] != 1:
@@ -594,6 +618,31 @@ def nvfp4_gemm_bias_residual_fp16(
594
  return out
595
 
596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
597
  def fp4_w4a4_gemv_warpsplit_bf16(
598
  a_packed: torch.Tensor,
599
  b_packed: torch.Tensor,
@@ -870,6 +919,7 @@ __all__ = [
870
  "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
 
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
 
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "sm120_warpsplit_mrows": (1, 16),
38
+ "sm120_warpsplit_mrows_n_alignment": 8,
39
+ "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
  "errors": "exceptions",
41
  }
42
 
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemm_warpsplit_mrows_bf16"))
105
+ def _gemm_warpsplit_mrows_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 2, stages: int = 6) -> None:
106
+ if a_packed.dim() != 2 or b_packed.dim() != 2:
107
+ raise RuntimeError("a_packed and b_packed must be rank-2")
108
+ if a_packed.shape[1] != b_packed.shape[1]:
109
+ raise RuntimeError("a_packed and b_packed must have the same K / 2")
110
+ if not 1 <= a_packed.shape[0] <= 16:
111
+ raise RuntimeError("the multi-row warp-split tier serves 1..16 rows")
112
+ if b_packed.shape[0] % 8:
113
+ raise RuntimeError("N must be a multiple of 8")
114
+ if warps not in (2, 4, 8):
115
+ raise RuntimeError("warps must be 2, 4 or 8")
116
+ if stages not in (3, 4, 6) or (warps == 8 and stages == 6):
117
+ raise RuntimeError("unsupported stages/warps combination")
118
+ if (a_packed.shape[1] * 2) % (64 * warps):
119
+ raise RuntimeError("K must be a multiple of 64*warps")
120
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
121
+ raise RuntimeError("out must have shape (M, N)")
122
+ return None
123
+
124
+
125
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
126
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
127
  if a_packed.shape[0] != 1:
 
618
  return out
619
 
620
 
621
+ def fp4_w4a4_gemm_warpsplit_mrows_bf16(
622
+ a_packed: torch.Tensor,
623
+ b_packed: torch.Tensor,
624
+ sfa: torch.Tensor,
625
+ sfb: torch.Tensor,
626
+ *,
627
+ alpha: float = 1.0,
628
+ warps: int = 2,
629
+ stages: int = 6,
630
+ out: Optional[torch.Tensor] = None,
631
+ ) -> torch.Tensor:
632
+ """Multi-row (M<=16) warp-split-K NVFP4 W4A4 GEMM (SM120).
633
+
634
+ The 16x8x64 block-scaled MMA atom computes a full 16-row output
635
+ tile, so up to sixteen rows ride one weight stream at near-GEMV
636
+ cost - the spec-verify block and its re-advance prefixes are the
637
+ customers. Same packed/scale layouts as the linear entry points;
638
+ deeper default stages hide the strided-B latency the extra A-row
639
+ loads expose."""
640
+ if out is None:
641
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
642
+ ops.fp4_w4a4_gemm_warpsplit_mrows_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
643
+ return out
644
+
645
+
646
  def fp4_w4a4_gemv_warpsplit_bf16(
647
  a_packed: torch.Tensor,
648
  b_packed: torch.Tensor,
 
919
  "fp4_repack_b_interleaved_sm120",
920
  "fp4_w4a4_gemv_warpsplit_bf16",
921
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
922
+ "fp4_w4a4_gemm_warpsplit_mrows_bf16",
923
  "nvfp4_gemm_bf16",
924
  "nvfp4_gemm_fp16",
925
  "nvfp4_gemm_variant_bf16",
build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_014dcce.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0d0f55ad6312f014e86e534fce4f0758b2f6f353a1db23c1e0fbe430d0f67921
3
- size 8296848
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b580778bebed486d4865d0e9b5dd09282d2545ea7417f961b0acdb3289874e5a
3
+ size 8564792
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_014dcce
3
- ops = torch.ops._fp4_gemm_cuda_014dcce
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_014dcce::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_e1f10b6
3
+ ops = torch.ops._fp4_gemm_cuda_e1f10b6
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_e1f10b6::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_014dcce",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,9 +14,9 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
18
- "_fp4_gemm_cuda_014dcce.abi3.so": "DQ9VrWMS8BToblNPzk8HWLL281Oh2yPB4PvkMND2eSE=",
19
- "_ops.py": "3SSvweMyei7LivOTJNIPAdX+/MPOur59X7Qf5gA8m0Q=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
@@ -27,7 +27,7 @@
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "014dcce673c8660d9e8932be3926f3b85340aac8",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_e1f10b6",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "x7M8jFG4z1Mx/zhAh30IsOFooDn28i8uyrkkNIjyG+c=",
18
+ "_fp4_gemm_cuda_e1f10b6.abi3.so": "tYB3i+vtSG1IZdDptd0JKC0lRep0F/lhsKzbMomHTlo=",
19
+ "_ops.py": "FGdLd66XdfjhguPY8EEn/zd8T/lGPpQP+r8VJnZaCPE=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
 
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "e1f10b68fd3d9f68ecd85895128283aa6768ea6f",
31
  "dirty": false
32
  }
33
  }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -34,6 +34,9 @@ def capabilities() -> dict[str, object]:
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
 
 
 
37
  "errors": "exceptions",
38
  }
39
 
@@ -98,6 +101,27 @@ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
98
  return None
99
 
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
102
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
103
  if a_packed.shape[0] != 1:
@@ -594,6 +618,31 @@ def nvfp4_gemm_bias_residual_fp16(
594
  return out
595
 
596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
597
  def fp4_w4a4_gemv_warpsplit_bf16(
598
  a_packed: torch.Tensor,
599
  b_packed: torch.Tensor,
@@ -870,6 +919,7 @@ __all__ = [
870
  "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
 
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
 
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "sm120_warpsplit_mrows": (1, 16),
38
+ "sm120_warpsplit_mrows_n_alignment": 8,
39
+ "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
  "errors": "exceptions",
41
  }
42
 
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemm_warpsplit_mrows_bf16"))
105
+ def _gemm_warpsplit_mrows_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 2, stages: int = 6) -> None:
106
+ if a_packed.dim() != 2 or b_packed.dim() != 2:
107
+ raise RuntimeError("a_packed and b_packed must be rank-2")
108
+ if a_packed.shape[1] != b_packed.shape[1]:
109
+ raise RuntimeError("a_packed and b_packed must have the same K / 2")
110
+ if not 1 <= a_packed.shape[0] <= 16:
111
+ raise RuntimeError("the multi-row warp-split tier serves 1..16 rows")
112
+ if b_packed.shape[0] % 8:
113
+ raise RuntimeError("N must be a multiple of 8")
114
+ if warps not in (2, 4, 8):
115
+ raise RuntimeError("warps must be 2, 4 or 8")
116
+ if stages not in (3, 4, 6) or (warps == 8 and stages == 6):
117
+ raise RuntimeError("unsupported stages/warps combination")
118
+ if (a_packed.shape[1] * 2) % (64 * warps):
119
+ raise RuntimeError("K must be a multiple of 64*warps")
120
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
121
+ raise RuntimeError("out must have shape (M, N)")
122
+ return None
123
+
124
+
125
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
126
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
127
  if a_packed.shape[0] != 1:
 
618
  return out
619
 
620
 
621
+ def fp4_w4a4_gemm_warpsplit_mrows_bf16(
622
+ a_packed: torch.Tensor,
623
+ b_packed: torch.Tensor,
624
+ sfa: torch.Tensor,
625
+ sfb: torch.Tensor,
626
+ *,
627
+ alpha: float = 1.0,
628
+ warps: int = 2,
629
+ stages: int = 6,
630
+ out: Optional[torch.Tensor] = None,
631
+ ) -> torch.Tensor:
632
+ """Multi-row (M<=16) warp-split-K NVFP4 W4A4 GEMM (SM120).
633
+
634
+ The 16x8x64 block-scaled MMA atom computes a full 16-row output
635
+ tile, so up to sixteen rows ride one weight stream at near-GEMV
636
+ cost - the spec-verify block and its re-advance prefixes are the
637
+ customers. Same packed/scale layouts as the linear entry points;
638
+ deeper default stages hide the strided-B latency the extra A-row
639
+ loads expose."""
640
+ if out is None:
641
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
642
+ ops.fp4_w4a4_gemm_warpsplit_mrows_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
643
+ return out
644
+
645
+
646
  def fp4_w4a4_gemv_warpsplit_bf16(
647
  a_packed: torch.Tensor,
648
  b_packed: torch.Tensor,
 
919
  "fp4_repack_b_interleaved_sm120",
920
  "fp4_w4a4_gemv_warpsplit_bf16",
921
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
922
+ "fp4_w4a4_gemm_warpsplit_mrows_bf16",
923
  "nvfp4_gemm_bf16",
924
  "nvfp4_gemm_fp16",
925
  "nvfp4_gemm_variant_bf16",
build/torch212-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_e9b9a96.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:15fbd01a3c9cb4ce4e32c97e64de74c60081fc8c81a1e4c72b04a6fe350140f4
3
- size 8299008
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f950eb405f45cdfa822aa2ee646f87ddcda0c0c6de4d26a81b51947dd50bf3d7
3
+ size 8566680
build/torch212-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_e9b9a96
3
- ops = torch.ops._fp4_gemm_cuda_e9b9a96
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_e9b9a96::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_e1f10b6
3
+ ops = torch.ops._fp4_gemm_cuda_e1f10b6
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_e1f10b6::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/fp4_gemm/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,10 +1,9 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_e9b9a96",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
- "kernel-depends": [],
8
  "backend": {
9
  "type": "cuda",
10
  "archs": [
@@ -15,19 +14,20 @@
15
  "digest": {
16
  "algorithm": "sha256",
17
  "files": {
18
- "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
19
- "_fp4_gemm_cuda_e9b9a96.abi3.so": "FfvQGjyctM5OMsl+ZN50xgCB/IyBoeTHKwSm/jUBQPQ=",
20
- "_ops.py": "5NPF3p4eWS4ZGXUoXeRYwRUbDsdKALOF1OB+lo7QqL8="
 
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
- "sha": "3a5c09d941a19f582fe551d038dc2b3bc6b0a03d",
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "e9b9a9675d6a82b82c4863a3dbf800dfc3960466",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_e1f10b6",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "x7M8jFG4z1Mx/zhAh30IsOFooDn28i8uyrkkNIjyG+c=",
18
+ "_fp4_gemm_cuda_e1f10b6.abi3.so": "+VDrQF9FzfqCKqLuZG+H3c2gwMbeTSaoG1GUfdUL89c=",
19
+ "_ops.py": "FGdLd66XdfjhguPY8EEn/zd8T/lGPpQP+r8VJnZaCPE=",
20
+ "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
+ "sha": "b39ca23f1b36383df00b27b3ffe1276cd5dbea85",
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "e1f10b68fd3d9f68ecd85895128283aa6768ea6f",
31
  "dirty": false
32
  }
33
  }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -34,6 +34,9 @@ def capabilities() -> dict[str, object]:
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
 
 
 
37
  "errors": "exceptions",
38
  }
39
 
@@ -98,6 +101,27 @@ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
98
  return None
99
 
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
102
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
103
  if a_packed.shape[0] != 1:
@@ -594,6 +618,31 @@ def nvfp4_gemm_bias_residual_fp16(
594
  return out
595
 
596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
597
  def fp4_w4a4_gemv_warpsplit_bf16(
598
  a_packed: torch.Tensor,
599
  b_packed: torch.Tensor,
@@ -870,6 +919,7 @@ __all__ = [
870
  "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
 
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
 
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "sm120_warpsplit_mrows": (1, 16),
38
+ "sm120_warpsplit_mrows_n_alignment": 8,
39
+ "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
  "errors": "exceptions",
41
  }
42
 
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemm_warpsplit_mrows_bf16"))
105
+ def _gemm_warpsplit_mrows_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 2, stages: int = 6) -> None:
106
+ if a_packed.dim() != 2 or b_packed.dim() != 2:
107
+ raise RuntimeError("a_packed and b_packed must be rank-2")
108
+ if a_packed.shape[1] != b_packed.shape[1]:
109
+ raise RuntimeError("a_packed and b_packed must have the same K / 2")
110
+ if not 1 <= a_packed.shape[0] <= 16:
111
+ raise RuntimeError("the multi-row warp-split tier serves 1..16 rows")
112
+ if b_packed.shape[0] % 8:
113
+ raise RuntimeError("N must be a multiple of 8")
114
+ if warps not in (2, 4, 8):
115
+ raise RuntimeError("warps must be 2, 4 or 8")
116
+ if stages not in (3, 4, 6) or (warps == 8 and stages == 6):
117
+ raise RuntimeError("unsupported stages/warps combination")
118
+ if (a_packed.shape[1] * 2) % (64 * warps):
119
+ raise RuntimeError("K must be a multiple of 64*warps")
120
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
121
+ raise RuntimeError("out must have shape (M, N)")
122
+ return None
123
+
124
+
125
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
126
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
127
  if a_packed.shape[0] != 1:
 
618
  return out
619
 
620
 
621
+ def fp4_w4a4_gemm_warpsplit_mrows_bf16(
622
+ a_packed: torch.Tensor,
623
+ b_packed: torch.Tensor,
624
+ sfa: torch.Tensor,
625
+ sfb: torch.Tensor,
626
+ *,
627
+ alpha: float = 1.0,
628
+ warps: int = 2,
629
+ stages: int = 6,
630
+ out: Optional[torch.Tensor] = None,
631
+ ) -> torch.Tensor:
632
+ """Multi-row (M<=16) warp-split-K NVFP4 W4A4 GEMM (SM120).
633
+
634
+ The 16x8x64 block-scaled MMA atom computes a full 16-row output
635
+ tile, so up to sixteen rows ride one weight stream at near-GEMV
636
+ cost - the spec-verify block and its re-advance prefixes are the
637
+ customers. Same packed/scale layouts as the linear entry points;
638
+ deeper default stages hide the strided-B latency the extra A-row
639
+ loads expose."""
640
+ if out is None:
641
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
642
+ ops.fp4_w4a4_gemm_warpsplit_mrows_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
643
+ return out
644
+
645
+
646
  def fp4_w4a4_gemv_warpsplit_bf16(
647
  a_packed: torch.Tensor,
648
  b_packed: torch.Tensor,
 
919
  "fp4_repack_b_interleaved_sm120",
920
  "fp4_w4a4_gemv_warpsplit_bf16",
921
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
922
+ "fp4_w4a4_gemm_warpsplit_mrows_bf16",
923
  "nvfp4_gemm_bf16",
924
  "nvfp4_gemm_fp16",
925
  "nvfp4_gemm_variant_bf16",
build/torch212-cxx11-cu132-x86_64-linux/{_fp4_gemm_cuda_e9b9a96.abi3.so → _fp4_gemm_cuda_e1f10b6.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:45eb22b74cbd7a27a38137319c3a16588ced4da2b5969cf577da74499e960acb
3
- size 8282672
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20b51c0058864befdf526936c452585e317d43341059d3a924ca6797250ae23e
3
+ size 8550344
build/torch212-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_e9b9a96
3
- ops = torch.ops._fp4_gemm_cuda_e9b9a96
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_e9b9a96::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_e1f10b6
3
+ ops = torch.ops._fp4_gemm_cuda_e1f10b6
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_e1f10b6::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/fp4_gemm/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,10 +1,9 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_e9b9a96",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
- "kernel-depends": [],
8
  "backend": {
9
  "type": "cuda",
10
  "archs": [
@@ -15,19 +14,20 @@
15
  "digest": {
16
  "algorithm": "sha256",
17
  "files": {
18
- "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
19
- "_fp4_gemm_cuda_e9b9a96.abi3.so": "Resit0y9eiejgTcxnDoWWIztTaK1lpz1d9p0SZ6WCss=",
20
- "_ops.py": "5NPF3p4eWS4ZGXUoXeRYwRUbDsdKALOF1OB+lo7QqL8="
 
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
- "sha": "3a5c09d941a19f582fe551d038dc2b3bc6b0a03d",
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "e9b9a9675d6a82b82c4863a3dbf800dfc3960466",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_e1f10b6",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "x7M8jFG4z1Mx/zhAh30IsOFooDn28i8uyrkkNIjyG+c=",
18
+ "_fp4_gemm_cuda_e1f10b6.abi3.so": "ILUcAFiGS+/fUmk2xFJYXjF9QzQQWdOpJMpnlyUK4j4=",
19
+ "_ops.py": "FGdLd66XdfjhguPY8EEn/zd8T/lGPpQP+r8VJnZaCPE=",
20
+ "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
+ "sha": "b39ca23f1b36383df00b27b3ffe1276cd5dbea85",
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "e1f10b68fd3d9f68ecd85895128283aa6768ea6f",
31
  "dirty": false
32
  }
33
  }
build/torch213-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -34,6 +34,9 @@ def capabilities() -> dict[str, object]:
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
 
 
 
37
  "errors": "exceptions",
38
  }
39
 
@@ -98,6 +101,27 @@ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
98
  return None
99
 
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
102
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
103
  if a_packed.shape[0] != 1:
@@ -594,6 +618,31 @@ def nvfp4_gemm_bias_residual_fp16(
594
  return out
595
 
596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
597
  def fp4_w4a4_gemv_warpsplit_bf16(
598
  a_packed: torch.Tensor,
599
  b_packed: torch.Tensor,
@@ -870,6 +919,7 @@ __all__ = [
870
  "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
 
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
 
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "sm120_warpsplit_mrows": (1, 16),
38
+ "sm120_warpsplit_mrows_n_alignment": 8,
39
+ "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
  "errors": "exceptions",
41
  }
42
 
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemm_warpsplit_mrows_bf16"))
105
+ def _gemm_warpsplit_mrows_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 2, stages: int = 6) -> None:
106
+ if a_packed.dim() != 2 or b_packed.dim() != 2:
107
+ raise RuntimeError("a_packed and b_packed must be rank-2")
108
+ if a_packed.shape[1] != b_packed.shape[1]:
109
+ raise RuntimeError("a_packed and b_packed must have the same K / 2")
110
+ if not 1 <= a_packed.shape[0] <= 16:
111
+ raise RuntimeError("the multi-row warp-split tier serves 1..16 rows")
112
+ if b_packed.shape[0] % 8:
113
+ raise RuntimeError("N must be a multiple of 8")
114
+ if warps not in (2, 4, 8):
115
+ raise RuntimeError("warps must be 2, 4 or 8")
116
+ if stages not in (3, 4, 6) or (warps == 8 and stages == 6):
117
+ raise RuntimeError("unsupported stages/warps combination")
118
+ if (a_packed.shape[1] * 2) % (64 * warps):
119
+ raise RuntimeError("K must be a multiple of 64*warps")
120
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
121
+ raise RuntimeError("out must have shape (M, N)")
122
+ return None
123
+
124
+
125
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
126
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
127
  if a_packed.shape[0] != 1:
 
618
  return out
619
 
620
 
621
+ def fp4_w4a4_gemm_warpsplit_mrows_bf16(
622
+ a_packed: torch.Tensor,
623
+ b_packed: torch.Tensor,
624
+ sfa: torch.Tensor,
625
+ sfb: torch.Tensor,
626
+ *,
627
+ alpha: float = 1.0,
628
+ warps: int = 2,
629
+ stages: int = 6,
630
+ out: Optional[torch.Tensor] = None,
631
+ ) -> torch.Tensor:
632
+ """Multi-row (M<=16) warp-split-K NVFP4 W4A4 GEMM (SM120).
633
+
634
+ The 16x8x64 block-scaled MMA atom computes a full 16-row output
635
+ tile, so up to sixteen rows ride one weight stream at near-GEMV
636
+ cost - the spec-verify block and its re-advance prefixes are the
637
+ customers. Same packed/scale layouts as the linear entry points;
638
+ deeper default stages hide the strided-B latency the extra A-row
639
+ loads expose."""
640
+ if out is None:
641
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
642
+ ops.fp4_w4a4_gemm_warpsplit_mrows_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
643
+ return out
644
+
645
+
646
  def fp4_w4a4_gemv_warpsplit_bf16(
647
  a_packed: torch.Tensor,
648
  b_packed: torch.Tensor,
 
919
  "fp4_repack_b_interleaved_sm120",
920
  "fp4_w4a4_gemv_warpsplit_bf16",
921
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
922
+ "fp4_w4a4_gemm_warpsplit_mrows_bf16",
923
  "nvfp4_gemm_bf16",
924
  "nvfp4_gemm_fp16",
925
  "nvfp4_gemm_variant_bf16",
build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_e1f10b6.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f37120b35de71e695142d6f77a82f68e97a0403647fd6b0a7f251a9e9d564101
3
+ size 8566520
build/torch213-cxx11-cu130-x86_64-linux/_fp4_gemm_cuda_e9b9a96.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:fc7617d8d02b6b38a52255d16f99e9b77b35b10132725d713c29b34bc82e38b7
3
- size 8298848
 
 
 
 
build/torch213-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_e9b9a96
3
- ops = torch.ops._fp4_gemm_cuda_e9b9a96
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_e9b9a96::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_e1f10b6
3
+ ops = torch.ops._fp4_gemm_cuda_e1f10b6
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_e1f10b6::{op_name}"
build/torch213-cxx11-cu130-x86_64-linux/fp4_gemm/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch213-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,10 +1,9 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_e9b9a96",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
- "kernel-depends": [],
8
  "backend": {
9
  "type": "cuda",
10
  "archs": [
@@ -15,19 +14,20 @@
15
  "digest": {
16
  "algorithm": "sha256",
17
  "files": {
18
- "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
19
- "_fp4_gemm_cuda_e9b9a96.abi3.so": "/HYX2NArazilIlXRb5npt3s1sQEycl1xPCmzS8guOLc=",
20
- "_ops.py": "5NPF3p4eWS4ZGXUoXeRYwRUbDsdKALOF1OB+lo7QqL8="
 
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
- "sha": "3a5c09d941a19f582fe551d038dc2b3bc6b0a03d",
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "e9b9a9675d6a82b82c4863a3dbf800dfc3960466",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_e1f10b6",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "x7M8jFG4z1Mx/zhAh30IsOFooDn28i8uyrkkNIjyG+c=",
18
+ "_fp4_gemm_cuda_e1f10b6.abi3.so": "83Egs13nHmlRQtb3eoL2jpegQDZH/WsKfyUanp1WQQE=",
19
+ "_ops.py": "FGdLd66XdfjhguPY8EEn/zd8T/lGPpQP+r8VJnZaCPE=",
20
+ "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
+ "sha": "b39ca23f1b36383df00b27b3ffe1276cd5dbea85",
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "e1f10b68fd3d9f68ecd85895128283aa6768ea6f",
31
  "dirty": false
32
  }
33
  }
build/torch213-cxx11-cu132-x86_64-linux/__init__.py CHANGED
@@ -34,6 +34,9 @@ def capabilities() -> dict[str, object]:
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
 
 
 
37
  "errors": "exceptions",
38
  }
39
 
@@ -98,6 +101,27 @@ def _bias_residual_fp16_fake(a, b, sfa, sfb, bias, residual, out) -> None:
98
  return None
99
 
100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
101
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
102
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
103
  if a_packed.shape[0] != 1:
@@ -594,6 +618,31 @@ def nvfp4_gemm_bias_residual_fp16(
594
  return out
595
 
596
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
597
  def fp4_w4a4_gemv_warpsplit_bf16(
598
  a_packed: torch.Tensor,
599
  b_packed: torch.Tensor,
@@ -870,6 +919,7 @@ __all__ = [
870
  "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
 
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
 
34
  "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
  "sm120_interleaved_gemv_m": 1,
36
  "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "sm120_warpsplit_mrows": (1, 16),
38
+ "sm120_warpsplit_mrows_n_alignment": 8,
39
+ "sm120_warpsplit_mrows_k_alignment": "64 * warps",
40
  "errors": "exceptions",
41
  }
42
 
 
101
  return None
102
 
103
 
104
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemm_warpsplit_mrows_bf16"))
105
+ def _gemm_warpsplit_mrows_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 2, stages: int = 6) -> None:
106
+ if a_packed.dim() != 2 or b_packed.dim() != 2:
107
+ raise RuntimeError("a_packed and b_packed must be rank-2")
108
+ if a_packed.shape[1] != b_packed.shape[1]:
109
+ raise RuntimeError("a_packed and b_packed must have the same K / 2")
110
+ if not 1 <= a_packed.shape[0] <= 16:
111
+ raise RuntimeError("the multi-row warp-split tier serves 1..16 rows")
112
+ if b_packed.shape[0] % 8:
113
+ raise RuntimeError("N must be a multiple of 8")
114
+ if warps not in (2, 4, 8):
115
+ raise RuntimeError("warps must be 2, 4 or 8")
116
+ if stages not in (3, 4, 6) or (warps == 8 and stages == 6):
117
+ raise RuntimeError("unsupported stages/warps combination")
118
+ if (a_packed.shape[1] * 2) % (64 * warps):
119
+ raise RuntimeError("K must be a multiple of 64*warps")
120
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
121
+ raise RuntimeError("out must have shape (M, N)")
122
+ return None
123
+
124
+
125
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_bf16"))
126
  def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0, warps: int = 4, stages: int = 4) -> None:
127
  if a_packed.shape[0] != 1:
 
618
  return out
619
 
620
 
621
+ def fp4_w4a4_gemm_warpsplit_mrows_bf16(
622
+ a_packed: torch.Tensor,
623
+ b_packed: torch.Tensor,
624
+ sfa: torch.Tensor,
625
+ sfb: torch.Tensor,
626
+ *,
627
+ alpha: float = 1.0,
628
+ warps: int = 2,
629
+ stages: int = 6,
630
+ out: Optional[torch.Tensor] = None,
631
+ ) -> torch.Tensor:
632
+ """Multi-row (M<=16) warp-split-K NVFP4 W4A4 GEMM (SM120).
633
+
634
+ The 16x8x64 block-scaled MMA atom computes a full 16-row output
635
+ tile, so up to sixteen rows ride one weight stream at near-GEMV
636
+ cost - the spec-verify block and its re-advance prefixes are the
637
+ customers. Same packed/scale layouts as the linear entry points;
638
+ deeper default stages hide the strided-B latency the extra A-row
639
+ loads expose."""
640
+ if out is None:
641
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
642
+ ops.fp4_w4a4_gemm_warpsplit_mrows_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(warps), int(stages))
643
+ return out
644
+
645
+
646
  def fp4_w4a4_gemv_warpsplit_bf16(
647
  a_packed: torch.Tensor,
648
  b_packed: torch.Tensor,
 
919
  "fp4_repack_b_interleaved_sm120",
920
  "fp4_w4a4_gemv_warpsplit_bf16",
921
  "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
922
+ "fp4_w4a4_gemm_warpsplit_mrows_bf16",
923
  "nvfp4_gemm_bf16",
924
  "nvfp4_gemm_fp16",
925
  "nvfp4_gemm_variant_bf16",
build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_e1f10b6.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a519e2bfab7a53c2a6564c2c932c91f8377213082a22471e45bf42a35c8c9cc8
3
+ size 8550192
build/torch213-cxx11-cu132-x86_64-linux/_fp4_gemm_cuda_e9b9a96.abi3.so DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e4182625c082dd70a0d83613c92c8fa5f526d2dee6c82903eddca5ea365f8363
3
- size 8282512
 
 
 
 
build/torch213-cxx11-cu132-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_e9b9a96
3
- ops = torch.ops._fp4_gemm_cuda_e9b9a96
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_e9b9a96::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_e1f10b6
3
+ ops = torch.ops._fp4_gemm_cuda_e1f10b6
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_e1f10b6::{op_name}"
build/torch213-cxx11-cu132-x86_64-linux/fp4_gemm/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch213-cxx11-cu132-x86_64-linux/metadata.json CHANGED
@@ -1,10 +1,9 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_e9b9a96",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
- "kernel-depends": [],
8
  "backend": {
9
  "type": "cuda",
10
  "archs": [
@@ -15,19 +14,20 @@
15
  "digest": {
16
  "algorithm": "sha256",
17
  "files": {
18
- "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
19
- "_fp4_gemm_cuda_e9b9a96.abi3.so": "5BgmJcCC3XCg2DYTySyPpfUm0t7myCkD7dyl6jZfg2M=",
20
- "_ops.py": "5NPF3p4eWS4ZGXUoXeRYwRUbDsdKALOF1OB+lo7QqL8="
 
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
- "sha": "3a5c09d941a19f582fe551d038dc2b3bc6b0a03d",
27
  "dirty": false
28
  },
29
  "kernel": {
30
- "sha": "e9b9a9675d6a82b82c4863a3dbf800dfc3960466",
31
  "dirty": false
32
  }
33
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_e1f10b6",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "x7M8jFG4z1Mx/zhAh30IsOFooDn28i8uyrkkNIjyG+c=",
18
+ "_fp4_gemm_cuda_e1f10b6.abi3.so": "pRniv6t6U8KmVkwskyyR+DdyEwgqIkceRb9Co1yMnMg=",
19
+ "_ops.py": "FGdLd66XdfjhguPY8EEn/zd8T/lGPpQP+r8VJnZaCPE=",
20
+ "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
  },
23
  "provenance": {
24
  "kernel-builder": {
25
  "version": "0.17.0-dev0",
26
+ "sha": "b39ca23f1b36383df00b27b3ffe1276cd5dbea85",
27
  "dirty": false
28
  },
29
  "kernel": {
30
+ "sha": "e1f10b68fd3d9f68ecd85895128283aa6768ea6f",
31
  "dirty": false
32
  }
33
  }