liangsu9988 commited on
Commit
87f1bbc
·
verified ·
1 Parent(s): 7ff2242

Uploaded using `kernel-builder`.

Browse files
build/torch211-cxx11-cu130-x86_64-linux/__init__.py CHANGED
@@ -7,6 +7,37 @@ import torch
7
  from ._ops import add_op_namespace_prefix, ops
8
 
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  def sfa_size_bytes(rows: int, dim: int) -> int:
11
  if rows <= 0 or dim <= 0 or dim % 16 != 0:
12
  raise ValueError("rows must be positive and dim must be positive/divisible by 16")
@@ -76,6 +107,34 @@ def _gemv_warpsplit_fake(a_packed, b_packed, sfa, sfb, out, alpha: float = 1.0,
76
  return None
77
 
78
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
80
  def _legacy_linear_fake(
81
  a_packed: torch.Tensor,
@@ -559,6 +618,89 @@ def fp4_w4a4_gemv_warpsplit_bf16(
559
  return out
560
 
561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
562
  def fp4_w4a16_linear_bf16(
563
  a_packed: torch.Tensor,
564
  b_packed: torch.Tensor,
@@ -714,16 +856,26 @@ def nvfp4_gemm_streamk_bias_bf16(
714
 
715
 
716
  __all__ = [
 
 
 
 
 
 
717
  "aligned_fp4_dim",
718
  "cutlass_fp4_gemm_geglu_il_hw_v10",
719
  "dequantize_fp4_sfa_fp16",
720
  "e0m3_weight_gemm_fp16",
721
  "fp4_w4a16_linear_bf16",
 
722
  "fp4_w4a4_gemv_warpsplit_bf16",
 
723
  "nvfp4_gemm_bf16",
724
  "nvfp4_gemm_fp16",
725
  "nvfp4_gemm_variant_bf16",
726
  "nvfp4_gemm_nvfp4",
 
 
727
  "nvfp4_gemm_geglu_nvfp4_fp16",
728
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
729
  "nvfp4_gemm_bias_residual_fp16",
 
7
  from ._ops import add_op_namespace_prefix, ops
8
 
9
 
10
+ NVFP4_BLOCK_SIZE = 16
11
+ NVFP4_SCALE_TILE_ROWS = 128
12
+ NVFP4_SCALE_TILE_BLOCKS = 4
13
+ SUPPORTED_LAYOUTS = ("row-major-packed-e2m1", "cutlass-sm1xx-blockscaled")
14
+ SUPPORTED_CUDA_CAPABILITIES = ("11.0a", "12.0a")
15
+
16
+
17
+ def capabilities() -> dict[str, object]:
18
+ """Return the public Tensor API contract used by runtime dispatchers."""
19
+ return {
20
+ "quantization": "W4A4 NVFP4 E2M1",
21
+ "block_size": NVFP4_BLOCK_SIZE,
22
+ "scale_layout": "pad128(rows) x pad4(dim/16), 512 bytes per tile",
23
+ "scale_size": "ceil(rows/128) * ceil((dim/16)/4) * 512 bytes",
24
+ "layouts": SUPPORTED_LAYOUTS,
25
+ "cuda_capabilities": SUPPORTED_CUDA_CAPABILITIES,
26
+ "public_m_alignment": 1,
27
+ "raw_sm120_tile_m": 128,
28
+ "sm120_m256_min_m": 512,
29
+ "sm120_m256_qualified_nk": (
30
+ (17408, 5120),
31
+ (5120, 17408),
32
+ (12288, 5120),
33
+ ),
34
+ "sm120_m256_diagnostic_nk": ((16384, 5120),),
35
+ "sm120_interleaved_gemv_m": 1,
36
+ "sm120_interleaved_weight_layout": "groups-of-8 x K/64 x 8 x 32B",
37
+ "errors": "exceptions",
38
+ }
39
+
40
+
41
  def sfa_size_bytes(rows: int, dim: int) -> int:
42
  if rows <= 0 or dim <= 0 or dim % 16 != 0:
43
  raise ValueError("rows must be positive and dim must be positive/divisible by 16")
 
107
  return None
108
 
109
 
110
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_repack_b_interleaved_sm120"))
111
+ def _repack_b_interleaved_fake(b_packed, b_interleaved) -> None:
112
+ if b_interleaved.shape != b_packed.shape:
113
+ raise RuntimeError("b_interleaved must have the same shape as b_packed")
114
+ return None
115
+
116
+
117
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a4_gemv_warpsplit_interleaved_bf16"))
118
+ def _gemv_warpsplit_interleaved_fake(
119
+ a_packed, b_interleaved, sfa, sfb, out,
120
+ alpha: float = 1.0, warps: int = 4, stages: int = 4,
121
+ ) -> None:
122
+ if a_packed.shape[0] != 1:
123
+ raise RuntimeError("interleaved warp-split GEMV serves M=1 only")
124
+ if out.shape != (1, b_interleaved.shape[0]):
125
+ raise RuntimeError("out must have shape (1, N)")
126
+ return None
127
+
128
+
129
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_m256_bf16"))
130
+ def _m256_fake(a_packed, b_packed, sfa, sfb, workspace, out, alpha: float = 1.0) -> None:
131
+ if a_packed.shape[0] < 512:
132
+ raise RuntimeError("the M256 tier requires M >= 512")
133
+ if out.shape != (a_packed.shape[0], b_packed.shape[0]):
134
+ raise RuntimeError("out must have shape (M, N)")
135
+ return None
136
+
137
+
138
  @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
139
  def _legacy_linear_fake(
140
  a_packed: torch.Tensor,
 
618
  return out
619
 
620
 
621
+ def fp4_repack_b_interleaved_sm120(
622
+ b_packed: torch.Tensor,
623
+ *,
624
+ out: Optional[torch.Tensor] = None,
625
+ ) -> torch.Tensor:
626
+ """Repack dense row-major packed FP4 weights for the SM120 M=1 GEMV.
627
+
628
+ This is a bind-time operation. Cache the returned tensor with the packed
629
+ weight; do not execute it in the decode hot path.
630
+ """
631
+ if out is None:
632
+ out = torch.empty_like(b_packed)
633
+ ops.fp4_repack_b_interleaved_sm120(b_packed, out)
634
+ return out
635
+
636
+
637
+ def fp4_w4a4_gemv_warpsplit_interleaved_bf16(
638
+ a_packed: torch.Tensor,
639
+ b_interleaved: torch.Tensor,
640
+ sfa: torch.Tensor,
641
+ sfb: torch.Tensor,
642
+ *,
643
+ alpha: float = 1.0,
644
+ warps: int = 8,
645
+ stages: int = 3,
646
+ out: Optional[torch.Tensor] = None,
647
+ ) -> torch.Tensor:
648
+ """M=1 SM120 W4A4 GEMV using bind-time interleaved FP4 weights."""
649
+ if out is None:
650
+ out = torch.empty(
651
+ (1, b_interleaved.shape[0]),
652
+ device=a_packed.device,
653
+ dtype=torch.bfloat16,
654
+ )
655
+ ops.fp4_w4a4_gemv_warpsplit_interleaved_bf16(
656
+ a_packed, b_interleaved, sfa, sfb, out,
657
+ float(alpha), int(warps), int(stages),
658
+ )
659
+ return out
660
+
661
+
662
+ def nvfp4_gemm_m256_workspace_size(
663
+ a_packed: torch.Tensor,
664
+ b_packed: torch.Tensor,
665
+ sfa: torch.Tensor,
666
+ sfb: torch.Tensor,
667
+ ) -> int:
668
+ """Return workspace bytes for the SM120 large-M tier.
669
+
670
+ Query and allocate once before CUDA Graph capture.
671
+ """
672
+ return int(ops.nvfp4_gemm_m256_workspace_size(a_packed, b_packed, sfa, sfb))
673
+
674
+
675
+ def nvfp4_gemm_m256_bf16(
676
+ a_packed: torch.Tensor,
677
+ b_packed: torch.Tensor,
678
+ sfa: torch.Tensor,
679
+ sfb: torch.Tensor,
680
+ *,
681
+ workspace: Optional[torch.Tensor] = None,
682
+ alpha: float = 1.0,
683
+ out: Optional[torch.Tensor] = None,
684
+ ) -> torch.Tensor:
685
+ """SM120 M>=512 NVFP4 GEMM with a caller-owned graph-stable workspace."""
686
+ if workspace is None:
687
+ workspace = torch.empty(
688
+ nvfp4_gemm_m256_workspace_size(a_packed, b_packed, sfa, sfb),
689
+ device=a_packed.device,
690
+ dtype=torch.uint8,
691
+ )
692
+ if out is None:
693
+ out = torch.empty(
694
+ (a_packed.shape[0], b_packed.shape[0]),
695
+ device=a_packed.device,
696
+ dtype=torch.bfloat16,
697
+ )
698
+ ops.nvfp4_gemm_m256_bf16(
699
+ a_packed, b_packed, sfa, sfb, workspace, out, float(alpha)
700
+ )
701
+ return out
702
+
703
+
704
  def fp4_w4a16_linear_bf16(
705
  a_packed: torch.Tensor,
706
  b_packed: torch.Tensor,
 
856
 
857
 
858
  __all__ = [
859
+ "NVFP4_BLOCK_SIZE",
860
+ "NVFP4_SCALE_TILE_ROWS",
861
+ "NVFP4_SCALE_TILE_BLOCKS",
862
+ "SUPPORTED_CUDA_CAPABILITIES",
863
+ "SUPPORTED_LAYOUTS",
864
+ "capabilities",
865
  "aligned_fp4_dim",
866
  "cutlass_fp4_gemm_geglu_il_hw_v10",
867
  "dequantize_fp4_sfa_fp16",
868
  "e0m3_weight_gemm_fp16",
869
  "fp4_w4a16_linear_bf16",
870
+ "fp4_repack_b_interleaved_sm120",
871
  "fp4_w4a4_gemv_warpsplit_bf16",
872
+ "fp4_w4a4_gemv_warpsplit_interleaved_bf16",
873
  "nvfp4_gemm_bf16",
874
  "nvfp4_gemm_fp16",
875
  "nvfp4_gemm_variant_bf16",
876
  "nvfp4_gemm_nvfp4",
877
+ "nvfp4_gemm_m256_bf16",
878
+ "nvfp4_gemm_m256_workspace_size",
879
  "nvfp4_gemm_geglu_nvfp4_fp16",
880
  "nvfp4_gemm_bias_gelu_nvfp4_fp16",
881
  "nvfp4_gemm_bias_residual_fp16",
build/torch211-cxx11-cu130-x86_64-linux/{_fp4_gemm_cuda_e79264b.abi3.so → _fp4_gemm_cuda_014dcce.abi3.so} RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:59783c76c8c360b0adc9177f8adef8bdd496d8db590419ee16964f923d8b100e
3
- size 7893704
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d0f55ad6312f014e86e534fce4f0758b2f6f353a1db23c1e0fbe430d0f67921
3
+ size 8296848
build/torch211-cxx11-cu130-x86_64-linux/_ops.py CHANGED
@@ -1,9 +1,9 @@
1
  import torch
2
- from . import _fp4_gemm_cuda_e79264b
3
- ops = torch.ops._fp4_gemm_cuda_e79264b
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
- return f"_fp4_gemm_cuda_e79264b::{op_name}"
 
1
  import torch
2
+ from . import _fp4_gemm_cuda_014dcce
3
+ ops = torch.ops._fp4_gemm_cuda_014dcce
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
  """
7
  Prefix op by namespace.
8
  """
9
+ return f"_fp4_gemm_cuda_014dcce::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/metadata.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "name": "fp4-gemm",
3
- "id": "_fp4_gemm_cuda_e79264b",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
@@ -14,10 +14,21 @@
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
- "__init__.py": "Mr1/bAUzs+af68D+bRXX6iONDof//dzKgn6QU5pKdu8=",
18
- "_fp4_gemm_cuda_e79264b.abi3.so": "WXg8dsjDYLCtyRd/it74vdSW2NtZBBnuFpZPkj2LEA4=",
19
- "_ops.py": "R8mQX+2vC6Q0eFM37W/8tEtdmbSjYNVDtv/BySdH8kM=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
 
 
 
 
 
 
 
 
 
 
 
22
  }
23
  }
 
1
  {
2
  "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_014dcce",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
 
14
  "digest": {
15
  "algorithm": "sha256",
16
  "files": {
17
+ "__init__.py": "nTluIQWuLs7LotHRt43tTkaA+FufNMoFyUhMfJfbB+c=",
18
+ "_fp4_gemm_cuda_014dcce.abi3.so": "DQ9VrWMS8BToblNPzk8HWLL281Oh2yPB4PvkMND2eSE=",
19
+ "_ops.py": "3SSvweMyei7LivOTJNIPAdX+/MPOur59X7Qf5gA8m0Q=",
20
  "fp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
21
  }
22
+ },
23
+ "provenance": {
24
+ "kernel-builder": {
25
+ "version": "0.17.0-dev0",
26
+ "sha": "b39ca23f1b36383df00b27b3ffe1276cd5dbea85",
27
+ "dirty": false
28
+ },
29
+ "kernel": {
30
+ "sha": "014dcce673c8660d9e8932be3926f3b85340aac8",
31
+ "dirty": false
32
+ }
33
  }
34
  }