liangsu9988 commited on
Commit
fdf2b52
·
verified ·
1 Parent(s): ecaeaaa

Add torch211-cxx11-cu130-aarch64-linux SM110 artifact

Browse files
build/torch211-cxx11-cu130-aarch64-linux/__init__.py ADDED
@@ -0,0 +1,254 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT FP4 GEMM kernels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import torch
6
+
7
+ from ._ops import add_op_namespace_prefix, ops
8
+
9
+
10
+ def sfa_size_bytes(rows: int, dim: int) -> int:
11
+ if rows <= 0 or dim <= 0 or dim % 16 != 0:
12
+ raise ValueError("rows must be positive and dim must be positive/divisible by 16")
13
+ n_blocks = dim // 16
14
+ n_row_super = (rows + 127) // 128
15
+ n_col_super = (n_blocks + 3) // 4
16
+ return n_row_super * n_col_super * 512
17
+
18
+
19
+ def _alloc_fp4(rows: int, dim: int, device: torch.device | str):
20
+ return (
21
+ torch.empty((rows, dim // 2), device=device, dtype=torch.uint8),
22
+ torch.empty((sfa_size_bytes(rows, dim),), device=device, dtype=torch.uint8),
23
+ )
24
+
25
+
26
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bf16"))
27
+ def _linear_fake(
28
+ a_packed: torch.Tensor,
29
+ b_packed: torch.Tensor,
30
+ sfa: torch.Tensor,
31
+ sfb: torch.Tensor,
32
+ out: torch.Tensor,
33
+ alpha: float = 1.0,
34
+ variant: int = -1,
35
+ ) -> None:
36
+ return None
37
+
38
+
39
+ @torch.library.register_fake(add_op_namespace_prefix("fp4_w4a16_linear_bf16"))
40
+ def _legacy_linear_fake(
41
+ a_packed: torch.Tensor,
42
+ b_packed: torch.Tensor,
43
+ sfa: torch.Tensor,
44
+ sfb: torch.Tensor,
45
+ out: torch.Tensor,
46
+ alpha: float = 1.0,
47
+ variant: int = -1,
48
+ ) -> None:
49
+ return None
50
+
51
+
52
+ @torch.library.register_fake(add_op_namespace_prefix("quantize_fp4_sfa_fp16"))
53
+ def _quant_fake(x: torch.Tensor, packed: torch.Tensor, sfa: torch.Tensor, is_sfb: bool = False) -> None:
54
+ return None
55
+
56
+
57
+ @torch.library.register_fake(add_op_namespace_prefix("dequantize_fp4_sfa_fp16"))
58
+ def _dequant_fake(packed: torch.Tensor, sfa: torch.Tensor, out: torch.Tensor, is_sfb: bool = False) -> None:
59
+ return None
60
+
61
+
62
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_residual_bf16"))
63
+ def _residual_fake(a, b, sfa, sfb, residual, out, alpha: float = 1.0) -> None:
64
+ return None
65
+
66
+
67
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_bf16"))
68
+ def _bias_gelu_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
69
+ return None
70
+
71
+
72
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_bias_gelu_nvfp4"))
73
+ def _bias_gelu_nvfp4_fake(
74
+ a, b, sfa, sfb, bias, out_packed, out_sfa, alpha: float = 1.0
75
+ ) -> None:
76
+ return None
77
+
78
+
79
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bf16"))
80
+ def _streamk_fake(a, b, sfa, sfb, out, alpha: float = 1.0) -> None:
81
+ return None
82
+
83
+
84
+ @torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemm_streamk_bias_bf16"))
85
+ def _streamk_bias_fake(a, b, sfa, sfb, bias, out, alpha: float = 1.0) -> None:
86
+ return None
87
+
88
+
89
+ def quantize_fp4_sfa_fp16(
90
+ x: torch.Tensor,
91
+ packed: torch.Tensor | None = None,
92
+ sfa: torch.Tensor | None = None,
93
+ is_sfb: bool = False,
94
+ ):
95
+ if packed is None or sfa is None:
96
+ packed, sfa = _alloc_fp4(x.shape[0], x.shape[1], x.device)
97
+ ops.quantize_fp4_sfa_fp16(x, packed, sfa, bool(is_sfb))
98
+ return packed, sfa
99
+
100
+
101
+ def dequantize_fp4_sfa_fp16(
102
+ packed: torch.Tensor,
103
+ sfa: torch.Tensor,
104
+ out: torch.Tensor | None = None,
105
+ is_sfb: bool = False,
106
+ ) -> torch.Tensor:
107
+ if out is None:
108
+ out = torch.empty((packed.shape[0], packed.shape[1] * 2), device=packed.device, dtype=torch.float16)
109
+ ops.dequantize_fp4_sfa_fp16(packed, sfa, out, bool(is_sfb))
110
+ return out
111
+
112
+
113
+ def nvfp4_gemm_bf16(
114
+ a_packed: torch.Tensor,
115
+ b_packed: torch.Tensor,
116
+ sfa: torch.Tensor,
117
+ sfb: torch.Tensor,
118
+ alpha: float = 1.0,
119
+ out: torch.Tensor | None = None,
120
+ variant: int = -1,
121
+ ) -> torch.Tensor:
122
+ if out is None:
123
+ out = torch.empty((a_packed.shape[0], b_packed.shape[0]), device=a_packed.device, dtype=torch.bfloat16)
124
+ ops.nvfp4_gemm_bf16(a_packed, b_packed, sfa, sfb, out, float(alpha), int(variant))
125
+ return out
126
+
127
+
128
+ def fp4_w4a16_linear_bf16(
129
+ a_packed: torch.Tensor,
130
+ b_packed: torch.Tensor,
131
+ sfa: torch.Tensor,
132
+ sfb: torch.Tensor,
133
+ alpha: float = 1.0,
134
+ out: torch.Tensor | None = None,
135
+ variant: int = -1,
136
+ ) -> torch.Tensor:
137
+ """Compatibility alias for :func:`nvfp4_gemm_bf16`."""
138
+ return nvfp4_gemm_bf16(
139
+ a_packed, b_packed, sfa, sfb, alpha=alpha, out=out, variant=variant
140
+ )
141
+
142
+
143
+ def nvfp4_gemm_residual_bf16(
144
+ a_packed: torch.Tensor,
145
+ b_packed: torch.Tensor,
146
+ sfa: torch.Tensor,
147
+ sfb: torch.Tensor,
148
+ residual: torch.Tensor,
149
+ alpha: float = 1.0,
150
+ out: torch.Tensor | None = None,
151
+ ) -> torch.Tensor:
152
+ if out is None:
153
+ out = torch.empty_like(residual)
154
+ ops.nvfp4_gemm_residual_bf16(
155
+ a_packed, b_packed, sfa, sfb, residual, out, float(alpha)
156
+ )
157
+ return out
158
+
159
+
160
+ def nvfp4_gemm_bias_gelu_bf16(
161
+ a_packed: torch.Tensor,
162
+ b_packed: torch.Tensor,
163
+ sfa: torch.Tensor,
164
+ sfb: torch.Tensor,
165
+ bias: torch.Tensor,
166
+ alpha: float = 1.0,
167
+ out: torch.Tensor | None = None,
168
+ ) -> torch.Tensor:
169
+ if out is None:
170
+ out = torch.empty(
171
+ (a_packed.shape[0], b_packed.shape[0]),
172
+ device=a_packed.device,
173
+ dtype=torch.bfloat16,
174
+ )
175
+ ops.nvfp4_gemm_bias_gelu_bf16(
176
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
177
+ )
178
+ return out
179
+
180
+
181
+ def nvfp4_gemm_bias_gelu_nvfp4(
182
+ a_packed: torch.Tensor,
183
+ b_packed: torch.Tensor,
184
+ sfa: torch.Tensor,
185
+ sfb: torch.Tensor,
186
+ bias: torch.Tensor,
187
+ alpha: float = 1.0,
188
+ out_packed: torch.Tensor | None = None,
189
+ out_sfa: torch.Tensor | None = None,
190
+ ) -> tuple[torch.Tensor, torch.Tensor]:
191
+ m, n = a_packed.shape[0], b_packed.shape[0]
192
+ if out_packed is None:
193
+ out_packed = torch.empty((m, n // 2), device=a_packed.device, dtype=torch.uint8)
194
+ if out_sfa is None:
195
+ out_sfa = torch.empty((sfa_size_bytes(m, n),), device=a_packed.device, dtype=torch.uint8)
196
+ ops.nvfp4_gemm_bias_gelu_nvfp4(
197
+ a_packed, b_packed, sfa, sfb, bias, out_packed, out_sfa, float(alpha)
198
+ )
199
+ return out_packed, out_sfa
200
+
201
+
202
+ def nvfp4_gemm_streamk_bf16(
203
+ a_packed: torch.Tensor,
204
+ b_packed: torch.Tensor,
205
+ sfa: torch.Tensor,
206
+ sfb: torch.Tensor,
207
+ alpha: float = 1.0,
208
+ out: torch.Tensor | None = None,
209
+ ) -> torch.Tensor:
210
+ if out is None:
211
+ out = torch.empty(
212
+ (a_packed.shape[0], b_packed.shape[0]),
213
+ device=a_packed.device,
214
+ dtype=torch.bfloat16,
215
+ )
216
+ ops.nvfp4_gemm_streamk_bf16(
217
+ a_packed, b_packed, sfa, sfb, out, float(alpha)
218
+ )
219
+ return out
220
+
221
+
222
+ def nvfp4_gemm_streamk_bias_bf16(
223
+ a_packed: torch.Tensor,
224
+ b_packed: torch.Tensor,
225
+ sfa: torch.Tensor,
226
+ sfb: torch.Tensor,
227
+ bias: torch.Tensor,
228
+ alpha: float = 1.0,
229
+ out: torch.Tensor | None = None,
230
+ ) -> torch.Tensor:
231
+ if out is None:
232
+ out = torch.empty(
233
+ (a_packed.shape[0], b_packed.shape[0]),
234
+ device=a_packed.device,
235
+ dtype=torch.bfloat16,
236
+ )
237
+ ops.nvfp4_gemm_streamk_bias_bf16(
238
+ a_packed, b_packed, sfa, sfb, bias, out, float(alpha)
239
+ )
240
+ return out
241
+
242
+
243
+ __all__ = [
244
+ "dequantize_fp4_sfa_fp16",
245
+ "fp4_w4a16_linear_bf16",
246
+ "nvfp4_gemm_bf16",
247
+ "nvfp4_gemm_bias_gelu_bf16",
248
+ "nvfp4_gemm_bias_gelu_nvfp4",
249
+ "nvfp4_gemm_residual_bf16",
250
+ "nvfp4_gemm_streamk_bf16",
251
+ "nvfp4_gemm_streamk_bias_bf16",
252
+ "quantize_fp4_sfa_fp16",
253
+ "sfa_size_bytes",
254
+ ]
build/torch211-cxx11-cu130-aarch64-linux/_fp4_gemm_cuda_7781728.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca5e0135a51929d6d3fdbb486ddfaf1de4b0626150eeb93a00de328036501acb
3
+ size 680560
build/torch211-cxx11-cu130-aarch64-linux/_ops.py ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _fp4_gemm_cuda_7781728
3
+ ops = torch.ops._fp4_gemm_cuda_7781728
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ return f"_fp4_gemm_cuda_7781728::{op_name}"
build/torch211-cxx11-cu130-aarch64-linux/fp4_gemm/__init__.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+
6
+ def _import_from_path(file_path: Path):
7
+ path_hash = '{:x}'.format(ctypes.c_size_t(hash(file_path.absolute())).value)
8
+ spec = importlib.util.spec_from_file_location(path_hash, file_path)
9
+ module = importlib.util.module_from_spec(spec)
10
+ sys.modules[path_hash] = module
11
+ spec.loader.exec_module(module)
12
+ return module
13
+
14
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / '__init__.py')))
build/torch211-cxx11-cu130-aarch64-linux/metadata.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "fp4-gemm",
3
+ "id": "_fp4_gemm_cuda_7781728",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "11.0a"
11
+ ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "EJ8HPhVwOKhifotSgdwCmB6MRbtnP1S0QfUGOVascy0=",
17
+ "_fp4_gemm_cuda_7781728.abi3.so": "yl4BNaUZKdbT/btIbd+vHeSwYmFQ7rk6AN4ygDZQGss=",
18
+ "_ops.py": "V90sTDYZObC3Ga+eak9fZiO8p6DsMdNZaWpcnYd61Yo=",
19
+ "fp4_gemm/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
+ }
21
+ },
22
+ "provenance": {
23
+ "kernel": {
24
+ "sha": "4c95454360b18c79bed75ec6346a7bac47ae2ca2",
25
+ "dirty": false
26
+ },
27
+ "validation": {
28
+ "torch": "2.11.0+cu130",
29
+ "cuda": "13.0"
30
+ }
31
+ }
32
+ }