liangsu9988 commited on
Commit
abed882
·
verified ·
1 Parent(s): af4f1aa

Add torch211-cxx11-cu130-aarch64-linux SM110 artifact

Browse files
build/torch211-cxx11-cu130-aarch64-linux/__init__.py CHANGED
@@ -55,6 +55,50 @@ def _fp8_gemm_bf16_fake(
55
  return None
56
 
57
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
59
  def _fp8_linear_bias_gelu_quant_bf16_fake(
60
  input: torch.Tensor,
@@ -101,6 +145,34 @@ def _fp8_gelu_mlp_bf16_fake(
101
  return None
102
 
103
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
105
  return torch.tensor([value], device=input.device, dtype=torch.float32)
106
 
@@ -129,6 +201,33 @@ def fp8_gemm_bf16(
129
  return out
130
 
131
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  def fp8_linear_bias_gelu_quant_bf16(
133
  input: torch.Tensor,
134
  weight: torch.Tensor,
@@ -220,8 +319,125 @@ def fp8_gelu_mlp_bf16(
220
  return out
221
 
222
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
223
  __all__ = [
 
 
224
  "fp8_gemm_bf16",
225
  "fp8_gelu_mlp_bf16",
 
226
  "fp8_linear_bias_gelu_quant_bf16",
227
  ]
 
55
  return None
56
 
57
 
58
+ @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_bf16"))
59
+ def _fp8_linear_bias_bf16_fake(
60
+ input: torch.Tensor,
61
+ weight: torch.Tensor,
62
+ bias: torch.Tensor,
63
+ input_scale: torch.Tensor,
64
+ weight_scale: torch.Tensor,
65
+ out: torch.Tensor,
66
+ ) -> None:
67
+ if input.dim() != 2 or weight.dim() != 2:
68
+ raise RuntimeError("input and weight must be rank-2 tensors")
69
+ if bias.shape != (weight.shape[0],):
70
+ raise RuntimeError("bias shape must be (weight.shape[0],)")
71
+ if out.shape != (input.shape[0], weight.shape[0]):
72
+ raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
73
+ return None
74
+
75
+
76
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_linear_bias_bf16"))
77
+ def _bf16_fp8_linear_bias_bf16_fake(
78
+ input: torch.Tensor,
79
+ weight: torch.Tensor,
80
+ bias: torch.Tensor,
81
+ input_scale: torch.Tensor,
82
+ weight_scale: torch.Tensor,
83
+ input_fp8: torch.Tensor,
84
+ out: torch.Tensor,
85
+ ) -> None:
86
+ if input.dim() != 2 or weight.dim() != 2:
87
+ raise RuntimeError("input and weight must be rank-2 tensors")
88
+ if input.shape[1] != weight.shape[1]:
89
+ raise RuntimeError("input.shape[1] must match weight.shape[1]")
90
+ padded_m = input_fp8.shape[0]
91
+ if padded_m < input.shape[0]:
92
+ raise RuntimeError("input_fp8 padded rows must cover input rows")
93
+ if input_fp8.shape != (padded_m, input.shape[1]):
94
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
95
+ if bias.shape != (weight.shape[0],):
96
+ raise RuntimeError("bias shape must be (weight.shape[0],)")
97
+ if out.shape != (padded_m, weight.shape[0]):
98
+ raise RuntimeError("out must have shape (padded_m, weight.shape[0])")
99
+ return None
100
+
101
+
102
  @torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
103
  def _fp8_linear_bias_gelu_quant_bf16_fake(
104
  input: torch.Tensor,
 
145
  return None
146
 
147
 
148
+ @torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
149
+ def _bf16_fp8_gelu_mlp_bf16_fake(
150
+ input: torch.Tensor,
151
+ up_weight: torch.Tensor,
152
+ up_bias: torch.Tensor,
153
+ down_weight: torch.Tensor,
154
+ down_bias: torch.Tensor,
155
+ input_scale: torch.Tensor,
156
+ up_weight_scale: torch.Tensor,
157
+ hidden_scale: torch.Tensor,
158
+ down_weight_scale: torch.Tensor,
159
+ input_fp8: torch.Tensor,
160
+ hidden_bf16: torch.Tensor,
161
+ hidden_fp8: torch.Tensor,
162
+ out: torch.Tensor,
163
+ ) -> None:
164
+ padded_m = input_fp8.shape[0]
165
+ if input_fp8.shape != (padded_m, input.shape[1]):
166
+ raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
167
+ if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
168
+ raise RuntimeError("hidden_bf16 has an invalid padded shape")
169
+ if hidden_fp8.shape != hidden_bf16.shape:
170
+ raise RuntimeError("hidden_fp8 must match hidden_bf16")
171
+ if out.shape != (padded_m, down_weight.shape[0]):
172
+ raise RuntimeError("out has an invalid padded shape")
173
+ return None
174
+
175
+
176
  def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
177
  return torch.tensor([value], device=input.device, dtype=torch.float32)
178
 
 
201
  return out
202
 
203
 
204
+ def fp8_linear_bias_bf16(
205
+ input: torch.Tensor,
206
+ weight: torch.Tensor,
207
+ bias: torch.Tensor,
208
+ input_scale: torch.Tensor,
209
+ weight_scale: torch.Tensor,
210
+ out: torch.Tensor | None = None,
211
+ ) -> torch.Tensor:
212
+ """Run an FP8-weight linear projection with BF16 bias and output.
213
+
214
+ ``input`` and ``weight`` use FP8 E4M3 with shapes ``(M, K)`` and
215
+ ``(N, K)``. Scales are device float32 scalar tensors. Pass ``out`` to keep
216
+ the call allocation-free for CUDA Graph capture.
217
+ """
218
+
219
+ if out is None:
220
+ out = torch.empty(
221
+ (input.shape[0], weight.shape[0]),
222
+ device=input.device,
223
+ dtype=torch.bfloat16,
224
+ )
225
+ ops.fp8_linear_bias_bf16(
226
+ input, weight, bias, input_scale, weight_scale, out
227
+ )
228
+ return out
229
+
230
+
231
  def fp8_linear_bias_gelu_quant_bf16(
232
  input: torch.Tensor,
233
  weight: torch.Tensor,
 
319
  return out
320
 
321
 
322
+ def _midm_padded_rows(input: torch.Tensor) -> int:
323
+ rows = input.shape[0]
324
+ if (
325
+ torch.version.hip is None
326
+ and input.is_cuda
327
+ and torch.cuda.get_device_capability(input.device) == (11, 0)
328
+ and 9 <= rows <= 128
329
+ ):
330
+ return ((rows + 63) // 64) * 64
331
+ return rows
332
+
333
+
334
+ def bf16_fp8_linear_bias_bf16(
335
+ input: torch.Tensor,
336
+ weight: torch.Tensor,
337
+ bias: torch.Tensor,
338
+ input_scale: torch.Tensor,
339
+ weight_scale: torch.Tensor,
340
+ input_fp8: torch.Tensor | None = None,
341
+ out: torch.Tensor | None = None,
342
+ *,
343
+ pad_to: int | None = None,
344
+ ) -> torch.Tensor:
345
+ """Run a static-scale FP8 linear projection from a BF16 boundary.
346
+
347
+ This traceable region entry quantizes ``input`` and computes the FP8
348
+ projection with BF16 bias/output. Thor SM110 pads M=9..128 to a multiple of
349
+ 64 by default. Preallocate ``input_fp8`` and ``out`` for allocation-free
350
+ CUDA Graph replay.
351
+ """
352
+
353
+ logical_m = input.shape[0]
354
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
355
+ if padded_m < logical_m:
356
+ raise ValueError("pad_to must be >= input.shape[0]")
357
+ if input_fp8 is None:
358
+ input_fp8 = torch.empty(
359
+ (padded_m, input.shape[1]), device=input.device, dtype=_fp8_dtype()
360
+ )
361
+ if out is None:
362
+ out = torch.empty(
363
+ (padded_m, weight.shape[0]),
364
+ device=input.device,
365
+ dtype=torch.bfloat16,
366
+ )
367
+ ops.bf16_fp8_linear_bias_bf16(
368
+ input, weight, bias, input_scale, weight_scale, input_fp8, out
369
+ )
370
+ return out[:logical_m]
371
+
372
+
373
+ def bf16_fp8_gelu_mlp_bf16(
374
+ input: torch.Tensor,
375
+ up_weight: torch.Tensor,
376
+ up_bias: torch.Tensor,
377
+ down_weight: torch.Tensor,
378
+ down_bias: torch.Tensor,
379
+ input_scale: torch.Tensor,
380
+ up_weight_scale: torch.Tensor,
381
+ hidden_scale: torch.Tensor,
382
+ down_weight_scale: torch.Tensor,
383
+ input_fp8: torch.Tensor | None = None,
384
+ hidden_bf16: torch.Tensor | None = None,
385
+ hidden_fp8: torch.Tensor | None = None,
386
+ out: torch.Tensor | None = None,
387
+ *,
388
+ pad_to: int | None = None,
389
+ ) -> torch.Tensor:
390
+ """Run a static-scale FP8 GELU MLP from a BF16 region boundary.
391
+
392
+ The op quantizes ``input`` and executes both FP8-weight GEMMs in one
393
+ traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
394
+ other targets keep the logical M unless ``pad_to`` is supplied. Pass
395
+ preallocated buffers for allocation-free CUDA Graph replay. This is one
396
+ custom-op boundary, not a single CUDA launch.
397
+ """
398
+
399
+ logical_m = input.shape[0]
400
+ padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
401
+ if padded_m < logical_m:
402
+ raise ValueError("pad_to must be >= input.shape[0]")
403
+ device = input.device
404
+ if input_fp8 is None:
405
+ input_fp8 = torch.empty(
406
+ (padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
407
+ )
408
+ if hidden_bf16 is None:
409
+ hidden_bf16 = torch.empty(
410
+ (padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
411
+ )
412
+ if hidden_fp8 is None:
413
+ hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
414
+ if out is None:
415
+ out = torch.empty(
416
+ (padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
417
+ )
418
+ ops.bf16_fp8_gelu_mlp_bf16(
419
+ input,
420
+ up_weight,
421
+ up_bias,
422
+ down_weight,
423
+ down_bias,
424
+ input_scale,
425
+ up_weight_scale,
426
+ hidden_scale,
427
+ down_weight_scale,
428
+ input_fp8,
429
+ hidden_bf16,
430
+ hidden_fp8,
431
+ out,
432
+ )
433
+ return out[:logical_m]
434
+
435
+
436
  __all__ = [
437
+ "bf16_fp8_linear_bias_bf16",
438
+ "bf16_fp8_gelu_mlp_bf16",
439
  "fp8_gemm_bf16",
440
  "fp8_gelu_mlp_bf16",
441
+ "fp8_linear_bias_bf16",
442
  "fp8_linear_bias_gelu_quant_bf16",
443
  ]
build/torch211-cxx11-cu130-aarch64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec0844b1d17575d1e324ef9aac70c4ba78d464615bee9a1a9f3ae4bca51e8a41
3
+ size 256720
build/torch211-cxx11-cu130-aarch64-linux/_ops.py CHANGED
@@ -1,6 +1,6 @@
1
  import torch
2
- from . import _flashrt_fp8_ffn_cuda_4a17df5
3
- ops = torch.ops._flashrt_fp8_ffn_cuda_4a17df5
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
- return f"_flashrt_fp8_ffn_cuda_4a17df5::{op_name}"
 
1
  import torch
2
+ from . import _flashrt_fp8_ffn_cuda_255584f
3
+ ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
4
 
5
  def add_op_namespace_prefix(op_name: str):
6
+ return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
build/torch211-cxx11-cu130-aarch64-linux/metadata.json CHANGED
@@ -1,14 +1,22 @@
1
  {
2
  "name": "flashrt-fp8-ffn",
3
- "id": "_flashrt_fp8_ffn_cuda_4a17df5",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
- "11.0",
11
- "12.1+PTX"
12
  ]
 
 
 
 
 
 
 
 
 
13
  }
14
  }
 
1
  {
2
  "name": "flashrt-fp8-ffn",
3
+ "id": "_flashrt_fp8_ffn_cuda_255584f",
4
  "version": 1,
5
  "license": "Apache-2.0",
6
  "python-depends": [],
7
  "backend": {
8
  "type": "cuda",
9
  "archs": [
10
+ "11.0"
 
11
  ]
12
+ },
13
+ "digest": {
14
+ "algorithm": "sha256",
15
+ "files": {
16
+ "__init__.py": "Lfpi4OMG3jS0AMFqtyCpL5H/nNuAzqtpwFaUnv2wgqo=",
17
+ "_flashrt_fp8_ffn_cuda_255584f.abi3.so": "7AhEsdF1ddHjJO+arHDEunjUZGFb7poanzrkvKUeikE=",
18
+ "_ops.py": "DjRVHDqNFmTPfak1/oS9Kq0Gncu1ODIrvbLkhemqRzI=",
19
+ "flashrt_fp8_ffn/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
20
+ }
21
  }
22
  }