Add torch211-cxx11-cu130-aarch64-linux SM110 artifact
Browse files
build/torch211-cxx11-cu130-aarch64-linux/__init__.py
CHANGED
|
@@ -55,6 +55,50 @@ def _fp8_gemm_bf16_fake(
|
|
| 55 |
return None
|
| 56 |
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
@torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
|
| 59 |
def _fp8_linear_bias_gelu_quant_bf16_fake(
|
| 60 |
input: torch.Tensor,
|
|
@@ -101,6 +145,34 @@ def _fp8_gelu_mlp_bf16_fake(
|
|
| 101 |
return None
|
| 102 |
|
| 103 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 105 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 106 |
|
|
@@ -129,6 +201,33 @@ def fp8_gemm_bf16(
|
|
| 129 |
return out
|
| 130 |
|
| 131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
def fp8_linear_bias_gelu_quant_bf16(
|
| 133 |
input: torch.Tensor,
|
| 134 |
weight: torch.Tensor,
|
|
@@ -220,8 +319,125 @@ def fp8_gelu_mlp_bf16(
|
|
| 220 |
return out
|
| 221 |
|
| 222 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
__all__ = [
|
|
|
|
|
|
|
| 224 |
"fp8_gemm_bf16",
|
| 225 |
"fp8_gelu_mlp_bf16",
|
|
|
|
| 226 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 227 |
]
|
|
|
|
| 55 |
return None
|
| 56 |
|
| 57 |
|
| 58 |
+
@torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_bf16"))
|
| 59 |
+
def _fp8_linear_bias_bf16_fake(
|
| 60 |
+
input: torch.Tensor,
|
| 61 |
+
weight: torch.Tensor,
|
| 62 |
+
bias: torch.Tensor,
|
| 63 |
+
input_scale: torch.Tensor,
|
| 64 |
+
weight_scale: torch.Tensor,
|
| 65 |
+
out: torch.Tensor,
|
| 66 |
+
) -> None:
|
| 67 |
+
if input.dim() != 2 or weight.dim() != 2:
|
| 68 |
+
raise RuntimeError("input and weight must be rank-2 tensors")
|
| 69 |
+
if bias.shape != (weight.shape[0],):
|
| 70 |
+
raise RuntimeError("bias shape must be (weight.shape[0],)")
|
| 71 |
+
if out.shape != (input.shape[0], weight.shape[0]):
|
| 72 |
+
raise RuntimeError("out shape must be (input.shape[0], weight.shape[0])")
|
| 73 |
+
return None
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_linear_bias_bf16"))
|
| 77 |
+
def _bf16_fp8_linear_bias_bf16_fake(
|
| 78 |
+
input: torch.Tensor,
|
| 79 |
+
weight: torch.Tensor,
|
| 80 |
+
bias: torch.Tensor,
|
| 81 |
+
input_scale: torch.Tensor,
|
| 82 |
+
weight_scale: torch.Tensor,
|
| 83 |
+
input_fp8: torch.Tensor,
|
| 84 |
+
out: torch.Tensor,
|
| 85 |
+
) -> None:
|
| 86 |
+
if input.dim() != 2 or weight.dim() != 2:
|
| 87 |
+
raise RuntimeError("input and weight must be rank-2 tensors")
|
| 88 |
+
if input.shape[1] != weight.shape[1]:
|
| 89 |
+
raise RuntimeError("input.shape[1] must match weight.shape[1]")
|
| 90 |
+
padded_m = input_fp8.shape[0]
|
| 91 |
+
if padded_m < input.shape[0]:
|
| 92 |
+
raise RuntimeError("input_fp8 padded rows must cover input rows")
|
| 93 |
+
if input_fp8.shape != (padded_m, input.shape[1]):
|
| 94 |
+
raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
|
| 95 |
+
if bias.shape != (weight.shape[0],):
|
| 96 |
+
raise RuntimeError("bias shape must be (weight.shape[0],)")
|
| 97 |
+
if out.shape != (padded_m, weight.shape[0]):
|
| 98 |
+
raise RuntimeError("out must have shape (padded_m, weight.shape[0])")
|
| 99 |
+
return None
|
| 100 |
+
|
| 101 |
+
|
| 102 |
@torch.library.register_fake(add_op_namespace_prefix("fp8_linear_bias_gelu_quant_bf16"))
|
| 103 |
def _fp8_linear_bias_gelu_quant_bf16_fake(
|
| 104 |
input: torch.Tensor,
|
|
|
|
| 145 |
return None
|
| 146 |
|
| 147 |
|
| 148 |
+
@torch.library.register_fake(add_op_namespace_prefix("bf16_fp8_gelu_mlp_bf16"))
|
| 149 |
+
def _bf16_fp8_gelu_mlp_bf16_fake(
|
| 150 |
+
input: torch.Tensor,
|
| 151 |
+
up_weight: torch.Tensor,
|
| 152 |
+
up_bias: torch.Tensor,
|
| 153 |
+
down_weight: torch.Tensor,
|
| 154 |
+
down_bias: torch.Tensor,
|
| 155 |
+
input_scale: torch.Tensor,
|
| 156 |
+
up_weight_scale: torch.Tensor,
|
| 157 |
+
hidden_scale: torch.Tensor,
|
| 158 |
+
down_weight_scale: torch.Tensor,
|
| 159 |
+
input_fp8: torch.Tensor,
|
| 160 |
+
hidden_bf16: torch.Tensor,
|
| 161 |
+
hidden_fp8: torch.Tensor,
|
| 162 |
+
out: torch.Tensor,
|
| 163 |
+
) -> None:
|
| 164 |
+
padded_m = input_fp8.shape[0]
|
| 165 |
+
if input_fp8.shape != (padded_m, input.shape[1]):
|
| 166 |
+
raise RuntimeError("input_fp8 must have shape (padded_m, input.shape[1])")
|
| 167 |
+
if hidden_bf16.shape != (padded_m, up_weight.shape[0]):
|
| 168 |
+
raise RuntimeError("hidden_bf16 has an invalid padded shape")
|
| 169 |
+
if hidden_fp8.shape != hidden_bf16.shape:
|
| 170 |
+
raise RuntimeError("hidden_fp8 must match hidden_bf16")
|
| 171 |
+
if out.shape != (padded_m, down_weight.shape[0]):
|
| 172 |
+
raise RuntimeError("out has an invalid padded shape")
|
| 173 |
+
return None
|
| 174 |
+
|
| 175 |
+
|
| 176 |
def _scalar_scale_like(input: torch.Tensor, value: float = 1.0) -> torch.Tensor:
|
| 177 |
return torch.tensor([value], device=input.device, dtype=torch.float32)
|
| 178 |
|
|
|
|
| 201 |
return out
|
| 202 |
|
| 203 |
|
| 204 |
+
def fp8_linear_bias_bf16(
|
| 205 |
+
input: torch.Tensor,
|
| 206 |
+
weight: torch.Tensor,
|
| 207 |
+
bias: torch.Tensor,
|
| 208 |
+
input_scale: torch.Tensor,
|
| 209 |
+
weight_scale: torch.Tensor,
|
| 210 |
+
out: torch.Tensor | None = None,
|
| 211 |
+
) -> torch.Tensor:
|
| 212 |
+
"""Run an FP8-weight linear projection with BF16 bias and output.
|
| 213 |
+
|
| 214 |
+
``input`` and ``weight`` use FP8 E4M3 with shapes ``(M, K)`` and
|
| 215 |
+
``(N, K)``. Scales are device float32 scalar tensors. Pass ``out`` to keep
|
| 216 |
+
the call allocation-free for CUDA Graph capture.
|
| 217 |
+
"""
|
| 218 |
+
|
| 219 |
+
if out is None:
|
| 220 |
+
out = torch.empty(
|
| 221 |
+
(input.shape[0], weight.shape[0]),
|
| 222 |
+
device=input.device,
|
| 223 |
+
dtype=torch.bfloat16,
|
| 224 |
+
)
|
| 225 |
+
ops.fp8_linear_bias_bf16(
|
| 226 |
+
input, weight, bias, input_scale, weight_scale, out
|
| 227 |
+
)
|
| 228 |
+
return out
|
| 229 |
+
|
| 230 |
+
|
| 231 |
def fp8_linear_bias_gelu_quant_bf16(
|
| 232 |
input: torch.Tensor,
|
| 233 |
weight: torch.Tensor,
|
|
|
|
| 319 |
return out
|
| 320 |
|
| 321 |
|
| 322 |
+
def _midm_padded_rows(input: torch.Tensor) -> int:
|
| 323 |
+
rows = input.shape[0]
|
| 324 |
+
if (
|
| 325 |
+
torch.version.hip is None
|
| 326 |
+
and input.is_cuda
|
| 327 |
+
and torch.cuda.get_device_capability(input.device) == (11, 0)
|
| 328 |
+
and 9 <= rows <= 128
|
| 329 |
+
):
|
| 330 |
+
return ((rows + 63) // 64) * 64
|
| 331 |
+
return rows
|
| 332 |
+
|
| 333 |
+
|
| 334 |
+
def bf16_fp8_linear_bias_bf16(
|
| 335 |
+
input: torch.Tensor,
|
| 336 |
+
weight: torch.Tensor,
|
| 337 |
+
bias: torch.Tensor,
|
| 338 |
+
input_scale: torch.Tensor,
|
| 339 |
+
weight_scale: torch.Tensor,
|
| 340 |
+
input_fp8: torch.Tensor | None = None,
|
| 341 |
+
out: torch.Tensor | None = None,
|
| 342 |
+
*,
|
| 343 |
+
pad_to: int | None = None,
|
| 344 |
+
) -> torch.Tensor:
|
| 345 |
+
"""Run a static-scale FP8 linear projection from a BF16 boundary.
|
| 346 |
+
|
| 347 |
+
This traceable region entry quantizes ``input`` and computes the FP8
|
| 348 |
+
projection with BF16 bias/output. Thor SM110 pads M=9..128 to a multiple of
|
| 349 |
+
64 by default. Preallocate ``input_fp8`` and ``out`` for allocation-free
|
| 350 |
+
CUDA Graph replay.
|
| 351 |
+
"""
|
| 352 |
+
|
| 353 |
+
logical_m = input.shape[0]
|
| 354 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 355 |
+
if padded_m < logical_m:
|
| 356 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 357 |
+
if input_fp8 is None:
|
| 358 |
+
input_fp8 = torch.empty(
|
| 359 |
+
(padded_m, input.shape[1]), device=input.device, dtype=_fp8_dtype()
|
| 360 |
+
)
|
| 361 |
+
if out is None:
|
| 362 |
+
out = torch.empty(
|
| 363 |
+
(padded_m, weight.shape[0]),
|
| 364 |
+
device=input.device,
|
| 365 |
+
dtype=torch.bfloat16,
|
| 366 |
+
)
|
| 367 |
+
ops.bf16_fp8_linear_bias_bf16(
|
| 368 |
+
input, weight, bias, input_scale, weight_scale, input_fp8, out
|
| 369 |
+
)
|
| 370 |
+
return out[:logical_m]
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
def bf16_fp8_gelu_mlp_bf16(
|
| 374 |
+
input: torch.Tensor,
|
| 375 |
+
up_weight: torch.Tensor,
|
| 376 |
+
up_bias: torch.Tensor,
|
| 377 |
+
down_weight: torch.Tensor,
|
| 378 |
+
down_bias: torch.Tensor,
|
| 379 |
+
input_scale: torch.Tensor,
|
| 380 |
+
up_weight_scale: torch.Tensor,
|
| 381 |
+
hidden_scale: torch.Tensor,
|
| 382 |
+
down_weight_scale: torch.Tensor,
|
| 383 |
+
input_fp8: torch.Tensor | None = None,
|
| 384 |
+
hidden_bf16: torch.Tensor | None = None,
|
| 385 |
+
hidden_fp8: torch.Tensor | None = None,
|
| 386 |
+
out: torch.Tensor | None = None,
|
| 387 |
+
*,
|
| 388 |
+
pad_to: int | None = None,
|
| 389 |
+
) -> torch.Tensor:
|
| 390 |
+
"""Run a static-scale FP8 GELU MLP from a BF16 region boundary.
|
| 391 |
+
|
| 392 |
+
The op quantizes ``input`` and executes both FP8-weight GEMMs in one
|
| 393 |
+
traceable package call. Thor SM110 pads rows 9..128 to a multiple of 64;
|
| 394 |
+
other targets keep the logical M unless ``pad_to`` is supplied. Pass
|
| 395 |
+
preallocated buffers for allocation-free CUDA Graph replay. This is one
|
| 396 |
+
custom-op boundary, not a single CUDA launch.
|
| 397 |
+
"""
|
| 398 |
+
|
| 399 |
+
logical_m = input.shape[0]
|
| 400 |
+
padded_m = _midm_padded_rows(input) if pad_to is None else pad_to
|
| 401 |
+
if padded_m < logical_m:
|
| 402 |
+
raise ValueError("pad_to must be >= input.shape[0]")
|
| 403 |
+
device = input.device
|
| 404 |
+
if input_fp8 is None:
|
| 405 |
+
input_fp8 = torch.empty(
|
| 406 |
+
(padded_m, input.shape[1]), device=device, dtype=_fp8_dtype()
|
| 407 |
+
)
|
| 408 |
+
if hidden_bf16 is None:
|
| 409 |
+
hidden_bf16 = torch.empty(
|
| 410 |
+
(padded_m, up_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 411 |
+
)
|
| 412 |
+
if hidden_fp8 is None:
|
| 413 |
+
hidden_fp8 = torch.empty_like(hidden_bf16, dtype=_fp8_dtype())
|
| 414 |
+
if out is None:
|
| 415 |
+
out = torch.empty(
|
| 416 |
+
(padded_m, down_weight.shape[0]), device=device, dtype=torch.bfloat16
|
| 417 |
+
)
|
| 418 |
+
ops.bf16_fp8_gelu_mlp_bf16(
|
| 419 |
+
input,
|
| 420 |
+
up_weight,
|
| 421 |
+
up_bias,
|
| 422 |
+
down_weight,
|
| 423 |
+
down_bias,
|
| 424 |
+
input_scale,
|
| 425 |
+
up_weight_scale,
|
| 426 |
+
hidden_scale,
|
| 427 |
+
down_weight_scale,
|
| 428 |
+
input_fp8,
|
| 429 |
+
hidden_bf16,
|
| 430 |
+
hidden_fp8,
|
| 431 |
+
out,
|
| 432 |
+
)
|
| 433 |
+
return out[:logical_m]
|
| 434 |
+
|
| 435 |
+
|
| 436 |
__all__ = [
|
| 437 |
+
"bf16_fp8_linear_bias_bf16",
|
| 438 |
+
"bf16_fp8_gelu_mlp_bf16",
|
| 439 |
"fp8_gemm_bf16",
|
| 440 |
"fp8_gelu_mlp_bf16",
|
| 441 |
+
"fp8_linear_bias_bf16",
|
| 442 |
"fp8_linear_bias_gelu_quant_bf16",
|
| 443 |
]
|
build/torch211-cxx11-cu130-aarch64-linux/_flashrt_fp8_ffn_cuda_255584f.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ec0844b1d17575d1e324ef9aac70c4ba78d464615bee9a1a9f3ae4bca51e8a41
|
| 3 |
+
size 256720
|
build/torch211-cxx11-cu130-aarch64-linux/_ops.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_fp8_ffn_cuda_255584f
|
| 3 |
+
ops = torch.ops._flashrt_fp8_ffn_cuda_255584f
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
return f"_flashrt_fp8_ffn_cuda_255584f::{op_name}"
|
build/torch211-cxx11-cu130-aarch64-linux/metadata.json
CHANGED
|
@@ -1,14 +1,22 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
| 7 |
"backend": {
|
| 8 |
"type": "cuda",
|
| 9 |
"archs": [
|
| 10 |
-
"11.0"
|
| 11 |
-
"12.1+PTX"
|
| 12 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
}
|
| 14 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-fp8-ffn",
|
| 3 |
+
"id": "_flashrt_fp8_ffn_cuda_255584f",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
| 7 |
"backend": {
|
| 8 |
"type": "cuda",
|
| 9 |
"archs": [
|
| 10 |
+
"11.0"
|
|
|
|
| 11 |
]
|
| 12 |
+
},
|
| 13 |
+
"digest": {
|
| 14 |
+
"algorithm": "sha256",
|
| 15 |
+
"files": {
|
| 16 |
+
"__init__.py": "Lfpi4OMG3jS0AMFqtyCpL5H/nNuAzqtpwFaUnv2wgqo=",
|
| 17 |
+
"_flashrt_fp8_ffn_cuda_255584f.abi3.so": "7AhEsdF1ddHjJO+arHDEunjUZGFb7poanzrkvKUeikE=",
|
| 18 |
+
"_ops.py": "DjRVHDqNFmTPfak1/oS9Kq0Gncu1ODIrvbLkhemqRzI=",
|
| 19 |
+
"flashrt_fp8_ffn/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
|
| 20 |
+
}
|
| 21 |
}
|
| 22 |
}
|