Promote latest kernel artifacts to main
Browse files- README.md +75 -0
- VALIDATION.md +59 -0
- build/torch211-cxx11-cu130-aarch64-linux/__init__.py +31 -0
- build/torch211-cxx11-cu130-aarch64-linux/{_flashrt_residual_norm_quant_cuda_5de4768.abi3.so → _flashrt_residual_norm_quant_cuda_b2fe8e6.abi3.so} +2 -2
- build/torch211-cxx11-cu130-aarch64-linux/_ops.py +3 -3
- build/torch211-cxx11-cu130-aarch64-linux/metadata.json +14 -4
- tests/test_residual_norm_quant.py +395 -0
README.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# flashrt-residual-norm-quant
|
| 2 |
+
|
| 3 |
+
Tensor-facing FlashRT residual, RMSNorm, and static FP8 quantization kernels for
|
| 4 |
+
Hugging Face `kernels`.
|
| 5 |
+
|
| 6 |
+
This package is runtime glue for model hot paths that need to produce the next
|
| 7 |
+
layer's FP8 activation without falling back to multiple PyTorch operations:
|
| 8 |
+
|
| 9 |
+
```text
|
| 10 |
+
BF16 residual/x -> residual add -> RMSNorm -> static-scale FP8 E4M3 activation
|
| 11 |
+
```
|
| 12 |
+
|
| 13 |
+
## Exported APIs
|
| 14 |
+
|
| 15 |
+
- `rms_norm_bf16(x, weight, eps=1e-6, out=None)`
|
| 16 |
+
- `rms_norm_quant_fp8_static_bf16(x, weight, scale, eps=1e-6, out=None)`
|
| 17 |
+
- `residual_add_rms_norm_quant_fp8_static_bf16(residual, x, weight, scale, eps=1e-6, out=None)`
|
| 18 |
+
- `residual_add_rms_norm_bf16(residual, x, weight, eps=1e-6, out=None)`
|
| 19 |
+
|
| 20 |
+
The residual API updates `residual` in place with `residual += x`, rounded to
|
| 21 |
+
BF16, then emits the normalized FP8 activation.
|
| 22 |
+
The no-quant twin preserves the same in-place BF16 residual contract and emits
|
| 23 |
+
BF16, for runtimes whose next consumer is not low precision.
|
| 24 |
+
|
| 25 |
+
## Tensor Conventions
|
| 26 |
+
|
| 27 |
+
- `x`: BF16 tensor, shape `(rows, dim)`
|
| 28 |
+
- `residual`: BF16 tensor, shape `(rows, dim)`, in-place updated
|
| 29 |
+
- `weight`: BF16 tensor, shape `(dim,)`
|
| 30 |
+
- `scale`: CUDA `float32` scalar tensor used for static FP8 quantization
|
| 31 |
+
- `out`: FP8 E4M3 tensor, shape `(rows, dim)`
|
| 32 |
+
|
| 33 |
+
The hidden dimension must be even because this version uses FlashRT's packed
|
| 34 |
+
BF16 pair path.
|
| 35 |
+
|
| 36 |
+
## Minimal Usage
|
| 37 |
+
|
| 38 |
+
```python
|
| 39 |
+
from kernels import get_kernel
|
| 40 |
+
import torch
|
| 41 |
+
|
| 42 |
+
ops = get_kernel(
|
| 43 |
+
"flashrt/flashrt-residual-norm-quant",
|
| 44 |
+
version=1,
|
| 45 |
+
trust_remote_code=True,
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
x = torch.randn((10, 1024), device="cuda", dtype=torch.bfloat16)
|
| 49 |
+
residual = torch.randn_like(x)
|
| 50 |
+
weight = torch.ones((1024,), device="cuda", dtype=torch.bfloat16)
|
| 51 |
+
scale = torch.tensor([0.04], device="cuda", dtype=torch.float32)
|
| 52 |
+
|
| 53 |
+
x_fp8 = ops.residual_add_rms_norm_quant_fp8_static_bf16(
|
| 54 |
+
residual,
|
| 55 |
+
x,
|
| 56 |
+
weight,
|
| 57 |
+
scale,
|
| 58 |
+
eps=1e-6,
|
| 59 |
+
)
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
## Validation
|
| 63 |
+
|
| 64 |
+
```bash
|
| 65 |
+
python flashrt-residual-norm-quant/tests/test_residual_norm_quant.py --backend source --mode full
|
| 66 |
+
python flashrt-residual-norm-quant/benchmarks/benchmark.py --backend source --shapes all
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
Current RTX 5090 source-extension rows pass the package's strict BF16/FP8
|
| 70 |
+
distribution gates across the PI0.5, VLA, video, and Cosmos3-Edge shape grid.
|
| 71 |
+
Built artifacts must pass the same gate before publication.
|
| 72 |
+
|
| 73 |
+
The no-quant BF16 path is also gated at the Cosmos3-Edge production hidden
|
| 74 |
+
size (`rows=128`, `dim=2048`) and preserves the in-place residual contract.
|
| 75 |
+
Its static-output CUDA Graph replay is checked bitwise.
|
VALIDATION.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Validation: flashrt-residual-norm-quant
|
| 2 |
+
|
| 3 |
+
Required before publishing this package:
|
| 4 |
+
|
| 5 |
+
1. Source-extension correctness:
|
| 6 |
+
|
| 7 |
+
```bash
|
| 8 |
+
python flashrt-residual-norm-quant/tests/test_residual_norm_quant.py --backend source --mode full
|
| 9 |
+
```
|
| 10 |
+
|
| 11 |
+
2. Source-extension benchmark:
|
| 12 |
+
|
| 13 |
+
```bash
|
| 14 |
+
python flashrt-residual-norm-quant/benchmarks/benchmark.py \
|
| 15 |
+
--backend source \
|
| 16 |
+
--shapes all \
|
| 17 |
+
--warmup 3 \
|
| 18 |
+
--iters 10
|
| 19 |
+
```
|
| 20 |
+
|
| 21 |
+
3. Kernel-builder artifact build:
|
| 22 |
+
|
| 23 |
+
```bash
|
| 24 |
+
kernel-builder build-and-copy flashrt-residual-norm-quant
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
4. Built-artifact correctness:
|
| 28 |
+
|
| 29 |
+
```bash
|
| 30 |
+
PYTHONPATH=<artifact-path> \
|
| 31 |
+
python flashrt-residual-norm-quant/tests/test_residual_norm_quant.py \
|
| 32 |
+
--backend installed \
|
| 33 |
+
--mode full
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
5. Built-artifact benchmark:
|
| 37 |
+
|
| 38 |
+
```bash
|
| 39 |
+
python flashrt-residual-norm-quant/benchmarks/benchmark.py \
|
| 40 |
+
--backend installed \
|
| 41 |
+
--artifact <artifact-path> \
|
| 42 |
+
--shapes all
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
6. Multi-hardware matrix:
|
| 46 |
+
|
| 47 |
+
Add hardware claims only after the same correctness and benchmark commands
|
| 48 |
+
pass on that machine.
|
| 49 |
+
|
| 50 |
+
## Thor validation
|
| 51 |
+
|
| 52 |
+
On 2026-08-06, source full passed on NVIDIA Thor (SM110), PyTorch
|
| 53 |
+
`2.13.0+cu130`, and CUDA 13.0. The new `residual_add_rms_norm_bf16` entry was
|
| 54 |
+
checked both out-of-place and with the production in-place residual contract
|
| 55 |
+
across PI0.5, GROOT, and video-prefill shapes. All rows passed the package's
|
| 56 |
+
strict BF16 operation-order reference.
|
| 57 |
+
|
| 58 |
+
Cosmos3-Edge coverage uses `(rows=128, dim=2048)` and additionally requires
|
| 59 |
+
bitwise-identical static-output CUDA Graph replay.
|
build/torch211-cxx11-cu130-aarch64-linux/__init__.py
CHANGED
|
@@ -78,6 +78,22 @@ def _residual_add_rms_norm_quant_fp8_static_bf16_fake(
|
|
| 78 |
return None
|
| 79 |
|
| 80 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
def rms_norm_bf16(
|
| 82 |
x: torch.Tensor,
|
| 83 |
weight: torch.Tensor,
|
|
@@ -145,9 +161,24 @@ def residual_add_rms_norm_quant_fp8_static_bf16(
|
|
| 145 |
return out
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
__all__ = [
|
| 149 |
"layer_norm_bf16",
|
| 150 |
"residual_add_rms_norm_quant_fp8_static_bf16",
|
|
|
|
| 151 |
"rms_norm_bf16",
|
| 152 |
"rms_norm_quant_fp8_static_bf16",
|
| 153 |
]
|
|
|
|
| 78 |
return None
|
| 79 |
|
| 80 |
|
| 81 |
+
@torch.library.register_fake(add_op_namespace_prefix("residual_add_rms_norm_bf16"))
|
| 82 |
+
def _residual_add_rms_norm_bf16_fake(
|
| 83 |
+
residual: torch.Tensor,
|
| 84 |
+
x: torch.Tensor,
|
| 85 |
+
weight: torch.Tensor,
|
| 86 |
+
eps: float,
|
| 87 |
+
out: torch.Tensor,
|
| 88 |
+
) -> None:
|
| 89 |
+
if residual.shape != x.shape:
|
| 90 |
+
raise RuntimeError("residual and x must have the same shape")
|
| 91 |
+
_check_rank2_same_shape(x, out, "out")
|
| 92 |
+
if weight.shape != (x.shape[1],):
|
| 93 |
+
raise RuntimeError("weight must have shape (x.shape[1],)")
|
| 94 |
+
return None
|
| 95 |
+
|
| 96 |
+
|
| 97 |
def rms_norm_bf16(
|
| 98 |
x: torch.Tensor,
|
| 99 |
weight: torch.Tensor,
|
|
|
|
| 161 |
return out
|
| 162 |
|
| 163 |
|
| 164 |
+
def residual_add_rms_norm_bf16(
|
| 165 |
+
residual: torch.Tensor,
|
| 166 |
+
x: torch.Tensor,
|
| 167 |
+
weight: torch.Tensor,
|
| 168 |
+
eps: float = 1e-6,
|
| 169 |
+
out: torch.Tensor | None = None,
|
| 170 |
+
) -> torch.Tensor:
|
| 171 |
+
"""In-place ``residual += x`` followed by weighted BF16 RMSNorm."""
|
| 172 |
+
if out is None:
|
| 173 |
+
out = torch.empty_like(x, dtype=torch.bfloat16)
|
| 174 |
+
ops.residual_add_rms_norm_bf16(residual, x, weight, float(eps), out)
|
| 175 |
+
return out
|
| 176 |
+
|
| 177 |
+
|
| 178 |
__all__ = [
|
| 179 |
"layer_norm_bf16",
|
| 180 |
"residual_add_rms_norm_quant_fp8_static_bf16",
|
| 181 |
+
"residual_add_rms_norm_bf16",
|
| 182 |
"rms_norm_bf16",
|
| 183 |
"rms_norm_quant_fp8_static_bf16",
|
| 184 |
]
|
build/torch211-cxx11-cu130-aarch64-linux/{_flashrt_residual_norm_quant_cuda_5de4768.abi3.so → _flashrt_residual_norm_quant_cuda_b2fe8e6.abi3.so}
RENAMED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:eb9795efc35b9829766fee9b6bd7a86e8da262600b2ca4ec6221824e908d4386
|
| 3 |
+
size 311320
|
build/torch211-cxx11-cu130-aarch64-linux/_ops.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import torch
|
| 2 |
-
from . import
|
| 3 |
-
ops = torch.ops.
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
-
return f"
|
|
|
|
| 1 |
import torch
|
| 2 |
+
from . import _flashrt_residual_norm_quant_cuda_b2fe8e6
|
| 3 |
+
ops = torch.ops._flashrt_residual_norm_quant_cuda_b2fe8e6
|
| 4 |
|
| 5 |
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
return f"_flashrt_residual_norm_quant_cuda_b2fe8e6::{op_name}"
|
build/torch211-cxx11-cu130-aarch64-linux/metadata.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-residual-norm-quant",
|
| 3 |
-
"id": "
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
@@ -13,10 +13,20 @@
|
|
| 13 |
"digest": {
|
| 14 |
"algorithm": "sha256",
|
| 15 |
"files": {
|
| 16 |
-
"__init__.py": "
|
| 17 |
-
"
|
| 18 |
-
"_ops.py": "
|
| 19 |
"flashrt_residual_norm_quant/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
|
| 20 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
}
|
| 22 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"name": "flashrt-residual-norm-quant",
|
| 3 |
+
"id": "_flashrt_residual_norm_quant_cuda_b2fe8e6",
|
| 4 |
"version": 1,
|
| 5 |
"license": "Apache-2.0",
|
| 6 |
"python-depends": [],
|
|
|
|
| 13 |
"digest": {
|
| 14 |
"algorithm": "sha256",
|
| 15 |
"files": {
|
| 16 |
+
"__init__.py": "snK1D78CVgyD7zmjN/6PuR+kxnjatRCR+69xTPG0E2k=",
|
| 17 |
+
"_flashrt_residual_norm_quant_cuda_b2fe8e6.abi3.so": "65eV78NbmCl2b+6ba9eobo2iYmALLKTsYiGCTpCNQ4Y=",
|
| 18 |
+
"_ops.py": "gutJ8KpG6JAkSVZA2ljBpKEV59nOBLNVYGqH8Y3PUPI=",
|
| 19 |
"flashrt_residual_norm_quant/__init__.py": "v6p5XMfQzddhi1fLSAw4HX9CyS0rQsidvu9VsT01xi4="
|
| 20 |
}
|
| 21 |
+
},
|
| 22 |
+
"provenance": {
|
| 23 |
+
"kernel": {
|
| 24 |
+
"sha": "b2fe8e62837df976eac61590192e6f5b451db840",
|
| 25 |
+
"dirty": false
|
| 26 |
+
},
|
| 27 |
+
"validation": {
|
| 28 |
+
"torch": "2.11.0+cu130",
|
| 29 |
+
"cuda": "13.0"
|
| 30 |
+
}
|
| 31 |
}
|
| 32 |
}
|
tests/test_residual_norm_quant.py
ADDED
|
@@ -0,0 +1,395 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Correctness tests for flashrt-residual-norm-quant."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import ctypes
|
| 8 |
+
import ctypes.util
|
| 9 |
+
import importlib
|
| 10 |
+
import math
|
| 11 |
+
import os
|
| 12 |
+
import sys
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
ROOT = Path(__file__).resolve().parents[2]
|
| 19 |
+
PACKAGE = ROOT / "flashrt-residual-norm-quant"
|
| 20 |
+
REGISTRATION_INCLUDE = (
|
| 21 |
+
ROOT.parent
|
| 22 |
+
/ "kernels"
|
| 23 |
+
/ "kernel-builder"
|
| 24 |
+
/ "src"
|
| 25 |
+
/ "pyproject"
|
| 26 |
+
/ "templates"
|
| 27 |
+
/ "torch"
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class SourceOps:
|
| 32 |
+
def __init__(self, namespace: str) -> None:
|
| 33 |
+
self._ops = getattr(torch.ops, namespace)
|
| 34 |
+
|
| 35 |
+
def rms_norm_bf16(self, x, weight, eps=1e-6, out=None):
|
| 36 |
+
if out is None:
|
| 37 |
+
out = torch.empty_like(x, dtype=torch.bfloat16)
|
| 38 |
+
self._ops.rms_norm_bf16(x, weight, float(eps), out)
|
| 39 |
+
return out
|
| 40 |
+
|
| 41 |
+
def rms_norm_quant_fp8_static_bf16(self, x, weight, scale, eps=1e-6, out=None):
|
| 42 |
+
if out is None:
|
| 43 |
+
out = torch.empty_like(x, dtype=torch.float8_e4m3fn)
|
| 44 |
+
self._ops.rms_norm_quant_fp8_static_bf16(x, weight, scale, float(eps), out)
|
| 45 |
+
return out
|
| 46 |
+
|
| 47 |
+
def layer_norm_bf16(self, x, weight, bias, eps=1e-5, out=None):
|
| 48 |
+
if out is None:
|
| 49 |
+
out = torch.empty_like(x, dtype=torch.bfloat16)
|
| 50 |
+
self._ops.layer_norm_bf16(x, weight, bias, float(eps), out)
|
| 51 |
+
return out
|
| 52 |
+
|
| 53 |
+
def residual_add_rms_norm_quant_fp8_static_bf16(
|
| 54 |
+
self, residual, x, weight, scale, eps=1e-6, out=None
|
| 55 |
+
):
|
| 56 |
+
if out is None:
|
| 57 |
+
out = torch.empty_like(x, dtype=torch.float8_e4m3fn)
|
| 58 |
+
self._ops.residual_add_rms_norm_quant_fp8_static_bf16(
|
| 59 |
+
residual, x, weight, scale, float(eps), out
|
| 60 |
+
)
|
| 61 |
+
return out
|
| 62 |
+
|
| 63 |
+
def residual_add_rms_norm_bf16(self, residual, x, weight, eps=1e-6, out=None):
|
| 64 |
+
if out is None:
|
| 65 |
+
out = torch.empty_like(x, dtype=torch.bfloat16)
|
| 66 |
+
self._ops.residual_add_rms_norm_bf16(residual, x, weight, float(eps), out)
|
| 67 |
+
return out
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def _preload_cublaslt() -> None:
|
| 71 |
+
for parent in Path(torch.__file__).resolve().parents:
|
| 72 |
+
candidate = parent / "nvidia" / "cublas" / "lib" / "libcublasLt.so.12"
|
| 73 |
+
if candidate.exists():
|
| 74 |
+
ctypes.CDLL(str(candidate), mode=ctypes.RTLD_GLOBAL)
|
| 75 |
+
return
|
| 76 |
+
library = ctypes.util.find_library("cublasLt")
|
| 77 |
+
if library:
|
| 78 |
+
ctypes.CDLL(library, mode=ctypes.RTLD_GLOBAL)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _current_arch_list() -> str:
|
| 82 |
+
major, minor = torch.cuda.get_device_capability(0)
|
| 83 |
+
return f"{major}.{minor}"
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def load_source_ops() -> SourceOps:
|
| 87 |
+
from torch.utils.cpp_extension import load
|
| 88 |
+
|
| 89 |
+
if not REGISTRATION_INCLUDE.is_dir():
|
| 90 |
+
raise RuntimeError(f"missing kernel-builder registration include: {REGISTRATION_INCLUDE}")
|
| 91 |
+
_preload_cublaslt()
|
| 92 |
+
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", _current_arch_list())
|
| 93 |
+
namespace = "flashrt_residual_norm_quant_test"
|
| 94 |
+
load(
|
| 95 |
+
name=namespace,
|
| 96 |
+
sources=[
|
| 97 |
+
str(PACKAGE / "torch-ext" / "torch_binding.cpp"),
|
| 98 |
+
str(PACKAGE / "csrc" / "residual_norm_quant.cu"),
|
| 99 |
+
],
|
| 100 |
+
extra_include_paths=[str(PACKAGE / "csrc"), str(REGISTRATION_INCLUDE)],
|
| 101 |
+
extra_cflags=["-O3", "-DCUDA_KERNEL"],
|
| 102 |
+
extra_cuda_cflags=["-O3", "--expt-relaxed-constexpr", "-DCUDA_KERNEL"],
|
| 103 |
+
verbose=False,
|
| 104 |
+
)
|
| 105 |
+
return SourceOps(namespace)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def load_installed_ops(artifact: str | None):
|
| 109 |
+
if artifact:
|
| 110 |
+
sys.path.insert(0, artifact)
|
| 111 |
+
try:
|
| 112 |
+
return importlib.import_module("flashrt_residual_norm_quant")
|
| 113 |
+
finally:
|
| 114 |
+
if artifact:
|
| 115 |
+
sys.path.remove(artifact)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def quantize_fp8(x: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
| 119 |
+
return torch.clamp(x.float() / scale.float(), -448.0, 448.0).to(torch.float8_e4m3fn)
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def ref_rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
| 123 |
+
rms = torch.rsqrt(torch.mean(x.float() * x.float(), dim=1, keepdim=True) + eps)
|
| 124 |
+
return x.float() * rms * weight.float()
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def ref_rms_norm_bf16(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
| 128 |
+
return ref_rms_norm(x, weight, eps).to(torch.bfloat16)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def ref_rms_norm_quant(x, weight, scale, eps) -> torch.Tensor:
|
| 132 |
+
return quantize_fp8(ref_rms_norm(x, weight, eps), scale)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def ref_layer_norm_bf16(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float) -> torch.Tensor:
|
| 136 |
+
return torch.nn.functional.layer_norm(
|
| 137 |
+
x.float(),
|
| 138 |
+
(x.shape[1],),
|
| 139 |
+
weight.float(),
|
| 140 |
+
bias.float(),
|
| 141 |
+
eps=eps,
|
| 142 |
+
).to(torch.bfloat16)
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def ref_residual_add_rms_norm_quant(residual, x, weight, scale, eps):
|
| 146 |
+
added = residual.float() + x.float()
|
| 147 |
+
residual_out = added.to(torch.bfloat16)
|
| 148 |
+
rms = torch.rsqrt(torch.mean(added * added, dim=1, keepdim=True) + eps)
|
| 149 |
+
# The production kernel rereads the BF16 residual value for the output pass.
|
| 150 |
+
norm = residual_out.float() * rms * weight.float()
|
| 151 |
+
return residual_out, quantize_fp8(norm, scale)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def make_case(rows: int, dim: int):
|
| 155 |
+
x = torch.randn((rows, dim), device="cuda", dtype=torch.bfloat16)
|
| 156 |
+
residual = torch.randn((rows, dim), device="cuda", dtype=torch.bfloat16)
|
| 157 |
+
weight = (1.0 + 0.1 * torch.randn((dim,), device="cuda", dtype=torch.bfloat16)).contiguous()
|
| 158 |
+
bias = (0.1 * torch.randn((dim,), device="cuda", dtype=torch.bfloat16)).contiguous()
|
| 159 |
+
scale = torch.tensor([0.04], device="cuda", dtype=torch.float32)
|
| 160 |
+
return x, residual, weight, bias, scale
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def percentile(x: torch.Tensor, q: float) -> torch.Tensor:
|
| 164 |
+
flat = x.flatten()
|
| 165 |
+
k = max(1, min(flat.numel(), math.ceil(q * flat.numel())))
|
| 166 |
+
return flat.kthvalue(k).values
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def distribution_metrics(got: torch.Tensor, expected: torch.Tensor):
|
| 170 |
+
diff = (got.float() - expected.float()).abs().flatten()
|
| 171 |
+
rel = diff / expected.float().abs().flatten().clamp_min(1.0)
|
| 172 |
+
return {
|
| 173 |
+
"max_abs": float(diff.max().item()),
|
| 174 |
+
"mean_abs": float(diff.mean().item()),
|
| 175 |
+
"p99_abs": float(percentile(diff, 0.99).item()),
|
| 176 |
+
"p99_rel": float(percentile(rel, 0.99).item()),
|
| 177 |
+
"cosine": float(
|
| 178 |
+
torch.nn.functional.cosine_similarity(
|
| 179 |
+
got.float().flatten(), expected.float().flatten(), dim=0
|
| 180 |
+
).item()
|
| 181 |
+
),
|
| 182 |
+
}
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def assert_close_distribution(
|
| 186 |
+
name: str,
|
| 187 |
+
got: torch.Tensor,
|
| 188 |
+
expected: torch.Tensor,
|
| 189 |
+
*,
|
| 190 |
+
p99_abs_limit: float,
|
| 191 |
+
p99_rel_limit: float,
|
| 192 |
+
) -> None:
|
| 193 |
+
m = distribution_metrics(got, expected)
|
| 194 |
+
if m["p99_abs"] > p99_abs_limit or m["p99_rel"] > p99_rel_limit:
|
| 195 |
+
raise AssertionError(
|
| 196 |
+
f"{name} failed: max_abs={m['max_abs']} mean_abs={m['mean_abs']} "
|
| 197 |
+
f"p99_abs={m['p99_abs']} p99_rel={m['p99_rel']} cosine={m['cosine']}"
|
| 198 |
+
)
|
| 199 |
+
print(
|
| 200 |
+
f"PASS {name}: max_abs={m['max_abs']:.6f} mean_abs={m['mean_abs']:.6f} "
|
| 201 |
+
f"p99_abs={m['p99_abs']:.6f} p99_rel={m['p99_rel']:.6f} "
|
| 202 |
+
f"cosine={m['cosine']:.8f}"
|
| 203 |
+
)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def assert_fp8_close(name: str, got: torch.Tensor, expected: torch.Tensor) -> None:
|
| 207 |
+
diff = (got.float() - expected.float()).abs().flatten()
|
| 208 |
+
mismatches = int((got.detach().cpu() != expected.detach().cpu()).sum().item())
|
| 209 |
+
mismatch_rate = mismatches / got.numel()
|
| 210 |
+
max_abs = float(diff.max().item())
|
| 211 |
+
p99_abs = float(percentile(diff, 0.99).item())
|
| 212 |
+
if p99_abs > 0.5 or mismatch_rate > 0.01:
|
| 213 |
+
raise AssertionError(
|
| 214 |
+
f"{name} failed: max_abs={max_abs} p99_abs={p99_abs} "
|
| 215 |
+
f"mismatch_rate={mismatch_rate}"
|
| 216 |
+
)
|
| 217 |
+
print(
|
| 218 |
+
f"PASS {name}: fp8_max_abs={max_abs:.6f} fp8_p99_abs={p99_abs:.6f} "
|
| 219 |
+
f"mismatches={mismatches} mismatch_rate={mismatch_rate:.8f}"
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def expect_runtime_error(label: str, fn) -> None:
|
| 224 |
+
try:
|
| 225 |
+
fn()
|
| 226 |
+
except RuntimeError as exc:
|
| 227 |
+
print(f"PASS {label}: rejected invalid input ({str(exc).splitlines()[0]})")
|
| 228 |
+
return
|
| 229 |
+
raise AssertionError(f"{label} failed: expected RuntimeError")
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def run_shape(ops, label: str, rows: int, dim: int, eps: float) -> None:
|
| 233 |
+
x, residual, weight, bias, scale = make_case(rows, dim)
|
| 234 |
+
|
| 235 |
+
got_norm = ops.rms_norm_bf16(x, weight, eps)
|
| 236 |
+
exp_norm = ref_rms_norm_bf16(x, weight, eps)
|
| 237 |
+
assert_close_distribution(
|
| 238 |
+
f"{label}/rms_norm_bf16",
|
| 239 |
+
got_norm,
|
| 240 |
+
exp_norm,
|
| 241 |
+
p99_abs_limit=0.015625,
|
| 242 |
+
p99_rel_limit=0.02,
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
got_ln = ops.layer_norm_bf16(x, weight, bias, eps)
|
| 246 |
+
exp_ln = ref_layer_norm_bf16(x, weight, bias, eps)
|
| 247 |
+
assert_close_distribution(
|
| 248 |
+
f"{label}/layer_norm_bf16",
|
| 249 |
+
got_ln,
|
| 250 |
+
exp_ln,
|
| 251 |
+
p99_abs_limit=0.015625,
|
| 252 |
+
p99_rel_limit=0.02,
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
got_fp8 = ops.rms_norm_quant_fp8_static_bf16(x, weight, scale, eps)
|
| 256 |
+
exp_fp8 = ref_rms_norm_quant(x, weight, scale, eps)
|
| 257 |
+
assert_fp8_close(f"{label}/rms_norm_quant_fp8_static_bf16", got_fp8, exp_fp8)
|
| 258 |
+
|
| 259 |
+
residual_in = residual.clone()
|
| 260 |
+
got_res_fp8 = ops.residual_add_rms_norm_quant_fp8_static_bf16(
|
| 261 |
+
residual, x, weight, scale, eps
|
| 262 |
+
)
|
| 263 |
+
exp_residual, exp_res_fp8 = ref_residual_add_rms_norm_quant(
|
| 264 |
+
residual_in, x, weight, scale, eps
|
| 265 |
+
)
|
| 266 |
+
assert_close_distribution(
|
| 267 |
+
f"{label}/residual_inplace",
|
| 268 |
+
residual,
|
| 269 |
+
exp_residual,
|
| 270 |
+
p99_abs_limit=0.0,
|
| 271 |
+
p99_rel_limit=0.0,
|
| 272 |
+
)
|
| 273 |
+
assert_fp8_close(
|
| 274 |
+
f"{label}/residual_add_rms_norm_quant_fp8_static_bf16",
|
| 275 |
+
got_res_fp8,
|
| 276 |
+
exp_res_fp8,
|
| 277 |
+
)
|
| 278 |
+
|
| 279 |
+
residual.copy_(residual_in)
|
| 280 |
+
got_res_bf16 = ops.residual_add_rms_norm_bf16(
|
| 281 |
+
residual, x, weight, eps
|
| 282 |
+
)
|
| 283 |
+
assert_close_distribution(
|
| 284 |
+
f"{label}/residual_add_rms_norm_bf16_inplace",
|
| 285 |
+
residual,
|
| 286 |
+
exp_residual,
|
| 287 |
+
p99_abs_limit=0.0,
|
| 288 |
+
p99_rel_limit=0.0,
|
| 289 |
+
)
|
| 290 |
+
added_fp32 = residual_in.float() + x.float()
|
| 291 |
+
added_rms = torch.rsqrt(added_fp32.square().mean(1, keepdim=True) + eps)
|
| 292 |
+
expected_bf16 = (exp_residual.float() * added_rms * weight.float()).to(torch.bfloat16)
|
| 293 |
+
assert_close_distribution(
|
| 294 |
+
f"{label}/residual_add_rms_norm_bf16",
|
| 295 |
+
got_res_bf16,
|
| 296 |
+
expected_bf16,
|
| 297 |
+
p99_abs_limit=0.015625,
|
| 298 |
+
p99_rel_limit=0.02,
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def run_rejection_tests(ops) -> None:
|
| 303 |
+
x, residual, weight, bias, scale = make_case(4, 128)
|
| 304 |
+
bad_x = torch.randn((4, 127), device="cuda", dtype=torch.bfloat16)
|
| 305 |
+
bad_weight = torch.randn((127,), device="cuda", dtype=torch.bfloat16)
|
| 306 |
+
bad_out = torch.empty((4, 128), device="cuda", dtype=torch.bfloat16).t()
|
| 307 |
+
bad_residual = torch.randn((4, 64), device="cuda", dtype=torch.bfloat16)
|
| 308 |
+
|
| 309 |
+
expect_runtime_error(
|
| 310 |
+
"reject odd hidden dim",
|
| 311 |
+
lambda: ops.rms_norm_bf16(bad_x, bad_weight),
|
| 312 |
+
)
|
| 313 |
+
expect_runtime_error(
|
| 314 |
+
"reject non-contiguous output",
|
| 315 |
+
lambda: ops.rms_norm_bf16(x, weight, out=bad_out),
|
| 316 |
+
)
|
| 317 |
+
expect_runtime_error(
|
| 318 |
+
"reject layer norm bias shape mismatch",
|
| 319 |
+
lambda: ops.layer_norm_bf16(x, weight, bad_weight),
|
| 320 |
+
)
|
| 321 |
+
expect_runtime_error(
|
| 322 |
+
"reject residual shape mismatch",
|
| 323 |
+
lambda: ops.residual_add_rms_norm_quant_fp8_static_bf16(
|
| 324 |
+
bad_residual, x, weight, scale
|
| 325 |
+
),
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def run_cosmos_edge_graph(ops, eps: float) -> None:
|
| 330 |
+
rows, dim = 128, 2048
|
| 331 |
+
x = torch.randn((rows, dim), device="cuda", dtype=torch.bfloat16)
|
| 332 |
+
residual_base = torch.randn_like(x)
|
| 333 |
+
weight = torch.randn((dim,), device="cuda", dtype=torch.bfloat16)
|
| 334 |
+
residual = residual_base.clone()
|
| 335 |
+
out = torch.empty_like(x)
|
| 336 |
+
|
| 337 |
+
graph = torch.cuda.CUDAGraph()
|
| 338 |
+
torch.cuda.synchronize()
|
| 339 |
+
with torch.cuda.graph(graph):
|
| 340 |
+
ops.residual_add_rms_norm_bf16(residual, x, weight, eps, out=out)
|
| 341 |
+
|
| 342 |
+
reference_residual = residual_base.clone()
|
| 343 |
+
reference_out = ops.residual_add_rms_norm_bf16(
|
| 344 |
+
reference_residual, x, weight, eps
|
| 345 |
+
)
|
| 346 |
+
residual.copy_(residual_base)
|
| 347 |
+
graph.replay()
|
| 348 |
+
torch.cuda.synchronize()
|
| 349 |
+
torch.testing.assert_close(residual, reference_residual, rtol=0.0, atol=0.0)
|
| 350 |
+
torch.testing.assert_close(out, reference_out, rtol=0.0, atol=0.0)
|
| 351 |
+
first_residual, first_out = residual.clone(), out.clone()
|
| 352 |
+
residual.copy_(residual_base)
|
| 353 |
+
graph.replay()
|
| 354 |
+
torch.cuda.synchronize()
|
| 355 |
+
torch.testing.assert_close(residual, first_residual, rtol=0.0, atol=0.0)
|
| 356 |
+
torch.testing.assert_close(out, first_out, rtol=0.0, atol=0.0)
|
| 357 |
+
print("PASS cosmos_edge/residual_add_rms_norm_bf16 CUDA Graph replay")
|
| 358 |
+
|
| 359 |
+
|
| 360 |
+
def run(args) -> None:
|
| 361 |
+
if not torch.cuda.is_available():
|
| 362 |
+
raise SystemExit("CUDA is required")
|
| 363 |
+
torch.manual_seed(23)
|
| 364 |
+
ops = load_source_ops() if args.backend == "source" else load_installed_ops(args.artifact)
|
| 365 |
+
|
| 366 |
+
shapes = {
|
| 367 |
+
"small": (16, 128),
|
| 368 |
+
"pi05_decoder": (10, 1024),
|
| 369 |
+
"pi05_vision": (512, 1152),
|
| 370 |
+
"groot_vl": (1024, 2048),
|
| 371 |
+
"cosmos_edge": (128, 2048),
|
| 372 |
+
"video_prefill": (2520, 2048),
|
| 373 |
+
}
|
| 374 |
+
if args.mode == "smoke":
|
| 375 |
+
shapes = {k: shapes[k] for k in ("small", "pi05_decoder")}
|
| 376 |
+
|
| 377 |
+
for label, (rows, dim) in shapes.items():
|
| 378 |
+
run_shape(ops, label, rows, dim, args.eps)
|
| 379 |
+
if args.mode == "full":
|
| 380 |
+
run_cosmos_edge_graph(ops, args.eps)
|
| 381 |
+
run_rejection_tests(ops)
|
| 382 |
+
|
| 383 |
+
|
| 384 |
+
def main() -> None:
|
| 385 |
+
parser = argparse.ArgumentParser()
|
| 386 |
+
parser.add_argument("--backend", choices=["source", "installed"], default="source")
|
| 387 |
+
parser.add_argument("--artifact", default=None)
|
| 388 |
+
parser.add_argument("--mode", choices=["smoke", "full"], default="full")
|
| 389 |
+
parser.add_argument("--eps", type=float, default=1e-6)
|
| 390 |
+
args = parser.parse_args()
|
| 391 |
+
run(args)
|
| 392 |
+
|
| 393 |
+
|
| 394 |
+
if __name__ == "__main__":
|
| 395 |
+
main()
|