Promote latest kernel artifacts to main
Browse files- .gitattributes +3 -35
- README.md +0 -9
- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +326 -0
- build/torch211-cxx11-cu128-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so +3 -0
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu128-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +326 -0
- build/torch211-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so +3 -0
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +326 -0
- build/torch212-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so +3 -0
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +326 -0
- build/torch212-cxx11-cu132-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so +3 -0
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu132-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
.gitattributes
CHANGED
|
@@ -1,35 +1,3 @@
|
|
| 1 |
-
*.
|
| 2 |
-
*.
|
| 3 |
-
*.
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 1 |
+
*.so filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.pyd filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.dylib filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
DELETED
|
@@ -1,9 +0,0 @@
|
|
| 1 |
-
# flashrt/flashrt-adarms-train
|
| 2 |
-
|
| 3 |
-
This repository is a compatibility mirror for older `kernels` clients
|
| 4 |
-
that resolve repositories through the default Hugging Face model repo API.
|
| 5 |
-
|
| 6 |
-
Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-adarms-train
|
| 7 |
-
|
| 8 |
-
Do not edit this mirror by hand. It is generated from the Kernel Hub
|
| 9 |
-
`vN` branches and contains the same `build/**` artifacts.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
|
| 2 |
+
forward and backward.
|
| 3 |
+
|
| 4 |
+
Operator (fp32 internal math):
|
| 5 |
+
rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
|
| 6 |
+
adaptive: y = xhat * (1 + scale) + shift
|
| 7 |
+
non-adaptive: y = xhat * (1 + weight)
|
| 8 |
+
The resgate variant computes r = x + h * gate_in first (r is a real output)
|
| 9 |
+
and normalizes r.
|
| 10 |
+
|
| 11 |
+
The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
|
| 12 |
+
fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
|
| 13 |
+
PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
|
| 14 |
+
the gate chunk stay in PyTorch, so their autograd is native.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _use_ops(namespace_ops) -> None:
|
| 32 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 33 |
+
global ops, _HAS_OPS
|
| 34 |
+
ops = namespace_ops
|
| 35 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 36 |
+
_register_fakes()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ---------------------------------------------------------------------------
|
| 40 |
+
# pure-PyTorch reference (also the CPU / unsupported-shape fallback)
|
| 41 |
+
# ---------------------------------------------------------------------------
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 45 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 46 |
+
return modulation.unsqueeze(1)
|
| 47 |
+
return modulation
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 51 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 55 |
+
compute_dtype = _compute_dtype(x)
|
| 56 |
+
xf = x.to(compute_dtype)
|
| 57 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 58 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def reference_adarms(
|
| 62 |
+
x: torch.Tensor,
|
| 63 |
+
modulation_or_weight: torch.Tensor,
|
| 64 |
+
eps: float = 1e-6,
|
| 65 |
+
adaptive: bool = True,
|
| 66 |
+
):
|
| 67 |
+
normed, rstd = _rms_norm(x, eps)
|
| 68 |
+
if adaptive:
|
| 69 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 70 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 71 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 72 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 73 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 74 |
+
return y.to(x.dtype), None, rstd
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def reference_resgate_adarms(
|
| 78 |
+
x: torch.Tensor,
|
| 79 |
+
h: torch.Tensor,
|
| 80 |
+
gate_in: torch.Tensor | None,
|
| 81 |
+
modulation_or_weight: torch.Tensor,
|
| 82 |
+
eps: float = 1e-6,
|
| 83 |
+
adaptive: bool = True,
|
| 84 |
+
):
|
| 85 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 86 |
+
y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
|
| 87 |
+
return residual, y, gate, rstd
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------------------
|
| 91 |
+
# CUDA autograd path
|
| 92 |
+
# ---------------------------------------------------------------------------
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
|
| 96 |
+
if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
|
| 97 |
+
return False
|
| 98 |
+
if x.dtype not in (torch.bfloat16, torch.float32):
|
| 99 |
+
return False
|
| 100 |
+
# the modulation/weight may be fp32 alongside bf16 activations (pi-style
|
| 101 |
+
# selective-bf16 keeps norm parameters in fp32)
|
| 102 |
+
if mod.dtype not in (x.dtype, torch.float32):
|
| 103 |
+
return False
|
| 104 |
+
lanes = 8 if x.dtype == torch.bfloat16 else 4
|
| 105 |
+
return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
|
| 109 |
+
if like.shape[1] == 1 and grad.shape[1] != 1:
|
| 110 |
+
return grad.sum(dim=1, keepdim=True)
|
| 111 |
+
return grad
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _register_fakes() -> None:
|
| 115 |
+
"""Fake (meta) kernels so torch.compile can trace through the ops."""
|
| 116 |
+
if not _HAS_OPS:
|
| 117 |
+
return
|
| 118 |
+
try:
|
| 119 |
+
namespace = ops.adarms_fwd.default.name().split("::")[0]
|
| 120 |
+
|
| 121 |
+
def _rstd_like(x):
|
| 122 |
+
return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
|
| 123 |
+
|
| 124 |
+
@torch.library.register_fake(f"{namespace}::adarms_fwd")
|
| 125 |
+
def _(x, scale, shift, weight, eps):
|
| 126 |
+
return torch.empty_like(x), _rstd_like(x)
|
| 127 |
+
|
| 128 |
+
@torch.library.register_fake(f"{namespace}::adarms_bwd")
|
| 129 |
+
def _(dy, x, scale, weight, rstd):
|
| 130 |
+
dmod = (
|
| 131 |
+
torch.empty_like(x)
|
| 132 |
+
if scale is not None
|
| 133 |
+
else x.new_empty((x.shape[-1],), dtype=torch.float32)
|
| 134 |
+
)
|
| 135 |
+
return torch.empty_like(x), dmod
|
| 136 |
+
|
| 137 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
|
| 138 |
+
def _(x, h, gate, scale, shift, weight, eps):
|
| 139 |
+
return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
|
| 140 |
+
|
| 141 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
|
| 142 |
+
def _(dy, dyr, r, h, gate, scale, weight, rstd):
|
| 143 |
+
dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
|
| 144 |
+
dmod = (
|
| 145 |
+
torch.empty_like(r)
|
| 146 |
+
if scale is not None
|
| 147 |
+
else r.new_empty((r.shape[-1],), dtype=torch.float32)
|
| 148 |
+
)
|
| 149 |
+
return torch.empty_like(r), torch.empty_like(r), dg, dmod
|
| 150 |
+
except Exception:
|
| 151 |
+
pass # fakes are a compile nicety, never a functional requirement
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
_register_fakes()
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class _AdaRMSFn(torch.autograd.Function):
|
| 158 |
+
@staticmethod
|
| 159 |
+
def forward(ctx, x, scale, shift, weight, eps):
|
| 160 |
+
y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
|
| 161 |
+
ctx.save_for_backward(x, scale, weight, rstd)
|
| 162 |
+
return y
|
| 163 |
+
|
| 164 |
+
@staticmethod
|
| 165 |
+
def backward(ctx, dy):
|
| 166 |
+
x, scale, weight, rstd = ctx.saved_tensors
|
| 167 |
+
dy = dy.contiguous()
|
| 168 |
+
dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
|
| 169 |
+
if scale is not None:
|
| 170 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 171 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 172 |
+
return dx, dscale, dshift, None, None
|
| 173 |
+
return dx, None, None, dmod, None
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class _ResGateAdaRMSFn(torch.autograd.Function):
|
| 177 |
+
@staticmethod
|
| 178 |
+
def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
|
| 179 |
+
r, y, rstd = ops.resgate_adarms_fwd(
|
| 180 |
+
x, h, gate_in, scale, shift, weight, float(eps)
|
| 181 |
+
)
|
| 182 |
+
ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
|
| 183 |
+
return r, y
|
| 184 |
+
|
| 185 |
+
@staticmethod
|
| 186 |
+
def backward(ctx, dr_out, dy):
|
| 187 |
+
h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
|
| 188 |
+
dy = dy.contiguous()
|
| 189 |
+
dyr = dr_out.contiguous() if dr_out is not None else None
|
| 190 |
+
dr, dh, dg, dmod = ops.resgate_adarms_bwd(
|
| 191 |
+
dy, dyr, r, h, gate_in, scale, weight, rstd
|
| 192 |
+
)
|
| 193 |
+
dg_out = dg if gate_in is not None else None
|
| 194 |
+
if scale is not None:
|
| 195 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 196 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 197 |
+
return dr, dh, dg_out, dscale, dshift, None, None
|
| 198 |
+
return dr, dh, dg_out, None, None, dmod, None
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
|
| 202 |
+
modulation = _broadcast_modulation(x, modulation)
|
| 203 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 204 |
+
return scale, shift, gate
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def adarms(
|
| 208 |
+
x: torch.Tensor,
|
| 209 |
+
modulation_or_weight: torch.Tensor,
|
| 210 |
+
eps: float = 1e-6,
|
| 211 |
+
adaptive: bool = True,
|
| 212 |
+
):
|
| 213 |
+
"""AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
|
| 214 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 215 |
+
y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 216 |
+
return y, gate
|
| 217 |
+
if adaptive:
|
| 218 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 219 |
+
y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
|
| 220 |
+
return y, gate.to(x.dtype)
|
| 221 |
+
y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
|
| 222 |
+
return y, None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def resgate_adarms(
|
| 226 |
+
x: torch.Tensor,
|
| 227 |
+
h: torch.Tensor,
|
| 228 |
+
gate_in: torch.Tensor | None,
|
| 229 |
+
modulation_or_weight: torch.Tensor,
|
| 230 |
+
eps: float = 1e-6,
|
| 231 |
+
adaptive: bool = True,
|
| 232 |
+
):
|
| 233 |
+
"""r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
|
| 234 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 235 |
+
r, y, gate, _ = reference_resgate_adarms(
|
| 236 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 237 |
+
)
|
| 238 |
+
return r, y, gate
|
| 239 |
+
gate_in_c = None
|
| 240 |
+
if gate_in is not None:
|
| 241 |
+
gate_in_c = gate_in.to(x.dtype)
|
| 242 |
+
gate_in_c = (
|
| 243 |
+
gate_in_c.expand_as(x).contiguous()
|
| 244 |
+
if gate_in_c.shape != x.shape
|
| 245 |
+
else gate_in_c.contiguous()
|
| 246 |
+
)
|
| 247 |
+
if adaptive:
|
| 248 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 249 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 250 |
+
x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
|
| 251 |
+
)
|
| 252 |
+
return r, y, gate.to(x.dtype)
|
| 253 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 254 |
+
x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
|
| 255 |
+
)
|
| 256 |
+
return r, y, None
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def adarms_forward(
|
| 260 |
+
x: torch.Tensor,
|
| 261 |
+
modulation_or_weight: torch.Tensor,
|
| 262 |
+
eps: float = 1e-6,
|
| 263 |
+
adaptive: bool = True,
|
| 264 |
+
):
|
| 265 |
+
"""Compatibility wrapper returning (y, gate_or_None, rstd)."""
|
| 266 |
+
y, gate = adarms(x, modulation_or_weight, eps, adaptive)
|
| 267 |
+
_, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 268 |
+
return y, gate, rstd
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def resgate_adarms_forward(
|
| 272 |
+
x: torch.Tensor,
|
| 273 |
+
h: torch.Tensor,
|
| 274 |
+
gate_in: torch.Tensor | None,
|
| 275 |
+
modulation_or_weight: torch.Tensor,
|
| 276 |
+
eps: float = 1e-6,
|
| 277 |
+
adaptive: bool = True,
|
| 278 |
+
):
|
| 279 |
+
"""Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
|
| 280 |
+
r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
|
| 281 |
+
_, _, _, rstd = reference_resgate_adarms(
|
| 282 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 283 |
+
)
|
| 284 |
+
return r, y, gate, rstd
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 288 |
+
"""Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
|
| 289 |
+
|
| 290 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 291 |
+
super().__init__()
|
| 292 |
+
self.dim = int(dim)
|
| 293 |
+
self.eps = float(eps)
|
| 294 |
+
self.cond_dim = cond_dim
|
| 295 |
+
if cond_dim is None:
|
| 296 |
+
self.weight = nn.Parameter(torch.zeros(dim))
|
| 297 |
+
self.dense = None
|
| 298 |
+
else:
|
| 299 |
+
self.weight = None
|
| 300 |
+
self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
|
| 301 |
+
nn.init.zeros_(self.dense.weight)
|
| 302 |
+
|
| 303 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 304 |
+
if self.dense is None:
|
| 305 |
+
return adarms(x, self.weight, self.eps, adaptive=False)
|
| 306 |
+
if cond is None:
|
| 307 |
+
raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 308 |
+
return adarms(x, self.dense(cond), self.eps, adaptive=True)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 312 |
+
if ops is None:
|
| 313 |
+
return x
|
| 314 |
+
return ops._flashrt_training_package_marker(x)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
__all__ = [
|
| 318 |
+
"adarms",
|
| 319 |
+
"resgate_adarms",
|
| 320 |
+
"adarms_forward",
|
| 321 |
+
"resgate_adarms_forward",
|
| 322 |
+
"reference_adarms",
|
| 323 |
+
"reference_resgate_adarms",
|
| 324 |
+
"FlashRTAdaRMSNorm",
|
| 325 |
+
"backend_marker",
|
| 326 |
+
]
|
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:59dac281038a7834220edae42869dfe6e8ac5854c227f7aa8e3333b88fbe2be0
|
| 3 |
+
size 20031168
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_cc234e4
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_cc234e4",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"10.1",
|
| 12 |
+
"12.0+PTX",
|
| 13 |
+
"7.0",
|
| 14 |
+
"7.2",
|
| 15 |
+
"7.5",
|
| 16 |
+
"8.0",
|
| 17 |
+
"8.6",
|
| 18 |
+
"8.7",
|
| 19 |
+
"8.9",
|
| 20 |
+
"9.0"
|
| 21 |
+
]
|
| 22 |
+
}
|
| 23 |
+
}
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
|
| 2 |
+
forward and backward.
|
| 3 |
+
|
| 4 |
+
Operator (fp32 internal math):
|
| 5 |
+
rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
|
| 6 |
+
adaptive: y = xhat * (1 + scale) + shift
|
| 7 |
+
non-adaptive: y = xhat * (1 + weight)
|
| 8 |
+
The resgate variant computes r = x + h * gate_in first (r is a real output)
|
| 9 |
+
and normalizes r.
|
| 10 |
+
|
| 11 |
+
The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
|
| 12 |
+
fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
|
| 13 |
+
PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
|
| 14 |
+
the gate chunk stay in PyTorch, so their autograd is native.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _use_ops(namespace_ops) -> None:
|
| 32 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 33 |
+
global ops, _HAS_OPS
|
| 34 |
+
ops = namespace_ops
|
| 35 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 36 |
+
_register_fakes()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ---------------------------------------------------------------------------
|
| 40 |
+
# pure-PyTorch reference (also the CPU / unsupported-shape fallback)
|
| 41 |
+
# ---------------------------------------------------------------------------
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 45 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 46 |
+
return modulation.unsqueeze(1)
|
| 47 |
+
return modulation
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 51 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 55 |
+
compute_dtype = _compute_dtype(x)
|
| 56 |
+
xf = x.to(compute_dtype)
|
| 57 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 58 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def reference_adarms(
|
| 62 |
+
x: torch.Tensor,
|
| 63 |
+
modulation_or_weight: torch.Tensor,
|
| 64 |
+
eps: float = 1e-6,
|
| 65 |
+
adaptive: bool = True,
|
| 66 |
+
):
|
| 67 |
+
normed, rstd = _rms_norm(x, eps)
|
| 68 |
+
if adaptive:
|
| 69 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 70 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 71 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 72 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 73 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 74 |
+
return y.to(x.dtype), None, rstd
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def reference_resgate_adarms(
|
| 78 |
+
x: torch.Tensor,
|
| 79 |
+
h: torch.Tensor,
|
| 80 |
+
gate_in: torch.Tensor | None,
|
| 81 |
+
modulation_or_weight: torch.Tensor,
|
| 82 |
+
eps: float = 1e-6,
|
| 83 |
+
adaptive: bool = True,
|
| 84 |
+
):
|
| 85 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 86 |
+
y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
|
| 87 |
+
return residual, y, gate, rstd
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------------------
|
| 91 |
+
# CUDA autograd path
|
| 92 |
+
# ---------------------------------------------------------------------------
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
|
| 96 |
+
if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
|
| 97 |
+
return False
|
| 98 |
+
if x.dtype not in (torch.bfloat16, torch.float32):
|
| 99 |
+
return False
|
| 100 |
+
# the modulation/weight may be fp32 alongside bf16 activations (pi-style
|
| 101 |
+
# selective-bf16 keeps norm parameters in fp32)
|
| 102 |
+
if mod.dtype not in (x.dtype, torch.float32):
|
| 103 |
+
return False
|
| 104 |
+
lanes = 8 if x.dtype == torch.bfloat16 else 4
|
| 105 |
+
return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
|
| 109 |
+
if like.shape[1] == 1 and grad.shape[1] != 1:
|
| 110 |
+
return grad.sum(dim=1, keepdim=True)
|
| 111 |
+
return grad
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _register_fakes() -> None:
|
| 115 |
+
"""Fake (meta) kernels so torch.compile can trace through the ops."""
|
| 116 |
+
if not _HAS_OPS:
|
| 117 |
+
return
|
| 118 |
+
try:
|
| 119 |
+
namespace = ops.adarms_fwd.default.name().split("::")[0]
|
| 120 |
+
|
| 121 |
+
def _rstd_like(x):
|
| 122 |
+
return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
|
| 123 |
+
|
| 124 |
+
@torch.library.register_fake(f"{namespace}::adarms_fwd")
|
| 125 |
+
def _(x, scale, shift, weight, eps):
|
| 126 |
+
return torch.empty_like(x), _rstd_like(x)
|
| 127 |
+
|
| 128 |
+
@torch.library.register_fake(f"{namespace}::adarms_bwd")
|
| 129 |
+
def _(dy, x, scale, weight, rstd):
|
| 130 |
+
dmod = (
|
| 131 |
+
torch.empty_like(x)
|
| 132 |
+
if scale is not None
|
| 133 |
+
else x.new_empty((x.shape[-1],), dtype=torch.float32)
|
| 134 |
+
)
|
| 135 |
+
return torch.empty_like(x), dmod
|
| 136 |
+
|
| 137 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
|
| 138 |
+
def _(x, h, gate, scale, shift, weight, eps):
|
| 139 |
+
return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
|
| 140 |
+
|
| 141 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
|
| 142 |
+
def _(dy, dyr, r, h, gate, scale, weight, rstd):
|
| 143 |
+
dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
|
| 144 |
+
dmod = (
|
| 145 |
+
torch.empty_like(r)
|
| 146 |
+
if scale is not None
|
| 147 |
+
else r.new_empty((r.shape[-1],), dtype=torch.float32)
|
| 148 |
+
)
|
| 149 |
+
return torch.empty_like(r), torch.empty_like(r), dg, dmod
|
| 150 |
+
except Exception:
|
| 151 |
+
pass # fakes are a compile nicety, never a functional requirement
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
_register_fakes()
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class _AdaRMSFn(torch.autograd.Function):
|
| 158 |
+
@staticmethod
|
| 159 |
+
def forward(ctx, x, scale, shift, weight, eps):
|
| 160 |
+
y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
|
| 161 |
+
ctx.save_for_backward(x, scale, weight, rstd)
|
| 162 |
+
return y
|
| 163 |
+
|
| 164 |
+
@staticmethod
|
| 165 |
+
def backward(ctx, dy):
|
| 166 |
+
x, scale, weight, rstd = ctx.saved_tensors
|
| 167 |
+
dy = dy.contiguous()
|
| 168 |
+
dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
|
| 169 |
+
if scale is not None:
|
| 170 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 171 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 172 |
+
return dx, dscale, dshift, None, None
|
| 173 |
+
return dx, None, None, dmod, None
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class _ResGateAdaRMSFn(torch.autograd.Function):
|
| 177 |
+
@staticmethod
|
| 178 |
+
def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
|
| 179 |
+
r, y, rstd = ops.resgate_adarms_fwd(
|
| 180 |
+
x, h, gate_in, scale, shift, weight, float(eps)
|
| 181 |
+
)
|
| 182 |
+
ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
|
| 183 |
+
return r, y
|
| 184 |
+
|
| 185 |
+
@staticmethod
|
| 186 |
+
def backward(ctx, dr_out, dy):
|
| 187 |
+
h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
|
| 188 |
+
dy = dy.contiguous()
|
| 189 |
+
dyr = dr_out.contiguous() if dr_out is not None else None
|
| 190 |
+
dr, dh, dg, dmod = ops.resgate_adarms_bwd(
|
| 191 |
+
dy, dyr, r, h, gate_in, scale, weight, rstd
|
| 192 |
+
)
|
| 193 |
+
dg_out = dg if gate_in is not None else None
|
| 194 |
+
if scale is not None:
|
| 195 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 196 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 197 |
+
return dr, dh, dg_out, dscale, dshift, None, None
|
| 198 |
+
return dr, dh, dg_out, None, None, dmod, None
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
|
| 202 |
+
modulation = _broadcast_modulation(x, modulation)
|
| 203 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 204 |
+
return scale, shift, gate
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def adarms(
|
| 208 |
+
x: torch.Tensor,
|
| 209 |
+
modulation_or_weight: torch.Tensor,
|
| 210 |
+
eps: float = 1e-6,
|
| 211 |
+
adaptive: bool = True,
|
| 212 |
+
):
|
| 213 |
+
"""AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
|
| 214 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 215 |
+
y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 216 |
+
return y, gate
|
| 217 |
+
if adaptive:
|
| 218 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 219 |
+
y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
|
| 220 |
+
return y, gate.to(x.dtype)
|
| 221 |
+
y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
|
| 222 |
+
return y, None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def resgate_adarms(
|
| 226 |
+
x: torch.Tensor,
|
| 227 |
+
h: torch.Tensor,
|
| 228 |
+
gate_in: torch.Tensor | None,
|
| 229 |
+
modulation_or_weight: torch.Tensor,
|
| 230 |
+
eps: float = 1e-6,
|
| 231 |
+
adaptive: bool = True,
|
| 232 |
+
):
|
| 233 |
+
"""r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
|
| 234 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 235 |
+
r, y, gate, _ = reference_resgate_adarms(
|
| 236 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 237 |
+
)
|
| 238 |
+
return r, y, gate
|
| 239 |
+
gate_in_c = None
|
| 240 |
+
if gate_in is not None:
|
| 241 |
+
gate_in_c = gate_in.to(x.dtype)
|
| 242 |
+
gate_in_c = (
|
| 243 |
+
gate_in_c.expand_as(x).contiguous()
|
| 244 |
+
if gate_in_c.shape != x.shape
|
| 245 |
+
else gate_in_c.contiguous()
|
| 246 |
+
)
|
| 247 |
+
if adaptive:
|
| 248 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 249 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 250 |
+
x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
|
| 251 |
+
)
|
| 252 |
+
return r, y, gate.to(x.dtype)
|
| 253 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 254 |
+
x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
|
| 255 |
+
)
|
| 256 |
+
return r, y, None
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def adarms_forward(
|
| 260 |
+
x: torch.Tensor,
|
| 261 |
+
modulation_or_weight: torch.Tensor,
|
| 262 |
+
eps: float = 1e-6,
|
| 263 |
+
adaptive: bool = True,
|
| 264 |
+
):
|
| 265 |
+
"""Compatibility wrapper returning (y, gate_or_None, rstd)."""
|
| 266 |
+
y, gate = adarms(x, modulation_or_weight, eps, adaptive)
|
| 267 |
+
_, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 268 |
+
return y, gate, rstd
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def resgate_adarms_forward(
|
| 272 |
+
x: torch.Tensor,
|
| 273 |
+
h: torch.Tensor,
|
| 274 |
+
gate_in: torch.Tensor | None,
|
| 275 |
+
modulation_or_weight: torch.Tensor,
|
| 276 |
+
eps: float = 1e-6,
|
| 277 |
+
adaptive: bool = True,
|
| 278 |
+
):
|
| 279 |
+
"""Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
|
| 280 |
+
r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
|
| 281 |
+
_, _, _, rstd = reference_resgate_adarms(
|
| 282 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 283 |
+
)
|
| 284 |
+
return r, y, gate, rstd
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 288 |
+
"""Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
|
| 289 |
+
|
| 290 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 291 |
+
super().__init__()
|
| 292 |
+
self.dim = int(dim)
|
| 293 |
+
self.eps = float(eps)
|
| 294 |
+
self.cond_dim = cond_dim
|
| 295 |
+
if cond_dim is None:
|
| 296 |
+
self.weight = nn.Parameter(torch.zeros(dim))
|
| 297 |
+
self.dense = None
|
| 298 |
+
else:
|
| 299 |
+
self.weight = None
|
| 300 |
+
self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
|
| 301 |
+
nn.init.zeros_(self.dense.weight)
|
| 302 |
+
|
| 303 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 304 |
+
if self.dense is None:
|
| 305 |
+
return adarms(x, self.weight, self.eps, adaptive=False)
|
| 306 |
+
if cond is None:
|
| 307 |
+
raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 308 |
+
return adarms(x, self.dense(cond), self.eps, adaptive=True)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 312 |
+
if ops is None:
|
| 313 |
+
return x
|
| 314 |
+
return ops._flashrt_training_package_marker(x)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
__all__ = [
|
| 318 |
+
"adarms",
|
| 319 |
+
"resgate_adarms",
|
| 320 |
+
"adarms_forward",
|
| 321 |
+
"resgate_adarms_forward",
|
| 322 |
+
"reference_adarms",
|
| 323 |
+
"reference_resgate_adarms",
|
| 324 |
+
"FlashRTAdaRMSNorm",
|
| 325 |
+
"backend_marker",
|
| 326 |
+
]
|
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:93bdb8707481aea267204dfa607ea9edd2a4e12dcf55059874221a19dc2afcef
|
| 3 |
+
size 19280512
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_cc234e4
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_cc234e4",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
|
| 2 |
+
forward and backward.
|
| 3 |
+
|
| 4 |
+
Operator (fp32 internal math):
|
| 5 |
+
rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
|
| 6 |
+
adaptive: y = xhat * (1 + scale) + shift
|
| 7 |
+
non-adaptive: y = xhat * (1 + weight)
|
| 8 |
+
The resgate variant computes r = x + h * gate_in first (r is a real output)
|
| 9 |
+
and normalizes r.
|
| 10 |
+
|
| 11 |
+
The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
|
| 12 |
+
fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
|
| 13 |
+
PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
|
| 14 |
+
the gate chunk stay in PyTorch, so their autograd is native.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _use_ops(namespace_ops) -> None:
|
| 32 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 33 |
+
global ops, _HAS_OPS
|
| 34 |
+
ops = namespace_ops
|
| 35 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 36 |
+
_register_fakes()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ---------------------------------------------------------------------------
|
| 40 |
+
# pure-PyTorch reference (also the CPU / unsupported-shape fallback)
|
| 41 |
+
# ---------------------------------------------------------------------------
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 45 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 46 |
+
return modulation.unsqueeze(1)
|
| 47 |
+
return modulation
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 51 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 55 |
+
compute_dtype = _compute_dtype(x)
|
| 56 |
+
xf = x.to(compute_dtype)
|
| 57 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 58 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def reference_adarms(
|
| 62 |
+
x: torch.Tensor,
|
| 63 |
+
modulation_or_weight: torch.Tensor,
|
| 64 |
+
eps: float = 1e-6,
|
| 65 |
+
adaptive: bool = True,
|
| 66 |
+
):
|
| 67 |
+
normed, rstd = _rms_norm(x, eps)
|
| 68 |
+
if adaptive:
|
| 69 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 70 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 71 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 72 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 73 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 74 |
+
return y.to(x.dtype), None, rstd
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def reference_resgate_adarms(
|
| 78 |
+
x: torch.Tensor,
|
| 79 |
+
h: torch.Tensor,
|
| 80 |
+
gate_in: torch.Tensor | None,
|
| 81 |
+
modulation_or_weight: torch.Tensor,
|
| 82 |
+
eps: float = 1e-6,
|
| 83 |
+
adaptive: bool = True,
|
| 84 |
+
):
|
| 85 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 86 |
+
y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
|
| 87 |
+
return residual, y, gate, rstd
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------------------
|
| 91 |
+
# CUDA autograd path
|
| 92 |
+
# ---------------------------------------------------------------------------
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
|
| 96 |
+
if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
|
| 97 |
+
return False
|
| 98 |
+
if x.dtype not in (torch.bfloat16, torch.float32):
|
| 99 |
+
return False
|
| 100 |
+
# the modulation/weight may be fp32 alongside bf16 activations (pi-style
|
| 101 |
+
# selective-bf16 keeps norm parameters in fp32)
|
| 102 |
+
if mod.dtype not in (x.dtype, torch.float32):
|
| 103 |
+
return False
|
| 104 |
+
lanes = 8 if x.dtype == torch.bfloat16 else 4
|
| 105 |
+
return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
|
| 109 |
+
if like.shape[1] == 1 and grad.shape[1] != 1:
|
| 110 |
+
return grad.sum(dim=1, keepdim=True)
|
| 111 |
+
return grad
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _register_fakes() -> None:
|
| 115 |
+
"""Fake (meta) kernels so torch.compile can trace through the ops."""
|
| 116 |
+
if not _HAS_OPS:
|
| 117 |
+
return
|
| 118 |
+
try:
|
| 119 |
+
namespace = ops.adarms_fwd.default.name().split("::")[0]
|
| 120 |
+
|
| 121 |
+
def _rstd_like(x):
|
| 122 |
+
return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
|
| 123 |
+
|
| 124 |
+
@torch.library.register_fake(f"{namespace}::adarms_fwd")
|
| 125 |
+
def _(x, scale, shift, weight, eps):
|
| 126 |
+
return torch.empty_like(x), _rstd_like(x)
|
| 127 |
+
|
| 128 |
+
@torch.library.register_fake(f"{namespace}::adarms_bwd")
|
| 129 |
+
def _(dy, x, scale, weight, rstd):
|
| 130 |
+
dmod = (
|
| 131 |
+
torch.empty_like(x)
|
| 132 |
+
if scale is not None
|
| 133 |
+
else x.new_empty((x.shape[-1],), dtype=torch.float32)
|
| 134 |
+
)
|
| 135 |
+
return torch.empty_like(x), dmod
|
| 136 |
+
|
| 137 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
|
| 138 |
+
def _(x, h, gate, scale, shift, weight, eps):
|
| 139 |
+
return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
|
| 140 |
+
|
| 141 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
|
| 142 |
+
def _(dy, dyr, r, h, gate, scale, weight, rstd):
|
| 143 |
+
dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
|
| 144 |
+
dmod = (
|
| 145 |
+
torch.empty_like(r)
|
| 146 |
+
if scale is not None
|
| 147 |
+
else r.new_empty((r.shape[-1],), dtype=torch.float32)
|
| 148 |
+
)
|
| 149 |
+
return torch.empty_like(r), torch.empty_like(r), dg, dmod
|
| 150 |
+
except Exception:
|
| 151 |
+
pass # fakes are a compile nicety, never a functional requirement
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
_register_fakes()
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class _AdaRMSFn(torch.autograd.Function):
|
| 158 |
+
@staticmethod
|
| 159 |
+
def forward(ctx, x, scale, shift, weight, eps):
|
| 160 |
+
y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
|
| 161 |
+
ctx.save_for_backward(x, scale, weight, rstd)
|
| 162 |
+
return y
|
| 163 |
+
|
| 164 |
+
@staticmethod
|
| 165 |
+
def backward(ctx, dy):
|
| 166 |
+
x, scale, weight, rstd = ctx.saved_tensors
|
| 167 |
+
dy = dy.contiguous()
|
| 168 |
+
dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
|
| 169 |
+
if scale is not None:
|
| 170 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 171 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 172 |
+
return dx, dscale, dshift, None, None
|
| 173 |
+
return dx, None, None, dmod, None
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class _ResGateAdaRMSFn(torch.autograd.Function):
|
| 177 |
+
@staticmethod
|
| 178 |
+
def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
|
| 179 |
+
r, y, rstd = ops.resgate_adarms_fwd(
|
| 180 |
+
x, h, gate_in, scale, shift, weight, float(eps)
|
| 181 |
+
)
|
| 182 |
+
ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
|
| 183 |
+
return r, y
|
| 184 |
+
|
| 185 |
+
@staticmethod
|
| 186 |
+
def backward(ctx, dr_out, dy):
|
| 187 |
+
h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
|
| 188 |
+
dy = dy.contiguous()
|
| 189 |
+
dyr = dr_out.contiguous() if dr_out is not None else None
|
| 190 |
+
dr, dh, dg, dmod = ops.resgate_adarms_bwd(
|
| 191 |
+
dy, dyr, r, h, gate_in, scale, weight, rstd
|
| 192 |
+
)
|
| 193 |
+
dg_out = dg if gate_in is not None else None
|
| 194 |
+
if scale is not None:
|
| 195 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 196 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 197 |
+
return dr, dh, dg_out, dscale, dshift, None, None
|
| 198 |
+
return dr, dh, dg_out, None, None, dmod, None
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
|
| 202 |
+
modulation = _broadcast_modulation(x, modulation)
|
| 203 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 204 |
+
return scale, shift, gate
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def adarms(
|
| 208 |
+
x: torch.Tensor,
|
| 209 |
+
modulation_or_weight: torch.Tensor,
|
| 210 |
+
eps: float = 1e-6,
|
| 211 |
+
adaptive: bool = True,
|
| 212 |
+
):
|
| 213 |
+
"""AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
|
| 214 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 215 |
+
y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 216 |
+
return y, gate
|
| 217 |
+
if adaptive:
|
| 218 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 219 |
+
y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
|
| 220 |
+
return y, gate.to(x.dtype)
|
| 221 |
+
y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
|
| 222 |
+
return y, None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def resgate_adarms(
|
| 226 |
+
x: torch.Tensor,
|
| 227 |
+
h: torch.Tensor,
|
| 228 |
+
gate_in: torch.Tensor | None,
|
| 229 |
+
modulation_or_weight: torch.Tensor,
|
| 230 |
+
eps: float = 1e-6,
|
| 231 |
+
adaptive: bool = True,
|
| 232 |
+
):
|
| 233 |
+
"""r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
|
| 234 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 235 |
+
r, y, gate, _ = reference_resgate_adarms(
|
| 236 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 237 |
+
)
|
| 238 |
+
return r, y, gate
|
| 239 |
+
gate_in_c = None
|
| 240 |
+
if gate_in is not None:
|
| 241 |
+
gate_in_c = gate_in.to(x.dtype)
|
| 242 |
+
gate_in_c = (
|
| 243 |
+
gate_in_c.expand_as(x).contiguous()
|
| 244 |
+
if gate_in_c.shape != x.shape
|
| 245 |
+
else gate_in_c.contiguous()
|
| 246 |
+
)
|
| 247 |
+
if adaptive:
|
| 248 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 249 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 250 |
+
x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
|
| 251 |
+
)
|
| 252 |
+
return r, y, gate.to(x.dtype)
|
| 253 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 254 |
+
x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
|
| 255 |
+
)
|
| 256 |
+
return r, y, None
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def adarms_forward(
|
| 260 |
+
x: torch.Tensor,
|
| 261 |
+
modulation_or_weight: torch.Tensor,
|
| 262 |
+
eps: float = 1e-6,
|
| 263 |
+
adaptive: bool = True,
|
| 264 |
+
):
|
| 265 |
+
"""Compatibility wrapper returning (y, gate_or_None, rstd)."""
|
| 266 |
+
y, gate = adarms(x, modulation_or_weight, eps, adaptive)
|
| 267 |
+
_, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 268 |
+
return y, gate, rstd
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def resgate_adarms_forward(
|
| 272 |
+
x: torch.Tensor,
|
| 273 |
+
h: torch.Tensor,
|
| 274 |
+
gate_in: torch.Tensor | None,
|
| 275 |
+
modulation_or_weight: torch.Tensor,
|
| 276 |
+
eps: float = 1e-6,
|
| 277 |
+
adaptive: bool = True,
|
| 278 |
+
):
|
| 279 |
+
"""Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
|
| 280 |
+
r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
|
| 281 |
+
_, _, _, rstd = reference_resgate_adarms(
|
| 282 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 283 |
+
)
|
| 284 |
+
return r, y, gate, rstd
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 288 |
+
"""Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
|
| 289 |
+
|
| 290 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 291 |
+
super().__init__()
|
| 292 |
+
self.dim = int(dim)
|
| 293 |
+
self.eps = float(eps)
|
| 294 |
+
self.cond_dim = cond_dim
|
| 295 |
+
if cond_dim is None:
|
| 296 |
+
self.weight = nn.Parameter(torch.zeros(dim))
|
| 297 |
+
self.dense = None
|
| 298 |
+
else:
|
| 299 |
+
self.weight = None
|
| 300 |
+
self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
|
| 301 |
+
nn.init.zeros_(self.dense.weight)
|
| 302 |
+
|
| 303 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 304 |
+
if self.dense is None:
|
| 305 |
+
return adarms(x, self.weight, self.eps, adaptive=False)
|
| 306 |
+
if cond is None:
|
| 307 |
+
raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 308 |
+
return adarms(x, self.dense(cond), self.eps, adaptive=True)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 312 |
+
if ops is None:
|
| 313 |
+
return x
|
| 314 |
+
return ops._flashrt_training_package_marker(x)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
__all__ = [
|
| 318 |
+
"adarms",
|
| 319 |
+
"resgate_adarms",
|
| 320 |
+
"adarms_forward",
|
| 321 |
+
"resgate_adarms_forward",
|
| 322 |
+
"reference_adarms",
|
| 323 |
+
"reference_resgate_adarms",
|
| 324 |
+
"FlashRTAdaRMSNorm",
|
| 325 |
+
"backend_marker",
|
| 326 |
+
]
|
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b08b7c49dfd208cb524627fabba868ea0a5591e8767f34e2191881c8e84f3b3a
|
| 3 |
+
size 19280416
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_cc234e4
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_cc234e4",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
|
| 2 |
+
forward and backward.
|
| 3 |
+
|
| 4 |
+
Operator (fp32 internal math):
|
| 5 |
+
rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
|
| 6 |
+
adaptive: y = xhat * (1 + scale) + shift
|
| 7 |
+
non-adaptive: y = xhat * (1 + weight)
|
| 8 |
+
The resgate variant computes r = x + h * gate_in first (r is a real output)
|
| 9 |
+
and normalizes r.
|
| 10 |
+
|
| 11 |
+
The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
|
| 12 |
+
fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
|
| 13 |
+
PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
|
| 14 |
+
the gate chunk stay in PyTorch, so their autograd is native.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import torch
|
| 20 |
+
from torch import nn
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from ._ops import ops
|
| 24 |
+
|
| 25 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 26 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 27 |
+
ops = None
|
| 28 |
+
_HAS_OPS = False
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _use_ops(namespace_ops) -> None:
|
| 32 |
+
"""Install a manually built extension (dev/testing path)."""
|
| 33 |
+
global ops, _HAS_OPS
|
| 34 |
+
ops = namespace_ops
|
| 35 |
+
_HAS_OPS = hasattr(ops, "adarms_fwd")
|
| 36 |
+
_register_fakes()
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ---------------------------------------------------------------------------
|
| 40 |
+
# pure-PyTorch reference (also the CPU / unsupported-shape fallback)
|
| 41 |
+
# ---------------------------------------------------------------------------
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 45 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 46 |
+
return modulation.unsqueeze(1)
|
| 47 |
+
return modulation
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 51 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 55 |
+
compute_dtype = _compute_dtype(x)
|
| 56 |
+
xf = x.to(compute_dtype)
|
| 57 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 58 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def reference_adarms(
|
| 62 |
+
x: torch.Tensor,
|
| 63 |
+
modulation_or_weight: torch.Tensor,
|
| 64 |
+
eps: float = 1e-6,
|
| 65 |
+
adaptive: bool = True,
|
| 66 |
+
):
|
| 67 |
+
normed, rstd = _rms_norm(x, eps)
|
| 68 |
+
if adaptive:
|
| 69 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 70 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 71 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 72 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 73 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 74 |
+
return y.to(x.dtype), None, rstd
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def reference_resgate_adarms(
|
| 78 |
+
x: torch.Tensor,
|
| 79 |
+
h: torch.Tensor,
|
| 80 |
+
gate_in: torch.Tensor | None,
|
| 81 |
+
modulation_or_weight: torch.Tensor,
|
| 82 |
+
eps: float = 1e-6,
|
| 83 |
+
adaptive: bool = True,
|
| 84 |
+
):
|
| 85 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 86 |
+
y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
|
| 87 |
+
return residual, y, gate, rstd
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------------------
|
| 91 |
+
# CUDA autograd path
|
| 92 |
+
# ---------------------------------------------------------------------------
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
|
| 96 |
+
if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
|
| 97 |
+
return False
|
| 98 |
+
if x.dtype not in (torch.bfloat16, torch.float32):
|
| 99 |
+
return False
|
| 100 |
+
# the modulation/weight may be fp32 alongside bf16 activations (pi-style
|
| 101 |
+
# selective-bf16 keeps norm parameters in fp32)
|
| 102 |
+
if mod.dtype not in (x.dtype, torch.float32):
|
| 103 |
+
return False
|
| 104 |
+
lanes = 8 if x.dtype == torch.bfloat16 else 4
|
| 105 |
+
return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
|
| 109 |
+
if like.shape[1] == 1 and grad.shape[1] != 1:
|
| 110 |
+
return grad.sum(dim=1, keepdim=True)
|
| 111 |
+
return grad
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _register_fakes() -> None:
|
| 115 |
+
"""Fake (meta) kernels so torch.compile can trace through the ops."""
|
| 116 |
+
if not _HAS_OPS:
|
| 117 |
+
return
|
| 118 |
+
try:
|
| 119 |
+
namespace = ops.adarms_fwd.default.name().split("::")[0]
|
| 120 |
+
|
| 121 |
+
def _rstd_like(x):
|
| 122 |
+
return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
|
| 123 |
+
|
| 124 |
+
@torch.library.register_fake(f"{namespace}::adarms_fwd")
|
| 125 |
+
def _(x, scale, shift, weight, eps):
|
| 126 |
+
return torch.empty_like(x), _rstd_like(x)
|
| 127 |
+
|
| 128 |
+
@torch.library.register_fake(f"{namespace}::adarms_bwd")
|
| 129 |
+
def _(dy, x, scale, weight, rstd):
|
| 130 |
+
dmod = (
|
| 131 |
+
torch.empty_like(x)
|
| 132 |
+
if scale is not None
|
| 133 |
+
else x.new_empty((x.shape[-1],), dtype=torch.float32)
|
| 134 |
+
)
|
| 135 |
+
return torch.empty_like(x), dmod
|
| 136 |
+
|
| 137 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
|
| 138 |
+
def _(x, h, gate, scale, shift, weight, eps):
|
| 139 |
+
return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
|
| 140 |
+
|
| 141 |
+
@torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
|
| 142 |
+
def _(dy, dyr, r, h, gate, scale, weight, rstd):
|
| 143 |
+
dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
|
| 144 |
+
dmod = (
|
| 145 |
+
torch.empty_like(r)
|
| 146 |
+
if scale is not None
|
| 147 |
+
else r.new_empty((r.shape[-1],), dtype=torch.float32)
|
| 148 |
+
)
|
| 149 |
+
return torch.empty_like(r), torch.empty_like(r), dg, dmod
|
| 150 |
+
except Exception:
|
| 151 |
+
pass # fakes are a compile nicety, never a functional requirement
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
_register_fakes()
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class _AdaRMSFn(torch.autograd.Function):
|
| 158 |
+
@staticmethod
|
| 159 |
+
def forward(ctx, x, scale, shift, weight, eps):
|
| 160 |
+
y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
|
| 161 |
+
ctx.save_for_backward(x, scale, weight, rstd)
|
| 162 |
+
return y
|
| 163 |
+
|
| 164 |
+
@staticmethod
|
| 165 |
+
def backward(ctx, dy):
|
| 166 |
+
x, scale, weight, rstd = ctx.saved_tensors
|
| 167 |
+
dy = dy.contiguous()
|
| 168 |
+
dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
|
| 169 |
+
if scale is not None:
|
| 170 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 171 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 172 |
+
return dx, dscale, dshift, None, None
|
| 173 |
+
return dx, None, None, dmod, None
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
class _ResGateAdaRMSFn(torch.autograd.Function):
|
| 177 |
+
@staticmethod
|
| 178 |
+
def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
|
| 179 |
+
r, y, rstd = ops.resgate_adarms_fwd(
|
| 180 |
+
x, h, gate_in, scale, shift, weight, float(eps)
|
| 181 |
+
)
|
| 182 |
+
ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
|
| 183 |
+
return r, y
|
| 184 |
+
|
| 185 |
+
@staticmethod
|
| 186 |
+
def backward(ctx, dr_out, dy):
|
| 187 |
+
h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
|
| 188 |
+
dy = dy.contiguous()
|
| 189 |
+
dyr = dr_out.contiguous() if dr_out is not None else None
|
| 190 |
+
dr, dh, dg, dmod = ops.resgate_adarms_bwd(
|
| 191 |
+
dy, dyr, r, h, gate_in, scale, weight, rstd
|
| 192 |
+
)
|
| 193 |
+
dg_out = dg if gate_in is not None else None
|
| 194 |
+
if scale is not None:
|
| 195 |
+
dscale = _reduce_broadcast(dmod, scale)
|
| 196 |
+
dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
|
| 197 |
+
return dr, dh, dg_out, dscale, dshift, None, None
|
| 198 |
+
return dr, dh, dg_out, None, None, dmod, None
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
|
| 202 |
+
modulation = _broadcast_modulation(x, modulation)
|
| 203 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 204 |
+
return scale, shift, gate
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def adarms(
|
| 208 |
+
x: torch.Tensor,
|
| 209 |
+
modulation_or_weight: torch.Tensor,
|
| 210 |
+
eps: float = 1e-6,
|
| 211 |
+
adaptive: bool = True,
|
| 212 |
+
):
|
| 213 |
+
"""AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
|
| 214 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 215 |
+
y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 216 |
+
return y, gate
|
| 217 |
+
if adaptive:
|
| 218 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 219 |
+
y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
|
| 220 |
+
return y, gate.to(x.dtype)
|
| 221 |
+
y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
|
| 222 |
+
return y, None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
def resgate_adarms(
|
| 226 |
+
x: torch.Tensor,
|
| 227 |
+
h: torch.Tensor,
|
| 228 |
+
gate_in: torch.Tensor | None,
|
| 229 |
+
modulation_or_weight: torch.Tensor,
|
| 230 |
+
eps: float = 1e-6,
|
| 231 |
+
adaptive: bool = True,
|
| 232 |
+
):
|
| 233 |
+
"""r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
|
| 234 |
+
if not _kernel_supported(x, modulation_or_weight):
|
| 235 |
+
r, y, gate, _ = reference_resgate_adarms(
|
| 236 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 237 |
+
)
|
| 238 |
+
return r, y, gate
|
| 239 |
+
gate_in_c = None
|
| 240 |
+
if gate_in is not None:
|
| 241 |
+
gate_in_c = gate_in.to(x.dtype)
|
| 242 |
+
gate_in_c = (
|
| 243 |
+
gate_in_c.expand_as(x).contiguous()
|
| 244 |
+
if gate_in_c.shape != x.shape
|
| 245 |
+
else gate_in_c.contiguous()
|
| 246 |
+
)
|
| 247 |
+
if adaptive:
|
| 248 |
+
scale, shift, gate = _split_modulation(x, modulation_or_weight)
|
| 249 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 250 |
+
x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
|
| 251 |
+
)
|
| 252 |
+
return r, y, gate.to(x.dtype)
|
| 253 |
+
r, y = _ResGateAdaRMSFn.apply(
|
| 254 |
+
x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
|
| 255 |
+
)
|
| 256 |
+
return r, y, None
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def adarms_forward(
|
| 260 |
+
x: torch.Tensor,
|
| 261 |
+
modulation_or_weight: torch.Tensor,
|
| 262 |
+
eps: float = 1e-6,
|
| 263 |
+
adaptive: bool = True,
|
| 264 |
+
):
|
| 265 |
+
"""Compatibility wrapper returning (y, gate_or_None, rstd)."""
|
| 266 |
+
y, gate = adarms(x, modulation_or_weight, eps, adaptive)
|
| 267 |
+
_, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
|
| 268 |
+
return y, gate, rstd
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def resgate_adarms_forward(
|
| 272 |
+
x: torch.Tensor,
|
| 273 |
+
h: torch.Tensor,
|
| 274 |
+
gate_in: torch.Tensor | None,
|
| 275 |
+
modulation_or_weight: torch.Tensor,
|
| 276 |
+
eps: float = 1e-6,
|
| 277 |
+
adaptive: bool = True,
|
| 278 |
+
):
|
| 279 |
+
"""Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
|
| 280 |
+
r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
|
| 281 |
+
_, _, _, rstd = reference_resgate_adarms(
|
| 282 |
+
x, h, gate_in, modulation_or_weight, eps, adaptive
|
| 283 |
+
)
|
| 284 |
+
return r, y, gate, rstd
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 288 |
+
"""Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
|
| 289 |
+
|
| 290 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 291 |
+
super().__init__()
|
| 292 |
+
self.dim = int(dim)
|
| 293 |
+
self.eps = float(eps)
|
| 294 |
+
self.cond_dim = cond_dim
|
| 295 |
+
if cond_dim is None:
|
| 296 |
+
self.weight = nn.Parameter(torch.zeros(dim))
|
| 297 |
+
self.dense = None
|
| 298 |
+
else:
|
| 299 |
+
self.weight = None
|
| 300 |
+
self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
|
| 301 |
+
nn.init.zeros_(self.dense.weight)
|
| 302 |
+
|
| 303 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 304 |
+
if self.dense is None:
|
| 305 |
+
return adarms(x, self.weight, self.eps, adaptive=False)
|
| 306 |
+
if cond is None:
|
| 307 |
+
raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 308 |
+
return adarms(x, self.dense(cond), self.eps, adaptive=True)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 312 |
+
if ops is None:
|
| 313 |
+
return x
|
| 314 |
+
return ops._flashrt_training_package_marker(x)
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
__all__ = [
|
| 318 |
+
"adarms",
|
| 319 |
+
"resgate_adarms",
|
| 320 |
+
"adarms_forward",
|
| 321 |
+
"resgate_adarms_forward",
|
| 322 |
+
"reference_adarms",
|
| 323 |
+
"reference_resgate_adarms",
|
| 324 |
+
"FlashRTAdaRMSNorm",
|
| 325 |
+
"backend_marker",
|
| 326 |
+
]
|
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:973507666b4c3c108d6f2590ef1fad193018fb9ad6f6df85e9720f8b3b9702fa
|
| 3 |
+
size 19284512
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_cc234e4
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_cc234e4",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|