liangsu9988 commited on
Commit
6758c49
·
verified ·
1 Parent(s): c6e6c5d

Promote latest kernel artifacts to main

Browse files
.gitattributes CHANGED
@@ -1,35 +1,3 @@
1
- *.7z filter=lfs diff=lfs merge=lfs -text
2
- *.arrow filter=lfs diff=lfs merge=lfs -text
3
- *.bin filter=lfs diff=lfs merge=lfs -text
4
- *.bz2 filter=lfs diff=lfs merge=lfs -text
5
- *.ckpt filter=lfs diff=lfs merge=lfs -text
6
- *.ftz filter=lfs diff=lfs merge=lfs -text
7
- *.gz filter=lfs diff=lfs merge=lfs -text
8
- *.h5 filter=lfs diff=lfs merge=lfs -text
9
- *.joblib filter=lfs diff=lfs merge=lfs -text
10
- *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
- *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
- *.model filter=lfs diff=lfs merge=lfs -text
13
- *.msgpack filter=lfs diff=lfs merge=lfs -text
14
- *.npy filter=lfs diff=lfs merge=lfs -text
15
- *.npz filter=lfs diff=lfs merge=lfs -text
16
- *.onnx filter=lfs diff=lfs merge=lfs -text
17
- *.ot filter=lfs diff=lfs merge=lfs -text
18
- *.parquet filter=lfs diff=lfs merge=lfs -text
19
- *.pb filter=lfs diff=lfs merge=lfs -text
20
- *.pickle filter=lfs diff=lfs merge=lfs -text
21
- *.pkl filter=lfs diff=lfs merge=lfs -text
22
- *.pt filter=lfs diff=lfs merge=lfs -text
23
- *.pth filter=lfs diff=lfs merge=lfs -text
24
- *.rar filter=lfs diff=lfs merge=lfs -text
25
- *.safetensors filter=lfs diff=lfs merge=lfs -text
26
- saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
- *.tar.* filter=lfs diff=lfs merge=lfs -text
28
- *.tar filter=lfs diff=lfs merge=lfs -text
29
- *.tflite filter=lfs diff=lfs merge=lfs -text
30
- *.tgz filter=lfs diff=lfs merge=lfs -text
31
- *.wasm filter=lfs diff=lfs merge=lfs -text
32
- *.xz filter=lfs diff=lfs merge=lfs -text
33
- *.zip filter=lfs diff=lfs merge=lfs -text
34
- *.zst filter=lfs diff=lfs merge=lfs -text
35
- *tfevents* filter=lfs diff=lfs merge=lfs -text
 
1
+ *.so filter=lfs diff=lfs merge=lfs -text
2
+ *.pyd filter=lfs diff=lfs merge=lfs -text
3
+ *.dylib filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
README.md DELETED
@@ -1,9 +0,0 @@
1
- # flashrt/flashrt-adarms-train
2
-
3
- This repository is a compatibility mirror for older `kernels` clients
4
- that resolve repositories through the default Hugging Face model repo API.
5
-
6
- Canonical Kernel Hub repo: https://huggingface.co/kernels/flashrt/flashrt-adarms-train
7
-
8
- Do not edit this mirror by hand. It is generated from the Kernel Hub
9
- `vN` branches and contains the same `build/**` artifacts.
 
 
 
 
 
 
 
 
 
 
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
2
+ forward and backward.
3
+
4
+ Operator (fp32 internal math):
5
+ rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
6
+ adaptive: y = xhat * (1 + scale) + shift
7
+ non-adaptive: y = xhat * (1 + weight)
8
+ The resgate variant computes r = x + h * gate_in first (r is a real output)
9
+ and normalizes r.
10
+
11
+ The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
12
+ fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
13
+ PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
14
+ the gate chunk stay in PyTorch, so their autograd is native.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import torch
20
+ from torch import nn
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ def _use_ops(namespace_ops) -> None:
32
+ """Install a manually built extension (dev/testing path)."""
33
+ global ops, _HAS_OPS
34
+ ops = namespace_ops
35
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
36
+ _register_fakes()
37
+
38
+
39
+ # ---------------------------------------------------------------------------
40
+ # pure-PyTorch reference (also the CPU / unsupported-shape fallback)
41
+ # ---------------------------------------------------------------------------
42
+
43
+
44
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
45
+ if x.dim() == 3 and modulation.dim() == 2:
46
+ return modulation.unsqueeze(1)
47
+ return modulation
48
+
49
+
50
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
51
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
52
+
53
+
54
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
55
+ compute_dtype = _compute_dtype(x)
56
+ xf = x.to(compute_dtype)
57
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
58
+ return xf * rstd, rstd.squeeze(-1)
59
+
60
+
61
+ def reference_adarms(
62
+ x: torch.Tensor,
63
+ modulation_or_weight: torch.Tensor,
64
+ eps: float = 1e-6,
65
+ adaptive: bool = True,
66
+ ):
67
+ normed, rstd = _rms_norm(x, eps)
68
+ if adaptive:
69
+ modulation = _broadcast_modulation(x, modulation_or_weight)
70
+ scale, shift, gate = modulation.chunk(3, dim=-1)
71
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
72
+ return y.to(x.dtype), gate.to(x.dtype), rstd
73
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
74
+ return y.to(x.dtype), None, rstd
75
+
76
+
77
+ def reference_resgate_adarms(
78
+ x: torch.Tensor,
79
+ h: torch.Tensor,
80
+ gate_in: torch.Tensor | None,
81
+ modulation_or_weight: torch.Tensor,
82
+ eps: float = 1e-6,
83
+ adaptive: bool = True,
84
+ ):
85
+ residual = x + h if gate_in is None else x + h * gate_in
86
+ y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
87
+ return residual, y, gate, rstd
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+ # CUDA autograd path
92
+ # ---------------------------------------------------------------------------
93
+
94
+
95
+ def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
96
+ if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
97
+ return False
98
+ if x.dtype not in (torch.bfloat16, torch.float32):
99
+ return False
100
+ # the modulation/weight may be fp32 alongside bf16 activations (pi-style
101
+ # selective-bf16 keeps norm parameters in fp32)
102
+ if mod.dtype not in (x.dtype, torch.float32):
103
+ return False
104
+ lanes = 8 if x.dtype == torch.bfloat16 else 4
105
+ return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
106
+
107
+
108
+ def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
109
+ if like.shape[1] == 1 and grad.shape[1] != 1:
110
+ return grad.sum(dim=1, keepdim=True)
111
+ return grad
112
+
113
+
114
+ def _register_fakes() -> None:
115
+ """Fake (meta) kernels so torch.compile can trace through the ops."""
116
+ if not _HAS_OPS:
117
+ return
118
+ try:
119
+ namespace = ops.adarms_fwd.default.name().split("::")[0]
120
+
121
+ def _rstd_like(x):
122
+ return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
123
+
124
+ @torch.library.register_fake(f"{namespace}::adarms_fwd")
125
+ def _(x, scale, shift, weight, eps):
126
+ return torch.empty_like(x), _rstd_like(x)
127
+
128
+ @torch.library.register_fake(f"{namespace}::adarms_bwd")
129
+ def _(dy, x, scale, weight, rstd):
130
+ dmod = (
131
+ torch.empty_like(x)
132
+ if scale is not None
133
+ else x.new_empty((x.shape[-1],), dtype=torch.float32)
134
+ )
135
+ return torch.empty_like(x), dmod
136
+
137
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
138
+ def _(x, h, gate, scale, shift, weight, eps):
139
+ return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
140
+
141
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
142
+ def _(dy, dyr, r, h, gate, scale, weight, rstd):
143
+ dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
144
+ dmod = (
145
+ torch.empty_like(r)
146
+ if scale is not None
147
+ else r.new_empty((r.shape[-1],), dtype=torch.float32)
148
+ )
149
+ return torch.empty_like(r), torch.empty_like(r), dg, dmod
150
+ except Exception:
151
+ pass # fakes are a compile nicety, never a functional requirement
152
+
153
+
154
+ _register_fakes()
155
+
156
+
157
+ class _AdaRMSFn(torch.autograd.Function):
158
+ @staticmethod
159
+ def forward(ctx, x, scale, shift, weight, eps):
160
+ y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
161
+ ctx.save_for_backward(x, scale, weight, rstd)
162
+ return y
163
+
164
+ @staticmethod
165
+ def backward(ctx, dy):
166
+ x, scale, weight, rstd = ctx.saved_tensors
167
+ dy = dy.contiguous()
168
+ dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
169
+ if scale is not None:
170
+ dscale = _reduce_broadcast(dmod, scale)
171
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
172
+ return dx, dscale, dshift, None, None
173
+ return dx, None, None, dmod, None
174
+
175
+
176
+ class _ResGateAdaRMSFn(torch.autograd.Function):
177
+ @staticmethod
178
+ def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
179
+ r, y, rstd = ops.resgate_adarms_fwd(
180
+ x, h, gate_in, scale, shift, weight, float(eps)
181
+ )
182
+ ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
183
+ return r, y
184
+
185
+ @staticmethod
186
+ def backward(ctx, dr_out, dy):
187
+ h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
188
+ dy = dy.contiguous()
189
+ dyr = dr_out.contiguous() if dr_out is not None else None
190
+ dr, dh, dg, dmod = ops.resgate_adarms_bwd(
191
+ dy, dyr, r, h, gate_in, scale, weight, rstd
192
+ )
193
+ dg_out = dg if gate_in is not None else None
194
+ if scale is not None:
195
+ dscale = _reduce_broadcast(dmod, scale)
196
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
197
+ return dr, dh, dg_out, dscale, dshift, None, None
198
+ return dr, dh, dg_out, None, None, dmod, None
199
+
200
+
201
+ def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
202
+ modulation = _broadcast_modulation(x, modulation)
203
+ scale, shift, gate = modulation.chunk(3, dim=-1)
204
+ return scale, shift, gate
205
+
206
+
207
+ def adarms(
208
+ x: torch.Tensor,
209
+ modulation_or_weight: torch.Tensor,
210
+ eps: float = 1e-6,
211
+ adaptive: bool = True,
212
+ ):
213
+ """AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
214
+ if not _kernel_supported(x, modulation_or_weight):
215
+ y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
216
+ return y, gate
217
+ if adaptive:
218
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
219
+ y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
220
+ return y, gate.to(x.dtype)
221
+ y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
222
+ return y, None
223
+
224
+
225
+ def resgate_adarms(
226
+ x: torch.Tensor,
227
+ h: torch.Tensor,
228
+ gate_in: torch.Tensor | None,
229
+ modulation_or_weight: torch.Tensor,
230
+ eps: float = 1e-6,
231
+ adaptive: bool = True,
232
+ ):
233
+ """r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
234
+ if not _kernel_supported(x, modulation_or_weight):
235
+ r, y, gate, _ = reference_resgate_adarms(
236
+ x, h, gate_in, modulation_or_weight, eps, adaptive
237
+ )
238
+ return r, y, gate
239
+ gate_in_c = None
240
+ if gate_in is not None:
241
+ gate_in_c = gate_in.to(x.dtype)
242
+ gate_in_c = (
243
+ gate_in_c.expand_as(x).contiguous()
244
+ if gate_in_c.shape != x.shape
245
+ else gate_in_c.contiguous()
246
+ )
247
+ if adaptive:
248
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
249
+ r, y = _ResGateAdaRMSFn.apply(
250
+ x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
251
+ )
252
+ return r, y, gate.to(x.dtype)
253
+ r, y = _ResGateAdaRMSFn.apply(
254
+ x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
255
+ )
256
+ return r, y, None
257
+
258
+
259
+ def adarms_forward(
260
+ x: torch.Tensor,
261
+ modulation_or_weight: torch.Tensor,
262
+ eps: float = 1e-6,
263
+ adaptive: bool = True,
264
+ ):
265
+ """Compatibility wrapper returning (y, gate_or_None, rstd)."""
266
+ y, gate = adarms(x, modulation_or_weight, eps, adaptive)
267
+ _, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
268
+ return y, gate, rstd
269
+
270
+
271
+ def resgate_adarms_forward(
272
+ x: torch.Tensor,
273
+ h: torch.Tensor,
274
+ gate_in: torch.Tensor | None,
275
+ modulation_or_weight: torch.Tensor,
276
+ eps: float = 1e-6,
277
+ adaptive: bool = True,
278
+ ):
279
+ """Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
280
+ r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
281
+ _, _, _, rstd = reference_resgate_adarms(
282
+ x, h, gate_in, modulation_or_weight, eps, adaptive
283
+ )
284
+ return r, y, gate, rstd
285
+
286
+
287
+ class FlashRTAdaRMSNorm(nn.Module):
288
+ """Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
289
+
290
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
291
+ super().__init__()
292
+ self.dim = int(dim)
293
+ self.eps = float(eps)
294
+ self.cond_dim = cond_dim
295
+ if cond_dim is None:
296
+ self.weight = nn.Parameter(torch.zeros(dim))
297
+ self.dense = None
298
+ else:
299
+ self.weight = None
300
+ self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
301
+ nn.init.zeros_(self.dense.weight)
302
+
303
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
304
+ if self.dense is None:
305
+ return adarms(x, self.weight, self.eps, adaptive=False)
306
+ if cond is None:
307
+ raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
308
+ return adarms(x, self.dense(cond), self.eps, adaptive=True)
309
+
310
+
311
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
312
+ if ops is None:
313
+ return x
314
+ return ops._flashrt_training_package_marker(x)
315
+
316
+
317
+ __all__ = [
318
+ "adarms",
319
+ "resgate_adarms",
320
+ "adarms_forward",
321
+ "resgate_adarms_forward",
322
+ "reference_adarms",
323
+ "reference_resgate_adarms",
324
+ "FlashRTAdaRMSNorm",
325
+ "backend_marker",
326
+ ]
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59dac281038a7834220edae42869dfe6e8ac5854c227f7aa8e3333b88fbe2be0
3
+ size 20031168
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_cc234e4
3
+ ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_cc234e4",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
2
+ forward and backward.
3
+
4
+ Operator (fp32 internal math):
5
+ rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
6
+ adaptive: y = xhat * (1 + scale) + shift
7
+ non-adaptive: y = xhat * (1 + weight)
8
+ The resgate variant computes r = x + h * gate_in first (r is a real output)
9
+ and normalizes r.
10
+
11
+ The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
12
+ fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
13
+ PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
14
+ the gate chunk stay in PyTorch, so their autograd is native.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import torch
20
+ from torch import nn
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ def _use_ops(namespace_ops) -> None:
32
+ """Install a manually built extension (dev/testing path)."""
33
+ global ops, _HAS_OPS
34
+ ops = namespace_ops
35
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
36
+ _register_fakes()
37
+
38
+
39
+ # ---------------------------------------------------------------------------
40
+ # pure-PyTorch reference (also the CPU / unsupported-shape fallback)
41
+ # ---------------------------------------------------------------------------
42
+
43
+
44
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
45
+ if x.dim() == 3 and modulation.dim() == 2:
46
+ return modulation.unsqueeze(1)
47
+ return modulation
48
+
49
+
50
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
51
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
52
+
53
+
54
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
55
+ compute_dtype = _compute_dtype(x)
56
+ xf = x.to(compute_dtype)
57
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
58
+ return xf * rstd, rstd.squeeze(-1)
59
+
60
+
61
+ def reference_adarms(
62
+ x: torch.Tensor,
63
+ modulation_or_weight: torch.Tensor,
64
+ eps: float = 1e-6,
65
+ adaptive: bool = True,
66
+ ):
67
+ normed, rstd = _rms_norm(x, eps)
68
+ if adaptive:
69
+ modulation = _broadcast_modulation(x, modulation_or_weight)
70
+ scale, shift, gate = modulation.chunk(3, dim=-1)
71
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
72
+ return y.to(x.dtype), gate.to(x.dtype), rstd
73
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
74
+ return y.to(x.dtype), None, rstd
75
+
76
+
77
+ def reference_resgate_adarms(
78
+ x: torch.Tensor,
79
+ h: torch.Tensor,
80
+ gate_in: torch.Tensor | None,
81
+ modulation_or_weight: torch.Tensor,
82
+ eps: float = 1e-6,
83
+ adaptive: bool = True,
84
+ ):
85
+ residual = x + h if gate_in is None else x + h * gate_in
86
+ y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
87
+ return residual, y, gate, rstd
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+ # CUDA autograd path
92
+ # ---------------------------------------------------------------------------
93
+
94
+
95
+ def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
96
+ if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
97
+ return False
98
+ if x.dtype not in (torch.bfloat16, torch.float32):
99
+ return False
100
+ # the modulation/weight may be fp32 alongside bf16 activations (pi-style
101
+ # selective-bf16 keeps norm parameters in fp32)
102
+ if mod.dtype not in (x.dtype, torch.float32):
103
+ return False
104
+ lanes = 8 if x.dtype == torch.bfloat16 else 4
105
+ return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
106
+
107
+
108
+ def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
109
+ if like.shape[1] == 1 and grad.shape[1] != 1:
110
+ return grad.sum(dim=1, keepdim=True)
111
+ return grad
112
+
113
+
114
+ def _register_fakes() -> None:
115
+ """Fake (meta) kernels so torch.compile can trace through the ops."""
116
+ if not _HAS_OPS:
117
+ return
118
+ try:
119
+ namespace = ops.adarms_fwd.default.name().split("::")[0]
120
+
121
+ def _rstd_like(x):
122
+ return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
123
+
124
+ @torch.library.register_fake(f"{namespace}::adarms_fwd")
125
+ def _(x, scale, shift, weight, eps):
126
+ return torch.empty_like(x), _rstd_like(x)
127
+
128
+ @torch.library.register_fake(f"{namespace}::adarms_bwd")
129
+ def _(dy, x, scale, weight, rstd):
130
+ dmod = (
131
+ torch.empty_like(x)
132
+ if scale is not None
133
+ else x.new_empty((x.shape[-1],), dtype=torch.float32)
134
+ )
135
+ return torch.empty_like(x), dmod
136
+
137
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
138
+ def _(x, h, gate, scale, shift, weight, eps):
139
+ return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
140
+
141
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
142
+ def _(dy, dyr, r, h, gate, scale, weight, rstd):
143
+ dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
144
+ dmod = (
145
+ torch.empty_like(r)
146
+ if scale is not None
147
+ else r.new_empty((r.shape[-1],), dtype=torch.float32)
148
+ )
149
+ return torch.empty_like(r), torch.empty_like(r), dg, dmod
150
+ except Exception:
151
+ pass # fakes are a compile nicety, never a functional requirement
152
+
153
+
154
+ _register_fakes()
155
+
156
+
157
+ class _AdaRMSFn(torch.autograd.Function):
158
+ @staticmethod
159
+ def forward(ctx, x, scale, shift, weight, eps):
160
+ y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
161
+ ctx.save_for_backward(x, scale, weight, rstd)
162
+ return y
163
+
164
+ @staticmethod
165
+ def backward(ctx, dy):
166
+ x, scale, weight, rstd = ctx.saved_tensors
167
+ dy = dy.contiguous()
168
+ dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
169
+ if scale is not None:
170
+ dscale = _reduce_broadcast(dmod, scale)
171
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
172
+ return dx, dscale, dshift, None, None
173
+ return dx, None, None, dmod, None
174
+
175
+
176
+ class _ResGateAdaRMSFn(torch.autograd.Function):
177
+ @staticmethod
178
+ def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
179
+ r, y, rstd = ops.resgate_adarms_fwd(
180
+ x, h, gate_in, scale, shift, weight, float(eps)
181
+ )
182
+ ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
183
+ return r, y
184
+
185
+ @staticmethod
186
+ def backward(ctx, dr_out, dy):
187
+ h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
188
+ dy = dy.contiguous()
189
+ dyr = dr_out.contiguous() if dr_out is not None else None
190
+ dr, dh, dg, dmod = ops.resgate_adarms_bwd(
191
+ dy, dyr, r, h, gate_in, scale, weight, rstd
192
+ )
193
+ dg_out = dg if gate_in is not None else None
194
+ if scale is not None:
195
+ dscale = _reduce_broadcast(dmod, scale)
196
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
197
+ return dr, dh, dg_out, dscale, dshift, None, None
198
+ return dr, dh, dg_out, None, None, dmod, None
199
+
200
+
201
+ def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
202
+ modulation = _broadcast_modulation(x, modulation)
203
+ scale, shift, gate = modulation.chunk(3, dim=-1)
204
+ return scale, shift, gate
205
+
206
+
207
+ def adarms(
208
+ x: torch.Tensor,
209
+ modulation_or_weight: torch.Tensor,
210
+ eps: float = 1e-6,
211
+ adaptive: bool = True,
212
+ ):
213
+ """AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
214
+ if not _kernel_supported(x, modulation_or_weight):
215
+ y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
216
+ return y, gate
217
+ if adaptive:
218
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
219
+ y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
220
+ return y, gate.to(x.dtype)
221
+ y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
222
+ return y, None
223
+
224
+
225
+ def resgate_adarms(
226
+ x: torch.Tensor,
227
+ h: torch.Tensor,
228
+ gate_in: torch.Tensor | None,
229
+ modulation_or_weight: torch.Tensor,
230
+ eps: float = 1e-6,
231
+ adaptive: bool = True,
232
+ ):
233
+ """r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
234
+ if not _kernel_supported(x, modulation_or_weight):
235
+ r, y, gate, _ = reference_resgate_adarms(
236
+ x, h, gate_in, modulation_or_weight, eps, adaptive
237
+ )
238
+ return r, y, gate
239
+ gate_in_c = None
240
+ if gate_in is not None:
241
+ gate_in_c = gate_in.to(x.dtype)
242
+ gate_in_c = (
243
+ gate_in_c.expand_as(x).contiguous()
244
+ if gate_in_c.shape != x.shape
245
+ else gate_in_c.contiguous()
246
+ )
247
+ if adaptive:
248
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
249
+ r, y = _ResGateAdaRMSFn.apply(
250
+ x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
251
+ )
252
+ return r, y, gate.to(x.dtype)
253
+ r, y = _ResGateAdaRMSFn.apply(
254
+ x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
255
+ )
256
+ return r, y, None
257
+
258
+
259
+ def adarms_forward(
260
+ x: torch.Tensor,
261
+ modulation_or_weight: torch.Tensor,
262
+ eps: float = 1e-6,
263
+ adaptive: bool = True,
264
+ ):
265
+ """Compatibility wrapper returning (y, gate_or_None, rstd)."""
266
+ y, gate = adarms(x, modulation_or_weight, eps, adaptive)
267
+ _, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
268
+ return y, gate, rstd
269
+
270
+
271
+ def resgate_adarms_forward(
272
+ x: torch.Tensor,
273
+ h: torch.Tensor,
274
+ gate_in: torch.Tensor | None,
275
+ modulation_or_weight: torch.Tensor,
276
+ eps: float = 1e-6,
277
+ adaptive: bool = True,
278
+ ):
279
+ """Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
280
+ r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
281
+ _, _, _, rstd = reference_resgate_adarms(
282
+ x, h, gate_in, modulation_or_weight, eps, adaptive
283
+ )
284
+ return r, y, gate, rstd
285
+
286
+
287
+ class FlashRTAdaRMSNorm(nn.Module):
288
+ """Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
289
+
290
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
291
+ super().__init__()
292
+ self.dim = int(dim)
293
+ self.eps = float(eps)
294
+ self.cond_dim = cond_dim
295
+ if cond_dim is None:
296
+ self.weight = nn.Parameter(torch.zeros(dim))
297
+ self.dense = None
298
+ else:
299
+ self.weight = None
300
+ self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
301
+ nn.init.zeros_(self.dense.weight)
302
+
303
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
304
+ if self.dense is None:
305
+ return adarms(x, self.weight, self.eps, adaptive=False)
306
+ if cond is None:
307
+ raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
308
+ return adarms(x, self.dense(cond), self.eps, adaptive=True)
309
+
310
+
311
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
312
+ if ops is None:
313
+ return x
314
+ return ops._flashrt_training_package_marker(x)
315
+
316
+
317
+ __all__ = [
318
+ "adarms",
319
+ "resgate_adarms",
320
+ "adarms_forward",
321
+ "resgate_adarms_forward",
322
+ "reference_adarms",
323
+ "reference_resgate_adarms",
324
+ "FlashRTAdaRMSNorm",
325
+ "backend_marker",
326
+ ]
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93bdb8707481aea267204dfa607ea9edd2a4e12dcf55059874221a19dc2afcef
3
+ size 19280512
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_cc234e4
3
+ ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_cc234e4",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
2
+ forward and backward.
3
+
4
+ Operator (fp32 internal math):
5
+ rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
6
+ adaptive: y = xhat * (1 + scale) + shift
7
+ non-adaptive: y = xhat * (1 + weight)
8
+ The resgate variant computes r = x + h * gate_in first (r is a real output)
9
+ and normalizes r.
10
+
11
+ The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
12
+ fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
13
+ PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
14
+ the gate chunk stay in PyTorch, so their autograd is native.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import torch
20
+ from torch import nn
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ def _use_ops(namespace_ops) -> None:
32
+ """Install a manually built extension (dev/testing path)."""
33
+ global ops, _HAS_OPS
34
+ ops = namespace_ops
35
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
36
+ _register_fakes()
37
+
38
+
39
+ # ---------------------------------------------------------------------------
40
+ # pure-PyTorch reference (also the CPU / unsupported-shape fallback)
41
+ # ---------------------------------------------------------------------------
42
+
43
+
44
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
45
+ if x.dim() == 3 and modulation.dim() == 2:
46
+ return modulation.unsqueeze(1)
47
+ return modulation
48
+
49
+
50
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
51
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
52
+
53
+
54
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
55
+ compute_dtype = _compute_dtype(x)
56
+ xf = x.to(compute_dtype)
57
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
58
+ return xf * rstd, rstd.squeeze(-1)
59
+
60
+
61
+ def reference_adarms(
62
+ x: torch.Tensor,
63
+ modulation_or_weight: torch.Tensor,
64
+ eps: float = 1e-6,
65
+ adaptive: bool = True,
66
+ ):
67
+ normed, rstd = _rms_norm(x, eps)
68
+ if adaptive:
69
+ modulation = _broadcast_modulation(x, modulation_or_weight)
70
+ scale, shift, gate = modulation.chunk(3, dim=-1)
71
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
72
+ return y.to(x.dtype), gate.to(x.dtype), rstd
73
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
74
+ return y.to(x.dtype), None, rstd
75
+
76
+
77
+ def reference_resgate_adarms(
78
+ x: torch.Tensor,
79
+ h: torch.Tensor,
80
+ gate_in: torch.Tensor | None,
81
+ modulation_or_weight: torch.Tensor,
82
+ eps: float = 1e-6,
83
+ adaptive: bool = True,
84
+ ):
85
+ residual = x + h if gate_in is None else x + h * gate_in
86
+ y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
87
+ return residual, y, gate, rstd
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+ # CUDA autograd path
92
+ # ---------------------------------------------------------------------------
93
+
94
+
95
+ def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
96
+ if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
97
+ return False
98
+ if x.dtype not in (torch.bfloat16, torch.float32):
99
+ return False
100
+ # the modulation/weight may be fp32 alongside bf16 activations (pi-style
101
+ # selective-bf16 keeps norm parameters in fp32)
102
+ if mod.dtype not in (x.dtype, torch.float32):
103
+ return False
104
+ lanes = 8 if x.dtype == torch.bfloat16 else 4
105
+ return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
106
+
107
+
108
+ def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
109
+ if like.shape[1] == 1 and grad.shape[1] != 1:
110
+ return grad.sum(dim=1, keepdim=True)
111
+ return grad
112
+
113
+
114
+ def _register_fakes() -> None:
115
+ """Fake (meta) kernels so torch.compile can trace through the ops."""
116
+ if not _HAS_OPS:
117
+ return
118
+ try:
119
+ namespace = ops.adarms_fwd.default.name().split("::")[0]
120
+
121
+ def _rstd_like(x):
122
+ return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
123
+
124
+ @torch.library.register_fake(f"{namespace}::adarms_fwd")
125
+ def _(x, scale, shift, weight, eps):
126
+ return torch.empty_like(x), _rstd_like(x)
127
+
128
+ @torch.library.register_fake(f"{namespace}::adarms_bwd")
129
+ def _(dy, x, scale, weight, rstd):
130
+ dmod = (
131
+ torch.empty_like(x)
132
+ if scale is not None
133
+ else x.new_empty((x.shape[-1],), dtype=torch.float32)
134
+ )
135
+ return torch.empty_like(x), dmod
136
+
137
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
138
+ def _(x, h, gate, scale, shift, weight, eps):
139
+ return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
140
+
141
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
142
+ def _(dy, dyr, r, h, gate, scale, weight, rstd):
143
+ dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
144
+ dmod = (
145
+ torch.empty_like(r)
146
+ if scale is not None
147
+ else r.new_empty((r.shape[-1],), dtype=torch.float32)
148
+ )
149
+ return torch.empty_like(r), torch.empty_like(r), dg, dmod
150
+ except Exception:
151
+ pass # fakes are a compile nicety, never a functional requirement
152
+
153
+
154
+ _register_fakes()
155
+
156
+
157
+ class _AdaRMSFn(torch.autograd.Function):
158
+ @staticmethod
159
+ def forward(ctx, x, scale, shift, weight, eps):
160
+ y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
161
+ ctx.save_for_backward(x, scale, weight, rstd)
162
+ return y
163
+
164
+ @staticmethod
165
+ def backward(ctx, dy):
166
+ x, scale, weight, rstd = ctx.saved_tensors
167
+ dy = dy.contiguous()
168
+ dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
169
+ if scale is not None:
170
+ dscale = _reduce_broadcast(dmod, scale)
171
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
172
+ return dx, dscale, dshift, None, None
173
+ return dx, None, None, dmod, None
174
+
175
+
176
+ class _ResGateAdaRMSFn(torch.autograd.Function):
177
+ @staticmethod
178
+ def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
179
+ r, y, rstd = ops.resgate_adarms_fwd(
180
+ x, h, gate_in, scale, shift, weight, float(eps)
181
+ )
182
+ ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
183
+ return r, y
184
+
185
+ @staticmethod
186
+ def backward(ctx, dr_out, dy):
187
+ h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
188
+ dy = dy.contiguous()
189
+ dyr = dr_out.contiguous() if dr_out is not None else None
190
+ dr, dh, dg, dmod = ops.resgate_adarms_bwd(
191
+ dy, dyr, r, h, gate_in, scale, weight, rstd
192
+ )
193
+ dg_out = dg if gate_in is not None else None
194
+ if scale is not None:
195
+ dscale = _reduce_broadcast(dmod, scale)
196
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
197
+ return dr, dh, dg_out, dscale, dshift, None, None
198
+ return dr, dh, dg_out, None, None, dmod, None
199
+
200
+
201
+ def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
202
+ modulation = _broadcast_modulation(x, modulation)
203
+ scale, shift, gate = modulation.chunk(3, dim=-1)
204
+ return scale, shift, gate
205
+
206
+
207
+ def adarms(
208
+ x: torch.Tensor,
209
+ modulation_or_weight: torch.Tensor,
210
+ eps: float = 1e-6,
211
+ adaptive: bool = True,
212
+ ):
213
+ """AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
214
+ if not _kernel_supported(x, modulation_or_weight):
215
+ y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
216
+ return y, gate
217
+ if adaptive:
218
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
219
+ y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
220
+ return y, gate.to(x.dtype)
221
+ y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
222
+ return y, None
223
+
224
+
225
+ def resgate_adarms(
226
+ x: torch.Tensor,
227
+ h: torch.Tensor,
228
+ gate_in: torch.Tensor | None,
229
+ modulation_or_weight: torch.Tensor,
230
+ eps: float = 1e-6,
231
+ adaptive: bool = True,
232
+ ):
233
+ """r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
234
+ if not _kernel_supported(x, modulation_or_weight):
235
+ r, y, gate, _ = reference_resgate_adarms(
236
+ x, h, gate_in, modulation_or_weight, eps, adaptive
237
+ )
238
+ return r, y, gate
239
+ gate_in_c = None
240
+ if gate_in is not None:
241
+ gate_in_c = gate_in.to(x.dtype)
242
+ gate_in_c = (
243
+ gate_in_c.expand_as(x).contiguous()
244
+ if gate_in_c.shape != x.shape
245
+ else gate_in_c.contiguous()
246
+ )
247
+ if adaptive:
248
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
249
+ r, y = _ResGateAdaRMSFn.apply(
250
+ x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
251
+ )
252
+ return r, y, gate.to(x.dtype)
253
+ r, y = _ResGateAdaRMSFn.apply(
254
+ x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
255
+ )
256
+ return r, y, None
257
+
258
+
259
+ def adarms_forward(
260
+ x: torch.Tensor,
261
+ modulation_or_weight: torch.Tensor,
262
+ eps: float = 1e-6,
263
+ adaptive: bool = True,
264
+ ):
265
+ """Compatibility wrapper returning (y, gate_or_None, rstd)."""
266
+ y, gate = adarms(x, modulation_or_weight, eps, adaptive)
267
+ _, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
268
+ return y, gate, rstd
269
+
270
+
271
+ def resgate_adarms_forward(
272
+ x: torch.Tensor,
273
+ h: torch.Tensor,
274
+ gate_in: torch.Tensor | None,
275
+ modulation_or_weight: torch.Tensor,
276
+ eps: float = 1e-6,
277
+ adaptive: bool = True,
278
+ ):
279
+ """Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
280
+ r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
281
+ _, _, _, rstd = reference_resgate_adarms(
282
+ x, h, gate_in, modulation_or_weight, eps, adaptive
283
+ )
284
+ return r, y, gate, rstd
285
+
286
+
287
+ class FlashRTAdaRMSNorm(nn.Module):
288
+ """Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
289
+
290
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
291
+ super().__init__()
292
+ self.dim = int(dim)
293
+ self.eps = float(eps)
294
+ self.cond_dim = cond_dim
295
+ if cond_dim is None:
296
+ self.weight = nn.Parameter(torch.zeros(dim))
297
+ self.dense = None
298
+ else:
299
+ self.weight = None
300
+ self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
301
+ nn.init.zeros_(self.dense.weight)
302
+
303
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
304
+ if self.dense is None:
305
+ return adarms(x, self.weight, self.eps, adaptive=False)
306
+ if cond is None:
307
+ raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
308
+ return adarms(x, self.dense(cond), self.eps, adaptive=True)
309
+
310
+
311
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
312
+ if ops is None:
313
+ return x
314
+ return ops._flashrt_training_package_marker(x)
315
+
316
+
317
+ __all__ = [
318
+ "adarms",
319
+ "resgate_adarms",
320
+ "adarms_forward",
321
+ "resgate_adarms_forward",
322
+ "reference_adarms",
323
+ "reference_resgate_adarms",
324
+ "FlashRTAdaRMSNorm",
325
+ "backend_marker",
326
+ ]
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b08b7c49dfd208cb524627fabba868ea0a5591e8767f34e2191881c8e84f3b3a
3
+ size 19280416
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_cc234e4
3
+ ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_cc234e4",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,326 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training kernels: fused AdaRMS / gated-residual+AdaRMS
2
+ forward and backward.
3
+
4
+ Operator (fp32 internal math):
5
+ rstd = rsqrt(mean(x^2, -1) + eps); xhat = x * rstd
6
+ adaptive: y = xhat * (1 + scale) + shift
7
+ non-adaptive: y = xhat * (1 + weight)
8
+ The resgate variant computes r = x + h * gate_in first (r is a real output)
9
+ and normalizes r.
10
+
11
+ The public API dispatches to the CUDA kernels when supported (CUDA, bf16 or
12
+ fp32, H a multiple of 8/4 up to the register-cached limit) and to the pure
13
+ PyTorch reference otherwise. The modulation projection (``dense(cond)``) and
14
+ the gate chunk stay in PyTorch, so their autograd is native.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import torch
20
+ from torch import nn
21
+
22
+ try:
23
+ from ._ops import ops
24
+
25
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
26
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
27
+ ops = None
28
+ _HAS_OPS = False
29
+
30
+
31
+ def _use_ops(namespace_ops) -> None:
32
+ """Install a manually built extension (dev/testing path)."""
33
+ global ops, _HAS_OPS
34
+ ops = namespace_ops
35
+ _HAS_OPS = hasattr(ops, "adarms_fwd")
36
+ _register_fakes()
37
+
38
+
39
+ # ---------------------------------------------------------------------------
40
+ # pure-PyTorch reference (also the CPU / unsupported-shape fallback)
41
+ # ---------------------------------------------------------------------------
42
+
43
+
44
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
45
+ if x.dim() == 3 and modulation.dim() == 2:
46
+ return modulation.unsqueeze(1)
47
+ return modulation
48
+
49
+
50
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
51
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
52
+
53
+
54
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
55
+ compute_dtype = _compute_dtype(x)
56
+ xf = x.to(compute_dtype)
57
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
58
+ return xf * rstd, rstd.squeeze(-1)
59
+
60
+
61
+ def reference_adarms(
62
+ x: torch.Tensor,
63
+ modulation_or_weight: torch.Tensor,
64
+ eps: float = 1e-6,
65
+ adaptive: bool = True,
66
+ ):
67
+ normed, rstd = _rms_norm(x, eps)
68
+ if adaptive:
69
+ modulation = _broadcast_modulation(x, modulation_or_weight)
70
+ scale, shift, gate = modulation.chunk(3, dim=-1)
71
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
72
+ return y.to(x.dtype), gate.to(x.dtype), rstd
73
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
74
+ return y.to(x.dtype), None, rstd
75
+
76
+
77
+ def reference_resgate_adarms(
78
+ x: torch.Tensor,
79
+ h: torch.Tensor,
80
+ gate_in: torch.Tensor | None,
81
+ modulation_or_weight: torch.Tensor,
82
+ eps: float = 1e-6,
83
+ adaptive: bool = True,
84
+ ):
85
+ residual = x + h if gate_in is None else x + h * gate_in
86
+ y, gate, rstd = reference_adarms(residual, modulation_or_weight, eps, adaptive)
87
+ return residual, y, gate, rstd
88
+
89
+
90
+ # ---------------------------------------------------------------------------
91
+ # CUDA autograd path
92
+ # ---------------------------------------------------------------------------
93
+
94
+
95
+ def _kernel_supported(x: torch.Tensor, mod: torch.Tensor) -> bool:
96
+ if not (_HAS_OPS and x.is_cuda and x.dim() == 3):
97
+ return False
98
+ if x.dtype not in (torch.bfloat16, torch.float32):
99
+ return False
100
+ # the modulation/weight may be fp32 alongside bf16 activations (pi-style
101
+ # selective-bf16 keeps norm parameters in fp32)
102
+ if mod.dtype not in (x.dtype, torch.float32):
103
+ return False
104
+ lanes = 8 if x.dtype == torch.bfloat16 else 4
105
+ return x.shape[-1] % lanes == 0 and x.shape[-1] <= 256 * lanes * 4
106
+
107
+
108
+ def _reduce_broadcast(grad: torch.Tensor, like: torch.Tensor) -> torch.Tensor:
109
+ if like.shape[1] == 1 and grad.shape[1] != 1:
110
+ return grad.sum(dim=1, keepdim=True)
111
+ return grad
112
+
113
+
114
+ def _register_fakes() -> None:
115
+ """Fake (meta) kernels so torch.compile can trace through the ops."""
116
+ if not _HAS_OPS:
117
+ return
118
+ try:
119
+ namespace = ops.adarms_fwd.default.name().split("::")[0]
120
+
121
+ def _rstd_like(x):
122
+ return x.new_empty((x.shape[0] * x.shape[1],), dtype=torch.float32)
123
+
124
+ @torch.library.register_fake(f"{namespace}::adarms_fwd")
125
+ def _(x, scale, shift, weight, eps):
126
+ return torch.empty_like(x), _rstd_like(x)
127
+
128
+ @torch.library.register_fake(f"{namespace}::adarms_bwd")
129
+ def _(dy, x, scale, weight, rstd):
130
+ dmod = (
131
+ torch.empty_like(x)
132
+ if scale is not None
133
+ else x.new_empty((x.shape[-1],), dtype=torch.float32)
134
+ )
135
+ return torch.empty_like(x), dmod
136
+
137
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_fwd")
138
+ def _(x, h, gate, scale, shift, weight, eps):
139
+ return torch.empty_like(x), torch.empty_like(x), _rstd_like(x)
140
+
141
+ @torch.library.register_fake(f"{namespace}::resgate_adarms_bwd")
142
+ def _(dy, dyr, r, h, gate, scale, weight, rstd):
143
+ dg = torch.empty_like(r) if gate is not None else r.new_empty((0,))
144
+ dmod = (
145
+ torch.empty_like(r)
146
+ if scale is not None
147
+ else r.new_empty((r.shape[-1],), dtype=torch.float32)
148
+ )
149
+ return torch.empty_like(r), torch.empty_like(r), dg, dmod
150
+ except Exception:
151
+ pass # fakes are a compile nicety, never a functional requirement
152
+
153
+
154
+ _register_fakes()
155
+
156
+
157
+ class _AdaRMSFn(torch.autograd.Function):
158
+ @staticmethod
159
+ def forward(ctx, x, scale, shift, weight, eps):
160
+ y, rstd = ops.adarms_fwd(x, scale, shift, weight, float(eps))
161
+ ctx.save_for_backward(x, scale, weight, rstd)
162
+ return y
163
+
164
+ @staticmethod
165
+ def backward(ctx, dy):
166
+ x, scale, weight, rstd = ctx.saved_tensors
167
+ dy = dy.contiguous()
168
+ dx, dmod = ops.adarms_bwd(dy, x, scale, weight, rstd)
169
+ if scale is not None:
170
+ dscale = _reduce_broadcast(dmod, scale)
171
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
172
+ return dx, dscale, dshift, None, None
173
+ return dx, None, None, dmod, None
174
+
175
+
176
+ class _ResGateAdaRMSFn(torch.autograd.Function):
177
+ @staticmethod
178
+ def forward(ctx, x, h, gate_in, scale, shift, weight, eps):
179
+ r, y, rstd = ops.resgate_adarms_fwd(
180
+ x, h, gate_in, scale, shift, weight, float(eps)
181
+ )
182
+ ctx.save_for_backward(h, gate_in, scale, weight, r, rstd)
183
+ return r, y
184
+
185
+ @staticmethod
186
+ def backward(ctx, dr_out, dy):
187
+ h, gate_in, scale, weight, r, rstd = ctx.saved_tensors
188
+ dy = dy.contiguous()
189
+ dyr = dr_out.contiguous() if dr_out is not None else None
190
+ dr, dh, dg, dmod = ops.resgate_adarms_bwd(
191
+ dy, dyr, r, h, gate_in, scale, weight, rstd
192
+ )
193
+ dg_out = dg if gate_in is not None else None
194
+ if scale is not None:
195
+ dscale = _reduce_broadcast(dmod, scale)
196
+ dshift = _reduce_broadcast(dy, scale).to(scale.dtype)
197
+ return dr, dh, dg_out, dscale, dshift, None, None
198
+ return dr, dh, dg_out, None, None, dmod, None
199
+
200
+
201
+ def _split_modulation(x: torch.Tensor, modulation: torch.Tensor):
202
+ modulation = _broadcast_modulation(x, modulation)
203
+ scale, shift, gate = modulation.chunk(3, dim=-1)
204
+ return scale, shift, gate
205
+
206
+
207
+ def adarms(
208
+ x: torch.Tensor,
209
+ modulation_or_weight: torch.Tensor,
210
+ eps: float = 1e-6,
211
+ adaptive: bool = True,
212
+ ):
213
+ """AdaRMS with fused CUDA fwd+bwd. Returns (y, gate_or_None)."""
214
+ if not _kernel_supported(x, modulation_or_weight):
215
+ y, gate, _ = reference_adarms(x, modulation_or_weight, eps, adaptive)
216
+ return y, gate
217
+ if adaptive:
218
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
219
+ y = _AdaRMSFn.apply(x.contiguous(), scale, shift, None, eps)
220
+ return y, gate.to(x.dtype)
221
+ y = _AdaRMSFn.apply(x.contiguous(), None, None, modulation_or_weight, eps)
222
+ return y, None
223
+
224
+
225
+ def resgate_adarms(
226
+ x: torch.Tensor,
227
+ h: torch.Tensor,
228
+ gate_in: torch.Tensor | None,
229
+ modulation_or_weight: torch.Tensor,
230
+ eps: float = 1e-6,
231
+ adaptive: bool = True,
232
+ ):
233
+ """r = x + h * gate_in, then AdaRMS(r). Returns (r, y, gate_or_None)."""
234
+ if not _kernel_supported(x, modulation_or_weight):
235
+ r, y, gate, _ = reference_resgate_adarms(
236
+ x, h, gate_in, modulation_or_weight, eps, adaptive
237
+ )
238
+ return r, y, gate
239
+ gate_in_c = None
240
+ if gate_in is not None:
241
+ gate_in_c = gate_in.to(x.dtype)
242
+ gate_in_c = (
243
+ gate_in_c.expand_as(x).contiguous()
244
+ if gate_in_c.shape != x.shape
245
+ else gate_in_c.contiguous()
246
+ )
247
+ if adaptive:
248
+ scale, shift, gate = _split_modulation(x, modulation_or_weight)
249
+ r, y = _ResGateAdaRMSFn.apply(
250
+ x.contiguous(), h.contiguous(), gate_in_c, scale, shift, None, eps
251
+ )
252
+ return r, y, gate.to(x.dtype)
253
+ r, y = _ResGateAdaRMSFn.apply(
254
+ x.contiguous(), h.contiguous(), gate_in_c, None, None, modulation_or_weight, eps
255
+ )
256
+ return r, y, None
257
+
258
+
259
+ def adarms_forward(
260
+ x: torch.Tensor,
261
+ modulation_or_weight: torch.Tensor,
262
+ eps: float = 1e-6,
263
+ adaptive: bool = True,
264
+ ):
265
+ """Compatibility wrapper returning (y, gate_or_None, rstd)."""
266
+ y, gate = adarms(x, modulation_or_weight, eps, adaptive)
267
+ _, _, rstd = reference_adarms(x, modulation_or_weight, eps, adaptive)
268
+ return y, gate, rstd
269
+
270
+
271
+ def resgate_adarms_forward(
272
+ x: torch.Tensor,
273
+ h: torch.Tensor,
274
+ gate_in: torch.Tensor | None,
275
+ modulation_or_weight: torch.Tensor,
276
+ eps: float = 1e-6,
277
+ adaptive: bool = True,
278
+ ):
279
+ """Compatibility wrapper returning (r, y, gate_or_None, rstd)."""
280
+ r, y, gate = resgate_adarms(x, h, gate_in, modulation_or_weight, eps, adaptive)
281
+ _, _, _, rstd = reference_resgate_adarms(
282
+ x, h, gate_in, modulation_or_weight, eps, adaptive
283
+ )
284
+ return r, y, gate, rstd
285
+
286
+
287
+ class FlashRTAdaRMSNorm(nn.Module):
288
+ """Drop-in for pi-style adaptive RMSNorm: forward(x, cond) -> (y, gate)."""
289
+
290
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
291
+ super().__init__()
292
+ self.dim = int(dim)
293
+ self.eps = float(eps)
294
+ self.cond_dim = cond_dim
295
+ if cond_dim is None:
296
+ self.weight = nn.Parameter(torch.zeros(dim))
297
+ self.dense = None
298
+ else:
299
+ self.weight = None
300
+ self.dense = nn.Linear(cond_dim, dim * 3, bias=True)
301
+ nn.init.zeros_(self.dense.weight)
302
+
303
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
304
+ if self.dense is None:
305
+ return adarms(x, self.weight, self.eps, adaptive=False)
306
+ if cond is None:
307
+ raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
308
+ return adarms(x, self.dense(cond), self.eps, adaptive=True)
309
+
310
+
311
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
312
+ if ops is None:
313
+ return x
314
+ return ops._flashrt_training_package_marker(x)
315
+
316
+
317
+ __all__ = [
318
+ "adarms",
319
+ "resgate_adarms",
320
+ "adarms_forward",
321
+ "resgate_adarms_forward",
322
+ "reference_adarms",
323
+ "reference_resgate_adarms",
324
+ "FlashRTAdaRMSNorm",
325
+ "backend_marker",
326
+ ]
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_adarms_train_cuda_cc234e4.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:973507666b4c3c108d6f2590ef1fad193018fb9ad6f6df85e9720f8b3b9702fa
3
+ size 19284512
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_cc234e4
3
+ ops = torch.ops._flashrt_adarms_train_cuda_cc234e4
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_cc234e4::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_cc234e4",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }