Uploaded using `kernel-builder`.
Browse files- build/torch211-cxx11-cu128-x86_64-linux/__init__.py +58 -0
- build/torch211-cxx11-cu128-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so +3 -0
- build/torch211-cxx11-cu128-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu128-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch211-cxx11-cu128-x86_64-linux/metadata.json +23 -0
- build/torch211-cxx11-cu130-x86_64-linux/__init__.py +58 -0
- build/torch211-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so +3 -0
- build/torch211-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch211-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch211-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu130-x86_64-linux/__init__.py +58 -0
- build/torch212-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so +3 -0
- build/torch212-cxx11-cu130-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch212-cxx11-cu130-x86_64-linux/metadata.json +22 -0
- build/torch212-cxx11-cu132-x86_64-linux/__init__.py +58 -0
- build/torch212-cxx11-cu132-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so +3 -0
- build/torch212-cxx11-cu132-x86_64-linux/_ops.py +9 -0
- build/torch212-cxx11-cu132-x86_64-linux/flashrt_adarms_train/__init__.py +26 -0
- build/torch212-cxx11-cu132-x86_64-linux/metadata.json +22 -0
build/torch211-cxx11-cu128-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training reference API."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
try:
|
| 6 |
+
from ._ops import ops
|
| 7 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 8 |
+
class _SourceOpsFallback:
|
| 9 |
+
def _flashrt_training_package_marker(self, x):
|
| 10 |
+
return x
|
| 11 |
+
ops = _SourceOpsFallback()
|
| 12 |
+
|
| 13 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 14 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 15 |
+
return modulation.unsqueeze(1)
|
| 16 |
+
return modulation
|
| 17 |
+
|
| 18 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 19 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 20 |
+
|
| 21 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 22 |
+
compute_dtype = _compute_dtype(x)
|
| 23 |
+
xf = x.to(compute_dtype)
|
| 24 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 25 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 26 |
+
|
| 27 |
+
def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 28 |
+
normed, rstd = _rms_norm(x, eps)
|
| 29 |
+
if adaptive:
|
| 30 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 31 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 32 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 33 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 34 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 35 |
+
return y.to(x.dtype), None, rstd
|
| 36 |
+
|
| 37 |
+
def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 38 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 39 |
+
y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
|
| 40 |
+
return residual, y, gate, rstd
|
| 41 |
+
|
| 42 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 43 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
|
| 46 |
+
if cond_dim is None:
|
| 47 |
+
self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
|
| 48 |
+
else:
|
| 49 |
+
self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
|
| 50 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 51 |
+
if self.dense is None:
|
| 52 |
+
return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
|
| 53 |
+
if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 54 |
+
return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
|
| 55 |
+
|
| 56 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 57 |
+
return ops._flashrt_training_package_marker(x)
|
| 58 |
+
__all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
|
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f5826ef955ea199558153ccb986f421c44fba58f383e988bcd30909e4070ae71
|
| 3 |
+
size 52848
|
build/torch211-cxx11-cu128-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_eefd4b8
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
|
build/torch211-cxx11-cu128-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu128-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_eefd4b8",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"10.1",
|
| 12 |
+
"12.0+PTX",
|
| 13 |
+
"7.0",
|
| 14 |
+
"7.2",
|
| 15 |
+
"7.5",
|
| 16 |
+
"8.0",
|
| 17 |
+
"8.6",
|
| 18 |
+
"8.7",
|
| 19 |
+
"8.9",
|
| 20 |
+
"9.0"
|
| 21 |
+
]
|
| 22 |
+
}
|
| 23 |
+
}
|
build/torch211-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training reference API."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
try:
|
| 6 |
+
from ._ops import ops
|
| 7 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 8 |
+
class _SourceOpsFallback:
|
| 9 |
+
def _flashrt_training_package_marker(self, x):
|
| 10 |
+
return x
|
| 11 |
+
ops = _SourceOpsFallback()
|
| 12 |
+
|
| 13 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 14 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 15 |
+
return modulation.unsqueeze(1)
|
| 16 |
+
return modulation
|
| 17 |
+
|
| 18 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 19 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 20 |
+
|
| 21 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 22 |
+
compute_dtype = _compute_dtype(x)
|
| 23 |
+
xf = x.to(compute_dtype)
|
| 24 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 25 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 26 |
+
|
| 27 |
+
def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 28 |
+
normed, rstd = _rms_norm(x, eps)
|
| 29 |
+
if adaptive:
|
| 30 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 31 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 32 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 33 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 34 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 35 |
+
return y.to(x.dtype), None, rstd
|
| 36 |
+
|
| 37 |
+
def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 38 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 39 |
+
y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
|
| 40 |
+
return residual, y, gate, rstd
|
| 41 |
+
|
| 42 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 43 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
|
| 46 |
+
if cond_dim is None:
|
| 47 |
+
self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
|
| 48 |
+
else:
|
| 49 |
+
self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
|
| 50 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 51 |
+
if self.dense is None:
|
| 52 |
+
return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
|
| 53 |
+
if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 54 |
+
return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
|
| 55 |
+
|
| 56 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 57 |
+
return ops._flashrt_training_package_marker(x)
|
| 58 |
+
__all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
|
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bfe6c08bec48cf279528df8db08ad36aea67ddfe69a1669bea699471c3dde8ce
|
| 3 |
+
size 56944
|
build/torch211-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_eefd4b8
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
|
build/torch211-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch211-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_eefd4b8",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu130-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training reference API."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
try:
|
| 6 |
+
from ._ops import ops
|
| 7 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 8 |
+
class _SourceOpsFallback:
|
| 9 |
+
def _flashrt_training_package_marker(self, x):
|
| 10 |
+
return x
|
| 11 |
+
ops = _SourceOpsFallback()
|
| 12 |
+
|
| 13 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 14 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 15 |
+
return modulation.unsqueeze(1)
|
| 16 |
+
return modulation
|
| 17 |
+
|
| 18 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 19 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 20 |
+
|
| 21 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 22 |
+
compute_dtype = _compute_dtype(x)
|
| 23 |
+
xf = x.to(compute_dtype)
|
| 24 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 25 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 26 |
+
|
| 27 |
+
def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 28 |
+
normed, rstd = _rms_norm(x, eps)
|
| 29 |
+
if adaptive:
|
| 30 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 31 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 32 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 33 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 34 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 35 |
+
return y.to(x.dtype), None, rstd
|
| 36 |
+
|
| 37 |
+
def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 38 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 39 |
+
y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
|
| 40 |
+
return residual, y, gate, rstd
|
| 41 |
+
|
| 42 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 43 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
|
| 46 |
+
if cond_dim is None:
|
| 47 |
+
self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
|
| 48 |
+
else:
|
| 49 |
+
self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
|
| 50 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 51 |
+
if self.dense is None:
|
| 52 |
+
return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
|
| 53 |
+
if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 54 |
+
return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
|
| 55 |
+
|
| 56 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 57 |
+
return ops._flashrt_training_package_marker(x)
|
| 58 |
+
__all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
|
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:955cb74cecb9574f3b3c0cd91dcc1c9be9353e4a2e97ec11db96c41f20066e6c
|
| 3 |
+
size 56864
|
build/torch212-cxx11-cu130-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_eefd4b8
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
|
build/torch212-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu130-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_eefd4b8",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|
build/torch212-cxx11-cu132-x86_64-linux/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""FlashRT AdaRMS training reference API."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import torch
|
| 4 |
+
from torch import nn
|
| 5 |
+
try:
|
| 6 |
+
from ._ops import ops
|
| 7 |
+
except Exception: # source-tree tests before kernel-builder creates _ops.py
|
| 8 |
+
class _SourceOpsFallback:
|
| 9 |
+
def _flashrt_training_package_marker(self, x):
|
| 10 |
+
return x
|
| 11 |
+
ops = _SourceOpsFallback()
|
| 12 |
+
|
| 13 |
+
def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
|
| 14 |
+
if x.dim() == 3 and modulation.dim() == 2:
|
| 15 |
+
return modulation.unsqueeze(1)
|
| 16 |
+
return modulation
|
| 17 |
+
|
| 18 |
+
def _compute_dtype(x: torch.Tensor) -> torch.dtype:
|
| 19 |
+
return torch.float64 if x.dtype == torch.float64 else torch.float32
|
| 20 |
+
|
| 21 |
+
def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
|
| 22 |
+
compute_dtype = _compute_dtype(x)
|
| 23 |
+
xf = x.to(compute_dtype)
|
| 24 |
+
rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
|
| 25 |
+
return xf * rstd, rstd.squeeze(-1)
|
| 26 |
+
|
| 27 |
+
def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 28 |
+
normed, rstd = _rms_norm(x, eps)
|
| 29 |
+
if adaptive:
|
| 30 |
+
modulation = _broadcast_modulation(x, modulation_or_weight)
|
| 31 |
+
scale, shift, gate = modulation.chunk(3, dim=-1)
|
| 32 |
+
y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
|
| 33 |
+
return y.to(x.dtype), gate.to(x.dtype), rstd
|
| 34 |
+
y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
|
| 35 |
+
return y.to(x.dtype), None, rstd
|
| 36 |
+
|
| 37 |
+
def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
|
| 38 |
+
residual = x + h if gate_in is None else x + h * gate_in
|
| 39 |
+
y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
|
| 40 |
+
return residual, y, gate, rstd
|
| 41 |
+
|
| 42 |
+
class FlashRTAdaRMSNorm(nn.Module):
|
| 43 |
+
def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
|
| 44 |
+
super().__init__()
|
| 45 |
+
self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
|
| 46 |
+
if cond_dim is None:
|
| 47 |
+
self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
|
| 48 |
+
else:
|
| 49 |
+
self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
|
| 50 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
|
| 51 |
+
if self.dense is None:
|
| 52 |
+
return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
|
| 53 |
+
if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
|
| 54 |
+
return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
|
| 55 |
+
|
| 56 |
+
def backend_marker(x: torch.Tensor) -> torch.Tensor:
|
| 57 |
+
return ops._flashrt_training_package_marker(x)
|
| 58 |
+
__all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
|
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f08cfb266fc45cbb36022599d184225fc7ca0a4afd006cffa099fac2a0e8868c
|
| 3 |
+
size 56864
|
build/torch212-cxx11-cu132-x86_64-linux/_ops.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import torch
|
| 2 |
+
from . import _flashrt_adarms_train_cuda_eefd4b8
|
| 3 |
+
ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
|
| 4 |
+
|
| 5 |
+
def add_op_namespace_prefix(op_name: str):
|
| 6 |
+
"""
|
| 7 |
+
Prefix op by namespace.
|
| 8 |
+
"""
|
| 9 |
+
return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
|
build/torch212-cxx11-cu132-x86_64-linux/flashrt_adarms_train/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ctypes
|
| 2 |
+
import importlib.util
|
| 3 |
+
import sys
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import ModuleType
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def _import_from_path(file_path: Path) -> ModuleType:
|
| 9 |
+
# We cannot use the module name as-is, after adding it to `sys.modules`,
|
| 10 |
+
# it would also be used for other imports. So, we make a module name that
|
| 11 |
+
# depends on the path for it to be unique using the hex-encoded hash of
|
| 12 |
+
# the path.
|
| 13 |
+
path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
|
| 14 |
+
module_name = path_hash
|
| 15 |
+
spec = importlib.util.spec_from_file_location(module_name, file_path)
|
| 16 |
+
if spec is None:
|
| 17 |
+
raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
|
| 18 |
+
module = importlib.util.module_from_spec(spec)
|
| 19 |
+
if module is None:
|
| 20 |
+
raise ImportError(f"Cannot load module {module_name} from spec")
|
| 21 |
+
sys.modules[module_name] = module
|
| 22 |
+
spec.loader.exec_module(module) # type: ignore
|
| 23 |
+
return module
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
|
build/torch212-cxx11-cu132-x86_64-linux/metadata.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"name": "flashrt-adarms-train",
|
| 3 |
+
"id": "_flashrt_adarms_train_cuda_eefd4b8",
|
| 4 |
+
"version": 1,
|
| 5 |
+
"license": "Apache-2.0",
|
| 6 |
+
"python-depends": [],
|
| 7 |
+
"backend": {
|
| 8 |
+
"type": "cuda",
|
| 9 |
+
"archs": [
|
| 10 |
+
"10.0",
|
| 11 |
+
"11.0",
|
| 12 |
+
"12.0",
|
| 13 |
+
"12.1+PTX",
|
| 14 |
+
"7.5",
|
| 15 |
+
"8.0",
|
| 16 |
+
"8.6",
|
| 17 |
+
"8.7",
|
| 18 |
+
"8.9",
|
| 19 |
+
"9.0"
|
| 20 |
+
]
|
| 21 |
+
}
|
| 22 |
+
}
|