liangsu9988 commited on
Commit
b83e11b
·
verified ·
1 Parent(s): 2ca1124

Uploaded using `kernel-builder`.

Browse files
build/torch211-cxx11-cu128-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training reference API."""
2
+ from __future__ import annotations
3
+ import torch
4
+ from torch import nn
5
+ try:
6
+ from ._ops import ops
7
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
8
+ class _SourceOpsFallback:
9
+ def _flashrt_training_package_marker(self, x):
10
+ return x
11
+ ops = _SourceOpsFallback()
12
+
13
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
14
+ if x.dim() == 3 and modulation.dim() == 2:
15
+ return modulation.unsqueeze(1)
16
+ return modulation
17
+
18
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
19
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
20
+
21
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
22
+ compute_dtype = _compute_dtype(x)
23
+ xf = x.to(compute_dtype)
24
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
25
+ return xf * rstd, rstd.squeeze(-1)
26
+
27
+ def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
28
+ normed, rstd = _rms_norm(x, eps)
29
+ if adaptive:
30
+ modulation = _broadcast_modulation(x, modulation_or_weight)
31
+ scale, shift, gate = modulation.chunk(3, dim=-1)
32
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
33
+ return y.to(x.dtype), gate.to(x.dtype), rstd
34
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
35
+ return y.to(x.dtype), None, rstd
36
+
37
+ def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
38
+ residual = x + h if gate_in is None else x + h * gate_in
39
+ y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
40
+ return residual, y, gate, rstd
41
+
42
+ class FlashRTAdaRMSNorm(nn.Module):
43
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
44
+ super().__init__()
45
+ self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
46
+ if cond_dim is None:
47
+ self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
48
+ else:
49
+ self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
50
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
51
+ if self.dense is None:
52
+ return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
53
+ if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
54
+ return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
55
+
56
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
57
+ return ops._flashrt_training_package_marker(x)
58
+ __all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
build/torch211-cxx11-cu128-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5826ef955ea199558153ccb986f421c44fba58f383e988bcd30909e4070ae71
3
+ size 52848
build/torch211-cxx11-cu128-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_eefd4b8
3
+ ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
build/torch211-cxx11-cu128-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu128-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_eefd4b8",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "10.1",
12
+ "12.0+PTX",
13
+ "7.0",
14
+ "7.2",
15
+ "7.5",
16
+ "8.0",
17
+ "8.6",
18
+ "8.7",
19
+ "8.9",
20
+ "9.0"
21
+ ]
22
+ }
23
+ }
build/torch211-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training reference API."""
2
+ from __future__ import annotations
3
+ import torch
4
+ from torch import nn
5
+ try:
6
+ from ._ops import ops
7
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
8
+ class _SourceOpsFallback:
9
+ def _flashrt_training_package_marker(self, x):
10
+ return x
11
+ ops = _SourceOpsFallback()
12
+
13
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
14
+ if x.dim() == 3 and modulation.dim() == 2:
15
+ return modulation.unsqueeze(1)
16
+ return modulation
17
+
18
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
19
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
20
+
21
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
22
+ compute_dtype = _compute_dtype(x)
23
+ xf = x.to(compute_dtype)
24
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
25
+ return xf * rstd, rstd.squeeze(-1)
26
+
27
+ def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
28
+ normed, rstd = _rms_norm(x, eps)
29
+ if adaptive:
30
+ modulation = _broadcast_modulation(x, modulation_or_weight)
31
+ scale, shift, gate = modulation.chunk(3, dim=-1)
32
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
33
+ return y.to(x.dtype), gate.to(x.dtype), rstd
34
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
35
+ return y.to(x.dtype), None, rstd
36
+
37
+ def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
38
+ residual = x + h if gate_in is None else x + h * gate_in
39
+ y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
40
+ return residual, y, gate, rstd
41
+
42
+ class FlashRTAdaRMSNorm(nn.Module):
43
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
44
+ super().__init__()
45
+ self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
46
+ if cond_dim is None:
47
+ self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
48
+ else:
49
+ self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
50
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
51
+ if self.dense is None:
52
+ return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
53
+ if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
54
+ return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
55
+
56
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
57
+ return ops._flashrt_training_package_marker(x)
58
+ __all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
build/torch211-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfe6c08bec48cf279528df8db08ad36aea67ddfe69a1669bea699471c3dde8ce
3
+ size 56944
build/torch211-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_eefd4b8
3
+ ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
build/torch211-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch211-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_eefd4b8",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu130-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training reference API."""
2
+ from __future__ import annotations
3
+ import torch
4
+ from torch import nn
5
+ try:
6
+ from ._ops import ops
7
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
8
+ class _SourceOpsFallback:
9
+ def _flashrt_training_package_marker(self, x):
10
+ return x
11
+ ops = _SourceOpsFallback()
12
+
13
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
14
+ if x.dim() == 3 and modulation.dim() == 2:
15
+ return modulation.unsqueeze(1)
16
+ return modulation
17
+
18
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
19
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
20
+
21
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
22
+ compute_dtype = _compute_dtype(x)
23
+ xf = x.to(compute_dtype)
24
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
25
+ return xf * rstd, rstd.squeeze(-1)
26
+
27
+ def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
28
+ normed, rstd = _rms_norm(x, eps)
29
+ if adaptive:
30
+ modulation = _broadcast_modulation(x, modulation_or_weight)
31
+ scale, shift, gate = modulation.chunk(3, dim=-1)
32
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
33
+ return y.to(x.dtype), gate.to(x.dtype), rstd
34
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
35
+ return y.to(x.dtype), None, rstd
36
+
37
+ def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
38
+ residual = x + h if gate_in is None else x + h * gate_in
39
+ y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
40
+ return residual, y, gate, rstd
41
+
42
+ class FlashRTAdaRMSNorm(nn.Module):
43
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
44
+ super().__init__()
45
+ self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
46
+ if cond_dim is None:
47
+ self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
48
+ else:
49
+ self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
50
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
51
+ if self.dense is None:
52
+ return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
53
+ if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
54
+ return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
55
+
56
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
57
+ return ops._flashrt_training_package_marker(x)
58
+ __all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
build/torch212-cxx11-cu130-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:955cb74cecb9574f3b3c0cd91dcc1c9be9353e4a2e97ec11db96c41f20066e6c
3
+ size 56864
build/torch212-cxx11-cu130-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_eefd4b8
3
+ ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
build/torch212-cxx11-cu130-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu130-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_eefd4b8",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }
build/torch212-cxx11-cu132-x86_64-linux/__init__.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FlashRT AdaRMS training reference API."""
2
+ from __future__ import annotations
3
+ import torch
4
+ from torch import nn
5
+ try:
6
+ from ._ops import ops
7
+ except Exception: # source-tree tests before kernel-builder creates _ops.py
8
+ class _SourceOpsFallback:
9
+ def _flashrt_training_package_marker(self, x):
10
+ return x
11
+ ops = _SourceOpsFallback()
12
+
13
+ def _broadcast_modulation(x: torch.Tensor, modulation: torch.Tensor) -> torch.Tensor:
14
+ if x.dim() == 3 and modulation.dim() == 2:
15
+ return modulation.unsqueeze(1)
16
+ return modulation
17
+
18
+ def _compute_dtype(x: torch.Tensor) -> torch.dtype:
19
+ return torch.float64 if x.dtype == torch.float64 else torch.float32
20
+
21
+ def _rms_norm(x: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor]:
22
+ compute_dtype = _compute_dtype(x)
23
+ xf = x.to(compute_dtype)
24
+ rstd = torch.rsqrt(torch.mean(xf * xf, dim=-1, keepdim=True) + float(eps))
25
+ return xf * rstd, rstd.squeeze(-1)
26
+
27
+ def adarms_forward(x: torch.Tensor, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
28
+ normed, rstd = _rms_norm(x, eps)
29
+ if adaptive:
30
+ modulation = _broadcast_modulation(x, modulation_or_weight)
31
+ scale, shift, gate = modulation.chunk(3, dim=-1)
32
+ y = normed * (1.0 + scale.to(normed.dtype)) + shift.to(normed.dtype)
33
+ return y.to(x.dtype), gate.to(x.dtype), rstd
34
+ y = normed * (1.0 + modulation_or_weight.to(normed.dtype))
35
+ return y.to(x.dtype), None, rstd
36
+
37
+ def resgate_adarms_forward(x: torch.Tensor, h: torch.Tensor, gate_in: torch.Tensor | None, modulation_or_weight: torch.Tensor, eps: float = 1e-6, adaptive: bool = True):
38
+ residual = x + h if gate_in is None else x + h * gate_in
39
+ y, gate, rstd = adarms_forward(residual, modulation_or_weight, eps, adaptive)
40
+ return residual, y, gate, rstd
41
+
42
+ class FlashRTAdaRMSNorm(nn.Module):
43
+ def __init__(self, dim: int, eps: float = 1e-6, cond_dim: int | None = None) -> None:
44
+ super().__init__()
45
+ self.dim = int(dim); self.eps = float(eps); self.cond_dim = cond_dim
46
+ if cond_dim is None:
47
+ self.weight = nn.Parameter(torch.zeros(dim)); self.dense = None
48
+ else:
49
+ self.weight = None; self.dense = nn.Linear(cond_dim, dim * 3, bias=True); nn.init.zeros_(self.dense.weight)
50
+ def forward(self, x: torch.Tensor, cond: torch.Tensor | None = None):
51
+ if self.dense is None:
52
+ return adarms_forward(x, self.weight, self.eps, adaptive=False)[:2]
53
+ if cond is None: raise ValueError("cond is required for adaptive FlashRTAdaRMSNorm")
54
+ return adarms_forward(x, self.dense(cond), self.eps, adaptive=True)[:2]
55
+
56
+ def backend_marker(x: torch.Tensor) -> torch.Tensor:
57
+ return ops._flashrt_training_package_marker(x)
58
+ __all__ = ["adarms_forward", "resgate_adarms_forward", "FlashRTAdaRMSNorm", "backend_marker"]
build/torch212-cxx11-cu132-x86_64-linux/_flashrt_adarms_train_cuda_eefd4b8.abi3.so ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f08cfb266fc45cbb36022599d184225fc7ca0a4afd006cffa099fac2a0e8868c
3
+ size 56864
build/torch212-cxx11-cu132-x86_64-linux/_ops.py ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ from . import _flashrt_adarms_train_cuda_eefd4b8
3
+ ops = torch.ops._flashrt_adarms_train_cuda_eefd4b8
4
+
5
+ def add_op_namespace_prefix(op_name: str):
6
+ """
7
+ Prefix op by namespace.
8
+ """
9
+ return f"_flashrt_adarms_train_cuda_eefd4b8::{op_name}"
build/torch212-cxx11-cu132-x86_64-linux/flashrt_adarms_train/__init__.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ctypes
2
+ import importlib.util
3
+ import sys
4
+ from pathlib import Path
5
+ from types import ModuleType
6
+
7
+
8
+ def _import_from_path(file_path: Path) -> ModuleType:
9
+ # We cannot use the module name as-is, after adding it to `sys.modules`,
10
+ # it would also be used for other imports. So, we make a module name that
11
+ # depends on the path for it to be unique using the hex-encoded hash of
12
+ # the path.
13
+ path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
14
+ module_name = path_hash
15
+ spec = importlib.util.spec_from_file_location(module_name, file_path)
16
+ if spec is None:
17
+ raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
18
+ module = importlib.util.module_from_spec(spec)
19
+ if module is None:
20
+ raise ImportError(f"Cannot load module {module_name} from spec")
21
+ sys.modules[module_name] = module
22
+ spec.loader.exec_module(module) # type: ignore
23
+ return module
24
+
25
+
26
+ globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))
build/torch212-cxx11-cu132-x86_64-linux/metadata.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "name": "flashrt-adarms-train",
3
+ "id": "_flashrt_adarms_train_cuda_eefd4b8",
4
+ "version": 1,
5
+ "license": "Apache-2.0",
6
+ "python-depends": [],
7
+ "backend": {
8
+ "type": "cuda",
9
+ "archs": [
10
+ "10.0",
11
+ "11.0",
12
+ "12.0",
13
+ "12.1+PTX",
14
+ "7.5",
15
+ "8.0",
16
+ "8.6",
17
+ "8.7",
18
+ "8.9",
19
+ "9.0"
20
+ ]
21
+ }
22
+ }