Download rivinity/normalization.py from RANITBAG/on_the_way: direct link, hf CLI and curl.
- Browser
- Download file 584 Bytes
-
https://huggingface.co/RANITBAG/on_the_way/resolve/main/rivinity/normalization.py
- Command line
-
hf download hf://RANITBAG/on_the_way/rivinity/normalization.py
-
curl -L -o normalization.py https://huggingface.co/RANITBAG/on_the_way/resolve/main/rivinity/normalization.py
584 Bytes
| """Rivinity RMSNorm (float32-internal computation for numerical stability).""" | |
| import torch | |
| import torch.nn as nn | |
| class RivinityRMSNorm(nn.Module): | |
| def __init__(self, dim: int, eps: float = 1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def _norm(self, x: torch.Tensor) -> torch.Tensor: | |
| return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| output = self._norm(x.float()) | |
| return (output.type_as(x)) * self.weight | |