on_the_way / rivinity /normalization.py
RANITBAG's picture
2nd run
1a1fc1f verified
Raw History Blame Contribute Delete
584 Bytes
"""Rivinity RMSNorm (float32-internal computation for numerical stability)."""
import torch
import torch.nn as nn
class RivinityRMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def _norm(self, x: torch.Tensor) -> torch.Tensor:
return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
def forward(self, x: torch.Tensor) -> torch.Tensor:
output = self._norm(x.float())
return (output.type_as(x)) * self.weight