File size: 584 Bytes
1a1fc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | """Rivinity RMSNorm (float32-internal computation for numerical stability)."""
import torch
import torch.nn as nn
class RivinityRMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def _norm(self, x: torch.Tensor) -> torch.Tensor:
return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
def forward(self, x: torch.Tensor) -> torch.Tensor:
output = self._norm(x.float())
return (output.type_as(x)) * self.weight
|