File size: 878 Bytes
27cd743 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | import torch
import torch.nn as nn
class ZeroCenteredRMSNorm(nn.Module):
"""RMSNorm with effective weight (1 + w), w initialized to zero.
Weight-decaying w pulls it toward 0, i.e. toward the identity transform
(effective weight 1), rather than toward zero gain as decaying a
conventional RMSNorm weight directly would. w is therefore intentionally
included in the weight-decayed AdamW group, unlike the usual
no-decay-for-norms convention.
"""
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.w = nn.Parameter(torch.zeros(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
dtype = x.dtype
x = x.float()
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
x = x * rms
return (x * (1.0 + self.w.float())).to(dtype)
|