TheFireHacker's picture
Upload folder using huggingface_hub
27cd743 verified
Raw History Blame Contribute Delete
878 Bytes
import torch
import torch.nn as nn
class ZeroCenteredRMSNorm(nn.Module):
"""RMSNorm with effective weight (1 + w), w initialized to zero.
Weight-decaying w pulls it toward 0, i.e. toward the identity transform
(effective weight 1), rather than toward zero gain as decaying a
conventional RMSNorm weight directly would. w is therefore intentionally
included in the weight-decayed AdamW group, unlike the usual
no-decay-for-norms convention.
"""
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.w = nn.Parameter(torch.zeros(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
dtype = x.dtype
x = x.float()
rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
x = x * rms
return (x * (1.0 + self.w.float())).to(dtype)