Download src/model/norm.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 878 Bytes
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/norm.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/model/norm.py
-
curl -L -o norm.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/norm.py
878 Bytes
| import torch | |
| import torch.nn as nn | |
| class ZeroCenteredRMSNorm(nn.Module): | |
| """RMSNorm with effective weight (1 + w), w initialized to zero. | |
| Weight-decaying w pulls it toward 0, i.e. toward the identity transform | |
| (effective weight 1), rather than toward zero gain as decaying a | |
| conventional RMSNorm weight directly would. w is therefore intentionally | |
| included in the weight-decayed AdamW group, unlike the usual | |
| no-decay-for-norms convention. | |
| """ | |
| def __init__(self, dim: int, eps: float = 1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.w = nn.Parameter(torch.zeros(dim)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| dtype = x.dtype | |
| x = x.float() | |
| rms = torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) | |
| x = x * rms | |
| return (x * (1.0 + self.w.float())).to(dtype) | |