Download src/part2/optimizers/mars.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 2.7 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/optimizers/mars.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part2/optimizers/mars.py
-
curl -L -o mars.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/optimizers/mars.py
2.7 kB
| """Category 2 -- variance-reduced AdamW variant: MARS (Yuan et al., 2024). | |
| AdamW's moments are built from raw minibatch gradients, so they carry the full | |
| minibatch noise. MARS applies a STORM-style variance-reduction correction | |
| before the moments are updated: it extrapolates along the difference between | |
| consecutive gradients, | |
| c_t = g_t + gamma * (b1 / (1 - b1)) * (g_t - g_{t-1}) | |
| which cancels part of the sampling noise that the two gradients share, since | |
| they differ only by the minibatch draw plus one small parameter step. The | |
| corrected gradient is then clipped to unit norm -- the correction is a | |
| difference of two noisy quantities and can blow up early in training -- and | |
| fed to an otherwise ordinary AdamW. | |
| Optimizer state: 3 tensors per parameter (m, v, previous gradient) -> 3x | |
| model size, the price of the variance reduction. | |
| """ | |
| import torch | |
| from .base import BaseOptimizer | |
| class MARS(BaseOptimizer): | |
| def __init__(self, params, lr=1e-3, betas=(0.95, 0.99), eps=1e-8, | |
| weight_decay=0.1, gamma=0.025, max_norm=1.0): | |
| defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, | |
| gamma=gamma, max_norm=max_norm) | |
| super().__init__(params, defaults) | |
| def step(self, closure=None): | |
| loss = self._closure(closure) | |
| for group in self.param_groups: | |
| b1, b2 = group["betas"] | |
| lr, eps, wd = group["lr"], group["eps"], group["weight_decay"] | |
| gamma, max_norm = group["gamma"], group["max_norm"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| g = p.grad | |
| st = self.state[p] | |
| if not st: | |
| st["step"] = 0 | |
| st["m"] = torch.zeros_like(p) | |
| st["v"] = torch.zeros_like(p) | |
| st["g_prev"] = torch.zeros_like(p) | |
| st["step"] += 1 | |
| t = st["step"] | |
| m, v, g_prev = st["m"], st["v"], st["g_prev"] | |
| # --- scaled gradient correction (the variance reduction) --- | |
| c = g + gamma * (b1 / (1 - b1)) * (g - g_prev) | |
| nrm = c.norm() | |
| if nrm > max_norm: | |
| c = c * (max_norm / (nrm + 1e-12)) | |
| g_prev.copy_(g) | |
| m.mul_(b1).add_(c, alpha=1 - b1) | |
| v.mul_(b2).addcmul_(c, c, value=1 - b2) | |
| m_hat = m / (1 - b1 ** t) | |
| v_hat = v / (1 - b2 ** t) | |
| if wd != 0: | |
| p.mul_(1 - lr * wd) | |
| p.addcdiv_(m_hat, v_hat.sqrt().add_(eps), value=-lr) | |
| return loss | |