"""Category 2 -- variance-reduced AdamW variant: MARS (Yuan et al., 2024). AdamW's moments are built from raw minibatch gradients, so they carry the full minibatch noise. MARS applies a STORM-style variance-reduction correction before the moments are updated: it extrapolates along the difference between consecutive gradients, c_t = g_t + gamma * (b1 / (1 - b1)) * (g_t - g_{t-1}) which cancels part of the sampling noise that the two gradients share, since they differ only by the minibatch draw plus one small parameter step. The corrected gradient is then clipped to unit norm -- the correction is a difference of two noisy quantities and can blow up early in training -- and fed to an otherwise ordinary AdamW. Optimizer state: 3 tensors per parameter (m, v, previous gradient) -> 3x model size, the price of the variance reduction. """ import torch from .base import BaseOptimizer class MARS(BaseOptimizer): def __init__(self, params, lr=1e-3, betas=(0.95, 0.99), eps=1e-8, weight_decay=0.1, gamma=0.025, max_norm=1.0): defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, gamma=gamma, max_norm=max_norm) super().__init__(params, defaults) @torch.no_grad() def step(self, closure=None): loss = self._closure(closure) for group in self.param_groups: b1, b2 = group["betas"] lr, eps, wd = group["lr"], group["eps"], group["weight_decay"] gamma, max_norm = group["gamma"], group["max_norm"] for p in group["params"]: if p.grad is None: continue g = p.grad st = self.state[p] if not st: st["step"] = 0 st["m"] = torch.zeros_like(p) st["v"] = torch.zeros_like(p) st["g_prev"] = torch.zeros_like(p) st["step"] += 1 t = st["step"] m, v, g_prev = st["m"], st["v"], st["g_prev"] # --- scaled gradient correction (the variance reduction) --- c = g + gamma * (b1 / (1 - b1)) * (g - g_prev) nrm = c.norm() if nrm > max_norm: c = c * (max_norm / (nrm + 1e-12)) g_prev.copy_(g) m.mul_(b1).add_(c, alpha=1 - b1) v.mul_(b2).addcmul_(c, c, value=1 - b2) m_hat = m / (1 - b1 ** t) v_hat = v / (1 - b2 ** t) if wd != 0: p.mul_(1 - lr * wd) p.addcdiv_(m_hat, v_hat.sqrt().add_(eps), value=-lr) return loss