sahoosk's picture
Upload folder using huggingface_hub
5407bdc verified
Raw History Blame Contribute Delete
2.7 kB
"""Category 2 -- variance-reduced AdamW variant: MARS (Yuan et al., 2024).
AdamW's moments are built from raw minibatch gradients, so they carry the full
minibatch noise. MARS applies a STORM-style variance-reduction correction
before the moments are updated: it extrapolates along the difference between
consecutive gradients,
c_t = g_t + gamma * (b1 / (1 - b1)) * (g_t - g_{t-1})
which cancels part of the sampling noise that the two gradients share, since
they differ only by the minibatch draw plus one small parameter step. The
corrected gradient is then clipped to unit norm -- the correction is a
difference of two noisy quantities and can blow up early in training -- and
fed to an otherwise ordinary AdamW.
Optimizer state: 3 tensors per parameter (m, v, previous gradient) -> 3x
model size, the price of the variance reduction.
"""
import torch
from .base import BaseOptimizer
class MARS(BaseOptimizer):
def __init__(self, params, lr=1e-3, betas=(0.95, 0.99), eps=1e-8,
weight_decay=0.1, gamma=0.025, max_norm=1.0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay,
gamma=gamma, max_norm=max_norm)
super().__init__(params, defaults)
@torch.no_grad()
def step(self, closure=None):
loss = self._closure(closure)
for group in self.param_groups:
b1, b2 = group["betas"]
lr, eps, wd = group["lr"], group["eps"], group["weight_decay"]
gamma, max_norm = group["gamma"], group["max_norm"]
for p in group["params"]:
if p.grad is None:
continue
g = p.grad
st = self.state[p]
if not st:
st["step"] = 0
st["m"] = torch.zeros_like(p)
st["v"] = torch.zeros_like(p)
st["g_prev"] = torch.zeros_like(p)
st["step"] += 1
t = st["step"]
m, v, g_prev = st["m"], st["v"], st["g_prev"]
# --- scaled gradient correction (the variance reduction) ---
c = g + gamma * (b1 / (1 - b1)) * (g - g_prev)
nrm = c.norm()
if nrm > max_norm:
c = c * (max_norm / (nrm + 1e-12))
g_prev.copy_(g)
m.mul_(b1).add_(c, alpha=1 - b1)
v.mul_(b2).addcmul_(c, c, value=1 - b2)
m_hat = m / (1 - b1 ** t)
v_hat = v / (1 - b2 ** t)
if wd != 0:
p.mul_(1 - lr * wd)
p.addcdiv_(m_hat, v_hat.sqrt().add_(eps), value=-lr)
return loss