"""Category 3 -- memory-efficient: Lion (Chen et al., 2023). Lion keeps *one* state tensor instead of AdamW's two, and its update is the sign of an interpolated momentum: u_t = sign(b1 * m_{t-1} + (1 - b1) * g_t) m_t = b2 * m_{t-1} + (1 - b2) * g_t Two distinct betas is the trick worth noticing: the direction is taken from a *fresher* blend (b1 = 0.9) than the one that is stored (b2 = 0.99), so the step responds to the current gradient while the memory stays long. Because ``sign`` makes every coordinate's step exactly +/- lr, the update norm is fixed and much larger than AdamW's for the same lr -- Lion needs a learning rate roughly 3-10x smaller, and correspondingly larger weight decay, which is why it gets its own sweep grid rather than inheriting AdamW's. Optimizer state: 1 tensor per parameter -> 1x model size, half of AdamW. """ import torch from .base import BaseOptimizer class Lion(BaseOptimizer): def __init__(self, params, lr=1e-4, betas=(0.9, 0.99), weight_decay=0.1): defaults = dict(lr=lr, betas=betas, weight_decay=weight_decay) super().__init__(params, defaults) @torch.no_grad() def step(self, closure=None): loss = self._closure(closure) for group in self.param_groups: b1, b2 = group["betas"] lr, wd = group["lr"], group["weight_decay"] for p in group["params"]: if p.grad is None: continue g = p.grad st = self.state[p] if not st: st["m"] = torch.zeros_like(p) m = st["m"] if wd != 0: p.mul_(1 - lr * wd) update = m.mul(b1).add_(g, alpha=1 - b1).sign_() p.add_(update, alpha=-lr) # The stored momentum uses the *other* beta. m.mul_(b2).add_(g, alpha=1 - b2) return loss