Download src/part2/optimizers/lion.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 1.93 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/optimizers/lion.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part2/optimizers/lion.py
-
curl -L -o lion.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/optimizers/lion.py
1.93 kB
| """Category 3 -- memory-efficient: Lion (Chen et al., 2023). | |
| Lion keeps *one* state tensor instead of AdamW's two, and its update is the | |
| sign of an interpolated momentum: | |
| u_t = sign(b1 * m_{t-1} + (1 - b1) * g_t) | |
| m_t = b2 * m_{t-1} + (1 - b2) * g_t | |
| Two distinct betas is the trick worth noticing: the direction is taken from a | |
| *fresher* blend (b1 = 0.9) than the one that is stored (b2 = 0.99), so the | |
| step responds to the current gradient while the memory stays long. | |
| Because ``sign`` makes every coordinate's step exactly +/- lr, the update norm | |
| is fixed and much larger than AdamW's for the same lr -- Lion needs a learning | |
| rate roughly 3-10x smaller, and correspondingly larger weight decay, which is | |
| why it gets its own sweep grid rather than inheriting AdamW's. | |
| Optimizer state: 1 tensor per parameter -> 1x model size, half of AdamW. | |
| """ | |
| import torch | |
| from .base import BaseOptimizer | |
| class Lion(BaseOptimizer): | |
| def __init__(self, params, lr=1e-4, betas=(0.9, 0.99), weight_decay=0.1): | |
| defaults = dict(lr=lr, betas=betas, weight_decay=weight_decay) | |
| super().__init__(params, defaults) | |
| def step(self, closure=None): | |
| loss = self._closure(closure) | |
| for group in self.param_groups: | |
| b1, b2 = group["betas"] | |
| lr, wd = group["lr"], group["weight_decay"] | |
| for p in group["params"]: | |
| if p.grad is None: | |
| continue | |
| g = p.grad | |
| st = self.state[p] | |
| if not st: | |
| st["m"] = torch.zeros_like(p) | |
| m = st["m"] | |
| if wd != 0: | |
| p.mul_(1 - lr * wd) | |
| update = m.mul(b1).add_(g, alpha=1 - b1).sign_() | |
| p.add_(update, alpha=-lr) | |
| # The stored momentum uses the *other* beta. | |
| m.mul_(b2).add_(g, alpha=1 - b2) | |
| return loss | |