File size: 6,908 Bytes
c0bce59 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 | """Optimizer, learning-rate schedule and EMA of weights."""
from __future__ import annotations
import math
from collections.abc import Collection, Iterator
from contextlib import contextmanager
import torch
from torch import Tensor, nn
from torch.optim.lr_scheduler import LambdaLR
from nida.train.config import TrainConfig
__all__ = ["EMA", "build_optimizer", "build_scheduler", "lr_factor"]
def lr_factor(step: int, *, warmup_steps: int, total_steps: int, min_lr_ratio: float) -> float:
"""Multiplier of the peak learning rate for the optimizer step number ``step`` (0-based).
Linear warmup ``(step + 1) / warmup_steps`` for ``step < warmup_steps`` (so the very
first step already has a non-zero rate and step ``warmup_steps - 1`` reaches 1.0), then
cosine decay from 1.0 at ``step == warmup_steps`` to ``min_lr_ratio`` at
``step == total_steps``; constant ``min_lr_ratio`` afterwards.
"""
if step < 0:
raise ValueError("step must be >= 0")
if step < warmup_steps:
return (step + 1) / warmup_steps
decay_steps = total_steps - warmup_steps
progress = 1.0 if decay_steps <= 0 else min(1.0, (step - warmup_steps) / decay_steps)
cosine = 0.5 * (1.0 + math.cos(math.pi * progress))
return min_lr_ratio + (1.0 - min_lr_ratio) * cosine
def build_optimizer(model: nn.Module, cfg: TrainConfig) -> torch.optim.AdamW:
"""AdamW with weight decay on weight matrices (``ndim >= 2``) only.
Biases, norm scales and embedding tables are exempt (decaying a lookup table shrinks
the rows of rare tokens/classes towards zero between their rare updates).
"""
embedding_ids = {
id(module.weight) for module in model.modules() if isinstance(module, nn.Embedding)
}
decay: list[nn.Parameter] = []
no_decay: list[nn.Parameter] = []
for _, param in model.named_parameters():
if param.requires_grad:
exempt = param.ndim < 2 or id(param) in embedding_ids
(no_decay if exempt else decay).append(param)
o = cfg.optim
groups = [
{"params": decay, "weight_decay": o.weight_decay},
{"params": no_decay, "weight_decay": 0.0},
]
return torch.optim.AdamW(groups, lr=o.lr, betas=o.betas, eps=o.eps)
def build_scheduler(optimizer: torch.optim.Optimizer, cfg: TrainConfig) -> LambdaLR:
"""``LambdaLR`` over :func:`lr_factor`; call ``scheduler.step()`` once per optimizer step."""
warmup, total, ratio = cfg.schedule.warmup_steps, cfg.steps, cfg.schedule.min_lr_ratio
def factor(step: int) -> float:
return lr_factor(step, warmup_steps=warmup, total_steps=total, min_lr_ratio=ratio)
return LambdaLR(optimizer, factor)
class EMA:
"""Exponential moving average of a model's weights.
``shadow = decay_n * shadow + (1 - decay_n) * param`` after every optimizer step, where
``decay_n = decay`` or, with ``warmup``, ``min(decay, (1 + n) / (10 + n))`` for the
``n``-th update (0-based) -- a short memory early on so the average is useful before
``1 / (1 - decay)`` steps have passed. Buffers are copied, not averaged. The shadow is
stored in float32 on the model's device.
"""
def __init__(self, model: nn.Module, *, decay: float, warmup: bool) -> None:
if not 0.0 <= decay < 1.0:
raise ValueError("decay must be in [0, 1)")
self.decay = decay
self.warmup = warmup
self.num_updates = 0
self.shadow: dict[str, Tensor] = {
name: tensor.detach().clone().float() for name, tensor in _state(model).items()
}
def current_decay(self) -> float:
"""Decay that the *next* :meth:`update` will use."""
if not self.warmup:
return self.decay
n = self.num_updates
return min(self.decay, (1.0 + n) / (10.0 + n))
@torch.no_grad()
def update(self, model: nn.Module) -> None:
decay = self.current_decay()
params = dict(model.named_parameters())
for name, tensor in _state(model).items():
shadow = self.shadow[name]
if name in params:
shadow.mul_(decay).add_(tensor.detach().float(), alpha=1.0 - decay)
else:
shadow.copy_(tensor.detach().float())
self.num_updates += 1
@torch.no_grad()
def copy_to(self, model: nn.Module) -> None:
"""Overwrite ``model``'s weights with the averaged ones."""
for name, tensor in _state(model).items():
tensor.copy_(self.shadow[name].to(tensor.dtype))
@contextmanager
def swapped_in(self, model: nn.Module) -> Iterator[None]:
"""Temporarily load the EMA weights into ``model`` (restored on exit, also on error)."""
backup = {name: t.detach().clone() for name, t in _state(model).items()}
self.copy_to(model)
try:
yield
finally:
with torch.no_grad():
for name, tensor in _state(model).items():
tensor.copy_(backup[name])
def state_dict(self) -> dict[str, object]:
return {
"decay": self.decay,
"warmup": self.warmup,
"num_updates": self.num_updates,
"shadow": {k: v.clone() for k, v in self.shadow.items()},
}
def load_state_dict(
self, state: dict[str, object], *, allow_missing: Collection[str] = ()
) -> None:
"""Load ``state``; its names must be exactly this EMA's names.
``allow_missing`` names tensors that ``state`` must NOT contain (a fine-tune that
adds parameters, :meth:`nida.train.loop.Trainer.init_from`): they keep their current
shadow value. The names missing from ``state`` have to be exactly this set.
Raises:
ValueError: names, shapes or ``num_updates`` do not match.
"""
shadow = state["shadow"]
allowed = set(allow_missing)
if not allowed <= set(self.shadow):
raise ValueError(
f"allow_missing names unknown to the EMA: {sorted(allowed - set(self.shadow))}"
)
if not isinstance(shadow, dict) or set(shadow) != set(self.shadow) - allowed:
raise ValueError("EMA state does not match the model's parameter names")
for name, tensor in shadow.items():
if tensor.shape != self.shadow[name].shape:
raise ValueError(f"EMA tensor {name!r} has shape {tuple(tensor.shape)}")
self.shadow[name].copy_(tensor)
num_updates = state["num_updates"]
if not isinstance(num_updates, int):
raise ValueError("EMA num_updates must be an int")
self.num_updates = num_updates
def _state(model: nn.Module) -> dict[str, Tensor]:
"""Parameters and buffers by name, in a deterministic order."""
out: dict[str, Tensor] = dict(model.named_parameters())
out.update(model.named_buffers())
return out
|