fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 4)
0d48417 verified
Raw History Blame Contribute Delete
1.89 kB
"""Learning-rate scheduler factory."""
from __future__ import annotations
from typing import Any, Dict, Optional
import torch
def build_scheduler(
cfg: Dict[str, Any],
optimizer: torch.optim.Optimizer,
steps_per_epoch: int,
max_epochs: int,
) -> Optional[torch.optim.lr_scheduler._LRScheduler]:
name = (cfg.get("name") or "none").lower()
params = dict(cfg.get("params", {}))
if name in {"none", "constant", "null"}:
return None
if name == "cosine":
t_max = int(params.pop("t_max", max_epochs))
return torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, t_max), **params)
if name == "step":
step_size = int(params.pop("step_size", 30))
gamma = float(params.pop("gamma", 0.1))
return torch.optim.lr_scheduler.StepLR(optimizer, step_size=max(1, step_size), gamma=gamma, **params)
if name == "exponential":
gamma = float(params.pop("gamma", 0.99))
return torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma, **params)
if name == "plateau":
mode = params.pop("mode", "min")
factor = float(params.pop("factor", 0.5))
patience = int(params.pop("patience", 10))
return torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, mode=mode, factor=factor, patience=patience, **params
)
if name == "onecycle":
max_lr = float(params.pop("max_lr", optimizer.param_groups[0]["lr"]))
total_steps = int(params.pop("total_steps", max_epochs * max(1, steps_per_epoch)))
return torch.optim.lr_scheduler.OneCycleLR(
optimizer, max_lr=max_lr, total_steps=max(1, total_steps), **params
)
raise ValueError(f"Unknown scheduler: {name}")
def is_plateau(scheduler) -> bool:
return scheduler is not None and isinstance(scheduler, torch.optim.lr_scheduler.ReduceLROnPlateau)