"""A small configurable PyTorch trainer with early stopping and checkpoint selection.""" from __future__ import annotations import contextlib import math import time from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, Optional import numpy as np import pandas as pd import torch from ..evaluation.decision import point_from_output from ..evaluation.metrics import mae, smape_plus_1 from ..utils import save_json from .callbacks import ( CheckpointManager, EarlyStopping, HistoryLogger, TensorBoardLogger, Timer, ) from .optimizers import build_optimizer from .schedulers import build_scheduler, is_plateau @dataclass class TrainResult: history: pd.DataFrame best_metrics: Dict[str, Any] best_path: Optional[str] last_path: Optional[str] epochs_run: int diverged: bool = False def _make_grad_scaler(enabled: bool): try: return torch.amp.GradScaler("cuda", enabled=enabled) except (TypeError, AttributeError): return torch.cuda.amp.GradScaler(enabled=enabled) def _autocast(enabled: bool): if not enabled: return contextlib.nullcontext() try: return torch.amp.autocast("cuda", dtype=torch.float16) except TypeError: return torch.cuda.amp.autocast(dtype=torch.float16) class Trainer: def __init__( self, cfg: Dict[str, Any], model: torch.nn.Module, train_loader, val_loader, device: torch.device, output_dir: str | Path, loss_fn, n_classes: int = 87, target_scaler=None, extra: Optional[Dict[str, Any]] = None, ): self.cfg = cfg self.model = model self.train_loader = train_loader self.val_loader = val_loader self.device = device self.output_dir = Path(output_dir) self.output_dir.mkdir(parents=True, exist_ok=True) self.loss_fn = loss_fn self.n_classes = int(n_classes) self.target_scaler = target_scaler self.extra = extra or {} tcfg = cfg["trainer"] self.max_epochs = int(tcfg.get("max_epochs", 100)) self.amp = bool(tcfg.get("amp", True)) and device.type == "cuda" self.scaler = _make_grad_scaler(self.amp) self.grad_clip = float(tcfg.get("grad_clip", 0.0) or 0.0) self.accumulate = max(1, int(tcfg.get("accumulate_grad_batches", 1))) self.max_minutes = float(tcfg.get("max_minutes", 0) or 0) self.stop_on_nan = bool(tcfg.get("stop_on_nan", True)) self.log_every = int(tcfg.get("log_every", 10)) self.optimizer = build_optimizer(cfg["optimizer"], model.parameters()) self.scheduler = build_scheduler( cfg["scheduler"], self.optimizer, steps_per_epoch=max(1, len(train_loader)), max_epochs=self.max_epochs, ) self.scheduler_interval = cfg["scheduler"].get("interval", "epoch") self.scheduler_monitor = cfg["scheduler"].get("params", {}).get("monitor", "val_loss") self.early = EarlyStopping(**cfg["trainer"]["early_stopping"]) ckpt_cfg = cfg["trainer"]["checkpoint"] self.ckpt = CheckpointManager( self.output_dir, monitor=ckpt_cfg.get("monitor", "val_smape"), mode=ckpt_cfg.get("mode", "min"), save_top_k=int(ckpt_cfg.get("save_top_k", 3)), save_last=bool(ckpt_cfg.get("save_last", True)), ) self.load_best_at_end = bool(ckpt_cfg.get("load_best_at_end", True)) self.history_logger = HistoryLogger(self.output_dir / "history.csv") self.tb = TensorBoardLogger(self.output_dir / "tensorboard") self.history = [] self.start_epoch = 0 self.diverged = False self._resume_if_needed() def _resume_if_needed(self) -> None: resume = self.cfg["trainer"].get("resume", "") if not resume: return path = Path(resume) if not path.exists(): raise FileNotFoundError(f"Resume checkpoint not found: {path}") payload = self.ckpt.load(path, self.model, self.optimizer, self.scheduler) self.start_epoch = int(payload.get("epoch", -1)) + 1 if "early_stopping" in payload.get("extra", {}): self.early.load_state_dict(payload["extra"]["early_stopping"]) print(f"Resumed from {path} at epoch {self.start_epoch}") def _move(self, batch): return {k: v.to(self.device, non_blocking=True) for k, v in batch.items()} def _train_epoch(self) -> float: self.model.train() total = 0.0 n = 0 self.optimizer.zero_grad(set_to_none=True) for step, batch in enumerate(self.train_loader): batch = self._move(batch) x, y, mask = batch["x"], batch["y"], batch["mask"] if self.target_scaler is not None: y = torch.from_numpy(self.target_scaler.transform(y.detach().cpu().numpy())).to(self.device).float() with _autocast(self.amp): pred = self.model(x) loss = self.loss_fn(pred, y, mask) if not math.isfinite(float(loss.detach().cpu())): if self.stop_on_nan: self.diverged = True break self.scaler.scale(loss / self.accumulate).backward() if (step + 1) % self.accumulate == 0: if self.grad_clip > 0: self.scaler.unscale_(self.optimizer) torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip) self.scaler.step(self.optimizer) self.scaler.update() self.optimizer.zero_grad(set_to_none=True) if self.scheduler is not None and self.scheduler_interval == "batch": self.scheduler.step() total += float(loss.detach().cpu()) * x.size(0) n += x.size(0) return total / max(1, n) @torch.no_grad() def _validate(self) -> tuple[float, Dict[str, float]]: self.model.eval() total = 0.0 n = 0 preds = [] targets = [] masks = [] for batch in self.val_loader: batch = self._move(batch) x, y, mask = batch["x"], batch["y"], batch["mask"] y_for_loss = y if self.target_scaler is not None: y_for_loss = torch.from_numpy(self.target_scaler.transform(y.cpu().numpy())).to(self.device).float() pred = self.model(x) loss = self.loss_fn(pred, y_for_loss, mask) point = point_from_output(pred, self.cfg["model"]["output_type"], self.n_classes) if self.target_scaler is not None and self.cfg["model"]["output_type"] == "regression": point = torch.from_numpy(self.target_scaler.inverse(point.cpu().numpy())).to(self.device).float() total += float(loss.detach().cpu()) * x.size(0) n += x.size(0) preds.append(point.detach().cpu()) targets.append(y.detach().cpu()) masks.append(mask.detach().cpu()) if not preds: return float("nan"), {"smape_plus_1": float("nan"), "mae": float("nan")} P = torch.cat(preds).numpy() Y = torch.cat(targets).numpy() M = torch.cat(masks).numpy().astype(bool) return total / max(1, n), { "smape_plus_1": smape_plus_1(Y[M], P[M]), "mae": mae(Y[M], P[M]), } @torch.no_grad() def predict(self, loader) -> tuple[np.ndarray, np.ndarray, np.ndarray]: self.model.eval() preds = [] targets = [] masks = [] for batch in loader: batch = self._move(batch) pred = self.model(batch["x"]) point = point_from_output(pred, self.cfg["model"]["output_type"], self.n_classes) if self.target_scaler is not None and self.cfg["model"]["output_type"] == "regression": point = torch.from_numpy(self.target_scaler.inverse(point.cpu().numpy())).to(self.device).float() preds.append(point.detach().cpu()) targets.append(batch["y"].detach().cpu()) masks.append(batch["mask"].detach().cpu()) return ( torch.cat(preds).numpy(), torch.cat(targets).numpy(), torch.cat(masks).numpy(), ) def fit(self) -> TrainResult: timer = Timer() best_metrics: Dict[str, Any] = {} for epoch in range(self.start_epoch, self.max_epochs): t0 = time.time() train_loss = self._train_epoch() if self.diverged: print("Diverged (non-finite loss); stopping.") break val_loss, val_metrics = self._validate() metrics = { "epoch": float(epoch), "train_loss": float(train_loss), "val_loss": float(val_loss), "val_smape": float(val_metrics.get("smape_plus_1", float("nan"))), "val_mae": float(val_metrics.get("mae", float("nan"))), "lr": float(self.optimizer.param_groups[0]["lr"]), "epoch_time": float(time.time() - t0), } self.history.append(metrics) self.history_logger.log(metrics) self.tb.log(metrics, epoch) print( f"epoch {epoch:03d} | train {metrics['train_loss']:.4f} | " f"val {metrics['val_loss']:.4f} | smape {metrics['val_smape']:.3f} | " f"lr {metrics['lr']:.2e} | {metrics['epoch_time']:.1f}s" ) if self.scheduler is not None: if is_plateau(self.scheduler): self.scheduler.step(metrics.get(self.scheduler_monitor, val_loss)) elif self.scheduler_interval != "batch": self.scheduler.step() if self.early.step(metrics, epoch): print(f"Early stopping at epoch {epoch} (best {self.early.best:.3f} @ {self.early.best_epoch})") break extra = dict(self.extra) extra["early_stopping"] = self.early.state_dict() self.ckpt.step(epoch, self.model, self.optimizer, self.scheduler, metrics, extra) if self.max_minutes > 0 and timer.elapsed_minutes() > self.max_minutes: print(f"Time limit reached ({self.max_minutes} min); stopping.") break if self.load_best_at_end and self.ckpt.best_path is not None: self.ckpt.load(self.ckpt.best_path, self.model) best_metrics = dict(self.ckpt.best_metrics) else: best_metrics = dict(self.history[-1]) if self.history else {} hist = pd.DataFrame(self.history) hist.to_csv(self.output_dir / "history.csv", index=False) save_json(best_metrics, self.output_dir / "best_metrics.json") self.tb.close() return TrainResult( history=hist, best_metrics=best_metrics, best_path=str(self.ckpt.best_path) if self.ckpt.best_path else None, last_path=str(self.ckpt.last_path) if self.ckpt.last_path.exists() else None, epochs_run=len(self.history), diverged=self.diverged, )