Download code/ampd/training/trainer.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 11.4 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/training/trainer.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/training/trainer.py
-
curl -L -o trainer.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/training/trainer.py
11.4 kB
| """A small configurable PyTorch trainer with early stopping and checkpoint selection.""" | |
| from __future__ import annotations | |
| import contextlib | |
| import math | |
| import time | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any, Dict, Optional | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from ..evaluation.decision import point_from_output | |
| from ..evaluation.metrics import mae, smape_plus_1 | |
| from ..utils import save_json | |
| from .callbacks import ( | |
| CheckpointManager, | |
| EarlyStopping, | |
| HistoryLogger, | |
| TensorBoardLogger, | |
| Timer, | |
| ) | |
| from .optimizers import build_optimizer | |
| from .schedulers import build_scheduler, is_plateau | |
| class TrainResult: | |
| history: pd.DataFrame | |
| best_metrics: Dict[str, Any] | |
| best_path: Optional[str] | |
| last_path: Optional[str] | |
| epochs_run: int | |
| diverged: bool = False | |
| def _make_grad_scaler(enabled: bool): | |
| try: | |
| return torch.amp.GradScaler("cuda", enabled=enabled) | |
| except (TypeError, AttributeError): | |
| return torch.cuda.amp.GradScaler(enabled=enabled) | |
| def _autocast(enabled: bool): | |
| if not enabled: | |
| return contextlib.nullcontext() | |
| try: | |
| return torch.amp.autocast("cuda", dtype=torch.float16) | |
| except TypeError: | |
| return torch.cuda.amp.autocast(dtype=torch.float16) | |
| class Trainer: | |
| def __init__( | |
| self, | |
| cfg: Dict[str, Any], | |
| model: torch.nn.Module, | |
| train_loader, | |
| val_loader, | |
| device: torch.device, | |
| output_dir: str | Path, | |
| loss_fn, | |
| n_classes: int = 87, | |
| target_scaler=None, | |
| extra: Optional[Dict[str, Any]] = None, | |
| ): | |
| self.cfg = cfg | |
| self.model = model | |
| self.train_loader = train_loader | |
| self.val_loader = val_loader | |
| self.device = device | |
| self.output_dir = Path(output_dir) | |
| self.output_dir.mkdir(parents=True, exist_ok=True) | |
| self.loss_fn = loss_fn | |
| self.n_classes = int(n_classes) | |
| self.target_scaler = target_scaler | |
| self.extra = extra or {} | |
| tcfg = cfg["trainer"] | |
| self.max_epochs = int(tcfg.get("max_epochs", 100)) | |
| self.amp = bool(tcfg.get("amp", True)) and device.type == "cuda" | |
| self.scaler = _make_grad_scaler(self.amp) | |
| self.grad_clip = float(tcfg.get("grad_clip", 0.0) or 0.0) | |
| self.accumulate = max(1, int(tcfg.get("accumulate_grad_batches", 1))) | |
| self.max_minutes = float(tcfg.get("max_minutes", 0) or 0) | |
| self.stop_on_nan = bool(tcfg.get("stop_on_nan", True)) | |
| self.log_every = int(tcfg.get("log_every", 10)) | |
| self.optimizer = build_optimizer(cfg["optimizer"], model.parameters()) | |
| self.scheduler = build_scheduler( | |
| cfg["scheduler"], | |
| self.optimizer, | |
| steps_per_epoch=max(1, len(train_loader)), | |
| max_epochs=self.max_epochs, | |
| ) | |
| self.scheduler_interval = cfg["scheduler"].get("interval", "epoch") | |
| self.scheduler_monitor = cfg["scheduler"].get("params", {}).get("monitor", "val_loss") | |
| self.early = EarlyStopping(**cfg["trainer"]["early_stopping"]) | |
| ckpt_cfg = cfg["trainer"]["checkpoint"] | |
| self.ckpt = CheckpointManager( | |
| self.output_dir, | |
| monitor=ckpt_cfg.get("monitor", "val_smape"), | |
| mode=ckpt_cfg.get("mode", "min"), | |
| save_top_k=int(ckpt_cfg.get("save_top_k", 3)), | |
| save_last=bool(ckpt_cfg.get("save_last", True)), | |
| ) | |
| self.load_best_at_end = bool(ckpt_cfg.get("load_best_at_end", True)) | |
| self.history_logger = HistoryLogger(self.output_dir / "history.csv") | |
| self.tb = TensorBoardLogger(self.output_dir / "tensorboard") | |
| self.history = [] | |
| self.start_epoch = 0 | |
| self.diverged = False | |
| self._resume_if_needed() | |
| def _resume_if_needed(self) -> None: | |
| resume = self.cfg["trainer"].get("resume", "") | |
| if not resume: | |
| return | |
| path = Path(resume) | |
| if not path.exists(): | |
| raise FileNotFoundError(f"Resume checkpoint not found: {path}") | |
| payload = self.ckpt.load(path, self.model, self.optimizer, self.scheduler) | |
| self.start_epoch = int(payload.get("epoch", -1)) + 1 | |
| if "early_stopping" in payload.get("extra", {}): | |
| self.early.load_state_dict(payload["extra"]["early_stopping"]) | |
| print(f"Resumed from {path} at epoch {self.start_epoch}") | |
| def _move(self, batch): | |
| return {k: v.to(self.device, non_blocking=True) for k, v in batch.items()} | |
| def _train_epoch(self) -> float: | |
| self.model.train() | |
| total = 0.0 | |
| n = 0 | |
| self.optimizer.zero_grad(set_to_none=True) | |
| for step, batch in enumerate(self.train_loader): | |
| batch = self._move(batch) | |
| x, y, mask = batch["x"], batch["y"], batch["mask"] | |
| if self.target_scaler is not None: | |
| y = torch.from_numpy(self.target_scaler.transform(y.detach().cpu().numpy())).to(self.device).float() | |
| with _autocast(self.amp): | |
| pred = self.model(x) | |
| loss = self.loss_fn(pred, y, mask) | |
| if not math.isfinite(float(loss.detach().cpu())): | |
| if self.stop_on_nan: | |
| self.diverged = True | |
| break | |
| self.scaler.scale(loss / self.accumulate).backward() | |
| if (step + 1) % self.accumulate == 0: | |
| if self.grad_clip > 0: | |
| self.scaler.unscale_(self.optimizer) | |
| torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.grad_clip) | |
| self.scaler.step(self.optimizer) | |
| self.scaler.update() | |
| self.optimizer.zero_grad(set_to_none=True) | |
| if self.scheduler is not None and self.scheduler_interval == "batch": | |
| self.scheduler.step() | |
| total += float(loss.detach().cpu()) * x.size(0) | |
| n += x.size(0) | |
| return total / max(1, n) | |
| def _validate(self) -> tuple[float, Dict[str, float]]: | |
| self.model.eval() | |
| total = 0.0 | |
| n = 0 | |
| preds = [] | |
| targets = [] | |
| masks = [] | |
| for batch in self.val_loader: | |
| batch = self._move(batch) | |
| x, y, mask = batch["x"], batch["y"], batch["mask"] | |
| y_for_loss = y | |
| if self.target_scaler is not None: | |
| y_for_loss = torch.from_numpy(self.target_scaler.transform(y.cpu().numpy())).to(self.device).float() | |
| pred = self.model(x) | |
| loss = self.loss_fn(pred, y_for_loss, mask) | |
| point = point_from_output(pred, self.cfg["model"]["output_type"], self.n_classes) | |
| if self.target_scaler is not None and self.cfg["model"]["output_type"] == "regression": | |
| point = torch.from_numpy(self.target_scaler.inverse(point.cpu().numpy())).to(self.device).float() | |
| total += float(loss.detach().cpu()) * x.size(0) | |
| n += x.size(0) | |
| preds.append(point.detach().cpu()) | |
| targets.append(y.detach().cpu()) | |
| masks.append(mask.detach().cpu()) | |
| if not preds: | |
| return float("nan"), {"smape_plus_1": float("nan"), "mae": float("nan")} | |
| P = torch.cat(preds).numpy() | |
| Y = torch.cat(targets).numpy() | |
| M = torch.cat(masks).numpy().astype(bool) | |
| return total / max(1, n), { | |
| "smape_plus_1": smape_plus_1(Y[M], P[M]), | |
| "mae": mae(Y[M], P[M]), | |
| } | |
| def predict(self, loader) -> tuple[np.ndarray, np.ndarray, np.ndarray]: | |
| self.model.eval() | |
| preds = [] | |
| targets = [] | |
| masks = [] | |
| for batch in loader: | |
| batch = self._move(batch) | |
| pred = self.model(batch["x"]) | |
| point = point_from_output(pred, self.cfg["model"]["output_type"], self.n_classes) | |
| if self.target_scaler is not None and self.cfg["model"]["output_type"] == "regression": | |
| point = torch.from_numpy(self.target_scaler.inverse(point.cpu().numpy())).to(self.device).float() | |
| preds.append(point.detach().cpu()) | |
| targets.append(batch["y"].detach().cpu()) | |
| masks.append(batch["mask"].detach().cpu()) | |
| return ( | |
| torch.cat(preds).numpy(), | |
| torch.cat(targets).numpy(), | |
| torch.cat(masks).numpy(), | |
| ) | |
| def fit(self) -> TrainResult: | |
| timer = Timer() | |
| best_metrics: Dict[str, Any] = {} | |
| for epoch in range(self.start_epoch, self.max_epochs): | |
| t0 = time.time() | |
| train_loss = self._train_epoch() | |
| if self.diverged: | |
| print("Diverged (non-finite loss); stopping.") | |
| break | |
| val_loss, val_metrics = self._validate() | |
| metrics = { | |
| "epoch": float(epoch), | |
| "train_loss": float(train_loss), | |
| "val_loss": float(val_loss), | |
| "val_smape": float(val_metrics.get("smape_plus_1", float("nan"))), | |
| "val_mae": float(val_metrics.get("mae", float("nan"))), | |
| "lr": float(self.optimizer.param_groups[0]["lr"]), | |
| "epoch_time": float(time.time() - t0), | |
| } | |
| self.history.append(metrics) | |
| self.history_logger.log(metrics) | |
| self.tb.log(metrics, epoch) | |
| print( | |
| f"epoch {epoch:03d} | train {metrics['train_loss']:.4f} | " | |
| f"val {metrics['val_loss']:.4f} | smape {metrics['val_smape']:.3f} | " | |
| f"lr {metrics['lr']:.2e} | {metrics['epoch_time']:.1f}s" | |
| ) | |
| if self.scheduler is not None: | |
| if is_plateau(self.scheduler): | |
| self.scheduler.step(metrics.get(self.scheduler_monitor, val_loss)) | |
| elif self.scheduler_interval != "batch": | |
| self.scheduler.step() | |
| if self.early.step(metrics, epoch): | |
| print(f"Early stopping at epoch {epoch} (best {self.early.best:.3f} @ {self.early.best_epoch})") | |
| break | |
| extra = dict(self.extra) | |
| extra["early_stopping"] = self.early.state_dict() | |
| self.ckpt.step(epoch, self.model, self.optimizer, self.scheduler, metrics, extra) | |
| if self.max_minutes > 0 and timer.elapsed_minutes() > self.max_minutes: | |
| print(f"Time limit reached ({self.max_minutes} min); stopping.") | |
| break | |
| if self.load_best_at_end and self.ckpt.best_path is not None: | |
| self.ckpt.load(self.ckpt.best_path, self.model) | |
| best_metrics = dict(self.ckpt.best_metrics) | |
| else: | |
| best_metrics = dict(self.history[-1]) if self.history else {} | |
| hist = pd.DataFrame(self.history) | |
| hist.to_csv(self.output_dir / "history.csv", index=False) | |
| save_json(best_metrics, self.output_dir / "best_metrics.json") | |
| self.tb.close() | |
| return TrainResult( | |
| history=hist, | |
| best_metrics=best_metrics, | |
| best_path=str(self.ckpt.best_path) if self.ckpt.best_path else None, | |
| last_path=str(self.ckpt.last_path) if self.ckpt.last_path.exists() else None, | |
| epochs_run=len(self.history), | |
| diverged=self.diverged, | |
| ) | |