Download code/ampd/training/callbacks.py from fryabcfry/cs4486-amp-pd-checkpoints: direct link, hf CLI and curl.
- Browser
- Download file 7.23 kB
-
https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/training/callbacks.py
- Command line
-
hf download hf://fryabcfry/cs4486-amp-pd-checkpoints/code/ampd/training/callbacks.py
-
curl -L -o callbacks.py https://huggingface.co/fryabcfry/cs4486-amp-pd-checkpoints/resolve/main/code/ampd/training/callbacks.py
7.23 kB
| """Early stopping, checkpointing, history, and TensorBoard logging.""" | |
| from __future__ import annotations | |
| import csv | |
| import math | |
| import shutil | |
| import time | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional | |
| import numpy as np | |
| import torch | |
| class EarlyStopping: | |
| def __init__( | |
| self, | |
| monitor: str = "val_smape", | |
| mode: str = "min", | |
| patience: int = 20, | |
| min_delta: float = 1e-4, | |
| enabled: bool = True, | |
| ): | |
| self.monitor = monitor | |
| self.mode = mode | |
| self.patience = int(patience) | |
| self.min_delta = float(min_delta) | |
| self.enabled = bool(enabled) | |
| self.best: Optional[float] = None | |
| self.best_epoch: int = -1 | |
| self.bad_epochs: int = 0 | |
| self.stopped: bool = False | |
| def _improved(self, value: float) -> bool: | |
| if self.best is None: | |
| return True | |
| if self.mode == "min": | |
| return value < self.best - self.min_delta | |
| return value > self.best + self.min_delta | |
| def step(self, metrics: Dict[str, float], epoch: int) -> bool: | |
| if not self.enabled: | |
| return False | |
| value = metrics.get(self.monitor) | |
| if value is None or not math.isfinite(float(value)): | |
| return False | |
| value = float(value) | |
| if self._improved(value): | |
| self.best = value | |
| self.best_epoch = int(epoch) | |
| self.bad_epochs = 0 | |
| else: | |
| self.bad_epochs += 1 | |
| self.stopped = self.bad_epochs >= self.patience | |
| return self.stopped | |
| def state_dict(self) -> Dict[str, Any]: | |
| return { | |
| "monitor": self.monitor, | |
| "mode": self.mode, | |
| "patience": self.patience, | |
| "min_delta": self.min_delta, | |
| "best": self.best, | |
| "best_epoch": self.best_epoch, | |
| "bad_epochs": self.bad_epochs, | |
| "stopped": self.stopped, | |
| } | |
| def load_state_dict(self, state: Dict[str, Any]) -> None: | |
| for k, v in state.items(): | |
| if hasattr(self, k): | |
| setattr(self, k, v) | |
| class CheckpointManager: | |
| def __init__( | |
| self, | |
| output_dir: str | Path, | |
| monitor: str = "val_smape", | |
| mode: str = "min", | |
| save_top_k: int = 3, | |
| save_last: bool = True, | |
| ): | |
| self.dir = Path(output_dir) / "checkpoints" | |
| self.topk_dir = self.dir / "topk" | |
| self.topk_dir.mkdir(parents=True, exist_ok=True) | |
| self.monitor = monitor | |
| self.mode = mode | |
| self.save_top_k = int(save_top_k) | |
| self.save_last = bool(save_last) | |
| self.topk: List[Dict[str, Any]] = [] | |
| self.best_metrics: Dict[str, Any] = {} | |
| def best_path(self) -> Optional[Path]: | |
| return Path(self.topk[0]["path"]) if self.topk else None | |
| def last_path(self) -> Path: | |
| return self.dir / "last.pt" | |
| def _better(self, a: float, b: float) -> bool: | |
| return a < b if self.mode == "min" else a > b | |
| def step( | |
| self, | |
| epoch: int, | |
| model: torch.nn.Module, | |
| optimizer: torch.optim.Optimizer, | |
| scheduler, | |
| metrics: Dict[str, float], | |
| extra: Dict[str, Any], | |
| ) -> None: | |
| payload = { | |
| "epoch": int(epoch), | |
| "model_state": model.state_dict(), | |
| "optimizer_state": optimizer.state_dict() if optimizer is not None else None, | |
| "scheduler_state": scheduler.state_dict() if scheduler is not None else None, | |
| "metrics": dict(metrics), | |
| "monitor": self.monitor, | |
| "mode": self.mode, | |
| "extra": extra, | |
| } | |
| if self.save_last: | |
| torch.save(payload, self.last_path) | |
| if self.save_top_k <= 0: | |
| return | |
| value = metrics.get(self.monitor) | |
| if value is None or not math.isfinite(float(value)): | |
| return | |
| value = float(value) | |
| worst = self.topk[-1]["score"] if len(self.topk) >= self.save_top_k else None | |
| should_save = len(self.topk) < self.save_top_k or (worst is not None and self._better(value, worst)) | |
| if not should_save: | |
| return | |
| path = self.topk_dir / f"epoch_{epoch:03d}_{self.monitor}_{value:.6f}.pt" | |
| torch.save(payload, path) | |
| self.topk.append({"score": value, "path": str(path), "epoch": int(epoch), "metrics": dict(metrics)}) | |
| self.topk.sort(key=lambda x: x["score"], reverse=(self.mode == "max")) | |
| while len(self.topk) > self.save_top_k: | |
| removed = self.topk.pop() | |
| try: | |
| Path(removed["path"]).unlink(missing_ok=True) | |
| except Exception: | |
| pass | |
| self.best_metrics = dict(self.topk[0]["metrics"]) | |
| try: | |
| import shutil | |
| shutil.copyfile(self.topk[0]["path"], self.dir / "best.pt") | |
| except Exception: | |
| pass | |
| def load(self, path: str | Path, model: torch.nn.Module, optimizer=None, scheduler=None): | |
| try: | |
| device = next(model.parameters()).device | |
| except StopIteration: | |
| device = "cpu" | |
| payload = torch.load(path, map_location=device, weights_only=False) | |
| model.load_state_dict(payload["model_state"]) | |
| if optimizer is not None and payload.get("optimizer_state") is not None: | |
| optimizer.load_state_dict(payload["optimizer_state"]) | |
| if scheduler is not None and payload.get("scheduler_state") is not None: | |
| scheduler.load_state_dict(payload["scheduler_state"]) | |
| return payload | |
| class HistoryLogger: | |
| def __init__(self, path: str | Path): | |
| self.path = Path(path) | |
| self.path.parent.mkdir(parents=True, exist_ok=True) | |
| self.fieldnames: Optional[List[str]] = None | |
| def log(self, row: Dict[str, Any]) -> None: | |
| if self.fieldnames is None: | |
| self.fieldnames = list(row.keys()) | |
| with open(self.path, "w", newline="", encoding="utf-8") as f: | |
| writer = csv.DictWriter(f, fieldnames=self.fieldnames) | |
| writer.writeheader() | |
| with open(self.path, "a", newline="", encoding="utf-8") as f: | |
| writer = csv.DictWriter(f, fieldnames=self.fieldnames) | |
| writer.writerow({k: row.get(k, "") for k in self.fieldnames}) | |
| class TensorBoardLogger: | |
| def __init__(self, log_dir: str | Path): | |
| self.writer = None | |
| try: | |
| from torch.utils.tensorboard import SummaryWriter | |
| self.writer = SummaryWriter(log_dir=str(log_dir)) | |
| except Exception: | |
| self.writer = None | |
| def log(self, metrics: Dict[str, float], step: int) -> None: | |
| if self.writer is None: | |
| return | |
| for k, v in metrics.items(): | |
| if isinstance(v, (int, float)) and math.isfinite(float(v)): | |
| self.writer.add_scalar(k, float(v), step) | |
| def close(self) -> None: | |
| if self.writer is not None: | |
| self.writer.close() | |
| class Timer: | |
| def __init__(self): | |
| self.start = time.time() | |
| def elapsed(self) -> float: | |
| return time.time() - self.start | |
| def elapsed_minutes(self) -> float: | |
| return self.elapsed() / 60.0 | |