"""Early stopping, checkpointing, history, and TensorBoard logging.""" from __future__ import annotations import csv import math import shutil import time from pathlib import Path from typing import Any, Dict, List, Optional import numpy as np import torch class EarlyStopping: def __init__( self, monitor: str = "val_smape", mode: str = "min", patience: int = 20, min_delta: float = 1e-4, enabled: bool = True, ): self.monitor = monitor self.mode = mode self.patience = int(patience) self.min_delta = float(min_delta) self.enabled = bool(enabled) self.best: Optional[float] = None self.best_epoch: int = -1 self.bad_epochs: int = 0 self.stopped: bool = False def _improved(self, value: float) -> bool: if self.best is None: return True if self.mode == "min": return value < self.best - self.min_delta return value > self.best + self.min_delta def step(self, metrics: Dict[str, float], epoch: int) -> bool: if not self.enabled: return False value = metrics.get(self.monitor) if value is None or not math.isfinite(float(value)): return False value = float(value) if self._improved(value): self.best = value self.best_epoch = int(epoch) self.bad_epochs = 0 else: self.bad_epochs += 1 self.stopped = self.bad_epochs >= self.patience return self.stopped def state_dict(self) -> Dict[str, Any]: return { "monitor": self.monitor, "mode": self.mode, "patience": self.patience, "min_delta": self.min_delta, "best": self.best, "best_epoch": self.best_epoch, "bad_epochs": self.bad_epochs, "stopped": self.stopped, } def load_state_dict(self, state: Dict[str, Any]) -> None: for k, v in state.items(): if hasattr(self, k): setattr(self, k, v) class CheckpointManager: def __init__( self, output_dir: str | Path, monitor: str = "val_smape", mode: str = "min", save_top_k: int = 3, save_last: bool = True, ): self.dir = Path(output_dir) / "checkpoints" self.topk_dir = self.dir / "topk" self.topk_dir.mkdir(parents=True, exist_ok=True) self.monitor = monitor self.mode = mode self.save_top_k = int(save_top_k) self.save_last = bool(save_last) self.topk: List[Dict[str, Any]] = [] self.best_metrics: Dict[str, Any] = {} @property def best_path(self) -> Optional[Path]: return Path(self.topk[0]["path"]) if self.topk else None @property def last_path(self) -> Path: return self.dir / "last.pt" def _better(self, a: float, b: float) -> bool: return a < b if self.mode == "min" else a > b def step( self, epoch: int, model: torch.nn.Module, optimizer: torch.optim.Optimizer, scheduler, metrics: Dict[str, float], extra: Dict[str, Any], ) -> None: payload = { "epoch": int(epoch), "model_state": model.state_dict(), "optimizer_state": optimizer.state_dict() if optimizer is not None else None, "scheduler_state": scheduler.state_dict() if scheduler is not None else None, "metrics": dict(metrics), "monitor": self.monitor, "mode": self.mode, "extra": extra, } if self.save_last: torch.save(payload, self.last_path) if self.save_top_k <= 0: return value = metrics.get(self.monitor) if value is None or not math.isfinite(float(value)): return value = float(value) worst = self.topk[-1]["score"] if len(self.topk) >= self.save_top_k else None should_save = len(self.topk) < self.save_top_k or (worst is not None and self._better(value, worst)) if not should_save: return path = self.topk_dir / f"epoch_{epoch:03d}_{self.monitor}_{value:.6f}.pt" torch.save(payload, path) self.topk.append({"score": value, "path": str(path), "epoch": int(epoch), "metrics": dict(metrics)}) self.topk.sort(key=lambda x: x["score"], reverse=(self.mode == "max")) while len(self.topk) > self.save_top_k: removed = self.topk.pop() try: Path(removed["path"]).unlink(missing_ok=True) except Exception: pass self.best_metrics = dict(self.topk[0]["metrics"]) try: import shutil shutil.copyfile(self.topk[0]["path"], self.dir / "best.pt") except Exception: pass def load(self, path: str | Path, model: torch.nn.Module, optimizer=None, scheduler=None): try: device = next(model.parameters()).device except StopIteration: device = "cpu" payload = torch.load(path, map_location=device, weights_only=False) model.load_state_dict(payload["model_state"]) if optimizer is not None and payload.get("optimizer_state") is not None: optimizer.load_state_dict(payload["optimizer_state"]) if scheduler is not None and payload.get("scheduler_state") is not None: scheduler.load_state_dict(payload["scheduler_state"]) return payload class HistoryLogger: def __init__(self, path: str | Path): self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) self.fieldnames: Optional[List[str]] = None def log(self, row: Dict[str, Any]) -> None: if self.fieldnames is None: self.fieldnames = list(row.keys()) with open(self.path, "w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=self.fieldnames) writer.writeheader() with open(self.path, "a", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=self.fieldnames) writer.writerow({k: row.get(k, "") for k in self.fieldnames}) class TensorBoardLogger: def __init__(self, log_dir: str | Path): self.writer = None try: from torch.utils.tensorboard import SummaryWriter self.writer = SummaryWriter(log_dir=str(log_dir)) except Exception: self.writer = None def log(self, metrics: Dict[str, float], step: int) -> None: if self.writer is None: return for k, v in metrics.items(): if isinstance(v, (int, float)) and math.isfinite(float(v)): self.writer.add_scalar(k, float(v), step) def close(self) -> None: if self.writer is not None: self.writer.close() class Timer: def __init__(self): self.start = time.time() def elapsed(self) -> float: return time.time() - self.start def elapsed_minutes(self) -> float: return self.elapsed() / 60.0