fryabcfry's picture
Publish AMP-PD checkpoints, portable loaders and verified metrics (part 4)
0d48417 verified
Raw History Blame Contribute Delete
7.23 kB
"""Early stopping, checkpointing, history, and TensorBoard logging."""
from __future__ import annotations
import csv
import math
import shutil
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
import numpy as np
import torch
class EarlyStopping:
def __init__(
self,
monitor: str = "val_smape",
mode: str = "min",
patience: int = 20,
min_delta: float = 1e-4,
enabled: bool = True,
):
self.monitor = monitor
self.mode = mode
self.patience = int(patience)
self.min_delta = float(min_delta)
self.enabled = bool(enabled)
self.best: Optional[float] = None
self.best_epoch: int = -1
self.bad_epochs: int = 0
self.stopped: bool = False
def _improved(self, value: float) -> bool:
if self.best is None:
return True
if self.mode == "min":
return value < self.best - self.min_delta
return value > self.best + self.min_delta
def step(self, metrics: Dict[str, float], epoch: int) -> bool:
if not self.enabled:
return False
value = metrics.get(self.monitor)
if value is None or not math.isfinite(float(value)):
return False
value = float(value)
if self._improved(value):
self.best = value
self.best_epoch = int(epoch)
self.bad_epochs = 0
else:
self.bad_epochs += 1
self.stopped = self.bad_epochs >= self.patience
return self.stopped
def state_dict(self) -> Dict[str, Any]:
return {
"monitor": self.monitor,
"mode": self.mode,
"patience": self.patience,
"min_delta": self.min_delta,
"best": self.best,
"best_epoch": self.best_epoch,
"bad_epochs": self.bad_epochs,
"stopped": self.stopped,
}
def load_state_dict(self, state: Dict[str, Any]) -> None:
for k, v in state.items():
if hasattr(self, k):
setattr(self, k, v)
class CheckpointManager:
def __init__(
self,
output_dir: str | Path,
monitor: str = "val_smape",
mode: str = "min",
save_top_k: int = 3,
save_last: bool = True,
):
self.dir = Path(output_dir) / "checkpoints"
self.topk_dir = self.dir / "topk"
self.topk_dir.mkdir(parents=True, exist_ok=True)
self.monitor = monitor
self.mode = mode
self.save_top_k = int(save_top_k)
self.save_last = bool(save_last)
self.topk: List[Dict[str, Any]] = []
self.best_metrics: Dict[str, Any] = {}
@property
def best_path(self) -> Optional[Path]:
return Path(self.topk[0]["path"]) if self.topk else None
@property
def last_path(self) -> Path:
return self.dir / "last.pt"
def _better(self, a: float, b: float) -> bool:
return a < b if self.mode == "min" else a > b
def step(
self,
epoch: int,
model: torch.nn.Module,
optimizer: torch.optim.Optimizer,
scheduler,
metrics: Dict[str, float],
extra: Dict[str, Any],
) -> None:
payload = {
"epoch": int(epoch),
"model_state": model.state_dict(),
"optimizer_state": optimizer.state_dict() if optimizer is not None else None,
"scheduler_state": scheduler.state_dict() if scheduler is not None else None,
"metrics": dict(metrics),
"monitor": self.monitor,
"mode": self.mode,
"extra": extra,
}
if self.save_last:
torch.save(payload, self.last_path)
if self.save_top_k <= 0:
return
value = metrics.get(self.monitor)
if value is None or not math.isfinite(float(value)):
return
value = float(value)
worst = self.topk[-1]["score"] if len(self.topk) >= self.save_top_k else None
should_save = len(self.topk) < self.save_top_k or (worst is not None and self._better(value, worst))
if not should_save:
return
path = self.topk_dir / f"epoch_{epoch:03d}_{self.monitor}_{value:.6f}.pt"
torch.save(payload, path)
self.topk.append({"score": value, "path": str(path), "epoch": int(epoch), "metrics": dict(metrics)})
self.topk.sort(key=lambda x: x["score"], reverse=(self.mode == "max"))
while len(self.topk) > self.save_top_k:
removed = self.topk.pop()
try:
Path(removed["path"]).unlink(missing_ok=True)
except Exception:
pass
self.best_metrics = dict(self.topk[0]["metrics"])
try:
import shutil
shutil.copyfile(self.topk[0]["path"], self.dir / "best.pt")
except Exception:
pass
def load(self, path: str | Path, model: torch.nn.Module, optimizer=None, scheduler=None):
try:
device = next(model.parameters()).device
except StopIteration:
device = "cpu"
payload = torch.load(path, map_location=device, weights_only=False)
model.load_state_dict(payload["model_state"])
if optimizer is not None and payload.get("optimizer_state") is not None:
optimizer.load_state_dict(payload["optimizer_state"])
if scheduler is not None and payload.get("scheduler_state") is not None:
scheduler.load_state_dict(payload["scheduler_state"])
return payload
class HistoryLogger:
def __init__(self, path: str | Path):
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
self.fieldnames: Optional[List[str]] = None
def log(self, row: Dict[str, Any]) -> None:
if self.fieldnames is None:
self.fieldnames = list(row.keys())
with open(self.path, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=self.fieldnames)
writer.writeheader()
with open(self.path, "a", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=self.fieldnames)
writer.writerow({k: row.get(k, "") for k in self.fieldnames})
class TensorBoardLogger:
def __init__(self, log_dir: str | Path):
self.writer = None
try:
from torch.utils.tensorboard import SummaryWriter
self.writer = SummaryWriter(log_dir=str(log_dir))
except Exception:
self.writer = None
def log(self, metrics: Dict[str, float], step: int) -> None:
if self.writer is None:
return
for k, v in metrics.items():
if isinstance(v, (int, float)) and math.isfinite(float(v)):
self.writer.add_scalar(k, float(v), step)
def close(self) -> None:
if self.writer is not None:
self.writer.close()
class Timer:
def __init__(self):
self.start = time.time()
def elapsed(self) -> float:
return time.time() - self.start
def elapsed_minutes(self) -> float:
return self.elapsed() / 60.0