"""Model selection on the inner-validation volunteers. Trainers evaluate the inner-validation set after every epoch and use :class:`InnerValSelector` for epoch selection / early stopping; LR schedulers step on the same inner-validation value. The test fold is evaluated once, with the selected checkpoint, after training. """ from __future__ import annotations import copy from dataclasses import dataclass, field from typing import Any, Dict, List, Optional @dataclass class InnerValSelector: metric: str mode: str = "max" # "max" or "min" patience: Optional[int] = None min_delta: float = 0.0 best_value: Optional[float] = None best_epoch: Optional[int] = None best_state: Any = None history: List[Dict[str, float]] = field(default_factory=list) _bad_epochs: int = 0 def __post_init__(self): if self.mode not in ("max", "min"): raise ValueError("mode must be 'max' or 'min'") def _better(self, v: float) -> bool: if self.best_value is None: return True if self.mode == "max": return v > self.best_value + self.min_delta return v < self.best_value - self.min_delta def update(self, epoch: int, inner_val_metrics: Dict[str, float], state: Any = None) -> bool: """Record one epoch; return True if it is the new best. ``state`` (e.g. ``model.state_dict()``) is deep-copied when it improves. """ v = float(inner_val_metrics[self.metric]) self.history.append({"epoch": epoch, **{k: float(x) for k, x in inner_val_metrics.items()}}) if self._better(v): self.best_value, self.best_epoch = v, epoch self.best_state = copy.deepcopy(state) if state is not None else None self._bad_epochs = 0 return True self._bad_epochs += 1 return False @property def should_stop(self) -> bool: return self.patience is not None and self._bad_epochs >= self.patience def summary(self) -> Dict[str, Any]: return {"criterion": f"inner_val/{self.metric} ({self.mode})", "best_epoch": self.best_epoch, "best_inner_val": self.best_value, "patience": self.patience, "epochs_run": len(self.history), "history": self.history}