File size: 2,303 Bytes
dffbc1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Model selection on the inner-validation volunteers.
Trainers evaluate the inner-validation set after every epoch and use
:class:`InnerValSelector` for epoch selection / early stopping; LR schedulers
step on the same inner-validation value. The test fold is evaluated once,
with the selected checkpoint, after training.
"""
from __future__ import annotations
import copy
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
@dataclass
class InnerValSelector:
metric: str
mode: str = "max" # "max" or "min"
patience: Optional[int] = None
min_delta: float = 0.0
best_value: Optional[float] = None
best_epoch: Optional[int] = None
best_state: Any = None
history: List[Dict[str, float]] = field(default_factory=list)
_bad_epochs: int = 0
def __post_init__(self):
if self.mode not in ("max", "min"):
raise ValueError("mode must be 'max' or 'min'")
def _better(self, v: float) -> bool:
if self.best_value is None:
return True
if self.mode == "max":
return v > self.best_value + self.min_delta
return v < self.best_value - self.min_delta
def update(self, epoch: int, inner_val_metrics: Dict[str, float], state: Any = None) -> bool:
"""Record one epoch; return True if it is the new best.
``state`` (e.g. ``model.state_dict()``) is deep-copied when it improves.
"""
v = float(inner_val_metrics[self.metric])
self.history.append({"epoch": epoch, **{k: float(x) for k, x in inner_val_metrics.items()}})
if self._better(v):
self.best_value, self.best_epoch = v, epoch
self.best_state = copy.deepcopy(state) if state is not None else None
self._bad_epochs = 0
return True
self._bad_epochs += 1
return False
@property
def should_stop(self) -> bool:
return self.patience is not None and self._bad_epochs >= self.patience
def summary(self) -> Dict[str, Any]:
return {"criterion": f"inner_val/{self.metric} ({self.mode})", "best_epoch": self.best_epoch,
"best_inner_val": self.best_value, "patience": self.patience,
"epochs_run": len(self.history), "history": self.history}
|