File size: 2,303 Bytes
dffbc1f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""Model selection on the inner-validation volunteers.

Trainers evaluate the inner-validation set after every epoch and use
:class:`InnerValSelector` for epoch selection / early stopping; LR schedulers
step on the same inner-validation value.  The test fold is evaluated once,
with the selected checkpoint, after training.
"""
from __future__ import annotations

import copy
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional


@dataclass
class InnerValSelector:
    metric: str
    mode: str = "max"            # "max" or "min"
    patience: Optional[int] = None
    min_delta: float = 0.0
    best_value: Optional[float] = None
    best_epoch: Optional[int] = None
    best_state: Any = None
    history: List[Dict[str, float]] = field(default_factory=list)
    _bad_epochs: int = 0

    def __post_init__(self):
        if self.mode not in ("max", "min"):
            raise ValueError("mode must be 'max' or 'min'")

    def _better(self, v: float) -> bool:
        if self.best_value is None:
            return True
        if self.mode == "max":
            return v > self.best_value + self.min_delta
        return v < self.best_value - self.min_delta

    def update(self, epoch: int, inner_val_metrics: Dict[str, float], state: Any = None) -> bool:
        """Record one epoch; return True if it is the new best.

        ``state`` (e.g. ``model.state_dict()``) is deep-copied when it improves.
        """
        v = float(inner_val_metrics[self.metric])
        self.history.append({"epoch": epoch, **{k: float(x) for k, x in inner_val_metrics.items()}})
        if self._better(v):
            self.best_value, self.best_epoch = v, epoch
            self.best_state = copy.deepcopy(state) if state is not None else None
            self._bad_epochs = 0
            return True
        self._bad_epochs += 1
        return False

    @property
    def should_stop(self) -> bool:
        return self.patience is not None and self._bad_epochs >= self.patience

    def summary(self) -> Dict[str, Any]:
        return {"criterion": f"inner_val/{self.metric} ({self.mode})", "best_epoch": self.best_epoch,
                "best_inner_val": self.best_value, "patience": self.patience,
                "epochs_run": len(self.history), "history": self.history}