File size: 4,422 Bytes
3e363e2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
"""Post-hoc confidence calibration for answer/path/verifier scores."""

from __future__ import annotations

import math
from dataclasses import dataclass

import torch
from torch import Tensor

#: Beyond this the fit is degenerate: the calibrator becomes a constant predictor.
MAX_TEMPERATURE = 100.0


@dataclass(frozen=True)
class TemperatureCalibrator:
    temperature: float = 1.0

    def transform(self, logits: Tensor) -> Tensor:
        if self.temperature <= 0:
            raise ValueError("temperature must be positive")
        return torch.sigmoid(logits / self.temperature)

    def fit(
        self, logits: Tensor, labels: Tensor, iterations: int = 100
    ) -> TemperatureCalibrator:
        value = torch.tensor(
            float(self.temperature),
            dtype=logits.dtype,
            device=logits.device,
            requires_grad=True,
        )
        optimizer = torch.optim.LBFGS(
            [value], max_iter=iterations, line_search_fn="strong_wolfe"
        )

        def closure() -> float:
            optimizer.zero_grad()
            loss = torch.nn.functional.binary_cross_entropy_with_logits(
                logits / value.clamp_min(1e-3), labels.float()
            )
            loss.backward()
            return float(loss.detach())

        optimizer.step(closure)
        fitted = float(value.detach().clamp_min(1e-3))
        if not math.isfinite(fitted) or fitted > MAX_TEMPERATURE:
            # A near-flat logit distribution drives temperature scaling to
            # divergence: on the 100M run LBFGS returned T=1954, which maps every
            # input to ~0.5 and is a degenerate fit rather than a calibration.
            # Report the failure by returning the identity so the promotion rule
            # compares against something meaningful instead of silently shipping a
            # constant predictor.
            return TemperatureCalibrator(1.0)
        return TemperatureCalibrator(fitted)


@dataclass(frozen=True)
class IsotonicCalibrator:
    """Dependency-free monotonic probability calibrator using PAVA."""

    thresholds: tuple[float, ...]
    values: tuple[float, ...]

    @classmethod
    def fit(cls, probabilities: Tensor, labels: Tensor) -> IsotonicCalibrator:
        rows = sorted(
            (float(probability), float(label))
            for probability, label in zip(
                probabilities.detach().cpu().tolist(),
                labels.detach().cpu().tolist(),
                strict=True,
            )
        )
        if not rows:
            raise ValueError("isotonic calibration requires at least one example")
        blocks: list[list[float]] = []
        for probability, label in rows:
            blocks.append([probability, probability, label, 1.0])
            while (
                len(blocks) >= 2
                and blocks[-2][2] / blocks[-2][3] > blocks[-1][2] / blocks[-1][3]
            ):
                right = blocks.pop()
                left = blocks.pop()
                blocks.append(
                    [left[0], right[1], left[2] + right[2], left[3] + right[3]]
                )
        return cls(
            tuple(block[1] for block in blocks),
            tuple(block[2] / block[3] for block in blocks),
        )

    def transform(self, probabilities: Tensor) -> Tensor:
        thresholds = torch.tensor(
            self.thresholds, device=probabilities.device, dtype=probabilities.dtype
        )
        values = torch.tensor(
            self.values, device=probabilities.device, dtype=probabilities.dtype
        )
        indices = torch.bucketize(probabilities, thresholds, right=True).clamp_max(
            len(values) - 1
        )
        return values[indices]


def calibration_metrics(
    probabilities: Tensor, labels: Tensor, bins: int = 10
) -> dict[str, float]:
    probabilities = probabilities.detach().float().clamp(0, 1)
    labels = labels.detach().float()
    brier = float(((probabilities - labels) ** 2).mean())
    ece = 0.0
    for index in range(bins):
        low, high = index / bins, (index + 1) / bins
        mask = (probabilities >= low) & (
            probabilities < high if index < bins - 1 else probabilities <= high
        )
        if mask.any():
            ece += float(mask.float().mean()) * abs(
                float(probabilities[mask].mean()) - float(labels[mask].mean())
            )
    return {"ece": ece, "brier": brier}