File size: 3,891 Bytes
4e09985
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Scores for a model that states how sure it is: accuracy with its uncertainty, calibration, and coverage.

Plain Python on purpose. These functions decide what the README claims, so they should be easy to read and to
check by hand, and they should not depend on the training stack.
"""

from __future__ import annotations

import math
from typing import Dict, List, Optional, Sequence, Tuple

Z95 = 1.959963984540054


def wilson(k: int, n: int, z: float = Z95) -> Tuple[float, float]:
    """The Wilson 95% interval for `k` right out of `n`.

    It stays inside [0, 1] and behaves near 0% and 100%, where the textbook normal interval does not.
    """
    if n <= 0:
        raise ValueError("n must be positive")
    if not 0 <= k <= n:
        raise ValueError("k must be between 0 and n")
    p = k / n
    denom = 1 + z * z / n
    centre = (p + z * z / (2 * n)) / denom
    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
    return max(0.0, centre - half), min(1.0, centre + half)


def ece(confidences: Sequence[float], correct: Sequence[bool], bins: int = 15) -> float:
    """Expected calibration error: the gap between stated confidence and accuracy, averaged over equal-width bins.

    A model that says 0.9 and is right 90% of the time scores 0. Confidence 1.0 falls in the last bin.
    """
    if len(confidences) != len(correct):
        raise ValueError("confidences and correct differ in length")
    n = len(confidences)
    if n == 0:
        raise ValueError("no predictions")
    sums: List[List[float]] = [[0, 0.0, 0.0] for _ in range(bins)]
    for c, ok in zip(confidences, correct):
        b = min(int(c * bins), bins - 1)
        sums[b][0] += 1
        sums[b][1] += c
        sums[b][2] += 1.0 if ok else 0.0
    return sum(abs(s[1] - s[2]) for s in sums if s[0]) / n


def selective(confidences: Sequence[float], correct: Sequence[bool], threshold: float) -> Dict[str, float]:
    """What happens when the model answers only at or above `threshold` and hands the rest to a person."""
    kept = [ok for c, ok in zip(confidences, correct) if c >= threshold]
    n = len(confidences)
    return {"threshold": threshold, "answered": len(kept), "coverage": len(kept) / n,
            "accuracy": (sum(kept) / len(kept)) if kept else None,
            "errors": len(kept) - sum(kept)}


def coverage_at_accuracy(confidences: Sequence[float], correct: Sequence[bool],
                         target: float) -> Optional[Dict[str, float]]:
    """The most messages the model could answer, most confident first, while staying at `target` accuracy or above.

    The cut is chosen on the same predictions it is scored on, so it describes this data and is optimistic for new
    data. Use it to draw the coverage curve; use `selective` with a threshold fixed beforehand to make a claim.
    A cut may only fall between two different confidences: messages with equal confidence are kept or dropped
    together, as a real threshold would. Returns None when no cut reaches the target.
    """
    order = sorted(range(len(confidences)), key=lambda i: -confidences[i])
    best = None
    right = 0
    for pos, i in enumerate(order):
        right += 1 if correct[i] else 0
        last_of_tie = pos + 1 == len(order) or confidences[order[pos + 1]] < confidences[i]
        if last_of_tie and right / (pos + 1) >= target:
            best = {"target": target, "threshold": confidences[i], "answered": pos + 1,
                    "coverage": (pos + 1) / len(order), "accuracy": right / (pos + 1)}
    return best


def mcnemar_exact(only_a: int, only_b: int) -> float:
    """Two-sided exact McNemar p-value from the two kinds of disagreement between paired models."""
    n = only_a + only_b
    if n == 0:
        return 1.0
    k = min(only_a, only_b)
    tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
    return min(1.0, 2 * tail)