"""Scores for a model that states how sure it is: accuracy with its uncertainty, calibration, and coverage. Plain Python on purpose. These functions decide what the README claims, so they should be easy to read and to check by hand, and they should not depend on the training stack. """ from __future__ import annotations import math from typing import Dict, List, Optional, Sequence, Tuple Z95 = 1.959963984540054 def wilson(k: int, n: int, z: float = Z95) -> Tuple[float, float]: """The Wilson 95% interval for `k` right out of `n`. It stays inside [0, 1] and behaves near 0% and 100%, where the textbook normal interval does not. """ if n <= 0: raise ValueError("n must be positive") if not 0 <= k <= n: raise ValueError("k must be between 0 and n") p = k / n denom = 1 + z * z / n centre = (p + z * z / (2 * n)) / denom half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom return max(0.0, centre - half), min(1.0, centre + half) def ece(confidences: Sequence[float], correct: Sequence[bool], bins: int = 15) -> float: """Expected calibration error: the gap between stated confidence and accuracy, averaged over equal-width bins. A model that says 0.9 and is right 90% of the time scores 0. Confidence 1.0 falls in the last bin. """ if len(confidences) != len(correct): raise ValueError("confidences and correct differ in length") n = len(confidences) if n == 0: raise ValueError("no predictions") sums: List[List[float]] = [[0, 0.0, 0.0] for _ in range(bins)] for c, ok in zip(confidences, correct): b = min(int(c * bins), bins - 1) sums[b][0] += 1 sums[b][1] += c sums[b][2] += 1.0 if ok else 0.0 return sum(abs(s[1] - s[2]) for s in sums if s[0]) / n def selective(confidences: Sequence[float], correct: Sequence[bool], threshold: float) -> Dict[str, float]: """What happens when the model answers only at or above `threshold` and hands the rest to a person.""" kept = [ok for c, ok in zip(confidences, correct) if c >= threshold] n = len(confidences) return {"threshold": threshold, "answered": len(kept), "coverage": len(kept) / n, "accuracy": (sum(kept) / len(kept)) if kept else None, "errors": len(kept) - sum(kept)} def coverage_at_accuracy(confidences: Sequence[float], correct: Sequence[bool], target: float) -> Optional[Dict[str, float]]: """The most messages the model could answer, most confident first, while staying at `target` accuracy or above. The cut is chosen on the same predictions it is scored on, so it describes this data and is optimistic for new data. Use it to draw the coverage curve; use `selective` with a threshold fixed beforehand to make a claim. A cut may only fall between two different confidences: messages with equal confidence are kept or dropped together, as a real threshold would. Returns None when no cut reaches the target. """ order = sorted(range(len(confidences)), key=lambda i: -confidences[i]) best = None right = 0 for pos, i in enumerate(order): right += 1 if correct[i] else 0 last_of_tie = pos + 1 == len(order) or confidences[order[pos + 1]] < confidences[i] if last_of_tie and right / (pos + 1) >= target: best = {"target": target, "threshold": confidences[i], "answered": pos + 1, "coverage": (pos + 1) / len(order), "accuracy": right / (pos + 1)} return best def mcnemar_exact(only_a: int, only_b: int) -> float: """Two-sided exact McNemar p-value from the two kinds of disagreement between paired models.""" n = only_a + only_b if n == 0: return 1.0 k = min(only_a, only_b) tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n return min(1.0, 2 * tail)