File size: 3,925 Bytes
ebeed67 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """Paired bootstrap of an AP difference between two models on one set.
Both models are scored on the SAME resampled frames (frames drawn with
replacement, ``rounds`` times), so what varies is the set, not the pairing:
the 95 % interval of ``AP(candidate) - AP(baseline)`` is the noise band a
"no regression" verdict is read against. Matching is per frame, so it is done
once (`metrics.frame_matches`) and a resample only re-weights each frame's
detections by how often the frame was drawn — the same AP as concatenating the
drawn frames (`metrics.ap_of_sample`), computed for every round at once.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Optional, Sequence
import numpy as np
from .metrics import COCO_IOUS, FrameMatches
@dataclass(frozen=True)
class _Flat:
"""One model's matches of one class at one IoU, score-sorted."""
frame: np.ndarray # frame index per detection, highest score first
tp: np.ndarray # bool per detection
positives: np.ndarray # ground-truth count per frame
def _flatten(matches: Sequence[FrameMatches]) -> _Flat:
scores, tps, frames = [], [], []
for i, m in enumerate(matches):
scores.extend(m.scores)
tps.extend(m.true_positive)
frames.extend([i] * len(m.scores))
order = np.argsort(-np.asarray(scores, dtype=np.float64), kind="stable")
return _Flat(
frame=np.asarray(frames, dtype=np.int64)[order],
tp=np.asarray(tps, dtype=bool)[order],
positives=np.asarray([m.positives for m in matches], dtype=np.float64),
)
def weighted_ap(flat: _Flat, weights: np.ndarray) -> np.ndarray:
"""AP per round; ``weights`` is ``[rounds, frames]`` (draw counts)."""
total = weights @ flat.positives
if flat.frame.size == 0:
return np.where(total > 0, 0.0, np.nan)
w = weights[:, flat.frame]
ctp = np.cumsum(w * flat.tp, axis=1)
cfp = np.cumsum(w * ~flat.tp, axis=1)
denom = ctp + cfp
precision = np.divide(ctp, denom, out=np.zeros_like(ctp), where=denom > 0)
envelope = np.maximum.accumulate(precision[:, ::-1], axis=1)[:, ::-1]
gained = (w * flat.tp * envelope).sum(axis=1)
return np.divide(gained, total, out=np.full_like(total, np.nan), where=total > 0)
@dataclass(frozen=True)
class Delta:
"""``candidate - baseline`` on the full set and its 95 % bootstrap interval."""
value: float
low: float
high: float
@property
def significant_drop(self) -> bool:
return self.high < 0
@property
def significant_gain(self) -> bool:
return self.low > 0
def as_dict(self) -> dict:
return {"delta": round(self.value, 4), "ci95": [round(self.low, 4), round(self.high, 4)]}
def draw_weights(frames: int, rounds: int, seed: int = 0) -> np.ndarray:
"""``[rounds + 1, frames]`` draw counts; row 0 is the full set (all ones)."""
rng = np.random.default_rng(seed)
draws = rng.integers(0, frames, size=(rounds, frames))
counts = np.zeros((rounds, frames), dtype=np.float64)
np.add.at(counts, (np.arange(rounds)[:, None], draws), 1.0)
return np.vstack([np.ones((1, frames)), counts])
def ap_rounds(matches_by_iou: dict[float, Sequence[FrameMatches]], weights: np.ndarray,
ious: Sequence[float]) -> np.ndarray:
"""Mean AP over ``ious`` per round."""
return np.nanmean(
np.stack([weighted_ap(_flatten(matches_by_iou[t]), weights) for t in ious]), axis=0
)
def paired_delta(baseline: np.ndarray, candidate: np.ndarray) -> Optional[Delta]:
"""From per-round values (row 0 = full set) of two models."""
diff = candidate - baseline
if np.isnan(diff[0]):
return None
rounds = diff[1:][~np.isnan(diff[1:])]
low, high = (np.percentile(rounds, [2.5, 97.5]) if rounds.size else (np.nan, np.nan))
return Delta(float(diff[0]), float(low), float(high))
AP50 = (0.5,)
AP50_95 = COCO_IOUS
|