camstack_train/_vendor/camstack_eval/bootstrap.py

"""Paired bootstrap of an AP difference between two models on one set.

Both models are scored on the SAME resampled frames (frames drawn with
replacement, ``rounds`` times), so what varies is the set, not the pairing:
the 95 % interval of ``AP(candidate) - AP(baseline)`` is the noise band a
"no regression" verdict is read against. Matching is per frame, so it is done
once (`metrics.frame_matches`) and a resample only re-weights each frame's
detections by how often the frame was drawn — the same AP as concatenating the
drawn frames (`metrics.ap_of_sample`), computed for every round at once.
"""
from __future__ import annotations

from dataclasses import dataclass
from typing import Optional, Sequence

import numpy as np

from .metrics import COCO_IOUS, FrameMatches


@dataclass(frozen=True)
class _Flat:
    """One model's matches of one class at one IoU, score-sorted."""

    frame: np.ndarray  # frame index per detection, highest score first
    tp: np.ndarray  # bool per detection
    positives: np.ndarray  # ground-truth count per frame


def _flatten(matches: Sequence[FrameMatches]) -> _Flat:
    scores, tps, frames = [], [], []
    for i, m in enumerate(matches):
        scores.extend(m.scores)
        tps.extend(m.true_positive)
        frames.extend([i] * len(m.scores))
    order = np.argsort(-np.asarray(scores, dtype=np.float64), kind="stable")
    return _Flat(
        frame=np.asarray(frames, dtype=np.int64)[order],
        tp=np.asarray(tps, dtype=bool)[order],
        positives=np.asarray([m.positives for m in matches], dtype=np.float64),
    )


def weighted_ap(flat: _Flat, weights: np.ndarray) -> np.ndarray:
    """AP per round; ``weights`` is ``[rounds, frames]`` (draw counts)."""
    total = weights @ flat.positives
    if flat.frame.size == 0:
        return np.where(total > 0, 0.0, np.nan)
    w = weights[:, flat.frame]
    ctp = np.cumsum(w * flat.tp, axis=1)
    cfp = np.cumsum(w * ~flat.tp, axis=1)
    denom = ctp + cfp
    precision = np.divide(ctp, denom, out=np.zeros_like(ctp), where=denom > 0)
    envelope = np.maximum.accumulate(precision[:, ::-1], axis=1)[:, ::-1]
    gained = (w * flat.tp * envelope).sum(axis=1)
    return np.divide(gained, total, out=np.full_like(total, np.nan), where=total > 0)


@dataclass(frozen=True)
class Delta:
    """``candidate - baseline`` on the full set and its 95 % bootstrap interval."""

    value: float
    low: float
    high: float

    @property
    def significant_drop(self) -> bool:
        return self.high < 0

    @property
    def significant_gain(self) -> bool:
        return self.low > 0

    def as_dict(self) -> dict:
        return {"delta": round(self.value, 4), "ci95": [round(self.low, 4), round(self.high, 4)]}


def draw_weights(frames: int, rounds: int, seed: int = 0) -> np.ndarray:
    """``[rounds + 1, frames]`` draw counts; row 0 is the full set (all ones)."""
    rng = np.random.default_rng(seed)
    draws = rng.integers(0, frames, size=(rounds, frames))
    counts = np.zeros((rounds, frames), dtype=np.float64)
    np.add.at(counts, (np.arange(rounds)[:, None], draws), 1.0)
    return np.vstack([np.ones((1, frames)), counts])


def ap_rounds(matches_by_iou: dict[float, Sequence[FrameMatches]], weights: np.ndarray,
              ious: Sequence[float]) -> np.ndarray:
    """Mean AP over ``ious`` per round."""
    return np.nanmean(
        np.stack([weighted_ap(_flatten(matches_by_iou[t]), weights) for t in ious]), axis=0
    )


def paired_delta(baseline: np.ndarray, candidate: np.ndarray) -> Optional[Delta]:
    """From per-round values (row 0 = full set) of two models."""
    diff = candidate - baseline
    if np.isnan(diff[0]):
        return None
    rounds = diff[1:][~np.isnan(diff[1:])]
    low, high = (np.percentile(rounds, [2.5, 97.5]) if rounds.size else (np.nan, np.nan))
    return Delta(float(diff[0]), float(low), float(high))


AP50 = (0.5,)
AP50_95 = COCO_IOUS