Download src/sigir/eval/bootstrap.py from inflaton/pakdd_mrd: direct link, hf CLI and curl.
- Browser
- Download file 3.93 kB
-
https://huggingface.co/spaces/inflaton/pakdd_mrd/resolve/main/src/sigir/eval/bootstrap.py
- Command line
-
hf download hf://spaces/inflaton/pakdd_mrd/src/sigir/eval/bootstrap.py
-
curl -L -o bootstrap.py https://huggingface.co/spaces/inflaton/pakdd_mrd/resolve/main/src/sigir/eval/bootstrap.py
3.93 kB
| """Paired entity-cluster bootstrap (plan §7). | |
| Resampling unit = entity_group. All perturbation variants of a query share its cluster, so replicas | |
| never count as new independent test entities. Paired differences resample the same clusters for both | |
| systems. | |
| """ | |
| from __future__ import annotations | |
| from typing import Callable, Sequence | |
| import numpy as np | |
| from .metrics import compute_metrics | |
| from .outcomes import ScoredInstance | |
| def _clusters(scored: Sequence[ScoredInstance]) -> dict[str, list[int]]: | |
| cl: dict[str, list[int]] = {} | |
| for i, s in enumerate(scored): | |
| cl.setdefault(s.entity_group, []).append(i) | |
| return cl | |
| def cluster_bootstrap( | |
| scored: Sequence[ScoredInstance], | |
| metric_keys: Sequence[str] = ("match_recall", "precision", "n_fmr", "f05", "coverage_auto"), | |
| n_boot: int = 1000, | |
| seed: int = 0, | |
| alpha: float = 0.05, | |
| ) -> dict[str, dict[str, float]]: | |
| rng = np.random.default_rng(seed) | |
| cl = _clusters(scored) | |
| keys = list(cl.keys()) | |
| idx_lists = [cl[k] for k in keys] | |
| point = compute_metrics(scored) | |
| samples: dict[str, list[float]] = {k: [] for k in metric_keys} | |
| n = len(keys) | |
| for _ in range(n_boot): | |
| draw = rng.integers(0, n, n) | |
| sub = [scored[i] for c in draw for i in idx_lists[c]] | |
| m = compute_metrics(sub) | |
| for k in metric_keys: | |
| samples[k].append(m[k]) | |
| out = {} | |
| for k in metric_keys: | |
| arr = np.asarray(samples[k], dtype=float) | |
| arr = arr[~np.isnan(arr)] | |
| out[k] = { | |
| "point": point[k], | |
| "lo": float(np.quantile(arr, alpha / 2)) if arr.size else float("nan"), | |
| "hi": float(np.quantile(arr, 1 - alpha / 2)) if arr.size else float("nan"), | |
| "n_clusters": n, | |
| } | |
| return out | |
| def paired_cluster_bootstrap( | |
| scored_a: Sequence[ScoredInstance], | |
| scored_b: Sequence[ScoredInstance], | |
| metric_keys: Sequence[str] = ("match_recall", "precision", "n_fmr", "f05"), | |
| n_boot: int = 1000, | |
| seed: int = 0, | |
| alpha: float = 0.05, | |
| ) -> dict[str, dict[str, float]]: | |
| """CI for metric(A) - metric(B), resampling the same entity clusters for both.""" | |
| by_id_b = {s.instance_id: s for s in scored_b} | |
| pairs = [(a, by_id_b[a.instance_id]) for a in scored_a if a.instance_id in by_id_b] | |
| cl: dict[str, list[int]] = {} | |
| for i, (a, _) in enumerate(pairs): | |
| cl.setdefault(a.entity_group, []).append(i) | |
| keys = list(cl.keys()) | |
| idx_lists = [cl[k] for k in keys] | |
| rng = np.random.default_rng(seed) | |
| pa = compute_metrics([a for a, _ in pairs]) | |
| pb = compute_metrics([b for _, b in pairs]) | |
| diffs: dict[str, list[float]] = {k: [] for k in metric_keys} | |
| n = len(keys) | |
| for _ in range(n_boot): | |
| draw = rng.integers(0, n, n) | |
| ia = [pairs[i][0] for c in draw for i in idx_lists[c]] | |
| ib = [pairs[i][1] for c in draw for i in idx_lists[c]] | |
| ma, mb = compute_metrics(ia), compute_metrics(ib) | |
| for k in metric_keys: | |
| diffs[k].append(ma[k] - mb[k]) | |
| out = {} | |
| for k in metric_keys: | |
| arr = np.asarray(diffs[k], dtype=float) | |
| arr = arr[~np.isnan(arr)] | |
| out[k] = { | |
| "diff": pa[k] - pb[k], | |
| "lo": float(np.quantile(arr, alpha / 2)) if arr.size else float("nan"), | |
| "hi": float(np.quantile(arr, 1 - alpha / 2)) if arr.size else float("nan"), | |
| "n_pairs": len(pairs), | |
| "n_clusters": n, | |
| } | |
| return out | |
| def seed_variability(metric_by_seed: dict[int, dict[str, float]], keys: Sequence[str]) -> dict[str, dict[str, float]]: | |
| """Training-seed variability, reported separately from bootstrap intervals.""" | |
| out = {} | |
| for k in keys: | |
| vals = np.asarray([m[k] for m in metric_by_seed.values()], dtype=float) | |
| out[k] = {"mean": float(np.nanmean(vals)), "std": float(np.nanstd(vals, ddof=1)) if len(vals) > 1 else 0.0, "n_seeds": len(vals)} | |
| return out | |