ResearchOS / scripts /significance.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
5.34 kB
"""Is the row 4 vs row 5 difference real, or is it noise?
The headline claim of this project is a delta between two reranker arms. On a
300-query test split a small delta can easily be sampling noise, so this script
tests it properly instead of eyeballing the means:
- per-query nDCG@10 for both arms, from pytrec_eval
- a paired bootstrap over queries (the pairing matters: both arms see the same
queries and the same candidate lists, so the comparison is within-query)
- a two-sided paired permutation test
Writes results/significance.json.
python scripts/significance.py --config configs/rerank_hard_ft.yaml
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
import numpy as np
import pytrec_eval
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.config import device, load_config, seed_everything
from src.data.loader import load_split
from src.rerank.model import Reranker
METRIC = "ndcg_cut_10"
def per_query_ndcg(qrels: dict, run: dict) -> dict[str, float]:
"""nDCG@10 for every query, using the official trec_eval implementation."""
clean_qrels = {q: {d: int(r) for d, r in rels.items()} for q, rels in qrels.items()}
evaluator = pytrec_eval.RelevanceEvaluator(clean_qrels, {"ndcg_cut.10"})
return {qid: scores[METRIC] for qid, scores in evaluator.evaluate(run).items()}
def paired_bootstrap(
diffs: np.ndarray, n_resamples: int, rng: np.random.Generator
) -> tuple[float, float, float]:
"""Percentile CI and the fraction of resamples that flip the sign."""
idx = rng.integers(0, len(diffs), size=(n_resamples, len(diffs)))
means = diffs[idx].mean(axis=1)
lo, hi = np.percentile(means, [2.5, 97.5])
observed = diffs.mean()
flipped = float((means <= 0).mean() if observed > 0 else (means >= 0).mean())
return float(lo), float(hi), flipped
def paired_permutation(diffs: np.ndarray, n_resamples: int, rng: np.random.Generator) -> float:
"""Two-sided p-value by randomly flipping the sign of each query's difference."""
observed = abs(diffs.mean())
signs = rng.choice([-1.0, 1.0], size=(n_resamples, len(diffs)))
null = np.abs((signs * diffs).mean(axis=1))
return float((null >= observed).mean())
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", default="configs/rerank_hard_ft.yaml")
parser.add_argument("--model-a", default="models/reranker-random/final", help="control arm")
parser.add_argument("--model-b", default="models/reranker-hard/final", help="treatment arm")
parser.add_argument("--resamples", type=int, default=10000)
parser.add_argument("--out", default="results/significance.json")
args = parser.parse_args()
cfg = load_config(args.config)
seed = seed_everything(cfg["seed"])
rng = np.random.default_rng(seed)
corpus, queries, qrels = load_split(
cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"]
)
fused = json.loads(Path(cfg["paths"]["fused_run"]).read_text(encoding="utf-8"))
top_k = cfg["rerank"]["top_k"]
scores: dict[str, dict[str, float]] = {}
for label, model_path in (("random", args.model_a), ("hard", args.model_b)):
reranker = Reranker(
model_name_or_path=model_path,
max_length=cfg["reranker"]["max_length"],
batch_size=cfg["reranker"]["batch_size"],
device=device(),
fp16=True,
)
run, _timings = reranker.rerank(queries, fused, corpus, top_k=top_k)
scores[label] = per_query_ndcg(qrels, run)
print(f"[sig] {label}: mean nDCG@10 = {np.mean(list(scores[label].values())):.4f}")
del reranker
qids = sorted(set(scores["random"]) & set(scores["hard"]))
a = np.array([scores["random"][q] for q in qids])
b = np.array([scores["hard"][q] for q in qids])
diffs = b - a
lo, hi, flip = paired_bootstrap(diffs, args.resamples, rng)
p_value = paired_permutation(diffs, args.resamples, rng)
wins = int((diffs > 1e-9).sum())
losses = int((diffs < -1e-9).sum())
ties = len(diffs) - wins - losses
result = {
"seed": seed,
"n_queries": len(qids),
"resamples": args.resamples,
"control": {"model": args.model_a, "mean_ndcg@10": round(float(a.mean()), 5)},
"treatment": {"model": args.model_b, "mean_ndcg@10": round(float(b.mean()), 5)},
"mean_difference": round(float(diffs.mean()), 5),
"bootstrap_95_ci": [round(lo, 5), round(hi, 5)],
"permutation_p_value": round(p_value, 4),
"per_query": {"treatment_better": wins, "control_better": losses, "tied": ties},
"significant_at_0.05": bool(p_value < 0.05),
}
print(f"\n[sig] mean difference : {result['mean_difference']:+.5f}")
print(f"[sig] 95% CI : [{lo:+.5f}, {hi:+.5f}]")
print(f"[sig] permutation p : {p_value:.4f}")
print(f"[sig] per-query : {wins} better / {losses} worse / {ties} tied")
print(f"[sig] significant : {result['significant_at_0.05']}")
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(result, indent=2), encoding="utf-8")
print(f"[sig] wrote {out}")
if __name__ == "__main__":
main()