Spaces:
Running on Zero
Running on Zero
Download scripts/significance.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 5.34 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/significance.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/significance.py
-
curl -L -o significance.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/significance.py
5.34 kB
| """Is the row 4 vs row 5 difference real, or is it noise? | |
| The headline claim of this project is a delta between two reranker arms. On a | |
| 300-query test split a small delta can easily be sampling noise, so this script | |
| tests it properly instead of eyeballing the means: | |
| - per-query nDCG@10 for both arms, from pytrec_eval | |
| - a paired bootstrap over queries (the pairing matters: both arms see the same | |
| queries and the same candidate lists, so the comparison is within-query) | |
| - a two-sided paired permutation test | |
| Writes results/significance.json. | |
| python scripts/significance.py --config configs/rerank_hard_ft.yaml | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import numpy as np | |
| import pytrec_eval | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from src.config import device, load_config, seed_everything | |
| from src.data.loader import load_split | |
| from src.rerank.model import Reranker | |
| METRIC = "ndcg_cut_10" | |
| def per_query_ndcg(qrels: dict, run: dict) -> dict[str, float]: | |
| """nDCG@10 for every query, using the official trec_eval implementation.""" | |
| clean_qrels = {q: {d: int(r) for d, r in rels.items()} for q, rels in qrels.items()} | |
| evaluator = pytrec_eval.RelevanceEvaluator(clean_qrels, {"ndcg_cut.10"}) | |
| return {qid: scores[METRIC] for qid, scores in evaluator.evaluate(run).items()} | |
| def paired_bootstrap( | |
| diffs: np.ndarray, n_resamples: int, rng: np.random.Generator | |
| ) -> tuple[float, float, float]: | |
| """Percentile CI and the fraction of resamples that flip the sign.""" | |
| idx = rng.integers(0, len(diffs), size=(n_resamples, len(diffs))) | |
| means = diffs[idx].mean(axis=1) | |
| lo, hi = np.percentile(means, [2.5, 97.5]) | |
| observed = diffs.mean() | |
| flipped = float((means <= 0).mean() if observed > 0 else (means >= 0).mean()) | |
| return float(lo), float(hi), flipped | |
| def paired_permutation(diffs: np.ndarray, n_resamples: int, rng: np.random.Generator) -> float: | |
| """Two-sided p-value by randomly flipping the sign of each query's difference.""" | |
| observed = abs(diffs.mean()) | |
| signs = rng.choice([-1.0, 1.0], size=(n_resamples, len(diffs))) | |
| null = np.abs((signs * diffs).mean(axis=1)) | |
| return float((null >= observed).mean()) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", default="configs/rerank_hard_ft.yaml") | |
| parser.add_argument("--model-a", default="models/reranker-random/final", help="control arm") | |
| parser.add_argument("--model-b", default="models/reranker-hard/final", help="treatment arm") | |
| parser.add_argument("--resamples", type=int, default=10000) | |
| parser.add_argument("--out", default="results/significance.json") | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| seed = seed_everything(cfg["seed"]) | |
| rng = np.random.default_rng(seed) | |
| corpus, queries, qrels = load_split( | |
| cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"] | |
| ) | |
| fused = json.loads(Path(cfg["paths"]["fused_run"]).read_text(encoding="utf-8")) | |
| top_k = cfg["rerank"]["top_k"] | |
| scores: dict[str, dict[str, float]] = {} | |
| for label, model_path in (("random", args.model_a), ("hard", args.model_b)): | |
| reranker = Reranker( | |
| model_name_or_path=model_path, | |
| max_length=cfg["reranker"]["max_length"], | |
| batch_size=cfg["reranker"]["batch_size"], | |
| device=device(), | |
| fp16=True, | |
| ) | |
| run, _timings = reranker.rerank(queries, fused, corpus, top_k=top_k) | |
| scores[label] = per_query_ndcg(qrels, run) | |
| print(f"[sig] {label}: mean nDCG@10 = {np.mean(list(scores[label].values())):.4f}") | |
| del reranker | |
| qids = sorted(set(scores["random"]) & set(scores["hard"])) | |
| a = np.array([scores["random"][q] for q in qids]) | |
| b = np.array([scores["hard"][q] for q in qids]) | |
| diffs = b - a | |
| lo, hi, flip = paired_bootstrap(diffs, args.resamples, rng) | |
| p_value = paired_permutation(diffs, args.resamples, rng) | |
| wins = int((diffs > 1e-9).sum()) | |
| losses = int((diffs < -1e-9).sum()) | |
| ties = len(diffs) - wins - losses | |
| result = { | |
| "seed": seed, | |
| "n_queries": len(qids), | |
| "resamples": args.resamples, | |
| "control": {"model": args.model_a, "mean_ndcg@10": round(float(a.mean()), 5)}, | |
| "treatment": {"model": args.model_b, "mean_ndcg@10": round(float(b.mean()), 5)}, | |
| "mean_difference": round(float(diffs.mean()), 5), | |
| "bootstrap_95_ci": [round(lo, 5), round(hi, 5)], | |
| "permutation_p_value": round(p_value, 4), | |
| "per_query": {"treatment_better": wins, "control_better": losses, "tied": ties}, | |
| "significant_at_0.05": bool(p_value < 0.05), | |
| } | |
| print(f"\n[sig] mean difference : {result['mean_difference']:+.5f}") | |
| print(f"[sig] 95% CI : [{lo:+.5f}, {hi:+.5f}]") | |
| print(f"[sig] permutation p : {p_value:.4f}") | |
| print(f"[sig] per-query : {wins} better / {losses} worse / {ties} tied") | |
| print(f"[sig] significant : {result['significant_at_0.05']}") | |
| out = Path(args.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(result, indent=2), encoding="utf-8") | |
| print(f"[sig] wrote {out}") | |
| if __name__ == "__main__": | |
| main() | |