"""Can the reranker stop early when the first candidates already look confident? python scripts/eval_early_exit.py --config configs/ml.yaml Reranking is the dominant search cost (linear in candidates scored; seconds on a 2-vCPU CPU). The idea: score only the first `n1` fused candidates, and if their top-5 mean score already clears the confidence bar `tau`, stop; otherwise score the rest. Easy queries - where the answer is obvious - then cost a fraction. This simulates every (n1, tau) policy offline from ONE full rerank per query, so the comparison is exact: the same scores, just a different subset available to the policy. For each policy it reports * recall@10 on the specific and vague query sets - what stopping early costs * the share of queries that exit early, and the mean candidates scored - what it saves (rerank latency is proportional to the latter) * top-10 overlap with the full rerank - whether the user would even notice """ from __future__ import annotations import argparse import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="replace") from src.config import load_config # noqa: E402 def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--config", default="configs/ml.yaml") ap.add_argument("--depth", type=int, default=30) ap.add_argument("--out", default="results/early_exit.json") args = ap.parse_args() from src.api.ml_pipeline import PSEUDO_QID, MLPipeline from src.retrieval.fusion import reciprocal_rank_fusion cfg = load_config(args.config) cfg["citation"]["allow_remote"] = False cfg.setdefault("diagnosis", {})["enabled"] = False p = MLPipeline(cfg) specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")] vague_q = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"] sets = {"specific": [(it["query"], it["gold_paper_id"]) for it in specific], "vague": [(q, pid) for pid, q in vague_q.items()]} # One full rerank per query: (fused order, score per chunk). scored: dict[str, list[tuple[list[str], dict[str, float]]]] = {} for name, items in sets.items(): scored[name] = [] for q, _ in items: m = {PSEUDO_QID: q} fused = reciprocal_rank_fusion( [p.bm25.search(m, top_k=args.depth), p.dense.search(m, top_k=args.depth)], k=p.rrf_k, top_k=args.depth) order = [c for c, _ in sorted(fused[PSEUDO_QID].items(), key=lambda kv: -kv[1])] ranked, _ = p.reranker.rerank(m, fused, p.corpus, top_k=args.depth) scored[name].append((order, ranked[PSEUDO_QID])) print(f"[early-exit] scored {len(items)} {name} queries", flush=True) def top_papers(scores: dict[str, float], k: int = 10) -> list[str]: best: dict[str, float] = {} for cid, s in scores.items(): pid = p.corpus[cid]["paper_id"] best[pid] = max(best.get(pid, float("-inf")), s) return [pid for pid, _ in sorted(best.items(), key=lambda kv: -kv[1])[:k]] def mean_top(scores: dict[str, float], n: int = 5) -> float: top = sorted(scores.values(), reverse=True)[:n] return sum(top) / len(top) if top else 0.0 def run(n1: int | None, tau: float | None) -> dict: out = {} for name, items in sets.items(): hits = overlap = overlap5 = early = scored_total = 0 for (q, gold), (order, scores) in zip(items, scored[name]): first = {c: scores[c] for c in order[:n1] if c in scores} if n1 else scores stop = n1 is not None and mean_top(first) >= tau used = first if stop else scores early += stop scored_total += len(first) if stop else len(scores) hits += gold in top_papers(used) full10, full5 = top_papers(scores), top_papers(scores, 5) # Normalised by the full list's own length: a query that yields # fewer than 10 papers must still be able to score 1.0. overlap += len(set(top_papers(used)) & set(full10)) / max(len(full10), 1) overlap5 += len(set(top_papers(used, 5)) & set(full5)) / max(len(full5), 1) n = len(items) out[name] = {"recall_at_10": round(hits / n, 4), "early_exit": round(early / n, 3), "mean_scored": round(scored_total / n, 1), "top10_overlap_with_full": round(overlap / n, 3), "top5_overlap_with_full": round(overlap5 / n, 3)} return out results = {"full": run(None, None)} for n1 in (10, 15, 20): for tau in (0.5, 0.6, 0.7, 0.8, 0.9): results[f"n1={n1},tau={tau}"] = run(n1, tau) Path(args.out).write_text(json.dumps(results, indent=2), encoding="utf-8") print(f"{'policy':<18}{'spec R@10':>10}{'vague R@10':>11}{'exit%':>7}{'scored':>8}" f"{'top10 same':>11}{'top5 same':>10}") for k, v in results.items(): s, g = v["specific"], v["vague"] exit_all = (s["early_exit"] * 163 + g["early_exit"] * 250) / 413 scored_all = (s["mean_scored"] * 163 + g["mean_scored"] * 250) / 413 ov = (s["top10_overlap_with_full"] * 163 + g["top10_overlap_with_full"] * 250) / 413 ov5 = (s["top5_overlap_with_full"] * 163 + g["top5_overlap_with_full"] * 250) / 413 print(f"{k:<18}{s['recall_at_10']:>10.3f}{g['recall_at_10']:>11.3f}" f"{exit_all:>7.0%}{scored_all:>8.1f}{ov:>11.3f}{ov5:>10.3f}") if __name__ == "__main__": main()