Spaces:
Running on Zero
Running on Zero
File size: 5,924 Bytes
e5c059f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | """Can the reranker stop early when the first candidates already look confident?
python scripts/eval_early_exit.py --config configs/ml.yaml
Reranking is the dominant search cost (linear in candidates scored; seconds on a
2-vCPU CPU). The idea: score only the first `n1` fused candidates, and if their
top-5 mean score already clears the confidence bar `tau`, stop; otherwise score
the rest. Easy queries - where the answer is obvious - then cost a fraction.
This simulates every (n1, tau) policy offline from ONE full rerank per query,
so the comparison is exact: the same scores, just a different subset available
to the policy. For each policy it reports
* recall@10 on the specific and vague query sets - what stopping early costs
* the share of queries that exit early, and the mean candidates scored -
what it saves (rerank latency is proportional to the latter)
* top-10 overlap with the full rerank - whether the user would even notice
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.config import load_config # noqa: E402
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--config", default="configs/ml.yaml")
ap.add_argument("--depth", type=int, default=30)
ap.add_argument("--out", default="results/early_exit.json")
args = ap.parse_args()
from src.api.ml_pipeline import PSEUDO_QID, MLPipeline
from src.retrieval.fusion import reciprocal_rank_fusion
cfg = load_config(args.config)
cfg["citation"]["allow_remote"] = False
cfg.setdefault("diagnosis", {})["enabled"] = False
p = MLPipeline(cfg)
specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
vague_q = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
sets = {"specific": [(it["query"], it["gold_paper_id"]) for it in specific],
"vague": [(q, pid) for pid, q in vague_q.items()]}
# One full rerank per query: (fused order, score per chunk).
scored: dict[str, list[tuple[list[str], dict[str, float]]]] = {}
for name, items in sets.items():
scored[name] = []
for q, _ in items:
m = {PSEUDO_QID: q}
fused = reciprocal_rank_fusion(
[p.bm25.search(m, top_k=args.depth), p.dense.search(m, top_k=args.depth)],
k=p.rrf_k, top_k=args.depth)
order = [c for c, _ in sorted(fused[PSEUDO_QID].items(), key=lambda kv: -kv[1])]
ranked, _ = p.reranker.rerank(m, fused, p.corpus, top_k=args.depth)
scored[name].append((order, ranked[PSEUDO_QID]))
print(f"[early-exit] scored {len(items)} {name} queries", flush=True)
def top_papers(scores: dict[str, float], k: int = 10) -> list[str]:
best: dict[str, float] = {}
for cid, s in scores.items():
pid = p.corpus[cid]["paper_id"]
best[pid] = max(best.get(pid, float("-inf")), s)
return [pid for pid, _ in sorted(best.items(), key=lambda kv: -kv[1])[:k]]
def mean_top(scores: dict[str, float], n: int = 5) -> float:
top = sorted(scores.values(), reverse=True)[:n]
return sum(top) / len(top) if top else 0.0
def run(n1: int | None, tau: float | None) -> dict:
out = {}
for name, items in sets.items():
hits = overlap = overlap5 = early = scored_total = 0
for (q, gold), (order, scores) in zip(items, scored[name]):
first = {c: scores[c] for c in order[:n1] if c in scores} if n1 else scores
stop = n1 is not None and mean_top(first) >= tau
used = first if stop else scores
early += stop
scored_total += len(first) if stop else len(scores)
hits += gold in top_papers(used)
full10, full5 = top_papers(scores), top_papers(scores, 5)
# Normalised by the full list's own length: a query that yields
# fewer than 10 papers must still be able to score 1.0.
overlap += len(set(top_papers(used)) & set(full10)) / max(len(full10), 1)
overlap5 += len(set(top_papers(used, 5)) & set(full5)) / max(len(full5), 1)
n = len(items)
out[name] = {"recall_at_10": round(hits / n, 4), "early_exit": round(early / n, 3),
"mean_scored": round(scored_total / n, 1),
"top10_overlap_with_full": round(overlap / n, 3),
"top5_overlap_with_full": round(overlap5 / n, 3)}
return out
results = {"full": run(None, None)}
for n1 in (10, 15, 20):
for tau in (0.5, 0.6, 0.7, 0.8, 0.9):
results[f"n1={n1},tau={tau}"] = run(n1, tau)
Path(args.out).write_text(json.dumps(results, indent=2), encoding="utf-8")
print(f"{'policy':<18}{'spec R@10':>10}{'vague R@10':>11}{'exit%':>7}{'scored':>8}"
f"{'top10 same':>11}{'top5 same':>10}")
for k, v in results.items():
s, g = v["specific"], v["vague"]
exit_all = (s["early_exit"] * 163 + g["early_exit"] * 250) / 413
scored_all = (s["mean_scored"] * 163 + g["mean_scored"] * 250) / 413
ov = (s["top10_overlap_with_full"] * 163 + g["top10_overlap_with_full"] * 250) / 413
ov5 = (s["top5_overlap_with_full"] * 163 + g["top5_overlap_with_full"] * 250) / 413
print(f"{k:<18}{s['recall_at_10']:>10.3f}{g['recall_at_10']:>11.3f}"
f"{exit_all:>7.0%}{scored_all:>8.1f}{ov:>11.3f}{ov5:>10.3f}")
if __name__ == "__main__":
main()
|