File size: 5,924 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
"""Can the reranker stop early when the first candidates already look confident?



    python scripts/eval_early_exit.py --config configs/ml.yaml



Reranking is the dominant search cost (linear in candidates scored; seconds on a

2-vCPU CPU). The idea: score only the first `n1` fused candidates, and if their

top-5 mean score already clears the confidence bar `tau`, stop; otherwise score

the rest. Easy queries - where the answer is obvious - then cost a fraction.



This simulates every (n1, tau) policy offline from ONE full rerank per query,

so the comparison is exact: the same scores, just a different subset available

to the policy. For each policy it reports



  * recall@10 on the specific and vague query sets - what stopping early costs

  * the share of queries that exit early, and the mean candidates scored -

    what it saves (rerank latency is proportional to the latter)

  * top-10 overlap with the full rerank - whether the user would even notice

"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from src.config import load_config  # noqa: E402


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--config", default="configs/ml.yaml")
    ap.add_argument("--depth", type=int, default=30)
    ap.add_argument("--out", default="results/early_exit.json")
    args = ap.parse_args()

    from src.api.ml_pipeline import PSEUDO_QID, MLPipeline
    from src.retrieval.fusion import reciprocal_rank_fusion

    cfg = load_config(args.config)
    cfg["citation"]["allow_remote"] = False
    cfg.setdefault("diagnosis", {})["enabled"] = False
    p = MLPipeline(cfg)

    specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
    vague_q = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
    sets = {"specific": [(it["query"], it["gold_paper_id"]) for it in specific],
            "vague": [(q, pid) for pid, q in vague_q.items()]}

    # One full rerank per query: (fused order, score per chunk).
    scored: dict[str, list[tuple[list[str], dict[str, float]]]] = {}
    for name, items in sets.items():
        scored[name] = []
        for q, _ in items:
            m = {PSEUDO_QID: q}
            fused = reciprocal_rank_fusion(
                [p.bm25.search(m, top_k=args.depth), p.dense.search(m, top_k=args.depth)],
                k=p.rrf_k, top_k=args.depth)
            order = [c for c, _ in sorted(fused[PSEUDO_QID].items(), key=lambda kv: -kv[1])]
            ranked, _ = p.reranker.rerank(m, fused, p.corpus, top_k=args.depth)
            scored[name].append((order, ranked[PSEUDO_QID]))
        print(f"[early-exit] scored {len(items)} {name} queries", flush=True)

    def top_papers(scores: dict[str, float], k: int = 10) -> list[str]:
        best: dict[str, float] = {}
        for cid, s in scores.items():
            pid = p.corpus[cid]["paper_id"]
            best[pid] = max(best.get(pid, float("-inf")), s)
        return [pid for pid, _ in sorted(best.items(), key=lambda kv: -kv[1])[:k]]

    def mean_top(scores: dict[str, float], n: int = 5) -> float:
        top = sorted(scores.values(), reverse=True)[:n]
        return sum(top) / len(top) if top else 0.0

    def run(n1: int | None, tau: float | None) -> dict:
        out = {}
        for name, items in sets.items():
            hits = overlap = overlap5 = early = scored_total = 0
            for (q, gold), (order, scores) in zip(items, scored[name]):
                first = {c: scores[c] for c in order[:n1] if c in scores} if n1 else scores
                stop = n1 is not None and mean_top(first) >= tau
                used = first if stop else scores
                early += stop
                scored_total += len(first) if stop else len(scores)
                hits += gold in top_papers(used)
                full10, full5 = top_papers(scores), top_papers(scores, 5)
                # Normalised by the full list's own length: a query that yields
                # fewer than 10 papers must still be able to score 1.0.
                overlap += len(set(top_papers(used)) & set(full10)) / max(len(full10), 1)
                overlap5 += len(set(top_papers(used, 5)) & set(full5)) / max(len(full5), 1)
            n = len(items)
            out[name] = {"recall_at_10": round(hits / n, 4), "early_exit": round(early / n, 3),
                         "mean_scored": round(scored_total / n, 1),
                         "top10_overlap_with_full": round(overlap / n, 3),
                         "top5_overlap_with_full": round(overlap5 / n, 3)}
        return out

    results = {"full": run(None, None)}
    for n1 in (10, 15, 20):
        for tau in (0.5, 0.6, 0.7, 0.8, 0.9):
            results[f"n1={n1},tau={tau}"] = run(n1, tau)
    Path(args.out).write_text(json.dumps(results, indent=2), encoding="utf-8")
    print(f"{'policy':<18}{'spec R@10':>10}{'vague R@10':>11}{'exit%':>7}{'scored':>8}"
          f"{'top10 same':>11}{'top5 same':>10}")
    for k, v in results.items():
        s, g = v["specific"], v["vague"]
        exit_all = (s["early_exit"] * 163 + g["early_exit"] * 250) / 413
        scored_all = (s["mean_scored"] * 163 + g["mean_scored"] * 250) / 413
        ov = (s["top10_overlap_with_full"] * 163 + g["top10_overlap_with_full"] * 250) / 413
        ov5 = (s["top5_overlap_with_full"] * 163 + g["top5_overlap_with_full"] * 250) / 413
        print(f"{k:<18}{s['recall_at_10']:>10.3f}{g['recall_at_10']:>11.3f}"
              f"{exit_all:>7.0%}{scored_all:>8.1f}{ov:>11.3f}{ov5:>10.3f}")


if __name__ == "__main__":
    main()