File size: 5,960 Bytes
e5c059f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"""Where does a search spend its time, and what does cutting depth cost in recall?



    python scripts/profile_latency.py --config configs/ml.cpu.yaml --threads 2 --depths 30 20 15

    python scripts/profile_latency.py --config configs/ml.yaml --recall --depths 60 30 20



Latency is measured per stage - BM25, dense query encode + FAISS, rank fusion,

cross-encoder rerank - over real queries, for each first-stage depth (how many

fused candidates the reranker scores). Rerank cost is linear in depth, so it is

the lever; recall@10 on the specific and vague query sets (see calibrate_gate.py)

says what pulling it costs. `--threads N` pins torch to N CPU threads to match

a 2-vCPU Space on a bigger machine.



Run one config per process: the pipeline holds the whole index in RAM.

"""

from __future__ import annotations

import argparse
import json
import statistics
import sys
import time
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")

from src.config import load_config  # noqa: E402


def pct(xs: list[float], q: float) -> float:
    xs = sorted(xs)
    return round(xs[min(len(xs) - 1, int(q * (len(xs) - 1) + 0.5))], 1)


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--config", default="configs/ml.cpu.yaml")
    ap.add_argument("--depths", type=int, nargs="+", default=[30])
    ap.add_argument("--queries", type=int, default=20, help="queries timed per depth")
    ap.add_argument("--threads", type=int, default=0)
    ap.add_argument("--recall", action="store_true", help="also measure recall@10 per depth")
    ap.add_argument("--out", default=None)
    args = ap.parse_args()

    import torch
    if args.threads:
        torch.set_num_threads(args.threads)

    from src.api.ml_pipeline import PSEUDO_QID, MLPipeline
    from src.retrieval.fusion import reciprocal_rank_fusion

    cfg = load_config(args.config)
    cfg["citation"]["allow_remote"] = False
    cfg.setdefault("diagnosis", {})["enabled"] = False
    t0 = time.time()
    p = MLPipeline(cfg)
    load_s = time.time() - t0

    specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
    vague_q = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
    vague = [{"query": q, "gold_paper_id": pid} for pid, q in vague_q.items()]
    timed = [it["query"] for it in (specific[::8] + vague[::12])][: args.queries]

    report = {"config": args.config, "threads": args.threads or None, "load_s": round(load_s, 1),
              "papers": len(p.papers_by_id), "passages": len(p.corpus), "depths": {}}
    for depth in args.depths:
        p.retrieve_k = depth
        st = {"bm25": [], "dense": [], "fuse": [], "rerank": [], "total": []}
        for q in timed:
            m = {PSEUDO_QID: q}
            a = time.perf_counter()
            b = p.bm25.search(m, top_k=depth)
            t1 = time.perf_counter()
            d = p.dense.search(m, top_k=depth)
            t2 = time.perf_counter()
            fused = reciprocal_rank_fusion([b, d], k=p.rrf_k, top_k=depth)
            t3 = time.perf_counter()
            p.reranker.rerank(m, fused, p.corpus, top_k=depth)
            t4 = time.perf_counter()
            for k, v in (("bm25", t1 - a), ("dense", t2 - t1), ("fuse", t3 - t2),
                         ("rerank", t4 - t3), ("total", t4 - a)):
                st[k].append(v * 1000)
        entry = {k: {"median": round(statistics.median(v), 1), "p95": pct(v, 0.95)}
                 for k, v in st.items()}
        if args.recall:
            for name, items in (("specific", specific), ("vague", vague)):
                hit = 0
                for it in items:
                    p._rank_cache.clear()
                    res = p.find_papers(it["query"], top_k=10)
                    hit += any(x["paper_id"] == it["gold_paper_id"] for x in res["papers"])
                entry[f"recall_at_10_{name}"] = round(hit / len(items), 4)
        report["depths"][str(depth)] = entry
        print(f"[profile] depth {depth}: " + json.dumps(entry), flush=True)

    # End to end through _ranked, which is what a request runs: full rerank vs
    # the adaptive policy from the config, cache cleared per query.
    policy = cfg["retrieval"].get("early_exit")
    p.retrieve_k = cfg["retrieval"]["top_k"]
    report["end_to_end_ms"] = {}
    for label, ee in (("full_rerank", None), ("adaptive", policy)):
        if label == "adaptive" and not policy:
            continue
        cfg["retrieval"]["early_exit"] = ee
        p.early_exits = 0
        times = []
        for q in timed:
            p._rank_cache.clear()
            a = time.perf_counter()
            p._ranked(q)
            times.append((time.perf_counter() - a) * 1000)
        report["end_to_end_ms"][label] = {
            "median": round(statistics.median(times), 1), "p95": pct(times, 0.95),
            "mean": round(statistics.mean(times), 1), "early_exits": p.early_exits,
            "of": len(times)}
        print(f"[profile] end-to-end {label}: {report['end_to_end_ms'][label]}", flush=True)
    cfg["retrieval"]["early_exit"] = policy

    # The cache: same query twice, as a user retrying or a shared link would.
    p._rank_cache.clear()
    q = timed[0]
    a = time.perf_counter(); p.find_papers(q, 10); cold = (time.perf_counter() - a) * 1000
    a = time.perf_counter(); p.find_papers(q, 10); warm = (time.perf_counter() - a) * 1000
    report["repeat_query_ms"] = {"cold": round(cold, 1), "cached": round(warm, 1)}
    print(json.dumps(report, indent=2))
    if args.out:
        Path(args.out).write_text(json.dumps(report, indent=2), encoding="utf-8")


if __name__ == "__main__":
    main()