ResearchOS / scripts /profile_latency.py
Kagzy's picture
v2 backend: figures, math prompts, live OpenAlex, 275k corpus via private dataset
e5c059f verified
Raw History Blame Contribute Delete
5.96 kB
"""Where does a search spend its time, and what does cutting depth cost in recall?
python scripts/profile_latency.py --config configs/ml.cpu.yaml --threads 2 --depths 30 20 15
python scripts/profile_latency.py --config configs/ml.yaml --recall --depths 60 30 20
Latency is measured per stage - BM25, dense query encode + FAISS, rank fusion,
cross-encoder rerank - over real queries, for each first-stage depth (how many
fused candidates the reranker scores). Rerank cost is linear in depth, so it is
the lever; recall@10 on the specific and vague query sets (see calibrate_gate.py)
says what pulling it costs. `--threads N` pins torch to N CPU threads to match
a 2-vCPU Space on a bigger machine.
Run one config per process: the pipeline holds the whole index in RAM.
"""
from __future__ import annotations
import argparse
import json
import statistics
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.config import load_config # noqa: E402
def pct(xs: list[float], q: float) -> float:
xs = sorted(xs)
return round(xs[min(len(xs) - 1, int(q * (len(xs) - 1) + 0.5))], 1)
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--config", default="configs/ml.cpu.yaml")
ap.add_argument("--depths", type=int, nargs="+", default=[30])
ap.add_argument("--queries", type=int, default=20, help="queries timed per depth")
ap.add_argument("--threads", type=int, default=0)
ap.add_argument("--recall", action="store_true", help="also measure recall@10 per depth")
ap.add_argument("--out", default=None)
args = ap.parse_args()
import torch
if args.threads:
torch.set_num_threads(args.threads)
from src.api.ml_pipeline import PSEUDO_QID, MLPipeline
from src.retrieval.fusion import reciprocal_rank_fusion
cfg = load_config(args.config)
cfg["citation"]["allow_remote"] = False
cfg.setdefault("diagnosis", {})["enabled"] = False
t0 = time.time()
p = MLPipeline(cfg)
load_s = time.time() - t0
specific = [json.loads(line) for line in open("data/generated_eval.jsonl", encoding="utf-8")]
vague_q = json.loads(Path("results/v2_rescue.cache.json").read_text(encoding="utf-8"))["queries"]
vague = [{"query": q, "gold_paper_id": pid} for pid, q in vague_q.items()]
timed = [it["query"] for it in (specific[::8] + vague[::12])][: args.queries]
report = {"config": args.config, "threads": args.threads or None, "load_s": round(load_s, 1),
"papers": len(p.papers_by_id), "passages": len(p.corpus), "depths": {}}
for depth in args.depths:
p.retrieve_k = depth
st = {"bm25": [], "dense": [], "fuse": [], "rerank": [], "total": []}
for q in timed:
m = {PSEUDO_QID: q}
a = time.perf_counter()
b = p.bm25.search(m, top_k=depth)
t1 = time.perf_counter()
d = p.dense.search(m, top_k=depth)
t2 = time.perf_counter()
fused = reciprocal_rank_fusion([b, d], k=p.rrf_k, top_k=depth)
t3 = time.perf_counter()
p.reranker.rerank(m, fused, p.corpus, top_k=depth)
t4 = time.perf_counter()
for k, v in (("bm25", t1 - a), ("dense", t2 - t1), ("fuse", t3 - t2),
("rerank", t4 - t3), ("total", t4 - a)):
st[k].append(v * 1000)
entry = {k: {"median": round(statistics.median(v), 1), "p95": pct(v, 0.95)}
for k, v in st.items()}
if args.recall:
for name, items in (("specific", specific), ("vague", vague)):
hit = 0
for it in items:
p._rank_cache.clear()
res = p.find_papers(it["query"], top_k=10)
hit += any(x["paper_id"] == it["gold_paper_id"] for x in res["papers"])
entry[f"recall_at_10_{name}"] = round(hit / len(items), 4)
report["depths"][str(depth)] = entry
print(f"[profile] depth {depth}: " + json.dumps(entry), flush=True)
# End to end through _ranked, which is what a request runs: full rerank vs
# the adaptive policy from the config, cache cleared per query.
policy = cfg["retrieval"].get("early_exit")
p.retrieve_k = cfg["retrieval"]["top_k"]
report["end_to_end_ms"] = {}
for label, ee in (("full_rerank", None), ("adaptive", policy)):
if label == "adaptive" and not policy:
continue
cfg["retrieval"]["early_exit"] = ee
p.early_exits = 0
times = []
for q in timed:
p._rank_cache.clear()
a = time.perf_counter()
p._ranked(q)
times.append((time.perf_counter() - a) * 1000)
report["end_to_end_ms"][label] = {
"median": round(statistics.median(times), 1), "p95": pct(times, 0.95),
"mean": round(statistics.mean(times), 1), "early_exits": p.early_exits,
"of": len(times)}
print(f"[profile] end-to-end {label}: {report['end_to_end_ms'][label]}", flush=True)
cfg["retrieval"]["early_exit"] = policy
# The cache: same query twice, as a user retrying or a shared link would.
p._rank_cache.clear()
q = timed[0]
a = time.perf_counter(); p.find_papers(q, 10); cold = (time.perf_counter() - a) * 1000
a = time.perf_counter(); p.find_papers(q, 10); warm = (time.perf_counter() - a) * 1000
report["repeat_query_ms"] = {"cold": round(cold, 1), "cached": round(warm, 1)}
print(json.dumps(report, indent=2))
if args.out:
Path(args.out).write_text(json.dumps(report, indent=2), encoding="utf-8")
if __name__ == "__main__":
main()