ResearchOS / scripts /benchmark_latency.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
4.73 kB
"""Stage 6b - benchmark PyTorch vs ONNX fp32 vs ONNX INT8.
Measures per-query rerank latency over a fixed sample of test queries, each with
the same candidate list, so the only variable is the inference backend.
python scripts/benchmark_latency.py --config configs/onnx.yaml
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.config import device, load_config, seed_everything
from src.data.loader import Corpus, load_split, passage_text
from src.protocols import Scorer
from src.rerank.evaluate import latency_stats
def build_pair_batches(
queries: dict[str, str],
fused: dict[str, dict[str, float]],
corpus: Corpus,
qids: list[str],
top_k: int,
) -> list[list[tuple[str, str]]]:
"""One list of (query, passage) pairs per query, identical across backends."""
batches: list[list[tuple[str, str]]] = []
for qid in qids:
ranked = sorted(fused[qid].items(), key=lambda kv: kv[1], reverse=True)[:top_k]
batches.append([(queries[qid], passage_text(corpus, d)) for d, _ in ranked])
return batches
def time_backend(
scorer: Scorer, batches: list[list[tuple[str, str]]], warmup: int
) -> dict[str, Any]:
"""Warm up, then time each query's rerank call."""
for batch in batches[:warmup]:
scorer.score(batch)
timings_ms: list[float] = []
total_pairs = 0
start_all = time.perf_counter()
for batch in batches:
start = time.perf_counter()
scorer.score(batch)
timings_ms.append((time.perf_counter() - start) * 1000.0)
total_pairs += len(batch)
wall_s = time.perf_counter() - start_all
stats = latency_stats(timings_ms)
stats["throughput_pairs_per_s"] = round(total_pairs / wall_s, 1)
stats["throughput_queries_per_s"] = round(len(batches) / wall_s, 2)
return stats
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", required=True)
args = parser.parse_args()
cfg = load_config(args.config)
seed = seed_everything(cfg["seed"])
ec, bc = cfg["export"], cfg["benchmark"]
corpus, queries, _qrels = load_split(
cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"]
)
fused = json.loads(Path(cfg["paths"]["fused_run"]).read_text(encoding="utf-8"))
qids = list(fused.keys())[: bc["num_queries"]]
batches = build_pair_batches(queries, fused, corpus, qids, bc["candidates_per_query"])
print(f"[bench] {len(batches)} queries x {bc['candidates_per_query']} candidates")
results: dict[str, Any] = {
"seed": seed,
"num_queries": len(batches),
"candidates_per_query": bc["candidates_per_query"],
"backends": {},
}
# --- PyTorch (GPU if available) ---
from src.rerank.model import Reranker
torch_model = Reranker(
model_name_or_path=ec["source_model"],
max_length=ec["max_length"],
batch_size=bc["batch_size"],
device=device(),
fp16=device() == "cuda",
)
results["backends"]["pytorch"] = {
"device": device(),
"precision": "fp16" if device() == "cuda" else "fp32",
**time_backend(torch_model, batches, bc["warmup_queries"]),
}
print(f"[bench] pytorch: {results['backends']['pytorch']}")
del torch_model
# --- ONNX Runtime, fp32 and int8, both on CPU ---
from src.rerank.onnx_runtime import OnnxReranker
for label, path in (("onnx_fp32", ec["fp32_path"]), ("onnx_int8", ec["int8_path"])):
if not Path(path).exists():
print(f"[bench] skipping {label}: {path} not found")
continue
onnx_model = OnnxReranker(
onnx_path=path,
tokenizer_path=ec["tokenizer_dir"],
max_length=ec["max_length"],
batch_size=bc["batch_size"],
)
results["backends"][label] = {
"device": "cpu",
"precision": "int8" if "int8" in label else "fp32",
"file_size_mb": round(Path(path).stat().st_size / 1e6, 1),
**time_backend(onnx_model, batches, bc["warmup_queries"]),
}
print(f"[bench] {label}: {results['backends'][label]}")
del onnx_model
out = Path(cfg["paths"]["latency"])
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
print(f"[bench] wrote {out}")
if __name__ == "__main__":
main()