Spaces:
Running on Zero
Running on Zero
Download scripts/benchmark_latency.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 4.73 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/benchmark_latency.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/benchmark_latency.py
-
curl -L -o benchmark_latency.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/benchmark_latency.py
4.73 kB
| """Stage 6b - benchmark PyTorch vs ONNX fp32 vs ONNX INT8. | |
| Measures per-query rerank latency over a fixed sample of test queries, each with | |
| the same candidate list, so the only variable is the inference backend. | |
| python scripts/benchmark_latency.py --config configs/onnx.yaml | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from src.config import device, load_config, seed_everything | |
| from src.data.loader import Corpus, load_split, passage_text | |
| from src.protocols import Scorer | |
| from src.rerank.evaluate import latency_stats | |
| def build_pair_batches( | |
| queries: dict[str, str], | |
| fused: dict[str, dict[str, float]], | |
| corpus: Corpus, | |
| qids: list[str], | |
| top_k: int, | |
| ) -> list[list[tuple[str, str]]]: | |
| """One list of (query, passage) pairs per query, identical across backends.""" | |
| batches: list[list[tuple[str, str]]] = [] | |
| for qid in qids: | |
| ranked = sorted(fused[qid].items(), key=lambda kv: kv[1], reverse=True)[:top_k] | |
| batches.append([(queries[qid], passage_text(corpus, d)) for d, _ in ranked]) | |
| return batches | |
| def time_backend( | |
| scorer: Scorer, batches: list[list[tuple[str, str]]], warmup: int | |
| ) -> dict[str, Any]: | |
| """Warm up, then time each query's rerank call.""" | |
| for batch in batches[:warmup]: | |
| scorer.score(batch) | |
| timings_ms: list[float] = [] | |
| total_pairs = 0 | |
| start_all = time.perf_counter() | |
| for batch in batches: | |
| start = time.perf_counter() | |
| scorer.score(batch) | |
| timings_ms.append((time.perf_counter() - start) * 1000.0) | |
| total_pairs += len(batch) | |
| wall_s = time.perf_counter() - start_all | |
| stats = latency_stats(timings_ms) | |
| stats["throughput_pairs_per_s"] = round(total_pairs / wall_s, 1) | |
| stats["throughput_queries_per_s"] = round(len(batches) / wall_s, 2) | |
| return stats | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", required=True) | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| seed = seed_everything(cfg["seed"]) | |
| ec, bc = cfg["export"], cfg["benchmark"] | |
| corpus, queries, _qrels = load_split( | |
| cfg["dataset"]["url"], cfg["dataset"]["split"], cfg["dataset"]["download_dir"] | |
| ) | |
| fused = json.loads(Path(cfg["paths"]["fused_run"]).read_text(encoding="utf-8")) | |
| qids = list(fused.keys())[: bc["num_queries"]] | |
| batches = build_pair_batches(queries, fused, corpus, qids, bc["candidates_per_query"]) | |
| print(f"[bench] {len(batches)} queries x {bc['candidates_per_query']} candidates") | |
| results: dict[str, Any] = { | |
| "seed": seed, | |
| "num_queries": len(batches), | |
| "candidates_per_query": bc["candidates_per_query"], | |
| "backends": {}, | |
| } | |
| # --- PyTorch (GPU if available) --- | |
| from src.rerank.model import Reranker | |
| torch_model = Reranker( | |
| model_name_or_path=ec["source_model"], | |
| max_length=ec["max_length"], | |
| batch_size=bc["batch_size"], | |
| device=device(), | |
| fp16=device() == "cuda", | |
| ) | |
| results["backends"]["pytorch"] = { | |
| "device": device(), | |
| "precision": "fp16" if device() == "cuda" else "fp32", | |
| **time_backend(torch_model, batches, bc["warmup_queries"]), | |
| } | |
| print(f"[bench] pytorch: {results['backends']['pytorch']}") | |
| del torch_model | |
| # --- ONNX Runtime, fp32 and int8, both on CPU --- | |
| from src.rerank.onnx_runtime import OnnxReranker | |
| for label, path in (("onnx_fp32", ec["fp32_path"]), ("onnx_int8", ec["int8_path"])): | |
| if not Path(path).exists(): | |
| print(f"[bench] skipping {label}: {path} not found") | |
| continue | |
| onnx_model = OnnxReranker( | |
| onnx_path=path, | |
| tokenizer_path=ec["tokenizer_dir"], | |
| max_length=ec["max_length"], | |
| batch_size=bc["batch_size"], | |
| ) | |
| results["backends"][label] = { | |
| "device": "cpu", | |
| "precision": "int8" if "int8" in label else "fp32", | |
| "file_size_mb": round(Path(path).stat().st_size / 1e6, 1), | |
| **time_backend(onnx_model, batches, bc["warmup_queries"]), | |
| } | |
| print(f"[bench] {label}: {results['backends'][label]}") | |
| del onnx_model | |
| out = Path(cfg["paths"]["latency"]) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(results, indent=2), encoding="utf-8") | |
| print(f"[bench] wrote {out}") | |
| if __name__ == "__main__": | |
| main() | |