ResearchOS / src /rerank /evaluate.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
3.28 kB
"""Evaluation with the official BEIR evaluator, and the results ledger.
results/metrics.json is the source of truth for the README table. Nothing in
this project is allowed to be print-only.
"""
from __future__ import annotations
import json
import statistics
from pathlib import Path
from typing import Any
from beir.retrieval.evaluation import EvaluateRetrieval
Run = dict[str, dict[str, float]]
Qrels = dict[str, dict[str, int]]
# Row order in the README table. Keys must match what the scripts write.
ROW_ORDER = [
"bm25",
"bm25_dense_rrf",
"offshelf_reranker",
"finetuned_random_negatives",
"finetuned_hard_negatives",
"finetuned_onnx_int8",
]
ROW_LABELS = {
"bm25": "BM25 only",
"bm25_dense_rrf": "BM25 + dense (RRF)",
"offshelf_reranker": "+ off-the-shelf reranker",
"finetuned_random_negatives": "+ fine-tuned, random negatives",
"finetuned_hard_negatives": "+ fine-tuned, hard negatives",
"finetuned_onnx_int8": "Fine-tuned, ONNX INT8",
}
def evaluate_run(qrels: Qrels, run: Run, k_values: list[int]) -> dict[str, float]:
"""Run the BEIR evaluator and flatten its output to a single dict."""
ndcg, _map, recall, precision = EvaluateRetrieval.evaluate(qrels, run, k_values)
flat: dict[str, float] = {}
for block in (ndcg, _map, recall, precision):
for key, value in block.items():
flat[key.lower().replace("_", "")] = round(float(value), 5)
return flat
def latency_stats(timings_ms: list[float]) -> dict[str, float]:
"""p50/p95/mean over per-query wall-clock timings."""
if not timings_ms:
return {}
ordered = sorted(timings_ms)
return {
"p50_ms": round(statistics.median(ordered), 2),
"p95_ms": round(ordered[max(0, int(0.95 * len(ordered)) - 1)], 2),
"mean_ms": round(sum(ordered) / len(ordered), 2),
"n_queries": len(ordered),
}
def write_row(row_id: str, payload: dict[str, Any], path: str = "results/metrics.json") -> None:
"""Merge one row into the metrics ledger, preserving the other rows."""
if row_id not in ROW_LABELS:
raise ValueError(f"Unknown row id {row_id!r}; expected one of {list(ROW_LABELS)}")
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
ledger: dict[str, Any] = {"rows": {}}
if target.exists():
ledger = json.loads(target.read_text(encoding="utf-8"))
ledger.setdefault("rows", {})
ledger["rows"][row_id] = {"label": ROW_LABELS[row_id], **payload}
ordered = {rid: ledger["rows"][rid] for rid in ROW_ORDER if rid in ledger["rows"]}
ledger["rows"] = ordered
target.write_text(json.dumps(ledger, indent=2), encoding="utf-8")
print(f"[results] wrote row {row_id!r} to {target}")
def read_rows(path: str = "results/metrics.json") -> dict[str, Any]:
"""Read the ledger back, or an empty dict if it does not exist yet."""
target = Path(path)
if not target.exists():
return {}
return json.loads(target.read_text(encoding="utf-8")).get("rows", {})
def summarise(metrics: dict[str, float]) -> str:
"""One-line headline for logs and commit messages."""
return f"nDCG@10 {metrics.get('ndcg@10', float('nan')):.4f} | Recall@50 {metrics.get('recall@50', float('nan')):.4f}"