Spaces:
Running on Zero
Running on Zero
Download src/rerank/evaluate.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 3.28 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/rerank/evaluate.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/rerank/evaluate.py
-
curl -L -o evaluate.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/rerank/evaluate.py
3.28 kB
| """Evaluation with the official BEIR evaluator, and the results ledger. | |
| results/metrics.json is the source of truth for the README table. Nothing in | |
| this project is allowed to be print-only. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import statistics | |
| from pathlib import Path | |
| from typing import Any | |
| from beir.retrieval.evaluation import EvaluateRetrieval | |
| Run = dict[str, dict[str, float]] | |
| Qrels = dict[str, dict[str, int]] | |
| # Row order in the README table. Keys must match what the scripts write. | |
| ROW_ORDER = [ | |
| "bm25", | |
| "bm25_dense_rrf", | |
| "offshelf_reranker", | |
| "finetuned_random_negatives", | |
| "finetuned_hard_negatives", | |
| "finetuned_onnx_int8", | |
| ] | |
| ROW_LABELS = { | |
| "bm25": "BM25 only", | |
| "bm25_dense_rrf": "BM25 + dense (RRF)", | |
| "offshelf_reranker": "+ off-the-shelf reranker", | |
| "finetuned_random_negatives": "+ fine-tuned, random negatives", | |
| "finetuned_hard_negatives": "+ fine-tuned, hard negatives", | |
| "finetuned_onnx_int8": "Fine-tuned, ONNX INT8", | |
| } | |
| def evaluate_run(qrels: Qrels, run: Run, k_values: list[int]) -> dict[str, float]: | |
| """Run the BEIR evaluator and flatten its output to a single dict.""" | |
| ndcg, _map, recall, precision = EvaluateRetrieval.evaluate(qrels, run, k_values) | |
| flat: dict[str, float] = {} | |
| for block in (ndcg, _map, recall, precision): | |
| for key, value in block.items(): | |
| flat[key.lower().replace("_", "")] = round(float(value), 5) | |
| return flat | |
| def latency_stats(timings_ms: list[float]) -> dict[str, float]: | |
| """p50/p95/mean over per-query wall-clock timings.""" | |
| if not timings_ms: | |
| return {} | |
| ordered = sorted(timings_ms) | |
| return { | |
| "p50_ms": round(statistics.median(ordered), 2), | |
| "p95_ms": round(ordered[max(0, int(0.95 * len(ordered)) - 1)], 2), | |
| "mean_ms": round(sum(ordered) / len(ordered), 2), | |
| "n_queries": len(ordered), | |
| } | |
| def write_row(row_id: str, payload: dict[str, Any], path: str = "results/metrics.json") -> None: | |
| """Merge one row into the metrics ledger, preserving the other rows.""" | |
| if row_id not in ROW_LABELS: | |
| raise ValueError(f"Unknown row id {row_id!r}; expected one of {list(ROW_LABELS)}") | |
| target = Path(path) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| ledger: dict[str, Any] = {"rows": {}} | |
| if target.exists(): | |
| ledger = json.loads(target.read_text(encoding="utf-8")) | |
| ledger.setdefault("rows", {}) | |
| ledger["rows"][row_id] = {"label": ROW_LABELS[row_id], **payload} | |
| ordered = {rid: ledger["rows"][rid] for rid in ROW_ORDER if rid in ledger["rows"]} | |
| ledger["rows"] = ordered | |
| target.write_text(json.dumps(ledger, indent=2), encoding="utf-8") | |
| print(f"[results] wrote row {row_id!r} to {target}") | |
| def read_rows(path: str = "results/metrics.json") -> dict[str, Any]: | |
| """Read the ledger back, or an empty dict if it does not exist yet.""" | |
| target = Path(path) | |
| if not target.exists(): | |
| return {} | |
| return json.loads(target.read_text(encoding="utf-8")).get("rows", {}) | |
| def summarise(metrics: dict[str, float]) -> str: | |
| """One-line headline for logs and commit messages.""" | |
| return f"nDCG@10 {metrics.get('ndcg@10', float('nan')):.4f} | Recall@50 {metrics.get('recall@50', float('nan')):.4f}" | |