reverie-api / app /eval.py
Mudit Nautiyal
Deploy Reverie backend
a72b630
Raw History Blame Contribute Delete
3.53 kB
"""Retrieval evaluation harness.
Reads a labelled question set from data/eval_questions.json and reports
per-question hit and recall@k, plus an overall hit rate and mean recall.
No LLM is involved.
python -m app.eval
python -m app.eval --top-k 10
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from app.config import DATA_DIR
from app.search import search
EVAL_PATH: Path = DATA_DIR / "eval_questions.json"
@dataclass
class QResult:
question: str
expected: list[int]
retrieved: list[int]
hit: int
recall: float
def _load_questions() -> list[dict]:
if not EVAL_PATH.exists():
raise FileNotFoundError(f"Eval set not found: {EVAL_PATH}")
with EVAL_PATH.open("r", encoding="utf-8") as f:
data = json.load(f)
if not isinstance(data, list):
raise ValueError("eval_questions.json must be a JSON array.")
for i, item in enumerate(data):
if not isinstance(item, dict) or "question" not in item or "expected_ids" not in item:
raise ValueError(f"Eval item {i} must have 'question' and 'expected_ids'.")
return data
def _evaluate_one(question: str, expected_ids: list[int], top_k: int) -> QResult:
expected_set = set(int(x) for x in expected_ids)
hits = search(question, top_k=top_k)
retrieved = [h.id for h in hits]
retrieved_set = set(retrieved)
overlap = expected_set & retrieved_set
hit = 1 if overlap else 0
recall = (len(overlap) / len(expected_set)) if expected_set else 0.0
return QResult(question=question, expected=list(expected_ids),
retrieved=retrieved, hit=hit, recall=recall)
def _format_id_list(ids: list[int], highlight: set[int]) -> str:
parts = []
for i in ids:
if i in highlight:
parts.append(f"\033[32m{i}\033[0m")
else:
parts.append(str(i))
return "[" + ", ".join(parts) + "]"
def run(top_k: int = 5) -> int:
questions = _load_questions()
print(f"\nReverie retrieval eval · k={top_k} · {len(questions)} questions")
print("=" * 78)
results: list[QResult] = []
for q in questions:
r = _evaluate_one(q["question"], q["expected_ids"], top_k)
results.append(r)
expected_set = set(r.expected)
retrieved_set = set(r.retrieved)
overlap = expected_set & retrieved_set
print()
print(f"Q: {r.question}")
print(f" expected: {_format_id_list(r.expected, overlap)}")
print(f" retrieved: {_format_id_list(r.retrieved, overlap)}")
print(
f" hit: {r.hit} recall@{top_k}: {r.recall:.2f} "
f"({len(overlap)}/{len(expected_set)} expected ids in top-{top_k})"
)
n = len(results)
if n == 0:
print("\nNo questions to evaluate.")
return 1
hit_rate = sum(r.hit for r in results) / n
mean_recall = sum(r.recall for r in results) / n
print()
print("-" * 78)
print(
f"Overall: hit rate = {hit_rate:.2f} "
f"({sum(r.hit for r in results)}/{n}) "
f"mean recall@{top_k} = {mean_recall:.2f}"
)
print()
return 0
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Reverie retrieval eval.")
parser.add_argument("--top-k", type=int, default=5)
args = parser.parse_args(argv)
return run(top_k=args.top_k)
if __name__ == "__main__":
sys.exit(main())