Spaces:
Sleeping
Sleeping
Download app/eval.py from Dragon-21/reverie-api: direct link, hf CLI and curl.
- Browser
- Download file 3.53 kB
-
https://huggingface.co/spaces/Dragon-21/reverie-api/resolve/main/app/eval.py
- Command line
-
hf download hf://spaces/Dragon-21/reverie-api/app/eval.py
-
curl -L -o eval.py https://huggingface.co/spaces/Dragon-21/reverie-api/resolve/main/app/eval.py
3.53 kB
| """Retrieval evaluation harness. | |
| Reads a labelled question set from data/eval_questions.json and reports | |
| per-question hit and recall@k, plus an overall hit rate and mean recall. | |
| No LLM is involved. | |
| python -m app.eval | |
| python -m app.eval --top-k 10 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from app.config import DATA_DIR | |
| from app.search import search | |
| EVAL_PATH: Path = DATA_DIR / "eval_questions.json" | |
| class QResult: | |
| question: str | |
| expected: list[int] | |
| retrieved: list[int] | |
| hit: int | |
| recall: float | |
| def _load_questions() -> list[dict]: | |
| if not EVAL_PATH.exists(): | |
| raise FileNotFoundError(f"Eval set not found: {EVAL_PATH}") | |
| with EVAL_PATH.open("r", encoding="utf-8") as f: | |
| data = json.load(f) | |
| if not isinstance(data, list): | |
| raise ValueError("eval_questions.json must be a JSON array.") | |
| for i, item in enumerate(data): | |
| if not isinstance(item, dict) or "question" not in item or "expected_ids" not in item: | |
| raise ValueError(f"Eval item {i} must have 'question' and 'expected_ids'.") | |
| return data | |
| def _evaluate_one(question: str, expected_ids: list[int], top_k: int) -> QResult: | |
| expected_set = set(int(x) for x in expected_ids) | |
| hits = search(question, top_k=top_k) | |
| retrieved = [h.id for h in hits] | |
| retrieved_set = set(retrieved) | |
| overlap = expected_set & retrieved_set | |
| hit = 1 if overlap else 0 | |
| recall = (len(overlap) / len(expected_set)) if expected_set else 0.0 | |
| return QResult(question=question, expected=list(expected_ids), | |
| retrieved=retrieved, hit=hit, recall=recall) | |
| def _format_id_list(ids: list[int], highlight: set[int]) -> str: | |
| parts = [] | |
| for i in ids: | |
| if i in highlight: | |
| parts.append(f"\033[32m{i}\033[0m") | |
| else: | |
| parts.append(str(i)) | |
| return "[" + ", ".join(parts) + "]" | |
| def run(top_k: int = 5) -> int: | |
| questions = _load_questions() | |
| print(f"\nReverie retrieval eval · k={top_k} · {len(questions)} questions") | |
| print("=" * 78) | |
| results: list[QResult] = [] | |
| for q in questions: | |
| r = _evaluate_one(q["question"], q["expected_ids"], top_k) | |
| results.append(r) | |
| expected_set = set(r.expected) | |
| retrieved_set = set(r.retrieved) | |
| overlap = expected_set & retrieved_set | |
| print() | |
| print(f"Q: {r.question}") | |
| print(f" expected: {_format_id_list(r.expected, overlap)}") | |
| print(f" retrieved: {_format_id_list(r.retrieved, overlap)}") | |
| print( | |
| f" hit: {r.hit} recall@{top_k}: {r.recall:.2f} " | |
| f"({len(overlap)}/{len(expected_set)} expected ids in top-{top_k})" | |
| ) | |
| n = len(results) | |
| if n == 0: | |
| print("\nNo questions to evaluate.") | |
| return 1 | |
| hit_rate = sum(r.hit for r in results) / n | |
| mean_recall = sum(r.recall for r in results) / n | |
| print() | |
| print("-" * 78) | |
| print( | |
| f"Overall: hit rate = {hit_rate:.2f} " | |
| f"({sum(r.hit for r in results)}/{n}) " | |
| f"mean recall@{top_k} = {mean_recall:.2f}" | |
| ) | |
| print() | |
| return 0 | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description="Reverie retrieval eval.") | |
| parser.add_argument("--top-k", type=int, default=5) | |
| args = parser.parse_args(argv) | |
| return run(top_k=args.top_k) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |