Download scripts/beir_eval.py from WickTech/lumen-rag: direct link, hf CLI and curl.
- Browser
- Download file 5.67 kB
-
https://huggingface.co/spaces/WickTech/lumen-rag/resolve/main/scripts/beir_eval.py
- Command line
-
hf download hf://spaces/WickTech/lumen-rag/scripts/beir_eval.py
-
curl -L -o beir_eval.py https://huggingface.co/spaces/WickTech/lumen-rag/resolve/main/scripts/beir_eval.py
5.67 kB
| #!/usr/bin/env python | |
| """Score the retrieval stack on BEIR SciFact, a public benchmark. | |
| SciFact: 5,183 scientific abstracts, 300 test claims, graded relevance judgements | |
| (``test.tsv``; any score > 0 counts as relevant). One chunk per abstract (title and | |
| abstract text), so chunk-level retrieval equals document retrieval. The data is | |
| downloaded from the Hugging Face Hub at run time and cached; nothing is committed. | |
| Metrics follow BEIR: nDCG@10 and recall@100. The reranked row reorders a pool of | |
| ``k * LUMEN_RERANK_POOL_FACTOR`` candidates, so recall@100 is not reported for it. | |
| Usage: | |
| python scripts/beir_eval.py # hashing rows only | |
| python scripts/beir_eval.py --embedder fastembed --rerank # adds bge-small + rerank | |
| python scripts/beir_eval.py --limit 50 # quick smoke run | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| ROOT = Path(__file__).parent.parent | |
| sys.path.insert(0, str(ROOT)) | |
| from lumen_rag.eval.metrics import ndcg_at_k, recall_at_k # noqa: E402 | |
| from lumen_rag.ingestion.pipeline import ingest_documents # noqa: E402 | |
| from lumen_rag.retrieval import Retriever # noqa: E402 | |
| from lumen_rag.store import VectorStore # noqa: E402 | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| from benchmark import RERANKER, make_embedder # noqa: E402 | |
| NDCG_K, RECALL_K = 10, 100 | |
| ONE_CHUNK = 1_000_000 # words: never splits an abstract | |
| OUT = ROOT / "benchmark_results_beir_scifact.json" | |
| def load_scifact() -> tuple[list[dict], dict[str, str], dict[str, set[str]]]: | |
| try: | |
| import pyarrow.parquet as pq | |
| from huggingface_hub import hf_hub_download | |
| except ImportError as exc: | |
| raise SystemExit("BEIR evaluation needs the 'beir' extra: pip install 'lumen-rag[beir]'") from exc | |
| def fetch(repo: str, name: str) -> str: | |
| return hf_hub_download(repo, name, repo_type="dataset") | |
| corpus = pq.read_table(fetch("BeIR/scifact", "corpus/corpus-00000-of-00001.parquet")).to_pylist() | |
| queries = pq.read_table(fetch("BeIR/scifact", "queries/queries-00000-of-00001.parquet")).to_pylist() | |
| docs = [ | |
| {"id": str(r["_id"]), "text": f"{r['title']}. {r['text']}" if r["title"] else r["text"]} | |
| for r in corpus | |
| ] | |
| query_text = {str(r["_id"]): r["text"] for r in queries} | |
| qrels: dict[str, set[str]] = {} | |
| for line in Path(fetch("BeIR/scifact-qrels", "test.tsv")).read_text().splitlines()[1:]: | |
| qid, did, score = line.split("\t") | |
| if int(score) > 0: | |
| qrels.setdefault(qid, set()).add(did) | |
| return docs, query_text, qrels | |
| def score(retriever: Retriever, mode: str, queries: list[tuple[str, set[str]]], rerank: bool) -> dict: | |
| ndcg, recall = [], [] | |
| start = time.perf_counter() | |
| for text, relevant in queries: | |
| k = NDCG_K if rerank else RECALL_K | |
| ids: list[str] = [] | |
| for sc in retriever.retrieve(text, k=k, mode=mode): | |
| if sc.chunk.doc_id not in ids: | |
| ids.append(sc.chunk.doc_id) | |
| ndcg.append(ndcg_at_k(ids, relevant, NDCG_K)) | |
| recall.append(recall_at_k(ids, relevant, RECALL_K)) | |
| n = len(queries) | |
| return { | |
| "ndcg@10": round(sum(ndcg) / n, 4), | |
| "recall@100": None if rerank else round(sum(recall) / n, 4), | |
| "ms_per_query": round((time.perf_counter() - start) * 1000 / n, 1), | |
| } | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description="Lumen RAG on BEIR SciFact") | |
| parser.add_argument("--embedder", choices=["hashing", "fastembed", "openai"], default="hashing") | |
| parser.add_argument("--rerank", action="store_true", help=f"add a hybrid + {RERANKER} row") | |
| parser.add_argument("--limit", type=int, default=0, help="only the first N test queries") | |
| args = parser.parse_args() | |
| docs, query_text, qrels = load_scifact() | |
| queries = [(query_text[q], rel) for q, rel in sorted(qrels.items())] | |
| if args.limit: | |
| queries = queries[: args.limit] | |
| print(f"SciFact: {len(docs)} abstracts, {len(queries)} test queries, embedder {args.embedder}") | |
| embedder = make_embedder(args.embedder) | |
| start = time.perf_counter() | |
| store = ingest_documents( | |
| docs, store=VectorStore(dim=embedder.dim), embedder=embedder, | |
| chunk_size=ONE_CHUNK, overlap=0, contextual=False, | |
| ) | |
| index_s = round(time.perf_counter() - start, 1) | |
| rows = [] | |
| plain = Retriever(store, embedder, reranker=None) | |
| for name, mode in (("vector-only", "vector"), ("BM25 only", "bm25"), ("hybrid (BM25 + RRF)", "hybrid")): | |
| rows.append({"config": name, **score(plain, mode, queries, rerank=False)}) | |
| if args.rerank: | |
| from lumen_rag.retrieval.rerank import CrossEncoderReranker | |
| reranked = Retriever(store, embedder, reranker=CrossEncoderReranker(RERANKER)) | |
| rows.append({"config": "hybrid + rerank", **score(reranked, "hybrid", queries, rerank=True)}) | |
| print(f"\nindexing: {index_s}s\n") | |
| print("| Configuration | nDCG@10 | recall@100 | ms/query |\n|---|---|---|---|") | |
| for r in rows: | |
| rec = "-" if r["recall@100"] is None else f"{r['recall@100']:.3f}" | |
| print(f"| {r['config']} | {r['ndcg@10']:.3f} | {rec} | {r['ms_per_query']:.1f} |") | |
| out = OUT if args.embedder == "hashing" else OUT.with_name(f"{OUT.stem}_{args.embedder}.json") | |
| if not args.limit: | |
| out.write_text(json.dumps({ | |
| "dataset": "BeIR/scifact", "n_docs": len(docs), "n_queries": len(queries), | |
| "embedder": args.embedder, "index_seconds": index_s, "results": rows, | |
| }, indent=2)) | |
| print(f"\nWrote {out}") | |
| if __name__ == "__main__": | |
| main() | |