File size: 5,666 Bytes
4f8feb6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#!/usr/bin/env python
"""Score the retrieval stack on BEIR SciFact, a public benchmark.

SciFact: 5,183 scientific abstracts, 300 test claims, graded relevance judgements
(``test.tsv``; any score > 0 counts as relevant). One chunk per abstract (title and
abstract text), so chunk-level retrieval equals document retrieval. The data is
downloaded from the Hugging Face Hub at run time and cached; nothing is committed.

Metrics follow BEIR: nDCG@10 and recall@100. The reranked row reorders a pool of
``k * LUMEN_RERANK_POOL_FACTOR`` candidates, so recall@100 is not reported for it.

Usage:
  python scripts/beir_eval.py                                  # hashing rows only
  python scripts/beir_eval.py --embedder fastembed --rerank    # adds bge-small + rerank
  python scripts/beir_eval.py --limit 50                       # quick smoke run
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT))

from lumen_rag.eval.metrics import ndcg_at_k, recall_at_k  # noqa: E402
from lumen_rag.ingestion.pipeline import ingest_documents  # noqa: E402
from lumen_rag.retrieval import Retriever  # noqa: E402
from lumen_rag.store import VectorStore  # noqa: E402

sys.path.insert(0, str(Path(__file__).parent))
from benchmark import RERANKER, make_embedder  # noqa: E402

NDCG_K, RECALL_K = 10, 100
ONE_CHUNK = 1_000_000  # words: never splits an abstract
OUT = ROOT / "benchmark_results_beir_scifact.json"


def load_scifact() -> tuple[list[dict], dict[str, str], dict[str, set[str]]]:
    try:
        import pyarrow.parquet as pq
        from huggingface_hub import hf_hub_download
    except ImportError as exc:
        raise SystemExit("BEIR evaluation needs the 'beir' extra: pip install 'lumen-rag[beir]'") from exc

    def fetch(repo: str, name: str) -> str:
        return hf_hub_download(repo, name, repo_type="dataset")

    corpus = pq.read_table(fetch("BeIR/scifact", "corpus/corpus-00000-of-00001.parquet")).to_pylist()
    queries = pq.read_table(fetch("BeIR/scifact", "queries/queries-00000-of-00001.parquet")).to_pylist()
    docs = [
        {"id": str(r["_id"]), "text": f"{r['title']}. {r['text']}" if r["title"] else r["text"]}
        for r in corpus
    ]
    query_text = {str(r["_id"]): r["text"] for r in queries}
    qrels: dict[str, set[str]] = {}
    for line in Path(fetch("BeIR/scifact-qrels", "test.tsv")).read_text().splitlines()[1:]:
        qid, did, score = line.split("\t")
        if int(score) > 0:
            qrels.setdefault(qid, set()).add(did)
    return docs, query_text, qrels


def score(retriever: Retriever, mode: str, queries: list[tuple[str, set[str]]], rerank: bool) -> dict:
    ndcg, recall = [], []
    start = time.perf_counter()
    for text, relevant in queries:
        k = NDCG_K if rerank else RECALL_K
        ids: list[str] = []
        for sc in retriever.retrieve(text, k=k, mode=mode):
            if sc.chunk.doc_id not in ids:
                ids.append(sc.chunk.doc_id)
        ndcg.append(ndcg_at_k(ids, relevant, NDCG_K))
        recall.append(recall_at_k(ids, relevant, RECALL_K))
    n = len(queries)
    return {
        "ndcg@10": round(sum(ndcg) / n, 4),
        "recall@100": None if rerank else round(sum(recall) / n, 4),
        "ms_per_query": round((time.perf_counter() - start) * 1000 / n, 1),
    }


def main() -> None:
    parser = argparse.ArgumentParser(description="Lumen RAG on BEIR SciFact")
    parser.add_argument("--embedder", choices=["hashing", "fastembed", "openai"], default="hashing")
    parser.add_argument("--rerank", action="store_true", help=f"add a hybrid + {RERANKER} row")
    parser.add_argument("--limit", type=int, default=0, help="only the first N test queries")
    args = parser.parse_args()

    docs, query_text, qrels = load_scifact()
    queries = [(query_text[q], rel) for q, rel in sorted(qrels.items())]
    if args.limit:
        queries = queries[: args.limit]
    print(f"SciFact: {len(docs)} abstracts, {len(queries)} test queries, embedder {args.embedder}")

    embedder = make_embedder(args.embedder)
    start = time.perf_counter()
    store = ingest_documents(
        docs, store=VectorStore(dim=embedder.dim), embedder=embedder,
        chunk_size=ONE_CHUNK, overlap=0, contextual=False,
    )
    index_s = round(time.perf_counter() - start, 1)

    rows = []
    plain = Retriever(store, embedder, reranker=None)
    for name, mode in (("vector-only", "vector"), ("BM25 only", "bm25"), ("hybrid (BM25 + RRF)", "hybrid")):
        rows.append({"config": name, **score(plain, mode, queries, rerank=False)})
    if args.rerank:
        from lumen_rag.retrieval.rerank import CrossEncoderReranker

        reranked = Retriever(store, embedder, reranker=CrossEncoderReranker(RERANKER))
        rows.append({"config": "hybrid + rerank", **score(reranked, "hybrid", queries, rerank=True)})

    print(f"\nindexing: {index_s}s\n")
    print("| Configuration | nDCG@10 | recall@100 | ms/query |\n|---|---|---|---|")
    for r in rows:
        rec = "-" if r["recall@100"] is None else f"{r['recall@100']:.3f}"
        print(f"| {r['config']} | {r['ndcg@10']:.3f} | {rec} | {r['ms_per_query']:.1f} |")

    out = OUT if args.embedder == "hashing" else OUT.with_name(f"{OUT.stem}_{args.embedder}.json")
    if not args.limit:
        out.write_text(json.dumps({
            "dataset": "BeIR/scifact", "n_docs": len(docs), "n_queries": len(queries),
            "embedder": args.embedder, "index_seconds": index_s, "results": rows,
        }, indent=2))
        print(f"\nWrote {out}")


if __name__ == "__main__":
    main()