ResearchOS / scripts /eval_retrieval.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
12.4 kB
"""Evaluate ResearchOS-ML retrieval, and compare configurations.
python scripts/eval_retrieval.py --config configs/ml.yaml
python scripts/eval_retrieval.py --benchmark generated --judge
TWO BENCHMARKS, AND WHY BOTH
----------------------------
arXiv ships no relevance judgements, so a test set had to be manufactured.
Citation-context mining - the principled route, kept in src/data/citation_eval.py
- yielded 12 usable pairs on this corpus, because papers from 2025-11 onward cite
mostly older work that is not indexed. Twelve queries is noise.
sentence A verbatim sentence from a paper's body is the query; gold is that
paper. The chunk it came from is HELD OUT, so the paper must be
found through its other passages. Free, exact labels - but the
query is literal text from the target, which hands BM25 an exact
lexical match. Absolute numbers are optimistic.
generated A model reads the paper and writes the question a researcher would
type to find it (scripts/generate_eval_queries.py). The label is
still structural - the question was written from that paper - but
the phrasing is natural, which is what users actually type. The
prompt forbids reusing the title, and queries that copy it anyway
are rejected.
The `generated` set is the better proxy for the Find-papers feature. The
`sentence` set is kept because it is label-noise-free and shows how much of any
result is a lexical artifact.
FALSE NEGATIVES, AND --judge
----------------------------
Both benchmarks mark exactly one paper relevant per query, so a genuinely good
result that is not the source paper counts as a miss. `--judge` pools the top-k
from every configuration, has a model judge only that pool, and adds the
positives. That recovers most of the loss - at the cost of being a SYNTHETIC
metric, so it is reported as a secondary row, never the headline.
"""
from __future__ import annotations
import argparse
import json
import random
import re
import sys
import time
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
from src.config import device, load_config, seed_everything
from src.data.arxiv_source import read_papers
from src.data.chunking import chunks_to_corpus, read_chunks
from src.rerank.evaluate import evaluate_run, latency_stats
from src.retrieval.bm25 import BM25Retriever
from src.retrieval.dense import DenseRetriever
from src.retrieval.fusion import reciprocal_rank_fusion
SENTENCE_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z])")
BODY_SECTIONS = {"method", "methods", "methodology", "approach", "experiments",
"experimental setup", "results", "analysis", "model", "architecture"}
MIN_WORDS, MAX_WORDS = 12, 45
K_VALUES = [1, 5, 10, 50]
# ------------------------------------------------------------------ benchmarks
def sentence_benchmark(
chunks_path: str, papers_path: str, n_queries: int, rng: random.Random
) -> tuple[dict[str, str], dict[str, dict[str, int]], dict[str, str]]:
"""Body sentence as query; gold is its paper; the source chunk is held out."""
chunks = read_chunks(chunks_path)
full_text = {p.paper_id for p in read_papers(papers_path) if p.full_text}
by_paper: dict[str, list[tuple[str, str]]] = {}
for chunk in chunks:
if chunk.paper_id in full_text and chunk.section in BODY_SECTIONS:
by_paper.setdefault(chunk.paper_id, []).append((chunk.chunk_id, chunk.text))
queries: dict[str, str] = {}
qrels: dict[str, dict[str, int]] = {}
sources: dict[str, str] = {}
candidates = sorted(by_paper)
rng.shuffle(candidates)
for paper_id in candidates:
if len(queries) >= n_queries:
break
if len(by_paper[paper_id]) < 2:
continue
sentences = [
(cid, s.strip())
for cid, text in by_paper[paper_id]
for s in SENTENCE_RE.split(text)
if MIN_WORDS <= len(s.split()) <= MAX_WORDS
]
sentences = [
(cid, s) for cid, s in sentences
if not s.lower().startswith(("in this section", "table", "figure"))
]
if not sentences:
continue
qid = f"s{len(queries)}"
source_chunk, sentence = rng.choice(sentences)
queries[qid], qrels[qid], sources[qid] = sentence, {paper_id: 1}, source_chunk
return queries, qrels, sources
def generated_benchmark(
path: str, n_queries: int
) -> tuple[dict[str, str], dict[str, dict[str, int]], dict[str, str]]:
"""Model-written natural-language queries; gold is the source paper."""
from src.data.query_gen import read_generated
items = read_generated(path)[:n_queries]
queries = {i.query_id: i.query for i in items}
qrels = {i.query_id: {i.gold_paper_id: 1} for i in items}
return queries, qrels, {}
# ------------------------------------------------------------------- machinery
def to_paper_run(
chunk_run: dict[str, dict[str, float]],
corpus: dict[str, dict[str, Any]],
top_k: int,
sources: dict[str, str] | None = None,
) -> dict[str, dict[str, float]]:
"""Collapse chunk-level hits to paper-level, scoring each paper by its best chunk."""
paper_run: dict[str, dict[str, float]] = {}
for qid, hits in chunk_run.items():
best: dict[str, float] = {}
held_out = (sources or {}).get(qid)
for chunk_id, score in hits.items():
if chunk_id == held_out:
continue
pid = corpus[chunk_id]["paper_id"]
if score > best.get(pid, float("-inf")):
best[pid] = float(score)
paper_run[qid] = dict(sorted(best.items(), key=lambda kv: kv[1], reverse=True)[:top_k])
return paper_run
def run_all_configs(
queries: dict[str, str],
corpus: dict[str, dict[str, Any]],
doc_ids: list[str],
doc_texts: list[str],
cfg: dict[str, Any],
sources: dict[str, str],
) -> dict[str, dict[str, Any]]:
"""Run every retrieval configuration over one query set."""
rc, paths = cfg["retrieval"], cfg["paths"]
top_k = rc["top_k"]
out: dict[str, dict[str, Any]] = {}
bm25 = BM25Retriever(k1=rc["bm25"]["k1"], b=rc["bm25"]["b"], stemmer_lang=rc["bm25"]["stemmer"])
bm25.index(doc_ids, doc_texts, cache_dir=paths["index_cache"])
t0 = time.perf_counter()
bm25_run = bm25.search(queries, top_k=top_k)
bm25_ms = [(time.perf_counter() - t0) * 1000.0 / len(queries)] * len(queries)
out["bm25"] = {"label": "BM25 only", "run": to_paper_run(bm25_run, corpus, top_k, sources),
"ms": bm25_ms, "params": None}
dense = DenseRetriever(
model_name=rc["dense"]["model"], batch_size=rc["dense"]["batch_size"],
max_length=rc["dense"]["max_length"], query_prefix=rc["dense"]["query_prefix"],
device=device(),
)
dense.index_corpus(doc_ids, doc_texts, cache_dir=paths["index_cache"])
t0 = time.perf_counter()
dense_run = dense.search(queries, top_k=top_k)
dense_ms = [(time.perf_counter() - t0) * 1000.0 / len(queries)] * len(queries)
out["dense"] = {"label": "Dense only (bge-small)", "run": to_paper_run(dense_run, corpus, top_k, sources),
"ms": dense_ms, "params": "33M"}
t0 = time.perf_counter()
fused = reciprocal_rank_fusion([bm25_run, dense_run], k=rc["fusion"]["rrf_k"], top_k=top_k)
fuse_ms = [(time.perf_counter() - t0) * 1000.0 / len(queries) + b + d
for b, d in zip(bm25_ms, dense_ms)]
out["rrf"] = {"label": "BM25 + dense (RRF)", "run": to_paper_run(fused, corpus, top_k, sources),
"ms": fuse_ms, "params": "33M"}
from src.rerank.model import Reranker
mc = cfg["reranker"]
reranker = Reranker(
model_name_or_path=mc["model"], max_length=mc["max_length"],
batch_size=mc["batch_size"], device=device(),
fp16=mc.get("fp16", True) and device() == "cuda",
)
reranked, rerank_ms = reranker.rerank(queries, fused, corpus, top_k=top_k)
out["rrf_rerank"] = {
"label": "+ cross-encoder reranker",
"run": to_paper_run(reranked, corpus, top_k, sources),
"ms": [f + r for f, r in zip(fuse_ms, rerank_ms)],
"params": "278M",
}
return out
def score(configs: dict[str, dict[str, Any]], qrels: dict[str, dict[str, int]], banner: str) -> dict[str, Any]:
print(f"\n--- {banner} ---")
rows: dict[str, Any] = {}
for name, data in configs.items():
metrics = evaluate_run(qrels, data["run"], K_VALUES)
rows[name] = {"label": data["label"], "metrics": metrics,
"latency": latency_stats(data["ms"]), "params": data["params"]}
print(f" {data['label']:32s} nDCG@10={metrics['ndcg@10']:.4f} "
f"R@10={metrics['recall@10']:.4f} MAP={metrics['map@10']:.4f}")
return rows
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", default="configs/ml.yaml")
parser.add_argument("--benchmark", choices=["sentence", "generated", "both"], default="both")
parser.add_argument("--n-queries", type=int, default=300)
parser.add_argument("--generated-path", default="data/generated_eval.jsonl")
parser.add_argument("--judge", action="store_true", help="pool + LLM-judge to recover false negatives")
parser.add_argument("--judge-depth", type=int, default=10)
parser.add_argument("--out", default="results/ml_metrics.json")
args = parser.parse_args()
cfg = load_config(args.config)
seed = seed_everything(cfg["seed"])
rng = random.Random(seed)
paths = cfg["paths"]
corpus = chunks_to_corpus(read_chunks(paths["chunks"]))
doc_ids = list(corpus)
doc_texts = [f"{corpus[d]['title']} {corpus[d]['text']}".strip() for d in doc_ids]
print(f"[eval] corpus: {len(doc_ids)} chunks")
results: dict[str, Any] = {"seed": seed, "benchmarks": {}}
wanted = ["sentence", "generated"] if args.benchmark == "both" else [args.benchmark]
for kind in wanted:
if kind == "sentence":
queries, qrels, sources = sentence_benchmark(
paths["chunks"], paths["papers"], args.n_queries, rng
)
note = "verbatim body sentence; source chunk held out; favours lexical matching"
else:
gen_path = Path(args.generated_path)
if not gen_path.exists():
print(f"[eval] skipping generated: {gen_path} not found "
"(run scripts/generate_eval_queries.py)")
continue
queries, qrels, sources = generated_benchmark(str(gen_path), args.n_queries)
note = "model-written natural-language query; label is structural"
if not queries:
continue
print(f"\n[eval] benchmark '{kind}': {len(queries)} queries")
configs = run_all_configs(queries, corpus, doc_ids, doc_texts, cfg, sources)
block: dict[str, Any] = {
"n_queries": len(queries),
"note": note,
"rows": score(configs, qrels, f"{kind}: structural labels (primary)"),
}
if args.judge and kind == "generated":
from src.data.judge import build_pool, judge_pool
papers_by_id = {p.paper_id: vars(p) for p in read_papers(paths["papers"])}
pool = build_pool({k: v["run"] for k, v in configs.items()}, depth=args.judge_depth)
merged = judge_pool(
pool, queries, papers_by_id, qrels,
model=cfg["answer"]["model"],
out_path="data/pooled_judgements.jsonl",
)
block["rows_pooled"] = score(
configs, merged, f"{kind}: + pooled LLM judgements (SECONDARY, synthetic)"
)
block["pooled_note"] = (
"Relevance judged by an LLM over the pooled top-k of every configuration. "
"Synthetic signal - secondary to the structural rows above."
)
results["benchmarks"][kind] = block
out = Path(args.out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
print(f"\n[eval] wrote {out}")
if __name__ == "__main__":
main()