Spaces:
Running on Zero
Running on Zero
Download scripts/eval_retrieval.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/eval_retrieval.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/eval_retrieval.py
-
curl -L -o eval_retrieval.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/eval_retrieval.py
12.4 kB
| """Evaluate ResearchOS-ML retrieval, and compare configurations. | |
| python scripts/eval_retrieval.py --config configs/ml.yaml | |
| python scripts/eval_retrieval.py --benchmark generated --judge | |
| TWO BENCHMARKS, AND WHY BOTH | |
| ---------------------------- | |
| arXiv ships no relevance judgements, so a test set had to be manufactured. | |
| Citation-context mining - the principled route, kept in src/data/citation_eval.py | |
| - yielded 12 usable pairs on this corpus, because papers from 2025-11 onward cite | |
| mostly older work that is not indexed. Twelve queries is noise. | |
| sentence A verbatim sentence from a paper's body is the query; gold is that | |
| paper. The chunk it came from is HELD OUT, so the paper must be | |
| found through its other passages. Free, exact labels - but the | |
| query is literal text from the target, which hands BM25 an exact | |
| lexical match. Absolute numbers are optimistic. | |
| generated A model reads the paper and writes the question a researcher would | |
| type to find it (scripts/generate_eval_queries.py). The label is | |
| still structural - the question was written from that paper - but | |
| the phrasing is natural, which is what users actually type. The | |
| prompt forbids reusing the title, and queries that copy it anyway | |
| are rejected. | |
| The `generated` set is the better proxy for the Find-papers feature. The | |
| `sentence` set is kept because it is label-noise-free and shows how much of any | |
| result is a lexical artifact. | |
| FALSE NEGATIVES, AND --judge | |
| ---------------------------- | |
| Both benchmarks mark exactly one paper relevant per query, so a genuinely good | |
| result that is not the source paper counts as a miss. `--judge` pools the top-k | |
| from every configuration, has a model judge only that pool, and adds the | |
| positives. That recovers most of the loss - at the cost of being a SYNTHETIC | |
| metric, so it is reported as a secondary row, never the headline. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import random | |
| import re | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Any | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| if hasattr(sys.stdout, "reconfigure"): | |
| sys.stdout.reconfigure(encoding="utf-8", errors="replace") | |
| from src.config import device, load_config, seed_everything | |
| from src.data.arxiv_source import read_papers | |
| from src.data.chunking import chunks_to_corpus, read_chunks | |
| from src.rerank.evaluate import evaluate_run, latency_stats | |
| from src.retrieval.bm25 import BM25Retriever | |
| from src.retrieval.dense import DenseRetriever | |
| from src.retrieval.fusion import reciprocal_rank_fusion | |
| SENTENCE_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z])") | |
| BODY_SECTIONS = {"method", "methods", "methodology", "approach", "experiments", | |
| "experimental setup", "results", "analysis", "model", "architecture"} | |
| MIN_WORDS, MAX_WORDS = 12, 45 | |
| K_VALUES = [1, 5, 10, 50] | |
| # ------------------------------------------------------------------ benchmarks | |
| def sentence_benchmark( | |
| chunks_path: str, papers_path: str, n_queries: int, rng: random.Random | |
| ) -> tuple[dict[str, str], dict[str, dict[str, int]], dict[str, str]]: | |
| """Body sentence as query; gold is its paper; the source chunk is held out.""" | |
| chunks = read_chunks(chunks_path) | |
| full_text = {p.paper_id for p in read_papers(papers_path) if p.full_text} | |
| by_paper: dict[str, list[tuple[str, str]]] = {} | |
| for chunk in chunks: | |
| if chunk.paper_id in full_text and chunk.section in BODY_SECTIONS: | |
| by_paper.setdefault(chunk.paper_id, []).append((chunk.chunk_id, chunk.text)) | |
| queries: dict[str, str] = {} | |
| qrels: dict[str, dict[str, int]] = {} | |
| sources: dict[str, str] = {} | |
| candidates = sorted(by_paper) | |
| rng.shuffle(candidates) | |
| for paper_id in candidates: | |
| if len(queries) >= n_queries: | |
| break | |
| if len(by_paper[paper_id]) < 2: | |
| continue | |
| sentences = [ | |
| (cid, s.strip()) | |
| for cid, text in by_paper[paper_id] | |
| for s in SENTENCE_RE.split(text) | |
| if MIN_WORDS <= len(s.split()) <= MAX_WORDS | |
| ] | |
| sentences = [ | |
| (cid, s) for cid, s in sentences | |
| if not s.lower().startswith(("in this section", "table", "figure")) | |
| ] | |
| if not sentences: | |
| continue | |
| qid = f"s{len(queries)}" | |
| source_chunk, sentence = rng.choice(sentences) | |
| queries[qid], qrels[qid], sources[qid] = sentence, {paper_id: 1}, source_chunk | |
| return queries, qrels, sources | |
| def generated_benchmark( | |
| path: str, n_queries: int | |
| ) -> tuple[dict[str, str], dict[str, dict[str, int]], dict[str, str]]: | |
| """Model-written natural-language queries; gold is the source paper.""" | |
| from src.data.query_gen import read_generated | |
| items = read_generated(path)[:n_queries] | |
| queries = {i.query_id: i.query for i in items} | |
| qrels = {i.query_id: {i.gold_paper_id: 1} for i in items} | |
| return queries, qrels, {} | |
| # ------------------------------------------------------------------- machinery | |
| def to_paper_run( | |
| chunk_run: dict[str, dict[str, float]], | |
| corpus: dict[str, dict[str, Any]], | |
| top_k: int, | |
| sources: dict[str, str] | None = None, | |
| ) -> dict[str, dict[str, float]]: | |
| """Collapse chunk-level hits to paper-level, scoring each paper by its best chunk.""" | |
| paper_run: dict[str, dict[str, float]] = {} | |
| for qid, hits in chunk_run.items(): | |
| best: dict[str, float] = {} | |
| held_out = (sources or {}).get(qid) | |
| for chunk_id, score in hits.items(): | |
| if chunk_id == held_out: | |
| continue | |
| pid = corpus[chunk_id]["paper_id"] | |
| if score > best.get(pid, float("-inf")): | |
| best[pid] = float(score) | |
| paper_run[qid] = dict(sorted(best.items(), key=lambda kv: kv[1], reverse=True)[:top_k]) | |
| return paper_run | |
| def run_all_configs( | |
| queries: dict[str, str], | |
| corpus: dict[str, dict[str, Any]], | |
| doc_ids: list[str], | |
| doc_texts: list[str], | |
| cfg: dict[str, Any], | |
| sources: dict[str, str], | |
| ) -> dict[str, dict[str, Any]]: | |
| """Run every retrieval configuration over one query set.""" | |
| rc, paths = cfg["retrieval"], cfg["paths"] | |
| top_k = rc["top_k"] | |
| out: dict[str, dict[str, Any]] = {} | |
| bm25 = BM25Retriever(k1=rc["bm25"]["k1"], b=rc["bm25"]["b"], stemmer_lang=rc["bm25"]["stemmer"]) | |
| bm25.index(doc_ids, doc_texts, cache_dir=paths["index_cache"]) | |
| t0 = time.perf_counter() | |
| bm25_run = bm25.search(queries, top_k=top_k) | |
| bm25_ms = [(time.perf_counter() - t0) * 1000.0 / len(queries)] * len(queries) | |
| out["bm25"] = {"label": "BM25 only", "run": to_paper_run(bm25_run, corpus, top_k, sources), | |
| "ms": bm25_ms, "params": None} | |
| dense = DenseRetriever( | |
| model_name=rc["dense"]["model"], batch_size=rc["dense"]["batch_size"], | |
| max_length=rc["dense"]["max_length"], query_prefix=rc["dense"]["query_prefix"], | |
| device=device(), | |
| ) | |
| dense.index_corpus(doc_ids, doc_texts, cache_dir=paths["index_cache"]) | |
| t0 = time.perf_counter() | |
| dense_run = dense.search(queries, top_k=top_k) | |
| dense_ms = [(time.perf_counter() - t0) * 1000.0 / len(queries)] * len(queries) | |
| out["dense"] = {"label": "Dense only (bge-small)", "run": to_paper_run(dense_run, corpus, top_k, sources), | |
| "ms": dense_ms, "params": "33M"} | |
| t0 = time.perf_counter() | |
| fused = reciprocal_rank_fusion([bm25_run, dense_run], k=rc["fusion"]["rrf_k"], top_k=top_k) | |
| fuse_ms = [(time.perf_counter() - t0) * 1000.0 / len(queries) + b + d | |
| for b, d in zip(bm25_ms, dense_ms)] | |
| out["rrf"] = {"label": "BM25 + dense (RRF)", "run": to_paper_run(fused, corpus, top_k, sources), | |
| "ms": fuse_ms, "params": "33M"} | |
| from src.rerank.model import Reranker | |
| mc = cfg["reranker"] | |
| reranker = Reranker( | |
| model_name_or_path=mc["model"], max_length=mc["max_length"], | |
| batch_size=mc["batch_size"], device=device(), | |
| fp16=mc.get("fp16", True) and device() == "cuda", | |
| ) | |
| reranked, rerank_ms = reranker.rerank(queries, fused, corpus, top_k=top_k) | |
| out["rrf_rerank"] = { | |
| "label": "+ cross-encoder reranker", | |
| "run": to_paper_run(reranked, corpus, top_k, sources), | |
| "ms": [f + r for f, r in zip(fuse_ms, rerank_ms)], | |
| "params": "278M", | |
| } | |
| return out | |
| def score(configs: dict[str, dict[str, Any]], qrels: dict[str, dict[str, int]], banner: str) -> dict[str, Any]: | |
| print(f"\n--- {banner} ---") | |
| rows: dict[str, Any] = {} | |
| for name, data in configs.items(): | |
| metrics = evaluate_run(qrels, data["run"], K_VALUES) | |
| rows[name] = {"label": data["label"], "metrics": metrics, | |
| "latency": latency_stats(data["ms"]), "params": data["params"]} | |
| print(f" {data['label']:32s} nDCG@10={metrics['ndcg@10']:.4f} " | |
| f"R@10={metrics['recall@10']:.4f} MAP={metrics['map@10']:.4f}") | |
| return rows | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", default="configs/ml.yaml") | |
| parser.add_argument("--benchmark", choices=["sentence", "generated", "both"], default="both") | |
| parser.add_argument("--n-queries", type=int, default=300) | |
| parser.add_argument("--generated-path", default="data/generated_eval.jsonl") | |
| parser.add_argument("--judge", action="store_true", help="pool + LLM-judge to recover false negatives") | |
| parser.add_argument("--judge-depth", type=int, default=10) | |
| parser.add_argument("--out", default="results/ml_metrics.json") | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| seed = seed_everything(cfg["seed"]) | |
| rng = random.Random(seed) | |
| paths = cfg["paths"] | |
| corpus = chunks_to_corpus(read_chunks(paths["chunks"])) | |
| doc_ids = list(corpus) | |
| doc_texts = [f"{corpus[d]['title']} {corpus[d]['text']}".strip() for d in doc_ids] | |
| print(f"[eval] corpus: {len(doc_ids)} chunks") | |
| results: dict[str, Any] = {"seed": seed, "benchmarks": {}} | |
| wanted = ["sentence", "generated"] if args.benchmark == "both" else [args.benchmark] | |
| for kind in wanted: | |
| if kind == "sentence": | |
| queries, qrels, sources = sentence_benchmark( | |
| paths["chunks"], paths["papers"], args.n_queries, rng | |
| ) | |
| note = "verbatim body sentence; source chunk held out; favours lexical matching" | |
| else: | |
| gen_path = Path(args.generated_path) | |
| if not gen_path.exists(): | |
| print(f"[eval] skipping generated: {gen_path} not found " | |
| "(run scripts/generate_eval_queries.py)") | |
| continue | |
| queries, qrels, sources = generated_benchmark(str(gen_path), args.n_queries) | |
| note = "model-written natural-language query; label is structural" | |
| if not queries: | |
| continue | |
| print(f"\n[eval] benchmark '{kind}': {len(queries)} queries") | |
| configs = run_all_configs(queries, corpus, doc_ids, doc_texts, cfg, sources) | |
| block: dict[str, Any] = { | |
| "n_queries": len(queries), | |
| "note": note, | |
| "rows": score(configs, qrels, f"{kind}: structural labels (primary)"), | |
| } | |
| if args.judge and kind == "generated": | |
| from src.data.judge import build_pool, judge_pool | |
| papers_by_id = {p.paper_id: vars(p) for p in read_papers(paths["papers"])} | |
| pool = build_pool({k: v["run"] for k, v in configs.items()}, depth=args.judge_depth) | |
| merged = judge_pool( | |
| pool, queries, papers_by_id, qrels, | |
| model=cfg["answer"]["model"], | |
| out_path="data/pooled_judgements.jsonl", | |
| ) | |
| block["rows_pooled"] = score( | |
| configs, merged, f"{kind}: + pooled LLM judgements (SECONDARY, synthetic)" | |
| ) | |
| block["pooled_note"] = ( | |
| "Relevance judged by an LLM over the pooled top-k of every configuration. " | |
| "Synthetic signal - secondary to the structural rows above." | |
| ) | |
| results["benchmarks"][kind] = block | |
| out = Path(args.out) | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(results, indent=2), encoding="utf-8") | |
| print(f"\n[eval] wrote {out}") | |
| if __name__ == "__main__": | |
| main() | |