ResearchOS / scripts /build_citation_eval.py
Kagzy's picture
ResearchOS
e8c5cc2 verified
Raw History Blame Contribute Delete
2.31 kB
"""Mine a retrieval evaluation set from citation contexts.
python scripts/build_citation_eval.py --config configs/ml.yaml
"""
from __future__ import annotations
import argparse
import collections
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.config import load_config, seed_everything
from src.data.arxiv_source import read_papers
from src.data.citation_eval import build_pairs, write_pairs
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--config", default="configs/ml.yaml")
parser.add_argument("--out", default="data/citation_eval.jsonl")
parser.add_argument("--threshold", type=float, default=0.78)
parser.add_argument("--max-per-paper", type=int, default=6)
args = parser.parse_args()
cfg = load_config(args.config)
seed_everything(cfg["seed"])
papers = read_papers(cfg["paths"]["papers"])
corpus_papers = {p.paper_id: vars(p) for p in papers}
with_text = [p for p in papers if p.full_text]
print(f"[eval] {len(papers)} papers, {len(with_text)} with full text")
pairs = build_pairs(
with_text,
corpus_papers,
match_threshold=args.threshold,
max_per_paper=args.max_per_paper,
)
if not pairs:
raise SystemExit(
"No citation pairs found. The corpus skews recent, so most references "
"point outside it. Try a lower --threshold or ingest more papers."
)
golds = collections.Counter(p.gold_paper_id for p in pairs)
citers = collections.Counter(p.citing_paper_id for p in pairs)
print(f"[eval] {len(pairs)} pairs | {len(golds)} distinct gold papers | {len(citers)} citing papers")
print(f"[eval] median query length: {sorted(len(p.query.split()) for p in pairs)[len(pairs) // 2]} words")
print("[eval] most-cited golds:")
for pid, n in golds.most_common(5):
title = next(p.gold_title for p in pairs if p.gold_paper_id == pid)
print(f" {n:>3}x {pid} {title[:58]}")
write_pairs(pairs, args.out)
print("\n[eval] sample:")
for pair in pairs[:3]:
print(f' Q: "{pair.query[:96]}"')
print(f" A: {pair.gold_paper_id} {pair.gold_title[:64]}\n")
if __name__ == "__main__":
main()