Spaces:
Running on Zero
Running on Zero
Download scripts/build_citation_eval.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 2.31 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/build_citation_eval.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/scripts/build_citation_eval.py
-
curl -L -o build_citation_eval.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/scripts/build_citation_eval.py
2.31 kB
| """Mine a retrieval evaluation set from citation contexts. | |
| python scripts/build_citation_eval.py --config configs/ml.yaml | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parents[1])) | |
| from src.config import load_config, seed_everything | |
| from src.data.arxiv_source import read_papers | |
| from src.data.citation_eval import build_pairs, write_pairs | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--config", default="configs/ml.yaml") | |
| parser.add_argument("--out", default="data/citation_eval.jsonl") | |
| parser.add_argument("--threshold", type=float, default=0.78) | |
| parser.add_argument("--max-per-paper", type=int, default=6) | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| seed_everything(cfg["seed"]) | |
| papers = read_papers(cfg["paths"]["papers"]) | |
| corpus_papers = {p.paper_id: vars(p) for p in papers} | |
| with_text = [p for p in papers if p.full_text] | |
| print(f"[eval] {len(papers)} papers, {len(with_text)} with full text") | |
| pairs = build_pairs( | |
| with_text, | |
| corpus_papers, | |
| match_threshold=args.threshold, | |
| max_per_paper=args.max_per_paper, | |
| ) | |
| if not pairs: | |
| raise SystemExit( | |
| "No citation pairs found. The corpus skews recent, so most references " | |
| "point outside it. Try a lower --threshold or ingest more papers." | |
| ) | |
| golds = collections.Counter(p.gold_paper_id for p in pairs) | |
| citers = collections.Counter(p.citing_paper_id for p in pairs) | |
| print(f"[eval] {len(pairs)} pairs | {len(golds)} distinct gold papers | {len(citers)} citing papers") | |
| print(f"[eval] median query length: {sorted(len(p.query.split()) for p in pairs)[len(pairs) // 2]} words") | |
| print("[eval] most-cited golds:") | |
| for pid, n in golds.most_common(5): | |
| title = next(p.gold_title for p in pairs if p.gold_paper_id == pid) | |
| print(f" {n:>3}x {pid} {title[:58]}") | |
| write_pairs(pairs, args.out) | |
| print("\n[eval] sample:") | |
| for pair in pairs[:3]: | |
| print(f' Q: "{pair.query[:96]}"') | |
| print(f" A: {pair.gold_paper_id} {pair.gold_title[:64]}\n") | |
| if __name__ == "__main__": | |
| main() | |