File size: 2,312 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""Mine a retrieval evaluation set from citation contexts.

    python scripts/build_citation_eval.py --config configs/ml.yaml
"""

from __future__ import annotations

import argparse
import collections
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from src.config import load_config, seed_everything
from src.data.arxiv_source import read_papers
from src.data.citation_eval import build_pairs, write_pairs


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--config", default="configs/ml.yaml")
    parser.add_argument("--out", default="data/citation_eval.jsonl")
    parser.add_argument("--threshold", type=float, default=0.78)
    parser.add_argument("--max-per-paper", type=int, default=6)
    args = parser.parse_args()

    cfg = load_config(args.config)
    seed_everything(cfg["seed"])

    papers = read_papers(cfg["paths"]["papers"])
    corpus_papers = {p.paper_id: vars(p) for p in papers}
    with_text = [p for p in papers if p.full_text]
    print(f"[eval] {len(papers)} papers, {len(with_text)} with full text")

    pairs = build_pairs(
        with_text,
        corpus_papers,
        match_threshold=args.threshold,
        max_per_paper=args.max_per_paper,
    )

    if not pairs:
        raise SystemExit(
            "No citation pairs found. The corpus skews recent, so most references "
            "point outside it. Try a lower --threshold or ingest more papers."
        )

    golds = collections.Counter(p.gold_paper_id for p in pairs)
    citers = collections.Counter(p.citing_paper_id for p in pairs)
    print(f"[eval] {len(pairs)} pairs | {len(golds)} distinct gold papers | {len(citers)} citing papers")
    print(f"[eval] median query length: {sorted(len(p.query.split()) for p in pairs)[len(pairs) // 2]} words")
    print("[eval] most-cited golds:")
    for pid, n in golds.most_common(5):
        title = next(p.gold_title for p in pairs if p.gold_paper_id == pid)
        print(f"         {n:>3}x  {pid}  {title[:58]}")

    write_pairs(pairs, args.out)
    print("\n[eval] sample:")
    for pair in pairs[:3]:
        print(f'  Q: "{pair.query[:96]}"')
        print(f"  A: {pair.gold_paper_id}  {pair.gold_title[:64]}\n")


if __name__ == "__main__":
    main()