File size: 8,035 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
"""Build a retrieval evaluation set from citation contexts.

arXiv ships no relevance judgements, so there is nothing to compute nDCG
against. This manufactures a test set from signal that is already in the
corpus and is genuinely human-authored:

    A paper writes  "...following the approach of [23]..."
    Reference [23]  resolves to arXiv paper 2301.08243
    => query  = that sentence
       gold   = 2301.08243

A researcher decided that citation belonged in that sentence. That is a real
relevance judgement, just not one collected for this purpose.

What this measures: "given a sentence that cites something, can the system
find the thing being cited?" That is exactly the Find-papers use case, so it
is a fair proxy for it. It is NOT a proxy for the Explain use case, and it is
weak supervision rather than a curated gold set - see the honesty notes in
README-ML.md.
"""

from __future__ import annotations

import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any

from src.data.arxiv_source import Paper
from src.data.citation import _build_idf, _tokens, _weighted_coverage, clean_citation

# Where the bibliography starts.
REFS_HEADER_RE = re.compile(r"^\s*(references|bibliography)\s*$", re.IGNORECASE | re.MULTILINE)
# "[23] Author, Title..." at the start of a reference entry.
NUMBERED_REF_RE = re.compile(r"\[(\d{1,3})\]\s*(.+?)(?=\n\s*\[\d{1,3}\]|\Z)", re.DOTALL)
# Inline "[23]" or "[12, 23]" or "[8-23]" markers in the body.
INLINE_MARKER_RE = re.compile(r"\[(\d{1,3}(?:\s*[,\-–]\s*\d{1,3})*)\]")
SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z])")

MIN_QUERY_WORDS = 8
MAX_QUERY_WORDS = 60


@dataclass
class CitationPair:
    """One (citing sentence -> cited paper) judgement."""

    query_id: str
    query: str
    gold_paper_id: str
    citing_paper_id: str
    marker: str
    gold_title: str


def split_body_and_refs(full_text: str) -> tuple[str, str]:
    """Return (body, references_block). Empty refs when no header is found."""
    matches = list(REFS_HEADER_RE.finditer(full_text))
    if not matches:
        return full_text, ""
    last = matches[-1]
    return full_text[: last.start()], full_text[last.end() :]


def parse_numbered_references(refs_block: str) -> dict[str, str]:
    """Map "23" -> the raw reference string."""
    out: dict[str, str] = {}
    for match in NUMBERED_REF_RE.finditer(refs_block):
        number, body = match.group(1), re.sub(r"\s+", " ", match.group(2)).strip()
        if 20 < len(body) < 600:
            out[number] = body
    return out


def _expand_markers(raw: str) -> list[str]:
    """"[12, 23]" -> ["12", "23"]. Ranges are skipped as too ambiguous."""
    if re.search(r"[\-–]", raw):
        return []
    return [p.strip() for p in raw.split(",") if p.strip().isdigit()]


def find_citing_sentences(body: str) -> list[tuple[str, str]]:
    """Return (marker_number, sentence) for every unambiguous inline citation."""
    found: list[tuple[str, str]] = []
    for sentence in SENTENCE_SPLIT_RE.split(body):
        clean = re.sub(r"\s+", " ", sentence).strip()
        words = clean.split()
        if not (MIN_QUERY_WORDS <= len(words) <= MAX_QUERY_WORDS):
            continue
        markers = INLINE_MARKER_RE.findall(clean)
        # A sentence citing many works at once is about all of them; a sentence
        # citing exactly one is about that one. Only the latter is a clean label.
        expanded = [n for m in markers for n in _expand_markers(m)]
        if len(expanded) != 1:
            continue
        found.append((expanded[0], clean))
    return found


def build_pairs(
    papers: list[Paper],
    corpus_papers: dict[str, dict[str, Any]],
    match_threshold: float = 0.78,
    max_per_paper: int = 6,
) -> list[CitationPair]:
    """Mine (citing sentence -> cited corpus paper) pairs from full-text papers.

    A pair is kept only when the cited work resolves to a paper that is IN the
    corpus - otherwise the retriever could not possibly find it and the query
    would score zero for reasons that say nothing about retrieval quality.

    `match_threshold` is deliberately stricter than the interactive lookup: a
    wrong label is worse than a missing one when building an evaluation set.
    """
    idf = _build_idf(corpus_papers)
    default_idf = max(idf.values(), default=1.0)
    titles = {meta["paper_id"]: meta.get("title", "") for meta in corpus_papers.values()}

    # Inverted index over title tokens. Scoring a reference against all 8,330
    # titles is 45M comparisons across the corpus and takes over ten minutes;
    # candidates that share no rare token cannot clear the threshold anyway, so
    # only papers sharing the query's rarest tokens are ever scored.
    postings: dict[str, list[str]] = {}
    for pid, title in titles.items():
        for token in _tokens(title):
            postings.setdefault(token, []).append(pid)

    def candidates_for(query: str, max_tokens: int = 6) -> set[str]:
        query_tokens = sorted(
            _tokens(query), key=lambda t: idf.get(t, default_idf), reverse=True
        )[:max_tokens]
        found: set[str] = set()
        for token in query_tokens:
            found.update(postings.get(token, ()))
        return found

    pairs: list[CitationPair] = []
    resolved_cache: dict[str, str | None] = {}

    for paper in papers:
        if not paper.full_text:
            continue
        body, refs_block = split_body_and_refs(paper.full_text)
        if not refs_block:
            continue
        references = parse_numbered_references(refs_block)
        if not references:
            continue

        kept = 0
        for marker, sentence in find_citing_sentences(body):
            if kept >= max_per_paper:
                break
            raw_ref = references.get(marker)
            if not raw_ref:
                continue

            cache_key = f"{paper.paper_id}:{marker}"
            if cache_key not in resolved_cache:
                title_guess = clean_citation(raw_ref)
                gold_id = None
                if len(_tokens(title_guess)) >= 3:
                    best_id, best_score = None, 0.0
                    for pid in candidates_for(title_guess):
                        score = _weighted_coverage(
                            title_guess, titles[pid], idf, default_idf
                        )
                        if score > best_score:
                            best_id, best_score = pid, score
                    if best_score >= match_threshold and best_id != paper.paper_id:
                        gold_id = best_id
                resolved_cache[cache_key] = gold_id

            gold_id = resolved_cache[cache_key]
            if not gold_id:
                continue

            # Strip the marker so the query cannot leak its own answer.
            query = INLINE_MARKER_RE.sub("", sentence).strip()
            query = re.sub(r"\s+", " ", query)
            if len(query.split()) < MIN_QUERY_WORDS:
                continue

            pairs.append(
                CitationPair(
                    query_id=f"{paper.paper_id}#{marker}#{kept}",
                    query=query,
                    gold_paper_id=gold_id,
                    citing_paper_id=paper.paper_id,
                    marker=marker,
                    gold_title=corpus_papers[gold_id].get("title", ""),
                )
            )
            kept += 1

    return pairs


def write_pairs(pairs: list[CitationPair], path: str | Path) -> None:
    target = Path(path)
    target.parent.mkdir(parents=True, exist_ok=True)
    with open(target, "w", encoding="utf-8") as handle:
        for pair in pairs:
            handle.write(json.dumps(asdict(pair), ensure_ascii=False) + "\n")
    print(f"[eval] wrote {len(pairs)} citation pairs -> {target}")


def read_pairs(path: str | Path) -> list[CitationPair]:
    with open(path, encoding="utf-8") as handle:
        return [CitationPair(**json.loads(line)) for line in handle if line.strip()]