Spaces:
Running on Zero
Running on Zero
File size: 8,035 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 | """Build a retrieval evaluation set from citation contexts.
arXiv ships no relevance judgements, so there is nothing to compute nDCG
against. This manufactures a test set from signal that is already in the
corpus and is genuinely human-authored:
A paper writes "...following the approach of [23]..."
Reference [23] resolves to arXiv paper 2301.08243
=> query = that sentence
gold = 2301.08243
A researcher decided that citation belonged in that sentence. That is a real
relevance judgement, just not one collected for this purpose.
What this measures: "given a sentence that cites something, can the system
find the thing being cited?" That is exactly the Find-papers use case, so it
is a fair proxy for it. It is NOT a proxy for the Explain use case, and it is
weak supervision rather than a curated gold set - see the honesty notes in
README-ML.md.
"""
from __future__ import annotations
import json
import re
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
from src.data.arxiv_source import Paper
from src.data.citation import _build_idf, _tokens, _weighted_coverage, clean_citation
# Where the bibliography starts.
REFS_HEADER_RE = re.compile(r"^\s*(references|bibliography)\s*$", re.IGNORECASE | re.MULTILINE)
# "[23] Author, Title..." at the start of a reference entry.
NUMBERED_REF_RE = re.compile(r"\[(\d{1,3})\]\s*(.+?)(?=\n\s*\[\d{1,3}\]|\Z)", re.DOTALL)
# Inline "[23]" or "[12, 23]" or "[8-23]" markers in the body.
INLINE_MARKER_RE = re.compile(r"\[(\d{1,3}(?:\s*[,\-–]\s*\d{1,3})*)\]")
SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z])")
MIN_QUERY_WORDS = 8
MAX_QUERY_WORDS = 60
@dataclass
class CitationPair:
"""One (citing sentence -> cited paper) judgement."""
query_id: str
query: str
gold_paper_id: str
citing_paper_id: str
marker: str
gold_title: str
def split_body_and_refs(full_text: str) -> tuple[str, str]:
"""Return (body, references_block). Empty refs when no header is found."""
matches = list(REFS_HEADER_RE.finditer(full_text))
if not matches:
return full_text, ""
last = matches[-1]
return full_text[: last.start()], full_text[last.end() :]
def parse_numbered_references(refs_block: str) -> dict[str, str]:
"""Map "23" -> the raw reference string."""
out: dict[str, str] = {}
for match in NUMBERED_REF_RE.finditer(refs_block):
number, body = match.group(1), re.sub(r"\s+", " ", match.group(2)).strip()
if 20 < len(body) < 600:
out[number] = body
return out
def _expand_markers(raw: str) -> list[str]:
""""[12, 23]" -> ["12", "23"]. Ranges are skipped as too ambiguous."""
if re.search(r"[\-–]", raw):
return []
return [p.strip() for p in raw.split(",") if p.strip().isdigit()]
def find_citing_sentences(body: str) -> list[tuple[str, str]]:
"""Return (marker_number, sentence) for every unambiguous inline citation."""
found: list[tuple[str, str]] = []
for sentence in SENTENCE_SPLIT_RE.split(body):
clean = re.sub(r"\s+", " ", sentence).strip()
words = clean.split()
if not (MIN_QUERY_WORDS <= len(words) <= MAX_QUERY_WORDS):
continue
markers = INLINE_MARKER_RE.findall(clean)
# A sentence citing many works at once is about all of them; a sentence
# citing exactly one is about that one. Only the latter is a clean label.
expanded = [n for m in markers for n in _expand_markers(m)]
if len(expanded) != 1:
continue
found.append((expanded[0], clean))
return found
def build_pairs(
papers: list[Paper],
corpus_papers: dict[str, dict[str, Any]],
match_threshold: float = 0.78,
max_per_paper: int = 6,
) -> list[CitationPair]:
"""Mine (citing sentence -> cited corpus paper) pairs from full-text papers.
A pair is kept only when the cited work resolves to a paper that is IN the
corpus - otherwise the retriever could not possibly find it and the query
would score zero for reasons that say nothing about retrieval quality.
`match_threshold` is deliberately stricter than the interactive lookup: a
wrong label is worse than a missing one when building an evaluation set.
"""
idf = _build_idf(corpus_papers)
default_idf = max(idf.values(), default=1.0)
titles = {meta["paper_id"]: meta.get("title", "") for meta in corpus_papers.values()}
# Inverted index over title tokens. Scoring a reference against all 8,330
# titles is 45M comparisons across the corpus and takes over ten minutes;
# candidates that share no rare token cannot clear the threshold anyway, so
# only papers sharing the query's rarest tokens are ever scored.
postings: dict[str, list[str]] = {}
for pid, title in titles.items():
for token in _tokens(title):
postings.setdefault(token, []).append(pid)
def candidates_for(query: str, max_tokens: int = 6) -> set[str]:
query_tokens = sorted(
_tokens(query), key=lambda t: idf.get(t, default_idf), reverse=True
)[:max_tokens]
found: set[str] = set()
for token in query_tokens:
found.update(postings.get(token, ()))
return found
pairs: list[CitationPair] = []
resolved_cache: dict[str, str | None] = {}
for paper in papers:
if not paper.full_text:
continue
body, refs_block = split_body_and_refs(paper.full_text)
if not refs_block:
continue
references = parse_numbered_references(refs_block)
if not references:
continue
kept = 0
for marker, sentence in find_citing_sentences(body):
if kept >= max_per_paper:
break
raw_ref = references.get(marker)
if not raw_ref:
continue
cache_key = f"{paper.paper_id}:{marker}"
if cache_key not in resolved_cache:
title_guess = clean_citation(raw_ref)
gold_id = None
if len(_tokens(title_guess)) >= 3:
best_id, best_score = None, 0.0
for pid in candidates_for(title_guess):
score = _weighted_coverage(
title_guess, titles[pid], idf, default_idf
)
if score > best_score:
best_id, best_score = pid, score
if best_score >= match_threshold and best_id != paper.paper_id:
gold_id = best_id
resolved_cache[cache_key] = gold_id
gold_id = resolved_cache[cache_key]
if not gold_id:
continue
# Strip the marker so the query cannot leak its own answer.
query = INLINE_MARKER_RE.sub("", sentence).strip()
query = re.sub(r"\s+", " ", query)
if len(query.split()) < MIN_QUERY_WORDS:
continue
pairs.append(
CitationPair(
query_id=f"{paper.paper_id}#{marker}#{kept}",
query=query,
gold_paper_id=gold_id,
citing_paper_id=paper.paper_id,
marker=marker,
gold_title=corpus_papers[gold_id].get("title", ""),
)
)
kept += 1
return pairs
def write_pairs(pairs: list[CitationPair], path: str | Path) -> None:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
with open(target, "w", encoding="utf-8") as handle:
for pair in pairs:
handle.write(json.dumps(asdict(pair), ensure_ascii=False) + "\n")
print(f"[eval] wrote {len(pairs)} citation pairs -> {target}")
def read_pairs(path: str | Path) -> list[CitationPair]:
with open(path, encoding="utf-8") as handle:
return [CitationPair(**json.loads(line)) for line in handle if line.strip()]
|