Spaces:
Running on Zero
Running on Zero
Download src/data/citation_eval.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 8.04 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/citation_eval.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/data/citation_eval.py
-
curl -L -o citation_eval.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/citation_eval.py
8.04 kB
| """Build a retrieval evaluation set from citation contexts. | |
| arXiv ships no relevance judgements, so there is nothing to compute nDCG | |
| against. This manufactures a test set from signal that is already in the | |
| corpus and is genuinely human-authored: | |
| A paper writes "...following the approach of [23]..." | |
| Reference [23] resolves to arXiv paper 2301.08243 | |
| => query = that sentence | |
| gold = 2301.08243 | |
| A researcher decided that citation belonged in that sentence. That is a real | |
| relevance judgement, just not one collected for this purpose. | |
| What this measures: "given a sentence that cites something, can the system | |
| find the thing being cited?" That is exactly the Find-papers use case, so it | |
| is a fair proxy for it. It is NOT a proxy for the Explain use case, and it is | |
| weak supervision rather than a curated gold set - see the honesty notes in | |
| README-ML.md. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| from src.data.arxiv_source import Paper | |
| from src.data.citation import _build_idf, _tokens, _weighted_coverage, clean_citation | |
| # Where the bibliography starts. | |
| REFS_HEADER_RE = re.compile(r"^\s*(references|bibliography)\s*$", re.IGNORECASE | re.MULTILINE) | |
| # "[23] Author, Title..." at the start of a reference entry. | |
| NUMBERED_REF_RE = re.compile(r"\[(\d{1,3})\]\s*(.+?)(?=\n\s*\[\d{1,3}\]|\Z)", re.DOTALL) | |
| # Inline "[23]" or "[12, 23]" or "[8-23]" markers in the body. | |
| INLINE_MARKER_RE = re.compile(r"\[(\d{1,3}(?:\s*[,\-–]\s*\d{1,3})*)\]") | |
| SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z])") | |
| MIN_QUERY_WORDS = 8 | |
| MAX_QUERY_WORDS = 60 | |
| class CitationPair: | |
| """One (citing sentence -> cited paper) judgement.""" | |
| query_id: str | |
| query: str | |
| gold_paper_id: str | |
| citing_paper_id: str | |
| marker: str | |
| gold_title: str | |
| def split_body_and_refs(full_text: str) -> tuple[str, str]: | |
| """Return (body, references_block). Empty refs when no header is found.""" | |
| matches = list(REFS_HEADER_RE.finditer(full_text)) | |
| if not matches: | |
| return full_text, "" | |
| last = matches[-1] | |
| return full_text[: last.start()], full_text[last.end() :] | |
| def parse_numbered_references(refs_block: str) -> dict[str, str]: | |
| """Map "23" -> the raw reference string.""" | |
| out: dict[str, str] = {} | |
| for match in NUMBERED_REF_RE.finditer(refs_block): | |
| number, body = match.group(1), re.sub(r"\s+", " ", match.group(2)).strip() | |
| if 20 < len(body) < 600: | |
| out[number] = body | |
| return out | |
| def _expand_markers(raw: str) -> list[str]: | |
| """"[12, 23]" -> ["12", "23"]. Ranges are skipped as too ambiguous.""" | |
| if re.search(r"[\-–]", raw): | |
| return [] | |
| return [p.strip() for p in raw.split(",") if p.strip().isdigit()] | |
| def find_citing_sentences(body: str) -> list[tuple[str, str]]: | |
| """Return (marker_number, sentence) for every unambiguous inline citation.""" | |
| found: list[tuple[str, str]] = [] | |
| for sentence in SENTENCE_SPLIT_RE.split(body): | |
| clean = re.sub(r"\s+", " ", sentence).strip() | |
| words = clean.split() | |
| if not (MIN_QUERY_WORDS <= len(words) <= MAX_QUERY_WORDS): | |
| continue | |
| markers = INLINE_MARKER_RE.findall(clean) | |
| # A sentence citing many works at once is about all of them; a sentence | |
| # citing exactly one is about that one. Only the latter is a clean label. | |
| expanded = [n for m in markers for n in _expand_markers(m)] | |
| if len(expanded) != 1: | |
| continue | |
| found.append((expanded[0], clean)) | |
| return found | |
| def build_pairs( | |
| papers: list[Paper], | |
| corpus_papers: dict[str, dict[str, Any]], | |
| match_threshold: float = 0.78, | |
| max_per_paper: int = 6, | |
| ) -> list[CitationPair]: | |
| """Mine (citing sentence -> cited corpus paper) pairs from full-text papers. | |
| A pair is kept only when the cited work resolves to a paper that is IN the | |
| corpus - otherwise the retriever could not possibly find it and the query | |
| would score zero for reasons that say nothing about retrieval quality. | |
| `match_threshold` is deliberately stricter than the interactive lookup: a | |
| wrong label is worse than a missing one when building an evaluation set. | |
| """ | |
| idf = _build_idf(corpus_papers) | |
| default_idf = max(idf.values(), default=1.0) | |
| titles = {meta["paper_id"]: meta.get("title", "") for meta in corpus_papers.values()} | |
| # Inverted index over title tokens. Scoring a reference against all 8,330 | |
| # titles is 45M comparisons across the corpus and takes over ten minutes; | |
| # candidates that share no rare token cannot clear the threshold anyway, so | |
| # only papers sharing the query's rarest tokens are ever scored. | |
| postings: dict[str, list[str]] = {} | |
| for pid, title in titles.items(): | |
| for token in _tokens(title): | |
| postings.setdefault(token, []).append(pid) | |
| def candidates_for(query: str, max_tokens: int = 6) -> set[str]: | |
| query_tokens = sorted( | |
| _tokens(query), key=lambda t: idf.get(t, default_idf), reverse=True | |
| )[:max_tokens] | |
| found: set[str] = set() | |
| for token in query_tokens: | |
| found.update(postings.get(token, ())) | |
| return found | |
| pairs: list[CitationPair] = [] | |
| resolved_cache: dict[str, str | None] = {} | |
| for paper in papers: | |
| if not paper.full_text: | |
| continue | |
| body, refs_block = split_body_and_refs(paper.full_text) | |
| if not refs_block: | |
| continue | |
| references = parse_numbered_references(refs_block) | |
| if not references: | |
| continue | |
| kept = 0 | |
| for marker, sentence in find_citing_sentences(body): | |
| if kept >= max_per_paper: | |
| break | |
| raw_ref = references.get(marker) | |
| if not raw_ref: | |
| continue | |
| cache_key = f"{paper.paper_id}:{marker}" | |
| if cache_key not in resolved_cache: | |
| title_guess = clean_citation(raw_ref) | |
| gold_id = None | |
| if len(_tokens(title_guess)) >= 3: | |
| best_id, best_score = None, 0.0 | |
| for pid in candidates_for(title_guess): | |
| score = _weighted_coverage( | |
| title_guess, titles[pid], idf, default_idf | |
| ) | |
| if score > best_score: | |
| best_id, best_score = pid, score | |
| if best_score >= match_threshold and best_id != paper.paper_id: | |
| gold_id = best_id | |
| resolved_cache[cache_key] = gold_id | |
| gold_id = resolved_cache[cache_key] | |
| if not gold_id: | |
| continue | |
| # Strip the marker so the query cannot leak its own answer. | |
| query = INLINE_MARKER_RE.sub("", sentence).strip() | |
| query = re.sub(r"\s+", " ", query) | |
| if len(query.split()) < MIN_QUERY_WORDS: | |
| continue | |
| pairs.append( | |
| CitationPair( | |
| query_id=f"{paper.paper_id}#{marker}#{kept}", | |
| query=query, | |
| gold_paper_id=gold_id, | |
| citing_paper_id=paper.paper_id, | |
| marker=marker, | |
| gold_title=corpus_papers[gold_id].get("title", ""), | |
| ) | |
| ) | |
| kept += 1 | |
| return pairs | |
| def write_pairs(pairs: list[CitationPair], path: str | Path) -> None: | |
| target = Path(path) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| with open(target, "w", encoding="utf-8") as handle: | |
| for pair in pairs: | |
| handle.write(json.dumps(asdict(pair), ensure_ascii=False) + "\n") | |
| print(f"[eval] wrote {len(pairs)} citation pairs -> {target}") | |
| def read_pairs(path: str | Path) -> list[CitationPair]: | |
| with open(path, encoding="utf-8") as handle: | |
| return [CitationPair(**json.loads(line)) for line in handle if line.strip()] | |