ResearchOS / src /data /citation.py
Kagzy's picture
answer any AI question; adaptive rerank + query cache; corpus loader with fallback; OpenAlex query fixes
2299777 verified
Raw History Blame Contribute Delete
20.2 kB
"""Resolve a citation you saw in a paper to the actual paper.
The use case: you are reading a paper, you hit "...following [23] (Assran et al.,
2023)...", and you want to know what that is without leaving what you are doing.
Accepted inputs, in the order they are tried:
arXiv id 2301.08243, arXiv:2301.08243v2, https://arxiv.org/abs/2301.08243
DOI 10.1109/CVPR.2016.90, https://doi.org/10.1109/CVPR.2016.90
bracketed ref [23] M. Assran et al. "Self-Supervised Learning from Images..."
bare title Self-Supervised Learning from Images with a Joint-Embedding...
Resolution is local-first: the ingested corpus is checked before any network
call, because most of what you are reading about is already indexed. Anything
not found locally is fetched on demand - arXiv first, then OpenAlex, which
covers the conference and journal papers that never had an arXiv preprint.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import Any
from src.data import openalex_source as openalex
from src.data.arxiv_source import (
ArxivUnavailable, Paper, download_pdf, extract_text, fetch_metadata_bounded,
)
# 2301.08243 / 2301.08243v2, with optional arXiv: prefix or abs/pdf URL
ARXIV_ID_RE = re.compile(
r"(?:arxiv[:\s/]*|/abs/|/pdf/)?(?P<id>\d{4}\.\d{4,5})(?:v\d+)?", re.IGNORECASE
)
# Leading "[23]" or "23." reference markers
REF_MARKER_RE = re.compile(r"^\s*[\[\(]?\d{1,3}[\]\).]?\s*")
# "Assran et al., 2023" style author-year tails
AUTHOR_YEAR_RE = re.compile(r"\b[A-Z][a-z]+\s+et\s+al\.?,?\s*\(?\d{4}\)?[.,]?", re.IGNORECASE)
# A leading author block terminated by "et al.", with or without initials:
# "M. Assran et al." / "Kaiming He et al." / "Vaswani et al.,"
AUTHOR_ETAL_RE = re.compile(
r"^(?:[A-Z][a-zA-Z.'-]*\s+)*?[A-Z][a-zA-Z'-]+,?\s+et\s+al\.?,?\s*", re.IGNORECASE
)
QUOTED_RE = re.compile(r"[\"β€œβ€˜']([^\"”’']{12,})[\"”’']")
def _safe_fetch(query: str, max_results: int) -> list[Paper]:
"""An arXiv lookup that never takes down - or stalls - the request.
Bounded by fetch_metadata_bounded: retries off, results cached, and the
request stops waiting after the budget. Disabling retries alone was not
enough - a throttled arXiv still held each call ~46s before its 429, and
resolve() can make two calls in a row. A failed lookup returns nothing
and the caller degrades.
"""
try:
return fetch_metadata_bounded(query, max_results=max_results)
except ArxivUnavailable as exc:
print(f"[arxiv] lookup skipped ({exc}); degrading")
return []
def _paper_from_openalex(work: dict[str, Any]) -> Paper:
"""An OpenAlex work in the Paper shape the rest of the pipeline expects."""
arxiv_id = work.get("arxiv_id")
return Paper(
paper_id=work["paper_id"],
version=1,
title=work["title"],
abstract=work["abstract"],
authors=work["authors"],
categories=[],
primary_category=work.get("venue", ""),
published=work.get("published", ""),
updated="",
# Only arXiv PDFs are ever fetched on the request path: a known host
# with stable, polite access. Other open-access PDFs are linked, not
# downloaded.
pdf_url=f"https://arxiv.org/pdf/{arxiv_id}" if arxiv_id else "",
metadata={
"source": "openalex",
"url": work.get("url", ""),
"venue": work.get("venue", ""),
"doi": work.get("doi") or "",
"cited_by_count": str(work.get("cited_by_count", 0)),
},
)
def _safe_openalex(fn, *args) -> Any:
try:
return fn(*args)
except openalex.OpenAlexUnavailable as exc:
print(f"[openalex] lookup skipped ({exc}); degrading")
return None
def paper_url(paper: Paper) -> str:
"""Where a reader should go for this paper, whichever source it came from."""
return (paper.metadata or {}).get("url") or f"https://arxiv.org/abs/{paper.paper_id}"
@dataclass
class Resolution:
"""What a citation string resolved to, and how."""
paper: Paper | None
method: str # arxiv_id | doi | local_title | arxiv_search | openalex_title | unresolved
query_used: str
from_corpus: bool
candidates: list[dict[str, str]]
QUESTION_STARTS = (
"what", "how", "why", "when", "where", "which", "who", "explain", "describe",
"compare", "does", "do ", "is ", "are ", "can ", "should", "tell me", "give me",
"summarise", "summarize", "define",
)
# What a citation actually looks like. Anything matching one of these is worth
# the cost of resolution; nothing else is.
_CITATION_SHAPES = (
re.compile(r"\bet\s+al\b", re.IGNORECASE), # Assran et al.
re.compile(r"^\s*[\[\(]\d{1,3}[\]\)]"), # [23] or (23)
re.compile(r"^\s*\d{1,3}\.\s+[A-Z]"), # 23. Author
re.compile(r"\((?:19|20)\d{2}\)"), # (2023)
re.compile(r",\s*(?:19|20)\d{2}\b"), # , 2023
re.compile(r"\b(?:CVPR|NeurIPS|ICML|ICLR|ACL|EMNLP|AAAI|SIGGRAPH)\b"),
re.compile(r"\barxiv\b", re.IGNORECASE),
re.compile(r"\b10\.\d{4,9}/\S+"), # a DOI
re.compile(r"[\"β€œβ€˜']([^\"”’']{15,})[\"”’']"), # a quoted title
)
def looks_like_a_citation(text: str) -> bool:
"""True only when the text has the shape of a reference.
The default matters. Selecting a phrase in a paper - "MLP block of
Transformers" - is asking about a concept, not naming a paper. Treating it
as a title costs two arXiv title searches that cannot succeed, and arXiv
rate-limits: one such selection took 34 seconds to fail. Citations announce
themselves (an id, "et al.", a bracketed number, a year, a venue); concepts
do not, so concepts are the default and resolution is opt-in.
"""
raw = (text or "").strip()
if not raw:
return False
if extract_arxiv_id(raw):
return True
if any(shape.search(raw) for shape in _CITATION_SHAPES):
return True
# A bare title: long, no trailing question mark, and mostly capitalised
# words - "Masked Autoencoders Are Scalable Vision Learners".
words = raw.split()
if 4 <= len(words) <= 25 and not raw.endswith("?"):
capitalised = sum(1 for w in words if w[:1].isupper())
if capitalised >= max(3, len(words) * 0.5):
return True
return False
def looks_like_a_question(text: str) -> bool:
"""Anything that is not shaped like a citation is treated as a question."""
raw = (text or "").strip()
if not raw:
return False
if raw.endswith("?") or raw.lower().startswith(QUESTION_STARTS):
return True
return not looks_like_a_citation(raw)
def extract_arxiv_id(text: str) -> str | None:
"""Pull an arXiv id out of free text, if there is one."""
match = ARXIV_ID_RE.search(text or "")
return match.group("id") if match else None
def clean_citation(text: str) -> str:
"""Strip reference markers, author-year tails and quotes down to a title-ish string."""
cleaned = REF_MARKER_RE.sub("", text or "").strip()
# A quoted span in a reference is almost always the title.
quoted = QUOTED_RE.search(cleaned)
if quoted:
return quoted.group(1).strip()
cleaned = AUTHOR_ETAL_RE.sub("", cleaned)
cleaned = AUTHOR_YEAR_RE.sub(" ", cleaned)
# Drop a leading author list: "M. Assran, Q. Duval, I. Misra. Title here"
cleaned = re.sub(r"^(?:[A-Z]\.\s*){1,4}[A-Z][a-z]+(?:,\s*(?:[A-Z]\.\s*)*[A-Z][a-z]+)*[.,]\s*", "", cleaned)
# A bare year left over once the author block is gone: "2017. Attention Is..."
cleaned = re.sub(r"^\(?(?:19|20)\d{2}\)?[.,]?\s*", "", cleaned)
# Drop trailing venue noise
cleaned = re.sub(r"\b(?:In\s+)?(?:Proceedings|Proc\.|CVPR|NeurIPS|ICML|ICLR|arXiv preprint)\b.*$", "", cleaned, flags=re.IGNORECASE)
return re.sub(r"\s+", " ", cleaned).strip(" .,;:")
def _normalise(text: str) -> str:
return re.sub(r"[^a-z0-9 ]+", " ", (text or "").lower())
STOPWORDS = {
"the", "a", "an", "of", "for", "with", "and", "in", "on", "to", "via", "using",
"all", "you", "need", "is", "are", "we", "our", "its", "that", "this", "from",
}
def _tokens(text: str) -> set[str]:
return {w for w in _normalise(text).split() if w not in STOPWORDS and len(w) > 2}
def _token_overlap(a: str, b: str) -> float:
"""Jaccard overlap on content words. Used for ranking arXiv search results."""
ta, tb = _tokens(a), _tokens(b)
if not ta or not tb:
return 0.0
return len(ta & tb) / len(ta | tb)
def _build_idf(papers_by_id: dict[str, dict[str, Any]]) -> dict[str, float]:
"""Inverse document frequency over corpus titles."""
import math
df: dict[str, int] = {}
for meta in papers_by_id.values():
for token in _tokens(meta.get("title", "")):
df[token] = df.get(token, 0) + 1
n = max(1, len(papers_by_id))
return {t: math.log(n / (1 + c)) for t, c in df.items()}
def _weighted_coverage(query: str, title: str, idf: dict[str, float], default_idf: float) -> float:
"""How much of the query's *information* the title covers.
Plain Jaccard fails badly on titles built from common words. "Attention Is
All You Need" against "All you need is log" scores 0.60 on Jaccard - three
throwaway tokens match and the one word that carries the meaning,
"attention", does not. Weighting each token by IDF makes the miss decisive:
a rare token that fails to match costs far more than three common ones that
do.
"""
qt = _tokens(query)
if not qt:
return 0.0
tt = _tokens(title)
total = sum(idf.get(t, default_idf) for t in qt)
if total <= 0:
return 0.0
matched = sum(idf.get(t, default_idf) for t in qt & tt)
# Penalise a title that is much longer than the query - it is a different
# paper that happens to contain these words.
length_penalty = min(1.0, len(qt) / max(1, len(tt))) ** 0.5
return (matched / total) * length_penalty
class TitleIndex:
"""Inverted index over corpus titles, built once per corpus.
find_in_corpus used to rebuild the IDF table and score every title on
every lookup: fine at 8k papers, seconds per lookup at 300k. This keeps
the scoring identical and only changes which titles get scored.
Pruning is exact, not heuristic. A title that contains none of a set S of
query tokens can match at most (total - w(S)) of the query's IDF weight,
so if w(S) >= (1 - floor) * total it cannot score above `floor`. Taking
S as the rarest query tokens until that holds, the postings of S contain
every title that could clear the floor - the same results as a full scan.
"""
def __init__(self, papers_by_id: dict[str, dict[str, Any]]) -> None:
import math
self.papers = papers_by_id
self.tokens: dict[str, set[str]] = {}
postings: dict[str, list[str]] = {}
for pid, meta in papers_by_id.items():
toks = _tokens(meta.get("title", ""))
self.tokens[pid] = toks
for t in toks:
postings.setdefault(t, []).append(pid)
self.postings = postings
n = max(1, len(papers_by_id))
self.idf = {t: math.log(n / (1 + len(ids))) for t, ids in postings.items()}
self.default_idf = max(self.idf.values(), default=1.0)
def search(self, query: str, floor: float = 0.2) -> list[tuple[float, dict[str, Any]]]:
qt = _tokens(query)
if not qt:
return []
weight = {t: self.idf.get(t, self.default_idf) for t in qt}
total = sum(weight.values())
needed, covered, pivots = (1.0 - floor) * total, 0.0, []
for t in sorted(qt, key=weight.get, reverse=True):
pivots.append(t)
covered += weight[t]
if covered >= needed:
break
candidates = {pid for t in pivots for pid in self.postings.get(t, ())}
scored = []
for pid in candidates:
tt = self.tokens[pid]
matched = sum(weight[t] for t in qt & tt)
score = (matched / total) * min(1.0, len(qt) / max(1, len(tt))) ** 0.5
scored.append((score, self.papers[pid]))
scored.sort(key=lambda kv: kv[0], reverse=True)
return scored
def title_coverage(self, query: str, title: str) -> float:
"""Share of the TITLE's IDF weight that the query covers - the
direction search() ignores."""
tt, qt = _tokens(title), _tokens(query)
if not tt:
return 0.0
w = {t: self.idf.get(t, self.default_idf) for t in tt}
return sum(w[t] for t in tt & qt) / sum(w.values())
_TITLE_INDEX: dict[int, tuple[int, TitleIndex]] = {}
AMBIGUITY_MARGIN = 0.05 # best must beat the runner-up by this much...
TITLE_COVERAGE_MIN = 0.80 # ...or the query must cover the title's own words
def _title_index(papers_by_id: dict[str, dict[str, Any]]) -> TitleIndex:
"""One index per corpus dict, rebuilt only if the corpus changes size."""
key = id(papers_by_id)
cached = _TITLE_INDEX.get(key)
if cached is None or cached[0] != len(papers_by_id):
_TITLE_INDEX.clear() # one live corpus per process
cached = (len(papers_by_id), TitleIndex(papers_by_id))
_TITLE_INDEX[key] = cached
return cached[1]
def find_in_corpus(
query: str, papers_by_id: dict[str, dict[str, Any]], threshold: float = 0.72
) -> tuple[dict[str, Any] | None, list[dict[str, str]]]:
"""Best local title match, plus runners-up for disambiguation.
A high score alone is not enough at scale. The score asks how much of
the query a title covers, so with 123k papers a cited title that is NOT
in the corpus ("Self-Supervised Learning from Images with a
Joint-Embedding Predictive Architecture") matched three different JEPA
papers at an identical 0.82. A match is accepted only when it is also
unambiguous - clearly ahead of the runner-up - or covers the title's own
words too; otherwise resolution falls through to arXiv/OpenAlex.
Measured on 600 corpus titles: recall unchanged on exact, word-dropped
and word-added citations (100 / 99.5 / 98.7%); the tie case is rejected.
"""
index = _title_index(papers_by_id)
scored = index.search(query, floor=0.2)
candidates = [
{"paper_id": m.get("paper_id", ""), "title": m.get("title", ""), "score": f"{s:.2f}"}
for s, m in scored[:5]
if s > 0.2
]
if scored and scored[0][0] >= threshold:
best_score, best = scored[0]
runner_up = scored[1][0] if len(scored) > 1 else 0.0
if (best_score - runner_up >= AMBIGUITY_MARGIN
or index.title_coverage(query, best.get("title", "")) >= TITLE_COVERAGE_MIN):
return best, candidates
return None, candidates
def resolve(
citation: str,
papers_by_id: dict[str, dict[str, Any]],
allow_remote: bool = True,
fetch_full_text: bool = True,
pdf_dir: str = "cache/arxiv/pdf",
text_dir: str = "cache/arxiv/text",
max_cache_gb: float = 5.0,
) -> Resolution:
"""Turn a citation string into a Paper. Local corpus first, then arXiv."""
raw = (citation or "").strip()
if not raw:
return Resolution(None, "unresolved", "", False, [])
# A DOI's digits can look like an arXiv id ("10.1145/3292500.3330701"
# contains "2500.33307"), so a non-arXiv DOI takes precedence over it.
doi = openalex.extract_doi(raw)
arxiv_doi = bool(doi and doi.startswith("10.48550/arxiv."))
# --- 1. explicit arXiv id ---
arxiv_id = extract_arxiv_id(raw) if (not doi or arxiv_doi) else None
if arxiv_id:
local = papers_by_id.get(arxiv_id)
if local:
return Resolution(Paper(**local), "arxiv_id", arxiv_id, True, [])
if allow_remote:
found = _safe_fetch(f"id:{arxiv_id}", 1)
if found:
paper = found[0]
if fetch_full_text:
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
return Resolution(paper, "arxiv_id", arxiv_id, False, [])
# arXiv down or throttling: every arXiv paper also has a DataCite
# DOI, which OpenAlex resolves - so an arXiv id never depends on
# arXiv's own API being up.
work = _safe_openalex(openalex.get_by_doi, f"10.48550/arxiv.{arxiv_id}")
if work and work.get("title"):
return Resolution(_paper_from_openalex(work), "arxiv_id", arxiv_id, False, [])
# --- 1b. DOI: conference and journal papers arXiv does not know ---
if doi and not arxiv_doi and allow_remote:
work = _safe_openalex(openalex.get_by_doi, doi)
if work and work.get("title"):
if work.get("arxiv_id") and work["arxiv_id"] in papers_by_id:
return Resolution(Paper(**papers_by_id[work["arxiv_id"]]), "doi", doi, True, [])
paper = _paper_from_openalex(work)
if fetch_full_text and paper.pdf_url:
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
return Resolution(paper, "doi", doi, False, [])
# --- 2. title match against the local corpus ---
title_query = clean_citation(raw)
local_match, candidates = find_in_corpus(title_query, papers_by_id)
if local_match:
return Resolution(Paper(**local_match), "local_title", title_query, True, candidates)
# --- 3. ask arXiv ---
if allow_remote and title_query:
escaped = title_query.replace('"', "").strip()
found = _safe_fetch(f'ti:"{escaped}"', 8)
if not found:
found = _safe_fetch(f"all:{escaped}", 8)
if found:
best = max(found, key=lambda p: _token_overlap(title_query, p.title))
if _token_overlap(title_query, best.title) >= 0.4:
if fetch_full_text:
_attach_text(best, pdf_dir, text_dir, max_cache_gb)
return Resolution(best, "arxiv_search", title_query, False, candidates)
candidates = [
{"paper_id": p.paper_id, "title": p.title, "score": f"{_token_overlap(title_query, p.title):.2f}"}
for p in found
] + candidates
# --- 4. ask OpenAlex: the paper may simply never have been on arXiv ---
if allow_remote and title_query:
works = _safe_openalex(openalex.search_title, title_query, 8) or []
if works:
best = max(works, key=lambda w: _token_overlap(title_query, w["title"]))
overlap = _token_overlap(title_query, best["title"])
if overlap >= 0.6 and best.get("abstract"):
paper = _paper_from_openalex(best)
if fetch_full_text and paper.pdf_url:
_attach_text(paper, pdf_dir, text_dir, max_cache_gb)
return Resolution(paper, "openalex_title", title_query, False, candidates)
candidates = candidates + [
{"paper_id": w["paper_id"], "title": w["title"],
"score": f"{_token_overlap(title_query, w['title']):.2f}"}
for w in works[:3]
]
return Resolution(None, "unresolved", title_query, False, candidates[:5])
def _attach_text(paper: Paper, pdf_dir: str, text_dir: str, max_cache_gb: float = 5.0) -> None:
"""Best-effort full text for an on-demand paper. Abstract-only is acceptable."""
pdf = download_pdf(paper, pdf_dir, max_cache_gb=max_cache_gb)
if pdf is not None:
text = extract_text(pdf, text_dir)
if text and len(text.split()) > 200:
paper.full_text = text