Spaces:
Running on Zero
Running on Zero
Download src/data/citation.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 20.2 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/citation.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/data/citation.py
-
curl -L -o citation.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/citation.py
20.2 kB
| """Resolve a citation you saw in a paper to the actual paper. | |
| The use case: you are reading a paper, you hit "...following [23] (Assran et al., | |
| 2023)...", and you want to know what that is without leaving what you are doing. | |
| Accepted inputs, in the order they are tried: | |
| arXiv id 2301.08243, arXiv:2301.08243v2, https://arxiv.org/abs/2301.08243 | |
| DOI 10.1109/CVPR.2016.90, https://doi.org/10.1109/CVPR.2016.90 | |
| bracketed ref [23] M. Assran et al. "Self-Supervised Learning from Images..." | |
| bare title Self-Supervised Learning from Images with a Joint-Embedding... | |
| Resolution is local-first: the ingested corpus is checked before any network | |
| call, because most of what you are reading about is already indexed. Anything | |
| not found locally is fetched on demand - arXiv first, then OpenAlex, which | |
| covers the conference and journal papers that never had an arXiv preprint. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from dataclasses import dataclass | |
| from typing import Any | |
| from src.data import openalex_source as openalex | |
| from src.data.arxiv_source import ( | |
| ArxivUnavailable, Paper, download_pdf, extract_text, fetch_metadata_bounded, | |
| ) | |
| # 2301.08243 / 2301.08243v2, with optional arXiv: prefix or abs/pdf URL | |
| ARXIV_ID_RE = re.compile( | |
| r"(?:arxiv[:\s/]*|/abs/|/pdf/)?(?P<id>\d{4}\.\d{4,5})(?:v\d+)?", re.IGNORECASE | |
| ) | |
| # Leading "[23]" or "23." reference markers | |
| REF_MARKER_RE = re.compile(r"^\s*[\[\(]?\d{1,3}[\]\).]?\s*") | |
| # "Assran et al., 2023" style author-year tails | |
| AUTHOR_YEAR_RE = re.compile(r"\b[A-Z][a-z]+\s+et\s+al\.?,?\s*\(?\d{4}\)?[.,]?", re.IGNORECASE) | |
| # A leading author block terminated by "et al.", with or without initials: | |
| # "M. Assran et al." / "Kaiming He et al." / "Vaswani et al.," | |
| AUTHOR_ETAL_RE = re.compile( | |
| r"^(?:[A-Z][a-zA-Z.'-]*\s+)*?[A-Z][a-zA-Z'-]+,?\s+et\s+al\.?,?\s*", re.IGNORECASE | |
| ) | |
| QUOTED_RE = re.compile(r"[\"ββ']([^\"ββ']{12,})[\"ββ']") | |
| def _safe_fetch(query: str, max_results: int) -> list[Paper]: | |
| """An arXiv lookup that never takes down - or stalls - the request. | |
| Bounded by fetch_metadata_bounded: retries off, results cached, and the | |
| request stops waiting after the budget. Disabling retries alone was not | |
| enough - a throttled arXiv still held each call ~46s before its 429, and | |
| resolve() can make two calls in a row. A failed lookup returns nothing | |
| and the caller degrades. | |
| """ | |
| try: | |
| return fetch_metadata_bounded(query, max_results=max_results) | |
| except ArxivUnavailable as exc: | |
| print(f"[arxiv] lookup skipped ({exc}); degrading") | |
| return [] | |
| def _paper_from_openalex(work: dict[str, Any]) -> Paper: | |
| """An OpenAlex work in the Paper shape the rest of the pipeline expects.""" | |
| arxiv_id = work.get("arxiv_id") | |
| return Paper( | |
| paper_id=work["paper_id"], | |
| version=1, | |
| title=work["title"], | |
| abstract=work["abstract"], | |
| authors=work["authors"], | |
| categories=[], | |
| primary_category=work.get("venue", ""), | |
| published=work.get("published", ""), | |
| updated="", | |
| # Only arXiv PDFs are ever fetched on the request path: a known host | |
| # with stable, polite access. Other open-access PDFs are linked, not | |
| # downloaded. | |
| pdf_url=f"https://arxiv.org/pdf/{arxiv_id}" if arxiv_id else "", | |
| metadata={ | |
| "source": "openalex", | |
| "url": work.get("url", ""), | |
| "venue": work.get("venue", ""), | |
| "doi": work.get("doi") or "", | |
| "cited_by_count": str(work.get("cited_by_count", 0)), | |
| }, | |
| ) | |
| def _safe_openalex(fn, *args) -> Any: | |
| try: | |
| return fn(*args) | |
| except openalex.OpenAlexUnavailable as exc: | |
| print(f"[openalex] lookup skipped ({exc}); degrading") | |
| return None | |
| def paper_url(paper: Paper) -> str: | |
| """Where a reader should go for this paper, whichever source it came from.""" | |
| return (paper.metadata or {}).get("url") or f"https://arxiv.org/abs/{paper.paper_id}" | |
| class Resolution: | |
| """What a citation string resolved to, and how.""" | |
| paper: Paper | None | |
| method: str # arxiv_id | doi | local_title | arxiv_search | openalex_title | unresolved | |
| query_used: str | |
| from_corpus: bool | |
| candidates: list[dict[str, str]] | |
| QUESTION_STARTS = ( | |
| "what", "how", "why", "when", "where", "which", "who", "explain", "describe", | |
| "compare", "does", "do ", "is ", "are ", "can ", "should", "tell me", "give me", | |
| "summarise", "summarize", "define", | |
| ) | |
| # What a citation actually looks like. Anything matching one of these is worth | |
| # the cost of resolution; nothing else is. | |
| _CITATION_SHAPES = ( | |
| re.compile(r"\bet\s+al\b", re.IGNORECASE), # Assran et al. | |
| re.compile(r"^\s*[\[\(]\d{1,3}[\]\)]"), # [23] or (23) | |
| re.compile(r"^\s*\d{1,3}\.\s+[A-Z]"), # 23. Author | |
| re.compile(r"\((?:19|20)\d{2}\)"), # (2023) | |
| re.compile(r",\s*(?:19|20)\d{2}\b"), # , 2023 | |
| re.compile(r"\b(?:CVPR|NeurIPS|ICML|ICLR|ACL|EMNLP|AAAI|SIGGRAPH)\b"), | |
| re.compile(r"\barxiv\b", re.IGNORECASE), | |
| re.compile(r"\b10\.\d{4,9}/\S+"), # a DOI | |
| re.compile(r"[\"ββ']([^\"ββ']{15,})[\"ββ']"), # a quoted title | |
| ) | |
| def looks_like_a_citation(text: str) -> bool: | |
| """True only when the text has the shape of a reference. | |
| The default matters. Selecting a phrase in a paper - "MLP block of | |
| Transformers" - is asking about a concept, not naming a paper. Treating it | |
| as a title costs two arXiv title searches that cannot succeed, and arXiv | |
| rate-limits: one such selection took 34 seconds to fail. Citations announce | |
| themselves (an id, "et al.", a bracketed number, a year, a venue); concepts | |
| do not, so concepts are the default and resolution is opt-in. | |
| """ | |
| raw = (text or "").strip() | |
| if not raw: | |
| return False | |
| if extract_arxiv_id(raw): | |
| return True | |
| if any(shape.search(raw) for shape in _CITATION_SHAPES): | |
| return True | |
| # A bare title: long, no trailing question mark, and mostly capitalised | |
| # words - "Masked Autoencoders Are Scalable Vision Learners". | |
| words = raw.split() | |
| if 4 <= len(words) <= 25 and not raw.endswith("?"): | |
| capitalised = sum(1 for w in words if w[:1].isupper()) | |
| if capitalised >= max(3, len(words) * 0.5): | |
| return True | |
| return False | |
| def looks_like_a_question(text: str) -> bool: | |
| """Anything that is not shaped like a citation is treated as a question.""" | |
| raw = (text or "").strip() | |
| if not raw: | |
| return False | |
| if raw.endswith("?") or raw.lower().startswith(QUESTION_STARTS): | |
| return True | |
| return not looks_like_a_citation(raw) | |
| def extract_arxiv_id(text: str) -> str | None: | |
| """Pull an arXiv id out of free text, if there is one.""" | |
| match = ARXIV_ID_RE.search(text or "") | |
| return match.group("id") if match else None | |
| def clean_citation(text: str) -> str: | |
| """Strip reference markers, author-year tails and quotes down to a title-ish string.""" | |
| cleaned = REF_MARKER_RE.sub("", text or "").strip() | |
| # A quoted span in a reference is almost always the title. | |
| quoted = QUOTED_RE.search(cleaned) | |
| if quoted: | |
| return quoted.group(1).strip() | |
| cleaned = AUTHOR_ETAL_RE.sub("", cleaned) | |
| cleaned = AUTHOR_YEAR_RE.sub(" ", cleaned) | |
| # Drop a leading author list: "M. Assran, Q. Duval, I. Misra. Title here" | |
| cleaned = re.sub(r"^(?:[A-Z]\.\s*){1,4}[A-Z][a-z]+(?:,\s*(?:[A-Z]\.\s*)*[A-Z][a-z]+)*[.,]\s*", "", cleaned) | |
| # A bare year left over once the author block is gone: "2017. Attention Is..." | |
| cleaned = re.sub(r"^\(?(?:19|20)\d{2}\)?[.,]?\s*", "", cleaned) | |
| # Drop trailing venue noise | |
| cleaned = re.sub(r"\b(?:In\s+)?(?:Proceedings|Proc\.|CVPR|NeurIPS|ICML|ICLR|arXiv preprint)\b.*$", "", cleaned, flags=re.IGNORECASE) | |
| return re.sub(r"\s+", " ", cleaned).strip(" .,;:") | |
| def _normalise(text: str) -> str: | |
| return re.sub(r"[^a-z0-9 ]+", " ", (text or "").lower()) | |
| STOPWORDS = { | |
| "the", "a", "an", "of", "for", "with", "and", "in", "on", "to", "via", "using", | |
| "all", "you", "need", "is", "are", "we", "our", "its", "that", "this", "from", | |
| } | |
| def _tokens(text: str) -> set[str]: | |
| return {w for w in _normalise(text).split() if w not in STOPWORDS and len(w) > 2} | |
| def _token_overlap(a: str, b: str) -> float: | |
| """Jaccard overlap on content words. Used for ranking arXiv search results.""" | |
| ta, tb = _tokens(a), _tokens(b) | |
| if not ta or not tb: | |
| return 0.0 | |
| return len(ta & tb) / len(ta | tb) | |
| def _build_idf(papers_by_id: dict[str, dict[str, Any]]) -> dict[str, float]: | |
| """Inverse document frequency over corpus titles.""" | |
| import math | |
| df: dict[str, int] = {} | |
| for meta in papers_by_id.values(): | |
| for token in _tokens(meta.get("title", "")): | |
| df[token] = df.get(token, 0) + 1 | |
| n = max(1, len(papers_by_id)) | |
| return {t: math.log(n / (1 + c)) for t, c in df.items()} | |
| def _weighted_coverage(query: str, title: str, idf: dict[str, float], default_idf: float) -> float: | |
| """How much of the query's *information* the title covers. | |
| Plain Jaccard fails badly on titles built from common words. "Attention Is | |
| All You Need" against "All you need is log" scores 0.60 on Jaccard - three | |
| throwaway tokens match and the one word that carries the meaning, | |
| "attention", does not. Weighting each token by IDF makes the miss decisive: | |
| a rare token that fails to match costs far more than three common ones that | |
| do. | |
| """ | |
| qt = _tokens(query) | |
| if not qt: | |
| return 0.0 | |
| tt = _tokens(title) | |
| total = sum(idf.get(t, default_idf) for t in qt) | |
| if total <= 0: | |
| return 0.0 | |
| matched = sum(idf.get(t, default_idf) for t in qt & tt) | |
| # Penalise a title that is much longer than the query - it is a different | |
| # paper that happens to contain these words. | |
| length_penalty = min(1.0, len(qt) / max(1, len(tt))) ** 0.5 | |
| return (matched / total) * length_penalty | |
| class TitleIndex: | |
| """Inverted index over corpus titles, built once per corpus. | |
| find_in_corpus used to rebuild the IDF table and score every title on | |
| every lookup: fine at 8k papers, seconds per lookup at 300k. This keeps | |
| the scoring identical and only changes which titles get scored. | |
| Pruning is exact, not heuristic. A title that contains none of a set S of | |
| query tokens can match at most (total - w(S)) of the query's IDF weight, | |
| so if w(S) >= (1 - floor) * total it cannot score above `floor`. Taking | |
| S as the rarest query tokens until that holds, the postings of S contain | |
| every title that could clear the floor - the same results as a full scan. | |
| """ | |
| def __init__(self, papers_by_id: dict[str, dict[str, Any]]) -> None: | |
| import math | |
| self.papers = papers_by_id | |
| self.tokens: dict[str, set[str]] = {} | |
| postings: dict[str, list[str]] = {} | |
| for pid, meta in papers_by_id.items(): | |
| toks = _tokens(meta.get("title", "")) | |
| self.tokens[pid] = toks | |
| for t in toks: | |
| postings.setdefault(t, []).append(pid) | |
| self.postings = postings | |
| n = max(1, len(papers_by_id)) | |
| self.idf = {t: math.log(n / (1 + len(ids))) for t, ids in postings.items()} | |
| self.default_idf = max(self.idf.values(), default=1.0) | |
| def search(self, query: str, floor: float = 0.2) -> list[tuple[float, dict[str, Any]]]: | |
| qt = _tokens(query) | |
| if not qt: | |
| return [] | |
| weight = {t: self.idf.get(t, self.default_idf) for t in qt} | |
| total = sum(weight.values()) | |
| needed, covered, pivots = (1.0 - floor) * total, 0.0, [] | |
| for t in sorted(qt, key=weight.get, reverse=True): | |
| pivots.append(t) | |
| covered += weight[t] | |
| if covered >= needed: | |
| break | |
| candidates = {pid for t in pivots for pid in self.postings.get(t, ())} | |
| scored = [] | |
| for pid in candidates: | |
| tt = self.tokens[pid] | |
| matched = sum(weight[t] for t in qt & tt) | |
| score = (matched / total) * min(1.0, len(qt) / max(1, len(tt))) ** 0.5 | |
| scored.append((score, self.papers[pid])) | |
| scored.sort(key=lambda kv: kv[0], reverse=True) | |
| return scored | |
| def title_coverage(self, query: str, title: str) -> float: | |
| """Share of the TITLE's IDF weight that the query covers - the | |
| direction search() ignores.""" | |
| tt, qt = _tokens(title), _tokens(query) | |
| if not tt: | |
| return 0.0 | |
| w = {t: self.idf.get(t, self.default_idf) for t in tt} | |
| return sum(w[t] for t in tt & qt) / sum(w.values()) | |
| _TITLE_INDEX: dict[int, tuple[int, TitleIndex]] = {} | |
| AMBIGUITY_MARGIN = 0.05 # best must beat the runner-up by this much... | |
| TITLE_COVERAGE_MIN = 0.80 # ...or the query must cover the title's own words | |
| def _title_index(papers_by_id: dict[str, dict[str, Any]]) -> TitleIndex: | |
| """One index per corpus dict, rebuilt only if the corpus changes size.""" | |
| key = id(papers_by_id) | |
| cached = _TITLE_INDEX.get(key) | |
| if cached is None or cached[0] != len(papers_by_id): | |
| _TITLE_INDEX.clear() # one live corpus per process | |
| cached = (len(papers_by_id), TitleIndex(papers_by_id)) | |
| _TITLE_INDEX[key] = cached | |
| return cached[1] | |
| def find_in_corpus( | |
| query: str, papers_by_id: dict[str, dict[str, Any]], threshold: float = 0.72 | |
| ) -> tuple[dict[str, Any] | None, list[dict[str, str]]]: | |
| """Best local title match, plus runners-up for disambiguation. | |
| A high score alone is not enough at scale. The score asks how much of | |
| the query a title covers, so with 123k papers a cited title that is NOT | |
| in the corpus ("Self-Supervised Learning from Images with a | |
| Joint-Embedding Predictive Architecture") matched three different JEPA | |
| papers at an identical 0.82. A match is accepted only when it is also | |
| unambiguous - clearly ahead of the runner-up - or covers the title's own | |
| words too; otherwise resolution falls through to arXiv/OpenAlex. | |
| Measured on 600 corpus titles: recall unchanged on exact, word-dropped | |
| and word-added citations (100 / 99.5 / 98.7%); the tie case is rejected. | |
| """ | |
| index = _title_index(papers_by_id) | |
| scored = index.search(query, floor=0.2) | |
| candidates = [ | |
| {"paper_id": m.get("paper_id", ""), "title": m.get("title", ""), "score": f"{s:.2f}"} | |
| for s, m in scored[:5] | |
| if s > 0.2 | |
| ] | |
| if scored and scored[0][0] >= threshold: | |
| best_score, best = scored[0] | |
| runner_up = scored[1][0] if len(scored) > 1 else 0.0 | |
| if (best_score - runner_up >= AMBIGUITY_MARGIN | |
| or index.title_coverage(query, best.get("title", "")) >= TITLE_COVERAGE_MIN): | |
| return best, candidates | |
| return None, candidates | |
| def resolve( | |
| citation: str, | |
| papers_by_id: dict[str, dict[str, Any]], | |
| allow_remote: bool = True, | |
| fetch_full_text: bool = True, | |
| pdf_dir: str = "cache/arxiv/pdf", | |
| text_dir: str = "cache/arxiv/text", | |
| max_cache_gb: float = 5.0, | |
| ) -> Resolution: | |
| """Turn a citation string into a Paper. Local corpus first, then arXiv.""" | |
| raw = (citation or "").strip() | |
| if not raw: | |
| return Resolution(None, "unresolved", "", False, []) | |
| # A DOI's digits can look like an arXiv id ("10.1145/3292500.3330701" | |
| # contains "2500.33307"), so a non-arXiv DOI takes precedence over it. | |
| doi = openalex.extract_doi(raw) | |
| arxiv_doi = bool(doi and doi.startswith("10.48550/arxiv.")) | |
| # --- 1. explicit arXiv id --- | |
| arxiv_id = extract_arxiv_id(raw) if (not doi or arxiv_doi) else None | |
| if arxiv_id: | |
| local = papers_by_id.get(arxiv_id) | |
| if local: | |
| return Resolution(Paper(**local), "arxiv_id", arxiv_id, True, []) | |
| if allow_remote: | |
| found = _safe_fetch(f"id:{arxiv_id}", 1) | |
| if found: | |
| paper = found[0] | |
| if fetch_full_text: | |
| _attach_text(paper, pdf_dir, text_dir, max_cache_gb) | |
| return Resolution(paper, "arxiv_id", arxiv_id, False, []) | |
| # arXiv down or throttling: every arXiv paper also has a DataCite | |
| # DOI, which OpenAlex resolves - so an arXiv id never depends on | |
| # arXiv's own API being up. | |
| work = _safe_openalex(openalex.get_by_doi, f"10.48550/arxiv.{arxiv_id}") | |
| if work and work.get("title"): | |
| return Resolution(_paper_from_openalex(work), "arxiv_id", arxiv_id, False, []) | |
| # --- 1b. DOI: conference and journal papers arXiv does not know --- | |
| if doi and not arxiv_doi and allow_remote: | |
| work = _safe_openalex(openalex.get_by_doi, doi) | |
| if work and work.get("title"): | |
| if work.get("arxiv_id") and work["arxiv_id"] in papers_by_id: | |
| return Resolution(Paper(**papers_by_id[work["arxiv_id"]]), "doi", doi, True, []) | |
| paper = _paper_from_openalex(work) | |
| if fetch_full_text and paper.pdf_url: | |
| _attach_text(paper, pdf_dir, text_dir, max_cache_gb) | |
| return Resolution(paper, "doi", doi, False, []) | |
| # --- 2. title match against the local corpus --- | |
| title_query = clean_citation(raw) | |
| local_match, candidates = find_in_corpus(title_query, papers_by_id) | |
| if local_match: | |
| return Resolution(Paper(**local_match), "local_title", title_query, True, candidates) | |
| # --- 3. ask arXiv --- | |
| if allow_remote and title_query: | |
| escaped = title_query.replace('"', "").strip() | |
| found = _safe_fetch(f'ti:"{escaped}"', 8) | |
| if not found: | |
| found = _safe_fetch(f"all:{escaped}", 8) | |
| if found: | |
| best = max(found, key=lambda p: _token_overlap(title_query, p.title)) | |
| if _token_overlap(title_query, best.title) >= 0.4: | |
| if fetch_full_text: | |
| _attach_text(best, pdf_dir, text_dir, max_cache_gb) | |
| return Resolution(best, "arxiv_search", title_query, False, candidates) | |
| candidates = [ | |
| {"paper_id": p.paper_id, "title": p.title, "score": f"{_token_overlap(title_query, p.title):.2f}"} | |
| for p in found | |
| ] + candidates | |
| # --- 4. ask OpenAlex: the paper may simply never have been on arXiv --- | |
| if allow_remote and title_query: | |
| works = _safe_openalex(openalex.search_title, title_query, 8) or [] | |
| if works: | |
| best = max(works, key=lambda w: _token_overlap(title_query, w["title"])) | |
| overlap = _token_overlap(title_query, best["title"]) | |
| if overlap >= 0.6 and best.get("abstract"): | |
| paper = _paper_from_openalex(best) | |
| if fetch_full_text and paper.pdf_url: | |
| _attach_text(paper, pdf_dir, text_dir, max_cache_gb) | |
| return Resolution(paper, "openalex_title", title_query, False, candidates) | |
| candidates = candidates + [ | |
| {"paper_id": w["paper_id"], "title": w["title"], | |
| "score": f"{_token_overlap(title_query, w['title']):.2f}"} | |
| for w in works[:3] | |
| ] | |
| return Resolution(None, "unresolved", title_query, False, candidates[:5]) | |
| def _attach_text(paper: Paper, pdf_dir: str, text_dir: str, max_cache_gb: float = 5.0) -> None: | |
| """Best-effort full text for an on-demand paper. Abstract-only is acceptable.""" | |
| pdf = download_pdf(paper, pdf_dir, max_cache_gb=max_cache_gb) | |
| if pdf is not None: | |
| text = extract_text(pdf, text_dir) | |
| if text and len(text.split()) > 200: | |
| paper.full_text = text | |