ResearchOS / src /data /arxiv_source.py
Kagzy's picture
Bound request-time arXiv calls: 5s budget, shared breaker
af38999 verified
Raw History Blame Contribute Delete
11.6 kB
"""arXiv ingestion: metadata, full text, and version dedup.
Design notes that follow from REQUIREMENTS-ML.md:
FR1.4 - resumable. Every PDF and every extracted text file is cached on disk by
arXiv id. Re-running ingestion skips anything already fetched, so an interrupted
run costs nothing.
FR6.2 - version dedup. arXiv ids carry a version suffix (2401.12345v3). The
index is keyed on the *versionless* id, so a later version replaces an earlier
one instead of sitting beside it as a near-duplicate. Duplicate near-identical
papers corrupt both retrieval and any evaluation run over the index.
"""
from __future__ import annotations
import json
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from dataclasses import asdict, dataclass, field
from pathlib import Path
import arxiv
# arXiv rate-limits hard (HTTP 429) and the client library retries internally
# with multi-second sleeps, so one unlucky interactive query could block for a
# minute and then fail. A small in-process cache makes repeated lookups free,
# which matters because the same paper gets asked about repeatedly.
_SEARCH_CACHE: dict[str, list["Paper"]] = {}
_SEARCH_CACHE_MAX = 512
# "http://arxiv.org/abs/2401.12345v3" -> ("2401.12345", 3)
_ENTRY_RE = re.compile(r"/abs/(?P<id>[^v]+(?:v\d+)?)")
_VERSION_RE = re.compile(r"^(?P<base>.+?)v(?P<version>\d+)$")
@dataclass
class Paper:
"""One arXiv paper, versionless id as the primary key."""
paper_id: str
version: int
title: str
abstract: str
authors: list[str]
categories: list[str]
primary_category: str
published: str
updated: str
pdf_url: str
full_text: str | None = None
metadata: dict[str, str] = field(default_factory=dict)
@property
def versioned_id(self) -> str:
return f"{self.paper_id}v{self.version}"
def _split_version(entry_id: str) -> tuple[str, int]:
"""Turn an arXiv entry URL into (versionless_id, version)."""
match = _ENTRY_RE.search(entry_id)
raw = match.group("id") if match else entry_id.rsplit("/", 1)[-1]
versioned = _VERSION_RE.match(raw)
if versioned:
return versioned.group("base"), int(versioned.group("version"))
return raw, 1
def _clean(text: str) -> str:
"""Collapse the whitespace arXiv abstracts arrive with."""
return re.sub(r"\s+", " ", text or "").strip()
def fetch_metadata(
query: str,
max_results: int,
page_size: int = 100,
delay_seconds: float = 3.0,
sort_by_relevance: bool = False,
num_retries: int = 1,
use_cache: bool = False,
) -> list[Paper]:
"""Run one arXiv API query and return papers.
Ingestion wants newest-first (build a current corpus). Citation lookup wants
relevance - sorting a title search by date returns the most *recent* papers
whose titles match, so a foundational paper like "Attention Is All You Need"
is buried under every recent paper that echoes its title.
"""
cache_key = f"{query}|{max_results}|{sort_by_relevance}"
if use_cache and cache_key in _SEARCH_CACHE:
return _SEARCH_CACHE[cache_key]
client = arxiv.Client(
page_size=page_size, delay_seconds=delay_seconds, num_retries=num_retries
)
search = arxiv.Search(
query=query,
max_results=max_results,
sort_by=(
arxiv.SortCriterion.Relevance
if sort_by_relevance
else arxiv.SortCriterion.SubmittedDate
),
)
papers: list[Paper] = []
for result in client.results(search):
paper_id, version = _split_version(result.entry_id)
papers.append(
Paper(
paper_id=paper_id,
version=version,
title=_clean(result.title),
abstract=_clean(result.summary),
authors=[a.name for a in result.authors][:20],
categories=list(result.categories),
primary_category=result.primary_category,
published=result.published.date().isoformat() if result.published else "",
updated=result.updated.date().isoformat() if result.updated else "",
pdf_url=result.pdf_url or "",
)
)
print(f"[arxiv] {len(papers):>5} results for: {query[:80]}")
if use_cache:
if len(_SEARCH_CACHE) >= _SEARCH_CACHE_MAX:
_SEARCH_CACHE.clear()
_SEARCH_CACHE[cache_key] = papers
return papers
def dedup_by_version(papers: list[Paper]) -> list[Paper]:
"""FR6.2 - keep only the highest version of each paper."""
best: dict[str, Paper] = {}
for paper in papers:
existing = best.get(paper.paper_id)
if existing is None or paper.version > existing.version:
best[paper.paper_id] = paper
dropped = len(papers) - len(best)
if dropped:
print(f"[arxiv] dedup: dropped {dropped} superseded/duplicate versions")
return list(best.values())
def prune_pdf_cache(pdf_dir: str | Path, max_gb: float) -> int:
"""Keep the PDF cache under `max_gb`, deleting least-recently-used first.
Citation lookup downloads a PDF per unresolved paper, so on a shared
deployment this directory grows without bound. Extracted text is kept - it
is ~1% of the size and is what retrieval actually uses, so evicting a PDF
costs nothing unless the same paper is re-parsed.
"""
directory = Path(pdf_dir)
if not directory.exists() or max_gb <= 0:
return 0
files = sorted(directory.glob("*.pdf"), key=lambda f: f.stat().st_atime)
total = sum(f.stat().st_size for f in files)
budget = max_gb * 1e9
removed = 0
for path in files:
if total <= budget:
break
size = path.stat().st_size
try:
path.unlink()
total -= size
removed += 1
except OSError:
continue
if removed:
print(f"[arxiv] pruned {removed} cached PDFs, cache now {total / 1e9:.1f} GB")
return removed
# ----------------------------------------------------------------------------
# Request-time lookups
#
# fetch_metadata above is built for ingestion, where waiting out a rate limit
# is correct. Inside a user's request it is not: the arxiv client has no
# request timeout, and a throttled arXiv does not fail fast - a lookup was
# measured holding the connection 46-50s before answering HTTP 429, with
# retries already disabled. Hugging Face Spaces share outbound IPs, so the
# deployed backend is throttled far more often than a laptop is.
#
# So request-time callers go through fetch_metadata_bounded: the call runs on
# a small pool and the request stops waiting at the budget. One breaker is
# shared by every caller - live search and citation lookup hit the same API,
# so if one is being throttled the other is too, and continuing to ask only
# extends the throttle.
# ----------------------------------------------------------------------------
REQUEST_BUDGET_S = 5.0
BREAKER_COOLDOWN_S = 300.0
_REQUEST_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="arxiv-request")
_BREAKER_LOCK = threading.Lock()
_BREAKER = {"open_until": 0.0}
class ArxivUnavailable(RuntimeError):
"""arXiv did not answer in time, refused, or is cooling down."""
def breaker_open() -> bool:
with _BREAKER_LOCK:
return time.monotonic() < _BREAKER["open_until"]
def _trip_breaker(reason: str) -> None:
with _BREAKER_LOCK:
_BREAKER["open_until"] = time.monotonic() + BREAKER_COOLDOWN_S
print(f"[arxiv] {reason} - pausing request-time lookups for "
f"{BREAKER_COOLDOWN_S:.0f}s", flush=True)
def fetch_metadata_bounded(
query: str,
max_results: int,
budget_s: float = REQUEST_BUDGET_S,
sort_by_relevance: bool = True,
) -> list[Paper]:
"""fetch_metadata for use inside a request: bounded, retry-free, cached.
Raises ArxivUnavailable instead of blocking. Callers treat that as "no
result" and degrade - these lookups are always a fallback or an
enrichment, never the only way to answer.
"""
if breaker_open():
raise ArxivUnavailable("cooling down after a recent failure")
future = _REQUEST_POOL.submit(
fetch_metadata, query, max_results=max_results,
sort_by_relevance=sort_by_relevance,
delay_seconds=0.0, num_retries=0, use_cache=True,
)
try:
return future.result(timeout=max(budget_s, 0.1))
except FuturesTimeout:
# The worker thread finishes on its own; the request just stops waiting.
_trip_breaker(f"no answer within {budget_s:.1f}s")
raise ArxivUnavailable(f"timed out after {budget_s:.1f}s") from None
except Exception as exc: # noqa: BLE001
_trip_breaker(f"{type(exc).__name__}: {str(exc)[:80]}")
raise ArxivUnavailable(str(exc)) from exc
def download_pdf(
paper: Paper,
pdf_dir: str | Path,
delay_seconds: float = 1.0,
max_cache_gb: float = 0.0,
) -> Path | None:
"""Fetch a PDF, reusing the cached copy when present. Returns None on failure."""
target = Path(pdf_dir) / f"{paper.versioned_id}.pdf"
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists() and target.stat().st_size > 0:
return target
if max_cache_gb > 0:
prune_pdf_cache(pdf_dir, max_cache_gb)
try:
import requests
response = requests.get(
paper.pdf_url,
timeout=60,
headers={"User-Agent": "ResearchOS-ML/1.0 (research assistant; arXiv ingest)"},
)
response.raise_for_status()
target.write_bytes(response.content)
time.sleep(delay_seconds) # be polite to arXiv
return target
except Exception as exc: # noqa: BLE001 - one bad PDF must not stop ingestion
print(f"[arxiv] pdf failed {paper.versioned_id}: {type(exc).__name__}")
return None
def extract_text(pdf_path: Path, text_dir: str | Path) -> str | None:
"""Extract text from a PDF, caching the result. Returns None on failure."""
cached = Path(text_dir) / f"{pdf_path.stem}.txt"
cached.parent.mkdir(parents=True, exist_ok=True)
if cached.exists():
return cached.read_text(encoding="utf-8")
try:
import pymupdf
with pymupdf.open(pdf_path) as doc:
text = "\n".join(page.get_text() for page in doc)
cached.write_text(text, encoding="utf-8")
return text
except Exception as exc: # noqa: BLE001
print(f"[arxiv] text extraction failed {pdf_path.name}: {type(exc).__name__}")
return None
def write_papers(papers: list[Paper], path: str | Path) -> None:
"""Persist papers as JSONL."""
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
with open(target, "w", encoding="utf-8") as handle:
for paper in papers:
handle.write(json.dumps(asdict(paper), ensure_ascii=False) + "\n")
print(f"[arxiv] wrote {len(papers)} papers -> {target}")
def read_papers(path: str | Path) -> list[Paper]:
"""Load papers back from JSONL."""
with open(path, encoding="utf-8") as handle:
return [Paper(**json.loads(line)) for line in handle if line.strip()]