"""arXiv ingestion: metadata, full text, and version dedup. Design notes that follow from REQUIREMENTS-ML.md: FR1.4 - resumable. Every PDF and every extracted text file is cached on disk by arXiv id. Re-running ingestion skips anything already fetched, so an interrupted run costs nothing. FR6.2 - version dedup. arXiv ids carry a version suffix (2401.12345v3). The index is keyed on the *versionless* id, so a later version replaces an earlier one instead of sitting beside it as a near-duplicate. Duplicate near-identical papers corrupt both retrieval and any evaluation run over the index. """ from __future__ import annotations import json import re import threading import time from concurrent.futures import ThreadPoolExecutor from concurrent.futures import TimeoutError as FuturesTimeout from dataclasses import asdict, dataclass, field from pathlib import Path import arxiv # arXiv rate-limits hard (HTTP 429) and the client library retries internally # with multi-second sleeps, so one unlucky interactive query could block for a # minute and then fail. A small in-process cache makes repeated lookups free, # which matters because the same paper gets asked about repeatedly. _SEARCH_CACHE: dict[str, list["Paper"]] = {} _SEARCH_CACHE_MAX = 512 # "http://arxiv.org/abs/2401.12345v3" -> ("2401.12345", 3) _ENTRY_RE = re.compile(r"/abs/(?P[^v]+(?:v\d+)?)") _VERSION_RE = re.compile(r"^(?P.+?)v(?P\d+)$") @dataclass class Paper: """One arXiv paper, versionless id as the primary key.""" paper_id: str version: int title: str abstract: str authors: list[str] categories: list[str] primary_category: str published: str updated: str pdf_url: str full_text: str | None = None metadata: dict[str, str] = field(default_factory=dict) @property def versioned_id(self) -> str: return f"{self.paper_id}v{self.version}" def _split_version(entry_id: str) -> tuple[str, int]: """Turn an arXiv entry URL into (versionless_id, version).""" match = _ENTRY_RE.search(entry_id) raw = match.group("id") if match else entry_id.rsplit("/", 1)[-1] versioned = _VERSION_RE.match(raw) if versioned: return versioned.group("base"), int(versioned.group("version")) return raw, 1 def _clean(text: str) -> str: """Collapse the whitespace arXiv abstracts arrive with.""" return re.sub(r"\s+", " ", text or "").strip() def fetch_metadata( query: str, max_results: int, page_size: int = 100, delay_seconds: float = 3.0, sort_by_relevance: bool = False, num_retries: int = 1, use_cache: bool = False, ) -> list[Paper]: """Run one arXiv API query and return papers. Ingestion wants newest-first (build a current corpus). Citation lookup wants relevance - sorting a title search by date returns the most *recent* papers whose titles match, so a foundational paper like "Attention Is All You Need" is buried under every recent paper that echoes its title. """ cache_key = f"{query}|{max_results}|{sort_by_relevance}" if use_cache and cache_key in _SEARCH_CACHE: return _SEARCH_CACHE[cache_key] client = arxiv.Client( page_size=page_size, delay_seconds=delay_seconds, num_retries=num_retries ) search = arxiv.Search( query=query, max_results=max_results, sort_by=( arxiv.SortCriterion.Relevance if sort_by_relevance else arxiv.SortCriterion.SubmittedDate ), ) papers: list[Paper] = [] for result in client.results(search): paper_id, version = _split_version(result.entry_id) papers.append( Paper( paper_id=paper_id, version=version, title=_clean(result.title), abstract=_clean(result.summary), authors=[a.name for a in result.authors][:20], categories=list(result.categories), primary_category=result.primary_category, published=result.published.date().isoformat() if result.published else "", updated=result.updated.date().isoformat() if result.updated else "", pdf_url=result.pdf_url or "", ) ) print(f"[arxiv] {len(papers):>5} results for: {query[:80]}") if use_cache: if len(_SEARCH_CACHE) >= _SEARCH_CACHE_MAX: _SEARCH_CACHE.clear() _SEARCH_CACHE[cache_key] = papers return papers def dedup_by_version(papers: list[Paper]) -> list[Paper]: """FR6.2 - keep only the highest version of each paper.""" best: dict[str, Paper] = {} for paper in papers: existing = best.get(paper.paper_id) if existing is None or paper.version > existing.version: best[paper.paper_id] = paper dropped = len(papers) - len(best) if dropped: print(f"[arxiv] dedup: dropped {dropped} superseded/duplicate versions") return list(best.values()) def prune_pdf_cache(pdf_dir: str | Path, max_gb: float) -> int: """Keep the PDF cache under `max_gb`, deleting least-recently-used first. Citation lookup downloads a PDF per unresolved paper, so on a shared deployment this directory grows without bound. Extracted text is kept - it is ~1% of the size and is what retrieval actually uses, so evicting a PDF costs nothing unless the same paper is re-parsed. """ directory = Path(pdf_dir) if not directory.exists() or max_gb <= 0: return 0 files = sorted(directory.glob("*.pdf"), key=lambda f: f.stat().st_atime) total = sum(f.stat().st_size for f in files) budget = max_gb * 1e9 removed = 0 for path in files: if total <= budget: break size = path.stat().st_size try: path.unlink() total -= size removed += 1 except OSError: continue if removed: print(f"[arxiv] pruned {removed} cached PDFs, cache now {total / 1e9:.1f} GB") return removed # ---------------------------------------------------------------------------- # Request-time lookups # # fetch_metadata above is built for ingestion, where waiting out a rate limit # is correct. Inside a user's request it is not: the arxiv client has no # request timeout, and a throttled arXiv does not fail fast - a lookup was # measured holding the connection 46-50s before answering HTTP 429, with # retries already disabled. Hugging Face Spaces share outbound IPs, so the # deployed backend is throttled far more often than a laptop is. # # So request-time callers go through fetch_metadata_bounded: the call runs on # a small pool and the request stops waiting at the budget. One breaker is # shared by every caller - live search and citation lookup hit the same API, # so if one is being throttled the other is too, and continuing to ask only # extends the throttle. # ---------------------------------------------------------------------------- REQUEST_BUDGET_S = 5.0 BREAKER_COOLDOWN_S = 300.0 _REQUEST_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="arxiv-request") _BREAKER_LOCK = threading.Lock() _BREAKER = {"open_until": 0.0} class ArxivUnavailable(RuntimeError): """arXiv did not answer in time, refused, or is cooling down.""" def breaker_open() -> bool: with _BREAKER_LOCK: return time.monotonic() < _BREAKER["open_until"] def _trip_breaker(reason: str) -> None: with _BREAKER_LOCK: _BREAKER["open_until"] = time.monotonic() + BREAKER_COOLDOWN_S print(f"[arxiv] {reason} - pausing request-time lookups for " f"{BREAKER_COOLDOWN_S:.0f}s", flush=True) def fetch_metadata_bounded( query: str, max_results: int, budget_s: float = REQUEST_BUDGET_S, sort_by_relevance: bool = True, ) -> list[Paper]: """fetch_metadata for use inside a request: bounded, retry-free, cached. Raises ArxivUnavailable instead of blocking. Callers treat that as "no result" and degrade - these lookups are always a fallback or an enrichment, never the only way to answer. """ if breaker_open(): raise ArxivUnavailable("cooling down after a recent failure") future = _REQUEST_POOL.submit( fetch_metadata, query, max_results=max_results, sort_by_relevance=sort_by_relevance, delay_seconds=0.0, num_retries=0, use_cache=True, ) try: return future.result(timeout=max(budget_s, 0.1)) except FuturesTimeout: # The worker thread finishes on its own; the request just stops waiting. _trip_breaker(f"no answer within {budget_s:.1f}s") raise ArxivUnavailable(f"timed out after {budget_s:.1f}s") from None except Exception as exc: # noqa: BLE001 _trip_breaker(f"{type(exc).__name__}: {str(exc)[:80]}") raise ArxivUnavailable(str(exc)) from exc def download_pdf( paper: Paper, pdf_dir: str | Path, delay_seconds: float = 1.0, max_cache_gb: float = 0.0, ) -> Path | None: """Fetch a PDF, reusing the cached copy when present. Returns None on failure.""" target = Path(pdf_dir) / f"{paper.versioned_id}.pdf" target.parent.mkdir(parents=True, exist_ok=True) if target.exists() and target.stat().st_size > 0: return target if max_cache_gb > 0: prune_pdf_cache(pdf_dir, max_cache_gb) try: import requests response = requests.get( paper.pdf_url, timeout=60, headers={"User-Agent": "ResearchOS-ML/1.0 (research assistant; arXiv ingest)"}, ) response.raise_for_status() target.write_bytes(response.content) time.sleep(delay_seconds) # be polite to arXiv return target except Exception as exc: # noqa: BLE001 - one bad PDF must not stop ingestion print(f"[arxiv] pdf failed {paper.versioned_id}: {type(exc).__name__}") return None def extract_text(pdf_path: Path, text_dir: str | Path) -> str | None: """Extract text from a PDF, caching the result. Returns None on failure.""" cached = Path(text_dir) / f"{pdf_path.stem}.txt" cached.parent.mkdir(parents=True, exist_ok=True) if cached.exists(): return cached.read_text(encoding="utf-8") try: import pymupdf with pymupdf.open(pdf_path) as doc: text = "\n".join(page.get_text() for page in doc) cached.write_text(text, encoding="utf-8") return text except Exception as exc: # noqa: BLE001 print(f"[arxiv] text extraction failed {pdf_path.name}: {type(exc).__name__}") return None def write_papers(papers: list[Paper], path: str | Path) -> None: """Persist papers as JSONL.""" target = Path(path) target.parent.mkdir(parents=True, exist_ok=True) with open(target, "w", encoding="utf-8") as handle: for paper in papers: handle.write(json.dumps(asdict(paper), ensure_ascii=False) + "\n") print(f"[arxiv] wrote {len(papers)} papers -> {target}") def read_papers(path: str | Path) -> list[Paper]: """Load papers back from JSONL.""" with open(path, encoding="utf-8") as handle: return [Paper(**json.loads(line)) for line in handle if line.strip()]