Spaces:
Running on Zero
Running on Zero
Download src/data/arxiv_source.py from Kagzy/ResearchOS: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/arxiv_source.py
- Command line
-
hf download hf://spaces/Kagzy/ResearchOS/src/data/arxiv_source.py
-
curl -L -o arxiv_source.py https://huggingface.co/spaces/Kagzy/ResearchOS/resolve/main/src/data/arxiv_source.py
11.6 kB
| """arXiv ingestion: metadata, full text, and version dedup. | |
| Design notes that follow from REQUIREMENTS-ML.md: | |
| FR1.4 - resumable. Every PDF and every extracted text file is cached on disk by | |
| arXiv id. Re-running ingestion skips anything already fetched, so an interrupted | |
| run costs nothing. | |
| FR6.2 - version dedup. arXiv ids carry a version suffix (2401.12345v3). The | |
| index is keyed on the *versionless* id, so a later version replaces an earlier | |
| one instead of sitting beside it as a near-duplicate. Duplicate near-identical | |
| papers corrupt both retrieval and any evaluation run over the index. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import re | |
| import threading | |
| import time | |
| from concurrent.futures import ThreadPoolExecutor | |
| from concurrent.futures import TimeoutError as FuturesTimeout | |
| from dataclasses import asdict, dataclass, field | |
| from pathlib import Path | |
| import arxiv | |
| # arXiv rate-limits hard (HTTP 429) and the client library retries internally | |
| # with multi-second sleeps, so one unlucky interactive query could block for a | |
| # minute and then fail. A small in-process cache makes repeated lookups free, | |
| # which matters because the same paper gets asked about repeatedly. | |
| _SEARCH_CACHE: dict[str, list["Paper"]] = {} | |
| _SEARCH_CACHE_MAX = 512 | |
| # "http://arxiv.org/abs/2401.12345v3" -> ("2401.12345", 3) | |
| _ENTRY_RE = re.compile(r"/abs/(?P<id>[^v]+(?:v\d+)?)") | |
| _VERSION_RE = re.compile(r"^(?P<base>.+?)v(?P<version>\d+)$") | |
| class Paper: | |
| """One arXiv paper, versionless id as the primary key.""" | |
| paper_id: str | |
| version: int | |
| title: str | |
| abstract: str | |
| authors: list[str] | |
| categories: list[str] | |
| primary_category: str | |
| published: str | |
| updated: str | |
| pdf_url: str | |
| full_text: str | None = None | |
| metadata: dict[str, str] = field(default_factory=dict) | |
| def versioned_id(self) -> str: | |
| return f"{self.paper_id}v{self.version}" | |
| def _split_version(entry_id: str) -> tuple[str, int]: | |
| """Turn an arXiv entry URL into (versionless_id, version).""" | |
| match = _ENTRY_RE.search(entry_id) | |
| raw = match.group("id") if match else entry_id.rsplit("/", 1)[-1] | |
| versioned = _VERSION_RE.match(raw) | |
| if versioned: | |
| return versioned.group("base"), int(versioned.group("version")) | |
| return raw, 1 | |
| def _clean(text: str) -> str: | |
| """Collapse the whitespace arXiv abstracts arrive with.""" | |
| return re.sub(r"\s+", " ", text or "").strip() | |
| def fetch_metadata( | |
| query: str, | |
| max_results: int, | |
| page_size: int = 100, | |
| delay_seconds: float = 3.0, | |
| sort_by_relevance: bool = False, | |
| num_retries: int = 1, | |
| use_cache: bool = False, | |
| ) -> list[Paper]: | |
| """Run one arXiv API query and return papers. | |
| Ingestion wants newest-first (build a current corpus). Citation lookup wants | |
| relevance - sorting a title search by date returns the most *recent* papers | |
| whose titles match, so a foundational paper like "Attention Is All You Need" | |
| is buried under every recent paper that echoes its title. | |
| """ | |
| cache_key = f"{query}|{max_results}|{sort_by_relevance}" | |
| if use_cache and cache_key in _SEARCH_CACHE: | |
| return _SEARCH_CACHE[cache_key] | |
| client = arxiv.Client( | |
| page_size=page_size, delay_seconds=delay_seconds, num_retries=num_retries | |
| ) | |
| search = arxiv.Search( | |
| query=query, | |
| max_results=max_results, | |
| sort_by=( | |
| arxiv.SortCriterion.Relevance | |
| if sort_by_relevance | |
| else arxiv.SortCriterion.SubmittedDate | |
| ), | |
| ) | |
| papers: list[Paper] = [] | |
| for result in client.results(search): | |
| paper_id, version = _split_version(result.entry_id) | |
| papers.append( | |
| Paper( | |
| paper_id=paper_id, | |
| version=version, | |
| title=_clean(result.title), | |
| abstract=_clean(result.summary), | |
| authors=[a.name for a in result.authors][:20], | |
| categories=list(result.categories), | |
| primary_category=result.primary_category, | |
| published=result.published.date().isoformat() if result.published else "", | |
| updated=result.updated.date().isoformat() if result.updated else "", | |
| pdf_url=result.pdf_url or "", | |
| ) | |
| ) | |
| print(f"[arxiv] {len(papers):>5} results for: {query[:80]}") | |
| if use_cache: | |
| if len(_SEARCH_CACHE) >= _SEARCH_CACHE_MAX: | |
| _SEARCH_CACHE.clear() | |
| _SEARCH_CACHE[cache_key] = papers | |
| return papers | |
| def dedup_by_version(papers: list[Paper]) -> list[Paper]: | |
| """FR6.2 - keep only the highest version of each paper.""" | |
| best: dict[str, Paper] = {} | |
| for paper in papers: | |
| existing = best.get(paper.paper_id) | |
| if existing is None or paper.version > existing.version: | |
| best[paper.paper_id] = paper | |
| dropped = len(papers) - len(best) | |
| if dropped: | |
| print(f"[arxiv] dedup: dropped {dropped} superseded/duplicate versions") | |
| return list(best.values()) | |
| def prune_pdf_cache(pdf_dir: str | Path, max_gb: float) -> int: | |
| """Keep the PDF cache under `max_gb`, deleting least-recently-used first. | |
| Citation lookup downloads a PDF per unresolved paper, so on a shared | |
| deployment this directory grows without bound. Extracted text is kept - it | |
| is ~1% of the size and is what retrieval actually uses, so evicting a PDF | |
| costs nothing unless the same paper is re-parsed. | |
| """ | |
| directory = Path(pdf_dir) | |
| if not directory.exists() or max_gb <= 0: | |
| return 0 | |
| files = sorted(directory.glob("*.pdf"), key=lambda f: f.stat().st_atime) | |
| total = sum(f.stat().st_size for f in files) | |
| budget = max_gb * 1e9 | |
| removed = 0 | |
| for path in files: | |
| if total <= budget: | |
| break | |
| size = path.stat().st_size | |
| try: | |
| path.unlink() | |
| total -= size | |
| removed += 1 | |
| except OSError: | |
| continue | |
| if removed: | |
| print(f"[arxiv] pruned {removed} cached PDFs, cache now {total / 1e9:.1f} GB") | |
| return removed | |
| # ---------------------------------------------------------------------------- | |
| # Request-time lookups | |
| # | |
| # fetch_metadata above is built for ingestion, where waiting out a rate limit | |
| # is correct. Inside a user's request it is not: the arxiv client has no | |
| # request timeout, and a throttled arXiv does not fail fast - a lookup was | |
| # measured holding the connection 46-50s before answering HTTP 429, with | |
| # retries already disabled. Hugging Face Spaces share outbound IPs, so the | |
| # deployed backend is throttled far more often than a laptop is. | |
| # | |
| # So request-time callers go through fetch_metadata_bounded: the call runs on | |
| # a small pool and the request stops waiting at the budget. One breaker is | |
| # shared by every caller - live search and citation lookup hit the same API, | |
| # so if one is being throttled the other is too, and continuing to ask only | |
| # extends the throttle. | |
| # ---------------------------------------------------------------------------- | |
| REQUEST_BUDGET_S = 5.0 | |
| BREAKER_COOLDOWN_S = 300.0 | |
| _REQUEST_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="arxiv-request") | |
| _BREAKER_LOCK = threading.Lock() | |
| _BREAKER = {"open_until": 0.0} | |
| class ArxivUnavailable(RuntimeError): | |
| """arXiv did not answer in time, refused, or is cooling down.""" | |
| def breaker_open() -> bool: | |
| with _BREAKER_LOCK: | |
| return time.monotonic() < _BREAKER["open_until"] | |
| def _trip_breaker(reason: str) -> None: | |
| with _BREAKER_LOCK: | |
| _BREAKER["open_until"] = time.monotonic() + BREAKER_COOLDOWN_S | |
| print(f"[arxiv] {reason} - pausing request-time lookups for " | |
| f"{BREAKER_COOLDOWN_S:.0f}s", flush=True) | |
| def fetch_metadata_bounded( | |
| query: str, | |
| max_results: int, | |
| budget_s: float = REQUEST_BUDGET_S, | |
| sort_by_relevance: bool = True, | |
| ) -> list[Paper]: | |
| """fetch_metadata for use inside a request: bounded, retry-free, cached. | |
| Raises ArxivUnavailable instead of blocking. Callers treat that as "no | |
| result" and degrade - these lookups are always a fallback or an | |
| enrichment, never the only way to answer. | |
| """ | |
| if breaker_open(): | |
| raise ArxivUnavailable("cooling down after a recent failure") | |
| future = _REQUEST_POOL.submit( | |
| fetch_metadata, query, max_results=max_results, | |
| sort_by_relevance=sort_by_relevance, | |
| delay_seconds=0.0, num_retries=0, use_cache=True, | |
| ) | |
| try: | |
| return future.result(timeout=max(budget_s, 0.1)) | |
| except FuturesTimeout: | |
| # The worker thread finishes on its own; the request just stops waiting. | |
| _trip_breaker(f"no answer within {budget_s:.1f}s") | |
| raise ArxivUnavailable(f"timed out after {budget_s:.1f}s") from None | |
| except Exception as exc: # noqa: BLE001 | |
| _trip_breaker(f"{type(exc).__name__}: {str(exc)[:80]}") | |
| raise ArxivUnavailable(str(exc)) from exc | |
| def download_pdf( | |
| paper: Paper, | |
| pdf_dir: str | Path, | |
| delay_seconds: float = 1.0, | |
| max_cache_gb: float = 0.0, | |
| ) -> Path | None: | |
| """Fetch a PDF, reusing the cached copy when present. Returns None on failure.""" | |
| target = Path(pdf_dir) / f"{paper.versioned_id}.pdf" | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| if target.exists() and target.stat().st_size > 0: | |
| return target | |
| if max_cache_gb > 0: | |
| prune_pdf_cache(pdf_dir, max_cache_gb) | |
| try: | |
| import requests | |
| response = requests.get( | |
| paper.pdf_url, | |
| timeout=60, | |
| headers={"User-Agent": "ResearchOS-ML/1.0 (research assistant; arXiv ingest)"}, | |
| ) | |
| response.raise_for_status() | |
| target.write_bytes(response.content) | |
| time.sleep(delay_seconds) # be polite to arXiv | |
| return target | |
| except Exception as exc: # noqa: BLE001 - one bad PDF must not stop ingestion | |
| print(f"[arxiv] pdf failed {paper.versioned_id}: {type(exc).__name__}") | |
| return None | |
| def extract_text(pdf_path: Path, text_dir: str | Path) -> str | None: | |
| """Extract text from a PDF, caching the result. Returns None on failure.""" | |
| cached = Path(text_dir) / f"{pdf_path.stem}.txt" | |
| cached.parent.mkdir(parents=True, exist_ok=True) | |
| if cached.exists(): | |
| return cached.read_text(encoding="utf-8") | |
| try: | |
| import pymupdf | |
| with pymupdf.open(pdf_path) as doc: | |
| text = "\n".join(page.get_text() for page in doc) | |
| cached.write_text(text, encoding="utf-8") | |
| return text | |
| except Exception as exc: # noqa: BLE001 | |
| print(f"[arxiv] text extraction failed {pdf_path.name}: {type(exc).__name__}") | |
| return None | |
| def write_papers(papers: list[Paper], path: str | Path) -> None: | |
| """Persist papers as JSONL.""" | |
| target = Path(path) | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| with open(target, "w", encoding="utf-8") as handle: | |
| for paper in papers: | |
| handle.write(json.dumps(asdict(paper), ensure_ascii=False) + "\n") | |
| print(f"[arxiv] wrote {len(papers)} papers -> {target}") | |
| def read_papers(path: str | Path) -> list[Paper]: | |
| """Load papers back from JSONL.""" | |
| with open(path, encoding="utf-8") as handle: | |
| return [Paper(**json.loads(line)) for line in handle if line.strip()] | |