File size: 11,588 Bytes
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
af38999
e8c5cc2
af38999
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
af38999
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e8c5cc2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
"""arXiv ingestion: metadata, full text, and version dedup.



Design notes that follow from REQUIREMENTS-ML.md:



FR1.4 - resumable. Every PDF and every extracted text file is cached on disk by

arXiv id. Re-running ingestion skips anything already fetched, so an interrupted

run costs nothing.



FR6.2 - version dedup. arXiv ids carry a version suffix (2401.12345v3). The

index is keyed on the *versionless* id, so a later version replaces an earlier

one instead of sitting beside it as a near-duplicate. Duplicate near-identical

papers corrupt both retrieval and any evaluation run over the index.

"""

from __future__ import annotations

import json
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import TimeoutError as FuturesTimeout
from dataclasses import asdict, dataclass, field
from pathlib import Path

import arxiv

# arXiv rate-limits hard (HTTP 429) and the client library retries internally
# with multi-second sleeps, so one unlucky interactive query could block for a
# minute and then fail. A small in-process cache makes repeated lookups free,
# which matters because the same paper gets asked about repeatedly.
_SEARCH_CACHE: dict[str, list["Paper"]] = {}
_SEARCH_CACHE_MAX = 512

# "http://arxiv.org/abs/2401.12345v3" -> ("2401.12345", 3)
_ENTRY_RE = re.compile(r"/abs/(?P<id>[^v]+(?:v\d+)?)")
_VERSION_RE = re.compile(r"^(?P<base>.+?)v(?P<version>\d+)$")


@dataclass
class Paper:
    """One arXiv paper, versionless id as the primary key."""

    paper_id: str
    version: int
    title: str
    abstract: str
    authors: list[str]
    categories: list[str]
    primary_category: str
    published: str
    updated: str
    pdf_url: str
    full_text: str | None = None
    metadata: dict[str, str] = field(default_factory=dict)

    @property
    def versioned_id(self) -> str:
        return f"{self.paper_id}v{self.version}"


def _split_version(entry_id: str) -> tuple[str, int]:
    """Turn an arXiv entry URL into (versionless_id, version)."""
    match = _ENTRY_RE.search(entry_id)
    raw = match.group("id") if match else entry_id.rsplit("/", 1)[-1]
    versioned = _VERSION_RE.match(raw)
    if versioned:
        return versioned.group("base"), int(versioned.group("version"))
    return raw, 1


def _clean(text: str) -> str:
    """Collapse the whitespace arXiv abstracts arrive with."""
    return re.sub(r"\s+", " ", text or "").strip()


def fetch_metadata(

    query: str,

    max_results: int,

    page_size: int = 100,

    delay_seconds: float = 3.0,

    sort_by_relevance: bool = False,

    num_retries: int = 1,

    use_cache: bool = False,

) -> list[Paper]:
    """Run one arXiv API query and return papers.



    Ingestion wants newest-first (build a current corpus). Citation lookup wants

    relevance - sorting a title search by date returns the most *recent* papers

    whose titles match, so a foundational paper like "Attention Is All You Need"

    is buried under every recent paper that echoes its title.

    """
    cache_key = f"{query}|{max_results}|{sort_by_relevance}"
    if use_cache and cache_key in _SEARCH_CACHE:
        return _SEARCH_CACHE[cache_key]

    client = arxiv.Client(
        page_size=page_size, delay_seconds=delay_seconds, num_retries=num_retries
    )
    search = arxiv.Search(
        query=query,
        max_results=max_results,
        sort_by=(
            arxiv.SortCriterion.Relevance
            if sort_by_relevance
            else arxiv.SortCriterion.SubmittedDate
        ),
    )

    papers: list[Paper] = []
    for result in client.results(search):
        paper_id, version = _split_version(result.entry_id)
        papers.append(
            Paper(
                paper_id=paper_id,
                version=version,
                title=_clean(result.title),
                abstract=_clean(result.summary),
                authors=[a.name for a in result.authors][:20],
                categories=list(result.categories),
                primary_category=result.primary_category,
                published=result.published.date().isoformat() if result.published else "",
                updated=result.updated.date().isoformat() if result.updated else "",
                pdf_url=result.pdf_url or "",
            )
        )
    print(f"[arxiv] {len(papers):>5} results for: {query[:80]}")
    if use_cache:
        if len(_SEARCH_CACHE) >= _SEARCH_CACHE_MAX:
            _SEARCH_CACHE.clear()
        _SEARCH_CACHE[cache_key] = papers
    return papers


def dedup_by_version(papers: list[Paper]) -> list[Paper]:
    """FR6.2 - keep only the highest version of each paper."""
    best: dict[str, Paper] = {}
    for paper in papers:
        existing = best.get(paper.paper_id)
        if existing is None or paper.version > existing.version:
            best[paper.paper_id] = paper
    dropped = len(papers) - len(best)
    if dropped:
        print(f"[arxiv] dedup: dropped {dropped} superseded/duplicate versions")
    return list(best.values())


def prune_pdf_cache(pdf_dir: str | Path, max_gb: float) -> int:
    """Keep the PDF cache under `max_gb`, deleting least-recently-used first.



    Citation lookup downloads a PDF per unresolved paper, so on a shared

    deployment this directory grows without bound. Extracted text is kept - it

    is ~1% of the size and is what retrieval actually uses, so evicting a PDF

    costs nothing unless the same paper is re-parsed.

    """
    directory = Path(pdf_dir)
    if not directory.exists() or max_gb <= 0:
        return 0
    files = sorted(directory.glob("*.pdf"), key=lambda f: f.stat().st_atime)
    total = sum(f.stat().st_size for f in files)
    budget = max_gb * 1e9
    removed = 0
    for path in files:
        if total <= budget:
            break
        size = path.stat().st_size
        try:
            path.unlink()
            total -= size
            removed += 1
        except OSError:
            continue
    if removed:
        print(f"[arxiv] pruned {removed} cached PDFs, cache now {total / 1e9:.1f} GB")
    return removed



# ----------------------------------------------------------------------------
# Request-time lookups
#
# fetch_metadata above is built for ingestion, where waiting out a rate limit
# is correct. Inside a user's request it is not: the arxiv client has no
# request timeout, and a throttled arXiv does not fail fast - a lookup was
# measured holding the connection 46-50s before answering HTTP 429, with
# retries already disabled. Hugging Face Spaces share outbound IPs, so the
# deployed backend is throttled far more often than a laptop is.
#
# So request-time callers go through fetch_metadata_bounded: the call runs on
# a small pool and the request stops waiting at the budget. One breaker is
# shared by every caller - live search and citation lookup hit the same API,
# so if one is being throttled the other is too, and continuing to ask only
# extends the throttle.
# ----------------------------------------------------------------------------

REQUEST_BUDGET_S = 5.0
BREAKER_COOLDOWN_S = 300.0

_REQUEST_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="arxiv-request")
_BREAKER_LOCK = threading.Lock()
_BREAKER = {"open_until": 0.0}


class ArxivUnavailable(RuntimeError):
    """arXiv did not answer in time, refused, or is cooling down."""


def breaker_open() -> bool:
    with _BREAKER_LOCK:
        return time.monotonic() < _BREAKER["open_until"]


def _trip_breaker(reason: str) -> None:
    with _BREAKER_LOCK:
        _BREAKER["open_until"] = time.monotonic() + BREAKER_COOLDOWN_S
    print(f"[arxiv] {reason} - pausing request-time lookups for "
          f"{BREAKER_COOLDOWN_S:.0f}s", flush=True)


def fetch_metadata_bounded(

    query: str,

    max_results: int,

    budget_s: float = REQUEST_BUDGET_S,

    sort_by_relevance: bool = True,

) -> list[Paper]:
    """fetch_metadata for use inside a request: bounded, retry-free, cached.



    Raises ArxivUnavailable instead of blocking. Callers treat that as "no

    result" and degrade - these lookups are always a fallback or an

    enrichment, never the only way to answer.

    """
    if breaker_open():
        raise ArxivUnavailable("cooling down after a recent failure")
    future = _REQUEST_POOL.submit(
        fetch_metadata, query, max_results=max_results,
        sort_by_relevance=sort_by_relevance,
        delay_seconds=0.0, num_retries=0, use_cache=True,
    )
    try:
        return future.result(timeout=max(budget_s, 0.1))
    except FuturesTimeout:
        # The worker thread finishes on its own; the request just stops waiting.
        _trip_breaker(f"no answer within {budget_s:.1f}s")
        raise ArxivUnavailable(f"timed out after {budget_s:.1f}s") from None
    except Exception as exc:  # noqa: BLE001
        _trip_breaker(f"{type(exc).__name__}: {str(exc)[:80]}")
        raise ArxivUnavailable(str(exc)) from exc


def download_pdf(

    paper: Paper,

    pdf_dir: str | Path,

    delay_seconds: float = 1.0,

    max_cache_gb: float = 0.0,

) -> Path | None:
    """Fetch a PDF, reusing the cached copy when present. Returns None on failure."""
    target = Path(pdf_dir) / f"{paper.versioned_id}.pdf"
    target.parent.mkdir(parents=True, exist_ok=True)
    if target.exists() and target.stat().st_size > 0:
        return target
    if max_cache_gb > 0:
        prune_pdf_cache(pdf_dir, max_cache_gb)

    try:
        import requests

        response = requests.get(
            paper.pdf_url,
            timeout=60,
            headers={"User-Agent": "ResearchOS-ML/1.0 (research assistant; arXiv ingest)"},
        )
        response.raise_for_status()
        target.write_bytes(response.content)
        time.sleep(delay_seconds)  # be polite to arXiv
        return target
    except Exception as exc:  # noqa: BLE001 - one bad PDF must not stop ingestion
        print(f"[arxiv] pdf failed {paper.versioned_id}: {type(exc).__name__}")
        return None


def extract_text(pdf_path: Path, text_dir: str | Path) -> str | None:
    """Extract text from a PDF, caching the result. Returns None on failure."""
    cached = Path(text_dir) / f"{pdf_path.stem}.txt"
    cached.parent.mkdir(parents=True, exist_ok=True)
    if cached.exists():
        return cached.read_text(encoding="utf-8")

    try:
        import pymupdf

        with pymupdf.open(pdf_path) as doc:
            text = "\n".join(page.get_text() for page in doc)
        cached.write_text(text, encoding="utf-8")
        return text
    except Exception as exc:  # noqa: BLE001
        print(f"[arxiv] text extraction failed {pdf_path.name}: {type(exc).__name__}")
        return None


def write_papers(papers: list[Paper], path: str | Path) -> None:
    """Persist papers as JSONL."""
    target = Path(path)
    target.parent.mkdir(parents=True, exist_ok=True)
    with open(target, "w", encoding="utf-8") as handle:
        for paper in papers:
            handle.write(json.dumps(asdict(paper), ensure_ascii=False) + "\n")
    print(f"[arxiv] wrote {len(papers)} papers -> {target}")


def read_papers(path: str | Path) -> list[Paper]:
    """Load papers back from JSONL."""
    with open(path, encoding="utf-8") as handle:
        return [Paper(**json.loads(line)) for line in handle if line.strip()]