Retry slow arXiv and PDF requests

app.py

@@ -5,6 +5,7 @@ import io
 import json
 import os
 import re
+import time
 import uuid
 import zipfile
 from datetime import datetime, timezone
@@ -54,6 +55,33 @@ def safe_slug(value: str) -> str:
     return (slug or "algorithm-topic")[:64]
 
 
+def http_get_with_retries(
+    url: str,
+    *,
+    headers: dict[str, str] | None = None,
+    read_timeout: float = 90,
+    attempts: int = 3,
+) -> httpx.Response:
+    last_error: Exception | None = None
+    timeout = httpx.Timeout(connect=20, read=read_timeout, write=30, pool=30)
+    for attempt in range(attempts):
+        try:
+            with httpx.Client(
+                timeout=timeout,
+                follow_redirects=True,
+                headers=headers,
+                http2=False,
+            ) as client:
+                response = client.get(url)
+                response.raise_for_status()
+                return response
+        except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as exc:
+            last_error = exc
+            if attempt + 1 < attempts:
+                time.sleep(2 + attempt * 3)
+    raise RuntimeError(f"External source did not respond after {attempts} attempts: {url}") from last_error
+
+
 def arxiv_search(topic: str, limit: int) -> list[dict[str, Any]]:
     query = quote_plus(f'all:"{topic}"')
     url = (
@@ -62,9 +90,7 @@ def arxiv_search(topic: str, limit: int) -> list[dict[str, Any]]:
         "&sortBy=submittedDate&sortOrder=descending"
     )
     headers = {"User-Agent": f"algorithm-research-deck/1.0 ({os.getenv('CONTACT_EMAIL', 'contact-not-set')})"}
-    with httpx.Client(timeout=30, follow_redirects=True, headers=headers) as client:
-        response = client.get(url)
-        response.raise_for_status()
+    response = http_get_with_retries(url, headers=headers, read_timeout=90, attempts=3)
     ns = {"a": "http://www.w3.org/2005/Atom"}
     root = ElementTree.fromstring(response.text)
     papers = []
@@ -167,11 +193,9 @@ def select_papers(request: ResearchRequest, papers: list[dict[str, Any]], screen
 
 
 def extract_pdf(paper: dict[str, Any], max_chars: int = 14000) -> str:
-    with httpx.Client(timeout=45, follow_redirects=True) as client:
-        response = client.get(paper["pdf_url"])
-        response.raise_for_status()
-        if len(response.content) > 20 * 1024 * 1024:
-            return "PDF omitted because it exceeds the 20 MB limit."
+    response = http_get_with_retries(paper["pdf_url"], read_timeout=120, attempts=3)
+    if len(response.content) > 20 * 1024 * 1024:
+        return "PDF omitted because it exceeds the 20 MB limit."
     reader = PdfReader(io.BytesIO(response.content))
     pages = []
     indexes = list(range(min(8, len(reader.pages))))
@@ -304,7 +328,14 @@ def run_research_ui(
     )
     budget = DEPTHS[request.depth]
     progress(0.08, desc="正在检索 arXiv")
-    papers = deduplicate(normalize_user_papers(request.paper_urls) + arxiv_search(request.topic, budget["discover"]))
+    user_papers = normalize_user_papers(request.paper_urls)
+    try:
+        discovered_papers = arxiv_search(request.topic, budget["discover"])
+    except Exception as exc:
+        if not user_papers:
+            raise gr.Error(f"arXiv 连续三次请求超时。请稍后重试，或填写至少一个 arXiv 论文链接。详情：{exc}") from exc
+        discovered_papers = []
+    papers = deduplicate(user_papers + discovered_papers)
     if not papers:
         raise gr.Error("没有检索到论文，请尝试英文关键词或提供 arXiv 链接。")
     screen = papers[: budget["screen"]]