Plan literature searches before retrieval

README.md

@@ -14,7 +14,7 @@ license: apache-2.0
 
 # Algorithm Research Deck
 
-Turn an algorithm research topic into an interactive, paper-backed HTML presentation. The Space searches arXiv on CPU, uses ZeroGPU for open-model synthesis, previews the result, and provides HTML, JSON, and ZIP downloads.
+Turn an algorithm research topic into an interactive, paper-backed HTML presentation. The model first expands the topic into historical, milestone, current, and benchmark-oriented search queries. The Space searches arXiv with OpenAlex fallback, uses ZeroGPU for open-model synthesis, previews the result, and provides HTML, JSON, and ZIP downloads.
 
 ## Space configuration
 

app.py

@@ -110,6 +110,48 @@ def arxiv_search(topic: str, limit: int) -> list[dict[str, Any]]:
     return papers
 
 
+def openalex_search(query: str, limit: int) -> list[dict[str, Any]]:
+    email = os.getenv("CONTACT_EMAIL", "").strip()
+    url = f"https://api.openalex.org/works?search={quote_plus(query)}&per-page={min(limit, 100)}"
+    if email:
+        url += f"&mailto={quote_plus(email)}"
+    response = http_get_with_retries(url, read_timeout=60, attempts=3)
+    results = response.json().get("results", [])
+    papers = []
+    for work in results:
+        primary = work.get("primary_location") or {}
+        source = primary.get("source") or {}
+        open_access = work.get("open_access") or {}
+        best_oa = work.get("best_oa_location") or {}
+        landing = primary.get("landing_page_url") or work.get("doi") or work.get("id") or ""
+        pdf_url = best_oa.get("pdf_url") or primary.get("pdf_url") or ""
+        authors = [
+            ((item.get("author") or {}).get("display_name") or "")
+            for item in work.get("authorships", [])
+        ]
+        inverted = work.get("abstract_inverted_index") or {}
+        ordered_words = sorted(
+            ((position, word) for word, positions in inverted.items() for position in positions),
+            key=lambda item: item[0],
+        )
+        abstract = " ".join(word for _, word in ordered_words)
+        papers.append({
+            "id": (work.get("doi") or work.get("id") or landing).rsplit("/", 1)[-1],
+            "title": work.get("title") or "Untitled",
+            "summary": abstract,
+            "authors": [x for x in authors if x],
+            "published": work.get("publication_date") or "",
+            "updated": "",
+            "url": landing,
+            "pdf_url": pdf_url,
+            "venue": source.get("display_name") or "",
+            "citation_count": work.get("cited_by_count") or 0,
+            "open_access": open_access.get("is_oa", False),
+            "source": "OpenAlex",
+        })
+    return papers
+
+
 def normalize_user_papers(urls: list[str]) -> list[dict[str, Any]]:
     result = []
     for raw in urls:
@@ -134,13 +176,58 @@ def deduplicate(papers: list[dict[str, Any]]) -> list[dict[str, Any]]:
     seen = set()
     output = []
     for paper in papers:
-        key = re.sub(r"v\d+$", "", paper["id"])
+        normalized_title = re.sub(r"[^a-z0-9]+", "", paper.get("title", "").lower())
+        key = normalized_title or re.sub(r"v\d+$", "", paper["id"])
         if key not in seen:
             seen.add(key)
             output.append(paper)
     return output
 
 
+def plan_search_queries(request: ResearchRequest) -> list[str]:
+    prompt = json.dumps({
+        "topic": request.topic,
+        "purpose": request.purpose,
+        "focus": request.focus,
+        "date_range": request.date_range,
+        "instruction": (
+            "Understand the technical topic before searching. Return JSON with search_queries containing 3 to 5 concise English academic queries. "
+            "Cover historical/classical terminology, milestone method families, current terminology, and one limitation or benchmark angle. "
+            "Queries must be suitable for arXiv and OpenAlex. Do not include site operators."
+        ),
+    }, ensure_ascii=False)
+    result = chat_json(
+        "You plan literature searches for an algorithm engineer. Expand ambiguous topic names into precise academic terminology.",
+        prompt,
+        650,
+    )
+    queries = [str(x).strip() for x in result.get("search_queries", []) if str(x).strip()]
+    if request.topic not in queries:
+        queries.append(request.topic)
+    return queries[:6]
+
+
+def search_literature(queries: list[str], total_limit: int) -> tuple[list[dict[str, Any]], list[str]]:
+    per_query = max(5, min(20, total_limit // max(1, len(queries))))
+    papers: list[dict[str, Any]] = []
+    warnings: list[str] = []
+    for query in queries:
+        try:
+            found = arxiv_search(query, per_query)
+            for paper in found:
+                paper["source"] = "arXiv"
+            papers.extend(found)
+        except Exception:
+            warnings.append(f"arXiv timeout: {query}")
+            try:
+                papers.extend(openalex_search(query, per_query))
+            except Exception:
+                warnings.append(f"OpenAlex timeout: {query}")
+        if len(deduplicate(papers)) >= total_limit:
+            break
+    return deduplicate(papers)[:total_limit], warnings
+
+
 @spaces.GPU(duration=120)
 def generate_text(system: str, prompt: str, max_tokens: int) -> str:
     global _model, _tokenizer
@@ -193,6 +280,8 @@ def select_papers(request: ResearchRequest, papers: list[dict[str, Any]], screen
 
 
 def extract_pdf(paper: dict[str, Any], max_chars: int = 14000) -> str:
+    if not paper.get("pdf_url"):
+        return "No open PDF URL was available from the academic index."
     response = http_get_with_retries(paper["pdf_url"], read_timeout=120, attempts=3)
     if len(response.content) > 20 * 1024 * 1024:
         return "PDF omitted because it exceeds the 20 MB limit."
@@ -327,14 +416,13 @@ def run_research_ui(
         paper_urls=[x.strip() for x in paper_urls.splitlines() if x.strip()],
     )
     budget = DEPTHS[request.depth]
-    progress(0.08, desc="正在检索 arXiv")
+    progress(0.05, desc="模型正在理解主题并规划检索词")
+    queries = plan_search_queries(request)
+    progress(0.16, desc="正在检索 arXiv 和 OpenAlex")
     user_papers = normalize_user_papers(request.paper_urls)
-    try:
-        discovered_papers = arxiv_search(request.topic, budget["discover"])
-    except Exception as exc:
-        if not user_papers:
-            raise gr.Error(f"arXiv 连续三次请求超时。请稍后重试，或填写至少一个 arXiv 论文链接。详情：{exc}") from exc
-        discovered_papers = []
+    discovered_papers, search_warnings = search_literature(queries, budget["discover"])
+    if not discovered_papers and not user_papers:
+        raise gr.Error("arXiv 和 OpenAlex 均未返回论文。请稍后重试，或填写至少一个 arXiv 论文链接。")
     papers = deduplicate(user_papers + discovered_papers)
     if not papers:
         raise gr.Error("没有检索到论文，请尝试英文关键词或提供 arXiv 链接。")
@@ -365,7 +453,11 @@ def run_research_ui(
     report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
     bundle_path = create_bundle(job_dir, html_path, report_path)
     progress(1, desc="完成")
-    status = f"✅ 已完成：检索 {len(papers)} 篇，筛选 {len(screen)} 篇，深读 {len(deep_texts)} 篇。"
+    warning_text = f" 检索降级：{'；'.join(search_warnings)}。" if search_warnings else ""
+    status = (
+        f"✅ 已完成：模型规划 {len(queries)} 组检索词，发现 {len(papers)} 篇，"
+        f"筛选 {len(screen)} 篇，深读 {len(deep_texts)} 篇。{warning_text}"
+    )
     return status, preview_frame(content), str(html_path), str(report_path), str(bundle_path)
 
 

research-ppt/references/academic-search.md

@@ -26,6 +26,8 @@ Build synonym sets for the task, mechanism, historical terminology, and benchmar
 
 Run a forward search from classical terms and a backward search from current terminology. Record queries that materially define coverage.
 
+For automated research, let the model interpret the topic and propose a small set of English academic queries before calling indexes. Cover classical terminology, milestone families, current terminology, and a benchmark or limitation angle. Use OpenAlex as a metadata fallback when arXiv is unavailable, and disclose degraded coverage rather than silently treating one failed index as no literature.
+
 ## Metadata rules
 
 - Use DOI links for published versions when available and arXiv links for preprint versions.