Create self_learning_graph.py

dvnc_ai_v2_hf/self_learning_graph.py

@@ -0,0 +1,1001 @@
+import html
+import os
+import re
+import urllib.parse
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Dict, List, Optional
+from urllib.parse import quote
+
+import gradio as gr
+import requests
+
+try:
+    import fitz  # PyMuPDF
+except Exception:
+    fitz = None
+
+try:
+    from bs4 import BeautifulSoup
+except Exception:
+    BeautifulSoup = None
+
+
+JOURNALS = [
+    {"name": "Nature", "url": "https://www.nature.com/search", "desc": "Flagship multidisciplinary research journal."},
+    {"name": "Science", "url": "https://www.science.org/search", "desc": "High-impact science journal and family."},
+    {"name": "Cell", "url": "https://www.cell.com/search", "desc": "Life sciences and translational biology."},
+    {"name": "The Lancet", "url": "https://www.thelancet.com/search", "desc": "Clinical and medical research."},
+    {"name": "IEEE Xplore", "url": "https://ieeexplore.ieee.org/search/searchresult.jsp", "desc": "Engineering, AI, signal processing, and systems."},
+]
+
+SEARCH_MODES = ["topic", "title", "doi", "link", "paper_name", "autonomous_web"]
+SOURCE_OPTIONS = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
+DEFAULT_SOURCES = ["arxiv", "openalex", "crossref", "semantic_scholar", "europe_pmc"]
+
+SEMANTIC_SCHOLAR_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY", "")
+GROBID_URL = os.getenv("GROBID_URL", "").strip()
+REQUEST_TIMEOUT = 25
+
+
+def safe_text(x, default=""):
+    return html.escape(str(x if x is not None else default))
+
+
+def norm_text(x: Optional[str]) -> str:
+    return re.sub(r"\s+", " ", (x or "")).strip()
+
+
+def detect_query_type(query: str) -> str:
+    q = (query or "").strip()
+    doi_pattern = r"^10\.\d{4,9}/[-._;()/:A-Z0-9]+$"
+    if re.match(doi_pattern, q, flags=re.I):
+        return "doi"
+    if q.startswith("http://") or q.startswith("https://"):
+        return "link"
+    return "topic"
+
+
+def ensure_list(x):
+    return x if isinstance(x, list) else []
+
+
+def build_learning_graph_html(nodes, edges, title="Self-Learning Knowledge Graph"):
+    if not nodes:
+        return """
+        <div class="panel brain-shell">
+          <div class="brain-header">
+            <div>
+              <p class="eyebrow">Learning Graph</p>
+              <h3>Self-Learning Knowledge Graph</h3>
+            </div>
+          </div>
+          <div class="brain-stage learning-empty">
+            <div class="empty-graph-copy">
+              <h4>No papers mapped yet</h4>
+              <p>Search papers, pick a topic, select candidates, or upload a PDF to grow the graph in real time.</p>
+            </div>
+          </div>
+        </div>
+        """
+
+    node_items = []
+    label_items = []
+    edge_items = []
+    coords = [(110, 110), (320, 80), (540, 130), (660, 270), (550, 410), (300, 450), (110, 340), (380, 260)]
+
+    graph_nodes = [dict(n) for n in nodes[:8]]
+    for i, node in enumerate(graph_nodes):
+        x, y = coords[i]
+        node["sx"] = x
+        node["sy"] = y
+
+    node_map = {n["id"]: n for n in graph_nodes}
+    for a, b in edges:
+        if a in node_map and b in node_map:
+            na = node_map[a]
+            nb = node_map[b]
+            edge_items.append(
+                f'<line class="learn-edge" x1="{na["sx"]}" y1="{na["sy"]}" x2="{nb["sx"]}" y2="{nb["sy"]}" />'
+            )
+
+    for node in graph_nodes:
+        kind = node.get("kind", "paper")
+        klass = f'learn-node {kind}'
+        radius = 24 if kind == "query" else 20
+        node_items.append(
+            f'<circle class="{klass}" cx="{node["sx"]}" cy="{node["sy"]}" r="{radius}" />'
+        )
+        label_items.append(
+            f'<text class="learn-label" x="{node["sx"] + 28}" y="{node["sy"] - 10}">{safe_text(node["label"][:44])}</text>'
+        )
+
+    return f"""
+    <div class="panel brain-shell">
+      <div class="brain-header">
+        <div>
+          <p class="eyebrow">Learning Graph</p>
+          <h3>{safe_text(title)}</h3>
+        </div>
+        <div class="brain-legend">
+          <span><i class="dot dot-query"></i> query</span>
+          <span><i class="dot dot-paper"></i> paper</span>
+          <span><i class="dot dot-upload"></i> uploaded PDF</span>
+        </div>
+      </div>
+      <div class="brain-stage">
+        <svg viewBox="0 0 760 520" class="brain-svg" role="img" aria-label="Self-learning knowledge graph">
+          {''.join(edge_items)}
+          {''.join(node_items)}
+          {''.join(label_items)}
+        </svg>
+      </div>
+    </div>
+    """
+
+
+def build_journal_html(query):
+    q = urllib.parse.quote_plus(query or "biomaterials cardiac repair")
+    rows = []
+    for journal in JOURNALS:
+        url = f"{journal['url']}?q={q}" if "?" not in journal["url"] else f"{journal['url']}&q={q}"
+        if "ieeexplore" in journal["url"]:
+            url = f"https://ieeexplore.ieee.org/search/searchresult.jsp?queryText={q}"
+        rows.append(
+            f"""
+            <a class="journal-card" href="{safe_text(url)}" target="_blank" rel="noopener noreferrer">
+              <div>
+                <h4>{safe_text(journal['name'])}</h4>
+                <p>{safe_text(journal['desc'])}</p>
+              </div>
+              <span>Open</span>
+            </a>
+            """
+        )
+    return '<div class="journal-grid">' + ''.join(rows) + '</div>'
+
+
+def search_arxiv(query, max_results=8):
+    encoded = urllib.parse.quote(query)
+    url = (
+        "http://export.arxiv.org/api/query?search_query=all:"
+        f"{encoded}&start=0&max_results={max_results}&sortBy=relevance&sortOrder=descending"
+    )
+    response = requests.get(url, timeout=REQUEST_TIMEOUT)
+    response.raise_for_status()
+    root = ET.fromstring(response.text)
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    papers = []
+    for entry in root.findall("atom:entry", ns):
+        title = " ".join((entry.findtext("atom:title", default="", namespaces=ns) or "").split())
+        summary = " ".join((entry.findtext("atom:summary", default="", namespaces=ns) or "").split())
+        published = entry.findtext("atom:published", default="", namespaces=ns)
+        paper_id = entry.findtext("atom:id", default="", namespaces=ns)
+        authors = [a.findtext("atom:name", default="", namespaces=ns) for a in entry.findall("atom:author", ns)]
+        pdf_url = ""
+        for link in entry.findall("atom:link", ns):
+            if link.attrib.get("title") == "pdf":
+                pdf_url = link.attrib.get("href", "")
+                break
+        papers.append(
+            {
+                "id": paper_id or title,
+                "title": title,
+                "summary": summary,
+                "abstract": summary,
+                "published": published[:10],
+                "authors": [a for a in authors[:8] if a],
+                "authors_text": ", ".join([a for a in authors[:4] if a]),
+                "url": paper_id,
+                "pdf": pdf_url,
+                "doi": "",
+                "venue": "arXiv",
+                "year": published[:4] if published else "",
+                "source": "arxiv",
+                "score": 0.76,
+                "open_access": True,
+                "external_ids": {"arxiv": (paper_id or "").split("/")[-1]},
+            }
+        )
+    return papers
+
+
+def search_crossref(query, mode="topic", max_results=8):
+    headers = {"User-Agent": "dvnc-ai-space/0.1"}
+    if mode == "doi":
+        url = f"https://api.crossref.org/works/{quote(query)}"
+        response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT)
+        if response.status_code != 200:
+            return []
+        items = [response.json().get("message", {})]
+    else:
+        params = {"rows": max_results}
+        if mode in ("title", "paper_name"):
+            params["query.title"] = query
+        else:
+            params["query.bibliographic"] = query
+        response = requests.get("https://api.crossref.org/works", params=params, headers=headers, timeout=REQUEST_TIMEOUT)
+        response.raise_for_status()
+        items = response.json().get("message", {}).get("items", [])
+
+    out = []
+    for item in items:
+        authors = []
+        for a in item.get("author", []) or []:
+            name = " ".join(filter(None, [a.get("given"), a.get("family")])).strip()
+            if name:
+                authors.append(name)
+        title = (item.get("title") or ["Untitled"])[0]
+        year = ""
+        for key in ["published-print", "published-online", "created"]:
+            if item.get(key, {}).get("date-parts"):
+                year = str(item[key]["date-parts"][0][0])
+                break
+        abstract = item.get("abstract") or ""
+        abstract = re.sub("<.*?>", "", abstract)
+        doi = item.get("DOI", "")
+        out.append({
+            "id": doi or title,
+            "title": norm_text(title),
+            "summary": norm_text(abstract)[:400],
+            "abstract": norm_text(abstract),
+            "published": year,
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": item.get("URL", ""),
+            "pdf": "",
+            "doi": doi,
+            "venue": (item.get("container-title") or [""])[0],
+            "year": year,
+            "source": "crossref",
+            "score": 0.72,
+            "open_access": None,
+            "external_ids": {"crossref": doi} if doi else {},
+        })
+    return out
+
+
+def search_openalex(query, mode="topic", max_results=8):
+    params = {"per-page": max_results}
+    if mode == "doi":
+        doi = query.lower().replace("https://doi.org/", "").replace("http://doi.org/", "")
+        params["filter"] = f"doi:https://doi.org/{doi}"
+    else:
+        params["search"] = query
+    response = requests.get("https://api.openalex.org/works", params=params, timeout=REQUEST_TIMEOUT)
+    response.raise_for_status()
+    items = response.json().get("results", [])
+    out = []
+    for item in items:
+        authors = []
+        for auth in item.get("authorships", [])[:8]:
+            author = auth.get("author") or {}
+            if author.get("display_name"):
+                authors.append(author["display_name"])
+        oa = item.get("open_access") or {}
+        doi = (item.get("doi") or "").replace("https://doi.org/", "")
+        out.append({
+            "id": item.get("id") or doi or item.get("title"),
+            "title": norm_text(item.get("title")),
+            "summary": "",
+            "abstract": "",
+            "published": str(item.get("publication_year") or ""),
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": (item.get("primary_location") or {}).get("landing_page_url") or item.get("id") or "",
+            "pdf": oa.get("oa_url") or "",
+            "doi": doi,
+            "venue": ((item.get("primary_location") or {}).get("source") or {}).get("display_name") or "",
+            "year": str(item.get("publication_year") or ""),
+            "source": "openalex",
+            "score": 0.80,
+            "open_access": oa.get("is_oa"),
+            "external_ids": item.get("ids") or {},
+        })
+    return out
+
+
+def search_semantic_scholar(query, mode="topic", max_results=8):
+    headers = {}
+    if SEMANTIC_SCHOLAR_API_KEY:
+        headers["x-api-key"] = SEMANTIC_SCHOLAR_API_KEY
+    fields = "title,authors,year,abstract,venue,externalIds,url,openAccessPdf"
+    if mode == "doi":
+        doi = query.lower().replace("https://doi.org/", "").replace("http://doi.org/", "")
+        url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{quote(doi)}"
+        response = requests.get(url, params={"fields": fields}, headers=headers, timeout=REQUEST_TIMEOUT)
+        if response.status_code != 200:
+            return []
+        items = [response.json()]
+    else:
+        response = requests.get(
+            "https://api.semanticscholar.org/graph/v1/paper/search",
+            params={"query": query, "limit": max_results, "fields": fields},
+            headers=headers,
+            timeout=REQUEST_TIMEOUT,
+        )
+        if response.status_code != 200:
+            return []
+        items = response.json().get("data", [])
+
+    out = []
+    for item in items:
+        external = item.get("externalIds") or {}
+        authors = [a.get("name") for a in item.get("authors", []) if a.get("name")]
+        out.append({
+            "id": external.get("CorpusId") or external.get("DOI") or item.get("title"),
+            "title": norm_text(item.get("title")),
+            "summary": norm_text(item.get("abstract", ""))[:400],
+            "abstract": norm_text(item.get("abstract", "")),
+            "published": str(item.get("year") or ""),
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": item.get("url") or "",
+            "pdf": (item.get("openAccessPdf") or {}).get("url") or "",
+            "doi": external.get("DOI", ""),
+            "venue": item.get("venue") or "",
+            "year": str(item.get("year") or ""),
+            "source": "semantic_scholar",
+            "score": 0.84,
+            "open_access": bool((item.get("openAccessPdf") or {}).get("url")),
+            "external_ids": external,
+        })
+    return out
+
+
+def search_europe_pmc(query, mode="topic", max_results=8):
+    epmc_query = f'DOI:"{query}"' if mode == "doi" else query
+    params = {
+        "query": epmc_query,
+        "format": "json",
+        "pageSize": max_results,
+        "resultType": "core",
+    }
+    response = requests.get("https://www.ebi.ac.uk/europepmc/webservices/rest/search", params=params, timeout=REQUEST_TIMEOUT)
+    if response.status_code != 200:
+        return []
+    items = response.json().get("resultList", {}).get("result", [])
+    out = []
+    for item in items:
+        author_string = item.get("authorString", "")
+        authors = [x.strip() for x in author_string.split(",")[:8] if x.strip()]
+        pmcid = item.get("pmcid", "")
+        pdf_url = f"https://europepmc.org/articles/{pmcid}?pdf=render" if pmcid else ""
+        landing_url = f"https://europepmc.org/article/PMC/{pmcid}" if pmcid else ""
+        out.append({
+            "id": item.get("id") or item.get("doi") or item.get("title"),
+            "title": norm_text(item.get("title")),
+            "summary": norm_text(item.get("abstractText", ""))[:400],
+            "abstract": norm_text(item.get("abstractText", "")),
+            "published": str(item.get("pubYear") or ""),
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": landing_url,
+            "pdf": pdf_url,
+            "doi": item.get("doi", ""),
+            "venue": item.get("journalTitle", ""),
+            "year": str(item.get("pubYear") or ""),
+            "source": "europe_pmc",
+            "score": 0.78,
+            "open_access": bool(pmcid),
+            "external_ids": {"pmid": item.get("pmid"), "pmcid": pmcid},
+        })
+    return out
+
+
+def resolve_link(query):
+    url = (query or "").strip()
+    if not url:
+        return []
+    try:
+        response = requests.get(
+            url,
+            timeout=REQUEST_TIMEOUT,
+            allow_redirects=True,
+            headers={"User-Agent": "dvnc-ai-space/0.1"},
+        )
+        content_type = response.headers.get("content-type", "")
+        if "pdf" in content_type or url.lower().endswith(".pdf"):
+            name = Path(url.split("?")[0]).name or "linked-paper.pdf"
+            return [{
+                "id": url,
+                "title": name,
+                "summary": "Direct PDF link detected.",
+                "abstract": "",
+                "published": "",
+                "authors": [],
+                "authors_text": "Unknown authors",
+                "url": url,
+                "pdf": url,
+                "doi": "",
+                "venue": "Direct PDF",
+                "year": "",
+                "source": "link",
+                "score": 0.66,
+                "open_access": True,
+                "external_ids": {},
+            }]
+
+        if BeautifulSoup is None:
+            return [{
+                "id": url,
+                "title": url,
+                "summary": "Web page resolved. Install beautifulsoup4 for DOI extraction.",
+                "abstract": "",
+                "published": "",
+                "authors": [],
+                "authors_text": "Unknown authors",
+                "url": url,
+                "pdf": "",
+                "doi": "",
+                "venue": "Web Link",
+                "year": "",
+                "source": "link",
+                "score": 0.48,
+                "open_access": None,
+                "external_ids": {},
+            }]
+
+        soup = BeautifulSoup(response.text, "html.parser")
+        doi = ""
+        for meta_name in ["citation_doi", "dc.identifier", "dc.Identifier"]:
+            tag = soup.find("meta", attrs={"name": meta_name})
+            if tag and tag.get("content"):
+                doi = tag["content"].strip()
+                break
+
+        title = soup.title.text.strip() if soup.title else url
+        pdf_link = ""
+        for a in soup.find_all("a", href=True):
+            href = a["href"]
+            if ".pdf" in href.lower():
+                pdf_link = href if href.startswith("http") else ""
+                break
+
+        if doi:
+            results = search_crossref(doi, mode="doi", max_results=1)
+            if results:
+                if pdf_link and not results[0].get("pdf"):
+                    results[0]["pdf"] = pdf_link
+                if url and not results[0].get("url"):
+                    results[0]["url"] = url
+                return results
+
+        return [{
+            "id": url,
+            "title": title,
+            "summary": "Landing page resolved from direct link.",
+            "abstract": "",
+            "published": "",
+            "authors": [],
+            "authors_text": "Unknown authors",
+            "url": url,
+            "pdf": pdf_link,
+            "doi": doi,
+            "venue": "Web Link",
+            "year": "",
+            "source": "link",
+            "score": 0.54,
+            "open_access": bool(pdf_link),
+            "external_ids": {},
+        }]
+    except Exception as e:
+        return [{
+            "id": url,
+            "title": "Link resolution error",
+            "summary": str(e),
+            "abstract": "",
+            "published": "",
+            "authors": [],
+            "authors_text": "Unknown authors",
+            "url": url,
+            "pdf": "",
+            "doi": "",
+            "venue": "Link",
+            "year": "",
+            "source": "link",
+            "score": 0.20,
+            "open_access": None,
+            "external_ids": {},
+        }]
+
+
+def dedupe_papers(items: List[Dict]) -> List[Dict]:
+    seen = {}
+    for item in items:
+        key = (
+            (item.get("doi") or "").lower().strip()
+            or (item.get("external_ids") or {}).get("arxiv")
+            or norm_text(item.get("title", "")).lower()
+            or item.get("id")
+        )
+        if not key:
+            key = f"{item.get('source')}::{item.get('title')}"
+        if key not in seen or float(item.get("score", 0)) > float(seen[key].get("score", 0)):
+            seen[key] = item
+    return sorted(seen.values(), key=lambda x: float(x.get("score", 0)), reverse=True)
+
+
+def discover_papers(query, mode, sources, max_results=10):
+    query = (query or "").strip()
+    if not query:
+        return []
+
+    mode = detect_query_type(query) if mode == "autonomous_web" else mode
+    selected_sources = ensure_list(sources) or DEFAULT_SOURCES
+    results = []
+
+    if mode == "link":
+        return dedupe_papers(resolve_link(query))
+
+    try:
+        if "arxiv" in selected_sources and mode != "doi":
+            results.extend(search_arxiv(query, max_results=max_results))
+    except Exception:
+        pass
+    try:
+        if "crossref" in selected_sources:
+            results.extend(search_crossref(query, mode=mode, max_results=max_results))
+    except Exception:
+        pass
+    try:
+        if "openalex" in selected_sources:
+            results.extend(search_openalex(query, mode=mode, max_results=max_results))
+    except Exception:
+        pass
+    try:
+        if "semantic_scholar" in selected_sources:
+            results.extend(search_semantic_scholar(query, mode=mode, max_results=max_results))
+    except Exception:
+        pass
+    try:
+        if "europe_pmc" in selected_sources:
+            results.extend(search_europe_pmc(query, mode=mode, max_results=max_results))
+    except Exception:
+        pass
+
+    return dedupe_papers(results)
+
+
+def format_papers_html(papers):
+    if not papers:
+        return '<div class="panel papers-panel" style="padding:18px"><p>No papers found yet.</p></div>'
+
+    items = []
+    for i, paper in enumerate(papers, start=1):
+        summary = safe_text((paper.get("summary") or "")[:280])
+        doi_line = f'<span class="paper-badge doi-badge">{safe_text(paper.get("doi"))}</span>' if paper.get("doi") else ""
+        pdf_link = paper.get("pdf") or "#"
+        abs_link = paper.get("url") or "#"
+        items.append(
+            f"""
+            <article class="paper-card">
+              <div class="paper-topline">
+                <span class="paper-badge">{safe_text(paper.get('source', 'paper'))}</span>
+                <span class="paper-badge alt">{safe_text(paper.get('published', '') or 'Paper')}</span>
+                {doi_line}
+              </div>
+              <h4>{i}. {safe_text(paper.get('title', 'Untitled'))}</h4>
+              <p>{summary or 'No abstract snippet available.'}</p>
+              <div class="paper-meta-stack">
+                <div><strong>Authors:</strong> {safe_text(paper.get('authors_text', 'Unknown authors'))}</div>
+                <div><strong>Venue:</strong> {safe_text(paper.get('venue', 'Unknown venue'))}</div>
+                <div><strong>Score:</strong> {safe_text(round(float(paper.get('score', 0)), 3))}</div>
+              </div>
+              <div class="paper-links">
+                <a href="{safe_text(abs_link)}" target="_blank" rel="noopener noreferrer">Abstract</a>
+                <a href="{safe_text(pdf_link)}" target="_blank" rel="noopener noreferrer">PDF</a>
+              </div>
+            </article>
+            """
+        )
+    return '<div class="papers-grid">' + ''.join(items) + '</div>'
+
+
+def format_selection_choices(papers):
+    choices = []
+    for i, paper in enumerate(papers):
+        label = f"[{paper.get('source', 'src')}] {paper.get('title', 'Untitled')} — {paper.get('authors_text', 'Unknown authors')[:80]}"
+        choices.append((label, str(i)))
+    return choices
+
+
+def uploaded_pdf_summary(file_obj):
+    if not file_obj:
+        return "No PDF uploaded yet."
+    path = getattr(file_obj, "name", None) or str(file_obj)
+    p = Path(path)
+    return f"Uploaded PDF ready for ingestion: {p.name}. Use Parse uploaded PDF to extract title, abstract, sections, and references."
+
+
+def build_learning_graph_state(query, papers, uploaded_name=None):
+    nodes = [{"id": "query", "label": query or "Research topic", "kind": "query"}]
+    edges = []
+    for i, paper in enumerate(papers[:5], start=1):
+        pid = f"paper_{i}"
+        nodes.append({"id": pid, "label": paper["title"], "kind": "paper"})
+        edges.append(("query", pid))
+    if uploaded_name:
+        nodes.append({"id": "upload", "label": uploaded_name, "kind": "upload"})
+        edges.append(("query", "upload"))
+        if len(nodes) > 2:
+            edges.append(("upload", "paper_1"))
+    return nodes, edges
+
+
+def graph_from_selected(query, selected_papers, uploaded_name=None):
+    nodes = [{"id": "query", "label": query or "Research topic", "kind": "query"}]
+    edges = []
+    for i, paper in enumerate(selected_papers[:6], start=1):
+        pid = f"paper_{i}"
+        nodes.append({"id": pid, "label": paper["title"], "kind": "paper"})
+        edges.append(("query", pid))
+        if paper.get("doi"):
+            doi_id = f"doi_{i}"
+            nodes.append({"id": doi_id, "label": f"DOI {paper['doi']}", "kind": "paper"})
+            edges.append((pid, doi_id))
+    if uploaded_name:
+        nodes.append({"id": "upload", "label": uploaded_name, "kind": "upload"})
+        edges.append(("query", "upload"))
+        if len(selected_papers) > 0:
+            edges.append(("upload", "paper_1"))
+    return nodes, edges
+
+
+def parse_pdf_with_grobid(pdf_path):
+    if not GROBID_URL:
+        raise RuntimeError("GROBID_URL is not set")
+
+    with open(pdf_path, "rb") as f:
+        files = {"input": (Path(pdf_path).name, f, "application/pdf")}
+        response = requests.post(
+            f"{GROBID_URL.rstrip('/')}/api/processFulltextDocument",
+            files=files,
+            data={"includeRawAffiliations": "1", "segmentSentences": "1"},
+            timeout=120,
+        )
+
+    response.raise_for_status()
+    tei_xml = response.text
+
+    if BeautifulSoup is None:
+        return {
+            "parser": "grobid",
+            "title": Path(pdf_path).name,
+            "abstract": "",
+            "authors": [],
+            "sections": [],
+            "references": [],
+            "raw_text": "",
+            "tei_xml": tei_xml[:30000],
+        }
+
+    soup = BeautifulSoup(tei_xml, "xml")
+    title_stmt = soup.find("titleStmt")
+    title_tag = title_stmt.find("title") if title_stmt else soup.find("title")
+    abstract_tag = soup.find("abstract")
+
+    authors = []
+    for author in soup.find_all("author"):
+        pers = author.find("persName")
+        if pers:
+            name = " ".join(
+                x.get_text(" ", strip=True)
+                for x in pers.find_all(["forename", "surname"])
+            ).strip()
+            if name:
+                authors.append(name)
+
+    sections = []
+    for div in soup.find_all("div"):
+        head = div.find("head")
+        paras = [p.get_text(" ", strip=True) for p in div.find_all("p")]
+        text = "\n".join([p for p in paras if p.strip()])
+        if head and text.strip():
+            sections.append({"heading": head.get_text(" ", strip=True), "text": text[:4000]})
+
+    references = []
+    for bibl in soup.find_all("biblStruct")[:30]:
+        ref_title = ""
+        ref_doi = ""
+        title_node = bibl.find("title")
+        if title_node:
+            ref_title = title_node.get_text(" ", strip=True)
+        doi_node = bibl.find("idno", attrs={"type": "DOI"})
+        if doi_node:
+            ref_doi = doi_node.get_text(" ", strip=True)
+        references.append({"title": ref_title, "doi": ref_doi})
+
+    return {
+        "parser": "grobid",
+        "title": title_tag.get_text(" ", strip=True) if title_tag else Path(pdf_path).name,
+        "abstract": abstract_tag.get_text(" ", strip=True) if abstract_tag else "",
+        "authors": authors,
+        "sections": sections,
+        "references": references,
+        "raw_text": "",
+        "tei_xml": tei_xml[:60000],
+    }
+
+
+def parse_pdf_with_pymupdf(pdf_path):
+    if fitz is None:
+        raise RuntimeError("PyMuPDF not installed")
+
+    doc = fitz.open(pdf_path)
+    pages = [page.get_text("text") for page in doc]
+    raw_text = "\n".join(pages).strip()
+    first_page = raw_text[:4000]
+    lines = [x.strip() for x in first_page.splitlines() if x.strip()]
+    title = lines[0][:300] if lines else Path(pdf_path).name
+
+    abstract = ""
+    match = re.search(r"abstract\s*(.+?)(?:\n\s*\n|\n1[\.\s]|introduction)", raw_text, re.I | re.S)
+    if match:
+        abstract = norm_text(match.group(1))[:2000]
+
+    return {
+        "parser": "pymupdf",
+        "title": title,
+        "abstract": abstract,
+        "authors": [],
+        "sections": [{"heading": "Full Text", "text": raw_text[:12000]}] if raw_text else [],
+        "references": [],
+        "raw_text": raw_text[:50000],
+        "tei_xml": "",
+    }
+
+
+def parse_pdf_with_docling(pdf_path):
+    try:
+        from docling.document_converter import DocumentConverter
+    except Exception as e:
+        raise RuntimeError(f"Docling import failed: {e}")
+
+    converter = DocumentConverter()
+    result = converter.convert(pdf_path)
+    doc = result.document
+    markdown = doc.export_to_markdown()
+
+    title = Path(pdf_path).name
+    first_nonempty = next((line.strip("# ").strip() for line in markdown.splitlines() if line.strip()), "")
+    if first_nonempty:
+        title = first_nonempty[:300]
+
+    return {
+        "parser": "docling",
+        "title": title,
+        "abstract": "",
+        "authors": [],
+        "sections": [{"heading": "Document", "text": markdown[:12000]}] if markdown else [],
+        "references": [],
+        "raw_text": markdown[:50000],
+        "tei_xml": "",
+    }
+
+
+def parse_uploaded_pdf(file_obj, parser_order):
+    if not file_obj:
+        return "No PDF uploaded yet.", {}
+
+    path = getattr(file_obj, "name", None) or str(file_obj)
+    parser_order = ensure_list(parser_order) or ["grobid", "docling", "pymupdf"]
+    errors = []
+
+    for parser_name in parser_order:
+        try:
+            if parser_name == "grobid":
+                result = parse_pdf_with_grobid(path)
+            elif parser_name == "docling":
+                result = parse_pdf_with_docling(path)
+            elif parser_name == "pymupdf":
+                result = parse_pdf_with_pymupdf(path)
+            else:
+                continue
+
+            summary = (
+                f"### Parsed PDF\n\n"
+                f"- Parser used: {result['parser']}\n"
+                f"- Title: {result.get('title') or 'Unknown'}\n"
+                f"- Authors: {', '.join(result.get('authors')[:6]) if result.get('authors') else 'Unknown'}\n"
+                f"- Abstract found: {'Yes' if result.get('abstract') else 'No'}\n"
+                f"- Sections extracted: {len(result.get('sections') or [])}\n"
+                f"- References extracted: {len(result.get('references') or [])}\n"
+            )
+            return summary, result
+        except Exception as e:
+            errors.append(f"{parser_name}: {e}")
+
+    fail_summary = "### PDF parsing failed\n\n" + "\n".join([f"- {x}" for x in errors])
+    return fail_summary, {"parser": None, "errors": errors}
+
+
+def render_parse_result(parsed):
+    if not parsed:
+        return '<div class="panel" style="padding:18px"><p>No parsed document yet.</p></div>'
+
+    sections_html = []
+    for section in parsed.get("sections", [])[:6]:
+        sections_html.append(
+            f"""
+            <details class="agent-step">
+              <summary class="agent-summary">
+                <div class="agent-index">§</div>
+                <div class="agent-head">
+                  <h4>{safe_text(section.get('heading', 'Section'))}</h4>
+                  <span>section</span>
+                </div>
+              </summary>
+              <div class="agent-copy">
+                <p>{safe_text(section.get('text', '')[:1800])}</p>
+              </div>
+            </details>
+            """
+        )
+
+    refs = parsed.get("references", [])[:12]
+    refs_html = "".join(
+        f"<li>{safe_text(r.get('title') or 'Untitled')} {'· DOI ' + safe_text(r.get('doi')) if r.get('doi') else ''}</li>"
+        for r in refs
+    ) or "<li>No references extracted.</li>"
+
+    title = safe_text(parsed.get("title") or "Parsed document")
+    abstract = safe_text((parsed.get("abstract") or "")[:2400]) or "No abstract extracted."
+    parser_name = safe_text(parsed.get("parser") or "unknown")
+
+    return f"""
+    <div class="panel" style="padding:18px">
+      <div class="brain-header">
+        <div>
+          <p class="eyebrow">PDF Parse</p>
+          <h3>{title}</h3>
+        </div>
+        <div class="brain-legend"><span><i class="dot dot-upload"></i> {parser_name}</span></div>
+      </div>
+      <div class="parse-grid">
+        <div class="parse-card">
+          <h4>Abstract</h4>
+          <p>{abstract}</p>
+        </div>
+        <div class="parse-card">
+          <h4>References</h4>
+          <ul class="ref-list">{refs_html}</ul>
+        </div>
+      </div>
+      <div class="timeline" style="margin-top:14px;">
+        {''.join(sections_html) if sections_html else '<div class="panel" style="padding:16px;"><p>No sections extracted.</p></div>'}
+      </div>
+    </div>
+    """
+
+
+def build_ingest_payload(query, selected_papers, parsed_pdf=None):
+    nodes = [{"id": "topic:query", "type": "Topic", "label": query or "Research topic"}]
+    edges = []
+
+    for i, p in enumerate(selected_papers, start=1):
+        paper_id = p.get("doi") or (p.get("external_ids") or {}).get("arxiv") or f"paper:{i}"
+        nodes.append({
+            "id": paper_id,
+            "type": "Paper",
+            "title": p.get("title"),
+            "year": p.get("year"),
+            "venue": p.get("venue"),
+            "doi": p.get("doi"),
+            "source": p.get("source"),
+            "url": p.get("url"),
+            "pdf": p.get("pdf"),
+        })
+        edges.append({"source": "topic:query", "target": paper_id, "type": "ABOUT", "weight": p.get("score", 0)})
+
+        for author in p.get("authors", [])[:8]:
+            author_id = f"author:{author.lower()}"
+            nodes.append({"id": author_id, "type": "Author", "label": author})
+            edges.append({"source": paper_id, "target": author_id, "type": "WRITTEN_BY"})
+
+    if parsed_pdf and parsed_pdf.get("title"):
+        doc_id = "upload:pdf"
+        nodes.append({
+            "id": doc_id,
+            "type": "UploadedPDF",
+            "title": parsed_pdf.get("title"),
+            "parser": parsed_pdf.get("parser"),
+        })
+        edges.append({"source": "topic:query", "target": doc_id, "type": "UPLOADED_SOURCE"})
+        for idx, section in enumerate(parsed_pdf.get("sections", [])[:8], start=1):
+            sec_id = f"{doc_id}:section:{idx}"
+            nodes.append({"id": sec_id, "type": "Section", "label": section.get("heading") or f"Section {idx}"})
+            edges.append({"source": doc_id, "target": sec_id, "type": "HAS_SECTION"})
+        for idx, ref in enumerate(parsed_pdf.get("references", [])[:12], start=1):
+            ref_id = f"{doc_id}:ref:{idx}"
+            nodes.append({"id": ref_id, "type": "Reference", "label": ref.get("title") or f"Reference {idx}", "doi": ref.get("doi")})
+            edges.append({"source": doc_id, "target": ref_id, "type": "CITES"})
+
+    return {"status": "ok", "nodes": nodes, "edges": edges}
+
+
+def run_paper_discovery(query, search_mode, sources, pdf_file):
+    query_text = (query or "").strip()
+
+    if not query_text and not pdf_file:
+        empty_graph = build_learning_graph_html([], [], "Self-Learning Knowledge Graph")
+        return (
+            empty_graph,
+            '<div class="panel papers-panel" style="padding:18px"><p>Enter a topic, title, DOI, link, or upload a PDF to start learning.</p></div>',
+            build_journal_html("biomaterials cardiac repair"),
+            "No PDF uploaded yet.",
+            gr.update(choices=[], value=[]),
+            [],
+            "### No discovery results yet.",
+        )
+
+    papers = []
+    if query_text:
+        try:
+            papers = discover_papers(query_text, search_mode, sources, max_results=10)
+        except Exception as e:
+            graph_nodes, graph_edges = build_learning_graph_state(
+                query_text,
+                [],
+                Path(getattr(pdf_file, "name", "uploaded.pdf")).name if pdf_file else None,
+            )
+            error_html = f'<div class="panel papers-panel" style="padding:18px"><p>Paper search failed: {safe_text(str(e))}</p></div>'
+            return (
+                build_learning_graph_html(graph_nodes, graph_edges),
+                error_html,
+                build_journal_html(query_text or "biomaterials cardiac repair"),
+                uploaded_pdf_summary(pdf_file),
+                gr.update(choices=[], value=[]),
+                [],
+                f"### Discovery failed.\n\n- Error: {safe_text(str(e))}",
+            )
+
+    uploaded_name = Path(getattr(pdf_file, "name", str(pdf_file))).name if pdf_file else None
+    graph_nodes, graph_edges = build_learning_graph_state(query_text, papers[:5], uploaded_name)
+    graph_html = build_learning_graph_html(graph_nodes, graph_edges)
+    papers_html = format_papers_html(papers)
+    journals_html = build_journal_html(query_text or "biomaterials cardiac repair")
+    pdf_summary = uploaded_pdf_summary(pdf_file)
+    choices = format_selection_choices(papers)
+
+    status_md = (
+        f"### Discovery results\n\n"
+        f"- Search mode: {search_mode}\n"
+        f"- Sources: {', '.join(ensure_list(sources) or DEFAULT_SOURCES)}\n"
+        f"- Candidates found: {len(papers)}\n"
+        f"- Select papers below, then click **Ingest selected into graph**.\n"
+    )
+    return graph_html, papers_html, journals_html, pdf_summary, gr.update(choices=choices, value=[]), papers, status_md
+
+
+def ingest_selected_papers(query, selected_indices, papers_state, pdf_file, parsed_state):
+    papers = ensure_list(papers_state)
+    selected_indices = ensure_list(selected_indices)
+
+    selected = []
+    for idx in selected_indices:
+        try:
+            selected.append(papers[int(idx)])
+        except Exception:
+            pass
+
+    uploaded_name = Path(getattr(pdf_file, "name", str(pdf_file))).name if pdf_file else None
+
+    if not selected and not parsed_state:
+        graph_html = build_learning_graph_html([], [], "Self-Learning Knowledge Graph")
+        return graph_html, "### Nothing ingested yet.\n\nSelect papers or parse an uploaded PDF first.", {"status": "empty", "nodes": [], "edges": []}
+
+    graph_nodes, graph_edges = graph_from_selected(query, selected, uploaded_name)
+    graph_html = build_learning_graph_html(graph_nodes, graph_edges, "Selected Research Graph")
+    payload = build_ingest_payload(query, selected, parsed_state if isinstance(parsed_state, dict) else None)
+
+    summary_lines = [
+        "### Graph ingest ready",
+        "",
+        f"- Topic: {query or 'Research topic'}",
+        f"- Selected papers: {len(selected)}",
+        f"- Uploaded PDF parsed: {'Yes' if parsed_state and isinstance(parsed_state, dict) and parsed_state.get('title') else 'No'}",
+        f"- Nodes created: {len(payload['nodes'])}",
+        f"- Edges created: {len(payload['edges'])}",
+    ]
+    return graph_html, "\n".join(summary_lines), payload