Create self_learning_graph.py

dvnc_ai_v2_hf/self_learning_graph.py

@@ -0,0 +1,1430 @@
+import html
+import os
+import re
+import time
+import urllib.parse
+import xml.etree.ElementTree as ET
+from collections import Counter
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import gradio as gr
+import requests
+
+try:
+    import fitz  # PyMuPDF
+except Exception:
+    fitz = None
+
+try:
+    from bs4 import BeautifulSoup
+except Exception:
+    BeautifulSoup = None
+
+
+JOURNALS = [
+    {"name": "Nature", "url": "https://www.nature.com/search", "desc": "Flagship multidisciplinary research journal."},
+    {"name": "Science", "url": "https://www.science.org/search", "desc": "High-impact science journal and family."},
+    {"name": "Cell", "url": "https://www.cell.com/search", "desc": "Life sciences and translational biology."},
+    {"name": "The Lancet", "url": "https://www.thelancet.com/search", "desc": "Clinical and medical research."},
+    {"name": "IEEE Xplore", "url": "https://ieeexplore.ieee.org/search/searchresult.jsp", "desc": "Engineering, AI, signal processing, and systems."},
+]
+
+SEARCH_MODES = ["topic", "title", "doi", "link", "paper_name", "autonomous_web"]
+SOURCE_OPTIONS = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
+DEFAULT_SOURCES = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
+
+SEMANTIC_SCHOLAR_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY", "").strip()
+GROBID_URL = os.getenv("GROBID_URL", "").strip()
+REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "25"))
+GRAPH_MAX_CONCEPTS = int(os.getenv("GRAPH_MAX_CONCEPTS", "10"))
+GRAPH_MAX_CLAIMS = int(os.getenv("GRAPH_MAX_CLAIMS", "6"))
+GRAPH_MAX_RESULTS = int(os.getenv("GRAPH_MAX_RESULTS", "10"))
+
+STOPWORDS = {
+    "a", "an", "and", "are", "as", "at", "be", "been", "being", "by", "can", "could", "did", "do", "does",
+    "for", "from", "had", "has", "have", "if", "in", "into", "is", "it", "its", "may", "might", "of", "on",
+    "or", "our", "such", "that", "the", "their", "there", "these", "this", "those", "to", "using", "use",
+    "used", "via", "was", "were", "will", "with", "within", "without", "we", "they", "you", "your", "study",
+    "paper", "research", "results", "method", "methods", "analysis", "approach", "toward", "towards",
+    "based", "new", "novel", "effect", "effects", "model", "models", "system", "systems", "show", "shows",
+}
+
+GRAPH_MEMORY = {
+    "papers": {},
+    "nodes": {},
+    "edges": [],
+    "concept_counts": Counter(),
+    "claim_counts": Counter(),
+    "queries": [],
+    "events": [],
+}
+
+
+def safe_text(x, default=""):
+    return html.escape(str(x if x is not None else default))
+
+
+def norm_text(x: Optional[str]) -> str:
+    return re.sub(r"\s+", " ", (x or "")).strip()
+
+
+def slugify(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")
+
+
+def ensure_list(x):
+    return x if isinstance(x, list) else []
+
+
+def normalize_doi(text: str) -> str:
+    text = (text or "").strip()
+    text = text.replace("https://doi.org/", "").replace("http://doi.org/", "")
+    return text.strip()
+
+
+def detect_query_type(query: str) -> str:
+    q = (query or "").strip()
+    doi_pattern = r"^10\.\d{4,9}/[-._;()/:A-Z0-9]+$"
+    if re.match(doi_pattern, q, flags=re.I):
+        return "doi"
+    if q.startswith("http://") or q.startswith("https://"):
+        return "link"
+    return "topic"
+
+
+def tokenize(text: str) -> List[str]:
+    return [t for t in re.findall(r"[a-zA-Z][a-zA-Z0-9\-]{2,}", (text or "").lower()) if t not in STOPWORDS]
+
+
+def unique_keep_order(items: List[str]) -> List[str]:
+    seen = set()
+    out = []
+    for item in items:
+        key = norm_text(item).lower()
+        if key and key not in seen:
+            seen.add(key)
+            out.append(norm_text(item))
+    return out
+
+
+def text_overlap_score(a: str, b: str) -> float:
+    sa = set(tokenize(a))
+    sb = set(tokenize(b))
+    if not sa or not sb:
+        return 0.0
+    return len(sa & sb) / max(1, len(sa | sb))
+
+
+def compute_recency_bonus(year: str) -> float:
+    try:
+        y = int(str(year)[:4])
+    except Exception:
+        return 0.0
+    current = time.gmtime().tm_year
+    age = max(current - y, 0)
+    return max(0.0, 0.12 - age * 0.015)
+
+
+def extract_candidate_phrases(text: str, max_terms: int = 20) -> List[str]:
+    text = norm_text(text)
+    if not text:
+        return []
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9\-]{2,}", text)
+    phrases = []
+
+    for n in (3, 2, 1):
+        for i in range(len(tokens) - n + 1):
+            phrase = " ".join(tokens[i:i + n]).strip().lower()
+            if len(phrase) < 4:
+                continue
+            parts = phrase.split()
+            if any(p in STOPWORDS for p in parts):
+                continue
+            if all(len(p) <= 2 for p in parts):
+                continue
+            phrases.append(phrase)
+
+    counts = Counter(phrases)
+    ranked = [p for p, _ in counts.most_common(max_terms * 3)]
+
+    filtered = []
+    for phrase in ranked:
+        if phrase in filtered:
+            continue
+        if any(phrase != other and phrase in other for other in filtered):
+            continue
+        filtered.append(phrase)
+        if len(filtered) >= max_terms:
+            break
+
+    return filtered
+
+
+def extract_concepts_from_text(text: str, max_terms: int = GRAPH_MAX_CONCEPTS) -> List[str]:
+    return extract_candidate_phrases(text, max_terms=max_terms)
+
+
+def extract_claim_like_sentences(text: str, max_items: int = GRAPH_MAX_CLAIMS) -> List[str]:
+    text = norm_text(text)
+    if not text:
+        return []
+
+    parts = re.split(r"(?<=[\.\!\?])\s+", text)
+    scored = []
+    for sentence in parts:
+        s = norm_text(sentence)
+        if len(s) < 40 or len(s) > 280:
+            continue
+        lower = s.lower()
+        score = 0.0
+        if any(k in lower for k in ["improves", "reduces", "increases", "suggests", "demonstrates", "shows", "reveals", "predicts", "achieves", "outperforms"]):
+            score += 2.0
+        if any(k in lower for k in ["significant", "associated", "correlated", "effective", "robust", "accurate", "validated"]):
+            score += 1.0
+        score += min(len(tokenize(s)) / 15.0, 2.0)
+        scored.append((score, s))
+    return [s for _, s in sorted(scored, key=lambda x: x[0], reverse=True)[:max_items]]
+
+
+def parse_openalex_abstract(inverted_index) -> str:
+    if not inverted_index or not isinstance(inverted_index, dict):
+        return ""
+    pos_to_word = {}
+    for word, positions in inverted_index.items():
+        for pos in positions:
+            pos_to_word[pos] = word
+    if not pos_to_word:
+        return ""
+    return " ".join(pos_to_word[i] for i in sorted(pos_to_word))
+
+
+def enrich_paper_semantics(query: str, paper: Dict) -> Dict:
+    paper = dict(paper)
+    title = paper.get("title", "")
+    abstract = paper.get("abstract", "") or paper.get("summary", "")
+    venue = paper.get("venue", "")
+    base_text = " ".join([title, abstract, venue]).strip()
+
+    concepts = extract_concepts_from_text(base_text, max_terms=GRAPH_MAX_CONCEPTS)
+    claims = extract_claim_like_sentences(abstract, max_items=GRAPH_MAX_CLAIMS)
+
+    rel = text_overlap_score(query, f"{title} {abstract}")
+    recency = compute_recency_bonus(paper.get("year"))
+    doi_bonus = 0.02 if paper.get("doi") else 0.0
+    oa_bonus = 0.03 if paper.get("open_access") else 0.0
+    concept_bonus = min(len(concepts), 8) * 0.01
+
+    learned_score = float(paper.get("score", 0)) + rel * 0.5 + recency + doi_bonus + oa_bonus + concept_bonus
+
+    paper["concepts"] = concepts[:GRAPH_MAX_CONCEPTS]
+    paper["claims"] = claims[:GRAPH_MAX_CLAIMS]
+    paper["relevance"] = round(rel, 4)
+    paper["learned_score"] = round(learned_score, 4)
+    return paper
+
+
+def paper_identity_key(paper: Dict) -> str:
+    return (
+        normalize_doi(paper.get("doi") or "")
+        or (paper.get("external_ids") or {}).get("arxiv")
+        or norm_text(paper.get("title", "")).lower()
+        or str(paper.get("id"))
+    )
+
+
+def journal_query_links(query: str):
+    q = urllib.parse.quote_plus(query or "biomaterials cardiac repair")
+    rows = []
+    for journal in JOURNALS:
+        url = f"{journal['url']}?q={q}" if "?" not in journal["url"] else f"{journal['url']}&q={q}"
+        if "ieeexplore" in journal["url"]:
+            url = f"https://ieeexplore.ieee.org/search/searchresult.jsp?queryText={q}"
+        rows.append({"name": journal["name"], "desc": journal["desc"], "url": url})
+    return rows
+
+
+def build_journal_html(query):
+    rows = []
+    for journal in journal_query_links(query):
+        rows.append(
+            f"""
+            <a class="journal-card" href="{safe_text(journal['url'])}" target="_blank" rel="noopener noreferrer">
+              <div>
+                <h4>{safe_text(journal['name'])}</h4>
+                <p>{safe_text(journal['desc'])}</p>
+              </div>
+              <span>Open</span>
+            </a>
+            """
+        )
+    return '<div class="journal-grid">' + ''.join(rows) + '</div>'
+
+
+def search_arxiv(query, max_results=8):
+    encoded = urllib.parse.quote(query)
+    url = (
+        "http://export.arxiv.org/api/query?search_query=all:"
+        f"{encoded}&start=0&max_results={max_results}&sortBy=relevance&sortOrder=descending"
+    )
+    response = requests.get(url, timeout=REQUEST_TIMEOUT)
+    response.raise_for_status()
+
+    root = ET.fromstring(response.text)
+    ns = {"atom": "http://www.w3.org/2005/Atom"}
+    papers = []
+
+    for entry in root.findall("atom:entry", ns):
+        title = " ".join((entry.findtext("atom:title", default="", namespaces=ns) or "").split())
+        summary = " ".join((entry.findtext("atom:summary", default="", namespaces=ns) or "").split())
+        published = entry.findtext("atom:published", default="", namespaces=ns)
+        paper_id = entry.findtext("atom:id", default="", namespaces=ns)
+        authors = [a.findtext("atom:name", default="", namespaces=ns) for a in entry.findall("atom:author", ns)]
+        pdf_url = ""
+
+        for link in entry.findall("atom:link", ns):
+            if link.attrib.get("title") == "pdf":
+                pdf_url = link.attrib.get("href", "")
+                break
+
+        papers.append({
+            "id": paper_id or title,
+            "title": title,
+            "summary": summary,
+            "abstract": summary,
+            "published": published[:10],
+            "authors": [a for a in authors[:8] if a],
+            "authors_text": ", ".join([a for a in authors[:4] if a]) or "Unknown authors",
+            "url": paper_id,
+            "pdf": pdf_url,
+            "doi": "",
+            "venue": "arXiv",
+            "year": published[:4] if published else "",
+            "source": "arxiv",
+            "score": 0.76,
+            "open_access": True,
+            "external_ids": {"arxiv": (paper_id or "").split("/")[-1]},
+        })
+
+    return papers
+
+
+def search_crossref(query, mode="topic", max_results=8):
+    headers = {"User-Agent": "dvnc-ai-space/0.3"}
+
+    if mode == "doi":
+        url = f"https://api.crossref.org/works/{urllib.parse.quote(query)}"
+        response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT)
+        if response.status_code != 200:
+            return []
+        items = [response.json().get("message", {})]
+    else:
+        params = {"rows": max_results}
+        if mode in ("title", "paper_name"):
+            params["query.title"] = query
+        else:
+            params["query.bibliographic"] = query
+        response = requests.get("https://api.crossref.org/works", params=params, headers=headers, timeout=REQUEST_TIMEOUT)
+        response.raise_for_status()
+        items = response.json().get("message", {}).get("items", [])
+
+    out = []
+    for item in items:
+        authors = []
+        for a in item.get("author", []) or []:
+            name = " ".join(filter(None, [a.get("given"), a.get("family")])).strip()
+            if name:
+                authors.append(name)
+
+        title = (item.get("title") or ["Untitled"])[0]
+        year = ""
+        for key in ["published-print", "published-online", "created"]:
+            if item.get(key, {}).get("date-parts"):
+                year = str(item[key]["date-parts"][0][0])
+                break
+
+        abstract = re.sub("<.*?>", "", item.get("abstract") or "")
+        doi = normalize_doi(item.get("DOI", ""))
+
+        out.append({
+            "id": doi or title,
+            "title": norm_text(title),
+            "summary": norm_text(abstract)[:500],
+            "abstract": norm_text(abstract),
+            "published": year,
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": item.get("URL", ""),
+            "pdf": "",
+            "doi": doi,
+            "venue": (item.get("container-title") or [""])[0],
+            "year": year,
+            "source": "crossref",
+            "score": 0.72,
+            "open_access": None,
+            "external_ids": {"crossref": doi} if doi else {},
+        })
+
+    return out
+
+
+def search_openalex(query, mode="topic", max_results=8):
+    params = {"per-page": max_results}
+    if mode == "doi":
+        doi = normalize_doi(query)
+        params["filter"] = f"doi:https://doi.org/{doi}"
+    else:
+        params["search"] = query
+
+    response = requests.get("https://api.openalex.org/works", params=params, timeout=REQUEST_TIMEOUT)
+    response.raise_for_status()
+    items = response.json().get("results", [])
+
+    out = []
+    for item in items:
+        authors = []
+        for auth in item.get("authorships", [])[:8]:
+            author = auth.get("author") or {}
+            if author.get("display_name"):
+                authors.append(author["display_name"])
+
+        oa = item.get("open_access") or {}
+        doi = normalize_doi(item.get("doi") or "")
+        abstract = parse_openalex_abstract(item.get("abstract_inverted_index"))
+
+        out.append({
+            "id": item.get("id") or doi or item.get("title"),
+            "title": norm_text(item.get("title")),
+            "summary": norm_text(abstract)[:500],
+            "abstract": norm_text(abstract),
+            "published": str(item.get("publication_year") or ""),
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": (item.get("primary_location") or {}).get("landing_page_url") or item.get("id") or "",
+            "pdf": oa.get("oa_url") or "",
+            "doi": doi,
+            "venue": ((item.get("primary_location") or {}).get("source") or {}).get("display_name") or "",
+            "year": str(item.get("publication_year") or ""),
+            "source": "openalex",
+            "score": 0.80,
+            "open_access": oa.get("is_oa"),
+            "external_ids": item.get("ids") or {},
+        })
+
+    return out
+
+
+def search_semantic_scholar(query, mode="topic", max_results=8):
+    headers = {}
+    if SEMANTIC_SCHOLAR_API_KEY:
+        headers["x-api-key"] = SEMANTIC_SCHOLAR_API_KEY
+
+    fields = "title,authors,year,abstract,venue,externalIds,url,openAccessPdf"
+
+    if mode == "doi":
+        doi = normalize_doi(query)
+        url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{urllib.parse.quote(doi)}"
+        response = requests.get(url, params={"fields": fields}, headers=headers, timeout=REQUEST_TIMEOUT)
+        if response.status_code != 200:
+            return []
+        items = [response.json()]
+    else:
+        response = requests.get(
+            "https://api.semanticscholar.org/graph/v1/paper/search",
+            params={"query": query, "limit": max_results, "fields": fields},
+            headers=headers,
+            timeout=REQUEST_TIMEOUT,
+        )
+        if response.status_code != 200:
+            return []
+        items = response.json().get("data", [])
+
+    out = []
+    for item in items:
+        external = item.get("externalIds") or {}
+        authors = [a.get("name") for a in item.get("authors", []) if a.get("name")]
+
+        out.append({
+            "id": external.get("CorpusId") or external.get("DOI") or item.get("title"),
+            "title": norm_text(item.get("title")),
+            "summary": norm_text(item.get("abstract", ""))[:500],
+            "abstract": norm_text(item.get("abstract", "")),
+            "published": str(item.get("year") or ""),
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": item.get("url") or "",
+            "pdf": (item.get("openAccessPdf") or {}).get("url") or "",
+            "doi": normalize_doi(external.get("DOI", "")),
+            "venue": item.get("venue") or "",
+            "year": str(item.get("year") or ""),
+            "source": "semantic_scholar",
+            "score": 0.84,
+            "open_access": bool((item.get("openAccessPdf") or {}).get("url")),
+            "external_ids": external,
+        })
+
+    return out
+
+
+def search_europe_pmc(query, mode="topic", max_results=8):
+    epmc_query = f'DOI:"{query}"' if mode == "doi" else query
+    params = {"query": epmc_query, "format": "json", "pageSize": max_results, "resultType": "core"}
+    response = requests.get("https://www.ebi.ac.uk/europepmc/webservices/rest/search", params=params, timeout=REQUEST_TIMEOUT)
+    if response.status_code != 200:
+        return []
+
+    items = response.json().get("resultList", {}).get("result", [])
+    out = []
+    for item in items:
+        author_string = item.get("authorString", "")
+        authors = [x.strip() for x in author_string.split(",")[:8] if x.strip()]
+        pmcid = item.get("pmcid", "")
+        pdf_url = f"https://europepmc.org/articles/{pmcid}?pdf=render" if pmcid else ""
+        landing_url = f"https://europepmc.org/article/PMC/{pmcid}" if pmcid else ""
+
+        out.append({
+            "id": item.get("id") or item.get("doi") or item.get("title"),
+            "title": norm_text(item.get("title")),
+            "summary": norm_text(item.get("abstractText", ""))[:500],
+            "abstract": norm_text(item.get("abstractText", "")),
+            "published": str(item.get("pubYear") or ""),
+            "authors": authors,
+            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
+            "url": landing_url,
+            "pdf": pdf_url,
+            "doi": normalize_doi(item.get("doi", "")),
+            "venue": item.get("journalTitle", ""),
+            "year": str(item.get("pubYear") or ""),
+            "source": "europe_pmc",
+            "score": 0.78,
+            "open_access": bool(pmcid),
+            "external_ids": {"pmid": item.get("pmid"), "pmcid": pmcid},
+        })
+
+    return out
+
+
+def resolve_link(query):
+    url = (query or "").strip()
+    if not url:
+        return []
+
+    try:
+        response = requests.get(
+            url,
+            timeout=REQUEST_TIMEOUT,
+            allow_redirects=True,
+            headers={"User-Agent": "dvnc-ai-space/0.3"},
+        )
+
+        content_type = response.headers.get("content-type", "")
+        if "pdf" in content_type or url.lower().endswith(".pdf"):
+            name = Path(url.split("?")[0]).name or "linked-paper.pdf"
+            return [{
+                "id": url,
+                "title": name,
+                "summary": "Direct PDF link detected.",
+                "abstract": "",
+                "published": "",
+                "authors": [],
+                "authors_text": "Unknown authors",
+                "url": url,
+                "pdf": url,
+                "doi": "",
+                "venue": "Direct PDF",
+                "year": "",
+                "source": "link",
+                "score": 0.66,
+                "open_access": True,
+                "external_ids": {},
+            }]
+
+        doi = ""
+        title = url
+        pdf_link = ""
+
+        if BeautifulSoup is not None:
+            soup = BeautifulSoup(response.text, "html.parser")
+            title = soup.title.text.strip() if soup.title else url
+
+            for meta_name in ["citation_doi", "dc.identifier", "dc.Identifier"]:
+                tag = soup.find("meta", attrs={"name": meta_name})
+                if tag and tag.get("content"):
+                    doi = normalize_doi(tag["content"].strip())
+                    break
+
+            for a in soup.find_all("a", href=True):
+                href = a["href"]
+                if ".pdf" in href.lower():
+                    pdf_link = href if href.startswith("http") else ""
+                    break
+
+        if doi:
+            results = search_crossref(doi, mode="doi", max_results=1)
+            if results:
+                if pdf_link and not results[0].get("pdf"):
+                    results[0]["pdf"] = pdf_link
+                if url and not results[0].get("url"):
+                    results[0]["url"] = url
+                return results
+
+        return [{
+            "id": url,
+            "title": title,
+            "summary": "Landing page resolved from direct link.",
+            "abstract": "",
+            "published": "",
+            "authors": [],
+            "authors_text": "Unknown authors",
+            "url": url,
+            "pdf": pdf_link,
+            "doi": doi,
+            "venue": "Web Link",
+            "year": "",
+            "source": "link",
+            "score": 0.54,
+            "open_access": bool(pdf_link),
+            "external_ids": {},
+        }]
+    except Exception as e:
+        return [{
+            "id": url,
+            "title": "Link resolution error",
+            "summary": str(e),
+            "abstract": "",
+            "published": "",
+            "authors": [],
+            "authors_text": "Unknown authors",
+            "url": url,
+            "pdf": "",
+            "doi": "",
+            "venue": "Link",
+            "year": "",
+            "source": "link",
+            "score": 0.20,
+            "open_access": None,
+            "external_ids": {},
+        }]
+
+
+def dedupe_papers(items: List[Dict]) -> List[Dict]:
+    seen = {}
+    for item in items:
+        key = paper_identity_key(item) or f"{item.get('source', 'src')}::{item.get('title', 'paper')}"
+        if key not in seen or float(item.get("score", 0)) > float(seen[key].get("score", 0)):
+            seen[key] = item
+    return sorted(seen.values(), key=lambda x: float(x.get("score", 0)), reverse=True)
+
+
+def discover_papers(query, mode, sources, max_results=10):
+    query = (query or "").strip()
+    if not query:
+        return []
+
+    mode = detect_query_type(query) if mode == "autonomous_web" else mode
+    selected_sources = ensure_list(sources) or DEFAULT_SOURCES
+    results = []
+
+    if mode == "link":
+        return dedupe_papers(resolve_link(query))
+
+    if "arxiv" in selected_sources and mode != "doi":
+        try:
+            results.extend(search_arxiv(query, max_results=min(max_results, GRAPH_MAX_RESULTS)))
+        except Exception:
+            pass
+
+    if "crossref" in selected_sources:
+        try:
+            results.extend(search_crossref(query, mode=mode, max_results=min(max_results, GRAPH_MAX_RESULTS)))
+        except Exception:
+            pass
+
+    if "openalex" in selected_sources:
+        try:
+            results.extend(search_openalex(query, mode=mode, max_results=min(max_results, GRAPH_MAX_RESULTS)))
+        except Exception:
+            pass
+
+    if "semantic_scholar" in selected_sources:
+        try:
+            results.extend(search_semantic_scholar(query, mode=mode, max_results=min(max_results, GRAPH_MAX_RESULTS)))
+        except Exception:
+            pass
+
+    if "europe_pmc" in selected_sources:
+        try:
+            results.extend(search_europe_pmc(query, mode=mode, max_results=min(max_results, GRAPH_MAX_RESULTS)))
+        except Exception:
+            pass
+
+    papers = dedupe_papers(results)
+    papers = [enrich_paper_semantics(query, p) for p in papers]
+    papers = sorted(papers, key=lambda x: float(x.get("learned_score", x.get("score", 0))), reverse=True)
+    return papers
+
+
+def paper_choice_value(index: int, paper: Dict) -> str:
+    doi = normalize_doi(paper.get("doi") or "")
+    title_slug = slugify(paper.get("title", ""))[:40]
+    return f"{index}|{doi}|{title_slug}"
+
+
+def paper_choice_label(index: int, paper: Dict) -> str:
+    score = round(float(paper.get("learned_score", paper.get("score", 0))), 3)
+    title = paper.get("title", "Untitled")
+    authors_text = paper.get("authors_text", "Unknown authors")[:80]
+    source = paper.get("source", "src")
+    return f"[{source}] {title} — {authors_text} — score {score}"
+
+
+def format_selection_choices(papers):
+    return [(paper_choice_label(i, paper), paper_choice_value(i, paper)) for i, paper in enumerate(papers)]
+
+
+def format_papers_html(papers):
+    if not papers:
+        return '<div class="panel papers-panel" style="padding:18px"><p>No papers found yet.</p></div>'
+
+    items = []
+    for i, paper in enumerate(papers, start=1):
+        summary = safe_text((paper.get("summary") or paper.get("abstract") or "")[:280])
+        doi_line = f'<span class="paper-badge doi-badge">{safe_text(paper.get("doi"))}</span>' if paper.get("doi") else ""
+        pdf_link = paper.get("pdf") or "#"
+        abs_link = paper.get("url") or "#"
+        concepts_text = ", ".join((paper.get("concepts") or [])[:4])
+
+        items.append(
+            f"""
+            <article class="paper-card">
+              <div class="paper-topline">
+                <span class="paper-badge">{safe_text(paper.get('source', 'paper'))}</span>
+                <span class="paper-badge alt">{safe_text(paper.get('published', '') or 'Paper')}</span>
+                {doi_line}
+              </div>
+              <h4>{i}. {safe_text(paper.get('title', 'Untitled'))}</h4>
+              <p>{summary or 'No abstract snippet available.'}</p>
+              <div class="paper-meta-stack">
+                <div><strong>Authors:</strong> {safe_text(paper.get('authors_text', 'Unknown authors'))}</div>
+                <div><strong>Venue:</strong> {safe_text(paper.get('venue', 'Unknown venue'))}</div>
+                <div><strong>Learned score:</strong> {safe_text(round(float(paper.get('learned_score', paper.get('score', 0))), 3))}</div>
+                <div><strong>Concepts:</strong> {safe_text(concepts_text or 'None extracted')}</div>
+              </div>
+              <div class="paper-links">
+                <a href="{safe_text(abs_link)}" target="_blank" rel="noopener noreferrer">Abstract</a>
+                <a href="{safe_text(pdf_link)}" target="_blank" rel="noopener noreferrer">PDF</a>
+              </div>
+            </article>
+            """
+        )
+
+    return '<div class="papers-grid">' + ''.join(items) + '</div>'
+
+
+def uploaded_pdf_summary(file_obj):
+    if not file_obj:
+        return "No PDF uploaded yet."
+    path = getattr(file_obj, "name", None) or str(file_obj)
+    p = Path(path)
+    return f"Uploaded PDF ready for ingestion: {p.name}. Use Parse uploaded PDF to extract title, abstract, sections, references, concepts, and claims."
+
+
+def build_learning_graph_html(nodes, edges, title="Self-Learning Knowledge Graph"):
+    if not nodes:
+        return """
+        <div class="panel brain-shell">
+          <div class="brain-header">
+            <div>
+              <p class="eyebrow">Learning Graph</p>
+              <h3>Self-Learning Knowledge Graph</h3>
+            </div>
+          </div>
+          <div class="brain-stage learning-empty">
+            <div class="empty-graph-copy">
+              <h4>No papers mapped yet</h4>
+              <p>Search papers, pick a topic, select candidates, or upload a PDF to grow the graph in real time.</p>
+            </div>
+          </div>
+        </div>
+        """
+
+    coords = [
+        (100, 90), (250, 60), (420, 75), (590, 115), (690, 250), (620, 395),
+        (455, 455), (280, 455), (110, 395), (60, 250), (215, 250), (365, 245),
+        (525, 250), (300, 145), (480, 340), (180, 340), (545, 175), (130, 170)
+    ]
+
+    graph_nodes = [dict(n) for n in nodes[:18]]
+    for i, node in enumerate(graph_nodes):
+        x, y = coords[i % len(coords)]
+        node["sx"] = x
+        node["sy"] = y
+
+    node_map = {n["id"]: n for n in graph_nodes}
+    edge_items = []
+    node_items = []
+    label_items = []
+
+    for edge in edges[:80]:
+        source = edge.get("source")
+        target = edge.get("target")
+        edge_type = edge.get("type", "")
+        if source in node_map and target in node_map:
+            a = node_map[source]
+            b = node_map[target]
+            edge_items.append(
+                f'<line class="learn-edge edge-{safe_text(edge_type.lower())}" x1="{a["sx"]}" y1="{a["sy"]}" x2="{b["sx"]}" y2="{b["sy"]}" />'
+            )
+
+    for node in graph_nodes:
+        kind = (node.get("kind") or node.get("type") or "paper").lower()
+        if kind == "topic":
+            kind = "query"
+        if kind == "uploadedpdf":
+            kind = "upload"
+
+        radius = 25 if kind == "query" else 18 if kind in {"concept", "author", "claim", "reference"} else 20
+        css_class = f"learn-node {kind}"
+
+        node_items.append(
+            f'<circle class="{css_class}" cx="{node["sx"]}" cy="{node["sy"]}" r="{radius}" />'
+        )
+
+        label = node.get("label") or node.get("title") or node.get("id")
+        label_items.append(
+            f'<text class="learn-label" x="{node["sx"] + 26}" y="{node["sy"] - 8}">{safe_text(str(label)[:46])}</text>'
+        )
+
+    return f"""
+    <div class="panel brain-shell">
+      <div class="brain-header">
+        <div>
+          <p class="eyebrow">Learning Graph</p>
+          <h3>{safe_text(title)}</h3>
+        </div>
+        <div class="brain-legend">
+          <span><i class="dot dot-query"></i> topic</span>
+          <span><i class="dot dot-paper"></i> paper</span>
+          <span><i class="dot dot-upload"></i> uploaded PDF</span>
+          <span><i class="dot dot-concept"></i> concept</span>
+          <span><i class="dot dot-author"></i> author</span>
+          <span><i class="dot dot-ref"></i> reference</span>
+        </div>
+      </div>
+      <div class="brain-stage">
+        <svg viewBox="0 0 760 520" class="brain-svg" role="img" aria-label="Self-learning knowledge graph">
+          {''.join(edge_items)}
+          {''.join(node_items)}
+          {''.join(label_items)}
+        </svg>
+      </div>
+    </div>
+    """
+
+
+def build_learning_graph_state(query, papers, uploaded_name=None):
+    nodes = [{"id": "query", "label": query or "Research topic", "kind": "query"}]
+    edges = []
+
+    for i, paper in enumerate(papers[:5], start=1):
+        pid = f"paper_{i}"
+        nodes.append({"id": pid, "label": paper.get("title", f"Paper {i}"), "kind": "paper"})
+        edges.append({"source": "query", "target": pid, "type": "ABOUT"})
+
+        for concept in (paper.get("concepts") or [])[:2]:
+            cid = f"concept_{i}_{slugify(concept)[:20]}"
+            nodes.append({"id": cid, "label": concept, "kind": "concept"})
+            edges.append({"source": pid, "target": cid, "type": "MENTIONS"})
+
+    if uploaded_name:
+        nodes.append({"id": "upload", "label": uploaded_name, "kind": "upload"})
+        edges.append({"source": "query", "target": "upload", "type": "UPLOADED_SOURCE"})
+
+    return nodes, edges
+
+
+def graph_from_selected(query, selected_papers, uploaded_name=None, parsed_state=None):
+    nodes = [{"id": "query", "label": query or "Research topic", "kind": "query"}]
+    edges = []
+
+    for i, paper in enumerate(selected_papers[:6], start=1):
+        pid = f"paper_{i}"
+        nodes.append({"id": pid, "label": paper.get("title", f"Paper {i}"), "kind": "paper"})
+        edges.append({"source": "query", "target": pid, "type": "ABOUT"})
+
+        for author in paper.get("authors", [])[:2]:
+            aid = f"author_{i}_{slugify(author)[:24]}"
+            nodes.append({"id": aid, "label": author, "kind": "author"})
+            edges.append({"source": pid, "target": aid, "type": "WRITTEN_BY"})
+
+        for concept in (paper.get("concepts") or [])[:2]:
+            cid = f"concept_{i}_{slugify(concept)[:24]}"
+            nodes.append({"id": cid, "label": concept, "kind": "concept"})
+            edges.append({"source": pid, "target": cid, "type": "MENTIONS"})
+
+        for claim in (paper.get("claims") or [])[:1]:
+            cid = f"claim_{i}_{slugify(claim)[:24]}"
+            nodes.append({"id": cid, "label": claim[:42], "kind": "claim"})
+            edges.append({"source": pid, "target": cid, "type": "ASSERTS"})
+
+    if uploaded_name:
+        nodes.append({"id": "upload", "label": uploaded_name, "kind": "upload"})
+        edges.append({"source": "query", "target": "upload", "type": "UPLOADED_SOURCE"})
+
+        if parsed_state and isinstance(parsed_state, dict):
+            for concept in (parsed_state.get("concepts") or [])[:3]:
+                cid = f"upload_concept_{slugify(concept)[:24]}"
+                nodes.append({"id": cid, "label": concept, "kind": "concept"})
+                edges.append({"source": "upload", "target": cid, "type": "MENTIONS"})
+
+    return nodes, edges
+
+
+def parse_pdf_with_grobid(pdf_path):
+    if not GROBID_URL:
+        raise RuntimeError("GROBID_URL is not set")
+
+    with open(pdf_path, "rb") as f:
+        files = {"input": (Path(pdf_path).name, f, "application/pdf")}
+        response = requests.post(
+            f"{GROBID_URL.rstrip('/')}/api/processFulltextDocument",
+            files=files,
+            data={"includeRawAffiliations": "1", "segmentSentences": "1"},
+            timeout=120,
+        )
+
+    response.raise_for_status()
+    tei_xml = response.text
+    root = ET.fromstring(tei_xml)
+    ns = {"tei": "http://www.tei-c.org/ns/1.0"}
+
+    title = root.findtext(".//tei:titleStmt/tei:title", default="", namespaces=ns)
+    if not title:
+        title = Path(pdf_path).name
+
+    abstract_parts = []
+    for p in root.findall(".//tei:profileDesc/tei:abstract//tei:p", ns):
+        abstract_parts.append(" ".join(list(p.itertext())))
+    abstract = norm_text(" ".join(abstract_parts))
+
+    authors = []
+    for author in root.findall(".//tei:sourceDesc//tei:author", ns):
+        parts = []
+        forename = author.findall(".//tei:forename", ns)
+        surname = author.findall(".//tei:surname", ns)
+        parts.extend([norm_text(" ".join(x.itertext())) for x in forename])
+        parts.extend([norm_text(" ".join(x.itertext())) for x in surname])
+        name = norm_text(" ".join(parts))
+        if name:
+            authors.append(name)
+
+    sections = []
+    text_pool = []
+    for div in root.findall(".//tei:text//tei:body//tei:div", ns):
+        head = div.findtext("./tei:head", default="", namespaces=ns)
+        paras = []
+        for p in div.findall(".//tei:p", ns):
+            para_text = norm_text(" ".join(list(p.itertext())))
+            if para_text:
+                paras.append(para_text)
+        joined = "\n".join(paras)
+        if head or joined:
+            sections.append({"heading": head or "Section", "text": joined[:4000]})
+            if joined:
+                text_pool.append(joined)
+
+    references = []
+    for bibl in root.findall(".//tei:listBibl//tei:biblStruct", ns)[:40]:
+        ref_title = bibl.findtext(".//tei:title", default="", namespaces=ns)
+        ref_doi = ""
+        for idno in bibl.findall(".//tei:idno", ns):
+            if (idno.attrib.get("type") or "").lower() == "doi":
+                ref_doi = norm_text(" ".join(idno.itertext()))
+                break
+        references.append({"title": norm_text(ref_title), "doi": normalize_doi(ref_doi)})
+
+    semantic_text = " ".join([abstract] + text_pool[:4])
+
+    return {
+        "parser": "grobid",
+        "title": norm_text(title),
+        "abstract": abstract,
+        "authors": authors[:12],
+        "sections": sections[:12],
+        "references": references[:40],
+        "claims": extract_claim_like_sentences(semantic_text, max_items=GRAPH_MAX_CLAIMS),
+        "concepts": extract_concepts_from_text(semantic_text, max_terms=GRAPH_MAX_CONCEPTS),
+        "raw_text": "",
+    }
+
+
+def parse_pdf_with_pymupdf(pdf_path):
+    if fitz is None:
+        raise RuntimeError("PyMuPDF not installed")
+
+    doc = fitz.open(pdf_path)
+    raw_text = "\n".join(page.get_text("text") for page in doc).strip()
+    first_page = raw_text[:4000]
+    lines = [x.strip() for x in first_page.splitlines() if x.strip()]
+    title = lines[0][:300] if lines else Path(pdf_path).name
+
+    abstract = ""
+    match = re.search(r"abstract\s*(.+?)(?:\n\s*\n|\n1[\.\s]|introduction)", raw_text, re.I | re.S)
+    if match:
+        abstract = norm_text(match.group(1))[:2500]
+
+    return {
+        "parser": "pymupdf",
+        "title": title,
+        "abstract": abstract,
+        "authors": [],
+        "sections": [{"heading": "Full Text", "text": raw_text[:12000]}] if raw_text else [],
+        "references": [],
+        "claims": extract_claim_like_sentences(raw_text, max_items=GRAPH_MAX_CLAIMS),
+        "concepts": extract_concepts_from_text(raw_text, max_terms=GRAPH_MAX_CONCEPTS),
+        "raw_text": raw_text[:50000],
+    }
+
+
+def parse_pdf_with_docling(pdf_path):
+    try:
+        from docling.document_converter import DocumentConverter
+    except Exception as e:
+        raise RuntimeError(f"Docling import failed: {e}")
+
+    converter = DocumentConverter()
+    result = converter.convert(pdf_path)
+    doc = result.document
+    markdown = doc.export_to_markdown()
+
+    title = Path(pdf_path).name
+    first_nonempty = next((line.strip("# ").strip() for line in markdown.splitlines() if line.strip()), "")
+    if first_nonempty:
+        title = first_nonempty[:300]
+
+    return {
+        "parser": "docling",
+        "title": title,
+        "abstract": "",
+        "authors": [],
+        "sections": [{"heading": "Document", "text": markdown[:12000]}] if markdown else [],
+        "references": [],
+        "claims": extract_claim_like_sentences(markdown, max_items=GRAPH_MAX_CLAIMS),
+        "concepts": extract_concepts_from_text(markdown, max_terms=GRAPH_MAX_CONCEPTS),
+        "raw_text": markdown[:50000],
+    }
+
+
+def parse_uploaded_pdf(file_obj, parser_order):
+    if not file_obj:
+        return "### PDF parse status\n\nNo PDF uploaded yet.", {}
+
+    path = getattr(file_obj, "name", None) or str(file_obj)
+    parser_order = ensure_list(parser_order) or ["grobid", "docling", "pymupdf"]
+    errors = []
+
+    for parser_name in parser_order:
+        try:
+            if parser_name == "grobid":
+                result = parse_pdf_with_grobid(path)
+            elif parser_name == "docling":
+                result = parse_pdf_with_docling(path)
+            elif parser_name == "pymupdf":
+                result = parse_pdf_with_pymupdf(path)
+            else:
+                continue
+
+            summary = (
+                f"### PDF parse status\n\n"
+                f"- Parser used: {result['parser']}\n"
+                f"- Title: {result.get('title') or 'Unknown'}\n"
+                f"- Authors: {', '.join(result.get('authors')[:6]) if result.get('authors') else 'Unknown'}\n"
+                f"- Abstract found: {'Yes' if result.get('abstract') else 'No'}\n"
+                f"- Sections extracted: {len(result.get('sections') or [])}\n"
+                f"- References extracted: {len(result.get('references') or [])}\n"
+                f"- Concepts extracted: {len(result.get('concepts') or [])}\n"
+                f"- Claims extracted: {len(result.get('claims') or [])}\n"
+            )
+            return summary, result
+        except Exception as e:
+            errors.append(f"{parser_name}: {e}")
+
+    fail_summary = "### PDF parse status\n\n" + "\n".join([f"- {x}" for x in errors])
+    return fail_summary, {"parser": None, "errors": errors}
+
+
+def render_parse_result(parsed):
+    if not parsed or not isinstance(parsed, dict) or (not parsed.get("title") and not parsed.get("sections")):
+        return '<div class="panel" style="padding:18px"><p>No parsed document yet.</p></div>'
+
+    sections_html = []
+    for section in parsed.get("sections", [])[:6]:
+        sections_html.append(
+            f"""
+            <details class="agent-step">
+              <summary class="agent-summary">
+                <div class="agent-index">§</div>
+                <div class="agent-head">
+                  <h4>{safe_text(section.get('heading', 'Section'))}</h4>
+                  <span>section</span>
+                </div>
+              </summary>
+              <div class="agent-copy">
+                <p>{safe_text(section.get('text', '')[:1800])}</p>
+              </div>
+            </details>
+            """
+        )
+
+    refs = parsed.get("references", [])[:12]
+    refs_html = "".join(
+        f"<li>{safe_text(r.get('title') or 'Untitled')} {'· DOI ' + safe_text(r.get('doi')) if r.get('doi') else ''}</li>"
+        for r in refs
+    ) or "<li>No references extracted.</li>"
+
+    concepts = parsed.get("concepts", [])[:10]
+    claims = parsed.get("claims", [])[:6]
+    concepts_html = "".join(f"<li>{safe_text(x)}</li>" for x in concepts) or "<li>No concepts extracted.</li>"
+    claims_html = "".join(f"<li>{safe_text(x)}</li>" for x in claims) or "<li>No claims extracted.</li>"
+
+    title = safe_text(parsed.get("title") or "Parsed document")
+    abstract = safe_text((parsed.get("abstract") or "")[:2400]) or "No abstract extracted."
+    parser_name = safe_text(parsed.get("parser") or "unknown")
+
+    return f"""
+    <div class="panel" style="padding:18px">
+      <div class="brain-header">
+        <div>
+          <p class="eyebrow">PDF Parse</p>
+          <h3>{title}</h3>
+        </div>
+        <div class="brain-legend"><span><i class="dot dot-upload"></i> {parser_name}</span></div>
+      </div>
+      <div class="parse-grid">
+        <div class="parse-card">
+          <h4>Abstract</h4>
+          <p>{abstract}</p>
+        </div>
+        <div class="parse-card">
+          <h4>References</h4>
+          <ul class="ref-list">{refs_html}</ul>
+        </div>
+        <div class="parse-card">
+          <h4>Concepts</h4>
+          <ul class="ref-list">{concepts_html}</ul>
+        </div>
+        <div class="parse-card">
+          <h4>Claims</h4>
+          <ul class="ref-list">{claims_html}</ul>
+        </div>
+      </div>
+      <div class="timeline" style="margin-top:14px;">
+        {''.join(sections_html) if sections_html else '<div class="panel" style="padding:16px;"><p>No sections extracted.</p></div>'}
+      </div>
+    </div>
+    """
+
+
+def add_node(nodes_by_id: Dict[str, Dict], node_id: str, node_type: str, label: str = "", **attrs):
+    if not node_id:
+        return
+    current = nodes_by_id.get(node_id, {})
+    merged = {"id": node_id, "type": node_type, "label": label or current.get("label", node_id)}
+    merged.update(current)
+    for key, value in attrs.items():
+        if value not in [None, ""]:
+            merged[key] = value
+    nodes_by_id[node_id] = merged
+
+
+def add_edge(edges: List[Dict], source: str, target: str, edge_type: str, **attrs):
+    if not source or not target or source == target:
+        return
+    edge = {"source": source, "target": target, "type": edge_type}
+    for key, value in attrs.items():
+        if value not in [None, ""]:
+            edge[key] = value
+    edges.append(edge)
+
+
+def build_ingest_payload(query, selected_papers, parsed_pdf=None):
+    nodes_by_id = {}
+    edges = []
+
+    topic_id = "topic:query"
+    add_node(nodes_by_id, topic_id, "Topic", label=query or "Research topic", query=query or "")
+
+    for i, paper in enumerate(selected_papers, start=1):
+        paper_id = normalize_doi(paper.get("doi")) or (paper.get("external_ids") or {}).get("arxiv") or f"paper:{i}:{slugify(paper.get('title', 'paper'))[:32]}"
+        add_node(
+            nodes_by_id,
+            paper_id,
+            "Paper",
+            label=paper.get("title") or f"Paper {i}",
+            title=paper.get("title"),
+            year=paper.get("year"),
+            venue=paper.get("venue"),
+            doi=normalize_doi(paper.get("doi")),
+            source=paper.get("source"),
+            url=paper.get("url"),
+            pdf=paper.get("pdf"),
+            score=paper.get("score"),
+            learned_score=paper.get("learned_score", paper.get("score")),
+            open_access=paper.get("open_access"),
+        )
+        add_edge(edges, topic_id, paper_id, "ABOUT", weight=paper.get("learned_score", paper.get("score", 0)))
+
+        for author in paper.get("authors", [])[:6]:
+            author_id = f"author:{slugify(author)[:64]}"
+            add_node(nodes_by_id, author_id, "Author", label=author, name=author)
+            add_edge(edges, paper_id, author_id, "WRITTEN_BY")
+
+        for concept in (paper.get("concepts") or [])[:6]:
+            concept_id = f"concept:{slugify(concept)[:72]}"
+            add_node(nodes_by_id, concept_id, "Concept", label=concept, name=concept)
+            add_edge(edges, paper_id, concept_id, "MENTIONS")
+
+        for claim in (paper.get("claims") or [])[:3]:
+            claim_id = f"claim:{slugify(claim)[:72]}"
+            add_node(nodes_by_id, claim_id, "Claim", label=claim[:120], text=claim)
+            add_edge(edges, paper_id, claim_id, "ASSERTS")
+
+    if parsed_pdf and isinstance(parsed_pdf, dict) and parsed_pdf.get("title"):
+        doc_id = "upload:pdf"
+        add_node(nodes_by_id, doc_id, "UploadedPDF", label=parsed_pdf.get("title"), title=parsed_pdf.get("title"), parser=parsed_pdf.get("parser"))
+        add_edge(edges, topic_id, doc_id, "UPLOADED_SOURCE")
+
+        for concept in (parsed_pdf.get("concepts") or [])[:6]:
+            concept_id = f"concept:{slugify(concept)[:72]}"
+            add_node(nodes_by_id, concept_id, "Concept", label=concept, name=concept)
+            add_edge(edges, doc_id, concept_id, "MENTIONS")
+
+        for claim in (parsed_pdf.get("claims") or [])[:4]:
+            claim_id = f"claim:{slugify(claim)[:72]}"
+            add_node(nodes_by_id, claim_id, "Claim", label=claim[:120], text=claim)
+            add_edge(edges, doc_id, claim_id, "ASSERTS")
+
+        for idx, ref in enumerate(parsed_pdf.get("references", [])[:12], start=1):
+            ref_title = ref.get("title") or f"Reference {idx}"
+            ref_doi = normalize_doi(ref.get("doi") or "")
+            ref_id = ref_doi or f"ref:{idx}:{slugify(ref_title)[:32]}"
+            add_node(nodes_by_id, ref_id, "Reference", label=ref_title, title=ref_title, doi=ref_doi)
+            add_edge(edges, doc_id, ref_id, "CITES")
+
+    return {"status": "ok", "nodes": list(nodes_by_id.values()), "edges": edges}
+
+
+def learn_from_payload(payload: Dict, query: str = "") -> Dict:
+    if not payload:
+        return GRAPH_MEMORY
+
+    GRAPH_MEMORY["queries"].append(query or "")
+    GRAPH_MEMORY["events"].append({
+        "ts": time.time(),
+        "query": query or "",
+        "nodes": len(payload.get("nodes", [])),
+        "edges": len(payload.get("edges", [])),
+    })
+
+    for node in payload.get("nodes", []):
+        node_id = node.get("id")
+        if not node_id:
+            continue
+        GRAPH_MEMORY["nodes"][node_id] = node
+
+        node_type = (node.get("type") or "").lower()
+        if node_type == "paper":
+            GRAPH_MEMORY["papers"][node_id] = node
+        if node_type == "concept" and node.get("label"):
+            GRAPH_MEMORY["concept_counts"][node["label"].lower()] += 1
+        if node_type == "claim" and node.get("label"):
+            GRAPH_MEMORY["claim_counts"][node["label"].lower()] += 1
+
+    GRAPH_MEMORY["edges"].extend(payload.get("edges", []))
+    return GRAPH_MEMORY
+
+
+def resolve_selected_papers(selected_indices, papers_state):
+    papers = ensure_list(papers_state)
+    selected_indices = ensure_list(selected_indices)
+    selected = []
+
+    if not selected_indices:
+        return selected
+
+    value_map = {paper_choice_value(i, paper): paper for i, paper in enumerate(papers)}
+    label_map = {paper_choice_label(i, paper): paper for i, paper in enumerate(papers)}
+
+    for idx in selected_indices:
+        try:
+            if isinstance(idx, int):
+                if 0 <= idx < len(papers):
+                    selected.append(papers[idx])
+                continue
+
+            idx_str = str(idx)
+            if idx_str in value_map:
+                selected.append(value_map[idx_str])
+                continue
+
+            if idx_str.isdigit():
+                num = int(idx_str)
+                if 0 <= num < len(papers):
+                    selected.append(papers[num])
+                    continue
+
+            if "|" in idx_str:
+                left = idx_str.split("|", 1)[0]
+                if left.isdigit():
+                    num = int(left)
+                    if 0 <= num < len(papers):
+                        selected.append(papers[num])
+                        continue
+
+            if idx_str in label_map:
+                selected.append(label_map[idx_str])
+                continue
+        except Exception:
+            continue
+
+    out = []
+    seen = set()
+    for paper in selected:
+        key = paper_identity_key(paper)
+        if key not in seen:
+            seen.add(key)
+            out.append(paper)
+    return out
+
+
+def summarize_learning_state(query_text, papers, selected_sources):
+    concept_pool = []
+    for paper in papers[:8]:
+        concept_pool.extend((paper.get("concepts") or [])[:3])
+
+    top_concepts = [c for c, _ in Counter([c.lower() for c in concept_pool]).most_common(6)]
+
+    return (
+        "### Discovery results\n\n"
+        f"- Query: {query_text}\n"
+        f"- Sources: {', '.join(selected_sources)}\n"
+        f"- Candidates found: {len(papers)}\n"
+        f"- Top learned concepts: {', '.join(top_concepts) if top_concepts else 'None'}\n"
+        "- Select papers below, then click **Ingest selected into graph**.\n"
+    )
+
+
+def run_paper_discovery(query, search_mode, sources, pdf_file):
+    query_text = norm_text(query or "")
+    selected_sources = ensure_list(sources) or DEFAULT_SOURCES
+
+    if not query_text and not pdf_file:
+        empty_graph = build_learning_graph_html([], [], "Self-Learning Knowledge Graph")
+        return (
+            empty_graph,
+            '<div class="panel papers-panel" style="padding:18px"><p>Enter a topic, title, DOI, link, or upload a PDF to start learning.</p></div>',
+            build_journal_html("biomaterials cardiac repair"),
+            "No PDF uploaded yet.",
+            gr.update(choices=[], value=[]),
+            [],
+            "### No discovery results yet.",
+        )
+
+    if not query_text and pdf_file:
+        uploaded_name = Path(getattr(pdf_file, "name", str(pdf_file))).name
+        graph_nodes, graph_edges = build_learning_graph_state("", [], uploaded_name)
+        return (
+            build_learning_graph_html(graph_nodes, graph_edges, "Uploaded PDF Waiting for Parse"),
+            '<div class="panel papers-panel" style="padding:18px"><p>No query yet. Parse the uploaded PDF or enter a research topic to begin discovery.</p></div>',
+            build_journal_html("biomaterials cardiac repair"),
+            uploaded_pdf_summary(pdf_file),
+            gr.update(choices=[], value=[]),
+            [],
+            "### Upload detected.\n\n- Parse the PDF to extract structure.\n- Or enter a topic to start discovery.",
+        )
+
+    uploaded_name = Path(getattr(pdf_file, "name", str(pdf_file))).name if pdf_file else None
+
+    try:
+        papers = discover_papers(query_text, search_mode, selected_sources, max_results=GRAPH_MAX_RESULTS)
+        graph_nodes, graph_edges = build_learning_graph_state(query_text, papers[:6], uploaded_name)
+        graph_html = build_learning_graph_html(graph_nodes, graph_edges, "Self-Learning Knowledge Graph")
+        papers_html = format_papers_html(papers)
+        journals_html = build_journal_html(query_text or "biomaterials cardiac repair")
+        pdf_summary = uploaded_pdf_summary(pdf_file)
+        choices = format_selection_choices(papers)
+        status_md = summarize_learning_state(query_text, papers, selected_sources)
+
+        return (
+            graph_html,
+            papers_html,
+            journals_html,
+            pdf_summary,
+            gr.update(choices=choices, value=[]),
+            papers,
+            status_md,
+        )
+
+    except Exception as e:
+        graph_nodes, graph_edges = build_learning_graph_state(query_text, [], uploaded_name)
+        error_html = f'<div class="panel papers-panel" style="padding:18px"><p>Paper search failed: {safe_text(str(e))}</p></div>'
+        return (
+            build_learning_graph_html(graph_nodes, graph_edges),
+            error_html,
+            build_journal_html(query_text or "biomaterials cardiac repair"),
+            uploaded_pdf_summary(pdf_file),
+            gr.update(choices=[], value=[]),
+            [],
+            f"### Discovery failed.\n\n- Error: {safe_text(str(e))}",
+        )
+
+
+def ingest_selected_papers(query, selected_indices, papers_state, pdf_file, parsed_state):
+    papers = ensure_list(papers_state)
+    selected = resolve_selected_papers(selected_indices, papers)
+    uploaded_name = Path(getattr(pdf_file, "name", str(pdf_file))).name if pdf_file else None
+
+    if not selected and parsed_state and isinstance(parsed_state, dict) and parsed_state.get("title") and papers:
+        selected = papers[:3]
+
+    if not selected and not (parsed_state and isinstance(parsed_state, dict) and parsed_state.get("title")):
+        graph_html = build_learning_graph_html([], [], "Self-Learning Knowledge Graph")
+        return (
+            graph_html,
+            "### Graph ingest status\n\nSelect papers or parse an uploaded PDF first.",
+            {"status": "empty", "nodes": [], "edges": []},
+        )
+
+    query_text = norm_text(query or "")
+    if not query_text and isinstance(parsed_state, dict):
+        query_text = parsed_state.get("title") or "Research topic"
+    if not query_text:
+        query_text = "Research topic"
+
+    selected = [enrich_paper_semantics(query_text, paper) for paper in selected]
+    graph_nodes, graph_edges = graph_from_selected(query_text, selected, uploaded_name, parsed_state if isinstance(parsed_state, dict) else None)
+    graph_html = build_learning_graph_html(graph_nodes, graph_edges, "Selected Research Graph")
+
+    payload = build_ingest_payload(query_text, selected, parsed_state if isinstance(parsed_state, dict) else None)
+    learn_from_payload(payload, query=query_text)
+
+    top_concepts = []
+    for paper in selected:
+        top_concepts.extend((paper.get("concepts") or [])[:3])
+    if isinstance(parsed_state, dict):
+        top_concepts.extend((parsed_state.get("concepts") or [])[:3])
+
+    summary_lines = [
+        "### Graph ingest status",
+        "",
+        f"- Topic: {query_text}",
+        f"- Selected papers: {len(selected)}",
+        f"- Uploaded PDF parsed: {'Yes' if parsed_state and isinstance(parsed_state, dict) and parsed_state.get('title') else 'No'}",
+        f"- Nodes created: {len(payload['nodes'])}",
+        f"- Edges created: {len(payload['edges'])}",
+        f"- Learned concepts: {', '.join(unique_keep_order(top_concepts)[:8]) if top_concepts else 'None'}",
+        f"- Memory papers stored: {len(GRAPH_MEMORY['papers'])}",
+        f"- Memory concepts stored: {len(GRAPH_MEMORY['concept_counts'])}",
+    ]
+
+    return graph_html, "\n".join(summary_lines), payload