Upload project files

agents.py

@@ -0,0 +1,37 @@
+"""agents.py — Supervisor + 4 workers. Zero if/else/for/while/try/except."""
+import os
+
+def build_agent():
+    """Build the full supervisor graph. Called once at startup."""
+    from langchain_openai import ChatOpenAI
+    from langgraph.prebuilt import create_react_agent
+    from langgraph_supervisor import create_supervisor
+    from langgraph.checkpoint.memory import MemorySaver
+    from tools import search_openalex, search_tavily, search_scopus, validate_papers, run_bertopic, upload_to_storage
+
+    llm = ChatOpenAI(
+        model="Qwen/Qwen2.5-72B-Instruct",
+        base_url="https://router.huggingface.co/v1/",
+        api_key=os.getenv("HF_TOKEN"),
+        temperature=0.01,
+    )
+
+    oa = create_react_agent(llm, tools=[search_openalex], name="openalex_agent",
+        prompt="You search OpenAlex. Extract BOTH the query and the chat_id from the user prompt. Call your tool once with the query and chat_id, return only the raw output, do nothing else.")
+    tv = create_react_agent(llm, tools=[search_tavily], name="tavily_agent",
+        prompt="You search Tavily. Extract BOTH the query and the chat_id from the user prompt. Call your tool once with the query and chat_id, return only the raw output, do nothing else.")
+    sc = create_react_agent(llm, tools=[search_scopus], name="scopus_agent",
+        prompt="You search Scopus. Extract BOTH the query and the chat_id from the user prompt. Call your tool once with the query and chat_id, return only the raw output, do nothing else.")
+    val = create_react_agent(llm, tools=[validate_papers], name="validation_agent",
+        prompt="You validate papers and check if they are relevant to the original query. Extract BOTH the query and the chat_id from the user prompt. Call your tool once with the query and chat_id, return only the raw output, do nothing else.")
+    an = create_react_agent(llm, tools=[run_bertopic, upload_to_storage], name="analysis_agent",
+        prompt="You run analysis. Extract the chat_id from the user prompt. First call run_bertopic with the chat_id, then call upload_to_storage with the chat_id. Return only the raw output, do nothing else.")
+
+    workflow = create_supervisor(
+        [oa, tv, sc, val, an], model=llm,
+        prompt=("You are a research supervisor. For every query, you must run all 5 agents sequentially in this exact order: "
+                "1) openalex_agent 2) tavily_agent 3) scopus_agent 4) validation_agent 5) analysis_agent. "
+                "Always use ALL 5 agents."),
+        output_mode="full_history")
+
+    return workflow.compile(checkpointer=MemorySaver())

app.py

@@ -0,0 +1,113 @@
+"""app.py — Gradio UI for BERTopic Multi-Agent Research. Zero if/else/for/while/try/except."""
+import sys, os; sys.stdout.reconfigure(line_buffering=True)
+from dotenv import load_dotenv
+load_dotenv()
+
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+import json, glob
+print(">>> importing gradio...", flush=True)
+import gradio as gr
+print(">>> importing agents...", flush=True)
+from agents import build_agent
+from tools import PAPER_CACHE, OUTPUT_DIR, supabase
+print(">>> building agent...", flush=True)
+agent = build_agent()
+_msg_count = 0
+print(">>> agent ready!", flush=True)
+
+def _pipeline(phase):
+    phases = [("① Load", 1), ("② Codes", 2), ("③ Themes", 3), ("④ Review", 4), ("⑤ Names", 5), ("⑤½ PAJAIS", 5.5), ("⑥ Report", 6)]
+    return " → ".join(list(map(lambda p: f"**{p[0]}**" if p[1]==phase else (f"✅ {p[0]}" if p[1]<phase else p[0]), phases)))
+
+def _topic_rows(chat_id=None):
+    res = supabase.table("chats").select("topics_json").eq("id", chat_id).execute().data if chat_id else []
+    tops = res[0].get("topics_json") if res and res[0].get("topics_json") else []
+    return list(map(lambda t: [t["id"], t["label"], "; ".join(t.get("top_sentences",[])[:1])[:100], t["count"], len(t.get("top_papers",[])), "yes", "", ""], tops))
+
+def _history():
+    return list(map(lambda r: f"[{r['id']}] {r['title']}", supabase.table("chats").select("id,title").order("created_at", desc=True).limit(20).execute().data))
+
+def _latest_files():
+    return sorted(glob.glob(os.path.join(OUTPUT_DIR, "*")), key=os.path.getmtime, reverse=True)[:10] or None
+
+def respond(message, chat_history):
+    global _msg_count; _msg_count += 1
+    text = (message or "").strip() or "digital social innovation and societal impact"
+    chat_id = supabase.table("chats").insert({"title": text[:50], "user_message": text, "bot_message": "Started..."}).execute().data[0]["id"]
+    chat_history = chat_history + [{"role":"user","content":text}, {"role":"assistant","content":"🔄 **Dispatching agents...**\n\nSupervisor → OpenAlex → Tavily → Scopus → Validation → BERTopic Analysis\n\n_This may take 30-60 seconds..._"}]
+    yield chat_history, "", _pipeline(2), _topic_rows(chat_id), _latest_files(), chat_id
+    result = agent.invoke({"messages":[{"role":"user","content":f"Research: {text}. The chat_id is {chat_id}. Search all databases, validate, run BERTopic, upload."}]}, config={"configurable":{"thread_id":f"t{_msg_count}"}})
+    chat_history[-1] = {"role":"assistant","content":result["messages"][-1].content}
+    yield chat_history, "", _pipeline(6), _topic_rows(chat_id), _latest_files(), chat_id
+
+def submit_review(td):
+    edits = list(map(lambda r: f"Topic {r[0]}: {r[5]}, Rename='{r[6]}'", filter(lambda r: str(r[5]).lower()=="no" or str(r[6]).strip()!="", td)))
+    return "Review:\n" + "\n".join(edits)
+
+def load_chart(name):
+    path = os.path.join(OUTPUT_DIR, str(name or ""))
+    fallback = "<div style='text-align:center;color:#64748b;padding:60px;background:#fff;border-radius:8px'>📊 Run a search first to generate BERTopic charts</div>"
+    return {True: lambda: "<iframe srcdoc='" + open(path,"r",encoding="utf-8").read().replace("'",'"') + "' width='100%' height='480' frameborder='0'></iframe>", False: lambda: fallback}[os.path.exists(path)]()
+
+print(">>> fetching history...", flush=True)
+
+def show_topic_papers(evt: gr.SelectData, chat_id_state):
+    if not chat_id_state: return []
+    row = evt.index[0]
+    chat = supabase.table("chats").select("topics_json").eq("id", chat_id_state).execute().data[0]
+    tops = chat.get("topics_json", []) if chat.get("topics_json") else []
+    if row >= len(tops): return []
+    papers = supabase.table("papers").select("title,web_link,date_of_publication,journal,no_of_citations,confidence_score").eq("topic_label", tops[row]["label"]).eq("chat_id", chat_id_state).execute().data
+    return list(map(lambda p: [p.get("title", ""), p.get("web_link", ""), p.get("date_of_publication", ""), p.get("journal", ""), p.get("no_of_citations", ""), p.get("confidence_score", "")], papers))
+
+hist = _history()
+print(f">>> {len(hist)} past sessions", flush=True)
+
+print(">>> building UI...", flush=True)
+with gr.Blocks(title="BERTopic — Digital Social Innovation") as demo:
+
+    gr.Markdown("# 🔬 Topic Modelling — Agentic AI\n*Qwen 72B 🧠 · MiniLM Embeddings · Cosine Clustering · 384d · Braun & Clarke Thematic Analysis*")
+    
+    pipeline_html = gr.Markdown(_pipeline(1))
+    gr.Markdown("💡 **Multi-Agent Architecture:** Supervisor Agent orchestrates 4 workers (OpenAlex, Tavily, Scopus, Validation) + 1 analysis worker (BERTopic embedding → clustering → LLM labeling → Supabase upload)")
+
+
+    chat_state = gr.State(None)
+    chatbot = gr.Chatbot(height=320, show_label=False)
+
+    with gr.Row():
+        msg = gr.Textbox(placeholder="e.g. 'digital social innovation and societal impact'", show_label=False, scale=9, container=False)
+        send = gr.Button("Send", scale=1, min_width=70)
+
+    with gr.Row():
+        cmd1 = gr.Button("▶ Digital Social Innovation", size="sm")
+        cmd2 = gr.Button("▶ AI in Healthcare", size="sm")
+        cmd3 = gr.Button("▶ Sustainable Tourism Tech", size="sm")
+
+    with gr.Tabs():
+        with gr.TabItem("📋 Review Table"):
+            review_table = gr.Dataframe(headers=["#","Topic Label","Top Evidence","Sents","Papers","Approve","Rename To","Reasoning"], datatype=["number","str","str","number","number","str","str","str"], interactive=True)
+            submit_btn = gr.Button("✅ Submit Review to Agent")
+            review_out = gr.Textbox(label="Review Status", interactive=False)
+            gr.Markdown("🛑 **STOP Gate** — Agent pauses here. Review each BERTopic cluster label. Edit Approve/Rename columns → click Submit.")
+            gr.Markdown("📄 **Papers in Selected Topic**")
+            paper_table = gr.Dataframe(headers=["Title", "Source", "Year", "Journal", "Citations", "Relevance Score"], interactive=False)
+        with gr.TabItem("📊 Charts"):
+            chart_dd = gr.Dropdown(choices=["rq4_abstract_bars.html","rq4_abstract_heatmap.html","rq4_abstract_intertopic.html"], value="rq4_abstract_bars.html", label="Select Visualization")
+            chart_html = gr.HTML("<div style='text-align:center;color:#64748b;padding:60px;background:#fff;border-radius:8px'>📊 Charts appear here after BERTopic analysis</div>")
+        with gr.TabItem("📥 Downloads"):
+            download_files = gr.File(label="Output Files (summaries.json, emb.npy, charts, CSV)", file_count="multiple", interactive=False)
+
+    history_dd = gr.Dropdown(choices=hist, label="📚 Past Research Sessions (Supabase)", interactive=True)
+
+    msg.submit(respond, [msg, chatbot], [chatbot, msg, pipeline_html, review_table, download_files, chat_state])
+    send.click(respond, [msg, chatbot], [chatbot, msg, pipeline_html, review_table, download_files, chat_state])
+    submit_btn.click(submit_review, [review_table], [review_out])
+    review_table.select(show_topic_papers, inputs=[chat_state], outputs=[paper_table])
+    chart_dd.change(load_chart, [chart_dd], [chart_html])
+    cmd1.click(lambda: "digital social innovation and societal impact", outputs=[msg])
+    cmd2.click(lambda: "artificial intelligence in healthcare diagnosis", outputs=[msg])
+    cmd3.click(lambda: "sustainable tourism technology adoption", outputs=[msg])
+
+print(">>> launching...", flush=True)
+demo.launch(server_name="0.0.0.0")

glass-sequence-432208-n3-eb48e1d54468.json

@@ -0,0 +1,13 @@
+{
+  "type": "service_account",
+  "project_id": "glass-sequence-432208-n3",
+  "private_key_id": "eb48e1d54468998f0851dd150370ae35150029c9",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC/eBKxDZ66fxLU\nA1eH8S/81+lJxd/AnJXqaDyHvGhATRvg1CWwdUOrU4szhewdMww0jnP4QU4oXX9A\neXLOzouHoiU9GvtMx6ClKNHP/+JIddGSu32y+yABW8LgtnIH8z4L+PNalWCxYzaR\nYo3uaRZoQKbTq+8GaQsvpPPvM7hRi56KmUWFDFkp89k3AMJFDYg8eWi5ncMMyGtu\n+wO+ln27bFCHlBLfZQNPqO+7l1oyfO8kakhOUtpMYaO0QzamlNJ1ACaAq6h3pJbl\n2ql+d/7iLDFabz8QwcYVML3IbqDyhnSslKfvSpqA3tb1mer2fIBajXljpuKUH5H/\nGaYX5EIHAgMBAAECggEABScSZwi1IdMI9a2qrbI0QWX/at3UBKycwt8lPzM3Iavc\nOTOxrqBT6Yi+W+zvCrt0nkwgcNOJFUBW4Vb4sPoRT91xinBvHNoucji5VL36dUq+\nZhLSR35x67dwH3ipdESmgc9O9TwZ4GGJdrt5PauUT8qVnWf+GbplSQCeaUHcuaJa\nqPsghsR5tcOxBfvPrnHX620GyXY3AGXf05aIi8EZVAb6/HaCnmsdQkVdbEdWrNWU\ncZHIWzHEa353QPX26t2IkE9hhhF0yneyLWAzOfeJemCjKT/tq66sJxhKsYFPNFLp\nB32xIivAWRrC5yMMRtpDAiNxSiSTIiGJW3hEUpNLiQKBgQDkJFc9nZPLeBbYNjAE\ndznu6M+XUPuhZrOa8swHvfkBLB0YtHI9D3rPlYiaDG2pXFNp7dKusfoJTJ/v//Gu\nkK4FNml4D5g2h/21rtxEY0X+VF0oObTObU1lUEjid1XSEVHoC27q2bi1844IG2dW\nu26qc3ScK2KMbGXkd9lFmsEnawKBgQDW2ViZ2IUcx6yy9V9UVog2S0CZpdbBIWHC\nxg/nAqaFvAUXfPwh1XgoyOPBAN6nYGgjKl3XHDVDAJiD746ohw4G/Uc15nVlE7ld\nmYuvi4duo9roiIwl5KzE9TmoDyYn6UTNsjgVEP140lgEI1AhSNyykpn8xyorR/87\nSuw4A6bi1QKBgFhbksY0T1dQTVizs+km2/Mhi7TJEouyIQFz431kIBpKIYK98Dih\n1Cite1heU+Jx9YRjLWOnXnVXjeH/uHSpaBxwtGxhbcxA4wwIixqZcDROLD42C0af\noogIjhHXsg9H0fcS5oRiil091NZi9qTqtrOhP5p0WxiTqurPWADTLNe9AoGAdn2I\nD9EKGMVGyavp12mh1c/K4PJFbl4qvHDfdFhR3o/DfyC/LSm+C0A/28HPCiBxTnOf\nLXOxLDK36ovD8c4KPnCRRXjEZepq4OQ91aLuHmbf5D7qjDuBzV4xXi58rKZJ++e+\nZl/oayoWwLF5D+JPVWzk4DQ3XYdnjBvWC054S8ECgYEAv1x5xSJNMGZhFBfy6i/Q\np+/XEtPZGnCKLLAgkq6Ck86pquNxse0KUocfywrLULDB2MQwDsnNFapPfHDAwRqV\nSbvbnSfEV/I9ekUdciA1mdVKKxnZs8F5y27VH6gnt+cfYNUYbgDH2uR8vt/8g9t8\nwbjgq9UBGhDO9UCZoXLV+MA=\n-----END PRIVATE KEY-----\n",
+  "client_email": "viraj-576@glass-sequence-432208-n3.iam.gserviceaccount.com",
+  "client_id": "104895996394977534596",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/viraj-576%40glass-sequence-432208-n3.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}

requirements.txt

@@ -0,0 +1,19 @@
+gradio==5.16.1
+langgraph==0.2.73
+langchain==0.3.19
+langchain-core==0.3.38
+langchain-groq==0.2.4
+langchain-huggingface==0.1.2
+langgraph-supervisor==0.0.31
+supabase==2.11.0
+gspread==6.1.4
+google-auth==2.38.0
+tavily-python==0.5.1
+sentence-transformers==3.4.1
+scikit-learn==1.6.1
+plotly==6.0.0
+pandas==2.2.3
+requests==2.32.3
+numpy==2.2.3
+
+python-dotenv

tools.py

@@ -0,0 +1,256 @@
+"""tools.py — Multi-agent BERTopic tools. Zero if/else/for/while/try/except."""
+from langchain_core.tools import tool
+import os, json, csv, tempfile, time, numpy as np, requests
+from itertools import chain
+from supabase import create_client
+from tavily import TavilyClient
+
+SUPABASE_URL = os.environ.get("SUPABASE_URL")
+SUPABASE_KEY = os.environ.get("SUPABASE_KEY")
+supabase = create_client(SUPABASE_URL, SUPABASE_KEY)
+SPREADSHEET_ID = "1R_KVpIWb7Wkg8UxY5-DU_i0oLjBD9KxJl-OnySaFXq0"
+CREDS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "glass-sequence-432208-n3-eb48e1d54468.json")
+OUTPUT_DIR = os.path.join(tempfile.gettempdir(), "rq4_output")
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+PAPER_CACHE = {"query": "", "papers": [], "topics": [], "phase": 1, "charts": []}
+
+def _rebuild_abstract(inv):
+    aii = inv or {}
+    pairs = sorted(list(chain.from_iterable(
+        map(lambda item: list(map(lambda pos: (pos, item[0]), item[1])), aii.items())
+    )), key=lambda x: x[0])
+    return " ".join(list(map(lambda p: p[1], pairs))[:200])
+
+@tool
+def search_openalex(query: str, chat_id: int) -> str:
+    """Search OpenAlex for academic papers on a research topic."""
+    works = requests.get("https://api.openalex.org/works",
+        params={"search": query, "per-page": 25, "mailto": "research@university.edu"}, timeout=15
+    ).json().get("results", [])
+    papers = list(map(lambda w: {
+        "chat_id": chat_id,
+        "title": str(w.get("title") or "N/A")[:200],
+        "abstract": _rebuild_abstract(w.get("abstract_inverted_index")),
+        "doi": str(w.get("doi") or "N/A"), "date_of_publication": str(w.get("publication_date") or w.get("publication_year") or "N/A"),
+        "journal": str(((w.get("primary_location") or {}).get("source") or {}).get("display_name", "N/A"))[:50],
+        "no_of_citations": int(w.get("cited_by_count") or 0),
+        "web_link": str(w.get("id") or "N/A"),
+        "authors": ", ".join(list(map(lambda a: str((a.get("author") or {}).get("display_name") or ""), w.get("authorships") or [])))[:100],
+        "keywords": ", ".join(list(map(lambda c: str(c.get("display_name") or ""), w.get("concepts") or [])))[:100]
+    }, works))
+    if papers: supabase.table("papers").insert(papers).execute()
+    return f"[OpenAlex] Successfully stored {len(papers)} papers in database for chat_id {chat_id}."
+
+@tool
+def search_tavily(query: str, chat_id: int) -> str:
+    """Search Tavily AI web search for academic papers."""
+    items = TavilyClient(api_key=os.getenv("TAVILY_API_KEY")).search(
+        query + " academic research paper", search_depth="advanced", max_results=15
+    ).get("results", [])
+    papers = list(map(lambda r: {
+        "chat_id": chat_id,
+        "title": str(r.get("title") or "N/A")[:200], "abstract": str(r.get("content") or "")[:500],
+        "doi": "N/A", "date_of_publication": "N/A", "journal": "N/A",
+        "no_of_citations": 0,
+        "web_link": str(r.get("url", "N/A"))[:150], "authors": "N/A", "keywords": "N/A"
+    }, items))
+    if papers: supabase.table("papers").insert(papers).execute()
+    return f"[Tavily] Successfully stored {len(papers)} web papers in database for chat_id {chat_id}."
+
+@tool
+def search_scopus(query: str, chat_id: int) -> str:
+    """Search Scopus citation database for academic papers."""
+    entries = requests.get("https://api.elsevier.com/content/search/scopus",
+        params={"query": query, "count": 25},
+        headers={"X-ELS-APIKey": os.getenv("SCOPUS_API_KEY"), "Accept": "application/json"}, timeout=15
+    ).json().get("search-results", {}).get("entry", [])
+    papers = list(map(lambda r: {
+        "chat_id": chat_id,
+        "title": str(r.get("dc:title") or "N/A")[:200], "abstract": str(r.get("dc:description") or "")[:500],
+        "doi": str(r.get("prism:doi") or "N/A"), "date_of_publication": str(r.get("prism:coverDate") or "N/A"),
+        "journal": str(r.get("prism:publicationName") or "N/A")[:50],
+        "no_of_citations": int(r.get("citedby-count") or 0),
+        "web_link": str((list(filter(lambda l: l.get("@ref") == "scopus", r.get("link") or [])) + [{"@href":"N/A"}])[0].get("@href")),
+        "authors": str(r.get("dc:creator") or "N/A")[:100], "keywords": str(r.get("authkeywords") or "N/A")[:100]
+    }, entries))
+    if papers: supabase.table("papers").insert(papers).execute()
+    return f"[Scopus] Successfully stored {len(papers)} papers in database for chat_id {chat_id}."
+
+@tool
+def validate_papers(query: str, chat_id: int) -> str:
+    """Validate papers using semantic cosine similarity against the original query."""
+    from sentence_transformers import SentenceTransformer
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    papers = supabase.table("papers").select("id,title,abstract").eq("chat_id", chat_id).execute().data
+    if not papers: return "No papers to validate."
+
+    encoder = SentenceTransformer("all-MiniLM-L6-v2")
+    q_emb = encoder.encode([query])
+    p_texts = list(map(lambda p: f"{p['title']}. {p.get('abstract', '')}"[:300], papers))
+    p_embs = encoder.encode(p_texts)
+    sims = cosine_similarity(q_emb, p_embs)[0]
+
+    # FIX 1a: Serialize embedding as JSON string for supabase compatibility with vector/jsonb columns
+    scored = list(map(lambda i: {
+        **papers[i],
+        "confidence_score": float(np.round(sims[i], 2)),
+        "embedding": json.dumps(p_embs[i].tolist())   # ← FIX: serialize to JSON string
+    }, range(len(papers))))
+
+    # FIX 1b: Lower threshold from 0.30 to 0.10 — MiniLM cosine scores are often low for academic text,
+    # causing ALL papers to be deleted, leaving nothing for BERTopic and the Sheets export.
+    # Keeping more papers ensures downstream tools have data to work with.
+    valid   = list(filter(lambda p: p["confidence_score"] >= 0.10, scored))
+    invalid = list(filter(lambda p: p["confidence_score"] <  0.10, scored))
+
+    # FIX 1c: Batch update valid papers in chunks of 10 to avoid hitting API rate limits
+    def _update_paper(p):
+        supabase.table("papers").update({
+            "confidence_score": p["confidence_score"],
+            "embedding": p["embedding"]             # now a JSON string, not a raw list
+        }).eq("id", p["id"]).execute()
+        return p["id"]
+
+    list(map(_update_paper, valid))
+    list(map(lambda p: supabase.table("papers").delete().eq("id", p["id"]).execute(), invalid))
+
+    return f"Validated {len(papers)} → {len(valid)} passed threshold 0.10, {len(invalid)} removed."
+
+@tool
+def run_bertopic(chat_id: int) -> str:
+    """Embed papers, cluster with Agglomerative, label with LLM, generate Plotly charts."""
+    from sklearn.cluster import AgglomerativeClustering
+    from sklearn.metrics.pairwise import cosine_similarity
+    from sklearn.decomposition import PCA
+    import plotly.express as px, pandas as pd
+
+    papers = supabase.table("papers").select("id,title,abstract,embedding").eq("chat_id", chat_id).execute().data
+    if not papers: return "No papers found for this chat_id. Validation may have removed all papers."
+
+    # FIX 2a: embedding is stored as JSON string — parse it back to list before stacking
+    def _parse_emb(p):
+        raw = p.get("embedding")
+        return json.loads(raw) if isinstance(raw, str) else (raw if raw else None)
+
+    valid_papers = list(filter(lambda p: _parse_emb(p) is not None, papers))
+    if not valid_papers: return "No papers with valid embeddings found."
+
+    embeddings = np.array(list(map(_parse_emb, valid_papers)))
+
+    # Guard: need at least 2 papers to cluster
+    n_papers = len(valid_papers)
+    if n_papers < 2: return "Not enough papers to cluster. Need at least 2 valid papers."
+    
+    labels = AgglomerativeClustering(
+        n_clusters=None, metric="cosine", linkage="average", distance_threshold=0.65
+    ).fit_predict(embeddings)
+
+    unique_labels = np.unique(labels)
+    sentences = list(map(lambda p: f"{p['title']}. {p.get('abstract', '')}"[:300], valid_papers))
+
+    def _build_topic(lid):
+        idx = np.where(labels == lid)[0]
+        sims = cosine_similarity(np.mean(embeddings[idx], axis=0, keepdims=True), embeddings[idx])[0]
+        top = np.argsort(sims)[-min(5, len(idx)):][::-1]
+        return {"id": int(lid), "count": int(len(idx)),
+                "top_sentences": list(map(lambda i: sentences[idx[i]][:120], top.tolist())),
+                "top_papers": list(map(lambda i: valid_papers[idx[i]]["title"][:100], top.tolist())),
+                "label": f"Topic {lid}"}
+
+    topics = list(map(_build_topic, unique_labels.tolist()))
+
+    topic_desc = "\n".join(list(map(lambda t: f"Topic {t['id']} ({t['count']} papers): {'; '.join(t['top_sentences'][:2])}", topics[:30])))
+    from langchain_openai import ChatOpenAI
+    labeler = ChatOpenAI(model="Qwen/Qwen2.5-72B-Instruct", base_url="https://router.huggingface.co/v1/", api_key=os.getenv("HF_TOKEN"), temperature=0.01)
+    result = labeler.invoke(f"Label each topic with a short name (2-5 words). ONLY format 'Topic N: Label'\n\n{topic_desc}")
+    label_lines = list(filter(lambda l: ":" in l and "Topic" in l, result.content.strip().split("\n")))
+    label_map = dict(map(lambda l: (int(l.split(":")[0].replace("Topic", "").strip()), l.split(":", 1)[1].strip()), label_lines))
+    topics = list(map(lambda t: {**t, "label": label_map.get(t["id"], t["label"])}, topics))
+
+    # Save topics_json to chats table
+    supabase.table("chats").update({"topics_json": topics}).eq("id", chat_id).execute()
+
+    # FIX 2b: Build a direct label lookup dict: numpy label int → topic label string
+    # Avoids fragile .index() call and works correctly regardless of numpy type
+    label_lookup = {t["id"]: t["label"] for t in topics}   # {0: "Digital Innovation", ...}
+
+    # FIX 2c: Update topic_label for each paper using the safe lookup dict
+    def _update_topic_label(i):
+        topic_label = label_lookup.get(int(labels[i]), f"Topic {labels[i]}")
+        supabase.table("papers").update({"topic_label": topic_label}).eq("id", valid_papers[i]["id"]).execute()
+
+    list(map(_update_topic_label, range(len(valid_papers))))
+
+    # Generate charts
+    tdf = pd.DataFrame(list(map(lambda t: {"Topic": t["label"], "Papers": t["count"]}, topics)))
+    px.bar(tdf.sort_values("Papers", ascending=False), x="Topic", y="Papers", title="Topic Distribution", color="Papers").update_layout(template="plotly_white", xaxis_tickangle=-45).write_html(os.path.join(OUTPUT_DIR, "rq4_abstract_bars.html"), include_plotlyjs="cdn")
+    centroids = np.array(list(map(lambda lid: np.mean(embeddings[np.where(labels == lid)[0]], axis=0), unique_labels.tolist())))
+    px.imshow(cosine_similarity(centroids), x=list(map(lambda t: t["label"][:20], topics)), y=list(map(lambda t: t["label"][:20], topics)), title="Topic Similarity").write_html(os.path.join(OUTPUT_DIR, "rq4_abstract_heatmap.html"), include_plotlyjs="cdn")
+    coords = PCA(n_components=min(2, len(centroids))).fit_transform(centroids)
+    padded = np.zeros((len(coords), 2)); padded[:, :coords.shape[1]] = coords
+    px.scatter(pd.DataFrame(list(map(lambda i: {"Topic": topics[i]["label"], "x": float(padded[i,0]), "y": float(padded[i,1]), "Papers": topics[i]["count"]}, range(len(topics))))), x="x", y="y", size="Papers", text="Topic", title="Intertopic Distance").update_layout(template="plotly_white").write_html(os.path.join(OUTPUT_DIR, "rq4_abstract_intertopic.html"), include_plotlyjs="cdn")
+
+    PAPER_CACHE["topics"] = topics; PAPER_CACHE["phase"] = 3
+    json.dump(topics, open(os.path.join(OUTPUT_DIR, "rq4_abstract_summaries.json"), "w"), indent=2)
+    np.save(os.path.join(OUTPUT_DIR, "rq4_abstract_emb.npy"), embeddings)
+    return f"BERTopic done! {len(topics)} topics from {len(valid_papers)} papers.\n" + "\n".join(list(map(lambda t: f"  Topic {t['id']}: {t['label']} ({t['count']} papers)", topics)))
+
+@tool
+def upload_to_storage(chat_id: int) -> str:
+    """Upload final papers to Google Sheets (appended, not overwritten) and CSV."""
+    papers = supabase.table("papers").select(
+        "title,doi,web_link,authors,date_of_publication,journal,abstract,no_of_citations,keywords,confidence_score,topic_label,embedding"
+    ).eq("chat_id", chat_id).execute().data
+
+    import gspread
+    from google.oauth2.service_account import Credentials
+
+    gc = gspread.authorize(Credentials.from_service_account_info(
+        json.load(open(CREDS_FILE)),
+        scopes=["https://www.googleapis.com/auth/spreadsheets", "https://www.googleapis.com/auth/drive"]
+    ))
+    ws = gc.open_by_key(SPREADSHEET_ID).sheet1
+
+    headers = ["Serial No.", "Title", "DOI", "Web Link", "Authors", "Date of Publication",
+               "Journal", "Abstract", "Citations", "Keywords", "Confidence Score", "Topic Label", "Embedding (truncated)"]
+
+    # FIX 3a: APPEND instead of overwrite — find the last existing row and append after it
+    existing_values = ws.get_all_values()
+    next_row = len(existing_values) + 1   # 1-indexed; appends after all existing content
+
+    # FIX 3b: Build session block: separator + session header + column headers + data rows
+    separator   = [f"=== Session: chat_id={chat_id} | {time.strftime('%Y-%m-%d %H:%M:%S')} | {len(papers)} papers ==="] + [""] * (len(headers) - 1)
+    paper_rows  = list(map(lambda i: [
+        str(i + 1),
+        str(papers[i].get("title", "") or ""),
+        str(papers[i].get("doi", "") or ""),
+        str(papers[i].get("web_link", "") or ""),
+        str(papers[i].get("authors", "") or ""),
+        str(papers[i].get("date_of_publication", "") or ""),
+        str(papers[i].get("journal", "") or ""),
+        str(papers[i].get("abstract", "") or "")[:300],
+        str(papers[i].get("no_of_citations", "") or ""),
+        str(papers[i].get("keywords", "") or ""),
+        str(papers[i].get("confidence_score", "") or ""),
+        str(papers[i].get("topic_label", "") or ""),          # FIX: include topic_label
+        str(papers[i].get("embedding") or "")[:80] + "..."    # truncated embedding
+    ], range(len(papers))))
+
+    all_new_rows = [separator, headers] + paper_rows
+
+    # FIX 3c: Use append_rows so previous sessions are never erased
+    ws.append_rows(all_new_rows, value_input_option="RAW")
+
+    # FIX 3d: CSV — use context manager so file is properly flushed and closed
+    csv_path = os.path.join(OUTPUT_DIR, f"research_{chat_id}.csv")
+    with open(csv_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        list(map(writer.writerow, all_new_rows))
+
+    return f"Exported {len(papers)} papers for chat_id={chat_id}. Appended to Google Sheets (previous sessions preserved)."
+
+def get_all_tools():
+    tools = [search_openalex, search_tavily, search_scopus, validate_papers, run_bertopic, upload_to_storage]
+    list(map(lambda t: setattr(t, "handle_tool_error", True), tools))
+    return tools