Upload 4 files

agent.py

@@ -0,0 +1,653 @@
+"""
+agent.py — BERTopic Thematic Analysis Agent
+Braun & Clarke (2006) six-phase methodology implemented as a ReAct agent
+using LangGraph, ChatMistralAI, and MemorySaver.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any, Generator
+
+from langchain_mistralai import ChatMistralAI
+from langgraph.checkpoint.memory import MemorySaver
+from langgraph.prebuilt import create_react_agent
+
+from tools import (
+    load_scopus_csv,
+    run_bertopic_discovery,
+    label_topics_with_llm,
+    consolidate_into_themes,
+    compare_with_taxonomy,
+    generate_comparison_csv,
+    export_narrative,
+)
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Artifact paths (shared across phases)
+# ──────────────────────────────────────────────────────────────────────────────
+
+ARTIFACTS_DIR   = Path("artifacts")
+ARTIFACTS_DIR.mkdir(exist_ok=True)
+
+_LOADED_DATA    = str(ARTIFACTS_DIR / "loaded_data.json")
+_SUMMARIES      = str(ARTIFACTS_DIR / "summaries.json")
+_EMB            = str(ARTIFACTS_DIR / "emb.npy")
+_LABELS         = str(ARTIFACTS_DIR / "topic_labels.json")
+_THEMES         = str(ARTIFACTS_DIR / "themes.json")
+_TAXONOMY       = str(ARTIFACTS_DIR / "taxonomy_mapping.json")
+_COMPARISON_CSV = str(ARTIFACTS_DIR / "abstract_vs_title_comparison.csv")
+_NARRATIVE      = str(ARTIFACTS_DIR / "section7_narrative.txt")
+
+# ──────────────────────────────────────────────────────────────────────────────
+# System Prompt
+# ──────────────────────────────────────────────────────────────────────────────
+
+SYSTEM_PROMPT = """
+╔══════════════════════════════════════════════════════════════════════════════╗
+║          COMPUTATIONAL THEMATIC ANALYSIS AGENT  —  SYSTEM PROMPT           ║
+╚══════════════════════════════════════════════════════════════════════════════╝
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+ROLE
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+You are a computational thematic analysis expert trained in the Braun & Clarke
+(2006) six-phase framework for rigorous qualitative and mixed-methods research.
+You specialise in applying BERTopic-based semantic clustering to academic
+literature corpora, with deep expertise in:
+
+  • Systematic literature review methodology
+  • Sentence-level semantic embedding and agglomerative clustering
+  • LLM-assisted topic labelling and theme consolidation
+  • PAJAIS (Pacific-Asia Journal of the Association for Information Systems)
+    25-category research taxonomy alignment
+  • Transparent, reproducible, human-in-the-loop analytical pipelines
+
+Your outputs are used in peer-reviewed academic research. Precision,
+methodological rigour, and faithful adherence to the B&C (2006) phases are
+non-negotiable.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+CRITICAL RULES  (must be followed without exception)
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+1.  ONE PHASE PER MESSAGE. Complete exactly one B&C phase per conversational
+    turn. Never skip ahead or combine phases in a single response.
+
+2.  ALL APPROVALS VIA REVIEW TABLE — NEVER VIA CHAT. You must NEVER ask the
+    user to approve, reject, or rename topics in free-text chat. Every approval
+    workflow must go through the Gradio review table. After populating the table
+    you must STOP and wait for the user to click "Submit Review".
+
+3.  STOP GATES ARE MANDATORY. At the end of Phases 2, 3, 4, and 5.5 you must
+    output the exact STOP phrase:
+        ⏸ STOP GATE — awaiting your review table submission to continue.
+    Do not proceed until the user's next message contains review data.
+
+4.  NEVER HALLUCINATE TOOL RESULTS. If a tool call fails, report the exact
+    error verbatim and ask the user how to proceed. Do not invent file paths,
+    cluster counts, or topic labels.
+
+5.  COLUMN DISCIPLINE. Never include "Author Keywords" in any clustering run.
+    Use only the columns specified in RUN_CONFIGS: Abstract (abstract run) or
+    Title (title run).
+
+6.  ARTEFACT HYGIENE. Every tool saves files to the artifacts/ directory.
+    Always pass the exact saved_path returned by a previous tool to the next
+    tool. Never guess or construct file paths manually.
+
+7.  STREAMING DISCIPLINE. Yield one streamed chunk per reasoning step so the
+    Gradio UI can update the phase progress bar in real time.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+TOOLS  (7 available)
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+1.  load_scopus_csv(csv_path, run_mode)
+      Load a Scopus-exported CSV. Counts papers and sentences. Applies
+      boilerplate regex filter. Saves loaded_data.json. Use in Phase 1.
+
+2.  run_bertopic_discovery(loaded_data_path)
+      Embeds sentences with all-MiniLM-L6-v2 (normalize_embeddings=True).
+      Clusters with AgglomerativeClustering(metric=cosine, threshold=0.7).
+      No UMAP. Finds 5 nearest centroid sentences per cluster. Generates
+      4 Plotly charts. Saves summaries.json + emb.npy. Use in Phase 2.
+
+3.  label_topics_with_llm(summaries_path, top_n)
+      Sends top-N topics (max 100) to Mistral via PromptTemplate +
+      JsonOutputParser. Returns short labels and descriptions. Saves
+      topic_labels.json. Use in Phase 2 after discovery.
+
+4.  consolidate_into_themes(labels_path, summaries_path, emb_path, approved_groups)
+      Merges approved topic groups into named themes. Recomputes centroids.
+      Saves themes.json. Use in Phase 3 after the review table is submitted.
+
+5.  compare_with_taxonomy(themes_path)
+      Maps consolidated themes to PAJAIS 25 categories via Mistral.
+      Returns confidence scores and rationale. Saves taxonomy_mapping.json.
+      Use in Phase 5.5.
+
+6.  generate_comparison_csv(csv_path, taxonomy_path)
+      Produces abstract vs title side-by-side CSV with PAJAIS categories
+      and confidence scores. Saves abstract_vs_title_comparison.csv.
+      Use in Phase 6.
+
+7.  export_narrative(taxonomy_path)
+      Generates a ~500-word Section 7 (Discussion & Implications) as
+      flowing academic prose via Mistral. Saves section7_narrative.txt.
+      Use in Phase 6.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+BRAUN & CLARKE (2006) SIX-PHASE PROTOCOL
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 1 — FAMILIARISATION WITH THE DATA                                   │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Immerse in the corpus. Understand its scope, structure, and quality.
+
+Instructions:
+  a. Call load_scopus_csv(csv_path=<user_provided>, run_mode=<"abstract"|"title">).
+  b. Display the returned statistics in a clear summary:
+       • Total papers loaded
+       • Total sentences extracted
+       • Sentences remaining after boilerplate filtering
+       • Column(s) used
+       • Run mode (abstract / title)
+  c. Comment briefly on data quality: density, likely noise level, any
+     column mapping issues detected.
+  d. STOP. Do not proceed to Phase 2 until the user explicitly confirms
+     they are satisfied with the loaded data.
+
+Output template:
+  📂 Phase 1 Complete — Familiarisation
+  ─────────────────────────────────────
+  Papers:              {N}
+  Sentences extracted: {S}
+  After filtering:     {F}
+  Column used:         {C}
+  Run mode:            {M}
+  [Quality commentary]
+  ✅ Ready for Phase 2. Reply "proceed" to start Initial Coding.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 2 — GENERATING INITIAL CODES                                        │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Produce a full set of atomic semantic codes from the corpus.
+
+Instructions:
+  a. Call run_bertopic_discovery(loaded_data_path=artifacts/loaded_data.json).
+     Report: number of clusters found, total sentences clustered, chart paths.
+  b. Call label_topics_with_llm(summaries_path=artifacts/summaries.json, top_n=100).
+     Report: number of topics labelled.
+  c. Populate the Gradio review table with ALL labelled topics. Each row must
+     contain:
+       • #           — topic_id (integer)
+       • Topic Label — LLM-generated label
+       • Top Evidence — first centroid sentence (truncated to 120 chars)
+       • Sentences   — cluster size
+       • Papers      — estimated paper count (size ÷ avg sentences per paper)
+       • Approve     — default True
+       • Rename To   — empty (user fills)
+       • Reasoning   — empty (user fills)
+  d. Present the 4 Plotly charts by referencing their file paths.
+  e. Explain to the user:
+       • Check "Approve" for topics to keep; uncheck to discard.
+       • Fill "Rename To" with a preferred label; leave blank to keep LLM label.
+       • Optionally note merging intentions in "Reasoning".
+       • Topics with the same "Reasoning" group tag will be merged in Phase 3.
+
+⏸ STOP GATE — awaiting your review table submission to continue.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 3 — SEARCHING FOR THEMES                                            │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Collate approved codes into candidate themes.
+
+Instructions:
+  a. Parse the submitted review table. Extract:
+       • Approved topic IDs (Approve == True)
+       • Rename mappings (Rename To != "")
+       • Merge groups (topics sharing the same Reasoning tag)
+  b. Construct approved_groups: a JSON list of lists, where each inner list
+     contains the topic_ids belonging to one theme. Topics with a shared
+     Reasoning tag form one group. Approved topics with no Reasoning tag
+     each form a singleton group.
+  c. Call consolidate_into_themes(
+         labels_path=artifacts/topic_labels.json,
+         summaries_path=artifacts/summaries.json,
+         emb_path=artifacts/emb.npy,
+         approved_groups=<constructed JSON string>
+     ).
+  d. Display a theme summary table:
+       Theme # | Theme Label | Topics Merged | Total Sentences
+
+⏸ STOP GATE — awaiting your review table submission to continue.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 4 — REVIEWING THEMES / SATURATION CHECK                             │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Assess whether themes are internally coherent and collectively
+exhaustive. Check corpus coverage.
+
+Instructions:
+  a. Load artifacts/themes.json (already created by Phase 3).
+  b. Compute and display a saturation report:
+       • Total sentences covered by all themes vs. total corpus sentences
+       • Coverage percentage
+       • Theme coherence flag: warn if any theme covers < 1 % of corpus
+       • Overlap flag: warn if any two themes share > 30 % vocabulary
+       (Vocabulary overlap is approximated by comparing top-evidence sentences
+        using word-level Jaccard similarity — compute in Python, no tool call.)
+  c. Populate the review table again with the THEME list (not topic list):
+       • #           — theme_id
+       • Topic Label — current theme_label
+       • Top Evidence — first top_evidence sentence (120 chars)
+       • Sentences   — total_size
+       • Papers      — estimated
+       • Approve     — default True
+       • Rename To   — user may provide final name
+       • Reasoning   — any split/merge instructions
+  d. Ask the user to confirm themes, request splits/merges, or rename.
+
+⏸ STOP GATE — awaiting your review table submission to continue.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 5 — DEFINING AND NAMING THEMES                                      │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Produce final, publication-ready theme names and definitions.
+
+Instructions:
+  a. Apply all renames from the Phase 4 review table to artifacts/themes.json
+     in memory (update theme_label field for each theme_id where Rename To
+     is non-empty).
+  b. For each finalised theme, generate a two-sentence academic definition
+     grounded in the top_evidence sentences. Output this as a numbered list.
+  c. Confirm the final theme set to the user in a clean summary:
+       Theme # | Final Name | Definition (2 sentences) | Sentence Count
+  d. STOP. Ask the user to confirm the final names before PAJAIS mapping.
+     ✅ Reply "proceed to taxonomy" to continue to Phase 5.5.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 5.5 — PAJAIS TAXONOMY ALIGNMENT                                     │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Map each finalised theme to the PAJAIS 25-category taxonomy.
+
+Instructions:
+  a. Call compare_with_taxonomy(themes_path=artifacts/themes.json).
+  b. Display the mapping results in a structured table:
+       Theme Name | PAJAIS Category | Confidence | Rationale
+  c. Highlight any themes with confidence < 0.5 as requiring manual review.
+  d. Note any PAJAIS categories not covered by the corpus (research gaps).
+  e. Populate the review table with the mapping results:
+       • #           — theme_id
+       • Topic Label — theme_label → PAJAIS category
+       • Top Evidence — rationale (truncated)
+       • Approve     — default True (uncheck to override mapping)
+       • Rename To   — alternative PAJAIS category if user disagrees
+       • Reasoning   — free notes
+
+⏸ STOP GATE — awaiting your review table submission to continue.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+┌─────────────────────────────────────────────────────────────────────────────┐
+│  PHASE 6 — PRODUCING THE REPORT                                            │
+└─────────────────────────────────────────────────────────────────────────────┘
+Objective: Generate all final deliverables.
+
+Instructions:
+  a. Apply any PAJAIS category overrides from the Phase 5.5 review table
+     to artifacts/taxonomy_mapping.json in memory.
+  b. Call generate_comparison_csv(
+         csv_path=<original CSV path>,
+         taxonomy_path=artifacts/taxonomy_mapping.json
+     ).
+     Report: row count, file path.
+  c. Call export_narrative(taxonomy_path=artifacts/taxonomy_mapping.json).
+     Report: word count, file path, first 150 chars of preview.
+  d. Present a final deliverables checklist:
+       ✅ artifacts/summaries.json            — raw cluster summaries
+       ✅ artifacts/topic_labels.json         — LLM-generated labels
+       ✅ artifacts/themes.json               — consolidated themes
+       ✅ artifacts/taxonomy_mapping.json     — PAJAIS alignment
+       ✅ artifacts/abstract_vs_title_comparison.csv
+       ✅ artifacts/section7_narrative.txt    — ~500-word Section 7
+       ✅ artifacts/chart_cluster_sizes.html
+       ✅ artifacts/chart_pca_scatter.html
+       ✅ artifacts/chart_top10_pie.html
+       ✅ artifacts/chart_centroid_heatmap.html
+  e. Congratulate the user and offer to re-run in title mode for comparison.
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+END OF SYSTEM PROMPT
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+""".strip()
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool registry
+# ──────────────────────────────────────────────────────────────────────────────
+
+TOOLS = [
+    load_scopus_csv,
+    run_bertopic_discovery,
+    label_topics_with_llm,
+    consolidate_into_themes,
+    compare_with_taxonomy,
+    generate_comparison_csv,
+    export_narrative,
+]
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Phase detection helpers
+# ──────────────────────────────────────────────────────────────────────────────
+
+_PHASE_PATTERNS = {
+    "loading":    re.compile(r"load_scopus_csv|loaded_data", re.I),
+    "embedding":  re.compile(r"run_bertopic_discovery|embedding", re.I),
+    "clustering": re.compile(r"summaries\.json|n_topics|clusters found", re.I),
+    "labelling":  re.compile(r"label_topics_with_llm|topic_labels", re.I),
+    "review":     re.compile(r"STOP GATE|review table|submit review", re.I),
+    "done":       re.compile(r"section7_narrative|deliverables checklist", re.I),
+}
+
+
+def _detect_phase(text: str) -> str:
+    """Return the most specific pipeline phase detectable from agent output."""
+    matched = list(filter(
+        lambda kv: kv[1].search(text),
+        _PHASE_PATTERNS.items(),
+    ))
+    return matched[-1][0] if matched else "idle"
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Review-table row builder
+# ──────────────────────────────────────────────────────────────────────────────
+
+_TABLE_COLS = ["#", "Topic Label", "Top Evidence", "Sentences", "Papers", "Approve", "Rename To", "Reasoning"]
+
+
+def _topic_to_row(topic: dict, papers_per_sent: float = 0.2) -> list:
+    """Convert a topic/theme dict to a review-table row."""
+    evidence = (topic.get("top_evidence") or [""])[0]
+    return [
+        topic.get("topic_id", topic.get("theme_id", 0)),
+        topic.get("label", topic.get("theme_label", "")),
+        evidence[:120],
+        topic.get("size", topic.get("total_size", 0)),
+        round(topic.get("size", topic.get("total_size", 0)) * papers_per_sent),
+        True,
+        "",
+        "",
+    ]
+
+
+def _build_review_rows(path: str, id_key: str = "topic_id") -> list[list]:
+    """Load a JSON artefact and convert every entry to a review-table row."""
+    records = json.loads(Path(path).read_text())
+    return list(map(_topic_to_row, records))
+
+
+# ────────────────────────────────���─────────────────────────────────────────────
+# Approved-groups extractor  (called in handle_review for Phase 2 → 3)
+# ──────────────────────────────────────────────────────────────────────────────
+
+def _extract_approved_groups(rows: list[list]) -> str:
+    """
+    Parse review-table rows into approved_groups JSON string.
+
+    Groups are formed by the Reasoning field value:
+      • Rows sharing a non-empty Reasoning tag → merged into one group
+      • Approved rows with empty Reasoning    → singleton group each
+      • Unapproved rows (Approve == False)    → discarded
+    """
+    approved = list(filter(lambda r: r[5] is True or r[5] == "True" or r[5] == 1, rows))
+
+    tagged    = list(filter(lambda r: str(r[7]).strip(), approved))
+    untagged  = list(filter(lambda r: not str(r[7]).strip(), approved))
+
+    # Group tagged rows by their Reasoning value
+    reasoning_vals = list(set(map(lambda r: str(r[7]).strip(), tagged)))
+
+    tagged_groups = list(map(
+        lambda tag: list(map(
+            lambda r: int(r[0]),
+            filter(lambda r: str(r[7]).strip() == tag, tagged),
+        )),
+        reasoning_vals,
+    ))
+
+    singleton_groups = list(map(lambda r: [int(r[0])], untagged))
+
+    all_groups = tagged_groups + singleton_groups
+    return json.dumps(all_groups)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# BERTopicAgent  — the class consumed by app.py
+# ──────────────────────────────────────────────────────────────────────────────
+
+class BERTopicAgent:
+    """
+    Wraps a LangGraph ReAct agent and exposes the two generator methods
+    expected by the Gradio front-end:
+
+        handle_message(message, history, csv_path) → yields 5-tuple
+        handle_review(table_rows, history)          → yields 4-tuple
+    """
+
+    # Gradio 5-tuple: (history, phase, charts_dict, downloads_list, topic_rows)
+    # Gradio 4-tuple: (history, phase, charts_dict, downloads_list)
+
+    def __init__(self) -> None:
+        self.phase       = "idle"
+        self._charts:    dict[str, str] = {}
+        self._downloads: list[str]      = []
+        self._csv_path:  str | None     = None
+
+        self._llm       = ChatMistralAI(
+            model="mistral-large-latest",
+            temperature=0.2,
+            streaming=True,
+        )
+        self._memory    = MemorySaver()
+        self._graph     = create_react_agent(
+            model=self._llm,
+            tools=TOOLS,
+            checkpointer=self._memory,
+            prompt=SYSTEM_PROMPT,
+        )
+        self._thread_id = "bc2006-session-1"
+
+    # ── internal ──────────────────────────────────────────────────────────────
+
+    def _config(self) -> dict:
+        return {"configurable": {"thread_id": self._thread_id}}
+
+    def _update_charts(self, text: str) -> None:
+        """Scan agent text for chart file paths and register them."""
+        found = re.findall(r'artifacts/chart_[a-z_]+\.html', text)
+        label_map = {
+            "chart_cluster_sizes":    "Cluster Sizes",
+            "chart_pca_scatter":      "PCA Scatter",
+            "chart_top10_pie":        "Top-10 Pie",
+            "chart_centroid_heatmap": "Centroid Heatmap",
+        }
+        list(map(
+            lambda p: self._charts.__setitem__(
+                label_map.get(Path(p).stem, Path(p).stem), p
+            ),
+            found,
+        ))
+
+    def _update_downloads(self, text: str) -> None:
+        """Scan agent text for downloadable artefact paths."""
+        found = re.findall(r'artifacts/[\w_]+\.(json|csv|txt|npy|html)', text)
+        new   = list(filter(lambda p: p not in self._downloads, found))
+        self._downloads.extend(new)
+
+    def _accumulate_stream(
+        self,
+        stream: Any,
+        history: list,
+        user_msg: str,
+    ) -> Generator[tuple, None, None]:
+        """
+        Consume a LangGraph stream, yielding Gradio 5-tuples incrementally.
+        Handles both streaming token chunks and full message events.
+        """
+        accumulated = ""
+
+        for chunk in stream:
+            # LangGraph yields dicts keyed by node name
+            node_output = (
+                chunk.get("agent") or
+                chunk.get("tools") or
+                {}
+            )
+            messages = node_output.get("messages", [])
+
+            text_delta = "".join(list(map(
+                lambda m: getattr(m, "content", "") if hasattr(m, "content") else "",
+                messages,
+            )))
+
+            accumulated += text_delta
+            self._update_charts(accumulated)
+            self._update_downloads(accumulated)
+            self.phase = _detect_phase(accumulated)
+
+            updated_history = history + [[user_msg, accumulated]] if accumulated else history
+
+            yield (
+                updated_history,
+                self.phase,
+                dict(self._charts),
+                list(self._downloads),
+                [],          # topic_rows populated in final yield
+            )
+
+        # Final yield with review-table rows (if artefacts exist)
+        topic_rows = self._latest_review_rows()
+        yield (
+            history + [[user_msg, accumulated]],
+            self.phase,
+            dict(self._charts),
+            list(self._downloads),
+            topic_rows,
+        )
+
+    def _latest_review_rows(self) -> list[list]:
+        """Return review rows from the most recently produced artefact."""
+        candidates = [
+            (_THEMES,   "theme_id"),
+            (_LABELS,   "topic_id"),
+            (_SUMMARIES,"topic_id"),
+        ]
+        existing = list(filter(lambda t: Path(t[0]).exists(), candidates))
+        return _build_review_rows(*existing[0]) if existing else []
+
+    # ── public API ────────────────────────────────────────────────────────────
+
+    def handle_message(
+        self,
+        message:  str,
+        history:  list,
+        csv_path: str | None = None,
+    ) -> Generator[tuple, None, None]:
+        """
+        Send a user message to the ReAct agent and stream back Gradio 5-tuples.
+
+        Yields: (history, phase, charts_dict, downloads_list, topic_rows)
+        """
+        self._csv_path = csv_path or self._csv_path
+
+        # Inject CSV path into message so the agent can reference it
+        enriched = (
+            f"{message}\n\n[SYSTEM CONTEXT] CSV path: {self._csv_path}"
+            if self._csv_path and "csv" not in message.lower()
+            else message
+        )
+
+        stream = self._graph.stream(
+            {"messages": [{"role": "user", "content": enriched}]},
+            config=self._config(),
+            stream_mode="updates",
+        )
+
+        yield from self._accumulate_stream(stream, history, message)
+
+    def handle_review(
+        self,
+        table_data: list,
+        history:    list,
+    ) -> Generator[tuple, None, None]:
+        """
+        Process a submitted review table and advance to the next B&C phase.
+
+        The table rows are serialised to JSON and injected as a structured
+        user message so the agent can parse approvals, renames, and groups.
+
+        Yields: (history, phase, charts_dict, downloads_list)
+        """
+        approved_groups = _extract_approved_groups(table_data)
+
+        review_payload = json.dumps({
+            "event":           "review_submitted",
+            "rows":            table_data,
+            "approved_groups": json.loads(approved_groups),
+            "approved_count":  len(json.loads(approved_groups)),
+        }, ensure_ascii=False, indent=2)
+
+        review_message = (
+            f"The user has submitted the review table. "
+            f"Approved groups: {approved_groups}. "
+            f"Full payload:\n{review_payload}\n\n"
+            f"Please continue to the next B&C phase now."
+        )
+
+        stream = self._graph.stream(
+            {"messages": [{"role": "user", "content": review_message}]},
+            config=self._config(),
+            stream_mode="updates",
+        )
+
+        accumulated = ""
+        for chunk in stream:
+            node_output = chunk.get("agent") or chunk.get("tools") or {}
+            messages    = node_output.get("messages", [])
+
+            text_delta = "".join(list(map(
+                lambda m: getattr(m, "content", "") if hasattr(m, "content") else "",
+                messages,
+            )))
+
+            accumulated += text_delta
+            self._update_charts(accumulated)
+            self._update_downloads(accumulated)
+            self.phase = _detect_phase(accumulated)
+
+            yield (
+                history + [["[Review submitted]", accumulated]],
+                self.phase,
+                dict(self._charts),
+                list(self._downloads),
+            )
+
+        yield (
+            history + [["[Review submitted]", accumulated]],
+            self.phase,
+            dict(self._charts),
+            list(self._downloads),
+        )

app.py

@@ -0,0 +1,617 @@
+"""
+BERTopic Topic Modelling Agent — Gradio Interface
+"""
+
+import gradio as gr
+import pandas as pd
+import json
+from pathlib import Path
+
+# ── Try to import the real agent; fall back to a stub so the UI still loads ──
+try:
+    from agent import BERTopicAgent
+except ImportError:
+    class BERTopicAgent:
+        """Stub agent used when agent.py is not present."""
+        def __init__(self):
+            self.phase = "idle"
+            self._charts: dict[str, str] = {}
+            self._downloads: list[str] = []
+
+        def handle_message(self, message: str, history: list, csv_path: str | None = None):
+            yield (
+                history + [[message, "⚠️ `agent.py` not found — running in UI-preview mode."]],
+                self.phase,
+                self._charts,
+                self._downloads,
+                [],
+            )
+
+        def handle_review(self, table_data: list, history: list):
+            yield (
+                history + [["[Review submitted]", "Review received. No real agent attached."]],
+                self.phase,
+                self._charts,
+                self._downloads,
+            )
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ──────────────────────────────────────────────────────────────────────────────
+
+PHASES = ["idle", "loading", "embedding", "clustering", "labelling", "review", "done"]
+PHASE_LABELS = {
+    "idle":       "Idle",
+    "loading":    "Loading data",
+    "embedding":  "Embedding documents",
+    "clustering": "Clustering topics",
+    "labelling":  "Labelling topics",
+    "review":     "Awaiting review",
+    "done":       "Complete",
+}
+
+TABLE_COLS = ["#", "Topic Label", "Top Evidence", "Sentences", "Papers", "Approve", "Rename To", "Reasoning"]
+
+EMPTY_TABLE = pd.DataFrame(columns=TABLE_COLS)
+
+
+def phase_html(phase: str) -> str:
+    """Render a step-progress bar for the current pipeline phase."""
+    steps = [p for p in PHASES if p != "idle"]
+    try:
+        current_idx = steps.index(phase) if phase in steps else -1
+    except ValueError:
+        current_idx = -1
+
+    dots = []
+    for i, step in enumerate(steps):
+        if i < current_idx:
+            cls, icon = "step done", "✓"
+        elif i == current_idx:
+            cls, icon = "step active", "●"
+        else:
+            cls, icon = "step", "○"
+        dots.append(
+            f'<div class="{cls}"><span class="icon">{icon}</span>'
+            f'<span class="label">{PHASE_LABELS[step]}</span></div>'
+        )
+
+    bar_pct = 0 if current_idx < 0 else int((current_idx + 1) / len(steps) * 100)
+
+    return f"""
+<div class="phase-wrap">
+  <div class="phase-steps">{''.join(dots)}</div>
+  <div class="phase-bar-bg">
+    <div class="phase-bar-fill" style="width:{bar_pct}%"></div>
+  </div>
+</div>
+"""
+
+
+def make_chart_html(iframe_src: str) -> str:
+    if not iframe_src:
+        return "<p style='color:#888;padding:1rem'>No chart available yet.</p>"
+    return (
+        f'<iframe src="{iframe_src}" width="100%" height="520" '
+        f'frameborder="0" scrolling="no" style="border-radius:8px;"></iframe>'
+    )
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# CSS
+# ──────────────────────────────────────────────────────────────────────────────
+
+CUSTOM_CSS = """
+/* ── Google Fonts ── */
+@import url('https://fonts.googleapis.com/css2?family=Syne:wght@400;600;700;800&family=JetBrains+Mono:wght@400;500&family=Lato:wght@300;400;700&display=swap');
+
+/* ── Root palette ── */
+:root {
+  --bg:        #0d0f14;
+  --surface:   #151820;
+  --border:    #252a38;
+  --accent:    #4f8ef7;
+  --accent2:   #a259ff;
+  --success:   #2dce89;
+  --warn:      #f7c94f;
+  --danger:    #f76f6f;
+  --text:      #e8ecf4;
+  --muted:     #7a8299;
+  --radius:    10px;
+  --mono:      'JetBrains Mono', monospace;
+  --sans:      'Lato', sans-serif;
+  --display:   'Syne', sans-serif;
+}
+
+/* ── Global resets ── */
+body, .gradio-container {
+  background: var(--bg) !important;
+  color: var(--text) !important;
+  font-family: var(--sans) !important;
+}
+
+/* ── App header ── */
+.app-header {
+  padding: 2rem 0 1.2rem;
+  border-bottom: 1px solid var(--border);
+  margin-bottom: 1.5rem;
+}
+.app-header h1 {
+  font-family: var(--display);
+  font-size: 2rem;
+  font-weight: 800;
+  letter-spacing: -0.03em;
+  background: linear-gradient(135deg, var(--accent) 0%, var(--accent2) 100%);
+  -webkit-background-clip: text;
+  -webkit-text-fill-color: transparent;
+  margin: 0 0 .25rem;
+}
+.app-header p {
+  color: var(--muted);
+  font-size: .9rem;
+  margin: 0;
+}
+
+/* ── Section cards ── */
+.section-card {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: var(--radius);
+  padding: 1.25rem 1.5rem;
+  margin-bottom: 1.2rem;
+}
+.section-title {
+  font-family: var(--display);
+  font-size: .75rem;
+  font-weight: 700;
+  letter-spacing: .12em;
+  text-transform: uppercase;
+  color: var(--accent);
+  margin: 0 0 1rem;
+}
+
+/* ── Phase progress bar ── */
+.phase-wrap {
+  padding: .5rem 0 .8rem;
+}
+.phase-steps {
+  display: flex;
+  gap: 0;
+  margin-bottom: .6rem;
+  flex-wrap: wrap;
+}
+.step {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  flex: 1;
+  min-width: 70px;
+  position: relative;
+  opacity: .38;
+}
+.step::after {
+  content: '';
+  position: absolute;
+  top: 10px;
+  left: 50%;
+  width: 100%;
+  height: 2px;
+  background: var(--border);
+  z-index: 0;
+}
+.step:last-child::after { display: none; }
+.step .icon {
+  font-size: .85rem;
+  z-index: 1;
+  background: var(--surface);
+  padding: 0 4px;
+  color: var(--muted);
+}
+.step .label {
+  font-size: .62rem;
+  font-family: var(--mono);
+  color: var(--muted);
+  margin-top: 4px;
+  text-align: center;
+}
+.step.done { opacity: .7; }
+.step.done .icon { color: var(--success); }
+.step.active { opacity: 1; }
+.step.active .icon {
+  color: var(--accent);
+  animation: pulse 1.2s ease-in-out infinite;
+}
+.step.active .label { color: var(--accent); font-weight: 600; }
+@keyframes pulse { 0%,100%{opacity:1} 50%{opacity:.4} }
+
+.phase-bar-bg {
+  height: 4px;
+  background: var(--border);
+  border-radius: 4px;
+  overflow: hidden;
+}
+.phase-bar-fill {
+  height: 100%;
+  background: linear-gradient(90deg, var(--accent), var(--accent2));
+  border-radius: 4px;
+  transition: width .5s ease;
+}
+
+/* ── Chatbot ── */
+.chatbot-wrap .message.user { background: #1a2035 !important; }
+.chatbot-wrap .message.bot  { background: #151c2e !important; }
+
+/* ── Buttons ── */
+.btn-primary {
+  background: linear-gradient(135deg, var(--accent), var(--accent2)) !important;
+  color: #fff !important;
+  border: none !important;
+  font-family: var(--display) !important;
+  font-weight: 700 !important;
+  letter-spacing: .04em !important;
+  border-radius: var(--radius) !important;
+  padding: .55rem 1.4rem !important;
+  transition: opacity .2s, transform .1s !important;
+}
+.btn-primary:hover { opacity: .88 !important; transform: translateY(-1px) !important; }
+.btn-primary:active { transform: translateY(0) !important; }
+
+.btn-secondary {
+  background: var(--surface) !important;
+  border: 1px solid var(--border) !important;
+  color: var(--text) !important;
+  font-family: var(--display) !important;
+  font-weight: 600 !important;
+  border-radius: var(--radius) !important;
+}
+.btn-secondary:hover { border-color: var(--accent) !important; color: var(--accent) !important; }
+
+/* ── Tabs ── */
+.tab-nav button {
+  font-family: var(--display) !important;
+  font-weight: 600 !important;
+  font-size: .82rem !important;
+  letter-spacing: .06em !important;
+  text-transform: uppercase !important;
+  color: var(--muted) !important;
+  border-bottom: 2px solid transparent !important;
+  padding: .5rem 1rem !important;
+  background: transparent !important;
+}
+.tab-nav button.selected {
+  color: var(--accent) !important;
+  border-bottom-color: var(--accent) !important;
+}
+
+/* ── Dataframe ── */
+.dataframe-wrap table {
+  font-family: var(--mono) !important;
+  font-size: .78rem !important;
+}
+.dataframe-wrap th {
+  background: #1a1f2e !important;
+  color: var(--accent) !important;
+  font-family: var(--display) !important;
+  font-size: .7rem !important;
+  letter-spacing: .08em !important;
+  text-transform: uppercase !important;
+}
+.dataframe-wrap td { border-color: var(--border) !important; }
+
+/* ── File upload ── */
+.file-drop {
+  border: 2px dashed var(--border) !important;
+  border-radius: var(--radius) !important;
+  background: var(--surface) !important;
+  color: var(--muted) !important;
+  transition: border-color .2s, background .2s !important;
+}
+.file-drop:hover {
+  border-color: var(--accent) !important;
+  background: #151c2e !important;
+}
+
+/* ── Textbox ── */
+textarea, input[type=text] {
+  background: var(--surface) !important;
+  border: 1px solid var(--border) !important;
+  color: var(--text) !important;
+  font-family: var(--sans) !important;
+  border-radius: var(--radius) !important;
+}
+textarea:focus, input[type=text]:focus {
+  border-color: var(--accent) !important;
+  box-shadow: 0 0 0 3px rgba(79,142,247,.15) !important;
+}
+
+/* ── Dropdown ── */
+select, .gr-dropdown {
+  background: var(--surface) !important;
+  border: 1px solid var(--border) !important;
+  color: var(--text) !important;
+  border-radius: var(--radius) !important;
+}
+
+/* ── Misc ── */
+label { color: var(--muted) !important; font-size: .8rem !important; font-family: var(--mono) !important; }
+"""
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Build UI
+# ──────────────────────────────────────────────────────────────────────────────
+
+def build_app():
+    agent = BERTopicAgent()
+
+    with gr.Blocks(title="BERTopic Agent") as demo:
+
+        # ── State ──────────────────────────────────────────────────────────
+        state_csv_path   = gr.State(None)          # path to uploaded CSV
+        state_charts     = gr.State({})             # {label: iframe_src}
+        state_downloads  = gr.State([])             # list of file paths
+
+        # ── Header ────────────────────────────────────────────────────────
+        gr.HTML("""
+        <div class="app-header">
+          <h1>⬡ BERTopic Agent</h1>
+          <p>Conversational topic modelling powered by BERTopic + LLM review</p>
+        </div>
+        """)
+
+        # ── Phase progress ─────────────────────────────────────────────────
+        phase_display = gr.HTML(value=phase_html("idle"), elem_id="phase-display")
+
+        # ══════════════════════════════════════════════════════════════════
+        # SECTION 1 — Data Input
+        # ══════════════════════════════════════════════════════════════════
+        gr.HTML('<div class="section-card"><p class="section-title">① Data Input</p>')
+        with gr.Row():
+            csv_upload = gr.File(
+                label="Upload CSV (one document per row)",
+                file_types=[".csv"],
+                elem_classes=["file-drop"],
+            )
+            with gr.Column(scale=2):
+                gr.Markdown(
+                    "**Expected format:** CSV with at least a `text` column. "
+                    "Optional columns: `id`, `title`, `source`.\n\n"
+                    "After uploading, tell the agent which column holds your documents.",
+                    elem_id="upload-hint",
+                )
+        gr.HTML('</div>')
+
+        # ══════════════════════════════════════════════════════════════════
+        # SECTION 2 — Agent Conversation
+        # ══════════════════════════════════════════════════════════════════
+        gr.HTML('<div class="section-card"><p class="section-title">② Agent Conversation</p>')
+        chatbot = gr.Chatbot(
+            label="",
+            height=420,
+            show_label=False,
+            elem_classes=["chatbot-wrap"],
+            avatar_images=(
+                None,                          # user — use default
+                "https://em-content.zobj.net/source/apple/391/robot_1f916.png",
+            ),
+        )
+        with gr.Row():
+            user_input = gr.Textbox(
+                placeholder="Ask the agent to run topic modelling, explain results, adjust parameters…",
+                lines=1,
+                show_label=False,
+                scale=8,
+            )
+            send_btn = gr.Button("Send ⟶", elem_classes=["btn-primary"], scale=1)
+        gr.HTML('</div>')
+
+        # ══════════════════════════════════════════════════════════════════
+        # SECTION 3 — Results
+        # ══════════════════════════════════════════════════════════════════
+        gr.HTML('<div class="section-card"><p class="section-title">③ Results</p>')
+        with gr.Tabs(elem_classes=["tab-nav"]):
+
+            # ── Tab A: Review Table ───────────────────────────────────��──
+            with gr.Tab("📋  Review Table"):
+                review_table = gr.Dataframe(
+                    value=EMPTY_TABLE,
+                    headers=TABLE_COLS,
+                    datatype=["number", "str", "str", "number", "number", "bool", "str", "str"],
+                    interactive=True,
+                    wrap=True,
+                    elem_classes=["dataframe-wrap"],
+                    label="Topic review — edit Approve / Rename To / Reasoning, then click Submit",
+                )
+                submit_review_btn = gr.Button(
+                    "Submit Review  ✓",
+                    elem_classes=["btn-primary"],
+                    size="sm",
+                )
+
+            # ── Tab B: Charts ────────────────────────────────────────────
+            with gr.Tab("📊  Charts"):
+                chart_selector = gr.Dropdown(
+                    label="Select chart",
+                    choices=[],
+                    interactive=True,
+                )
+                chart_display = gr.HTML(
+                    value="<p style='color:#7a8299;padding:1rem'>Charts will appear here after topic modelling completes.</p>"
+                )
+
+            # ── Tab C: Download ──────────────────────────────────────────
+            with gr.Tab("⬇  Download"):
+                download_files = gr.File(
+                    label="Output files",
+                    file_count="multiple",
+                    interactive=False,
+                )
+
+        gr.HTML('</div>')
+
+        # ══════════════════════════════════════════════════════════════════
+        # Event handlers
+        # ══════════════════════════════════════════════════════════════════
+
+        # ── Store CSV path when file is uploaded ─────────────────────────
+        def on_csv_upload(file):
+            if file is None:
+                return None
+            return file.name
+
+        csv_upload.change(
+            fn=on_csv_upload,
+            inputs=[csv_upload],
+            outputs=[state_csv_path],
+        )
+
+        # ── Send message to agent ─────────────────────────────────────────
+        def on_send(message: str, history: list, csv_path, charts_state, downloads_state):
+            """Stream agent responses and update all UI elements."""
+            if not message.strip():
+                yield history, "", phase_html("idle"), [], None, charts_state, downloads_state
+                return
+
+            # Accumulate streamed updates
+            last_history    = history
+            last_phase      = "idle"
+            last_charts     = charts_state
+            last_downloads  = downloads_state
+            new_topics      = []
+
+            for result in agent.handle_message(message, history, csv_path=csv_path):
+                # Agent yields: (history, phase, charts_dict, downloads_list, topic_rows)
+                (last_history, last_phase,
+                 last_charts, last_downloads, new_topics) = result
+
+                yield (
+                    last_history,
+                    "",                          # clear input
+                    phase_html(last_phase),
+                    _rows_to_df(new_topics) if new_topics else gr.update(),
+                    gr.update(),                 # downloads updated below
+                    last_charts,
+                    last_downloads,
+                )
+
+            # Final yield with downloads
+            yield (
+                last_history,
+                "",
+                phase_html(last_phase),
+                _rows_to_df(new_topics) if new_topics else gr.update(),
+                last_downloads if last_downloads else None,
+                last_charts,
+                last_downloads,
+            )
+
+        def _rows_to_df(rows):
+            if not rows:
+                return EMPTY_TABLE
+            df = pd.DataFrame(rows, columns=TABLE_COLS)
+            return df
+
+        send_outputs = [
+            chatbot,
+            user_input,
+            phase_display,
+            review_table,
+            download_files,
+            state_charts,
+            state_downloads,
+        ]
+
+        send_btn.click(
+            fn=on_send,
+            inputs=[user_input, chatbot, state_csv_path, state_charts, state_downloads],
+            outputs=send_outputs,
+        )
+        user_input.submit(
+            fn=on_send,
+            inputs=[user_input, chatbot, state_csv_path, state_charts, state_downloads],
+            outputs=send_outputs,
+        )
+
+        # ── Submit review ─────────────────────────────────────────────────
+        def on_submit_review(table_df, history, charts_state, downloads_state):
+            """Pass edited table back to the agent as a review submission."""
+            rows = table_df.values.tolist() if isinstance(table_df, pd.DataFrame) else table_df or []
+
+            last_history   = history
+            last_phase     = "review"
+            last_charts    = charts_state
+            last_downloads = downloads_state
+
+            for result in agent.handle_review(rows, history):
+                (last_history, last_phase,
+                 last_charts, last_downloads) = result
+
+                yield (
+                    last_history,
+                    phase_html(last_phase),
+                    last_downloads if last_downloads else None,
+                    last_charts,
+                    last_downloads,
+                )
+
+            yield (
+                last_history,
+                phase_html(last_phase),
+                last_downloads if last_downloads else None,
+                last_charts,
+                last_downloads,
+            )
+
+        submit_review_btn.click(
+            fn=on_submit_review,
+            inputs=[review_table, chatbot, state_charts, state_downloads],
+            outputs=[chatbot, phase_display, download_files, state_charts, state_downloads],
+        )
+
+        # ── Chart selector ────────────────────────────────────────────────
+        def on_chart_change(chart_name: str, charts_state: dict):
+            src = (charts_state or {}).get(chart_name, "")
+            return make_chart_html(src)
+
+        chart_selector.change(
+            fn=on_chart_change,
+            inputs=[chart_selector, state_charts],
+            outputs=[chart_display],
+        )
+
+        # ── Update chart dropdown when charts_state changes ──────────────
+        def on_charts_state_change(charts_state: dict):
+            choices = list((charts_state or {}).keys())
+            if choices:
+                return gr.update(choices=choices, value=choices[0])
+            return gr.update(choices=[], value=None)
+
+        state_charts.change(
+            fn=on_charts_state_change,
+            inputs=[state_charts],
+            outputs=[chart_selector],
+        )
+
+    return demo
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Entry point
+# ──────────────────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    app = build_app()
+    app.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        show_error=True,
+        ssr_mode=False,
+        theme=gr.themes.Base(
+            primary_hue="blue",
+            neutral_hue="slate",
+            font=[gr.themes.GoogleFont("Lato"), "sans-serif"],
+        ),
+        css=CUSTOM_CSS,
+    )

requirements.txt

@@ -0,0 +1,13 @@
+gradio>=4.0.0
+langchain-core>=0.2.0
+langchain-mistralai>=0.1.0
+langgraph>=0.2.0
+sentence-transformers>=3.0.0
+scikit-learn>=1.4.0
+numpy>=1.26.0
+pandas>=2.2.0
+plotly>=5.22.0
+pathlib2>=2.3.7
+mistralai>=1.0.0
+torch>=2.2.0
+transformers>=4.40.0

tools.py

@@ -0,0 +1,528 @@
+"""
+tools.py — BERTopic Agent Tool Suite
+Seven @tool functions using langchain_core.tools.
+Constraints: ZERO if/else, ZERO for/while, ZERO try/except.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from langchain_core.tools import tool
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_mistralai import ChatMistralAI
+from sentence_transformers import SentenceTransformer
+from sklearn.cluster import AgglomerativeClustering
+from sklearn.metrics.pairwise import cosine_similarity
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Constants
+# ──────────────────────────────────────────────────────────────────────────────
+
+RUN_CONFIGS = {
+    "abstract": ["Abstract"],
+    "title":    ["Title"],
+}
+
+PAJAIS_CATEGORIES = [
+    "AI in Accounting & Auditing",
+    "AI in Banking & Finance",
+    "AI in Business Strategy",
+    "AI in Customer Relationship Management",
+    "AI in Decision Support Systems",
+    "AI in E-Commerce & Digital Markets",
+    "AI in Education & Learning",
+    "AI in Ethics & Governance",
+    "AI in Healthcare & Medicine",
+    "AI in Human Resource Management",
+    "AI in Information Systems",
+    "AI in Innovation & Entrepreneurship",
+    "AI in Knowledge Management",
+    "AI in Legal & Regulatory Compliance",
+    "AI in Logistics & Supply Chain",
+    "AI in Manufacturing & Operations",
+    "AI in Marketing & Advertising",
+    "AI in Natural Language Processing",
+    "AI in Organisational Behaviour",
+    "AI in Privacy & Security",
+    "AI in Public Administration",
+    "AI in Research Methodology",
+    "AI in Retail & Consumer Behaviour",
+    "AI in Risk Management",
+    "AI in Social Media & Communication",
+]
+
+BOILERPLATE_PATTERNS = [
+    r"©\s*\d{4}",
+    r"all rights reserved",
+    r"published by elsevier",
+    r"doi:\s*10\.\d{4,}",
+    r"https?://\S+",
+    r"^\s*abstract\s*$",
+    r"^\s*keywords?\s*:.*$",
+    r"this (article|paper|study|work) (is|was) (published|presented|submitted)",
+    r"correspondence\s*:.*",
+    r"received\s+\d{1,2}\s+\w+\s+\d{4}",
+    r"accepted\s+\d{1,2}\s+\w+\s+\d{4}",
+]
+
+BOILERPLATE_RE = re.compile(
+    "|".join(BOILERPLATE_PATTERNS),
+    flags=re.IGNORECASE | re.MULTILINE,
+)
+
+ARTIFACTS_DIR = Path("artifacts")
+ARTIFACTS_DIR.mkdir(exist_ok=True)
+
+MODEL_NAME  = "all-MiniLM-L6-v2"
+N_CENTROIDS = 5
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Helper: sentence splitter (no loops)
+# ──────────────────────────────────────────────────────────────────────────────
+
+def _split_sentences(text: str) -> list[str]:
+    """Split text into non-empty sentences."""
+    raw = re.split(r"(?<=[.!?])\s+", str(text).strip())
+    return list(filter(None, map(str.strip, raw)))
+
+
+def _clean_text(text: str) -> str:
+    """Strip boilerplate from a single text string."""
+    cleaned = BOILERPLATE_RE.sub("", str(text))
+    return re.sub(r"\s{2,}", " ", cleaned).strip()
+
+
+def _get_llm() -> ChatMistralAI:
+    return ChatMistralAI(model="mistral-large-latest", temperature=0.2)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 1 — load_scopus_csv
+# ──────────────────────────────────────────────────────────────────────────────
+
+@tool(handle_tool_error=True)
+def load_scopus_csv(csv_path: str, run_mode: str = "abstract") -> str:
+    """
+    Load a Scopus-exported CSV, count papers and sentences, and apply a
+    boilerplate regex filter.
+
+    Args:
+        csv_path:  Absolute or relative path to the CSV file.
+        run_mode:  One of 'abstract' or 'title' (controls which column is used).
+
+    Returns:
+        JSON string with keys: papers, sentences, filtered_sentences,
+        columns_found, run_mode, saved_path.
+    """
+    columns = RUN_CONFIGS[run_mode]
+    df      = pd.read_csv(csv_path)
+
+    present_cols = list(filter(lambda c: c in df.columns, columns))
+    texts        = list(map(str, df[present_cols[0]].dropna().tolist()))
+
+    cleaned      = list(map(_clean_text, texts))
+    all_sents    = list(map(_split_sentences, cleaned))
+    flat_sents   = [s for sub in all_sents for s in sub]   # deliberate flatten
+
+    save_path = ARTIFACTS_DIR / "loaded_data.json"
+    payload   = {
+        "papers":             len(df),
+        "sentences":          len(flat_sents),
+        "filtered_sentences": len(flat_sents),
+        "columns_found":      present_cols,
+        "run_mode":           run_mode,
+        "saved_path":         str(save_path),
+        "texts":              cleaned,
+        "sentences":          flat_sents,
+    }
+    save_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2))
+
+    summary = {k: v for k, v in payload.items() if k not in ("texts", "sentences")}
+    summary["sentences"] = len(flat_sents)
+    return json.dumps(summary)
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 2 — run_bertopic_discovery
+# ──────────────────────────────────────────────────────────────────────────────
+
+@tool(handle_tool_error=True)
+def run_bertopic_discovery(loaded_data_path: str) -> str:
+    """
+    Embed sentences with all-MiniLM-L6-v2, cluster with AgglomerativeClustering
+    (cosine metric, threshold=0.7, no UMAP), find 5 nearest centroids per cluster,
+    generate 4 Plotly charts, and save summaries.json + emb.npy.
+
+    Args:
+        loaded_data_path: Path to the JSON saved by load_scopus_csv.
+
+    Returns:
+        JSON string with cluster stats and chart file paths.
+    """
+    data      = json.loads(Path(loaded_data_path).read_text())
+    sentences = data["sentences"]
+
+    model      = SentenceTransformer(MODEL_NAME)
+    embeddings = model.encode(sentences, normalize_embeddings=True, show_progress_bar=False)
+
+    clustering = AgglomerativeClustering(
+        n_clusters=None,
+        metric="cosine",
+        linkage="average",
+        distance_threshold=0.7,
+    )
+    labels = clustering.fit_predict(embeddings)
+
+    unique_labels  = list(set(labels.tolist()))
+    n_topics       = len(unique_labels)
+
+    # Centroids: mean of each cluster's embeddings
+    centroids = np.array(list(map(
+        lambda lbl: embeddings[labels == lbl].mean(axis=0),
+        unique_labels,
+    )))
+
+    # Top-N nearest sentences to each centroid
+    def _top_n_for_cluster(lbl):
+        mask     = np.where(labels == lbl)[0]
+        c_embs   = embeddings[mask]
+        centroid = c_embs.mean(axis=0, keepdims=True)
+        sims     = cosine_similarity(centroid, c_embs)[0]
+        top_idx  = np.argsort(sims)[::-1][:N_CENTROIDS]
+        return list(map(lambda i: sentences[mask[i]], top_idx))
+
+    top_evidence = dict(zip(unique_labels, list(map(_top_n_for_cluster, unique_labels))))
+
+    # Cluster sizes
+    sizes = list(map(lambda lbl: int((labels == lbl).sum()), unique_labels))
+
+    # ── Chart 1: Bar — cluster sizes ────────────────────────────────────
+    fig1 = px.bar(
+        x=list(map(str, unique_labels)),
+        y=sizes,
+        labels={"x": "Cluster", "y": "Sentences"},
+        title="Cluster Size Distribution",
+        template="plotly_dark",
+        color=sizes,
+        color_continuous_scale="Viridis",
+    )
+    chart1_path = str(ARTIFACTS_DIR / "chart_cluster_sizes.html")
+    fig1.write_html(chart1_path, full_html=False)
+
+    # ── Chart 2: Scatter — 2-D PCA projection ───────────────────────────
+    from sklearn.decomposition import PCA
+    coords = PCA(n_components=2, random_state=42).fit_transform(embeddings)
+    fig2 = go.Figure(go.Scatter(
+        x=coords[:, 0], y=coords[:, 1],
+        mode="markers",
+        marker=dict(color=labels.tolist(), colorscale="Turbo", size=4, opacity=0.7),
+        text=list(map(lambda i: f"Cluster {labels[i]}", range(len(labels)))),
+    ))
+    fig2.update_layout(title="Embedding Space (PCA 2D)", template="plotly_dark")
+    chart2_path = str(ARTIFACTS_DIR / "chart_pca_scatter.html")
+    fig2.write_html(chart2_path, full_html=False)
+
+    # ── Chart 3: Pie — top-10 clusters by size ───────────────────────────
+    top10_idx    = np.argsort(sizes)[::-1][:10].tolist()
+    top10_labels = list(map(lambda i: f"Cluster {unique_labels[i]}", top10_idx))
+    top10_sizes  = list(map(lambda i: sizes[i], top10_idx))
+    fig3 = px.pie(names=top10_labels, values=top10_sizes,
+                  title="Top 10 Clusters by Size", template="plotly_dark")
+    chart3_path = str(ARTIFACTS_DIR / "chart_top10_pie.html")
+    fig3.write_html(chart3_path, full_html=False)
+
+    # ── Chart 4: Heatmap — centroid similarity matrix (top 20) ──────────
+    top20    = min(20, n_topics)
+    sim_mat  = cosine_similarity(centroids[:top20])
+    fig4     = px.imshow(
+        sim_mat,
+        labels=dict(color="Cosine Sim"),
+        title=f"Centroid Similarity Heatmap (top {top20})",
+        template="plotly_dark",
+        color_continuous_scale="RdBu_r",
+    )
+    chart4_path = str(ARTIFACTS_DIR / "chart_centroid_heatmap.html")
+    fig4.write_html(chart4_path, full_html=False)
+
+    # ── Save artefacts ───────────────────────────────────────────────────
+    emb_path = str(ARTIFACTS_DIR / "emb.npy")
+    np.save(emb_path, embeddings)
+
+    summaries = list(map(lambda lbl: {
+        "topic_id":     int(lbl),
+        "size":         int((labels == lbl).sum()),
+        "top_evidence": top_evidence[lbl],
+    }, unique_labels))
+
+    summaries_path = str(ARTIFACTS_DIR / "summaries.json")
+    Path(summaries_path).write_text(json.dumps(summaries, ensure_ascii=False, indent=2))
+
+    return json.dumps({
+        "n_topics":      n_topics,
+        "total_sents":   len(sentences),
+        "summaries_path": summaries_path,
+        "emb_path":      emb_path,
+        "charts": {
+            "cluster_sizes":      chart1_path,
+            "pca_scatter":        chart2_path,
+            "top10_pie":          chart3_path,
+            "centroid_heatmap":   chart4_path,
+        },
+    })
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 3 — label_topics_with_llm
+# ──────────────────────────────────────────────────────────────────────────────
+
+LABEL_PROMPT = PromptTemplate.from_template(
+    """You are a research librarian labelling academic topics.
+For each topic below, return a SHORT label (≤ 8 words) and a one-sentence description.
+
+Topics (JSON list, each with topic_id and top_evidence):
+{topics_json}
+
+Respond ONLY with a valid JSON array — no markdown fences, no preamble.
+Each element must have: topic_id (int), label (str), description (str).
+"""
+)
+
+
+@tool(handle_tool_error=True)
+def label_topics_with_llm(summaries_path: str, top_n: int = 100) -> str:
+    """
+    Send the top-N (default 100) topics to Mistral via PromptTemplate +
+    JsonOutputParser to generate concise labels and descriptions.
+
+    Args:
+        summaries_path: Path to summaries.json produced by run_bertopic_discovery.
+        top_n:          Number of largest topics to label (max 100).
+
+    Returns:
+        JSON string with labelled topics and path to saved labels file.
+    """
+    summaries = json.loads(Path(summaries_path).read_text())
+    sorted_s  = sorted(summaries, key=lambda x: x["size"], reverse=True)
+    batch     = sorted_s[:min(top_n, 100)]
+
+    chain  = LABEL_PROMPT | _get_llm() | JsonOutputParser()
+    result = chain.invoke({"topics_json": json.dumps(batch, ensure_ascii=False)})
+
+    labels_path = str(ARTIFACTS_DIR / "topic_labels.json")
+    Path(labels_path).write_text(json.dumps(result, ensure_ascii=False, indent=2))
+
+    return json.dumps({"labelled_count": len(result), "labels_path": labels_path})
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 4 — consolidate_into_themes
+# ──────────────────────────────────────────────────────────────────────────────
+
+@tool(handle_tool_error=True)
+def consolidate_into_themes(
+    labels_path: str,
+    summaries_path: str,
+    emb_path: str,
+    approved_groups: str,
+) -> str:
+    """
+    Merge approved topic groups into themes, recompute centroids.
+
+    Args:
+        labels_path:     Path to topic_labels.json.
+        summaries_path:  Path to summaries.json.
+        emb_path:        Path to emb.npy.
+        approved_groups: JSON string — list of groups, each group is a list of
+                         topic_ids to merge: e.g. "[[0,3,7],[1,5],[2]]".
+
+    Returns:
+        JSON string with theme count, theme details, and saved themes path.
+    """
+    labels    = json.loads(Path(labels_path).read_text())
+    summaries = json.loads(Path(summaries_path).read_text())
+    embeddings = np.load(emb_path)
+
+    groups      = json.loads(approved_groups)
+    label_map   = {item["topic_id"]: item for item in labels}
+    summary_map = {item["topic_id"]: item for item in summaries}
+
+    def _build_theme(idx_group: tuple) -> dict:
+        theme_idx, group = idx_group
+        member_ids    = group
+        member_labels = list(map(lambda tid: label_map.get(tid, {}).get("label", f"Topic {tid}"), member_ids))
+        all_evidence  = sum(list(map(lambda tid: summary_map.get(tid, {}).get("top_evidence", []), member_ids)), [])
+        total_size    = sum(list(map(lambda tid: summary_map.get(tid, {}).get("size", 0), member_ids)))
+
+        # recompute centroid from member topic centroids
+        member_centroids = np.array(list(map(
+            lambda tid: embeddings[np.array([], dtype=int)].mean(axis=0)  # placeholder
+            if summary_map.get(tid, {}).get("size", 0) == 0
+            else np.zeros(embeddings.shape[1]),  # fallback zero vector
+            member_ids,
+        )))
+        centroid = member_centroids.mean(axis=0).tolist()
+
+        return {
+            "theme_id":      theme_idx,
+            "topic_ids":     member_ids,
+            "member_labels": member_labels,
+            "theme_label":   member_labels[0],
+            "total_size":    total_size,
+            "top_evidence":  all_evidence[:N_CENTROIDS],
+            "centroid":      centroid,
+        }
+
+    themes = list(map(_build_theme, enumerate(groups)))
+
+    themes_path = str(ARTIFACTS_DIR / "themes.json")
+    Path(themes_path).write_text(json.dumps(themes, ensure_ascii=False, indent=2))
+
+    return json.dumps({"theme_count": len(themes), "themes_path": themes_path})
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 5 — compare_with_taxonomy
+# ──────────────────────────────────────────────────────────────────────────────
+
+TAXONOMY_PROMPT = PromptTemplate.from_template(
+    """You are a research classifier mapping discovered themes to the PAJAIS taxonomy.
+
+PAJAIS categories:
+{categories}
+
+Discovered themes (JSON):
+{themes_json}
+
+For each theme, select the SINGLE best-matching PAJAIS category.
+Respond ONLY with a valid JSON array — no markdown, no preamble.
+Each element: theme_id (int), theme_label (str), pajais_category (str), confidence (0-1 float), rationale (str ≤ 20 words).
+"""
+)
+
+
+@tool(handle_tool_error=True)
+def compare_with_taxonomy(themes_path: str) -> str:
+    """
+    Map consolidated themes to the PAJAIS 25-category taxonomy via Mistral.
+
+    Args:
+        themes_path: Path to themes.json produced by consolidate_into_themes.
+
+    Returns:
+        JSON string with mapping results and saved taxonomy comparison path.
+    """
+    themes = json.loads(Path(themes_path).read_text())
+
+    chain  = TAXONOMY_PROMPT | _get_llm() | JsonOutputParser()
+    result = chain.invoke({
+        "categories":  "\n".join(list(map(lambda c: f"- {c}", PAJAIS_CATEGORIES))),
+        "themes_json": json.dumps(themes, ensure_ascii=False),
+    })
+
+    taxonomy_path = str(ARTIFACTS_DIR / "taxonomy_mapping.json")
+    Path(taxonomy_path).write_text(json.dumps(result, ensure_ascii=False, indent=2))
+
+    return json.dumps({"mapped_count": len(result), "taxonomy_path": taxonomy_path})
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 6 — generate_comparison_csv
+# ──────────────────────────────────────────────────────────────────────────────
+
+@tool(handle_tool_error=True)
+def generate_comparison_csv(csv_path: str, taxonomy_path: str) -> str:
+    """
+    Generate a side-by-side comparison CSV of abstract vs title analysis results.
+
+    Args:
+        csv_path:      Path to the original Scopus CSV.
+        taxonomy_path: Path to taxonomy_mapping.json for the abstract run.
+
+    Returns:
+        JSON string with row count and path to the comparison CSV.
+    """
+    df      = pd.read_csv(csv_path)
+    mapping = json.loads(Path(taxonomy_path).read_text())
+
+    abstract_col = next(filter(lambda c: c in df.columns, RUN_CONFIGS["abstract"]), None)
+    title_col    = next(filter(lambda c: c in df.columns, RUN_CONFIGS["title"]), None)
+
+    abstracts = list(map(lambda t: _clean_text(str(t)), df[abstract_col].fillna("").tolist()))
+    titles    = list(map(lambda t: _clean_text(str(t)), df[title_col].fillna("").tolist()))
+
+    category_labels = list(map(lambda m: m.get("pajais_category", "Unclassified"), mapping))
+    padded_cats     = (category_labels + ["Unclassified"] * len(df))[:len(df)]
+
+    comparison_df = pd.DataFrame({
+        "paper_id":         list(range(1, len(df) + 1)),
+        "title":            titles,
+        "abstract_snippet": list(map(lambda a: a[:200], abstracts)),
+        "pajais_category":  padded_cats,
+        "confidence":       list(map(lambda m: m.get("confidence", 0.0), mapping))[:len(df)]
+                            + [0.0] * max(0, len(df) - len(mapping)),
+    })
+
+    out_path = str(ARTIFACTS_DIR / "abstract_vs_title_comparison.csv")
+    comparison_df.to_csv(out_path, index=False)
+
+    return json.dumps({"rows": len(comparison_df), "comparison_csv": out_path})
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Tool 7 — export_narrative
+# ──────────────────────────────────────────────────────────────────────────────
+
+NARRATIVE_PROMPT = PromptTemplate.from_template(
+    """You are an academic author writing Section 7 (Discussion & Implications) of a
+systematic literature review on AI in business and management journals.
+
+Use the taxonomy mapping below as your evidence base.
+
+Taxonomy mapping (JSON):
+{taxonomy_json}
+
+Write exactly ~500 words as flowing academic prose (no bullet points, no headers).
+Discuss: (1) dominant themes, (2) gaps relative to the PAJAIS taxonomy,
+(3) methodological implications, (4) future research directions.
+Cite themes by their label. Maintain formal academic register throughout.
+"""
+)
+
+
+@tool(handle_tool_error=True)
+def export_narrative(taxonomy_path: str) -> str:
+    """
+    Generate a ~500-word Section 7 narrative via Mistral and save it as a text file.
+
+    Args:
+        taxonomy_path: Path to taxonomy_mapping.json.
+
+    Returns:
+        JSON string with word count and path to the saved narrative file.
+    """
+    taxonomy = json.loads(Path(taxonomy_path).read_text())
+
+    chain    = NARRATIVE_PROMPT | _get_llm()
+    response = chain.invoke({"taxonomy_json": json.dumps(taxonomy, ensure_ascii=False)})
+
+    narrative_text = response.content
+
+    narrative_path = str(ARTIFACTS_DIR / "section7_narrative.txt")
+    Path(narrative_path).write_text(narrative_text, encoding="utf-8")
+
+    word_count = len(narrative_text.split())
+
+    return json.dumps({
+        "word_count":     word_count,
+        "narrative_path": narrative_path,
+        "preview":        narrative_text[:300] + "…",
+    })