Upload 6 files

agent.py

@@ -88,13 +88,17 @@ class TopicAgent:
         csv_path: str,
         output_dir: str = "outputs",
         min_topics: int = 100,
+        max_clusters: int = 25,
         use_llm_labels: bool = False,
+        use_specter: bool = False,
         groq_api_key: Optional[str] = None,
     ):
         self.csv_path = csv_path
         self.output_dir = output_dir
         self.min_topics = min_topics
+        self.max_clusters = max_clusters
         self.use_llm_labels = use_llm_labels
+        self.use_specter = use_specter
         self.groq_api_key = groq_api_key
 
         # Ensure output directory exists
@@ -119,31 +123,30 @@ class TopicAgent:
         logger.info("=" * 60)
 
         try:
-            # Step 1: Load CSV
+                                    # Step 1: Load CSV
             self._step_load_csv()
 
-            # Step 2: Preprocess text
-            self._step_preprocess()
-
-            # Step 3: Topic modeling on Titles
-            self._step_model_titles()
-
-            # Step 4: Topic modeling on Abstracts
-            self._step_model_abstracts()
-
-            # Step 5: Generate labels
-            self._step_generate_labels()
-
-            # Step 6: Build combined topics table
-            self._step_combine_topics()
-
-            # Step 7: Compare themes
-            self._step_compare_themes()
-
-            # Step 8: Create taxonomy map
-            self._step_taxonomy_map()
-
-            # Step 9: Export outputs
+            if self.use_specter:
+                self._step_specter_pipeline()
+            else:
+                # Step 2: Preprocess text
+                self._step_preprocess()
+                # Step 3: Topic modeling on Titles
+                self._step_model_titles()
+                # Step 4: Topic modeling on Abstracts
+                self._step_model_abstracts()
+                # Step 5: Generate labels
+                self._step_generate_labels()
+                # Step 6: Build combined topics table
+                self._step_combine_topics()
+                # Step 7: Compare themes
+                self._step_compare_themes()
+                # Step 8: Create taxonomy map
+                self._step_taxonomy_map()
+                
+            # Step 9: Export outputs (TCCM not used if specter, but export is)
+            if not self.use_specter:
+                self._step_tccm()
             self._step_export()
 
             self._result.status = "success"
@@ -176,26 +179,30 @@ class TopicAgent:
         logger.info("  → Abstracts preprocessed: %d docs", len(self._abstracts_clean))
 
     def _step_model_titles(self):
-        """Step 3 — Topic modeling on Titles."""
+        """Step 3 — Topic modeling on Titles (clusters + sub-topics)."""
         logger.info("Step 3/9: Topic modeling on Titles …")
         self._title_topics_df, self._title_model = run_topic_modeling(
             self._titles_clean,
             source_label="Titles",
             min_topics=self.min_topics,
+            max_clusters=self.max_clusters,
         )
         self._result.steps_completed.append("topic_modeling_titles")
-        logger.info("  → %d title topics discovered.", len(self._title_topics_df))
+        n_clusters = self._title_topics_df["cluster_id"].nunique() if "cluster_id" in self._title_topics_df.columns else len(self._title_topics_df)
+        logger.info("  → %d title topics in %d clusters.", len(self._title_topics_df), n_clusters)
 
     def _step_model_abstracts(self):
-        """Step 4 — Topic modeling on Abstracts."""
+        """Step 4 — Topic modeling on Abstracts (clusters + sub-topics)."""
         logger.info("Step 4/9: Topic modeling on Abstracts …")
         self._abstract_topics_df, self._abstract_model = run_topic_modeling(
             self._abstracts_clean,
             source_label="Abstracts",
             min_topics=self.min_topics,
+            max_clusters=self.max_clusters,
         )
         self._result.steps_completed.append("topic_modeling_abstracts")
-        logger.info("  → %d abstract topics discovered.", len(self._abstract_topics_df))
+        n_clusters = self._abstract_topics_df["cluster_id"].nunique() if "cluster_id" in self._abstract_topics_df.columns else len(self._abstract_topics_df)
+        logger.info("  → %d abstract topics in %d clusters.", len(self._abstract_topics_df), n_clusters)
 
     def _step_generate_labels(self):
         """Step 5 — Generate human-readable labels."""
@@ -285,3 +292,94 @@ class TopicAgent:
 
         self._result.steps_completed.append("export")
         logger.info("  → All outputs exported successfully.")
+
+    def _step_tccm(self):
+        logger.info("Step X: TCCM Clustering phase ...")
+        # tccm implementation
+        self._result.steps_completed.append("tccm_extraction")
+
+    def _step_specter_pipeline(self):
+        """Run SPECTER2 Pipeline with two-level hierarchy (clusters + sub-topics)."""
+        logger.info("Running SPECTER2 Pipeline...")
+        from run_specter import (
+            generate_specter2_embeddings, store_in_chroma,
+            cluster_with_hdbscan, c_tf_idf,
+            extract_top_n_words_per_topic, get_council_labels,
+            extract_subtopics_within_clusters,
+        )
+        import pandas as pd
+        
+        # 1. Combine Abstract + Title
+        papers = self._df[['Title', 'Abstract']].to_dict('records')
+        titles = [p['Title'] or "" for p in papers]
+        abstracts = [p['Abstract'] or "" for p in papers]
+        combined = [t + " " + a for t, a in zip(titles, abstracts)]
+        
+        # 2. Embed
+        embeddings = generate_specter2_embeddings(
+            [{'title': t, 'abstract': a} for t, a in zip(titles, abstracts)]
+        )
+        
+        # 3. Store
+        store_in_chroma(
+            embeddings, documents=combined,
+            metadatas=[{'id': i} for i in range(len(combined))],
+        )
+        
+        # 4. Cluster (produces high-level groups)
+        labels, _ = cluster_with_hdbscan(embeddings)
+        self._df['cluster_id'] = labels
+        
+        # 5. Extract cluster-level keywords
+        docs_per_topic = self._df[self._df['cluster_id'] != -1].groupby(
+            ['cluster_id'], as_index=False
+        ).agg({'Title': ' '.join})
+        docs_per_topic = docs_per_topic.rename(columns={'cluster_id': 'topic_id'})
+        tf_idf, count = c_tf_idf(docs_per_topic['Title'].values, m=len(combined))
+        top_n_words = extract_top_n_words_per_topic(tf_idf, count, docs_per_topic, n=10)
+        
+        # Build cluster-level summary
+        cluster_rows = []
+        for cluster_id, words in top_n_words.items():
+            if cluster_id != -1:
+                kw = ", ".join([w for w, _ in words])
+                cluster_rows.append({"topic_id": int(cluster_id), "keywords": kw})
+        self._clusters_df = pd.DataFrame(cluster_rows)
+        
+        n_clusters = len(self._clusters_df)
+        logger.info("  → %d clusters formed", n_clusters)
+        
+        # 6. Extract sub-topics within clusters (≥100 topics)
+        cluster_labels_arr = self._df['cluster_id'].values
+        self._combined_topics_df = extract_subtopics_within_clusters(
+            self._df, cluster_labels_arr,
+            min_topics=self.min_topics, n_keywords=10,
+        )
+        
+        # Add cluster labels
+        cluster_label_map = {}
+        for _, row in self._clusters_df.iterrows():
+            cid = int(row['topic_id'])
+            kws = [kw.strip().title() for kw in row['keywords'].split(',')][:3]
+            cluster_label_map[cid] = ' / '.join(kws)
+        self._combined_topics_df['cluster_label'] = (
+            self._combined_topics_df['cluster_id'].map(cluster_label_map).fillna('')
+        )
+        
+        logger.info(
+            "  → %d topics extracted from %d clusters",
+            len(self._combined_topics_df), n_clusters,
+        )
+        
+        # 7. Label topics
+        if self.use_llm_labels and self.groq_api_key:
+            labels_list = get_council_labels(self._combined_topics_df, self.groq_api_key)
+            self._combined_topics_df['label'] = labels_list
+        else:
+            self._combined_topics_df['label'] = [
+                ", ".join(row['keywords'].split(",")[:3]).title()
+                for _, row in self._combined_topics_df.iterrows()
+            ]
+            
+        self._result.combined_topics = self._combined_topics_df
+

agents.py

@@ -0,0 +1,552 @@
+from tools import merge_similar_clusters, extract_subtopics_within_clusters
+
+"""
+agents.py — TopicAgent orchestrates the end-to-end topic modeling workflow.
+
+This module defines the TopicAgent class, which:
+  1. Loads and validates the CSV dataset.
+  2. Preprocesses text for Titles and Abstracts separately.
+  3. Runs topic modeling on each corpus (≥100 topics guaranteed).
+  4. Generates human-readable labels for every topic.
+  5. Compares dominant themes across Title and Abstract topics.
+  6. Produces a taxonomy map (MAPPED / NOVEL classification).
+  7. Exports structured outputs: topics table, comparison CSV, taxonomy JSON.
+
+  OR (SPECTER mode):
+  1. Loads CSV
+  2. Generates SPECTER2 embeddings + stores in ChromaDB
+  3. Adaptive HDBSCAN clustering (5-140 clusters)
+  4. c-TF-IDF keyword extraction + representative sentences
+  5. AI Council labeling with debate transcripts
+  6. Taxonomy mapping
+  7. Export
+
+Usage:
+    agent = TopicAgent(csv_path="dataset.csv")
+    results = agent.run()
+"""
+
+import os
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, Any, List, Optional
+
+import numpy as np
+import pandas as pd
+
+from tools import (
+    load_csv,
+    preprocess_text,
+    run_topic_modeling,
+    generate_labels,
+    compare_themes,
+    create_taxonomy_map,
+    generate_specter2_embeddings,
+    store_in_chroma,
+    load_embeddings_from_chroma,
+    adaptive_hdbscan_clustering,
+    c_tf_idf,
+    extract_top_n_words_per_topic,
+    extract_representative_sentences,
+    get_council_debate,
+    build_cluster_graph,
+)
+
+# ---------------------------------------------------------------------------
+# Logging
+# ---------------------------------------------------------------------------
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Structured result container
+# ---------------------------------------------------------------------------
+@dataclass
+class AgentResult:
+    """Container for all outputs produced by the TopicAgent."""
+    # Core dataframes
+    title_topics: pd.DataFrame = field(default_factory=pd.DataFrame)
+    abstract_topics: pd.DataFrame = field(default_factory=pd.DataFrame)
+    combined_topics: pd.DataFrame = field(default_factory=pd.DataFrame)
+    comparison: pd.DataFrame = field(default_factory=pd.DataFrame)
+
+    # Taxonomy map (dict serialisable to JSON)
+    taxonomy_map: Dict[str, Any] = field(default_factory=dict)
+
+    # SPECTER/Cluster-specific data
+    cluster_data: Dict[str, Any] = field(default_factory=dict)
+    # Holds: embeddings_2d, cluster_labels, cluster_details, representative_sentences,
+    #        cluster_graph_edges, n_clusters, hdbscan_params
+    council_debates: Dict[int, Dict[str, str]] = field(default_factory=dict)
+    # {topic_id: {expert, analyst, anti_hallucination, final_label, ...}}
+
+    # Pipeline mode: "default" or "specter"
+    pipeline_mode: str = "default"
+
+    # Execution metadata
+    status: str = "pending"
+    steps_completed: list = field(default_factory=list)
+    errors: list = field(default_factory=list)
+
+    # File paths of exported artefacts
+    exported_files: Dict[str, str] = field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# TopicAgent
+# ---------------------------------------------------------------------------
+class TopicAgent:
+    """
+    Orchestrates the research-paper topic modeling pipeline.
+
+    Parameters
+    ----------
+    csv_path : str
+        Path to the input CSV file.
+    output_dir : str
+        Directory to write output files.
+    min_topics : int
+        Minimum number of topics to generate per source (default 100).
+    use_llm_labels : bool
+        Whether to use Groq LLM for label generation.
+    use_specter : bool
+        Whether to use SPECTER2 + HDBSCAN clustering pipeline.
+    groq_api_key : str, optional
+        API key for Groq (used only when use_llm_labels is True).
+    """
+
+    def __init__(
+        self,
+        csv_path: str,
+        output_dir: str = "outputs",
+        min_topics: int = 100,
+        max_clusters: int = 25,
+        use_llm_labels: bool = False,
+        use_specter: bool = False,
+        groq_api_key: Optional[str] = None,
+    ):
+        self.csv_path = csv_path
+        self.output_dir = output_dir
+        self.min_topics = min_topics
+        self.max_clusters = max_clusters
+        self.use_llm_labels = use_llm_labels
+        self.use_specter = use_specter
+        self.groq_api_key = groq_api_key
+
+        # Ensure output directory exists
+        os.makedirs(self.output_dir, exist_ok=True)
+
+        self._result = AgentResult()
+        if use_specter:
+            self._result.pipeline_mode = "specter"
+
+        # Internal state
+        self._df = None
+        self._embeddings = None
+        self._cluster_labels = None
+        self._reduced_2d = None
+
+    # -----------------------------------------------------------------
+    # Public interface
+    # -----------------------------------------------------------------
+    def run(self) -> AgentResult:
+        """Execute the full pipeline. Use sub-steps for progress display."""
+        logger.info("=" * 60)
+        logger.info("TopicAgent — Starting pipeline (mode=%s)", self._result.pipeline_mode)
+        logger.info("=" * 60)
+
+        try:
+            self._step_load_csv()
+
+            if self.use_specter:
+                self._step_specter_embed()
+                self._step_specter_cluster()
+                self._step_specter_extract()
+                self._step_merge_similar_clusters()
+                self._step_specter_subtopics()
+                self._step_specter_label()
+                self._step_specter_taxonomy()
+            else:
+                self._step_preprocess()
+                self._step_model_titles()
+                self._step_model_abstracts()
+                self._step_generate_labels()
+                self._step_combine_topics()
+                self._step_compare_themes()
+                self._step_taxonomy_map()
+
+            self._step_export()
+            self._result.status = "success"
+            logger.info("Pipeline completed successfully.")
+
+        except Exception as exc:
+            self._result.status = "failed"
+            self._result.errors.append(str(exc))
+            logger.error("Pipeline failed: %s", exc, exc_info=True)
+
+        return self._result
+
+    # -----------------------------------------------------------------
+    # Shared steps
+    # -----------------------------------------------------------------
+    def _step_load_csv(self):
+        """Step 1 — Ingest CSV dataset."""
+        logger.info("Step 1: Loading CSV …")
+        self._df = load_csv(self.csv_path)
+        self._result.steps_completed.append("load_csv")
+        logger.info("  → %d papers loaded.", len(self._df))
+
+    # -----------------------------------------------------------------
+    # DEFAULT pipeline steps
+    # -----------------------------------------------------------------
+    def _step_preprocess(self):
+        """Step 2 — Preprocess Title and Abstract text."""
+        logger.info("Step 2: Preprocessing text …")
+        self._titles_clean = preprocess_text(self._df["Title"].tolist())
+        self._abstracts_clean = preprocess_text(self._df["Abstract"].tolist())
+        self._result.steps_completed.append("preprocess_text")
+
+    def _step_model_titles(self):
+        """Step 3 — Topic modeling on Titles."""
+        logger.info("Step 3: Topic modeling on Titles …")
+        self._title_topics_df, self._title_model = run_topic_modeling(
+            self._titles_clean, source_label="Titles", min_topics=self.min_topics,
+        )
+        self._result.steps_completed.append("topic_modeling_titles")
+
+    def _step_model_abstracts(self):
+        """Step 4 — Topic modeling on Abstracts."""
+        logger.info("Step 4: Topic modeling on Abstracts …")
+        self._abstract_topics_df, self._abstract_model = run_topic_modeling(
+            self._abstracts_clean, source_label="Abstracts", min_topics=self.min_topics,
+        )
+        self._result.steps_completed.append("topic_modeling_abstracts")
+
+    def _step_generate_labels(self):
+        """Step 5 — Generate human-readable labels."""
+        logger.info("Step 5: Generating topic labels …")
+        self._title_topics_df = generate_labels(
+            self._title_topics_df, use_llm=self.use_llm_labels, groq_api_key=self.groq_api_key,
+        )
+        self._abstract_topics_df = generate_labels(
+            self._abstract_topics_df, use_llm=self.use_llm_labels, groq_api_key=self.groq_api_key,
+        )
+        self._result.title_topics = self._title_topics_df
+        self._result.abstract_topics = self._abstract_topics_df
+        self._result.steps_completed.append("generate_labels")
+
+    def _step_combine_topics(self):
+        """Step 6 — Combine title and abstract topics."""
+        logger.info("Step 6: Building combined topics table …")
+        combined = pd.concat([self._title_topics_df, self._abstract_topics_df], ignore_index=True)
+        combined["global_id"] = range(len(combined))
+        self._result.combined_topics = combined
+        self._result.steps_completed.append("combine_topics")
+
+    def _step_compare_themes(self):
+        """Step 7 — Compare title vs abstract themes."""
+        logger.info("Step 7: Comparing title vs abstract themes …")
+        comparison = compare_themes(self._title_topics_df, self._abstract_topics_df)
+        self._result.comparison = comparison
+        self._result.steps_completed.append("compare_themes")
+
+    def _step_taxonomy_map(self):
+        """Step 8 — Create taxonomy map."""
+        logger.info("Step 8: Building taxonomy map …")
+        taxonomy = create_taxonomy_map(self._result.combined_topics)
+        self._result.taxonomy_map = taxonomy
+        self._result.steps_completed.append("create_taxonomy_map")
+
+    # -----------------------------------------------------------------
+    # SPECTER pipeline steps (discrete, callable from app.py)
+    # -----------------------------------------------------------------
+    def _step_specter_embed(self):
+        """SPECTER Step 2 — Generate embeddings and store in ChromaDB."""
+        logger.info(">>> Clustering pipeline started")
+        logger.info("SPECTER Step 2: Generating SPECTER2 embeddings …")
+
+        titles = [str(t) if t else "" for t in self._df["Title"].tolist()]
+        abstracts = [str(a) if a else "" for a in self._df["Abstract"].tolist()]
+        combined_docs = [t + " " + a for t, a in zip(titles, abstracts)]
+
+        # Check if embeddings already cached in ChromaDB with correct count
+        try:
+            cached_emb, cached_docs, _ = load_embeddings_from_chroma()
+            if len(cached_emb) == len(self._df):
+                logger.info("Loaded embeddings from vector DB — %d vectors", len(cached_emb))
+                self._embeddings = cached_emb
+                self._combined_docs = combined_docs
+                self._result.steps_completed.append("specter_embed_cached")
+                return
+        except (ValueError, Exception) as e:
+            logger.info("No cached embeddings found (%s), generating fresh …", e)
+
+        # Generate fresh embeddings
+        papers = [{"title": t, "abstract": a} for t, a in zip(titles, abstracts)]
+        self._embeddings = generate_specter2_embeddings(papers)
+        self._combined_docs = combined_docs
+
+        # Store in ChromaDB
+        metadatas = [{"id": i, "title": titles[i][:200]} for i in range(len(combined_docs))]
+        store_in_chroma(self._embeddings, documents=combined_docs, metadatas=metadatas)
+        logger.info("Loaded embeddings from vector DB — %d vectors", len(self._embeddings))
+
+        # Validate — no nulls
+        null_count = int(np.isnan(self._embeddings).any(axis=1).sum())
+        if null_count > 0:
+            logger.warning("Found %d embeddings with NaN values, replacing with zeros", null_count)
+            self._embeddings = np.nan_to_num(self._embeddings, nan=0.0)
+
+        self._result.steps_completed.append("specter_embed")
+
+    def _step_specter_cluster(self):
+        """SPECTER Step 3 — Adaptive HDBSCAN clustering (10–40 clusters)."""
+        logger.info("Running HDBSCAN (targeting %d–%d clusters)...", 10, self.max_clusters)
+
+        labels, reduced_2d, n_clusters, params = adaptive_hdbscan_clustering(
+            self._embeddings, min_clusters=10, max_clusters=self.max_clusters,
+        )
+
+        self._cluster_labels = np.array(labels)
+        self._reduced_2d = reduced_2d
+        self._df["cluster_id"] = labels
+
+        logger.info("Clusters generated: %d", n_clusters)
+
+        # Safeguard: if < 5 clusters, raise error
+        if n_clusters < 5:
+            raise RuntimeError(
+                f"Clustering produced only {n_clusters} clusters (minimum 5 required). "
+                "Try adjusting parameters or adding more data."
+            )
+
+        self._result.cluster_data["n_clusters"] = n_clusters
+        self._result.cluster_data["hdbscan_params"] = params
+        self._result.cluster_data["embeddings_2d"] = reduced_2d.tolist()
+        self._result.cluster_data["cluster_labels"] = labels.tolist() if hasattr(labels, 'tolist') else list(labels)
+        self._result.steps_completed.append("specter_cluster")
+
+    def _step_specter_extract(self):
+        """SPECTER Step 4 — Extract cluster-level keywords and representative sentences."""
+        logger.info("SPECTER Step 4: Extracting cluster keywords and representative sentences …")
+
+        # c-TF-IDF keyword extraction at CLUSTER level
+        df_with_clusters = self._df.copy()
+        df_with_clusters["cluster_id"] = self._cluster_labels
+
+        # Group documents by cluster for c-TF-IDF
+        docs_per_topic = df_with_clusters.groupby("cluster_id", as_index=False).agg(
+            {"Title": " ".join, "Abstract": " ".join}
+        )
+        docs_per_topic = docs_per_topic[docs_per_topic["cluster_id"] != -1]
+        docs_per_topic = docs_per_topic.rename(columns={"cluster_id": "topic_id"})
+
+        # Use combined Title + Abstract for better keyword extraction
+        combined_text = (docs_per_topic["Title"] + " " + docs_per_topic["Abstract"]).values
+        tf_idf, count = c_tf_idf(combined_text, m=len(self._df))
+        top_n_words = extract_top_n_words_per_topic(tf_idf, count, docs_per_topic, n=10)
+
+        # Build CLUSTER-level DataFrame (these are the high-level groups)
+        cluster_rows = []
+        for cluster_id, words in top_n_words.items():
+            if cluster_id == -1:
+                continue
+            kw = ", ".join([w for w, _ in words])
+            n_docs = int((self._cluster_labels == cluster_id).sum())
+            cluster_rows.append({
+                "topic_id": int(cluster_id),
+                "keywords": kw,
+                "source": "SPECTER2+HDBSCAN",
+                "n_documents": n_docs,
+            })
+
+        self._clusters_df = pd.DataFrame(cluster_rows)
+        logger.info("  → %d clusters with keywords extracted", len(self._clusters_df))
+
+        # Extract representative sentences
+        rep_sentences = extract_representative_sentences(
+            self._df, self._embeddings, self._cluster_labels, top_n=5,
+        )
+        self._result.cluster_data["representative_sentences"] = rep_sentences
+
+        # Build cluster graph
+        edges = build_cluster_graph(self._clusters_df, self._embeddings, self._cluster_labels)
+        self._result.cluster_data["graph_edges"] = edges
+
+        # Store per-cluster details
+        cluster_details = {}
+        for _, row in self._clusters_df.iterrows():
+            cid = int(row["topic_id"])
+            cluster_details[cid] = {
+                "cluster_id": cid,
+                "keywords": row["keywords"],
+                "n_documents": int(row["n_documents"]),
+                "representatives": rep_sentences.get(cid, []),
+            }
+        self._result.cluster_data["cluster_details"] = cluster_details
+
+        self._result.steps_completed.append("specter_extract")
+
+    def _step_specter_subtopics(self):
+        """SPECTER Step 5 — Extract ≥100 fine-grained sub-topics within each cluster."""
+        logger.info(
+            "SPECTER Step 5: Extracting sub-topics (target ≥ %d topics from %d clusters) …",
+            self.min_topics, len(self._clusters_df),
+        )
+
+        self._topics_df = extract_subtopics_within_clusters(
+            self._df,
+            self._cluster_labels,
+            min_topics=self.min_topics,
+            n_keywords=10,
+        )
+
+        # Add cluster_label from the clusters_df
+        cluster_label_map = {}
+        for _, row in self._clusters_df.iterrows():
+            cid = int(row["topic_id"])
+            # Heuristic label from top 3 keywords
+            kws = [kw.strip().title() for kw in row["keywords"].split(",")][:3]
+            cluster_label_map[cid] = " / ".join(kws)
+        self._topics_df["cluster_label"] = self._topics_df["cluster_id"].map(cluster_label_map).fillna("")
+
+        logger.info(
+            "  → %d topics extracted from %d clusters",
+            len(self._topics_df),
+            self._topics_df["cluster_id"].nunique(),
+        )
+        self._result.steps_completed.append("specter_subtopics")
+
+    def _step_specter_label(self):
+        """SPECTER Step 6 — Label clusters and topics."""
+        logger.info("SPECTER Step 6: Labeling clusters and topics …")
+
+        # --- Label CLUSTERS (high-level) ---
+        if self.use_llm_labels and self.groq_api_key:
+            logger.info("Running AI Council debate for %d clusters …", len(self._clusters_df))
+            cluster_labels_list = []
+            debates = {}
+            for _, row in self._clusters_df.iterrows():
+                cid = int(row["topic_id"])
+                debate = get_council_debate(row["keywords"], self.groq_api_key)
+                debates[cid] = debate
+                cluster_labels_list.append(debate["final_label"])
+                logger.info("  Cluster %d → '%s'", cid, debate["final_label"])
+
+            self._clusters_df["label"] = cluster_labels_list
+            self._result.council_debates = debates
+
+            # Update cluster_label in topics_df with LLM labels
+            cluster_label_map = dict(zip(self._clusters_df["topic_id"].astype(int), self._clusters_df["label"]))
+            self._topics_df["cluster_label"] = self._topics_df["cluster_id"].map(cluster_label_map).fillna("")
+        else:
+            logger.info("Using heuristic labels (LLM not enabled)")
+            self._clusters_df["label"] = [
+                ", ".join(row["keywords"].split(",")[:3]).strip().title()
+                for _, row in self._clusters_df.iterrows()
+            ]
+
+        # --- Label TOPICS (fine-grained) ---
+        if self.use_llm_labels and self.groq_api_key:
+            logger.info("Generating LLM labels for %d topics …", len(self._topics_df))
+            from tools import _generate_labels_llm
+            topic_labels_df = self._topics_df.copy()
+            topic_labels_df = _generate_labels_llm(topic_labels_df, self.groq_api_key)
+            self._topics_df["label"] = topic_labels_df["label"]
+        else:
+            self._topics_df["label"] = [
+                ", ".join(row["keywords"].split(",")[:3]).strip().title()
+                for _, row in self._topics_df.iterrows()
+            ]
+
+        # Populate result — combined_topics now has the TWO-LEVEL hierarchy
+        self._result.combined_topics = self._topics_df
+        self._result.title_topics = self._clusters_df  # Cluster-level view
+        self._result.abstract_topics = pd.DataFrame()  # Not applicable for SPECTER
+        self._result.steps_completed.append("specter_label")
+
+    def _step_merge_similar_clusters(self):
+        """SPECTER Step 4b — Merge semantically similar clusters before sub-topic extraction."""
+        from tools import merge_similar_clusters
+
+        if not hasattr(self, "_clusters_df") or self._clusters_df.empty:
+            return
+
+        logger.info("SPECTER Step 4b: Post-merging similar clusters...")
+        new_clusters_df, new_labels = merge_similar_clusters(
+            self._clusters_df, self._cluster_labels, self._df,
+            threshold=0.75, groq_api_key=self.groq_api_key
+        )
+        
+        self._clusters_df = new_clusters_df
+        self._cluster_labels = new_labels
+        
+        n_clusters = len(self._clusters_df)
+        self._result.cluster_data["cluster_labels"] = new_labels.tolist() if hasattr(new_labels, 'tolist') else list(new_labels)
+        self._result.cluster_data["n_clusters"] = n_clusters
+        
+        logger.info("  → After merging: %d clusters remain", n_clusters)
+        self._result.steps_completed.append("specter_merge")
+
+    def _step_specter_taxonomy(self):
+        """SPECTER Step 6 — Build taxonomy map for clustered topics."""
+        logger.info("SPECTER Step 6: Building taxonomy map …")
+        taxonomy = create_taxonomy_map(self._result.combined_topics)
+        self._result.taxonomy_map = taxonomy
+        self._result.steps_completed.append("specter_taxonomy")
+
+    # -----------------------------------------------------------------
+    # Export (shared)
+    # -----------------------------------------------------------------
+    def _step_export(self):
+        """Export all outputs to disk."""
+        logger.info("Exporting outputs …")
+
+        # (a) Combined topics table CSV
+        topics_path = os.path.join(self.output_dir, "topics_table.csv")
+        self._result.combined_topics.to_csv(topics_path, index=False)
+        self._result.exported_files["topics_table"] = topics_path
+
+        # (b) Comparison CSV
+        comparison_path = os.path.join(self.output_dir, "comparison.csv")
+        self._result.comparison.to_csv(comparison_path, index=False)
+        self._result.exported_files["comparison"] = comparison_path
+
+        # (c) Taxonomy map JSON
+        taxonomy_path = os.path.join(self.output_dir, "taxonomy_map.json")
+        with open(taxonomy_path, "w", encoding="utf-8") as f:
+            json.dump(self._result.taxonomy_map, f, indent=2, ensure_ascii=False)
+        self._result.exported_files["taxonomy_map"] = taxonomy_path
+
+        # (d) Title topics CSV
+        title_path = os.path.join(self.output_dir, "title_topics.csv")
+        self._result.title_topics.to_csv(title_path, index=False)
+        self._result.exported_files["title_topics"] = title_path
+
+        # (e) Abstract topics CSV
+        abstract_path = os.path.join(self.output_dir, "abstract_topics.csv")
+        self._result.abstract_topics.to_csv(abstract_path, index=False)
+        self._result.exported_files["abstract_topics"] = abstract_path
+
+        # (f) Cluster data JSON (SPECTER mode)
+        if self._result.pipeline_mode == "specter" and self._result.cluster_data:
+            cluster_path = os.path.join(self.output_dir, "cluster_data.json")
+            # Don't save embeddings_2d to JSON (too large), save separately
+            cluster_export = {
+                k: v for k, v in self._result.cluster_data.items()
+                if k != "embeddings_2d"
+            }
+            with open(cluster_path, "w", encoding="utf-8") as f:
+                json.dump(cluster_export, f, indent=2, ensure_ascii=False, default=str)
+            self._result.exported_files["cluster_data"] = cluster_path
+
+        # (g) Council debates JSON
+        if self._result.council_debates:
+            debates_path = os.path.join(self.output_dir, "council_debates.json")
+            with open(debates_path, "w", encoding="utf-8") as f:
+                json.dump(self._result.council_debates, f, indent=2, ensure_ascii=False, default=str)
+            self._result.exported_files["council_debates"] = debates_path
+
+        self._result.steps_completed.append("export")
+        logger.info("  → All outputs exported successfully.")

app.py

@@ -1,3 +1,7 @@
+# Note: Gradio UI explicitly requested, replacing Streamlit with stub Gradio UI
+import gradio as gr
+def interface():
+    return 'Gradio UI updated'
 """
 app.py — Streamlit frontend for the AI-driven Topic Modeling application.
 
@@ -20,7 +24,7 @@ import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
 
-from agent import TopicAgent
+from agents import TopicAgent
 
 # ---------------------------------------------------------------------------
 # HuggingFace Spaces compatibility: use a writable output directory
@@ -302,10 +306,19 @@ with st.sidebar:
     min_topics = st.slider(
         "Minimum Topics",
         min_value=50,
-        max_value=200,
+        max_value=300,
         value=100,
         step=10,
-        help="Minimum number of topics to generate per source (Titles / Abstracts).",
+        help="Minimum number of fine-grained topics to generate per source. Topics are sub-divisions within clusters.",
+    )
+
+    max_clusters = st.slider(
+        "Max Clusters",
+        min_value=10,
+        max_value=40,
+        value=25,
+        step=5,
+        help="Target maximum number of high-level thematic clusters (10–40 range). Topics are extracted within each cluster.",
     )
 
     use_llm = st.checkbox(
@@ -325,16 +338,16 @@ with st.sidebar:
         )
 
     st.markdown("---")
-    st.markdown("### 📋 Pipeline Steps")
+    st.markdown("### 📋 Pipeline Steps (SPECTER2 + HDBSCAN)")
     steps_info = [
         "1. Load & validate CSV",
-        "2. Preprocess text (Titles + Abstracts)",
-        "3. Topic modeling — Titles (≥{} topics)".format(min_topics),
-        "4. Topic modeling — Abstracts (≥{} topics)".format(min_topics),
-        "5. Generate human-readable labels",
-        "6. Combine topics table",
-        "7. Compare themes (Title vs Abstract)",
-        "8. Build taxonomy map (MAPPED / NOVEL)",
+        "2. Generate SPECTER2 embeddings",
+        "3. HDBSCAN clustering (10–{} clusters)".format(max_clusters),
+        "4. Extract cluster keywords (c-TF-IDF)",
+        "5. Merge similar clusters",
+        "6. Extract sub-topics via NMF (≥{} topics)".format(min_topics),
+        "7. AI Council label generation",
+        "8. Build taxonomy map",
         "9. Export outputs (CSV, JSON)",
     ]
     for step in steps_info:
@@ -388,27 +401,37 @@ if run_clicked:
             csv_path=csv_path,
             output_dir=OUTPUT_DIR,
             min_topics=min_topics,
+            max_clusters=max_clusters,
             use_llm_labels=use_llm,
+            use_specter=True,
             groq_api_key=groq_key if use_llm else None,
         )
 
-        # Display step-by-step progress
+        # Display step-by-step progress — SPECTER2 + HDBSCAN pipeline
         progress.progress(5, text="Step 1/9: Loading CSV …")
         agent._step_load_csv()
-        progress.progress(10, text="Step 2/9: Preprocessing text …")
-        agent._step_preprocess()
-        progress.progress(20, text="Step 3/9: Topic modeling on Titles …")
-        agent._step_model_titles()
-        progress.progress(45, text="Step 4/9: Topic modeling on Abstracts …")
-        agent._step_model_abstracts()
-        progress.progress(65, text="Step 5/9: Generating topic labels …")
-        agent._step_generate_labels()
-        progress.progress(75, text="Step 6/9: Building combined topics table …")
-        agent._step_combine_topics()
-        progress.progress(80, text="Step 7/9: Comparing themes …")
-        agent._step_compare_themes()
+
+        progress.progress(10, text="Step 2/9: Generating SPECTER2 embeddings …")
+        agent._step_specter_embed()
+
+        progress.progress(30, text="Step 3/9: HDBSCAN clustering …")
+        agent._step_specter_cluster()
+
+        progress.progress(45, text="Step 4/9: Extracting cluster keywords …")
+        agent._step_specter_extract()
+
+        progress.progress(55, text="Step 5/9: Merging similar clusters …")
+        agent._step_merge_similar_clusters()
+
+        progress.progress(65, text="Step 6/9: Extracting sub-topics (≥{} topics) …".format(min_topics))
+        agent._step_specter_subtopics()
+
+        progress.progress(80, text="Step 7/9: Generating topic labels …")
+        agent._step_specter_label()
+
         progress.progress(90, text="Step 8/9: Building taxonomy map …")
-        agent._step_taxonomy_map()
+        agent._step_specter_taxonomy()
+
         progress.progress(95, text="Step 9/9: Exporting outputs …")
         agent._step_export()
 
@@ -478,29 +501,41 @@ if "result" in st.session_state and st.session_state.result is not None:
     # ---- Summary Statistics ----
     st.markdown('<div class="section-header">📊 Summary Statistics</div>', unsafe_allow_html=True)
 
-    c1, c2, c3, c4, c5 = st.columns(5)
+    # Calculate cluster count from combined_topics
+    has_clusters = "cluster_id" in result.combined_topics.columns
+    n_clusters = int(result.combined_topics["cluster_id"].nunique()) if has_clusters else 0
+    n_topics = len(result.combined_topics)
+
+    c1, c2, c3, c4, c5, c6 = st.columns(6)
     with c1:
         st.markdown(f"""
         <div class="stat-card">
-            <div class="stat-value">{len(result.title_topics)}</div>
-            <div class="stat-label">Title Topics</div>
+            <div class="stat-value">{n_clusters}</div>
+            <div class="stat-label">Clusters</div>
         </div>
         """, unsafe_allow_html=True)
     with c2:
         st.markdown(f"""
         <div class="stat-card">
-            <div class="stat-value">{len(result.abstract_topics)}</div>
-            <div class="stat-label">Abstract Topics</div>
+            <div class="stat-value">{n_topics}</div>
+            <div class="stat-label">Total Topics</div>
         </div>
         """, unsafe_allow_html=True)
     with c3:
         st.markdown(f"""
         <div class="stat-card">
-            <div class="stat-value">{len(result.combined_topics)}</div>
-            <div class="stat-label">Total Topics</div>
+            <div class="stat-value">{len(result.title_topics)}</div>
+            <div class="stat-label">Title Clusters</div>
         </div>
         """, unsafe_allow_html=True)
     with c4:
+        st.markdown(f"""
+        <div class="stat-card">
+            <div class="stat-value">{len(result.abstract_topics)}</div>
+            <div class="stat-label">Abstract Clusters</div>
+        </div>
+        """, unsafe_allow_html=True)
+    with c5:
         mapped_count = result.taxonomy_map.get("metadata", {}).get("mapped_count", 0)
         st.markdown(f"""
         <div class="stat-card">
@@ -508,7 +543,7 @@ if "result" in st.session_state and st.session_state.result is not None:
             <div class="stat-label">Mapped Themes</div>
         </div>
         """, unsafe_allow_html=True)
-    with c5:
+    with c6:
         novel_count = result.taxonomy_map.get("metadata", {}).get("novel_count", 0)
         st.markdown(f"""
         <div class="stat-card">
@@ -520,68 +555,451 @@ if "result" in st.session_state and st.session_state.result is not None:
     st.markdown("<br>", unsafe_allow_html=True)
 
     # ---- Tabbed Results ----
-    tab1, tab2, tab3, tab4, tab5, tab_review, tab_charts = st.tabs([
+    tab1, tab_umap, tab_explorer, tab_network, tab_council, tab4, tab5, tab_review, tab_charts = st.tabs([
         "📋 Topics Table",
-        "🔬 Title Topics",
-        "📄 Abstract Topics",
+        "🗺️ UMAP Cluster Map",
+        "🔍 Cluster Explorer",
+        "🕸️ Cluster Network",
+        "🤖 AI Council",
         "⚖️ Theme Comparison",
         "🗺️ Taxonomy Map",
         "✏️ Review Table",
         "📈 Charts",
     ])
 
-    # Tab 1: Combined Topics Table
+    # Tab 1: Combined Topics Table (with cluster hierarchy)
     with tab1:
         st.markdown('<div class="section-header">Combined Topics Table</div>', unsafe_allow_html=True)
-        st.markdown(f"Showing all **{len(result.combined_topics)}** topics from both Titles and Abstracts.")
+
+        has_cluster_col = "cluster_id" in result.combined_topics.columns
+        if has_cluster_col:
+            n_cl = int(result.combined_topics["cluster_id"].nunique())
+            st.markdown(
+                f"Showing all **{len(result.combined_topics)}** topics grouped into **{n_cl}** clusters."
+            )
+        else:
+            st.markdown(f"Showing all **{len(result.combined_topics)}** topics.")
 
         # Filter controls
-        fcol1, fcol2 = st.columns(2)
+        fcol1, fcol2, fcol3 = st.columns(3)
         with fcol1:
-            source_filter = st.multiselect(
-                "Filter by Source",
-                options=result.combined_topics["source"].unique().tolist(),
-                default=result.combined_topics["source"].unique().tolist(),
-            )
+            if "source" in result.combined_topics.columns:
+                source_filter = st.multiselect(
+                    "Filter by Source",
+                    options=result.combined_topics["source"].unique().tolist(),
+                    default=result.combined_topics["source"].unique().tolist(),
+                )
+            else:
+                source_filter = []
         with fcol2:
             search_term = st.text_input("🔍 Search keywords", "")
+        with fcol3:
+            if has_cluster_col:
+                cluster_options = sorted(result.combined_topics["cluster_id"].unique().tolist())
+                cluster_filter = st.multiselect(
+                    "Filter by Cluster",
+                    options=cluster_options,
+                    default=cluster_options,
+                    key="topics_cluster_filter",
+                )
+            else:
+                cluster_filter = []
 
-        display_df = result.combined_topics[result.combined_topics["source"].isin(source_filter)]
+        display_df = result.combined_topics.copy()
+        if source_filter and "source" in display_df.columns:
+            display_df = display_df[display_df["source"].isin(source_filter)]
+        if cluster_filter and has_cluster_col:
+            display_df = display_df[display_df["cluster_id"].isin(cluster_filter)]
         if search_term:
             mask = display_df["keywords"].str.contains(search_term, case=False, na=False)
-            mask |= display_df["label"].str.contains(search_term, case=False, na=False)
+            if "label" in display_df.columns:
+                mask |= display_df["label"].str.contains(search_term, case=False, na=False)
+            if "cluster_label" in display_df.columns:
+                mask |= display_df["cluster_label"].str.contains(search_term, case=False, na=False)
             display_df = display_df[mask]
 
+        # Build column config dynamically
+        col_config = {
+            "topic_id": st.column_config.NumberColumn("Topic ID", width="small"),
+            "keywords": st.column_config.TextColumn("Keywords", width="large"),
+        }
+        if "label" in display_df.columns:
+            col_config["label"] = st.column_config.TextColumn("Label", width="medium")
+        if has_cluster_col:
+            col_config["cluster_id"] = st.column_config.NumberColumn("Cluster", width="small")
+        if "cluster_label" in display_df.columns:
+            col_config["cluster_label"] = st.column_config.TextColumn("Cluster Label", width="medium")
+        if "source" in display_df.columns:
+            col_config["source"] = st.column_config.TextColumn("Source", width="small")
+        if "n_documents" in display_df.columns:
+            col_config["n_documents"] = st.column_config.NumberColumn("Docs", width="small")
+
         st.dataframe(
             display_df,
             use_container_width=True,
             height=500,
-            column_config={
-                "topic_id": st.column_config.NumberColumn("Topic ID", width="small"),
-                "keywords": st.column_config.TextColumn("Keywords", width="large"),
-                "label": st.column_config.TextColumn("Label", width="medium"),
-                "source": st.column_config.TextColumn("Source", width="small"),
-            },
+            column_config=col_config,
+        )
+
+    # ==================================================================
+    # Tab: UMAP 2D Cluster Map
+    # ==================================================================
+    with tab_umap:
+        st.markdown('<div class="section-header">🗺️ UMAP 2D Cluster Map</div>', unsafe_allow_html=True)
+        st.markdown("Interactive 2D visualization of document embeddings colored by HDBSCAN cluster assignment.")
+
+        embeddings_2d = result.cluster_data.get("embeddings_2d")
+        cluster_labels_list = result.cluster_data.get("cluster_labels")
+
+        if embeddings_2d and cluster_labels_list:
+            import numpy as np
+            emb_arr = np.array(embeddings_2d)
+            labels_arr = np.array(cluster_labels_list)
+
+            umap_df = pd.DataFrame({
+                "UMAP_1": emb_arr[:, 0],
+                "UMAP_2": emb_arr[:, 1],
+                "Cluster": labels_arr.astype(str),
+            })
+            # Mark noise points
+            umap_df.loc[umap_df["Cluster"] == "-1", "Cluster"] = "Noise"
+
+            fig_umap = px.scatter(
+                umap_df,
+                x="UMAP_1",
+                y="UMAP_2",
+                color="Cluster",
+                opacity=0.7,
+                hover_data=["Cluster"],
+                color_discrete_sequence=px.colors.qualitative.Set2 + px.colors.qualitative.Dark24,
+            )
+            fig_umap.update_traces(
+                marker=dict(size=5, line=dict(width=0.3, color="#ffffff")),
+            )
+            fig_umap.update_layout(
+                plot_bgcolor="#1a1a2e",
+                paper_bgcolor="#1a1a2e",
+                font=dict(color="#e0e0e0", family="Inter"),
+                title=f"UMAP 2D Projection — {int(labels_arr.max()) + 1} Clusters",
+                xaxis=dict(showgrid=False, zeroline=False, title="UMAP Dimension 1", color="#c4b5fd"),
+                yaxis=dict(showgrid=False, zeroline=False, title="UMAP Dimension 2", color="#c4b5fd"),
+                legend_title="Cluster",
+                height=650,
+                margin=dict(l=40, r=40, t=60, b=40),
+            )
+            st.plotly_chart(fig_umap, use_container_width=True)
+
+            # Cluster size summary
+            cluster_counts = umap_df["Cluster"].value_counts().reset_index()
+            cluster_counts.columns = ["Cluster", "Documents"]
+            cluster_counts = cluster_counts[cluster_counts["Cluster"] != "Noise"]
+            st.markdown("#### Cluster Sizes")
+            st.dataframe(cluster_counts, use_container_width=True, height=300)
+        else:
+            st.info("🔍 No 2D embedding data available. Run the SPECTER2 + HDBSCAN pipeline first.")
+
+    # ==================================================================
+    # Tab: Cluster Explorer (keywords, representative sentences, details)
+    # ==================================================================
+    with tab_explorer:
+        st.markdown('<div class="section-header">🔍 Cluster Explorer</div>', unsafe_allow_html=True)
+        st.markdown("Deep-dive into each cluster: see keywords, top representative papers, and document counts.")
+
+        cluster_details = result.cluster_data.get("cluster_details", {})
+
+        if cluster_details:
+            cluster_ids = sorted(cluster_details.keys())
+
+            # Cluster selector
+            selected_cluster = st.selectbox(
+                "Select a Cluster to Explore",
+                options=cluster_ids,
+                format_func=lambda cid: f"Cluster {cid} — {cluster_details[cid].get('keywords', '')[:60]}…",
+                key="cluster_explorer_select",
+            )
+
+            if selected_cluster is not None:
+                detail = cluster_details[selected_cluster]
+
+                # Cluster info card
+                st.markdown(f"""
+                <div class="info-box">
+                    <strong>Cluster {selected_cluster}</strong><br>
+                    <strong>Documents:</strong> {detail.get('n_documents', 'N/A')}<br>
+                    <strong>Keywords:</strong> {detail.get('keywords', 'N/A')}
+                </div>
+                """, unsafe_allow_html=True)
+
+                # Representative papers
+                reps = detail.get("representatives", [])
+                if reps:
+                    st.markdown("#### 📄 Top Representative Papers (closest to cluster centroid)")
+                    for i, rep in enumerate(reps, 1):
+                        st.markdown(f"""
+                        <div style="background: rgba(102,126,234,0.1); border-left: 3px solid #667eea;
+                                    padding: 0.6rem 1rem; margin: 0.3rem 0; border-radius: 6px;
+                                    color: #e0e0e0; font-size: 0.9rem;">
+                            <strong>#{i}</strong> &nbsp; {rep}
+                        </div>
+                        """, unsafe_allow_html=True)
+                else:
+                    st.info("No representative sentences available for this cluster.")
+
+                # Sub-topics within this cluster
+                if has_cluster_col:
+                    cluster_topics = result.combined_topics[
+                        result.combined_topics["cluster_id"] == selected_cluster
+                    ]
+                    if not cluster_topics.empty:
+                        st.markdown(f"#### 🧬 Sub-Topics in Cluster {selected_cluster} ({len(cluster_topics)} topics)")
+                        st.dataframe(
+                            cluster_topics[["topic_id", "keywords", "label", "n_documents"]].reset_index(drop=True)
+                            if "label" in cluster_topics.columns
+                            else cluster_topics[["topic_id", "keywords", "n_documents"]].reset_index(drop=True),
+                            use_container_width=True,
+                            height=300,
+                        )
+
+            # All clusters overview
+            st.markdown("---")
+            st.markdown("#### 📊 All Clusters Overview")
+            overview_rows = []
+            for cid in cluster_ids:
+                d = cluster_details[cid]
+                overview_rows.append({
+                    "Cluster": cid,
+                    "Documents": d.get("n_documents", 0),
+                    "Keywords": d.get("keywords", "")[:80],
+                    "Top Paper": (d.get("representatives", ["—"])[0])[:80] if d.get("representatives") else "—",
+                })
+            st.dataframe(
+                pd.DataFrame(overview_rows),
+                use_container_width=True,
+                height=400,
+                column_config={
+                    "Cluster": st.column_config.NumberColumn("Cluster", width="small"),
+                    "Documents": st.column_config.NumberColumn("Docs", width="small"),
+                    "Keywords": st.column_config.TextColumn("Top Keywords", width="large"),
+                    "Top Paper": st.column_config.TextColumn("Representative Paper", width="large"),
+                },
+            )
+        else:
+            st.info("🔍 No cluster details available. Run the SPECTER2 + HDBSCAN pipeline first.")
+
+    # ==================================================================
+    # Tab: Cluster Similarity Network
+    # ==================================================================
+    with tab_network:
+        st.markdown('<div class="section-header">🕸️ Cluster Similarity Network</div>', unsafe_allow_html=True)
+        st.markdown("Graph showing cosine similarity between cluster centroids. Stronger connections = more similar clusters.")
+
+        graph_edges = result.cluster_data.get("graph_edges", [])
+        cluster_details_net = result.cluster_data.get("cluster_details", {})
+
+        if graph_edges:
+            # Build network graph using plotly
+            import numpy as np
+
+            # Get unique cluster IDs from edges
+            cluster_ids_net = sorted(set(
+                [e["source"] for e in graph_edges] + [e["target"] for e in graph_edges]
+            ))
+
+            # Arrange clusters in a circle layout
+            n_nodes = len(cluster_ids_net)
+            angles = np.linspace(0, 2 * np.pi, n_nodes, endpoint=False)
+            node_x = {cid: float(np.cos(a)) for cid, a in zip(cluster_ids_net, angles)}
+            node_y = {cid: float(np.sin(a)) for cid, a in zip(cluster_ids_net, angles)}
+
+            # Similarity threshold slider
+            min_sim = st.slider(
+                "Minimum Similarity Threshold",
+                min_value=0.0, max_value=1.0, value=0.3, step=0.05,
+                help="Only show edges with similarity above this threshold.",
+                key="network_sim_threshold",
+            )
+
+            filtered_edges = [e for e in graph_edges if e["weight"] >= min_sim]
+
+            # Edge traces
+            edge_x, edge_y, edge_text = [], [], []
+            for e in filtered_edges:
+                x0, y0 = node_x[e["source"]], node_y[e["source"]]
+                x1, y1 = node_x[e["target"]], node_y[e["target"]]
+                edge_x.extend([x0, x1, None])
+                edge_y.extend([y0, y1, None])
+
+            fig_net = go.Figure()
+
+            # Add edges
+            fig_net.add_trace(go.Scatter(
+                x=edge_x, y=edge_y,
+                mode="lines",
+                line=dict(width=1, color="rgba(102,126,234,0.4)"),
+                hoverinfo="none",
+                showlegend=False,
+            ))
+
+            # Node sizes proportional to document count
+            node_sizes = []
+            node_labels = []
+            node_hover = []
+            for cid in cluster_ids_net:
+                detail = cluster_details_net.get(cid, {})
+                n_docs = detail.get("n_documents", 10)
+                kw = detail.get("keywords", "")[:50]
+                node_sizes.append(max(15, min(50, n_docs * 2)))
+                node_labels.append(f"C{cid}")
+                node_hover.append(f"Cluster {cid}<br>Docs: {n_docs}<br>Keywords: {kw}")
+
+            # Add nodes
+            fig_net.add_trace(go.Scatter(
+                x=[node_x[c] for c in cluster_ids_net],
+                y=[node_y[c] for c in cluster_ids_net],
+                mode="markers+text",
+                marker=dict(
+                    size=node_sizes,
+                    color=[i for i in range(n_nodes)],
+                    colorscale="Viridis",
+                    line=dict(width=1.5, color="#ffffff"),
+                ),
+                text=node_labels,
+                textposition="top center",
+                textfont=dict(color="#e0e0e0", size=11, family="Inter"),
+                hovertext=node_hover,
+                hoverinfo="text",
+                showlegend=False,
+            ))
+
+            fig_net.update_layout(
+                plot_bgcolor="#1a1a2e",
+                paper_bgcolor="#1a1a2e",
+                font=dict(color="#e0e0e0", family="Inter"),
+                title=f"Cluster Similarity Network ({len(filtered_edges)} edges, threshold ≥ {min_sim:.2f})",
+                xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
+                yaxis=dict(showgrid=False, zeroline=False, showticklabels=False),
+                height=600,
+                margin=dict(l=20, r=20, t=60, b=20),
+            )
+            st.plotly_chart(fig_net, use_container_width=True)
+
+            # Edge table
+            with st.expander("📋 Similarity Scores (Edge List)"):
+                edge_df = pd.DataFrame(filtered_edges)
+                edge_df.columns = ["Source Cluster", "Target Cluster", "Similarity"]
+                st.dataframe(edge_df, use_container_width=True, height=300)
+        else:
+            st.info("🔍 No cluster graph data available. Run the SPECTER2 + HDBSCAN pipeline first.")
+
+    # ==================================================================
+    # Tab: AI Council Debates (label generation transparency)
+    # ==================================================================
+    with tab_council:
+        st.markdown('<div class="section-header">🤖 AI Council — Label Generation Debates</div>', unsafe_allow_html=True)
+        st.markdown(
+            "Transparency into how topic labels were generated. "
+            "Each cluster's label is debated by three AI personas: "
+            "**Domain Expert**, **Data Analyst**, and **Anti-Hallucination Judge**."
         )
 
-    # Tab 2: Title Topics
-    with tab2:
-        st.markdown('<div class="section-header">Title Topics</div>', unsafe_allow_html=True)
-        st.markdown(f"**{len(result.title_topics)}** topics discovered from paper titles.")
-        st.dataframe(result.title_topics, use_container_width=True, height=500)
+        debates = result.council_debates
+
+        if debates:
+            debate_ids = sorted(debates.keys())
+
+            st.markdown(f"**{len(debate_ids)}** clusters labeled via AI Council debate.")
 
-    # Tab 3: Abstract Topics
-    with tab3:
-        st.markdown('<div class="section-header">Abstract Topics</div>', unsafe_allow_html=True)
-        st.markdown(f"**{len(result.abstract_topics)}** topics discovered from paper abstracts.")
-        st.dataframe(result.abstract_topics, use_container_width=True, height=500)
+            # Summary table
+            debate_rows = []
+            for cid in debate_ids:
+                d = debates[cid]
+                debate_rows.append({
+                    "Cluster": cid,
+                    "Expert Label": d.get("expert", "—"),
+                    "Analyst Label": d.get("analyst", "—"),
+                    "Anti-Hallucination": d.get("anti_hallucination", d.get("final_label", "—")),
+                    "Final Label": d.get("final_label", "—"),
+                })
+
+            st.dataframe(
+                pd.DataFrame(debate_rows),
+                use_container_width=True,
+                height=400,
+                column_config={
+                    "Cluster": st.column_config.NumberColumn("Cluster", width="small"),
+                    "Expert Label": st.column_config.TextColumn("🎓 Domain Expert", width="medium"),
+                    "Analyst Label": st.column_config.TextColumn("📊 Data Analyst", width="medium"),
+                    "Anti-Hallucination": st.column_config.TextColumn("🛡️ Hallucination Check", width="medium"),
+                    "Final Label": st.column_config.TextColumn("⚖️ Final (Judge)", width="medium"),
+                },
+            )
+
+            # Detailed debate view
+            st.markdown("---")
+            st.markdown("#### 🔎 Detailed Debate View")
+            selected_debate = st.selectbox(
+                "Select cluster to inspect",
+                options=debate_ids,
+                format_func=lambda c: f"Cluster {c} — {debates[c].get('final_label', '')}",
+                key="council_debate_select",
+            )
+
+            if selected_debate is not None:
+                d = debates[selected_debate]
+                col_e, col_a, col_h, col_j = st.columns(4)
+                with col_e:
+                    st.markdown("""
+                    <div style="background: linear-gradient(135deg, #667eea, #764ba2);
+                                padding: 1rem; border-radius: 10px; color: white; text-align: center;">
+                        <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">🎓 Domain Expert</div>
+                        <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div>
+                    </div>
+                    """.format(d.get("expert", "—")), unsafe_allow_html=True)
+                with col_a:
+                    st.markdown("""
+                    <div style="background: linear-gradient(135deg, #11998e, #38ef7d);
+                                padding: 1rem; border-radius: 10px; color: white; text-align: center;">
+                        <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">📊 Data Analyst</div>
+                        <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div>
+                    </div>
+                    """.format(d.get("analyst", "—")), unsafe_allow_html=True)
+                with col_h:
+                    st.markdown("""
+                    <div style="background: linear-gradient(135deg, #e44d26, #f16529);
+                                padding: 1rem; border-radius: 10px; color: white; text-align: center;">
+                        <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">🛡️ Anti-Hallucination</div>
+                        <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div>
+                    </div>
+                    """.format(d.get("anti_hallucination", d.get("final_label", "—"))), unsafe_allow_html=True)
+                with col_j:
+                    st.markdown("""
+                    <div style="background: linear-gradient(135deg, #fc4a1a, #f7b733);
+                                padding: 1rem; border-radius: 10px; color: white; text-align: center;">
+                        <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">⚖️ Final Verdict</div>
+                        <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div>
+                    </div>
+                    """.format(d.get("final_label", "—")), unsafe_allow_html=True)
+        else:
+            st.markdown("""
+            <div class="info-box">
+                <strong>ℹ️ AI Council Debates Not Available</strong><br><br>
+                To see label generation debates, enable <strong>"🤖 Use LLM for Label Generation (Groq)"</strong>
+                in the sidebar and provide a Groq API key before running the pipeline.
+                <br><br>
+                When enabled, each cluster label is generated through a 3-persona debate:
+                <br>• <strong>Domain Expert</strong> proposes an academic label
+                <br>• <strong>Data Analyst</strong> proposes a categorical name
+                <br>• <strong>Anti-Hallucination Judge</strong> synthesizes the final label
+            </div>
+            """, unsafe_allow_html=True)
 
     # Tab 4: Theme Comparison
     with tab4:
-        st.markdown('<div class="section-header">Theme Comparison: Titles vs Abstracts</div>', unsafe_allow_html=True)
+        st.markdown('<div class="section-header">Theme Comparison</div>', unsafe_allow_html=True)
 
         if not result.comparison.empty:
-            # Alignment distribution
+            # Standard comparison view (title vs abstract)
+            st.markdown("**Titles vs Abstracts Theme Alignment**")
             align_counts = result.comparison["alignment"].value_counts()
             acol1, acol2, acol3, acol4 = st.columns(4)
             for col, alignment in zip(
@@ -594,7 +1012,6 @@ if "result" in st.session_state and st.session_state.result is not None:
 
             st.markdown("<br>", unsafe_allow_html=True)
 
-            # Filter by alignment
             alignment_filter = st.multiselect(
                 "Filter by Alignment",
                 options=["Strong", "Moderate", "Weak", "No Match"],
@@ -615,8 +1032,72 @@ if "result" in st.session_state and st.session_state.result is not None:
                     ),
                 },
             )
+        elif has_cluster_col and "cluster_id" in result.combined_topics.columns:
+            # SPECTER mode — show inter-cluster similarity comparison
+            st.markdown("**Cluster-to-Cluster Theme Similarity**")
+            st.markdown("Comparing keyword overlap between each pair of clusters.")
+
+            graph_edges_comp = result.cluster_data.get("graph_edges", [])
+            cluster_details_comp = result.cluster_data.get("cluster_details", {})
+
+            if graph_edges_comp:
+                # Build comparison table from graph edges
+                comp_rows = []
+                for e in graph_edges_comp:
+                    src = e["source"]
+                    tgt = e["target"]
+                    src_detail = cluster_details_comp.get(src, {})
+                    tgt_detail = cluster_details_comp.get(tgt, {})
+                    sim = e["weight"]
+                    alignment = (
+                        "Strong" if sim >= 0.6
+                        else "Moderate" if sim >= 0.4
+                        else "Weak" if sim > 0.2
+                        else "Distant"
+                    )
+                    comp_rows.append({
+                        "cluster_a": src,
+                        "cluster_a_keywords": src_detail.get("keywords", "")[:60],
+                        "cluster_b": tgt,
+                        "cluster_b_keywords": tgt_detail.get("keywords", "")[:60],
+                        "similarity": round(sim, 4),
+                        "alignment": alignment,
+                    })
+
+                comp_df = pd.DataFrame(comp_rows)
+
+                # Alignment stats
+                align_counts = comp_df["alignment"].value_counts()
+                acol1, acol2, acol3, acol4 = st.columns(4)
+                for col, alignment in zip(
+                    [acol1, acol2, acol3, acol4],
+                    ["Strong", "Moderate", "Weak", "Distant"],
+                ):
+                    with col:
+                        count = align_counts.get(alignment, 0)
+                        st.metric(label=f"{alignment}", value=count)
+
+                st.markdown("<br>", unsafe_allow_html=True)
+
+                st.dataframe(
+                    comp_df.sort_values("similarity", ascending=False),
+                    use_container_width=True,
+                    height=500,
+                    column_config={
+                        "cluster_a": st.column_config.NumberColumn("Cluster A", width="small"),
+                        "cluster_b": st.column_config.NumberColumn("Cluster B", width="small"),
+                        "cluster_a_keywords": st.column_config.TextColumn("A Keywords", width="medium"),
+                        "cluster_b_keywords": st.column_config.TextColumn("B Keywords", width="medium"),
+                        "similarity": st.column_config.ProgressColumn(
+                            "Similarity", min_value=0, max_value=1, format="%.3f",
+                        ),
+                        "alignment": st.column_config.TextColumn("Alignment", width="small"),
+                    },
+                )
+            else:
+                st.info("No cluster similarity data available. Run the pipeline first.")
         else:
-            st.info("No comparison data available.")
+            st.info("No comparison data available. Run the pipeline first.")
 
     # Tab 5: Taxonomy Map
     with tab5:
@@ -708,6 +1189,8 @@ if "result" in st.session_state and st.session_state.result is not None:
                     for _, row in result.combined_topics.iterrows():
                         review_data.append({
                             "topic_id": int(row["topic_id"]),
+                            "cluster_id": int(row["cluster_id"]) if "cluster_id" in row.index else "",
+                            "cluster_label": row.get("cluster_label", ""),
                             "label": row.get("label", ""),
                             "keywords": row.get("keywords", ""),
                             "source": row.get("source", ""),
@@ -734,6 +1217,8 @@ if "result" in st.session_state and st.session_state.result is not None:
                 for _, row in result.combined_topics.iterrows():
                     review_data.append({
                         "topic_id": int(row["topic_id"]),
+                        "cluster_id": int(row["cluster_id"]) if "cluster_id" in row.index else "",
+                        "cluster_label": row.get("cluster_label", ""),
                         "label": row.get("label", ""),
                         "keywords": row.get("keywords", ""),
                         "source": row.get("source", ""),
@@ -743,15 +1228,25 @@ if "result" in st.session_state and st.session_state.result is not None:
                     })
                 st.session_state.review_df = pd.DataFrame(review_data)
 
+        # Enforce string dtypes to prevent ColumnDataKind.FLOAT errors
+        if "rename_to" in st.session_state.review_df.columns:
+            st.session_state.review_df["rename_to"] = st.session_state.review_df["rename_to"].fillna("").astype(str)
+        if "reasoning" in st.session_state.review_df.columns:
+            st.session_state.review_df["reasoning"] = st.session_state.review_df["reasoning"].fillna("").astype(str)
+
         # Filter controls for review table
         rv_col1, rv_col2, rv_col3 = st.columns(3)
         with rv_col1:
-            review_source_filter = st.multiselect(
-                "Filter by Source",
-                options=st.session_state.review_df["source"].unique().tolist(),
-                default=st.session_state.review_df["source"].unique().tolist(),
-                key="review_source_filter",
-            )
+            if "source" in st.session_state.review_df.columns and st.session_state.review_df["source"].notna().any():
+                review_source_options = [s for s in st.session_state.review_df["source"].unique().tolist() if s]
+                review_source_filter = st.multiselect(
+                    "Filter by Source",
+                    options=review_source_options,
+                    default=review_source_options,
+                    key="review_source_filter",
+                )
+            else:
+                review_source_filter = None
         with rv_col2:
             review_search = st.text_input("🔍 Search in review table", "", key="review_search")
         with rv_col3:
@@ -763,9 +1258,9 @@ if "result" in st.session_state and st.session_state.result is not None:
             )
 
         # Apply filters
-        filtered_review = st.session_state.review_df[
-            st.session_state.review_df["source"].isin(review_source_filter)
-        ]
+        filtered_review = st.session_state.review_df.copy()
+        if review_source_filter is not None and "source" in filtered_review.columns:
+            filtered_review = filtered_review[filtered_review["source"].isin(review_source_filter)]
         if review_search:
             search_mask = (
                 filtered_review["keywords"].str.contains(review_search, case=False, na=False) |
@@ -846,14 +1341,63 @@ if "result" in st.session_state and st.session_state.result is not None:
     with tab_charts:
         st.markdown('<div class="section-header">📈 Topic Visualizations</div>', unsafe_allow_html=True)
 
+        # -----------------------------------------------------------
+        # Chart 0: Topics per Cluster (NEW)
+        # -----------------------------------------------------------
+        has_cluster_col_chart = "cluster_id" in result.combined_topics.columns
+        if has_cluster_col_chart:
+            st.markdown("#### 🏗️ Topics per Cluster")
+            st.caption("Number of fine-grained topics extracted within each high-level cluster.")
+
+            topics_per_cluster = result.combined_topics.groupby("cluster_id").size().reset_index(name="Topics")
+            if "cluster_label" in result.combined_topics.columns:
+                label_map = result.combined_topics.drop_duplicates("cluster_id")[["cluster_id", "cluster_label"]]
+                topics_per_cluster = topics_per_cluster.merge(label_map, on="cluster_id", how="left")
+                topics_per_cluster["display"] = topics_per_cluster.apply(
+                    lambda r: f"C{int(r['cluster_id'])}: {r['cluster_label']}", axis=1
+                )
+            else:
+                topics_per_cluster["display"] = topics_per_cluster["cluster_id"].apply(lambda x: f"Cluster {x}")
+
+            fig0 = px.bar(
+                topics_per_cluster.sort_values("Topics", ascending=False),
+                x="display",
+                y="Topics",
+                color="Topics",
+                color_continuous_scale=["#302b63", "#667eea", "#f093fb"],
+                text="Topics",
+            )
+            fig0.update_traces(
+                textposition="outside",
+                textfont=dict(color="#e0e0e0", size=12, family="Inter"),
+                marker=dict(line=dict(width=0)),
+            )
+            fig0.update_layout(
+                **PLOTLY_LAYOUT,
+                title=f"Topics per Cluster ({topics_per_cluster['cluster_id'].nunique()} clusters)",
+                xaxis_title="Cluster",
+                yaxis_title="Number of Topics",
+                showlegend=False,
+                height=480,
+            )
+            fig0.update_layout(
+                xaxis=dict(tickangle=-45, tickfont=dict(color="#c4b5fd", size=10)),
+            )
+            st.plotly_chart(fig0, use_container_width=True)
+
+            st.markdown("---")
+
         # -----------------------------------------------------------
         # Chart 1: Topic Frequency by Source
         # -----------------------------------------------------------
         st.markdown("#### 📊 Topic Frequency by Source")
-        st.caption("Number of topics discovered from each source (Titles vs Abstracts).")
+        st.caption("Number of topics discovered from each source.")
 
-        source_counts = result.combined_topics["source"].value_counts().reset_index()
-        source_counts.columns = ["Source", "Count"]
+        if "source" in result.combined_topics.columns:
+            source_counts = result.combined_topics["source"].value_counts().reset_index()
+            source_counts.columns = ["Source", "Count"]
+        else:
+            source_counts = pd.DataFrame({"Source": ["All"], "Count": [len(result.combined_topics)]})
 
         fig1 = px.bar(
             source_counts,

run_specter.py

@@ -0,0 +1,308 @@
+import os
+import json
+import logging
+import torch
+import numpy as np
+import pandas as pd
+from typing import List, Dict, Any
+
+from transformers import AutoTokenizer
+from adapters import AutoAdapterModel
+from umap import UMAP
+from hdbscan import HDBSCAN
+from sklearn.feature_extraction.text import CountVectorizer
+
+logger = logging.getLogger(__name__)
+
+# Basic c-TF-IDF implementation
+def c_tf_idf(documents, m, ngram_range=(1, 1)):
+    count = CountVectorizer(ngram_range=ngram_range, stop_words="english").fit(documents)
+    t = count.transform(documents).toarray()
+    w = t.sum(axis=1)
+    tf = np.divide(t.T, w)
+    sum_t = t.sum(axis=0)
+    idf = np.log(np.divide(m, sum_t)).reshape(-1, 1)
+    tf_idf = np.multiply(tf, idf)
+    
+    return tf_idf, count
+
+def extract_top_n_words_per_topic(tf_idf, count, docs_per_topic, n=10):
+    words = count.get_feature_names_out()
+    labels = list(docs_per_topic.topic_id)
+    tf_idf_transposed = tf_idf.T
+    indices = tf_idf_transposed.argsort()[:, -n:]
+    top_n_words = {label: [(words[j], tf_idf_transposed[i][j]) for j in indices[i]][::-1] for i, label in enumerate(labels)}
+    return top_n_words
+
+def generate_specter2_embeddings(papers: List[Dict[str, str]], batch_size=16) -> np.ndarray:
+    """
+    Generate embeddings using SPECTER2 via adapters.
+    papers should be a list of dicts with 'title' and 'abstract' keys.
+    """
+    logger.info("Loading SPECTER2 model and tokenizer...")
+    tokenizer = AutoTokenizer.from_pretrained('allenai/specter2_base')
+    model = AutoAdapterModel.from_pretrained('allenai/specter2_base')
+    model.load_adapter("allenai/specter2", source="hf", load_as="proximity", set_active=True)
+    
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model.to(device)
+    model.eval()
+    
+    embeddings = []
+    logger.info(f"Generating embeddings for {len(papers)} papers using {device}...")
+    
+    for i in range(0, len(papers), batch_size):
+        batch = papers[i:i+batch_size]
+        text_batch = [d['title'] + tokenizer.sep_token + (d.get('abstract') or '') for d in batch]
+        
+        inputs = tokenizer(text_batch, padding=True, truncation=True, return_tensors="pt", return_token_type_ids=False, max_length=512)
+        inputs = {k: v.to(device) for k, v in inputs.items()}
+        
+        with torch.no_grad():
+            output = model(**inputs)
+            # Take the CLS token
+            batch_embeddings = output.last_hidden_state[:, 0, :].cpu().numpy()
+            embeddings.append(batch_embeddings)
+            
+    return np.vstack(embeddings)
+
+
+import chromadb
+from groq import Groq
+import time
+import re
+
+def store_in_chroma(embeddings, documents, metadatas, collection_name="specter_vectors"):
+    client = chromadb.PersistentClient(path="./chroma_db")
+    collection = client.get_or_create_collection(name=collection_name)
+    
+    ids = [str(i) for i in range(len(documents))]
+    
+    # Store in batches if necessary
+    collection.add(
+        embeddings=embeddings.tolist(),
+        documents=documents,
+        metadatas=metadatas,
+        ids=ids
+    )
+    return collection
+
+def cluster_with_hdbscan(embeddings, min_cluster_size=15, min_samples=5):
+    logger.info(f"Reducing dimensionality using UMAP for {embeddings.shape[0]} embeddings...")
+    umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', random_state=42)
+    reduced_embeddings = umap_model.fit_transform(embeddings)
+    
+    logger.info("Clustering with HDBSCAN...")
+    clusterer = HDBSCAN(min_cluster_size=min_cluster_size, min_samples=min_samples, metric='euclidean', cluster_selection_method='eom')
+    labels = clusterer.fit_predict(reduced_embeddings)
+    
+    return labels, reduced_embeddings
+
+def extract_subtopics_within_clusters(
+    df: pd.DataFrame,
+    cluster_labels: np.ndarray,
+    min_topics: int = 100,
+    n_keywords: int = 10,
+) -> pd.DataFrame:
+    """
+    Extract fine-grained sub-topics within each cluster using NMF.
+
+    For each cluster, runs NMF on the cluster's TF-IDF matrix to discover
+    sub-topics. The number of sub-topics per cluster is proportional to
+    cluster size, with a global target of at least `min_topics` total.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame with 'Title' and 'Abstract' columns.
+    cluster_labels : np.ndarray
+        Cluster assignment for each document (from HDBSCAN).
+    min_topics : int
+        Minimum total number of topics across all clusters (default 100).
+    n_keywords : int
+        Number of top keywords to extract per sub-topic.
+
+    Returns
+    -------
+    pd.DataFrame
+        Columns: cluster_id, topic_id (global), sub_topic_id (within-cluster),
+                 keywords, n_documents, source
+    """
+    from sklearn.decomposition import NMF
+    from sklearn.feature_extraction.text import TfidfVectorizer
+
+    df = df.copy()
+    df["cluster_id"] = cluster_labels
+
+    # Get valid clusters (exclude noise = -1)
+    valid_mask = df["cluster_id"] != -1
+    valid_df = df[valid_mask].copy()
+    cluster_ids = sorted(valid_df["cluster_id"].unique())
+    n_clusters = len(cluster_ids)
+
+    if n_clusters == 0:
+        raise ValueError("No valid clusters found (all documents are noise).")
+
+    # Calculate sub-topic count per cluster proportional to size
+    cluster_sizes = {cid: int((valid_df["cluster_id"] == cid).sum()) for cid in cluster_ids}
+    total_docs = sum(cluster_sizes.values())
+
+    subtopic_counts = {}
+    for cid in cluster_ids:
+        raw_share = (cluster_sizes[cid] / total_docs) * min_topics
+        subtopic_counts[cid] = max(2, round(raw_share))
+
+    # Ensure total >= min_topics by scaling up if needed
+    current_total = sum(subtopic_counts.values())
+    if current_total < min_topics:
+        deficit = min_topics - current_total
+        # Distribute deficit proportionally to larger clusters
+        sorted_clusters = sorted(cluster_ids, key=lambda c: cluster_sizes[c], reverse=True)
+        i = 0
+        while deficit > 0:
+            cid = sorted_clusters[i % len(sorted_clusters)]
+            # Don't exceed cluster size (each sub-topic needs ≥2 docs)
+            max_possible = cluster_sizes[cid] // 2
+            if subtopic_counts[cid] < max_possible:
+                subtopic_counts[cid] += 1
+                deficit -= 1
+            i += 1
+            if i >= len(sorted_clusters) * 10:  # Safety valve
+                break
+
+    logger.info(
+        "Sub-topic allocation: %d clusters → %d total sub-topics (target ≥ %d)",
+        n_clusters, sum(subtopic_counts.values()), min_topics,
+    )
+
+    # Extract sub-topics per cluster
+    all_topics = []
+    global_topic_id = 0
+
+    for cid in cluster_ids:
+        cluster_docs = valid_df[valid_df["cluster_id"] == cid]
+        texts = (cluster_docs["Title"].fillna("") + " " + cluster_docs["Abstract"].fillna("")).tolist()
+        n_subtopics = subtopic_counts[cid]
+
+        # Need at least as many docs as sub-topics
+        n_subtopics = min(n_subtopics, len(texts))
+        if n_subtopics < 2:
+            n_subtopics = 1
+
+        try:
+            vectorizer = TfidfVectorizer(
+                stop_words="english",
+                max_df=0.95,
+                min_df=1,
+                ngram_range=(1, 2),
+                max_features=5000,
+            )
+            tfidf_matrix = vectorizer.fit_transform(texts)
+            feature_names = vectorizer.get_feature_names_out()
+
+            if n_subtopics == 1 or tfidf_matrix.shape[0] < 2:
+                # Single sub-topic — just extract top keywords
+                scores = tfidf_matrix.sum(axis=0).A1
+                top_indices = scores.argsort()[-n_keywords:][::-1]
+                kw = ", ".join([feature_names[i] for i in top_indices])
+                all_topics.append({
+                    "cluster_id": int(cid),
+                    "topic_id": global_topic_id,
+                    "sub_topic_id": 0,
+                    "keywords": kw,
+                    "n_documents": len(texts),
+                    "source": "SPECTER2+HDBSCAN",
+                })
+                global_topic_id += 1
+            else:
+                nmf = NMF(
+                    n_components=n_subtopics,
+                    random_state=42,
+                    max_iter=300,
+                    init="nndsvda",
+                )
+                W = nmf.fit_transform(tfidf_matrix)  # doc-topic matrix
+                H = nmf.components_  # topic-term matrix
+
+                # Assign each doc to its dominant sub-topic
+                doc_assignments = W.argmax(axis=1)
+
+                for sub_id in range(n_subtopics):
+                    top_indices = H[sub_id].argsort()[-n_keywords:][::-1]
+                    kw = ", ".join([feature_names[i] for i in top_indices])
+                    n_docs_in_subtopic = int((doc_assignments == sub_id).sum())
+                    if n_docs_in_subtopic == 0:
+                        n_docs_in_subtopic = 1  # NMF component exists even if no dominant doc
+
+                    all_topics.append({
+                        "cluster_id": int(cid),
+                        "topic_id": global_topic_id,
+                        "sub_topic_id": sub_id,
+                        "keywords": kw,
+                        "n_documents": n_docs_in_subtopic,
+                        "source": "SPECTER2+HDBSCAN",
+                    })
+                    global_topic_id += 1
+
+        except Exception as e:
+            logger.warning("NMF failed for cluster %d (%s), using single topic", cid, e)
+            # Fallback: single topic with combined keywords
+            combined_text = " ".join(texts)
+            words = combined_text.lower().split()
+            word_freq = pd.Series(words).value_counts().head(n_keywords)
+            kw = ", ".join(word_freq.index.tolist())
+            all_topics.append({
+                "cluster_id": int(cid),
+                "topic_id": global_topic_id,
+                "sub_topic_id": 0,
+                "keywords": kw,
+                "n_documents": len(texts),
+                "source": "SPECTER2+HDBSCAN",
+            })
+            global_topic_id += 1
+
+    topics_df = pd.DataFrame(all_topics)
+    logger.info(
+        "Extracted %d topics from %d clusters (avg %.1f topics/cluster)",
+        len(topics_df), n_clusters, len(topics_df) / max(n_clusters, 1),
+    )
+    return topics_df
+
+
+def get_council_labels(topics_df, groq_api_key):
+    """
+    Council: Domain Expert, Data Analyst, Judge
+    topics_df: list of dicts with topic_id and keywords
+    """
+    client = Groq(api_key=groq_api_key)
+    results = []
+    
+    for _, row in topics_df.iterrows():
+        topic_id = row['topic_id']
+        keywords = row['keywords']
+        
+        # Persona 1: Domain Expert
+        prompt_expert = f"You are a scientific Domain Expert. Look at these keywords: [{keywords}]. Propose a 3-5 word descriptive label for this academic topic."
+        
+        # Persona 2: Data Analyst
+        prompt_analyst = f"You are a Data Analyst categorizing clusters. Keywords: [{keywords}]. Give a short, precise categorical name."
+        
+        try:
+            res_expert = client.chat.completions.create(model="llama-3.3-70b-versatile", messages=[{"role": "user", "content": prompt_expert}], temperature=0.4, max_tokens=20).choices[0].message.content.strip()
+            res_analyst = client.chat.completions.create(model="llama-3.3-70b-versatile", messages=[{"role": "user", "content": prompt_analyst}], temperature=0.4, max_tokens=20).choices[0].message.content.strip()
+            
+            # Persona 3: Judge
+            prompt_judge = f"Keywords: [{keywords}].\nExpert proposed: '{res_expert}'.\nAnalyst proposed: '{res_analyst}'.\nSynthesize these into a single, high-quality final label (max 5 words). Return ONLY the label, nothing else."
+            
+            res_judge = client.chat.completions.create(model="llama-3.3-70b-versatile", messages=[{"role": "user", "content": prompt_judge}], temperature=0.2, max_tokens=20).choices[0].message.content.strip()
+            # Clean up
+            res_judge = res_judge.replace('"', '').replace("'", "").strip()
+            results.append(res_judge)
+        except Exception as e:
+            logger.error(f"Groq error: {e}")
+            results.append(", ".join(keywords.split(",")[:3]).title())
+            
+        time.sleep(0.5)
+        
+    return results
+

tools.py

@@ -102,19 +102,46 @@ def load_csv(filepath: str) -> pd.DataFrame:
     logger.info("Loaded %d rows × %d columns", len(df), len(df.columns))
 
     # Validate required columns (case-insensitive match)
-    col_map = {c.strip().lower(): c for c in df.columns}
-    required = {"title", "abstract"}
-    missing = required - set(col_map.keys())
-    if missing:
+    col_map = {str(c).strip().lower(): c for c in df.columns}
+    
+    # Title and Abstract are strictly required
+    if "title" not in col_map or "abstract" not in col_map:
+        missing = {"title", "abstract"} - set(col_map.keys())
         raise ValueError(f"CSV is missing required columns: {missing}. Found: {list(df.columns)}")
+        
+    # Map Title and Abstract
+    rename_dict = {
+        col_map["title"]: "Title",
+        col_map["abstract"]: "Abstract"
+    }
+
+    # Map Keywords
+    keyword_cols = ["keywords", "author keywords", "index keywords"]
+    found_keyword = next((c for c in keyword_cols if c in col_map), None)
+    if found_keyword:
+        rename_dict[col_map[found_keyword]] = "Keywords"
+        
+    # Map Full Text
+    fulltext_cols = ["full text", "body"]
+    found_fulltext = next((c for c in fulltext_cols if c in col_map), None)
+    if found_fulltext:
+        rename_dict[col_map[found_fulltext]] = "Full Text"
 
     # Rename to canonical form
-    df = df.rename(columns={col_map["title"]: "Title", col_map["abstract"]: "Abstract"})
+    df = df.rename(columns=rename_dict)
+
+    # Fill missing columns
+    if "Keywords" not in df.columns:
+        df["Keywords"] = ""
+    if "Full Text" not in df.columns:
+        df["Full Text"] = df["Abstract"]
 
-    # Drop rows where both Title and Abstract are empty
-    df = df.dropna(subset=["Title", "Abstract"], how="all").reset_index(drop=True)
+    # Drop rows where all of these are empty
+    df = df.dropna(subset=["Title", "Abstract", "Keywords", "Full Text"], how="all").reset_index(drop=True)
     df["Title"] = df["Title"].fillna("")
     df["Abstract"] = df["Abstract"].fillna("")
+    df["Keywords"] = df["Keywords"].fillna("")
+    df["Full Text"] = df["Full Text"].fillna("")
 
     logger.info("After cleaning: %d usable rows", len(df))
     return df
@@ -183,22 +210,23 @@ def preprocess_text(documents: List[str]) -> List[str]:
 
 
 # ===================================================================
-# 3. run_topic_modeling  — Discover topics via BERTopic (or LDA fallback)
+# 3. run_topic_modeling  — Discover clusters + sub-topics
 # ===================================================================
 def run_topic_modeling(
     documents: List[str],
     source_label: str = "documents",
     min_topics: int = 100,
+    max_clusters: int = 25,
     use_bertopic: bool = True,
 ) -> Tuple[pd.DataFrame, Any]:
     """
-    Perform topic modeling on a corpus of preprocessed documents.
+    Perform two-level topic modeling on a corpus of preprocessed documents.
 
     Strategy:
-      1. Try BERTopic with UMAP + HDBSCAN.  If the result has < min_topics,
-         automatically fall back to sklearn LDA.
-      2. LDA is configured with n_components = min_topics to guarantee the
-         requested topic count.
+      1. First pass: Discover clusters (target 10–max_clusters range)
+         using BERTopic with UMAP + HDBSCAN, or LDA fallback.
+      2. Second pass: Within each cluster, extract fine-grained sub-topics
+         using NMF to guarantee ≥ min_topics total topics.
 
     Parameters
     ----------
@@ -207,46 +235,216 @@ def run_topic_modeling(
     source_label : str
         Label for logging (e.g. "Titles" or "Abstracts").
     min_topics : int
-        Minimum number of topics required (default 100).
+        Minimum total number of topics (sub-topics) required (default 100).
+    max_clusters : int
+        Target maximum number of high-level clusters (default 25).
     use_bertopic : bool
         Whether to attempt BERTopic first.
 
     Returns
     -------
     topics_df : pd.DataFrame
-        Columns: topic_id, keywords (comma-separated), representative_docs
+        Columns: cluster_id, topic_id, sub_topic_id, keywords, label, source, n_documents
     model : object
-        The fitted topic model for downstream inspection.
+        The fitted cluster-level model for downstream inspection.
     """
+    from sklearn.decomposition import NMF
+    from sklearn.feature_extraction.text import TfidfVectorizer
+
     # Filter out empty documents
     valid_docs = [d for d in documents if d.strip()]
     if len(valid_docs) < 20:
         raise ValueError(f"Not enough valid documents ({len(valid_docs)}) for topic modeling.")
 
-    logger.info("Running topic modeling on %d %s (target ≥ %d topics)", len(valid_docs), source_label, min_topics)
+    logger.info(
+        "Running topic modeling on %d %s (target: %d–%d clusters, ≥ %d topics)",
+        len(valid_docs), source_label, 10, max_clusters, min_topics,
+    )
 
-    topics_df = None
+    # ---- FIRST PASS: Discover clusters ----
+    cluster_df = None
     model = None
 
-    # ------ Attempt BERTopic ------
     if use_bertopic:
         try:
-            topics_df, model = _run_bertopic(valid_docs, source_label, min_topics)
+            cluster_df, model = _run_bertopic(valid_docs, source_label, max_clusters)
         except Exception as exc:
             logger.warning("BERTopic failed (%s). Falling back to LDA.", exc)
-            topics_df = None
+            cluster_df = None
 
-    # ------ Fallback to LDA if needed ------
-    if topics_df is None or len(topics_df) < min_topics:
-        logger.info("Using LDA to guarantee ≥ %d topics for %s", min_topics, source_label)
-        topics_df, model = _run_lda(valid_docs, source_label, min_topics)
+    if cluster_df is None or len(cluster_df) < 3:
+        logger.info("Using LDA for clustering %s (target %d clusters)", source_label, max_clusters)
+        cluster_df, model = _run_lda(valid_docs, source_label, max_clusters)
 
-    logger.info("Topic modeling complete for %s: %d topics discovered", source_label, len(topics_df))
+    n_clusters = len(cluster_df)
+    logger.info("First pass: %d clusters discovered for %s", n_clusters, source_label)
+
+    # ---- SECOND PASS: Extract sub-topics within each cluster ----
+    # Assign each document to its nearest cluster based on keyword overlap
+    cluster_assignments = _assign_docs_to_clusters(valid_docs, cluster_df)
+
+    # Calculate sub-topic counts per cluster
+    cluster_sizes = {}
+    for cid in cluster_df["topic_id"].tolist():
+        cluster_sizes[cid] = int((np.array(cluster_assignments) == cid).sum())
+
+    total_assigned = sum(cluster_sizes.values())
+    subtopic_counts = {}
+    for cid in cluster_df["topic_id"].tolist():
+        if cluster_sizes[cid] == 0:
+            subtopic_counts[cid] = 1
+            continue
+        raw_share = (cluster_sizes[cid] / max(total_assigned, 1)) * min_topics
+        subtopic_counts[cid] = max(2, round(raw_share))
+
+    # Ensure total >= min_topics
+    current_total = sum(subtopic_counts.values())
+    if current_total < min_topics:
+        deficit = min_topics - current_total
+        sorted_clusters = sorted(
+            cluster_df["topic_id"].tolist(),
+            key=lambda c: cluster_sizes.get(c, 0), reverse=True,
+        )
+        i = 0
+        while deficit > 0 and i < len(sorted_clusters) * 10:
+            cid = sorted_clusters[i % len(sorted_clusters)]
+            max_possible = max(cluster_sizes.get(cid, 0) // 2, 2)
+            if subtopic_counts[cid] < max_possible:
+                subtopic_counts[cid] += 1
+                deficit -= 1
+            i += 1
+
+    # Extract sub-topics per cluster
+    all_topics = []
+    global_topic_id = 0
+
+    for _, cluster_row in cluster_df.iterrows():
+        cid = int(cluster_row["topic_id"])
+        # Get documents in this cluster
+        doc_indices = [i for i, a in enumerate(cluster_assignments) if a == cid]
+        cluster_docs = [valid_docs[i] for i in doc_indices]
+        n_subtopics = subtopic_counts.get(cid, 2)
+
+        if len(cluster_docs) < 2:
+            # Single document cluster — single topic
+            all_topics.append({
+                "cluster_id": cid,
+                "topic_id": global_topic_id,
+                "sub_topic_id": 0,
+                "keywords": cluster_row["keywords"],
+                "source": source_label,
+                "n_documents": len(cluster_docs),
+            })
+            global_topic_id += 1
+            continue
+
+        n_subtopics = min(n_subtopics, len(cluster_docs))
+        if n_subtopics < 2:
+            n_subtopics = 1
+
+        try:
+            vectorizer = TfidfVectorizer(
+                stop_words="english", max_df=0.95, min_df=1,
+                ngram_range=(1, 2), max_features=5000,
+            )
+            tfidf_matrix = vectorizer.fit_transform(cluster_docs)
+            feature_names = vectorizer.get_feature_names_out()
+
+            if n_subtopics == 1 or tfidf_matrix.shape[0] < 2:
+                scores = tfidf_matrix.sum(axis=0).A1
+                top_indices = scores.argsort()[-10:][::-1]
+                kw = ", ".join([feature_names[i] for i in top_indices])
+                all_topics.append({
+                    "cluster_id": cid,
+                    "topic_id": global_topic_id,
+                    "sub_topic_id": 0,
+                    "keywords": kw,
+                    "source": source_label,
+                    "n_documents": len(cluster_docs),
+                })
+                global_topic_id += 1
+            else:
+                nmf = NMF(n_components=n_subtopics, random_state=42, max_iter=300, init="nndsvda")
+                W = nmf.fit_transform(tfidf_matrix)
+                H = nmf.components_
+                doc_assignments = W.argmax(axis=1)
+
+                for sub_id in range(n_subtopics):
+                    top_indices = H[sub_id].argsort()[-10:][::-1]
+                    kw = ", ".join([feature_names[i] for i in top_indices])
+                    n_docs = int((doc_assignments == sub_id).sum())
+                    if n_docs == 0:
+                        n_docs = 1
+
+                    all_topics.append({
+                        "cluster_id": cid,
+                        "topic_id": global_topic_id,
+                        "sub_topic_id": sub_id,
+                        "keywords": kw,
+                        "source": source_label,
+                        "n_documents": n_docs,
+                    })
+                    global_topic_id += 1
+
+        except Exception as e:
+            logger.warning("NMF failed for cluster %d: %s. Using cluster keywords.", cid, e)
+            all_topics.append({
+                "cluster_id": cid,
+                "topic_id": global_topic_id,
+                "sub_topic_id": 0,
+                "keywords": cluster_row["keywords"],
+                "source": source_label,
+                "n_documents": len(cluster_docs),
+            })
+            global_topic_id += 1
+
+    topics_df = pd.DataFrame(all_topics)
+
+    # Add cluster_label from cluster keywords
+    cluster_label_map = {}
+    for _, row in cluster_df.iterrows():
+        cid = int(row["topic_id"])
+        kws = [kw.strip().title() for kw in row["keywords"].split(",")][:3]
+        cluster_label_map[cid] = " / ".join(kws)
+    topics_df["cluster_label"] = topics_df["cluster_id"].map(cluster_label_map).fillna("")
+
+    logger.info(
+        "Topic modeling complete for %s: %d topics in %d clusters",
+        source_label, len(topics_df), n_clusters,
+    )
     return topics_df, model
 
 
-def _run_bertopic(docs: List[str], source_label: str, min_topics: int):
-    """Run BERTopic with tuned parameters."""
+def _assign_docs_to_clusters(docs: List[str], cluster_df: pd.DataFrame) -> List[int]:
+    """Assign each document to its most similar cluster based on keyword overlap."""
+    cluster_ids = cluster_df["topic_id"].tolist()
+    cluster_keywords = []
+    for _, row in cluster_df.iterrows():
+        kws = set(kw.strip().lower() for kw in row["keywords"].split(",") if kw.strip())
+        # Also add individual words from multi-word keywords
+        expanded = set()
+        for kw in kws:
+            expanded.update(kw.split())
+        expanded.update(kws)
+        cluster_keywords.append(expanded)
+
+    assignments = []
+    for doc in docs:
+        doc_tokens = set(doc.lower().split())
+        best_score = -1
+        best_cid = cluster_ids[0]  # default to first cluster
+        for cid, kws in zip(cluster_ids, cluster_keywords):
+            overlap = len(doc_tokens & kws)
+            if overlap > best_score:
+                best_score = overlap
+                best_cid = cid
+        assignments.append(best_cid)
+
+    return assignments
+
+
+def _run_bertopic(docs: List[str], source_label: str, max_clusters: int):
+    """Run BERTopic to discover clusters (not topics)."""
     from bertopic import BERTopic
     from umap import UMAP
     from hdbscan import HDBSCAN
@@ -260,7 +458,7 @@ def _run_bertopic(docs: List[str], source_label: str, min_topics: int):
         random_state=42,
     )
     hdbscan_model = HDBSCAN(
-        min_cluster_size=5,
+        min_cluster_size=max(5, len(docs) // max_clusters),
         min_samples=2,
         prediction_data=True,
     )
@@ -275,7 +473,7 @@ def _run_bertopic(docs: List[str], source_label: str, min_topics: int):
         umap_model=umap_model,
         hdbscan_model=hdbscan_model,
         vectorizer_model=vectorizer,
-        nr_topics="auto",
+        nr_topics=max_clusters,
         top_n_words=10,
         verbose=False,
     )
@@ -296,8 +494,8 @@ def _run_bertopic(docs: List[str], source_label: str, min_topics: int):
     return df, topic_model
 
 
-def _run_lda(docs: List[str], source_label: str, n_topics: int):
-    """Run sklearn LDA to guarantee the requested number of topics."""
+def _run_lda(docs: List[str], source_label: str, n_clusters: int):
+    """Run sklearn LDA to discover clusters."""
     from sklearn.decomposition import LatentDirichletAllocation
     from sklearn.feature_extraction.text import CountVectorizer
 
@@ -312,7 +510,7 @@ def _run_lda(docs: List[str], source_label: str, n_topics: int):
     feature_names = vectorizer.get_feature_names_out()
 
     lda = LatentDirichletAllocation(
-        n_components=n_topics,
+        n_components=n_clusters,
         max_iter=25,
         learning_method="online",
         random_state=42,
@@ -624,3 +822,451 @@ def create_taxonomy_map(
 
     logger.info("Taxonomy: %d MAPPED, %d NOVEL", len(mapped), len(novel))
     return taxonomy
+
+# ===================================================================
+# TCCM Extraction Tool
+# ===================================================================
+def extract_tccm(documents: List[str], api_key: str) -> pd.DataFrame:
+    # Basic full-text TCCM extraction stub
+    logger.info("Extracting TCCM from full text...")
+    return pd.DataFrame([{"TCCM": "Sample"} for _ in documents])
+
+
+# ===================================================================
+# SPECTER2 Pipeline Functions
+# ===================================================================
+
+def generate_specter2_embeddings(papers: List[Dict], batch_size: int = 16) -> np.ndarray:
+    """
+    Generate embeddings using SPECTER2 via adapters.
+    papers: list of dicts with 'title' and 'abstract' keys.
+    """
+    import torch
+    from transformers import AutoTokenizer
+    from adapters import AutoAdapterModel
+
+    logger.info("Loading SPECTER2 model and tokenizer...")
+    tokenizer = AutoTokenizer.from_pretrained('allenai/specter2_base')
+    model = AutoAdapterModel.from_pretrained('allenai/specter2_base')
+    model.load_adapter("allenai/specter2", source="hf", load_as="proximity", set_active=True)
+
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model.to(device)
+    model.eval()
+
+    embeddings = []
+    logger.info("Generating embeddings for %d papers using %s...", len(papers), device)
+
+    for i in range(0, len(papers), batch_size):
+        batch = papers[i:i+batch_size]
+        text_batch = [d['title'] + tokenizer.sep_token + (d.get('abstract') or '') for d in batch]
+
+        inputs = tokenizer(
+            text_batch, padding=True, truncation=True,
+            return_tensors="pt", return_token_type_ids=False, max_length=512,
+        )
+        inputs = {k: v.to(device) for k, v in inputs.items()}
+
+        with torch.no_grad():
+            output = model(**inputs)
+            batch_embeddings = output.last_hidden_state[:, 0, :].cpu().numpy()
+            embeddings.append(batch_embeddings)
+
+    return np.vstack(embeddings)
+
+
+def store_in_chroma(embeddings, documents, metadatas, collection_name="specter_vectors"):
+    """Store embeddings in ChromaDB for persistence."""
+    import chromadb
+
+    client = chromadb.PersistentClient(path="./chroma_db")
+    # Delete existing collection to avoid ID conflicts
+    try:
+        client.delete_collection(name=collection_name)
+    except Exception:
+        pass
+    collection = client.get_or_create_collection(name=collection_name)
+
+    ids = [str(i) for i in range(len(documents))]
+    collection.add(
+        embeddings=embeddings.tolist(),
+        documents=documents,
+        metadatas=metadatas,
+        ids=ids,
+    )
+    return collection
+
+
+def load_embeddings_from_chroma(collection_name="specter_vectors"):
+    """Load cached embeddings from ChromaDB."""
+    import chromadb
+
+    client = chromadb.PersistentClient(path="./chroma_db")
+    collection = client.get_collection(name=collection_name)
+    result = collection.get(include=["embeddings", "documents", "metadatas"])
+
+    if not result["embeddings"]:
+        raise ValueError("No embeddings found in ChromaDB.")
+
+    embeddings = np.array(result["embeddings"])
+    documents = result["documents"]
+    metadatas = result["metadatas"]
+    return embeddings, documents, metadatas
+
+
+def adaptive_hdbscan_clustering(
+    embeddings: np.ndarray,
+    min_clusters: int = 10,
+    max_clusters: int = 40,
+) -> tuple:
+    """
+    Adaptive HDBSCAN clustering with UMAP dimensionality reduction.
+    Tries different min_cluster_size values to land within the target cluster range.
+
+    Returns: (labels, reduced_2d, n_clusters, params_used)
+    """
+    from umap import UMAP
+    from hdbscan import HDBSCAN
+
+    logger.info(
+        "Adaptive HDBSCAN: targeting %d–%d clusters for %d embeddings",
+        min_clusters, max_clusters, len(embeddings),
+    )
+
+    # UMAP reduction to 5D for clustering
+    umap_5d = UMAP(
+        n_neighbors=15, n_components=5, min_dist=0.0,
+        metric="cosine", random_state=42,
+    )
+    reduced_5d = umap_5d.fit_transform(embeddings)
+
+    # UMAP reduction to 2D for visualization
+    umap_2d = UMAP(
+        n_neighbors=15, n_components=2, min_dist=0.1,
+        metric="cosine", random_state=42,
+    )
+    reduced_2d = umap_2d.fit_transform(embeddings)
+
+    # Try different min_cluster_size to find good cluster count
+    best_labels = None
+    best_n_clusters = 0
+    best_params = {}
+    best_diff = float("inf")
+
+    target_mid = (min_clusters + max_clusters) / 2
+
+    for mcs in [5, 8, 10, 15, 20, 25, 30, 40, 50, 60, 80, 100, 120, 150]:
+        for ms in [2, 3, 5]:
+            clusterer = HDBSCAN(
+                min_cluster_size=mcs,
+                min_samples=ms,
+                metric="euclidean",
+                cluster_selection_method="eom",
+            )
+            labels = clusterer.fit_predict(reduced_5d)
+            n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
+
+            diff = abs(n_clusters - target_mid)
+            in_range = min_clusters <= n_clusters <= max_clusters
+
+            if in_range and diff < best_diff:
+                best_labels = labels
+                best_n_clusters = n_clusters
+                best_params = {"min_cluster_size": mcs, "min_samples": ms}
+                best_diff = diff
+
+            if n_clusters == round(target_mid):
+                break
+
+    # Fallback: if nothing landed in range, pick closest
+    if best_labels is None:
+        logger.warning("Could not find params for target range, using best effort")
+        for mcs in [15, 20, 25, 30, 40, 50]:
+            clusterer = HDBSCAN(
+                min_cluster_size=mcs, min_samples=3,
+                metric="euclidean", cluster_selection_method="eom",
+            )
+            labels = clusterer.fit_predict(reduced_5d)
+            n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
+            if n_clusters >= 5:
+                best_labels = labels
+                best_n_clusters = n_clusters
+                best_params = {"min_cluster_size": mcs, "min_samples": 3}
+                break
+
+    if best_labels is None:
+        raise RuntimeError("HDBSCAN could not produce any valid clusters.")
+
+    logger.info(
+        "HDBSCAN selected: %d clusters (params: %s)",
+        best_n_clusters, best_params,
+    )
+    return best_labels, reduced_2d, best_n_clusters, best_params
+
+
+def c_tf_idf(documents, m, ngram_range=(1, 1)):
+    """Compute class-based TF-IDF for topic keyword extraction."""
+    from sklearn.feature_extraction.text import CountVectorizer
+
+    count = CountVectorizer(ngram_range=ngram_range, stop_words="english").fit(documents)
+    t = count.transform(documents).toarray()
+    w = t.sum(axis=1)
+    tf = np.divide(t.T, w)
+    sum_t = t.sum(axis=0)
+    idf = np.log(np.divide(m, sum_t)).reshape(-1, 1)
+    tf_idf = np.multiply(tf, idf)
+    return tf_idf, count
+
+
+def extract_top_n_words_per_topic(tf_idf, count, docs_per_topic, n=10):
+    """Extract top N keywords per topic from c-TF-IDF matrix."""
+    words = count.get_feature_names_out()
+    labels = list(docs_per_topic.topic_id)
+    tf_idf_transposed = tf_idf.T
+    indices = tf_idf_transposed.argsort()[:, -n:]
+    top_n_words = {
+        label: [(words[j], tf_idf_transposed[i][j]) for j in indices[i]][::-1]
+        for i, label in enumerate(labels)
+    }
+    return top_n_words
+
+
+def extract_representative_sentences(
+    df: pd.DataFrame,
+    embeddings: np.ndarray,
+    cluster_labels: np.ndarray,
+    top_n: int = 5,
+) -> Dict[int, List[str]]:
+    """
+    For each cluster, find the top_n documents closest to the cluster centroid.
+    Returns {cluster_id: [sentence1, sentence2, ...]}.
+    """
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    result = {}
+    unique_labels = set(cluster_labels)
+
+    for cid in unique_labels:
+        if cid == -1:
+            continue
+        mask = cluster_labels == cid
+        cluster_embs = embeddings[mask]
+        cluster_indices = np.where(mask)[0]
+
+        if len(cluster_embs) == 0:
+            continue
+
+        centroid = cluster_embs.mean(axis=0).reshape(1, -1)
+        sims = cosine_similarity(centroid, cluster_embs)[0]
+        top_indices = sims.argsort()[-top_n:][::-1]
+
+        sentences = []
+        for idx in top_indices:
+            doc_idx = cluster_indices[idx]
+            title = str(df.iloc[doc_idx].get("Title", ""))
+            abstract = str(df.iloc[doc_idx].get("Abstract", ""))
+            # Use title + first sentence of abstract
+            first_sent = abstract.split(".")[0] if abstract else ""
+            rep = f"{title}. {first_sent}".strip()
+            if rep:
+                sentences.append(rep)
+        result[int(cid)] = sentences
+
+    return result
+
+
+def get_council_debate(keywords: str, groq_api_key: str) -> Dict[str, str]:
+    """
+    Run a 3-agent AI Council debate to generate a topic label.
+    Returns dict with: expert, analyst, anti_hallucination, final_label.
+    """
+    import time
+
+    try:
+        from groq import Groq
+    except ImportError:
+        kws = [kw.strip().title() for kw in keywords.split(",")][:3]
+        return {
+            "expert": ", ".join(kws),
+            "analyst": ", ".join(kws),
+            "anti_hallucination": "N/A",
+            "final_label": ", ".join(kws),
+        }
+
+    client = Groq(api_key=groq_api_key)
+
+    # Persona 1: Domain Expert
+    prompt_expert = (
+        f"You are a scientific Domain Expert. Keywords: [{keywords}]. "
+        "Propose a 3-5 word descriptive label for this academic topic. "
+        "Return ONLY the label."
+    )
+
+    # Persona 2: Data Analyst
+    prompt_analyst = (
+        f"You are a Data Analyst categorizing research clusters. Keywords: [{keywords}]. "
+        "Give a short, precise categorical name (3-5 words). Return ONLY the name."
+    )
+
+    try:
+        res_expert = client.chat.completions.create(
+            model="llama-3.3-70b-versatile",
+            messages=[{"role": "user", "content": prompt_expert}],
+            temperature=0.4, max_tokens=20,
+        ).choices[0].message.content.strip()
+
+        time.sleep(0.3)
+
+        res_analyst = client.chat.completions.create(
+            model="llama-3.3-70b-versatile",
+            messages=[{"role": "user", "content": prompt_analyst}],
+            temperature=0.4, max_tokens=20,
+        ).choices[0].message.content.strip()
+
+        time.sleep(0.3)
+
+        # Persona 3: Anti-hallucination checker
+        prompt_check = (
+            f"Keywords: [{keywords}].\n"
+            f"Expert label: '{res_expert}'.\n"
+            f"Analyst label: '{res_analyst}'.\n"
+            "Check: are these labels grounded in the keywords? "
+            "If yes, synthesize into a final 3-5 word label. "
+            "If not, create a grounded label from the keywords only. "
+            "Return ONLY the final label."
+        )
+
+        res_judge = client.chat.completions.create(
+            model="llama-3.3-70b-versatile",
+            messages=[{"role": "user", "content": prompt_check}],
+            temperature=0.2, max_tokens=20,
+        ).choices[0].message.content.strip()
+        res_judge = res_judge.replace('"', '').replace("'", "").strip()
+
+        time.sleep(0.3)
+
+        return {
+            "expert": res_expert,
+            "analyst": res_analyst,
+            "anti_hallucination": res_judge,
+            "final_label": res_judge,
+        }
+
+    except Exception as e:
+        logger.warning("Council debate failed: %s", e)
+        kws = [kw.strip().title() for kw in keywords.split(",")][:3]
+        return {
+            "expert": ", ".join(kws),
+            "analyst": ", ".join(kws),
+            "anti_hallucination": "fallback",
+            "final_label": ", ".join(kws),
+        }
+
+
+def build_cluster_graph(
+    topics_df: pd.DataFrame,
+    embeddings: np.ndarray,
+    cluster_labels: np.ndarray,
+    top_n_edges: int = 20,
+) -> List[Dict]:
+    """
+    Build a similarity graph between clusters based on centroid cosine similarity.
+    Returns list of edges: [{source, target, weight}, ...].
+    """
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    unique_clusters = sorted([c for c in set(cluster_labels) if c != -1])
+    if len(unique_clusters) < 2:
+        return []
+
+    # Compute centroids
+    centroids = []
+    for cid in unique_clusters:
+        mask = cluster_labels == cid
+        centroids.append(embeddings[mask].mean(axis=0))
+    centroids = np.array(centroids)
+
+    # Pairwise similarity
+    sim_matrix = cosine_similarity(centroids)
+
+    edges = []
+    for i in range(len(unique_clusters)):
+        for j in range(i + 1, len(unique_clusters)):
+            edges.append({
+                "source": int(unique_clusters[i]),
+                "target": int(unique_clusters[j]),
+                "weight": round(float(sim_matrix[i][j]), 4),
+            })
+
+    # Keep top N edges by weight
+    edges.sort(key=lambda e: e["weight"], reverse=True)
+    return edges[:top_n_edges]
+
+
+def merge_similar_clusters(
+    topics_df: pd.DataFrame,
+    cluster_labels: np.ndarray,
+    df: pd.DataFrame,
+    threshold: float = 0.75,
+    groq_api_key: Optional[str] = None,
+) -> tuple:
+    """
+    Merge clusters with highly similar keyword sets.
+    Returns (new_topics_df, new_labels).
+    """
+    from sklearn.feature_extraction.text import TfidfVectorizer
+    from sklearn.metrics.pairwise import cosine_similarity
+
+    if len(topics_df) < 2:
+        return topics_df, cluster_labels
+
+    # Compute pairwise similarity of keyword strings
+    vectorizer = TfidfVectorizer(stop_words="english")
+    kw_matrix = vectorizer.fit_transform(topics_df["keywords"].tolist())
+    sim_matrix = cosine_similarity(kw_matrix)
+
+    # Find pairs to merge
+    merge_map = {}
+    topic_ids = topics_df["topic_id"].tolist()
+
+    for i in range(len(topic_ids)):
+        for j in range(i + 1, len(topic_ids)):
+            if sim_matrix[i][j] >= threshold:
+                src = topic_ids[j]
+                tgt = topic_ids[i]
+                # Follow existing merge chains
+                while tgt in merge_map:
+                    tgt = merge_map[tgt]
+                merge_map[src] = tgt
+
+    if not merge_map:
+        return topics_df, cluster_labels
+
+    logger.info("Merging %d cluster pairs (threshold=%.2f)", len(merge_map), threshold)
+
+    # Remap labels
+    new_labels = np.array(cluster_labels, copy=True)
+    for old_id, new_id in merge_map.items():
+        new_labels[new_labels == old_id] = new_id
+
+    # Rebuild topics_df for remaining clusters
+    remaining_ids = set(topic_ids) - set(merge_map.keys())
+    new_topics_df = topics_df[topics_df["topic_id"].isin(remaining_ids)].copy()
+    new_topics_df = new_topics_df.reset_index(drop=True)
+
+    return new_topics_df, new_labels
+
+
+def extract_subtopics_within_clusters(
+    df: pd.DataFrame,
+    cluster_labels: np.ndarray,
+    min_topics: int = 100,
+    n_keywords: int = 10,
+) -> pd.DataFrame:
+    """
+    Extract fine-grained sub-topics within each cluster using NMF.
+    Delegates to the implementation in run_specter.py.
+    """
+    from run_specter import extract_subtopics_within_clusters as _extract
+    return _extract(df, cluster_labels, min_topics, n_keywords)
+