Blame for app.py

import os
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import streamlit as st
import pandas as pd
import warnings
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from bertopic import BERTopic
from bertopic.representation import MaximalMarginalRelevance, KeyBERTInspired
from sentence_transformers import SentenceTransformer, models
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from umap import UMAP
from hdbscan import HDBSCAN
import gensim.corpora as corpora
from gensim.models.coherencemodel import CoherenceModel

warnings.filterwarnings("ignore")

# ==========================================
# 1. PAGE CONFIGURATION & MAPPINGS
# ==========================================
st.set_page_config(page_title="Topic Modeling Pipeline", layout="wide", initial_sidebar_state="collapsed")

EMBEDDING_MAP = {
    "MiniLM (Fast & Lightweight)": "sentence-transformers/all-MiniLM-L6-v2",
    "MPNet (High Accuracy)": "sentence-transformers/all-mpnet-base-v2",
    "Specter2 (Scientific/Academic)": "allenai/specter2_base"
}

POOLING_MAP = {
    "Mean (Smooth context)": "mean",
    "Max (Sharp keywords)": "max",
    "CLS (Classification)": "cls",
    "Mean-Max (Combined)": "mean-max"
}

@st.cache_resource
def load_embedder(model_name, pool_strat):
    word_emb = models.Transformer(model_name)
    pool_model = models.Pooling(
        word_emb.get_word_embedding_dimension(),
        pooling_mode_mean_tokens=("mean" in pool_strat),
        pooling_mode_max_tokens=("max" in pool_strat),
        pooling_mode_cls_token=("cls" in pool_strat)
    )
    return SentenceTransformer(modules=[word_emb, pool_model])

# ==========================================
# 2. THE GUIDED UI
# ==========================================
st.title("🧠 Advanced Topic Modeling Pipeline")

try:
    st.image("pipeline.png", use_container_width=True)
except FileNotFoundError:
    pass

st.divider()

st.header("📥 Step 0: Input Data & Core Settings")
st.info("💡 **Preprocessing Note:** You do not need to manually lowercase or strip punctuation. The `CountVectorizer` algorithm and the `Uncased` BERT Neural Networks handle casing and token normalization autonomously at the mathematical level.")

data_source = st.radio("Choose Data Source:", ["Use Sample ACM Abstract", "Paste Text"], horizontal=True)

sample_abstract = """

Students who registered for the Mapping with Google massive open online course (MOOC) 

were asked several questions during the registration process to identify prior 

experience with eleven skills as well as their goals for registering for the course. 

At the end of the course, we compared students' self reports of goal achievement 

with behavioral click-stream analysis. In addition, we assessed how well prior 

skill in a subject predicts a student's course completion and found no correlation. 

Our research shows that students who completed course activities were more likely 

to earn certificates of completion than peers who did not.

"""

raw_data = st.text_area("Text Data:", value=sample_abstract if data_source == "Use Sample ACM Abstract" else "", height=150)

col_a, col_b = st.columns(2)
with col_a:
    n_themes = st.slider("Target Number of Themes", 2, 20, 3)
with col_b:
    words_per_theme = st.slider("Words to Output per Theme", 3, 10, 5)

# --- THE VERTICAL CONFIGURATION WIZARD ---
st.header("⚙️ Model Configuration")

with st.expander("1️⃣ Semantic Layer (Embeddings & Pooling)", expanded=True):
    st.markdown("*💡 **BERTopic Default:** Uses `all-MiniLM-L6-v2` with `Mean` pooling.*")
    ui_embedding = st.selectbox("Embedding Model (Override Default):", list(EMBEDDING_MAP.keys()))
    ui_pooling = st.selectbox("Pooling Strategy (Override Default):", list(POOLING_MAP.keys()))

with st.expander("2️⃣ Geometry Layer (Dimensionality Reduction)", expanded=True):
    st.markdown("*💡 **BERTopic Default:** Uses `UMAP` with `Cosine` distance to reduce 384D to 5D space.*")
    ui_algo = st.selectbox("Algorithm", ["UMAP (Complex geometry)", "PCA (Fast/Deterministic)"])
    if "UMAP" in ui_algo:
        ui_metric = st.selectbox("Distance Metric", ["cosine", "euclidean", "manhattan"])
    else:
        ui_metric = "euclidean"
        st.info("PCA inherently uses Variance (Euclidean math), so distance metrics are bypassed.")

with st.expander("3️⃣ Clustering Layer (Grouping)", expanded=True):
    st.markdown("*💡 **BERTopic Default:** Uses `HDBSCAN` exclusively (which crashes on tiny datasets).*")
    st.markdown("""

    *Our Engine mathematically draws boundaries around similar sentences using a smart-routing system:*

    * **Primary Engine (HDBSCAN):** Runs on datasets $\ge$ 15 sentences. Automatically filters outliers and finds dense semantic clouds. *(Defaults: min_cluster_size=10)*

    * **Fallback Engine (K-Means):** Runs on datasets $<$ 15 sentences. Forces all sentences into buckets to prevent math crashes on tiny text samples.

    """)

with st.expander("4️⃣ Vocabulary Layer (Vectorization)", expanded=True):
    st.markdown("*💡 **BERTopic Default:** Uses `Unigrams` (1 word) and does **not** filter redundant dataset noise.*")
    ngram_range = st.slider("N-Gram Range", 1, 3, (1, 2), help="1=Unigrams, 2=Bigrams (e.g., 'machine learning')")
    auto_noise = st.checkbox("Auto-Remove Redundant Noise (max_df)", value=True, help="Mathematically deletes words appearing in >85% of documents.")

with st.expander("5️⃣ Extraction Layer (Representation)", expanded=True):
    st.markdown("*💡 **BERTopic Default:** ALWAYS extracts baseline words using **c-TF-IDF** (Word Frequency).*")
    ui_extraction = st.selectbox("Apply Advanced Filter on top of c-TF-IDF:", ["None (Base c-TF-IDF only)", "KeyBERTInspired (Semantic cosine)", "MMR (Reduce redundancy)"])
    if "MMR" in ui_extraction:
        mmr_diversity = st.slider("MMR Diversity Penalty", 0.0, 1.0, 0.3)
    else:
        mmr_diversity = None

st.header("📊 Evaluation Metrics")
eval_metrics = st.multiselect(
    "Select KPIs to generate a final report card:",
    ["Topic Diversity", "NPMI Coherence", "UMass Coherence", "Silhouette Score"],
    default=["Topic Diversity", "NPMI Coherence", "UMass Coherence", "Silhouette Score"]
)

st.divider()

# ==========================================
# 3. ENGINE EXECUTION
# ==========================================
if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_width=True):
    
    if not raw_data or len(raw_data) < 20:
        st.error("Please provide more text data!")
        st.stop()

    with st.spinner("Processing Semantic Pipeline... (Models are cached to prevent crashes)"):
        
        sentences = [s.strip() for s in raw_data.split('.') if len(s.strip()) > 10]
        dataset_size = len(sentences)
        
        academic_noise = ['students', 'course', 'research', 'paper', 'found', 'likely', 'did']
        from sklearn.feature_extraction import text
        stop_w = list(text.ENGLISH_STOP_WORDS.union(academic_noise))
        
        vectorizer_model = CountVectorizer(stop_words=stop_w, ngram_range=ngram_range, max_df=0.85 if auto_noise and dataset_size > 10 else 1.0)
        
        custom_embedder = load_embedder(EMBEDDING_MAP[ui_embedding], POOLING_MAP[ui_pooling])
        embeddings = custom_embedder.encode(sentences)
        
        is_fallback = False
        if dataset_size < 15 or "PCA" in ui_algo:
            safe_n_themes = min(n_themes, dataset_size)
            dim_model = PCA(n_components=2, random_state=42)
            cluster_model = KMeans(n_clusters=safe_n_themes, random_state=42)
            reduce_topics = None
            is_fallback = True
            algo_used = "PCA"
            cluster_algo = "K-Means"
        else:
            dim_model = UMAP(n_neighbors=15, n_components=5, metric=ui_metric, random_state=42)
            clustering_model = HDBSCAN(min_cluster_size=10, metric='euclidean', cluster_selection_method='eom')
            reduce_topics = n_themes
            algo_used = "UMAP"
            cluster_algo = "HDBSCAN"

        if "MMR" in ui_extraction:
            rep_model = MaximalMarginalRelevance(diversity=mmr_diversity, top_n_words=words_per_theme)
        elif "KeyBERT" in ui_extraction:
            rep_model = KeyBERTInspired(top_n_words=words_per_theme)
        else:
            rep_model = None

        topic_model = BERTopic(
            embedding_model=custom_embedder,
            umap_model=dim_model,
            hdbscan_model=cluster_model,
            vectorizer_model=vectorizer_model,
            representation_model=rep_model,
            nr_topics=reduce_topics,
            top_n_words=words_per_theme,
            language="english"
        )
        topics, _ = topic_model.fit_transform(sentences)

    # ==========================================
    # 4. UI DISPLAY & METRICS
    # ==========================================
    st.success("Analysis Complete!")
    
    if is_fallback:
        if safe_n_themes < n_themes:
            st.warning(f"⚠️ **Reduced requested themes from {n_themes} to {safe_n_themes}.**\n\n"
                       f"*The Math Explanation:* BERTopic clusters complete sentences to preserve context. "
                       f"You cannot sort {dataset_size} sentences into {n_themes} buckets without leaving empty buckets, "
                       f"which mathematically breaks the clustering algorithms!")
        else:
            st.info(f"ℹ️ Auto-switched to PCA/K-Means due to small dataset size ({dataset_size} sentences).")
            
    st.markdown("### 🏆 Discovered Themes")
    topic_info = topic_model.get_topic_info()
    all_words = []
    
    cols = st.columns(3)
    col_idx = 0
    for t_id in topic_info['Topic']:
        if t_id == -1: continue
        theme_w = [w[0] for w in topic_model.get_topic(t_id)]
        all_words.append(theme_w)
        with cols[col_idx % 3]:
            st.info(f"**Theme {t_id + 1}**\n\n" + "\n".join([f"🔹 {w}" for w in theme_w]))
        col_idx += 1

    # --- METRICS CALCULATIONS ---
    div_val, npmi_val, umass_val, sil_val = 0.0, 0.0, 0.0, 0.0
    div_status, npmi_status, umass_status, sil_status = "Skipped", "Skipped", "Skipped", "Skipped"
    
    if len(eval_metrics) > 0:
        with st.spinner("Calculating mathematical metrics... (NPMI requires building a dictionary and takes a moment)"):
            
            # Diversity
            if "Topic Diversity" in eval_metrics and len(all_words) > 0:
                u_words = set([w for t in all_words for w in t])
                t_words = sum([len(t) for t in all_words])
                div_val = float(len(u_words) / t_words) if t_words > 0 else 0.0
                div_status = f"{div_val:.2f}"
            
            # Coherence Models (NPMI and UMass)
            if "NPMI Coherence" in eval_metrics or "UMass Coherence" in eval_metrics:
                try:
                    tokenized = [vectorizer_model.build_analyzer()(s) for s in sentences]
                    dictionary = corpora.Dictionary(tokenized)
                    
                    if "NPMI Coherence" in eval_metrics:
                        cm_npmi = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='c_npmi')
                        temp_npmi = cm_npmi.get_coherence()
                        npmi_status = "N/A (Too few words)" if np.isnan(temp_npmi) else f"{float(temp_npmi):.2f}"
                    
                    if "UMass Coherence" in eval_metrics:
                        cm_umass = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='u_mass')
                        temp_umass = cm_umass.get_coherence()
                        umass_status = "N/A (Too few words)" if np.isnan(temp_umass) else f"{float(temp_umass):.2f}"
                except Exception:
                    npmi_status = "Skipped (Data too small)"
                    umass_status = "Skipped (Data too small)"

            # Silhouette
            if "Silhouette Score" in eval_metrics:
                valid_idx = [i for i, t in enumerate(topics) if t != -1]
                unique_topics = set([topics[i] for i in valid_idx])
                if 1 < len(unique_topics) < len(valid_idx):
                    sil_val = float(silhouette_score(
                        np.array([embeddings[i] for i in valid_idx]), 
                            [topics[i] for i in valid_idx], 
                            metric='cosine'
                        ))
                        sil_status = f"{sil_val:.2f}"
                    else:
                        sil_status = "Skipped (Themes need ≥2 sentences)"

        # --- RENDER KPI DASHBOARD (UI draws instantly here!) ---
        st.markdown("### 📊 Key Performance Indicators (KPI)")
        kpi_cols = st.columns(len(eval_metrics))
        
        for idx, metric in enumerate(eval_metrics):
            with kpi_cols[idx]:
                if metric == "Topic Diversity":
                    st.metric("Topic Diversity (Target: 1.0)", div_status)
                elif metric == "NPMI Coherence":
                    st.metric("NPMI Coherence (Target: >0.1)", npmi_status)
                elif metric == "UMass Coherence":
                    st.metric("UMass Coherence (Target: closer to 0)", umass_status)
                elif metric == "Silhouette Score":
                    st.metric("Silhouette Score (Target: >0.0)", sil_status)

    # ==========================================
    # 5. XAI VISUALIZATION GRAPH
    # ==========================================
    st.markdown("### 📈 Explainable AI (XAI) Architecture Map")
    
    with st.spinner("Rendering Explainable AI Dashboard..."):
        sns.set_theme(style="whitegrid")
        fig = plt.figure(figsize=(16, 14)) 
        fig.suptitle(f"Topic Modeling Pipeline Analytics\n(Pooling: {ui_pooling.split()[0]} | Rep: {ui_extraction.split()[0]})", fontsize=20, fontweight='bold', y=0.98)
        box_style = dict(boxstyle="round,pad=0.4", facecolor='lightyellow', edgecolor='orange', alpha=0.9)

        # 1. Embeddings
        ax1 = plt.subplot(3, 2, 1)
        sns.heatmap(embeddings[:, :50], cmap="viridis", cbar=False, ax=ax1)
        ax1.set_title("STEP 1: Embeddings & Pooling", fontsize=13, fontweight='bold')
        ax1.set_ylabel("Sentences")
        ax1.set_xlabel("Vector Dimensions (First 50 shown)")
        ax1.text(0.5, -0.25, f"Math: {ui_embedding.split('/')[-1]} encodes text into 384D.\nPooling '{ui_pooling.split()[0]}' squashes word vectors into 1 sentence vector.", 
                 fontsize=10, ha='center', va='top', transform=ax1.transAxes, bbox=box_style)

        # 2. Geometry
        ax2 = plt.subplot(3, 2, 2)
        reduced_embeddings = topic_model.umap_model.transform(embeddings)
        ax2.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c='grey', s=100, alpha=0.6, edgecolor='k')
        ax2.set_title(f"STEP 2: Geometry ({algo_used})", fontsize=13, fontweight='bold')
        ax2.text(0.5, -0.25, f"Math: {algo_used} reduces 384D vectors into a 2D map.\nPlaces similar sentences close together.", 
                 fontsize=10, ha='center', va='top', transform=ax2.transAxes, bbox=box_style)

        # 3. Clustering
        ax3 = plt.subplot(3, 2, 3)
        ax3.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c=topics, cmap='tab10', s=150, edgecolor='k')
        ax3.set_title(f"STEP 3: Clustering ({cluster_algo})", fontsize=13, fontweight='bold')
        ax3.text(0.5, -0.25, f"Math: {cluster_algo} scans the 2D space to draw boundaries.\nColors represent assigned semantic clusters.", 
                 fontsize=10, ha='center', va='top', transform=ax3.transAxes, bbox=box_style)

        # 4. Representation
        ax4 = plt.subplot(3, 2, 4)
        theme_1_data = topic_model.get_topic(0)
        if theme_1_data:
            words = [x[0] for x in theme_1_data][::-1] 
            scores = [x[1] for x in theme_1_data][::-1]
            ax4.barh(words, scores, color='coral', edgecolor='black')
            ax4.set_title(f"STEP 4: Topic Representation ({ui_extraction.split()[0]})", fontsize=13, fontweight='bold')
            ax4.text(0.5, -0.25, f"Math: Applies {ui_extraction.split()[0]} to rank vocabulary.\nLonger bars = higher semantic relevance.", 
                     fontsize=10, ha='center', va='top', transform=ax4.transAxes, bbox=box_style)
        else:
            ax4.text(0.5, 0.5, "Theme not found", ha='center', transform=ax4.transAxes)

        # 5. KPI Dashboard (Updated with UMass)
        ax5 = plt.subplot(3, 2, 5)
        ax5.axis('off') 
        ax5.set_title("STEP 5: Key Performance Indicators (KPI)", fontsize=13, fontweight='bold', y=0.9)
        
        kpi_text = (
            f"📊 Topic Diversity:  {div_status} (Target: 1.0)\n\n"
            f"🧠 NPMI Coherence:  {npmi_status} (Target: >0.1)\n\n"
            f"📚 UMass Coherence:  {umass_status} (Target: Closer to 0)\n\n"
            f"📐 Silhouette Score:  {sil_status} (Target: >0.0)"
        )
        ax5.text(0.5, 0.4, kpi_text, fontsize=11, va='center', ha='center', 
                 bbox=dict(boxstyle="square,pad=1.5", facecolor='#e6f2ff', edgecolor='#377eb8', lw=2))
        ax5.text(0.5, -0.15, "Math: Since pipeline algorithms don't use 'Training Loss',\nthese KPIs provide the absolute mathematical grade of the topics.", 
                 fontsize=10, ha='center', va='top', transform=ax5.transAxes, bbox=box_style)

        # 6. Summary Panel
        ax6 = plt.subplot(3, 2, 6)
        ax6.axis('off') 
        summary_text = (
            "=== PIPELINE ARCHITECTURE ===\n\n"
            f"1. Embeddings: {ui_embedding.split()[0]}\n"
            f"2. Pooling: {ui_pooling.split()[0]}\n"
            f"3. N-Grams: {ngram_range}\n"
            f"4. Geometry: {algo_used}\n"
            f"5. Clustering: {cluster_algo}\n"
            f"6. Representation: {ui_extraction.split()[0]}\n\n"
            "This modular pipeline successfully transforms unstructured text\n"
            "into mathematically validated semantic domains."
        )
        ax6.text(0.1, 0.5, summary_text, fontsize=12, va='center', ha='left', 
                 bbox=dict(boxstyle="square,pad=1", facecolor='#f0f0f0', edgecolor='grey', lw=2))

        plt.subplots_adjust(hspace=0.6, wspace=0.3)
        st.pyplot(fig, use_container_width=True)