Blame for app.py

import os
# --- ANTI-CRASH ENVIRONMENT VARIABLES ---
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["TOKENIZERS_PARALLELISM"] = "false"

import streamlit as st
import pandas as pd
import warnings
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from bertopic import BERTopic
from bertopic.representation import MaximalMarginalRelevance, KeyBERTInspired
from sentence_transformers import SentenceTransformer, models
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from umap import UMAP
from hdbscan import HDBSCAN
import gensim.corpora as corpora
from gensim.models.coherencemodel import CoherenceModel

warnings.filterwarnings("ignore")

# ==========================================
# 1. PAGE CONFIGURATION & MAPPINGS
# ==========================================
st.set_page_config(page_title="Topic Modeling Pipeline", layout="wide", initial_sidebar_state="collapsed")

EMBEDDING_MAP = {
    "MiniLM (Fast & Lightweight)": "sentence-transformers/all-MiniLM-L6-v2",
    "MPNet (High Accuracy)": "sentence-transformers/all-mpnet-base-v2",
    "Specter2 (Scientific/Academic)": "allenai/specter2_base"
}

POOLING_MAP = {
    "Mean (Smooth context)": "mean",
    "Max (Sharp keywords)": "max",
    "CLS (Classification)": "cls",
    "Mean-Max (Combined)": "mean-max"
}

# --- CACHE THE NEURAL NETWORK ---
@st.cache_resource
def load_embedder(model_name, pool_strat):
    word_emb = models.Transformer(model_name)
    pool_model = models.Pooling(
        word_emb.get_word_embedding_dimension(),
        pooling_mode_mean_tokens=("mean" in pool_strat),
        pooling_mode_max_tokens=("max" in pool_strat),
        pooling_mode_cls_token=("cls" in pool_strat)
    )
    return SentenceTransformer(modules=[word_emb, pool_model])

# ==========================================
# 2. THE GUIDED UI (MAIN PAGE)
# ==========================================
st.title("🧠 BERTopic Topic Modeling Pipeline")

try:
    st.image("pipeline.png", use_container_width=True)
except FileNotFoundError:
    pass

st.divider()

# --- STEP 0: DATA SETTINGS ---
st.header("📥 Step 0: Input Data & Core Settings")
data_source = st.radio("Choose Data Source:", ["Use Sample ACM Abstract", "Paste Text"], horizontal=True)

sample_abstract = """

Students who registered for the Mapping with Google massive open online course (MOOC) 

were asked several questions during the registration process to identify prior 

experience with eleven skills as well as their goals for registering for the course. 

At the end of the course, we compared students' self reports of goal achievement 

with behavioral click-stream analysis. In addition, we assessed how well prior 

skill in a subject predicts a student's course completion and found no correlation. 

Our research shows that students who completed course activities were more likely 

to earn certificates of completion than peers who did not.

"""

raw_data = st.text_area("Text Data:", value=sample_abstract if data_source == "Use Sample ACM Abstract" else "", height=150)

col_a, col_b = st.columns(2)
with col_a:
    n_themes = st.slider("Target Number of Themes", 2, 20, 3)
with col_b:
    words_per_theme = st.slider("Words to Output per Theme", 3, 10, 5)

# --- THE VERTICAL CONFIGURATION WIZARD ---
st.header("⚙️ Model Configuration")

with st.expander("1️⃣ Semantic Layer (Embeddings & Pooling)", expanded=True):
    ui_embedding = st.selectbox("Embedding Model", list(EMBEDDING_MAP.keys()))
    ui_pooling = st.selectbox("Pooling Strategy", list(POOLING_MAP.keys()))

with st.expander("2️⃣ Geometry Layer (Dimensionality Reduction)", expanded=True):
    ui_algo = st.selectbox("Algorithm", ["UMAP (Complex geometry)", "PCA (Fast/Deterministic)"])
    if "UMAP" in ui_algo:
        ui_metric = st.selectbox("Distance Metric", ["cosine", "euclidean", "manhattan"])
    else:
        ui_metric = "euclidean"
        st.info("PCA inherently uses Variance (Euclidean math), so distance metrics are bypassed.")

with st.expander("3️⃣ Clustering Layer (Grouping)", expanded=True):
    st.markdown("""

    *Clustering mathematically draws boundaries around similar sentences.*

    * **Primary Engine (HDBSCAN):** Runs on datasets $\ge$ 15 sentences. Automatically filters outliers and finds dense semantic clouds. 

      *(Defaults: min_cluster_size=10, cluster_selection_method='eom', metric='euclidean')*

    * **Fallback Engine (K-Means):** Runs on datasets $<$ 15 sentences. Forces all sentences into buckets to prevent math crashes on tiny text samples. 

      *(Defaults: n_clusters = Target Themes, random_state=42)*

    """)

with st.expander("4️⃣ Vocabulary Layer (Vectorization)", expanded=True):
    ngram_range = st.slider("N-Gram Range", 1, 3, (1, 2), help="1=Unigrams, 2=Bigrams (e.g., 'machine learning')")
    # Added the explanation here!
    auto_noise = st.checkbox(
        "Auto-Remove Redundant Noise (max_df)", 
        value=True,
        help="Mathematically deletes words that appear in more than 85% of the documents."
    )
    st.caption("Deletes overly common words (like 'paper' or 'study') that appear everywhere, preventing generic filler from dominating your themes.")

with st.expander("5️⃣ Extraction Layer (Representation)", expanded=True):
    ui_extraction = st.selectbox("Strategy", ["c-TF-IDF (Word frequency)", "KeyBERTInspired (Semantic cosine)", "MMR (Reduce redundancy)"])
    if "MMR" in ui_extraction:
        mmr_diversity = st.slider("MMR Diversity Penalty", 0.0, 1.0, 0.3)
    else:
        mmr_diversity = None

# --- EVALUATION METRICS ---
st.header("📊 Evaluation Metrics")
eval_metrics = st.multiselect(
    "Select KPIs to generate a final report card:",
    ["Topic Diversity", "NPMI Coherence", "Silhouette Score"],
    default=["Topic Diversity", "NPMI Coherence", "Silhouette Score"]
)

st.divider()

# ==========================================
# 3. ENGINE EXECUTION
# ==========================================
if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_width=True):
    
    if not raw_data or len(raw_data) < 20:
        st.error("Please provide more text data!")
        st.stop()

    # --- MATH EXECUTION (Inside Spinner) ---
    with st.spinner("Processing Semantic Pipeline... (Models are cached to prevent crashes)"):
        
        sentences = [s.strip() for s in raw_data.split('.') if len(s.strip()) > 10]
        dataset_size = len(sentences)
        
        academic_noise = ['students', 'course', 'research', 'paper', 'found', 'likely', 'did']
        from sklearn.feature_extraction import text
        stop_w = list(text.ENGLISH_STOP_WORDS.union(academic_noise))
        
        vectorizer_model = CountVectorizer(stop_words=stop_w, ngram_range=ngram_range, max_df=0.85 if auto_noise and dataset_size > 10 else 1.0)
        
        custom_embedder = load_embedder(EMBEDDING_MAP[ui_embedding], POOLING_MAP[ui_pooling])
        embeddings = custom_embedder.encode(sentences)
        
        # Fallback Logic (Step 3 representation in code)
        is_fallback = False
        if dataset_size < 15 or "PCA" in ui_algo:
            safe_n_themes = min(n_themes, dataset_size)
            dim_model = PCA(n_components=2, random_state=42)
            cluster_model = KMeans(n_clusters=safe_n_themes, random_state=42)
            reduce_topics = None
            is_fallback = True
            algo_used = "PCA"
            cluster_algo = "K-Means"
        else:
            dim_model = UMAP(n_neighbors=15, n_components=5, metric=ui_metric, random_state=42)
            clustering_model = HDBSCAN(min_cluster_size=10, metric='euclidean', cluster_selection_method='eom')
            reduce_topics = n_themes
            algo_used = "UMAP"
            cluster_algo = "HDBSCAN"

        # Representation
        if "MMR" in ui_extraction:
            rep_model = MaximalMarginalRelevance(diversity=mmr_diversity, top_n_words=words_per_theme)
        elif "KeyBERT" in ui_extraction:
            rep_model = KeyBERTInspired(top_n_words=words_per_theme)
        else:
            rep_model = None

        topic_model = BERTopic(
            embedding_model=custom_embedder,
            umap_model=dim_model,
            hdbscan_model=cluster_model,
            vectorizer_model=vectorizer_model,
            representation_model=rep_model,
            nr_topics=reduce_topics,
            top_n_words=words_per_theme,
            language="english"
        )
        topics, _ = topic_model.fit_transform(sentences)

    # ==========================================
    # 4. UI DISPLAY & METRICS (Outside Spinner)
    # ==========================================
    st.success("Analysis Complete!")
    
    if is_fallback:
        if safe_n_themes < n_themes:
            st.warning(f"⚠️ **Reduced requested themes from {n_themes} to {safe_n_themes}.**\n\n"
                       f"*The Math Explanation:* BERTopic clusters complete sentences to preserve context. "
                       f"You cannot sort {dataset_size} sentences into {n_themes} buckets without leaving empty buckets, "
                       f"which mathematically breaks the clustering algorithms!")
        else:
            st.info(f"ℹ️ Auto-switched to PCA/K-Means due to small dataset size ({dataset_size} sentences).")
            
    st.markdown("### 🏆 Discovered Themes")
    topic_info = topic_model.get_topic_info()
    all_words = []
    
    cols = st.columns(3)
    col_idx = 0
    for t_id in topic_info['Topic']:
        if t_id == -1: continue
        theme_w = [w[0] for w in topic_model.get_topic(t_id)]
        all_words.append(theme_w)
        with cols[col_idx % 3]:
            st.info(f"**Theme {t_id + 1}**\n\n" + "\n".join([f"🔹 {w}" for w in theme_w]))
        col_idx += 1

    # --- METRICS CALCULATIONS ---
    div_val, npmi_val, sil_val = 0.0, 0.0, 0.0
    
    if len(eval_metrics) > 0:
        st.markdown("### 📊 Key Performance Indicators (KPI)")
        
        with st.spinner("Calculating mathematical metrics... (NPMI requires building a dictionary and takes a moment)"):
            
            for metric in eval_metrics:
                if "Diversity" in metric:
                    if len(all_words) > 0:
                        u_words = set([w for t in all_words for w in t])
                        t_words = sum([len(t) for t in all_words])
                        div_val = len(u_words) / t_words if t_words > 0 else 0
                        st.metric("Topic Diversity (Target: 1.0)", f"{div_val:.2f}")
                    else:
                        st.metric("Topic Diversity", "Skipped")
                        
                elif "NPMI" in metric:
                    try:
                        tokenized = [vectorizer_model.build_analyzer()(s) for s in sentences]
                        dictionary = corpora.Dictionary(tokenized)
                        cm = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='c_npmi')
                        temp_npmi = cm.get_coherence()
                        if np.isnan(temp_npmi):
                            st.metric("NPMI Coherence", "N/A (Too few words)")
                        else:
                            npmi_val = float(temp_npmi)
                            st.metric("NPMI Coherence (Target: >0.1)", f"{npmi_val:.2f}")
                    except Exception:
                        st.metric("NPMI Coherence", "Skipped (Data too small)")
                        
                elif "Silhouette" in metric:
                    valid_idx = [i for i, t in enumerate(topics) if t != -1]
                    unique_topics = set([topics[i] for i in valid_idx])
                    
                    if 1 < len(unique_topics) < len(valid_idx):
                        sil_val = float(silhouette_score(
                            np.array([embeddings[i] for i in valid_idx]), 
                            [topics[i] for i in valid_idx], 
                            metric='cosine'
                        ))
                        st.metric("Silhouette Score (Target: >0.0)", f"{sil_val:.2f}")
                    else:
                        st.metric("Silhouette Score", "Skipped (Themes need ≥2 sentences each)")

    # ==========================================
    # 5. XAI VISUALIZATION GRAPH
    # ==========================================
    st.markdown("### 📈 Explainable AI (XAI) Architecture Map")
    
    with st.spinner("Rendering Explainable AI Dashboard..."):
        sns.set_theme(style="whitegrid")
        fig = plt.figure(figsize=(16, 14)) 
        fig.suptitle(f"Topic Modeling Pipeline Analytics\n(Pooling: {ui_pooling.split()[0]} | Rep: {ui_extraction.split()[0]})", fontsize=20, fontweight='bold', y=0.98)
        box_style = dict(boxstyle="round,pad=0.4", facecolor='lightyellow', edgecolor='orange', alpha=0.9)

        # 1. Embeddings
        ax1 = plt.subplot(3, 2, 1)
        sns.heatmap(embeddings[:, :50], cmap="viridis", cbar=False, ax=ax1)
        ax1.set_title("STEP 1: Embeddings & Pooling", fontsize=13, fontweight='bold')
        ax1.set_ylabel("Sentences")
        ax1.set_xlabel("Vector Dimensions (First 50 shown)")
        ax1.text(0.5, -0.25, f"Math: {ui_embedding.split()[0]} encodes text into 384D.\nPooling '{ui_pooling.split()[0]}' squashes word vectors into 1 sentence vector.", 
                 fontsize=10, ha='center', va='top', transform=ax1.transAxes, bbox=box_style)

        # 2. Geometry
        ax2 = plt.subplot(3, 2, 2)
        reduced_embeddings = topic_model.umap_model.transform(embeddings)
        ax2.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c='grey', s=100, alpha=0.6, edgecolor='k')
        ax2.set_title(f"STEP 2: Geometry ({algo_used})", fontsize=13, fontweight='bold')
        ax2.text(0.5, -0.25, f"Math: {algo_used} reduces 384D vectors into a 2D map.\nPlaces similar sentences close together.", 
                 fontsize=10, ha='center', va='top', transform=ax2.transAxes, bbox=box_style)

        # 3. Clustering
        ax3 = plt.subplot(3, 2, 3)
        ax3.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c=topics, cmap='tab10', s=150, edgecolor='k')
        ax3.set_title(f"STEP 3: Clustering ({cluster_algo})", fontsize=13, fontweight='bold')
        ax3.text(0.5, -0.25, f"Math: {cluster_algo} scans the 2D space to draw boundaries.\nColors represent assigned semantic clusters.", 
                 fontsize=10, ha='center', va='top', transform=ax3.transAxes, bbox=box_style)

        # 4. Representation
        ax4 = plt.subplot(3, 2, 4)
        theme_1_data = topic_model.get_topic(0)
        if theme_1_data:
            words = [x[0] for x in theme_1_data][::-1] 
            scores = [x[1] for x in theme_1_data][::-1]
            ax4.barh(words, scores, color='coral', edgecolor='black')
            ax4.set_title(f"STEP 4: Topic Representation ({ui_extraction.split()[0]})", fontsize=13, fontweight='bold')
            ax4.text(0.5, -0.25, f"Math: Applies {ui_extraction.split()[0]} to rank vocabulary.\nLonger bars = higher semantic relevance.", 
                     fontsize=10, ha='center', va='top', transform=ax4.transAxes, bbox=box_style)
        else:
            ax4.text(0.5, 0.5, "Theme not found", ha='center', transform=ax4.transAxes)

        # 5. KPI Dashboard
        ax5 = plt.subplot(3, 2, 5)
        ax5.axis('off') 
        ax5.set_title("STEP 5: Key Performance Indicators (KPI)", fontsize=13, fontweight='bold', y=0.9)
        
        div_str = f"{div_val:.2f}" if div_val > 0 else "Skipped"
        npmi_str = f"{npmi_val:.2f}" if npmi_val != 0.0 else "Skipped"
        sil_str = f"{sil_val:.2f}" if sil_val != 0.0 else "Skipped"

        kpi_text = (
            f"📊 Topic Diversity:  {div_str} (Target: 1.0)\n\n"
            f"🧠 NPMI Coherence:  {npmi_str} (Target: >0.1)\n\n"
            f"📐 Silhouette Score:  {sil_str} (Target: >0.0)"
        )
        ax5.text(0.5, 0.4, kpi_text, fontsize=12, va='center', ha='center', 
                 bbox=dict(boxstyle="square,pad=1.5", facecolor='#e6f2ff', edgecolor='#377eb8', lw=2))
        ax5.text(0.5, -0.15, "Math: Since pipeline algorithms don't use 'Training Loss',\nthese KPIs provide the absolute mathematical grade of the topics.", 
                 fontsize=10, ha='center', va='top', transform=ax5.transAxes, bbox=box_style)

        # 6. Summary Panel
        ax6 = plt.subplot(3, 2, 6)
        ax6.axis('off') 
        summary_text = (
            "=== PIPELINE ARCHITECTURE ===\n\n"
            f"1. Embeddings: {ui_embedding.split()[0]}\n"
            f"2. Pooling: {ui_pooling.split()[0]}\n"
            f"3. N-Grams: {ngram_range}\n"
            f"4. Geometry: {algo_used}\n"
            f"5. Clustering: {cluster_algo}\n"
            f"6. Representation: {ui_extraction.split()[0]}\n\n"
            "This modular pipeline successfully transforms unstructured text\n"
            "into mathematically validated semantic domains."
        )
        ax6.text(0.1, 0.5, summary_text, fontsize=12, va='center', ha='left', 
                 bbox=dict(boxstyle="square,pad=1", facecolor='#f0f0f0', edgecolor='grey', lw=2))

        plt.subplots_adjust(hspace=0.6, wspace=0.3)
        st.pyplot(fig)