Update app.py

app.py

@@ -1,5 +1,4 @@
 import os
-# --- ANTI-CRASH ENVIRONMENT VARIABLES ---
 os.environ["OMP_NUM_THREADS"] = "1"
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
@@ -27,17 +26,7 @@ warnings.filterwarnings("ignore")
 # ==========================================
 # 1. PAGE CONFIGURATION & MAPPINGS
 # ==========================================
-st.set_page_config(page_title="Topic Modeling Pipeline", layout="wide", initial_sidebar_state="expanded")
-
-# Add this right below set_page_config to stop the screen from jumping left/right
-st.markdown("""
-    <style>
-        /* Force the vertical scrollbar to always show so the page width never changes */
-        html { overflow-y: scroll; }
-        /* Prevent horizontal scrolling */
-        .block-container { max-width: 100%; overflow-x: hidden; }
-    </style>
-    """, unsafe_allow_html=True)
+st.set_page_config(page_title="Topic Modeling Pipeline", layout="wide", initial_sidebar_state="collapsed")
 
 EMBEDDING_MAP = {
     "MiniLM (Fast & Lightweight)": "sentence-transformers/all-MiniLM-L6-v2",
@@ -52,7 +41,6 @@ POOLING_MAP = {
     "Mean-Max (Combined)": "mean-max"
 }
 
-# --- CACHE THE NEURAL NETWORK ---
 @st.cache_resource
 def load_embedder(model_name, pool_strat):
     word_emb = models.Transformer(model_name)
@@ -62,18 +50,12 @@ def load_embedder(model_name, pool_strat):
         pooling_mode_max_tokens=("max" in pool_strat),
         pooling_mode_cls_token=("cls" in pool_strat)
     )
-
-    modules = [word_emb, pool_model]
-
-    # NEW MATH STEP: If user checked the box, append the Normalization layer!
-    if apply_normalization:
-        modules.append(models.Normalize())
     return SentenceTransformer(modules=[word_emb, pool_model])
 
 # ==========================================
-# 2. THE GUIDED UI (MAIN PAGE)
+# 2. THE GUIDED UI
 # ==========================================
-st.title("🧠 BERTopic - Topic Modeling Pipeline Math Visualization")
+st.title("🧠 Advanced Topic Modeling Pipeline")
 
 try:
     st.image("pipeline.png", use_container_width=True)
@@ -82,8 +64,9 @@ except FileNotFoundError:
 
 st.divider()
 
-# --- STEP 0: DATA SETTINGS ---
 st.header("📥 Step 0: Input Data & Core Settings")
+st.info("💡 **Preprocessing Note:** You do not need to manually lowercase or strip punctuation. The `CountVectorizer` algorithm and the `Uncased` BERT Neural Networks handle casing and token normalization autonomously at the mathematical level.")
+
 data_source = st.radio("Choose Data Source:", ["Use Sample ACM Abstract", "Paste Text"], horizontal=True)
 
 sample_abstract = """
@@ -105,60 +88,43 @@ with col_a:
 with col_b:
     words_per_theme = st.slider("Words to Output per Theme", 3, 10, 5)
 
-# --- THE VERTICAL CONFIGURATION WIZARD ---
 st.header("⚙️ Model Configuration")
 
-with st.expander("1️⃣ Semantic Layer (Embeddings & Pooling)", expanded=True):
-    ui_embedding = st.selectbox("Embedding Model", list(EMBEDDING_MAP.keys()))
-    ui_pooling = st.selectbox("Pooling Strategy", list(POOLING_MAP.keys()))
-    ui_normalize = st.checkbox("Apply L2 Normalization (Recommended)", value=True, help="Forces all vectors to have a length of 1.0 so length doesn't bias the clustering.")
-
-with st.expander("2️⃣ Geometry Layer (Dimensionality Reduction)", expanded=True):
-    ui_algo = st.selectbox("Algorithm", ["UMAP (Complex geometry)", "PCA (Fast/Deterministic)"])
-    if "UMAP" in ui_algo:
-        ui_metric = st.selectbox("Distance Metric", ["cosine", "euclidean", "manhattan"])
-    else:
-        ui_metric = "euclidean"
-        st.info("PCA inherently uses Variance (Euclidean math), so distance metrics are bypassed.")
-
-with st.expander("3️⃣ Clustering Layer (Grouping)", expanded=True):
-    st.markdown("""
-    *Clustering mathematically draws boundaries around similar sentences.*
-    * **Primary Engine (HDBSCAN):** Runs on datasets $\ge$ 15 sentences. Automatically filters outliers and finds dense semantic clouds. 
-      *(Defaults: min_cluster_size=10, cluster_selection_method='eom', metric='euclidean')*
-    * **Fallback Engine (K-Means):** Runs on datasets $<$ 15 sentences. Forces all sentences into buckets to prevent math crashes on tiny text samples. 
-      *(Defaults: n_clusters = Target Themes, random_state=42)*
-    """)
-
-with st.expander("4️⃣ Vocabulary Layer (Vectorization)", expanded=True):
-    ngram_range = st.slider("N-Gram Range", 1, 3, (1, 2), help="1=Unigrams, 2=Bigrams (e.g., 'machine learning')")
-    # Added the explanation here!
-    auto_noise = st.checkbox(
-        "Auto-Remove Redundant Noise (max_df)", 
-        value=True,
-        help="Mathematically deletes words that appear in more than 85% of the documents."
-    )
-    st.caption("Deletes overly common words (like 'paper' or 'study') that appear everywhere, preventing generic filler from dominating your themes.")
-
-with st.expander("4️⃣ Extraction Layer (Representation)", expanded=True):
-    st.markdown("*BERTopic ALWAYS extracts baseline words using **c-TF-IDF** (Word Frequency). Do you want to apply an advanced mathematical filter on top of it?*")
-    
-    ui_extraction = st.selectbox(
-        "Apply Advanced Filter:", 
-        ["None (Base c-TF-IDF only)", "KeyBERTInspired (Semantic alignment)", "MMR (Reduce redundancy)"]
-    )
-    
-    if "MMR" in ui_extraction:
-        mmr_diversity = st.slider("MMR Diversity Penalty", 0.0, 1.0, 0.3)
-    else:
-        mmr_diversity = None
+col1, col2 = st.columns(2)
+with col1:
+    with st.expander("1️⃣ Semantic Layer (Embeddings)", expanded=True):
+        ui_embedding = st.selectbox("Embedding Model", list(EMBEDDING_MAP.keys()))
+        ui_pooling = st.selectbox("Pooling Strategy", list(POOLING_MAP.keys()))
+
+with col2:
+    with st.expander("2️⃣ Geometry Layer (Dimensionality Reduction)", expanded=True):
+        ui_algo = st.selectbox("Algorithm", ["UMAP (Complex geometry)", "PCA (Fast/Deterministic)"])
+        ui_metric = st.selectbox("Distance Metric", ["cosine", "euclidean", "manhattan"]) if "UMAP" in ui_algo else "euclidean"
+
+col3, col4 = st.columns(2)
+with col3:
+    with st.expander("3️⃣ Clustering Layer (Grouping)", expanded=True):
+        st.markdown("""
+        *Clustering mathematically draws boundaries around similar sentences.*
+        * **Primary (HDBSCAN):** $\ge$ 15 sentences. Filters outliers. *(Defaults: min_cluster_size=10)*
+        * **Fallback (K-Means):** $<$ 15 sentences. Forces buckets to prevent math crashes on tiny text.
+        """)
+
+with col4:
+    with st.expander("4️⃣ Vocabulary Layer (Vectorization)", expanded=True):
+        ngram_range = st.slider("N-Gram Range", 1, 3, (1, 2))
+        auto_noise = st.checkbox("Auto-Remove Redundant Noise", value=True, help="Mathematically deletes words appearing in >85% of documents.")
+
+with st.expander("5️⃣ Extraction Layer (Representation)", expanded=True):
+    st.markdown("*BERTopic ALWAYS extracts baseline words using **c-TF-IDF**. Do you want to apply an advanced mathematical filter on top of it?*")
+    ui_extraction = st.selectbox("Apply Advanced Filter:", ["None (Base c-TF-IDF only)", "KeyBERTInspired (Semantic alignment)", "MMR (Reduce redundancy)"])
+    mmr_diversity = st.slider("MMR Diversity Penalty", 0.0, 1.0, 0.3) if "MMR" in ui_extraction else None
 
-# --- EVALUATION METRICS ---
 st.header("📊 Evaluation Metrics")
 eval_metrics = st.multiselect(
     "Select KPIs to generate a final report card:",
-    ["Topic Diversity", "NPMI Coherence", "Silhouette Score"],
-    default=["Topic Diversity", "NPMI Coherence", "Silhouette Score"]
+    ["Topic Diversity", "NPMI Coherence", "UMass Coherence", "Silhouette Score"],
+    default=["Topic Diversity", "NPMI Coherence", "UMass Coherence", "Silhouette Score"]
 )
 
 st.divider()
@@ -172,7 +138,6 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
         st.error("Please provide more text data!")
         st.stop()
 
-    # --- MATH EXECUTION (Inside Spinner) ---
     with st.spinner("Processing Semantic Pipeline... (Models are cached to prevent crashes)"):
         
         sentences = [s.strip() for s in raw_data.split('.') if len(s.strip()) > 10]
@@ -184,12 +149,9 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
         
         vectorizer_model = CountVectorizer(stop_words=stop_w, ngram_range=ngram_range, max_df=0.85 if auto_noise and dataset_size > 10 else 1.0)
         
-        #custom_embedder = load_embedder(EMBEDDING_MAP[ui_embedding], POOLING_MAP[ui_pooling])
-        # Ensure you pass the new variable into the function!
-        custom_embedder = load_embedder(EMBEDDING_MAP[ui_embedding], POOLING_MAP[ui_pooling], ui_normalize)
+        custom_embedder = load_embedder(EMBEDDING_MAP[ui_embedding], POOLING_MAP[ui_pooling])
         embeddings = custom_embedder.encode(sentences)
         
-        # Fallback Logic (Step 3 representation in code)
         is_fallback = False
         if dataset_size < 15 or "PCA" in ui_algo:
             safe_n_themes = min(n_themes, dataset_size)
@@ -206,7 +168,6 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
             algo_used = "UMAP"
             cluster_algo = "HDBSCAN"
 
-        # Representation
         if "MMR" in ui_extraction:
             rep_model = MaximalMarginalRelevance(diversity=mmr_diversity, top_n_words=words_per_theme)
         elif "KeyBERT" in ui_extraction:
@@ -227,7 +188,7 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
         topics, _ = topic_model.fit_transform(sentences)
 
     # ==========================================
-    # 4. UI DISPLAY & METRICS (Outside Spinner)
+    # 4. UI DISPLAY & METRICS
     # ==========================================
     st.success("Analysis Complete!")
     
@@ -254,49 +215,54 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
             st.info(f"**Theme {t_id + 1}**\n\n" + "\n".join([f"��� {w}" for w in theme_w]))
         col_idx += 1
 
-    # --- METRICS CALCULATIONS (Math First, UI Second) ---
-    div_val, npmi_val, sil_val = 0.0, 0.0, 0.0
-    div_status, npmi_status, sil_status = "Skipped", "Skipped", "Skipped"
+    # --- METRICS CALCULATIONS ---
+    div_val, npmi_val, umass_val, sil_val = 0.0, 0.0, 0.0, 0.0
+    div_status, npmi_status, umass_status, sil_status = "Skipped", "Skipped", "Skipped", "Skipped"
     
     if len(eval_metrics) > 0:
-        # 1. DO ALL THE MATH INVISIBLY FIRST
         with st.spinner("Calculating mathematical metrics... (NPMI requires building a dictionary and takes a moment)"):
             
-            if "Topic Diversity" in eval_metrics:
-                if len(all_words) > 0:
-                    u_words = set([w for t in all_words for w in t])
-                    t_words = sum([len(t) for t in all_words])
-                    div_val = float(len(u_words) / t_words) if t_words > 0 else 0.0
-                    div_status = f"{div_val:.2f}"
-
-            if "NPMI Coherence" in eval_metrics:
+            # Diversity
+            if "Topic Diversity" in eval_metrics and len(all_words) > 0:
+                u_words = set([w for t in all_words for w in t])
+                t_words = sum([len(t) for t in all_words])
+                div_val = float(len(u_words) / t_words) if t_words > 0 else 0.0
+                div_status = f"{div_val:.2f}"
+            
+            # Coherence Models (NPMI and UMass)
+            if "NPMI Coherence" in eval_metrics or "UMass Coherence" in eval_metrics:
                 try:
                     tokenized = [vectorizer_model.build_analyzer()(s) for s in sentences]
                     dictionary = corpora.Dictionary(tokenized)
-                    cm = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='c_npmi')
-                    temp_npmi = cm.get_coherence()
-                    if np.isnan(temp_npmi):
-                        npmi_status = "N/A (Too few words)"
-                    else:
-                        npmi_val = float(temp_npmi)
-                        npmi_status = f"{npmi_val:.2f}"
+                    
+                    if "NPMI Coherence" in eval_metrics:
+                        cm_npmi = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='c_npmi')
+                        temp_npmi = cm_npmi.get_coherence()
+                        npmi_status = "N/A (Too few words)" if np.isnan(temp_npmi) else f"{float(temp_npmi):.2f}"
+                    
+                    if "UMass Coherence" in eval_metrics:
+                        cm_umass = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='u_mass')
+                        temp_umass = cm_umass.get_coherence()
+                        umass_status = "N/A (Too few words)" if np.isnan(temp_umass) else f"{float(temp_umass):.2f}"
                 except Exception:
                     npmi_status = "Skipped (Data too small)"
+                    umass_status = "Skipped (Data too small)"
 
+            # Silhouette
             if "Silhouette Score" in eval_metrics:
                 valid_idx = [i for i, t in enumerate(topics) if t != -1]
                 unique_topics = set([topics[i] for i in valid_idx])
                 if 1 < len(unique_topics) < len(valid_idx):
                     sil_val = float(silhouette_score(
                         np.array([embeddings[i] for i in valid_idx]), 
-                        [topics[i] for i in valid_idx], 
-                        metric='cosine'
-                    ))
-                    sil_status = f"{sil_val:.2f}"
-                else:
-                    sil_status = "Skipped (Themes need ≥2 sentences)"
-
-        # 2. DRAW THE UI INSTANTLY (No more jumping!)
+                            [topics[i] for i in valid_idx], 
+                            metric='cosine'
+                        ))
+                        sil_status = f"{sil_val:.2f}"
+                    else:
+                        sil_status = "Skipped (Themes need ≥2 sentences)"
+
+        # --- RENDER KPI DASHBOARD (UI draws instantly here!) ---
         st.markdown("### 📊 Key Performance Indicators (KPI)")
         kpi_cols = st.columns(len(eval_metrics))
         
@@ -306,10 +272,11 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
                     st.metric("Topic Diversity (Target: 1.0)", div_status)
                 elif metric == "NPMI Coherence":
                     st.metric("NPMI Coherence (Target: >0.1)", npmi_status)
+                elif metric == "UMass Coherence":
+                    st.metric("UMass Coherence (Target: closer to 0)", umass_status)
                 elif metric == "Silhouette Score":
                     st.metric("Silhouette Score (Target: >0.0)", sil_status)
 
-                    
     # ==========================================
     # 5. XAI VISUALIZATION GRAPH
     # ==========================================
@@ -327,7 +294,7 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
         ax1.set_title("STEP 1: Embeddings & Pooling", fontsize=13, fontweight='bold')
         ax1.set_ylabel("Sentences")
         ax1.set_xlabel("Vector Dimensions (First 50 shown)")
-        ax1.text(0.5, -0.25, f"Math: {ui_embedding.split()[0]} encodes text into 384D.\nPooling '{ui_pooling.split()[0]}' squashes word vectors into 1 sentence vector.", 
+        ax1.text(0.5, -0.25, f"Math: {ui_embedding.split('/')[-1]} encodes text into 384D.\nPooling '{ui_pooling.split()[0]}' squashes word vectors into 1 sentence vector.", 
                  fontsize=10, ha='center', va='top', transform=ax1.transAxes, bbox=box_style)
 
         # 2. Geometry
@@ -358,21 +325,18 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
         else:
             ax4.text(0.5, 0.5, "Theme not found", ha='center', transform=ax4.transAxes)
 
-        # 5. KPI Dashboard
+        # 5. KPI Dashboard (Updated with UMass)
         ax5 = plt.subplot(3, 2, 5)
         ax5.axis('off') 
         ax5.set_title("STEP 5: Key Performance Indicators (KPI)", fontsize=13, fontweight='bold', y=0.9)
         
-        div_str = f"{div_val:.2f}" if div_val > 0 else "Skipped"
-        npmi_str = f"{npmi_val:.2f}" if npmi_val != 0.0 else "Skipped"
-        sil_str = f"{sil_val:.2f}" if sil_val != 0.0 else "Skipped"
-
         kpi_text = (
-            f"📊 Topic Diversity:  {div_str} (Target: 1.0)\n\n"
-            f"🧠 NPMI Coherence:  {npmi_str} (Target: >0.1)\n\n"
-            f"📐 Silhouette Score:  {sil_str} (Target: >0.0)"
+            f"📊 Topic Diversity:  {div_status} (Target: 1.0)\n\n"
+            f"🧠 NPMI Coherence:  {npmi_status} (Target: >0.1)\n\n"
+            f"📚 UMass Coherence:  {umass_status} (Target: Closer to 0)\n\n"
+            f"📐 Silhouette Score:  {sil_status} (Target: >0.0)"
         )
-        ax5.text(0.5, 0.4, kpi_text, fontsize=12, va='center', ha='center', 
+        ax5.text(0.5, 0.4, kpi_text, fontsize=11, va='center', ha='center', 
                  bbox=dict(boxstyle="square,pad=1.5", facecolor='#e6f2ff', edgecolor='#377eb8', lw=2))
         ax5.text(0.5, -0.15, "Math: Since pipeline algorithms don't use 'Training Loss',\nthese KPIs provide the absolute mathematical grade of the topics.", 
                  fontsize=10, ha='center', va='top', transform=ax5.transAxes, bbox=box_style)