Update app.py

app.py

@@ -224,9 +224,10 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
             st.info(f"**Theme {t_id + 1}**\n\n" + "\n".join([f"🔹 {w}" for w in theme_w]))
         col_idx += 1
 
-    # --- METRICS CALCULATIONS ---
+# --- METRICS CALCULATIONS ---
     div_val, npmi_val, umass_val, sil_val = 0.0, 0.0, 0.0, 0.0
     div_status, npmi_status, umass_status, sil_status = "Skipped", "Skipped", "Skipped", "Skipped"
+    u_words_len, t_words_len = 0, 0
     
     if len(eval_metrics) > 0:
         with st.spinner("Calculating mathematical metrics... (NPMI requires building a dictionary and takes a moment)"):
@@ -235,10 +236,11 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
             if "Topic Diversity" in eval_metrics and len(all_words) > 0:
                 u_words = set([w for t in all_words for w in t])
                 t_words = sum([len(t) for t in all_words])
+                u_words_len, t_words_len = len(u_words), t_words
                 div_val = float(len(u_words) / t_words) if t_words > 0 else 0.0
                 div_status = f"{div_val:.2f}"
             
-            # Coherence Models (NPMI and UMass)
+            # Coherence Models
             if "NPMI Coherence" in eval_metrics or "UMass Coherence" in eval_metrics:
                 try:
                     tokenized = [vectorizer_model.build_analyzer()(s) for s in sentences]
@@ -247,21 +249,28 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
                     if "NPMI Coherence" in eval_metrics:
                         cm_npmi = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='c_npmi')
                         temp_npmi = cm_npmi.get_coherence()
-                        npmi_status = "N/A (Too few words)" if np.isnan(temp_npmi) else f"{float(temp_npmi):.2f}"
+                        if np.isnan(temp_npmi):
+                            npmi_status = "N/A"
+                        else:
+                            npmi_val = float(temp_npmi)
+                            npmi_status = f"{npmi_val:.2f}"
                     
                     if "UMass Coherence" in eval_metrics:
                         cm_umass = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='u_mass')
                         temp_umass = cm_umass.get_coherence()
-                        umass_status = "N/A (Too few words)" if np.isnan(temp_umass) else f"{float(temp_umass):.2f}"
+                        if np.isnan(temp_umass):
+                            umass_status = "N/A"
+                        else:
+                            umass_val = float(temp_umass)
+                            umass_status = f"{umass_val:.2f}"
                 except Exception:
-                    npmi_status = "Skipped (Data too small)"
-                    umass_status = "Skipped (Data too small)"
+                    npmi_status = "Skipped"
+                    umass_status = "Skipped"
 
             # Silhouette
             if "Silhouette Score" in eval_metrics:
                 valid_idx = [i for i, t in enumerate(topics) if t != -1]
                 unique_topics = set([topics[i] for i in valid_idx])
-                
                 if 1 < len(unique_topics) < len(valid_idx):
                     sil_val = float(silhouette_score(
                         np.array([embeddings[i] for i in valid_idx]), 
@@ -270,59 +279,98 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
                     ))
                     sil_status = f"{sil_val:.2f}"
                 else:
-                    sil_status = "Skipped (Themes need ≥2 sentences)"
+                    sil_status = "Skipped"
 
-        # --- RENDER KPI DASHBOARD (UI draws instantly here!) ---
+        # --- RENDER KPI DASHBOARD WITH TOOLTIPS ---
         st.markdown("### 📊 Key Performance Indicators (KPI)")
         kpi_cols = st.columns(len(eval_metrics))
         
         for idx, metric in enumerate(eval_metrics):
             with kpi_cols[idx]:
                 if metric == "Topic Diversity":
-                    st.metric("Topic Diversity (Target: 1.0)", div_status)
+                    st.metric("Topic Diversity", div_status, help="Math: Unique Words / Total Words.\nTarget: 1.0 (No redundant words across themes).")
                 elif metric == "NPMI Coherence":
-                    st.metric("NPMI Coherence (Target: >0.1)", npmi_status)
+                    st.metric("NPMI Coherence", npmi_status, help="Math: Normalized Pointwise Mutual Information.\nCalculates joint probability of words existing together.\nTarget: >0.1")
                 elif metric == "UMass Coherence":
-                    st.metric("UMass Coherence (Target: closer to 0)", umass_status)
+                    st.metric("UMass Coherence", umass_status, help="Math: Internal log-conditional probability.\nEvaluates if words co-occur strictly inside your uploaded dataset.\nTarget: Closer to 0.")
                 elif metric == "Silhouette Score":
-                    st.metric("Silhouette Score (Target: >0.0)", sil_status)
+                    st.metric("Silhouette Score", sil_status, help="Math: (b - a) / max(a,b).\nMeasures intra-cluster density (a) vs nearest-cluster distance (b).\nTarget: >0.0")
 
-    # ==========================================
-    # 5. XAI VISUALIZATION GRAPH
+                    
+# ==========================================
+    # 5. XAI VISUALIZATION GRAPH (With Live Math & Matrices)
     # ==========================================
     st.markdown("### 📈 Explainable AI (XAI) Architecture Map")
     
-    with st.spinner("Rendering Explainable AI Dashboard..."):
+    with st.spinner("Rendering Mathematical Dashboard..."):
         sns.set_theme(style="whitegrid")
-        fig = plt.figure(figsize=(16, 14)) 
-        fig.suptitle(f"Topic Modeling Pipeline Analytics\n(Pooling: {ui_pooling.split()[0]} | Rep: {ui_extraction.split()[0]})", fontsize=20, fontweight='bold', y=0.98)
-        box_style = dict(boxstyle="round,pad=0.4", facecolor='lightyellow', edgecolor='orange', alpha=0.9)
+        fig = plt.figure(figsize=(18, 16)) 
+        fig.suptitle(f"Topic Modeling Mathematical Pipeline\n(Pooling: {ui_pooling.split()[0]} | Rep: {ui_extraction.split()[0]})", fontsize=20, fontweight='bold', y=0.98)
+        
+        # Style for the Math/Data boxes
+        box_style = dict(boxstyle="round,pad=0.5", facecolor='#f8f9fa', edgecolor='#4b72b8', alpha=0.95, lw=2)
 
-        # 1. Embeddings
+        # --------------------------------------------------
+        # 1. Embeddings & Pooling
+        # --------------------------------------------------
         ax1 = plt.subplot(3, 2, 1)
         sns.heatmap(embeddings[:, :50], cmap="viridis", cbar=False, ax=ax1)
         ax1.set_title("STEP 1: Embeddings & Pooling", fontsize=13, fontweight='bold')
-        ax1.set_ylabel("Sentences")
+        ax1.set_ylabel("Sentences (Docs)")
         ax1.set_xlabel("Vector Dimensions (First 50 shown)")
-        ax1.text(0.5, -0.25, f"Math: {ui_embedding.split('/')[-1]} encodes text into 384D.\nPooling '{ui_pooling.split()[0]}' squashes word vectors into 1 sentence vector.", 
-                 fontsize=10, ha='center', va='top', transform=ax1.transAxes, bbox=box_style)
+        
+        # Live Data Extraction
+        emb_shape = embeddings.shape
+        emb_sample = np.round(embeddings[0, :5], 3).tolist() # First 5 numbers of Doc 1
+        
+        math_text_1 = (
+            r"$\mathbf{Math (Mean Pool):} \quad v = \frac{1}{N} \sum_{i=1}^{N} \text{BERT}(w_i)$" + "\n"
+            f"Matrix Shape: {emb_shape} (Docs x Dims)\n"
+            f"Doc 1 [Dims 1-5]: {emb_sample}..."
+        )
+        ax1.text(0.5, -0.25, math_text_1, fontsize=11, ha='center', va='top', transform=ax1.transAxes, bbox=box_style)
 
-        # 2. Geometry
+        # --------------------------------------------------
+        # 2. Geometry (Dimensionality Reduction)
+        # --------------------------------------------------
         ax2 = plt.subplot(3, 2, 2)
         reduced_embeddings = topic_model.umap_model.transform(embeddings)
         ax2.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c='grey', s=100, alpha=0.6, edgecolor='k')
         ax2.set_title(f"STEP 2: Geometry ({algo_used})", fontsize=13, fontweight='bold')
-        ax2.text(0.5, -0.25, f"Math: {algo_used} reduces 384D vectors into a 2D map.\nPlaces similar sentences close together.", 
-                 fontsize=10, ha='center', va='top', transform=ax2.transAxes, bbox=box_style)
+        
+        # Live Data Extraction
+        red_shape = reduced_embeddings.shape
+        red_sample = np.round(reduced_embeddings[0, :2], 3).tolist() # X, Y coord of Doc 1
+        
+        eq_2 = r"$\mathbf{Math (PCA):} \quad Z = X \cdot W_{2D}$" if algo_used == "PCA" else r"$\mathbf{Math (UMAP):} \quad \text{Topological Manifold Approx.}$"
+        math_text_2 = (
+            f"{eq_2}\n"
+            f"Matrix Shape: {red_shape} (Docs x 2D Coordinates)\n"
+            f"Doc 1 Coordinate: [X: {red_sample[0]}, Y: {red_sample[1]}]"
+        )
+        ax2.text(0.5, -0.25, math_text_2, fontsize=11, ha='center', va='top', transform=ax2.transAxes, bbox=box_style)
 
+        # --------------------------------------------------
         # 3. Clustering
+        # --------------------------------------------------
         ax3 = plt.subplot(3, 2, 3)
         ax3.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c=topics, cmap='tab10', s=150, edgecolor='k')
         ax3.set_title(f"STEP 3: Clustering ({cluster_algo})", fontsize=13, fontweight='bold')
-        ax3.text(0.5, -0.25, f"Math: {cluster_algo} scans the 2D space to draw boundaries.\nColors represent assigned semantic clusters.", 
-                 fontsize=10, ha='center', va='top', transform=ax3.transAxes, bbox=box_style)
+        
+        # Live Data Extraction
+        topic_sample = topics[:5] # First 5 cluster assignments
+        
+        eq_3 = r"$\mathbf{Math (K-Means):} \quad \arg\min_S \sum ||x_i - \mu_c||^2$" if cluster_algo == "K-Means" else r"$\mathbf{Math (HDBSCAN):} \quad \text{Density} = \frac{1}{\text{core\_dist}(x)}$"
+        math_text_3 = (
+            f"{eq_3}\n"
+            f"Output Array Shape: ({len(topics)},) (1 Label per Doc)\n"
+            f"Doc 1-5 Assignments: {topic_sample}..."
+        )
+        ax3.text(0.5, -0.25, math_text_3, fontsize=11, ha='center', va='top', transform=ax3.transAxes, bbox=box_style)
 
+        # --------------------------------------------------
         # 4. Representation
+        # --------------------------------------------------
         ax4 = plt.subplot(3, 2, 4)
         theme_1_data = topic_model.get_topic(0)
         if theme_1_data:
@@ -330,43 +378,62 @@ if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_w
             scores = [x[1] for x in theme_1_data][::-1]
             ax4.barh(words, scores, color='coral', edgecolor='black')
             ax4.set_title(f"STEP 4: Topic Representation ({ui_extraction.split()[0]})", fontsize=13, fontweight='bold')
-            ax4.text(0.5, -0.25, f"Math: Applies {ui_extraction.split()[0]} to rank vocabulary.\nLonger bars = higher semantic relevance.", 
-                     fontsize=10, ha='center', va='top', transform=ax4.transAxes, bbox=box_style)
+            
+            # Live Data Extraction
+            top_word_score = round(scores[-1], 4)
+            vocab_len = len(vectorizer_model.vocabulary_) if hasattr(vectorizer_model, 'vocabulary_') else 'N/A'
+            
+            math_text_4 = (
+                r"$\mathbf{Math (c-TF-IDF):} \quad W_{t,c} = tf_{t,c} \times \log\left(1 + \frac{A}{df_t}\right)$" + "\n"
+                f"Global Vocab Extracted: {vocab_len} terms\n"
+                f"Top Word ('{words[-1]}') Score: {top_word_score}"
+            )
+            ax4.text(0.5, -0.25, math_text_4, fontsize=11, ha='center', va='top', transform=ax4.transAxes, bbox=box_style)
         else:
             ax4.text(0.5, 0.5, "Theme not found", ha='center', transform=ax4.transAxes)
 
-        # 5. KPI Dashboard (Updated with UMass)
+# --------------------------------------------------
+        # 5. KPI Dashboard (Formulas & Live Calculations)
+        # --------------------------------------------------
         ax5 = plt.subplot(3, 2, 5)
         ax5.axis('off') 
-        ax5.set_title("STEP 5: Key Performance Indicators (KPI)", fontsize=13, fontweight='bold', y=0.9)
+        ax5.set_title("STEP 5: Post-Hoc Evaluation (Math & Live Data)", fontsize=13, fontweight='bold', y=0.9)
         
-        kpi_text = (
-            f"📊 Topic Diversity:  {div_status} (Target: 1.0)\n\n"
-            f"🧠 NPMI Coherence:  {npmi_status} (Target: >0.1)\n\n"
-            f"📚 UMass Coherence:  {umass_status} (Target: Closer to 0)\n\n"
-            f"📐 Silhouette Score:  {sil_status} (Target: >0.0)"
+        # Get the actual distance metric used for the label
+        dist_used = ui_metric if algo_used == "UMAP" else "euclidean"
+        
+        kpi_math = (
+            r"$\mathbf{Topic \ Diversity:} \quad D = \frac{| \text{Unique Words} |}{| \text{Total Words} |}$" + "\n"
+            f"Live Calc: {u_words_len} Unique / {t_words_len} Total = {div_status}\n\n"
+            
+            r"$\mathbf{Silhouette \ Score:} \quad S = \frac{b - a}{\max(a, b)}$" + "\n"
+            f"Live Calc: Score = {sil_status} (Using {dist_used} distance)\n\n"
+            
+            r"$\mathbf{NPMI \ Coherence:} \quad \text{NPMI}(x, y) = \frac{\log \frac{P(x,y)}{P(x)P(y)}}{-\log P(x,y)}$" + "\n"
+            f"Live Calc: Score = {npmi_status} across {len(all_words)} valid themes"
         )
-        ax5.text(0.5, 0.4, kpi_text, fontsize=11, va='center', ha='center', 
+        
+        ax5.text(0.5, 0.4, kpi_math, fontsize=11, va='center', ha='center', 
                  bbox=dict(boxstyle="square,pad=1.5", facecolor='#e6f2ff', edgecolor='#377eb8', lw=2))
-        ax5.text(0.5, -0.15, "Math: Since pipeline algorithms don't use 'Training Loss',\nthese KPIs provide the absolute mathematical grade of the topics.", 
-                 fontsize=10, ha='center', va='top', transform=ax5.transAxes, bbox=box_style)
-
-        # 6. Summary Panel
+        
+        # --------------------------------------------------
+        # 6. Summary Matrix Transformations
+        # --------------------------------------------------
         ax6 = plt.subplot(3, 2, 6)
         ax6.axis('off') 
         summary_text = (
-            "=== PIPELINE ARCHITECTURE ===\n\n"
-            f"1. Embeddings: {ui_embedding.split()[0]}\n"
-            f"2. Pooling: {ui_pooling.split()[0]}\n"
-            f"3. N-Grams: {ngram_range}\n"
-            f"4. Geometry: {algo_used}\n"
-            f"5. Clustering: {cluster_algo}\n"
-            f"6. Representation: {ui_extraction.split()[0]}\n\n"
-            "This modular pipeline successfully transforms unstructured text\n"
-            "into mathematically validated semantic domains."
+            "=== THE MATRIX TRANSFORMATION LIFECYCLE ===\n\n"
+            f"1. Raw Text $\\rightarrow$ {emb_shape} Matrix (Dense Meaning)\n"
+            f"2. {emb_shape} $\\rightarrow$ {red_shape} Matrix (Geometric Compression)\n"
+            f"3. {red_shape} $\\rightarrow$ ({len(topics)},) Array (Discrete Bucketing)\n"
+            f"4. ({len(topics)},) $\\rightarrow$ c-TF-IDF Matrix (Word Extraction)\n"
+            f"5. c-TF-IDF $\\rightarrow$ {CONFIG['words_per_theme']} Output Words (Per Theme)\n\n"
+            "This proves Topic Modeling is a sequence of \ndimensionality reductions and matrix multiplications."
         )
         ax6.text(0.1, 0.5, summary_text, fontsize=12, va='center', ha='left', 
                  bbox=dict(boxstyle="square,pad=1", facecolor='#f0f0f0', edgecolor='grey', lw=2))
 
-        plt.subplots_adjust(hspace=0.6, wspace=0.3)
-        st.pyplot(fig, use_container_width=True)
+        plt.subplots_adjust(hspace=0.7, wspace=0.3)
+        st.pyplot(fig, use_container_width=True)
+
+