| import os
|
| os.environ["OMP_NUM_THREADS"] = "1"
|
| os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
|
|
| import streamlit as st
|
| import pandas as pd
|
| import warnings
|
| import numpy as np
|
| import matplotlib.pyplot as plt
|
| import seaborn as sns
|
|
|
| from bertopic import BERTopic
|
| from bertopic.representation import MaximalMarginalRelevance, KeyBERTInspired
|
| from sentence_transformers import SentenceTransformer, models
|
| from sklearn.feature_extraction.text import CountVectorizer
|
| from sklearn.decomposition import PCA
|
| from sklearn.cluster import KMeans
|
| from sklearn.metrics import silhouette_score
|
| from umap import UMAP
|
| from hdbscan import HDBSCAN
|
| import gensim.corpora as corpora
|
| from gensim.models.coherencemodel import CoherenceModel
|
|
|
| warnings.filterwarnings("ignore")
|
|
|
|
|
|
|
|
|
| st.set_page_config(page_title="Topic Modeling Pipeline", layout="wide", initial_sidebar_state="collapsed")
|
|
|
| EMBEDDING_MAP = {
|
| "MiniLM (Fast & Lightweight)": "sentence-transformers/all-MiniLM-L6-v2",
|
| "MPNet (High Accuracy)": "sentence-transformers/all-mpnet-base-v2",
|
| "Specter2 (Scientific/Academic)": "allenai/specter2_base"
|
| }
|
|
|
| POOLING_MAP = {
|
| "Mean (Smooth context)": "mean",
|
| "Max (Sharp keywords)": "max",
|
| "CLS (Classification)": "cls",
|
| "Mean-Max (Combined)": "mean-max"
|
| }
|
|
|
|
|
| @st.cache_resource
|
| def load_embedder(model_name, pool_strat):
|
| word_emb = models.Transformer(model_name)
|
| pool_model = models.Pooling(
|
| word_emb.get_word_embedding_dimension(),
|
| pooling_mode_mean_tokens=("mean" in pool_strat),
|
| pooling_mode_max_tokens=("max" in pool_strat),
|
| pooling_mode_cls_token=("cls" in pool_strat)
|
| )
|
|
|
|
|
| return SentenceTransformer(modules=[word_emb, pool_model, models.Normalize()])
|
|
|
|
|
|
|
|
|
| st.title("🧠 Advanced Topic Modeling Pipeline")
|
|
|
| try:
|
| st.image("pipeline.png", use_container_width=True)
|
| except FileNotFoundError:
|
| pass
|
|
|
| st.divider()
|
|
|
| st.header("📥 Step 0: Input Data & Core Settings")
|
| st.info("💡 **Preprocessing Note:** You do not need to manually lowercase or strip punctuation. The `CountVectorizer` algorithm and the `Uncased` BERT Neural Networks handle casing and token normalization autonomously at the mathematical level.")
|
|
|
| data_source = st.radio("Choose Data Source:", ["Use Sample ACM Abstract", "Paste Text"], horizontal=True)
|
|
|
| sample_abstract = """
|
| Students who registered for the Mapping with Google massive open online course (MOOC)
|
| were asked several questions during the registration process to identify prior
|
| experience with eleven skills as well as their goals for registering for the course.
|
| At the end of the course, we compared students' self reports of goal achievement
|
| with behavioral click-stream analysis. In addition, we assessed how well prior
|
| skill in a subject predicts a student's course completion and found no correlation.
|
| Our research shows that students who completed course activities were more likely
|
| to earn certificates of completion than peers who did not.
|
| """
|
|
|
| raw_data = st.text_area("Text Data:", value=sample_abstract if data_source == "Use Sample ACM Abstract" else "", height=150)
|
|
|
| col_a, col_b = st.columns(2)
|
| with col_a:
|
| n_themes = st.slider("Target Number of Themes", 2, 20, 3)
|
| with col_b:
|
| words_per_theme = st.slider("Words to Output per Theme", 3, 10, 5)
|
|
|
|
|
| st.header("⚙️ Model Configuration")
|
|
|
| with st.expander("1️⃣ Semantic Layer (Embeddings & Pooling)", expanded=True):
|
| st.markdown("*💡 **BERTopic Default:** Uses `all-MiniLM-L6-v2` with `Mean` pooling.*")
|
| ui_embedding = st.selectbox("Embedding Model (Override Default):", list(EMBEDDING_MAP.keys()))
|
| ui_pooling = st.selectbox("Pooling Strategy (Override Default):", list(POOLING_MAP.keys()))
|
|
|
| with st.expander("2️⃣ Geometry Layer (Dimensionality Reduction)", expanded=True):
|
| st.markdown("*💡 **BERTopic Default:** Uses `UMAP` with `Cosine` distance to reduce 384D to 5D space.*")
|
| ui_algo = st.selectbox("Algorithm", ["UMAP (Complex geometry)", "PCA (Fast/Deterministic)"])
|
| if "UMAP" in ui_algo:
|
| ui_metric = st.selectbox("Distance Metric", ["cosine", "euclidean", "manhattan"])
|
| else:
|
| ui_metric = "euclidean"
|
| st.info("PCA inherently uses Variance (Euclidean math), so distance metrics are bypassed.")
|
|
|
| with st.expander("3️⃣ Clustering Layer (Grouping)", expanded=True):
|
| st.markdown("*💡 **BERTopic Default:** Uses `HDBSCAN` exclusively (which crashes on tiny datasets).*")
|
| st.markdown("""
|
| *The model mathematically draws boundaries around similar sentences. *
|
| * **Primary clustering algorithm (HDBSCAN):** Runs on datasets $\ge$ 15 sentences. Automatically filters outliers and finds dense semantic clouds. *(Defaults: min_cluster_size=10)*
|
| * **Fallback clustering algorithm (K-Means):** Runs on datasets $<$ 15 sentences. Forces all sentences into buckets to prevent math crashes on tiny text samples.
|
| """)
|
|
|
| with st.expander("4️⃣ Vocabulary Layer (Vectorization)", expanded=True):
|
| st.markdown("*💡 **BERTopic Default:** Uses `Unigrams` (1 word) and does **not** filter redundant dataset noise.*")
|
| ngram_range = st.slider("N-Gram Range", 1, 3, (1, 2), help="1=Unigrams, 2=Bigrams (e.g., 'machine learning')")
|
| auto_noise = st.checkbox("Auto-Remove Redundant Noise (max_df)", value=True, help="Mathematically deletes words appearing in >85% of documents.")
|
|
|
| with st.expander("5️⃣ Extraction Layer (Representation)", expanded=True):
|
| st.markdown("*💡 **BERTopic Default:** ALWAYS extracts baseline words using **c-TF-IDF** (Word Frequency).*")
|
| ui_extraction = st.selectbox("Apply Advanced Filter on top of c-TF-IDF:", ["None (Base c-TF-IDF only)", "KeyBERTInspired (Semantic cosine)", "MMR (Reduce redundancy)"])
|
| if "MMR" in ui_extraction:
|
| mmr_diversity = st.slider("MMR Diversity Penalty", 0.0, 1.0, 0.3)
|
| else:
|
| mmr_diversity = None
|
|
|
| st.header("📊 Evaluation Metrics")
|
| eval_metrics = st.multiselect(
|
| "Select KPIs to generate a final report card:",
|
| ["Topic Diversity", "NPMI Coherence", "UMass Coherence", "Silhouette Score"],
|
| default=["Topic Diversity", "NPMI Coherence", "UMass Coherence", "Silhouette Score"]
|
| )
|
|
|
| st.divider()
|
|
|
|
|
|
|
|
|
| if st.button("🚀 Run Topic Modeling Pipeline", type="primary", use_container_width=True):
|
|
|
| if not raw_data or len(raw_data) < 20:
|
| st.error("Please provide more text data!")
|
| st.stop()
|
|
|
| with st.spinner("Processing Semantic Pipeline... (Models are cached to prevent crashes)"):
|
|
|
| sentences = [s.strip() for s in raw_data.split('.') if len(s.strip()) > 10]
|
| dataset_size = len(sentences)
|
|
|
| academic_noise = ['students', 'course', 'research', 'paper', 'found', 'likely', 'did']
|
| from sklearn.feature_extraction import text
|
| stop_w = list(text.ENGLISH_STOP_WORDS.union(academic_noise))
|
|
|
| vectorizer_model = CountVectorizer(stop_words=stop_w, ngram_range=ngram_range, max_df=0.85 if auto_noise and dataset_size > 10 else 1.0)
|
|
|
| custom_embedder = load_embedder(EMBEDDING_MAP[ui_embedding], POOLING_MAP[ui_pooling])
|
| embeddings = custom_embedder.encode(sentences)
|
|
|
| is_fallback = False
|
| if dataset_size < 15 or "PCA" in ui_algo:
|
| safe_n_themes = min(n_themes, dataset_size)
|
| dim_model = PCA(n_components=2, random_state=42)
|
| cluster_model = KMeans(n_clusters=safe_n_themes, random_state=42)
|
| reduce_topics = None
|
| is_fallback = True
|
| algo_used = "PCA"
|
| cluster_algo = "K-Means"
|
| else:
|
| dim_model = UMAP(n_neighbors=15, n_components=5, metric=ui_metric, random_state=42)
|
| clustering_model = HDBSCAN(min_cluster_size=10, metric='euclidean', cluster_selection_method='eom')
|
| reduce_topics = n_themes
|
| algo_used = "UMAP"
|
| cluster_algo = "HDBSCAN"
|
|
|
| if "MMR" in ui_extraction:
|
| rep_model = MaximalMarginalRelevance(diversity=mmr_diversity, top_n_words=words_per_theme)
|
| elif "KeyBERT" in ui_extraction:
|
| rep_model = KeyBERTInspired(top_n_words=words_per_theme)
|
| else:
|
| rep_model = None
|
|
|
| topic_model = BERTopic(
|
| embedding_model=custom_embedder,
|
| umap_model=dim_model,
|
| hdbscan_model=cluster_model,
|
| vectorizer_model=vectorizer_model,
|
| representation_model=rep_model,
|
| nr_topics=reduce_topics,
|
| top_n_words=words_per_theme,
|
| language="english"
|
| )
|
| topics, _ = topic_model.fit_transform(sentences)
|
|
|
|
|
|
|
|
|
| st.success("Analysis Complete!")
|
|
|
| if is_fallback:
|
| if safe_n_themes < n_themes:
|
| st.warning(f"⚠️ **Reduced requested themes from {n_themes} to {safe_n_themes}.**\n\n"
|
| f"*The Math Explanation:* BERTopic clusters complete sentences to preserve context. "
|
| f"You cannot sort {dataset_size} sentences into {n_themes} buckets without leaving empty buckets, "
|
| f"which mathematically breaks the clustering algorithms!")
|
| else:
|
| st.info(f"ℹ️ Auto-switched to PCA/K-Means due to small dataset size ({dataset_size} sentences).")
|
|
|
| st.markdown("### 🏆 Discovered Themes")
|
| topic_info = topic_model.get_topic_info()
|
| all_words = []
|
|
|
| cols = st.columns(3)
|
| col_idx = 0
|
| for t_id in topic_info['Topic']:
|
| if t_id == -1: continue
|
| theme_w = [w[0] for w in topic_model.get_topic(t_id)]
|
| all_words.append(theme_w)
|
| with cols[col_idx % 3]:
|
| st.info(f"**Theme {t_id + 1}**\n\n" + "\n".join([f"🔹 {w}" for w in theme_w]))
|
| col_idx += 1
|
|
|
|
|
| div_val, npmi_val, umass_val, sil_val = 0.0, 0.0, 0.0, 0.0
|
| div_status, npmi_status, umass_status, sil_status = "Skipped", "Skipped", "Skipped", "Skipped"
|
|
|
| if len(eval_metrics) > 0:
|
| with st.spinner("Calculating mathematical metrics... (NPMI requires building a dictionary and takes a moment)"):
|
|
|
|
|
| if "Topic Diversity" in eval_metrics and len(all_words) > 0:
|
| u_words = set([w for t in all_words for w in t])
|
| t_words = sum([len(t) for t in all_words])
|
| div_val = float(len(u_words) / t_words) if t_words > 0 else 0.0
|
| div_status = f"{div_val:.2f}"
|
|
|
|
|
| if "NPMI Coherence" in eval_metrics or "UMass Coherence" in eval_metrics:
|
| try:
|
| tokenized = [vectorizer_model.build_analyzer()(s) for s in sentences]
|
| dictionary = corpora.Dictionary(tokenized)
|
|
|
| if "NPMI Coherence" in eval_metrics:
|
| cm_npmi = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='c_npmi')
|
| temp_npmi = cm_npmi.get_coherence()
|
| npmi_status = "N/A (Too few words)" if np.isnan(temp_npmi) else f"{float(temp_npmi):.2f}"
|
|
|
| if "UMass Coherence" in eval_metrics:
|
| cm_umass = CoherenceModel(topics=all_words, texts=tokenized, dictionary=dictionary, coherence='u_mass')
|
| temp_umass = cm_umass.get_coherence()
|
| umass_status = "N/A (Too few words)" if np.isnan(temp_umass) else f"{float(temp_umass):.2f}"
|
| except Exception:
|
| npmi_status = "Skipped (Data too small)"
|
| umass_status = "Skipped (Data too small)"
|
|
|
|
|
| if "Silhouette Score" in eval_metrics:
|
| valid_idx = [i for i, t in enumerate(topics) if t != -1]
|
| unique_topics = set([topics[i] for i in valid_idx])
|
|
|
| if 1 < len(unique_topics) < len(valid_idx):
|
| sil_val = float(silhouette_score(
|
| np.array([embeddings[i] for i in valid_idx]),
|
| [topics[i] for i in valid_idx],
|
| metric='cosine'
|
| ))
|
| sil_status = f"{sil_val:.2f}"
|
| else:
|
| sil_status = "Skipped (Themes need ≥2 sentences)"
|
|
|
|
|
| st.markdown("### 📊 Key Performance Indicators (KPI)")
|
| kpi_cols = st.columns(len(eval_metrics))
|
|
|
| for idx, metric in enumerate(eval_metrics):
|
| with kpi_cols[idx]:
|
| if metric == "Topic Diversity":
|
| st.metric("Topic Diversity (Target: 1.0)", div_status)
|
| elif metric == "NPMI Coherence":
|
| st.metric("NPMI Coherence (Target: >0.1)", npmi_status)
|
| elif metric == "UMass Coherence":
|
| st.metric("UMass Coherence (Target: closer to 0)", umass_status)
|
| elif metric == "Silhouette Score":
|
| st.metric("Silhouette Score (Target: >0.0)", sil_status)
|
|
|
|
|
|
|
|
|
| st.markdown("### 📈 Explainable AI (XAI) Architecture Map")
|
|
|
| with st.spinner("Rendering Explainable AI Dashboard..."):
|
| sns.set_theme(style="whitegrid")
|
| fig = plt.figure(figsize=(16, 14))
|
| fig.suptitle(f"Topic Modeling Pipeline Analytics\n(Pooling: {ui_pooling.split()[0]} | Rep: {ui_extraction.split()[0]})", fontsize=20, fontweight='bold', y=0.98)
|
| box_style = dict(boxstyle="round,pad=0.4", facecolor='lightyellow', edgecolor='orange', alpha=0.9)
|
|
|
|
|
| ax1 = plt.subplot(3, 2, 1)
|
| sns.heatmap(embeddings[:, :50], cmap="viridis", cbar=False, ax=ax1)
|
| ax1.set_title("STEP 1: Embeddings & Pooling", fontsize=13, fontweight='bold')
|
| ax1.set_ylabel("Sentences")
|
| ax1.set_xlabel("Vector Dimensions (First 50 shown)")
|
| ax1.text(0.5, -0.25, f"Math: {ui_embedding.split('/')[-1]} encodes text into 384D.\nPooling '{ui_pooling.split()[0]}' squashes word vectors into 1 sentence vector.",
|
| fontsize=10, ha='center', va='top', transform=ax1.transAxes, bbox=box_style)
|
|
|
|
|
| ax2 = plt.subplot(3, 2, 2)
|
| reduced_embeddings = topic_model.umap_model.transform(embeddings)
|
| ax2.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c='grey', s=100, alpha=0.6, edgecolor='k')
|
| ax2.set_title(f"STEP 2: Geometry ({algo_used})", fontsize=13, fontweight='bold')
|
| ax2.text(0.5, -0.25, f"Math: {algo_used} reduces 384D vectors into a 2D map.\nPlaces similar sentences close together.",
|
| fontsize=10, ha='center', va='top', transform=ax2.transAxes, bbox=box_style)
|
|
|
|
|
| ax3 = plt.subplot(3, 2, 3)
|
| ax3.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], c=topics, cmap='tab10', s=150, edgecolor='k')
|
| ax3.set_title(f"STEP 3: Clustering ({cluster_algo})", fontsize=13, fontweight='bold')
|
| ax3.text(0.5, -0.25, f"Math: {cluster_algo} scans the 2D space to draw boundaries.\nColors represent assigned semantic clusters.",
|
| fontsize=10, ha='center', va='top', transform=ax3.transAxes, bbox=box_style)
|
|
|
|
|
| ax4 = plt.subplot(3, 2, 4)
|
| theme_1_data = topic_model.get_topic(0)
|
| if theme_1_data:
|
| words = [x[0] for x in theme_1_data][::-1]
|
| scores = [x[1] for x in theme_1_data][::-1]
|
| ax4.barh(words, scores, color='coral', edgecolor='black')
|
| ax4.set_title(f"STEP 4: Topic Representation ({ui_extraction.split()[0]})", fontsize=13, fontweight='bold')
|
| ax4.text(0.5, -0.25, f"Math: Applies {ui_extraction.split()[0]} to rank vocabulary.\nLonger bars = higher semantic relevance.",
|
| fontsize=10, ha='center', va='top', transform=ax4.transAxes, bbox=box_style)
|
| else:
|
| ax4.text(0.5, 0.5, "Theme not found", ha='center', transform=ax4.transAxes)
|
|
|
|
|
| ax5 = plt.subplot(3, 2, 5)
|
| ax5.axis('off')
|
| ax5.set_title("STEP 5: Key Performance Indicators (KPI)", fontsize=13, fontweight='bold', y=0.9)
|
|
|
| kpi_text = (
|
| f"📊 Topic Diversity: {div_status} (Target: 1.0)\n\n"
|
| f"🧠 NPMI Coherence: {npmi_status} (Target: >0.1)\n\n"
|
| f"📚 UMass Coherence: {umass_status} (Target: Closer to 0)\n\n"
|
| f"📐 Silhouette Score: {sil_status} (Target: >0.0)"
|
| )
|
| ax5.text(0.5, 0.4, kpi_text, fontsize=11, va='center', ha='center',
|
| bbox=dict(boxstyle="square,pad=1.5", facecolor='#e6f2ff', edgecolor='#377eb8', lw=2))
|
| ax5.text(0.5, -0.15, "Math: Since pipeline algorithms don't use 'Training Loss',\nthese KPIs provide the absolute mathematical grade of the topics.",
|
| fontsize=10, ha='center', va='top', transform=ax5.transAxes, bbox=box_style)
|
|
|
|
|
| ax6 = plt.subplot(3, 2, 6)
|
| ax6.axis('off')
|
| summary_text = (
|
| "=== PIPELINE ARCHITECTURE ===\n\n"
|
| f"1. Embeddings: {ui_embedding.split()[0]}\n"
|
| f"2. Pooling: {ui_pooling.split()[0]}\n"
|
| f"3. N-Grams: {ngram_range}\n"
|
| f"4. Geometry: {algo_used}\n"
|
| f"5. Clustering: {cluster_algo}\n"
|
| f"6. Representation: {ui_extraction.split()[0]}\n\n"
|
| "This modular pipeline successfully transforms unstructured text\n"
|
| "into mathematically validated semantic domains."
|
| )
|
| ax6.text(0.1, 0.5, summary_text, fontsize=12, va='center', ha='left',
|
| bbox=dict(boxstyle="square,pad=1", facecolor='#f0f0f0', edgecolor='grey', lw=2))
|
|
|
| plt.subplots_adjust(hspace=0.6, wspace=0.3)
|
| st.pyplot(fig, use_container_width=True) |
