def interface(): return 'Gradio UI updated' """ app.py — Streamlit frontend for the AI-driven Topic Modeling application. This module provides an interactive web interface that allows users to: 1. Upload a CSV file containing research paper Titles and Abstracts. 2. Configure pipeline parameters (min topics, LLM label generation). 3. Run the TopicAgent pipeline with a single click. 4. View and explore results: topics table, comparison, taxonomy map. 5. Review topics with an editable review table. 6. Visualize topic distributions with interactive Plotly charts. 7. Download all generated outputs (CSV, JSON). """ import os import json import tempfile import streamlit as st import pandas as pd import plotly.express as px import plotly.graph_objects as go from agents import TopicAgent from computation_clusters import run_computation_clustering from research_chatbot import build_context_summary, chat_with_groq, SUGGESTED_PROMPTS, get_context_stats # --------------------------------------------------------------------------- # HuggingFace Spaces compatibility: use a writable output directory # On HF Spaces the working directory can be read-only, so fall back to /tmp # --------------------------------------------------------------------------- OUTPUT_DIR = "outputs" try: os.makedirs(OUTPUT_DIR, exist_ok=True) # Test write access _test_path = os.path.join(OUTPUT_DIR, ".write_test") with open(_test_path, "w") as _f: _f.write("ok") os.remove(_test_path) except (OSError, PermissionError): OUTPUT_DIR = os.path.join(tempfile.gettempdir(), "topic_modeler_outputs") os.makedirs(OUTPUT_DIR, exist_ok=True) # --------------------------------------------------------------------------- # Page configuration # --------------------------------------------------------------------------- st.set_page_config( page_title="Research Topic Modeler — AI Agent", page_icon="🔬", layout="wide", initial_sidebar_state="expanded", ) # --------------------------------------------------------------------------- # Custom CSS for a polished, professional look with dark-safe text colors # --------------------------------------------------------------------------- st.markdown(""" """, unsafe_allow_html=True) # --------------------------------------------------------------------------- # Header # --------------------------------------------------------------------------- st.markdown("""

🔬 Research Topic Modeler

AI-powered topic modeling agent for research papers — discover, compare, and classify themes across Titles and Abstracts

""", unsafe_allow_html=True) # --------------------------------------------------------------------------- # Sidebar — Configuration # --------------------------------------------------------------------------- with st.sidebar: st.markdown("## ⚙️ Configuration") st.markdown("---") # File upload st.markdown("### 📁 Dataset") uploaded_file = st.file_uploader( "Upload CSV with Title & Abstract columns", type=["csv"], help="The CSV must contain at least 'Title' and 'Abstract' columns.", ) # Or use default dataset use_default = st.checkbox( "Use default dataset (dataset.csv)", value=True if not uploaded_file else False, help="Use the bundled dataset.csv file in the project directory.", ) st.markdown("---") st.markdown("### 🎯 Parameters") min_topics = st.slider( "Minimum Topics", min_value=50, max_value=300, value=100, step=10, help="Minimum number of fine-grained topics to generate per source. Topics are sub-divisions within clusters.", ) max_clusters = st.slider( "Max Clusters", min_value=10, max_value=40, value=25, step=5, help="Target maximum number of high-level thematic clusters (10–40 range). Topics are extracted within each cluster.", ) use_llm = st.checkbox( "🤖 Use LLM for Label Generation (Groq)", value=False, help="Use Groq's LLaMA model to generate contextual topic labels. " "Falls back to keyword heuristic if unchecked.", ) groq_key = os.environ.get("GROQ_API_KEY", "") if use_llm: groq_key = st.text_input( "Groq API Key", value=groq_key, type="password", help="Your Groq API key for LLM label generation.", ) st.markdown("---") st.markdown("### 📋 Pipeline Steps (SPECTER2 + HDBSCAN)") steps_info = [ "1. Load & validate CSV", "2. Generate SPECTER2 embeddings", "3. HDBSCAN clustering (10–{} clusters)".format(max_clusters), "4. Extract cluster keywords (c-TF-IDF)", "5. Merge similar clusters", "6. Extract sub-topics via NMF (≥{} topics)".format(min_topics), "7. AI Council label generation", "8. Build taxonomy map", "9. Export outputs (CSV, JSON)", ] for step in steps_info: st.markdown(f'

{step}

', unsafe_allow_html=True) # --------------------------------------------------------------------------- # Main area — Run button and results # --------------------------------------------------------------------------- col_run, col_status = st.columns([2, 3]) with col_run: run_clicked = st.button("🚀 Run Topic Modeling Agent", use_container_width=True, type="primary") with col_status: if "result" in st.session_state and st.session_state.result is not None: res = st.session_state.result if res.status == "success": st.markdown('✓ Pipeline Complete', unsafe_allow_html=True) elif res.status == "failed": st.markdown('✗ Pipeline Failed', unsafe_allow_html=True) else: st.markdown('● Awaiting Input', unsafe_allow_html=True) # --------------------------------------------------------------------------- # Execute pipeline # --------------------------------------------------------------------------- if run_clicked: # Determine CSV path csv_path = None if uploaded_file is not None: # Save uploaded file to a temp location with tempfile.NamedTemporaryFile(delete=False, suffix=".csv", dir=".") as tmp: tmp.write(uploaded_file.getvalue()) csv_path = tmp.name elif use_default: csv_path = "dataset.csv" if not os.path.exists(csv_path): st.error("❌ Default dataset.csv not found in the project directory.") st.stop() else: st.error("❌ Please upload a CSV file or select the default dataset.") st.stop() # Run the agent with st.spinner("🔄 Running the Topic Modeling Agent … this may take a few minutes."): progress = st.progress(0, text="Initializing …") agent = TopicAgent( csv_path=csv_path, output_dir=OUTPUT_DIR, min_topics=min_topics, max_clusters=max_clusters, use_llm_labels=use_llm, use_specter=True, groq_api_key=groq_key if use_llm else None, ) # Display step-by-step progress — SPECTER2 + HDBSCAN pipeline progress.progress(5, text="Step 1/9: Loading CSV …") agent._step_load_csv() progress.progress(10, text="Step 2/9: Generating SPECTER2 embeddings …") agent._step_specter_embed() progress.progress(30, text="Step 3/9: HDBSCAN clustering …") agent._step_specter_cluster() progress.progress(45, text="Step 4/9: Extracting cluster keywords …") agent._step_specter_extract() progress.progress(55, text="Step 5/9: Merging similar clusters …") agent._step_merge_similar_clusters() progress.progress(65, text="Step 6/9: Extracting sub-topics (≥{} topics) …".format(min_topics)) agent._step_specter_subtopics() progress.progress(80, text="Step 7/9: Generating topic labels …") agent._step_specter_label() progress.progress(90, text="Step 8/9: Building taxonomy map …") agent._step_specter_taxonomy() progress.progress(95, text="Step 9/9: Exporting outputs …") agent._step_export() agent._result.status = "success" progress.progress(100, text="✅ Pipeline complete!") st.session_state.result = agent._result # Clean up temp file if uploaded_file is not None and csv_path and os.path.exists(csv_path): try: os.unlink(csv_path) except Exception: pass st.rerun() # --------------------------------------------------------------------------- # Helper: Plotly chart theme (dark background, readable text) # --------------------------------------------------------------------------- PLOTLY_LAYOUT = dict( paper_bgcolor="rgba(26, 26, 46, 0.95)", plot_bgcolor="rgba(22, 33, 62, 0.95)", font=dict(family="Inter, sans-serif", size=13, color="#e0e0e0"), title_font=dict(size=18, color="#ffffff"), legend=dict( font=dict(color="#e0e0e0"), bgcolor="rgba(26, 26, 46, 0.7)", bordercolor="#667eea", borderwidth=1, ), xaxis=dict( gridcolor="rgba(102, 126, 234, 0.15)", zerolinecolor="rgba(102, 126, 234, 0.25)", tickfont=dict(color="#c4b5fd"), title_font=dict(color="#e0e0e0"), ), yaxis=dict( gridcolor="rgba(102, 126, 234, 0.15)", zerolinecolor="rgba(102, 126, 234, 0.25)", tickfont=dict(color="#c4b5fd"), title_font=dict(color="#e0e0e0"), ), margin=dict(l=60, r=30, t=60, b=60), ) # Gradient-like color sequence CHART_COLORS = [ "#667eea", "#764ba2", "#f093fb", "#f5576c", "#4facfe", "#00f2fe", "#43e97b", "#38f9d7", "#fa709a", "#fee140", "#a18cd1", "#fbc2eb", "#ff9a9e", "#fad0c4", "#ffecd2", "#fcb69f", ] # --------------------------------------------------------------------------- # Display results # --------------------------------------------------------------------------- if "result" in st.session_state and st.session_state.result is not None: result = st.session_state.result if result.status == "failed": st.error(f"Pipeline failed with errors: {result.errors}") st.stop() # ---- Summary Statistics ---- st.markdown('

📊 Summary Statistics

', unsafe_allow_html=True) # Calculate cluster count from combined_topics has_clusters = "cluster_id" in result.combined_topics.columns n_clusters = int(result.combined_topics["cluster_id"].nunique()) if has_clusters else 0 n_topics = len(result.combined_topics) c1, c2, c3, c4, c5, c6 = st.columns(6) with c1: st.markdown(f"""

{n_clusters}

Clusters

""", unsafe_allow_html=True) with c2: st.markdown(f"""

{n_topics}

Total Topics

""", unsafe_allow_html=True) with c3: st.markdown(f"""

{len(result.title_topics)}

Title Clusters

""", unsafe_allow_html=True) with c4: st.markdown(f"""

{len(result.abstract_topics)}

Abstract Clusters

""", unsafe_allow_html=True) with c5: mapped_count = result.taxonomy_map.get("metadata", {}).get("mapped_count", 0) st.markdown(f"""

{mapped_count}

Mapped Themes

""", unsafe_allow_html=True) with c6: novel_count = result.taxonomy_map.get("metadata", {}).get("novel_count", 0) st.markdown(f"""

{novel_count}

Novel Themes

""", unsafe_allow_html=True) st.markdown("
", unsafe_allow_html=True) # ---- Tabbed Results ---- tab1, tab_umap, tab_explorer, tab_network, tab_council, tab4, tab5, tab_review, tab_charts, tab_comp, tab_chat = st.tabs([ "📋 Topics Table", "🗺️ UMAP Cluster Map", "🔍 Cluster Explorer", "🕸️ Cluster Network", "🤖 AI Council", "⚖️ Theme Comparison", "🗺️ Taxonomy Map", "✏️ Review Table", "📈 Charts", "🧮 Comp Strategy Map", "🧠 Research Chatbot", ]) # Tab 1: Combined Topics Table (with cluster hierarchy) with tab1: st.markdown('

Combined Topics Table

', unsafe_allow_html=True) has_cluster_col = "cluster_id" in result.combined_topics.columns if has_cluster_col: n_cl = int(result.combined_topics["cluster_id"].nunique()) st.markdown( f"Showing all **{len(result.combined_topics)}** topics grouped into **{n_cl}** clusters." ) else: st.markdown(f"Showing all **{len(result.combined_topics)}** topics.") # Filter controls fcol1, fcol2, fcol3 = st.columns(3) with fcol1: if "source" in result.combined_topics.columns: source_filter = st.multiselect( "Filter by Source", options=result.combined_topics["source"].unique().tolist(), default=result.combined_topics["source"].unique().tolist(), ) else: source_filter = [] with fcol2: search_term = st.text_input("🔍 Search keywords", "") with fcol3: if has_cluster_col: cluster_options = sorted(result.combined_topics["cluster_id"].unique().tolist()) cluster_filter = st.multiselect( "Filter by Cluster", options=cluster_options, default=cluster_options, key="topics_cluster_filter", ) else: cluster_filter = [] display_df = result.combined_topics.copy() if source_filter and "source" in display_df.columns: display_df = display_df[display_df["source"].isin(source_filter)] if cluster_filter and has_cluster_col: display_df = display_df[display_df["cluster_id"].isin(cluster_filter)] if search_term: mask = display_df["keywords"].str.contains(search_term, case=False, na=False) if "label" in display_df.columns: mask |= display_df["label"].str.contains(search_term, case=False, na=False) if "cluster_label" in display_df.columns: mask |= display_df["cluster_label"].str.contains(search_term, case=False, na=False) display_df = display_df[mask] # Build column config dynamically col_config = { "topic_id": st.column_config.NumberColumn("Topic ID", width="small"), "keywords": st.column_config.TextColumn("Keywords", width="large"), } if "label" in display_df.columns: col_config["label"] = st.column_config.TextColumn("Label", width="medium") if has_cluster_col: col_config["cluster_id"] = st.column_config.NumberColumn("Cluster", width="small") if "cluster_label" in display_df.columns: col_config["cluster_label"] = st.column_config.TextColumn("Cluster Label", width="medium") if "source" in display_df.columns: col_config["source"] = st.column_config.TextColumn("Source", width="small") if "n_documents" in display_df.columns: col_config["n_documents"] = st.column_config.NumberColumn("Docs", width="small") st.dataframe( display_df, use_container_width=True, height=500, column_config=col_config, ) # ================================================================== # Tab: UMAP 2D Cluster Map # ================================================================== with tab_umap: st.markdown('

🗺️ UMAP 2D Cluster Map

', unsafe_allow_html=True) st.markdown("Interactive 2D visualization of document embeddings colored by HDBSCAN cluster assignment.") embeddings_2d = result.cluster_data.get("embeddings_2d") cluster_labels_list = result.cluster_data.get("cluster_labels") if embeddings_2d and cluster_labels_list: import numpy as np emb_arr = np.array(embeddings_2d) labels_arr = np.array(cluster_labels_list) umap_df = pd.DataFrame({ "UMAP_1": emb_arr[:, 0], "UMAP_2": emb_arr[:, 1], "Cluster": labels_arr.astype(str), }) # Mark noise points umap_df.loc[umap_df["Cluster"] == "-1", "Cluster"] = "Noise" fig_umap = px.scatter( umap_df, x="UMAP_1", y="UMAP_2", color="Cluster", opacity=0.7, hover_data=["Cluster"], color_discrete_sequence=px.colors.qualitative.Set2 + px.colors.qualitative.Dark24, ) fig_umap.update_traces( marker=dict(size=5, line=dict(width=0.3, color="#ffffff")), ) fig_umap.update_layout( plot_bgcolor="#1a1a2e", paper_bgcolor="#1a1a2e", font=dict(color="#e0e0e0", family="Inter"), title=f"UMAP 2D Projection — {int(labels_arr.max()) + 1} Clusters", xaxis=dict(showgrid=False, zeroline=False, title="UMAP Dimension 1", color="#c4b5fd"), yaxis=dict(showgrid=False, zeroline=False, title="UMAP Dimension 2", color="#c4b5fd"), legend_title="Cluster", height=650, margin=dict(l=40, r=40, t=60, b=40), ) st.plotly_chart(fig_umap, use_container_width=True) # Cluster size summary cluster_counts = umap_df["Cluster"].value_counts().reset_index() cluster_counts.columns = ["Cluster", "Documents"] cluster_counts = cluster_counts[cluster_counts["Cluster"] != "Noise"] st.markdown("#### Cluster Sizes") st.dataframe(cluster_counts, use_container_width=True, height=300) else: st.info("🔍 No 2D embedding data available. Run the SPECTER2 + HDBSCAN pipeline first.") # ================================================================== # Tab: Cluster Explorer (keywords, representative sentences, details) # ================================================================== with tab_explorer: st.markdown('

🔍 Cluster Explorer

', unsafe_allow_html=True) st.markdown("Deep-dive into each cluster: see keywords, top representative papers, and document counts.") cluster_details = result.cluster_data.get("cluster_details", {}) if cluster_details: cluster_ids = sorted(cluster_details.keys()) # Cluster selector selected_cluster = st.selectbox( "Select a Cluster to Explore", options=cluster_ids, format_func=lambda cid: f"Cluster {cid} — {cluster_details[cid].get('keywords', '')[:60]}…", key="cluster_explorer_select", ) if selected_cluster is not None: detail = cluster_details[selected_cluster] # Cluster info card st.markdown(f"""

Cluster {selected_cluster}
Documents: {detail.get('n_documents', 'N/A')}
Keywords: {detail.get('keywords', 'N/A')}

""", unsafe_allow_html=True) # Representative papers reps = detail.get("representatives", []) if reps: st.markdown("#### 📄 Top Representative Papers (closest to cluster centroid)") for i, rep in enumerate(reps, 1): st.markdown(f"""

#{i} {rep}

""", unsafe_allow_html=True) else: st.info("No representative sentences available for this cluster.") # Sub-topics within this cluster if has_cluster_col: cluster_topics = result.combined_topics[ result.combined_topics["cluster_id"] == selected_cluster ] if not cluster_topics.empty: st.markdown(f"#### 🧬 Sub-Topics in Cluster {selected_cluster} ({len(cluster_topics)} topics)") st.dataframe( cluster_topics[["topic_id", "keywords", "label", "n_documents"]].reset_index(drop=True) if "label" in cluster_topics.columns else cluster_topics[["topic_id", "keywords", "n_documents"]].reset_index(drop=True), use_container_width=True, height=300, ) # All clusters overview st.markdown("---") st.markdown("#### 📊 All Clusters Overview") overview_rows = [] for cid in cluster_ids: d = cluster_details[cid] overview_rows.append({ "Cluster": cid, "Documents": d.get("n_documents", 0), "Keywords": d.get("keywords", "")[:80], "Top Paper": (d.get("representatives", ["—"])[0])[:80] if d.get("representatives") else "—", }) st.dataframe( pd.DataFrame(overview_rows), use_container_width=True, height=400, column_config={ "Cluster": st.column_config.NumberColumn("Cluster", width="small"), "Documents": st.column_config.NumberColumn("Docs", width="small"), "Keywords": st.column_config.TextColumn("Top Keywords", width="large"), "Top Paper": st.column_config.TextColumn("Representative Paper", width="large"), }, ) else: st.info("🔍 No cluster details available. Run the SPECTER2 + HDBSCAN pipeline first.") # ================================================================== # Tab: Cluster Similarity Network # ================================================================== with tab_network: st.markdown('

🕸️ Cluster Similarity Network

', unsafe_allow_html=True) st.markdown("Graph showing cosine similarity between cluster centroids. Stronger connections = more similar clusters.") graph_edges = result.cluster_data.get("graph_edges", []) cluster_details_net = result.cluster_data.get("cluster_details", {}) if graph_edges: # Build network graph using plotly import numpy as np # Get unique cluster IDs from edges cluster_ids_net = sorted(set( [e["source"] for e in graph_edges] + [e["target"] for e in graph_edges] )) # Arrange clusters in a circle layout n_nodes = len(cluster_ids_net) angles = np.linspace(0, 2 * np.pi, n_nodes, endpoint=False) node_x = {cid: float(np.cos(a)) for cid, a in zip(cluster_ids_net, angles)} node_y = {cid: float(np.sin(a)) for cid, a in zip(cluster_ids_net, angles)} # Similarity threshold slider min_sim = st.slider( "Minimum Similarity Threshold", min_value=0.0, max_value=1.0, value=0.3, step=0.05, help="Only show edges with similarity above this threshold.", key="network_sim_threshold", ) filtered_edges = [e for e in graph_edges if e["weight"] >= min_sim] # Edge traces edge_x, edge_y, edge_text = [], [], [] for e in filtered_edges: x0, y0 = node_x[e["source"]], node_y[e["source"]] x1, y1 = node_x[e["target"]], node_y[e["target"]] edge_x.extend([x0, x1, None]) edge_y.extend([y0, y1, None]) fig_net = go.Figure() # Add edges fig_net.add_trace(go.Scatter( x=edge_x, y=edge_y, mode="lines", line=dict(width=1, color="rgba(102,126,234,0.4)"), hoverinfo="none", showlegend=False, )) # Node sizes proportional to document count node_sizes = [] node_labels = [] node_hover = [] for cid in cluster_ids_net: detail = cluster_details_net.get(cid, {}) n_docs = detail.get("n_documents", 10) kw = detail.get("keywords", "")[:50] node_sizes.append(max(15, min(50, n_docs * 2))) node_labels.append(f"C{cid}") node_hover.append(f"Cluster {cid}
Docs: {n_docs}
Keywords: {kw}") # Add nodes fig_net.add_trace(go.Scatter( x=[node_x[c] for c in cluster_ids_net], y=[node_y[c] for c in cluster_ids_net], mode="markers+text", marker=dict( size=node_sizes, color=[i for i in range(n_nodes)], colorscale="Viridis", line=dict(width=1.5, color="#ffffff"), ), text=node_labels, textposition="top center", textfont=dict(color="#e0e0e0", size=11, family="Inter"), hovertext=node_hover, hoverinfo="text", showlegend=False, )) fig_net.update_layout( plot_bgcolor="#1a1a2e", paper_bgcolor="#1a1a2e", font=dict(color="#e0e0e0", family="Inter"), title=f"Cluster Similarity Network ({len(filtered_edges)} edges, threshold ≥ {min_sim:.2f})", xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), height=600, margin=dict(l=20, r=20, t=60, b=20), ) st.plotly_chart(fig_net, use_container_width=True) # Edge table with st.expander("📋 Similarity Scores (Edge List)"): edge_df = pd.DataFrame(filtered_edges) edge_df.columns = ["Source Cluster", "Target Cluster", "Similarity"] st.dataframe(edge_df, use_container_width=True, height=300) else: st.info("🔍 No cluster graph data available. Run the SPECTER2 + HDBSCAN pipeline first.") # ================================================================== # Tab: AI Council Debates (label generation transparency) # ================================================================== with tab_council: st.markdown('

🤖 AI Council — Label Generation Debates

', unsafe_allow_html=True) st.markdown( "Transparency into how topic labels were generated. " "Each cluster's label is debated by three AI personas: " "**Domain Expert**, **Data Analyst**, and **Anti-Hallucination Judge**." ) debates = result.council_debates if debates: debate_ids = sorted(debates.keys()) st.markdown(f"**{len(debate_ids)}** clusters labeled via AI Council debate.") # Summary table debate_rows = [] for cid in debate_ids: d = debates[cid] debate_rows.append({ "Cluster": cid, "Expert Label": d.get("expert", "—"), "Analyst Label": d.get("analyst", "—"), "Anti-Hallucination": d.get("anti_hallucination", d.get("final_label", "—")), "Final Label": d.get("final_label", "—"), }) st.dataframe( pd.DataFrame(debate_rows), use_container_width=True, height=400, column_config={ "Cluster": st.column_config.NumberColumn("Cluster", width="small"), "Expert Label": st.column_config.TextColumn("🎓 Domain Expert", width="medium"), "Analyst Label": st.column_config.TextColumn("📊 Data Analyst", width="medium"), "Anti-Hallucination": st.column_config.TextColumn("🛡️ Hallucination Check", width="medium"), "Final Label": st.column_config.TextColumn("⚖️ Final (Judge)", width="medium"), }, ) # Detailed debate view st.markdown("---") st.markdown("#### 🔎 Detailed Debate View") selected_debate = st.selectbox( "Select cluster to inspect", options=debate_ids, format_func=lambda c: f"Cluster {c} — {debates[c].get('final_label', '')}", key="council_debate_select", ) if selected_debate is not None: d = debates[selected_debate] col_e, col_a, col_h, col_j = st.columns(4) with col_e: st.markdown("""

🎓 Domain Expert

{}

""".format(d.get("expert", "—")), unsafe_allow_html=True) with col_a: st.markdown("""

📊 Data Analyst

{}

""".format(d.get("analyst", "—")), unsafe_allow_html=True) with col_h: st.markdown("""

🛡️ Anti-Hallucination

{}

""".format(d.get("anti_hallucination", d.get("final_label", "—"))), unsafe_allow_html=True) with col_j: st.markdown("""

⚖️ Final Verdict

{}

""".format(d.get("final_label", "—")), unsafe_allow_html=True) else: st.markdown("""

ℹ️ AI Council Debates Not Available

To see label generation debates, enable "🤖 Use LLM for Label Generation (Groq)" in the sidebar and provide a Groq API key before running the pipeline.

When enabled, each cluster label is generated through a 3-persona debate:
• Domain Expert proposes an academic label
• Data Analyst proposes a categorical name
• Anti-Hallucination Judge synthesizes the final label

""", unsafe_allow_html=True) # Tab 4: Theme Comparison with tab4: st.markdown('

Theme Comparison

', unsafe_allow_html=True) if not result.comparison.empty: # Standard comparison view (title vs abstract) st.markdown("**Titles vs Abstracts Theme Alignment**") align_counts = result.comparison["alignment"].value_counts() acol1, acol2, acol3, acol4 = st.columns(4) for col, alignment in zip( [acol1, acol2, acol3, acol4], ["Strong", "Moderate", "Weak", "No Match"], ): with col: count = align_counts.get(alignment, 0) st.metric(label=f"{alignment} Alignment", value=count) st.markdown("
", unsafe_allow_html=True) alignment_filter = st.multiselect( "Filter by Alignment", options=["Strong", "Moderate", "Weak", "No Match"], default=["Strong", "Moderate", "Weak", "No Match"], ) filtered_comp = result.comparison[result.comparison["alignment"].isin(alignment_filter)] st.dataframe( filtered_comp, use_container_width=True, height=500, column_config={ "similarity": st.column_config.ProgressColumn( "Similarity", min_value=0, max_value=1, format="%.2f", ), }, ) elif has_cluster_col and "cluster_id" in result.combined_topics.columns: # SPECTER mode — show inter-cluster similarity comparison st.markdown("**Cluster-to-Cluster Theme Similarity**") st.markdown("Comparing keyword overlap between each pair of clusters.") graph_edges_comp = result.cluster_data.get("graph_edges", []) cluster_details_comp = result.cluster_data.get("cluster_details", {}) if graph_edges_comp: # Build comparison table from graph edges comp_rows = [] for e in graph_edges_comp: src = e["source"] tgt = e["target"] src_detail = cluster_details_comp.get(src, {}) tgt_detail = cluster_details_comp.get(tgt, {}) sim = e["weight"] alignment = ( "Strong" if sim >= 0.6 else "Moderate" if sim >= 0.4 else "Weak" if sim > 0.2 else "Distant" ) comp_rows.append({ "cluster_a": src, "cluster_a_keywords": src_detail.get("keywords", "")[:60], "cluster_b": tgt, "cluster_b_keywords": tgt_detail.get("keywords", "")[:60], "similarity": round(sim, 4), "alignment": alignment, }) comp_df = pd.DataFrame(comp_rows) # Alignment stats align_counts = comp_df["alignment"].value_counts() acol1, acol2, acol3, acol4 = st.columns(4) for col, alignment in zip( [acol1, acol2, acol3, acol4], ["Strong", "Moderate", "Weak", "Distant"], ): with col: count = align_counts.get(alignment, 0) st.metric(label=f"{alignment}", value=count) st.markdown("
", unsafe_allow_html=True) st.dataframe( comp_df.sort_values("similarity", ascending=False), use_container_width=True, height=500, column_config={ "cluster_a": st.column_config.NumberColumn("Cluster A", width="small"), "cluster_b": st.column_config.NumberColumn("Cluster B", width="small"), "cluster_a_keywords": st.column_config.TextColumn("A Keywords", width="medium"), "cluster_b_keywords": st.column_config.TextColumn("B Keywords", width="medium"), "similarity": st.column_config.ProgressColumn( "Similarity", min_value=0, max_value=1, format="%.3f", ), "alignment": st.column_config.TextColumn("Alignment", width="small"), }, ) else: st.info("No cluster similarity data available. Run the pipeline first.") else: st.info("No comparison data available. Run the pipeline first.") # Tab 5: Taxonomy Map with tab5: st.markdown('

Taxonomy Map

', unsafe_allow_html=True) taxonomy = result.taxonomy_map meta = taxonomy.get("metadata", {}) st.markdown(f"""

Classification Summary:
Total Topics: {meta.get('total_topics', 0)} | MAPPED: {meta.get('mapped_count', 0)} | NOVEL: {meta.get('novel_count', 0)} | Threshold: {meta.get('threshold', 0.15)}

""", unsafe_allow_html=True) tax_tab1, tax_tab2 = st.tabs(["✅ Mapped Themes", "🆕 Novel Themes"]) with tax_tab1: mapped_list = taxonomy.get("mapped", []) if mapped_list: mapped_df = pd.DataFrame(mapped_list) st.dataframe( mapped_df, use_container_width=True, height=400, column_config={ "score": st.column_config.ProgressColumn( "Match Score", min_value=0, max_value=1, format="%.3f", ), }, ) else: st.info("No mapped themes found.") with tax_tab2: novel_list = taxonomy.get("novel", []) if novel_list: novel_df = pd.DataFrame(novel_list) st.dataframe( novel_df, use_container_width=True, height=400, column_config={ "score": st.column_config.ProgressColumn( "Match Score", min_value=0, max_value=1, format="%.3f", ), }, ) else: st.info("No novel themes found.") # ================================================================== # Tab 6: Editable Review Table # ================================================================== with tab_review: st.markdown("""

✏️ Topic Review Table

Review, approve, rename, and annotate each topic. Changes are saved to outputs/review_table.csv.

""", unsafe_allow_html=True) # Build review dataframe from combined topics # Load existing review table if available to preserve edits review_csv_path = os.path.join(OUTPUT_DIR, "review_table.csv") if "review_df" not in st.session_state: if os.path.exists(review_csv_path): # Load previously saved review table existing_review = pd.read_csv(review_csv_path) # Merge with current topics to ensure all topics are represented current_ids = set(result.combined_topics["topic_id"].tolist()) existing_ids = set(existing_review["topic_id"].tolist()) if "topic_id" in existing_review.columns else set() if current_ids == existing_ids or existing_ids.issuperset(current_ids): st.session_state.review_df = existing_review else: # Rebuild from current topics, but preserve existing edits review_data = [] for _, row in result.combined_topics.iterrows(): review_data.append({ "topic_id": int(row["topic_id"]), "cluster_id": int(row["cluster_id"]) if "cluster_id" in row.index else "", "cluster_label": row.get("cluster_label", ""), "label": row.get("label", ""), "keywords": row.get("keywords", ""), "source": row.get("source", ""), "approve": False, "rename_to": "", "reasoning": "", }) new_review_df = pd.DataFrame(review_data) # Merge existing edits if not existing_review.empty and "topic_id" in existing_review.columns: for _, erow in existing_review.iterrows(): mask = new_review_df["topic_id"] == erow["topic_id"] if mask.any(): if "approve" in erow: new_review_df.loc[mask, "approve"] = erow["approve"] if "rename_to" in erow and pd.notna(erow["rename_to"]): new_review_df.loc[mask, "rename_to"] = erow["rename_to"] if "reasoning" in erow and pd.notna(erow["reasoning"]): new_review_df.loc[mask, "reasoning"] = erow["reasoning"] st.session_state.review_df = new_review_df else: # Build fresh review table review_data = [] for _, row in result.combined_topics.iterrows(): review_data.append({ "topic_id": int(row["topic_id"]), "cluster_id": int(row["cluster_id"]) if "cluster_id" in row.index else "", "cluster_label": row.get("cluster_label", ""), "label": row.get("label", ""), "keywords": row.get("keywords", ""), "source": row.get("source", ""), "approve": False, "rename_to": "", "reasoning": "", }) st.session_state.review_df = pd.DataFrame(review_data) # Enforce string dtypes to prevent ColumnDataKind.FLOAT errors if "rename_to" in st.session_state.review_df.columns: st.session_state.review_df["rename_to"] = st.session_state.review_df["rename_to"].fillna("").astype(str) if "reasoning" in st.session_state.review_df.columns: st.session_state.review_df["reasoning"] = st.session_state.review_df["reasoning"].fillna("").astype(str) # Filter controls for review table rv_col1, rv_col2, rv_col3 = st.columns(3) with rv_col1: if "source" in st.session_state.review_df.columns and st.session_state.review_df["source"].notna().any(): review_source_options = [s for s in st.session_state.review_df["source"].unique().tolist() if s] review_source_filter = st.multiselect( "Filter by Source", options=review_source_options, default=review_source_options, key="review_source_filter", ) else: review_source_filter = None with rv_col2: review_search = st.text_input("🔍 Search in review table", "", key="review_search") with rv_col3: review_approval_filter = st.selectbox( "Show", options=["All Topics", "Approved Only", "Not Approved"], index=0, key="review_approval_filter", ) # Apply filters filtered_review = st.session_state.review_df.copy() if review_source_filter is not None and "source" in filtered_review.columns: filtered_review = filtered_review[filtered_review["source"].isin(review_source_filter)] if review_search: search_mask = ( filtered_review["keywords"].str.contains(review_search, case=False, na=False) | filtered_review["label"].str.contains(review_search, case=False, na=False) ) filtered_review = filtered_review[search_mask] if review_approval_filter == "Approved Only": filtered_review = filtered_review[filtered_review["approve"] == True] elif review_approval_filter == "Not Approved": filtered_review = filtered_review[filtered_review["approve"] == False] # Editable data editor edited_df = st.data_editor( filtered_review, use_container_width=True, height=500, num_rows="fixed", key="review_editor", column_config={ "topic_id": st.column_config.NumberColumn( "Topic ID", width="small", disabled=True ), "label": st.column_config.TextColumn( "Label", width="medium", ), "keywords": st.column_config.TextColumn( "Keywords", width="large", disabled=True, ), "source": st.column_config.TextColumn( "Source", width="small", disabled=True, ), "approve": st.column_config.CheckboxColumn( "✅ Approve", width="small", default=False, ), "rename_to": st.column_config.TextColumn( "Rename To", width="medium", ), "reasoning": st.column_config.TextColumn( "Reasoning / Notes", width="large", ), }, column_order=["topic_id", "label", "keywords", "approve", "rename_to", "reasoning", "source"], ) # Update session state with edits if edited_df is not None: # Merge edits back into the full review dataframe for idx, erow in edited_df.iterrows(): mask = st.session_state.review_df.index == idx if mask.any(): for col in ["label", "approve", "rename_to", "reasoning"]: if col in erow: st.session_state.review_df.loc[mask, col] = erow[col] # Save button sv_col1, sv_col2, sv_col3 = st.columns([1, 1, 2]) with sv_col1: if st.button("💾 Save Review Table", use_container_width=True, type="primary"): os.makedirs(OUTPUT_DIR, exist_ok=True) st.session_state.review_df.to_csv(review_csv_path, index=False) st.markdown( '

✅ Review table saved to outputs/review_table.csv

', unsafe_allow_html=True, ) with sv_col2: approved_count = int(st.session_state.review_df["approve"].sum()) if "approve" in st.session_state.review_df.columns else 0 total_count = len(st.session_state.review_df) st.markdown(f"""

{approved_count}/{total_count}

Topics Approved

""", unsafe_allow_html=True) # ================================================================== # Tab 7: Charts # ================================================================== with tab_charts: st.markdown('

📈 Topic Visualizations

', unsafe_allow_html=True) # ----------------------------------------------------------- # Chart 0: Topics per Cluster (NEW) # ----------------------------------------------------------- has_cluster_col_chart = "cluster_id" in result.combined_topics.columns if has_cluster_col_chart: st.markdown("#### 🏗️ Topics per Cluster") st.caption("Number of fine-grained topics extracted within each high-level cluster.") topics_per_cluster = result.combined_topics.groupby("cluster_id").size().reset_index(name="Topics") if "cluster_label" in result.combined_topics.columns: label_map = result.combined_topics.drop_duplicates("cluster_id")[["cluster_id", "cluster_label"]] topics_per_cluster = topics_per_cluster.merge(label_map, on="cluster_id", how="left") topics_per_cluster["display"] = topics_per_cluster.apply( lambda r: f"C{int(r['cluster_id'])}: {r['cluster_label']}", axis=1 ) else: topics_per_cluster["display"] = topics_per_cluster["cluster_id"].apply(lambda x: f"Cluster {x}") fig0 = px.bar( topics_per_cluster.sort_values("Topics", ascending=False), x="display", y="Topics", color="Topics", color_continuous_scale=["#302b63", "#667eea", "#f093fb"], text="Topics", ) fig0.update_traces( textposition="outside", textfont=dict(color="#e0e0e0", size=12, family="Inter"), marker=dict(line=dict(width=0)), ) fig0.update_layout( **PLOTLY_LAYOUT, title=f"Topics per Cluster ({topics_per_cluster['cluster_id'].nunique()} clusters)", xaxis_title="Cluster", yaxis_title="Number of Topics", showlegend=False, height=480, ) fig0.update_layout( xaxis=dict(tickangle=-45, tickfont=dict(color="#c4b5fd", size=10)), ) st.plotly_chart(fig0, use_container_width=True) st.markdown("---") # ----------------------------------------------------------- # Chart 1: Topic Frequency by Source # ----------------------------------------------------------- st.markdown("#### 📊 Topic Frequency by Source") st.caption("Number of topics discovered from each source.") if "source" in result.combined_topics.columns: source_counts = result.combined_topics["source"].value_counts().reset_index() source_counts.columns = ["Source", "Count"] else: source_counts = pd.DataFrame({"Source": ["All"], "Count": [len(result.combined_topics)]}) fig1 = px.bar( source_counts, x="Source", y="Count", color="Source", color_discrete_sequence=["#667eea", "#764ba2"], text="Count", ) fig1.update_traces( textposition="outside", textfont=dict(color="#e0e0e0", size=14, family="Inter"), marker=dict( line=dict(width=0), ), ) fig1.update_layout( **PLOTLY_LAYOUT, title="Topic Count by Source", xaxis_title="Source", yaxis_title="Number of Topics", showlegend=False, height=420, ) st.plotly_chart(fig1, use_container_width=True) st.markdown("---") # ----------------------------------------------------------- # Chart 2: Top Keywords Across All Topics # ----------------------------------------------------------- st.markdown("#### 🔤 Top Keywords Across All Topics") st.caption("Most frequently occurring keywords across all discovered topics.") # Extract all keywords, count frequencies all_keywords = [] for kw_str in result.combined_topics["keywords"].dropna(): for kw in kw_str.split(","): kw_clean = kw.strip().lower() if kw_clean and len(kw_clean) > 2: all_keywords.append(kw_clean) kw_counts = pd.Series(all_keywords).value_counts().head(25).reset_index() kw_counts.columns = ["Keyword", "Frequency"] fig2 = px.bar( kw_counts, x="Frequency", y="Keyword", orientation="h", color="Frequency", color_continuous_scale=["#302b63", "#667eea", "#f093fb", "#f5576c"], ) fig2.update_traces( marker=dict(line=dict(width=0)), ) fig2.update_layout( **PLOTLY_LAYOUT, title="Top 25 Keywords by Frequency", xaxis_title="Frequency (across all topics)", yaxis_title="", height=700, coloraxis_colorbar=dict( title="Freq", tickfont=dict(color="#c4b5fd"), title_font=dict(color="#e0e0e0"), ), ) # Override yaxis separately to avoid duplicate keyword with PLOTLY_LAYOUT fig2.update_layout( yaxis=dict( autorange="reversed", gridcolor="rgba(102, 126, 234, 0.1)", tickfont=dict(color="#c4b5fd", size=12), ), ) st.plotly_chart(fig2, use_container_width=True) st.markdown("---") # ----------------------------------------------------------- # Chart 3: Taxonomy Distribution (Mapped vs Novel) # ----------------------------------------------------------- st.markdown("#### 🧬 Taxonomy Classification Distribution") st.caption("How topics are classified against the known research taxonomy.") tax_meta = result.taxonomy_map.get("metadata", {}) tax_data = pd.DataFrame({ "Classification": ["MAPPED", "NOVEL"], "Count": [tax_meta.get("mapped_count", 0), tax_meta.get("novel_count", 0)], }) chart3_col1, chart3_col2 = st.columns(2) with chart3_col1: fig3a = px.pie( tax_data, values="Count", names="Classification", color="Classification", color_discrete_map={ "MAPPED": "#38ef7d", "NOVEL": "#f7b733", }, hole=0.55, ) fig3a.update_traces( textfont=dict(color="#ffffff", size=14), textinfo="percent+label", marker=dict(line=dict(color="#1a1a2e", width=3)), ) fig3a.update_layout( paper_bgcolor="rgba(26, 26, 46, 0.95)", plot_bgcolor="rgba(22, 33, 62, 0.95)", font=dict(family="Inter, sans-serif", size=13, color="#e0e0e0"), title=dict(text="Mapped vs Novel", font=dict(size=16, color="#ffffff")), legend=dict(font=dict(color="#e0e0e0")), height=380, margin=dict(l=20, r=20, t=50, b=20), ) st.plotly_chart(fig3a, use_container_width=True) with chart3_col2: fig3b = px.bar( tax_data, x="Classification", y="Count", color="Classification", color_discrete_map={ "MAPPED": "#38ef7d", "NOVEL": "#f7b733", }, text="Count", ) fig3b.update_traces( textposition="outside", textfont=dict(color="#e0e0e0", size=16, family="Inter"), marker=dict(line=dict(width=0)), ) fig3b.update_layout( **PLOTLY_LAYOUT, title="Classification Count", xaxis_title="", yaxis_title="Number of Topics", showlegend=False, height=380, ) st.plotly_chart(fig3b, use_container_width=True) st.markdown("---") # ----------------------------------------------------------- # Chart 4: Alignment Distribution (from comparisons) # ----------------------------------------------------------- if not result.comparison.empty: st.markdown("#### ⚖️ Theme Alignment Distribution") st.caption("Distribution of alignment strength between Title and Abstract topics.") alignment_data = result.comparison["alignment"].value_counts().reset_index() alignment_data.columns = ["Alignment", "Count"] # Define order and colors align_order = ["Strong", "Moderate", "Weak", "No Match"] align_colors = { "Strong": "#38ef7d", "Moderate": "#4facfe", "Weak": "#f7971e", "No Match": "#f5576c", } fig4 = px.bar( alignment_data, x="Alignment", y="Count", color="Alignment", color_discrete_map=align_colors, text="Count", category_orders={"Alignment": align_order}, ) fig4.update_traces( textposition="outside", textfont=dict(color="#e0e0e0", size=14, family="Inter"), marker=dict(line=dict(width=0)), ) fig4.update_layout( **PLOTLY_LAYOUT, title="Title ↔ Abstract Alignment Distribution", xaxis_title="Alignment Level", yaxis_title="Number of Topic Pairs", showlegend=False, height=420, ) st.plotly_chart(fig4, use_container_width=True) st.markdown("---") # ----------------------------------------------------------- # Chart 5: Similarity Score Histogram # ----------------------------------------------------------- st.markdown("#### 📐 Similarity Score Distribution") st.caption("Distribution of Jaccard similarity scores between matched Title and Abstract topics.") fig5 = px.histogram( result.comparison, x="similarity", nbins=30, color_discrete_sequence=["#667eea"], marginal="box", ) fig5.update_traces( marker=dict( line=dict(width=1, color="#b8b5ff"), ), selector=dict(type="histogram"), ) fig5.update_layout( **PLOTLY_LAYOUT, title="Similarity Score Histogram", xaxis_title="Jaccard Similarity Score", yaxis_title="Count", height=420, bargap=0.05, ) st.plotly_chart(fig5, use_container_width=True) # ================================================================== # Tab 10: Computational Strategy Map # ================================================================== with tab_comp: st.markdown('

🧮 Computational Strategy Map

', unsafe_allow_html=True) st.markdown("SPECTER2 semantic clustering of computational strategies — powered by HDBSCAN + UMAP") COMP_CSV = "computation.csv" if not os.path.exists(COMP_CSV): st.warning("⚠️ `computation.csv` not found in the project directory.") else: # Run / cache pipeline if "comp_cluster_data" not in st.session_state: with st.spinner("🔄 Clustering computation.csv … (cached after first run)"): try: comp_data = run_computation_clustering(COMP_CSV) st.session_state.comp_cluster_data = comp_data except Exception as exc: st.error(f"Clustering failed: {exc}") comp_data = None if "comp_cluster_data" in st.session_state: comp = st.session_state.comp_cluster_data comp_df = comp["df"] cluster_names_comp = comp["cluster_names"] cluster_keywords_comp = comp["cluster_keywords"] strategy_types_comp = comp["strategy_types"] # ---- Summary stats ---- sc1, sc2, sc3 = st.columns(3) with sc1: st.markdown(f"""

{comp['n_papers']}

Papers

""", unsafe_allow_html=True) with sc2: st.markdown(f"""

{comp['n_clusters']}

Semantic Clusters

""", unsafe_allow_html=True) with sc3: st.markdown(f"""

{len(strategy_types_comp)}

Strategy Types

""", unsafe_allow_html=True) st.markdown("
", unsafe_allow_html=True) # ---- Sub-tabs ---- csub1, csub2, csub3, csub4 = st.tabs([ "🗺️ Cluster Map", "📊 Statistics", "📋 Papers", "✏️ Review", ]) # -- Cluster Map -- with csub1: fc1, fc2, fc3 = st.columns(3) with fc1: comp_search = st.text_input("🔍 Search title", "", key="comp_search") with fc2: comp_strat_f = st.multiselect("Strategy Type", options=strategy_types_comp, default=strategy_types_comp, key="comp_strategy_filter") with fc3: cl_opts = sorted(cluster_names_comp.keys()) comp_cl_f = st.multiselect("Cluster", options=cl_opts, default=cl_opts, format_func=lambda c: f"C{c}: {cluster_names_comp[c]}", key="comp_cluster_filter") plot_df = comp_df.copy() if comp_search: plot_df = plot_df[plot_df["Paper Title"].str.contains(comp_search, case=False, na=False)] if comp_strat_f: plot_df = plot_df[plot_df["Strategy Type"].isin(comp_strat_f)] if comp_cl_f: plot_df = plot_df[plot_df["cluster_id"].isin(comp_cl_f)] if len(plot_df) == 0: st.warning("No papers match filters.") else: plot_df = plot_df.copy() plot_df["hover_text"] = plot_df.apply( lambda r: ( f"{r['Paper Title'][:80]}
" f"Authors: {r['Authors'][:60]}
" f"Strategy: {str(r.get('Computational Strategies Used',''))[:60]}
" f"Type: {r['Strategy Type']}
" f"Cluster: {r['cluster_name']}" ), axis=1) plot_df["Cluster"] = plot_df["cluster_name"].astype(str) fig_comp = px.scatter( plot_df, x="x", y="y", color="Cluster", hover_name="Paper Title", custom_data=["hover_text"], color_discrete_sequence=px.colors.qualitative.Set2 + px.colors.qualitative.Dark24, ) fig_comp.update_traces( marker=dict(size=12, line=dict(width=1.5, color="#ffffff")), hovertemplate="%{customdata[0]}", ) fig_comp.update_layout( plot_bgcolor="#1a1a2e", paper_bgcolor="#1a1a2e", font=dict(color="#e0e0e0", family="Inter, sans-serif"), title=dict(text=f"Cluster Map — {len(plot_df)} papers, {comp['n_clusters']} clusters", font=dict(size=18, color="#ffffff")), xaxis=dict(showgrid=False, zeroline=False, title="UMAP Dim 1", tickfont=dict(color="#c4b5fd")), yaxis=dict(showgrid=False, zeroline=False, title="UMAP Dim 2", tickfont=dict(color="#c4b5fd")), legend=dict(title="Cluster", font=dict(color="#e0e0e0"), bgcolor="rgba(26,26,46,0.7)", bordercolor="#667eea", borderwidth=1), height=650, margin=dict(l=40, r=40, t=60, b=40), ) st.plotly_chart(fig_comp, use_container_width=True) st.markdown("---") export_cols = [c for c in ["Paper Title","Authors","Computational Strategies Used", "Strategy Type","Supporting Keywords / Evidence", "cluster_id","cluster_name"] if c in comp_df.columns] st.download_button("⬇️ Export Clustered CSV", data=comp_df[export_cols].to_csv(index=False), file_name="computation_clustered.csv", mime="text/csv", use_container_width=True, key="comp_export_csv") # -- Statistics -- with csub2: for cid in sorted(cluster_names_comp.keys()): csub = comp_df[comp_df["cluster_id"] == cid] kws = cluster_keywords_comp.get(cid, []) strat_bd = csub["Strategy Type"].value_counts().to_dict() st.markdown(f"""

Cluster {cid}: {cluster_names_comp[cid]}
Papers: {len(csub)}
Keywords: {', '.join(kws)}
Types: {', '.join(f'{k} ({v})' for k,v in strat_bd.items() if k.strip())}

""", unsafe_allow_html=True) with st.expander(f"📄 Papers in Cluster {cid}", expanded=False): dcols = [c for c in ["Paper Title","Authors","Strategy Type"] if c in csub.columns] st.dataframe(csub[dcols].reset_index(drop=True), use_container_width=True) st.markdown("---") st.markdown("#### Cluster Size Distribution") sz = comp_df.groupby("cluster_name").size().reset_index(name="Papers") fig_pie = px.pie(sz, values="Papers", names="cluster_name", color_discrete_sequence=CHART_COLORS, hole=0.5) fig_pie.update_traces(textfont=dict(color="#ffffff", size=12), textinfo="percent+label", marker=dict(line=dict(color="#1a1a2e", width=2))) fig_pie.update_layout( paper_bgcolor="rgba(26,26,46,0.95)", plot_bgcolor="rgba(22,33,62,0.95)", font=dict(family="Inter, sans-serif", size=13, color="#e0e0e0"), title=dict(text="Cluster Size", font=dict(size=16, color="#ffffff")), legend=dict(font=dict(color="#e0e0e0")), height=420, margin=dict(l=20,r=20,t=50,b=20), ) st.plotly_chart(fig_pie, use_container_width=True) # -- Papers Table -- with csub3: tf1, tf2 = st.columns(2) with tf1: tbl_s = st.text_input("🔍 Search", "", key="comp_tbl_search") with tf2: tbl_c = st.multiselect("Cluster", options=sorted(cluster_names_comp.keys()), default=sorted(cluster_names_comp.keys()), format_func=lambda c: f"C{c}: {cluster_names_comp[c]}", key="comp_tbl_cluster") tdf = comp_df.copy() if tbl_s: tdf = tdf[tdf["Paper Title"].str.contains(tbl_s, case=False, na=False)] if tbl_c: tdf = tdf[tdf["cluster_id"].isin(tbl_c)] dcols = [c for c in ["Paper Title","Authors","Computational Strategies Used", "Strategy Type","cluster_name"] if c in tdf.columns] st.dataframe(tdf[dcols].reset_index(drop=True), use_container_width=True, height=500) # -- Review -- with csub4: st.markdown("""

✏️ Computational Strategy Review

Review and annotate cluster assignments. Saved to outputs/comp_review_table.csv.

""", unsafe_allow_html=True) comp_rev_path = os.path.join(OUTPUT_DIR, "comp_review_table.csv") if "comp_review_df" not in st.session_state: if os.path.exists(comp_rev_path): st.session_state.comp_review_df = pd.read_csv(comp_rev_path) else: rows = [] for _, r in comp_df.iterrows(): rows.append({ "Paper Title": r.get("Paper Title",""), "Strategy Type": r.get("Strategy Type",""), "cluster_name": r.get("cluster_name",""), "approve": False, "notes": "", }) st.session_state.comp_review_df = pd.DataFrame(rows) if "notes" in st.session_state.comp_review_df.columns: st.session_state.comp_review_df["notes"] = st.session_state.comp_review_df["notes"].fillna("").astype(str) cr1, cr2 = st.columns(2) with cr1: crs = st.text_input("🔍 Search", "", key="comp_review_search") with cr2: crsh = st.selectbox("Show", ["All","Approved","Not Approved"], key="comp_review_show") fcr = st.session_state.comp_review_df.copy() if crs: fcr = fcr[fcr["Paper Title"].str.contains(crs, case=False, na=False)] if crsh == "Approved": fcr = fcr[fcr["approve"] == True] elif crsh == "Not Approved": fcr = fcr[fcr["approve"] == False] edited = st.data_editor( fcr, use_container_width=True, height=500, num_rows="fixed", key="comp_review_editor", column_config={ "Paper Title": st.column_config.TextColumn("Paper Title", width="large", disabled=True), "Strategy Type": st.column_config.TextColumn("Type", width="small", disabled=True), "cluster_name": st.column_config.TextColumn("Cluster", width="medium", disabled=True), "approve": st.column_config.CheckboxColumn("✅", width="small", default=False), "notes": st.column_config.TextColumn("Notes", width="large"), }, column_order=["Paper Title","Strategy Type","cluster_name","approve","notes"], ) if edited is not None: for idx, erow in edited.iterrows(): mask = st.session_state.comp_review_df.index == idx if mask.any(): for col in ["approve","notes"]: if col in erow: st.session_state.comp_review_df.loc[mask, col] = erow[col] sv1, sv2 = st.columns([1, 2]) with sv1: if st.button("💾 Save Review", use_container_width=True, type="primary", key="comp_save_review"): os.makedirs(OUTPUT_DIR, exist_ok=True) st.session_state.comp_review_df.to_csv(comp_rev_path, index=False) st.markdown('

✅ Saved to outputs/comp_review_table.csv

', unsafe_allow_html=True) with sv2: appr = int(st.session_state.comp_review_df["approve"].sum()) if "approve" in st.session_state.comp_review_df.columns else 0 st.markdown(f"""

{appr}/{len(st.session_state.comp_review_df)}

Papers Approved

""", unsafe_allow_html=True) # ================================================================== # Tab 11: Research Intelligence Chatbot # ================================================================== with tab_chat: # ---- Custom CSS for chatbot ---- st.markdown(""" """, unsafe_allow_html=True) st.markdown('

🧠 Research Intelligence Chatbot

', unsafe_allow_html=True) st.markdown("Interpret clusters · Identify gaps · Optimize parameters · Analyze strategies — powered by Groq LLM") # ---- Resolve Groq key ---- chat_groq_key = groq_key if groq_key else os.environ.get("GROQ_API_KEY", "") if not chat_groq_key: st.markdown("""

🔑 API Key Required

To use the Research Intelligence Chatbot, enter a Groq API key in the sidebar by enabling the "🤖 Use LLM for Label Generation" checkbox, or set the GROQ_API_KEY environment variable.

The chatbot uses Groq's LLaMA 3.3 70B model for fast, intelligent responses grounded in your analysis data.

""", unsafe_allow_html=True) else: # ---- Build / cache context summary ---- if "chat_context" not in st.session_state: comp_d = st.session_state.get("comp_cluster_data") st.session_state.chat_context = build_context_summary(result, comp_d) # Compute context stats for right panel comp_d = st.session_state.get("comp_cluster_data") ctx_stats = get_context_stats(result, comp_d) # ---- Initialize session state ---- if "chat_messages" not in st.session_state: st.session_state.chat_messages = [] if "chat_prompt_history" not in st.session_state: st.session_state.chat_prompt_history = [] # ---- Layout: Left (prompts) | Center (chat) | Right (context) ---- chat_left, chat_center, chat_right = st.columns([1.1, 3.2, 1.1]) # ---- LEFT: Suggested prompts + history ---- with chat_left: st.markdown("#### 💡 Suggested Prompts") for i, prompt in enumerate(SUGGESTED_PROMPTS): display_label = prompt[:42] + "…" if len(prompt) > 42 else prompt if st.button(display_label, key=f"suggest_{i}", use_container_width=True): st.session_state.chat_messages.append({"role": "user", "content": prompt}) # Track in history if prompt not in st.session_state.chat_prompt_history: st.session_state.chat_prompt_history.insert(0, prompt) if len(st.session_state.chat_prompt_history) > 20: st.session_state.chat_prompt_history = st.session_state.chat_prompt_history[:20] with st.spinner("Thinking …"): reply = chat_with_groq( st.session_state.chat_messages, st.session_state.chat_context, chat_groq_key, ) st.session_state.chat_messages.append({"role": "assistant", "content": reply}) st.rerun() st.markdown("---") # Controls use_ctx = st.toggle("📎 Use analysis context", value=True, key="chat_use_ctx") ctrl_c1, ctrl_c2 = st.columns(2) with ctrl_c1: if st.button("🗑️ Clear", key="chat_clear", use_container_width=True): st.session_state.chat_messages = [] st.rerun() with ctrl_c2: if st.button("🔄 Refresh", key="chat_refresh_ctx", use_container_width=True, help="Rebuild context from latest analysis"): comp_d = st.session_state.get("comp_cluster_data") st.session_state.chat_context = build_context_summary(result, comp_d) st.rerun() # Prompt history if st.session_state.chat_prompt_history: st.markdown("---") st.markdown("#### 📜 History") for hp in st.session_state.chat_prompt_history[:8]: st.markdown( f'

{hp[:45]}

', unsafe_allow_html=True, ) # ---- CENTER: Chat interface ---- with chat_center: # Display conversation chat_container = st.container(height=540) with chat_container: if not st.session_state.chat_messages: # Welcome message st.markdown("""

👋 Welcome to the Research Intelligence Chatbot

I'm your AI research assistant, powered by your full analysis context. I can help you:

🔍 Interpret Clusters 📊 Analyze Quality ⚙️ Optimize Parameters 🔬 Find Research Gaps 📈 Identify Trends 🎯 Detect Outliers 🧬 Compare Strategies 📝 Academic Summaries

Try a suggested prompt on the left, or type your question below. Toggle "📎 Use analysis context" to control whether the chatbot references your pipeline outputs.

""", unsafe_allow_html=True) else: for msg in st.session_state.chat_messages: with st.chat_message(msg["role"]): st.markdown(msg["content"]) # Chat input user_input = st.chat_input("Ask about your research analysis …", key="chat_input") if user_input: st.session_state.chat_messages.append({"role": "user", "content": user_input}) # Track in history if user_input not in st.session_state.chat_prompt_history: st.session_state.chat_prompt_history.insert(0, user_input) if len(st.session_state.chat_prompt_history) > 20: st.session_state.chat_prompt_history = st.session_state.chat_prompt_history[:20] ctx = st.session_state.chat_context if use_ctx else "No analysis context provided — answer based on general knowledge." with st.spinner("Thinking …"): reply = chat_with_groq( st.session_state.chat_messages, ctx, chat_groq_key, ) st.session_state.chat_messages.append({"role": "assistant", "content": reply}) st.rerun() # ---- RIGHT: Live context summary ---- with chat_right: st.markdown("#### 📊 Live Context") # Pipeline stats st.markdown(f"""

🔬 Pipeline
{ctx_stats['n_clusters']} clusters · {ctx_stats['n_topics']} topics
Mapped: {ctx_stats['n_mapped']} · Novel: {ctx_stats['n_novel']}

""", unsafe_allow_html=True) # Clustering quality sil_display = f"{ctx_stats['silhouette']:.3f}" if ctx_stats.get('silhouette') else "N/A" quality_display = ctx_stats.get('quality_rating', '—') balance_display = ctx_stats.get('balance_quality', '—') quality_color = { "Excellent": "#38ef7d", "Good": "#4facfe", "Fair": "#f7971e", "Poor": "#f5576c", }.get(quality_display, "#e0e0e0") st.markdown(f"""

📐 Quality
Silhouette: {sil_display}
Rating: {quality_display}
Balance: {balance_display}

""", unsafe_allow_html=True) # Warnings / issues n_overlaps = ctx_stats.get("n_overlaps", 0) n_outliers = ctx_stats.get("n_outliers", 0) if n_overlaps > 0 or n_outliers > 0: warn_parts = [] if n_overlaps > 0: warn_parts.append(f"🔗 {n_overlaps} overlapping pairs") if n_outliers > 0: warn_parts.append(f"⚡ {n_outliers} outlier clusters") st.markdown(f"""

⚠️ Alerts
{'
'.join(warn_parts)}

""", unsafe_allow_html=True) # Comp strategies (if available) if ctx_stats.get("comp_papers", 0) > 0: st.markdown(f"""

🧮 Comp Strategies
{ctx_stats['comp_papers']} papers · {ctx_stats['comp_clusters']} clusters
Types: {ctx_stats['comp_strategy_types']}

""", unsafe_allow_html=True) # Chat session stats n_msgs = len(st.session_state.get('chat_messages', [])) n_user = sum(1 for m in st.session_state.get('chat_messages', []) if m['role'] == 'user') st.markdown(f"""

💬 Session
Messages: {n_msgs} ({n_user} questions)
Context: {'Active ✅' if use_ctx else 'Off ❌'}

""", unsafe_allow_html=True) # ---- Downloads Section ---- st.markdown('

📥 Download Outputs

', unsafe_allow_html=True) dcol1, dcol2, dcol3, dcol4 = st.columns(4) with dcol1: csv_data = result.combined_topics.to_csv(index=False) st.download_button( "⬇️ Topics Table (CSV)", data=csv_data, file_name="topics_table.csv", mime="text/csv", use_container_width=True, ) with dcol2: comp_data = result.comparison.to_csv(index=False) st.download_button( "⬇️ Comparison (CSV)", data=comp_data, file_name="comparison.csv", mime="text/csv", use_container_width=True, ) with dcol3: json_data = json.dumps(result.taxonomy_map, indent=2, ensure_ascii=False) st.download_button( "⬇️ Taxonomy Map (JSON)", data=json_data, file_name="taxonomy_map.json", mime="application/json", use_container_width=True, ) with dcol4: # Download review table if it exists review_path = os.path.join(OUTPUT_DIR, "review_table.csv") if os.path.exists(review_path): with open(review_path, "r") as f: review_data = f.read() st.download_button( "⬇️ Review Table (CSV)", data=review_data, file_name="review_table.csv", mime="text/csv", use_container_width=True, ) else: st.download_button( "⬇️ Review Table (CSV)", data="Not saved yet. Go to Review Table tab and click Save.", file_name="review_table.csv", mime="text/csv", use_container_width=True, disabled=True, ) # ---- Auto-save comparison.csv and taxonomy_map.json to outputs ---- os.makedirs(OUTPUT_DIR, exist_ok=True) result.comparison.to_csv(os.path.join(OUTPUT_DIR, "comparison.csv"), index=False) with open(os.path.join(OUTPUT_DIR, "taxonomy_map.json"), "w", encoding="utf-8") as f: json.dump(result.taxonomy_map, f, indent=2, ensure_ascii=False) # ---- Pipeline Log ---- with st.expander("📜 Pipeline Execution Log"): st.markdown(f"**Status:** `{result.status}`") st.markdown(f"**Steps Completed:** {len(result.steps_completed)}/9") for i, step in enumerate(result.steps_completed, 1): st.markdown(f" ✅ Step {i}: `{step}`") if result.errors: st.markdown("**Errors:**") for err in result.errors: st.error(err) st.markdown("**Exported Files:**") for name, path in result.exported_files.items(): st.markdown(f" 📄 `{name}` → `{path}`") else: # ---- Welcome / instructions when no results ---- st.markdown("""

👋 Welcome!

This application uses an AI agent to perform comprehensive topic modeling on research papers.

How to use:
1️⃣ Upload a CSV file with Title and Abstract columns (or use the default dataset).
2️⃣ Configure the minimum number of topics and label generation method in the sidebar.
3️⃣ Click "🚀 Run Topic Modeling Agent" to start the analysis.
4️⃣ Explore topics, comparisons, and taxonomy classification in the results tabs.
5️⃣ Review and annotate topics in the ✏️ Review Table tab.
6️⃣ View interactive charts in the 📈 Charts tab.
7️⃣ Download all outputs as CSV and JSON files.

""", unsafe_allow_html=True) st.markdown("
", unsafe_allow_html=True) # Show a preview if default dataset exists if os.path.exists("dataset.csv"): with st.expander("👀 Preview Default Dataset", expanded=False): try: preview_df = pd.read_csv("dataset.csv", nrows=10) st.markdown(f"**Columns:** {', '.join(preview_df.columns.tolist())}") if "Title" in preview_df.columns: st.dataframe(preview_df[["Title", "Abstract"]].head(10) if "Abstract" in preview_df.columns else preview_df[["Title"]].head(10), use_container_width=True) else: st.dataframe(preview_df.head(10), use_container_width=True) except Exception as e: st.warning(f"Could not preview dataset: {e}")