Spaces:
Running
Running
| def interface(): | |
| return 'Gradio UI updated' | |
| """ | |
| app.py — Streamlit frontend for the AI-driven Topic Modeling application. | |
| This module provides an interactive web interface that allows users to: | |
| 1. Upload a CSV file containing research paper Titles and Abstracts. | |
| 2. Configure pipeline parameters (min topics, LLM label generation). | |
| 3. Run the TopicAgent pipeline with a single click. | |
| 4. View and explore results: topics table, comparison, taxonomy map. | |
| 5. Review topics with an editable review table. | |
| 6. Visualize topic distributions with interactive Plotly charts. | |
| 7. Download all generated outputs (CSV, JSON). | |
| """ | |
| import os | |
| import json | |
| import tempfile | |
| import streamlit as st | |
| import pandas as pd | |
| import plotly.express as px | |
| import plotly.graph_objects as go | |
| from agents import TopicAgent | |
| from computation_clusters import run_computation_clustering | |
| from research_chatbot import build_context_summary, chat_with_groq, SUGGESTED_PROMPTS, get_context_stats | |
| # --------------------------------------------------------------------------- | |
| # HuggingFace Spaces compatibility: use a writable output directory | |
| # On HF Spaces the working directory can be read-only, so fall back to /tmp | |
| # --------------------------------------------------------------------------- | |
| OUTPUT_DIR = "outputs" | |
| try: | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| # Test write access | |
| _test_path = os.path.join(OUTPUT_DIR, ".write_test") | |
| with open(_test_path, "w") as _f: | |
| _f.write("ok") | |
| os.remove(_test_path) | |
| except (OSError, PermissionError): | |
| OUTPUT_DIR = os.path.join(tempfile.gettempdir(), "topic_modeler_outputs") | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| # --------------------------------------------------------------------------- | |
| # Page configuration | |
| # --------------------------------------------------------------------------- | |
| st.set_page_config( | |
| page_title="Research Topic Modeler — AI Agent", | |
| page_icon="🔬", | |
| layout="wide", | |
| initial_sidebar_state="expanded", | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Custom CSS for a polished, professional look with dark-safe text colors | |
| # --------------------------------------------------------------------------- | |
| st.markdown(""" | |
| <style> | |
| /* Import Google Font */ | |
| @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap'); | |
| /* Global */ | |
| html, body, [class*="css"] { | |
| font-family: 'Inter', sans-serif; | |
| } | |
| /* Header gradient banner */ | |
| .main-header { | |
| background: linear-gradient(135deg, #0f0c29 0%, #302b63 50%, #24243e 100%); | |
| padding: 2rem 2.5rem; | |
| border-radius: 16px; | |
| margin-bottom: 1.5rem; | |
| box-shadow: 0 8px 32px rgba(48, 43, 99, 0.3); | |
| } | |
| .main-header h1 { | |
| color: #ffffff; | |
| font-size: 2.2rem; | |
| font-weight: 700; | |
| margin: 0; | |
| letter-spacing: -0.5px; | |
| } | |
| .main-header p { | |
| color: #b8b5ff; | |
| font-size: 1.05rem; | |
| margin: 0.5rem 0 0 0; | |
| font-weight: 300; | |
| } | |
| /* Stat cards */ | |
| .stat-card { | |
| background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); | |
| padding: 1.25rem 1.5rem; | |
| border-radius: 12px; | |
| color: white; | |
| text-align: center; | |
| box-shadow: 0 4px 15px rgba(102, 126, 234, 0.3); | |
| transition: transform 0.2s ease; | |
| } | |
| .stat-card:hover { | |
| transform: translateY(-2px); | |
| } | |
| .stat-card .stat-value { | |
| font-size: 2rem; | |
| font-weight: 700; | |
| line-height: 1.2; | |
| color: #ffffff; | |
| } | |
| .stat-card .stat-label { | |
| font-size: 0.85rem; | |
| opacity: 0.85; | |
| margin-top: 0.3rem; | |
| font-weight: 400; | |
| color: #e8e6ff; | |
| } | |
| /* Status badge */ | |
| .status-badge { | |
| display: inline-block; | |
| padding: 0.3rem 1rem; | |
| border-radius: 20px; | |
| font-size: 0.8rem; | |
| font-weight: 600; | |
| text-transform: uppercase; | |
| letter-spacing: 0.5px; | |
| } | |
| .status-success { | |
| background: linear-gradient(135deg, #11998e, #38ef7d); | |
| color: #ffffff; | |
| } | |
| .status-failed { | |
| background: linear-gradient(135deg, #eb3349, #f45c43); | |
| color: #ffffff; | |
| } | |
| .status-running { | |
| background: linear-gradient(135deg, #f7971e, #ffd200); | |
| color: #1a1a2e; | |
| } | |
| /* Section headers — always readable on both light and dark backgrounds */ | |
| .section-header { | |
| font-size: 1.3rem; | |
| font-weight: 600; | |
| color: #c4b5fd; | |
| margin: 1.5rem 0 0.75rem 0; | |
| padding-bottom: 0.5rem; | |
| border-bottom: 2px solid #667eea; | |
| display: inline-block; | |
| } | |
| /* Taxonomy badges */ | |
| .mapped-badge { | |
| display: inline-block; | |
| background: linear-gradient(135deg, #11998e, #38ef7d); | |
| color: #ffffff; | |
| padding: 0.2rem 0.7rem; | |
| border-radius: 12px; | |
| font-size: 0.75rem; | |
| font-weight: 600; | |
| } | |
| .novel-badge { | |
| display: inline-block; | |
| background: linear-gradient(135deg, #fc4a1a, #f7b733); | |
| color: #ffffff; | |
| padding: 0.2rem 0.7rem; | |
| border-radius: 12px; | |
| font-size: 0.75rem; | |
| font-weight: 600; | |
| } | |
| /* Sidebar styling */ | |
| section[data-testid="stSidebar"] { | |
| background: linear-gradient(180deg, #1a1a2e 0%, #16213e 100%); | |
| } | |
| section[data-testid="stSidebar"] .stMarkdown { | |
| color: #e0e0e0; | |
| } | |
| section[data-testid="stSidebar"] label { | |
| color: #e0e0e0 !important; | |
| } | |
| section[data-testid="stSidebar"] .stSlider label { | |
| color: #e0e0e0 !important; | |
| } | |
| /* Data table enhancements */ | |
| .stDataFrame { | |
| border-radius: 8px; | |
| overflow: hidden; | |
| } | |
| /* Info box — dark-safe: dark background with light text */ | |
| .info-box { | |
| background: linear-gradient(135deg, #1e1e3f 0%, #2d2b55 100%); | |
| padding: 1rem 1.5rem; | |
| border-radius: 10px; | |
| border-left: 4px solid #667eea; | |
| margin: 0.75rem 0; | |
| color: #e0e0e0; | |
| } | |
| .info-box strong { | |
| color: #ffffff; | |
| } | |
| .info-box code { | |
| background: rgba(102, 126, 234, 0.2); | |
| color: #b8b5ff; | |
| padding: 0.1rem 0.4rem; | |
| border-radius: 4px; | |
| } | |
| /* Pipeline step */ | |
| .step-item { | |
| padding: 0.5rem 1rem; | |
| margin: 0.3rem 0; | |
| border-radius: 8px; | |
| background: rgba(102, 126, 234, 0.15); | |
| border-left: 3px solid #667eea; | |
| font-size: 0.9rem; | |
| color: #e0e0e0; | |
| } | |
| /* Chart container styling */ | |
| .chart-container { | |
| background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); | |
| border-radius: 12px; | |
| padding: 1rem; | |
| margin: 0.5rem 0; | |
| box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2); | |
| } | |
| /* Review section header */ | |
| .review-header { | |
| background: linear-gradient(135deg, #11998e 0%, #38ef7d 100%); | |
| padding: 1rem 1.5rem; | |
| border-radius: 12px; | |
| margin-bottom: 1rem; | |
| box-shadow: 0 4px 15px rgba(17, 153, 142, 0.3); | |
| } | |
| .review-header h3 { | |
| color: #ffffff; | |
| margin: 0; | |
| font-weight: 600; | |
| } | |
| .review-header p { | |
| color: #e0fff8; | |
| margin: 0.3rem 0 0 0; | |
| font-size: 0.9rem; | |
| } | |
| /* Save confirmation */ | |
| .save-confirm { | |
| background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); | |
| color: #ffffff; | |
| padding: 0.75rem 1.25rem; | |
| border-radius: 10px; | |
| margin-top: 0.5rem; | |
| font-weight: 500; | |
| } | |
| /* Ensure tab labels are readable */ | |
| .stTabs [data-baseweb="tab-list"] button { | |
| color: #c4b5fd; | |
| } | |
| .stTabs [data-baseweb="tab-list"] button[aria-selected="true"] { | |
| color: #ffffff; | |
| } | |
| </style> | |
| """, unsafe_allow_html=True) | |
| # --------------------------------------------------------------------------- | |
| # Header | |
| # --------------------------------------------------------------------------- | |
| st.markdown(""" | |
| <div class="main-header"> | |
| <h1>🔬 Research Topic Modeler</h1> | |
| <p>AI-powered topic modeling agent for research papers — discover, compare, and classify themes across Titles and Abstracts</p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # --------------------------------------------------------------------------- | |
| # Sidebar — Configuration | |
| # --------------------------------------------------------------------------- | |
| with st.sidebar: | |
| st.markdown("## ⚙️ Configuration") | |
| st.markdown("---") | |
| # File upload | |
| st.markdown("### 📁 Dataset") | |
| uploaded_file = st.file_uploader( | |
| "Upload CSV with Title & Abstract columns", | |
| type=["csv"], | |
| help="The CSV must contain at least 'Title' and 'Abstract' columns.", | |
| ) | |
| # Or use default dataset | |
| use_default = st.checkbox( | |
| "Use default dataset (dataset.csv)", | |
| value=True if not uploaded_file else False, | |
| help="Use the bundled dataset.csv file in the project directory.", | |
| ) | |
| st.markdown("---") | |
| st.markdown("### 🎯 Parameters") | |
| min_topics = st.slider( | |
| "Minimum Topics", | |
| min_value=50, | |
| max_value=300, | |
| value=100, | |
| step=10, | |
| help="Minimum number of fine-grained topics to generate per source. Topics are sub-divisions within clusters.", | |
| ) | |
| max_clusters = st.slider( | |
| "Max Clusters", | |
| min_value=10, | |
| max_value=40, | |
| value=25, | |
| step=5, | |
| help="Target maximum number of high-level thematic clusters (10–40 range). Topics are extracted within each cluster.", | |
| ) | |
| use_llm = st.checkbox( | |
| "🤖 Use LLM for Label Generation (Groq)", | |
| value=False, | |
| help="Use Groq's LLaMA model to generate contextual topic labels. " | |
| "Falls back to keyword heuristic if unchecked.", | |
| ) | |
| groq_key = os.environ.get("GROQ_API_KEY", "") | |
| if use_llm: | |
| groq_key = st.text_input( | |
| "Groq API Key", | |
| value=groq_key, | |
| type="password", | |
| help="Your Groq API key for LLM label generation.", | |
| ) | |
| st.markdown("---") | |
| st.markdown("### 📋 Pipeline Steps (SPECTER2 + HDBSCAN)") | |
| steps_info = [ | |
| "1. Load & validate CSV", | |
| "2. Generate SPECTER2 embeddings", | |
| "3. HDBSCAN clustering (10–{} clusters)".format(max_clusters), | |
| "4. Extract cluster keywords (c-TF-IDF)", | |
| "5. Merge similar clusters", | |
| "6. Extract sub-topics via NMF (≥{} topics)".format(min_topics), | |
| "7. AI Council label generation", | |
| "8. Build taxonomy map", | |
| "9. Export outputs (CSV, JSON)", | |
| ] | |
| for step in steps_info: | |
| st.markdown(f'<div class="step-item">{step}</div>', unsafe_allow_html=True) | |
| # --------------------------------------------------------------------------- | |
| # Main area — Run button and results | |
| # --------------------------------------------------------------------------- | |
| col_run, col_status = st.columns([2, 3]) | |
| with col_run: | |
| run_clicked = st.button("🚀 Run Topic Modeling Agent", use_container_width=True, type="primary") | |
| with col_status: | |
| if "result" in st.session_state and st.session_state.result is not None: | |
| res = st.session_state.result | |
| if res.status == "success": | |
| st.markdown('<span class="status-badge status-success">✓ Pipeline Complete</span>', unsafe_allow_html=True) | |
| elif res.status == "failed": | |
| st.markdown('<span class="status-badge status-failed">✗ Pipeline Failed</span>', unsafe_allow_html=True) | |
| else: | |
| st.markdown('<span class="status-badge status-running">● Awaiting Input</span>', unsafe_allow_html=True) | |
| # --------------------------------------------------------------------------- | |
| # Execute pipeline | |
| # --------------------------------------------------------------------------- | |
| if run_clicked: | |
| # Determine CSV path | |
| csv_path = None | |
| if uploaded_file is not None: | |
| # Save uploaded file to a temp location | |
| with tempfile.NamedTemporaryFile(delete=False, suffix=".csv", dir=".") as tmp: | |
| tmp.write(uploaded_file.getvalue()) | |
| csv_path = tmp.name | |
| elif use_default: | |
| csv_path = "dataset.csv" | |
| if not os.path.exists(csv_path): | |
| st.error("❌ Default dataset.csv not found in the project directory.") | |
| st.stop() | |
| else: | |
| st.error("❌ Please upload a CSV file or select the default dataset.") | |
| st.stop() | |
| # Run the agent | |
| with st.spinner("🔄 Running the Topic Modeling Agent … this may take a few minutes."): | |
| progress = st.progress(0, text="Initializing …") | |
| agent = TopicAgent( | |
| csv_path=csv_path, | |
| output_dir=OUTPUT_DIR, | |
| min_topics=min_topics, | |
| max_clusters=max_clusters, | |
| use_llm_labels=use_llm, | |
| use_specter=True, | |
| groq_api_key=groq_key if use_llm else None, | |
| ) | |
| # Display step-by-step progress — SPECTER2 + HDBSCAN pipeline | |
| progress.progress(5, text="Step 1/9: Loading CSV …") | |
| agent._step_load_csv() | |
| progress.progress(10, text="Step 2/9: Generating SPECTER2 embeddings …") | |
| agent._step_specter_embed() | |
| progress.progress(30, text="Step 3/9: HDBSCAN clustering …") | |
| agent._step_specter_cluster() | |
| progress.progress(45, text="Step 4/9: Extracting cluster keywords …") | |
| agent._step_specter_extract() | |
| progress.progress(55, text="Step 5/9: Merging similar clusters …") | |
| agent._step_merge_similar_clusters() | |
| progress.progress(65, text="Step 6/9: Extracting sub-topics (≥{} topics) …".format(min_topics)) | |
| agent._step_specter_subtopics() | |
| progress.progress(80, text="Step 7/9: Generating topic labels …") | |
| agent._step_specter_label() | |
| progress.progress(90, text="Step 8/9: Building taxonomy map …") | |
| agent._step_specter_taxonomy() | |
| progress.progress(95, text="Step 9/9: Exporting outputs …") | |
| agent._step_export() | |
| agent._result.status = "success" | |
| progress.progress(100, text="✅ Pipeline complete!") | |
| st.session_state.result = agent._result | |
| # Clean up temp file | |
| if uploaded_file is not None and csv_path and os.path.exists(csv_path): | |
| try: | |
| os.unlink(csv_path) | |
| except Exception: | |
| pass | |
| st.rerun() | |
| # --------------------------------------------------------------------------- | |
| # Helper: Plotly chart theme (dark background, readable text) | |
| # --------------------------------------------------------------------------- | |
| PLOTLY_LAYOUT = dict( | |
| paper_bgcolor="rgba(26, 26, 46, 0.95)", | |
| plot_bgcolor="rgba(22, 33, 62, 0.95)", | |
| font=dict(family="Inter, sans-serif", size=13, color="#e0e0e0"), | |
| title_font=dict(size=18, color="#ffffff"), | |
| legend=dict( | |
| font=dict(color="#e0e0e0"), | |
| bgcolor="rgba(26, 26, 46, 0.7)", | |
| bordercolor="#667eea", | |
| borderwidth=1, | |
| ), | |
| xaxis=dict( | |
| gridcolor="rgba(102, 126, 234, 0.15)", | |
| zerolinecolor="rgba(102, 126, 234, 0.25)", | |
| tickfont=dict(color="#c4b5fd"), | |
| title_font=dict(color="#e0e0e0"), | |
| ), | |
| yaxis=dict( | |
| gridcolor="rgba(102, 126, 234, 0.15)", | |
| zerolinecolor="rgba(102, 126, 234, 0.25)", | |
| tickfont=dict(color="#c4b5fd"), | |
| title_font=dict(color="#e0e0e0"), | |
| ), | |
| margin=dict(l=60, r=30, t=60, b=60), | |
| ) | |
| # Gradient-like color sequence | |
| CHART_COLORS = [ | |
| "#667eea", "#764ba2", "#f093fb", "#f5576c", | |
| "#4facfe", "#00f2fe", "#43e97b", "#38f9d7", | |
| "#fa709a", "#fee140", "#a18cd1", "#fbc2eb", | |
| "#ff9a9e", "#fad0c4", "#ffecd2", "#fcb69f", | |
| ] | |
| # --------------------------------------------------------------------------- | |
| # Display results | |
| # --------------------------------------------------------------------------- | |
| if "result" in st.session_state and st.session_state.result is not None: | |
| result = st.session_state.result | |
| if result.status == "failed": | |
| st.error(f"Pipeline failed with errors: {result.errors}") | |
| st.stop() | |
| # ---- Summary Statistics ---- | |
| st.markdown('<div class="section-header">📊 Summary Statistics</div>', unsafe_allow_html=True) | |
| # Calculate cluster count from combined_topics | |
| has_clusters = "cluster_id" in result.combined_topics.columns | |
| n_clusters = int(result.combined_topics["cluster_id"].nunique()) if has_clusters else 0 | |
| n_topics = len(result.combined_topics) | |
| c1, c2, c3, c4, c5, c6 = st.columns(6) | |
| with c1: | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{n_clusters}</div> | |
| <div class="stat-label">Clusters</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with c2: | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{n_topics}</div> | |
| <div class="stat-label">Total Topics</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with c3: | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{len(result.title_topics)}</div> | |
| <div class="stat-label">Title Clusters</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with c4: | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{len(result.abstract_topics)}</div> | |
| <div class="stat-label">Abstract Clusters</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with c5: | |
| mapped_count = result.taxonomy_map.get("metadata", {}).get("mapped_count", 0) | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{mapped_count}</div> | |
| <div class="stat-label">Mapped Themes</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with c6: | |
| novel_count = result.taxonomy_map.get("metadata", {}).get("novel_count", 0) | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{novel_count}</div> | |
| <div class="stat-label">Novel Themes</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| st.markdown("<br>", unsafe_allow_html=True) | |
| # ---- Tabbed Results ---- | |
| tab1, tab_umap, tab_explorer, tab_network, tab_council, tab4, tab5, tab_review, tab_charts, tab_comp, tab_chat = st.tabs([ | |
| "📋 Topics Table", | |
| "🗺️ UMAP Cluster Map", | |
| "🔍 Cluster Explorer", | |
| "🕸️ Cluster Network", | |
| "🤖 AI Council", | |
| "⚖️ Theme Comparison", | |
| "🗺️ Taxonomy Map", | |
| "✏️ Review Table", | |
| "📈 Charts", | |
| "🧮 Comp Strategy Map", | |
| "🧠 Research Chatbot", | |
| ]) | |
| # Tab 1: Combined Topics Table (with cluster hierarchy) | |
| with tab1: | |
| st.markdown('<div class="section-header">Combined Topics Table</div>', unsafe_allow_html=True) | |
| has_cluster_col = "cluster_id" in result.combined_topics.columns | |
| if has_cluster_col: | |
| n_cl = int(result.combined_topics["cluster_id"].nunique()) | |
| st.markdown( | |
| f"Showing all **{len(result.combined_topics)}** topics grouped into **{n_cl}** clusters." | |
| ) | |
| else: | |
| st.markdown(f"Showing all **{len(result.combined_topics)}** topics.") | |
| # Filter controls | |
| fcol1, fcol2, fcol3 = st.columns(3) | |
| with fcol1: | |
| if "source" in result.combined_topics.columns: | |
| source_filter = st.multiselect( | |
| "Filter by Source", | |
| options=result.combined_topics["source"].unique().tolist(), | |
| default=result.combined_topics["source"].unique().tolist(), | |
| ) | |
| else: | |
| source_filter = [] | |
| with fcol2: | |
| search_term = st.text_input("🔍 Search keywords", "") | |
| with fcol3: | |
| if has_cluster_col: | |
| cluster_options = sorted(result.combined_topics["cluster_id"].unique().tolist()) | |
| cluster_filter = st.multiselect( | |
| "Filter by Cluster", | |
| options=cluster_options, | |
| default=cluster_options, | |
| key="topics_cluster_filter", | |
| ) | |
| else: | |
| cluster_filter = [] | |
| display_df = result.combined_topics.copy() | |
| if source_filter and "source" in display_df.columns: | |
| display_df = display_df[display_df["source"].isin(source_filter)] | |
| if cluster_filter and has_cluster_col: | |
| display_df = display_df[display_df["cluster_id"].isin(cluster_filter)] | |
| if search_term: | |
| mask = display_df["keywords"].str.contains(search_term, case=False, na=False) | |
| if "label" in display_df.columns: | |
| mask |= display_df["label"].str.contains(search_term, case=False, na=False) | |
| if "cluster_label" in display_df.columns: | |
| mask |= display_df["cluster_label"].str.contains(search_term, case=False, na=False) | |
| display_df = display_df[mask] | |
| # Build column config dynamically | |
| col_config = { | |
| "topic_id": st.column_config.NumberColumn("Topic ID", width="small"), | |
| "keywords": st.column_config.TextColumn("Keywords", width="large"), | |
| } | |
| if "label" in display_df.columns: | |
| col_config["label"] = st.column_config.TextColumn("Label", width="medium") | |
| if has_cluster_col: | |
| col_config["cluster_id"] = st.column_config.NumberColumn("Cluster", width="small") | |
| if "cluster_label" in display_df.columns: | |
| col_config["cluster_label"] = st.column_config.TextColumn("Cluster Label", width="medium") | |
| if "source" in display_df.columns: | |
| col_config["source"] = st.column_config.TextColumn("Source", width="small") | |
| if "n_documents" in display_df.columns: | |
| col_config["n_documents"] = st.column_config.NumberColumn("Docs", width="small") | |
| st.dataframe( | |
| display_df, | |
| use_container_width=True, | |
| height=500, | |
| column_config=col_config, | |
| ) | |
| # ================================================================== | |
| # Tab: UMAP 2D Cluster Map | |
| # ================================================================== | |
| with tab_umap: | |
| st.markdown('<div class="section-header">🗺️ UMAP 2D Cluster Map</div>', unsafe_allow_html=True) | |
| st.markdown("Interactive 2D visualization of document embeddings colored by HDBSCAN cluster assignment.") | |
| embeddings_2d = result.cluster_data.get("embeddings_2d") | |
| cluster_labels_list = result.cluster_data.get("cluster_labels") | |
| if embeddings_2d and cluster_labels_list: | |
| import numpy as np | |
| emb_arr = np.array(embeddings_2d) | |
| labels_arr = np.array(cluster_labels_list) | |
| umap_df = pd.DataFrame({ | |
| "UMAP_1": emb_arr[:, 0], | |
| "UMAP_2": emb_arr[:, 1], | |
| "Cluster": labels_arr.astype(str), | |
| }) | |
| # Mark noise points | |
| umap_df.loc[umap_df["Cluster"] == "-1", "Cluster"] = "Noise" | |
| fig_umap = px.scatter( | |
| umap_df, | |
| x="UMAP_1", | |
| y="UMAP_2", | |
| color="Cluster", | |
| opacity=0.7, | |
| hover_data=["Cluster"], | |
| color_discrete_sequence=px.colors.qualitative.Set2 + px.colors.qualitative.Dark24, | |
| ) | |
| fig_umap.update_traces( | |
| marker=dict(size=5, line=dict(width=0.3, color="#ffffff")), | |
| ) | |
| fig_umap.update_layout( | |
| plot_bgcolor="#1a1a2e", | |
| paper_bgcolor="#1a1a2e", | |
| font=dict(color="#e0e0e0", family="Inter"), | |
| title=f"UMAP 2D Projection — {int(labels_arr.max()) + 1} Clusters", | |
| xaxis=dict(showgrid=False, zeroline=False, title="UMAP Dimension 1", color="#c4b5fd"), | |
| yaxis=dict(showgrid=False, zeroline=False, title="UMAP Dimension 2", color="#c4b5fd"), | |
| legend_title="Cluster", | |
| height=650, | |
| margin=dict(l=40, r=40, t=60, b=40), | |
| ) | |
| st.plotly_chart(fig_umap, use_container_width=True) | |
| # Cluster size summary | |
| cluster_counts = umap_df["Cluster"].value_counts().reset_index() | |
| cluster_counts.columns = ["Cluster", "Documents"] | |
| cluster_counts = cluster_counts[cluster_counts["Cluster"] != "Noise"] | |
| st.markdown("#### Cluster Sizes") | |
| st.dataframe(cluster_counts, use_container_width=True, height=300) | |
| else: | |
| st.info("🔍 No 2D embedding data available. Run the SPECTER2 + HDBSCAN pipeline first.") | |
| # ================================================================== | |
| # Tab: Cluster Explorer (keywords, representative sentences, details) | |
| # ================================================================== | |
| with tab_explorer: | |
| st.markdown('<div class="section-header">🔍 Cluster Explorer</div>', unsafe_allow_html=True) | |
| st.markdown("Deep-dive into each cluster: see keywords, top representative papers, and document counts.") | |
| cluster_details = result.cluster_data.get("cluster_details", {}) | |
| if cluster_details: | |
| cluster_ids = sorted(cluster_details.keys()) | |
| # Cluster selector | |
| selected_cluster = st.selectbox( | |
| "Select a Cluster to Explore", | |
| options=cluster_ids, | |
| format_func=lambda cid: f"Cluster {cid} — {cluster_details[cid].get('keywords', '')[:60]}…", | |
| key="cluster_explorer_select", | |
| ) | |
| if selected_cluster is not None: | |
| detail = cluster_details[selected_cluster] | |
| # Cluster info card | |
| st.markdown(f""" | |
| <div class="info-box"> | |
| <strong>Cluster {selected_cluster}</strong><br> | |
| <strong>Documents:</strong> {detail.get('n_documents', 'N/A')}<br> | |
| <strong>Keywords:</strong> {detail.get('keywords', 'N/A')} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Representative papers | |
| reps = detail.get("representatives", []) | |
| if reps: | |
| st.markdown("#### 📄 Top Representative Papers (closest to cluster centroid)") | |
| for i, rep in enumerate(reps, 1): | |
| st.markdown(f""" | |
| <div style="background: rgba(102,126,234,0.1); border-left: 3px solid #667eea; | |
| padding: 0.6rem 1rem; margin: 0.3rem 0; border-radius: 6px; | |
| color: #e0e0e0; font-size: 0.9rem;"> | |
| <strong>#{i}</strong> {rep} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| else: | |
| st.info("No representative sentences available for this cluster.") | |
| # Sub-topics within this cluster | |
| if has_cluster_col: | |
| cluster_topics = result.combined_topics[ | |
| result.combined_topics["cluster_id"] == selected_cluster | |
| ] | |
| if not cluster_topics.empty: | |
| st.markdown(f"#### 🧬 Sub-Topics in Cluster {selected_cluster} ({len(cluster_topics)} topics)") | |
| st.dataframe( | |
| cluster_topics[["topic_id", "keywords", "label", "n_documents"]].reset_index(drop=True) | |
| if "label" in cluster_topics.columns | |
| else cluster_topics[["topic_id", "keywords", "n_documents"]].reset_index(drop=True), | |
| use_container_width=True, | |
| height=300, | |
| ) | |
| # All clusters overview | |
| st.markdown("---") | |
| st.markdown("#### 📊 All Clusters Overview") | |
| overview_rows = [] | |
| for cid in cluster_ids: | |
| d = cluster_details[cid] | |
| overview_rows.append({ | |
| "Cluster": cid, | |
| "Documents": d.get("n_documents", 0), | |
| "Keywords": d.get("keywords", "")[:80], | |
| "Top Paper": (d.get("representatives", ["—"])[0])[:80] if d.get("representatives") else "—", | |
| }) | |
| st.dataframe( | |
| pd.DataFrame(overview_rows), | |
| use_container_width=True, | |
| height=400, | |
| column_config={ | |
| "Cluster": st.column_config.NumberColumn("Cluster", width="small"), | |
| "Documents": st.column_config.NumberColumn("Docs", width="small"), | |
| "Keywords": st.column_config.TextColumn("Top Keywords", width="large"), | |
| "Top Paper": st.column_config.TextColumn("Representative Paper", width="large"), | |
| }, | |
| ) | |
| else: | |
| st.info("🔍 No cluster details available. Run the SPECTER2 + HDBSCAN pipeline first.") | |
| # ================================================================== | |
| # Tab: Cluster Similarity Network | |
| # ================================================================== | |
| with tab_network: | |
| st.markdown('<div class="section-header">🕸️ Cluster Similarity Network</div>', unsafe_allow_html=True) | |
| st.markdown("Graph showing cosine similarity between cluster centroids. Stronger connections = more similar clusters.") | |
| graph_edges = result.cluster_data.get("graph_edges", []) | |
| cluster_details_net = result.cluster_data.get("cluster_details", {}) | |
| if graph_edges: | |
| # Build network graph using plotly | |
| import numpy as np | |
| # Get unique cluster IDs from edges | |
| cluster_ids_net = sorted(set( | |
| [e["source"] for e in graph_edges] + [e["target"] for e in graph_edges] | |
| )) | |
| # Arrange clusters in a circle layout | |
| n_nodes = len(cluster_ids_net) | |
| angles = np.linspace(0, 2 * np.pi, n_nodes, endpoint=False) | |
| node_x = {cid: float(np.cos(a)) for cid, a in zip(cluster_ids_net, angles)} | |
| node_y = {cid: float(np.sin(a)) for cid, a in zip(cluster_ids_net, angles)} | |
| # Similarity threshold slider | |
| min_sim = st.slider( | |
| "Minimum Similarity Threshold", | |
| min_value=0.0, max_value=1.0, value=0.3, step=0.05, | |
| help="Only show edges with similarity above this threshold.", | |
| key="network_sim_threshold", | |
| ) | |
| filtered_edges = [e for e in graph_edges if e["weight"] >= min_sim] | |
| # Edge traces | |
| edge_x, edge_y, edge_text = [], [], [] | |
| for e in filtered_edges: | |
| x0, y0 = node_x[e["source"]], node_y[e["source"]] | |
| x1, y1 = node_x[e["target"]], node_y[e["target"]] | |
| edge_x.extend([x0, x1, None]) | |
| edge_y.extend([y0, y1, None]) | |
| fig_net = go.Figure() | |
| # Add edges | |
| fig_net.add_trace(go.Scatter( | |
| x=edge_x, y=edge_y, | |
| mode="lines", | |
| line=dict(width=1, color="rgba(102,126,234,0.4)"), | |
| hoverinfo="none", | |
| showlegend=False, | |
| )) | |
| # Node sizes proportional to document count | |
| node_sizes = [] | |
| node_labels = [] | |
| node_hover = [] | |
| for cid in cluster_ids_net: | |
| detail = cluster_details_net.get(cid, {}) | |
| n_docs = detail.get("n_documents", 10) | |
| kw = detail.get("keywords", "")[:50] | |
| node_sizes.append(max(15, min(50, n_docs * 2))) | |
| node_labels.append(f"C{cid}") | |
| node_hover.append(f"Cluster {cid}<br>Docs: {n_docs}<br>Keywords: {kw}") | |
| # Add nodes | |
| fig_net.add_trace(go.Scatter( | |
| x=[node_x[c] for c in cluster_ids_net], | |
| y=[node_y[c] for c in cluster_ids_net], | |
| mode="markers+text", | |
| marker=dict( | |
| size=node_sizes, | |
| color=[i for i in range(n_nodes)], | |
| colorscale="Viridis", | |
| line=dict(width=1.5, color="#ffffff"), | |
| ), | |
| text=node_labels, | |
| textposition="top center", | |
| textfont=dict(color="#e0e0e0", size=11, family="Inter"), | |
| hovertext=node_hover, | |
| hoverinfo="text", | |
| showlegend=False, | |
| )) | |
| fig_net.update_layout( | |
| plot_bgcolor="#1a1a2e", | |
| paper_bgcolor="#1a1a2e", | |
| font=dict(color="#e0e0e0", family="Inter"), | |
| title=f"Cluster Similarity Network ({len(filtered_edges)} edges, threshold ≥ {min_sim:.2f})", | |
| xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), | |
| yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), | |
| height=600, | |
| margin=dict(l=20, r=20, t=60, b=20), | |
| ) | |
| st.plotly_chart(fig_net, use_container_width=True) | |
| # Edge table | |
| with st.expander("📋 Similarity Scores (Edge List)"): | |
| edge_df = pd.DataFrame(filtered_edges) | |
| edge_df.columns = ["Source Cluster", "Target Cluster", "Similarity"] | |
| st.dataframe(edge_df, use_container_width=True, height=300) | |
| else: | |
| st.info("🔍 No cluster graph data available. Run the SPECTER2 + HDBSCAN pipeline first.") | |
| # ================================================================== | |
| # Tab: AI Council Debates (label generation transparency) | |
| # ================================================================== | |
| with tab_council: | |
| st.markdown('<div class="section-header">🤖 AI Council — Label Generation Debates</div>', unsafe_allow_html=True) | |
| st.markdown( | |
| "Transparency into how topic labels were generated. " | |
| "Each cluster's label is debated by three AI personas: " | |
| "**Domain Expert**, **Data Analyst**, and **Anti-Hallucination Judge**." | |
| ) | |
| debates = result.council_debates | |
| if debates: | |
| debate_ids = sorted(debates.keys()) | |
| st.markdown(f"**{len(debate_ids)}** clusters labeled via AI Council debate.") | |
| # Summary table | |
| debate_rows = [] | |
| for cid in debate_ids: | |
| d = debates[cid] | |
| debate_rows.append({ | |
| "Cluster": cid, | |
| "Expert Label": d.get("expert", "—"), | |
| "Analyst Label": d.get("analyst", "—"), | |
| "Anti-Hallucination": d.get("anti_hallucination", d.get("final_label", "—")), | |
| "Final Label": d.get("final_label", "—"), | |
| }) | |
| st.dataframe( | |
| pd.DataFrame(debate_rows), | |
| use_container_width=True, | |
| height=400, | |
| column_config={ | |
| "Cluster": st.column_config.NumberColumn("Cluster", width="small"), | |
| "Expert Label": st.column_config.TextColumn("🎓 Domain Expert", width="medium"), | |
| "Analyst Label": st.column_config.TextColumn("📊 Data Analyst", width="medium"), | |
| "Anti-Hallucination": st.column_config.TextColumn("🛡️ Hallucination Check", width="medium"), | |
| "Final Label": st.column_config.TextColumn("⚖️ Final (Judge)", width="medium"), | |
| }, | |
| ) | |
| # Detailed debate view | |
| st.markdown("---") | |
| st.markdown("#### 🔎 Detailed Debate View") | |
| selected_debate = st.selectbox( | |
| "Select cluster to inspect", | |
| options=debate_ids, | |
| format_func=lambda c: f"Cluster {c} — {debates[c].get('final_label', '')}", | |
| key="council_debate_select", | |
| ) | |
| if selected_debate is not None: | |
| d = debates[selected_debate] | |
| col_e, col_a, col_h, col_j = st.columns(4) | |
| with col_e: | |
| st.markdown(""" | |
| <div style="background: linear-gradient(135deg, #667eea, #764ba2); | |
| padding: 1rem; border-radius: 10px; color: white; text-align: center;"> | |
| <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">🎓 Domain Expert</div> | |
| <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div> | |
| </div> | |
| """.format(d.get("expert", "—")), unsafe_allow_html=True) | |
| with col_a: | |
| st.markdown(""" | |
| <div style="background: linear-gradient(135deg, #11998e, #38ef7d); | |
| padding: 1rem; border-radius: 10px; color: white; text-align: center;"> | |
| <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">📊 Data Analyst</div> | |
| <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div> | |
| </div> | |
| """.format(d.get("analyst", "—")), unsafe_allow_html=True) | |
| with col_h: | |
| st.markdown(""" | |
| <div style="background: linear-gradient(135deg, #e44d26, #f16529); | |
| padding: 1rem; border-radius: 10px; color: white; text-align: center;"> | |
| <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">🛡️ Anti-Hallucination</div> | |
| <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div> | |
| </div> | |
| """.format(d.get("anti_hallucination", d.get("final_label", "—"))), unsafe_allow_html=True) | |
| with col_j: | |
| st.markdown(""" | |
| <div style="background: linear-gradient(135deg, #fc4a1a, #f7b733); | |
| padding: 1rem; border-radius: 10px; color: white; text-align: center;"> | |
| <div style="font-size: 0.7rem; opacity: 0.85; text-transform: uppercase; letter-spacing: 1px;">⚖️ Final Verdict</div> | |
| <div style="font-size: 1.1rem; font-weight: 600; margin-top: 0.5rem;">{}</div> | |
| </div> | |
| """.format(d.get("final_label", "—")), unsafe_allow_html=True) | |
| else: | |
| st.markdown(""" | |
| <div class="info-box"> | |
| <strong>ℹ️ AI Council Debates Not Available</strong><br><br> | |
| To see label generation debates, enable <strong>"🤖 Use LLM for Label Generation (Groq)"</strong> | |
| in the sidebar and provide a Groq API key before running the pipeline. | |
| <br><br> | |
| When enabled, each cluster label is generated through a 3-persona debate: | |
| <br>• <strong>Domain Expert</strong> proposes an academic label | |
| <br>• <strong>Data Analyst</strong> proposes a categorical name | |
| <br>• <strong>Anti-Hallucination Judge</strong> synthesizes the final label | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Tab 4: Theme Comparison | |
| with tab4: | |
| st.markdown('<div class="section-header">Theme Comparison</div>', unsafe_allow_html=True) | |
| if not result.comparison.empty: | |
| # Standard comparison view (title vs abstract) | |
| st.markdown("**Titles vs Abstracts Theme Alignment**") | |
| align_counts = result.comparison["alignment"].value_counts() | |
| acol1, acol2, acol3, acol4 = st.columns(4) | |
| for col, alignment in zip( | |
| [acol1, acol2, acol3, acol4], | |
| ["Strong", "Moderate", "Weak", "No Match"], | |
| ): | |
| with col: | |
| count = align_counts.get(alignment, 0) | |
| st.metric(label=f"{alignment} Alignment", value=count) | |
| st.markdown("<br>", unsafe_allow_html=True) | |
| alignment_filter = st.multiselect( | |
| "Filter by Alignment", | |
| options=["Strong", "Moderate", "Weak", "No Match"], | |
| default=["Strong", "Moderate", "Weak", "No Match"], | |
| ) | |
| filtered_comp = result.comparison[result.comparison["alignment"].isin(alignment_filter)] | |
| st.dataframe( | |
| filtered_comp, | |
| use_container_width=True, | |
| height=500, | |
| column_config={ | |
| "similarity": st.column_config.ProgressColumn( | |
| "Similarity", | |
| min_value=0, | |
| max_value=1, | |
| format="%.2f", | |
| ), | |
| }, | |
| ) | |
| elif has_cluster_col and "cluster_id" in result.combined_topics.columns: | |
| # SPECTER mode — show inter-cluster similarity comparison | |
| st.markdown("**Cluster-to-Cluster Theme Similarity**") | |
| st.markdown("Comparing keyword overlap between each pair of clusters.") | |
| graph_edges_comp = result.cluster_data.get("graph_edges", []) | |
| cluster_details_comp = result.cluster_data.get("cluster_details", {}) | |
| if graph_edges_comp: | |
| # Build comparison table from graph edges | |
| comp_rows = [] | |
| for e in graph_edges_comp: | |
| src = e["source"] | |
| tgt = e["target"] | |
| src_detail = cluster_details_comp.get(src, {}) | |
| tgt_detail = cluster_details_comp.get(tgt, {}) | |
| sim = e["weight"] | |
| alignment = ( | |
| "Strong" if sim >= 0.6 | |
| else "Moderate" if sim >= 0.4 | |
| else "Weak" if sim > 0.2 | |
| else "Distant" | |
| ) | |
| comp_rows.append({ | |
| "cluster_a": src, | |
| "cluster_a_keywords": src_detail.get("keywords", "")[:60], | |
| "cluster_b": tgt, | |
| "cluster_b_keywords": tgt_detail.get("keywords", "")[:60], | |
| "similarity": round(sim, 4), | |
| "alignment": alignment, | |
| }) | |
| comp_df = pd.DataFrame(comp_rows) | |
| # Alignment stats | |
| align_counts = comp_df["alignment"].value_counts() | |
| acol1, acol2, acol3, acol4 = st.columns(4) | |
| for col, alignment in zip( | |
| [acol1, acol2, acol3, acol4], | |
| ["Strong", "Moderate", "Weak", "Distant"], | |
| ): | |
| with col: | |
| count = align_counts.get(alignment, 0) | |
| st.metric(label=f"{alignment}", value=count) | |
| st.markdown("<br>", unsafe_allow_html=True) | |
| st.dataframe( | |
| comp_df.sort_values("similarity", ascending=False), | |
| use_container_width=True, | |
| height=500, | |
| column_config={ | |
| "cluster_a": st.column_config.NumberColumn("Cluster A", width="small"), | |
| "cluster_b": st.column_config.NumberColumn("Cluster B", width="small"), | |
| "cluster_a_keywords": st.column_config.TextColumn("A Keywords", width="medium"), | |
| "cluster_b_keywords": st.column_config.TextColumn("B Keywords", width="medium"), | |
| "similarity": st.column_config.ProgressColumn( | |
| "Similarity", min_value=0, max_value=1, format="%.3f", | |
| ), | |
| "alignment": st.column_config.TextColumn("Alignment", width="small"), | |
| }, | |
| ) | |
| else: | |
| st.info("No cluster similarity data available. Run the pipeline first.") | |
| else: | |
| st.info("No comparison data available. Run the pipeline first.") | |
| # Tab 5: Taxonomy Map | |
| with tab5: | |
| st.markdown('<div class="section-header">Taxonomy Map</div>', unsafe_allow_html=True) | |
| taxonomy = result.taxonomy_map | |
| meta = taxonomy.get("metadata", {}) | |
| st.markdown(f""" | |
| <div class="info-box"> | |
| <strong>Classification Summary:</strong><br> | |
| Total Topics: <strong>{meta.get('total_topics', 0)}</strong> | | |
| <span class="mapped-badge">MAPPED: {meta.get('mapped_count', 0)}</span> | | |
| <span class="novel-badge">NOVEL: {meta.get('novel_count', 0)}</span> | | |
| Threshold: {meta.get('threshold', 0.15)} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| tax_tab1, tax_tab2 = st.tabs(["✅ Mapped Themes", "🆕 Novel Themes"]) | |
| with tax_tab1: | |
| mapped_list = taxonomy.get("mapped", []) | |
| if mapped_list: | |
| mapped_df = pd.DataFrame(mapped_list) | |
| st.dataframe( | |
| mapped_df, | |
| use_container_width=True, | |
| height=400, | |
| column_config={ | |
| "score": st.column_config.ProgressColumn( | |
| "Match Score", | |
| min_value=0, | |
| max_value=1, | |
| format="%.3f", | |
| ), | |
| }, | |
| ) | |
| else: | |
| st.info("No mapped themes found.") | |
| with tax_tab2: | |
| novel_list = taxonomy.get("novel", []) | |
| if novel_list: | |
| novel_df = pd.DataFrame(novel_list) | |
| st.dataframe( | |
| novel_df, | |
| use_container_width=True, | |
| height=400, | |
| column_config={ | |
| "score": st.column_config.ProgressColumn( | |
| "Match Score", | |
| min_value=0, | |
| max_value=1, | |
| format="%.3f", | |
| ), | |
| }, | |
| ) | |
| else: | |
| st.info("No novel themes found.") | |
| # ================================================================== | |
| # Tab 6: Editable Review Table | |
| # ================================================================== | |
| with tab_review: | |
| st.markdown(""" | |
| <div class="review-header"> | |
| <h3>✏️ Topic Review Table</h3> | |
| <p>Review, approve, rename, and annotate each topic. Changes are saved to outputs/review_table.csv.</p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Build review dataframe from combined topics | |
| # Load existing review table if available to preserve edits | |
| review_csv_path = os.path.join(OUTPUT_DIR, "review_table.csv") | |
| if "review_df" not in st.session_state: | |
| if os.path.exists(review_csv_path): | |
| # Load previously saved review table | |
| existing_review = pd.read_csv(review_csv_path) | |
| # Merge with current topics to ensure all topics are represented | |
| current_ids = set(result.combined_topics["topic_id"].tolist()) | |
| existing_ids = set(existing_review["topic_id"].tolist()) if "topic_id" in existing_review.columns else set() | |
| if current_ids == existing_ids or existing_ids.issuperset(current_ids): | |
| st.session_state.review_df = existing_review | |
| else: | |
| # Rebuild from current topics, but preserve existing edits | |
| review_data = [] | |
| for _, row in result.combined_topics.iterrows(): | |
| review_data.append({ | |
| "topic_id": int(row["topic_id"]), | |
| "cluster_id": int(row["cluster_id"]) if "cluster_id" in row.index else "", | |
| "cluster_label": row.get("cluster_label", ""), | |
| "label": row.get("label", ""), | |
| "keywords": row.get("keywords", ""), | |
| "source": row.get("source", ""), | |
| "approve": False, | |
| "rename_to": "", | |
| "reasoning": "", | |
| }) | |
| new_review_df = pd.DataFrame(review_data) | |
| # Merge existing edits | |
| if not existing_review.empty and "topic_id" in existing_review.columns: | |
| for _, erow in existing_review.iterrows(): | |
| mask = new_review_df["topic_id"] == erow["topic_id"] | |
| if mask.any(): | |
| if "approve" in erow: | |
| new_review_df.loc[mask, "approve"] = erow["approve"] | |
| if "rename_to" in erow and pd.notna(erow["rename_to"]): | |
| new_review_df.loc[mask, "rename_to"] = erow["rename_to"] | |
| if "reasoning" in erow and pd.notna(erow["reasoning"]): | |
| new_review_df.loc[mask, "reasoning"] = erow["reasoning"] | |
| st.session_state.review_df = new_review_df | |
| else: | |
| # Build fresh review table | |
| review_data = [] | |
| for _, row in result.combined_topics.iterrows(): | |
| review_data.append({ | |
| "topic_id": int(row["topic_id"]), | |
| "cluster_id": int(row["cluster_id"]) if "cluster_id" in row.index else "", | |
| "cluster_label": row.get("cluster_label", ""), | |
| "label": row.get("label", ""), | |
| "keywords": row.get("keywords", ""), | |
| "source": row.get("source", ""), | |
| "approve": False, | |
| "rename_to": "", | |
| "reasoning": "", | |
| }) | |
| st.session_state.review_df = pd.DataFrame(review_data) | |
| # Enforce string dtypes to prevent ColumnDataKind.FLOAT errors | |
| if "rename_to" in st.session_state.review_df.columns: | |
| st.session_state.review_df["rename_to"] = st.session_state.review_df["rename_to"].fillna("").astype(str) | |
| if "reasoning" in st.session_state.review_df.columns: | |
| st.session_state.review_df["reasoning"] = st.session_state.review_df["reasoning"].fillna("").astype(str) | |
| # Filter controls for review table | |
| rv_col1, rv_col2, rv_col3 = st.columns(3) | |
| with rv_col1: | |
| if "source" in st.session_state.review_df.columns and st.session_state.review_df["source"].notna().any(): | |
| review_source_options = [s for s in st.session_state.review_df["source"].unique().tolist() if s] | |
| review_source_filter = st.multiselect( | |
| "Filter by Source", | |
| options=review_source_options, | |
| default=review_source_options, | |
| key="review_source_filter", | |
| ) | |
| else: | |
| review_source_filter = None | |
| with rv_col2: | |
| review_search = st.text_input("🔍 Search in review table", "", key="review_search") | |
| with rv_col3: | |
| review_approval_filter = st.selectbox( | |
| "Show", | |
| options=["All Topics", "Approved Only", "Not Approved"], | |
| index=0, | |
| key="review_approval_filter", | |
| ) | |
| # Apply filters | |
| filtered_review = st.session_state.review_df.copy() | |
| if review_source_filter is not None and "source" in filtered_review.columns: | |
| filtered_review = filtered_review[filtered_review["source"].isin(review_source_filter)] | |
| if review_search: | |
| search_mask = ( | |
| filtered_review["keywords"].str.contains(review_search, case=False, na=False) | | |
| filtered_review["label"].str.contains(review_search, case=False, na=False) | |
| ) | |
| filtered_review = filtered_review[search_mask] | |
| if review_approval_filter == "Approved Only": | |
| filtered_review = filtered_review[filtered_review["approve"] == True] | |
| elif review_approval_filter == "Not Approved": | |
| filtered_review = filtered_review[filtered_review["approve"] == False] | |
| # Editable data editor | |
| edited_df = st.data_editor( | |
| filtered_review, | |
| use_container_width=True, | |
| height=500, | |
| num_rows="fixed", | |
| key="review_editor", | |
| column_config={ | |
| "topic_id": st.column_config.NumberColumn( | |
| "Topic ID", width="small", disabled=True | |
| ), | |
| "label": st.column_config.TextColumn( | |
| "Label", width="medium", | |
| ), | |
| "keywords": st.column_config.TextColumn( | |
| "Keywords", width="large", disabled=True, | |
| ), | |
| "source": st.column_config.TextColumn( | |
| "Source", width="small", disabled=True, | |
| ), | |
| "approve": st.column_config.CheckboxColumn( | |
| "✅ Approve", width="small", default=False, | |
| ), | |
| "rename_to": st.column_config.TextColumn( | |
| "Rename To", width="medium", | |
| ), | |
| "reasoning": st.column_config.TextColumn( | |
| "Reasoning / Notes", width="large", | |
| ), | |
| }, | |
| column_order=["topic_id", "label", "keywords", "approve", "rename_to", "reasoning", "source"], | |
| ) | |
| # Update session state with edits | |
| if edited_df is not None: | |
| # Merge edits back into the full review dataframe | |
| for idx, erow in edited_df.iterrows(): | |
| mask = st.session_state.review_df.index == idx | |
| if mask.any(): | |
| for col in ["label", "approve", "rename_to", "reasoning"]: | |
| if col in erow: | |
| st.session_state.review_df.loc[mask, col] = erow[col] | |
| # Save button | |
| sv_col1, sv_col2, sv_col3 = st.columns([1, 1, 2]) | |
| with sv_col1: | |
| if st.button("💾 Save Review Table", use_container_width=True, type="primary"): | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| st.session_state.review_df.to_csv(review_csv_path, index=False) | |
| st.markdown( | |
| '<div class="save-confirm">✅ Review table saved to outputs/review_table.csv</div>', | |
| unsafe_allow_html=True, | |
| ) | |
| with sv_col2: | |
| approved_count = int(st.session_state.review_df["approve"].sum()) if "approve" in st.session_state.review_df.columns else 0 | |
| total_count = len(st.session_state.review_df) | |
| st.markdown(f""" | |
| <div class="stat-card" style="padding: 0.75rem 1rem;"> | |
| <div class="stat-value" style="font-size: 1.4rem;">{approved_count}/{total_count}</div> | |
| <div class="stat-label">Topics Approved</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # ================================================================== | |
| # Tab 7: Charts | |
| # ================================================================== | |
| with tab_charts: | |
| st.markdown('<div class="section-header">📈 Topic Visualizations</div>', unsafe_allow_html=True) | |
| # ----------------------------------------------------------- | |
| # Chart 0: Topics per Cluster (NEW) | |
| # ----------------------------------------------------------- | |
| has_cluster_col_chart = "cluster_id" in result.combined_topics.columns | |
| if has_cluster_col_chart: | |
| st.markdown("#### 🏗️ Topics per Cluster") | |
| st.caption("Number of fine-grained topics extracted within each high-level cluster.") | |
| topics_per_cluster = result.combined_topics.groupby("cluster_id").size().reset_index(name="Topics") | |
| if "cluster_label" in result.combined_topics.columns: | |
| label_map = result.combined_topics.drop_duplicates("cluster_id")[["cluster_id", "cluster_label"]] | |
| topics_per_cluster = topics_per_cluster.merge(label_map, on="cluster_id", how="left") | |
| topics_per_cluster["display"] = topics_per_cluster.apply( | |
| lambda r: f"C{int(r['cluster_id'])}: {r['cluster_label']}", axis=1 | |
| ) | |
| else: | |
| topics_per_cluster["display"] = topics_per_cluster["cluster_id"].apply(lambda x: f"Cluster {x}") | |
| fig0 = px.bar( | |
| topics_per_cluster.sort_values("Topics", ascending=False), | |
| x="display", | |
| y="Topics", | |
| color="Topics", | |
| color_continuous_scale=["#302b63", "#667eea", "#f093fb"], | |
| text="Topics", | |
| ) | |
| fig0.update_traces( | |
| textposition="outside", | |
| textfont=dict(color="#e0e0e0", size=12, family="Inter"), | |
| marker=dict(line=dict(width=0)), | |
| ) | |
| fig0.update_layout( | |
| **PLOTLY_LAYOUT, | |
| title=f"Topics per Cluster ({topics_per_cluster['cluster_id'].nunique()} clusters)", | |
| xaxis_title="Cluster", | |
| yaxis_title="Number of Topics", | |
| showlegend=False, | |
| height=480, | |
| ) | |
| fig0.update_layout( | |
| xaxis=dict(tickangle=-45, tickfont=dict(color="#c4b5fd", size=10)), | |
| ) | |
| st.plotly_chart(fig0, use_container_width=True) | |
| st.markdown("---") | |
| # ----------------------------------------------------------- | |
| # Chart 1: Topic Frequency by Source | |
| # ----------------------------------------------------------- | |
| st.markdown("#### 📊 Topic Frequency by Source") | |
| st.caption("Number of topics discovered from each source.") | |
| if "source" in result.combined_topics.columns: | |
| source_counts = result.combined_topics["source"].value_counts().reset_index() | |
| source_counts.columns = ["Source", "Count"] | |
| else: | |
| source_counts = pd.DataFrame({"Source": ["All"], "Count": [len(result.combined_topics)]}) | |
| fig1 = px.bar( | |
| source_counts, | |
| x="Source", | |
| y="Count", | |
| color="Source", | |
| color_discrete_sequence=["#667eea", "#764ba2"], | |
| text="Count", | |
| ) | |
| fig1.update_traces( | |
| textposition="outside", | |
| textfont=dict(color="#e0e0e0", size=14, family="Inter"), | |
| marker=dict( | |
| line=dict(width=0), | |
| ), | |
| ) | |
| fig1.update_layout( | |
| **PLOTLY_LAYOUT, | |
| title="Topic Count by Source", | |
| xaxis_title="Source", | |
| yaxis_title="Number of Topics", | |
| showlegend=False, | |
| height=420, | |
| ) | |
| st.plotly_chart(fig1, use_container_width=True) | |
| st.markdown("---") | |
| # ----------------------------------------------------------- | |
| # Chart 2: Top Keywords Across All Topics | |
| # ----------------------------------------------------------- | |
| st.markdown("#### 🔤 Top Keywords Across All Topics") | |
| st.caption("Most frequently occurring keywords across all discovered topics.") | |
| # Extract all keywords, count frequencies | |
| all_keywords = [] | |
| for kw_str in result.combined_topics["keywords"].dropna(): | |
| for kw in kw_str.split(","): | |
| kw_clean = kw.strip().lower() | |
| if kw_clean and len(kw_clean) > 2: | |
| all_keywords.append(kw_clean) | |
| kw_counts = pd.Series(all_keywords).value_counts().head(25).reset_index() | |
| kw_counts.columns = ["Keyword", "Frequency"] | |
| fig2 = px.bar( | |
| kw_counts, | |
| x="Frequency", | |
| y="Keyword", | |
| orientation="h", | |
| color="Frequency", | |
| color_continuous_scale=["#302b63", "#667eea", "#f093fb", "#f5576c"], | |
| ) | |
| fig2.update_traces( | |
| marker=dict(line=dict(width=0)), | |
| ) | |
| fig2.update_layout( | |
| **PLOTLY_LAYOUT, | |
| title="Top 25 Keywords by Frequency", | |
| xaxis_title="Frequency (across all topics)", | |
| yaxis_title="", | |
| height=700, | |
| coloraxis_colorbar=dict( | |
| title="Freq", | |
| tickfont=dict(color="#c4b5fd"), | |
| title_font=dict(color="#e0e0e0"), | |
| ), | |
| ) | |
| # Override yaxis separately to avoid duplicate keyword with PLOTLY_LAYOUT | |
| fig2.update_layout( | |
| yaxis=dict( | |
| autorange="reversed", | |
| gridcolor="rgba(102, 126, 234, 0.1)", | |
| tickfont=dict(color="#c4b5fd", size=12), | |
| ), | |
| ) | |
| st.plotly_chart(fig2, use_container_width=True) | |
| st.markdown("---") | |
| # ----------------------------------------------------------- | |
| # Chart 3: Taxonomy Distribution (Mapped vs Novel) | |
| # ----------------------------------------------------------- | |
| st.markdown("#### 🧬 Taxonomy Classification Distribution") | |
| st.caption("How topics are classified against the known research taxonomy.") | |
| tax_meta = result.taxonomy_map.get("metadata", {}) | |
| tax_data = pd.DataFrame({ | |
| "Classification": ["MAPPED", "NOVEL"], | |
| "Count": [tax_meta.get("mapped_count", 0), tax_meta.get("novel_count", 0)], | |
| }) | |
| chart3_col1, chart3_col2 = st.columns(2) | |
| with chart3_col1: | |
| fig3a = px.pie( | |
| tax_data, | |
| values="Count", | |
| names="Classification", | |
| color="Classification", | |
| color_discrete_map={ | |
| "MAPPED": "#38ef7d", | |
| "NOVEL": "#f7b733", | |
| }, | |
| hole=0.55, | |
| ) | |
| fig3a.update_traces( | |
| textfont=dict(color="#ffffff", size=14), | |
| textinfo="percent+label", | |
| marker=dict(line=dict(color="#1a1a2e", width=3)), | |
| ) | |
| fig3a.update_layout( | |
| paper_bgcolor="rgba(26, 26, 46, 0.95)", | |
| plot_bgcolor="rgba(22, 33, 62, 0.95)", | |
| font=dict(family="Inter, sans-serif", size=13, color="#e0e0e0"), | |
| title=dict(text="Mapped vs Novel", font=dict(size=16, color="#ffffff")), | |
| legend=dict(font=dict(color="#e0e0e0")), | |
| height=380, | |
| margin=dict(l=20, r=20, t=50, b=20), | |
| ) | |
| st.plotly_chart(fig3a, use_container_width=True) | |
| with chart3_col2: | |
| fig3b = px.bar( | |
| tax_data, | |
| x="Classification", | |
| y="Count", | |
| color="Classification", | |
| color_discrete_map={ | |
| "MAPPED": "#38ef7d", | |
| "NOVEL": "#f7b733", | |
| }, | |
| text="Count", | |
| ) | |
| fig3b.update_traces( | |
| textposition="outside", | |
| textfont=dict(color="#e0e0e0", size=16, family="Inter"), | |
| marker=dict(line=dict(width=0)), | |
| ) | |
| fig3b.update_layout( | |
| **PLOTLY_LAYOUT, | |
| title="Classification Count", | |
| xaxis_title="", | |
| yaxis_title="Number of Topics", | |
| showlegend=False, | |
| height=380, | |
| ) | |
| st.plotly_chart(fig3b, use_container_width=True) | |
| st.markdown("---") | |
| # ----------------------------------------------------------- | |
| # Chart 4: Alignment Distribution (from comparisons) | |
| # ----------------------------------------------------------- | |
| if not result.comparison.empty: | |
| st.markdown("#### ⚖️ Theme Alignment Distribution") | |
| st.caption("Distribution of alignment strength between Title and Abstract topics.") | |
| alignment_data = result.comparison["alignment"].value_counts().reset_index() | |
| alignment_data.columns = ["Alignment", "Count"] | |
| # Define order and colors | |
| align_order = ["Strong", "Moderate", "Weak", "No Match"] | |
| align_colors = { | |
| "Strong": "#38ef7d", | |
| "Moderate": "#4facfe", | |
| "Weak": "#f7971e", | |
| "No Match": "#f5576c", | |
| } | |
| fig4 = px.bar( | |
| alignment_data, | |
| x="Alignment", | |
| y="Count", | |
| color="Alignment", | |
| color_discrete_map=align_colors, | |
| text="Count", | |
| category_orders={"Alignment": align_order}, | |
| ) | |
| fig4.update_traces( | |
| textposition="outside", | |
| textfont=dict(color="#e0e0e0", size=14, family="Inter"), | |
| marker=dict(line=dict(width=0)), | |
| ) | |
| fig4.update_layout( | |
| **PLOTLY_LAYOUT, | |
| title="Title ↔ Abstract Alignment Distribution", | |
| xaxis_title="Alignment Level", | |
| yaxis_title="Number of Topic Pairs", | |
| showlegend=False, | |
| height=420, | |
| ) | |
| st.plotly_chart(fig4, use_container_width=True) | |
| st.markdown("---") | |
| # ----------------------------------------------------------- | |
| # Chart 5: Similarity Score Histogram | |
| # ----------------------------------------------------------- | |
| st.markdown("#### 📐 Similarity Score Distribution") | |
| st.caption("Distribution of Jaccard similarity scores between matched Title and Abstract topics.") | |
| fig5 = px.histogram( | |
| result.comparison, | |
| x="similarity", | |
| nbins=30, | |
| color_discrete_sequence=["#667eea"], | |
| marginal="box", | |
| ) | |
| fig5.update_traces( | |
| marker=dict( | |
| line=dict(width=1, color="#b8b5ff"), | |
| ), | |
| selector=dict(type="histogram"), | |
| ) | |
| fig5.update_layout( | |
| **PLOTLY_LAYOUT, | |
| title="Similarity Score Histogram", | |
| xaxis_title="Jaccard Similarity Score", | |
| yaxis_title="Count", | |
| height=420, | |
| bargap=0.05, | |
| ) | |
| st.plotly_chart(fig5, use_container_width=True) | |
| # ================================================================== | |
| # Tab 10: Computational Strategy Map | |
| # ================================================================== | |
| with tab_comp: | |
| st.markdown('<div class="section-header">🧮 Computational Strategy Map</div>', unsafe_allow_html=True) | |
| st.markdown("SPECTER2 semantic clustering of computational strategies — powered by HDBSCAN + UMAP") | |
| COMP_CSV = "computation.csv" | |
| if not os.path.exists(COMP_CSV): | |
| st.warning("⚠️ `computation.csv` not found in the project directory.") | |
| else: | |
| # Run / cache pipeline | |
| if "comp_cluster_data" not in st.session_state: | |
| with st.spinner("🔄 Clustering computation.csv … (cached after first run)"): | |
| try: | |
| comp_data = run_computation_clustering(COMP_CSV) | |
| st.session_state.comp_cluster_data = comp_data | |
| except Exception as exc: | |
| st.error(f"Clustering failed: {exc}") | |
| comp_data = None | |
| if "comp_cluster_data" in st.session_state: | |
| comp = st.session_state.comp_cluster_data | |
| comp_df = comp["df"] | |
| cluster_names_comp = comp["cluster_names"] | |
| cluster_keywords_comp = comp["cluster_keywords"] | |
| strategy_types_comp = comp["strategy_types"] | |
| # ---- Summary stats ---- | |
| sc1, sc2, sc3 = st.columns(3) | |
| with sc1: | |
| st.markdown(f""" | |
| <div class="stat-card"> | |
| <div class="stat-value">{comp['n_papers']}</div> | |
| <div class="stat-label">Papers</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with sc2: | |
| st.markdown(f""" | |
| <div class="stat-card" style="background: linear-gradient(135deg, #11998e 0%, #38ef7d 100%);"> | |
| <div class="stat-value">{comp['n_clusters']}</div> | |
| <div class="stat-label">Semantic Clusters</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with sc3: | |
| st.markdown(f""" | |
| <div class="stat-card" style="background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);"> | |
| <div class="stat-value">{len(strategy_types_comp)}</div> | |
| <div class="stat-label">Strategy Types</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| st.markdown("<br>", unsafe_allow_html=True) | |
| # ---- Sub-tabs ---- | |
| csub1, csub2, csub3, csub4 = st.tabs([ | |
| "🗺️ Cluster Map", "📊 Statistics", "📋 Papers", "✏️ Review", | |
| ]) | |
| # -- Cluster Map -- | |
| with csub1: | |
| fc1, fc2, fc3 = st.columns(3) | |
| with fc1: | |
| comp_search = st.text_input("🔍 Search title", "", key="comp_search") | |
| with fc2: | |
| comp_strat_f = st.multiselect("Strategy Type", options=strategy_types_comp, | |
| default=strategy_types_comp, key="comp_strategy_filter") | |
| with fc3: | |
| cl_opts = sorted(cluster_names_comp.keys()) | |
| comp_cl_f = st.multiselect("Cluster", options=cl_opts, default=cl_opts, | |
| format_func=lambda c: f"C{c}: {cluster_names_comp[c]}", | |
| key="comp_cluster_filter") | |
| plot_df = comp_df.copy() | |
| if comp_search: | |
| plot_df = plot_df[plot_df["Paper Title"].str.contains(comp_search, case=False, na=False)] | |
| if comp_strat_f: | |
| plot_df = plot_df[plot_df["Strategy Type"].isin(comp_strat_f)] | |
| if comp_cl_f: | |
| plot_df = plot_df[plot_df["cluster_id"].isin(comp_cl_f)] | |
| if len(plot_df) == 0: | |
| st.warning("No papers match filters.") | |
| else: | |
| plot_df = plot_df.copy() | |
| plot_df["hover_text"] = plot_df.apply( | |
| lambda r: ( | |
| f"<b>{r['Paper Title'][:80]}</b><br>" | |
| f"<b>Authors:</b> {r['Authors'][:60]}<br>" | |
| f"<b>Strategy:</b> {str(r.get('Computational Strategies Used',''))[:60]}<br>" | |
| f"<b>Type:</b> {r['Strategy Type']}<br>" | |
| f"<b>Cluster:</b> {r['cluster_name']}" | |
| ), axis=1) | |
| plot_df["Cluster"] = plot_df["cluster_name"].astype(str) | |
| fig_comp = px.scatter( | |
| plot_df, x="x", y="y", color="Cluster", hover_name="Paper Title", | |
| custom_data=["hover_text"], | |
| color_discrete_sequence=px.colors.qualitative.Set2 + px.colors.qualitative.Dark24, | |
| ) | |
| fig_comp.update_traces( | |
| marker=dict(size=12, line=dict(width=1.5, color="#ffffff")), | |
| hovertemplate="%{customdata[0]}<extra></extra>", | |
| ) | |
| fig_comp.update_layout( | |
| plot_bgcolor="#1a1a2e", paper_bgcolor="#1a1a2e", | |
| font=dict(color="#e0e0e0", family="Inter, sans-serif"), | |
| title=dict(text=f"Cluster Map — {len(plot_df)} papers, {comp['n_clusters']} clusters", | |
| font=dict(size=18, color="#ffffff")), | |
| xaxis=dict(showgrid=False, zeroline=False, title="UMAP Dim 1", | |
| tickfont=dict(color="#c4b5fd")), | |
| yaxis=dict(showgrid=False, zeroline=False, title="UMAP Dim 2", | |
| tickfont=dict(color="#c4b5fd")), | |
| legend=dict(title="Cluster", font=dict(color="#e0e0e0"), | |
| bgcolor="rgba(26,26,46,0.7)", bordercolor="#667eea", borderwidth=1), | |
| height=650, margin=dict(l=40, r=40, t=60, b=40), | |
| ) | |
| st.plotly_chart(fig_comp, use_container_width=True) | |
| st.markdown("---") | |
| export_cols = [c for c in ["Paper Title","Authors","Computational Strategies Used", | |
| "Strategy Type","Supporting Keywords / Evidence", | |
| "cluster_id","cluster_name"] if c in comp_df.columns] | |
| st.download_button("⬇️ Export Clustered CSV", data=comp_df[export_cols].to_csv(index=False), | |
| file_name="computation_clustered.csv", mime="text/csv", | |
| use_container_width=True, key="comp_export_csv") | |
| # -- Statistics -- | |
| with csub2: | |
| for cid in sorted(cluster_names_comp.keys()): | |
| csub = comp_df[comp_df["cluster_id"] == cid] | |
| kws = cluster_keywords_comp.get(cid, []) | |
| strat_bd = csub["Strategy Type"].value_counts().to_dict() | |
| st.markdown(f""" | |
| <div class="info-box" style="margin-bottom:0.75rem;"> | |
| <strong>Cluster {cid}: {cluster_names_comp[cid]}</strong><br> | |
| <strong>Papers:</strong> {len(csub)}<br> | |
| <strong>Keywords:</strong> {', '.join(kws)}<br> | |
| <strong>Types:</strong> {', '.join(f'{k} ({v})' for k,v in strat_bd.items() if k.strip())} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with st.expander(f"📄 Papers in Cluster {cid}", expanded=False): | |
| dcols = [c for c in ["Paper Title","Authors","Strategy Type"] if c in csub.columns] | |
| st.dataframe(csub[dcols].reset_index(drop=True), use_container_width=True) | |
| st.markdown("---") | |
| st.markdown("#### Cluster Size Distribution") | |
| sz = comp_df.groupby("cluster_name").size().reset_index(name="Papers") | |
| fig_pie = px.pie(sz, values="Papers", names="cluster_name", | |
| color_discrete_sequence=CHART_COLORS, hole=0.5) | |
| fig_pie.update_traces(textfont=dict(color="#ffffff", size=12), textinfo="percent+label", | |
| marker=dict(line=dict(color="#1a1a2e", width=2))) | |
| fig_pie.update_layout( | |
| paper_bgcolor="rgba(26,26,46,0.95)", plot_bgcolor="rgba(22,33,62,0.95)", | |
| font=dict(family="Inter, sans-serif", size=13, color="#e0e0e0"), | |
| title=dict(text="Cluster Size", font=dict(size=16, color="#ffffff")), | |
| legend=dict(font=dict(color="#e0e0e0")), height=420, margin=dict(l=20,r=20,t=50,b=20), | |
| ) | |
| st.plotly_chart(fig_pie, use_container_width=True) | |
| # -- Papers Table -- | |
| with csub3: | |
| tf1, tf2 = st.columns(2) | |
| with tf1: | |
| tbl_s = st.text_input("🔍 Search", "", key="comp_tbl_search") | |
| with tf2: | |
| tbl_c = st.multiselect("Cluster", options=sorted(cluster_names_comp.keys()), | |
| default=sorted(cluster_names_comp.keys()), | |
| format_func=lambda c: f"C{c}: {cluster_names_comp[c]}", | |
| key="comp_tbl_cluster") | |
| tdf = comp_df.copy() | |
| if tbl_s: | |
| tdf = tdf[tdf["Paper Title"].str.contains(tbl_s, case=False, na=False)] | |
| if tbl_c: | |
| tdf = tdf[tdf["cluster_id"].isin(tbl_c)] | |
| dcols = [c for c in ["Paper Title","Authors","Computational Strategies Used", | |
| "Strategy Type","cluster_name"] if c in tdf.columns] | |
| st.dataframe(tdf[dcols].reset_index(drop=True), use_container_width=True, height=500) | |
| # -- Review -- | |
| with csub4: | |
| st.markdown(""" | |
| <div class="review-header"> | |
| <h3>✏️ Computational Strategy Review</h3> | |
| <p>Review and annotate cluster assignments. Saved to outputs/comp_review_table.csv.</p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| comp_rev_path = os.path.join(OUTPUT_DIR, "comp_review_table.csv") | |
| if "comp_review_df" not in st.session_state: | |
| if os.path.exists(comp_rev_path): | |
| st.session_state.comp_review_df = pd.read_csv(comp_rev_path) | |
| else: | |
| rows = [] | |
| for _, r in comp_df.iterrows(): | |
| rows.append({ | |
| "Paper Title": r.get("Paper Title",""), | |
| "Strategy Type": r.get("Strategy Type",""), | |
| "cluster_name": r.get("cluster_name",""), | |
| "approve": False, "notes": "", | |
| }) | |
| st.session_state.comp_review_df = pd.DataFrame(rows) | |
| if "notes" in st.session_state.comp_review_df.columns: | |
| st.session_state.comp_review_df["notes"] = st.session_state.comp_review_df["notes"].fillna("").astype(str) | |
| cr1, cr2 = st.columns(2) | |
| with cr1: | |
| crs = st.text_input("🔍 Search", "", key="comp_review_search") | |
| with cr2: | |
| crsh = st.selectbox("Show", ["All","Approved","Not Approved"], key="comp_review_show") | |
| fcr = st.session_state.comp_review_df.copy() | |
| if crs: | |
| fcr = fcr[fcr["Paper Title"].str.contains(crs, case=False, na=False)] | |
| if crsh == "Approved": | |
| fcr = fcr[fcr["approve"] == True] | |
| elif crsh == "Not Approved": | |
| fcr = fcr[fcr["approve"] == False] | |
| edited = st.data_editor( | |
| fcr, use_container_width=True, height=500, num_rows="fixed", | |
| key="comp_review_editor", | |
| column_config={ | |
| "Paper Title": st.column_config.TextColumn("Paper Title", width="large", disabled=True), | |
| "Strategy Type": st.column_config.TextColumn("Type", width="small", disabled=True), | |
| "cluster_name": st.column_config.TextColumn("Cluster", width="medium", disabled=True), | |
| "approve": st.column_config.CheckboxColumn("✅", width="small", default=False), | |
| "notes": st.column_config.TextColumn("Notes", width="large"), | |
| }, | |
| column_order=["Paper Title","Strategy Type","cluster_name","approve","notes"], | |
| ) | |
| if edited is not None: | |
| for idx, erow in edited.iterrows(): | |
| mask = st.session_state.comp_review_df.index == idx | |
| if mask.any(): | |
| for col in ["approve","notes"]: | |
| if col in erow: | |
| st.session_state.comp_review_df.loc[mask, col] = erow[col] | |
| sv1, sv2 = st.columns([1, 2]) | |
| with sv1: | |
| if st.button("💾 Save Review", use_container_width=True, type="primary", key="comp_save_review"): | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| st.session_state.comp_review_df.to_csv(comp_rev_path, index=False) | |
| st.markdown('<div class="save-confirm">✅ Saved to outputs/comp_review_table.csv</div>', | |
| unsafe_allow_html=True) | |
| with sv2: | |
| appr = int(st.session_state.comp_review_df["approve"].sum()) if "approve" in st.session_state.comp_review_df.columns else 0 | |
| st.markdown(f""" | |
| <div class="stat-card" style="padding:0.75rem 1rem;"> | |
| <div class="stat-value" style="font-size:1.4rem;">{appr}/{len(st.session_state.comp_review_df)}</div> | |
| <div class="stat-label">Papers Approved</div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # ================================================================== | |
| # Tab 11: Research Intelligence Chatbot | |
| # ================================================================== | |
| with tab_chat: | |
| # ---- Custom CSS for chatbot ---- | |
| st.markdown(""" | |
| <style> | |
| /* Chat message styling */ | |
| .chat-welcome { | |
| background: linear-gradient(135deg, #1e1e3f 0%, #2d2b55 100%); | |
| padding: 1.5rem 2rem; | |
| border-radius: 14px; | |
| border-left: 4px solid #667eea; | |
| color: #e0e0e0; | |
| margin: 1rem 0; | |
| } | |
| .chat-welcome h3 { | |
| color: #ffffff; | |
| margin: 0 0 0.8rem 0; | |
| font-size: 1.2rem; | |
| } | |
| .chat-welcome .capability { | |
| display: inline-block; | |
| background: rgba(102, 126, 234, 0.2); | |
| border: 1px solid rgba(102, 126, 234, 0.3); | |
| color: #c4b5ff; | |
| padding: 0.25rem 0.65rem; | |
| border-radius: 16px; | |
| font-size: 0.75rem; | |
| margin: 0.2rem 0.15rem; | |
| font-weight: 500; | |
| } | |
| /* Context cards */ | |
| .ctx-card { | |
| padding: 0.7rem 0.85rem; | |
| border-radius: 8px; | |
| font-size: 0.8rem; | |
| color: #e0e0e0; | |
| margin-bottom: 0.45rem; | |
| line-height: 1.4; | |
| } | |
| .ctx-card strong { | |
| color: #ffffff; | |
| font-size: 0.82rem; | |
| } | |
| .ctx-card .metric-val { | |
| font-size: 1.1rem; | |
| font-weight: 700; | |
| color: #ffffff; | |
| } | |
| .ctx-card .metric-label { | |
| font-size: 0.7rem; | |
| opacity: 0.8; | |
| } | |
| .ctx-pipeline { background: rgba(102,126,234,0.15); border-left: 3px solid #667eea; } | |
| .ctx-quality { background: rgba(17,153,142,0.15); border-left: 3px solid #11998e; } | |
| .ctx-comp { background: rgba(240,147,251,0.12); border-left: 3px solid #f093fb; } | |
| .ctx-chat { background: rgba(245,87,108,0.12); border-left: 3px solid #f5576c; } | |
| .ctx-warn { background: rgba(247,183,51,0.12); border-left: 3px solid #f7b733; } | |
| /* Prompt history item */ | |
| .prompt-hist-item { | |
| background: rgba(102, 126, 234, 0.08); | |
| border: 1px solid rgba(102, 126, 234, 0.15); | |
| border-radius: 6px; | |
| padding: 0.35rem 0.5rem; | |
| margin-bottom: 0.3rem; | |
| font-size: 0.72rem; | |
| color: #b8b5ff; | |
| cursor: default; | |
| overflow: hidden; | |
| text-overflow: ellipsis; | |
| white-space: nowrap; | |
| } | |
| </style> | |
| """, unsafe_allow_html=True) | |
| st.markdown('<div class="section-header">🧠 Research Intelligence Chatbot</div>', unsafe_allow_html=True) | |
| st.markdown("Interpret clusters · Identify gaps · Optimize parameters · Analyze strategies — powered by Groq LLM") | |
| # ---- Resolve Groq key ---- | |
| chat_groq_key = groq_key if groq_key else os.environ.get("GROQ_API_KEY", "") | |
| if not chat_groq_key: | |
| st.markdown(""" | |
| <div class="chat-welcome"> | |
| <h3>🔑 API Key Required</h3> | |
| <p>To use the Research Intelligence Chatbot, enter a <strong>Groq API key</strong> | |
| in the sidebar by enabling the <em>"🤖 Use LLM for Label Generation"</em> checkbox, | |
| or set the <code>GROQ_API_KEY</code> environment variable.</p> | |
| <p style="margin-top:0.8rem; opacity:0.7;"> | |
| The chatbot uses Groq's LLaMA 3.3 70B model for fast, intelligent responses | |
| grounded in your analysis data. | |
| </p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| else: | |
| # ---- Build / cache context summary ---- | |
| if "chat_context" not in st.session_state: | |
| comp_d = st.session_state.get("comp_cluster_data") | |
| st.session_state.chat_context = build_context_summary(result, comp_d) | |
| # Compute context stats for right panel | |
| comp_d = st.session_state.get("comp_cluster_data") | |
| ctx_stats = get_context_stats(result, comp_d) | |
| # ---- Initialize session state ---- | |
| if "chat_messages" not in st.session_state: | |
| st.session_state.chat_messages = [] | |
| if "chat_prompt_history" not in st.session_state: | |
| st.session_state.chat_prompt_history = [] | |
| # ---- Layout: Left (prompts) | Center (chat) | Right (context) ---- | |
| chat_left, chat_center, chat_right = st.columns([1.1, 3.2, 1.1]) | |
| # ---- LEFT: Suggested prompts + history ---- | |
| with chat_left: | |
| st.markdown("#### 💡 Suggested Prompts") | |
| for i, prompt in enumerate(SUGGESTED_PROMPTS): | |
| display_label = prompt[:42] + "…" if len(prompt) > 42 else prompt | |
| if st.button(display_label, key=f"suggest_{i}", use_container_width=True): | |
| st.session_state.chat_messages.append({"role": "user", "content": prompt}) | |
| # Track in history | |
| if prompt not in st.session_state.chat_prompt_history: | |
| st.session_state.chat_prompt_history.insert(0, prompt) | |
| if len(st.session_state.chat_prompt_history) > 20: | |
| st.session_state.chat_prompt_history = st.session_state.chat_prompt_history[:20] | |
| with st.spinner("Thinking …"): | |
| reply = chat_with_groq( | |
| st.session_state.chat_messages, | |
| st.session_state.chat_context, | |
| chat_groq_key, | |
| ) | |
| st.session_state.chat_messages.append({"role": "assistant", "content": reply}) | |
| st.rerun() | |
| st.markdown("---") | |
| # Controls | |
| use_ctx = st.toggle("📎 Use analysis context", value=True, key="chat_use_ctx") | |
| ctrl_c1, ctrl_c2 = st.columns(2) | |
| with ctrl_c1: | |
| if st.button("🗑️ Clear", key="chat_clear", use_container_width=True): | |
| st.session_state.chat_messages = [] | |
| st.rerun() | |
| with ctrl_c2: | |
| if st.button("🔄 Refresh", key="chat_refresh_ctx", use_container_width=True, | |
| help="Rebuild context from latest analysis"): | |
| comp_d = st.session_state.get("comp_cluster_data") | |
| st.session_state.chat_context = build_context_summary(result, comp_d) | |
| st.rerun() | |
| # Prompt history | |
| if st.session_state.chat_prompt_history: | |
| st.markdown("---") | |
| st.markdown("#### 📜 History") | |
| for hp in st.session_state.chat_prompt_history[:8]: | |
| st.markdown( | |
| f'<div class="prompt-hist-item" title="{hp}">{hp[:45]}</div>', | |
| unsafe_allow_html=True, | |
| ) | |
| # ---- CENTER: Chat interface ---- | |
| with chat_center: | |
| # Display conversation | |
| chat_container = st.container(height=540) | |
| with chat_container: | |
| if not st.session_state.chat_messages: | |
| # Welcome message | |
| st.markdown(""" | |
| <div class="chat-welcome"> | |
| <h3>👋 Welcome to the Research Intelligence Chatbot</h3> | |
| <p>I'm your AI research assistant, powered by your full analysis context. | |
| I can help you:</p> | |
| <div style="margin: 0.8rem 0;"> | |
| <span class="capability">🔍 Interpret Clusters</span> | |
| <span class="capability">📊 Analyze Quality</span> | |
| <span class="capability">⚙️ Optimize Parameters</span> | |
| <span class="capability">🔬 Find Research Gaps</span> | |
| <span class="capability">📈 Identify Trends</span> | |
| <span class="capability">🎯 Detect Outliers</span> | |
| <span class="capability">🧬 Compare Strategies</span> | |
| <span class="capability">📝 Academic Summaries</span> | |
| </div> | |
| <p style="margin-top:0.8rem; opacity:0.8; font-size:0.85rem;"> | |
| Try a suggested prompt on the left, or type your question below. | |
| Toggle <strong>"📎 Use analysis context"</strong> to control whether | |
| the chatbot references your pipeline outputs. | |
| </p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| else: | |
| for msg in st.session_state.chat_messages: | |
| with st.chat_message(msg["role"]): | |
| st.markdown(msg["content"]) | |
| # Chat input | |
| user_input = st.chat_input("Ask about your research analysis …", key="chat_input") | |
| if user_input: | |
| st.session_state.chat_messages.append({"role": "user", "content": user_input}) | |
| # Track in history | |
| if user_input not in st.session_state.chat_prompt_history: | |
| st.session_state.chat_prompt_history.insert(0, user_input) | |
| if len(st.session_state.chat_prompt_history) > 20: | |
| st.session_state.chat_prompt_history = st.session_state.chat_prompt_history[:20] | |
| ctx = st.session_state.chat_context if use_ctx else "No analysis context provided — answer based on general knowledge." | |
| with st.spinner("Thinking …"): | |
| reply = chat_with_groq( | |
| st.session_state.chat_messages, ctx, chat_groq_key, | |
| ) | |
| st.session_state.chat_messages.append({"role": "assistant", "content": reply}) | |
| st.rerun() | |
| # ---- RIGHT: Live context summary ---- | |
| with chat_right: | |
| st.markdown("#### 📊 Live Context") | |
| # Pipeline stats | |
| st.markdown(f""" | |
| <div class="ctx-card ctx-pipeline"> | |
| <strong>🔬 Pipeline</strong><br> | |
| <span class="metric-val">{ctx_stats['n_clusters']}</span> | |
| <span class="metric-label">clusters</span> · | |
| <span class="metric-val">{ctx_stats['n_topics']}</span> | |
| <span class="metric-label">topics</span><br> | |
| Mapped: <strong>{ctx_stats['n_mapped']}</strong> · | |
| Novel: <strong>{ctx_stats['n_novel']}</strong> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Clustering quality | |
| sil_display = f"{ctx_stats['silhouette']:.3f}" if ctx_stats.get('silhouette') else "N/A" | |
| quality_display = ctx_stats.get('quality_rating', '—') | |
| balance_display = ctx_stats.get('balance_quality', '—') | |
| quality_color = { | |
| "Excellent": "#38ef7d", "Good": "#4facfe", | |
| "Fair": "#f7971e", "Poor": "#f5576c", | |
| }.get(quality_display, "#e0e0e0") | |
| st.markdown(f""" | |
| <div class="ctx-card ctx-quality"> | |
| <strong>📐 Quality</strong><br> | |
| Silhouette: <strong style="color:{quality_color}">{sil_display}</strong><br> | |
| Rating: <strong style="color:{quality_color}">{quality_display}</strong><br> | |
| Balance: <strong>{balance_display}</strong> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Warnings / issues | |
| n_overlaps = ctx_stats.get("n_overlaps", 0) | |
| n_outliers = ctx_stats.get("n_outliers", 0) | |
| if n_overlaps > 0 or n_outliers > 0: | |
| warn_parts = [] | |
| if n_overlaps > 0: | |
| warn_parts.append(f"🔗 {n_overlaps} overlapping pairs") | |
| if n_outliers > 0: | |
| warn_parts.append(f"⚡ {n_outliers} outlier clusters") | |
| st.markdown(f""" | |
| <div class="ctx-card ctx-warn"> | |
| <strong>⚠️ Alerts</strong><br> | |
| {'<br>'.join(warn_parts)} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Comp strategies (if available) | |
| if ctx_stats.get("comp_papers", 0) > 0: | |
| st.markdown(f""" | |
| <div class="ctx-card ctx-comp"> | |
| <strong>🧮 Comp Strategies</strong><br> | |
| <span class="metric-val">{ctx_stats['comp_papers']}</span> | |
| <span class="metric-label">papers</span> · | |
| <span class="metric-val">{ctx_stats['comp_clusters']}</span> | |
| <span class="metric-label">clusters</span><br> | |
| Types: <strong>{ctx_stats['comp_strategy_types']}</strong> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Chat session stats | |
| n_msgs = len(st.session_state.get('chat_messages', [])) | |
| n_user = sum(1 for m in st.session_state.get('chat_messages', []) if m['role'] == 'user') | |
| st.markdown(f""" | |
| <div class="ctx-card ctx-chat"> | |
| <strong>💬 Session</strong><br> | |
| Messages: <strong>{n_msgs}</strong> ({n_user} questions)<br> | |
| Context: {'<strong style="color:#38ef7d">Active ✅</strong>' if use_ctx else '<strong style="color:#f5576c">Off ❌</strong>'} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # ---- Downloads Section ---- | |
| st.markdown('<div class="section-header">📥 Download Outputs</div>', unsafe_allow_html=True) | |
| dcol1, dcol2, dcol3, dcol4 = st.columns(4) | |
| with dcol1: | |
| csv_data = result.combined_topics.to_csv(index=False) | |
| st.download_button( | |
| "⬇️ Topics Table (CSV)", | |
| data=csv_data, | |
| file_name="topics_table.csv", | |
| mime="text/csv", | |
| use_container_width=True, | |
| ) | |
| with dcol2: | |
| comp_data = result.comparison.to_csv(index=False) | |
| st.download_button( | |
| "⬇️ Comparison (CSV)", | |
| data=comp_data, | |
| file_name="comparison.csv", | |
| mime="text/csv", | |
| use_container_width=True, | |
| ) | |
| with dcol3: | |
| json_data = json.dumps(result.taxonomy_map, indent=2, ensure_ascii=False) | |
| st.download_button( | |
| "⬇️ Taxonomy Map (JSON)", | |
| data=json_data, | |
| file_name="taxonomy_map.json", | |
| mime="application/json", | |
| use_container_width=True, | |
| ) | |
| with dcol4: | |
| # Download review table if it exists | |
| review_path = os.path.join(OUTPUT_DIR, "review_table.csv") | |
| if os.path.exists(review_path): | |
| with open(review_path, "r") as f: | |
| review_data = f.read() | |
| st.download_button( | |
| "⬇️ Review Table (CSV)", | |
| data=review_data, | |
| file_name="review_table.csv", | |
| mime="text/csv", | |
| use_container_width=True, | |
| ) | |
| else: | |
| st.download_button( | |
| "⬇️ Review Table (CSV)", | |
| data="Not saved yet. Go to Review Table tab and click Save.", | |
| file_name="review_table.csv", | |
| mime="text/csv", | |
| use_container_width=True, | |
| disabled=True, | |
| ) | |
| # ---- Auto-save comparison.csv and taxonomy_map.json to outputs ---- | |
| os.makedirs(OUTPUT_DIR, exist_ok=True) | |
| result.comparison.to_csv(os.path.join(OUTPUT_DIR, "comparison.csv"), index=False) | |
| with open(os.path.join(OUTPUT_DIR, "taxonomy_map.json"), "w", encoding="utf-8") as f: | |
| json.dump(result.taxonomy_map, f, indent=2, ensure_ascii=False) | |
| # ---- Pipeline Log ---- | |
| with st.expander("📜 Pipeline Execution Log"): | |
| st.markdown(f"**Status:** `{result.status}`") | |
| st.markdown(f"**Steps Completed:** {len(result.steps_completed)}/9") | |
| for i, step in enumerate(result.steps_completed, 1): | |
| st.markdown(f" ✅ Step {i}: `{step}`") | |
| if result.errors: | |
| st.markdown("**Errors:**") | |
| for err in result.errors: | |
| st.error(err) | |
| st.markdown("**Exported Files:**") | |
| for name, path in result.exported_files.items(): | |
| st.markdown(f" 📄 `{name}` → `{path}`") | |
| else: | |
| # ---- Welcome / instructions when no results ---- | |
| st.markdown(""" | |
| <div class="info-box"> | |
| <strong>👋 Welcome!</strong><br><br> | |
| This application uses an AI agent to perform comprehensive topic modeling on research papers. | |
| <br><br> | |
| <strong>How to use:</strong><br> | |
| 1️⃣ Upload a CSV file with <code>Title</code> and <code>Abstract</code> columns (or use the default dataset).<br> | |
| 2️⃣ Configure the minimum number of topics and label generation method in the sidebar.<br> | |
| 3️⃣ Click <strong>"🚀 Run Topic Modeling Agent"</strong> to start the analysis.<br> | |
| 4️⃣ Explore topics, comparisons, and taxonomy classification in the results tabs.<br> | |
| 5️⃣ Review and annotate topics in the <strong>✏️ Review Table</strong> tab.<br> | |
| 6️⃣ View interactive charts in the <strong>📈 Charts</strong> tab.<br> | |
| 7️⃣ Download all outputs as CSV and JSON files. | |
| </div> | |
| """, unsafe_allow_html=True) | |
| st.markdown("<br>", unsafe_allow_html=True) | |
| # Show a preview if default dataset exists | |
| if os.path.exists("dataset.csv"): | |
| with st.expander("👀 Preview Default Dataset", expanded=False): | |
| try: | |
| preview_df = pd.read_csv("dataset.csv", nrows=10) | |
| st.markdown(f"**Columns:** {', '.join(preview_df.columns.tolist())}") | |
| if "Title" in preview_df.columns: | |
| st.dataframe(preview_df[["Title", "Abstract"]].head(10) if "Abstract" in preview_df.columns else preview_df[["Title"]].head(10), use_container_width=True) | |
| else: | |
| st.dataframe(preview_df.head(10), use_container_width=True) | |
| except Exception as e: | |
| st.warning(f"Could not preview dataset: {e}") | |