Spaces:

anushka3306
/

topic_modelling

Sleeping

Update app.py
5697d29 verified 6 months ago
31.8 kB
"""
app.py — Gradio UI for BERTopic Thematic Analysis Agent.
UPGRADED: SPECTER2 embeddings, HDBSCAN+UMAP, 15-30 clusters, 3-LLM voting.
Preserves original UI theme while adding:
  - Scopus CSV file upload
  - Interactive chart viewer (5 charts)
  - Cluster review table with approve/rename/reasoning
  - Phase progress tracker
"""

import gradio as gr
import json
import os
import shutil
import pandas as pd
from pathlib import Path

from agent import invoke_agent

# ── Constants ──────────────────────────────────────────────────────────────────

CHAT_THREAD = "thematic_analysis"
SUMMARIES_FILE = "summaries.json"
PAPERS_FILE = "papers_embedded.csv"

CHARTS = [
    ("🗺️ UMAP Paper Landscape", "chart_umap_scatter.html"),
    ("📊 Cluster Sizes & Cohesion", "chart_cluster_sizes.html"),
    ("🌳 Topic Treemap", "chart_treemap.html"),
    ("📈 Cumulative Coverage", "chart_cumulative.html"),
    ("🎯 Cohesion Distribution", "chart_cohesion_hist.html"),
]

THEMES_CHARTS = [
    ("🌐 Themes Sunburst", "chart_themes_detail.html"),
    ("📊 Themes Comparison", "chart_themes_radar.html"),
]

DOWNLOAD_FILES = [
    ("📄 comparison.csv", "comparison.csv"),
    ("📝 narrative.txt", "narrative.txt"),
    ("🗂️ summaries.json", "summaries.json"),
    ("🗂️ themes.json", "themes.json"),
    ("🗂️ taxonomy_map.json", "taxonomy_map.json"),
]

PHASE_LABELS = [
    "⬜ Phase 1: Load Data",
    "⬜ Phase 2a: Embed & Cluster",
    "⬜ Phase 2b: Label Clusters",
    "⬜ Phase 3: Form Themes",
    "⬜ Phase 4: Review Themes",
    "⬜ Phase 5: Define Themes",
    "⬜ Phase 5.5: PAJAIS Mapping",
    "⬜ Phase 6: Export Report",
]

# ── Helpers ────────────────────────────────────────────────────────────────────

def _read_html(path: str) -> str:
    """Read an HTML chart file or return placeholder."""
    p = Path(path)
    if p.exists():
        return p.read_text(encoding="utf-8")
    return "<div style='padding:40px;text-align:center;color:#888;font-family:sans-serif;'>Chart not yet generated. Run Phase 2 first.</div>"


def _load_summaries_as_df() -> pd.DataFrame:
    """Load summaries.json into a DataFrame for the review table."""
    if not Path(SUMMARIES_FILE).exists():
        return pd.DataFrame(columns=["topic_id", "label", "size", "cohesion_score", "top_sentences", "approved", "rename_to", "reasoning"])
    with open(SUMMARIES_FILE) as f:
        data = json.load(f)
    rows = []
    for t in data:
        rows.append({
            "topic_id": t.get("topic_id", ""),
            "label": t.get("label", ""),
            "size": t.get("size", 0),
            "cohesion_score": round(t.get("cohesion_score", 0.0), 3),
            "top_sentences": " | ".join(t.get("top_sentences", [])[:2])[:300],
            "approved": t.get("approved", False),
            "rename_to": t.get("rename_to", ""),
            "reasoning": t.get("reasoning", ""),
        })
    return pd.DataFrame(rows)


def _save_review_table(df: pd.DataFrame) -> str:
    """Save edited review table back to summaries.json and return a review message for the agent."""
    if not Path(SUMMARIES_FILE).exists():
        return "⚠️ No summaries.json found. Please run Phase 2 first."
    with open(SUMMARIES_FILE) as f:
        summaries = json.load(f)
    # Build lookup from df
    df_dict = {int(row["topic_id"]): row for _, row in df.iterrows()}
    updated = []
    for t in summaries:
        tid = t["topic_id"]
        if tid in df_dict:
            row = df_dict[tid]
            t["approved"] = bool(row.get("approved", False))
            t["rename_to"] = str(row.get("rename_to", "")).strip()
            t["reasoning"] = str(row.get("reasoning", "")).strip()
        updated.append(t)
    with open(SUMMARIES_FILE, "w") as f:
        json.dump(updated, f, indent=2)
    approved_count = sum(1 for t in updated if t.get("approved"))
    return f"✅ Review saved. {approved_count} clusters approved."


def _build_review_message(df: pd.DataFrame) -> str:
    """Build a structured message to send to the agent after review."""
    approved = df[df["approved"] == True]
    lines = ["[REVIEW TABLE SUBMITTED]\n"]
    lines.append(f"Total clusters reviewed: {len(df)}")
    lines.append(f"Approved clusters: {len(approved)}\n")
    for _, row in approved.iterrows():
        name = row.get("rename_to") or row.get("label")
        lines.append(
            f"- Cluster {row['topic_id']}: '{name}' "
            f"(size={row['size']}, cohesion={row['cohesion_score']}) "
            f"| Reasoning: {row.get('reasoning','')}"
        )
    lines.append("\nPlease proceed to Phase 3: Searching for Themes.")
    return "\n".join(lines)


# ── Core UI functions ──────────────────────────────────────────────────────────

def upload_scopus(file_obj):
    """Copy uploaded file to working directory as scopus_data.csv."""
    if file_obj is None:
        return "⚠️ No file uploaded."
    shutil.copy(file_obj.name, "scopus_data.csv")
    df = pd.read_csv("scopus_data.csv", encoding="latin1", nrows=3)
    preview = df.head(3).to_string(max_cols=6, max_colwidth=40)
    cols = list(df.columns)
    return f"✅ **scopus_data.csv** uploaded successfully.\n\nDetected columns: {cols}\n\nPreview:\n```\n{preview}\n```"


def chat(user_message, history):
    """Send message to agent and return updated history."""

    if history is None:
        history = []

    if not user_message.strip():
        return history, ""

    response = invoke_agent(user_message, thread_id=CHAT_THREAD)

    history.append({
        "role": "user",
        "content": user_message
    })

    history.append({
        "role": "assistant",
        "content": response
    })

    return history, ""

def submit_review_and_chat(review_df, history):
    """Save the review table, then send review message to agent."""

    if history is None:
        history = []

    save_msg = _save_review_table(review_df)
    agent_msg = _build_review_message(review_df)

    response = invoke_agent(agent_msg, thread_id=CHAT_THREAD)

    history.append({
        "role": "user",
        "content": f"[Review Submitted] {save_msg}"
    })

    history.append({
        "role": "assistant",
        "content": response
    })

    return history, _load_summaries_as_df()

def refresh_charts():
    """Reload all chart HTML files."""
    return tuple(_read_html(path) for _, path in CHARTS)


def refresh_review_table():
    return _load_summaries_as_df()


def check_downloads():
    """Return list of available download files."""
    available = [(label, path) for label, path in DOWNLOAD_FILES if Path(path).exists()]
    if not available:
        return "No output files yet. Complete the analysis first."
    return "\n".join([f"{label} → `{path}`" for label, path in available])


def _load_themes_json_html() -> str:
    """Render themes.json as a rich HTML card grid for the Themes tab."""
    themes_path = Path("themes.json")
    if not themes_path.exists():
        return (
            "<div style='padding:48px;text-align:center;color:#888;font-family:sans-serif;'>"
            "<h3>📭 No themes.json yet</h3>"
            "<p>Complete Phase 3 (Cluster Review → Submit) to generate themes.</p>"
            "</div>"
        )
    with open(themes_path) as f:
        themes = json.load(f)
    if not themes:
        return "<div style='padding:32px;color:#888;'>themes.json is empty.</div>"

    palette = [
        "#1a73e8","#0d47a1","#e53935","#7b1fa2","#00897b",
        "#f57c00","#2e7d32","#4527a0","#c62828","#00695c",
        "#1565c0","#6a1b9a","#558b2f","#ef6c00","#283593",
        "#ad1457","#00838f","#4e342e","#37474f","#5d4037",
        "#0277bd","#6d4c41","#827717","#1b5e20","#bf360c",
        "#880e4f","#004d40","#212121","#0d47a1","#b71c1c",
    ]

    cards_html = []
    for i, t in enumerate(themes):
        color = palette[i % len(palette)]
        name = t.get("name", "Unnamed")
        size = t.get("size", 0)
        cohesion = t.get("cohesion_score", 0)
        reasoning = t.get("reasoning", "—")
        evidence_items = t.get("evidence", [])
        dois = t.get("top_dois", [])

        # Cohesion badge color
        coh_color = "#2e7d32" if cohesion >= 0.55 else ("#f57c00" if cohesion >= 0.45 else "#c62828")

        ev_html = "".join([
            f"<li style='margin:4px 0;color:#444;font-size:0.82em;line-height:1.45;'>{ev[:160]}{'…' if len(ev)>160 else ''}</li>"
            for ev in evidence_items[:3]
        ])
        doi_html = "".join([
            f"<span style='display:inline-block;background:#e8f0fe;color:#1a73e8;border-radius:4px;"
            f"padding:2px 7px;font-size:0.75em;margin:2px;font-family:monospace;'>{d[:40]}</span>"
            for d in dois[:3]
        ])
        candidates = t.get("label_candidates", [])
        cand_html = (
            "<div style='margin-top:6px;'>"
            + "".join([
                f"<span style='background:#f3e5f5;color:#6a1b9a;border-radius:3px;"
                f"padding:1px 6px;font-size:0.73em;margin:2px;display:inline-block;'>{c}</span>"
                for c in candidates if c != name
            ])
            + "</div>"
        ) if candidates else ""

        cards_html.append(f"""
        <div style="
            background:white; border:1px solid #e0e0e0; border-radius:12px;
            overflow:hidden; box-shadow:0 2px 8px rgba(0,0,0,0.07);
            display:flex; flex-direction:column;
        ">
            <div style="background:{color};padding:14px 16px;">
                <div style="color:white;font-weight:700;font-size:1.0em;line-height:1.3;">
                    {name}
                </div>
                <div style="margin-top:6px;display:flex;gap:8px;align-items:center;flex-wrap:wrap;">
                    <span style="background:rgba(255,255,255,0.22);color:white;padding:2px 9px;
                        border-radius:10px;font-size:0.78em;font-weight:600;">
                        📄 {size} papers
                    </span>
                    <span style="background:rgba(255,255,255,0.22);color:white;padding:2px 9px;
                        border-radius:10px;font-size:0.78em;font-weight:600;"
                        title="Intra-cluster mean cosine similarity">
                        🎯 cohesion {cohesion:.3f}
                    </span>
                    <span style="background:rgba(255,255,255,0.22);color:white;padding:2px 9px;
                        border-radius:10px;font-size:0.78em;">
                        ID #{t.get('theme_id','?')}
                    </span>
                </div>
            </div>
            <div style="padding:14px 16px;flex:1;">
                <div style="font-size:0.8em;color:#555;font-style:italic;margin-bottom:10px;
                    border-left:3px solid {color};padding-left:8px;">
                    {reasoning[:200]}{'…' if len(reasoning)>200 else ''}
                </div>
                <div style="font-size:0.78em;font-weight:600;color:#333;margin-bottom:4px;">
                    📝 Representative Evidence:
                </div>
                <ul style="margin:0;padding-left:16px;">{ev_html}</ul>
                {f'<div style="margin-top:10px;font-size:0.78em;font-weight:600;color:#333;">🔗 Top DOIs:</div><div style="margin-top:4px;">{doi_html}</div>' if doi_html else ''}
                {f'<div style="margin-top:8px;font-size:0.77em;color:#777;">Other candidate labels:{cand_html}</div>' if cand_html else ''}
            </div>
        </div>
        """)

    grid = "\n".join(cards_html)
    return f"""
    <div style="padding:12px;">
        <div style="margin-bottom:16px;padding:10px 16px;background:#e8f5e9;border:1px solid #a5d6a7;
            border-radius:8px;font-size:0.88em;">
            <b>✅ {len(themes)} themes loaded from themes.json</b> ·
            Total papers: {sum(t.get('size',0) for t in themes)} ·
            Avg cohesion: {sum(t.get('cohesion_score',0) for t in themes)/len(themes):.3f}
        </div>
        <div style="
            display:grid;
            grid-template-columns:repeat(auto-fill, minmax(340px, 1fr));
            gap:16px;
        ">
            {grid}
        </div>
    </div>
    """


def refresh_themes_charts():
    return tuple(_read_html(path) for _, path in THEMES_CHARTS)


def refresh_themes_cards():
    return _load_themes_json_html()


# ── Gradio UI ─────────────────────────────────────────────────────────────────

CSS = """
/* ── Global ── */
body { font-family: 'Inter', sans-serif; }
.gradio-container { max-width: 1400px !important; margin: auto; }
/* ── Header ── */
.app-header {
    background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);
    color: white;
    padding: 28px 36px;
    border-radius: 16px;
    margin-bottom: 20px;
    box-shadow: 0 8px 32px rgba(0,0,0,0.3);
}
.app-header h1 { margin: 0; font-size: 1.9em; font-weight: 700; letter-spacing: -0.5px;color:white; }
.app-header p  { margin: 6px 0 0; color: #a8b8d8; font-size: 0.95em; }
.badge {
    display: inline-block;
    background: rgba(255,255,255,0.12);
    border: 1px solid rgba(255,255,255,0.2);
    color: #e0eaff;
    padding: 3px 10px;
    border-radius: 20px;
    font-size: 0.78em;
    margin: 4px 3px 0;
}
/* ── Upload zone ── */
.upload-box { border: 2px dashed #4a90d9 !important; border-radius: 12px !important; }
/* ── Chat ── */
.chatbot-wrap .message { border-radius: 12px !important; }
.chatbot-wrap .bot { background: #f0f4ff !important; }
/* ── Phase tracker ── */
.phase-tracker {
    background: #f8faff;
    border: 1px solid #dce6f9;
    border-radius: 10px;
    padding: 14px 18px;
    font-size: 0.85em;
    line-height: 1.8;
}
/* ── Chart tabs ── */
.chart-tab iframe, .chart-tab .html-container { height: 620px; border: none; border-radius: 10px; }
/* ── Review table ── */
#review-table .gradio-dataframe { font-size: 0.82em; }
/* ── Buttons ── */
.btn-primary {
    background: linear-gradient(135deg, #1a73e8, #0d47a1) !important;
    color: white !important; border-radius: 8px !important; font-weight: 600 !important;
}
.btn-secondary {
    background: #f1f3f4 !important; color: #333 !important;
    border-radius: 8px !important;
}
.btn-success {
    background: linear-gradient(135deg, #2e7d32, #1b5e20) !important;
    color: white !important; border-radius: 8px !important; font-weight: 600 !important;
}
/* ── Tabs ── */
.tab-nav button { font-weight: 600; font-size: 0.9em; }
/* ── Section titles ── */
.section-title {
    font-size: 1.05em; font-weight: 700; color: #1a1a2e;
    border-left: 4px solid #1a73e8; padding-left: 10px; margin: 8px 0;
}
"""

def build_ui():
    with gr.Blocks(title="Thematic Analysis Agent — SPECTER2") as demo:

        # ── Header ────────────────────────────────────────────────────────────
        gr.HTML("""
        <div class="app-header">
            <h1>🔬 Computational Thematic Analysis Agent</h1>
            <p>Braun &amp; Clarke (2006) · SPECTER2 Scientific Embeddings · HDBSCAN+UMAP Clustering · 3-LLM Voting</p>
            <span class="badge">📚 SPECTER2 (Allen AI)</span>
            <span class="badge">🗂️ Scopus CSV</span>
            <span class="badge">🎯 15–30 Clusters</span>
            <span class="badge">📰 IJEC Journal</span>
            <span class="badge">🤖 Mistral AI</span>
        </div>
        """)

        with gr.Tabs():

            # ════════════════════════════════════════════════════════
            # TAB 1 — Upload & Chat
            # ════════════════════════════════════════════════════════
            with gr.TabItem("💬 Analysis Chat", id="tab_chat"):
                with gr.Row():

                    # ── Left column: upload + phase tracker ───────
                    with gr.Column(scale=1, min_width=280):
                        gr.HTML('<div class="section-title">📂 Upload Scopus Export</div>')
                        file_input = gr.File(
                            label="Drop scopus_data.csv here",
                            file_types=[".csv"],
                            elem_classes=["upload-box"],
                        )
                        upload_btn = gr.Button("⬆️ Upload & Validate", elem_classes=["btn-primary"])
                        upload_status = gr.Markdown("*No file uploaded yet.*")

                        gr.HTML('<div class="section-title" style="margin-top:18px">📋 Phase Progress</div>')
                        phase_display = gr.HTML(
                            '<div class="phase-tracker">'
                            + "<br>".join(PHASE_LABELS)
                            + "</div>"
                        )

                        gr.HTML('<div class="section-title" style="margin-top:18px">⚡ Quick Actions</div>')
                        with gr.Row():
                            btn_p1 = gr.Button("▶ Start Phase 1", size="sm", elem_classes=["btn-secondary"])
                            btn_p2 = gr.Button("▶ Phase 2", size="sm", elem_classes=["btn-secondary"])
                        with gr.Row():
                            btn_p4 = gr.Button("▶ Phase 4", size="sm", elem_classes=["btn-secondary"])
                            btn_p5 = gr.Button("▶ Phase 5", size="sm", elem_classes=["btn-secondary"])
                        with gr.Row():
                            btn_p55 = gr.Button("▶ Phase 5.5", size="sm", elem_classes=["btn-secondary"])
                            btn_p6 = gr.Button("▶ Phase 6", size="sm", elem_classes=["btn-secondary"])

                        gr.HTML('<div class="section-title" style="margin-top:18px">💾 Downloads</div>')
                        download_info = gr.Markdown("*Run analysis to generate outputs.*")
                        refresh_dl_btn = gr.Button("🔄 Check Outputs", size="sm", elem_classes=["btn-secondary"])

                    # ── Right column: chatbot ──────────────────────
                    with gr.Column(scale=3):
                        gr.HTML('<div class="section-title">🤖 Agent Conversation</div>')
                        chatbot = gr.Chatbot(
                                label="Thematic Analysis Agent",
                                height=560,
                                
                                elem_classes=["chatbot-wrap"],
                                avatar_images=(None, "https://em-content.zobj.net/source/twitter/376/robot_1f916.png"),
                            )
                        with gr.Row():
                            msg_box = gr.Textbox(
                                placeholder="Type a message or use Quick Actions above…",
                                label="", show_label=False, scale=5,
                                container=False,
                            )
                            send_btn = gr.Button("Send ➤", scale=1, elem_classes=["btn-primary"])
                        clear_btn = gr.Button("🗑️ Clear Chat", size="sm", elem_classes=["btn-secondary"])

            # ════════════════════════════════════════════════════════
            # TAB 2 — Cluster Review Table
            # ════════════════════════════════════════════════════════
            with gr.TabItem("🔍 Cluster Review", id="tab_review"):
                gr.HTML("""
                <div style="background:#fff8e1;border:1px solid #ffe082;border-radius:10px;padding:14px 18px;margin-bottom:14px;">
                    <b>📋 Instructions:</b> After Phase 2 completes, this table will populate with all discovered clusters.
                    Check <b>approved</b> for clusters you want to keep. Optionally rename them in <b>rename_to</b>
                    and add <b>reasoning</b>. Then click <b>Submit Review to Agent</b> to trigger Phase 3.
                </div>
                """)
                with gr.Row():
                    refresh_table_btn = gr.Button("🔄 Refresh Table from Phase 2", elem_classes=["btn-secondary"])
                    submit_review_btn = gr.Button("✅ Submit Review → Phase 3", elem_classes=["btn-success"])

                review_table = gr.Dataframe(
                    headers=["topic_id", "label", "size", "cohesion_score", "top_sentences", "approved", "rename_to", "reasoning"],
                    datatype=["number", "str", "number", "number", "str", "bool", "str", "str"],
                    interactive=True,
                    wrap=True,
                    elem_id="review-table",
                    label="Cluster Review Table (edit inline)",
                   
                )
                review_status = gr.Markdown("")

            # ════════════════════════════════════════════════════════
            # TAB 3 — Cluster Charts & Visualisations
            # ════════════════════════════════════════════════════════
            with gr.TabItem("📊 Cluster Visualisations", id="tab_charts"):
                gr.HTML("""
                <div style="background:#e8f5e9;border:1px solid #a5d6a7;border-radius:10px;padding:12px 18px;margin-bottom:12px;">
                    <b>📈 5 interactive cluster charts generated after Phase 2b (LLM labeling).</b>
                    Charts auto-update with real cluster names after each phase.
                    <b>Hover</b> any point/bar to see cluster name, paper count, DOI &amp; title snippet.
                    Click legend items to show/hide clusters. Zoom, pan, download as PNG.
                </div>
                """)
                refresh_charts_btn = gr.Button("🔄 Refresh All Charts", elem_classes=["btn-secondary"])

                with gr.Tabs(elem_classes=["chart-tab"]):
                    chart_components = []
                    for label, path in CHARTS:
                        with gr.TabItem(label):
                            c = gr.HTML(_read_html(path))
                            chart_components.append(c)

            # ════════════════════════════════════════════════════════
            # TAB 4 — Themes Viewer (themes.json cards + charts)
            # ════════════════════════════════════════════════════════
            with gr.TabItem("🎨 Themes Viewer", id="tab_themes"):
                gr.HTML("""
                <div style="background:#fce4ec;border:1px solid #f48fb1;border-radius:10px;padding:12px 18px;margin-bottom:12px;">
                    <b>🎨 Visual explorer for themes.json</b> — generated after Phase 3 (Submit Review).
                    Card grid shows every theme with name, paper count, cohesion score, reasoning, evidence &amp; DOIs.
                    Charts show the sunburst structure and side-by-side comparison.
                </div>
                """)
                with gr.Row():
                    refresh_themes_cards_btn = gr.Button("🔄 Refresh Theme Cards", elem_classes=["btn-secondary"])
                    refresh_themes_charts_btn = gr.Button("🔄 Refresh Theme Charts", elem_classes=["btn-secondary"])

                # Card grid
                themes_cards_html = gr.HTML(_load_themes_json_html())

                # Theme-level charts in sub-tabs
                with gr.Tabs(elem_classes=["chart-tab"]):
                    themes_chart_components = []
                    for label, path in THEMES_CHARTS:
                        with gr.TabItem(label):
                            tc = gr.HTML(_read_html(path))
                            themes_chart_components.append(tc)

            # ════════════════════════════════════════════════════════
            # TAB 5 — Methodology Info
            # ════════════════════════════════════════════════════════
            with gr.TabItem("ℹ️ Methodology", id="tab_info"):
                gr.Markdown("""
## 🔬 Methodology Reference
### Embedding: SPECTER2 (Allen Institute for AI)
- **Model**: `allenai/specter2_base`
- **Input**: One combined `Title [SEP] Abstract` text per paper → one 768-dim vector
- **Advantage over general models**: Pre-trained on 146M citation links from Semantic Scholar.
  Captures scientific discourse, domain-specific terminology, and paper-level semantics.
- **Previous version (SPECTER)** used in published papers; SPECTER2 improves on it with
  multi-task adapter architecture.
### Dimensionality Reduction: UMAP
- **Parameters**: `n_components=10`, `n_neighbors=15`, `metric=cosine`, `random_state=42`
- Preserves both local cluster structure and global topology
- Cosine metric appropriate for normalized scientific embeddings
### Clustering: HDBSCAN
- **Type**: Density-based hierarchical clustering (no fixed cluster count)
- `min_cluster_size=5` (minimum papers per cluster)
- Max ~100 papers per cluster enforced via parameter search
- Papers not assignable to any cluster → labeled as **Noise** (-1)
- **Target intra-cluster cosine similarity**: 0.50–0.60
### Cluster Count Constraint: 15–30
- Automatic parameter grid search (min_cluster_size × min_samples combinations)
- Selects parameters yielding 15–30 valid clusters for meaningful IJEC discussion section
### Labeling: 3-LLM Majority Vote
- Each cluster's top-3 representative paper titles sent to Mistral 3 independent times
- Most common label wins (majority vote)
- Ensures robust, reproducible labels unaffected by single LLM stochasticity
### Braun & Clarke (2006) 6-Phase Framework
1. Familiarisation → 2. Generating Initial Codes → 3. Searching for Themes →
4. Reviewing Themes → 5. Defining & Naming Themes → 6. Producing the Report
### Visualization Pipeline
| Chart | Generated After | Shows |
|-------|----------------|-------|
| UMAP Paper Landscape | Phase 2a (updated after 2b & 3) | All papers as scatter points, colored+labeled by cluster name at centroid |
| Cluster Sizes & Cohesion | Phase 2a (updated after 2b & 3) | Horizontal bars with real names + cohesion diamonds |
| Topic Treemap | Phase 2a (updated after 2b & 3) | Area=papers, color=cohesion, real names in cells |
| Cumulative Coverage | Phase 2a (updated after 2b & 3) | Coverage curve with top-8 cluster name annotations |
| Cohesion Distribution | Phase 2a (updated after 2b & 3) | Histogram + per-cluster labeled dots |
| Themes Sunburst | Phase 3 | theme → evidence sentences, color=cohesion |
| Themes Comparison | Phase 3 | Final themes bar chart with reasoning on hover |
### Output Files
| File | Description |
|------|-------------|
| `comparison.csv` | All themes with PAJAIS mapping, cohesion, DOIs |
| `narrative.txt` | Academic Section 7 (IJEC tone) |
| `summaries.json` | Full cluster data with top sentences |
| `themes.json` | Approved themes (viewed in Themes Viewer tab) |
| `taxonomy_map.json` | PAJAIS-mapped themes |
| `chart_umap_scatter.html` | UMAP paper landscape with centroid labels |
| `chart_cluster_sizes.html` | Cluster sizes + cohesion overlay (horizontal) |
| `chart_treemap.html` | Treemap colored by cohesion |
| `chart_cumulative.html` | Cumulative coverage with name annotations |
| `chart_cohesion_hist.html` | Cohesion distribution with per-cluster dots |
| `chart_themes_detail.html` | Themes sunburst (structure + evidence) |
| `chart_themes_radar.html` | Themes comparison bar chart |
                """)

        # ── Event Wiring ───────────────────────────────────────────────────────

        # Upload
        upload_btn.click(upload_scopus, inputs=[file_input], outputs=[upload_status])

        # Chat
        send_btn.click(chat, inputs=[msg_box, chatbot], outputs=[chatbot, msg_box])
        msg_box.submit(chat, inputs=[msg_box, chatbot], outputs=[chatbot, msg_box])
        clear_btn.click(
            lambda: ([], ""),
            outputs=[chatbot, msg_box]
        )

        # Quick action buttons
        btn_p1.click(
            lambda h: chat("Please start Phase 1: load the Scopus data.", h),
            inputs=[chatbot], outputs=[chatbot, msg_box]
        )
        btn_p2.click(
            lambda h: chat("Please proceed to Phase 2: run embedding and clustering.", h),
            inputs=[chatbot], outputs=[chatbot, msg_box]
        )
        btn_p4.click(
            lambda h: chat("Please proceed to Phase 4: review the themes.", h),
            inputs=[chatbot], outputs=[chatbot, msg_box]
        )
        btn_p5.click(
            lambda h: chat("Please proceed to Phase 5: define and name the themes.", h),
            inputs=[chatbot], outputs=[chatbot, msg_box]
        )
        btn_p55.click(
            lambda h: chat("Please proceed to Phase 5.5: PAJAIS taxonomy mapping.", h),
            inputs=[chatbot], outputs=[chatbot, msg_box]
        )
        btn_p6.click(
            lambda h: chat("Please proceed to Phase 6: produce the final report.", h),
            inputs=[chatbot], outputs=[chatbot, msg_box]
        )

        # Review table
        refresh_table_btn.click(refresh_review_table, outputs=[review_table])
        submit_review_btn.click(
            submit_review_and_chat,
            inputs=[review_table, chatbot],
            outputs=[chatbot, review_table],
        )

        # Charts
        refresh_charts_btn.click(refresh_charts, outputs=chart_components)

        # Themes
        refresh_themes_cards_btn.click(refresh_themes_cards, outputs=[themes_cards_html])
        refresh_themes_charts_btn.click(refresh_themes_charts, outputs=themes_chart_components)

        # Downloads
        refresh_dl_btn.click(check_downloads, outputs=[download_info])

    return demo


# ── Entry point ───────────────────────────────────────────────────────────────

if __name__ == "__main__":
    app = build_ui()
    app.launch(
        server_name="0.0.0.0",
        server_port=7860,
        share=True,
        show_error=True,
    )