Delete app.py

app.py

@@ -1,944 +0,0 @@
-"""
-EGM Research Assistant — RAG + Visualisations
-Gradio 5.x | sentence-transformers → ChromaDB → Claude + Plotly via HTML
-"""
-
-import os, re
-import pandas as pd
-import chromadb
-from chromadb.utils import embedding_functions
-from anthropic import Anthropic
-import gradio as gr
-import plotly.graph_objects as go
-from plotly.subplots import make_subplots
-from collections import Counter
-
-# ---------------------------------------------------------------------------
-# Config
-# ---------------------------------------------------------------------------
-EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
-COLLECTION  = "egm_studies"
-MAX_TOKENS  = 2000
-BATCH_SIZE  = 50
-
-REGION_COLOURS = {
-    "Sub-Saharan Africa":           "#2ecc71",
-    "Latin America and Caribbean":  "#3498db",
-    "East Asia and Pacific":        "#e74c3c",
-    "South Asia":                   "#f39c12",
-    "Middle East and North Africa": "#9b59b6",
-    "Europe and Central Asia":      "#1abc9c",
-    "Multi-continent":              "#95a5a6",
-}
-
-# ---------------------------------------------------------------------------
-# Load dataset
-# ---------------------------------------------------------------------------
-print("Loading dataset...")
-df = pd.read_csv("dataset.csv").fillna("")
-
-# Normalise column name for PSE strategy (long name)
-PSE_COL  = "pse_strategy_multi_select_separate_values_with_single_vertical_bar"
-CONF_COL = "overall_how_much_confidence_do_you_have_in_the_methods_used_to_analyse_the_findings_relative_to_the_primary_question_addressed_in_the_review"
-
-df["year_of_publication"] = (
-    df["year_of_publication"].astype(str).str.replace(r"\.0$", "", regex=True)
-)
-RECORDS = df.to_dict("records")
-TOTAL   = len(RECORDS)
-print(f"Loaded {TOTAL} studies.")
-
-# ---------------------------------------------------------------------------
-# Build embedding text — rich multi-field document
-# ---------------------------------------------------------------------------
-def make_doc(r):
-    parts = [
-        r.get("title", ""),
-        "Authors: "                + r.get("authors", ""),
-        "Intervention: "           + r.get("intervention", ""),
-        "Intervention detail: "    + r.get("intervention_description", "")[:200],
-        "Intervention category: "  + r.get("intervention_category", ""),
-        "Outcome: "                + r.get("outcome", ""),
-        "Outcome category: "       + r.get("outcome_categories_assigned", ""),
-        "Theme: "                  + r.get("primary_theme", ""),
-        "Sub-theme: "              + r.get("sub_primary_theme", ""),
-        "Sector: "                 + r.get("sector_name", ""),
-        "SDG: "                    + r.get("un_sustainable_development_goal", ""),
-        "Evaluation design: "      + r.get("evaluation_design", ""),
-        "Evaluation method: "      + r.get("evaluation_method", ""),
-        "PSE strategy: "           + r.get(PSE_COL, ""),
-        "Country: "                + r.get("country", ""),
-        "Region: "                 + r.get("region", ""),
-        "Income level: "           + r.get("income_level", ""),
-        "Equity focus: "           + r.get("equity_focus", ""),
-        "Population: "             + r.get("population_description", "")[:150],
-        "Record type: "            + r.get("record_type", ""),
-        "Method eval: "            + r.get("method_eval", ""),
-        "Review confidence: "      + r.get(CONF_COL, ""),
-        "Keywords: "               + r.get("keywords", ""),
-        "Other topics: "           + r.get("other_topics", ""),
-        r.get("abstract", "")[:600],
-    ]
-    return " | ".join(p for p in parts if p.strip() and not p.strip().endswith(": "))
-
-DOCS  = [make_doc(r) for r in RECORDS]
-IDS   = [str(i) for i in range(TOTAL)]
-METAS = [
-    {
-        "title":   r["title"][:200],
-        "authors": r["authors"][:100],
-        "year":    r["year_of_publication"],
-        "country": r["country"][:80],
-        "url":     r.get("url_link", "")[:200] if str(r.get("url_link","")).startswith("http") else "",
-    }
-    for i, r in enumerate(RECORDS)
-]
-
-# ---------------------------------------------------------------------------
-# ChromaDB vector index
-# ---------------------------------------------------------------------------
-print("Building vector index...")
-ef     = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=EMBED_MODEL)
-chroma = chromadb.Client()
-
-try:
-    chroma.delete_collection(COLLECTION)
-except Exception:
-    pass
-
-col = chroma.create_collection(
-    name=COLLECTION,
-    embedding_function=ef,
-    metadata={"hnsw:space": "cosine"},
-)
-
-for start in range(0, TOTAL, BATCH_SIZE):
-    end = min(start + BATCH_SIZE, TOTAL)
-    col.add(ids=IDS[start:end], documents=DOCS[start:end], metadatas=METAS[start:end])
-    print(f"  Indexed {end}/{TOTAL} studies")
-
-print("Vector index ready ✓")
-
-# ---------------------------------------------------------------------------
-# Anthropic client + system prompt
-# ---------------------------------------------------------------------------
-anthropic_client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY", ""))
-
-SYSTEM = f"""You are a research synthesis assistant for an Evidence Gap Map (EGM) \
-containing {TOTAL} studies on private sector engagement and development interventions.
-
-STRICT RULES:
-1. Answer ONLY using the studies explicitly provided in each message. Never use outside knowledge.
-2. Only state what is explicitly present in the study data — do not generalise or assume.
-3. Respect the diversity of study designs: studies may be Experimental, Quasi-experimental, 
-   or Qualitative. Never describe the whole dataset as quantitative or experimental.
-4. Every factual claim must have an inline [N] citation to a specific provided study.
-5. If evidence is insufficient, say so and describe what the retrieved studies do cover.
-
-RESPONSE FORMAT:
-**Overview** — 2–3 sentences summarising what the evidence shows, noting study design diversity.
-**Key Findings** — bullets grouped by theme/intervention, each with [N] citations.
-**Study Designs & Methods** — note the methodological mix (RCTs, DiD, qualitative, etc.)
-**Geographic & Context Coverage** — regions, income levels, fragile/conflict-affected states if relevant.
-**Equity & Population Focus** — note gender, equity dimensions, and target populations if relevant.
-**Gaps & Caveats** — what is not well evidenced in the retrieved studies.
-**Key Takeaways** — 2–3 concise conclusions from the provided studies only.
-
-Use **bold** for key terms. Use [N] inline for every claim."""
-
-# ---------------------------------------------------------------------------
-# Retrieval + context builder
-# ---------------------------------------------------------------------------
-def retrieve(query, top_k):
-    top_k = min(int(top_k), TOTAL)
-    results = col.query(query_texts=[query], n_results=top_k,
-                        include=["metadatas", "distances"])
-    studies = []
-    for i, doc_id in enumerate(results["ids"][0]):
-        rec = RECORDS[int(doc_id)].copy()
-        rec["_rank"] = i + 1
-        studies.append(rec)
-    return studies
-
-
-def build_context(studies):
-    """
-    Two-tier context to stay within token limits:
-    - All studies: compact single-line format (~40 tokens each)
-    - <=80 studies: also include a short abstract snippet
-    This lets the full 358-study dataset fit within the 30k token limit.
-    """
-    include_abstract = len(studies) <= 80
-    parts = []
-    for i, s in enumerate(studies, 1):
-        line = (
-            f"[{i}] \"{s['title']}\" | "
-            f"{s['authors']} ({s['year_of_publication']}) | "
-            f"{s['country']} | {s['region']} | {s['income_level']} | "
-            f"{s['record_type']} | {s['method_eval']} | "
-            f"{s['evaluation_design']} | {s['evaluation_method']} | "
-            f"Intervention: {s['intervention']} [{s['intervention_category']}] | "
-            f"Outcome: {s['outcome']} [{s['outcome_categories_assigned']}] | "
-            f"Theme: {s['primary_theme']} > {s['sub_primary_theme']} | "
-            f"Sector: {s['sector_name']} | Equity: {s['equity_focus']} | "
-            f"Confidence: {s[CONF_COL]}"
-        )
-        if include_abstract and s["abstract"].strip():
-            line += f"\n  Abstract: {s['abstract'][:250]}"
-        parts.append(line)
-    return "\n".join(parts)
-
-
-# ---------------------------------------------------------------------------
-# Charts rendered as HTML (avoids gr.Plot version conflicts)
-# ---------------------------------------------------------------------------
-CHART_BG   = "#f8f9fa"
-GRID_COLOR = "#e9ecef"
-
-def make_chart_html(studies):
-    # Interventions
-    ivs = []
-    for s in studies:
-        for iv in s.get("intervention", "").split("|"):
-            iv = iv.strip()
-            if iv and iv.lower() not in ("", "not applicable", "nan"):
-                ivs.append(iv[:45] + ("…" if len(iv) > 45 else ""))
-    iv_counts = Counter(ivs).most_common(8)
-
-    # Regions
-    regions = []
-    for s in studies:
-        r = s.get("region", "").split("|")[0].strip()
-        if r and r.lower() not in ("", "not applicable", "nan"):
-            regions.append(r)
-    region_counts = Counter(regions)
-
-    # Study designs
-    designs = [s.get("evaluation_design","").strip() for s in studies
-               if s.get("evaluation_design","").strip() not in ("","Not applicable","nan")]
-    design_counts = Counter(designs)
-
-    # Years
-    years = []
-    for s in studies:
-        try: years.append(int(float(s.get("year_of_publication",""))))
-        except: pass
-    sorted_years = sorted(Counter(years).items())
-
-    fig = make_subplots(
-        rows=2, cols=2,
-        subplot_titles=("Top interventions","Regional distribution",
-                        "Study designs","Publication years"),
-        specs=[[{"type":"xy"},{"type":"domain"}],
-               [{"type":"domain"},{"type":"xy"}]],
-        vertical_spacing=0.20, horizontal_spacing=0.10,
-    )
-
-    if iv_counts:
-        labels = [x[0] for x in iv_counts][::-1]
-        values = [x[1] for x in iv_counts][::-1]
-        fig.add_trace(go.Bar(x=values, y=labels, orientation="h",
-            marker_color="#3498db", marker_line_width=0,
-            hovertemplate="%{y}: %{x} studies<extra></extra>"), row=1, col=1)
-
-    if region_counts:
-        rl = list(region_counts.keys())
-        rv = list(region_counts.values())
-        fig.add_trace(go.Pie(labels=rl, values=rv, hole=0.45, showlegend=False,
-            marker_colors=[REGION_COLOURS.get(r,"#bdc3c7") for r in rl],
-            textinfo="label+percent",
-            hovertemplate="%{label}: %{value}<extra></extra>"), row=1, col=2)
-
-    if design_counts:
-        dl = list(design_counts.keys())
-        dv = list(design_counts.values())
-        dc = {"Experimental":"#e74c3c","Quasi-experimental":"#3498db","Qualitative":"#2ecc71"}
-        fig.add_trace(go.Pie(labels=dl, values=dv, hole=0.45, showlegend=False,
-            marker_colors=[dc.get(d,"#95a5a6") for d in dl],
-            textinfo="label+percent",
-            hovertemplate="%{label}: %{value}<extra></extra>"), row=2, col=1)
-
-    if sorted_years:
-        fig.add_trace(go.Scatter(
-            x=[x[0] for x in sorted_years], y=[x[1] for x in sorted_years],
-            mode="lines+markers", fill="tozeroy",
-            line=dict(color="#e74c3c", width=2.5), marker=dict(size=7, color="#e74c3c"),
-            fillcolor="rgba(231,76,60,0.10)",
-            hovertemplate="Year %{x}: %{y} studies<extra></extra>"), row=2, col=2)
-
-    fig.update_layout(height=560, paper_bgcolor=CHART_BG, plot_bgcolor=CHART_BG,
-        font=dict(family="Inter,sans-serif", size=12, color="#2c3e50"),
-        margin=dict(t=50, b=20, l=10, r=10), showlegend=False)
-    fig.update_xaxes(showgrid=True, gridcolor=GRID_COLOR, zeroline=False)
-    fig.update_yaxes(showgrid=True, gridcolor=GRID_COLOR, zeroline=False)
-    for ann in fig.layout.annotations:
-        ann.font = dict(size=12, color="#2c3e50")
-
-    try:
-        return fig.to_html(full_html=False, include_plotlyjs=False, config={"responsive":True})
-    except Exception as e:
-        return f"<p style='color:red'>Chart error: {e}</p>"
-
-
-EMPTY_HTML = """<div style="height:180px;display:flex;align-items:center;justify-content:center;
-background:#f8f9fa;border-radius:8px;color:#adb5bd;font-size:14px;font-family:Inter,sans-serif;">
-Ask a question to see the evidence visualised here</div>"""
-
-# ---------------------------------------------------------------------------
-# Chat — Gradio 5.x messages format
-# ---------------------------------------------------------------------------
-def chat(user_message, history, top_k):
-    if not user_message.strip():
-        yield history, EMPTY_HTML
-        return
-
-    studies  = retrieve(user_message, top_k)
-    context  = build_context(studies)
-    augmented = (
-        f"RETRIEVED STUDIES ({len(studies)} of {TOTAL} total):\n\n{context}\n\n---\n"
-        f"Using ONLY the studies above, answer:\n{user_message}"
-    )
-
-    api_messages = []
-    for turn in history:
-        role    = turn.get("role", "")
-        content = turn.get("content", "")
-        if isinstance(content, list):
-            content = " ".join(b.get("text","") if isinstance(b,dict) else str(b) for b in content)
-        if role in ("user","assistant") and content:
-            api_messages.append({"role": role, "content": content})
-    api_messages.append({"role": "user", "content": augmented})
-
-    history = history + [
-        {"role": "user",      "content": user_message},
-        {"role": "assistant", "content": ""},
-    ]
-
-    partial = ""
-    try:
-        stream_ctx = anthropic_client.messages.stream(
-            model="claude-sonnet-4-20250514",
-            max_tokens=MAX_TOKENS,
-            system=SYSTEM,
-            messages=api_messages,
-        )
-        with stream_ctx as stream:
-            for text in stream.text_stream:
-                partial += text
-                history[-1]["content"] = partial
-                yield history, EMPTY_HTML
-    except Exception as e:
-        err = str(e)
-        if "rate_limit" in err or "429" in err:
-            history[-1]["content"] = "⚠️ **Rate limit reached.** Too many studies were sent at once. Please reduce the **Studies retrieved** slider to 20–25 and try again."
-        else:
-            history[-1]["content"] = f"⚠️ **API error:** {err[:300]}"
-        yield history, EMPTY_HTML
-        return
-
-    # Append reference list
-    cited = sorted(set(int(n) for n in re.findall(r"\[(\d+)\]", partial)))
-    if cited:
-        refs = ["\n\n---\n**Studies cited:**"]
-        for n in cited:
-            if 1 <= n <= len(studies):
-                s   = studies[n-1]
-                url = s.get("url_link","")
-                lnk = f"[{s['title']}]({url})" if url and url.startswith("http") else s["title"]
-                refs.append(
-                    f"**[{n}]** {lnk}  \n"
-                    f"*{s['authors']} ({s['year_of_publication']}) · "
-                    f"{s['country']} · {s['evaluation_design']}*"
-                )
-        history[-1]["content"] += "\n".join(refs)
-
-    try:
-        chart_html = make_chart_html(studies)
-    except Exception as e:
-        chart_html = f"<p style='color:#e74c3c;font-family:sans-serif'>Chart could not be generated: {e}</p>"
-    yield history, chart_html
-
-
-def clear_chat():
-    return [], EMPTY_HTML
-
-# ---------------------------------------------------------------------------
-# Gradio 5.x UI
-# ---------------------------------------------------------------------------
-
-# ---------------------------------------------------------------------------
-# Dataset Summary — direct column counts, no RAG/LLM needed
-# ---------------------------------------------------------------------------
-
-# Columns available for summary, with friendly labels
-SUMMARY_COLS = {
-    "Intervention Category":    "intervention_category",
-    "Outcome Category":         "outcome_categories_assigned",
-    "Primary Theme":            "primary_theme",
-    "Sub-theme":                "sub_primary_theme",
-    "Sector":                   "sector_name",
-    "Evaluation Design":        "evaluation_design",
-    "Method / Study Type":      "method_eval",
-    "Region":                   "region",
-    "Country":                  "country",
-    "Income Level":             "income_level",
-    "Equity Focus":             "equity_focus",
-    "Record Type":              "record_type",
-    "Publication Type":         "publication_type",
-    "SDG":                      "un_sustainable_development_goal",
-    "Funding Agency":           "program_funding_agency",
-    "Implementation Agency":    "implementation_agencies",
-}
-
-SUMMARY_COLOURS = [
-    "#3498db","#2ecc71","#e74c3c","#f39c12","#9b59b6",
-    "#1abc9c","#e67e22","#34495e","#e91e63","#00bcd4",
-    "#8bc34a","#ff5722","#607d8b","#795548","#9c27b0",
-]
-
-def make_summary_chart(col_label):
-    col = SUMMARY_COLS.get(col_label)
-    if not col:
-        return "<p>Select a column to summarise.</p>"
-
-    series = df[col].replace("", pd.NA).dropna()
-    if series.empty:
-        return f"<p>No data available for <b>{col_label}</b>.</p>"
-
-    all_vals = []
-    for v in series:
-        for part in str(v).split("|"):
-            part = part.strip()
-            if part and part.lower() not in ("not applicable", "nan", "n/a", "multi-country"):
-                all_vals.append(part)
-
-    counts   = Counter(all_vals).most_common(20)
-    if not counts:
-        return f"<p>No data available for <b>{col_label}</b>.</p>"
-
-    total    = sum(v for _, v in counts)
-    n_cats   = len(Counter(all_vals))
-    coverage = len(series)
-    missing  = TOTAL - coverage
-    top1     = counts[0]
-    top2     = counts[1] if len(counts) > 1 else None
-    top3     = counts[2] if len(counts) > 2 else None
-    top3_sum = sum(v for _, v in counts[:3])
-
-    # Full paragraph description
-    cat_word = "category" if n_cats == 1 else "categories"
-    rem_word = "category" if n_cats - 3 == 1 else "categories"
-    para = (
-        f"Across {coverage} studies with available data, "
-        f"<b>{col_label}</b> spans <b>{n_cats} distinct {"category" if n_cats == 1 else "categories"}</b>. "
-        f"The most represented is <b>{top1[0]}</b> with {top1[1]} studies ({top1[1]/total*100:.1f}% of all values)"
-    )
-    if top2:
-        para += f", followed by <b>{top2[0]}</b> ({top2[1]} studies, {top2[1]/total*100:.1f}%)"
-    if top3:
-        para += f" and <b>{top3[0]}</b> ({top3[1]} studies, {top3[1]/total*100:.1f}%)"
-    para += f". Together, the top 3 categories account for <b>{top3_sum/total*100:.0f}%</b> of all values"
-    if n_cats > 3:
-        para += f", while the remaining {n_cats - 3} {"category" if n_cats - 3 == 1 else "categories"} collectively represent {(total - top3_sum)/total*100:.0f}%"
-    para += ". "
-    if missing > 0:
-        para += f"Note: {missing} of {TOTAL} studies have no value recorded for this field."
-    else:
-        para += f"Data is complete — all {TOTAL} studies have a value for this field."
-
-    desc_html = f"""<div style="background:linear-gradient(135deg,#0a2342 0%,#1a3a5c 100%);
-        border-radius:10px;padding:18px 22px;margin-bottom:14px;
-        font-family:Inter,sans-serif;font-size:14px;line-height:1.8;color:#ffffff;
-        box-shadow:0 4px 16px rgba(10,35,66,0.25);">
-        <div style="font-size:12px;font-weight:700;color:rgba(255,255,255,0.7);
-            margin-bottom:8px;text-transform:uppercase;letter-spacing:0.8px;">
-            {col_label} &nbsp;·&nbsp; {n_cats} categories &nbsp;·&nbsp; {coverage} studies
-        </div>
-        <span style="color:#ffffff;">{para}</span>
-    </div>"""
-
-    # World map for Country
-    if col == "country":
-        country_counts = Counter(all_vals)
-        name_map = {
-            "Egypt, Arab Rep.": "Egypt", "Iran, Islamic Rep.": "Iran",
-            "Korea, Rep.": "South Korea", "Congo, Dem. Rep.": "Democratic Republic of the Congo",
-            "Congo, Rep.": "Republic of the Congo", "Gambia, The": "Gambia",
-            "Yemen, Rep.": "Yemen", "West Bank and Gaza": "Palestine",
-            "Russian Federation": "Russia", "Côte d’Ivoire": "Ivory Coast",
-        }
-        map_c = [name_map.get(c, c) for c in country_counts.keys()]
-        map_v = list(country_counts.values())
-
-        fig = go.Figure(go.Choropleth(
-            locations=map_c, locationmode="country names", z=map_v,
-            colorscale=[[0.0,"#d4e6f1"],[0.15,"#85c1e9"],[0.35,"#3498db"],
-                        [0.60,"#1a5276"],[1.0,"#0a2342"]],
-            colorbar=dict(title="Studies", thickness=14, len=0.6, tickfont=dict(size=11)),
-            hovertemplate="<b>%{location}</b><br>Studies: %{z}<extra></extra>",
-            marker_line_color="white", marker_line_width=0.5,
-        ))
-        fig.update_layout(
-            height=460, paper_bgcolor="#f8f9fa",
-            geo=dict(showframe=False, showcoastlines=True, coastlinecolor="#ccc",
-                     showland=True, landcolor="#f0f0f0", showocean=True,
-                     oceancolor="#eaf4fb", showlakes=False,
-                     projection_type="natural earth", bgcolor="#f8f9fa"),
-            margin=dict(t=10, b=10, l=0, r=0),
-            font=dict(family="Inter,sans-serif", size=12),
-        )
-        try:
-            return desc_html + fig.to_html(full_html=False, include_plotlyjs=False,
-                                           config={"responsive": True})
-        except Exception as e:
-            return desc_html + f"<p style='color:red'>Map error: {e}</p>"
-
-    # Bar chart for all other columns
-    labels = [x[0] for x in counts][::-1]
-    values = [x[1] for x in counts][::-1]
-    pcts   = [f"{v/total*100:.1f}%" for v in values]
-    colors = [SUMMARY_COLOURS[i % len(SUMMARY_COLOURS)] for i in range(len(labels))]
-
-    fig = go.Figure()
-    fig.add_trace(go.Bar(
-        x=values, y=labels, orientation="h",
-        marker_color=colors, marker_line_width=0,
-        text=pcts, textposition="outside",
-        hovertemplate="%{y}: %{x} studies (%{text})<extra></extra>",
-    ))
-    fig.update_layout(
-        height=max(300, len(labels) * 36 + 80),
-        paper_bgcolor="#f8f9fa", plot_bgcolor="#f8f9fa",
-        font=dict(family="Inter,sans-serif", size=12, color="#2c3e50"),
-        margin=dict(t=10, b=20, l=10, r=90),
-        xaxis=dict(showgrid=True, gridcolor="#e9ecef", zeroline=False, title="Number of studies"),
-        yaxis=dict(showgrid=False),
-    )
-    try:
-        return desc_html + fig.to_html(full_html=False, include_plotlyjs=False,
-                                       config={"responsive": True})
-    except Exception as e:
-        return desc_html + f"<p style='color:red'>Chart error: {e}</p>"
-
-
-# ---------------------------------------------------------------------------
-# Evidence Gap Analysis — intervention x outcome and SDG x intervention
-# ---------------------------------------------------------------------------
-
-INTERVENTIONS = [
-    "Capacity development of private, public, or hybrid sector actors",
-    "Public-private partnerships (PPPs)",
-    "Multicomponent",
-    "Financing with private sector actors",
-    "Financing of private sector actors",
-    "Matchmaking and consulting",
-    "Structured dialogue",
-]
-
-OUTCOMES = [
-    "Work-Labor/Social", "Jobs and Income", "Market/Sector Growth",
-    "Coordination/Reforms", "Environmental", "Energy/Extractives",
-]
-
-SDGS = [
-    "Decent Work and Economic Growth", "Good Health and Well-being",
-    "Quality Education", "No Poverty",
-    "Industry, Innovation and Infrastructure", "Reduced Inequality",
-    "Gender Equality", "Zero Hunger", "Climate Action",
-]
-
-# Short labels for axes
-INT_SHORT = {
-    "Capacity development of private, public, or hybrid sector actors": "Capacity Dev.",
-    "Public-private partnerships (PPPs)": "PPPs",
-    "Multicomponent": "Multicomponent",
-    "Financing with private sector actors": "Financing (with PSA)",
-    "Financing of private sector actors": "Financing (of PSA)",
-    "Matchmaking and consulting": "Matchmaking",
-    "Structured dialogue": "Structured Dialogue",
-}
-SDG_SHORT = {
-    "Decent Work and Economic Growth": "SDG 8: Decent Work",
-    "Good Health and Well-being": "SDG 3: Health",
-    "Quality Education": "SDG 4: Education",
-    "No Poverty": "SDG 1: No Poverty",
-    "Industry, Innovation and Infrastructure": "SDG 9: Industry",
-    "Reduced Inequality": "SDG 10: Inequality",
-    "Gender Equality": "SDG 5: Gender",
-    "Zero Hunger": "SDG 2: Zero Hunger",
-    "Climate Action": "SDG 13: Climate",
-}
-
-def build_matrix(row_keys, col_keys, row_col, col_col, split_col=False, dff=None):
-    if dff is None:
-        dff = df
-    mat = pd.DataFrame(0, index=row_keys, columns=col_keys)
-    for _, row in dff.iterrows():
-        rval = str(row[row_col]).strip()
-        if not rval or rval not in row_keys:
-            continue
-        cvals = str(row[col_col]).split("|") if split_col else [str(row[col_col])]
-        for cv in cvals:
-            cv = cv.strip()
-            if cv in col_keys:
-                mat.loc[rval, cv] += 1
-    return mat
-
-
-def make_gap_heatmap(matrix, row_labels, col_labels, title, height=420):
-    import numpy as np
-    z    = matrix.values.tolist()
-    rows = [row_labels.get(r, r) for r in matrix.index]
-    cols = [col_labels.get(c, c) for c in matrix.columns]
-
-    # Custom colour: 0 = red (gap), low = amber, high = dark navy
-    colorscale = [
-        [0.00, "#e74c3c"],   # 0 studies — clear gap
-        [0.01, "#f39c12"],   # 1–2 studies — limited
-        [0.10, "#f1c40f"],   # some evidence
-        [0.35, "#3498db"],   # moderate
-        [1.00, "#0a2342"],   # strong evidence
-    ]
-
-    # Annotations — show count or "GAP"
-    annotations = []
-    max_val = max(max(row) for row in z) or 1
-    for i, row in enumerate(z):
-        for j, val in enumerate(row):
-            text      = "GAP" if val == 0 else str(val)
-            textcolor = "white" if val == 0 or val/max_val > 0.3 else "#2c3e50"
-            annotations.append(dict(
-                x=j, y=i, text=text,
-                font=dict(size=11, color=textcolor, family="Inter,sans-serif"),
-                showarrow=False,
-            ))
-
-    fig = go.Figure(go.Heatmap(
-        z=z, x=cols, y=rows,
-        colorscale=colorscale,
-        showscale=True,
-        colorbar=dict(
-            title="Studies",
-            thickness=14, len=0.7,
-            tickfont=dict(size=10),
-        ),
-        hovertemplate="<b>%{y}</b> × <b>%{x}</b><br>Studies: %{z}<extra></extra>",
-    ))
-
-    fig.update_layout(
-        title=dict(text=title, font=dict(size=14, color="#0a2342", family="Inter,sans-serif")),
-        height=height,
-        paper_bgcolor="#f8f9fa", plot_bgcolor="#f8f9fa",
-        font=dict(family="Inter,sans-serif", size=11, color="#2c3e50"),
-        margin=dict(t=50, b=120, l=160, r=20),
-        xaxis=dict(tickangle=-35, side="bottom", tickfont=dict(size=10)),
-        yaxis=dict(tickfont=dict(size=10), autorange="reversed"),
-        annotations=annotations,
-    )
-    return fig
-
-
-def make_gap_summary(matrix, row_labels, col_labels, type_label):
-    """Generate a navy-card paragraph summarising the gap analysis."""
-    total_cells   = matrix.size
-    gap_cells     = int((matrix == 0).sum().sum())
-    strong_cells  = int((matrix >= 10).sum().sum())
-    limited_cells = int(((matrix >= 1) & (matrix < 5)).sum().sum())
-
-    # Identify top gaps
-    gaps = []
-    for r in matrix.index:
-        for c in matrix.columns:
-            if matrix.loc[r, c] == 0:
-                gaps.append((row_labels.get(r, r), col_labels.get(c, c)))
-
-    # Identify strongest combinations
-    flat    = matrix.stack().reset_index()
-    flat.columns = ["intervention", "other", "count"]
-    flat    = flat[flat["count"] > 0].sort_values("count", ascending=False)
-    top3    = flat.head(3)
-
-    gap_pct    = gap_cells / total_cells * 100
-    strong_pct = strong_cells / total_cells * 100
-
-    para = (
-        f"The {type_label} matrix covers <b>{total_cells} possible combinations</b> "
-        f"({len(matrix.index)} intervention categories × {len(matrix.columns)} {type_label.lower().split(' x ')[1]}). "
-        f"Of these, <b>{gap_cells} combinations ({gap_pct:.0f}%) have no evidence</b> in the dataset — "
-        f"representing clear evidence gaps where future research is needed. "
-        f"Only <b>{strong_cells} combinations ({strong_pct:.0f}%)</b> have 10 or more studies, "
-        f"indicating strong evidence. "
-    )
-    if not top3.empty:
-        t = top3.iloc[0]
-        para += (
-            f"The best-evidenced combination is "
-            f"<b>{row_labels.get(t['intervention'], t['intervention'])}</b> × "
-            f"<b>{col_labels.get(t['other'], t['other'])}</b> "
-            f"({int(t['count'])} studies). "
-        )
-    if gaps:
-        gap_examples = "; ".join(f"{r} × {c}" for r, c in gaps[:3])
-        para += f"Key gaps include: {gap_examples}."
-
-    return f"""<div style="background:linear-gradient(135deg,#0a2342 0%,#1a3a5c 100%);
-        border-radius:10px;padding:18px 22px;margin-bottom:14px;
-        font-family:Inter,sans-serif;font-size:14px;line-height:1.8;color:#ffffff;
-        box-shadow:0 4px 16px rgba(10,35,66,0.25);">
-        <div style="font-size:12px;font-weight:700;color:rgba(255,255,255,0.7);
-            margin-bottom:8px;text-transform:uppercase;letter-spacing:0.8px;">
-            Evidence Gap Analysis &nbsp;·&nbsp; {type_label}
-        </div>
-        {para}
-    </div>
-    <div style="display:flex;gap:12px;margin-bottom:14px;flex-wrap:wrap;">
-        <div style="background:#e74c3c;color:white;border-radius:8px;padding:10px 16px;
-            font-family:Inter,sans-serif;font-size:13px;font-weight:600;">
-            🔴 {gap_cells} Gaps (no evidence)
-        </div>
-        <div style="background:#f39c12;color:white;border-radius:8px;padding:10px 16px;
-            font-family:Inter,sans-serif;font-size:13px;font-weight:600;">
-            🟡 {limited_cells} Limited (&lt;5 studies)
-        </div>
-        <div style="background:#0a2342;color:white;border-radius:8px;padding:10px 16px;
-            font-family:Inter,sans-serif;font-size:13px;font-weight:600;">
-            🔵 {strong_cells} Strong (10+ studies)
-        </div>
-    </div>"""
-
-
-def make_gap_tab_content(analysis_type, method_filter="All"):
-    # Filter dataframe by method type
-    if method_filter == "IE Quantitative":
-        dff = df[df["method_eval"] == "IE Quantitative"]
-    elif method_filter == "IE Qualitative":
-        dff = df[df["method_eval"] == "IE Qualitative"]
-    elif method_filter == "SR (Systematic Review)":
-        dff = df[df["method_eval"] == "SR"]
-    else:
-        dff = df
-
-    n_filtered = len(dff)
-    method_note = "" if method_filter == "All" else f" [{method_filter}, {n_filtered} studies]"
-
-    if analysis_type == "Intervention × Outcome":
-        mat       = build_matrix(INTERVENTIONS, OUTCOMES,
-                                 "intervention_category", "outcome_categories_assigned",
-                                 split_col=True, dff=dff)
-        row_lab   = INT_SHORT
-        col_lab   = {o: o for o in OUTCOMES}
-        title     = f"Evidence Map: Intervention × Outcome{method_note}"
-        type_label = "Intervention × Outcome"
-        height    = 380
-    else:
-        mat       = build_matrix(INTERVENTIONS, SDGS,
-                                 "intervention_category", "un_sustainable_development_goal",
-                                 split_col=True, dff=dff)
-        row_lab   = INT_SHORT
-        col_lab   = SDG_SHORT
-        title     = f"Evidence Map: Intervention × SDG{method_note}"
-        type_label = "Intervention × SDG"
-        height    = 400
-
-    summary_html = make_gap_summary(mat, row_lab, col_lab, type_label)
-    fig          = make_gap_heatmap(mat, row_lab, col_lab, title, height=height)
-    try:
-        chart_html = fig.to_html(full_html=False, include_plotlyjs=False,
-                                 config={"responsive": True})
-    except Exception as e:
-        chart_html = f"<p style='color:red'>Chart error: {e}</p>"
-    return summary_html + chart_html
-
-
-# ---------------------------------------------------------------------------
-# Custom CSS — 3ie professional dark navy theme
-# ---------------------------------------------------------------------------
-CUSTOM_CSS = """
-@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap');
-
-* { font-family: 'Inter', sans-serif !important; }
-
-/* Main background */
-.gradio-container { background: #f0f2f5 !important; max-width: 1200px !important; margin: 0 auto !important; }
-
-/* Header banner */
-.egm-header {
-    background: linear-gradient(135deg, #0a2342 0%, #1a3a5c 60%, #1e5280 100%);
-    color: white;
-    padding: 28px 32px 22px;
-    border-radius: 12px;
-    margin-bottom: 20px;
-    box-shadow: 0 4px 20px rgba(10,35,66,0.25);
-}
-.egm-header h1 { font-size: 24px; font-weight: 700; margin: 0 0 4px; letter-spacing: -0.3px; }
-.egm-header p  { font-size: 13px; opacity: 0.75; margin: 0; font-weight: 400; }
-.egm-badges { display: flex; gap: 8px; margin-top: 12px; flex-wrap: wrap; }
-.egm-badge {
-    background: rgba(255,255,255,0.12);
-    border: 1px solid rgba(255,255,255,0.2);
-    border-radius: 20px;
-    padding: 3px 12px;
-    font-size: 11px;
-    font-weight: 500;
-    color: rgba(255,255,255,0.9);
-}
-
-/* Tabs */
-.tab-nav button {
-    font-weight: 600 !important;
-    font-size: 13px !important;
-    color: #4a5568 !important;
-    border-radius: 8px 8px 0 0 !important;
-}
-.tab-nav button.selected {
-    color: #0a2342 !important;
-    border-bottom: 3px solid #0a2342 !important;
-    background: white !important;
-}
-
-/* Chat bubbles */
-.message.user > div  { background: #0a2342 !important; color: white !important; border-radius: 14px 14px 4px 14px !important; }
-.message.bot > div   { background: white !important; border: 1px solid #e2e8f0 !important; border-radius: 14px 14px 14px 4px !important; box-shadow: 0 1px 4px rgba(0,0,0,0.06) !important; }
-
-/* Input box */
-textarea, input[type=text] {
-    border: 1.5px solid #cbd5e0 !important;
-    border-radius: 10px !important;
-    background: white !important;
-    font-size: 14px !important;
-    transition: border-color 0.2s !important;
-}
-textarea:focus, input[type=text]:focus { border-color: #0a2342 !important; box-shadow: 0 0 0 3px rgba(10,35,66,0.08) !important; }
-
-/* Buttons */
-.gr-button-primary {
-    background: #0a2342 !important;
-    border: none !important;
-    border-radius: 10px !important;
-    font-weight: 600 !important;
-    letter-spacing: 0.2px !important;
-    transition: background 0.2s !important;
-}
-.gr-button-primary:hover { background: #1a3a5c !important; }
-.gr-button-secondary { border-color: #cbd5e0 !important; border-radius: 10px !important; font-weight: 500 !important; }
-
-/* Panels / blocks */
-.gr-panel, .gr-box { border-radius: 12px !important; border: 1px solid #e2e8f0 !important; box-shadow: 0 1px 6px rgba(0,0,0,0.05) !important; }
-
-/* Slider */
-.gr-slider input[type=range]::-webkit-slider-thumb { background: #0a2342 !important; }
-
-/* Dropdown */
-.gr-dropdown { border-radius: 10px !important; }
-
-/* Footer */
-.egm-footer { font-size: 11px; color: #94a3b8; text-align: center; padding: 12px 0 4px; }
-"""
-
-with gr.Blocks(title="EGM Research Assistant", theme=gr.themes.Base(), css=CUSTOM_CSS) as demo:
-    gr.HTML(f"""
-    <div class="egm-header">
-        <h1>📚 EGM Research Assistant</h1>
-        <p>Evidence Gap Map · AI-powered research synthesis · Dataset-only responses</p>
-        <div class="egm-badges">
-            <span class="egm-badge">📚 {TOTAL} studies</span>
-            <span class="egm-badge">🔍 Semantic RAG</span>
-            <span class="egm-badge">✅ Dataset-only responses</span>
-            <span class="egm-badge">📊 Evidence visualised</span>
-        </div>
-    </div>
-    """)
-
-    with gr.Tabs():
-
-        # ── Tab 1: Chat ──────────────────────────────────────────────────
-        with gr.TabItem("💬 Research Chat"):
-            with gr.Row():
-                with gr.Column(scale=4):
-                    chatbot = gr.Chatbot(label="Research conversation",
-                                        height=480, type="messages")
-                    with gr.Row():
-                        msg = gr.Textbox(
-                            placeholder="e.g. What does the evidence say about cash transfers on education outcomes?",
-                            label="Your question", lines=2, scale=5)
-                        send_btn = gr.Button("Send ↗", variant="primary", scale=1)
-
-                with gr.Column(scale=1, min_width=220):
-                    gr.Markdown("### ⚙️ Settings")
-                    top_k = gr.Slider(minimum=10, maximum=358, value=50, step=10,
-                        label="Studies retrieved per query",
-                        info="Full dataset = 358 studies")
-                    clear_btn = gr.Button("🗑 Clear", variant="secondary")
-                    gr.Markdown("### 💡 Try asking")
-                    gr.Examples(examples=[
-                        ["What interventions improve employment outcomes?"],
-                        ["Evidence on cash transfers for education in Sub-Saharan Africa"],
-                        ["How effective is microfinance for women's empowerment?"],
-                        ["Which studies focus on fragile or conflict-affected states?"],
-                        ["What qualitative studies exist in the dataset?"],
-                    ], inputs=msg)
-
-            gr.HTML('<script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script>')
-            chat_chart = gr.HTML(value=EMPTY_HTML, label="Evidence snapshot")
-
-            send_btn.click(fn=chat, inputs=[msg, chatbot, top_k],
-                outputs=[chatbot, chat_chart]).then(lambda: "", outputs=msg)
-            msg.submit(fn=chat, inputs=[msg, chatbot, top_k],
-                outputs=[chatbot, chat_chart]).then(lambda: "", outputs=msg)
-            clear_btn.click(fn=clear_chat, outputs=[chatbot, chat_chart])
-
-        # ── Tab 2: Dataset Summary ────────────────────────────────────────
-        with gr.TabItem("📊 Dataset Summary"):
-            # Load Plotly JS once — shared by all charts in this tab
-            gr.HTML('<script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script>')
-            gr.Markdown("""
-### Explore the spread of any column in the dataset
-Select a column below to instantly see how studies are distributed across categories.
-No AI involved — this reads directly from the dataset.
-            """)
-            col_dropdown = gr.Dropdown(
-                choices=list(SUMMARY_COLS.keys()),
-                value="Intervention Category",
-                label="Select column to summarise",
-                interactive=True,
-            )
-            summary_chart = gr.HTML(
-                value=make_summary_chart("Intervention Category"),
-                label="Column distribution"
-            )
-            col_dropdown.change(
-                fn=make_summary_chart,
-                inputs=col_dropdown,
-                outputs=summary_chart,
-            )
-
-
-        # ── Tab 3: Evidence Gaps ─────────────────────────────────────────
-        with gr.TabItem("🔴 Evidence Gaps"):
-            gr.HTML('<script src="https://cdn.plot.ly/plotly-2.27.0.min.js"></script>')
-            gr.Markdown("""
-### Evidence Gap Analysis
-Heatmaps showing where evidence exists and where it is missing.
-🔴 **GAP** = no studies &nbsp;|&nbsp; 🟡 amber = limited evidence &nbsp;|&nbsp; 🔵 dark navy = strong evidence
-            """)
-            with gr.Row():
-                gap_type = gr.Radio(
-                    choices=["Intervention × Outcome", "Intervention × SDG"],
-                    value="Intervention × Outcome",
-                    label="Analysis type",
-                    interactive=True,
-                )
-                gap_method = gr.Radio(
-                    choices=["All", "IE Quantitative", "IE Qualitative", "SR (Systematic Review)"],
-                    value="All",
-                    label="Filter by method type",
-                    interactive=True,
-                )
-            gap_output = gr.HTML(value=make_gap_tab_content("Intervention × Outcome", "All"))
-            gap_type.change(fn=make_gap_tab_content, inputs=[gap_type, gap_method], outputs=gap_output)
-            gap_method.change(fn=make_gap_tab_content, inputs=[gap_type, gap_method], outputs=gap_output)
-
-    gr.HTML('<div class="egm-footer">Responses grounded exclusively in the EGM dataset · sentence-transformers + ChromaDB + Claude </div>')
-
-if __name__ == "__main__":
-    demo.launch()