Spaces:

EdyVision
/

praxa-behavioral-alignment-analysis

Running

praxa-behavioral-alignment-analysis / app.py
Persist Hub metrics on disk so ordinary Space loads skip re-download.
d6f66fc 11 days ago
52.7 kB
"""Praxa Behavioral Alignment Analysis — Dash Space.
Reads published metrics from EdyVision/praxa-behavioral-experiments (HF_TOKEN).
"""

from __future__ import annotations

import dash
import dash_mantine_components as dmc
import pandas as pd
from dash import Input, Output, State, dash_table, dcc, html
from dash_iconify import DashIconify

from charts import (
    ICON_ACCENT,
    CONTRAST_LABELS,
    edge_metric_bar,
    empty_figure,
    ladder_heatmap,
    learned_vs_frozen_bar,
    lift_bar,
    bootstrap_contrast_forest,
    rfdt_encoder_disagreement_bar,
    rfdt_encoder_slice_edge_heatmap,
    rfdt_vs_encoder_bar,
    slice_ladder_heatmap,
)
from data import (
    ALL_CONDITIONS,
    DOMAIN_LABELS,
    MATCHED_8B_KEYS,
    best_by_family,
    bootstrap_contrasts_frame,
    disagreement_by_slice,
    disagreement_summary,
    encoder_comparator_frame,
    generative_ladder_frame,
    get_analysis,
    get_metrics,
    mcnemar_contrasts_frame,
    refresh_cache,
    rfdt_encoder_disagreement,
)
from narrative import (
    BLOG_POST,
    PAPER_STATUS,
    PAPER_TITLE,
    PRAXA_LIBRARY,
    SHOW_BLOG_POST,
    SHOW_PRAXA_LIBRARY,
    about_card,
    methodology_modal_body,
    narrative_bundle,
    published_models_modal_body,
)
from report import build_alignment_report_pdf, report_filename

app = dash.Dash(__name__)
app.title = "Praxa Behavioral Alignment"
server = app.server

LOADING_FIGURE = empty_figure("Loading published metrics…")


def _graph(graph_id: str) -> dcc.Graph:
    """Graph with a dark loading placeholder so panels are never blank white."""
    return dcc.Graph(
        id=graph_id,
        figure=LOADING_FIGURE,
        config={"displayModeBar": False},
        style={"width": "100%", "minHeight": "280px"},
    )

FA = "fa-solid"
ICON = ICON_ACCENT
ACCENT = "blue"
RESEARCHER_HF = "https://huggingface.co/EdyVision"

TABLE_COLS = [
    ("model", "Model"),
    ("condition_label", "Condition"),
    ("macro_f1", "Macro-F1"),
    ("accuracy", "Accuracy"),
    ("answer_f1", "Answer F1"),
    ("refuse_f1", "Refuse F1"),
    ("allowed_edge", "Allowed-edge"),
    ("denied_edge_refuse_recall", "Denied refuse rec."),
]


def _fmt(value: float | None) -> str:
    if value is None or (isinstance(value, float) and pd.isna(value)):
        return "—"
    return f"{float(value):.3f}"


def _fa(name: str, size: int = 14) -> DashIconify:
    return DashIconify(icon=f"{FA}:{name}", width=size, color=ICON)


def _btn_label(icon: str, text: str) -> html.Span:
    """Icon + label with the same gap used on info-card titles."""
    return html.Span(
        [_fa(icon, 14), html.Span(text)],
        className="btn-with-icon",
    )


# Formula + why blurbs for section-title info tips (CSS hover, all tabs).
SECTION_FORMULAS: dict[str, str] = {
    "ladder": (
        "Formula: macro-F1 = (F1_answer + F1_refuse) / 2, where "
        "F1_c = 2·P_c·R_c / (P_c + R_c) on the frozen held-out split "
        "(invalid generative parses counted as refuse). "
        "Why: primary paper metric; averages both policy classes so a method "
        "cannot look strong by always answering or always refusing."
    ),
    "encoder_comparator": (
        "Formula: same macro-F1 on identical sample_ids and the retrieved-policy "
        "input contract for RFDT-LoRA vs ModernBERT Encoder-FT. "
        "Why: isolates representation-first LoRA against a conventional encoder "
        "under a matched task, not a weaker truncated-input baseline."
    ),
    "slice_heatmap": (
        "Formula: Acc_s = (# correct in stratum s) / n_s for "
        "s ∈ {allowed_base, allowed_edge, denied_base, denied_edge}. "
        "Why: overall macro-F1 can hide edge failures; stratum accuracy shows "
        "where policy behavior breaks."
    ),
    "disagreement_bar": (
        "Formula (paired on sample_id): counts of "
        "(RFDT✓∧Enc✓), (RFDT✗∧Enc✗), (RFDT✓∧Enc✗), (RFDT✗∧Enc✓). "
        "Why: mean F1 gaps do not show whether RFDT wins the same examples "
        "Encoder misses, or merely different error patterns."
    ),
    "disagreement_slice": (
        "Formula: edge_s = (n_RFDT-only correct − n_Encoder-only correct) / n_s "
        "within query-type stratum s. "
        "Why: localizes which policy strata drive the RFDT vs Encoder advantage."
    ),
    "lift": (
        "Formula: Lift(model, condition) = macro-F1(condition) − macro-F1(Base) "
        "within the same instruct model family. "
        "Why: subtracts each model’s unadapted baseline so gains are adaptation "
        "effects, not cross-family absolute score differences."
    ),
    "learned_vs_frozen": (
        "Formula: compare macro-F1(best frozen prompt) vs macro-F1(LoRA), "
        "macro-F1(Behavioral KV), and macro-F1(RFDT-LoRA) on matched-8B models. "
        "Why: asks whether learned adaptation beats the strongest inference-time "
        "control under the same split and supervision."
    ),
    "allowed_edge": (
        "Formula: Acc_allowed-edge = (# correct | query_type=allowed_edge) / n. "
        "Why: allowed-edge items are permit-but-hard; this checks over-refusal "
        "when answering is policy-legal."
    ),
    "denied_edge": (
        "Formula: RefuseRecall_denied-edge = TP_refuse / (TP_refuse + FN_refuse) "
        "on gold-REFUSE denied-edge items. "
        "Why: measures catching prohibited asks without diluting the score with "
        "easy denied-base refusals."
    ),
    "metrics_table": (
        "Formula: published metrics.json per freeze cell — "
        "macro-F1, Acc=(# correct)/n, class F1s, edge stratum rates. "
        "Why: tabular audit of the same numbers behind the charts, keyed to "
        "immutable Hub evaluation bundles."
    ),
    "bootstrap": (
        "Formula: Δ = macro-F1(A) − macro-F1(B). Draw B=10,000 resamples of "
        "leakage groups with replacement; 95% CI = [P_2.5(Δ*), P_97.5(Δ*)]. "
        "Two-sided bootstrap p = min(1, 2·min(mean(Δ*≤0), mean(Δ*≥0))). "
        "Holm within each named contrast (m=6 model–domain cells): sort "
        "p_(1)≤…≤p_(m); p̃_(k) = max_{j≤k} min(1, (m−j+1)·p_(j)). "
        "Why: groups (not rows) respect leakage structure; percentile CIs avoid "
        "normal approximations; Holm controls family-wise error across the six "
        "matched cells for that contrast."
    ),
    "mcnemar": (
        "Formula: on matched IDs, b = A-only correct, c = B-only correct; "
        "exact two-sided p = BinomialTest(b; n=b+c, π=1/2) (McNemar). "
        "Holm across the same six cells as bootstrap. "
        "Why: tests whether A wins more discordant examples than B; supports "
        "macro-F1 inference without replacing it."
    ),
    "error_atlas": (
        "Formula: confusion C_{g,ŷ} = count(gold=g, pred=ŷ) for "
        "g,ŷ∈{answer,refuse}; Acc_s = (# correct)/n_s by query_type; "
        "invalid rates from metrics.json. "
        "Why: decomposes aggregate F1 into which class and stratum errors dominate."
    ),
    "judge_sensitivity": (
        "Formula: Agree = mean(1[ŷ_primary = ŷ_indep]). "
        "Acc_j = mean(1[ŷ_j = gold]). "
        "On disagreements: Acc_opt = mean(1[ŷ_primary=gold ∨ ŷ_indep=gold]); "
        "Acc_pes = mean(1[ŷ_primary=gold ∧ ŷ_indep=gold]). "
        "Ordering: sign(macro-F1_LoRA − macro-F1_KV) under each judge’s labels. "
        "Why: generative scores depend on a measurement adapter; this bounds "
        "judge-choice sensitivity and checks whether LoRA>KV is judge-fragile."
    ),
    "release_audit": (
        "Formula: status = pass iff n_hits = 0 after scanning publishable "
        "findings for private hosts, URLs, credentials, and local paths. "
        "Why: public Hub/OSF artifacts must not leak lab infrastructure."
    ),
}


def section_heading(title: str, formula_key: str) -> html.Div:
    """Section title with a spaced info icon explaining the calculation.
    Uses a CSS hover tip (not Mantine Tooltip) so it works inside every tab
    panel, including panels that stay mounted while hidden.
    """
    formula = SECTION_FORMULAS.get(formula_key, "").strip()
    tip_children: list = [
        DashIconify(icon=f"{FA}:circle-info", width=15, color="#60a5fa"),
    ]
    if formula:
        tip_children.append(html.Span(formula, className="panel-heading-tip"))
    icon = html.Span(
        tip_children,
        className="panel-heading-info",
        tabIndex=0,
        role="img",
        title=formula or None,
        **{"aria-label": f"How {title} is calculated"},
    )
    return html.Div([html.H2(title), icon], className="panel-heading")


def metrics_table(df: pd.DataFrame) -> dash_table.DataTable:
    if df.empty:
        records: list[dict] = []
    else:
        show = df.copy()
        order = {c: i for i, c in enumerate(ALL_CONDITIONS)}
        show["_ord"] = show["condition"].map(lambda c: order.get(c, 99))
        show = show.sort_values(["model", "_ord"])
        records = []
        for row in show.itertuples(index=False):
            records.append(
                {
                    "model": row.model,
                    "condition_label": row.condition_label,
                    "macro_f1": _fmt(getattr(row, "macro_f1", None)),
                    "accuracy": _fmt(getattr(row, "accuracy", None)),
                    "answer_f1": _fmt(getattr(row, "answer_f1", None)),
                    "refuse_f1": _fmt(getattr(row, "refuse_f1", None)),
                    "allowed_edge": _fmt(getattr(row, "allowed_edge", None)),
                    "denied_edge_refuse_recall": _fmt(
                        getattr(row, "denied_edge_refuse_recall", None)
                    ),
                }
            )

    return dash_table.DataTable(
        id="metrics-table",
        columns=[{"name": label, "id": key} for key, label in TABLE_COLS],
        data=records,
        sort_action="native",
        filter_action="native",
        page_size=16,
        style_table={"overflowX": "auto"},
        style_header={
            "backgroundColor": "#12203a",
            "color": "#93c5fd",
            "fontWeight": "600",
            "border": "1px solid #1e3a5f",
            "fontFamily": "IBM Plex Sans, Helvetica, sans-serif",
            "fontSize": "13px",
        },
        style_cell={
            "backgroundColor": "#0c1220",
            "color": "#dbeafe",
            "border": "1px solid #1e3a5f",
            "fontFamily": "IBM Plex Sans, Helvetica, sans-serif",
            "fontSize": "13px",
            "padding": "8px 10px",
            "minWidth": "90px",
        },
        style_filter={
            "backgroundColor": "#0a1628",
            "color": "#dbeafe",
            "border": "1px solid #1e3a5f",
        },
        style_data_conditional=[
            {"if": {"row_index": "odd"}, "backgroundColor": "#0a1628"},
        ],
    )


def bootstrap_table(df: pd.DataFrame) -> dash_table.DataTable:
    """Compact table for published leakage-group bootstrap contrasts."""
    records: list[dict] = []
    if not df.empty:
        show = df.copy()
        show = show.sort_values(["family", "model_key", "contrast_id"])
        for row in show.itertuples(index=False):
            holm = getattr(row, "holm_p", None)
            records.append(
                {
                    "family": row.family,
                    "model_key": row.model_key,
                    "contrast": CONTRAST_LABELS.get(
                        str(row.contrast_id), str(row.contrast_id)
                    ),
                    "delta": _fmt(getattr(row, "delta_macro_f1", None)),
                    "ci": (
                        f"[{_fmt(getattr(row, 'ci_low', None))}, "
                        f"{_fmt(getattr(row, 'ci_high', None))}]"
                    ),
                    "p": _fmt(getattr(row, "bootstrap_p", None)),
                    "holm_p": _fmt(holm) if holm is not None and not pd.isna(holm) else "—",
                }
            )

    cols = [
        ("family", "Family"),
        ("model_key", "Model"),
        ("contrast", "Contrast"),
        ("delta", "Δ macro-F1"),
        ("ci", "95% CI"),
        ("p", "p"),
        ("holm_p", "Holm p"),
    ]
    return dash_table.DataTable(
        id="bootstrap-table",
        columns=[{"name": label, "id": key} for key, label in cols],
        data=records,
        sort_action="native",
        page_size=12,
        style_table={"overflowX": "auto"},
        style_header={
            "backgroundColor": "#12203a",
            "color": "#93c5fd",
            "fontWeight": "600",
            "border": "1px solid #1e3a5f",
            "fontFamily": "IBM Plex Sans, Helvetica, sans-serif",
            "fontSize": "13px",
        },
        style_cell={
            "backgroundColor": "#0c1220",
            "color": "#dbeafe",
            "border": "1px solid #1e3a5f",
            "fontFamily": "IBM Plex Sans, Helvetica, sans-serif",
            "fontSize": "13px",
            "padding": "8px 10px",
            "minWidth": "80px",
        },
        style_data_conditional=[
            {"if": {"row_index": "odd"}, "backgroundColor": "#0a1628"},
        ],
    )


def mcnemar_section(domain: str) -> html.Section | html.Div:
    df = mcnemar_contrasts_frame(domain)
    if df.empty:
        return html.Div()
    records: list[dict] = []
    show = df.sort_values(["family", "model_key", "contrast_id"])
    for row in show.itertuples(index=False):
        records.append(
            {
                "family": row.family,
                "model_key": row.model_key,
                "contrast": CONTRAST_LABELS.get(
                    str(row.contrast_id), str(row.contrast_id)
                ),
                "b": int(getattr(row, "n_a_only", 0)),
                "c": int(getattr(row, "n_b_only", 0)),
                "p": _fmt(getattr(row, "exact_p", None)),
                "holm_p": (
                    _fmt(getattr(row, "holm_p", None))
                    if getattr(row, "holm_p", None) is not None
                    and not pd.isna(getattr(row, "holm_p", None))
                    else "—"
                ),
            }
        )
    cols = [
        ("family", "Family"),
        ("model_key", "Model"),
        ("contrast", "Contrast"),
        ("b", "A only correct"),
        ("c", "B only correct"),
        ("p", "Exact p"),
        ("holm_p", "Holm p"),
    ]
    return html.Section(
        [
            section_heading("McNemar (matched correctness)", "mcnemar"),
            html.P(
                "Exact tests on paired correct/incorrect labels for the same "
                "examples. Discordant cells show where only one method is right.",
                className="panel-note",
            ),
            dash_table.DataTable(
                id="mcnemar-table",
                columns=[{"name": label, "id": key} for key, label in cols],
                data=records,
                sort_action="native",
                page_size=12,
                style_table={"overflowX": "auto"},
                style_header={
                    "backgroundColor": "#12203a",
                    "color": "#93c5fd",
                    "fontWeight": "600",
                    "border": "1px solid #1e3a5f",
                    "fontFamily": "IBM Plex Sans, Helvetica, sans-serif",
                    "fontSize": "13px",
                },
                style_cell={
                    "backgroundColor": "#0c1220",
                    "color": "#dbeafe",
                    "border": "1px solid #1e3a5f",
                    "fontFamily": "IBM Plex Sans, Helvetica, sans-serif",
                    "fontSize": "13px",
                    "padding": "8px 10px",
                    "minWidth": "80px",
                },
                style_data_conditional=[
                    {"if": {"row_index": "odd"}, "backgroundColor": "#0a1628"},
                ],
            ),
        ],
        className="panel",
    )


def error_atlas_section(domain: str) -> html.Section | html.Div:
    blob = get_analysis("error_atlas")
    if not blob:
        return html.Div()
    payload = blob.get("payload") or {}
    domains = payload.get("domains") or {}
    block = domains.get(domain)
    if not block:
        return html.Div()
    children: list = [
        section_heading("Error atlas", "error_atlas"),
        html.P(
            "Confusion counts, accuracy by query type, and invalid-output rates "
            "for the reported methods. Counts only; no query text.",
            className="panel-note",
        ),
    ]
    summary = block.get("summary") or {}
    if summary.get("n_cells") is not None:
        children.append(
            html.P(
                f"{summary.get('n_cells')} method cells across "
                f"{summary.get('n_models', '—')} models.",
                className="panel-note",
            )
        )
    for table in (block.get("tables") or [])[:4]:
        title = table.get("title") or "Table"
        rows = table.get("rows") or []
        if not rows:
            continue
        frame = pd.DataFrame(rows)
        children.append(html.H3(title, className="panel-subhead"))
        children.append(
            dash_table.DataTable(
                columns=[{"name": c, "id": c} for c in frame.columns],
                data=frame.to_dict("records"),
                page_size=8,
                style_table={"overflowX": "auto", "marginBottom": "12px"},
                style_header={
                    "backgroundColor": "#12203a",
                    "color": "#93c5fd",
                    "fontWeight": "600",
                    "border": "1px solid #1e3a5f",
                    "fontSize": "12px",
                },
                style_cell={
                    "backgroundColor": "#0c1220",
                    "color": "#dbeafe",
                    "border": "1px solid #1e3a5f",
                    "fontSize": "12px",
                    "padding": "6px 8px",
                },
            )
        )
    if len(children) <= 2:
        return html.Div()
    return html.Section(children, className="panel")


def bootstrap_section_children(
    boot_fig,
    boot_table: dash_table.DataTable,
) -> html.Section:
    return html.Section(
        [
            section_heading("Leakage-group bootstrap", "bootstrap"),
            html.P(
                "Paired macro-F1 differences for predeclared contrasts, "
                "resampling leakage groups (not rows) with percentile "
                "95% CIs. Holm adjusts the six matched domain×model tests "
                "within each named contrast.",
                className="panel-note",
            ),
            dcc.Graph(
                figure=boot_fig,
                config={"displayModeBar": False},
                style={"width": "100%"},
            ),
            boot_table,
        ],
        className="panel",
    )


def judge_sensitivity_section(domain: str) -> html.Section | html.Div:
    blob = get_analysis("judge_sensitivity")
    if not blob:
        return html.Div()
    payload = blob.get("payload") or {}
    overall = payload.get("overall") or {}
    bounds = payload.get("optimistic_pessimistic_bounds") or {}
    ordering = payload.get("lora_vs_kv_ordering") or {}
    cells = [
        row
        for row in (payload.get("headline_cells") or [])
        if str(row.get("domain")) == domain
    ]
    order_cells = [
        row
        for row in (ordering.get("cells") or [])
        if str(row.get("domain")) == domain
    ]
    if not cells and not order_cells and not overall:
        return html.Div()

    children: list = [
        section_heading("Judge sensitivity", "judge_sensitivity"),
        html.P(
            "Independent AI judge (v1.2 Gemini) vs frozen primary judge (v1.3) "
            "on the same free-form outputs. Checks whether LoRA−KV rankings "
            "depend on which grader is used.",
            className="panel-note",
        ),
    ]
    if overall.get("n"):
        children.append(
            html.P(
                f"Agreement {_fmt(overall.get('primary_vs_independent_agreement'))} · "
                f"primary vs gold {_fmt(overall.get('primary_vs_gold_accuracy'))} · "
                f"independent vs gold {_fmt(overall.get('independent_vs_gold_accuracy'))} · "
                f"optimistic / pessimistic "
                f"{_fmt(bounds.get('optimistic_accuracy'))} / "
                f"{_fmt(bounds.get('pessimistic_accuracy'))} "
                f"(all generative rows: {overall.get('n')}).",
                className="panel-note",
            )
        )
    if order_cells:
        children.append(html.H3("LoRA vs KV under each judge", className="panel-subhead"))
        order_records = []
        for row in sorted(order_cells, key=lambda r: str(r.get("model_key"))):
            order_records.append(
                {
                    "model_key": row.get("model_key"),
                    "primary_delta": _fmt(row.get("primary_delta_lora_minus_kv")),
                    "independent_delta": _fmt(
                        row.get("independent_delta_lora_minus_kv")
                    ),
                    "changed": "yes" if row.get("ordering_changed") else "no",
                }
            )
        children.append(
            dash_table.DataTable(
                id="judge-sensitivity-ordering",
                columns=[
                    {"name": "Model", "id": "model_key"},
                    {"name": "Primary Δ (LoRA−KV)", "id": "primary_delta"},
                    {"name": "Independent Δ", "id": "independent_delta"},
                    {"name": "Order changed?", "id": "changed"},
                ],
                data=order_records,
                style_table={"overflowX": "auto", "marginBottom": "12px"},
                style_header={
                    "backgroundColor": "#12203a",
                    "color": "#93c5fd",
                    "fontWeight": "600",
                    "border": "1px solid #1e3a5f",
                    "fontSize": "12px",
                },
                style_cell={
                    "backgroundColor": "#0c1220",
                    "color": "#dbeafe",
                    "border": "1px solid #1e3a5f",
                    "fontSize": "12px",
                    "padding": "6px 8px",
                },
            )
        )
    if cells:
        children.append(html.H3("Per-condition macro-F1", className="panel-subhead"))
        cell_records = []
        for row in sorted(
            cells, key=lambda r: (str(r.get("model_key")), str(r.get("condition")))
        ):
            cell_records.append(
                {
                    "model_key": row.get("model_key"),
                    "condition": row.get("condition"),
                    "primary_f1": _fmt(row.get("primary_macro_f1")),
                    "independent_f1": _fmt(row.get("independent_macro_f1")),
                    "agree": _fmt(row.get("judge_agreement")),
                }
            )
        children.append(
            dash_table.DataTable(
                id="judge-sensitivity-cells",
                columns=[
                    {"name": "Model", "id": "model_key"},
                    {"name": "Condition", "id": "condition"},
                    {"name": "Primary F1", "id": "primary_f1"},
                    {"name": "Independent F1", "id": "independent_f1"},
                    {"name": "Agree", "id": "agree"},
                ],
                data=cell_records,
                page_size=10,
                sort_action="native",
                style_table={"overflowX": "auto"},
                style_header={
                    "backgroundColor": "#12203a",
                    "color": "#93c5fd",
                    "fontWeight": "600",
                    "border": "1px solid #1e3a5f",
                    "fontSize": "12px",
                },
                style_cell={
                    "backgroundColor": "#0c1220",
                    "color": "#dbeafe",
                    "border": "1px solid #1e3a5f",
                    "fontSize": "12px",
                    "padding": "6px 8px",
                },
            )
        )
    return html.Section(children, className="panel")


def release_audit_section() -> html.Section | html.Div:
    blob = get_analysis("release_audit")
    if not blob:
        return html.Div()
    payload = blob.get("payload") or {}
    status = payload.get("status") or "—"
    n_hits = payload.get("n_hits")
    n_files = payload.get("n_files_scanned")
    return html.Section(
        [
            section_heading("Release safety audit", "release_audit"),
            html.P(
                f"Status: {status}. Scanned {n_files if n_files is not None else '—'} "
                f"publishable finding files; private-infra hits: "
                f"{n_hits if n_hits is not None else '—'}.",
                className="panel-note",
            ),
        ],
        className="panel",
    )


def summary_stack(stats: dict) -> html.Div:
    """Vertical stack of best-method cards (left column of headline row)."""
    cells = []
    items = [
        (
            "Best frozen",
            (
                f"{_fmt(stats['best_frozen']['value'])} · {stats['best_frozen']['model']}"
                if stats.get("best_frozen")
                else "—"
            ),
            "snowflake",
        ),
        (
            "Best LoRA",
            (
                f"{_fmt(stats['best_lora']['value'])} · {stats['best_lora']['model']}"
                if stats.get("best_lora")
                else "—"
            ),
            "layer-group",
        ),
        (
            "Best KV",
            (
                f"{_fmt(stats['best_kv']['value'])} · {stats['best_kv']['model']}"
                if stats.get("best_kv")
                else "—"
            ),
            "microchip",
        ),
        (
            "Best RFDT",
            (
                f"{_fmt(stats['best_rfdt']['value'])} · {stats['best_rfdt']['model']}"
                if stats.get("best_rfdt")
                else "—"
            ),
            "bullseye",
        ),
        (
            "Encoder-FT",
            (
                f"{_fmt(stats['best_encoder']['value'])} · {stats['best_encoder']['model']}"
                if stats.get("best_encoder")
                else "—"
            ),
            "robot",
        ),
    ]
    for title, value, icon in items:
        cells.append(
            html.Div(
                [
                    html.Div(
                        [_fa(icon, 14), html.Span(title, className="summary-label")],
                        className="summary-title",
                    ),
                    html.Div(value, className="summary-value"),
                ],
                className="summary-cell",
            )
        )
    return html.Div(cells, className="summary-stack")


def render_about(domain: str) -> html.Div:
    card = about_card(domain)
    return html.Div(
        [
            html.Div(
                [_fa(card.icon, 14), html.Span(card.title)],
                className="about-card-title",
            ),
            html.P(
                [
                    html.Em(PAPER_TITLE),
                    f" ({PAPER_STATUS}). ",
                ],
                className="about-card-paper",
            ),
            html.P(card.body, className="about-card-body"),
        ],
        className="about-card",
    )


def render_narrative_card(card, *, class_name: str = "info-card") -> html.Div | None:
    if card is None:
        return None
    paragraphs = [
        html.P(part.strip(), className="info-card-body")
        for part in str(card.body).split("\n\n")
        if part.strip()
    ]
    return html.Div(
        [
            html.Div(
                [_fa(card.icon, 14), html.Span(card.title)],
                className="info-card-title",
            ),
            *paragraphs,
        ],
        className=class_name,
    )


def _narrative_block(card) -> list:
    if card is None:
        return []
    paragraphs = [
        html.P(part.strip(), className="info-card-body")
        for part in str(card.body).split("\n\n")
        if part.strip()
    ]
    return [
        html.Div(
            [_fa(card.icon, 14), html.Span(card.title)],
            className="info-card-title",
        ),
        *paragraphs,
    ]


def render_story_card(findings, implications) -> html.Div | None:
    """Single headline card: findings then implications, labels unchanged."""
    blocks = _narrative_block(findings)
    implication_blocks = _narrative_block(implications)
    if implication_blocks:
        if blocks:
            blocks.append(html.Div(className="story-divider"))
        blocks.extend(implication_blocks)
    if not blocks:
        return None
    return html.Div(blocks, className="info-card findings-panel story-card")



def footer() -> html.Footer:
    resources: list = [
        html.Span("Resources", className="footer-label"),
        html.Span("·", className="footer-sep"),
        html.Button(
            "Methodology",
            id="methodology-open",
            n_clicks=0,
            className="footer-link",
        ),
    ]
    if SHOW_BLOG_POST:
        resources.extend(
            [
                html.Span("·", className="footer-sep"),
                html.A(
                    BLOG_POST["label"],
                    href=BLOG_POST["href"],
                    target="_blank",
                    className="footer-link",
                    title=BLOG_POST["note"],
                ),
            ]
        )
    if SHOW_PRAXA_LIBRARY:
        resources.extend(
            [
                html.Span("·", className="footer-sep"),
                html.A(
                    PRAXA_LIBRARY["label"],
                    href=PRAXA_LIBRARY["href"],
                    target="_blank",
                    className="footer-link",
                    title=PRAXA_LIBRARY["note"],
                    rel="noopener noreferrer",
                ),
            ]
        )
    resources.extend(
        [
            html.Span("·", className="footer-sep"),
            html.Button(
                "Published artifacts",
                id="weights-open",
                n_clicks=0,
                className="footer-link",
            ),
        ]
    )

    return html.Footer(
        [
            html.Div(resources, className="footer-row"),
            html.Div(
                [
                    html.Span("Researcher · "),
                    html.A(
                        "Eidan J. Rosado, PhD",
                        href=RESEARCHER_HF,
                        target="_blank",
                        className="footer-link",
                    ),
                ],
                className="footer-credit",
            ),
        ],
        className="site-footer",
    )


app.layout = dmc.MantineProvider(
    [
        html.Link(
            href=(
                "https://fonts.googleapis.com/css2?"
                "family=IBM+Plex+Sans:wght@400;500;600;700&display=swap"
            ),
            rel="stylesheet",
        ),
        dcc.Store(id="refresh-nonce", data=0),
        dcc.Download(id="download-report"),
        dmc.Modal(
            id="methodology-modal",
            title="Methodology",
            opened=False,
            size="lg",
            centered=True,
            children=methodology_modal_body(),
        ),
        dmc.Modal(
            id="weights-modal",
            title="Published artifacts",
            opened=False,
            size="lg",
            centered=True,
            children=published_models_modal_body(),
        ),
        html.Div(
            [
                html.Header(
                    [
                        html.Div(
                            [
                                html.H1("Praxa Behavioral Alignment"),
                                html.P("Policy-alignment experiment results using COMPASS"),
                            ],
                            className="title-block",
                        ),
                        html.Div(
                            [
                                dmc.Button(
                                    _btn_label("rotate", "Refresh"),
                                    id="refresh-btn",
                                    variant="subtle",
                                    color=ACCENT,
                                    size="xs",
                                ),
                                dmc.Button(
                                    _btn_label("file-pdf", "Export PDF"),
                                    id="export-pdf-btn",
                                    variant="light",
                                    color=ACCENT,
                                    size="xs",
                                ),
                            ],
                            className="toolbar",
                        ),
                    ],
                    className="topbar",
                ),
                html.Div(id="about-card"),
                html.Div(
                    [
                        dmc.SegmentedControl(
                            id="domain-select",
                            data=[
                                {"value": key, "label": label.split(" (")[0]}
                                for key, label in DOMAIN_LABELS.items()
                            ],
                            value="medicareplus",
                            color=ACCENT,
                            size="sm",
                        ),
                    ],
                    className="domain-bar",
                ),
                dcc.Loading(
                    id="main-loading",
                    type="circle",
                    color="#38bdf8",
                    className="main-loading",
                    children=[
                html.Div(
                    [
                        html.Div(id="summary-stack"),
                        html.Div(id="findings-card"),
                    ],
                    className="headline-row",
                ),
                dmc.Tabs(
                    id="view-tabs",
                    value="overview",
                    color=ACCENT,
                    variant="pills",
                    radius="md",
                    className="view-tabs",
                    children=[
                        dmc.TabsList(
                            [
                                dmc.TabsTab("Overview", value="overview"),
                                dmc.TabsTab("Slices & edges", value="slices"),
                                dmc.TabsTab("Statistics", value="statistics"),
                                dmc.TabsTab("Data", value="data"),
                            ],
                            className="view-tabs-list",
                        ),
                        dmc.TabsPanel(
                            [
                                html.Section(
                                    [
                                        section_heading(
                                            "Macro-F1 ladder (autoregressive)",
                                            "ladder",
                                        ),
                                        html.P(
                                            "Instruct / autoregressive 8B models only: frozen "
                                            "prompting, generative LoRA, Behavioral KV, and RFDT "
                                            "(decision-token). ModernBERT Encoder-FT is a sequence "
                                            "classifier, so it is not a ladder column; see edge "
                                            "charts and the comparator panel for that run.",
                                            className="panel-note",
                                        ),
                                        _graph("ladder-heatmap"),
                                    ],
                                    className="panel",
                                ),
                                html.Section(
                                    [
                                        section_heading(
                                            "RFDT-LoRA vs Encoder-FT (macro-F1)",
                                            "encoder_comparator",
                                        ),
                                        html.P(
                                            "Head-to-head on the same held-out IDs and "
                                            "retrieved-policy contract. RFDT-LoRA = decision-token "
                                            "scoring on 8B instruct models. Encoder-FT = fully "
                                            "fine-tuned ModernBERT-base classifier.",
                                            className="panel-note",
                                        ),
                                        _graph("encoder-comparator-chart"),
                                    ],
                                    className="panel",
                                ),
                            ],
                            value="overview",
                        ),
                        dmc.TabsPanel(
                            [
                                html.Section(
                                    [
                                        section_heading(
                                            "Accuracy by query-type slice",
                                            "slice_heatmap",
                                        ),
                                        html.P(
                                            "Published by_query_type accuracy for Base, 4-shot, "
                                            "LoRA, Behavioral KV, RFDT-LoRA, and Encoder-FT. "
                                            "Denied-edge is usually the hard column.",
                                            className="panel-note",
                                        ),
                                        _graph("slice-heatmap"),
                                    ],
                                    className="panel",
                                ),
                                html.Div(
                                    [
                                        html.Section(
                                            [
                                                section_heading(
                                                    "RFDT vs Encoder: who wins",
                                                    "disagreement_bar",
                                                ),
                                                html.P(
                                                    "Paired on sample_id. Both correct / both wrong "
                                                    "/ only RFDT / only Encoder across the test set.",
                                                    className="panel-note",
                                                ),
                                                _graph("disagreement-bar"),
                                            ],
                                            className="panel",
                                        ),
                                        html.Section(
                                            [
                                                section_heading(
                                                    "Net RFDT edge by slice",
                                                    "disagreement_slice",
                                                ),
                                                html.P(
                                                    "Share RFDT-only correct minus Encoder-only "
                                                    "correct. Positive = RFDT wins the slice; "
                                                    "negative = Encoder wins.",
                                                    className="panel-note",
                                                ),
                                                _graph("disagreement-slice-heatmap"),
                                            ],
                                            className="panel",
                                        ),
                                    ],
                                    className="two-col",
                                ),
                                html.Div(
                                    [
                                        html.Section(
                                            [
                                                section_heading(
                                                    "Lift vs Base (autoregressive)",
                                                    "lift",
                                                ),
                                                html.P(
                                                    "Δ macro-F1 vs each instruct model's Base. "
                                                    "Encoder-FT has no Base rung, so it is omitted here.",
                                                    className="panel-note",
                                                ),
                                                _graph("lift-chart"),
                                            ],
                                            className="panel",
                                        ),
                                        html.Section(
                                            [
                                                section_heading(
                                                    "Learned vs best frozen (autoregressive)",
                                                    "learned_vs_frozen",
                                                ),
                                                html.P(
                                                    "Best frozen prompt vs LoRA / Behavioral KV / "
                                                    "RFDT-LoRA on the matched-8B instruct models.",
                                                    className="panel-note",
                                                ),
                                                _graph("learned-chart"),
                                            ],
                                            className="panel",
                                        ),
                                    ],
                                    className="two-col",
                                ),
                                html.Div(
                                    [
                                        html.Section(
                                            [
                                                section_heading(
                                                    "Allowed-edge accuracy",
                                                    "allowed_edge",
                                                ),
                                                html.P(
                                                    "Includes ModernBERT Encoder-FT alongside "
                                                    "autoregressive conditions when published.",
                                                    className="panel-note",
                                                ),
                                                _graph("allowed-edge-chart"),
                                            ],
                                            className="panel",
                                        ),
                                        html.Section(
                                            [
                                                section_heading(
                                                    "Denied-edge refuse recall",
                                                    "denied_edge",
                                                ),
                                                html.P(
                                                    "Includes ModernBERT Encoder-FT alongside "
                                                    "autoregressive conditions when published.",
                                                    className="panel-note",
                                                ),
                                                _graph("denied-edge-chart"),
                                            ],
                                            className="panel",
                                        ),
                                    ],
                                    className="two-col",
                                ),
                            ],
                            value="slices",
                        ),
                        dmc.TabsPanel(
                            [
                                html.Div(id="statistics-panels"),
                            ],
                            value="statistics",
                        ),
                        dmc.TabsPanel(
                            [
                                html.Section(
                                    [
                                        section_heading("Metrics table", "metrics_table"),
                                        html.P(
                                            "Autoregressive ladder rows plus Encoder-FT for the "
                                            "matched-primary 8B trio and ModernBERT.",
                                            className="panel-note",
                                        ),
                                        html.Div(id="metrics-table-wrap"),
                                    ],
                                    className="panel",
                                ),
                            ],
                            value="data",
                        ),
                    ],
                ),
                    ],
                ),
                footer(),
            ],
            className="page",
        ),
    ],
    forceColorScheme="dark",
    theme={
        "primaryColor": ACCENT,
        "colors": {
            "dark": [
                "#eff6ff",
                "#dbeafe",
                "#bfdbfe",
                "#93c5fd",
                "#60a5fa",
                "#3b82f6",
                "#1e3a5f",
                "#12203a",
                "#0c1220",
                "#070b14",
            ]
        },
    },
)


@app.callback(
    Output("methodology-modal", "opened"),
    Input("methodology-open", "n_clicks"),
    State("methodology-modal", "opened"),
    prevent_initial_call=True,
)
def toggle_methodology(n_clicks, opened):
    return not bool(opened)


@app.callback(
    Output("weights-modal", "opened"),
    Input("weights-open", "n_clicks"),
    State("weights-modal", "opened"),
    prevent_initial_call=True,
)
def toggle_weights(n_clicks, opened):
    return not bool(opened)


@app.callback(
    Output("refresh-nonce", "data"),
    Input("refresh-btn", "n_clicks"),
    State("refresh-nonce", "data"),
    prevent_initial_call=True,
)
def on_refresh(n_clicks, nonce):
    refresh_cache(force=True)
    return (nonce or 0) + 1


@app.callback(
    Output("download-report", "data"),
    Input("export-pdf-btn", "n_clicks"),
    prevent_initial_call=True,
)
def on_export_pdf(n_clicks):
    pdf_bytes = build_alignment_report_pdf(force_refresh=False)
    name = report_filename()
    return dcc.send_bytes(pdf_bytes, filename=name)


@app.callback(
    Output("about-card", "children"),
    Input("domain-select", "value"),
)
def update_about(domain):
    return render_about(domain)


def statistics_panels(domain: str) -> html.Div:
    """Build Statistics tab contents; omit sections with nothing to show."""
    boot_df = bootstrap_contrasts_frame(domain)
    panels: list = []
    if not boot_df.empty:
        panels.append(
            bootstrap_section_children(
                bootstrap_contrast_forest(boot_df),
                bootstrap_table(boot_df),
            )
        )
    mcnemar = mcnemar_section(domain)
    if getattr(mcnemar, "className", None) == "panel":
        panels.append(mcnemar)
    atlas = error_atlas_section(domain)
    if getattr(atlas, "className", None) == "panel":
        panels.append(atlas)
    judge = judge_sensitivity_section(domain)
    if getattr(judge, "className", None) == "panel":
        panels.append(judge)
    audit = release_audit_section()
    if getattr(audit, "className", None) == "panel":
        panels.append(audit)
    if not panels:
        return html.Div(
            html.P(
                "No statistical summaries for this domain yet.",
                className="panel-note",
            )
        )
    return html.Div(panels)


@app.callback(
    Output("summary-stack", "children"),
    Output("findings-card", "children"),
    Output("ladder-heatmap", "figure"),
    Output("encoder-comparator-chart", "figure"),
    Output("slice-heatmap", "figure"),
    Output("disagreement-bar", "figure"),
    Output("disagreement-slice-heatmap", "figure"),
    Output("lift-chart", "figure"),
    Output("learned-chart", "figure"),
    Output("allowed-edge-chart", "figure"),
    Output("denied-edge-chart", "figure"),
    Output("statistics-panels", "children"),
    Output("metrics-table-wrap", "children"),
    Input("domain-select", "value"),
    Input("refresh-nonce", "data"),
)
def update_views(domain, _nonce):
    # Matched-primary 8B ladder + ModernBERT Encoder-FT. No toggle: charts
    # already separate AR ladder from the classifier comparator.
    df_raw = get_metrics(domain, matched_8b_only=False)
    if not df_raw.empty:
        keep = df_raw["model_key"].isin(MATCHED_8B_KEYS) | (
            df_raw["family"] == "encoder"
        )
        df_raw = df_raw[keep].reset_index(drop=True)

    df_ladder = generative_ladder_frame(df_raw)
    df_comp = encoder_comparator_frame(df_raw)

    df_narrative = pd.concat(
        [df_ladder, df_raw[df_raw["family"] == "encoder"]],
        ignore_index=True,
    )
    if not df_narrative.empty:
        df_narrative = df_narrative.drop_duplicates(
            subset=["model_key", "condition"], keep="last"
        )

    df_slices = df_narrative if not df_narrative.empty else df_ladder

    disagree = rfdt_encoder_disagreement(domain, matched_8b_only=True)
    disagree_sum = disagreement_summary(disagree)
    disagree_slice = disagreement_by_slice(disagree)

    stats_panels = statistics_panels(domain)

    stats = best_by_family(df_narrative, metric="macro_f1")
    strip = summary_stack(stats)
    bundle = narrative_bundle(df_narrative, domain)
    story = render_story_card(bundle.findings, bundle.implications)
    table_df = df_narrative if not df_narrative.empty else df_ladder
    table = metrics_table(table_df)

    if df_ladder.empty and df_comp.empty:
        msg = "No metrics loaded for this domain."
        empty = empty_figure(msg)
        return (
            strip,
            story,
            empty,
            empty,
            empty,
            empty,
            empty,
            empty,
            empty,
            empty,
            empty,
            stats_panels,
            table,
        )

    return (
        strip,
        story,
        ladder_heatmap(df_ladder, metric="macro_f1"),
        rfdt_vs_encoder_bar(df_comp, metric="macro_f1"),
        slice_ladder_heatmap(df_slices),
        rfdt_encoder_disagreement_bar(disagree_sum),
        rfdt_encoder_slice_edge_heatmap(disagree_slice),
        lift_bar(df_ladder, metric="macro_f1"),
        learned_vs_frozen_bar(df_ladder, metric="macro_f1"),
        edge_metric_bar(
            table_df,
            metric="allowed_edge",
            title="Allowed-edge accuracy",
        ),
        edge_metric_bar(
            table_df,
            metric="denied_edge_refuse_recall",
            title="Denied-edge refuse recall",
        ),
        stats_panels,
        table,
    )


# Warm memory/disk cache when the worker boots so the first page hit is local.
try:
    refresh_cache(force=False)
except Exception:
    pass


if __name__ == "__main__":
    app.run(debug=True, port=8050)