"""Praxa Behavioral Alignment Analysis — Dash Space. Reads published metrics from EdyVision/praxa-behavioral-experiments (HF_TOKEN). """ from __future__ import annotations import dash import dash_mantine_components as dmc import pandas as pd from dash import Input, Output, State, dash_table, dcc, html from dash_iconify import DashIconify from charts import ( ICON_ACCENT, CONTRAST_LABELS, edge_metric_bar, empty_figure, ladder_heatmap, learned_vs_frozen_bar, lift_bar, bootstrap_contrast_forest, rfdt_encoder_disagreement_bar, rfdt_encoder_slice_edge_heatmap, rfdt_vs_encoder_bar, slice_ladder_heatmap, ) from data import ( ALL_CONDITIONS, DOMAIN_LABELS, MATCHED_8B_KEYS, best_by_family, bootstrap_contrasts_frame, disagreement_by_slice, disagreement_summary, encoder_comparator_frame, generative_ladder_frame, get_analysis, get_metrics, mcnemar_contrasts_frame, refresh_cache, rfdt_encoder_disagreement, ) from narrative import ( BLOG_POST, PAPER_STATUS, PAPER_TITLE, PRAXA_LIBRARY, SHOW_BLOG_POST, SHOW_PRAXA_LIBRARY, about_card, methodology_modal_body, narrative_bundle, published_models_modal_body, ) from report import build_alignment_report_pdf, report_filename app = dash.Dash(__name__) app.title = "Praxa Behavioral Alignment" server = app.server LOADING_FIGURE = empty_figure("Loading published metrics…") def _graph(graph_id: str) -> dcc.Graph: """Graph with a dark loading placeholder so panels are never blank white.""" return dcc.Graph( id=graph_id, figure=LOADING_FIGURE, config={"displayModeBar": False}, style={"width": "100%", "minHeight": "280px"}, ) FA = "fa-solid" ICON = ICON_ACCENT ACCENT = "blue" RESEARCHER_HF = "https://huggingface.co/EdyVision" TABLE_COLS = [ ("model", "Model"), ("condition_label", "Condition"), ("macro_f1", "Macro-F1"), ("accuracy", "Accuracy"), ("answer_f1", "Answer F1"), ("refuse_f1", "Refuse F1"), ("allowed_edge", "Allowed-edge"), ("denied_edge_refuse_recall", "Denied refuse rec."), ] def _fmt(value: float | None) -> str: if value is None or (isinstance(value, float) and pd.isna(value)): return "—" return f"{float(value):.3f}" def _fa(name: str, size: int = 14) -> DashIconify: return DashIconify(icon=f"{FA}:{name}", width=size, color=ICON) def _btn_label(icon: str, text: str) -> html.Span: """Icon + label with the same gap used on info-card titles.""" return html.Span( [_fa(icon, 14), html.Span(text)], className="btn-with-icon", ) # Formula + why blurbs for section-title info tips (CSS hover, all tabs). SECTION_FORMULAS: dict[str, str] = { "ladder": ( "Formula: macro-F1 = (F1_answer + F1_refuse) / 2, where " "F1_c = 2·P_c·R_c / (P_c + R_c) on the frozen held-out split " "(invalid generative parses counted as refuse). " "Why: primary paper metric; averages both policy classes so a method " "cannot look strong by always answering or always refusing." ), "encoder_comparator": ( "Formula: same macro-F1 on identical sample_ids and the retrieved-policy " "input contract for RFDT-LoRA vs ModernBERT Encoder-FT. " "Why: isolates representation-first LoRA against a conventional encoder " "under a matched task, not a weaker truncated-input baseline." ), "slice_heatmap": ( "Formula: Acc_s = (# correct in stratum s) / n_s for " "s ∈ {allowed_base, allowed_edge, denied_base, denied_edge}. " "Why: overall macro-F1 can hide edge failures; stratum accuracy shows " "where policy behavior breaks." ), "disagreement_bar": ( "Formula (paired on sample_id): counts of " "(RFDT✓∧Enc✓), (RFDT✗∧Enc✗), (RFDT✓∧Enc✗), (RFDT✗∧Enc✓). " "Why: mean F1 gaps do not show whether RFDT wins the same examples " "Encoder misses, or merely different error patterns." ), "disagreement_slice": ( "Formula: edge_s = (n_RFDT-only correct − n_Encoder-only correct) / n_s " "within query-type stratum s. " "Why: localizes which policy strata drive the RFDT vs Encoder advantage." ), "lift": ( "Formula: Lift(model, condition) = macro-F1(condition) − macro-F1(Base) " "within the same instruct model family. " "Why: subtracts each model’s unadapted baseline so gains are adaptation " "effects, not cross-family absolute score differences." ), "learned_vs_frozen": ( "Formula: compare macro-F1(best frozen prompt) vs macro-F1(LoRA), " "macro-F1(Behavioral KV), and macro-F1(RFDT-LoRA) on matched-8B models. " "Why: asks whether learned adaptation beats the strongest inference-time " "control under the same split and supervision." ), "allowed_edge": ( "Formula: Acc_allowed-edge = (# correct | query_type=allowed_edge) / n. " "Why: allowed-edge items are permit-but-hard; this checks over-refusal " "when answering is policy-legal." ), "denied_edge": ( "Formula: RefuseRecall_denied-edge = TP_refuse / (TP_refuse + FN_refuse) " "on gold-REFUSE denied-edge items. " "Why: measures catching prohibited asks without diluting the score with " "easy denied-base refusals." ), "metrics_table": ( "Formula: published metrics.json per freeze cell — " "macro-F1, Acc=(# correct)/n, class F1s, edge stratum rates. " "Why: tabular audit of the same numbers behind the charts, keyed to " "immutable Hub evaluation bundles." ), "bootstrap": ( "Formula: Δ = macro-F1(A) − macro-F1(B). Draw B=10,000 resamples of " "leakage groups with replacement; 95% CI = [P_2.5(Δ*), P_97.5(Δ*)]. " "Two-sided bootstrap p = min(1, 2·min(mean(Δ*≤0), mean(Δ*≥0))). " "Holm within each named contrast (m=6 model–domain cells): sort " "p_(1)≤…≤p_(m); p̃_(k) = max_{j≤k} min(1, (m−j+1)·p_(j)). " "Why: groups (not rows) respect leakage structure; percentile CIs avoid " "normal approximations; Holm controls family-wise error across the six " "matched cells for that contrast." ), "mcnemar": ( "Formula: on matched IDs, b = A-only correct, c = B-only correct; " "exact two-sided p = BinomialTest(b; n=b+c, π=1/2) (McNemar). " "Holm across the same six cells as bootstrap. " "Why: tests whether A wins more discordant examples than B; supports " "macro-F1 inference without replacing it." ), "error_atlas": ( "Formula: confusion C_{g,ŷ} = count(gold=g, pred=ŷ) for " "g,ŷ∈{answer,refuse}; Acc_s = (# correct)/n_s by query_type; " "invalid rates from metrics.json. " "Why: decomposes aggregate F1 into which class and stratum errors dominate." ), "judge_sensitivity": ( "Formula: Agree = mean(1[ŷ_primary = ŷ_indep]). " "Acc_j = mean(1[ŷ_j = gold]). " "On disagreements: Acc_opt = mean(1[ŷ_primary=gold ∨ ŷ_indep=gold]); " "Acc_pes = mean(1[ŷ_primary=gold ∧ ŷ_indep=gold]). " "Ordering: sign(macro-F1_LoRA − macro-F1_KV) under each judge’s labels. " "Why: generative scores depend on a measurement adapter; this bounds " "judge-choice sensitivity and checks whether LoRA>KV is judge-fragile." ), "release_audit": ( "Formula: status = pass iff n_hits = 0 after scanning publishable " "findings for private hosts, URLs, credentials, and local paths. " "Why: public Hub/OSF artifacts must not leak lab infrastructure." ), } def section_heading(title: str, formula_key: str) -> html.Div: """Section title with a spaced info icon explaining the calculation. Uses a CSS hover tip (not Mantine Tooltip) so it works inside every tab panel, including panels that stay mounted while hidden. """ formula = SECTION_FORMULAS.get(formula_key, "").strip() tip_children: list = [ DashIconify(icon=f"{FA}:circle-info", width=15, color="#60a5fa"), ] if formula: tip_children.append(html.Span(formula, className="panel-heading-tip")) icon = html.Span( tip_children, className="panel-heading-info", tabIndex=0, role="img", title=formula or None, **{"aria-label": f"How {title} is calculated"}, ) return html.Div([html.H2(title), icon], className="panel-heading") def metrics_table(df: pd.DataFrame) -> dash_table.DataTable: if df.empty: records: list[dict] = [] else: show = df.copy() order = {c: i for i, c in enumerate(ALL_CONDITIONS)} show["_ord"] = show["condition"].map(lambda c: order.get(c, 99)) show = show.sort_values(["model", "_ord"]) records = [] for row in show.itertuples(index=False): records.append( { "model": row.model, "condition_label": row.condition_label, "macro_f1": _fmt(getattr(row, "macro_f1", None)), "accuracy": _fmt(getattr(row, "accuracy", None)), "answer_f1": _fmt(getattr(row, "answer_f1", None)), "refuse_f1": _fmt(getattr(row, "refuse_f1", None)), "allowed_edge": _fmt(getattr(row, "allowed_edge", None)), "denied_edge_refuse_recall": _fmt( getattr(row, "denied_edge_refuse_recall", None) ), } ) return dash_table.DataTable( id="metrics-table", columns=[{"name": label, "id": key} for key, label in TABLE_COLS], data=records, sort_action="native", filter_action="native", page_size=16, style_table={"overflowX": "auto"}, style_header={ "backgroundColor": "#12203a", "color": "#93c5fd", "fontWeight": "600", "border": "1px solid #1e3a5f", "fontFamily": "IBM Plex Sans, Helvetica, sans-serif", "fontSize": "13px", }, style_cell={ "backgroundColor": "#0c1220", "color": "#dbeafe", "border": "1px solid #1e3a5f", "fontFamily": "IBM Plex Sans, Helvetica, sans-serif", "fontSize": "13px", "padding": "8px 10px", "minWidth": "90px", }, style_filter={ "backgroundColor": "#0a1628", "color": "#dbeafe", "border": "1px solid #1e3a5f", }, style_data_conditional=[ {"if": {"row_index": "odd"}, "backgroundColor": "#0a1628"}, ], ) def bootstrap_table(df: pd.DataFrame) -> dash_table.DataTable: """Compact table for published leakage-group bootstrap contrasts.""" records: list[dict] = [] if not df.empty: show = df.copy() show = show.sort_values(["family", "model_key", "contrast_id"]) for row in show.itertuples(index=False): holm = getattr(row, "holm_p", None) records.append( { "family": row.family, "model_key": row.model_key, "contrast": CONTRAST_LABELS.get( str(row.contrast_id), str(row.contrast_id) ), "delta": _fmt(getattr(row, "delta_macro_f1", None)), "ci": ( f"[{_fmt(getattr(row, 'ci_low', None))}, " f"{_fmt(getattr(row, 'ci_high', None))}]" ), "p": _fmt(getattr(row, "bootstrap_p", None)), "holm_p": _fmt(holm) if holm is not None and not pd.isna(holm) else "—", } ) cols = [ ("family", "Family"), ("model_key", "Model"), ("contrast", "Contrast"), ("delta", "Δ macro-F1"), ("ci", "95% CI"), ("p", "p"), ("holm_p", "Holm p"), ] return dash_table.DataTable( id="bootstrap-table", columns=[{"name": label, "id": key} for key, label in cols], data=records, sort_action="native", page_size=12, style_table={"overflowX": "auto"}, style_header={ "backgroundColor": "#12203a", "color": "#93c5fd", "fontWeight": "600", "border": "1px solid #1e3a5f", "fontFamily": "IBM Plex Sans, Helvetica, sans-serif", "fontSize": "13px", }, style_cell={ "backgroundColor": "#0c1220", "color": "#dbeafe", "border": "1px solid #1e3a5f", "fontFamily": "IBM Plex Sans, Helvetica, sans-serif", "fontSize": "13px", "padding": "8px 10px", "minWidth": "80px", }, style_data_conditional=[ {"if": {"row_index": "odd"}, "backgroundColor": "#0a1628"}, ], ) def mcnemar_section(domain: str) -> html.Section | html.Div: df = mcnemar_contrasts_frame(domain) if df.empty: return html.Div() records: list[dict] = [] show = df.sort_values(["family", "model_key", "contrast_id"]) for row in show.itertuples(index=False): records.append( { "family": row.family, "model_key": row.model_key, "contrast": CONTRAST_LABELS.get( str(row.contrast_id), str(row.contrast_id) ), "b": int(getattr(row, "n_a_only", 0)), "c": int(getattr(row, "n_b_only", 0)), "p": _fmt(getattr(row, "exact_p", None)), "holm_p": ( _fmt(getattr(row, "holm_p", None)) if getattr(row, "holm_p", None) is not None and not pd.isna(getattr(row, "holm_p", None)) else "—" ), } ) cols = [ ("family", "Family"), ("model_key", "Model"), ("contrast", "Contrast"), ("b", "A only correct"), ("c", "B only correct"), ("p", "Exact p"), ("holm_p", "Holm p"), ] return html.Section( [ section_heading("McNemar (matched correctness)", "mcnemar"), html.P( "Exact tests on paired correct/incorrect labels for the same " "examples. Discordant cells show where only one method is right.", className="panel-note", ), dash_table.DataTable( id="mcnemar-table", columns=[{"name": label, "id": key} for key, label in cols], data=records, sort_action="native", page_size=12, style_table={"overflowX": "auto"}, style_header={ "backgroundColor": "#12203a", "color": "#93c5fd", "fontWeight": "600", "border": "1px solid #1e3a5f", "fontFamily": "IBM Plex Sans, Helvetica, sans-serif", "fontSize": "13px", }, style_cell={ "backgroundColor": "#0c1220", "color": "#dbeafe", "border": "1px solid #1e3a5f", "fontFamily": "IBM Plex Sans, Helvetica, sans-serif", "fontSize": "13px", "padding": "8px 10px", "minWidth": "80px", }, style_data_conditional=[ {"if": {"row_index": "odd"}, "backgroundColor": "#0a1628"}, ], ), ], className="panel", ) def error_atlas_section(domain: str) -> html.Section | html.Div: blob = get_analysis("error_atlas") if not blob: return html.Div() payload = blob.get("payload") or {} domains = payload.get("domains") or {} block = domains.get(domain) if not block: return html.Div() children: list = [ section_heading("Error atlas", "error_atlas"), html.P( "Confusion counts, accuracy by query type, and invalid-output rates " "for the reported methods. Counts only; no query text.", className="panel-note", ), ] summary = block.get("summary") or {} if summary.get("n_cells") is not None: children.append( html.P( f"{summary.get('n_cells')} method cells across " f"{summary.get('n_models', '—')} models.", className="panel-note", ) ) for table in (block.get("tables") or [])[:4]: title = table.get("title") or "Table" rows = table.get("rows") or [] if not rows: continue frame = pd.DataFrame(rows) children.append(html.H3(title, className="panel-subhead")) children.append( dash_table.DataTable( columns=[{"name": c, "id": c} for c in frame.columns], data=frame.to_dict("records"), page_size=8, style_table={"overflowX": "auto", "marginBottom": "12px"}, style_header={ "backgroundColor": "#12203a", "color": "#93c5fd", "fontWeight": "600", "border": "1px solid #1e3a5f", "fontSize": "12px", }, style_cell={ "backgroundColor": "#0c1220", "color": "#dbeafe", "border": "1px solid #1e3a5f", "fontSize": "12px", "padding": "6px 8px", }, ) ) if len(children) <= 2: return html.Div() return html.Section(children, className="panel") def bootstrap_section_children( boot_fig, boot_table: dash_table.DataTable, ) -> html.Section: return html.Section( [ section_heading("Leakage-group bootstrap", "bootstrap"), html.P( "Paired macro-F1 differences for predeclared contrasts, " "resampling leakage groups (not rows) with percentile " "95% CIs. Holm adjusts the six matched domain×model tests " "within each named contrast.", className="panel-note", ), dcc.Graph( figure=boot_fig, config={"displayModeBar": False}, style={"width": "100%"}, ), boot_table, ], className="panel", ) def judge_sensitivity_section(domain: str) -> html.Section | html.Div: blob = get_analysis("judge_sensitivity") if not blob: return html.Div() payload = blob.get("payload") or {} overall = payload.get("overall") or {} bounds = payload.get("optimistic_pessimistic_bounds") or {} ordering = payload.get("lora_vs_kv_ordering") or {} cells = [ row for row in (payload.get("headline_cells") or []) if str(row.get("domain")) == domain ] order_cells = [ row for row in (ordering.get("cells") or []) if str(row.get("domain")) == domain ] if not cells and not order_cells and not overall: return html.Div() children: list = [ section_heading("Judge sensitivity", "judge_sensitivity"), html.P( "Independent AI judge (v1.2 Gemini) vs frozen primary judge (v1.3) " "on the same free-form outputs. Checks whether LoRA−KV rankings " "depend on which grader is used.", className="panel-note", ), ] if overall.get("n"): children.append( html.P( f"Agreement {_fmt(overall.get('primary_vs_independent_agreement'))} · " f"primary vs gold {_fmt(overall.get('primary_vs_gold_accuracy'))} · " f"independent vs gold {_fmt(overall.get('independent_vs_gold_accuracy'))} · " f"optimistic / pessimistic " f"{_fmt(bounds.get('optimistic_accuracy'))} / " f"{_fmt(bounds.get('pessimistic_accuracy'))} " f"(all generative rows: {overall.get('n')}).", className="panel-note", ) ) if order_cells: children.append(html.H3("LoRA vs KV under each judge", className="panel-subhead")) order_records = [] for row in sorted(order_cells, key=lambda r: str(r.get("model_key"))): order_records.append( { "model_key": row.get("model_key"), "primary_delta": _fmt(row.get("primary_delta_lora_minus_kv")), "independent_delta": _fmt( row.get("independent_delta_lora_minus_kv") ), "changed": "yes" if row.get("ordering_changed") else "no", } ) children.append( dash_table.DataTable( id="judge-sensitivity-ordering", columns=[ {"name": "Model", "id": "model_key"}, {"name": "Primary Δ (LoRA−KV)", "id": "primary_delta"}, {"name": "Independent Δ", "id": "independent_delta"}, {"name": "Order changed?", "id": "changed"}, ], data=order_records, style_table={"overflowX": "auto", "marginBottom": "12px"}, style_header={ "backgroundColor": "#12203a", "color": "#93c5fd", "fontWeight": "600", "border": "1px solid #1e3a5f", "fontSize": "12px", }, style_cell={ "backgroundColor": "#0c1220", "color": "#dbeafe", "border": "1px solid #1e3a5f", "fontSize": "12px", "padding": "6px 8px", }, ) ) if cells: children.append(html.H3("Per-condition macro-F1", className="panel-subhead")) cell_records = [] for row in sorted( cells, key=lambda r: (str(r.get("model_key")), str(r.get("condition"))) ): cell_records.append( { "model_key": row.get("model_key"), "condition": row.get("condition"), "primary_f1": _fmt(row.get("primary_macro_f1")), "independent_f1": _fmt(row.get("independent_macro_f1")), "agree": _fmt(row.get("judge_agreement")), } ) children.append( dash_table.DataTable( id="judge-sensitivity-cells", columns=[ {"name": "Model", "id": "model_key"}, {"name": "Condition", "id": "condition"}, {"name": "Primary F1", "id": "primary_f1"}, {"name": "Independent F1", "id": "independent_f1"}, {"name": "Agree", "id": "agree"}, ], data=cell_records, page_size=10, sort_action="native", style_table={"overflowX": "auto"}, style_header={ "backgroundColor": "#12203a", "color": "#93c5fd", "fontWeight": "600", "border": "1px solid #1e3a5f", "fontSize": "12px", }, style_cell={ "backgroundColor": "#0c1220", "color": "#dbeafe", "border": "1px solid #1e3a5f", "fontSize": "12px", "padding": "6px 8px", }, ) ) return html.Section(children, className="panel") def release_audit_section() -> html.Section | html.Div: blob = get_analysis("release_audit") if not blob: return html.Div() payload = blob.get("payload") or {} status = payload.get("status") or "—" n_hits = payload.get("n_hits") n_files = payload.get("n_files_scanned") return html.Section( [ section_heading("Release safety audit", "release_audit"), html.P( f"Status: {status}. Scanned {n_files if n_files is not None else '—'} " f"publishable finding files; private-infra hits: " f"{n_hits if n_hits is not None else '—'}.", className="panel-note", ), ], className="panel", ) def summary_stack(stats: dict) -> html.Div: """Vertical stack of best-method cards (left column of headline row).""" cells = [] items = [ ( "Best frozen", ( f"{_fmt(stats['best_frozen']['value'])} · {stats['best_frozen']['model']}" if stats.get("best_frozen") else "—" ), "snowflake", ), ( "Best LoRA", ( f"{_fmt(stats['best_lora']['value'])} · {stats['best_lora']['model']}" if stats.get("best_lora") else "—" ), "layer-group", ), ( "Best KV", ( f"{_fmt(stats['best_kv']['value'])} · {stats['best_kv']['model']}" if stats.get("best_kv") else "—" ), "microchip", ), ( "Best RFDT", ( f"{_fmt(stats['best_rfdt']['value'])} · {stats['best_rfdt']['model']}" if stats.get("best_rfdt") else "—" ), "bullseye", ), ( "Encoder-FT", ( f"{_fmt(stats['best_encoder']['value'])} · {stats['best_encoder']['model']}" if stats.get("best_encoder") else "—" ), "robot", ), ] for title, value, icon in items: cells.append( html.Div( [ html.Div( [_fa(icon, 14), html.Span(title, className="summary-label")], className="summary-title", ), html.Div(value, className="summary-value"), ], className="summary-cell", ) ) return html.Div(cells, className="summary-stack") def render_about(domain: str) -> html.Div: card = about_card(domain) return html.Div( [ html.Div( [_fa(card.icon, 14), html.Span(card.title)], className="about-card-title", ), html.P( [ html.Em(PAPER_TITLE), f" ({PAPER_STATUS}). ", ], className="about-card-paper", ), html.P(card.body, className="about-card-body"), ], className="about-card", ) def render_narrative_card(card, *, class_name: str = "info-card") -> html.Div | None: if card is None: return None paragraphs = [ html.P(part.strip(), className="info-card-body") for part in str(card.body).split("\n\n") if part.strip() ] return html.Div( [ html.Div( [_fa(card.icon, 14), html.Span(card.title)], className="info-card-title", ), *paragraphs, ], className=class_name, ) def _narrative_block(card) -> list: if card is None: return [] paragraphs = [ html.P(part.strip(), className="info-card-body") for part in str(card.body).split("\n\n") if part.strip() ] return [ html.Div( [_fa(card.icon, 14), html.Span(card.title)], className="info-card-title", ), *paragraphs, ] def render_story_card(findings, implications) -> html.Div | None: """Single headline card: findings then implications, labels unchanged.""" blocks = _narrative_block(findings) implication_blocks = _narrative_block(implications) if implication_blocks: if blocks: blocks.append(html.Div(className="story-divider")) blocks.extend(implication_blocks) if not blocks: return None return html.Div(blocks, className="info-card findings-panel story-card") def footer() -> html.Footer: resources: list = [ html.Span("Resources", className="footer-label"), html.Span("·", className="footer-sep"), html.Button( "Methodology", id="methodology-open", n_clicks=0, className="footer-link", ), ] if SHOW_BLOG_POST: resources.extend( [ html.Span("·", className="footer-sep"), html.A( BLOG_POST["label"], href=BLOG_POST["href"], target="_blank", className="footer-link", title=BLOG_POST["note"], ), ] ) if SHOW_PRAXA_LIBRARY: resources.extend( [ html.Span("·", className="footer-sep"), html.A( PRAXA_LIBRARY["label"], href=PRAXA_LIBRARY["href"], target="_blank", className="footer-link", title=PRAXA_LIBRARY["note"], rel="noopener noreferrer", ), ] ) resources.extend( [ html.Span("·", className="footer-sep"), html.Button( "Published artifacts", id="weights-open", n_clicks=0, className="footer-link", ), ] ) return html.Footer( [ html.Div(resources, className="footer-row"), html.Div( [ html.Span("Researcher · "), html.A( "Eidan J. Rosado, PhD", href=RESEARCHER_HF, target="_blank", className="footer-link", ), ], className="footer-credit", ), ], className="site-footer", ) app.layout = dmc.MantineProvider( [ html.Link( href=( "https://fonts.googleapis.com/css2?" "family=IBM+Plex+Sans:wght@400;500;600;700&display=swap" ), rel="stylesheet", ), dcc.Store(id="refresh-nonce", data=0), dcc.Download(id="download-report"), dmc.Modal( id="methodology-modal", title="Methodology", opened=False, size="lg", centered=True, children=methodology_modal_body(), ), dmc.Modal( id="weights-modal", title="Published artifacts", opened=False, size="lg", centered=True, children=published_models_modal_body(), ), html.Div( [ html.Header( [ html.Div( [ html.H1("Praxa Behavioral Alignment"), html.P("Policy-alignment experiment results using COMPASS"), ], className="title-block", ), html.Div( [ dmc.Button( _btn_label("rotate", "Refresh"), id="refresh-btn", variant="subtle", color=ACCENT, size="xs", ), dmc.Button( _btn_label("file-pdf", "Export PDF"), id="export-pdf-btn", variant="light", color=ACCENT, size="xs", ), ], className="toolbar", ), ], className="topbar", ), html.Div(id="about-card"), html.Div( [ dmc.SegmentedControl( id="domain-select", data=[ {"value": key, "label": label.split(" (")[0]} for key, label in DOMAIN_LABELS.items() ], value="medicareplus", color=ACCENT, size="sm", ), ], className="domain-bar", ), dcc.Loading( id="main-loading", type="circle", color="#38bdf8", className="main-loading", children=[ html.Div( [ html.Div(id="summary-stack"), html.Div(id="findings-card"), ], className="headline-row", ), dmc.Tabs( id="view-tabs", value="overview", color=ACCENT, variant="pills", radius="md", className="view-tabs", children=[ dmc.TabsList( [ dmc.TabsTab("Overview", value="overview"), dmc.TabsTab("Slices & edges", value="slices"), dmc.TabsTab("Statistics", value="statistics"), dmc.TabsTab("Data", value="data"), ], className="view-tabs-list", ), dmc.TabsPanel( [ html.Section( [ section_heading( "Macro-F1 ladder (autoregressive)", "ladder", ), html.P( "Instruct / autoregressive 8B models only: frozen " "prompting, generative LoRA, Behavioral KV, and RFDT " "(decision-token). ModernBERT Encoder-FT is a sequence " "classifier, so it is not a ladder column; see edge " "charts and the comparator panel for that run.", className="panel-note", ), _graph("ladder-heatmap"), ], className="panel", ), html.Section( [ section_heading( "RFDT-LoRA vs Encoder-FT (macro-F1)", "encoder_comparator", ), html.P( "Head-to-head on the same held-out IDs and " "retrieved-policy contract. RFDT-LoRA = decision-token " "scoring on 8B instruct models. Encoder-FT = fully " "fine-tuned ModernBERT-base classifier.", className="panel-note", ), _graph("encoder-comparator-chart"), ], className="panel", ), ], value="overview", ), dmc.TabsPanel( [ html.Section( [ section_heading( "Accuracy by query-type slice", "slice_heatmap", ), html.P( "Published by_query_type accuracy for Base, 4-shot, " "LoRA, Behavioral KV, RFDT-LoRA, and Encoder-FT. " "Denied-edge is usually the hard column.", className="panel-note", ), _graph("slice-heatmap"), ], className="panel", ), html.Div( [ html.Section( [ section_heading( "RFDT vs Encoder: who wins", "disagreement_bar", ), html.P( "Paired on sample_id. Both correct / both wrong " "/ only RFDT / only Encoder across the test set.", className="panel-note", ), _graph("disagreement-bar"), ], className="panel", ), html.Section( [ section_heading( "Net RFDT edge by slice", "disagreement_slice", ), html.P( "Share RFDT-only correct minus Encoder-only " "correct. Positive = RFDT wins the slice; " "negative = Encoder wins.", className="panel-note", ), _graph("disagreement-slice-heatmap"), ], className="panel", ), ], className="two-col", ), html.Div( [ html.Section( [ section_heading( "Lift vs Base (autoregressive)", "lift", ), html.P( "Δ macro-F1 vs each instruct model's Base. " "Encoder-FT has no Base rung, so it is omitted here.", className="panel-note", ), _graph("lift-chart"), ], className="panel", ), html.Section( [ section_heading( "Learned vs best frozen (autoregressive)", "learned_vs_frozen", ), html.P( "Best frozen prompt vs LoRA / Behavioral KV / " "RFDT-LoRA on the matched-8B instruct models.", className="panel-note", ), _graph("learned-chart"), ], className="panel", ), ], className="two-col", ), html.Div( [ html.Section( [ section_heading( "Allowed-edge accuracy", "allowed_edge", ), html.P( "Includes ModernBERT Encoder-FT alongside " "autoregressive conditions when published.", className="panel-note", ), _graph("allowed-edge-chart"), ], className="panel", ), html.Section( [ section_heading( "Denied-edge refuse recall", "denied_edge", ), html.P( "Includes ModernBERT Encoder-FT alongside " "autoregressive conditions when published.", className="panel-note", ), _graph("denied-edge-chart"), ], className="panel", ), ], className="two-col", ), ], value="slices", ), dmc.TabsPanel( [ html.Div(id="statistics-panels"), ], value="statistics", ), dmc.TabsPanel( [ html.Section( [ section_heading("Metrics table", "metrics_table"), html.P( "Autoregressive ladder rows plus Encoder-FT for the " "matched-primary 8B trio and ModernBERT.", className="panel-note", ), html.Div(id="metrics-table-wrap"), ], className="panel", ), ], value="data", ), ], ), ], ), footer(), ], className="page", ), ], forceColorScheme="dark", theme={ "primaryColor": ACCENT, "colors": { "dark": [ "#eff6ff", "#dbeafe", "#bfdbfe", "#93c5fd", "#60a5fa", "#3b82f6", "#1e3a5f", "#12203a", "#0c1220", "#070b14", ] }, }, ) @app.callback( Output("methodology-modal", "opened"), Input("methodology-open", "n_clicks"), State("methodology-modal", "opened"), prevent_initial_call=True, ) def toggle_methodology(n_clicks, opened): return not bool(opened) @app.callback( Output("weights-modal", "opened"), Input("weights-open", "n_clicks"), State("weights-modal", "opened"), prevent_initial_call=True, ) def toggle_weights(n_clicks, opened): return not bool(opened) @app.callback( Output("refresh-nonce", "data"), Input("refresh-btn", "n_clicks"), State("refresh-nonce", "data"), prevent_initial_call=True, ) def on_refresh(n_clicks, nonce): refresh_cache(force=True) return (nonce or 0) + 1 @app.callback( Output("download-report", "data"), Input("export-pdf-btn", "n_clicks"), prevent_initial_call=True, ) def on_export_pdf(n_clicks): pdf_bytes = build_alignment_report_pdf(force_refresh=False) name = report_filename() return dcc.send_bytes(pdf_bytes, filename=name) @app.callback( Output("about-card", "children"), Input("domain-select", "value"), ) def update_about(domain): return render_about(domain) def statistics_panels(domain: str) -> html.Div: """Build Statistics tab contents; omit sections with nothing to show.""" boot_df = bootstrap_contrasts_frame(domain) panels: list = [] if not boot_df.empty: panels.append( bootstrap_section_children( bootstrap_contrast_forest(boot_df), bootstrap_table(boot_df), ) ) mcnemar = mcnemar_section(domain) if getattr(mcnemar, "className", None) == "panel": panels.append(mcnemar) atlas = error_atlas_section(domain) if getattr(atlas, "className", None) == "panel": panels.append(atlas) judge = judge_sensitivity_section(domain) if getattr(judge, "className", None) == "panel": panels.append(judge) audit = release_audit_section() if getattr(audit, "className", None) == "panel": panels.append(audit) if not panels: return html.Div( html.P( "No statistical summaries for this domain yet.", className="panel-note", ) ) return html.Div(panels) @app.callback( Output("summary-stack", "children"), Output("findings-card", "children"), Output("ladder-heatmap", "figure"), Output("encoder-comparator-chart", "figure"), Output("slice-heatmap", "figure"), Output("disagreement-bar", "figure"), Output("disagreement-slice-heatmap", "figure"), Output("lift-chart", "figure"), Output("learned-chart", "figure"), Output("allowed-edge-chart", "figure"), Output("denied-edge-chart", "figure"), Output("statistics-panels", "children"), Output("metrics-table-wrap", "children"), Input("domain-select", "value"), Input("refresh-nonce", "data"), ) def update_views(domain, _nonce): # Matched-primary 8B ladder + ModernBERT Encoder-FT. No toggle: charts # already separate AR ladder from the classifier comparator. df_raw = get_metrics(domain, matched_8b_only=False) if not df_raw.empty: keep = df_raw["model_key"].isin(MATCHED_8B_KEYS) | ( df_raw["family"] == "encoder" ) df_raw = df_raw[keep].reset_index(drop=True) df_ladder = generative_ladder_frame(df_raw) df_comp = encoder_comparator_frame(df_raw) df_narrative = pd.concat( [df_ladder, df_raw[df_raw["family"] == "encoder"]], ignore_index=True, ) if not df_narrative.empty: df_narrative = df_narrative.drop_duplicates( subset=["model_key", "condition"], keep="last" ) df_slices = df_narrative if not df_narrative.empty else df_ladder disagree = rfdt_encoder_disagreement(domain, matched_8b_only=True) disagree_sum = disagreement_summary(disagree) disagree_slice = disagreement_by_slice(disagree) stats_panels = statistics_panels(domain) stats = best_by_family(df_narrative, metric="macro_f1") strip = summary_stack(stats) bundle = narrative_bundle(df_narrative, domain) story = render_story_card(bundle.findings, bundle.implications) table_df = df_narrative if not df_narrative.empty else df_ladder table = metrics_table(table_df) if df_ladder.empty and df_comp.empty: msg = "No metrics loaded for this domain." empty = empty_figure(msg) return ( strip, story, empty, empty, empty, empty, empty, empty, empty, empty, empty, stats_panels, table, ) return ( strip, story, ladder_heatmap(df_ladder, metric="macro_f1"), rfdt_vs_encoder_bar(df_comp, metric="macro_f1"), slice_ladder_heatmap(df_slices), rfdt_encoder_disagreement_bar(disagree_sum), rfdt_encoder_slice_edge_heatmap(disagree_slice), lift_bar(df_ladder, metric="macro_f1"), learned_vs_frozen_bar(df_ladder, metric="macro_f1"), edge_metric_bar( table_df, metric="allowed_edge", title="Allowed-edge accuracy", ), edge_metric_bar( table_df, metric="denied_edge_refuse_recall", title="Denied-edge refuse recall", ), stats_panels, table, ) # Warm memory/disk cache when the worker boots so the first page hit is local. try: refresh_cache(force=False) except Exception: pass if __name__ == "__main__": app.run(debug=True, port=8050)