"""Shared data, palette, and rendering for the Space. Reads only the exported snapshot in ./data. Runs under regular Streamlit (local) and stlite (the static Hugging Face Space, in the browser).""" from __future__ import annotations import html import json import re from pathlib import Path import altair as alt import pandas as pd import streamlit as st DATA = Path(__file__).parent / "data" FIELDS = ["Symptoms", "Diagnosis", "History of Patient", "Plan of Action"] # Fixed order; a model keeps its color on every page, and rows are drawn in this order. Slots follow the # reference palette's validated adjacent order (blue, orange, aqua, yellow); every chart also labels each row # by name, so color is never the only cue. Archived BART-large shares a slot only if a snapshot includes it. MODELS = { "biobart": {"label": "BioBART-base", "slot": 0, "params": "166M", "about": "BART pretrained on PubMed text"}, "t5_base": {"label": "T5-base", "slot": 1, "params": "223M", "about": "text-to-text, general pretraining"}, "biobart_large": {"label": "BioBART-large", "slot": 2, "params": "406M", "about": "larger biomedical BART"}, "pegasus_large": {"label": "PEGASUS-large", "slot": 3, "params": "568M", "about": "gap-sentence pretraining (pretrained-only checkpoint)"}, "bart_large_cnn": {"label": "BART-large-CNN (archived)", "slot": 2, "params": "406M", "about": "news summarizer"}, "bart_large_cnn_minlen": {"label": "BART-large-CNN, min_length=10 (archived)", "slot": 2, "params": "406M", "about": "same model, minimum decode length"}, } SERIES = {"light": ["#2a78d6", "#eb6834", "#1baf7a", "#eda100"], "dark": ["#3987e5", "#d95926", "#199e70", "#c98500"]} INK = { "light": {"primary": "#0b0b0b", "secondary": "#52514e", "muted": "#898781", "grid": "#e1e0d9", "axis": "#c3c2b7", "reference": "#52514e", "mark": "#fff3bf", "surface": "#fcfcfb"}, "dark": {"primary": "#f0efec", "secondary": "#c3c2b7", "muted": "#898781", "grid": "#2c2c2a", "axis": "#383835", "reference": "#c3c2b7", "mark": "#5c4a00", "surface": "#1a1a19"}, } STATUS = {"good": "#0ca30c", "warning": "#fab219", "serious": "#ec835a", "critical": "#d03b3b"} LABEL_STYLE = { # judge label -> (status color, icon, text): status colors always ship with icon + text "supported": ("good", "✓", "supported"), "unsupported": ("critical", "✕", "not in dialogue"), "contradicted": ("critical", "✕", "contradicts dialogue"), "misplaced": ("warning", "↔", "wrong field"), } def mode() -> str: try: return "dark" if st.context.theme.type == "dark" else "light" except Exception: return "light" def color(family: str) -> str: return SERIES[mode()][MODELS.get(family, {"slot": 0})["slot"]] def label(family: str) -> str: return MODELS.get(family, {"label": family})["label"] @st.cache_data def load() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]: # JSON, not parquet: the static Space runs in the browser (stlite/Pyodide), where pyarrow may be missing. examples = pd.read_json(DATA / "examples.json", orient="records") notes = pd.read_json(DATA / "notes.json", orient="records") runs = pd.read_json(DATA / "runs.json", orient="records") summary = json.loads((DATA / "summary.json").read_text()) for df, col in ((notes, "verdict_json"), (examples, "ref_verdict_json")): df[col.replace("_json", "")] = [json.loads(v) if isinstance(v, str) else None for v in df[col]] for col in ("unsupported_terms", "filled_na_fields", "missed_fields"): notes[col] = notes[col].apply(lambda v: list(v) if v is not None else []) examples["ref_unsupported_terms"] = examples["ref_unsupported_terms"].apply(lambda v: list(v) if v is not None else []) notes["label"] = notes["family"].map(label) order = [f for f in MODELS if f in set(notes["family"])] return examples, notes, runs, {**summary, "family_order": order} def css() -> None: ink = INK[mode()] st.markdown(f"""""", unsafe_allow_html=True) def _highlight(text: str, terms: list[str]) -> str: out = html.escape(text) for t in sorted(set(terms), key=len, reverse=True): out = re.sub(rf"(?\1', out, flags=re.IGNORECASE) return out JUDGE_NAMES = {"databricks-gpt-oss-120b": "gpt-oss-120b", "databricks-qwen3-next-80b-a3b-instruct": "Qwen3-Next-80B"} def judge_name(endpoint: str | None) -> str: return JUDGE_NAMES.get(endpoint or "", (endpoint or "").replace("databricks-", "")) def _ok(v) -> bool: return v is not None and v == v # not None / NaN def note_html(title: str, family: str | None, text: str, fields_json: str | None, terms: list[str], meta: str = "", verdict: dict | None = None, filled_na: list[str] | None = None, second: dict | None = None) -> str: """second: {"name", "faithful", "score", "primary_faithful"} for the second judge's one-line verdict.""" c = color(family) if family else INK[mode()]["reference"] if not (text or "").strip(): body = '
(empty note: the model generated nothing)
' elif isinstance(fields_json, str): # missing values arrive as NaN from JSON, which is truthy fields = json.loads(fields_json) rows = [] for f in FIELDS: v = fields.get(f, "") is_na = v.strip().rstrip(".").lower() in ("n/a", "na", "none", "") flag = (' ↔ clinician wrote N/A' if filled_na and f in filled_na else "") val = f'{html.escape(v)}' if is_na else _highlight(v, terms) rows.append(f'
{f}: {val}{flag}
') body = "".join(rows) else: # Shown as written, line by line, so invented or repeated field labels are visible. lines = "".join(f'
{_highlight(line, terms)}
' for line in text.splitlines() if line.strip()) body = (lines + f'
⚠ not in the 4-field ' 'templatea field label is missing, repeated, out of order, or invented
') claims = "" if verdict: bad = [cl for cl in verdict.get("claims", []) if cl.get("label") != "supported"] omit = [o for o in verdict.get("omissions", []) if o.get("severity") == "major"] items = [] for cl in bad: key, icon, txt = LABEL_STYLE.get(cl["label"], ("warning", "?", cl["label"])) sev = "major" if cl.get("severity") == "major" else "minor" items.append(f'
{icon} {txt} · {sev}' f'{html.escape(cl.get("field", ""))}: {html.escape(cl.get("claim", ""))}
') for o in omit: items.append(f'
∅ missing · major' f'{html.escape(o.get("fact", ""))}
') if not items: items.append(f'
✓ judge found no problems
') claims = '
LLM judge
' + "".join(items) if second and _ok(second.get("faithful")): key = "good" if second["faithful"] else "serious" mark = "✓ faithful" if second["faithful"] else "✕ not faithful" disagree = (_ok(second.get("primary_faithful")) and bool(second["primary_faithful"]) != bool(second["faithful"])) claims += (f'
Second judge ({html.escape(second["name"])}): ' f'{mark}' + (f'score {second["score"]:.0f}/5 ' if _ok(second.get("score")) else "") + (f'⚠ judges disagree' if disagree else "") + "
") return (f'
{html.escape(title)}
' + (f'
{meta}
' if meta else "") + body + claims + "
") def dialogue_html(text: str) -> str: turns = [] for line in text.splitlines(): if not line.strip(): continue who, sep, said = line.partition(":") if sep and len(who) < 25: turns.append(f'
{html.escape(who.strip())}: {html.escape(said.strip())}
') else: turns.append(f'
{html.escape(line)}
') return "".join(turns) def chart_config(chart: alt.Chart) -> alt.Chart: ink = INK[mode()] return (chart.configure(background="transparent", font="system-ui, -apple-system, 'Segoe UI', sans-serif") .configure_view(stroke=None) .configure_axis(gridColor=ink["grid"], domainColor=ink["axis"], tickColor=ink["axis"], labelColor=ink["secondary"], titleColor=ink["secondary"], labelFontSize=12, titleFontSize=12, titleFontWeight="normal") .configure_title(color=ink["primary"], fontSize=14, anchor="start", fontWeight=600) .configure_legend(labelColor=ink["secondary"], titleColor=ink["secondary"])) def rate_chart(df: pd.DataFrame, value: str, title: str, reference: float | None = None, fmt: str = ".0%", domain=(0, 1), seed_dots: pd.DataFrame | None = None, height: int | None = None) -> alt.Chart: """Dot + seed-spread chart: one row per model, mean dot, ±sd bar, faint per-seed dots, gray reference rule.""" order = [label(f) for f in df["family"]] height_px = height or 46 * len(df) + 20 d = df.assign(model=df["family"].map(label), color=df["family"].map(color), shape=["diamond" if f.endswith("_minlen") else "circle" for f in df["family"]], lo=(df[value] - df[f"{value}_sd"]).clip(lower=domain[0]), hi=(df[value] + df[f"{value}_sd"]).clip(upper=domain[1])) y = alt.Y("model:N", sort=order, title=None, axis=alt.Axis(labelLimit=220)) x_scale = alt.Scale(domain=list(domain)) tip = [alt.Tooltip("model:N", title="Model"), alt.Tooltip(f"{value}:Q", title="Mean over seeds", format=fmt), alt.Tooltip(f"{value}_sd:Q", title="± sd", format=fmt)] bars = alt.Chart(d).mark_rule(strokeWidth=2, opacity=.55).encode(y=y, x=alt.X("lo:Q", scale=x_scale, title=None, axis=alt.Axis(format=fmt, tickCount=5)), x2="hi:Q", color=alt.Color("color:N", scale=None)) dots = alt.Chart(d).mark_point(filled=True, size=140, opacity=1, strokeWidth=2).encode( y=y, x=f"{value}:Q", color=alt.Color("color:N", scale=None), tooltip=tip, shape=alt.Shape("shape:N", scale=None)) text = alt.Chart(d).mark_text(align="left", dx=10, dy=-10, fontSize=12, color=INK[mode()]["primary"]).encode( y=y, x=f"{value}:Q", text=alt.Text(f"{value}:Q", format=fmt)) layers = [bars, dots, text] if seed_dots is not None and value in seed_dots: s = seed_dots.assign(model=seed_dots["family"].map(label), color=seed_dots["family"].map(color)) layers.insert(0, alt.Chart(s).mark_point(size=40, opacity=.45, filled=False, strokeWidth=1.5).encode( y=y, x=f"{value}:Q", color=alt.Color("color:N", scale=None), tooltip=[alt.Tooltip("experiment:N", title="Run"), alt.Tooltip(f"{value}:Q", format=fmt)])) if reference is not None: r = pd.DataFrame({"x": [reference]}) layers.append(alt.Chart(r).mark_rule(color=INK[mode()]["reference"], strokeWidth=1.5).encode( x="x:Q", tooltip=[alt.Tooltip("x:Q", title="Clinician reference", format=fmt)])) subtitle = f"gray line: clinician reference, {format(reference, fmt)}" if reference is not None else " " ink = INK[mode()] return alt.layer(*layers).properties( title=alt.TitleParams(title, subtitle=subtitle, subtitleColor=ink["secondary"], subtitleFontSize=11), height=height_px) def families_df(summary: dict) -> pd.DataFrame: df = pd.DataFrame(summary["families"]) order = [f for f in summary["family_order"] if f in set(df["family"])] return df.set_index("family").loc[order].reset_index()