File size: 13,856 Bytes
a1ed1e9 25aa382 a1ed1e9 25aa382 a1ed1e9 8c6e749 a1ed1e9 8c6e749 a1ed1e9 25aa382 a1ed1e9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 | """Shared data, palette, and rendering for the Space. Reads only the exported snapshot in ./data.
Runs under regular Streamlit (local) and stlite (the static Hugging Face Space, in the browser)."""
from __future__ import annotations
import html
import json
import re
from pathlib import Path
import altair as alt
import pandas as pd
import streamlit as st
DATA = Path(__file__).parent / "data"
FIELDS = ["Symptoms", "Diagnosis", "History of Patient", "Plan of Action"]
# Fixed order; a model keeps its color on every page, and rows are drawn in this order. Slots follow the
# reference palette's validated adjacent order (blue, orange, aqua, yellow); every chart also labels each row
# by name, so color is never the only cue. Archived BART-large shares a slot only if a snapshot includes it.
MODELS = {
"biobart": {"label": "BioBART-base", "slot": 0, "params": "166M", "about": "BART pretrained on PubMed text"},
"t5_base": {"label": "T5-base", "slot": 1, "params": "223M", "about": "text-to-text, general pretraining"},
"biobart_large": {"label": "BioBART-large", "slot": 2, "params": "406M", "about": "larger biomedical BART"},
"pegasus_large": {"label": "PEGASUS-large", "slot": 3, "params": "568M",
"about": "gap-sentence pretraining (pretrained-only checkpoint)"},
"bart_large_cnn": {"label": "BART-large-CNN (archived)", "slot": 2, "params": "406M", "about": "news summarizer"},
"bart_large_cnn_minlen": {"label": "BART-large-CNN, min_length=10 (archived)", "slot": 2, "params": "406M",
"about": "same model, minimum decode length"},
}
SERIES = {"light": ["#2a78d6", "#eb6834", "#1baf7a", "#eda100"], "dark": ["#3987e5", "#d95926", "#199e70", "#c98500"]}
INK = {
"light": {"primary": "#0b0b0b", "secondary": "#52514e", "muted": "#898781", "grid": "#e1e0d9",
"axis": "#c3c2b7", "reference": "#52514e", "mark": "#fff3bf", "surface": "#fcfcfb"},
"dark": {"primary": "#f0efec", "secondary": "#c3c2b7", "muted": "#898781", "grid": "#2c2c2a",
"axis": "#383835", "reference": "#c3c2b7", "mark": "#5c4a00", "surface": "#1a1a19"},
}
STATUS = {"good": "#0ca30c", "warning": "#fab219", "serious": "#ec835a", "critical": "#d03b3b"}
LABEL_STYLE = { # judge label -> (status color, icon, text): status colors always ship with icon + text
"supported": ("good", "✓", "supported"),
"unsupported": ("critical", "✕", "not in dialogue"),
"contradicted": ("critical", "✕", "contradicts dialogue"),
"misplaced": ("warning", "↔", "wrong field"),
}
def mode() -> str:
try:
return "dark" if st.context.theme.type == "dark" else "light"
except Exception:
return "light"
def color(family: str) -> str:
return SERIES[mode()][MODELS.get(family, {"slot": 0})["slot"]]
def label(family: str) -> str:
return MODELS.get(family, {"label": family})["label"]
@st.cache_data
def load() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, dict]:
# JSON, not parquet: the static Space runs in the browser (stlite/Pyodide), where pyarrow may be missing.
examples = pd.read_json(DATA / "examples.json", orient="records")
notes = pd.read_json(DATA / "notes.json", orient="records")
runs = pd.read_json(DATA / "runs.json", orient="records")
summary = json.loads((DATA / "summary.json").read_text())
for df, col in ((notes, "verdict_json"), (examples, "ref_verdict_json")):
df[col.replace("_json", "")] = [json.loads(v) if isinstance(v, str) else None for v in df[col]]
for col in ("unsupported_terms", "filled_na_fields", "missed_fields"):
notes[col] = notes[col].apply(lambda v: list(v) if v is not None else [])
examples["ref_unsupported_terms"] = examples["ref_unsupported_terms"].apply(lambda v: list(v) if v is not None else [])
notes["label"] = notes["family"].map(label)
order = [f for f in MODELS if f in set(notes["family"])]
return examples, notes, runs, {**summary, "family_order": order}
def css() -> None:
ink = INK[mode()]
st.markdown(f"""<style>
.note {{ border-left: 3px solid var(--c); padding: .55rem .85rem; margin: .25rem 0 .9rem; border-radius: 4px;
background: color-mix(in srgb, var(--c) 6%, transparent); }}
.note h5 {{ margin: 0 0 .35rem; font-size: .95rem; }}
.note .meta {{ color: {ink['secondary']}; font-size: .8rem; margin-bottom: .4rem; }}
.field {{ margin: .15rem 0; line-height: 1.45; }}
.field b {{ color: {ink['secondary']}; font-weight: 600; }}
.na {{ color: {ink['muted']}; }}
mark.uns {{ background: {ink['mark']}; color: inherit; padding: 0 .15em; border-radius: 3px;
text-decoration: underline wavy {STATUS['critical']}; text-underline-offset: 3px; }}
.claim {{ font-size: .85rem; margin: .1rem 0; }}
.tag {{ display: inline-block; font-size: .72rem; font-weight: 600; padding: .05rem .4rem; border-radius: 999px;
border: 1px solid var(--t); color: {ink['primary']}; margin-right: .35rem; white-space: nowrap; }}
.turn {{ margin: .2rem 0; line-height: 1.45; }}
.who {{ font-weight: 600; color: {ink['secondary']}; }}
.empty {{ color: {ink['muted']}; font-style: italic; }}
</style>""", unsafe_allow_html=True)
def _highlight(text: str, terms: list[str]) -> str:
out = html.escape(text)
for t in sorted(set(terms), key=len, reverse=True):
out = re.sub(rf"(?<![\w-])({re.escape(html.escape(t))})(?![\w-])", r'<mark class="uns">\1</mark>', out,
flags=re.IGNORECASE)
return out
JUDGE_NAMES = {"databricks-gpt-oss-120b": "gpt-oss-120b", "databricks-qwen3-next-80b-a3b-instruct": "Qwen3-Next-80B"}
def judge_name(endpoint: str | None) -> str:
return JUDGE_NAMES.get(endpoint or "", (endpoint or "").replace("databricks-", ""))
def _ok(v) -> bool:
return v is not None and v == v # not None / NaN
def note_html(title: str, family: str | None, text: str, fields_json: str | None, terms: list[str],
meta: str = "", verdict: dict | None = None, filled_na: list[str] | None = None,
second: dict | None = None) -> str:
"""second: {"name", "faithful", "score", "primary_faithful"} for the second judge's one-line verdict."""
c = color(family) if family else INK[mode()]["reference"]
if not (text or "").strip():
body = '<div class="empty">(empty note: the model generated nothing)</div>'
elif isinstance(fields_json, str): # missing values arrive as NaN from JSON, which is truthy
fields = json.loads(fields_json)
rows = []
for f in FIELDS:
v = fields.get(f, "")
is_na = v.strip().rstrip(".").lower() in ("n/a", "na", "none", "")
flag = (' <span class="tag" style="--t:' + STATUS["warning"] + '">↔ clinician wrote N/A</span>'
if filled_na and f in filled_na else "")
val = f'<span class="na">{html.escape(v)}</span>' if is_na else _highlight(v, terms)
rows.append(f'<div class="field"><b>{f}:</b> {val}{flag}</div>')
body = "".join(rows)
else:
# Shown as written, line by line, so invented or repeated field labels are visible.
lines = "".join(f'<div class="field">{_highlight(line, terms)}</div>' for line in text.splitlines() if line.strip())
body = (lines + f'<div class="meta"><span class="tag" style="--t:{STATUS["warning"]}">⚠ not in the 4-field '
'template</span>a field label is missing, repeated, out of order, or invented</div>')
claims = ""
if verdict:
bad = [cl for cl in verdict.get("claims", []) if cl.get("label") != "supported"]
omit = [o for o in verdict.get("omissions", []) if o.get("severity") == "major"]
items = []
for cl in bad:
key, icon, txt = LABEL_STYLE.get(cl["label"], ("warning", "?", cl["label"]))
sev = "major" if cl.get("severity") == "major" else "minor"
items.append(f'<div class="claim"><span class="tag" style="--t:{STATUS[key]}">{icon} {txt} · {sev}</span>'
f'{html.escape(cl.get("field", ""))}: {html.escape(cl.get("claim", ""))}</div>')
for o in omit:
items.append(f'<div class="claim"><span class="tag" style="--t:{STATUS["serious"]}">∅ missing · major</span>'
f'{html.escape(o.get("fact", ""))}</div>')
if not items:
items.append(f'<div class="claim"><span class="tag" style="--t:{STATUS["good"]}">✓ judge found no problems</span></div>')
claims = '<div class="meta" style="margin-top:.5rem">LLM judge</div>' + "".join(items)
if second and _ok(second.get("faithful")):
key = "good" if second["faithful"] else "serious"
mark = "✓ faithful" if second["faithful"] else "✕ not faithful"
disagree = (_ok(second.get("primary_faithful")) and bool(second["primary_faithful"]) != bool(second["faithful"]))
claims += (f'<div class="claim" style="margin-top:.35rem"><span class="meta">Second judge ({html.escape(second["name"])}):</span> '
f'<span class="tag" style="--t:{STATUS[key]}">{mark}</span>'
+ (f'score {second["score"]:.0f}/5 ' if _ok(second.get("score")) else "")
+ (f'<span class="tag" style="--t:{STATUS["warning"]}">⚠ judges disagree</span>' if disagree else "")
+ "</div>")
return (f'<div class="note" style="--c:{c}"><h5>{html.escape(title)}</h5>'
+ (f'<div class="meta">{meta}</div>' if meta else "") + body + claims + "</div>")
def dialogue_html(text: str) -> str:
turns = []
for line in text.splitlines():
if not line.strip():
continue
who, sep, said = line.partition(":")
if sep and len(who) < 25:
turns.append(f'<div class="turn"><span class="who">{html.escape(who.strip())}:</span> {html.escape(said.strip())}</div>')
else:
turns.append(f'<div class="turn">{html.escape(line)}</div>')
return "".join(turns)
def chart_config(chart: alt.Chart) -> alt.Chart:
ink = INK[mode()]
return (chart.configure(background="transparent", font="system-ui, -apple-system, 'Segoe UI', sans-serif")
.configure_view(stroke=None)
.configure_axis(gridColor=ink["grid"], domainColor=ink["axis"], tickColor=ink["axis"],
labelColor=ink["secondary"], titleColor=ink["secondary"], labelFontSize=12,
titleFontSize=12, titleFontWeight="normal")
.configure_title(color=ink["primary"], fontSize=14, anchor="start", fontWeight=600)
.configure_legend(labelColor=ink["secondary"], titleColor=ink["secondary"]))
def rate_chart(df: pd.DataFrame, value: str, title: str, reference: float | None = None, fmt: str = ".0%",
domain=(0, 1), seed_dots: pd.DataFrame | None = None, height: int | None = None) -> alt.Chart:
"""Dot + seed-spread chart: one row per model, mean dot, ±sd bar, faint per-seed dots, gray reference rule."""
order = [label(f) for f in df["family"]]
height_px = height or 46 * len(df) + 20
d = df.assign(model=df["family"].map(label), color=df["family"].map(color),
shape=["diamond" if f.endswith("_minlen") else "circle" for f in df["family"]],
lo=(df[value] - df[f"{value}_sd"]).clip(lower=domain[0]),
hi=(df[value] + df[f"{value}_sd"]).clip(upper=domain[1]))
y = alt.Y("model:N", sort=order, title=None, axis=alt.Axis(labelLimit=220))
x_scale = alt.Scale(domain=list(domain))
tip = [alt.Tooltip("model:N", title="Model"), alt.Tooltip(f"{value}:Q", title="Mean over seeds", format=fmt),
alt.Tooltip(f"{value}_sd:Q", title="± sd", format=fmt)]
bars = alt.Chart(d).mark_rule(strokeWidth=2, opacity=.55).encode(y=y, x=alt.X("lo:Q", scale=x_scale, title=None,
axis=alt.Axis(format=fmt, tickCount=5)),
x2="hi:Q", color=alt.Color("color:N", scale=None))
dots = alt.Chart(d).mark_point(filled=True, size=140, opacity=1, strokeWidth=2).encode(
y=y, x=f"{value}:Q", color=alt.Color("color:N", scale=None), tooltip=tip,
shape=alt.Shape("shape:N", scale=None))
text = alt.Chart(d).mark_text(align="left", dx=10, dy=-10, fontSize=12, color=INK[mode()]["primary"]).encode(
y=y, x=f"{value}:Q", text=alt.Text(f"{value}:Q", format=fmt))
layers = [bars, dots, text]
if seed_dots is not None and value in seed_dots:
s = seed_dots.assign(model=seed_dots["family"].map(label), color=seed_dots["family"].map(color))
layers.insert(0, alt.Chart(s).mark_point(size=40, opacity=.45, filled=False, strokeWidth=1.5).encode(
y=y, x=f"{value}:Q", color=alt.Color("color:N", scale=None),
tooltip=[alt.Tooltip("experiment:N", title="Run"), alt.Tooltip(f"{value}:Q", format=fmt)]))
if reference is not None:
r = pd.DataFrame({"x": [reference]})
layers.append(alt.Chart(r).mark_rule(color=INK[mode()]["reference"], strokeWidth=1.5).encode(
x="x:Q", tooltip=[alt.Tooltip("x:Q", title="Clinician reference", format=fmt)]))
subtitle = f"gray line: clinician reference, {format(reference, fmt)}" if reference is not None else " "
ink = INK[mode()]
return alt.layer(*layers).properties(
title=alt.TitleParams(title, subtitle=subtitle, subtitleColor=ink["secondary"], subtitleFontSize=11),
height=height_px)
def families_df(summary: dict) -> pd.DataFrame:
df = pd.DataFrame(summary["families"])
order = [f for f in summary["family_order"] if f in set(df["family"])]
return df.set_index("family").loc[order].reset_index()
|