parsa-hke's picture
Stroke Decision Copilot: page, bank of 195 cases (held-out by alias), EXP-66 results (part 2)
713f736 verified
Raw History Blame Contribute Delete
6.97 kB
"""Vision A metrics: Dice, volume error and decision-threshold crossings, per fold (plan §8.1, §9).
The same functions score the threshold baseline and nnU-Net. Reference volumes come from
native_volumes.csv, measured before resampling, while Dice is on the slim grid. Dice is reported on
the whole infarct and on the part with reliable Tmax, because a model cannot find infarct it has no
perfusion input for (docs/DATA.md). A case whose reference and prediction are both empty has no Dice.
"""
import statistics
import numpy as np
from strokecopilot.vision.perfusion.volumes import mask_ml
# Volume cut-offs the rules use: DAWN 21, 31 and 51 mL; DEFUSE-3 and the extended-window trials 70 mL.
DECISION_CUTOFFS_ML = (21.0, 31.0, 51.0, 70.0)
def dice(pred, truth):
pred, truth = np.asarray(pred, dtype=bool), np.asarray(truth, dtype=bool)
total = np.count_nonzero(pred) + np.count_nonzero(truth)
if total == 0:
return None
return 2.0 * np.count_nonzero(pred & truth) / total
def crosses(pred_ml, ref_ml, cutoff):
"""True when prediction and reference fall on different sides of a cut-off (below vs not below)."""
return (pred_ml < cutoff) != (ref_ml < cutoff)
def case_metrics(pred, infarct, valid, spacing_mm, ref_ml):
"""One case's scores for a predicted mask. ref_ml is the native infarct volume."""
pred_ml = mask_ml(pred, spacing_mm)
row = {
"dice": dice(pred, infarct),
"dice_reliable": dice(pred & valid, infarct & valid),
"pred_ml": pred_ml,
"ref_ml": ref_ml,
"error_ml": pred_ml - ref_ml,
"abs_error_ml": abs(pred_ml - ref_ml),
"infarct_reliable_share": (float(np.count_nonzero(infarct & valid) / np.count_nonzero(infarct)) if np.any(infarct) else None),
}
for cutoff in DECISION_CUTOFFS_ML:
row[f"crosses_{cutoff:g}"] = crosses(pred_ml, ref_ml, cutoff)
return row
def _mean_std(values):
values = [v for v in values if v is not None]
if not values:
return {"mean": None, "std": None, "n": 0}
return {"mean": statistics.fmean(values), "std": statistics.stdev(values) if len(values) > 1 else 0.0, "n": len(values)}
def _percentiles(values, points=(5, 50, 95)):
values = [v for v in values if v is not None]
return {f"p{p}": float(np.percentile(values, p)) for p in points} if values else None
def summarise(rows, metrics=("dice", "dice_reliable", "abs_error_ml", "error_ml")):
"""Per-fold means, then mean ± std over folds, plus pooled percentiles and crossing counts.
rows: dicts from case_metrics with a `fold` key added.
"""
folds = sorted({row["fold"] for row in rows})
per_fold = {}
for fold in folds:
in_fold = [row for row in rows if row["fold"] == fold]
per_fold[str(fold)] = {m: _mean_std([row[m] for row in in_fold]) for m in metrics}
per_fold[str(fold)]["cases"] = len(in_fold)
across = {m: _mean_std([per_fold[str(f)][m]["mean"] for f in folds]) for m in metrics}
crossings = {
f"{cutoff:g}": {"cases": sum(row[f"crosses_{cutoff:g}"] for row in rows), "of": len(rows)}
for cutoff in DECISION_CUTOFFS_ML
}
return {
"cases": len(rows),
"cases_without_dice": sum(row["dice"] is None for row in rows),
"per_fold": per_fold,
"mean_std_over_folds": across,
"pooled_percentiles": {m: _percentiles([row[m] for row in rows]) for m in metrics},
"threshold_crossings": crossings,
}
def load_per_case(path, variant=None):
"""Rows of a per_case.csv (nnU-Net run or threshold baseline), with numbers and booleans parsed. `variant` keeps
only that baseline variant (the baseline file holds one row per case per variant)."""
import csv
def parse(key, value):
if value in ("", "None", "nan"):
return None
if key.startswith("crosses_"):
return value in ("True", "true", "1", "1.0")
try:
return float(value)
except ValueError:
return value
with open(path, newline="") as f:
rows = [{k: parse(k, v) for k, v in row.items()} for row in csv.DictReader(f)]
if variant is not None:
rows = [row for row in rows if row.get("variant") == variant]
for row in rows:
row["fold"] = int(row["fold"])
return rows
def paired_comparison(model_rows, baseline_rows, n_boot=5000, seed=0):
"""Model against baseline on the same cases (B5, plan §8.1). For Dice and absolute volume error: both means, the
mean paired difference (model − baseline) with a bootstrap 95% CI over cases, how many cases each wins, and a
Wilcoxon signed-rank p. For each decision cut-off: cases only the model puts on the wrong side, only the baseline,
both, and an exact McNemar p on the discordant ones. Cases are matched by `case`; Dice skips cases where either is
None. Returns no case IDs."""
from scipy import stats
base = {row["case"]: row for row in baseline_rows}
pairs = [(row, base[row["case"]]) for row in model_rows if row["case"] in base]
rng = np.random.default_rng(seed)
out = {"cases": len(pairs), "metrics": {}, "crossings": {}}
for metric, higher_better in (("dice", True), ("abs_error_ml", False)):
m = np.array([a[metric] for a, b in pairs if a[metric] is not None and b[metric] is not None], dtype=float)
b = np.array([b[metric] for a, b in pairs if a[metric] is not None and b[metric] is not None], dtype=float)
diff = m - b
boot = diff[rng.integers(0, len(diff), size=(n_boot, len(diff)))].mean(axis=1)
nonzero = diff[diff != 0]
out["metrics"][metric] = {
"n": int(len(diff)), "model_mean": float(m.mean()), "baseline_mean": float(b.mean()),
"model_median": float(np.median(m)), "baseline_median": float(np.median(b)),
"mean_difference": float(diff.mean()),
"ci95": [float(np.percentile(boot, 2.5)), float(np.percentile(boot, 97.5))],
"model_better": int(np.sum(diff > 0) if higher_better else np.sum(diff < 0)),
"baseline_better": int(np.sum(diff < 0) if higher_better else np.sum(diff > 0)),
"ties": int(np.sum(diff == 0)),
"wilcoxon_p": float(stats.wilcoxon(nonzero).pvalue) if len(nonzero) > 0 else None,
}
for cutoff in DECISION_CUTOFFS_ML:
key = f"crosses_{cutoff:g}"
only_model = sum(a[key] and not b[key] for a, b in pairs)
only_base = sum(b[key] and not a[key] for a, b in pairs)
discordant = only_model + only_base
out["crossings"][f"{cutoff:g}"] = {
"model": sum(a[key] for a, _ in pairs), "baseline": sum(b[key] for _, b in pairs),
"only_model": only_model, "only_baseline": only_base, "both": sum(a[key] and b[key] for a, b in pairs),
"mcnemar_p": float(stats.binomtest(only_model, discordant, 0.5).pvalue) if discordant else None,
}
return out