Spaces:
Running
Running
File size: 6,971 Bytes
713f736 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | """Vision A metrics: Dice, volume error and decision-threshold crossings, per fold (plan §8.1, §9).
The same functions score the threshold baseline and nnU-Net. Reference volumes come from
native_volumes.csv, measured before resampling, while Dice is on the slim grid. Dice is reported on
the whole infarct and on the part with reliable Tmax, because a model cannot find infarct it has no
perfusion input for (docs/DATA.md). A case whose reference and prediction are both empty has no Dice.
"""
import statistics
import numpy as np
from strokecopilot.vision.perfusion.volumes import mask_ml
# Volume cut-offs the rules use: DAWN 21, 31 and 51 mL; DEFUSE-3 and the extended-window trials 70 mL.
DECISION_CUTOFFS_ML = (21.0, 31.0, 51.0, 70.0)
def dice(pred, truth):
pred, truth = np.asarray(pred, dtype=bool), np.asarray(truth, dtype=bool)
total = np.count_nonzero(pred) + np.count_nonzero(truth)
if total == 0:
return None
return 2.0 * np.count_nonzero(pred & truth) / total
def crosses(pred_ml, ref_ml, cutoff):
"""True when prediction and reference fall on different sides of a cut-off (below vs not below)."""
return (pred_ml < cutoff) != (ref_ml < cutoff)
def case_metrics(pred, infarct, valid, spacing_mm, ref_ml):
"""One case's scores for a predicted mask. ref_ml is the native infarct volume."""
pred_ml = mask_ml(pred, spacing_mm)
row = {
"dice": dice(pred, infarct),
"dice_reliable": dice(pred & valid, infarct & valid),
"pred_ml": pred_ml,
"ref_ml": ref_ml,
"error_ml": pred_ml - ref_ml,
"abs_error_ml": abs(pred_ml - ref_ml),
"infarct_reliable_share": (float(np.count_nonzero(infarct & valid) / np.count_nonzero(infarct)) if np.any(infarct) else None),
}
for cutoff in DECISION_CUTOFFS_ML:
row[f"crosses_{cutoff:g}"] = crosses(pred_ml, ref_ml, cutoff)
return row
def _mean_std(values):
values = [v for v in values if v is not None]
if not values:
return {"mean": None, "std": None, "n": 0}
return {"mean": statistics.fmean(values), "std": statistics.stdev(values) if len(values) > 1 else 0.0, "n": len(values)}
def _percentiles(values, points=(5, 50, 95)):
values = [v for v in values if v is not None]
return {f"p{p}": float(np.percentile(values, p)) for p in points} if values else None
def summarise(rows, metrics=("dice", "dice_reliable", "abs_error_ml", "error_ml")):
"""Per-fold means, then mean ± std over folds, plus pooled percentiles and crossing counts.
rows: dicts from case_metrics with a `fold` key added.
"""
folds = sorted({row["fold"] for row in rows})
per_fold = {}
for fold in folds:
in_fold = [row for row in rows if row["fold"] == fold]
per_fold[str(fold)] = {m: _mean_std([row[m] for row in in_fold]) for m in metrics}
per_fold[str(fold)]["cases"] = len(in_fold)
across = {m: _mean_std([per_fold[str(f)][m]["mean"] for f in folds]) for m in metrics}
crossings = {
f"{cutoff:g}": {"cases": sum(row[f"crosses_{cutoff:g}"] for row in rows), "of": len(rows)}
for cutoff in DECISION_CUTOFFS_ML
}
return {
"cases": len(rows),
"cases_without_dice": sum(row["dice"] is None for row in rows),
"per_fold": per_fold,
"mean_std_over_folds": across,
"pooled_percentiles": {m: _percentiles([row[m] for row in rows]) for m in metrics},
"threshold_crossings": crossings,
}
def load_per_case(path, variant=None):
"""Rows of a per_case.csv (nnU-Net run or threshold baseline), with numbers and booleans parsed. `variant` keeps
only that baseline variant (the baseline file holds one row per case per variant)."""
import csv
def parse(key, value):
if value in ("", "None", "nan"):
return None
if key.startswith("crosses_"):
return value in ("True", "true", "1", "1.0")
try:
return float(value)
except ValueError:
return value
with open(path, newline="") as f:
rows = [{k: parse(k, v) for k, v in row.items()} for row in csv.DictReader(f)]
if variant is not None:
rows = [row for row in rows if row.get("variant") == variant]
for row in rows:
row["fold"] = int(row["fold"])
return rows
def paired_comparison(model_rows, baseline_rows, n_boot=5000, seed=0):
"""Model against baseline on the same cases (B5, plan §8.1). For Dice and absolute volume error: both means, the
mean paired difference (model − baseline) with a bootstrap 95% CI over cases, how many cases each wins, and a
Wilcoxon signed-rank p. For each decision cut-off: cases only the model puts on the wrong side, only the baseline,
both, and an exact McNemar p on the discordant ones. Cases are matched by `case`; Dice skips cases where either is
None. Returns no case IDs."""
from scipy import stats
base = {row["case"]: row for row in baseline_rows}
pairs = [(row, base[row["case"]]) for row in model_rows if row["case"] in base]
rng = np.random.default_rng(seed)
out = {"cases": len(pairs), "metrics": {}, "crossings": {}}
for metric, higher_better in (("dice", True), ("abs_error_ml", False)):
m = np.array([a[metric] for a, b in pairs if a[metric] is not None and b[metric] is not None], dtype=float)
b = np.array([b[metric] for a, b in pairs if a[metric] is not None and b[metric] is not None], dtype=float)
diff = m - b
boot = diff[rng.integers(0, len(diff), size=(n_boot, len(diff)))].mean(axis=1)
nonzero = diff[diff != 0]
out["metrics"][metric] = {
"n": int(len(diff)), "model_mean": float(m.mean()), "baseline_mean": float(b.mean()),
"model_median": float(np.median(m)), "baseline_median": float(np.median(b)),
"mean_difference": float(diff.mean()),
"ci95": [float(np.percentile(boot, 2.5)), float(np.percentile(boot, 97.5))],
"model_better": int(np.sum(diff > 0) if higher_better else np.sum(diff < 0)),
"baseline_better": int(np.sum(diff < 0) if higher_better else np.sum(diff > 0)),
"ties": int(np.sum(diff == 0)),
"wilcoxon_p": float(stats.wilcoxon(nonzero).pvalue) if len(nonzero) > 0 else None,
}
for cutoff in DECISION_CUTOFFS_ML:
key = f"crosses_{cutoff:g}"
only_model = sum(a[key] and not b[key] for a, b in pairs)
only_base = sum(b[key] and not a[key] for a, b in pairs)
discordant = only_model + only_base
out["crossings"][f"{cutoff:g}"] = {
"model": sum(a[key] for a, _ in pairs), "baseline": sum(b[key] for _, b in pairs),
"only_model": only_model, "only_baseline": only_base, "both": sum(a[key] and b[key] for a, b in pairs),
"mcnemar_p": float(stats.binomtest(only_model, discordant, 0.5).pvalue) if discordant else None,
}
return out
|