Spaces:
Running
Running
File size: 11,533 Bytes
713f736 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | """Assemble vision_output.json for the 129 development cases (C7; plan §6, §7). No model runs here: each field is
read from a finished run, joined by case, and every object is validated before it is written.
Where each field comes from (development folds only; held-out cases are refused):
- core_ml: nnU-Net's validation prediction from the fold model that did not train on the case (DEC-25), the
`pred_ml` column of the five fold runs (EXP-14, EXP-18 to EXP-21).
- penumbra_ml: the Tmax > 6 s volume, core included (DEC-7, DEC-26), EXP-1's `contract_penumbra_ml`
(variant within_tmax6; the penumbra does not depend on the variant).
- mismatch_ratio, mismatch_volume_ml: derived from the rounded volumes (volumes.contract_volumes).
- occlusion_site: the contract rule (volumes.occlusion_site: ICA or basilar within 5 mm, else unknown) applied to
EXP-1's occlusion evidence columns.
- haemorrhage: EXP-16's calibrated scan confidence on 5 mm slabs (`slab_5mm_confidence`, EXP-6's aggregation and
Platt calibration); present = confidence >= 0.175 (DEC-28, infer.CONTRACT_THRESHOLD).
- confidence: the cross-fitted values of EXP-32 (DEC-27, confidence.development_confidence).
- aspects: null; no model produces it (DEC-9).
- warnings: confidence.core_warnings (DEC-27: a core under 1 mL).
Ground-truth contracts (the ground-truth arm of DEC-11; build_ground_truth_all): the same cases with what vision would
report if it were right, so the ground-truth vs predicted arms isolate the effect of vision errors:
- core_ml: the reference final infarct volume (`ref_ml`, from native_volumes.csv through the nnU-Net tables). It is
final infarct, not acute core (plan §8.1), the only label ISLES'24 has.
- haemorrhage: {present: false, confidence: 0.0}. ISLES'24 has no bleed labels; these patients went on to
reperfusion therapy, so they are presumed bleed-free (as in EXP-16).
- penumbra_ml, occlusion_site: the same as the predicted contracts (no penumbra label; the site already comes from
the reference occlusion mask).
- confidence: {core: 1.0, penumbra: 1.0}; warnings: [] (nothing is estimated); aspects: null (DEC-9).
Usage: python -m strokecopilot.contract.build <Drive runs folder> <output folder> [--model-version TEXT] [--ground-truth]
Writes one <case>.json per case into the output folder (case IDs: Drive only) and prints counts only.
"""
import argparse
import json
from collections import Counter
from pathlib import Path
import pandas as pd
from strokecopilot.contract.validate import validate_vision_output
from strokecopilot.data.splits import check_ids
from strokecopilot.vision.haemorrhage.infer import CONTRACT_THRESHOLD
from strokecopilot.vision.perfusion import confidence as conf
from strokecopilot.vision.perfusion import volumes
SCHEMA_VERSION = "0.1"
NNUNET_RUNS = {0: "EXP-14_nnunet_fold0", 1: "EXP-18_nnunet_fold1", 2: "EXP-19_nnunet_fold2", 3: "EXP-20_nnunet_fold3",
4: "EXP-21_nnunet_fold4"}
THRESHOLD_RUN = "EXP-1_thresholds_baseline"
THRESHOLD_VARIANT = "within_tmax6"
HAEMORRHAGE_RUN = "EXP-16_visionB_isles_check"
HAEMORRHAGE_COLUMN = "slab_5mm_confidence"
CONFIDENCE_RUN = "EXP-32_visionA_confidence_dec27"
EXPECTED_CASES = 129
def build_case(case_id, model_version, core_ml, penumbra_ml, evidence, haemorrhage_confidence, confidence):
"""One validated vision_output dict. evidence: volumes.occlusion_evidence keys (reason, site, site_share,
distance_mm); confidence: {"core", "penumbra"}. Raises ContractError if the result does not validate."""
if haemorrhage_confidence is None or not 0.0 <= float(haemorrhage_confidence) <= 1.0:
raise ValueError("The haemorrhage confidence must be a probability.")
site, _ = volumes.occlusion_site(evidence)
obj = {
"schema_version": SCHEMA_VERSION,
"case_id": case_id,
"model_version": model_version,
"haemorrhage": {"present": bool(float(haemorrhage_confidence) >= CONTRACT_THRESHOLD),
"confidence": round(float(haemorrhage_confidence), 4)},
**volumes.contract_volumes(core_ml, penumbra_ml),
"occlusion_site": site,
"aspects": None,
"confidence": {"core": round(float(confidence["core"]), 4), "penumbra": round(float(confidence["penumbra"]), 4)},
"warnings": conf.core_warnings(core_ml),
}
validate_vision_output(obj)
return obj
def _evidence(row):
def num(key):
value = row.get(key)
return None if value is None or pd.isna(value) else float(value)
reason = row.get("occlusion_reason")
return {"reason": reason if isinstance(reason, str) else "missing", "site": row.get("occlusion_site"),
"site_share": num("occlusion_site_share"), "distance_mm": num("occlusion_distance_mm")}
def load_sources(runs_dir, folds=None, expected=EXPECTED_CASES):
"""{case: inputs for build_case} from the finished runs. Refuses repeated cases, a case missing from any source,
a fold that differs between sources, and held-out cases (splits.check_ids). Messages carry counts only."""
runs = Path(runs_dir)
nnunet = []
for fold, name in NNUNET_RUNS.items():
table = pd.read_csv(runs / name / "per_case.csv")
if set(table["fold"]) != {fold}:
raise ValueError(f"{name} holds folds {sorted(set(table['fold']))}, expected {fold}.")
nnunet.append(table[["case", "fold", "pred_ml"]])
nnunet = pd.concat(nnunet, ignore_index=True)
threshold = pd.read_csv(runs / THRESHOLD_RUN / "per_case.csv")
threshold = threshold[threshold["variant"] == THRESHOLD_VARIANT]
haemorrhage = pd.read_csv(runs / HAEMORRHAGE_RUN / "per_case.csv")
if haemorrhage["error"].notna().any():
raise ValueError(f"{int(haemorrhage['error'].notna().sum())} Vision B scans failed in {HAEMORRHAGE_RUN}.")
tables = {"nnU-Net": nnunet, THRESHOLD_RUN: threshold, HAEMORRHAGE_RUN: haemorrhage}
for name, table in tables.items():
if table["case"].duplicated().any():
raise ValueError(f"{name} repeats {int(table['case'].duplicated().sum())} cases.")
cases = set(nnunet["case"])
for name, table in tables.items():
if set(table["case"]) != cases:
raise ValueError(f"{name} has {len(set(table['case']) ^ cases)} cases that the nnU-Net runs do not, or the reverse.")
if expected is not None and len(cases) != expected:
raise ValueError(f"{len(cases)} cases, expected {expected}.")
check_ids(sorted(cases), folds)
fold_of = dict(zip(nnunet["case"], nnunet["fold"]))
if any(fold_of[c] != f for c, f in zip(threshold["case"], threshold["fold"])):
raise ValueError("The nnU-Net and threshold runs disagree on a case's fold.")
confidence = conf.development_confidence(runs / CONFIDENCE_RUN / "per_case.csv", folds=folds)
if set(confidence) != cases:
raise ValueError(f"{CONFIDENCE_RUN} has {len(set(confidence) ^ cases)} cases that the nnU-Net runs do not, or the reverse.")
core = dict(zip(nnunet["case"], nnunet["pred_ml"]))
thr = threshold.set_index("case")
haem = dict(zip(haemorrhage["case"], haemorrhage[HAEMORRHAGE_COLUMN]))
return {case: {"core_ml": float(core[case]), "penumbra_ml": float(thr.at[case, "contract_penumbra_ml"]),
"evidence": _evidence(thr.loc[case].to_dict()), "haemorrhage_confidence": float(haem[case]),
"confidence": confidence[case], "fold": int(fold_of[case])}
for case in sorted(cases)}
def summarise(outputs):
"""Counts only, no case IDs."""
values = list(outputs.values())
def count(key):
return sum(v[key] is None for v in values)
return {
"cases": len(values),
"haemorrhage_present": sum(v["haemorrhage"]["present"] for v in values),
"occlusion_site": dict(sorted(Counter(v["occlusion_site"] for v in values).items())),
"null": {k: count(k) for k in ("core_ml", "penumbra_ml", "mismatch_ratio", "mismatch_volume_ml", "aspects")},
"negative_mismatch_volume": sum((v["mismatch_volume_ml"] or 0) < 0 for v in values),
"warnings": dict(Counter(w for v in values for w in v["warnings"])),
"core_ml_median": float(pd.Series([v["core_ml"] for v in values]).median()),
"penumbra_ml_median": float(pd.Series([v["penumbra_ml"] for v in values]).median()),
"confidence_core_median": float(pd.Series([v["confidence"]["core"] for v in values]).median()),
}
def build_ground_truth_case(case_id, model_version, ref_ml, penumbra_ml, evidence):
"""One validated ground-truth vision_output dict (see the module docstring)."""
site, _ = volumes.occlusion_site(evidence)
obj = {
"schema_version": SCHEMA_VERSION,
"case_id": case_id,
"model_version": model_version,
"haemorrhage": {"present": False, "confidence": 0.0},
**volumes.contract_volumes(ref_ml, penumbra_ml),
"occlusion_site": site,
"aspects": None,
"confidence": {"core": 1.0, "penumbra": 1.0},
"warnings": [],
}
validate_vision_output(obj)
return obj
def load_reference_volumes(runs_dir):
"""{case: ref_ml} from the nnU-Net fold tables (native reference volumes); refuses a case listed twice."""
refs = {}
for name in NNUNET_RUNS.values():
table = pd.read_csv(Path(runs_dir) / name / "per_case.csv")
for case, ref in zip(table["case"], table["ref_ml"]):
if case in refs:
raise ValueError("A case appears in two nnU-Net fold tables.")
refs[case] = float(ref)
return refs
def build_ground_truth_all(runs_dir, out_dir, model_version, folds=None, expected=EXPECTED_CASES):
"""Ground-truth contracts for every development case, validated and written like build_all; returns summarise()."""
sources = load_sources(runs_dir, folds, expected)
refs = load_reference_volumes(runs_dir)
outputs = {case: build_ground_truth_case(case, model_version, refs[case], s["penumbra_ml"], s["evidence"])
for case, s in sources.items()}
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
for case, obj in outputs.items():
(out / f"{case}.json").write_text(json.dumps(obj, indent=1) + "\n")
return summarise(outputs)
def build_all(runs_dir, out_dir, model_version, folds=None, expected=EXPECTED_CASES):
"""Build, validate and write every development case; returns summarise(). Nothing is written unless all build."""
sources = load_sources(runs_dir, folds, expected)
outputs = {case: build_case(case, model_version, s["core_ml"], s["penumbra_ml"], s["evidence"],
s["haemorrhage_confidence"], s["confidence"]) for case, s in sources.items()}
out = Path(out_dir)
out.mkdir(parents=True, exist_ok=True)
for case, obj in outputs.items():
(out / f"{case}.json").write_text(json.dumps(obj, indent=1) + "\n")
return summarise(outputs)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("runs_dir")
parser.add_argument("out_dir")
parser.add_argument("--model-version", required=True)
parser.add_argument("--ground-truth", action="store_true", help="build the ground-truth contracts instead")
args = parser.parse_args(argv)
builder = build_ground_truth_all if args.ground_truth else build_all
print(json.dumps(builder(args.runs_dir, args.out_dir, args.model_version), indent=1))
if __name__ == "__main__":
main()
|