Spaces:
Runtime error
Runtime error
File size: 6,442 Bytes
1cf3825 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 | """Brutal per-classifier OOD audit.
Runs every classifier (cnn, transfer, vit) AND v8 segmentation on every
OOD sample. Reports per-classifier accuracy WITHOUT cascade smoothing.
This is the unfiltered view the dashboard's classifier comparison panel
displays — i.e. what the user actually sees when something looks wrong.
"""
from __future__ import annotations
import sys
import time
from pathlib import Path
import numpy as np
from PIL import Image
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from scripts.eval_ood_cascade import (
SEG_ONNX, CLF_ONNX, MIN_TUMOR_AREA,
_sess, _preprocess_seg, seg_tta, classify_all, modality_of, GT,
)
SAMPLES_DIR = ROOT / 'samples' / 'ood'
def main():
seg = _sess(SEG_ONNX)
clfs = {n: _sess(p) for n, p in CLF_ONNX.items()}
samples = sorted(p for p in SAMPLES_DIR.rglob('*')
if p.suffix.lower() in ('.png', '.jpg', '.jpeg')
and p.parent.name in GT)
print(f'[init] {len(samples)} OOD samples across {len(GT)} sources')
print(f'[init] each classifier called individually, no consensus, no cascade.\n')
rows = []
t0 = time.perf_counter()
for p in samples:
img = Image.open(p)
gt = GT.get(p.parent.name, 'unknown')
probs = classify_all(clfs, img)
prob_map = seg_tta(seg, _preprocess_seg(img))
rows.append({
'source': p.parent.name,
'file': p.name,
'gt': gt,
'p_cnn': probs['cnn'],
'p_transfer': probs['transfer'],
'p_vit': probs['vit'],
'v8_pmax': float(prob_map.max()),
'v8_area_020': int((prob_map >= 0.20).sum()),
'v8_area_030': int((prob_map >= 0.30).sum()),
})
print(f'[done] {len(rows)} samples in {time.perf_counter()-t0:.0f}s\n')
# =================== per-classifier brutal scorecard ===================
print('='*78)
print('PER-CLASSIFIER ACCURACY ON OOD (no cascade, no consensus, no overrides)')
print('='*78)
for clf in ('cnn', 'transfer', 'vit'):
pkey = f'p_{clf}'
# Standard 0.5 threshold for binary classification.
TP = sum(1 for r in rows if r['gt']=='tumor' and r[pkey]>=0.5)
FN = sum(1 for r in rows if r['gt']=='tumor' and r[pkey]<0.5)
FP = sum(1 for r in rows if r['gt']=='no_tumor' and r[pkey]>=0.5)
TN = sum(1 for r in rows if r['gt']=='no_tumor' and r[pkey]<0.5)
recall = TP/(TP+FN) if TP+FN else 0
fpr = FP/(FP+TN) if FP+TN else 0
acc = (TP+TN)/len(rows)
print(f'\n {clf.upper():12s} TP={TP:2d} FN={FN:2d} FP={FP:2d} TN={TN:2d} '
f'recall={recall:.0%} FPR={fpr:.0%} accuracy={acc:.0%}')
# show worst FN cases (real tumors confidently called no_tumor)
confidently_wrong = sorted(
[r for r in rows if r['gt']=='tumor' and r[pkey]<0.2],
key=lambda r: r[pkey])[:5]
if confidently_wrong:
print(f' {len(confidently_wrong)} cases this classifier said p<0.20 on a real tumor:')
for r in confidently_wrong:
print(f' p={r[pkey]:.2f} {r["source"][:35]:35s} {r["file"][:40]}')
# =================== per-source recall ===============================
print('\n' + '='*78)
print('PER-SOURCE: how often each classifier sees the tumor (recall on GT=tumor)')
print('='*78)
by_src = {}
for r in rows:
by_src.setdefault(r['source'], []).append(r)
print(f'\n{"source":48s} GT n {"cnn":>5s} {"trans":>5s} {"vit":>5s} {"v8seg":>6s}')
for src in sorted(by_src):
rs = by_src[src]
gt = rs[0]['gt']
n = len(rs)
cells = []
for clf in ('p_cnn', 'p_transfer', 'p_vit'):
hits = sum(1 for r in rs if r[clf] >= 0.5)
metric = hits/n
cells.append(f'{metric:.0%}'.rjust(5))
v8_hits = sum(1 for r in rs if r['v8_area_020'] >= MIN_TUMOR_AREA)
v8_metric = v8_hits/n
kind = 'recall' if gt == 'tumor' else 'FP rate'
print(f' {src:46s} {gt[:6]:6s} {n:3d} {cells[0]} {cells[1]} {cells[2]} '
f'{v8_metric:.0%}'.rjust(6) + f' <- {kind}')
# =================== consensus vote breakdown ========================
print('\n' + '='*78)
print('CLASSIFIER CONSENSUS BREAKDOWN ON GT=tumor (the cases that matter most)')
print('='*78)
tum = [r for r in rows if r['gt'] == 'tumor']
print(f'\n Total OOD tumor samples: {len(tum)}')
n_all_no = sum(1 for r in tum if r['p_cnn']<0.5 and r['p_transfer']<0.5 and r['p_vit']<0.5)
n_all_yes = sum(1 for r in tum if r['p_cnn']>=0.5 and r['p_transfer']>=0.5 and r['p_vit']>=0.5)
n_split = len(tum) - n_all_no - n_all_yes
print(f' ALL 3 say "no_tumor": {n_all_no:3d} / {len(tum)} ({n_all_no/len(tum):.0%}) <- catastrophic miss')
print(f' ALL 3 say "tumor": {n_all_yes:3d} / {len(tum)} ({n_all_yes/len(tum):.0%}) <- clean detect')
print(f' SPLIT (some yes, some no): {n_split:3d} / {len(tum)} ({n_split/len(tum):.0%}) <- needs review')
print('\n Cases where ALL 3 classifiers confidently miss the tumor:')
all_miss = sorted([r for r in tum if r['p_cnn']<0.5 and r['p_transfer']<0.5 and r['p_vit']<0.5],
key=lambda r: max(r['p_cnn'], r['p_transfer'], r['p_vit']))
for r in all_miss[:10]:
print(f' cnn={r["p_cnn"]:.2f} trans={r["p_transfer"]:.2f} vit={r["p_vit"]:.2f} '
f'v8_pmax={r["v8_pmax"]:.2f} v8_area={r["v8_area_020"]:5d} {r["file"][:50]}')
# =================== v8-rescues-classifiers ==========================
print('\n' + '='*78)
print('THE v8 RESCUE TEST: when ALL 3 classifiers miss, does v8 still find the tumor?')
print('='*78)
rescued = sum(1 for r in all_miss
if r['v8_area_020'] >= MIN_TUMOR_AREA and r['v8_pmax'] >= 0.70)
print(f'\n All-classifier-miss cases: {len(all_miss)}')
print(f' Of those, v8 still produces a STRONG positive (area>=50 AND pmax>=0.70): '
f'{rescued} / {len(all_miss)} ({rescued/max(1,len(all_miss)):.0%})')
print(f' These are the cases the v8_strong override rule (shipped today) catches.')
if __name__ == '__main__':
main()
|