Spaces:
Runtime error
Runtime error
File size: 6,543 Bytes
1cf3825 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | """OOD audit on v8-balanced classifiers (OpenNeuro-augmented + pos_weight).
Three-way comparison:
- OLD: real_eval_current/ (Kaggle 4-class only)
- v8-RAW: real_eval_v8_retrained/ (dataset_v8 as-is, 78% positive)
- v8-BAL: real_eval_v8_balanced/ (+ OpenNeuro healthy, pos_weight=0.49)
Hypothesis: v8-balanced fixes the 100% OOD-healthy FPR of v8-raw while
keeping the recall recovery from broader training distribution.
"""
from __future__ import annotations
import sys
import time
from pathlib import Path
import numpy as np
from PIL import Image
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from scripts.eval_ood_cascade import SEG_ONNX, MIN_TUMOR_AREA, _sess, _preprocess_seg, _preprocess_clf, seg_tta, GT
CLF_SETS = {
'OLD': {'cnn': ROOT/'real_eval_current'/'cnn'/'best_weights.onnx',
'transfer': ROOT/'real_eval_current'/'transfer'/'best_weights.onnx',
'vit': ROOT/'real_eval_current'/'vit'/'best_weights.onnx'},
'v8-RAW': {'cnn': ROOT/'real_eval_v8_retrained'/'cnn'/'best_weights.onnx',
'transfer': ROOT/'real_eval_v8_retrained'/'transfer'/'best_weights.onnx',
'vit': ROOT/'real_eval_v8_retrained'/'vit'/'best_weights.onnx'},
'v8-BAL': {'cnn': ROOT/'real_eval_v8_balanced'/'cnn'/'best_weights.onnx',
'transfer': ROOT/'real_eval_v8_balanced'/'transfer'/'best_weights.onnx',
'vit': ROOT/'real_eval_v8_balanced'/'vit'/'best_weights.onnx'},
}
NORMALIZE_IMAGENET = {'cnn': False, 'transfer': True, 'vit': True}
SAMPLES_DIR = ROOT / 'samples' / 'ood'
def classify(sess, img, normalise):
chw = _preprocess_clf(img, normalise)
logit = float(sess.run(None, {sess.get_inputs()[0].name: chw[None]})[0].reshape(-1)[0])
return 1.0 / (1.0 + np.exp(-logit))
def stats_at_thr(rows, pkey, thr=0.5):
TP = sum(1 for r in rows if r['gt']=='tumor' and r[pkey]>=thr)
FN = sum(1 for r in rows if r['gt']=='tumor' and r[pkey]<thr)
FP = sum(1 for r in rows if r['gt']=='no_tumor' and r[pkey]>=thr)
TN = sum(1 for r in rows if r['gt']=='no_tumor' and r[pkey]<thr)
recall = TP/(TP+FN) if TP+FN else 0
fpr = FP/(FP+TN) if FP+TN else 0
acc = (TP+TN)/(TP+FN+FP+TN) if (TP+FN+FP+TN) else 0
return TP, FN, FP, TN, recall, fpr, acc
def main():
# Load all 3 model sets
sets = {}
for tag, paths in CLF_SETS.items():
if not all(p.exists() for p in paths.values()):
print(f'[skip] {tag}: missing weights')
continue
sets[tag] = {n: _sess(p) for n, p in paths.items()}
print(f'[init] loaded {len(sets)} classifier sets: {list(sets)}')
samples = sorted(p for p in SAMPLES_DIR.rglob('*')
if p.suffix.lower() in ('.png', '.jpg', '.jpeg')
and p.parent.name in GT)
print(f'[init] {len(samples)} OOD samples\n')
# Compute all probs once
rows = []
t0 = time.perf_counter()
for p in samples:
img = Image.open(p)
gt = GT[p.parent.name]
rec = {'source': p.parent.name, 'file': p.name, 'gt': gt}
for tag, clfs in sets.items():
for n, sess in clfs.items():
rec[f'{tag}__{n}'] = classify(sess, img, NORMALIZE_IMAGENET[n])
rows.append(rec)
print(f'[done] {len(rows)} samples in {time.perf_counter()-t0:.0f}s\n')
# ==================== summary table ===========================
print('='*78)
print('PER-CLASSIFIER SCORECARD (OLD vs v8-RAW vs v8-BAL)')
print('='*78)
for clf in ('cnn', 'transfer', 'vit'):
print(f'\n {clf.upper()}:')
print(f' {"set":10s} recall FPR acc F1')
for tag in ('OLD', 'v8-RAW', 'v8-BAL'):
if tag not in sets: continue
pkey = f'{tag}__{clf}'
TP, FN, FP, TN, re, fpr, acc = stats_at_thr(rows, pkey)
f1 = 2*TP/(2*TP+FP+FN) if 2*TP+FP+FN else 0
print(f' {tag:10s} {re:.0%} {fpr:.0%} {acc:.0%} {f1:.2f}')
# ==================== per-source ===============================
print('\n' + '='*78)
print('PER-SOURCE on v8-BAL only (the candidate)')
print('='*78)
by_src = {}
for r in rows:
by_src.setdefault(r['source'], []).append(r)
print(f'\n{"source":48s} GT n {"cnn":>5s} {"trans":>5s} {"vit":>5s}')
for src in sorted(by_src):
rs = by_src[src]; gt = rs[0]['gt']; n = len(rs)
kind = 'recall' if gt == 'tumor' else 'FPR'
cells = []
for clf in ('cnn', 'transfer', 'vit'):
pkey = f'v8-BAL__{clf}'
hits = sum(1 for r in rs if r[pkey] >= 0.5)
cells.append(f'{hits/n:.0%}'.rjust(5))
print(f' {src:46s} {gt[:6]:6s} {n:3d} {cells[0]} {cells[1]} {cells[2]} <- {kind}')
# ==================== consensus on tumor =======================
print('\n' + '='*78)
print('CLASSIFIER CONSENSUS on 36 OOD TUMOR SAMPLES')
print('='*78)
tum = [r for r in rows if r['gt'] == 'tumor']
for tag in ('OLD', 'v8-RAW', 'v8-BAL'):
if tag not in sets: continue
n_all_yes = sum(1 for r in tum if all(r[f'{tag}__{c}']>=0.5 for c in ('cnn','transfer','vit')))
n_all_no = sum(1 for r in tum if all(r[f'{tag}__{c}']<0.5 for c in ('cnn','transfer','vit')))
n_split = len(tum) - n_all_yes - n_all_no
print(f' {tag:10s} all_tumor={n_all_yes:3d} ({n_all_yes/len(tum):.0%}) '
f'split={n_split:3d} ({n_split/len(tum):.0%}) '
f'all_no={n_all_no:3d} ({n_all_no/len(tum):.0%})')
# ==================== consensus on healthy =====================
neg = [r for r in rows if r['gt'] == 'no_tumor']
print(f'\nCLASSIFIER CONSENSUS on {len(neg)} OOD HEALTHY SAMPLES')
print('-'*78)
for tag in ('OLD', 'v8-RAW', 'v8-BAL'):
if tag not in sets: continue
n_all_yes = sum(1 for r in neg if all(r[f'{tag}__{c}']>=0.5 for c in ('cnn','transfer','vit')))
n_all_no = sum(1 for r in neg if all(r[f'{tag}__{c}']<0.5 for c in ('cnn','transfer','vit')))
n_split = len(neg) - n_all_yes - n_all_no
print(f' {tag:10s} all_no={n_all_no:3d} ({n_all_no/len(neg):.0%}, correct) '
f'split={n_split:3d} ({n_split/len(neg):.0%}) '
f'all_yes(FP)={n_all_yes:3d} ({n_all_yes/len(neg):.0%})')
if __name__ == '__main__':
main()
|