{ "cells": [ { "cell_type": "markdown", "id": "eafcaa72", "metadata": { "id": "eafcaa72" }, "source": [ "### Intracranial Hemorrhage detection and classification with Yolo11n\n", "\n", "Note:\n", "- This code is meant to be run on Google Colab up till step 15.\n", "- Refer to README.txt in downloaded ZIP file (ich_local_demo) for full instructions to run app locally" ] }, { "cell_type": "markdown", "id": "62740665", "metadata": { "id": "62740665" }, "source": [ "1. Package Installation" ] }, { "cell_type": "code", "execution_count": null, "id": "885e6411", "metadata": { "id": "885e6411" }, "outputs": [], "source": [ "import subprocess, sys\n", "subprocess.run([sys.executable, '-m', 'pip', 'install', '-q',\n", " 'ultralytics>=8.4.0,<9', 'pydicom>=3.0,<4',\n", " 'pylibjpeg>=2,<3', 'pylibjpeg-libjpeg>=2,<3', 'pylibjpeg-openjpeg>=2,<3',\n", " 'gradio>=5.0,<6', 'kaggle>=1.7,<2', 'pandas>=2,<3',\n", " 'scikit-learn>=1.5,<2', 'matplotlib>=3.8,<4', 'Pillow>=10,<12',\n", " 'PyYAML>=6,<7', 'requests>=2.32,<3', 'tqdm>=4.66,<5'], check=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "eded5d0a", "metadata": { "id": "eded5d0a" }, "outputs": [], "source": [ "from pathlib import Path\n", "import ast, hashlib, io, json, os, re, shutil, subprocess, sys, zipfile\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import pydicom, requests, torch, yaml\n", "from PIL import Image\n", "from tqdm.auto import tqdm\n", "from sklearn.model_selection import GroupShuffleSplit\n", "from ultralytics import YOLO\n", "from IPython.display import display\n", "\n", "SEED = 42 #For Reproducibility\n", "ROOT = Path('/content/ich_yolo11')\n", "RAW = ROOT / 'raw'\n", "DICOMS = RAW / 'selected_dicoms'\n", "ARCHIVES = RAW / 'cq500_download'\n", "DATA = ROOT / 'dataset'\n", "ARTIFACTS = ROOT / 'artifacts'\n", "for folder in [ROOT, RAW, DICOMS, ARCHIVES, ARTIFACTS]:\n", " folder.mkdir(parents=True, exist_ok=True)\n", "NAMES = ['intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'epidural']\n", "CLASS_MAP = {'Intraparenchymal': 0, 'Intraventricular': 1, 'Subarachnoid': 2,\n", " 'Subdural': 3, 'Chronic': 3, 'Epidural': 4}\n", "IMGSZ, BATCH, EPOCHS = 512, 8, 100\n", "DEVICE = 0 if torch.cuda.is_available() else 'cpu'\n", "print('Device:', torch.cuda.get_device_name(0) if DEVICE == 0 else 'CPU, training will be slow')\n", "print('Free disk GB:', round(shutil.disk_usage(ROOT).free / 1e9, 1))\n", "(ARTIFACTS / 'environment.txt').write_text(\n", " subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'], text=True))\n", "\n", "RUNS = ROOT / 'runs'\n", "RUNS.mkdir(parents=True, exist_ok=True)" ] }, { "cell_type": "markdown", "id": "919c8375", "metadata": { "id": "919c8375" }, "source": [ "2. Download detection dataset from CQ500 and BHX" ] }, { "cell_type": "code", "execution_count": null, "id": "5938380b", "metadata": { "id": "5938380b" }, "outputs": [], "source": [ "DATA_MODE = 'torrent' # 'torrent' or 'existing'\n", "EXISTING_CQ500 = Path('/content/drive/MyDrive/CQ500')\n", "NEGATIVE_CSV = RAW / 'reviewed_negative_uids.csv'\n", "\n", "def download(url, dest):\n", " dest = Path(dest)\n", " if dest.exists() and dest.stat().st_size > 0:\n", " return dest\n", " temp = dest.with_suffix(dest.suffix + '.part')\n", " with requests.get(url, stream=True, timeout=(30, 120)) as response:\n", " response.raise_for_status()\n", " with temp.open('wb') as output:\n", " for block in response.iter_content(1024 * 1024):\n", " if block:\n", " output.write(block)\n", " temp.replace(dest)\n", " return dest\n", "\n", "BHX_CSV = download('https://physionet.org/files/bhx-brain-bounding-box/1.1/1_Initial_Manual_Labeling.csv',\n", " RAW / '1_Initial_Manual_Labeling.csv')\n", "annotations = pd.read_csv(BHX_CSV, dtype=str)\n", "required = {'SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID', 'data', 'labelName'}\n", "assert required <= set(annotations.columns), annotations.columns.tolist()\n", "assert annotations[list(required)].notna().all().all(), 'Missing annotation fields'\n", "unknown = set(annotations.labelName) - set(CLASS_MAP)\n", "assert not unknown, f'Unrecognized source classes: {unknown}'\n", "annotations['class_id'] = annotations.labelName.map(CLASS_MAP).astype(int)\n", "annotations = annotations.drop_duplicates(subset=['SOPInstanceUID', 'labelName', 'data']).copy()\n", "annotations.to_csv(ARTIFACTS / 'source_annotations.csv', index=False)\n", "\n", "negative_uids = set()\n", "if NEGATIVE_CSV.exists():\n", " negatives = pd.read_csv(NEGATIVE_CSV, dtype=str).fillna('')\n", " assert {'SOPInstanceUID', 'reviewed_negative', 'reviewer', 'provenance'} <= set(negatives.columns)\n", " assert negatives.reviewed_negative.eq('1').all()\n", " assert negatives.reviewer.str.strip().ne('').all() and negatives.provenance.str.strip().ne('').all()\n", " negative_uids = set(negatives.SOPInstanceUID)\n", " assert negative_uids.isdisjoint(set(annotations.SOPInstanceUID)), 'Conflicting positive/negative labels'\n", " negatives.to_csv(ARTIFACTS / 'negative_provenance.csv', index=False)\n", "TARGET_UIDS = set(annotations.SOPInstanceUID) | negative_uids\n", "\n", "if DATA_MODE == 'torrent':\n", " if shutil.disk_usage(ROOT).free < 38e9:\n", " raise RuntimeError('Allow at least 38 GB free space for compressed CQ500 and selected files.')\n", " if not shutil.which('aria2c'):\n", " subprocess.run(['apt-get', '-qq', 'update'], check=True)\n", " subprocess.run(['apt-get', '-qq', 'install', '-y', 'aria2'], check=True)\n", " torrent = download('https://academictorrents.com/download/47e9d8aab761e75fd0a81982fa62bddf3a173831.torrent',\n", " RAW / 'cq500.torrent')\n", " # Resume incomplete downloads on rerun. Disable seeding after completion.\n", " subprocess.run(['aria2c', '--seed-time=0', '--continue=true', '--file-allocation=none',\n", " '--summary-interval=60', '--dir=' + str(ARCHIVES), str(torrent)], check=True)\n", " SOURCE_ROOT = ARCHIVES\n", "elif DATA_MODE == 'existing':\n", " SOURCE_ROOT = EXISTING_CQ500\n", " assert SOURCE_ROOT.is_dir(), f'Place CQ500 files in {SOURCE_ROOT}'\n", "else:\n", " raise ValueError('DATA_MODE must be torrent or existing')\n", "print('Source folder:', SOURCE_ROOT)\n", "print('Annotated slices:', annotations.SOPInstanceUID.nunique(), 'Reviewed negative slices:', len(negative_uids))" ] }, { "cell_type": "markdown", "id": "274bd5a5", "metadata": { "id": "274bd5a5" }, "source": [ "3. Inspecting dataset before training" ] }, { "cell_type": "code", "execution_count": null, "id": "be9f2813", "metadata": { "id": "be9f2813" }, "outputs": [], "source": [ "print('Original annotation counts:')\n", "display(annotations.groupby('labelName').agg(boxes=('SOPInstanceUID','size'),\n", " slices=('SOPInstanceUID','nunique'), studies=('StudyInstanceUID','nunique')))\n", "print('Columns:', annotations.columns.tolist())\n", "print('Source types:', annotations.labelType.value_counts().to_dict() if 'labelType' in annotations else {})\n", "print('BHX CSV SHA256:', hashlib.sha256(BHX_CSV.read_bytes()).hexdigest())\n", "\n", "INDEX_CSV = RAW / 'dicom_index.csv'\n", "records = {}\n", "if INDEX_CSV.exists():\n", " old = pd.read_csv(INDEX_CSV, dtype=str).fillna('')\n", " records = {r['uid']: r for r in old.to_dict('records')\n", " if Path(r['path']).exists() and r['uid'] in TARGET_UIDS}\n", "\n", "def inspect_dicom(stream, source_name, raw_bytes=None, source_path=None):\n", " try:\n", " ds = pydicom.dcmread(stream, stop_before_pixels=True)\n", " except pydicom.errors.InvalidDicomError:\n", " return\n", " uid = str(getattr(ds, 'SOPInstanceUID', ''))\n", " if uid not in TARGET_UIDS or uid in records:\n", " return\n", " assert str(getattr(ds, 'Modality', '')) == 'CT', f'Non-CT target: {uid}'\n", " patient = str(getattr(ds, 'PatientID', '')).strip()\n", " match = re.search(r'CQ500[-_ ]*CT[-_ ]*(\\d+)', source_name, flags=re.I)\n", " case = 'CQ500-' + match.group(1) if match else ''\n", " assert patient or case, f'Missing patient/case identifier: {source_name}'\n", " dest = DICOMS / (uid + '.dcm')\n", " if raw_bytes is not None:\n", " dest.write_bytes(raw_bytes)\n", " elif Path(source_path).resolve() != dest.resolve():\n", " shutil.copy2(source_path, dest)\n", " records[uid] = dict(uid=uid, path=str(dest), patient=patient, case=case,\n", " study=str(ds.StudyInstanceUID), series=str(ds.SeriesInstanceUID),\n", " rows=int(ds.Rows), cols=int(ds.Columns), source=source_name)\n", "\n", "if set(records) != TARGET_UIDS:\n", " archives = sorted(SOURCE_ROOT.rglob('*.zip'))\n", " for archive in tqdm(archives, desc='Indexing CQ500 archives'):\n", " try:\n", " with zipfile.ZipFile(archive) as z:\n", " for member in z.infolist():\n", " if member.is_dir() or Path(member.filename).suffix.lower() in {'.csv','.txt','.json','.png','.jpg','.pdf'}:\n", " continue\n", " if member.file_size > 100 * 1024 * 1024:\n", " raise RuntimeError(f'Unexpected large/nested archive member: {member.filename}. Extract separately.')\n", " # Header reads avoid decompressing all pixels for non-target slices.\n", " with z.open(member) as stream:\n", " try:\n", " head = pydicom.dcmread(stream, stop_before_pixels=True)\n", " except pydicom.errors.InvalidDicomError:\n", " continue\n", " uid = str(getattr(head, 'SOPInstanceUID', ''))\n", " if uid in TARGET_UIDS and uid not in records:\n", " payload = z.read(member)\n", " inspect_dicom(io.BytesIO(payload), str(archive) + '/' + member.filename, raw_bytes=payload)\n", " except zipfile.BadZipFile:\n", " print(f'Warning: Skipping corrupted or invalid zip file: {archive}')\n", " continue\n", " if records:\n", " pd.DataFrame(records.values()).to_csv(INDEX_CSV, index=False)\n", " for path in tqdm(list(SOURCE_ROOT.rglob('*')), desc='Indexing extracted files'):\n", " if path.is_file() and path.suffix.lower() in {'.dcm', '.dicom', ''}:\n", " inspect_dicom(path, str(path), source_path=path)\n", " if records:\n", " pd.DataFrame(records.values()).to_csv(INDEX_CSV, index=False)\n", "\n", "missing = TARGET_UIDS - set(records)\n", "(ARTIFACTS / 'missing_uids.txt').write_text('\\n'.join(sorted(missing)))\n", "assert not missing, f'{len(missing)} target UIDs missing. Complete CQ500 download or check the chosen source. See missing_uids.txt.'\n", "index = pd.DataFrame(records.values())\n", "index[['rows','cols']] = index[['rows','cols']].astype(int)\n", "joined = annotations.merge(index, left_on='SOPInstanceUID', right_on='uid', validate='many_to_one')\n", "assert joined.StudyInstanceUID.eq(joined.study).all(), 'Study UID mismatch'\n", "assert joined.SeriesInstanceUID.eq(joined.series).all(), 'Series UID mismatch'\n", "display(index[['uid','rows','cols']].head())\n", "print('Matched images:', len(index), 'Studies:', index.study.nunique())" ] }, { "cell_type": "markdown", "id": "51dbb164", "metadata": { "id": "51dbb164" }, "source": [ "3.1 Dataset preprocessing (fixed brain window, centre image, image grayscale)" ] }, { "cell_type": "code", "execution_count": null, "id": "1f753082", "metadata": { "id": "1f753082" }, "outputs": [], "source": [ "from pathlib import Path\n", "import numpy as np\n", "import pydicom\n", "from PIL import Image\n", "\n", "def dicom_to_rgb(path):\n", " ds = pydicom.dcmread(path)\n", " if str(getattr(ds, 'Modality', '')) != 'CT':\n", " raise ValueError('Upload a CT DICOM slice.')\n", " if int(getattr(ds, 'NumberOfFrames', 1)) != 1:\n", " raise ValueError('Upload a single-frame CT slice.')\n", " if not hasattr(ds, 'RescaleSlope') or not hasattr(ds, 'RescaleIntercept'):\n", " raise ValueError('Missing CT rescale metadata. Convert with verified HU calibration first.')\n", " pixels = ds.pixel_array\n", " if pixels.ndim != 2:\n", " raise ValueError('Expected one grayscale CT slice.')\n", " if str(getattr(ds, 'PhotometricInterpretation', '')) not in {'MONOCHROME1', 'MONOCHROME2'}:\n", " raise ValueError('Expected monochrome CT data.')\n", " hu = pixels.astype(np.float32) * float(ds.RescaleSlope) + float(ds.RescaleIntercept)\n", " # Explicit HU window, independent of the source display LUT or photometric polarity.\n", " gray = np.rint(np.clip((hu - 0.0) / 80.0, 0, 1) * 255).astype(np.uint8)\n", " if hasattr(ds, 'PixelPaddingValue'):\n", " a = float(ds.PixelPaddingValue)\n", " b = float(getattr(ds, 'PixelPaddingRangeLimit', a))\n", " gray[(pixels >= min(a, b)) & (pixels <= max(a, b))] = 0\n", " return np.repeat(gray[..., None], 3, axis=2)\n", "\n", "def load_input(path):\n", " path = Path(path)\n", " if path.suffix.lower() in {'.png', '.jpg', '.jpeg'}:\n", " image = Image.open(path)\n", " if image.mode not in {'RGB','RGBA','L'}:\n", " raise ValueError('Use an 8-bit brain-windowed PNG/JPEG or original DICOM.')\n", " return np.asarray(image.convert('RGB'))\n", " return dicom_to_rgb(path)" ] }, { "cell_type": "markdown", "id": "fb03f53f", "metadata": { "id": "fb03f53f" }, "source": [ "4. Inspect a real label" ] }, { "cell_type": "code", "execution_count": null, "id": "a3f23dfc", "metadata": { "id": "a3f23dfc" }, "outputs": [], "source": [ "def source_box(value):\n", " d = ast.literal_eval(value) if isinstance(value, str) else value\n", " assert isinstance(d, dict) and {'x','y','width','height'} <= set(d), d\n", " box = np.array([d['x'], d['y'], d['width'], d['height']], dtype=float)\n", " assert np.isfinite(box).all() and (box[2:] > 0).all(), d\n", " return box\n", "\n", "def to_yolo(box, width, height):\n", " x, y, w, h = box\n", " # Permit minor edge overrun, then clip to the actual pixel canvas.\n", " assert x >= -2 and y >= -2 and x+w <= width+2 and y+h <= height+2, box\n", " x1, y1 = max(0., x), max(0., y)\n", " x2, y2 = min(float(width), x+w), min(float(height), y+h)\n", " assert x2 > x1 and y2 > y1, box\n", " return np.array([(x1+x2)/(2*width), (y1+y2)/(2*height),\n", " (x2-x1)/width, (y2-y1)/height])\n", "\n", "real = annotations.iloc[0]\n", "display(real.to_frame('source value'))\n", "record = index.set_index('uid').loc[real.SOPInstanceUID]\n", "print('YOLO row:', int(real.class_id), *to_yolo(source_box(real.data), record.cols, record.rows))\n", "\n", "from matplotlib.patches import Rectangle\n", "fig, axes = plt.subplots(1, 5, figsize=(20, 5))\n", "for class_id, ax in enumerate(axes):\n", " uid = annotations.loc[annotations.class_id.eq(class_id), 'SOPInstanceUID'].iloc[0]\n", " record = index.set_index('uid').loc[uid]\n", " ax.imshow(dicom_to_rgb(record.path))\n", " for row in annotations.loc[annotations.SOPInstanceUID.eq(uid)].itertuples():\n", " x, y, w, h = source_box(row.data)\n", " ax.add_patch(Rectangle((x,y),w,h,fill=False,edgecolor='lime',linewidth=1))\n", " ax.text(x,y,NAMES[row.class_id],color='yellow',fontsize=7)\n", " ax.set_title(NAMES[class_id]); ax.axis('off')\n", "plt.tight_layout()\n", "plt.savefig(ARTIFACTS / 'real_label_overlays.png', dpi=160)\n", "plt.show()" ] }, { "cell_type": "markdown", "id": "06b8e14e", "metadata": { "id": "06b8e14e" }, "source": [ "5. Creating Train/Validation/Test folders (70%, 15%, 15%)" ] }, { "cell_type": "code", "execution_count": null, "id": "fd5bf6b6", "metadata": { "id": "fd5bf6b6" }, "outputs": [], "source": [ "parent = {uid: uid for uid in index.uid}\n", "def find(x):\n", " while parent[x] != x:\n", " parent[x] = parent[parent[x]]\n", " x = parent[x]\n", " return x\n", "\n", "def union(a, b):\n", " parent[find(a)] = find(b)\n", "\n", "for field in ['patient', 'case', 'study']:\n", " for value, group in index.loc[index[field].ne('')].groupby(field):\n", " ids = group.uid.tolist()\n", " for uid in ids[1:]:\n", " union(ids[0], uid)\n", "index['group'] = index.uid.map(lambda uid: hashlib.sha256(find(uid).encode()).hexdigest()[:16])\n", "assert index.group.nunique() >= 10, 'Too few patient groups. Check generic/reused PatientID values against CQ500 provenance.'\n", "class_sets = annotations.groupby('SOPInstanceUID').class_id.agg(set).to_dict()\n", "index['negative'] = index.uid.isin(negative_uids)\n", "\n", "def covered(frame):\n", " present = set().union(*(class_sets.get(uid, set()) for uid in frame.uid))\n", " return present == set(range(5))\n", "\n", "SPLIT_CSV = ARTIFACTS / 'split_manifest.csv'\n", "if SPLIT_CSV.exists():\n", " previous = pd.read_csv(SPLIT_CSV, dtype={'uid':str, 'group':str})\n", " assert set(previous.uid) == set(index.uid), 'Dataset changed. Use a new ROOT for a new experiment.'\n", " assert previous.set_index('uid').group.to_dict() == index.set_index('uid').group.to_dict(), 'Grouping changed.'\n", " index['split'] = index.uid.map(previous.set_index('uid').split)\n", "else:\n", " for split_seed in range(SEED, SEED + 500):\n", " train_idx, hold_idx = next(GroupShuffleSplit(n_splits=1, test_size=.30,\n", " random_state=split_seed).split(index, groups=index.group))\n", " hold = index.iloc[hold_idx]\n", " val_rel, test_rel = next(GroupShuffleSplit(n_splits=1, test_size=.50,\n", " random_state=split_seed).split(hold, groups=hold.group))\n", " parts = [index.iloc[train_idx], hold.iloc[val_rel], hold.iloc[test_rel]]\n", " if all(covered(part) for part in parts):\n", " # If enough reviewed negative patients exist, require negatives in every split.\n", " if index.loc[index.negative, 'group'].nunique() >= 3 and not all(p.negative.any() for p in parts):\n", " continue\n", " index['split'] = ''\n", " for name, part in zip(['train','val','test'], parts):\n", " index.loc[part.index, 'split'] = name\n", " print('Split seed:', split_seed)\n", " break\n", " else:\n", " raise RuntimeError('No split covers every subtype. Add cases or revise the study design before training.')\n", "assert set(index.split) == {'train','val','test'}\n", "assert index.groupby('group').split.nunique().max() == 1\n", "assert index.groupby('study').split.nunique().max() == 1\n", "index.to_csv(SPLIT_CSV, index=False)\n", "for split in ['train','val','test']:\n", " for kind in ['images','labels']:\n", " (DATA / kind / split).mkdir(parents=True, exist_ok=True)\n", "display(index.groupby('split').agg(images=('uid','size'), patients=('group','nunique'), negatives=('negative','sum')))" ] }, { "cell_type": "markdown", "id": "3b40535f", "metadata": { "id": "3b40535f" }, "source": [ "6. Copy Images and Labels" ] }, { "cell_type": "code", "execution_count": null, "id": "791e0ba9", "metadata": { "id": "791e0ba9" }, "outputs": [], "source": [ "by_uid = {uid: group for uid, group in annotations.groupby('SOPInstanceUID')}\n", "pixel_hashes = []\n", "for row in tqdm(index.itertuples(), total=len(index), desc='Preparing YOLO pairs'):\n", " rgb = dicom_to_rgb(row.path)\n", " assert rgb.shape[:2] == (row.rows, row.cols)\n", " image_path = DATA / 'images' / row.split / (row.uid + '.png')\n", " label_path = DATA / 'labels' / row.split / (row.uid + '.txt')\n", " Image.fromarray(rgb).save(image_path)\n", " lines = []\n", " if row.uid in by_uid:\n", " for label in by_uid[row.uid].itertuples():\n", " coords = to_yolo(source_box(label.data), row.cols, row.rows)\n", " assert np.isfinite(coords).all() and (coords >= 0).all() and (coords <= 1).all()\n", " lines.append(str(label.class_id) + ' ' + ' '.join(f'{v:.8f}' for v in coords))\n", " else:\n", " assert row.uid in negative_uids, 'Unreviewed background image'\n", " label_path.write_text('\\n'.join(sorted(set(lines))) + ('\\n' if lines else ''))\n", " pixel_hashes.append({'uid':row.uid, 'split':row.split,\n", " 'sha256':hashlib.sha256(rgb.tobytes()).hexdigest()})\n", "hashes = pd.DataFrame(pixel_hashes)\n", "hashes.to_csv(ARTIFACTS / 'pixel_hashes.csv', index=False)\n", "assert hashes.groupby('sha256').split.nunique().max() == 1, 'Duplicate image content crosses splits. Regroup before training.'\n", "# Remove stale Ultralytics caches only. Image/label identity is checked next.\n", "for cache in DATA.rglob('*.cache'):\n", " cache.unlink()\n", "print('Paired PNG images and YOLO labels prepared.')" ] }, { "cell_type": "markdown", "id": "6dfbbc82", "metadata": { "id": "6dfbbc82" }, "source": [ "7. Tell YOLO where everything is" ] }, { "cell_type": "code", "execution_count": null, "id": "ab5d5e40", "metadata": { "id": "ab5d5e40" }, "outputs": [], "source": [ "DATA_YAML = DATA / 'data.yaml'\n", "config = {'path':str(DATA.resolve()), 'train':'images/train', 'val':'images/val',\n", " 'test':'images/test', 'names':dict(enumerate(NAMES))}\n", "DATA_YAML.write_text(yaml.safe_dump(config, sort_keys=False))\n", "print(DATA_YAML.read_text())" ] }, { "cell_type": "markdown", "id": "bb144383", "metadata": { "id": "bb144383" }, "source": [ "8. Count Everything" ] }, { "cell_type": "code", "execution_count": null, "id": "3cbde089", "metadata": { "id": "3cbde089" }, "outputs": [], "source": [ "counts = []\n", "for split in ['train','val','test']:\n", " images = {p.stem:p for p in (DATA/'images'/split).glob('*.png')}\n", " labels = {p.stem:p for p in (DATA/'labels'/split).glob('*.txt')}\n", " expected = set(index.loc[index.split.eq(split), 'uid'])\n", " assert set(images) == set(labels) == expected, f'Pair mismatch or stale files: {split}'\n", " box_counts = np.zeros(5, dtype=int)\n", " image_counts = np.zeros(5, dtype=int)\n", " negative_count = 0\n", " for uid, label in labels.items():\n", " present = set()\n", " for line in label.read_text().splitlines():\n", " tokens = line.split()\n", " assert len(tokens) == 5\n", " c = int(tokens[0]); xywh = np.array(tokens[1:], dtype=float)\n", " assert c in range(5) and np.isfinite(xywh).all()\n", " assert (xywh >= 0).all() and (xywh <= 1).all() and (xywh[2:] > 0).all()\n", " assert (xywh[:2]-xywh[2:]/2 >= -1e-6).all()\n", " assert (xywh[:2]+xywh[2:]/2 <= 1+1e-6).all()\n", " box_counts[c] += 1; present.add(c)\n", " for c in present:\n", " image_counts[c] += 1\n", " if not present:\n", " assert uid in negative_uids\n", " negative_count += 1\n", " for c in range(5):\n", " uids = set(annotations.loc[annotations.class_id.eq(c),'SOPInstanceUID'])\n", " patients = index.loc[index.split.eq(split) & index.uid.isin(uids),'group'].nunique()\n", " counts.append(dict(split=split, subtype=NAMES[c], boxes=box_counts[c],\n", " images=image_counts[c], patients=patients))\n", " assert (box_counts > 0).all(), f'Missing subtype in {split}'\n", " print(split, 'images:', len(images), 'labels:',len(labels), 'reviewed negatives:',negative_count)\n", "count_table = pd.DataFrame(counts)\n", "count_table.to_csv(ARTIFACTS/'class_support.csv', index=False)\n", "display(count_table)\n", "if not negative_uids:\n", " print('Positive-slice localization experiment. Specificity and negative predictive value are unavailable.')" ] }, { "cell_type": "markdown", "id": "c546531b", "metadata": { "id": "c546531b" }, "source": [ "9. Introducing YOLO11n" ] }, { "cell_type": "code", "execution_count": null, "id": "1b1fe551", "metadata": { "id": "1b1fe551" }, "outputs": [], "source": [ "WEIGHTS_URL = 'https://github.com/ultralytics/assets/releases/download/v8.4.0/yolo11n.pt'\n", "PRETRAINED = download(WEIGHTS_URL, ROOT/'yolo11n.pt')\n", "model = YOLO(str(PRETRAINED), task='detect')\n", "print('Checkpoint SHA256:', hashlib.sha256(PRETRAINED.read_bytes()).hexdigest())\n", "model.info()" ] }, { "cell_type": "markdown", "id": "f2e577e7", "metadata": { "id": "f2e577e7" }, "source": [ "10. Fine-tune YOLO11n" ] }, { "cell_type": "code", "execution_count": null, "id": "4146159b", "metadata": { "id": "4146159b" }, "outputs": [], "source": [ "train_result = model.train(\n", " data=str(DATA_YAML), epochs=EPOCHS, imgsz=IMGSZ, batch=BATCH,\n", " device=DEVICE, workers=2, seed=SEED, deterministic=True,\n", " project=str(RUNS), name='ich_yolo11n', exist_ok=False,\n", " optimizer='AdamW', lr0=0.001, weight_decay=0.0005,\n", " patience=12, save=True, save_period=5, cache=False, amp=True,\n", " mosaic=0.0, mixup=0.0, copy_paste=0.0,\n", " hsv_h=0.0, hsv_s=0.0, hsv_v=0.0,\n", " flipud=0.0, fliplr=0.0, degrees=5.0, translate=0.03, scale=0.10,\n", " shear=0.0, perspective=0.0, plots=True\n", ")" ] }, { "cell_type": "markdown", "id": "2d12cbb1", "metadata": { "id": "2d12cbb1" }, "source": [ "11. Find and preserve Trained model" ] }, { "cell_type": "code", "execution_count": null, "id": "926e3ffd", "metadata": { "id": "926e3ffd" }, "outputs": [], "source": [ "RUN_DIR = Path(model.trainer.save_dir)\n", "BEST = RUN_DIR/'weights'/'best.pt'\n", "assert BEST.is_file(), f'No best checkpoint at {BEST}'\n", "shutil.copy2(BEST, ARTIFACTS/'best.pt')\n", "for name in ['args.yaml','results.csv','results.png']:\n", " if (RUN_DIR/name).exists():\n", " shutil.copy2(RUN_DIR/name, ARTIFACTS/name)\n", "detector = YOLO(str(BEST))\n", "assert [detector.names[i] for i in range(5)] == NAMES\n", "print('Best model:', BEST)\n", "if (RUN_DIR/'results.png').exists():\n", " display(Image.open(RUN_DIR/'results.png'))" ] }, { "cell_type": "markdown", "id": "a79ba345", "metadata": { "id": "a79ba345" }, "source": [ "12. Evaluate on held-out test images" ] }, { "cell_type": "code", "execution_count": null, "id": "0bdbf066", "metadata": { "id": "0bdbf066" }, "outputs": [], "source": [ "test_metrics = detector.val(data=str(DATA_YAML), split='test', imgsz=IMGSZ,\n", " batch=BATCH, device=DEVICE, workers=2, conf=0.001, iou=0.7,\n", " plots=True, save_json=True, project=str(RUNS), name='heldout_test', exist_ok=False)\n", "TEST_RUN = Path(test_metrics.save_dir)\n", "print('Held-out evaluation outputs:', TEST_RUN)" ] }, { "cell_type": "markdown", "id": "3597b06c", "metadata": { "id": "3597b06c" }, "source": [ "13. Object Detection and Classification Scoring" ] }, { "cell_type": "code", "execution_count": null, "id": "63aea9e2", "metadata": { "id": "63aea9e2" }, "outputs": [], "source": [ "overall = {'mAP50':float(test_metrics.box.map50), 'mAP50_95':float(test_metrics.box.map),\n", " 'precision':float(test_metrics.box.mp), 'recall':float(test_metrics.box.mr),\n", " 'test_images':int(index.split.eq('test').sum()),\n", " 'test_patients':int(index.loc[index.split.eq('test'),'group'].nunique()),\n", " 'test_negative_images':int((index.split.eq('test') & index.negative).sum()),\n", " 'cohort':'manual BHX positives plus reviewed negatives' if negative_uids else 'manual BHX positive slices only'}\n", "(ARTIFACTS/'test_metrics.json').write_text(json.dumps(overall, indent=2))\n", "display(pd.DataFrame([overall]))\n", "rows = []\n", "# ap_class_index aligns the per-class arrays with actual class IDs.\n", "for j, class_id in enumerate(test_metrics.box.ap_class_index):\n", " rows.append({'subtype':NAMES[int(class_id)],\n", " 'mAP50':float(test_metrics.box.ap50[j]), 'mAP50_95':float(test_metrics.box.ap[j]),\n", " 'precision':float(test_metrics.box.p[j]), 'recall':float(test_metrics.box.r[j])})\n", "per_class = pd.DataFrame(rows).merge(count_table.query(\"split == 'test'\"), on='subtype')\n", "per_class.to_csv(ARTIFACTS/'test_per_class.csv', index=False)\n", "display(per_class)\n", "for filename in ['confusion_matrix_normalized.png','BoxPR_curve.png','PR_curve.png']:\n", " path = TEST_RUN/filename\n", " if path.exists():\n", " display(Image.open(path))" ] }, { "cell_type": "markdown", "source": [ "13.1 Setup and compatibility checks" ], "metadata": { "id": "tyjf4ANmeaOw" }, "id": "tyjf4ANmeaOw" }, { "cell_type": "code", "execution_count": null, "id": "2039f495", "metadata": { "id": "2039f495" }, "outputs": [], "source": [ "INSTALL_DEPENDENCIES = False # True only in a fresh environment.\n", "if INSTALL_DEPENDENCIES:\n", " import subprocess, sys\n", " subprocess.run([sys.executable, '-m', 'pip', 'install', '-q',\n", " 'ultralytics>=8.4.0,<9', 'pandas>=2,<3', 'scikit-learn>=1.5,<2',\n", " 'matplotlib>=3.8,<4', 'Pillow>=10,<12', 'PyYAML>=6,<7'], check=True)\n", "\n", "from pathlib import Path\n", "import hashlib, json, shutil\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import torch, yaml\n", "from PIL import Image\n", "from ultralytics import YOLO\n", "from sklearn.metrics import roc_auc_score, average_precision_score, roc_curve, precision_recall_curve\n", "from IPython.display import display\n", "\n", "METRIC_ROOT = Path(globals().get('ROOT', '/content/ich_yolo11')) # Edit for a restored experiment.\n", "m_data = METRIC_ROOT / 'dataset'\n", "m_artifacts = METRIC_ROOT / 'artifacts'\n", "m_out = m_artifacts / 'extended_metrics'\n", "m_out.mkdir(parents=True, exist_ok=True)\n", "m_best = Path(globals().get('BEST', m_artifacts / 'best.pt'))\n", "m_names = ['intraparenchymal','intraventricular','subarachnoid','subdural','epidural']\n", "m_imgsz = int(globals().get('IMGSZ', 512))\n", "m_batch = int(globals().get('BATCH', 8))\n", "m_device = globals().get('DEVICE', 0 if torch.cuda.is_available() else 'cpu')\n", "m_model = YOLO(str(m_best))\n", "assert [m_model.names[k] for k in range(5)] == m_names, 'Model class order differs from the original notebook.'\n", "m_index = pd.read_csv(m_artifacts/'split_manifest.csv', dtype={'uid':str,'group':str}).fillna('')\n", "assert {'uid','group','split','negative'} <= set(m_index)\n", "assert not m_index.uid.duplicated().any()\n", "assert m_index.group.ne('').all()\n", "assert set(m_index.split) == {'train','val','test'}\n", "assert m_index.groupby('group').split.nunique().max() == 1, 'Patient overlap across splits'\n", "if 'study' in m_index:\n", " assert m_index.groupby('study').split.nunique().max() == 1\n", "assert m_index.negative.astype(str).str.lower().isin(['true','false','1','0']).all()\n", "m_index['negative'] = m_index.negative.astype(str).str.lower().isin(['true','1'])\n", "for split in ['train','val','test']:\n", " expected = set(m_index.loc[m_index.split.eq(split),'uid'])\n", " for kind, ext in [('images','png'),('labels','txt')]:\n", " assert {p.stem for p in (m_data/kind/split).glob('*.'+ext)} == expected, f'{split}: missing/stale {kind}'\n", "# Regenerate absolute paths for restored experiments without changing the original data.yaml.\n", "m_yaml = m_out/'evaluation_data.yaml'\n", "m_yaml.write_text(yaml.safe_dump({'path':str(m_data.resolve()),'train':'images/train',\n", " 'val':'images/val','test':'images/test','names':dict(enumerate(m_names))},sort_keys=False))\n", "M_FLOOR, M_NMS_IOU, M_MAX_DET, M_MATCH_IOU = 0.001, 0.7, 300, 0.50\n", "M_GRID = np.round(np.arange(.05,.951,.05), 2)\n", "print('Model:',m_best)\n", "print('Output:',m_out)\n", "display(m_index.groupby('split').agg(images=('uid','size'),patients=('group','nunique'),negative_images=('negative','sum')))" ] }, { "cell_type": "markdown", "source": [ "13.2 Collection of validation predictions and definition of evaluators" ], "metadata": { "id": "qHqc9-hfeeX0" }, "id": "qHqc9-hfeeX0" }, { "cell_type": "code", "source": [ "def m_div(a, b):\n", " return float(a/b) if b else np.nan\n", "\n", "def m_prf(tp, fp, fn):\n", " return {'precision':m_div(tp,tp+fp), 'recall_sensitivity':m_div(tp,tp+fn),\n", " 'F1':m_div(2*tp,2*tp+fp+fn)}\n", "\n", "def m_iou(box, boxes):\n", " if len(boxes) == 0:\n", " return np.empty(0)\n", " lo = np.maximum(box[:2],boxes[:,:2]); hi = np.minimum(box[2:],boxes[:,2:])\n", " inter = np.clip(hi-lo,0,None).prod(axis=1)\n", " area_a = np.clip(box[2:]-box[:2],0,None).prod()\n", " area_b = np.clip(boxes[:,2:]-boxes[:,:2],0,None).prod(axis=1)\n", " return inter/np.maximum(area_a+area_b-inter,1e-12)\n", "\n", "def m_match(item, threshold, match_iou=M_MATCH_IOU):\n", " counts = np.zeros((5,3),dtype=int) # columns TP, FP, FN\n", " for c in range(5):\n", " truth = item['gt_box'][item['gt_cls']==c]\n", " sel = np.where((item['pred_cls']==c) & (item['score']>=threshold))[0]\n", " sel = sel[np.argsort(-item['score'][sel],kind='stable')]\n", " used = np.zeros(len(truth),dtype=bool)\n", " for j in sel:\n", " overlaps = m_iou(item['pred_box'][j],truth)\n", " overlaps[used] = -1\n", " if len(overlaps) and overlaps.max() >= match_iou:\n", " used[int(overlaps.argmax())] = True\n", " counts[c,0] += 1\n", " else:\n", " counts[c,1] += 1\n", " counts[c,2] = (~used).sum()\n", " return counts\n", "\n", "def m_collect(split):\n", " records = []\n", " frame = m_index.loc[m_index.split.eq(split)].sort_values('uid')\n", " rows = list(frame.itertuples(index=False))\n", " paths = [str(m_data/'images'/split/(r.uid+'.png')) for r in rows]\n", " results = m_model.predict(source=paths,stream=True,batch=m_batch,imgsz=m_imgsz,\n", " device=m_device,conf=M_FLOOR,iou=M_NMS_IOU,max_det=M_MAX_DET,\n", " agnostic_nms=False,verbose=False)\n", " for row, result in zip(rows,results):\n", " assert Path(result.path).stem == row.uid, 'Prediction order mismatch'\n", " h,w = result.orig_shape\n", " lines = (m_data/'labels'/split/(row.uid+'.txt')).read_text().splitlines()\n", " gt_cls, gt_box = [], []\n", " for line in lines:\n", " values = np.asarray(line.split(),dtype=float)\n", " assert values.shape == (5,) and np.isfinite(values).all()\n", " c,x,y,bw,bh = values\n", " assert c == int(c) and 0 <= c < 5 and bw>0 and bh>0\n", " assert min(x-bw/2,y-bh/2)>=-1e-6 and max(x+bw/2,y+bh/2)<=1+1e-6\n", " gt_cls.append(int(c))\n", " gt_box.append([(x-bw/2)*w,(y-bh/2)*h,(x+bw/2)*w,(y+bh/2)*h])\n", " assert bool(row.negative) == (len(gt_cls)==0), 'Unreviewed or inconsistent empty label'\n", " rec = dict(uid=row.uid,group=row.group,\n", " gt_cls=np.asarray(gt_cls,dtype=int),gt_box=np.asarray(gt_box,dtype=float).reshape(-1,4),\n", " pred_cls=result.boxes.cls.cpu().numpy().astype(int),\n", " pred_box=result.boxes.xyxy.cpu().numpy(),score=result.boxes.conf.cpu().numpy())\n", " assert np.isfinite(rec['pred_box']).all() and np.isfinite(rec['score']).all()\n", " assert set(rec['pred_cls']).issubset(set(range(5)))\n", " records.append(rec)\n", " assert len(records)==len(rows) and len(records)>0\n", " print(split, 'images:',len(records),'images at max_det:',sum(len(r['score'])>=M_MAX_DET for r in records))\n", " return records\n", "\n", "def m_slice_arrays(records):\n", " y = np.zeros((len(records),6),dtype=int)\n", " scores = np.zeros((len(records),6),dtype=float)\n", " for i,r in enumerate(records):\n", " y[i,np.unique(r['gt_cls'])] = 1\n", " for c in range(5):\n", " vals = r['score'][r['pred_cls']==c]\n", " scores[i,c] = vals.max() if len(vals) else 0.\n", " y[i,5] = y[i,:5].max()\n", " scores[i,5] = scores[i,:5].max()\n", " return y,scores\n", "\n", "m_val = m_collect('val')\n", "m_yval,m_sval = m_slice_arrays(m_val)" ], "metadata": { "id": "Q1RSIUPlekcp" }, "id": "Q1RSIUPlekcp", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.3 Freeze thresholds using validation only" ], "metadata": { "id": "xfEmRaedetic" }, "id": "xfEmRaedetic" }, { "cell_type": "code", "source": [ "m_fingerprint = hashlib.sha256()\n", "for p in [m_best,m_artifacts/'split_manifest.csv']:\n", " with p.open('rb') as f:\n", " for chunk in iter(lambda:f.read(1024*1024),b''):\n", " m_fingerprint.update(chunk)\n", "# Bind lock to evaluation images and labels, including preprocessing changes.\n", "for split in ['val','test']:\n", " for row in m_index.loc[m_index.split.eq(split)].sort_values('uid').itertuples():\n", " for kind,ext in [('images','png'),('labels','txt')]:\n", " p=m_data/kind/split/(row.uid+'.'+ext)\n", " m_fingerprint.update((split+'/'+kind+'/'+p.name).encode())\n", " m_fingerprint.update(p.read_bytes())\n", "m_settings=dict(imgsz=m_imgsz,confidence_floor=M_FLOOR,nms_iou=M_NMS_IOU,\n", " max_det=M_MAX_DET,matching_iou=M_MATCH_IOU,threshold_grid=M_GRID.tolist(),policy_version=1)\n", "m_fingerprint.update(json.dumps(m_settings,sort_keys=True).encode())\n", "m_signature=m_fingerprint.hexdigest()\n", "m_lock=m_out/'thresholds.json'\n", "if m_lock.exists():\n", " m_thresholds=json.loads(m_lock.read_text())\n", " assert m_thresholds['signature']==m_signature, 'Evaluation inputs/settings changed. Use a new output directory.'\n", "else:\n", " sweep=[]\n", " for t in M_GRID:\n", " ct=sum((m_match(r,t) for r in m_val),start=np.zeros((5,3),dtype=int))\n", " box_f1=m_prf(*ct.sum(axis=0))['F1']\n", " f1s=[]\n", " for c in range(5):\n", " y=m_yval[:,c].astype(bool); pred=m_sval[:,c]>=t\n", " f1s.append(m_prf((y&pred).sum(),(~y&pred).sum(),(y&~pred).sum())['F1'])\n", " y=m_yval[:,5].astype(bool); pred=m_sval[:,5]>=t\n", " any_f1=m_prf((y&pred).sum(),(~y&pred).sum(),(y&~pred).sum())['F1']\n", " sweep.append(dict(threshold=float(t),box_micro_F1=box_f1,\n", " subtype_macro_F1=float(np.nanmean(f1s)),any_F1=any_f1))\n", " m_sweep=pd.DataFrame(sweep)\n", " def choose(column):\n", " return float(m_sweep.sort_values([column,'threshold'],ascending=[False,False]).iloc[0].threshold)\n", " both_classes=np.unique(m_yval[:,5]).size==2\n", " m_thresholds={'signature':m_signature,'box':choose('box_micro_F1'),\n", " 'subtype':choose('subtype_macro_F1'),'any':choose('any_F1') if both_classes else choose('box_micro_F1'),\n", " 'any_policy':'validation binary F1' if both_classes else 'box-threshold fallback, no negative validation slices',\n", " 'settings':m_settings}\n", " m_sweep.to_csv(m_out/'validation_threshold_sweep.csv',index=False)\n", " m_lock.write_text(json.dumps(m_thresholds,indent=2))\n", "DEMO_CONF=float(m_thresholds['any']) # Used by original Steps 14–16.\n", "display(pd.DataFrame([m_thresholds]).drop(columns=['signature','settings']))\n", "# Test inference starts only after operating thresholds are fixed.\n", "m_test=m_collect('test')\n", "m_ytest,m_stest=m_slice_arrays(m_test)" ], "metadata": { "id": "QxCL9Fl2e0nM" }, "id": "QxCL9Fl2e0nM", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.4 Box-level precision + recall/sensitivity + F1 + FP" ], "metadata": { "id": "M8bMP9LTfHWL" }, "id": "M8bMP9LTfHWL" }, { "cell_type": "code", "source": [ "m_counts=np.stack([m_match(r,m_thresholds['box']) for r in m_test])\n", "m_detection_rows=[]\n", "for c,name in enumerate(m_names):\n", " tp,fp,fn=m_counts[:,c,:].sum(axis=0)\n", " support_groups={r['group'] for r in m_test if c in r['gt_cls']}\n", " m_detection_rows.append(dict(subtype=name,TP=int(tp),FP=int(fp),FN=int(fn),\n", " ground_truth_boxes=int(tp+fn),positive_patients=len(support_groups),\n", " FP_per_image=float(fp/len(m_test)),**m_prf(tp,fp,fn)))\n", "m_detection=pd.DataFrame(m_detection_rows)\n", "tp,fp,fn=m_counts.sum(axis=(0,1))\n", "m_micro={'average':'micro','TP':int(tp),'FP':int(fp),'FN':int(fn),\n", " 'FP_per_image':float(fp/len(m_test)),**m_prf(tp,fp,fn)}\n", "m_macro={'average':'macro',**{k:float(m_detection[k].mean()) for k in ['precision','recall_sensitivity','F1']}}\n", "m_detection_summary=pd.DataFrame([m_micro,m_macro])\n", "display(m_detection)\n", "display(m_detection_summary)\n", "print('Confidence:',m_thresholds['box'],'Matching IoU:',M_MATCH_IOU)\n", "m_detection.to_csv(m_out/'detection_per_subtype.csv',index=False)\n", "m_detection_summary.to_csv(m_out/'detection_summary.csv',index=False)" ], "metadata": { "id": "MUzNZ-d5fNLm" }, "id": "MUzNZ-d5fNLm", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.5 Detection on mAP@50, mAP@75 & mAP@50-95" ], "metadata": { "id": "BhNDqkmlfRSv" }, "id": "BhNDqkmlfRSv" }, { "cell_type": "code", "source": [ "m_ap=m_model.val(data=str(m_yaml),split='test',imgsz=m_imgsz,batch=m_batch,\n", " device=m_device,workers=2,conf=M_FLOOR,iou=M_NMS_IOU,max_det=M_MAX_DET,\n", " agnostic_nms=False,plots=True,project=str(m_out),name='ultralytics_test',exist_ok=False)\n", "m_ap_summary=pd.DataFrame([{'mAP50':float(m_ap.box.map50),'mAP75':float(m_ap.box.map75),\n", " 'mAP50_95':float(m_ap.box.map)}])\n", "m_ap_rows=[]\n", "for j,c in enumerate(m_ap.box.ap_class_index):\n", " m_ap_rows.append({'subtype':m_names[int(c)],'AP50':float(m_ap.box.all_ap[j,0]),\n", " 'AP75':float(m_ap.box.all_ap[j,5]),'AP50_95':float(m_ap.box.ap[j])})\n", "m_ap_table=pd.DataFrame({'subtype':m_names}).merge(pd.DataFrame(m_ap_rows,columns=['subtype','AP50','AP75','AP50_95']),on='subtype',how='left')\n", "display(m_ap_summary); display(m_ap_table)\n", "m_ap_summary.to_csv(m_out/'detection_mAP.csv',index=False)\n", "m_ap_table.to_csv(m_out/'detection_AP_per_subtype.csv',index=False)" ], "metadata": { "id": "OGtFydjFfckJ" }, "id": "OGtFydjFfckJ", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.6 Slice-level multilabel classification metrics" ], "metadata": { "id": "NigkVJuyff5O" }, "id": "NigkVJuyff5O" }, { "cell_type": "code", "source": [ "def m_binary_counts(y,p):\n", " y=np.asarray(y,dtype=bool); p=np.asarray(p,dtype=bool)\n", " return np.array([(y&p).sum(),(~y&p).sum(),(y&~p).sum(),(~y&~p).sum()],dtype=int)\n", "\n", "def m_binary_metrics(tp,fp,fn,tn):\n", " output=m_prf(tp,fp,fn)\n", " output.update(specificity=m_div(tn,tn+fp),NPV=m_div(tn,tn+fn),\n", " accuracy=m_div(tp+tn,tp+fp+fn+tn))\n", " output['balanced_accuracy']=(output['recall_sensitivity']+output['specificity'])/2\n", " return output\n", "\n", "m_targets=m_names+['any_hemorrhage']\n", "m_cutoffs=np.array([m_thresholds['subtype']]*5+[m_thresholds['any']])\n", "m_pred=m_stest>=m_cutoffs\n", "m_class_rows=[]\n", "for c,name in enumerate(m_targets):\n", " y=m_ytest[:,c]; s=m_stest[:,c]; pred=m_pred[:,c]\n", " tp,fp,fn,tn=m_binary_counts(y,pred)\n", " both=np.unique(y).size==2\n", " m_class_rows.append(dict(target=name,threshold=float(m_cutoffs[c]),\n", " TP=int(tp),FP=int(fp),FN=int(fn),TN=int(tn),positive_slices=int(y.sum()),\n", " negative_slices=int((1-y).sum()),\n", " positive_patients=len({r['group'] for i,r in enumerate(m_test) if y[i]}),\n", " AUROC=float(roc_auc_score(y,s)) if both else np.nan,\n", " average_precision=float(average_precision_score(y,s)) if both else np.nan,\n", " ranking_metric_status='defined' if both else 'undefined: single-class target',\n", " **m_binary_metrics(tp,fp,fn,tn)))\n", "m_classification=pd.DataFrame(m_class_rows)\n", "m_sub=m_classification.iloc[:5]\n", "m_flat_counts=m_binary_counts(m_ytest[:,:5].ravel(),m_pred[:,:5].ravel())\n", "m_class_summary=pd.DataFrame([\n", " {'average':'macro_subtype',**{k:float(m_sub[k].mean()) for k in\n", " ['precision','recall_sensitivity','F1','specificity','AUROC','average_precision']},\n", " 'defined_AUROC_classes':int(m_sub.AUROC.notna().sum()),\n", " 'defined_AP_classes':int(m_sub.average_precision.notna().sum())},\n", " {'average':'micro_subtype',**m_binary_metrics(*m_flat_counts)}])\n", "display(m_classification); display(m_class_summary)\n", "m_classification.to_csv(m_out/'slice_metrics_per_target.csv',index=False)\n", "m_class_summary.to_csv(m_out/'slice_metrics_summary.csv',index=False)\n", "m_slice_export=pd.DataFrame({'uid':[r['uid'] for r in m_test],'group':[r['group'] for r in m_test]})\n", "for c,name in enumerate(m_targets):\n", " m_slice_export[name+'_truth']=m_ytest[:,c]\n", " m_slice_export[name+'_score']=m_stest[:,c]\n", " m_slice_export[name+'_prediction']=m_pred[:,c].astype(int)\n", "m_slice_export.to_csv(m_out/'slice_test_predictions.csv',index=False)" ], "metadata": { "id": "pQ8DGV4Pfk36" }, "id": "pQ8DGV4Pfk36", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.7 Confusion matrices, ROC curves, Precision-Recall curves" ], "metadata": { "id": "X3R3lbeDfptQ" }, "id": "X3R3lbeDfptQ" }, { "cell_type": "code", "source": [ "fig,axes=plt.subplots(2,3,figsize=(12,8))\n", "for ax,row in zip(axes.ravel(),m_classification.itertuples()):\n", " cm=np.array([[row.TN,row.FP],[row.FN,row.TP]])\n", " ax.imshow(cm,cmap='Blues')\n", " for (i,j),value in np.ndenumerate(cm):\n", " ax.text(j,i,str(value),ha='center',va='center',color='black',\n", " bbox={'facecolor':'white','alpha':.7,'edgecolor':'none'})\n", " ax.set(xticks=[0,1],yticks=[0,1],xticklabels=['Absent','Present'],\n", " yticklabels=['Absent','Present'],xlabel='Predicted',ylabel='True',title=row.target)\n", "plt.tight_layout(); plt.savefig(m_out/'slice_confusion_matrices.png',dpi=160); plt.show()\n", "fig,axes=plt.subplots(1,2,figsize=(12,5))\n", "m_curve_count=0\n", "for c,name in enumerate(m_targets):\n", " if np.unique(m_ytest[:,c]).size<2:\n", " continue\n", " fpr,tpr,_=roc_curve(m_ytest[:,c],m_stest[:,c])\n", " precision,recall,_=precision_recall_curve(m_ytest[:,c],m_stest[:,c])\n", " axes[0].plot(fpr,tpr,label=name)\n", " axes[1].plot(recall,precision,label=name)\n", " m_curve_count+=1\n", "axes[0].plot([0,1],[0,1],'k--',alpha=.4)\n", "axes[0].set(xlabel='False positive rate',ylabel='Sensitivity',title='Slice-level ROC')\n", "axes[1].set(xlabel='Recall',ylabel='Precision',title='Slice-level precision-recall')\n", "for ax in axes:\n", " ax.set_xlim(0,1); ax.set_ylim(0,1.02)\n", " if m_curve_count: ax.legend(fontsize=8)\n", "plt.tight_layout(); plt.savefig(m_out/'slice_ROC_PR.png',dpi=160); plt.show()" ], "metadata": { "id": "pu33C_OzfuAJ" }, "id": "pu33C_OzfuAJ", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.8 Patient-cluster bootstrap 95% confidence intervals" ], "metadata": { "id": "0YBji70hf5uQ" }, "id": "0YBji70hf5uQ" }, { "cell_type": "code", "source": [ "M_BOOTSTRAPS=500\n", "m_rng=np.random.default_rng(42)\n", "m_groups=np.array([r['group'] for r in m_test])\n", "m_unique=np.unique(m_groups)\n", "m_group_det=np.stack([m_counts[m_groups==g].sum(axis=0) for g in m_unique])\n", "m_group_class=np.array([[m_binary_counts(m_ytest[m_groups==g,c],m_pred[m_groups==g,c])\n", " for c in range(6)] for g in m_unique])\n", "m_boot_values={}\n", "for _ in range(M_BOOTSTRAPS):\n", " chosen=m_rng.integers(0,len(m_unique),size=len(m_unique))\n", " det=m_group_det[chosen].sum(axis=0)\n", " cl=m_group_class[chosen].sum(axis=0)\n", " for name,ct in list(zip(m_names,det))+[('micro',det.sum(axis=0))]:\n", " for metric,value in m_prf(*ct).items():\n", " m_boot_values.setdefault(('detection',name,metric),[]).append(value)\n", " for name,ct in zip(m_targets,cl):\n", " for metric,value in m_binary_metrics(*ct).items():\n", " if metric in ['precision','recall_sensitivity','F1','specificity']:\n", " m_boot_values.setdefault(('slice',name,metric),[]).append(value)\n", "m_ci_rows=[]\n", "for (level,target,metric),values in m_boot_values.items():\n", " values=np.asarray(values); finite=values[np.isfinite(values)]\n", " if level=='detection':\n", " c=m_names.index(target) if target!='micro' else None\n", " point=(m_detection.set_index('subtype').loc[target,metric] if c is not None else m_micro[metric])\n", " ct=m_group_det[:,c,:] if c is not None else m_group_det.sum(axis=1)\n", " positives=int(((ct[:,0]+ct[:,2])>0).sum())\n", " denominator_groups=int(((ct[:,0]+ct[:,1])>0).sum()) if metric=='precision' else positives\n", " else:\n", " c=m_targets.index(target); point=m_classification.set_index('target').loc[target,metric]\n", " ct=m_group_class[:,c,:]\n", " positives=int(((ct[:,0]+ct[:,2])>0).sum())\n", " if metric=='specificity': denominator_groups=int(((ct[:,1]+ct[:,3])>0).sum())\n", " elif metric=='precision': denominator_groups=int(((ct[:,0]+ct[:,1])>0).sum())\n", " else: denominator_groups=positives\n", " enough=denominator_groups>=2 and (metric=='specificity' or positives>=2)\n", " enough=enough and len(finite)>=int(.8*M_BOOTSTRAPS) and np.isfinite(point)\n", " lo,hi=np.percentile(finite,[2.5,97.5]) if enough else (np.nan,np.nan)\n", " m_ci_rows.append(dict(level=level,target=target,metric=metric,estimate=point,\n", " lower_95=lo,upper_95=hi,valid_bootstraps=len(finite),total_bootstraps=M_BOOTSTRAPS,\n", " supporting_groups=denominator_groups,status='reported' if enough else 'insufficient support or undefined replicates'))\n", "m_ci=pd.DataFrame(m_ci_rows)\n", "m_ci.to_csv(m_out/'patient_bootstrap_CI.csv',index=False)\n", "display(m_ci)" ], "metadata": { "id": "5oYiECD9f-X0" }, "id": "5oYiECD9f-X0", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "13.9 Exporting analysis results" ], "metadata": { "id": "Sjpyf5_MgWVw" }, "id": "Sjpyf5_MgWVw" }, { "cell_type": "code", "source": [ "from importlib.metadata import version\n", "m_metadata={'model':str(m_best),'input_signature':m_signature,\n", " 'thresholds':m_thresholds,'class_names':m_names,'test_images':len(m_test),\n", " 'test_patient_groups':len(m_unique),'test_negative_images':int((m_ytest[:,5]==0).sum()),\n", " 'box_matching':'confidence-ordered, same class, one-to-one, highest unused IoU',\n", " 'classification':'per-slice multilabel, maximum retained detection confidence per subtype',\n", " 'subdural':'acute and chronic merged, as in initial notebook',\n", " 'uncertainty':'patient-cluster percentile bootstrap, fixed model and thresholds',\n", " 'versions':{p:version(p) for p in ['ultralytics','torch','numpy','pandas','scikit-learn']}}\n", "(m_out/'evaluation_metadata.json').write_text(json.dumps(m_metadata,indent=2))\n", "m_serial=[]\n", "for split,records in [('val',m_val),('test',m_test)]:\n", " for r in records:\n", " m_serial.append({'split':split,**{k:(v.tolist() if isinstance(v,np.ndarray) else v) for k,v in r.items()}})\n", "(m_out/'box_predictions_and_truth.json').write_text(json.dumps(m_serial))\n", "m_zip=Path(shutil.make_archive(str(m_artifacts/'extended_metrics_results'),'zip',m_out))\n", "print('Results:',m_zip)\n", "print('Continue with original Step 14. DEMO_CONF =',DEMO_CONF)\n", "DOWNLOAD_METRIC_RESULTS=True\n", "if DOWNLOAD_METRIC_RESULTS:\n", " try:\n", " from google.colab import files\n", " except ImportError:\n", " print('Outside Colab: open the ZIP path printed above.')\n", " else:\n", " files.download(str(m_zip))" ], "metadata": { "id": "0fU5go4Agh3f" }, "id": "0fU5go4Agh3f", "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "id": "f173857e", "metadata": { "id": "f173857e" }, "source": [ "14. Testing on unseen image (on Goolgle Colab)" ] }, { "cell_type": "code", "execution_count": null, "id": "339eb3ae", "metadata": { "id": "339eb3ae" }, "outputs": [], "source": [ "from google.colab import files\n", "uploaded = files.upload()\n", "assert len(uploaded) == 1, 'Upload exactly one image.'\n", "filename, contents = next(iter(uploaded.items()))\n", "UNSEEN = ROOT / ('unseen_input' + Path(filename).suffix.lower())\n", "UNSEEN.write_bytes(contents)\n", "del uploaded, contents\n", "if UNSEEN.suffix.lower() not in {'.png','.jpg','.jpeg'}:\n", " header = pydicom.dcmread(UNSEEN, stop_before_pixels=True)\n", " assert str(header.SOPInstanceUID) not in set(index.uid), 'This slice belongs to the dataset.'\n", " patient = str(getattr(header,'PatientID','')).strip()\n", " assert not patient or patient not in set(index.patient), 'This patient belongs to the dataset.'\n", "unseen_rgb = load_input(UNSEEN)\n", "assert hashlib.sha256(unseen_rgb.tobytes()).hexdigest() not in set(hashes.sha256), 'Duplicate dataset image.'\n", "# Passing a PIL RGB image avoids NumPy RGB/BGR ambiguity in Ultralytics.\n", "unseen_result = detector.predict(Image.fromarray(unseen_rgb), imgsz=IMGSZ,\n", " conf=DEMO_CONF, iou=.7, device=DEVICE, verbose=False)[0]" ] }, { "cell_type": "markdown", "id": "d75db598", "metadata": { "id": "d75db598" }, "source": [ "14.1 Show bounding boxes and confidence" ] }, { "cell_type": "code", "execution_count": null, "id": "d3ed85ad", "metadata": { "id": "d3ed85ad" }, "outputs": [], "source": [ "def prediction_table(result):\n", " rows = []\n", " for box in result.boxes:\n", " x1,y1,x2,y2 = box.xyxy[0].cpu().tolist()\n", " rows.append([result.names[int(box.cls.item())], float(box.conf.item()), x1,y1,x2,y2])\n", " return pd.DataFrame(rows, columns=['subtype','confidence','x1','y1','x2','y2'])\n", "\n", "annotated_rgb = unseen_result.plot()[..., ::-1] # Ultralytics plot returns BGR.\n", "display(Image.fromarray(annotated_rgb))\n", "display(prediction_table(unseen_result))\n", "Image.fromarray(annotated_rgb).save(ARTIFACTS/'unseen_prediction.png')\n", "prediction_table(unseen_result).to_csv(ARTIFACTS/'unseen_prediction.csv',index=False)\n", "if not len(unseen_result.boxes):\n", " print('No detection above the selected threshold. Hemorrhage is not excluded.')" ] }, { "cell_type": "markdown", "id": "d1fe621e", "metadata": { "id": "d1fe621e" }, "source": [ "15. Saving the experiment as an app to run locally" ] }, { "cell_type": "code", "execution_count": null, "id": "e37ced3f", "metadata": { "id": "e37ced3f" }, "outputs": [], "source": [ "APP_SOURCE = r'''\n", "from pathlib import Path\n", "import json\n", "import numpy as np\n", "import pandas as pd\n", "import torch\n", "import gradio as gr\n", "from PIL import Image\n", "from ultralytics import YOLO\n", "from preprocessing import load_input\n", "\n", "BASE = Path(__file__).resolve().parent\n", "config = json.loads((BASE/'config.json').read_text())\n", "model = YOLO(str(BASE/'best.pt'))\n", "device = 0 if torch.cuda.is_available() else 'cpu'\n", "COLUMNS = ['subtype','confidence','x1','y1','x2','y2']\n", "\n", "def predict(path, threshold):\n", " if not path:\n", " return None, pd.DataFrame(columns=COLUMNS), 'Upload one head CT slice.'\n", " try:\n", " rgb = load_input(path)\n", " result = model.predict(Image.fromarray(rgb), imgsz=config['imgsz'],\n", " conf=float(threshold), iou=.7, device=device, verbose=False)[0]\n", " rows = []\n", " for box in result.boxes:\n", " rows.append([result.names[int(box.cls.item())], float(box.conf.item()),\n", " *box.xyxy[0].cpu().tolist()])\n", " status = f'{len(rows)} region(s) detected.' if rows else 'No detection above threshold. Hemorrhage is not excluded.'\n", " return result.plot()[..., ::-1], pd.DataFrame(rows, columns=COLUMNS), status\n", " except Exception as error:\n", " raise gr.Error('Unable to process this slice: ' + str(error))\n", "\n", "with gr.Blocks(title='Intracranial hemorrhage research demo') as demo:\n", " gr.Markdown('# Intracranial hemorrhage research demo')\n", " gr.Markdown('Single-slice research output. Confidence scores are uncalibrated. No clinical diagnosis.')\n", " upload = gr.File(label='Upload head CT DICOM or brain-windowed PNG/JPEG', type='filepath',\n", " file_types=['.dcm','.dicom','.png','.jpg','.jpeg'])\n", " threshold = gr.Slider(.01,.95,value=config['confidence'],step=.01,label='Display confidence threshold')\n", " with gr.Row():\n", " predict_button = gr.Button('Predict',variant='primary')\n", " clear_button = gr.Button('Clear')\n", " output = gr.Image(label='Prediction: bounding boxes and confidence',type='numpy')\n", " table = gr.Dataframe(headers=COLUMNS,label='Detected regions',interactive=False)\n", " status = gr.Textbox(label='Result',interactive=False)\n", " predict_button.click(predict,inputs=[upload,threshold],outputs=[output,table,status],concurrency_limit=1)\n", " clear_button.click(lambda: (None,config['confidence'],None,pd.DataFrame(columns=COLUMNS),''),\n", " outputs=[upload,threshold,output,table,status])\n", "\n", "if __name__ == '__main__':\n", " demo.launch(server_name='127.0.0.1',server_port=7860,share=False,inbrowser=True)\n", "'''\n", "\n", "PREPROCESS_SOURCE = r'''\n", "from pathlib import Path\n", "import numpy as np\n", "import pydicom\n", "from PIL import Image\n", "\n", "def dicom_to_rgb(path):\n", " ds = pydicom.dcmread(path)\n", " if str(getattr(ds, 'Modality', '')) != 'CT':\n", " raise ValueError('Upload a CT DICOM slice.')\n", " if int(getattr(ds, 'NumberOfFrames', 1)) != 1:\n", " raise ValueError('Upload a single-frame CT slice.')\n", " if not hasattr(ds, 'RescaleSlope') or not hasattr(ds, 'RescaleIntercept'):\n", " raise ValueError('Missing CT rescale metadata. Convert with verified HU calibration first.')\n", " pixels = ds.pixel_array\n", " if pixels.ndim != 2:\n", " raise ValueError('Expected one grayscale CT slice.')\n", " if str(getattr(ds, 'PhotometricInterpretation', '')) not in {'MONOCHROME1', 'MONOCHROME2'}:\n", " raise ValueError('Expected monochrome CT data.')\n", " hu = pixels.astype(np.float32) * float(ds.RescaleSlope) + float(ds.RescaleIntercept)\n", " # Explicit HU window, independent of the source display LUT or photometric polarity.\n", " gray = np.rint(np.clip((hu - 0.0) / 80.0, 0, 1) * 255).astype(np.uint8)\n", " if hasattr(ds, 'PixelPaddingValue'):\n", " a = float(ds.PixelPaddingValue)\n", " b = float(getattr(ds, 'PixelPaddingRangeLimit', a))\n", " gray[(pixels >= min(a, b)) & (pixels <= max(a, b))] = 0\n", " return np.repeat(gray[..., None], 3, axis=2)\n", "\n", "def load_input(path):\n", " path = Path(path)\n", " if path.suffix.lower() in {'.png', '.jpg', '.jpeg'}:\n", " image = Image.open(path)\n", " if image.mode not in {'RGB','RGBA','L'}:\n", " raise ValueError('Use an 8-bit brain-windowed PNG/JPEG or original DICOM.')\n", " return np.asarray(image.convert('RGB'))\n", " return dicom_to_rgb(path)\n", "'''\n", "APP_DIR = ARTIFACTS/'local_demo'\n", "APP_DIR.mkdir(exist_ok=True)\n", "(APP_DIR/'app.py').write_text(APP_SOURCE)\n", "(APP_DIR/'preprocessing.py').write_text(PREPROCESS_SOURCE)\n", "shutil.copy2(BEST,APP_DIR/'best.pt')\n", "(APP_DIR/'config.json').write_text(json.dumps({'imgsz':IMGSZ,'confidence':DEMO_CONF,'names':NAMES},indent=2))\n", "# Pin runtime-resolved direct dependencies for the exported app.\n", "from importlib.metadata import version\n", "packages = ['ultralytics','gradio','pydicom','pylibjpeg','pylibjpeg-libjpeg',\n", " 'pylibjpeg-openjpeg','numpy','pandas','Pillow','torch','torchvision']\n", "(APP_DIR/'requirements.txt').write_text('\\n'.join(f\"{p}=={version(p).split('+')[0]}\" for p in packages)+'\\n')\n", "(APP_DIR/'README.txt').write_text(\n", " 'Input: de-identified single-frame CT DICOM or matching 8-bit brain-windowed PNG/JPEG.\\n'\n", " 'Subdural includes acute and chronic collections. Research use only.\\n'\n", " 'No detection does not exclude hemorrhage. Scores are not calibrated disease probabilities.\\n'\n", " 'IMPORTANT: INSTRUCTIONS TO RUN LOCALLY \\n'\n", " '1. Download app from Google Colab notebook, step 15 (file name: ich_local_demo.zip) \\n'\n", " '2. Extract ZIP (extract all) \\n'\n", " '===> ensure folder contains app.py, preprocessing.py, best.pt, config.json, and requirements.txt. \\n'\n", " '===> note down path (e.g. path: C:\\Users\\hslim\\Downloads\\ich_local_demo) \\n'\n", " '3. Creating environment in Anaconda powershell \\n'\n", " '-- conda create --name ich-demo python=3.12 pip -y \\n'\n", " '-- conda activate ich-demo \\n'\n", " '4. Enter app folder \\n'\n", " '-- cd \"C:\\Users\\hslim\\Downloads\\ich_local_demo\" \\n'\n", " '===> (keep quotation marks, replace content to your folder path) \\n'\n", " '-- dir \\n'\n", " '===>(to check for app.py and requirements.txt) \\n'\n", " '5. Install required packages \\n'\n", " '-- python -m pip install --upgrade pip \\n'\n", " '-- python -m pip install -r requirements.txt \\n'\n", " '6. Check environment \\n'\n", " '-- python -c \"import sys; print(sys.executable)\" \\n'\n", " '===> (should contain envs\\ich-demo\\python.exe) \\n'\n", " '-- python -c \"import torch, gradio, ultralytics, pydicom; print('Imports successful'); print('GPU available:', torch.cuda.is_available())\" \\n'\n", " '7. Starting the app \\n'\n", " '-- python app.py \\n'\n", " '8. Stopping/Restarting app \\n'\n", " '-- (stop) press Ctrl + C \\n'\n", " '-- (start) conda activate ich-demo \\n'\n", " '-- cd /d \"path\" \\n'\n", " '-- python app.py \\n')\n", "APP_ZIP = Path(shutil.make_archive(str(ARTIFACTS/'ich_local_demo'),'zip',APP_DIR))\n", "print('Local app bundle:',APP_ZIP)\n", "files.download(str(APP_ZIP))" ] }, { "cell_type": "markdown", "id": "a58bd48b", "metadata": { "id": "a58bd48b" }, "source": [ "(Optional) 15.1 Preview within Google Colab" ] }, { "cell_type": "code", "execution_count": null, "id": "7722a497", "metadata": { "id": "7722a497" }, "outputs": [], "source": [ "PREVIEW_IN_COLAB = False # Change to True if needed\n", "if PREVIEW_IN_COLAB:\n", " import importlib.util\n", " sys.path.insert(0,str(APP_DIR))\n", " spec = importlib.util.spec_from_file_location('ich_demo',APP_DIR/'app.py')\n", " app_module = importlib.util.module_from_spec(spec)\n", " spec.loader.exec_module(app_module)\n", " app_module.demo.launch(server_name='0.0.0.0',server_port=7860,\n", " share=False,inline=False,prevent_thread_lock=True)\n", " from google.colab import output\n", " output.serve_kernel_port_as_iframe(7860,height=850)" ] }, { "cell_type": "markdown", "id": "8b0245b1", "metadata": { "id": "8b0245b1" }, "source": [ "16. Saving Experiment into ZIP file" ] }, { "cell_type": "code", "execution_count": null, "id": "8b50e97c", "metadata": { "id": "8b50e97c" }, "outputs": [], "source": [ "experiment = {'seed':SEED,'imgsz':IMGSZ,'epochs_requested':EPOCHS,'batch':BATCH,\n", " 'pretrained_url':WEIGHTS_URL,'pretrained_sha256':hashlib.sha256(PRETRAINED.read_bytes()).hexdigest(),\n", " 'annotation_sha256':hashlib.sha256(BHX_CSV.read_bytes()).hexdigest(),\n", " 'data':'CQ500 with BHX v1.1 manual boxes','subdural_policy':'merge Chronic into Subdural',\n", " 'window':{'center_HU':40,'width_HU':80},'class_names':NAMES,\n", " 'best_checkpoint':str(BEST),'demo_confidence':DEMO_CONF}\n", "(ARTIFACTS/'experiment.json').write_text(json.dumps(experiment,indent=2))\n", "# The archive destination is outside ARTIFACTS to avoid including itself.\n", "RESULTS_ZIP = Path(shutil.make_archive(str(ROOT/'ich_experiment'),'zip',ARTIFACTS))\n", "files.download(str(RESULTS_ZIP))" ] } ], "metadata": { "language_info": { "name": "python" }, "colab": { "provenance": [], "gpuType": "T4" }, "accelerator": "GPU", "kernelspec": { "name": "python3", "display_name": "Python 3" } }, "nbformat": 4, "nbformat_minor": 5 }