parsa-hke's picture
Stroke Decision Copilot: page, bank of 195 cases (held-out by alias), EXP-66 results (part 2)
713f736 verified
Raw History Blame Contribute Delete
8.89 kB
"""Raw RSNA DICOM -> windowed JPEG tar shards plus a scan-level index (plan §5.2, DEC-17).
Each slice is read once, so the index comes from the same pass as the images. Slices are assigned
to shards by a hash of their ID, so a shard's contents never depend on the order of the input.
Patient-level splits are made later, from the index. JPEG channels are brain, subdural, bone, in
the order cv2 stores them.
"""
import hashlib
import io
import json
import os
import tarfile
import time
from multiprocessing import Pool
import cv2
import numpy as np
import pandas as pd
import pydicom
SLIM_VERSION = "rsna_slim_v1"
WINDOWS = ((40, 80), (80, 200), (600, 2800)) # (centre, width): brain, subdural, bone
LABEL_NAMES = ("any", "epidural", "intraparenchymal", "intraventricular", "subarachnoid", "subdural")
IMAGE_SIZE = 320
JPEG_QUALITY = 95
N_SHARDS = 4
MAX_TOTAL_GB = 19.0 # all shards must fit on Kaggle's local disk during training
INDEX_COLUMNS = ["image_id", "shard", "ok", "error", "patient_id", "study_id", "series_id", "z_mm", "pixel_spacing_mm"]
def find_data(root):
"""Return (labels csv, image folder) under root, without listing the image folders."""
for folder, subfolders, files in os.walk(root):
subfolders[:] = [d for d in subfolders if not d.startswith("stage_2_")]
if "stage_2_train.csv" in files:
return os.path.join(folder, "stage_2_train.csv"), os.path.join(folder, "stage_2_train")
raise FileNotFoundError(f"stage_2_train.csv not found under {root}. Add the competition data with Add Input.")
def load_labels(csv_path):
"""Read stage_2_train.csv into one row per slice, with one column per label."""
rows = pd.read_csv(csv_path).drop_duplicates("ID")
parts = rows["ID"].str.rsplit("_", n=1, expand=True)
rows = rows.assign(image_id=parts[0], label_name=parts[1])
labels = rows.pivot(index="image_id", columns="label_name", values="Label")
missing = set(LABEL_NAMES) - set(labels.columns)
if missing:
raise ValueError(f"labels file is missing {sorted(missing)}")
if labels[list(LABEL_NAMES)].isna().any().any():
raise ValueError("some slices do not have all six labels")
return labels[list(LABEL_NAMES)].astype(int).sort_index().reset_index().rename_axis(columns=None)
def shard_of(image_id, n_shards=N_SHARDS):
return int(hashlib.md5(image_id.encode()).hexdigest(), 16) % n_shards
def window_image(pixels, slope, intercept, size=IMAGE_SIZE):
"""Raw pixels -> size x size x 3 uint8 image, one CT window per channel."""
hu = pixels.astype("float32") * float(slope) + float(intercept)
channels = [np.clip((hu - (centre - width / 2)) / width, 0, 1) for centre, width in WINDOWS]
image = (np.stack(channels, axis=-1) * 255).round().astype("uint8")
return cv2.resize(image, (size, size), interpolation=cv2.INTER_AREA)
def _read_slice(job):
"""Worker: read one DICOM. Returns (image_id, jpeg bytes or None, index fields)."""
image_id, path, size, quality = job
fields = {"ok": False, "error": ""}
try:
ds = pydicom.dcmread(path)
image = window_image(ds.pixel_array, ds.get("RescaleSlope", 1), ds.get("RescaleIntercept", 0), size)
encoded, jpeg = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, quality])
if not encoded:
raise ValueError("JPEG encoding failed")
position = ds.get("ImagePositionPatient")
spacing = ds.get("PixelSpacing")
fields.update(
ok=True,
patient_id=str(ds.get("PatientID", "")),
study_id=str(ds.get("StudyInstanceUID", "")),
series_id=str(ds.get("SeriesInstanceUID", "")),
z_mm=float(position[2]) if position else None,
pixel_spacing_mm=float(spacing[0]) if spacing else None,
)
return image_id, jpeg.tobytes(), fields
except Exception as error:
fields["error"] = f"{type(error).__name__}: {str(error)[:120]}"
return image_id, None, fields
def _init_worker():
# RSNA's de-identified IDs are not valid DICOM UIDs; validating each one floods the log.
pydicom.config.settings.reading_validation_mode = pydicom.config.IGNORE
cv2.setNumThreads(1)
def _write_jpeg(tar, image_id, jpeg):
info = tarfile.TarInfo(f"{image_id}.jpg")
info.size = len(jpeg)
tar.addfile(info, io.BytesIO(jpeg)) # mtime stays 0, so reruns give identical archives
def shard_paths(out_dir, shard, size=IMAGE_SIZE):
stem = f"{SLIM_VERSION}_{size}px_s{shard:02d}"
return os.path.join(out_dir, f"{stem}.tar"), os.path.join(out_dir, f"{stem}_index.csv.gz")
def process_shard(labels, image_dir, out_dir, shard, n_shards=N_SHARDS, size=IMAGE_SIZE, quality=JPEG_QUALITY, workers=None, limit=None, progress_every=20000):
"""Write one shard's tar and index, and return a report.
limit processes only the first slices, for a quick size check. The tar keeps a .part name until
it is complete, so an interrupted run never leaves a finished-looking file.
"""
selected = labels[labels["image_id"].map(lambda i: shard_of(i, n_shards)) == shard]
if selected.empty:
raise ValueError(f"shard {shard} has no slices")
complete = limit is None or limit >= len(selected)
selected = selected.iloc[:limit] if limit else selected
os.makedirs(out_dir, exist_ok=True)
tar_path, index_path = shard_paths(out_dir, shard, size)
jobs = [(i, os.path.join(image_dir, f"{i}.dcm"), size, quality) for i in selected["image_id"]]
workers = workers if workers is not None else (os.cpu_count() or 2)
pool = Pool(workers, initializer=_init_worker) if workers > 0 else None
if pool is None:
_init_worker()
results = pool.imap(_read_slice, jobs, chunksize=16) if pool else map(_read_slice, jobs)
start, jpeg_bytes, rows = time.perf_counter(), 0, []
part_path = f"{tar_path}.part"
try:
with tarfile.open(part_path, "w") as tar:
for done, (image_id, jpeg, fields) in enumerate(results, 1):
rows.append({"image_id": image_id, "shard": shard, **fields})
if jpeg is not None:
_write_jpeg(tar, image_id, jpeg)
jpeg_bytes += len(jpeg)
if done % progress_every == 0:
seconds = time.perf_counter() - start
print(f"{done:,} / {len(jobs):,} slices, {done / seconds:.0f} per second, {jpeg_bytes / 1024 ** 3:.2f} GB")
finally:
if pool:
pool.close()
pool.join()
os.replace(part_path, tar_path)
index = pd.DataFrame(rows).reindex(columns=INDEX_COLUMNS).merge(selected, on="image_id", validate="one_to_one")
index.to_csv(index_path, index=False)
n_ok = int(index["ok"].sum())
seconds = time.perf_counter() - start
report = {
"version": SLIM_VERSION, "shard": shard, "n_shards": n_shards, "image_size": size, "jpeg_quality": quality,
"slices_in_labels": len(labels), "slices_processed": len(jobs), "slices_ok": n_ok, "slices_failed": len(jobs) - n_ok,
"errors": index.loc[~index["ok"], "error"].str.split(":").str[0].value_counts().to_dict(),
"complete_shard": complete,
"seconds": round(seconds, 1), "slices_per_second": round(len(jobs) / seconds, 1),
"tar_gb": round(os.path.getsize(tar_path) / 1024 ** 3, 3),
"projected_total_gb": round(jpeg_bytes / max(n_ok, 1) * len(labels) / 1024 ** 3, 2),
"index_file": os.path.basename(index_path),
}
return report
def verify_shard(tar_path, index_path, size=IMAGE_SIZE, sample=200):
"""Check a shard against its index, and time reading it straight through. Raises if they differ."""
index = pd.read_csv(index_path, keep_default_na=False)
expected = {f"{i}.jpg" for i in index.loc[index["ok"].astype(str) == "True", "image_id"]}
start, found, decoded = time.perf_counter(), set(), 0
with tarfile.open(tar_path) as tar:
for member in tar:
data = tar.extractfile(member).read()
found.add(member.name)
if decoded < sample:
image = cv2.imdecode(np.frombuffer(data, "uint8"), cv2.IMREAD_COLOR)
if image is None or image.shape != (size, size, 3):
raise ValueError(f"bad image in shard: {member.name}")
decoded += 1
if found != expected:
raise ValueError(f"tar and index differ: {len(found - expected)} extra, {len(expected - found)} missing")
seconds = time.perf_counter() - start
return {"images": len(found), "decoded_sample": decoded, "read_images_per_second": round(len(found) / max(seconds, 1e-6), 1)}
def save_report(report, out_dir, shard):
path = os.path.join(out_dir, f"{SLIM_VERSION}_s{shard:02d}_report.json")
with open(path, "w") as f:
json.dump(report, f, indent=2)
return path