Spaces:
Running
Running
File size: 8,411 Bytes
713f736 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | """RSNA ICH shard reader and CT windowing, for Kaggle notebooks (plan §5.2, DEC-17).
The index holds slice, patient and study IDs, so nothing here prints them, and none of this data may
leave Kaggle (docs/DATA_USE.md). Image channels are brain, subdural, bone, as `prep_rsna` stored them.
Kaggle unpacks archives when a dataset is made, so an attached shard can be a folder of JPEGs and a
plain CSV instead of a tar and a .csv.gz. First reads from such a folder run at about 25 slices per
second, so `pack_shard` stores a shard as one tar named .shard, which Kaggle should leave alone. All
layouts are read the same way, and a single file wins if both are attached.
"""
import os
import random
import re
import shutil
import subprocess
import tarfile
import time
from concurrent.futures import ThreadPoolExecutor
from multiprocessing.pool import ThreadPool
import cv2
import numpy as np
import pandas as pd
from strokecopilot.data.prep_rsna import IMAGE_SIZE, LABEL_NAMES, N_SHARDS, SLIM_VERSION, _write_jpeg, window_image
DATA_RE = re.compile(rf"^{SLIM_VERSION}_\d+px_s(\d+)(\.tar|\.shard)?$")
INDEX_RE = re.compile(rf"^{SLIM_VERSION}_\d+px_s(\d+)_index\.csv(\.gz)?$")
def find_shards(root, n_shards=N_SHARDS):
"""{shard: (file or folder path, index path)} under root, such as /kaggle/input. Raises if any is missing."""
single_files, folders, indexes = {}, {}, {}
for folder, subfolders, files in os.walk(root):
for name in files:
if (match := DATA_RE.match(name)) and match.group(2):
single_files[int(match.group(1))] = os.path.join(folder, name)
elif match := INDEX_RE.match(name):
indexes[int(match.group(1))] = os.path.join(folder, name)
for name in subfolders:
if (match := DATA_RE.match(name)) and not match.group(2):
folders[int(match.group(1))] = os.path.join(folder, name)
subfolders[:] = [d for d in subfolders if not DATA_RE.match(d)] # 188,000 JPEGs need no listing here
data = {**folders, **single_files}
complete = {shard: (data[shard], indexes[shard]) for shard in data if shard in indexes}
missing = sorted(set(range(n_shards)) - set(complete))
if missing:
raise FileNotFoundError(f"Shards {missing} lack images or an index under {root}. Attach every rsna-slim-v1 dataset with Add Input.")
return dict(sorted(complete.items()))
def load_index(shards):
"""One row per readable slice across shards. Failed slices (`ok` false) are dropped."""
frames = [pd.read_csv(index_path, low_memory=False) for _, index_path in shards.values()]
index = pd.concat(frames, ignore_index=True)
return index[index["ok"].astype(str) == "True"].reset_index(drop=True)
def scan_table(index):
"""One row per scan (DICOM series): patient, slice count and a label per type, positive if any slice is."""
labels = index.groupby("series_id")[list(LABEL_NAMES)].max()
info = index.groupby("series_id").agg(patient_id=("patient_id", "first"), slices=("image_id", "size"))
return info.join(labels).reset_index()
def slices_of_scan(index, series_id):
"""A scan's rows, ordered by position along the body. Spacing comes from z_mm (plan §5.2)."""
return index[index["series_id"] == series_id].sort_values("z_mm")
class ShardReader:
"""Random access to one uncompressed tar. Reads are thread-safe, because each states its offset.
Listing a tar reads every member's header, which may be very slow on an attached dataset, so list
a local copy and pass its offsets to read the same file elsewhere.
"""
def __init__(self, tar_path, offsets=None):
self.path = tar_path
self.offsets = offsets if offsets is not None else self._list(tar_path)
self._fd = os.open(tar_path, os.O_RDONLY)
@staticmethod
def _list(tar_path):
with tarfile.open(tar_path) as tar:
return {
os.path.basename(m.name)[: -len(".jpg")]: (m.offset_data, m.size)
for m in tar
if m.isfile() and m.name.endswith(".jpg") and not os.path.basename(m.name).startswith("._") # macOS metadata
}
def __getstate__(self): # a worker process opens its own file handle
return {"path": self.path, "offsets": self.offsets}
def __setstate__(self, state):
self.__init__(state["path"], state["offsets"])
def __contains__(self, image_id):
return image_id in self.offsets
def __len__(self):
return len(self.offsets)
def ids(self):
return set(self.offsets)
def read_bytes(self, image_id):
offset, size = self.offsets[image_id]
return os.pread(self._fd, size, offset)
def read(self, image_id):
return decode(self.read_bytes(image_id))
def close(self):
os.close(self._fd)
class FolderReader:
"""The same reads from a shard Kaggle unpacked into a folder of JPEGs."""
def __init__(self, folder):
self.path = folder
self.paths = {}
for parent, _, files in os.walk(folder):
for name in files:
if name.endswith(".jpg"):
self.paths[name[: -len(".jpg")]] = os.path.join(parent, name)
def __contains__(self, image_id):
return image_id in self.paths
def __len__(self):
return len(self.paths)
def ids(self):
return set(self.paths)
def read_bytes(self, image_id):
with open(self.paths[image_id], "rb") as f:
return f.read()
def read(self, image_id):
return decode(self.read_bytes(image_id))
def close(self):
pass
def open_shard(path):
return FolderReader(path) if os.path.isdir(path) else ShardReader(path)
def decode(jpeg):
"""JPEG bytes -> height x width x 3 uint8 array: brain, subdural, bone."""
image = cv2.imdecode(np.frombuffer(jpeg, "uint8"), cv2.IMREAD_COLOR)
if image is None:
raise ValueError("Could not decode a slice.")
return image
def window_hu(hu, size=IMAGE_SIZE):
"""HU slice -> the same three windows as training, for running on ISLES'24 NCCT (plan §8.2)."""
return window_image(hu, 1, 0, size)
def copy_shard(path, dest_dir):
"""Put a shard on local disk as one tar, and return the new path and seconds taken. A folder is packed."""
os.makedirs(dest_dir, exist_ok=True)
started = time.perf_counter()
dest = os.path.join(dest_dir, f"{os.path.basename(path).removesuffix('.tar').removesuffix('.shard')}.tar")
if os.path.isdir(path):
subprocess.run(["tar", "-cf", dest, "-C", path, "."], check=True)
else:
shutil.copyfile(path, dest)
return dest, time.perf_counter() - started
def measure_read(reader, n=2000, threads=1, seed=0):
"""Random reads and decodes per second. Returns numbers only."""
ids = random.Random(seed).sample(sorted(reader.ids()), min(n, len(reader)))
started = time.perf_counter()
if threads == 1:
for image_id in ids:
reader.read(image_id)
else:
with ThreadPoolExecutor(threads) as pool:
list(pool.map(reader.read, ids))
seconds = time.perf_counter() - started
return {"slices": len(ids), "threads": threads, "slices_per_second": round(len(ids) / max(seconds, 1e-6), 1)}
def pack_shard(folder, out_path, threads=16, every=20000):
"""Store a shard's folder as one tar, reading with threads. Returns numbers only.
Reads may finish out of order, but they are written in sorted order, so reruns give identical
files. The output keeps a .part name until it is complete.
"""
reader = FolderReader(folder)
ids = sorted(reader.ids())
part = f"{out_path}.part"
started = time.perf_counter()
with ThreadPool(threads) as pool, tarfile.open(part, "w") as tar:
for done, (image_id, jpeg) in enumerate(zip(ids, pool.imap(reader.read_bytes, ids, chunksize=32)), 1):
_write_jpeg(tar, image_id, jpeg)
if done % every == 0:
print(f"{done:,} of {len(ids):,} slices, {done / (time.perf_counter() - started):.0f} per second", flush=True)
os.replace(part, out_path)
seconds = time.perf_counter() - started
return {"slices": len(ids), "threads": threads, "seconds": round(seconds), "slices_per_second": round(len(ids) / max(seconds, 1e-6), 1), "gib": round(os.path.getsize(out_path) / 1024 ** 3, 3)}
|