omkorde7470's picture
download
raw
9.04 kB
"""
dataset.py
PyTorch Dataset and DataLoader module for 10-second multi-channel EEG windows.
Optimized for low-RAM systems using disk memory-mapping (np.memmap / np.load mmap_mode).
"""
import os
import json
import zipfile
import numpy as np
import pandas as pd
import torch
from torch.utils.data import Dataset, DataLoader
class EEGWindowDataset(Dataset):
"""
Dataset class for EEG windows loaded via memory-mapping.
Prevents loading the full 7GB raw float32 array into RAM.
"""
def __init__(
self,
mmap_windows_array,
metadata_df,
partition_indices,
transform=None
):
self.windows = mmap_windows_array # Memory-mapped array of shape (N, 18, 2560)
self.metadata = metadata_df.iloc[partition_indices].reset_index(drop=True)
self.global_indices = np.array(partition_indices, dtype=np.int64)
self.labels = self.metadata["label"].to_numpy(dtype=np.int64)
self.window_ids = self.metadata["window_id"].to_list()
self.transform = transform
def __len__(self):
return len(self.global_indices)
def __getitem__(self, idx):
global_idx = self.global_indices[idx]
# Access slice from memory-mapped array and copy to avoid keeping page locks
window = np.array(self.windows[global_idx], dtype=np.float32)
label = self.labels[idx]
window_id = self.window_ids[idx]
if self.transform is not None:
window = self.transform(window)
# Return as PyTorch tensor
tensor_x = torch.from_numpy(window) # shape: (18, 2560)
tensor_y = torch.tensor(label, dtype=torch.long)
return tensor_x, tensor_y, window_id
class EEGTrainAugmentation:
"""
On-the-fly temporal and amplitude augmentations for EEG training windows.
Applied exclusively during training to combat inter-subject memorization and overfitting.
"""
def __init__(
self,
p_scale=0.5,
scale_min=0.85,
scale_max=1.15,
p_shift=0.5,
max_shift=128,
p_noise=0.3,
noise_std_ratio=0.02,
p_channel_drop=0.1
):
self.p_scale = p_scale
self.scale_min = scale_min
self.scale_max = scale_max
self.p_shift = p_shift
self.max_shift = max_shift
self.p_noise = p_noise
self.noise_std_ratio = noise_std_ratio
self.p_channel_drop = p_channel_drop
def __call__(self, x):
x = x.copy()
# 1. Amplitude scaling (simulates electrode impedance variations across patients)
if self.p_scale > 0 and np.random.rand() < self.p_scale:
scale = np.random.uniform(self.scale_min, self.scale_max)
x = x * scale
# 2. Temporal shift (circular roll up to +/- max_shift samples)
if self.p_shift > 0 and np.random.rand() < self.p_shift:
shift = np.random.randint(-self.max_shift, self.max_shift)
x = np.roll(x, shift, axis=1)
# 3. Additive Gaussian noise (prevents memorizing microscopic noise patterns)
if self.p_noise > 0 and np.random.rand() < self.p_noise:
std = np.std(x, axis=1, keepdims=True) + 1e-8
noise = np.random.normal(0, self.noise_std_ratio * std, size=x.shape).astype(np.float32)
x = x + noise
# 4. Single channel dropout (encourages spatially distributed representation)
if self.p_channel_drop > 0 and np.random.rand() < self.p_channel_drop:
ch = np.random.randint(0, x.shape[0])
x[ch, :] = 0.0
return x
def ensure_mmap_archive_extracted(npz_path, target_dir):
"""
Extracts 'windows.npy' from the NPZ archive to disk if not already extracted.
Uses chunked zip extraction to avoid loading the 7GB array into RAM.
"""
os.makedirs(target_dir, exist_ok=True)
mmap_path = os.path.join(target_dir, "windows.npy")
if os.path.exists(mmap_path):
size_mb = os.path.getsize(mmap_path) / (1024**2)
print(f"Memory-map file already exists: {mmap_path} ({size_mb:.2f} MB)")
return mmap_path
print(f"Extracting 'windows.npy' from {npz_path} for zero-RAM memory-mapping...")
with zipfile.ZipFile(npz_path, "r") as z:
if "windows.npy" not in z.namelist():
raise FileNotFoundError(f"'windows.npy' member not found inside {npz_path}!")
z.extract("windows.npy", target_dir)
print(f"Extraction complete: {mmap_path} ({os.path.getsize(mmap_path) / (1024**2):.2f} MB)")
return mmap_path
def compute_class_weights(train_labels, method="balanced"):
"""
Compute class weights using ONLY the training partition labels.
Formula: N / (num_classes * count_c)
"""
classes, counts = np.unique(train_labels, return_counts=True)
total_samples = len(train_labels)
num_classes = len(classes)
weights = np.zeros(num_classes, dtype=np.float32)
for c, count in zip(classes, counts):
if count > 0:
weights[c] = total_samples / (num_classes * count)
else:
weights[c] = 1.0
print(f"Training Class Counts: {dict(zip(classes, counts))}")
print(f"Calculated Class Weights: {weights.tolist()}")
return weights
def get_data_loaders(config):
"""
Factory function to prepare Datasets and DataLoaders for train, val, and test partitions.
"""
data_cfg = config["data"]
train_cfg = config["training"]
hardware_cfg = config.get("hardware", {})
npz_path = data_cfg["raw_archive_path"]
metadata_path = data_cfg["metadata_path"]
data_dir = os.path.dirname(npz_path)
# 1. Ensure memory-mapped file exists
mmap_path = ensure_mmap_archive_extracted(npz_path, data_dir)
mmap_windows = np.load(mmap_path, mmap_mode="r")
# 2. Load metadata
df_meta = pd.read_csv(metadata_path)
# 3. Determine splits
train_subs = set(data_cfg["splits"]["train_subjects"])
val_subs = set(data_cfg["splits"]["val_subjects"])
test_subs = set(data_cfg["splits"]["test_subjects"])
exclude_ambiguous = data_cfg.get("exclude_ambiguous", True)
def get_partition_indices(subs):
cond = df_meta["subject_id"].isin(subs)
if exclude_ambiguous:
cond = cond & (df_meta["label"].isin([0, 1]))
return df_meta[cond].index.to_numpy()
train_indices = get_partition_indices(train_subs)
val_indices = get_partition_indices(val_subs)
test_indices = get_partition_indices(test_subs)
print(f"Dataset partitions prepared:")
print(f" Train windows (eligible): {len(train_indices)}")
print(f" Val windows (eligible): {len(val_indices)}")
print(f" Test windows (eligible): {len(test_indices)}")
# 4. Create Datasets (with optional training augmentation)
train_transform = None
if train_cfg.get("use_augmentation", False):
train_transform = EEGTrainAugmentation(
p_scale=train_cfg.get("aug_p_scale", 0.5),
scale_min=train_cfg.get("aug_scale_min", 0.85),
scale_max=train_cfg.get("aug_scale_max", 1.15),
p_shift=train_cfg.get("aug_p_shift", 0.5),
max_shift=train_cfg.get("aug_max_shift", 128),
p_noise=train_cfg.get("aug_p_noise", 0.3),
noise_std_ratio=train_cfg.get("aug_noise_ratio", 0.02),
p_channel_drop=train_cfg.get("aug_p_channel_drop", 0.1)
)
print("EEG training data augmentation ENABLED for training partition.")
train_dataset = EEGWindowDataset(mmap_windows, df_meta, train_indices, transform=train_transform)
val_dataset = EEGWindowDataset(mmap_windows, df_meta, val_indices)
test_dataset = EEGWindowDataset(mmap_windows, df_meta, test_indices)
# 5. Compute Training Class Weights
class_weights = compute_class_weights(train_dataset.labels)
# 6. Create DataLoaders
batch_size = train_cfg.get("batch_size", 64)
num_workers = hardware_cfg.get("num_workers", 0)
pin_memory = hardware_cfg.get("pin_memory", False) and (hardware_cfg.get("device", "cpu") == "cuda")
train_loader = DataLoader(
train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
pin_memory=pin_memory,
drop_last=False
)
val_loader = DataLoader(
val_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=pin_memory,
drop_last=False
)
test_loader = DataLoader(
test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=pin_memory,
drop_last=False
)
return {
"train_loader": train_loader,
"val_loader": val_loader,
"test_loader": test_loader,
"train_dataset": train_dataset,
"val_dataset": val_dataset,
"test_dataset": test_dataset,
"class_weights": class_weights,
"mmap_windows": mmap_windows,
"df_meta": df_meta
}

Xet Storage Details

Size:
9.04 kB
·
Xet hash:
28f0e1b4215e50ee74f8de2a4a6397f5a85d63246c0a4c56afef44d87f198d2e

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.