Buckets:
| """ | |
| dataset.py | |
| PyTorch Dataset and DataLoader module for 10-second multi-channel EEG windows. | |
| Optimized for low-RAM systems using disk memory-mapping (np.memmap / np.load mmap_mode). | |
| """ | |
| import os | |
| import json | |
| import zipfile | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from torch.utils.data import Dataset, DataLoader | |
| class EEGWindowDataset(Dataset): | |
| """ | |
| Dataset class for EEG windows loaded via memory-mapping. | |
| Prevents loading the full 7GB raw float32 array into RAM. | |
| """ | |
| def __init__( | |
| self, | |
| mmap_windows_array, | |
| metadata_df, | |
| partition_indices, | |
| transform=None | |
| ): | |
| self.windows = mmap_windows_array # Memory-mapped array of shape (N, 18, 2560) | |
| self.metadata = metadata_df.iloc[partition_indices].reset_index(drop=True) | |
| self.global_indices = np.array(partition_indices, dtype=np.int64) | |
| self.labels = self.metadata["label"].to_numpy(dtype=np.int64) | |
| self.window_ids = self.metadata["window_id"].to_list() | |
| self.transform = transform | |
| def __len__(self): | |
| return len(self.global_indices) | |
| def __getitem__(self, idx): | |
| global_idx = self.global_indices[idx] | |
| # Access slice from memory-mapped array and copy to avoid keeping page locks | |
| window = np.array(self.windows[global_idx], dtype=np.float32) | |
| label = self.labels[idx] | |
| window_id = self.window_ids[idx] | |
| if self.transform is not None: | |
| window = self.transform(window) | |
| # Return as PyTorch tensor | |
| tensor_x = torch.from_numpy(window) # shape: (18, 2560) | |
| tensor_y = torch.tensor(label, dtype=torch.long) | |
| return tensor_x, tensor_y, window_id | |
| class EEGTrainAugmentation: | |
| """ | |
| On-the-fly temporal and amplitude augmentations for EEG training windows. | |
| Applied exclusively during training to combat inter-subject memorization and overfitting. | |
| """ | |
| def __init__( | |
| self, | |
| p_scale=0.5, | |
| scale_min=0.85, | |
| scale_max=1.15, | |
| p_shift=0.5, | |
| max_shift=128, | |
| p_noise=0.3, | |
| noise_std_ratio=0.02, | |
| p_channel_drop=0.1 | |
| ): | |
| self.p_scale = p_scale | |
| self.scale_min = scale_min | |
| self.scale_max = scale_max | |
| self.p_shift = p_shift | |
| self.max_shift = max_shift | |
| self.p_noise = p_noise | |
| self.noise_std_ratio = noise_std_ratio | |
| self.p_channel_drop = p_channel_drop | |
| def __call__(self, x): | |
| x = x.copy() | |
| # 1. Amplitude scaling (simulates electrode impedance variations across patients) | |
| if self.p_scale > 0 and np.random.rand() < self.p_scale: | |
| scale = np.random.uniform(self.scale_min, self.scale_max) | |
| x = x * scale | |
| # 2. Temporal shift (circular roll up to +/- max_shift samples) | |
| if self.p_shift > 0 and np.random.rand() < self.p_shift: | |
| shift = np.random.randint(-self.max_shift, self.max_shift) | |
| x = np.roll(x, shift, axis=1) | |
| # 3. Additive Gaussian noise (prevents memorizing microscopic noise patterns) | |
| if self.p_noise > 0 and np.random.rand() < self.p_noise: | |
| std = np.std(x, axis=1, keepdims=True) + 1e-8 | |
| noise = np.random.normal(0, self.noise_std_ratio * std, size=x.shape).astype(np.float32) | |
| x = x + noise | |
| # 4. Single channel dropout (encourages spatially distributed representation) | |
| if self.p_channel_drop > 0 and np.random.rand() < self.p_channel_drop: | |
| ch = np.random.randint(0, x.shape[0]) | |
| x[ch, :] = 0.0 | |
| return x | |
| def ensure_mmap_archive_extracted(npz_path, target_dir): | |
| """ | |
| Extracts 'windows.npy' from the NPZ archive to disk if not already extracted. | |
| Uses chunked zip extraction to avoid loading the 7GB array into RAM. | |
| """ | |
| os.makedirs(target_dir, exist_ok=True) | |
| mmap_path = os.path.join(target_dir, "windows.npy") | |
| if os.path.exists(mmap_path): | |
| size_mb = os.path.getsize(mmap_path) / (1024**2) | |
| print(f"Memory-map file already exists: {mmap_path} ({size_mb:.2f} MB)") | |
| return mmap_path | |
| print(f"Extracting 'windows.npy' from {npz_path} for zero-RAM memory-mapping...") | |
| with zipfile.ZipFile(npz_path, "r") as z: | |
| if "windows.npy" not in z.namelist(): | |
| raise FileNotFoundError(f"'windows.npy' member not found inside {npz_path}!") | |
| z.extract("windows.npy", target_dir) | |
| print(f"Extraction complete: {mmap_path} ({os.path.getsize(mmap_path) / (1024**2):.2f} MB)") | |
| return mmap_path | |
| def compute_class_weights(train_labels, method="balanced"): | |
| """ | |
| Compute class weights using ONLY the training partition labels. | |
| Formula: N / (num_classes * count_c) | |
| """ | |
| classes, counts = np.unique(train_labels, return_counts=True) | |
| total_samples = len(train_labels) | |
| num_classes = len(classes) | |
| weights = np.zeros(num_classes, dtype=np.float32) | |
| for c, count in zip(classes, counts): | |
| if count > 0: | |
| weights[c] = total_samples / (num_classes * count) | |
| else: | |
| weights[c] = 1.0 | |
| print(f"Training Class Counts: {dict(zip(classes, counts))}") | |
| print(f"Calculated Class Weights: {weights.tolist()}") | |
| return weights | |
| def get_data_loaders(config): | |
| """ | |
| Factory function to prepare Datasets and DataLoaders for train, val, and test partitions. | |
| """ | |
| data_cfg = config["data"] | |
| train_cfg = config["training"] | |
| hardware_cfg = config.get("hardware", {}) | |
| npz_path = data_cfg["raw_archive_path"] | |
| metadata_path = data_cfg["metadata_path"] | |
| data_dir = os.path.dirname(npz_path) | |
| # 1. Ensure memory-mapped file exists | |
| mmap_path = ensure_mmap_archive_extracted(npz_path, data_dir) | |
| mmap_windows = np.load(mmap_path, mmap_mode="r") | |
| # 2. Load metadata | |
| df_meta = pd.read_csv(metadata_path) | |
| # 3. Determine splits | |
| train_subs = set(data_cfg["splits"]["train_subjects"]) | |
| val_subs = set(data_cfg["splits"]["val_subjects"]) | |
| test_subs = set(data_cfg["splits"]["test_subjects"]) | |
| exclude_ambiguous = data_cfg.get("exclude_ambiguous", True) | |
| def get_partition_indices(subs): | |
| cond = df_meta["subject_id"].isin(subs) | |
| if exclude_ambiguous: | |
| cond = cond & (df_meta["label"].isin([0, 1])) | |
| return df_meta[cond].index.to_numpy() | |
| train_indices = get_partition_indices(train_subs) | |
| val_indices = get_partition_indices(val_subs) | |
| test_indices = get_partition_indices(test_subs) | |
| print(f"Dataset partitions prepared:") | |
| print(f" Train windows (eligible): {len(train_indices)}") | |
| print(f" Val windows (eligible): {len(val_indices)}") | |
| print(f" Test windows (eligible): {len(test_indices)}") | |
| # 4. Create Datasets (with optional training augmentation) | |
| train_transform = None | |
| if train_cfg.get("use_augmentation", False): | |
| train_transform = EEGTrainAugmentation( | |
| p_scale=train_cfg.get("aug_p_scale", 0.5), | |
| scale_min=train_cfg.get("aug_scale_min", 0.85), | |
| scale_max=train_cfg.get("aug_scale_max", 1.15), | |
| p_shift=train_cfg.get("aug_p_shift", 0.5), | |
| max_shift=train_cfg.get("aug_max_shift", 128), | |
| p_noise=train_cfg.get("aug_p_noise", 0.3), | |
| noise_std_ratio=train_cfg.get("aug_noise_ratio", 0.02), | |
| p_channel_drop=train_cfg.get("aug_p_channel_drop", 0.1) | |
| ) | |
| print("EEG training data augmentation ENABLED for training partition.") | |
| train_dataset = EEGWindowDataset(mmap_windows, df_meta, train_indices, transform=train_transform) | |
| val_dataset = EEGWindowDataset(mmap_windows, df_meta, val_indices) | |
| test_dataset = EEGWindowDataset(mmap_windows, df_meta, test_indices) | |
| # 5. Compute Training Class Weights | |
| class_weights = compute_class_weights(train_dataset.labels) | |
| # 6. Create DataLoaders | |
| batch_size = train_cfg.get("batch_size", 64) | |
| num_workers = hardware_cfg.get("num_workers", 0) | |
| pin_memory = hardware_cfg.get("pin_memory", False) and (hardware_cfg.get("device", "cpu") == "cuda") | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=batch_size, | |
| shuffle=True, | |
| num_workers=num_workers, | |
| pin_memory=pin_memory, | |
| drop_last=False | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=batch_size, | |
| shuffle=False, | |
| num_workers=num_workers, | |
| pin_memory=pin_memory, | |
| drop_last=False | |
| ) | |
| test_loader = DataLoader( | |
| test_dataset, | |
| batch_size=batch_size, | |
| shuffle=False, | |
| num_workers=num_workers, | |
| pin_memory=pin_memory, | |
| drop_last=False | |
| ) | |
| return { | |
| "train_loader": train_loader, | |
| "val_loader": val_loader, | |
| "test_loader": test_loader, | |
| "train_dataset": train_dataset, | |
| "val_dataset": val_dataset, | |
| "test_dataset": test_dataset, | |
| "class_weights": class_weights, | |
| "mmap_windows": mmap_windows, | |
| "df_meta": df_meta | |
| } | |
Xet Storage Details
- Size:
- 9.04 kB
- Xet hash:
- 28f0e1b4215e50ee74f8de2a4a6397f5a85d63246c0a4c56afef44d87f198d2e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.