RareLungNoduleFL / postprocessing /postproccessing_and_initial_training.py
hephaestusonfire's picture
Upload 3 files
d730046 verified
Raw History Blame Contribute Delete
59.6 kB
# -*- coding: utf-8 -*- """lidc_step2_step3_step4.ipynb Automatically generated by Colab. Original file is located at https://colab.research.google.com/drive/19ND1Fka8Cromhk7B_hYQzGWayIbTl2eH """ # -*- coding: utf-8 -*- """ Step 2 & 3: PyTorch Dataset Loader with Data Augmentation + FL Client Splits CORRECTED VERSION - With Excel-based nodule size classification """ # ============================================================================ # STEP 0: MOUNT GOOGLE DRIVE # ============================================================================ from google.colab import drive drive.mount('/content/drive', force_remount=True) print("βœ… Google Drive mounted successfully!") print("πŸ“ Your files are accessible at: /content/drive/MyDrive/") # ============================================================================ # INSTALL CORRECT DEPENDENCIES # ============================================================================ print("πŸ“¦ Installing compatible dependencies...") # Uninstall any existing conflicting versions !pip uninstall -y torch torchvision torchaudio monai -q # Install PyTorch 2.6.0 with CUDA 12.1 (Colab default) !pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu121 -q # Install MONAI 1.5.0 (supports PyTorch 2.6.0) !pip install monai==1.5.0 -q # Install other dependencies (including openpyxl for Excel reading) !pip install numpy==1.26.4 pandas==2.2.0 scikit-learn==1.4.0 openpyxl==3.1.2 -q print("βœ… Dependencies installed!") print("⚠️ IMPORTANT: Click 'Runtime' β†’ 'Restart runtime' before proceeding!") # ============================================================================ # VERIFY INSTALLATIONS (Run after restart) # ============================================================================ import torch import monai import numpy as np import pandas as pd from sklearn import __version__ as sklearn_version print("="*70) print("DEPENDENCY VERIFICATION") print("="*70) print(f"βœ… PyTorch version: {torch.__version__}") print(f"βœ… MONAI version: {monai.__version__}") print(f"βœ… NumPy version: {np.__version__}") print(f"βœ… Pandas version: {pd.__version__}") print(f"βœ… Scikit-learn version: {sklearn_version}") print(f"βœ… CUDA available: {torch.cuda.is_available()}") if torch.cuda.is_available(): print(f"βœ… CUDA version: {torch.version.cuda}") print(f"βœ… GPU: {torch.cuda.get_device_name(0)}") print("="*70) # ============================================================================ # IMPORTS # ============================================================================ from pathlib import Path from typing import Dict, List, Tuple, Optional from torch.utils.data import Dataset, DataLoader from sklearn.model_selection import train_test_split import warnings warnings.filterwarnings('ignore') # MONAI transforms for medical imaging from monai.transforms import ( Compose, RandFlipd, RandRotate90d, RandAffined, RandAdjustContrastd, RandGaussianNoised, ToTensord, ScaleIntensityd, ) print("βœ… All imports successful!") # ============================================================================ # LOAD NODULE SIZE INFORMATION FROM EXCEL # ============================================================================ def load_nodule_sizes(excel_path: str) -> Dict[str, float]: """ Load nodule size information from LIDC-IDRI Excel file. Args: excel_path: Path to lidc-idri-nodule-counts-6-23-2015.xlsx Returns: Dictionary mapping (patient_id, nodule_id) -> diameter_mm """ print(f"\nπŸ“Š Loading nodule size data from Excel...") print(f" Path: {excel_path}") try: # Read Excel file df_excel = pd.read_excel(excel_path) print(f" βœ… Loaded Excel with shape: {df_excel.shape}") print(f" πŸ“‹ Columns: {list(df_excel.columns)}") # Create mapping dictionary # Adjust column names based on actual Excel structure nodule_sizes = {} # Common column name variations in LIDC-IDRI files patient_col = None nodule_col = None diameter_col = None # Find correct column names (case-insensitive search) for col in df_excel.columns: col_lower = str(col).lower() if 'patient' in col_lower or 'pid' in col_lower or 'subject' in col_lower: patient_col = col elif 'nodule' in col_lower and 'no' in col_lower: nodule_col = col elif 'diameter' in col_lower or 'size' in col_lower: diameter_col = col if patient_col and nodule_col and diameter_col: print(f" πŸ” Found columns: Patient='{patient_col}', Nodule='{nodule_col}', Diameter='{diameter_col}'") for _, row in df_excel.iterrows(): patient_id = str(row[patient_col]) nodule_id = f"nodule-{int(row[nodule_col])}" diameter = float(row[diameter_col]) # Ensure patient_id format matches (LIDC-IDRI-XXXX) if not patient_id.startswith('LIDC-IDRI'): patient_id = f"LIDC-IDRI-{patient_id.zfill(4)}" key = (patient_id, nodule_id) nodule_sizes[key] = diameter print(f" βœ… Loaded {len(nodule_sizes)} nodule size entries") # Statistics diameters = list(nodule_sizes.values()) small_count = sum(1 for d in diameters if d < 10.0) large_count = sum(1 for d in diameters if d >= 10.0) print(f" πŸ“ˆ Statistics:") print(f" β€’ Small nodules (<10mm): {small_count} ({small_count/len(diameters)*100:.1f}%)") print(f" β€’ Large nodules (β‰₯10mm): {large_count} ({large_count/len(diameters)*100:.1f}%)") print(f" β€’ Mean diameter: {np.mean(diameters):.2f}mm") print(f" β€’ Median diameter: {np.median(diameters):.2f}mm") print(f" β€’ Min diameter: {np.min(diameters):.2f}mm") print(f" β€’ Max diameter: {np.max(diameters):.2f}mm") else: print(f" ⚠️ Could not find standard columns. Available columns:") print(f" {list(df_excel.columns)}") print(f" ℹ️ Please check the Excel file structure.") # Show first few rows print(f"\n First 5 rows:") print(df_excel.head()) return nodule_sizes except Exception as e: print(f" ⚠️ Error loading Excel: {e}") print(f" ℹ️ Will use default classification (all nodules as small)") return {} # ============================================================================ # STEP 2: CUSTOM PYTORCH DATASET CLASS # ============================================================================ class LIDCNoduleDataset(Dataset): """ PyTorch Dataset for LIDC-IDRI lung nodule NPY files with augmentation. Features: - Loads preprocessed NPY files (images + masks) - Applies medical imaging augmentations (flip, rotate, scale, noise) - Supports heterogeneous augmentations per client - Implements rare nodule boosting """ def __init__( self, data_df: pd.DataFrame, data_dir: Path, transform: Optional[Compose] = None, is_train: bool = True, augment_rare: bool = False, client_noise_sigma: float = 15.0, # Poisson noise Οƒ=15HU ): """ Args: data_df: DataFrame with columns ['fname', 'patient_id', 'nodule_id', 'is_small', 'label'] data_dir: Path to directory containing NPY files transform: MONAI Compose transform pipeline is_train: If True, apply augmentations augment_rare: If True, oversample small nodules (<10mm) client_noise_sigma: Heterogeneous noise level per client """ self.data_dir = Path(data_dir) self.is_train = is_train self.client_noise_sigma = client_noise_sigma # Apply rare nodule boosting (duplicate small nodules x3) if augment_rare and is_train: rare_nodules = data_df[data_df['is_small'] == 1] # Duplicate rare samples 2 additional times (3x total) data_df = pd.concat([data_df, rare_nodules, rare_nodules], ignore_index=True) print(f"βœ… Rare nodule boosting: {len(rare_nodules)} small nodules duplicated 3x") self.data_df = data_df.reset_index(drop=True) # Set up transforms if transform is None: self.transform = self._get_default_transforms(is_train) else: self.transform = transform print(f"πŸ“Š Dataset initialized: {len(self.data_df)} samples") def _get_default_transforms(self, is_train: bool) -> Compose: """ Create default MONAI transform pipeline. Augmentations: flip/rotate/scale + Poisson noise Οƒ=15HU + contrast Β±15% """ if is_train: # Training augmentations transforms = [ # Geometric augmentations RandFlipd(keys=["image", "mask"], prob=0.5, spatial_axis=0), RandFlipd(keys=["image", "mask"], prob=0.5, spatial_axis=1), RandRotate90d(keys=["image", "mask"], prob=0.5, max_k=3), RandAffined( keys=["image", "mask"], prob=0.5, rotate_range=(np.pi / 18, np.pi / 18), # Β±10 degrees scale_range=(0.1, 0.1), # Β±10% scale mode=["bilinear", "nearest"], padding_mode="border", ), # Intensity augmentations (heterogeneous per client) RandGaussianNoised( keys=["image"], prob=0.5, mean=0.0, std=self.client_noise_sigma / 255.0, # Normalize to [0,1] range ), RandAdjustContrastd( keys=["image"], prob=0.5, gamma=(0.85, 1.15), # Β±15% contrast ), # Ensure intensity range [0, 1] ScaleIntensityd(keys=["image"], minv=0.0, maxv=1.0), # Convert to PyTorch tensors ToTensord(keys=["image", "mask"]), ] else: # Validation/test: no augmentation transforms = [ ScaleIntensityd(keys=["image"], minv=0.0, maxv=1.0), ToTensord(keys=["image", "mask"]), ] return Compose(transforms) def __len__(self) -> int: return len(self.data_df) def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]: """ Load and return a single sample. Returns: Dictionary with keys: 'image', 'mask', 'label', 'is_small', 'fname' """ row = self.data_df.iloc[idx] fname = row['fname'] # Load NPY files (matching your preprocessing output format) img_path = self.data_dir / f"{fname}_img.npy" mask_path = self.data_dir / f"{fname}_mask.npy" try: image = np.load(img_path).astype(np.float32) # Shape: (1, H, W) mask = np.load(mask_path).astype(np.float32) # Shape: (1, H, W) except Exception as e: print(f"⚠️ Error loading {fname}: {e}") # Return empty tensors as fallback image = np.zeros((1, 256, 256), dtype=np.float32) mask = np.zeros((1, 256, 256), dtype=np.float32) # Prepare data dict for MONAI transforms data_dict = { "image": image, "mask": mask, } # Apply transforms if self.transform: data_dict = self.transform(data_dict) # Add metadata data_dict["label"] = torch.tensor(row['label'], dtype=torch.long) data_dict["is_small"] = torch.tensor(row['is_small'], dtype=torch.float32) data_dict["fname"] = fname return data_dict # ============================================================================ # STEP 3: DATA SPLITTING FOR FEDERATED LEARNING # ============================================================================ class FederatedDataSplitter: """ Split preprocessed LIDC-IDRI data for federated learning experiments. Supports: - IID: Random stratified split across clients - Non-IID: Skewed by nodule size (small vs large) - Heterogeneous: Different noise levels per client """ def __init__( self, csv_path: str, data_dir: str, excel_path: str, num_clients: int = 5, test_size: float = 0.2, random_seed: int = 42, ): """ Args: csv_path: Path to dataset_ready.csv from preprocessing data_dir: Path to directory with NPY files excel_path: Path to LIDC-IDRI Excel file with nodule sizes num_clients: Number of federated clients (default 5) test_size: Fraction of data for global test set (default 0.2) random_seed: Random seed for reproducibility """ self.csv_path = Path(csv_path) self.data_dir = Path(data_dir) self.num_clients = num_clients self.test_size = test_size self.random_seed = random_seed # Load nodule sizes from Excel self.nodule_sizes = load_nodule_sizes(excel_path) # Load dataset print(f"\nπŸ“‚ Loading CSV from: {csv_path}") self.df = pd.read_csv(csv_path) print(f"πŸ“Š Loaded dataset: {len(self.df)} samples from {self.df['patient_id'].nunique()} patients") # Update is_small column based on Excel data self._update_nodule_sizes() def _update_nodule_sizes(self): """ Update is_small column in dataframe based on Excel nodule sizes. Small nodules: diameter < 10mm Large nodules: diameter >= 10mm """ print(f"\nπŸ”„ Updating nodule size classifications...") if not self.nodule_sizes: print(f" ⚠️ No Excel data available. Using default (all as small nodules)") self.df['is_small'] = 1 return updated_count = 0 small_count = 0 large_count = 0 for idx, row in self.df.iterrows(): patient_id = row['patient_id'] nodule_id = row['nodule_id'] key = (patient_id, nodule_id) if key in self.nodule_sizes: diameter = self.nodule_sizes[key] is_small = 1 if diameter < 10.0 else 0 self.df.at[idx, 'is_small'] = is_small updated_count += 1 if is_small: small_count += 1 else: large_count += 1 print(f" βœ… Updated {updated_count}/{len(self.df)} samples") print(f" πŸ“Š Classification results:") print(f" β€’ Small nodules (<10mm): {small_count} ({small_count/len(self.df)*100:.1f}%)") print(f" β€’ Large nodules (β‰₯10mm): {large_count} ({large_count/len(self.df)*100:.1f}%)") # Save updated CSV updated_csv_path = self.csv_path.parent / "dataset_ready_with_sizes.csv" self.df.to_csv(updated_csv_path, index=False) print(f" πŸ’Ύ Saved updated CSV to: {updated_csv_path}") def split_iid(self) -> Tuple[List[pd.DataFrame], pd.DataFrame]: """ IID split: Random stratified distribution across clients. Returns: Tuple of (client_dfs, test_df) """ print("\nπŸ”„ Creating IID split...") # Split train/test globally (stratified by patient) unique_patients = self.df['patient_id'].unique() train_patients, test_patients = train_test_split( unique_patients, test_size=self.test_size, random_state=self.random_seed, ) train_df = self.df[self.df['patient_id'].isin(train_patients)].copy() test_df = self.df[self.df['patient_id'].isin(test_patients)].copy() print(f" Train: {len(train_df)} samples ({len(train_patients)} patients)") print(f" Test: {len(test_df)} samples ({len(test_patients)} patients)") # Split train data equally across clients (random) train_df = train_df.sample(frac=1, random_state=self.random_seed).reset_index(drop=True) client_dfs = [] samples_per_client = len(train_df) // self.num_clients for i in range(self.num_clients): start_idx = i * samples_per_client if i == self.num_clients - 1: # Last client gets remaining samples client_df = train_df.iloc[start_idx:].copy() else: client_df = train_df.iloc[start_idx:start_idx + samples_per_client].copy() client_dfs.append(client_df) small_ratio = (client_df['is_small'] == 1).sum() / len(client_df) * 100 print(f" Client {i}: {len(client_df)} samples ({small_ratio:.1f}% small)") return client_dfs, test_df def split_non_iid(self, alpha: float = 0.5) -> Tuple[List[pd.DataFrame], pd.DataFrame]: """ Non-IID split: Skewed by nodule size (small vs large). Each client has different distribution of small/large nodules. Args: alpha: Dirichlet concentration parameter (lower = more skewed) 0.1 = highly non-IID, 1.0 = moderately non-IID Returns: Tuple of (client_dfs, test_df) """ print(f"\nπŸ”„ Creating Non-IID split (Ξ±={alpha})...") # Split train/test globally unique_patients = self.df['patient_id'].unique() train_patients, test_patients = train_test_split( unique_patients, test_size=self.test_size, random_state=self.random_seed, ) train_df = self.df[self.df['patient_id'].isin(train_patients)].copy() test_df = self.df[self.df['patient_id'].isin(test_patients)].copy() print(f" Train: {len(train_df)} samples ({len(train_patients)} patients)") print(f" Test: {len(test_df)} samples ({len(test_patients)} patients)") # Separate by nodule size small_nodules = train_df[train_df['is_small'] == 1].reset_index(drop=True) large_nodules = train_df[train_df['is_small'] == 0].reset_index(drop=True) print(f" Small nodules (<10mm): {len(small_nodules)}") print(f" Large nodules (β‰₯10mm): {len(large_nodules)}") # Use Dirichlet distribution for non-IID split np.random.seed(self.random_seed) # Proportions for small/large nodules across clients small_proportions = np.random.dirichlet([alpha] * self.num_clients) large_proportions = np.random.dirichlet([alpha] * self.num_clients) # Shuffle data small_nodules = small_nodules.sample(frac=1, random_state=self.random_seed).reset_index(drop=True) large_nodules = large_nodules.sample(frac=1, random_state=self.random_seed).reset_index(drop=True) # Distribute to clients client_dfs = [] small_start = 0 large_start = 0 for i in range(self.num_clients): # Calculate how many samples for this client small_count = int(small_proportions[i] * len(small_nodules)) large_count = int(large_proportions[i] * len(large_nodules)) # Handle last client (get remaining) if i == self.num_clients - 1: small_count = len(small_nodules) - small_start large_count = len(large_nodules) - large_start # Slice data client_small = small_nodules.iloc[small_start:small_start + small_count] client_large = large_nodules.iloc[large_start:large_start + large_count] client_df = pd.concat([client_small, client_large], ignore_index=True) client_df = client_df.sample(frac=1, random_state=self.random_seed + i).reset_index(drop=True) client_dfs.append(client_df) small_ratio = len(client_small) / len(client_df) * 100 if len(client_df) > 0 else 0 print(f" Client {i}: {len(client_df)} samples ({small_ratio:.1f}% small nodules)") small_start += small_count large_start += large_count return client_dfs, test_df def create_dataloaders( self, client_dfs: List[pd.DataFrame], test_df: pd.DataFrame, batch_size: int = 16, num_workers: int = 2, heterogeneous_noise: bool = True, ) -> Tuple[List[DataLoader], DataLoader]: """ Create DataLoaders for FL training. Args: client_dfs: List of DataFrames, one per client test_df: Global test DataFrame batch_size: Batch size for training num_workers: Number of workers for data loading heterogeneous_noise: If True, each client gets different noise level Returns: Tuple of (client_loaders, test_loader) """ print("\nπŸ”§ Creating DataLoaders...") client_loaders = [] for i, client_df in enumerate(client_dfs): # Heterogeneous noise: different sigma per client if heterogeneous_noise: noise_sigma = 10.0 + i * 5.0 # 10, 15, 20, 25, 30 HU else: noise_sigma = 15.0 # Standard Οƒ=15HU dataset = LIDCNoduleDataset( data_df=client_df, data_dir=self.data_dir, is_train=True, augment_rare=True, # Apply rare nodule boosting client_noise_sigma=noise_sigma, ) loader = DataLoader( dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=False, ) client_loaders.append(loader) print(f" Client {i} DataLoader: {len(dataset)} samples, noise Οƒ={noise_sigma:.1f}HU") # Global test loader (no augmentation) test_dataset = LIDCNoduleDataset( data_df=test_df, data_dir=self.data_dir, is_train=False, augment_rare=False, ) test_loader = DataLoader( test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=True, ) print(f" Test DataLoader: {len(test_dataset)} samples") return client_loaders, test_loader # ============================================================================ # USAGE EXAMPLE - WITH EXCEL FILE INTEGRATION # ============================================================================ if __name__ == "__main__": # Paths (update if different) csv_path = "/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/dataset_ready.csv" data_dir = "/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/" excel_path = "/content/drive/MyDrive/Rare Lung Nodules/lidc-idri-nodule-counts-6-23-2015.xlsx" # Verify paths exist print("\nπŸ” Verifying file paths...") if not Path(csv_path).exists(): print(f"❌ ERROR: CSV not found at {csv_path}") print("Please run preprocessing first!") else: print(f"βœ… CSV found: {csv_path}") if not Path(data_dir).exists(): print(f"❌ ERROR: Data directory not found at {data_dir}") else: print(f"βœ… Data directory found: {data_dir}") # Count NPY files npy_files = list(Path(data_dir).glob("*.npy")) print(f"βœ… Found {len(npy_files)} NPY files") if not Path(excel_path).exists(): print(f"⚠️ WARNING: Excel file not found at {excel_path}") print(" Will use default nodule classification (all as small)") else: print(f"βœ… Excel file found: {excel_path}") # Initialize splitter splitter = FederatedDataSplitter( csv_path=csv_path, data_dir=data_dir, excel_path=excel_path, num_clients=5, test_size=0.2, random_seed=42, ) # ======================================================================== # EXPERIMENT 1: IID Split # ======================================================================== print("\n" + "="*70) print("EXPERIMENT 1: IID (Independent & Identically Distributed)") print("="*70) client_dfs_iid, test_df_iid = splitter.split_iid() client_loaders_iid, test_loader_iid = splitter.create_dataloaders( client_dfs_iid, test_df_iid, batch_size=16, num_workers=2, heterogeneous_noise=False, # Same noise for IID baseline ) # Test loading a batch print("\nπŸ§ͺ Testing IID DataLoader...") sample_batch = next(iter(client_loaders_iid[0])) print(f" Image shape: {sample_batch['image'].shape}") print(f" Mask shape: {sample_batch['mask'].shape}") print(f" Labels: {sample_batch['label']}") print(f" Is small: {sample_batch['is_small']}") print(f" Small nodules in batch: {sample_batch['is_small'].sum().item()}/{len(sample_batch['is_small'])}") # ======================================================================== # EXPERIMENT 2: Non-IID Split # ======================================================================== print("\n" + "="*70) print("EXPERIMENT 2: Non-IID (Skewed by nodule size)") print("="*70) client_dfs_noniid, test_df_noniid = splitter.split_non_iid(alpha=0.5) client_loaders_noniid, test_loader_noniid = splitter.create_dataloaders( client_dfs_noniid, test_df_noniid, batch_size=16, num_workers=2, heterogeneous_noise=True, # Different noise per client ) # Test loading a batch print("\nπŸ§ͺ Testing Non-IID DataLoader...") sample_batch = next(iter(client_loaders_noniid[0])) print(f" Image shape: {sample_batch['image'].shape}") print(f" Mask shape: {sample_batch['mask'].shape}") # ======================================================================== # SAVE SPLITS FOR REPRODUCIBILITY # ======================================================================== print("\nπŸ’Ύ Saving splits to CSV...") output_dir = Path(data_dir) # Save IID splits for i, df in enumerate(client_dfs_iid): df.to_csv(output_dir / f"client_{i}_iid.csv", index=False) test_df_iid.to_csv(output_dir / "test_iid.csv", index=False) # Save Non-IID splits for i, df in enumerate(client_dfs_noniid): df.to_csv(output_dir / f"client_{i}_noniid.csv", index=False) test_df_noniid.to_csv(output_dir / "test_noniid.csv", index=False) print(f"βœ… Splits saved to: {output_dir}") print("\n" + "="*70) print("βœ… DATASET PREPARATION COMPLETE!") print("="*70) print("\nπŸ“Š Summary:") print(f" β€’ Excel file integrated for accurate nodule size classification") print(f" β€’ Small nodules (<10mm) correctly identified") print(f" β€’ IID splits: 5 clients + 1 global test set") print(f" β€’ Non-IID splits: 5 clients + 1 global test set (size-skewed)") print(f" β€’ Rare nodule boosting: 3x duplication enabled") print(f" β€’ Heterogeneous noise: Οƒ = 10, 15, 20, 25, 30 HU per client") print("\nNext steps:") print("1. Use client_loaders_iid for IID federated learning baseline") print("2. Use client_loaders_noniid for Non-IID + heterogeneous experiments") print("3. Use test_loader for global model evaluation") print("\nβœ… Ready for Step 4: U-Net model implementation!") # -*- coding: utf-8 -*- """ STEP 4: Train All 3 Baseline Models (WITH CHECKPOINT RESUMPTION) Auto-resumes from last saved checkpoint if training was interrupted """ # ============================================================================ # ADDITIONAL IMPORTS FOR STEP 4 # ============================================================================ import torch import torch.nn as nn from torch.optim import AdamW from torch.optim.lr_scheduler import CosineAnnealingLR from tqdm import tqdm from monai.networks.nets import UNet, BasicUNet, SwinUNETR from monai.networks.layers import Norm from monai.losses import DiceCELoss import pandas as pd from pathlib import Path from torch.utils.data import DataLoader import json print("βœ… Step 4 imports loaded!") # ============================================================================ # LOSS FUNCTIONS # ============================================================================ class CombinedLoss(nn.Module): def __init__(self, dice_weight=0.7, ce_weight=0.3, sigmoid=True): super().__init__() self.dice_ce_loss = DiceCELoss( sigmoid=sigmoid, softmax=False, include_background=False, to_onehot_y=False, lambda_dice=dice_weight, lambda_ce=ce_weight, ) def forward(self, pred, target): return self.dice_ce_loss(pred, target) class RUSLoss(nn.Module): def __init__(self, weight=0.1, num_bins=10): super().__init__() self.weight = weight self.num_bins = num_bins def compute_entropy(self, probs): return -torch.sum(probs * torch.log(probs + 1e-8), dim=1) def compute_ece(self, probs, targets): confidences, predictions = torch.max(probs, dim=1) targets_binary = (targets.squeeze(1) > 0.5).float() confidences_flat = confidences.flatten() predictions_flat = predictions.flatten().float() targets_flat = targets_binary.flatten() bin_boundaries = torch.linspace(0, 1, self.num_bins + 1, device=probs.device) ece = 0.0 for i in range(self.num_bins): in_bin = (confidences_flat > bin_boundaries[i]) & (confidences_flat <= bin_boundaries[i + 1]) if in_bin.sum() > 0: accuracy_in_bin = (predictions_flat[in_bin] == targets_flat[in_bin]).float().mean() avg_confidence_in_bin = confidences_flat[in_bin].mean() ece += torch.abs(avg_confidence_in_bin - accuracy_in_bin) * in_bin.sum().float() return ece / confidences_flat.numel() def forward(self, probs, targets, is_small): entropy = self.compute_entropy(probs) ece = self.compute_ece(probs, targets) is_small_mask = is_small.float().unsqueeze(1).unsqueeze(2).expand_as(entropy) return (is_small_mask * entropy).mean() + ece * self.weight def compute_metrics(preds, targets, threshold=0.5): preds_binary = (torch.sigmoid(preds) > threshold).float() targets_binary = (targets > 0.5).float() preds_flat = preds_binary.view(-1) targets_flat = targets_binary.view(-1) tp = (preds_flat * targets_flat).sum() fp = (preds_flat * (1 - targets_flat)).sum() fn = ((1 - preds_flat) * targets_flat).sum() tn = ((1 - preds_flat) * (1 - targets_flat)).sum() return { 'dice': ((2 * tp + 1e-8) / (2 * tp + fp + fn + 1e-8)).item(), 'sensitivity': ((tp + 1e-8) / (tp + fn + 1e-8)).item(), 'specificity': ((tn + 1e-8) / (tn + fp + 1e-8)).item(), 'precision': ((tp + 1e-8) / (tp + fp + 1e-8)).item(), 'iou': ((tp + 1e-8) / (tp + fp + fn + 1e-8)).item(), } def train_epoch(model, dataloader, optimizer, dice_ce_loss, rus_loss, device, use_rus, epoch): model.train() total_loss = total_dice = total_sens = 0.0 num_batches = 0 for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Train]"): images = batch['image'].to(device) masks = batch['mask'].to(device) is_small = batch['is_small'].to(device) optimizer.zero_grad() logits = model(images) loss_dice_ce = dice_ce_loss(logits, masks) loss_rus = 0.0 if use_rus: probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1) loss_rus = rus_loss(probs, masks, is_small) total_loss_batch = loss_dice_ce + loss_rus total_loss_batch.backward() optimizer.step() metrics = compute_metrics(logits, masks) total_loss += total_loss_batch.item() total_dice += metrics['dice'] total_sens += metrics['sensitivity'] num_batches += 1 return {'loss': total_loss/num_batches, 'dice': total_dice/num_batches, 'sensitivity': total_sens/num_batches} def validate_epoch(model, dataloader, dice_ce_loss, rus_loss, device, use_rus, epoch): model.eval() total_loss = total_dice = total_sens = total_spec = total_prec = total_iou = 0.0 num_batches = 0 with torch.no_grad(): for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Val]"): images = batch['image'].to(device) masks = batch['mask'].to(device) is_small = batch['is_small'].to(device) logits = model(images) loss_dice_ce = dice_ce_loss(logits, masks) loss_rus = 0.0 if use_rus: probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1) loss_rus = rus_loss(probs, masks, is_small) total_loss_batch = loss_dice_ce + loss_rus metrics = compute_metrics(logits, masks) total_loss += total_loss_batch.item() total_dice += metrics['dice'] total_sens += metrics['sensitivity'] total_spec += metrics['specificity'] total_prec += metrics['precision'] total_iou += metrics['iou'] num_batches += 1 return { 'loss': total_loss/num_batches, 'dice': total_dice/num_batches, 'sensitivity': total_sens/num_batches, 'specificity': total_spec/num_batches, 'precision': total_prec/num_batches, 'iou': total_iou/num_batches, } # ============================================================================ # DEFINE 3 MODEL ARCHITECTURES # ============================================================================ class Model1_UNet(nn.Module): def __init__(self): super().__init__() self.unet = UNet(spatial_dims=2, in_channels=1, out_channels=1, channels=(32, 64, 128, 256, 512), strides=(2, 2, 2, 2), num_res_units=2, norm=Norm.BATCH, dropout=0.1) def forward(self, x): return self.unet(x) class Model2_nnUNet(nn.Module): def __init__(self): super().__init__() self.basicunet = BasicUNet(spatial_dims=2, in_channels=1, out_channels=1, features=(32, 64, 128, 256, 512, 32), dropout=0.1, norm='batch') def forward(self, x): return self.basicunet(x) class Model3_SwinUNet(nn.Module): def __init__(self): super().__init__() self.swinunetr = SwinUNETR(img_size=(128, 128), in_channels=1, out_channels=1, feature_size=24, depths=(2, 2, 2, 2), num_heads=(3, 6, 12, 24), norm_name='batch', dropout_path_rate=0.1) def forward(self, x): return self.swinunetr(x) # ============================================================================ # TRAINING FUNCTION WITH CHECKPOINT RESUMPTION # ============================================================================ def train_model(model, train_loader, val_loader, model_name, num_epochs=50): device = torch.device('cuda') model = model.to(device) save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/") save_dir.mkdir(parents=True, exist_ok=True) checkpoint_path = save_dir / f"{model_name}_checkpoint.pth" best_model_path = save_dir / f"{model_name}_best.pth" history_path = save_dir / f"{model_name}_history.json" # ======================================================================== # CHECKPOINT RESUMPTION LOGIC # ======================================================================== start_epoch = 1 best_dice = 0.0 history = {'val_dice': [], 'val_sensitivity': [], 'val_specificity': [], 'val_precision': [], 'val_iou': [], 'train_dice': [], 'train_loss': []} # Try to load checkpoint if checkpoint_path.exists(): print(f"\nπŸ”„ Found checkpoint! Resuming from: {checkpoint_path}") checkpoint = torch.load(checkpoint_path) model.load_state_dict(checkpoint['model_state_dict']) start_epoch = checkpoint['epoch'] + 1 best_dice = checkpoint['best_dice'] # Load history if exists if history_path.exists(): with open(history_path, 'r') as f: history = json.load(f) print(f"βœ… Resumed from epoch {checkpoint['epoch']}") print(f" Best Dice so far: {best_dice:.4f}") print(f" Continuing from epoch {start_epoch}/{num_epochs}") else: print(f"\nπŸ†• Starting fresh training: {model_name}") print(f"{'='*70}") print(f"TRAINING: {model_name}") print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}") print(f"Epochs: {start_epoch} to {num_epochs}") print(f"{'='*70}") # Initialize optimizer and scheduler dice_ce_loss = CombinedLoss() rus_loss_fn = RUSLoss() optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5) scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-6) # Restore optimizer and scheduler if resuming if checkpoint_path.exists(): optimizer.load_state_dict(checkpoint['optimizer_state_dict']) scheduler.load_state_dict(checkpoint['scheduler_state_dict']) # ======================================================================== # TRAINING LOOP # ======================================================================== for epoch in range(start_epoch, num_epochs + 1): train_metrics = train_epoch(model, train_loader, optimizer, dice_ce_loss, rus_loss_fn, device, True, epoch) val_metrics = validate_epoch(model, val_loader, dice_ce_loss, rus_loss_fn, device, True, epoch) scheduler.step() # Print progress print(f"Epoch {epoch}/{num_epochs} - Train: L={train_metrics['loss']:.4f}, D={train_metrics['dice']:.4f} | " f"Val: D={val_metrics['dice']:.4f}, S={val_metrics['sensitivity']:.4f}, P={val_metrics['precision']:.4f}") # Update history history['train_loss'].append(train_metrics['loss']) history['train_dice'].append(train_metrics['dice']) for k in ['val_dice', 'val_sensitivity', 'val_specificity', 'val_precision', 'val_iou']: history[k].append(val_metrics[k.replace('val_', '')]) # Save best model if val_metrics['dice'] > best_dice: best_dice = val_metrics['dice'] torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'dice': best_dice, 'metrics': val_metrics, }, best_model_path) print(f" βœ… New best model saved! Dice: {best_dice:.4f}") # ==================================================================== # SAVE CHECKPOINT EVERY EPOCH (for resumption) # ==================================================================== torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'best_dice': best_dice, }, checkpoint_path) # Save history as JSON with open(history_path, 'w') as f: json.dump(history, f) print(f" πŸ’Ύ Checkpoint saved (epoch {epoch})") print(f"\nβœ… Training complete! Best Dice: {best_dice:.4f}") return model, history # ============================================================================ # PREPARE CENTRAL TRAINING DATA # ============================================================================ print("\nπŸ”„ Preparing central training data...") data_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/") # Combine all 5 IID client data all_train_data = [] for i in range(5): df = pd.read_csv(data_dir / f"client_{i}_iid.csv") all_train_data.append(df) print(f"βœ… Client {i}: {len(df)} samples") central_train_df = pd.concat(all_train_data, ignore_index=True) test_df = pd.read_csv(data_dir / "test_iid.csv") print(f"\nβœ… Central: {len(central_train_df)} train, {len(test_df)} test") # Create central dataloaders central_train_dataset = LIDCNoduleDataset(central_train_df, data_dir, is_train=True, augment_rare=True) central_train_loader = DataLoader(central_train_dataset, batch_size=16, shuffle=True, num_workers=2, pin_memory=True) central_test_dataset = LIDCNoduleDataset(test_df, data_dir, is_train=False, augment_rare=False) central_test_loader = DataLoader(central_test_dataset, batch_size=16, shuffle=False, num_workers=2, pin_memory=True) # ============================================================================ # TRAIN ALL 3 MODELS (WITH AUTO-RESUME) # ============================================================================ print("\n" + "="*70) print("STEP 4: TRAINING ALL 3 MODELS (WITH CHECKPOINT RESUMPTION)") print("="*70) # 4.1: U-Net print("\nπŸ”΅ Model 1: U-Net") model1, hist1 = train_model(Model1_UNet(), central_train_loader, central_test_loader, "step4.1_unet", 50) # 4.2: nnU-Net print("\n🟒 Model 2: nnU-Net") model2, hist2 = train_model(Model2_nnUNet(), central_train_loader, central_test_loader, "step4.2_nnunet", 50) # 4.3: Swin-UNet print("\nπŸ”΄ Model 3: Swin-UNet") model3, hist3 = train_model(Model3_SwinUNet(), central_train_loader, central_test_loader, "step4.3_swinunet", 50) # ============================================================================ # COMPARE RESULTS # ============================================================================ comparison = pd.DataFrame({ 'Model': ['4.1: U-Net', '4.2: nnU-Net', '4.3: Swin-UNet'], 'Dice': [max(hist1['val_dice']), max(hist2['val_dice']), max(hist3['val_dice'])], 'Sensitivity': [max(hist1['val_sensitivity']), max(hist2['val_sensitivity']), max(hist3['val_sensitivity'])], 'Precision': [max(hist1['val_precision']), max(hist2['val_precision']), max(hist3['val_precision'])], 'IoU': [max(hist1['val_iou']), max(hist2['val_iou']), max(hist3['val_iou'])], }) print("\n" + "="*70) print("FINAL COMPARISON") print("="*70) print(comparison.to_string(index=False)) save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/") comparison.to_csv(save_dir / 'step4_comparison.csv', index=False) print(f"\nβœ… STEP 4 COMPLETE!") print(f"πŸ† Best Model: {comparison.loc[comparison['Dice'].idxmax(), 'Model']}") print(f" Dice: {comparison['Dice'].max():.4f}") print(f" Sensitivity: {comparison.loc[comparison['Dice'].idxmax(), 'Sensitivity']:.4f}") print(f"πŸ“ Saved to: {save_dir}") print("\nπŸ“Š Files saved:") print(" β€’ step4.1_unet_best.pth (best model)") print(" β€’ step4.1_unet_checkpoint.pth (resume from here)") print(" β€’ step4.1_unet_history.json (training curves)") print(" β€’ step4.2_nnunet_* (same structure)") print(" β€’ step4.3_swinunet_* (same structure)") # -*- coding: utf-8 -*- """ STEP 4: Train All 3 Baseline Models (WITH CHECKPOINT RESUMPTION) Auto-resumes from last saved checkpoint if training was interrupted """ # ============================================================================ # ADDITIONAL IMPORTS FOR STEP 4 # ============================================================================ import torch import torch.nn as nn from torch.optim import AdamW from torch.optim.lr_scheduler import CosineAnnealingLR from tqdm import tqdm from monai.networks.nets import UNet, BasicUNet, SwinUNETR from monai.networks.layers import Norm from monai.losses import DiceCELoss import pandas as pd from pathlib import Path from torch.utils.data import DataLoader import json print("βœ… Step 4 imports loaded!") # ============================================================================ # LOSS FUNCTIONS # ============================================================================ class CombinedLoss(nn.Module): def __init__(self, dice_weight=0.7, ce_weight=0.3, sigmoid=True): super().__init__() self.dice_ce_loss = DiceCELoss( sigmoid=sigmoid, softmax=False, include_background=False, to_onehot_y=False, lambda_dice=dice_weight, lambda_ce=ce_weight, ) def forward(self, pred, target): return self.dice_ce_loss(pred, target) class RUSLoss(nn.Module): def __init__(self, weight=0.1, num_bins=10): super().__init__() self.weight = weight self.num_bins = num_bins def compute_entropy(self, probs): return -torch.sum(probs * torch.log(probs + 1e-8), dim=1) def compute_ece(self, probs, targets): confidences, predictions = torch.max(probs, dim=1) targets_binary = (targets.squeeze(1) > 0.5).float() confidences_flat = confidences.flatten() predictions_flat = predictions.flatten().float() targets_flat = targets_binary.flatten() bin_boundaries = torch.linspace(0, 1, self.num_bins + 1, device=probs.device) ece = 0.0 for i in range(self.num_bins): in_bin = (confidences_flat > bin_boundaries[i]) & (confidences_flat <= bin_boundaries[i + 1]) if in_bin.sum() > 0: accuracy_in_bin = (predictions_flat[in_bin] == targets_flat[in_bin]).float().mean() avg_confidence_in_bin = confidences_flat[in_bin].mean() ece += torch.abs(avg_confidence_in_bin - accuracy_in_bin) * in_bin.sum().float() return ece / confidences_flat.numel() def forward(self, probs, targets, is_small): entropy = self.compute_entropy(probs) ece = self.compute_ece(probs, targets) is_small_mask = is_small.float().unsqueeze(1).unsqueeze(2).expand_as(entropy) return (is_small_mask * entropy).mean() + ece * self.weight def compute_metrics(preds, targets, threshold=0.5): preds_binary = (torch.sigmoid(preds) > threshold).float() targets_binary = (targets > 0.5).float() preds_flat = preds_binary.view(-1) targets_flat = targets_binary.view(-1) tp = (preds_flat * targets_flat).sum() fp = (preds_flat * (1 - targets_flat)).sum() fn = ((1 - preds_flat) * targets_flat).sum() tn = ((1 - preds_flat) * (1 - targets_flat)).sum() return { 'dice': ((2 * tp + 1e-8) / (2 * tp + fp + fn + 1e-8)).item(), 'sensitivity': ((tp + 1e-8) / (tp + fn + 1e-8)).item(), 'specificity': ((tn + 1e-8) / (tn + fp + 1e-8)).item(), 'precision': ((tp + 1e-8) / (tp + fp + 1e-8)).item(), 'iou': ((tp + 1e-8) / (tp + fp + fn + 1e-8)).item(), } def train_epoch(model, dataloader, optimizer, dice_ce_loss, rus_loss, device, use_rus, epoch): model.train() total_loss = total_dice = total_sens = 0.0 num_batches = 0 for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Train]"): images = batch['image'].to(device) masks = batch['mask'].to(device) is_small = batch['is_small'].to(device) optimizer.zero_grad() logits = model(images) loss_dice_ce = dice_ce_loss(logits, masks) loss_rus = 0.0 if use_rus: probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1) loss_rus = rus_loss(probs, masks, is_small) total_loss_batch = loss_dice_ce + loss_rus total_loss_batch.backward() optimizer.step() metrics = compute_metrics(logits, masks) total_loss += total_loss_batch.item() total_dice += metrics['dice'] total_sens += metrics['sensitivity'] num_batches += 1 return {'loss': total_loss/num_batches, 'dice': total_dice/num_batches, 'sensitivity': total_sens/num_batches} def validate_epoch(model, dataloader, dice_ce_loss, rus_loss, device, use_rus, epoch): model.eval() total_loss = total_dice = total_sens = total_spec = total_prec = total_iou = 0.0 num_batches = 0 with torch.no_grad(): for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Val]"): images = batch['image'].to(device) masks = batch['mask'].to(device) is_small = batch['is_small'].to(device) logits = model(images) loss_dice_ce = dice_ce_loss(logits, masks) loss_rus = 0.0 if use_rus: probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1) loss_rus = rus_loss(probs, masks, is_small) total_loss_batch = loss_dice_ce + loss_rus metrics = compute_metrics(logits, masks) total_loss += total_loss_batch.item() total_dice += metrics['dice'] total_sens += metrics['sensitivity'] total_spec += metrics['specificity'] total_prec += metrics['precision'] total_iou += metrics['iou'] num_batches += 1 return { 'loss': total_loss/num_batches, 'dice': total_dice/num_batches, 'sensitivity': total_sens/num_batches, 'specificity': total_spec/num_batches, 'precision': total_prec/num_batches, 'iou': total_iou/num_batches, } # ============================================================================ # DEFINE 3 MODEL ARCHITECTURES (FIXED SwinUNETR) # ============================================================================ class Model1_UNet(nn.Module): def __init__(self): super().__init__() self.unet = UNet(spatial_dims=2, in_channels=1, out_channels=1, channels=(32, 64, 128, 256, 512), strides=(2, 2, 2, 2), num_res_units=2, norm=Norm.BATCH, dropout=0.1) def forward(self, x): return self.unet(x) class Model2_nnUNet(nn.Module): def __init__(self): super().__init__() self.basicunet = BasicUNet(spatial_dims=2, in_channels=1, out_channels=1, features=(32, 64, 128, 256, 512, 32), dropout=0.1, norm='batch') def forward(self, x): return self.basicunet(x) class Model3_SwinUNet(nn.Module): def __init__(self): super().__init__() self.swinunetr = SwinUNETR( spatial_dims=2, # βœ… FIXED: Added spatial_dims for 2D in_channels=1, # βœ… FIXED: Removed img_size (not supported in MONAI >=1.5) out_channels=1, feature_size=24, depths=(2, 2, 2, 2), num_heads=(3, 6, 12, 24), norm_name='batch', dropout_path_rate=0.1 ) def forward(self, x): return self.swinunetr(x) # ============================================================================ # TRAINING FUNCTION WITH CHECKPOINT RESUMPTION # ============================================================================ def train_model(model, train_loader, val_loader, model_name, num_epochs=50): device = torch.device('cuda') model = model.to(device) save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/") save_dir.mkdir(parents=True, exist_ok=True) checkpoint_path = save_dir / f"{model_name}_checkpoint.pth" best_model_path = save_dir / f"{model_name}_best.pth" history_path = save_dir / f"{model_name}_history.json" # ======================================================================== # CHECKPOINT RESUMPTION LOGIC # ======================================================================== start_epoch = 1 best_dice = 0.0 history = {'val_dice': [], 'val_sensitivity': [], 'val_specificity': [], 'val_precision': [], 'val_iou': [], 'train_dice': [], 'train_loss': []} # Try to load checkpoint if checkpoint_path.exists(): print(f"\nπŸ”„ Found checkpoint! Resuming from: {checkpoint_path}") checkpoint = torch.load(checkpoint_path) model.load_state_dict(checkpoint['model_state_dict']) start_epoch = checkpoint['epoch'] + 1 best_dice = checkpoint['best_dice'] # Load history if exists if history_path.exists(): with open(history_path, 'r') as f: history = json.load(f) print(f"βœ… Resumed from epoch {checkpoint['epoch']}") print(f" Best Dice so far: {best_dice:.4f}") print(f" Continuing from epoch {start_epoch}/{num_epochs}") else: print(f"\nπŸ†• Starting fresh training: {model_name}") print(f"{'='*70}") print(f"TRAINING: {model_name}") print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}") print(f"Epochs: {start_epoch} to {num_epochs}") print(f"{'='*70}") # Initialize optimizer and scheduler dice_ce_loss = CombinedLoss() rus_loss_fn = RUSLoss() optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5) scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-6) # Restore optimizer and scheduler if resuming if checkpoint_path.exists(): optimizer.load_state_dict(checkpoint['optimizer_state_dict']) scheduler.load_state_dict(checkpoint['scheduler_state_dict']) # ======================================================================== # TRAINING LOOP # ======================================================================== for epoch in range(start_epoch, num_epochs + 1): train_metrics = train_epoch(model, train_loader, optimizer, dice_ce_loss, rus_loss_fn, device, True, epoch) val_metrics = validate_epoch(model, val_loader, dice_ce_loss, rus_loss_fn, device, True, epoch) scheduler.step() # Print progress print(f"Epoch {epoch}/{num_epochs} - Train: L={train_metrics['loss']:.4f}, D={train_metrics['dice']:.4f} | " f"Val: D={val_metrics['dice']:.4f}, S={val_metrics['sensitivity']:.4f}, P={val_metrics['precision']:.4f}") # Update history history['train_loss'].append(train_metrics['loss']) history['train_dice'].append(train_metrics['dice']) for k in ['val_dice', 'val_sensitivity', 'val_specificity', 'val_precision', 'val_iou']: history[k].append(val_metrics[k.replace('val_', '')]) # Save best model if val_metrics['dice'] > best_dice: best_dice = val_metrics['dice'] torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'dice': best_dice, 'metrics': val_metrics, }, best_model_path) print(f" βœ… New best model saved! Dice: {best_dice:.4f}") # ==================================================================== # SAVE CHECKPOINT EVERY EPOCH (for resumption) # ==================================================================== torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'best_dice': best_dice, }, checkpoint_path) # Save history as JSON with open(history_path, 'w') as f: json.dump(history, f) print(f" πŸ’Ύ Checkpoint saved (epoch {epoch})") print(f"\nβœ… Training complete! Best Dice: {best_dice:.4f}") return model, history # ============================================================================ # PREPARE CENTRAL TRAINING DATA # ============================================================================ print("\nπŸ”„ Preparing central training data...") data_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/") # Combine all 5 IID client data all_train_data = [] for i in range(5): df = pd.read_csv(data_dir / f"client_{i}_iid.csv") all_train_data.append(df) print(f"βœ… Client {i}: {len(df)} samples") central_train_df = pd.concat(all_train_data, ignore_index=True) test_df = pd.read_csv(data_dir / "test_iid.csv") print(f"\nβœ… Central: {len(central_train_df)} train, {len(test_df)} test") # Create central dataloaders central_train_dataset = LIDCNoduleDataset(central_train_df, data_dir, is_train=True, augment_rare=True) central_train_loader = DataLoader(central_train_dataset, batch_size=16, shuffle=True, num_workers=2, pin_memory=True) central_test_dataset = LIDCNoduleDataset(test_df, data_dir, is_train=False, augment_rare=False) central_test_loader = DataLoader(central_test_dataset, batch_size=16, shuffle=False, num_workers=2, pin_memory=True) # ============================================================================ # TRAIN ALL 3 MODELS (WITH AUTO-RESUME) # ============================================================================ print("\n" + "="*70) print("STEP 4: TRAINING ALL 3 MODELS (WITH CHECKPOINT RESUMPTION)") print("="*70) # 4.1: U-Net print("\nπŸ”΅ Model 1: U-Net") model1, hist1 = train_model(Model1_UNet(), central_train_loader, central_test_loader, "step4.1_unet", 50) # 4.2: nnU-Net print("\n🟒 Model 2: nnU-Net") model2, hist2 = train_model(Model2_nnUNet(), central_train_loader, central_test_loader, "step4.2_nnunet", 50) # 4.3: Swin-UNet (FIXED) print("\nπŸ”΄ Model 3: Swin-UNet") model3, hist3 = train_model(Model3_SwinUNet(), central_train_loader, central_test_loader, "step4.3_swinunet", 50) # ============================================================================ # COMPARE RESULTS # ============================================================================ comparison = pd.DataFrame({ 'Model': ['4.1: U-Net', '4.2: nnU-Net', '4.3: Swin-UNet'], 'Dice': [max(hist1['val_dice']), max(hist2['val_dice']), max(hist3['val_dice'])], 'Sensitivity': [max(hist1['val_sensitivity']), max(hist2['val_sensitivity']), max(hist3['val_sensitivity'])], 'Precision': [max(hist1['val_precision']), max(hist2['val_precision']), max(hist3['val_precision'])], 'IoU': [max(hist1['val_iou']), max(hist2['val_iou']), max(hist3['val_iou'])], }) print("\n" + "="*70) print("FINAL COMPARISON") print("="*70) print(comparison.to_string(index=False)) save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/") comparison.to_csv(save_dir / 'step4_comparison.csv', index=False) print(f"\nβœ… STEP 4 COMPLETE!") print(f"πŸ† Best Model: {comparison.loc[comparison['Dice'].idxmax(), 'Model']}") print(f" Dice: {comparison['Dice'].max():.4f}") print(f" Sensitivity: {comparison.loc[comparison['Dice'].idxmax(), 'Sensitivity']:.4f}") print(f"πŸ“ Saved to: {save_dir}") print("\nπŸ“Š Files saved:") print(" β€’ step4.1_unet_best.pth (best model)") print(" β€’ step4.1_unet_checkpoint.pth (resume from here)") print(" β€’ step4.1_unet_history.json (training curves)") print(" β€’ step4.2_nnunet_* (same structure)") print(" β€’ step4.3_swinunet_* (same structure)")
# -*- coding: utf-8 -*-
"""lidc_step2_step3_step4.ipynb
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/19ND1Fka8Cromhk7B_hYQzGWayIbTl2eH
"""
# -*- coding: utf-8 -*-
"""
Step 2 & 3: PyTorch Dataset Loader with Data Augmentation + FL Client Splits
CORRECTED VERSION - With Excel-based nodule size classification
"""
# ============================================================================
# STEP 0: MOUNT GOOGLE DRIVE
# ============================================================================
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
print("βœ… Google Drive mounted successfully!")
print("πŸ“ Your files are accessible at: /content/drive/MyDrive/")
# ============================================================================
# INSTALL CORRECT DEPENDENCIES
# ============================================================================
print("πŸ“¦ Installing compatible dependencies...")
# Uninstall any existing conflicting versions
!pip uninstall -y torch torchvision torchaudio monai -q
# Install PyTorch 2.6.0 with CUDA 12.1 (Colab default)
!pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu121 -q
# Install MONAI 1.5.0 (supports PyTorch 2.6.0)
!pip install monai==1.5.0 -q
# Install other dependencies (including openpyxl for Excel reading)
!pip install numpy==1.26.4 pandas==2.2.0 scikit-learn==1.4.0 openpyxl==3.1.2 -q
print("βœ… Dependencies installed!")
print("⚠️ IMPORTANT: Click 'Runtime' β†’ 'Restart runtime' before proceeding!")
# ============================================================================
# VERIFY INSTALLATIONS (Run after restart)
# ============================================================================
import torch
import monai
import numpy as np
import pandas as pd
from sklearn import __version__ as sklearn_version
print("="*70)
print("DEPENDENCY VERIFICATION")
print("="*70)
print(f"βœ… PyTorch version: {torch.__version__}")
print(f"βœ… MONAI version: {monai.__version__}")
print(f"βœ… NumPy version: {np.__version__}")
print(f"βœ… Pandas version: {pd.__version__}")
print(f"βœ… Scikit-learn version: {sklearn_version}")
print(f"βœ… CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
print(f"βœ… CUDA version: {torch.version.cuda}")
print(f"βœ… GPU: {torch.cuda.get_device_name(0)}")
print("="*70)
# ============================================================================
# IMPORTS
# ============================================================================
from pathlib import Path
from typing import Dict, List, Tuple, Optional
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings('ignore')
# MONAI transforms for medical imaging
from monai.transforms import (
Compose,
RandFlipd,
RandRotate90d,
RandAffined,
RandAdjustContrastd,
RandGaussianNoised,
ToTensord,
ScaleIntensityd,
)
print("βœ… All imports successful!")
# ============================================================================
# LOAD NODULE SIZE INFORMATION FROM EXCEL
# ============================================================================
def load_nodule_sizes(excel_path: str) -> Dict[str, float]:
"""
Load nodule size information from LIDC-IDRI Excel file.
Args: