Spaces:
Sleeping
Sleeping
Download postprocessing/postproccessing_and_initial_training.py from hephaestusonfire/RareLungNoduleFL: direct link, hf CLI and curl.
- Browser
- Download file 59.6 kB
-
https://huggingface.co/spaces/hephaestusonfire/RareLungNoduleFL/resolve/main/postprocessing/postproccessing_and_initial_training.py
- Command line
-
hf download hf://spaces/hephaestusonfire/RareLungNoduleFL/postprocessing/postproccessing_and_initial_training.py
-
curl -L -o postproccessing_and_initial_training.py https://huggingface.co/spaces/hephaestusonfire/RareLungNoduleFL/resolve/main/postprocessing/postproccessing_and_initial_training.py
59.6 kB
# -*- coding: utf-8 -*-
"""lidc_step2_step3_step4.ipynb
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/19ND1Fka8Cromhk7B_hYQzGWayIbTl2eH
"""
# -*- coding: utf-8 -*-
"""
Step 2 & 3: PyTorch Dataset Loader with Data Augmentation + FL Client Splits
CORRECTED VERSION - With Excel-based nodule size classification
"""
# ============================================================================
# STEP 0: MOUNT GOOGLE DRIVE
# ============================================================================
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
print("β
Google Drive mounted successfully!")
print("π Your files are accessible at: /content/drive/MyDrive/")
# ============================================================================
# INSTALL CORRECT DEPENDENCIES
# ============================================================================
print("π¦ Installing compatible dependencies...")
# Uninstall any existing conflicting versions
!pip uninstall -y torch torchvision torchaudio monai -q
# Install PyTorch 2.6.0 with CUDA 12.1 (Colab default)
!pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu121 -q
# Install MONAI 1.5.0 (supports PyTorch 2.6.0)
!pip install monai==1.5.0 -q
# Install other dependencies (including openpyxl for Excel reading)
!pip install numpy==1.26.4 pandas==2.2.0 scikit-learn==1.4.0 openpyxl==3.1.2 -q
print("β
Dependencies installed!")
print("β οΈ IMPORTANT: Click 'Runtime' β 'Restart runtime' before proceeding!")
# ============================================================================
# VERIFY INSTALLATIONS (Run after restart)
# ============================================================================
import torch
import monai
import numpy as np
import pandas as pd
from sklearn import __version__ as sklearn_version
print("="*70)
print("DEPENDENCY VERIFICATION")
print("="*70)
print(f"β
PyTorch version: {torch.__version__}")
print(f"β
MONAI version: {monai.__version__}")
print(f"β
NumPy version: {np.__version__}")
print(f"β
Pandas version: {pd.__version__}")
print(f"β
Scikit-learn version: {sklearn_version}")
print(f"β
CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
print(f"β
CUDA version: {torch.version.cuda}")
print(f"β
GPU: {torch.cuda.get_device_name(0)}")
print("="*70)
# ============================================================================
# IMPORTS
# ============================================================================
from pathlib import Path
from typing import Dict, List, Tuple, Optional
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings('ignore')
# MONAI transforms for medical imaging
from monai.transforms import (
Compose,
RandFlipd,
RandRotate90d,
RandAffined,
RandAdjustContrastd,
RandGaussianNoised,
ToTensord,
ScaleIntensityd,
)
print("β
All imports successful!")
# ============================================================================
# LOAD NODULE SIZE INFORMATION FROM EXCEL
# ============================================================================
def load_nodule_sizes(excel_path: str) -> Dict[str, float]:
"""
Load nodule size information from LIDC-IDRI Excel file.
Args:
excel_path: Path to lidc-idri-nodule-counts-6-23-2015.xlsx
Returns:
Dictionary mapping (patient_id, nodule_id) -> diameter_mm
"""
print(f"\nπ Loading nodule size data from Excel...")
print(f" Path: {excel_path}")
try:
# Read Excel file
df_excel = pd.read_excel(excel_path)
print(f" β
Loaded Excel with shape: {df_excel.shape}")
print(f" π Columns: {list(df_excel.columns)}")
# Create mapping dictionary
# Adjust column names based on actual Excel structure
nodule_sizes = {}
# Common column name variations in LIDC-IDRI files
patient_col = None
nodule_col = None
diameter_col = None
# Find correct column names (case-insensitive search)
for col in df_excel.columns:
col_lower = str(col).lower()
if 'patient' in col_lower or 'pid' in col_lower or 'subject' in col_lower:
patient_col = col
elif 'nodule' in col_lower and 'no' in col_lower:
nodule_col = col
elif 'diameter' in col_lower or 'size' in col_lower:
diameter_col = col
if patient_col and nodule_col and diameter_col:
print(f" π Found columns: Patient='{patient_col}', Nodule='{nodule_col}', Diameter='{diameter_col}'")
for _, row in df_excel.iterrows():
patient_id = str(row[patient_col])
nodule_id = f"nodule-{int(row[nodule_col])}"
diameter = float(row[diameter_col])
# Ensure patient_id format matches (LIDC-IDRI-XXXX)
if not patient_id.startswith('LIDC-IDRI'):
patient_id = f"LIDC-IDRI-{patient_id.zfill(4)}"
key = (patient_id, nodule_id)
nodule_sizes[key] = diameter
print(f" β
Loaded {len(nodule_sizes)} nodule size entries")
# Statistics
diameters = list(nodule_sizes.values())
small_count = sum(1 for d in diameters if d < 10.0)
large_count = sum(1 for d in diameters if d >= 10.0)
print(f" π Statistics:")
print(f" β’ Small nodules (<10mm): {small_count} ({small_count/len(diameters)*100:.1f}%)")
print(f" β’ Large nodules (β₯10mm): {large_count} ({large_count/len(diameters)*100:.1f}%)")
print(f" β’ Mean diameter: {np.mean(diameters):.2f}mm")
print(f" β’ Median diameter: {np.median(diameters):.2f}mm")
print(f" β’ Min diameter: {np.min(diameters):.2f}mm")
print(f" β’ Max diameter: {np.max(diameters):.2f}mm")
else:
print(f" β οΈ Could not find standard columns. Available columns:")
print(f" {list(df_excel.columns)}")
print(f" βΉοΈ Please check the Excel file structure.")
# Show first few rows
print(f"\n First 5 rows:")
print(df_excel.head())
return nodule_sizes
except Exception as e:
print(f" β οΈ Error loading Excel: {e}")
print(f" βΉοΈ Will use default classification (all nodules as small)")
return {}
# ============================================================================
# STEP 2: CUSTOM PYTORCH DATASET CLASS
# ============================================================================
class LIDCNoduleDataset(Dataset):
"""
PyTorch Dataset for LIDC-IDRI lung nodule NPY files with augmentation.
Features:
- Loads preprocessed NPY files (images + masks)
- Applies medical imaging augmentations (flip, rotate, scale, noise)
- Supports heterogeneous augmentations per client
- Implements rare nodule boosting
"""
def __init__(
self,
data_df: pd.DataFrame,
data_dir: Path,
transform: Optional[Compose] = None,
is_train: bool = True,
augment_rare: bool = False,
client_noise_sigma: float = 15.0, # Poisson noise Ο=15HU
):
"""
Args:
data_df: DataFrame with columns ['fname', 'patient_id', 'nodule_id', 'is_small', 'label']
data_dir: Path to directory containing NPY files
transform: MONAI Compose transform pipeline
is_train: If True, apply augmentations
augment_rare: If True, oversample small nodules (<10mm)
client_noise_sigma: Heterogeneous noise level per client
"""
self.data_dir = Path(data_dir)
self.is_train = is_train
self.client_noise_sigma = client_noise_sigma
# Apply rare nodule boosting (duplicate small nodules x3)
if augment_rare and is_train:
rare_nodules = data_df[data_df['is_small'] == 1]
# Duplicate rare samples 2 additional times (3x total)
data_df = pd.concat([data_df, rare_nodules, rare_nodules], ignore_index=True)
print(f"β
Rare nodule boosting: {len(rare_nodules)} small nodules duplicated 3x")
self.data_df = data_df.reset_index(drop=True)
# Set up transforms
if transform is None:
self.transform = self._get_default_transforms(is_train)
else:
self.transform = transform
print(f"π Dataset initialized: {len(self.data_df)} samples")
def _get_default_transforms(self, is_train: bool) -> Compose:
"""
Create default MONAI transform pipeline.
Augmentations: flip/rotate/scale + Poisson noise Ο=15HU + contrast Β±15%
"""
if is_train:
# Training augmentations
transforms = [
# Geometric augmentations
RandFlipd(keys=["image", "mask"], prob=0.5, spatial_axis=0),
RandFlipd(keys=["image", "mask"], prob=0.5, spatial_axis=1),
RandRotate90d(keys=["image", "mask"], prob=0.5, max_k=3),
RandAffined(
keys=["image", "mask"],
prob=0.5,
rotate_range=(np.pi / 18, np.pi / 18), # Β±10 degrees
scale_range=(0.1, 0.1), # Β±10% scale
mode=["bilinear", "nearest"],
padding_mode="border",
),
# Intensity augmentations (heterogeneous per client)
RandGaussianNoised(
keys=["image"],
prob=0.5,
mean=0.0,
std=self.client_noise_sigma / 255.0, # Normalize to [0,1] range
),
RandAdjustContrastd(
keys=["image"],
prob=0.5,
gamma=(0.85, 1.15), # Β±15% contrast
),
# Ensure intensity range [0, 1]
ScaleIntensityd(keys=["image"], minv=0.0, maxv=1.0),
# Convert to PyTorch tensors
ToTensord(keys=["image", "mask"]),
]
else:
# Validation/test: no augmentation
transforms = [
ScaleIntensityd(keys=["image"], minv=0.0, maxv=1.0),
ToTensord(keys=["image", "mask"]),
]
return Compose(transforms)
def __len__(self) -> int:
return len(self.data_df)
def __getitem__(self, idx: int) -> Dict[str, torch.Tensor]:
"""
Load and return a single sample.
Returns:
Dictionary with keys: 'image', 'mask', 'label', 'is_small', 'fname'
"""
row = self.data_df.iloc[idx]
fname = row['fname']
# Load NPY files (matching your preprocessing output format)
img_path = self.data_dir / f"{fname}_img.npy"
mask_path = self.data_dir / f"{fname}_mask.npy"
try:
image = np.load(img_path).astype(np.float32) # Shape: (1, H, W)
mask = np.load(mask_path).astype(np.float32) # Shape: (1, H, W)
except Exception as e:
print(f"β οΈ Error loading {fname}: {e}")
# Return empty tensors as fallback
image = np.zeros((1, 256, 256), dtype=np.float32)
mask = np.zeros((1, 256, 256), dtype=np.float32)
# Prepare data dict for MONAI transforms
data_dict = {
"image": image,
"mask": mask,
}
# Apply transforms
if self.transform:
data_dict = self.transform(data_dict)
# Add metadata
data_dict["label"] = torch.tensor(row['label'], dtype=torch.long)
data_dict["is_small"] = torch.tensor(row['is_small'], dtype=torch.float32)
data_dict["fname"] = fname
return data_dict
# ============================================================================
# STEP 3: DATA SPLITTING FOR FEDERATED LEARNING
# ============================================================================
class FederatedDataSplitter:
"""
Split preprocessed LIDC-IDRI data for federated learning experiments.
Supports:
- IID: Random stratified split across clients
- Non-IID: Skewed by nodule size (small vs large)
- Heterogeneous: Different noise levels per client
"""
def __init__(
self,
csv_path: str,
data_dir: str,
excel_path: str,
num_clients: int = 5,
test_size: float = 0.2,
random_seed: int = 42,
):
"""
Args:
csv_path: Path to dataset_ready.csv from preprocessing
data_dir: Path to directory with NPY files
excel_path: Path to LIDC-IDRI Excel file with nodule sizes
num_clients: Number of federated clients (default 5)
test_size: Fraction of data for global test set (default 0.2)
random_seed: Random seed for reproducibility
"""
self.csv_path = Path(csv_path)
self.data_dir = Path(data_dir)
self.num_clients = num_clients
self.test_size = test_size
self.random_seed = random_seed
# Load nodule sizes from Excel
self.nodule_sizes = load_nodule_sizes(excel_path)
# Load dataset
print(f"\nπ Loading CSV from: {csv_path}")
self.df = pd.read_csv(csv_path)
print(f"π Loaded dataset: {len(self.df)} samples from {self.df['patient_id'].nunique()} patients")
# Update is_small column based on Excel data
self._update_nodule_sizes()
def _update_nodule_sizes(self):
"""
Update is_small column in dataframe based on Excel nodule sizes.
Small nodules: diameter < 10mm
Large nodules: diameter >= 10mm
"""
print(f"\nπ Updating nodule size classifications...")
if not self.nodule_sizes:
print(f" β οΈ No Excel data available. Using default (all as small nodules)")
self.df['is_small'] = 1
return
updated_count = 0
small_count = 0
large_count = 0
for idx, row in self.df.iterrows():
patient_id = row['patient_id']
nodule_id = row['nodule_id']
key = (patient_id, nodule_id)
if key in self.nodule_sizes:
diameter = self.nodule_sizes[key]
is_small = 1 if diameter < 10.0 else 0
self.df.at[idx, 'is_small'] = is_small
updated_count += 1
if is_small:
small_count += 1
else:
large_count += 1
print(f" β
Updated {updated_count}/{len(self.df)} samples")
print(f" π Classification results:")
print(f" β’ Small nodules (<10mm): {small_count} ({small_count/len(self.df)*100:.1f}%)")
print(f" β’ Large nodules (β₯10mm): {large_count} ({large_count/len(self.df)*100:.1f}%)")
# Save updated CSV
updated_csv_path = self.csv_path.parent / "dataset_ready_with_sizes.csv"
self.df.to_csv(updated_csv_path, index=False)
print(f" πΎ Saved updated CSV to: {updated_csv_path}")
def split_iid(self) -> Tuple[List[pd.DataFrame], pd.DataFrame]:
"""
IID split: Random stratified distribution across clients.
Returns:
Tuple of (client_dfs, test_df)
"""
print("\nπ Creating IID split...")
# Split train/test globally (stratified by patient)
unique_patients = self.df['patient_id'].unique()
train_patients, test_patients = train_test_split(
unique_patients,
test_size=self.test_size,
random_state=self.random_seed,
)
train_df = self.df[self.df['patient_id'].isin(train_patients)].copy()
test_df = self.df[self.df['patient_id'].isin(test_patients)].copy()
print(f" Train: {len(train_df)} samples ({len(train_patients)} patients)")
print(f" Test: {len(test_df)} samples ({len(test_patients)} patients)")
# Split train data equally across clients (random)
train_df = train_df.sample(frac=1, random_state=self.random_seed).reset_index(drop=True)
client_dfs = []
samples_per_client = len(train_df) // self.num_clients
for i in range(self.num_clients):
start_idx = i * samples_per_client
if i == self.num_clients - 1:
# Last client gets remaining samples
client_df = train_df.iloc[start_idx:].copy()
else:
client_df = train_df.iloc[start_idx:start_idx + samples_per_client].copy()
client_dfs.append(client_df)
small_ratio = (client_df['is_small'] == 1).sum() / len(client_df) * 100
print(f" Client {i}: {len(client_df)} samples ({small_ratio:.1f}% small)")
return client_dfs, test_df
def split_non_iid(self, alpha: float = 0.5) -> Tuple[List[pd.DataFrame], pd.DataFrame]:
"""
Non-IID split: Skewed by nodule size (small vs large).
Each client has different distribution of small/large nodules.
Args:
alpha: Dirichlet concentration parameter (lower = more skewed)
0.1 = highly non-IID, 1.0 = moderately non-IID
Returns:
Tuple of (client_dfs, test_df)
"""
print(f"\nπ Creating Non-IID split (Ξ±={alpha})...")
# Split train/test globally
unique_patients = self.df['patient_id'].unique()
train_patients, test_patients = train_test_split(
unique_patients,
test_size=self.test_size,
random_state=self.random_seed,
)
train_df = self.df[self.df['patient_id'].isin(train_patients)].copy()
test_df = self.df[self.df['patient_id'].isin(test_patients)].copy()
print(f" Train: {len(train_df)} samples ({len(train_patients)} patients)")
print(f" Test: {len(test_df)} samples ({len(test_patients)} patients)")
# Separate by nodule size
small_nodules = train_df[train_df['is_small'] == 1].reset_index(drop=True)
large_nodules = train_df[train_df['is_small'] == 0].reset_index(drop=True)
print(f" Small nodules (<10mm): {len(small_nodules)}")
print(f" Large nodules (β₯10mm): {len(large_nodules)}")
# Use Dirichlet distribution for non-IID split
np.random.seed(self.random_seed)
# Proportions for small/large nodules across clients
small_proportions = np.random.dirichlet([alpha] * self.num_clients)
large_proportions = np.random.dirichlet([alpha] * self.num_clients)
# Shuffle data
small_nodules = small_nodules.sample(frac=1, random_state=self.random_seed).reset_index(drop=True)
large_nodules = large_nodules.sample(frac=1, random_state=self.random_seed).reset_index(drop=True)
# Distribute to clients
client_dfs = []
small_start = 0
large_start = 0
for i in range(self.num_clients):
# Calculate how many samples for this client
small_count = int(small_proportions[i] * len(small_nodules))
large_count = int(large_proportions[i] * len(large_nodules))
# Handle last client (get remaining)
if i == self.num_clients - 1:
small_count = len(small_nodules) - small_start
large_count = len(large_nodules) - large_start
# Slice data
client_small = small_nodules.iloc[small_start:small_start + small_count]
client_large = large_nodules.iloc[large_start:large_start + large_count]
client_df = pd.concat([client_small, client_large], ignore_index=True)
client_df = client_df.sample(frac=1, random_state=self.random_seed + i).reset_index(drop=True)
client_dfs.append(client_df)
small_ratio = len(client_small) / len(client_df) * 100 if len(client_df) > 0 else 0
print(f" Client {i}: {len(client_df)} samples ({small_ratio:.1f}% small nodules)")
small_start += small_count
large_start += large_count
return client_dfs, test_df
def create_dataloaders(
self,
client_dfs: List[pd.DataFrame],
test_df: pd.DataFrame,
batch_size: int = 16,
num_workers: int = 2,
heterogeneous_noise: bool = True,
) -> Tuple[List[DataLoader], DataLoader]:
"""
Create DataLoaders for FL training.
Args:
client_dfs: List of DataFrames, one per client
test_df: Global test DataFrame
batch_size: Batch size for training
num_workers: Number of workers for data loading
heterogeneous_noise: If True, each client gets different noise level
Returns:
Tuple of (client_loaders, test_loader)
"""
print("\nπ§ Creating DataLoaders...")
client_loaders = []
for i, client_df in enumerate(client_dfs):
# Heterogeneous noise: different sigma per client
if heterogeneous_noise:
noise_sigma = 10.0 + i * 5.0 # 10, 15, 20, 25, 30 HU
else:
noise_sigma = 15.0 # Standard Ο=15HU
dataset = LIDCNoduleDataset(
data_df=client_df,
data_dir=self.data_dir,
is_train=True,
augment_rare=True, # Apply rare nodule boosting
client_noise_sigma=noise_sigma,
)
loader = DataLoader(
dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers,
pin_memory=True,
drop_last=False,
)
client_loaders.append(loader)
print(f" Client {i} DataLoader: {len(dataset)} samples, noise Ο={noise_sigma:.1f}HU")
# Global test loader (no augmentation)
test_dataset = LIDCNoduleDataset(
data_df=test_df,
data_dir=self.data_dir,
is_train=False,
augment_rare=False,
)
test_loader = DataLoader(
test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True,
)
print(f" Test DataLoader: {len(test_dataset)} samples")
return client_loaders, test_loader
# ============================================================================
# USAGE EXAMPLE - WITH EXCEL FILE INTEGRATION
# ============================================================================
if __name__ == "__main__":
# Paths (update if different)
csv_path = "/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/dataset_ready.csv"
data_dir = "/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/"
excel_path = "/content/drive/MyDrive/Rare Lung Nodules/lidc-idri-nodule-counts-6-23-2015.xlsx"
# Verify paths exist
print("\nπ Verifying file paths...")
if not Path(csv_path).exists():
print(f"β ERROR: CSV not found at {csv_path}")
print("Please run preprocessing first!")
else:
print(f"β
CSV found: {csv_path}")
if not Path(data_dir).exists():
print(f"β ERROR: Data directory not found at {data_dir}")
else:
print(f"β
Data directory found: {data_dir}")
# Count NPY files
npy_files = list(Path(data_dir).glob("*.npy"))
print(f"β
Found {len(npy_files)} NPY files")
if not Path(excel_path).exists():
print(f"β οΈ WARNING: Excel file not found at {excel_path}")
print(" Will use default nodule classification (all as small)")
else:
print(f"β
Excel file found: {excel_path}")
# Initialize splitter
splitter = FederatedDataSplitter(
csv_path=csv_path,
data_dir=data_dir,
excel_path=excel_path,
num_clients=5,
test_size=0.2,
random_seed=42,
)
# ========================================================================
# EXPERIMENT 1: IID Split
# ========================================================================
print("\n" + "="*70)
print("EXPERIMENT 1: IID (Independent & Identically Distributed)")
print("="*70)
client_dfs_iid, test_df_iid = splitter.split_iid()
client_loaders_iid, test_loader_iid = splitter.create_dataloaders(
client_dfs_iid,
test_df_iid,
batch_size=16,
num_workers=2,
heterogeneous_noise=False, # Same noise for IID baseline
)
# Test loading a batch
print("\nπ§ͺ Testing IID DataLoader...")
sample_batch = next(iter(client_loaders_iid[0]))
print(f" Image shape: {sample_batch['image'].shape}")
print(f" Mask shape: {sample_batch['mask'].shape}")
print(f" Labels: {sample_batch['label']}")
print(f" Is small: {sample_batch['is_small']}")
print(f" Small nodules in batch: {sample_batch['is_small'].sum().item()}/{len(sample_batch['is_small'])}")
# ========================================================================
# EXPERIMENT 2: Non-IID Split
# ========================================================================
print("\n" + "="*70)
print("EXPERIMENT 2: Non-IID (Skewed by nodule size)")
print("="*70)
client_dfs_noniid, test_df_noniid = splitter.split_non_iid(alpha=0.5)
client_loaders_noniid, test_loader_noniid = splitter.create_dataloaders(
client_dfs_noniid,
test_df_noniid,
batch_size=16,
num_workers=2,
heterogeneous_noise=True, # Different noise per client
)
# Test loading a batch
print("\nπ§ͺ Testing Non-IID DataLoader...")
sample_batch = next(iter(client_loaders_noniid[0]))
print(f" Image shape: {sample_batch['image'].shape}")
print(f" Mask shape: {sample_batch['mask'].shape}")
# ========================================================================
# SAVE SPLITS FOR REPRODUCIBILITY
# ========================================================================
print("\nπΎ Saving splits to CSV...")
output_dir = Path(data_dir)
# Save IID splits
for i, df in enumerate(client_dfs_iid):
df.to_csv(output_dir / f"client_{i}_iid.csv", index=False)
test_df_iid.to_csv(output_dir / "test_iid.csv", index=False)
# Save Non-IID splits
for i, df in enumerate(client_dfs_noniid):
df.to_csv(output_dir / f"client_{i}_noniid.csv", index=False)
test_df_noniid.to_csv(output_dir / "test_noniid.csv", index=False)
print(f"β
Splits saved to: {output_dir}")
print("\n" + "="*70)
print("β
DATASET PREPARATION COMPLETE!")
print("="*70)
print("\nπ Summary:")
print(f" β’ Excel file integrated for accurate nodule size classification")
print(f" β’ Small nodules (<10mm) correctly identified")
print(f" β’ IID splits: 5 clients + 1 global test set")
print(f" β’ Non-IID splits: 5 clients + 1 global test set (size-skewed)")
print(f" β’ Rare nodule boosting: 3x duplication enabled")
print(f" β’ Heterogeneous noise: Ο = 10, 15, 20, 25, 30 HU per client")
print("\nNext steps:")
print("1. Use client_loaders_iid for IID federated learning baseline")
print("2. Use client_loaders_noniid for Non-IID + heterogeneous experiments")
print("3. Use test_loader for global model evaluation")
print("\nβ
Ready for Step 4: U-Net model implementation!")
# -*- coding: utf-8 -*-
"""
STEP 4: Train All 3 Baseline Models (WITH CHECKPOINT RESUMPTION)
Auto-resumes from last saved checkpoint if training was interrupted
"""
# ============================================================================
# ADDITIONAL IMPORTS FOR STEP 4
# ============================================================================
import torch
import torch.nn as nn
from torch.optim import AdamW
from torch.optim.lr_scheduler import CosineAnnealingLR
from tqdm import tqdm
from monai.networks.nets import UNet, BasicUNet, SwinUNETR
from monai.networks.layers import Norm
from monai.losses import DiceCELoss
import pandas as pd
from pathlib import Path
from torch.utils.data import DataLoader
import json
print("β
Step 4 imports loaded!")
# ============================================================================
# LOSS FUNCTIONS
# ============================================================================
class CombinedLoss(nn.Module):
def __init__(self, dice_weight=0.7, ce_weight=0.3, sigmoid=True):
super().__init__()
self.dice_ce_loss = DiceCELoss(
sigmoid=sigmoid, softmax=False, include_background=False,
to_onehot_y=False, lambda_dice=dice_weight, lambda_ce=ce_weight,
)
def forward(self, pred, target):
return self.dice_ce_loss(pred, target)
class RUSLoss(nn.Module):
def __init__(self, weight=0.1, num_bins=10):
super().__init__()
self.weight = weight
self.num_bins = num_bins
def compute_entropy(self, probs):
return -torch.sum(probs * torch.log(probs + 1e-8), dim=1)
def compute_ece(self, probs, targets):
confidences, predictions = torch.max(probs, dim=1)
targets_binary = (targets.squeeze(1) > 0.5).float()
confidences_flat = confidences.flatten()
predictions_flat = predictions.flatten().float()
targets_flat = targets_binary.flatten()
bin_boundaries = torch.linspace(0, 1, self.num_bins + 1, device=probs.device)
ece = 0.0
for i in range(self.num_bins):
in_bin = (confidences_flat > bin_boundaries[i]) & (confidences_flat <= bin_boundaries[i + 1])
if in_bin.sum() > 0:
accuracy_in_bin = (predictions_flat[in_bin] == targets_flat[in_bin]).float().mean()
avg_confidence_in_bin = confidences_flat[in_bin].mean()
ece += torch.abs(avg_confidence_in_bin - accuracy_in_bin) * in_bin.sum().float()
return ece / confidences_flat.numel()
def forward(self, probs, targets, is_small):
entropy = self.compute_entropy(probs)
ece = self.compute_ece(probs, targets)
is_small_mask = is_small.float().unsqueeze(1).unsqueeze(2).expand_as(entropy)
return (is_small_mask * entropy).mean() + ece * self.weight
def compute_metrics(preds, targets, threshold=0.5):
preds_binary = (torch.sigmoid(preds) > threshold).float()
targets_binary = (targets > 0.5).float()
preds_flat = preds_binary.view(-1)
targets_flat = targets_binary.view(-1)
tp = (preds_flat * targets_flat).sum()
fp = (preds_flat * (1 - targets_flat)).sum()
fn = ((1 - preds_flat) * targets_flat).sum()
tn = ((1 - preds_flat) * (1 - targets_flat)).sum()
return {
'dice': ((2 * tp + 1e-8) / (2 * tp + fp + fn + 1e-8)).item(),
'sensitivity': ((tp + 1e-8) / (tp + fn + 1e-8)).item(),
'specificity': ((tn + 1e-8) / (tn + fp + 1e-8)).item(),
'precision': ((tp + 1e-8) / (tp + fp + 1e-8)).item(),
'iou': ((tp + 1e-8) / (tp + fp + fn + 1e-8)).item(),
}
def train_epoch(model, dataloader, optimizer, dice_ce_loss, rus_loss, device, use_rus, epoch):
model.train()
total_loss = total_dice = total_sens = 0.0
num_batches = 0
for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Train]"):
images = batch['image'].to(device)
masks = batch['mask'].to(device)
is_small = batch['is_small'].to(device)
optimizer.zero_grad()
logits = model(images)
loss_dice_ce = dice_ce_loss(logits, masks)
loss_rus = 0.0
if use_rus:
probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1)
loss_rus = rus_loss(probs, masks, is_small)
total_loss_batch = loss_dice_ce + loss_rus
total_loss_batch.backward()
optimizer.step()
metrics = compute_metrics(logits, masks)
total_loss += total_loss_batch.item()
total_dice += metrics['dice']
total_sens += metrics['sensitivity']
num_batches += 1
return {'loss': total_loss/num_batches, 'dice': total_dice/num_batches, 'sensitivity': total_sens/num_batches}
def validate_epoch(model, dataloader, dice_ce_loss, rus_loss, device, use_rus, epoch):
model.eval()
total_loss = total_dice = total_sens = total_spec = total_prec = total_iou = 0.0
num_batches = 0
with torch.no_grad():
for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Val]"):
images = batch['image'].to(device)
masks = batch['mask'].to(device)
is_small = batch['is_small'].to(device)
logits = model(images)
loss_dice_ce = dice_ce_loss(logits, masks)
loss_rus = 0.0
if use_rus:
probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1)
loss_rus = rus_loss(probs, masks, is_small)
total_loss_batch = loss_dice_ce + loss_rus
metrics = compute_metrics(logits, masks)
total_loss += total_loss_batch.item()
total_dice += metrics['dice']
total_sens += metrics['sensitivity']
total_spec += metrics['specificity']
total_prec += metrics['precision']
total_iou += metrics['iou']
num_batches += 1
return {
'loss': total_loss/num_batches, 'dice': total_dice/num_batches,
'sensitivity': total_sens/num_batches, 'specificity': total_spec/num_batches,
'precision': total_prec/num_batches, 'iou': total_iou/num_batches,
}
# ============================================================================
# DEFINE 3 MODEL ARCHITECTURES
# ============================================================================
class Model1_UNet(nn.Module):
def __init__(self):
super().__init__()
self.unet = UNet(spatial_dims=2, in_channels=1, out_channels=1,
channels=(32, 64, 128, 256, 512), strides=(2, 2, 2, 2),
num_res_units=2, norm=Norm.BATCH, dropout=0.1)
def forward(self, x):
return self.unet(x)
class Model2_nnUNet(nn.Module):
def __init__(self):
super().__init__()
self.basicunet = BasicUNet(spatial_dims=2, in_channels=1, out_channels=1,
features=(32, 64, 128, 256, 512, 32), dropout=0.1, norm='batch')
def forward(self, x):
return self.basicunet(x)
class Model3_SwinUNet(nn.Module):
def __init__(self):
super().__init__()
self.swinunetr = SwinUNETR(img_size=(128, 128), in_channels=1, out_channels=1,
feature_size=24, depths=(2, 2, 2, 2),
num_heads=(3, 6, 12, 24), norm_name='batch', dropout_path_rate=0.1)
def forward(self, x):
return self.swinunetr(x)
# ============================================================================
# TRAINING FUNCTION WITH CHECKPOINT RESUMPTION
# ============================================================================
def train_model(model, train_loader, val_loader, model_name, num_epochs=50):
device = torch.device('cuda')
model = model.to(device)
save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/")
save_dir.mkdir(parents=True, exist_ok=True)
checkpoint_path = save_dir / f"{model_name}_checkpoint.pth"
best_model_path = save_dir / f"{model_name}_best.pth"
history_path = save_dir / f"{model_name}_history.json"
# ========================================================================
# CHECKPOINT RESUMPTION LOGIC
# ========================================================================
start_epoch = 1
best_dice = 0.0
history = {'val_dice': [], 'val_sensitivity': [], 'val_specificity': [],
'val_precision': [], 'val_iou': [], 'train_dice': [], 'train_loss': []}
# Try to load checkpoint
if checkpoint_path.exists():
print(f"\nπ Found checkpoint! Resuming from: {checkpoint_path}")
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint['model_state_dict'])
start_epoch = checkpoint['epoch'] + 1
best_dice = checkpoint['best_dice']
# Load history if exists
if history_path.exists():
with open(history_path, 'r') as f:
history = json.load(f)
print(f"β
Resumed from epoch {checkpoint['epoch']}")
print(f" Best Dice so far: {best_dice:.4f}")
print(f" Continuing from epoch {start_epoch}/{num_epochs}")
else:
print(f"\nπ Starting fresh training: {model_name}")
print(f"{'='*70}")
print(f"TRAINING: {model_name}")
print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")
print(f"Epochs: {start_epoch} to {num_epochs}")
print(f"{'='*70}")
# Initialize optimizer and scheduler
dice_ce_loss = CombinedLoss()
rus_loss_fn = RUSLoss()
optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5)
scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-6)
# Restore optimizer and scheduler if resuming
if checkpoint_path.exists():
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
# ========================================================================
# TRAINING LOOP
# ========================================================================
for epoch in range(start_epoch, num_epochs + 1):
train_metrics = train_epoch(model, train_loader, optimizer, dice_ce_loss,
rus_loss_fn, device, True, epoch)
val_metrics = validate_epoch(model, val_loader, dice_ce_loss, rus_loss_fn,
device, True, epoch)
scheduler.step()
# Print progress
print(f"Epoch {epoch}/{num_epochs} - Train: L={train_metrics['loss']:.4f}, D={train_metrics['dice']:.4f} | "
f"Val: D={val_metrics['dice']:.4f}, S={val_metrics['sensitivity']:.4f}, P={val_metrics['precision']:.4f}")
# Update history
history['train_loss'].append(train_metrics['loss'])
history['train_dice'].append(train_metrics['dice'])
for k in ['val_dice', 'val_sensitivity', 'val_specificity', 'val_precision', 'val_iou']:
history[k].append(val_metrics[k.replace('val_', '')])
# Save best model
if val_metrics['dice'] > best_dice:
best_dice = val_metrics['dice']
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'dice': best_dice,
'metrics': val_metrics,
}, best_model_path)
print(f" β
New best model saved! Dice: {best_dice:.4f}")
# ====================================================================
# SAVE CHECKPOINT EVERY EPOCH (for resumption)
# ====================================================================
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scheduler_state_dict': scheduler.state_dict(),
'best_dice': best_dice,
}, checkpoint_path)
# Save history as JSON
with open(history_path, 'w') as f:
json.dump(history, f)
print(f" πΎ Checkpoint saved (epoch {epoch})")
print(f"\nβ
Training complete! Best Dice: {best_dice:.4f}")
return model, history
# ============================================================================
# PREPARE CENTRAL TRAINING DATA
# ============================================================================
print("\nπ Preparing central training data...")
data_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/")
# Combine all 5 IID client data
all_train_data = []
for i in range(5):
df = pd.read_csv(data_dir / f"client_{i}_iid.csv")
all_train_data.append(df)
print(f"β
Client {i}: {len(df)} samples")
central_train_df = pd.concat(all_train_data, ignore_index=True)
test_df = pd.read_csv(data_dir / "test_iid.csv")
print(f"\nβ
Central: {len(central_train_df)} train, {len(test_df)} test")
# Create central dataloaders
central_train_dataset = LIDCNoduleDataset(central_train_df, data_dir, is_train=True, augment_rare=True)
central_train_loader = DataLoader(central_train_dataset, batch_size=16, shuffle=True, num_workers=2, pin_memory=True)
central_test_dataset = LIDCNoduleDataset(test_df, data_dir, is_train=False, augment_rare=False)
central_test_loader = DataLoader(central_test_dataset, batch_size=16, shuffle=False, num_workers=2, pin_memory=True)
# ============================================================================
# TRAIN ALL 3 MODELS (WITH AUTO-RESUME)
# ============================================================================
print("\n" + "="*70)
print("STEP 4: TRAINING ALL 3 MODELS (WITH CHECKPOINT RESUMPTION)")
print("="*70)
# 4.1: U-Net
print("\nπ΅ Model 1: U-Net")
model1, hist1 = train_model(Model1_UNet(), central_train_loader, central_test_loader, "step4.1_unet", 50)
# 4.2: nnU-Net
print("\nπ’ Model 2: nnU-Net")
model2, hist2 = train_model(Model2_nnUNet(), central_train_loader, central_test_loader, "step4.2_nnunet", 50)
# 4.3: Swin-UNet
print("\nπ΄ Model 3: Swin-UNet")
model3, hist3 = train_model(Model3_SwinUNet(), central_train_loader, central_test_loader, "step4.3_swinunet", 50)
# ============================================================================
# COMPARE RESULTS
# ============================================================================
comparison = pd.DataFrame({
'Model': ['4.1: U-Net', '4.2: nnU-Net', '4.3: Swin-UNet'],
'Dice': [max(hist1['val_dice']), max(hist2['val_dice']), max(hist3['val_dice'])],
'Sensitivity': [max(hist1['val_sensitivity']), max(hist2['val_sensitivity']), max(hist3['val_sensitivity'])],
'Precision': [max(hist1['val_precision']), max(hist2['val_precision']), max(hist3['val_precision'])],
'IoU': [max(hist1['val_iou']), max(hist2['val_iou']), max(hist3['val_iou'])],
})
print("\n" + "="*70)
print("FINAL COMPARISON")
print("="*70)
print(comparison.to_string(index=False))
save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/")
comparison.to_csv(save_dir / 'step4_comparison.csv', index=False)
print(f"\nβ
STEP 4 COMPLETE!")
print(f"π Best Model: {comparison.loc[comparison['Dice'].idxmax(), 'Model']}")
print(f" Dice: {comparison['Dice'].max():.4f}")
print(f" Sensitivity: {comparison.loc[comparison['Dice'].idxmax(), 'Sensitivity']:.4f}")
print(f"π Saved to: {save_dir}")
print("\nπ Files saved:")
print(" β’ step4.1_unet_best.pth (best model)")
print(" β’ step4.1_unet_checkpoint.pth (resume from here)")
print(" β’ step4.1_unet_history.json (training curves)")
print(" β’ step4.2_nnunet_* (same structure)")
print(" β’ step4.3_swinunet_* (same structure)")
# -*- coding: utf-8 -*-
"""
STEP 4: Train All 3 Baseline Models (WITH CHECKPOINT RESUMPTION)
Auto-resumes from last saved checkpoint if training was interrupted
"""
# ============================================================================
# ADDITIONAL IMPORTS FOR STEP 4
# ============================================================================
import torch
import torch.nn as nn
from torch.optim import AdamW
from torch.optim.lr_scheduler import CosineAnnealingLR
from tqdm import tqdm
from monai.networks.nets import UNet, BasicUNet, SwinUNETR
from monai.networks.layers import Norm
from monai.losses import DiceCELoss
import pandas as pd
from pathlib import Path
from torch.utils.data import DataLoader
import json
print("β
Step 4 imports loaded!")
# ============================================================================
# LOSS FUNCTIONS
# ============================================================================
class CombinedLoss(nn.Module):
def __init__(self, dice_weight=0.7, ce_weight=0.3, sigmoid=True):
super().__init__()
self.dice_ce_loss = DiceCELoss(
sigmoid=sigmoid, softmax=False, include_background=False,
to_onehot_y=False, lambda_dice=dice_weight, lambda_ce=ce_weight,
)
def forward(self, pred, target):
return self.dice_ce_loss(pred, target)
class RUSLoss(nn.Module):
def __init__(self, weight=0.1, num_bins=10):
super().__init__()
self.weight = weight
self.num_bins = num_bins
def compute_entropy(self, probs):
return -torch.sum(probs * torch.log(probs + 1e-8), dim=1)
def compute_ece(self, probs, targets):
confidences, predictions = torch.max(probs, dim=1)
targets_binary = (targets.squeeze(1) > 0.5).float()
confidences_flat = confidences.flatten()
predictions_flat = predictions.flatten().float()
targets_flat = targets_binary.flatten()
bin_boundaries = torch.linspace(0, 1, self.num_bins + 1, device=probs.device)
ece = 0.0
for i in range(self.num_bins):
in_bin = (confidences_flat > bin_boundaries[i]) & (confidences_flat <= bin_boundaries[i + 1])
if in_bin.sum() > 0:
accuracy_in_bin = (predictions_flat[in_bin] == targets_flat[in_bin]).float().mean()
avg_confidence_in_bin = confidences_flat[in_bin].mean()
ece += torch.abs(avg_confidence_in_bin - accuracy_in_bin) * in_bin.sum().float()
return ece / confidences_flat.numel()
def forward(self, probs, targets, is_small):
entropy = self.compute_entropy(probs)
ece = self.compute_ece(probs, targets)
is_small_mask = is_small.float().unsqueeze(1).unsqueeze(2).expand_as(entropy)
return (is_small_mask * entropy).mean() + ece * self.weight
def compute_metrics(preds, targets, threshold=0.5):
preds_binary = (torch.sigmoid(preds) > threshold).float()
targets_binary = (targets > 0.5).float()
preds_flat = preds_binary.view(-1)
targets_flat = targets_binary.view(-1)
tp = (preds_flat * targets_flat).sum()
fp = (preds_flat * (1 - targets_flat)).sum()
fn = ((1 - preds_flat) * targets_flat).sum()
tn = ((1 - preds_flat) * (1 - targets_flat)).sum()
return {
'dice': ((2 * tp + 1e-8) / (2 * tp + fp + fn + 1e-8)).item(),
'sensitivity': ((tp + 1e-8) / (tp + fn + 1e-8)).item(),
'specificity': ((tn + 1e-8) / (tn + fp + 1e-8)).item(),
'precision': ((tp + 1e-8) / (tp + fp + 1e-8)).item(),
'iou': ((tp + 1e-8) / (tp + fp + fn + 1e-8)).item(),
}
def train_epoch(model, dataloader, optimizer, dice_ce_loss, rus_loss, device, use_rus, epoch):
model.train()
total_loss = total_dice = total_sens = 0.0
num_batches = 0
for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Train]"):
images = batch['image'].to(device)
masks = batch['mask'].to(device)
is_small = batch['is_small'].to(device)
optimizer.zero_grad()
logits = model(images)
loss_dice_ce = dice_ce_loss(logits, masks)
loss_rus = 0.0
if use_rus:
probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1)
loss_rus = rus_loss(probs, masks, is_small)
total_loss_batch = loss_dice_ce + loss_rus
total_loss_batch.backward()
optimizer.step()
metrics = compute_metrics(logits, masks)
total_loss += total_loss_batch.item()
total_dice += metrics['dice']
total_sens += metrics['sensitivity']
num_batches += 1
return {'loss': total_loss/num_batches, 'dice': total_dice/num_batches, 'sensitivity': total_sens/num_batches}
def validate_epoch(model, dataloader, dice_ce_loss, rus_loss, device, use_rus, epoch):
model.eval()
total_loss = total_dice = total_sens = total_spec = total_prec = total_iou = 0.0
num_batches = 0
with torch.no_grad():
for batch in tqdm(dataloader, desc=f"Epoch {epoch} [Val]"):
images = batch['image'].to(device)
masks = batch['mask'].to(device)
is_small = batch['is_small'].to(device)
logits = model(images)
loss_dice_ce = dice_ce_loss(logits, masks)
loss_rus = 0.0
if use_rus:
probs = torch.softmax(torch.cat([1 - torch.sigmoid(logits), torch.sigmoid(logits)], dim=1), dim=1)
loss_rus = rus_loss(probs, masks, is_small)
total_loss_batch = loss_dice_ce + loss_rus
metrics = compute_metrics(logits, masks)
total_loss += total_loss_batch.item()
total_dice += metrics['dice']
total_sens += metrics['sensitivity']
total_spec += metrics['specificity']
total_prec += metrics['precision']
total_iou += metrics['iou']
num_batches += 1
return {
'loss': total_loss/num_batches, 'dice': total_dice/num_batches,
'sensitivity': total_sens/num_batches, 'specificity': total_spec/num_batches,
'precision': total_prec/num_batches, 'iou': total_iou/num_batches,
}
# ============================================================================
# DEFINE 3 MODEL ARCHITECTURES (FIXED SwinUNETR)
# ============================================================================
class Model1_UNet(nn.Module):
def __init__(self):
super().__init__()
self.unet = UNet(spatial_dims=2, in_channels=1, out_channels=1,
channels=(32, 64, 128, 256, 512), strides=(2, 2, 2, 2),
num_res_units=2, norm=Norm.BATCH, dropout=0.1)
def forward(self, x):
return self.unet(x)
class Model2_nnUNet(nn.Module):
def __init__(self):
super().__init__()
self.basicunet = BasicUNet(spatial_dims=2, in_channels=1, out_channels=1,
features=(32, 64, 128, 256, 512, 32), dropout=0.1, norm='batch')
def forward(self, x):
return self.basicunet(x)
class Model3_SwinUNet(nn.Module):
def __init__(self):
super().__init__()
self.swinunetr = SwinUNETR(
spatial_dims=2, # β
FIXED: Added spatial_dims for 2D
in_channels=1, # β
FIXED: Removed img_size (not supported in MONAI >=1.5)
out_channels=1,
feature_size=24,
depths=(2, 2, 2, 2),
num_heads=(3, 6, 12, 24),
norm_name='batch',
dropout_path_rate=0.1
)
def forward(self, x):
return self.swinunetr(x)
# ============================================================================
# TRAINING FUNCTION WITH CHECKPOINT RESUMPTION
# ============================================================================
def train_model(model, train_loader, val_loader, model_name, num_epochs=50):
device = torch.device('cuda')
model = model.to(device)
save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/")
save_dir.mkdir(parents=True, exist_ok=True)
checkpoint_path = save_dir / f"{model_name}_checkpoint.pth"
best_model_path = save_dir / f"{model_name}_best.pth"
history_path = save_dir / f"{model_name}_history.json"
# ========================================================================
# CHECKPOINT RESUMPTION LOGIC
# ========================================================================
start_epoch = 1
best_dice = 0.0
history = {'val_dice': [], 'val_sensitivity': [], 'val_specificity': [],
'val_precision': [], 'val_iou': [], 'train_dice': [], 'train_loss': []}
# Try to load checkpoint
if checkpoint_path.exists():
print(f"\nπ Found checkpoint! Resuming from: {checkpoint_path}")
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint['model_state_dict'])
start_epoch = checkpoint['epoch'] + 1
best_dice = checkpoint['best_dice']
# Load history if exists
if history_path.exists():
with open(history_path, 'r') as f:
history = json.load(f)
print(f"β
Resumed from epoch {checkpoint['epoch']}")
print(f" Best Dice so far: {best_dice:.4f}")
print(f" Continuing from epoch {start_epoch}/{num_epochs}")
else:
print(f"\nπ Starting fresh training: {model_name}")
print(f"{'='*70}")
print(f"TRAINING: {model_name}")
print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")
print(f"Epochs: {start_epoch} to {num_epochs}")
print(f"{'='*70}")
# Initialize optimizer and scheduler
dice_ce_loss = CombinedLoss()
rus_loss_fn = RUSLoss()
optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5)
scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-6)
# Restore optimizer and scheduler if resuming
if checkpoint_path.exists():
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
# ========================================================================
# TRAINING LOOP
# ========================================================================
for epoch in range(start_epoch, num_epochs + 1):
train_metrics = train_epoch(model, train_loader, optimizer, dice_ce_loss,
rus_loss_fn, device, True, epoch)
val_metrics = validate_epoch(model, val_loader, dice_ce_loss, rus_loss_fn,
device, True, epoch)
scheduler.step()
# Print progress
print(f"Epoch {epoch}/{num_epochs} - Train: L={train_metrics['loss']:.4f}, D={train_metrics['dice']:.4f} | "
f"Val: D={val_metrics['dice']:.4f}, S={val_metrics['sensitivity']:.4f}, P={val_metrics['precision']:.4f}")
# Update history
history['train_loss'].append(train_metrics['loss'])
history['train_dice'].append(train_metrics['dice'])
for k in ['val_dice', 'val_sensitivity', 'val_specificity', 'val_precision', 'val_iou']:
history[k].append(val_metrics[k.replace('val_', '')])
# Save best model
if val_metrics['dice'] > best_dice:
best_dice = val_metrics['dice']
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'dice': best_dice,
'metrics': val_metrics,
}, best_model_path)
print(f" β
New best model saved! Dice: {best_dice:.4f}")
# ====================================================================
# SAVE CHECKPOINT EVERY EPOCH (for resumption)
# ====================================================================
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scheduler_state_dict': scheduler.state_dict(),
'best_dice': best_dice,
}, checkpoint_path)
# Save history as JSON
with open(history_path, 'w') as f:
json.dump(history, f)
print(f" πΎ Checkpoint saved (epoch {epoch})")
print(f"\nβ
Training complete! Best Dice: {best_dice:.4f}")
return model, history
# ============================================================================
# PREPARE CENTRAL TRAINING DATA
# ============================================================================
print("\nπ Preparing central training data...")
data_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/preprocessed_lidc/")
# Combine all 5 IID client data
all_train_data = []
for i in range(5):
df = pd.read_csv(data_dir / f"client_{i}_iid.csv")
all_train_data.append(df)
print(f"β
Client {i}: {len(df)} samples")
central_train_df = pd.concat(all_train_data, ignore_index=True)
test_df = pd.read_csv(data_dir / "test_iid.csv")
print(f"\nβ
Central: {len(central_train_df)} train, {len(test_df)} test")
# Create central dataloaders
central_train_dataset = LIDCNoduleDataset(central_train_df, data_dir, is_train=True, augment_rare=True)
central_train_loader = DataLoader(central_train_dataset, batch_size=16, shuffle=True, num_workers=2, pin_memory=True)
central_test_dataset = LIDCNoduleDataset(test_df, data_dir, is_train=False, augment_rare=False)
central_test_loader = DataLoader(central_test_dataset, batch_size=16, shuffle=False, num_workers=2, pin_memory=True)
# ============================================================================
# TRAIN ALL 3 MODELS (WITH AUTO-RESUME)
# ============================================================================
print("\n" + "="*70)
print("STEP 4: TRAINING ALL 3 MODELS (WITH CHECKPOINT RESUMPTION)")
print("="*70)
# 4.1: U-Net
print("\nπ΅ Model 1: U-Net")
model1, hist1 = train_model(Model1_UNet(), central_train_loader, central_test_loader, "step4.1_unet", 50)
# 4.2: nnU-Net
print("\nπ’ Model 2: nnU-Net")
model2, hist2 = train_model(Model2_nnUNet(), central_train_loader, central_test_loader, "step4.2_nnunet", 50)
# 4.3: Swin-UNet (FIXED)
print("\nπ΄ Model 3: Swin-UNet")
model3, hist3 = train_model(Model3_SwinUNet(), central_train_loader, central_test_loader, "step4.3_swinunet", 50)
# ============================================================================
# COMPARE RESULTS
# ============================================================================
comparison = pd.DataFrame({
'Model': ['4.1: U-Net', '4.2: nnU-Net', '4.3: Swin-UNet'],
'Dice': [max(hist1['val_dice']), max(hist2['val_dice']), max(hist3['val_dice'])],
'Sensitivity': [max(hist1['val_sensitivity']), max(hist2['val_sensitivity']), max(hist3['val_sensitivity'])],
'Precision': [max(hist1['val_precision']), max(hist2['val_precision']), max(hist3['val_precision'])],
'IoU': [max(hist1['val_iou']), max(hist2['val_iou']), max(hist3['val_iou'])],
})
print("\n" + "="*70)
print("FINAL COMPARISON")
print("="*70)
print(comparison.to_string(index=False))
save_dir = Path("/content/drive/MyDrive/Rare Lung Nodules/models/")
comparison.to_csv(save_dir / 'step4_comparison.csv', index=False)
print(f"\nβ
STEP 4 COMPLETE!")
print(f"π Best Model: {comparison.loc[comparison['Dice'].idxmax(), 'Model']}")
print(f" Dice: {comparison['Dice'].max():.4f}")
print(f" Sensitivity: {comparison.loc[comparison['Dice'].idxmax(), 'Sensitivity']:.4f}")
print(f"π Saved to: {save_dir}")
print("\nπ Files saved:")
print(" β’ step4.1_unet_best.pth (best model)")
print(" β’ step4.1_unet_checkpoint.pth (resume from here)")
print(" β’ step4.1_unet_history.json (training curves)")
print(" β’ step4.2_nnunet_* (same structure)")
print(" β’ step4.3_swinunet_* (same structure)")
# -*- coding: utf-8 -*-
"""lidc_step2_step3_step4.ipynb
Automatically generated by Colab.
Original file is located at
https://colab.research.google.com/drive/19ND1Fka8Cromhk7B_hYQzGWayIbTl2eH
"""
# -*- coding: utf-8 -*-
"""
Step 2 & 3: PyTorch Dataset Loader with Data Augmentation + FL Client Splits
CORRECTED VERSION - With Excel-based nodule size classification
"""
# ============================================================================
# STEP 0: MOUNT GOOGLE DRIVE
# ============================================================================
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
print("β
Google Drive mounted successfully!")
print("π Your files are accessible at: /content/drive/MyDrive/")
# ============================================================================
# INSTALL CORRECT DEPENDENCIES
# ============================================================================
print("π¦ Installing compatible dependencies...")
# Uninstall any existing conflicting versions
!pip uninstall -y torch torchvision torchaudio monai -q
# Install PyTorch 2.6.0 with CUDA 12.1 (Colab default)
!pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu121 -q
# Install MONAI 1.5.0 (supports PyTorch 2.6.0)
!pip install monai==1.5.0 -q
# Install other dependencies (including openpyxl for Excel reading)
!pip install numpy==1.26.4 pandas==2.2.0 scikit-learn==1.4.0 openpyxl==3.1.2 -q
print("β
Dependencies installed!")
print("β οΈ IMPORTANT: Click 'Runtime' β 'Restart runtime' before proceeding!")
# ============================================================================
# VERIFY INSTALLATIONS (Run after restart)
# ============================================================================
import torch
import monai
import numpy as np
import pandas as pd
from sklearn import __version__ as sklearn_version
print("="*70)
print("DEPENDENCY VERIFICATION")
print("="*70)
print(f"β
PyTorch version: {torch.__version__}")
print(f"β
MONAI version: {monai.__version__}")
print(f"β
NumPy version: {np.__version__}")
print(f"β
Pandas version: {pd.__version__}")
print(f"β
Scikit-learn version: {sklearn_version}")
print(f"β
CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
print(f"β
CUDA version: {torch.version.cuda}")
print(f"β
GPU: {torch.cuda.get_device_name(0)}")
print("="*70)
# ============================================================================
# IMPORTS
# ============================================================================
from pathlib import Path
from typing import Dict, List, Tuple, Optional
from torch.utils.data import Dataset, DataLoader
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings('ignore')
# MONAI transforms for medical imaging
from monai.transforms import (
Compose,
RandFlipd,
RandRotate90d,
RandAffined,
RandAdjustContrastd,
RandGaussianNoised,
ToTensord,
ScaleIntensityd,
)
print("β
All imports successful!")
# ============================================================================
# LOAD NODULE SIZE INFORMATION FROM EXCEL
# ============================================================================
def load_nodule_sizes(excel_path: str) -> Dict[str, float]:
"""
Load nodule size information from LIDC-IDRI Excel file.
Args: