Spaces:
Sleeping
Sleeping
Download src/training/optimization.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/training/optimization.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/training/optimization.py
-
curl -L -o optimization.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/training/optimization.py
11.6 kB
| """Training utilities for small dataset optimization. | |
| Implements advanced techniques: | |
| - Exponential Moving Average (EMA) weights | |
| - Checkpoint averaging | |
| - Curriculum learning | |
| - Gradient accumulation | |
| - Warmup scheduling | |
| """ | |
| from __future__ import annotations | |
| import torch | |
| import torch.nn as nn | |
| from pathlib import Path | |
| from typing import Optional, List | |
| import copy | |
| class EMA(nn.Module): | |
| """Exponential Moving Average for model weights. | |
| Maintains a running average of model weights for better generalization, | |
| especially important for small datasets. | |
| """ | |
| def __init__(self, model: nn.Module, decay: float = 0.999): | |
| """Initialize EMA wrapper. | |
| Args: | |
| model: Model to apply EMA to | |
| decay: EMA decay rate (higher = slower updates) | |
| """ | |
| super().__init__() | |
| self.decay = decay | |
| self.model = model | |
| self.shadow = copy.deepcopy(model) | |
| self.shadow.eval() | |
| for param in self.shadow.parameters(): | |
| param.requires_grad = False | |
| def update(self) -> None: | |
| """Update shadow weights with EMA.""" | |
| for model_param, shadow_param in zip(self.model.parameters(), self.shadow.parameters()): | |
| shadow_param.data = self.decay * shadow_param.data + (1 - self.decay) * model_param.data.detach() | |
| def forward(self, *args, **kwargs): | |
| """Forward pass through shadow model (during inference).""" | |
| return self.shadow(*args, **kwargs) | |
| def swap(self) -> None: | |
| """Swap model and shadow parameters for evaluation.""" | |
| self.model, self.shadow = self.shadow, self.model | |
| def restore(self) -> None: | |
| """Restore original model and shadow after evaluation.""" | |
| self.swap() | |
| class CheckpointAveraging: | |
| """Average weights from multiple checkpoints for improved generalization.""" | |
| def __init__(self, num_checkpoints: int = 5): | |
| """Initialize checkpoint averaging. | |
| Args: | |
| num_checkpoints: Number of last checkpoints to average | |
| """ | |
| self.num_checkpoints = num_checkpoints | |
| self.checkpoints: List[dict] = [] | |
| def add_checkpoint(self, state_dict: dict) -> None: | |
| """Add checkpoint state dict. | |
| Args: | |
| state_dict: Model state dict to add | |
| """ | |
| import copy | |
| self.checkpoints.append(copy.deepcopy(state_dict)) | |
| # Keep only recent checkpoints | |
| if len(self.checkpoints) > self.num_checkpoints: | |
| self.checkpoints.pop(0) | |
| def get_averaged_weights(self) -> dict: | |
| """Compute average of checkpoint weights. | |
| Returns: | |
| Averaged state dict | |
| """ | |
| if not self.checkpoints: | |
| raise ValueError("No checkpoints to average") | |
| averaged = {} | |
| num_checkpoints = len(self.checkpoints) | |
| # Get keys from first checkpoint | |
| for key in self.checkpoints[0].keys(): | |
| # Average weights | |
| weight_sum = torch.zeros_like(self.checkpoints[0][key]) | |
| for ckpt in self.checkpoints: | |
| weight_sum += ckpt[key] | |
| averaged[key] = weight_sum / num_checkpoints | |
| return averaged | |
| class WarmupScheduler: | |
| """Learning rate warmup + decay scheduler. | |
| Gradually increases LR during warmup phase, then decays. | |
| Important for small datasets to prevent divergence. | |
| """ | |
| def __init__( | |
| self, | |
| optimizer: torch.optim.Optimizer, | |
| warmup_epochs: int = 5, | |
| total_epochs: int = 50, | |
| base_lr: float = 1e-4, | |
| min_lr: float = 1e-6, | |
| ): | |
| """Initialize scheduler. | |
| Args: | |
| optimizer: PyTorch optimizer | |
| warmup_epochs: Number of epochs for warmup | |
| total_epochs: Total number of training epochs | |
| base_lr: Base learning rate | |
| min_lr: Minimum learning rate | |
| """ | |
| self.optimizer = optimizer | |
| self.warmup_epochs = warmup_epochs | |
| self.total_epochs = total_epochs | |
| self.base_lr = base_lr | |
| self.min_lr = min_lr | |
| self.epoch = 0 | |
| def step(self) -> None: | |
| """Update learning rate for current epoch.""" | |
| if self.epoch < self.warmup_epochs: | |
| # Linear warmup | |
| lr = self.base_lr * (self.epoch + 1) / self.warmup_epochs | |
| else: | |
| # Cosine annealing decay | |
| progress = (self.epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs) | |
| lr = self.min_lr + (self.base_lr - self.min_lr) * 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item()) | |
| for param_group in self.optimizer.param_groups: | |
| param_group["lr"] = lr | |
| self.epoch += 1 | |
| def get_lr(self) -> float: | |
| """Get current learning rate.""" | |
| return self.optimizer.param_groups[0]["lr"] | |
| class CurriculumLearning: | |
| """Curriculum learning scheduler for small datasets. | |
| Gradually increases task difficulty (e.g., more modality dropout). | |
| Helps prevent overfitting on small datasets. | |
| """ | |
| def __init__( | |
| self, | |
| initial_dropout_rate: float = 0.0, | |
| final_dropout_rate: float = 0.2, | |
| total_epochs: int = 50, | |
| ): | |
| """Initialize curriculum. | |
| Args: | |
| initial_dropout_rate: Starting dropout rate (easy) | |
| final_dropout_rate: Final dropout rate (hard) | |
| total_epochs: Total training epochs | |
| """ | |
| self.initial_dropout = initial_dropout_rate | |
| self.final_dropout = final_dropout_rate | |
| self.total_epochs = total_epochs | |
| self.epoch = 0 | |
| def get_current_difficulty(self) -> float: | |
| """Get current difficulty level (0 to 1).""" | |
| return self.epoch / self.total_epochs | |
| def get_modality_dropout_rate(self) -> float: | |
| """Get modality dropout rate for current epoch.""" | |
| difficulty = self.get_current_difficulty() | |
| return self.initial_dropout + (self.final_dropout - self.initial_dropout) * difficulty | |
| def get_temporal_dropout_rate(self) -> float: | |
| """Get temporal dropout rate for current epoch.""" | |
| difficulty = self.get_current_difficulty() | |
| # Start with less temporal dropout, increase over time | |
| return difficulty * 0.15 | |
| def step(self) -> None: | |
| """Advance to next epoch.""" | |
| self.epoch += 1 | |
| class GradientAccumulation: | |
| """Gradient accumulation for large effective batch sizes on small datasets.""" | |
| def __init__(self, accumulation_steps: int = 4): | |
| """Initialize gradient accumulation. | |
| Args: | |
| accumulation_steps: Number of steps to accumulate gradients | |
| """ | |
| self.accumulation_steps = accumulation_steps | |
| self.step_count = 0 | |
| def should_accumulate(self) -> bool: | |
| """Check if should accumulate gradient.""" | |
| return (self.step_count + 1) % self.accumulation_steps != 0 | |
| def should_step(self) -> bool: | |
| """Check if should perform optimizer step.""" | |
| return (self.step_count + 1) % self.accumulation_steps == 0 | |
| def step(self) -> None: | |
| """Advance accumulation counter.""" | |
| self.step_count += 1 | |
| def reset(self) -> None: | |
| """Reset accumulation counter.""" | |
| self.step_count = 0 | |
| class MixupAugmentation: | |
| """Mixup data augmentation for small datasets. | |
| Creates interpolated samples between training examples. | |
| """ | |
| def __init__(self, alpha: float = 0.2): | |
| """Initialize mixup. | |
| Args: | |
| alpha: Beta distribution parameter for mixing | |
| """ | |
| self.alpha = alpha | |
| def __call__( | |
| self, | |
| features1: torch.Tensor, | |
| features2: torch.Tensor, | |
| target1: float, | |
| target2: float, | |
| ) -> tuple[torch.Tensor, float]: | |
| """Apply mixup to two samples. | |
| Args: | |
| features1: First feature tensor | |
| features2: Second feature tensor | |
| target1: First target value | |
| target2: Second target value | |
| Returns: | |
| Tuple of (mixed_features, mixed_target) | |
| """ | |
| lam = torch.distributions.Beta(self.alpha, self.alpha).sample() | |
| mixed_features = lam * features1 + (1 - lam) * features2 | |
| mixed_target = lam * target1 + (1 - lam) * target2 | |
| return mixed_features, mixed_target | |
| class ManifoldMixup: | |
| """Manifold Mixup: apply mixup in hidden layer space. | |
| More effective than input-space mixup for small datasets. | |
| """ | |
| def __init__(self, alpha: float = 0.2): | |
| """Initialize manifold mixup. | |
| Args: | |
| alpha: Beta distribution parameter | |
| """ | |
| self.alpha = alpha | |
| def __call__( | |
| self, | |
| hidden1: torch.Tensor, | |
| hidden2: torch.Tensor, | |
| target1: float, | |
| target2: float, | |
| ) -> tuple[torch.Tensor, float]: | |
| """Apply mixup to hidden representations. | |
| Args: | |
| hidden1: First hidden representation | |
| hidden2: Second hidden representation | |
| target1: First target value | |
| target2: Second target value | |
| Returns: | |
| Tuple of (mixed_hidden, mixed_target) | |
| """ | |
| lam = torch.distributions.Beta(self.alpha, self.alpha).sample() | |
| mixed_hidden = lam * hidden1 + (1 - lam) * hidden2 | |
| mixed_target = lam * target1 + (1 - lam) * target2 | |
| return mixed_hidden, mixed_target | |
| class StochasticDepth(nn.Module): | |
| """Stochastic depth (drop path) for regularization. | |
| Randomly drops entire residual connections during training. | |
| """ | |
| def __init__(self, drop_prob: float = 0.1): | |
| """Initialize stochastic depth. | |
| Args: | |
| drop_prob: Probability of dropping path | |
| """ | |
| super().__init__() | |
| self.drop_prob = drop_prob | |
| def forward(self, x: torch.Tensor, residual: torch.Tensor) -> torch.Tensor: | |
| """Apply stochastic depth. | |
| Args: | |
| x: Main path output | |
| residual: Residual connection | |
| Returns: | |
| Output with stochastic depth applied | |
| """ | |
| if not self.training or self.drop_prob == 0: | |
| return x + residual | |
| # Randomly drop path | |
| keep_prob = 1 - self.drop_prob | |
| shape = (x.shape[0],) + (1,) * (x.ndim - 1) | |
| random_tensor = torch.rand(shape, device=x.device) | |
| random_tensor = (random_tensor < keep_prob).float() / keep_prob | |
| return (x + residual) * random_tensor | |
| class FocalLoss(nn.Module): | |
| """Focal loss for class imbalance in binary classification. | |
| Downweights easy examples, focuses on hard ones. | |
| """ | |
| def __init__(self, alpha: float = 0.25, gamma: float = 2.0): | |
| """Initialize focal loss. | |
| Args: | |
| alpha: Weighting factor in [0, 1] | |
| gamma: Focusing parameter >= 0 | |
| """ | |
| super().__init__() | |
| self.alpha = alpha | |
| self.gamma = gamma | |
| def forward( | |
| self, | |
| predictions: torch.Tensor, | |
| targets: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """Compute focal loss. | |
| Args: | |
| predictions: Logits [batch_size] | |
| targets: Binary targets [batch_size] | |
| Returns: | |
| Focal loss | |
| """ | |
| # Convert logits to probabilities | |
| p = torch.sigmoid(predictions) | |
| # Compute cross entropy | |
| ce = torch.nn.functional.binary_cross_entropy_with_logits(predictions, targets, reduction="none") | |
| # Compute focal loss | |
| p_t = p * targets + (1 - p) * (1 - targets) | |
| focal_loss = self.alpha * ((1 - p_t) ** self.gamma) * ce | |
| return focal_loss.mean() | |