File size: 11,623 Bytes
8d7b4b1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
"""Training utilities for small dataset optimization.

Implements advanced techniques:
- Exponential Moving Average (EMA) weights
- Checkpoint averaging
- Curriculum learning
- Gradient accumulation
- Warmup scheduling
"""

from __future__ import annotations

import torch
import torch.nn as nn
from pathlib import Path
from typing import Optional, List
import copy


class EMA(nn.Module):
    """Exponential Moving Average for model weights.

    Maintains a running average of model weights for better generalization,
    especially important for small datasets.
    """

    def __init__(self, model: nn.Module, decay: float = 0.999):
        """Initialize EMA wrapper.

        Args:
            model: Model to apply EMA to
            decay: EMA decay rate (higher = slower updates)
        """
        super().__init__()
        self.decay = decay
        self.model = model
        self.shadow = copy.deepcopy(model)
        self.shadow.eval()

        for param in self.shadow.parameters():
            param.requires_grad = False

    def update(self) -> None:
        """Update shadow weights with EMA."""
        for model_param, shadow_param in zip(self.model.parameters(), self.shadow.parameters()):
            shadow_param.data = self.decay * shadow_param.data + (1 - self.decay) * model_param.data.detach()

    def forward(self, *args, **kwargs):
        """Forward pass through shadow model (during inference)."""
        return self.shadow(*args, **kwargs)

    def swap(self) -> None:
        """Swap model and shadow parameters for evaluation."""
        self.model, self.shadow = self.shadow, self.model

    def restore(self) -> None:
        """Restore original model and shadow after evaluation."""
        self.swap()


class CheckpointAveraging:
    """Average weights from multiple checkpoints for improved generalization."""

    def __init__(self, num_checkpoints: int = 5):
        """Initialize checkpoint averaging.

        Args:
            num_checkpoints: Number of last checkpoints to average
        """
        self.num_checkpoints = num_checkpoints
        self.checkpoints: List[dict] = []

    def add_checkpoint(self, state_dict: dict) -> None:
        """Add checkpoint state dict.

        Args:
            state_dict: Model state dict to add
        """
        import copy
        self.checkpoints.append(copy.deepcopy(state_dict))
        # Keep only recent checkpoints
        if len(self.checkpoints) > self.num_checkpoints:
            self.checkpoints.pop(0)

    def get_averaged_weights(self) -> dict:
        """Compute average of checkpoint weights.

        Returns:
            Averaged state dict
        """
        if not self.checkpoints:
            raise ValueError("No checkpoints to average")

        averaged = {}
        num_checkpoints = len(self.checkpoints)

        # Get keys from first checkpoint
        for key in self.checkpoints[0].keys():
            # Average weights
            weight_sum = torch.zeros_like(self.checkpoints[0][key])
            for ckpt in self.checkpoints:
                weight_sum += ckpt[key]
            averaged[key] = weight_sum / num_checkpoints

        return averaged


class WarmupScheduler:
    """Learning rate warmup + decay scheduler.

    Gradually increases LR during warmup phase, then decays.
    Important for small datasets to prevent divergence.
    """

    def __init__(
        self,
        optimizer: torch.optim.Optimizer,
        warmup_epochs: int = 5,
        total_epochs: int = 50,
        base_lr: float = 1e-4,
        min_lr: float = 1e-6,
    ):
        """Initialize scheduler.

        Args:
            optimizer: PyTorch optimizer
            warmup_epochs: Number of epochs for warmup
            total_epochs: Total number of training epochs
            base_lr: Base learning rate
            min_lr: Minimum learning rate
        """
        self.optimizer = optimizer
        self.warmup_epochs = warmup_epochs
        self.total_epochs = total_epochs
        self.base_lr = base_lr
        self.min_lr = min_lr
        self.epoch = 0

    def step(self) -> None:
        """Update learning rate for current epoch."""
        if self.epoch < self.warmup_epochs:
            # Linear warmup
            lr = self.base_lr * (self.epoch + 1) / self.warmup_epochs
        else:
            # Cosine annealing decay
            progress = (self.epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs)
            lr = self.min_lr + (self.base_lr - self.min_lr) * 0.5 * (1 + torch.cos(torch.tensor(progress * 3.14159)).item())

        for param_group in self.optimizer.param_groups:
            param_group["lr"] = lr

        self.epoch += 1

    def get_lr(self) -> float:
        """Get current learning rate."""
        return self.optimizer.param_groups[0]["lr"]


class CurriculumLearning:
    """Curriculum learning scheduler for small datasets.

    Gradually increases task difficulty (e.g., more modality dropout).
    Helps prevent overfitting on small datasets.
    """

    def __init__(
        self,
        initial_dropout_rate: float = 0.0,
        final_dropout_rate: float = 0.2,
        total_epochs: int = 50,
    ):
        """Initialize curriculum.

        Args:
            initial_dropout_rate: Starting dropout rate (easy)
            final_dropout_rate: Final dropout rate (hard)
            total_epochs: Total training epochs
        """
        self.initial_dropout = initial_dropout_rate
        self.final_dropout = final_dropout_rate
        self.total_epochs = total_epochs
        self.epoch = 0

    def get_current_difficulty(self) -> float:
        """Get current difficulty level (0 to 1)."""
        return self.epoch / self.total_epochs

    def get_modality_dropout_rate(self) -> float:
        """Get modality dropout rate for current epoch."""
        difficulty = self.get_current_difficulty()
        return self.initial_dropout + (self.final_dropout - self.initial_dropout) * difficulty

    def get_temporal_dropout_rate(self) -> float:
        """Get temporal dropout rate for current epoch."""
        difficulty = self.get_current_difficulty()
        # Start with less temporal dropout, increase over time
        return difficulty * 0.15

    def step(self) -> None:
        """Advance to next epoch."""
        self.epoch += 1


class GradientAccumulation:
    """Gradient accumulation for large effective batch sizes on small datasets."""

    def __init__(self, accumulation_steps: int = 4):
        """Initialize gradient accumulation.

        Args:
            accumulation_steps: Number of steps to accumulate gradients
        """
        self.accumulation_steps = accumulation_steps
        self.step_count = 0

    def should_accumulate(self) -> bool:
        """Check if should accumulate gradient."""
        return (self.step_count + 1) % self.accumulation_steps != 0

    def should_step(self) -> bool:
        """Check if should perform optimizer step."""
        return (self.step_count + 1) % self.accumulation_steps == 0

    def step(self) -> None:
        """Advance accumulation counter."""
        self.step_count += 1

    def reset(self) -> None:
        """Reset accumulation counter."""
        self.step_count = 0


class MixupAugmentation:
    """Mixup data augmentation for small datasets.

    Creates interpolated samples between training examples.
    """

    def __init__(self, alpha: float = 0.2):
        """Initialize mixup.

        Args:
            alpha: Beta distribution parameter for mixing
        """
        self.alpha = alpha

    def __call__(
        self,
        features1: torch.Tensor,
        features2: torch.Tensor,
        target1: float,
        target2: float,
    ) -> tuple[torch.Tensor, float]:
        """Apply mixup to two samples.

        Args:
            features1: First feature tensor
            features2: Second feature tensor
            target1: First target value
            target2: Second target value

        Returns:
            Tuple of (mixed_features, mixed_target)
        """
        lam = torch.distributions.Beta(self.alpha, self.alpha).sample()
        mixed_features = lam * features1 + (1 - lam) * features2
        mixed_target = lam * target1 + (1 - lam) * target2
        return mixed_features, mixed_target


class ManifoldMixup:
    """Manifold Mixup: apply mixup in hidden layer space.

    More effective than input-space mixup for small datasets.
    """

    def __init__(self, alpha: float = 0.2):
        """Initialize manifold mixup.

        Args:
            alpha: Beta distribution parameter
        """
        self.alpha = alpha

    def __call__(
        self,
        hidden1: torch.Tensor,
        hidden2: torch.Tensor,
        target1: float,
        target2: float,
    ) -> tuple[torch.Tensor, float]:
        """Apply mixup to hidden representations.

        Args:
            hidden1: First hidden representation
            hidden2: Second hidden representation
            target1: First target value
            target2: Second target value

        Returns:
            Tuple of (mixed_hidden, mixed_target)
        """
        lam = torch.distributions.Beta(self.alpha, self.alpha).sample()
        mixed_hidden = lam * hidden1 + (1 - lam) * hidden2
        mixed_target = lam * target1 + (1 - lam) * target2
        return mixed_hidden, mixed_target


class StochasticDepth(nn.Module):
    """Stochastic depth (drop path) for regularization.

    Randomly drops entire residual connections during training.
    """

    def __init__(self, drop_prob: float = 0.1):
        """Initialize stochastic depth.

        Args:
            drop_prob: Probability of dropping path
        """
        super().__init__()
        self.drop_prob = drop_prob

    def forward(self, x: torch.Tensor, residual: torch.Tensor) -> torch.Tensor:
        """Apply stochastic depth.

        Args:
            x: Main path output
            residual: Residual connection

        Returns:
            Output with stochastic depth applied
        """
        if not self.training or self.drop_prob == 0:
            return x + residual

        # Randomly drop path
        keep_prob = 1 - self.drop_prob
        shape = (x.shape[0],) + (1,) * (x.ndim - 1)
        random_tensor = torch.rand(shape, device=x.device)
        random_tensor = (random_tensor < keep_prob).float() / keep_prob
        return (x + residual) * random_tensor


class FocalLoss(nn.Module):
    """Focal loss for class imbalance in binary classification.

    Downweights easy examples, focuses on hard ones.
    """

    def __init__(self, alpha: float = 0.25, gamma: float = 2.0):
        """Initialize focal loss.

        Args:
            alpha: Weighting factor in [0, 1]
            gamma: Focusing parameter >= 0
        """
        super().__init__()
        self.alpha = alpha
        self.gamma = gamma

    def forward(
        self,
        predictions: torch.Tensor,
        targets: torch.Tensor,
    ) -> torch.Tensor:
        """Compute focal loss.

        Args:
            predictions: Logits [batch_size]
            targets: Binary targets [batch_size]

        Returns:
            Focal loss
        """
        # Convert logits to probabilities
        p = torch.sigmoid(predictions)

        # Compute cross entropy
        ce = torch.nn.functional.binary_cross_entropy_with_logits(predictions, targets, reduction="none")

        # Compute focal loss
        p_t = p * targets + (1 - p) * (1 - targets)
        focal_loss = self.alpha * ((1 - p_t) ** self.gamma) * ce

        return focal_loss.mean()