Spaces:
Sleeping
Sleeping
File size: 10,865 Bytes
8d7b4b1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 | from __future__ import annotations
import time
import torch
import torch.nn as nn
from pathlib import Path
from typing import Dict, Optional
from torch.utils.data import DataLoader
from src.training.losses import MultitaskLoss
from src.training.metrics import regression_metrics
import logging
logger = logging.getLogger(__name__)
class EarlyStopping:
def __init__(self, patience: int = 8, min_delta: float = 0.001):
self.patience = patience
self.min_delta = min_delta
self.best = float("inf")
self.counter = 0
def step(self, value: float) -> bool:
if value + self.min_delta < self.best:
self.best = value
self.counter = 0
return False
self.counter += 1
return self.counter >= self.patience
class Trainer:
def __init__(
self,
model: nn.Module,
train_loader: DataLoader,
val_loader: DataLoader,
optimizer: torch.optim.Optimizer,
scheduler: Optional[torch.optim.lr_scheduler._LRScheduler],
config: dict,
device: Optional[torch.device] = None,
):
self.model = model
self.train_loader = train_loader
self.val_loader = val_loader
self.optimizer = optimizer
self.scheduler = scheduler
self.device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model.to(self.device)
self.max_epochs = config.get("max_epochs", 50)
self.gradient_clip_norm = config.get("grad_clip_norm", 1.0)
self.use_amp = config.get("use_amp", True)
self.early_stopping = EarlyStopping(config.get("early_stopping_patience", 8), config.get("early_stopping_min_delta", 0.001))
self.scaler = torch.cuda.amp.GradScaler(enabled=self.use_amp and self.device.type == "cuda")
self.loss_fn = MultitaskLoss(
regression_weight=config.get("regression_weight", 1.0),
classification_weight=config.get("classification_weight", 0.5),
question_weight=config.get("question_weight", 1.0),
entropy_weight=config.get("entropy_weight", 0.02),
ccc_weight=config.get("ccc_weight", 0.2),
confidence_weight=config.get("confidence_weight", 0.05),
gate_balance_weight=config.get("gate_balance_weight", 0.5),
text_gate_weight=config.get("text_gate_weight", 0.0),
text_gate_target=config.get("text_gate_target", 0.40),
)
self.mixup_alpha = float(config.get("mixup_alpha", 0.0))
self.checkpoint_dir = Path(config.get("checkpoint_dir", "checkpoints"))
self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
# EMA for better generalization on small datasets
from src.training.optimization import EMA
self.ema = EMA(self.model, decay=0.999)
self.best_ccc = -float("inf")
def train(self) -> Dict[str, float]:
history = {
"train_loss": [],
"val_loss": [],
"val_ccc": [],
"val_rmse": [],
"val_mae": [],
}
logger.info("Starting training loop...")
best_path = self.checkpoint_dir / "best_model.pt"
best_ccc = -float("inf")
for epoch in range(1, self.max_epochs + 1):
train_loss = self._train_epoch()
# Use EMA weights for validation
self.ema.swap()
val_loss, val_metrics = self._validate_epoch()
self.ema.restore()
val_ccc = val_metrics.get("ccc", val_metrics.get("total_ccc", 0.0))
history["train_loss"].append(train_loss)
history["val_loss"].append(val_loss)
history["val_rmse"].append(val_metrics.get("rmse", 0.0))
history["val_mae"].append(val_metrics.get("mae", 0.0))
history["val_ccc"].append(val_ccc)
gate_str = ""
if "gate_audio" in val_metrics:
gate_str = f" | Gates: A:{val_metrics['gate_audio']:.2f} V:{val_metrics['gate_video']:.2f} T:{val_metrics['gate_text']:.2f}"
logger.info(
f"Epoch {epoch:03d}/{self.max_epochs} | "
f"Train Loss: {train_loss:.4f} | "
f"Val Loss: {val_loss:.4f} | "
f"Val CCC: {val_ccc:.4f} | "
f"Val RMSE: {val_metrics.get('rmse', val_metrics.get('total_rmse', 0.0)):.4f}{gate_str}"
)
if self.scheduler is not None:
self.scheduler.step(val_loss)
if val_ccc > best_ccc:
best_ccc = val_ccc
self.best_ccc = best_ccc
logger.info(f" --> New best model saved with CCC: {val_ccc:.4f}")
# Save EMA weights as the best model
self.ema.swap()
torch.save(self.model.state_dict(), best_path)
self.ema.restore()
if self.early_stopping.step(-val_ccc): # Early stopping based on CCC (negative because it expects min_delta to be lower)
logger.info(f"Early stopping triggered at epoch {epoch}")
break
return {"best_ccc": best_ccc, **history}
def _train_epoch(self) -> float:
self.model.train()
total_loss = 0.0
for batch in self.train_loader:
self.optimizer.zero_grad()
loss = self._compute_batch_loss(batch)
self.scaler.scale(loss).backward()
self.scaler.unscale_(self.optimizer)
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.gradient_clip_norm)
self.scaler.step(self.optimizer)
self.scaler.update()
# Update EMA weights
self.ema.update()
total_loss += float(loss.detach().cpu())
return total_loss / max(1, len(self.train_loader))
def _validate_epoch(self) -> tuple[float, Dict[str, float]]:
self.model.eval()
total_loss = 0.0
y_true = []
y_pred = []
all_modality_scores = []
with torch.no_grad():
for batch in self.val_loader:
output = self._forward(batch)
ms_val = output.get("modality_scores")
gate_w_val = None
if ms_val is not None and isinstance(ms_val, dict):
raw_v = ms_val.get("raw_weights")
gate_w_val = raw_v if raw_v is not None else torch.stack(
[ms_val["audio"], ms_val["video"], ms_val["text"]], dim=1
).to(self.device)
loss = self.loss_fn(
output["phq_total"],
batch["phq_total"].to(self.device),
output["phq_questions"],
batch["phq_questions"].to(self.device),
output["classification"],
batch["classification"].to(self.device),
output["entropy"],
confidence_pred=output.get("modality_confidence"),
gate_weights=gate_w_val,
)
total_loss += float(loss.cpu())
y_true.extend(batch["phq_total"].tolist())
y_pred.extend(output["phq_total"].cpu().tolist())
if "modality_scores" in output and output["modality_scores"] is not None:
ms = output["modality_scores"]
if isinstance(ms, dict):
gate_tensor = torch.stack([ms["audio"], ms["video"], ms["text"]], dim=1).cpu()
all_modality_scores.append(gate_tensor)
else:
all_modality_scores.append(ms.cpu())
metrics = regression_metrics(y_true, y_pred)
if all_modality_scores:
avg_scores = torch.cat(all_modality_scores, dim=0).mean(dim=0).tolist()
if len(avg_scores) >= 3:
metrics["gate_audio"] = avg_scores[0]
metrics["gate_video"] = avg_scores[1]
metrics["gate_text"] = avg_scores[2]
return total_loss / max(1, len(self.val_loader)), metrics
def _compute_batch_loss(self, batch: Dict[str, torch.Tensor]) -> torch.Tensor:
# Optional mixup augmentation — interpolate between random sample pairs
if self.model.training and self.mixup_alpha > 0.0 and len(batch["phq_total"]) > 1:
batch = self._apply_mixup(batch)
output = self._forward(batch)
# Use raw_weights (has gradients) so gate_balance loss can backprop
gate_weights = None
ms = output.get("modality_scores")
if ms is not None and isinstance(ms, dict):
raw = ms.get("raw_weights")
if raw is not None:
gate_weights = raw
else:
gate_weights = torch.stack(
[ms["audio"], ms["video"], ms["text"]], dim=1
).to(self.device)
loss = self.loss_fn(
output["phq_total"],
batch["phq_total"].to(self.device),
output["phq_questions"],
batch["phq_questions"].to(self.device),
output["classification"],
batch["classification"].to(self.device),
output["entropy"],
confidence_pred=output.get("modality_confidence"),
gate_weights=gate_weights,
)
return loss
def _apply_mixup(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""Mix pairs of samples in the batch (label-preserving interpolation)."""
import numpy as np
B = batch["phq_total"].shape[0]
lam = float(np.random.beta(self.mixup_alpha, self.mixup_alpha))
idx = torch.randperm(B)
mixed = dict(batch) # shallow copy
for key in ("phq_total", "phq_questions", "classification"):
mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx]
# Feature tensors: mix along time dim with the same lambda
for key in ("audio", "video", "text"):
if batch.get(key) is not None:
mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx]
return mixed
def _forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
audio = batch["audio"].to(self.device) if batch.get("audio") is not None else None
video = batch["video"].to(self.device) if batch.get("video") is not None else None
text = batch["text"].to(self.device) if batch.get("text") is not None else None
audio_mask = batch["audio_mask"].to(self.device)
video_mask = batch["video_mask"].to(self.device)
text_mask = batch["text_mask"].to(self.device)
return self.model(audio, video, text, audio_mask, video_mask, text_mask)
|