mindscope-Model / src /training /trainer.py
koushik2109's picture
Model Changes
8d7b4b1
Raw History Blame Contribute Delete
10.9 kB
from __future__ import annotations
import time
import torch
import torch.nn as nn
from pathlib import Path
from typing import Dict, Optional
from torch.utils.data import DataLoader
from src.training.losses import MultitaskLoss
from src.training.metrics import regression_metrics
import logging
logger = logging.getLogger(__name__)
class EarlyStopping:
def __init__(self, patience: int = 8, min_delta: float = 0.001):
self.patience = patience
self.min_delta = min_delta
self.best = float("inf")
self.counter = 0
def step(self, value: float) -> bool:
if value + self.min_delta < self.best:
self.best = value
self.counter = 0
return False
self.counter += 1
return self.counter >= self.patience
class Trainer:
def __init__(
self,
model: nn.Module,
train_loader: DataLoader,
val_loader: DataLoader,
optimizer: torch.optim.Optimizer,
scheduler: Optional[torch.optim.lr_scheduler._LRScheduler],
config: dict,
device: Optional[torch.device] = None,
):
self.model = model
self.train_loader = train_loader
self.val_loader = val_loader
self.optimizer = optimizer
self.scheduler = scheduler
self.device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model.to(self.device)
self.max_epochs = config.get("max_epochs", 50)
self.gradient_clip_norm = config.get("grad_clip_norm", 1.0)
self.use_amp = config.get("use_amp", True)
self.early_stopping = EarlyStopping(config.get("early_stopping_patience", 8), config.get("early_stopping_min_delta", 0.001))
self.scaler = torch.cuda.amp.GradScaler(enabled=self.use_amp and self.device.type == "cuda")
self.loss_fn = MultitaskLoss(
regression_weight=config.get("regression_weight", 1.0),
classification_weight=config.get("classification_weight", 0.5),
question_weight=config.get("question_weight", 1.0),
entropy_weight=config.get("entropy_weight", 0.02),
ccc_weight=config.get("ccc_weight", 0.2),
confidence_weight=config.get("confidence_weight", 0.05),
gate_balance_weight=config.get("gate_balance_weight", 0.5),
text_gate_weight=config.get("text_gate_weight", 0.0),
text_gate_target=config.get("text_gate_target", 0.40),
)
self.mixup_alpha = float(config.get("mixup_alpha", 0.0))
self.checkpoint_dir = Path(config.get("checkpoint_dir", "checkpoints"))
self.checkpoint_dir.mkdir(parents=True, exist_ok=True)
# EMA for better generalization on small datasets
from src.training.optimization import EMA
self.ema = EMA(self.model, decay=0.999)
self.best_ccc = -float("inf")
def train(self) -> Dict[str, float]:
history = {
"train_loss": [],
"val_loss": [],
"val_ccc": [],
"val_rmse": [],
"val_mae": [],
}
logger.info("Starting training loop...")
best_path = self.checkpoint_dir / "best_model.pt"
best_ccc = -float("inf")
for epoch in range(1, self.max_epochs + 1):
train_loss = self._train_epoch()
# Use EMA weights for validation
self.ema.swap()
val_loss, val_metrics = self._validate_epoch()
self.ema.restore()
val_ccc = val_metrics.get("ccc", val_metrics.get("total_ccc", 0.0))
history["train_loss"].append(train_loss)
history["val_loss"].append(val_loss)
history["val_rmse"].append(val_metrics.get("rmse", 0.0))
history["val_mae"].append(val_metrics.get("mae", 0.0))
history["val_ccc"].append(val_ccc)
gate_str = ""
if "gate_audio" in val_metrics:
gate_str = f" | Gates: A:{val_metrics['gate_audio']:.2f} V:{val_metrics['gate_video']:.2f} T:{val_metrics['gate_text']:.2f}"
logger.info(
f"Epoch {epoch:03d}/{self.max_epochs} | "
f"Train Loss: {train_loss:.4f} | "
f"Val Loss: {val_loss:.4f} | "
f"Val CCC: {val_ccc:.4f} | "
f"Val RMSE: {val_metrics.get('rmse', val_metrics.get('total_rmse', 0.0)):.4f}{gate_str}"
)
if self.scheduler is not None:
self.scheduler.step(val_loss)
if val_ccc > best_ccc:
best_ccc = val_ccc
self.best_ccc = best_ccc
logger.info(f" --> New best model saved with CCC: {val_ccc:.4f}")
# Save EMA weights as the best model
self.ema.swap()
torch.save(self.model.state_dict(), best_path)
self.ema.restore()
if self.early_stopping.step(-val_ccc): # Early stopping based on CCC (negative because it expects min_delta to be lower)
logger.info(f"Early stopping triggered at epoch {epoch}")
break
return {"best_ccc": best_ccc, **history}
def _train_epoch(self) -> float:
self.model.train()
total_loss = 0.0
for batch in self.train_loader:
self.optimizer.zero_grad()
loss = self._compute_batch_loss(batch)
self.scaler.scale(loss).backward()
self.scaler.unscale_(self.optimizer)
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.gradient_clip_norm)
self.scaler.step(self.optimizer)
self.scaler.update()
# Update EMA weights
self.ema.update()
total_loss += float(loss.detach().cpu())
return total_loss / max(1, len(self.train_loader))
def _validate_epoch(self) -> tuple[float, Dict[str, float]]:
self.model.eval()
total_loss = 0.0
y_true = []
y_pred = []
all_modality_scores = []
with torch.no_grad():
for batch in self.val_loader:
output = self._forward(batch)
ms_val = output.get("modality_scores")
gate_w_val = None
if ms_val is not None and isinstance(ms_val, dict):
raw_v = ms_val.get("raw_weights")
gate_w_val = raw_v if raw_v is not None else torch.stack(
[ms_val["audio"], ms_val["video"], ms_val["text"]], dim=1
).to(self.device)
loss = self.loss_fn(
output["phq_total"],
batch["phq_total"].to(self.device),
output["phq_questions"],
batch["phq_questions"].to(self.device),
output["classification"],
batch["classification"].to(self.device),
output["entropy"],
confidence_pred=output.get("modality_confidence"),
gate_weights=gate_w_val,
)
total_loss += float(loss.cpu())
y_true.extend(batch["phq_total"].tolist())
y_pred.extend(output["phq_total"].cpu().tolist())
if "modality_scores" in output and output["modality_scores"] is not None:
ms = output["modality_scores"]
if isinstance(ms, dict):
gate_tensor = torch.stack([ms["audio"], ms["video"], ms["text"]], dim=1).cpu()
all_modality_scores.append(gate_tensor)
else:
all_modality_scores.append(ms.cpu())
metrics = regression_metrics(y_true, y_pred)
if all_modality_scores:
avg_scores = torch.cat(all_modality_scores, dim=0).mean(dim=0).tolist()
if len(avg_scores) >= 3:
metrics["gate_audio"] = avg_scores[0]
metrics["gate_video"] = avg_scores[1]
metrics["gate_text"] = avg_scores[2]
return total_loss / max(1, len(self.val_loader)), metrics
def _compute_batch_loss(self, batch: Dict[str, torch.Tensor]) -> torch.Tensor:
# Optional mixup augmentation — interpolate between random sample pairs
if self.model.training and self.mixup_alpha > 0.0 and len(batch["phq_total"]) > 1:
batch = self._apply_mixup(batch)
output = self._forward(batch)
# Use raw_weights (has gradients) so gate_balance loss can backprop
gate_weights = None
ms = output.get("modality_scores")
if ms is not None and isinstance(ms, dict):
raw = ms.get("raw_weights")
if raw is not None:
gate_weights = raw
else:
gate_weights = torch.stack(
[ms["audio"], ms["video"], ms["text"]], dim=1
).to(self.device)
loss = self.loss_fn(
output["phq_total"],
batch["phq_total"].to(self.device),
output["phq_questions"],
batch["phq_questions"].to(self.device),
output["classification"],
batch["classification"].to(self.device),
output["entropy"],
confidence_pred=output.get("modality_confidence"),
gate_weights=gate_weights,
)
return loss
def _apply_mixup(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
"""Mix pairs of samples in the batch (label-preserving interpolation)."""
import numpy as np
B = batch["phq_total"].shape[0]
lam = float(np.random.beta(self.mixup_alpha, self.mixup_alpha))
idx = torch.randperm(B)
mixed = dict(batch) # shallow copy
for key in ("phq_total", "phq_questions", "classification"):
mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx]
# Feature tensors: mix along time dim with the same lambda
for key in ("audio", "video", "text"):
if batch.get(key) is not None:
mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx]
return mixed
def _forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
audio = batch["audio"].to(self.device) if batch.get("audio") is not None else None
video = batch["video"].to(self.device) if batch.get("video") is not None else None
text = batch["text"].to(self.device) if batch.get("text") is not None else None
audio_mask = batch["audio_mask"].to(self.device)
video_mask = batch["video_mask"].to(self.device)
text_mask = batch["text_mask"].to(self.device)
return self.model(audio, video, text, audio_mask, video_mask, text_mask)