Spaces:
Sleeping
Sleeping
Download src/training/trainer.py from koushik2109/mindscope-Model: direct link, hf CLI and curl.
- Browser
- Download file 10.9 kB
-
https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/training/trainer.py
- Command line
-
hf download hf://spaces/koushik2109/mindscope-Model/src/training/trainer.py
-
curl -L -o trainer.py https://huggingface.co/spaces/koushik2109/mindscope-Model/resolve/main/src/training/trainer.py
10.9 kB
| from __future__ import annotations | |
| import time | |
| import torch | |
| import torch.nn as nn | |
| from pathlib import Path | |
| from typing import Dict, Optional | |
| from torch.utils.data import DataLoader | |
| from src.training.losses import MultitaskLoss | |
| from src.training.metrics import regression_metrics | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| class EarlyStopping: | |
| def __init__(self, patience: int = 8, min_delta: float = 0.001): | |
| self.patience = patience | |
| self.min_delta = min_delta | |
| self.best = float("inf") | |
| self.counter = 0 | |
| def step(self, value: float) -> bool: | |
| if value + self.min_delta < self.best: | |
| self.best = value | |
| self.counter = 0 | |
| return False | |
| self.counter += 1 | |
| return self.counter >= self.patience | |
| class Trainer: | |
| def __init__( | |
| self, | |
| model: nn.Module, | |
| train_loader: DataLoader, | |
| val_loader: DataLoader, | |
| optimizer: torch.optim.Optimizer, | |
| scheduler: Optional[torch.optim.lr_scheduler._LRScheduler], | |
| config: dict, | |
| device: Optional[torch.device] = None, | |
| ): | |
| self.model = model | |
| self.train_loader = train_loader | |
| self.val_loader = val_loader | |
| self.optimizer = optimizer | |
| self.scheduler = scheduler | |
| self.device = device or torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| self.model.to(self.device) | |
| self.max_epochs = config.get("max_epochs", 50) | |
| self.gradient_clip_norm = config.get("grad_clip_norm", 1.0) | |
| self.use_amp = config.get("use_amp", True) | |
| self.early_stopping = EarlyStopping(config.get("early_stopping_patience", 8), config.get("early_stopping_min_delta", 0.001)) | |
| self.scaler = torch.cuda.amp.GradScaler(enabled=self.use_amp and self.device.type == "cuda") | |
| self.loss_fn = MultitaskLoss( | |
| regression_weight=config.get("regression_weight", 1.0), | |
| classification_weight=config.get("classification_weight", 0.5), | |
| question_weight=config.get("question_weight", 1.0), | |
| entropy_weight=config.get("entropy_weight", 0.02), | |
| ccc_weight=config.get("ccc_weight", 0.2), | |
| confidence_weight=config.get("confidence_weight", 0.05), | |
| gate_balance_weight=config.get("gate_balance_weight", 0.5), | |
| text_gate_weight=config.get("text_gate_weight", 0.0), | |
| text_gate_target=config.get("text_gate_target", 0.40), | |
| ) | |
| self.mixup_alpha = float(config.get("mixup_alpha", 0.0)) | |
| self.checkpoint_dir = Path(config.get("checkpoint_dir", "checkpoints")) | |
| self.checkpoint_dir.mkdir(parents=True, exist_ok=True) | |
| # EMA for better generalization on small datasets | |
| from src.training.optimization import EMA | |
| self.ema = EMA(self.model, decay=0.999) | |
| self.best_ccc = -float("inf") | |
| def train(self) -> Dict[str, float]: | |
| history = { | |
| "train_loss": [], | |
| "val_loss": [], | |
| "val_ccc": [], | |
| "val_rmse": [], | |
| "val_mae": [], | |
| } | |
| logger.info("Starting training loop...") | |
| best_path = self.checkpoint_dir / "best_model.pt" | |
| best_ccc = -float("inf") | |
| for epoch in range(1, self.max_epochs + 1): | |
| train_loss = self._train_epoch() | |
| # Use EMA weights for validation | |
| self.ema.swap() | |
| val_loss, val_metrics = self._validate_epoch() | |
| self.ema.restore() | |
| val_ccc = val_metrics.get("ccc", val_metrics.get("total_ccc", 0.0)) | |
| history["train_loss"].append(train_loss) | |
| history["val_loss"].append(val_loss) | |
| history["val_rmse"].append(val_metrics.get("rmse", 0.0)) | |
| history["val_mae"].append(val_metrics.get("mae", 0.0)) | |
| history["val_ccc"].append(val_ccc) | |
| gate_str = "" | |
| if "gate_audio" in val_metrics: | |
| gate_str = f" | Gates: A:{val_metrics['gate_audio']:.2f} V:{val_metrics['gate_video']:.2f} T:{val_metrics['gate_text']:.2f}" | |
| logger.info( | |
| f"Epoch {epoch:03d}/{self.max_epochs} | " | |
| f"Train Loss: {train_loss:.4f} | " | |
| f"Val Loss: {val_loss:.4f} | " | |
| f"Val CCC: {val_ccc:.4f} | " | |
| f"Val RMSE: {val_metrics.get('rmse', val_metrics.get('total_rmse', 0.0)):.4f}{gate_str}" | |
| ) | |
| if self.scheduler is not None: | |
| self.scheduler.step(val_loss) | |
| if val_ccc > best_ccc: | |
| best_ccc = val_ccc | |
| self.best_ccc = best_ccc | |
| logger.info(f" --> New best model saved with CCC: {val_ccc:.4f}") | |
| # Save EMA weights as the best model | |
| self.ema.swap() | |
| torch.save(self.model.state_dict(), best_path) | |
| self.ema.restore() | |
| if self.early_stopping.step(-val_ccc): # Early stopping based on CCC (negative because it expects min_delta to be lower) | |
| logger.info(f"Early stopping triggered at epoch {epoch}") | |
| break | |
| return {"best_ccc": best_ccc, **history} | |
| def _train_epoch(self) -> float: | |
| self.model.train() | |
| total_loss = 0.0 | |
| for batch in self.train_loader: | |
| self.optimizer.zero_grad() | |
| loss = self._compute_batch_loss(batch) | |
| self.scaler.scale(loss).backward() | |
| self.scaler.unscale_(self.optimizer) | |
| torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.gradient_clip_norm) | |
| self.scaler.step(self.optimizer) | |
| self.scaler.update() | |
| # Update EMA weights | |
| self.ema.update() | |
| total_loss += float(loss.detach().cpu()) | |
| return total_loss / max(1, len(self.train_loader)) | |
| def _validate_epoch(self) -> tuple[float, Dict[str, float]]: | |
| self.model.eval() | |
| total_loss = 0.0 | |
| y_true = [] | |
| y_pred = [] | |
| all_modality_scores = [] | |
| with torch.no_grad(): | |
| for batch in self.val_loader: | |
| output = self._forward(batch) | |
| ms_val = output.get("modality_scores") | |
| gate_w_val = None | |
| if ms_val is not None and isinstance(ms_val, dict): | |
| raw_v = ms_val.get("raw_weights") | |
| gate_w_val = raw_v if raw_v is not None else torch.stack( | |
| [ms_val["audio"], ms_val["video"], ms_val["text"]], dim=1 | |
| ).to(self.device) | |
| loss = self.loss_fn( | |
| output["phq_total"], | |
| batch["phq_total"].to(self.device), | |
| output["phq_questions"], | |
| batch["phq_questions"].to(self.device), | |
| output["classification"], | |
| batch["classification"].to(self.device), | |
| output["entropy"], | |
| confidence_pred=output.get("modality_confidence"), | |
| gate_weights=gate_w_val, | |
| ) | |
| total_loss += float(loss.cpu()) | |
| y_true.extend(batch["phq_total"].tolist()) | |
| y_pred.extend(output["phq_total"].cpu().tolist()) | |
| if "modality_scores" in output and output["modality_scores"] is not None: | |
| ms = output["modality_scores"] | |
| if isinstance(ms, dict): | |
| gate_tensor = torch.stack([ms["audio"], ms["video"], ms["text"]], dim=1).cpu() | |
| all_modality_scores.append(gate_tensor) | |
| else: | |
| all_modality_scores.append(ms.cpu()) | |
| metrics = regression_metrics(y_true, y_pred) | |
| if all_modality_scores: | |
| avg_scores = torch.cat(all_modality_scores, dim=0).mean(dim=0).tolist() | |
| if len(avg_scores) >= 3: | |
| metrics["gate_audio"] = avg_scores[0] | |
| metrics["gate_video"] = avg_scores[1] | |
| metrics["gate_text"] = avg_scores[2] | |
| return total_loss / max(1, len(self.val_loader)), metrics | |
| def _compute_batch_loss(self, batch: Dict[str, torch.Tensor]) -> torch.Tensor: | |
| # Optional mixup augmentation — interpolate between random sample pairs | |
| if self.model.training and self.mixup_alpha > 0.0 and len(batch["phq_total"]) > 1: | |
| batch = self._apply_mixup(batch) | |
| output = self._forward(batch) | |
| # Use raw_weights (has gradients) so gate_balance loss can backprop | |
| gate_weights = None | |
| ms = output.get("modality_scores") | |
| if ms is not None and isinstance(ms, dict): | |
| raw = ms.get("raw_weights") | |
| if raw is not None: | |
| gate_weights = raw | |
| else: | |
| gate_weights = torch.stack( | |
| [ms["audio"], ms["video"], ms["text"]], dim=1 | |
| ).to(self.device) | |
| loss = self.loss_fn( | |
| output["phq_total"], | |
| batch["phq_total"].to(self.device), | |
| output["phq_questions"], | |
| batch["phq_questions"].to(self.device), | |
| output["classification"], | |
| batch["classification"].to(self.device), | |
| output["entropy"], | |
| confidence_pred=output.get("modality_confidence"), | |
| gate_weights=gate_weights, | |
| ) | |
| return loss | |
| def _apply_mixup(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: | |
| """Mix pairs of samples in the batch (label-preserving interpolation).""" | |
| import numpy as np | |
| B = batch["phq_total"].shape[0] | |
| lam = float(np.random.beta(self.mixup_alpha, self.mixup_alpha)) | |
| idx = torch.randperm(B) | |
| mixed = dict(batch) # shallow copy | |
| for key in ("phq_total", "phq_questions", "classification"): | |
| mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx] | |
| # Feature tensors: mix along time dim with the same lambda | |
| for key in ("audio", "video", "text"): | |
| if batch.get(key) is not None: | |
| mixed[key] = lam * batch[key] + (1 - lam) * batch[key][idx] | |
| return mixed | |
| def _forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: | |
| audio = batch["audio"].to(self.device) if batch.get("audio") is not None else None | |
| video = batch["video"].to(self.device) if batch.get("video") is not None else None | |
| text = batch["text"].to(self.device) if batch.get("text") is not None else None | |
| audio_mask = batch["audio_mask"].to(self.device) | |
| video_mask = batch["video_mask"].to(self.device) | |
| text_mask = batch["text_mask"].to(self.device) | |
| return self.model(audio, video, text, audio_mask, video_mask, text_mask) | |