"""Trainer: train/val loop, LR scheduling, early stopping, checkpointing.""" from __future__ import annotations import csv from pathlib import Path import torch import torch.nn as nn from torch.utils.data import DataLoader from src.models.caption_model import CaptionModel from src.utils.config import save_config class Trainer: def __init__( self, model: CaptionModel, train_loader: DataLoader, val_loader: DataLoader, config: dict, vocab_size: int, device: str = "cpu", ): self.model = model.to(device) self.train_loader = train_loader self.val_loader = val_loader self.config = config self.device = device train_cfg = config["training"] self.criterion = nn.CrossEntropyLoss(ignore_index=0) # ignore self.optimizer = torch.optim.Adam( self.model.parameters(), lr=train_cfg["lr"], weight_decay=train_cfg.get("weight_decay", 0.0), ) self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( self.optimizer, mode="min", factor=train_cfg["lr_scheduler_factor"], patience=train_cfg["lr_scheduler_patience"], ) self.vocab_size = vocab_size self.early_stop_patience = train_cfg["early_stop_patience"] self.epochs = train_cfg["epochs"] self.run_dir = Path(config["paths"]["models_dir"]) / config["run_name"] self.run_dir.mkdir(parents=True, exist_ok=True) self.log_path = self.run_dir / "training_log.csv" self.best_val_loss = float("inf") self.patience_counter = 0 self.start_epoch = 0 def _run_epoch(self, loader: DataLoader, train: bool) -> float: self.model.train(mode=train) total_loss = 0.0 n_batches = 0 context = torch.enable_grad() if train else torch.no_grad() with context: for batch in loader: image_features = batch["image_feature"].to(self.device) input_seq = batch["input_seq"].to(self.device) target_seq = batch["target_seq"].to(self.device) if train: self.optimizer.zero_grad() outputs = self.model(image_features, input_seq) loss = self.criterion( outputs.reshape(-1, self.vocab_size), target_seq.reshape(-1), ) if train: loss.backward() grad_clip = self.config["training"].get("grad_clip_norm", 0.0) if grad_clip > 0: torch.nn.utils.clip_grad_norm_(self.model.parameters(), grad_clip) self.optimizer.step() total_loss += loss.item() n_batches += 1 return total_loss / max(n_batches, 1) def save_checkpoint(self, path: Path, epoch: int, val_loss: float) -> None: torch.save( { "model_state_dict": self.model.state_dict(), "optimizer_state_dict": self.optimizer.state_dict(), "epoch": epoch, "val_loss": val_loss, "vocab_size": self.vocab_size, "config": {k: v for k, v in self.config.items() if not k.startswith("_")}, }, path, ) def load_checkpoint(self, path: Path, resume_training: bool = True) -> None: checkpoint = torch.load(path, map_location=self.device, weights_only=False) self.model.load_state_dict(checkpoint["model_state_dict"]) if resume_training: self.optimizer.load_state_dict(checkpoint["optimizer_state_dict"]) self.start_epoch = checkpoint["epoch"] + 1 self.best_val_loss = checkpoint["val_loss"] def fit(self, resume_from: Path | None = None) -> None: if resume_from is not None and resume_from.exists(): print(f"Resuming from checkpoint: {resume_from}") self.load_checkpoint(resume_from, resume_training=True) save_config(self.config, self.run_dir / "config.yaml") write_header = not self.log_path.exists() log_file = open(self.log_path, "a", newline="") log_writer = csv.writer(log_file) if write_header: log_writer.writerow(["epoch", "train_loss", "val_loss", "lr"]) for epoch in range(self.start_epoch, self.epochs): train_loss = self._run_epoch(self.train_loader, train=True) val_loss = self._run_epoch(self.val_loader, train=False) self.scheduler.step(val_loss) current_lr = self.optimizer.param_groups[0]["lr"] print( f"Epoch {epoch + 1}/{self.epochs} | " f"train_loss={train_loss:.4f} | val_loss={val_loss:.4f} | lr={current_lr:.2e}" ) log_writer.writerow([epoch + 1, train_loss, val_loss, current_lr]) log_file.flush() self.save_checkpoint(self.run_dir / "last_checkpoint.pt", epoch, val_loss) if val_loss < self.best_val_loss: self.best_val_loss = val_loss self.patience_counter = 0 self.save_checkpoint(self.run_dir / "best_model.pt", epoch, val_loss) print(f" -> new best model saved (val_loss={val_loss:.4f})") else: self.patience_counter += 1 if self.patience_counter >= self.early_stop_patience: print(f"Early stopping triggered at epoch {epoch + 1}") break log_file.close()