trainer.py

import torch
from models.image.clip_detection.clip_detection import SynthScopeCLIPModel
from models.image.cnn_detection.cnn_detection import SynthScopeCNNModel
from models.image.dino_detection.dino_detection import SynthScopeDINOModel, SynthScopeDINOv3Model
from models.image.adapted_dino.adapted_dino import SynthScopeAdaptedDINOModel
from models.image.safe.safe import SynthScopeSAFEModel
from models.image.fat_former.fat_former import SynthScopeFatFormerModel
from models.image.co_spy.co_spy import SynthScopeCoSpyModel
from models.image.aide.aide import SynthScopeAIDEModel
from models.image.spai.spai import SPAI
from models.image.vib_net.vib_net import SynthScopeVIBNetModel
from models.image.npr.npr import NPR
from models.image.pgc.pgc import SynthScopePGCModel
from models.image.cure.cure import SynthScopeCuReModel
from models.video.d3.d3 import SynthScopeD3Model
from models.video.restrav.restrav import SynthScopeReStraVModel
from models.video.stall.stall import SynthScopeSTALLModel
from metrics import AIGIMetrics
from dataset import AIGIDatasetModule
from omegaconf import DictConfig
from accelerate import Accelerator
from transformers import get_scheduler
import os
from logger import build_logger
from tqdm import tqdm

class SynthScopeModelTrainer:
    def __init__(self, config: DictConfig, dataset_module: AIGIDatasetModule):
        self.model = self._build_model(config)
        self.optimizer = torch.optim.AdamW(
            self._filter_trainable_parameters(),
            lr=config.TrainerConfig.lr,
            betas=(0.9, 0.999),
            weight_decay=config.TrainerConfig.get("weight_decay", 0.01),
        )
        self.criterion = torch.nn.CrossEntropyLoss(
            label_smoothing=config.TrainerConfig.get("label_smoothing", 0.0)
        )
        self.train_loader = dataset_module.train_dataloader()
        self.val_loader = dataset_module.val_dataloader()
        self.test_loader = dataset_module.test_dataloader()
        self.config = config
        self.train_metrics = AIGIMetrics(config.metrics)
        self.val_metrics = AIGIMetrics(config.metrics)
        self.test_metrics = AIGIMetrics(config.metrics)
        self.accelerator = Accelerator()
        self.use_metrics = len(config.metrics) > 0
        self.scheduler = None
        warmup = config.TrainerConfig.warmup
        if warmup.mode != "off":
            num_training_steps = config.TrainerConfig.epochs * (len(self.train_loader) // self.accelerator.gradient_accumulation_steps)
            self.scheduler = get_scheduler(warmup.lr_scheduler, optimizer=self.optimizer, num_warmup_steps=int(warmup.warmup_ratio * num_training_steps), num_training_steps=num_training_steps)
        self.model, self.optimizer, self.scheduler, self.train_loader, self.val_loader, self.test_loader = self.accelerator.prepare(
            self.model, self.optimizer, self.scheduler, self.train_loader, self.val_loader, self.test_loader)
        self.train_metrics = self.train_metrics.to(self.accelerator.device)
        self.val_metrics = self.val_metrics.to(self.accelerator.device)
        self.test_metrics = self.test_metrics.to(self.accelerator.device)
        self.global_step = 0
        self.best_val_loss = float("inf")
        self.best_test_loss = float("inf")
        self.logger = build_logger(config, self.accelerator)
        if self.accelerator.is_main_process:    
            self.print_trainable_parameters()
    
    def print_trainable_parameters(self):
        trainable_parameters = [n for (n, p) in self.model.named_parameters() if p.requires_grad]
        self.accelerator.print(f"Trainable Parameters: {trainable_parameters}")

    def _build_model(self, config: DictConfig):
        model_type = config.ModelConfig.get("model_type", "dino").lower()
        if model_type == "safe":
            return SynthScopeSAFEModel(config)
        if model_type == "fat_former":
            return SynthScopeFatFormerModel(config)
        if model_type == "co_spy":
            return SynthScopeCoSpyModel(config)
        if model_type == "aide":
            return SynthScopeAIDEModel(config)
        if model_type == "spai":
            return SPAI(config)
        if model_type == "vib_net":
            return SynthScopeVIBNetModel(config)
        if model_type == "npr":
            return NPR(config)
        if model_type == "pgc":
            return SynthScopePGCModel(config)
        if model_type == "cure":
            return SynthScopeCuReModel(config)
        if model_type == "d3":
            return SynthScopeD3Model(config)
        if model_type == "restrav":
            return SynthScopeReStraVModel(config)
        if model_type == "stall":
            return SynthScopeSTALLModel(config)
        if model_type == "cnn":
            return SynthScopeCNNModel(config)
        if model_type == "adapted_dino":
            return SynthScopeAdaptedDINOModel(config)
        if model_type == "dinov2":
            return SynthScopeDINOModel(config)
        if model_type == "dinov3":
            return SynthScopeDINOv3Model(config)
        # Explicit model_type wins above; the CLIP-prefixed backbone is only a last-resort
        # fallback, so e.g. model_type=cnn with the default CLIP:ViT-L/14 backbone is not hijacked.
        if model_type == "clip" or config.ModelConfig.backbone_name.startswith("CLIP:"):
            return SynthScopeCLIPModel(config)
        raise ValueError(f"Invalid model_type: {model_type}")
    
    def _filter_trainable_parameters(self):
        return [p for p in self.model.parameters() if p.requires_grad]

    def train_step(self, batch):
        loss_dict = self.model.compute_loss(batch)
        loss = loss_dict["loss"]
        outputs = loss_dict["outputs"]
        labels = batch[1]
        folder_ids = batch[2]
        if self.use_metrics:    
            outputs_g, labels_g, folder_ids_g = self.accelerator.gather_for_metrics((outputs.detach(), labels.detach(), folder_ids.detach()))
            self.train_metrics(outputs_g, labels_g, folder_ids_g)  
        return loss

    def val_step(self, batch):
        loss_dict = self.model.compute_loss(batch)
        loss = loss_dict["loss"]
        outputs = loss_dict["outputs"]
        labels = batch[1]
        folder_ids = batch[2]
        if self.use_metrics:
            outputs_g, labels_g, folder_ids_g = self.accelerator.gather_for_metrics((outputs.detach(), labels.detach(), folder_ids.detach()))
            self.val_metrics(outputs_g, labels_g, folder_ids_g)
        return loss
    
    def test_step(self, batch):
        loss_dict = self.model.compute_loss(batch)
        loss = loss_dict["loss"]
        outputs = loss_dict["outputs"]
        labels = batch[1]
        folder_ids = batch[2]
        if self.use_metrics:
            outputs_g, labels_g, folder_ids_g = self.accelerator.gather_for_metrics((outputs.detach(), labels.detach(), folder_ids.detach()))
            self.test_metrics(outputs_g, labels_g, folder_ids_g)
        return loss

    def train_epoch(self):
        self.model.train()
        total_loss = 0
        for batch in tqdm(self.train_loader, desc="Training"):
            self.optimizer.zero_grad()
            loss  = self.train_step(batch)
            self.accelerator.backward(loss)
            self.optimizer.step()
            if self.scheduler is not None:
                self.scheduler.step()
            if self.accelerator.sync_gradients:
                self.global_step += 1
            loss_sum = self.accelerator.gather(loss.detach().unsqueeze(0)).mean()
            total_loss += loss_sum.item()
        if self.use_metrics:
            total_metrics_dict = {
                metric: self.train_metrics.metrics_dict[metric].compute().item()
                for metric in self.train_metrics.metrics_dict
            }
            self.train_metrics._reset_metrics_dict()
        else:
            total_metrics_dict = {}
        return total_loss / len(self.train_loader), total_metrics_dict
    
    def val_epoch(self):
        self.model.eval()
        total_loss = 0
        with torch.no_grad():
            for batch in tqdm(self.val_loader, desc="Validating"):
                loss = self.val_step(batch)
                loss_sum = self.accelerator.gather(loss.detach().unsqueeze(0)).mean()
                total_loss += loss_sum.item()
        if self.use_metrics:
            total_metrics_dict = {
                metric: self.val_metrics.metrics_dict[metric].compute().item()
                for metric in self.val_metrics.metrics_dict
            }
            self.val_metrics._reset_metrics_dict()
        else:
            total_metrics_dict = {}
        return total_loss / len(self.val_loader), total_metrics_dict
    
    def test_epoch(self):
        self.model.eval()
        total_loss = 0
        with torch.no_grad():
            for batch in tqdm(self.test_loader, desc="Testing"):
                loss = self.test_step(batch)
                loss_sum = self.accelerator.gather(loss.detach().unsqueeze(0)).mean()
                total_loss += loss_sum.item()
           
        if self.use_metrics:
            total_metrics_dict = {
                metric: self.test_metrics.metrics_dict[metric].compute().item()
                for metric in self.test_metrics.metrics_dict
            }
            self.test_metrics._reset_metrics_dict()
        else:
            total_metrics_dict = {}
        return total_loss / len(self.test_loader), total_metrics_dict


    def _save_model(self, epoch, save_path):
        self.accelerator.wait_for_everyone()
        unwrapped_model = self.accelerator.unwrap_model(self.model)
        if self.accelerator.is_main_process:
            os.makedirs(save_path, exist_ok=True)
            self.accelerator.save(unwrapped_model.state_dict(), f"{save_path}/model_{epoch}.pth")

    def _save(self, val_loss, epoch):
        if self.config.TrainerConfig.save_policy == "max":
            if val_loss > self.best_val_loss:
                self.best_val_loss = val_loss
                self._save_model(epoch, self.config.TrainerConfig.save_path)
        elif self.config.TrainerConfig.save_policy == "min":
            if val_loss < self.best_val_loss:
                self.best_val_loss = val_loss
                self._save_model(epoch, self.config.TrainerConfig.save_path)
        self.accelerator.print(f"Epoch {epoch}, Model Saved")

    def train(self):
        for epoch in range(self.config.TrainerConfig.epochs):
            if epoch % self.config.TrainerConfig.val_interval == 0:
                val_loss, val_metrics_dict = self.val()
                self.accelerator.print(f"Epoch {epoch}, Val Loss: {val_loss}, Val Metrics: {val_metrics_dict}")
                self.logger.log_metrics({"val/loss": val_loss, **{f"val/{metric}": value for metric, value in val_metrics_dict.items()}}, self.global_step)
            if epoch % self.config.TrainerConfig.test_interval == 0:
                test_loss, test_metrics_dict = self.test()   
                self.accelerator.print(f"Epoch {epoch}, Test Loss: {test_loss}, Test Metrics: {test_metrics_dict}")
                self.logger.log_metrics({"test/loss": test_loss, **{f"test/{metric}": value for metric, value in test_metrics_dict.items()}}, self.global_step)
            train_loss, train_metrics_dict = self.train_epoch()
            self.accelerator.print(f"Epoch {epoch}, Train Loss: {train_loss}, Train Metrics: {train_metrics_dict}")
            self.logger.log_metrics({"train/loss": train_loss, **{f"train/{metric}": value for metric, value in train_metrics_dict.items()}}, self.global_step)
            self.accelerator.print(f"Epoch{epoch}, Training Loss: {train_loss}, Training Metrics: {train_metrics_dict}")
            if epoch % self.config.TrainerConfig.save_interval == 0:
                if self.config.TrainerConfig.save:
                    self._save(val_loss, epoch)
        self.logger.finish()
    
    def test(self):
        test_loss, test_metrics_dict = self.test_epoch()
        self.accelerator.print(f"Test Loss: {test_loss}, Test Metrics: {test_metrics_dict}")
        self.logger.log_metrics({"test/loss": test_loss, **{f"test/{metric}": value for metric, value in test_metrics_dict.items()}}, self.global_step)
        return test_loss, test_metrics_dict
    
    def val(self):
        val_loss, val_metrics_dict = self.val_epoch()
        self.accelerator.print(f"Val Loss: {val_loss}, Val Metrics: {val_metrics_dict}")
        self.logger.log_metrics({"val/loss": val_loss, **{f"val/{metric}": value for metric, value in val_metrics_dict.items()}}, self.global_step)
        return val_loss, val_metrics_dict