"""Shared generator runtime used by distribution matching objectives. This module contains rollout mechanics only. Distribution objectives belong in their concrete model classes; frozen-encoder losses and statistics are not part of the generator runtime. """ from typing import Optional import torch import torch.distributed as dist from model.base import SelfForcingModel class GeneratorRuntimeBase(SelfForcingModel): """Initialize the generator and provide common differentiable rollouts.""" uses_critic = False uses_vae_for_loss = True def __init__(self, args, device): super().__init__(args, device) self.causal_generator = bool(getattr(args, "causal", True)) self.num_frame_per_block = getattr(args, "num_frame_per_block", 1) self.same_step_across_blocks = getattr( args, "same_step_across_blocks", True) self.num_training_frames = getattr(args, "num_training_frames", 21) if self.causal_generator and self.num_frame_per_block > 1: self.generator.model.num_frame_per_block = self.num_frame_per_block self.independent_first_frame = getattr( args, "independent_first_frame", False) if self.causal_generator and self.independent_first_frame: self.generator.model.independent_first_frame = True if args.gradient_checkpointing: self.generator.enable_gradient_checkpointing() self.inference_pipeline = None self.generation_guidance_scale = float( getattr(args, "guidance_scale", 1.0)) if not hasattr(self, "denoising_step_list"): raise ValueError( "few-step generation requires denoising_step_list") denoising_step_list = self.denoising_step_list.to(device=device) if ( denoising_step_list.numel() > 0 and denoising_step_list[-1].item() == 0 ): denoising_step_list = denoising_step_list[:-1] self.generation_denoising_step_list = denoising_step_list @staticmethod def _apply_gradient_mask( latents: torch.Tensor, gradient_mask: Optional[torch.Tensor], ) -> torch.Tensor: if gradient_mask is None: return latents frame_mask = gradient_mask.flatten(2).any(dim=2).view( latents.shape[0], latents.shape[1], 1, 1, 1) return torch.where(frame_mask, latents, latents.detach()) def _predict_x0_with_cfg( self, noisy_latent: torch.Tensor, conditional_dict: dict, unconditional_dict: dict, timestep: torch.Tensor, ) -> torch.Tensor: _, x0_cond = self.generator( noisy_image_or_video=noisy_latent, conditional_dict=conditional_dict, timestep=timestep, ) if self.generation_guidance_scale == 1.0: return x0_cond _, x0_uncond = self.generator( noisy_image_or_video=noisy_latent, conditional_dict=unconditional_dict, timestep=timestep, ) return x0_uncond + self.generation_guidance_scale * ( x0_cond - x0_uncond) def _run_few_step_generator( self, image_or_video_shape, conditional_dict: dict, unconditional_dict: dict, exit_index: Optional[int] = None, ) -> torch.Tensor: """Roll out detached steps, then differentiate one selected exit step.""" latent = torch.randn( image_or_video_shape, device=self.device, dtype=self.dtype, ) batch_size, num_frames = latent.shape[:2] denoising_steps = self.generation_denoising_step_list.to( device=self.device) num_steps = len(denoising_steps) if exit_index is None: if num_steps <= 1: exit_index = 0 else: index_tensor = torch.randint( 0, num_steps, (1,), device=self.device) if dist.is_available() and dist.is_initialized(): dist.broadcast(index_tensor, src=0) exit_index = int(index_tensor.item()) for index in range(num_steps): current_timestep = denoising_steps[index] timestep = torch.ones( [batch_size, num_frames], device=self.device, dtype=current_timestep.dtype, ) * current_timestep if index < exit_index: with torch.no_grad(): x0_pred = self._predict_x0_with_cfg( noisy_latent=latent, conditional_dict=conditional_dict, unconditional_dict=unconditional_dict, timestep=timestep, ) next_timestep = denoising_steps[index + 1] latent = self.scheduler.add_noise( x0_pred.flatten(0, 1), torch.randn_like(x0_pred.flatten(0, 1)), next_timestep * torch.ones( [batch_size * num_frames], device=self.device, dtype=next_timestep.dtype, ), ).unflatten(0, x0_pred.shape[:2]).to(self.dtype) else: return self._predict_x0_with_cfg( noisy_latent=latent, conditional_dict=conditional_dict, unconditional_dict=unconditional_dict, timestep=timestep, ) return latent