import argparse import json import torch import os from omegaconf import OmegaConf from tqdm import tqdm from torchvision import transforms import imageio.v2 as imageio from einops import rearrange import torch.distributed as dist from torch.utils.data import DataLoader, SequentialSampler, Subset from safetensors import safe_open from model.predictor_v4 import SelfForcingPredictorV4 from pipeline import ( CausalDiffusionInferencePipeline, CausalInferencePipeline, ) from utils.dataset import TextDataset, TextImagePairDataset from utils.misc import set_seed from demo_utils.memory import get_cuda_free_memory_gb, DynamicSwapInstaller def load_predictor_checkpoint(path: str) -> tuple[dict[str, torch.Tensor], dict]: """Load compact Predictor weights together with required architecture metadata.""" with safe_open(path, framework="pt", device="cpu") as handle: metadata = handle.metadata() or {} state_dict = {name: handle.get_tensor(name) for name in handle.keys()} if metadata.get("format") != "self_forcing_predictor_v4": raise ValueError( f"{path} is not a self_forcing_predictor_v4 safetensors checkpoint" ) try: checkpoint_config = json.loads(metadata["config"]) except (KeyError, json.JSONDecodeError) as exc: raise ValueError(f"{path} has invalid Predictor metadata") from exc return state_dict, checkpoint_config parser = argparse.ArgumentParser() parser.add_argument("--config_path", type=str, help="Path to the config file") parser.add_argument("--checkpoint_path", type=str, help="Path to the checkpoint folder") parser.add_argument("--data_path", type=str, help="Path to the dataset") parser.add_argument("--extended_prompt_path", type=str, help="Path to the extended prompt") parser.add_argument("--output_folder", type=str, help="Output folder") parser.add_argument("--num_output_frames", type=int, default=21, help="Number of overlap frames between sliding windows") parser.add_argument("--i2v", action="store_true", help="Whether to perform I2V (or T2V by default)") parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA parameters") parser.add_argument("--seed", type=int, default=0, help="Random seed") parser.add_argument("--num_samples", type=int, default=1, help="Number of samples to generate per prompt") parser.add_argument("--max_prompts", type=int, default=None, help="Only run inference on the first N prompts") parser.add_argument( "--reuse_first_step_velocity", action="store_true", help=( "Enable velocity reuse after an all-Full chunk 0; later chunks use " "--reuse_first_step_velocity_schedule" ), ) parser.add_argument( "--reuse_first_step_velocity_schedule", choices=("frrf", "frrr"), default="frrf", help="Velocity-reuse schedule for chunks after the all-Full chunk 0", ) parser.add_argument( "--predictor_checkpoint", type=str, default=None, help="Compact Predictor-v4 safetensors checkpoint; enables F-P-P-F/P", ) parser.add_argument( "--predictor_source_blocks", type=str, default="1,28", help="Two comma-separated Wan Teacher block ids used by Predictor-v4", ) parser.add_argument( "--predictor_schedule", choices=("fppf", "fppp"), default="fppf", help="Predictor deployment schedule after the initial FFFF chunk", ) parser.add_argument("--save_with_index", action="store_true", help="Whether to save the video using the index or prompt as the filename") args = parser.parse_args() # Initialize distributed inference if "LOCAL_RANK" in os.environ: dist.init_process_group(backend='nccl') local_rank = int(os.environ["LOCAL_RANK"]) torch.cuda.set_device(local_rank) device = torch.device(f"cuda:{local_rank}") world_size = dist.get_world_size() set_seed(args.seed) else: device = torch.device("cuda") local_rank = 0 world_size = 1 set_seed(args.seed) print(f'Free VRAM {get_cuda_free_memory_gb(device)} GB') low_memory = get_cuda_free_memory_gb(device) < 40 torch.set_grad_enabled(False) config = OmegaConf.load(args.config_path) default_config = OmegaConf.load("configs/default_config.yaml") config = OmegaConf.merge(default_config, config) if args.reuse_first_step_velocity: config.reuse_first_step_velocity = True config.reuse_first_step_velocity_schedule = ( args.reuse_first_step_velocity_schedule ) # Initialize pipeline if hasattr(config, 'denoising_step_list'): # Few-step inference pipeline = CausalInferencePipeline(config, device=device) else: # Multi-step diffusion inference pipeline = CausalDiffusionInferencePipeline(config, device=device) if args.checkpoint_path: state_dict = torch.load(args.checkpoint_path, map_location="cpu") pipeline.generator.load_state_dict(state_dict['generator' if not args.use_ema else 'generator_ema']) del state_dict if args.predictor_checkpoint: if not isinstance(pipeline, CausalInferencePipeline): raise ValueError("Predictor-v4 requires the few-step CausalInferencePipeline") if args.reuse_first_step_velocity: raise ValueError( "--predictor_checkpoint and --reuse_first_step_velocity are mutually exclusive" ) source_blocks = tuple( int(value.strip()) for value in args.predictor_source_blocks.split(",") if value.strip() ) if len(source_blocks) != 2: raise ValueError("--predictor_source_blocks must contain exactly two ids") predictor_state, predictor_metadata = load_predictor_checkpoint( args.predictor_checkpoint ) saved_source_blocks = tuple( int(value) for value in predictor_metadata.get("source_block_ids", ()) ) if saved_source_blocks != source_blocks: raise ValueError( "Predictor checkpoint source blocks do not match the requested blocks: " f"checkpoint={saved_source_blocks}, requested={source_blocks}" ) expected_teacher_key = "generator_ema" if args.use_ema else "generator" saved_teacher_key = predictor_metadata.get("teacher_checkpoint_key") if saved_teacher_key is not None and saved_teacher_key != expected_teacher_key: raise ValueError( "Predictor checkpoint was initialized from " f"{saved_teacher_key}, but inference loaded {expected_teacher_key}" ) predictor = SelfForcingPredictorV4.from_teacher( pipeline.generator.model, source_block_ids=source_blocks, spatial_grid=(30, 52), ) predictor.load_trainable_state_dict( predictor_state, strict=True, ) saved_architecture = predictor_metadata.get("predictor_config") if saved_architecture is not None: current_architecture = predictor.config_dict normalized_current = json.loads(json.dumps(current_architecture)) if saved_architecture != normalized_current: raise ValueError( "Predictor checkpoint architecture does not match the loaded Teacher: " f"checkpoint={saved_architecture}, current={normalized_current}" ) pipeline.enable_predictor_v4( predictor, schedule=args.predictor_schedule, ) pipeline = pipeline.to(dtype=torch.bfloat16) if low_memory: DynamicSwapInstaller.install_model(pipeline.text_encoder, device=device) else: pipeline.text_encoder.to(device=device) pipeline.generator.to(device=device) pipeline.vae.to(device=device) if pipeline.predictor_v4 is not None: pipeline.predictor_v4.to(device=device) # Create dataset if args.i2v: assert not dist.is_initialized(), "I2V does not support distributed inference yet" transform = transforms.Compose([ transforms.Resize((480, 832)), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]) ]) dataset = TextImagePairDataset(args.data_path, transform=transform) else: dataset = TextDataset(prompt_path=args.data_path, extended_prompt_path=args.extended_prompt_path) if args.max_prompts is not None: if args.max_prompts <= 0: raise ValueError("--max_prompts must be a positive integer") dataset = Subset(dataset, range(min(args.max_prompts, len(dataset)))) num_prompts = len(dataset) print(f"Number of prompts: {num_prompts}") if dist.is_initialized(): # Assign every prompt to exactly one rank without padding or dropping the # tail when the prompt count is not divisible by world size. sampler = list(range(local_rank, len(dataset), world_size)) else: sampler = SequentialSampler(dataset) dataloader = DataLoader(dataset, batch_size=1, sampler=sampler, num_workers=0, drop_last=False) # Create output directory (only on main process to avoid race conditions) if local_rank == 0: os.makedirs(args.output_folder, exist_ok=True) if dist.is_initialized(): dist.barrier(device_ids=[local_rank]) def encode(self, videos: torch.Tensor) -> torch.Tensor: device, dtype = videos[0].device, videos[0].dtype scale = [self.mean.to(device=device, dtype=dtype), 1.0 / self.std.to(device=device, dtype=dtype)] output = [ self.model.encode(u.unsqueeze(0), scale).float().squeeze(0) for u in videos ] output = torch.stack(output, dim=0) return output def write_video(output_path: str, video: torch.Tensor, fps: int = 16) -> None: video = video.clamp(0, 255).to(torch.uint8).cpu().numpy() imageio.mimsave(output_path, video, fps=fps, codec="libx264") for i, batch_data in tqdm(enumerate(dataloader), disable=(local_rank != 0)): idx = batch_data['idx'].item() # Keep the experiment invariant to rank assignment and resume position: # every prompt starts from the requested seed (seed 0 in our benchmarks). set_seed(args.seed) # For DataLoader batch_size=1, the batch_data is already a single item, but in a batch container # Unpack the batch data for convenience if isinstance(batch_data, dict): batch = batch_data elif isinstance(batch_data, list): batch = batch_data[0] # First (and only) item in the batch all_video = [] num_generated_frames = 0 # Number of generated (latent) frames if args.i2v: # For image-to-video, batch contains image and caption prompt = batch['prompts'][0] # Get caption from batch prompts = [prompt] * args.num_samples # Process the image image = batch['image'].squeeze(0).unsqueeze(0).unsqueeze(2).to(device=device, dtype=torch.bfloat16) # Encode the input image as the first latent initial_latent = pipeline.vae.encode_to_latent(image).to(device=device, dtype=torch.bfloat16) initial_latent = initial_latent.repeat(args.num_samples, 1, 1, 1, 1) sampled_noise = torch.randn( [args.num_samples, args.num_output_frames - 1, 16, 60, 104], device=device, dtype=torch.bfloat16 ) else: # For text-to-video, batch is just the text prompt prompt = batch['prompts'][0] extended_prompt = batch['extended_prompts'][0] if 'extended_prompts' in batch else None if extended_prompt is not None: prompts = [extended_prompt] * args.num_samples else: prompts = [prompt] * args.num_samples initial_latent = None sampled_noise = torch.randn( [args.num_samples, args.num_output_frames, 16, 60, 104], device=device, dtype=torch.bfloat16 ) # Generate 81 frames video, latents = pipeline.inference( noise=sampled_noise, text_prompts=prompts, return_latents=True, initial_latent=initial_latent, low_memory=low_memory, ) current_video = rearrange(video, 'b t c h w -> b t h w c').cpu() all_video.append(current_video) num_generated_frames += latents.shape[1] # Final output video video = 255.0 * torch.cat(all_video, dim=1) # Clear VAE cache pipeline.vae.model.clear_cache() # Save the video if the current prompt is not a dummy prompt if idx < num_prompts: model = "regular" if not args.use_ema else "ema" for seed_idx in range(args.num_samples): # All processes save their videos if args.save_with_index: output_path = os.path.join(args.output_folder, f'{idx}-{seed_idx}_{model}.mp4') else: output_path = os.path.join(args.output_folder, f'{prompt[:100]}-{seed_idx}.mp4') write_video(output_path, video[seed_idx], fps=16)