Upload pipeline source
Browse files- pipeline/__init__.py +13 -0
- pipeline/bidirectional_diffusion_inference.py +110 -0
- pipeline/bidirectional_inference.py +71 -0
- pipeline/causal_diffusion_inference.py +342 -0
- pipeline/causal_inference.py +312 -0
- pipeline/self_forcing_training.py +267 -0
pipeline/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .bidirectional_diffusion_inference import BidirectionalDiffusionInferencePipeline
|
| 2 |
+
from .bidirectional_inference import BidirectionalInferencePipeline
|
| 3 |
+
from .causal_diffusion_inference import CausalDiffusionInferencePipeline
|
| 4 |
+
from .causal_inference import CausalInferencePipeline
|
| 5 |
+
from .self_forcing_training import SelfForcingTrainingPipeline
|
| 6 |
+
|
| 7 |
+
__all__ = [
|
| 8 |
+
"BidirectionalDiffusionInferencePipeline",
|
| 9 |
+
"BidirectionalInferencePipeline",
|
| 10 |
+
"CausalDiffusionInferencePipeline",
|
| 11 |
+
"CausalInferencePipeline",
|
| 12 |
+
"SelfForcingTrainingPipeline"
|
| 13 |
+
]
|
pipeline/bidirectional_diffusion_inference.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from tqdm import tqdm
|
| 2 |
+
from typing import List
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from wan.utils.fm_solvers import FlowDPMSolverMultistepScheduler, get_sampling_sigmas, retrieve_timesteps
|
| 6 |
+
from wan.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler
|
| 7 |
+
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class BidirectionalDiffusionInferencePipeline(torch.nn.Module):
|
| 11 |
+
def __init__(
|
| 12 |
+
self,
|
| 13 |
+
args,
|
| 14 |
+
device,
|
| 15 |
+
generator=None,
|
| 16 |
+
text_encoder=None,
|
| 17 |
+
vae=None
|
| 18 |
+
):
|
| 19 |
+
super().__init__()
|
| 20 |
+
# Step 1: Initialize all models
|
| 21 |
+
self.generator = WanDiffusionWrapper(
|
| 22 |
+
**getattr(args, "model_kwargs", {}), is_causal=False) if generator is None else generator
|
| 23 |
+
self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
|
| 24 |
+
self.vae = WanVAEWrapper() if vae is None else vae
|
| 25 |
+
|
| 26 |
+
# Step 2: Initialize scheduler
|
| 27 |
+
self.num_train_timesteps = args.num_train_timestep
|
| 28 |
+
self.sampling_steps = 50
|
| 29 |
+
self.sample_solver = 'unipc'
|
| 30 |
+
self.shift = 8.0
|
| 31 |
+
|
| 32 |
+
self.args = args
|
| 33 |
+
|
| 34 |
+
def inference(
|
| 35 |
+
self,
|
| 36 |
+
noise: torch.Tensor,
|
| 37 |
+
text_prompts: List[str],
|
| 38 |
+
return_latents=False
|
| 39 |
+
) -> torch.Tensor:
|
| 40 |
+
"""
|
| 41 |
+
Perform inference on the given noise and text prompts.
|
| 42 |
+
Inputs:
|
| 43 |
+
noise (torch.Tensor): The input noise tensor of shape
|
| 44 |
+
(batch_size, num_frames, num_channels, height, width).
|
| 45 |
+
text_prompts (List[str]): The list of text prompts.
|
| 46 |
+
Outputs:
|
| 47 |
+
video (torch.Tensor): The generated video tensor of shape
|
| 48 |
+
(batch_size, num_frames, num_channels, height, width). It is normalized to be in the range [0, 1].
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
conditional_dict = self.text_encoder(
|
| 52 |
+
text_prompts=text_prompts
|
| 53 |
+
)
|
| 54 |
+
unconditional_dict = self.text_encoder(
|
| 55 |
+
text_prompts=[self.args.negative_prompt] * len(text_prompts)
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
latents = noise
|
| 59 |
+
|
| 60 |
+
sample_scheduler = self._initialize_sample_scheduler(noise)
|
| 61 |
+
for _, t in enumerate(tqdm(sample_scheduler.timesteps)):
|
| 62 |
+
latent_model_input = latents
|
| 63 |
+
timestep = t * torch.ones([latents.shape[0], 21], device=noise.device, dtype=torch.float32)
|
| 64 |
+
|
| 65 |
+
flow_pred_cond, _ = self.generator(latent_model_input, conditional_dict, timestep)
|
| 66 |
+
flow_pred_uncond, _ = self.generator(latent_model_input, unconditional_dict, timestep)
|
| 67 |
+
|
| 68 |
+
flow_pred = flow_pred_uncond + self.args.guidance_scale * (
|
| 69 |
+
flow_pred_cond - flow_pred_uncond)
|
| 70 |
+
|
| 71 |
+
temp_x0 = sample_scheduler.step(
|
| 72 |
+
flow_pred.unsqueeze(0),
|
| 73 |
+
t,
|
| 74 |
+
latents.unsqueeze(0),
|
| 75 |
+
return_dict=False)[0]
|
| 76 |
+
latents = temp_x0.squeeze(0)
|
| 77 |
+
|
| 78 |
+
x0 = latents
|
| 79 |
+
video = self.vae.decode_to_pixel(x0)
|
| 80 |
+
video = (video * 0.5 + 0.5).clamp(0, 1)
|
| 81 |
+
|
| 82 |
+
del sample_scheduler
|
| 83 |
+
|
| 84 |
+
if return_latents:
|
| 85 |
+
return video, latents
|
| 86 |
+
else:
|
| 87 |
+
return video
|
| 88 |
+
|
| 89 |
+
def _initialize_sample_scheduler(self, noise):
|
| 90 |
+
if self.sample_solver == 'unipc':
|
| 91 |
+
sample_scheduler = FlowUniPCMultistepScheduler(
|
| 92 |
+
num_train_timesteps=self.num_train_timesteps,
|
| 93 |
+
shift=1,
|
| 94 |
+
use_dynamic_shifting=False)
|
| 95 |
+
sample_scheduler.set_timesteps(
|
| 96 |
+
self.sampling_steps, device=noise.device, shift=self.shift)
|
| 97 |
+
self.timesteps = sample_scheduler.timesteps
|
| 98 |
+
elif self.sample_solver == 'dpm++':
|
| 99 |
+
sample_scheduler = FlowDPMSolverMultistepScheduler(
|
| 100 |
+
num_train_timesteps=self.num_train_timesteps,
|
| 101 |
+
shift=1,
|
| 102 |
+
use_dynamic_shifting=False)
|
| 103 |
+
sampling_sigmas = get_sampling_sigmas(self.sampling_steps, self.shift)
|
| 104 |
+
self.timesteps, _ = retrieve_timesteps(
|
| 105 |
+
sample_scheduler,
|
| 106 |
+
device=noise.device,
|
| 107 |
+
sigmas=sampling_sigmas)
|
| 108 |
+
else:
|
| 109 |
+
raise NotImplementedError("Unsupported solver.")
|
| 110 |
+
return sample_scheduler
|
pipeline/bidirectional_inference.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class BidirectionalInferencePipeline(torch.nn.Module):
|
| 8 |
+
def __init__(
|
| 9 |
+
self,
|
| 10 |
+
args,
|
| 11 |
+
device,
|
| 12 |
+
generator=None,
|
| 13 |
+
text_encoder=None,
|
| 14 |
+
vae=None
|
| 15 |
+
):
|
| 16 |
+
super().__init__()
|
| 17 |
+
# Step 1: Initialize all models
|
| 18 |
+
self.generator = WanDiffusionWrapper(
|
| 19 |
+
**getattr(args, "model_kwargs", {}), is_causal=False) if generator is None else generator
|
| 20 |
+
self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
|
| 21 |
+
self.vae = WanVAEWrapper() if vae is None else vae
|
| 22 |
+
|
| 23 |
+
# Step 2: Initialize all bidirectional wan hyperparmeters
|
| 24 |
+
self.scheduler = self.generator.get_scheduler()
|
| 25 |
+
self.denoising_step_list = torch.tensor(
|
| 26 |
+
args.denoising_step_list, dtype=torch.long, device=device)
|
| 27 |
+
if self.denoising_step_list[-1] == 0:
|
| 28 |
+
self.denoising_step_list = self.denoising_step_list[:-1] # remove the zero timestep for inference
|
| 29 |
+
if args.warp_denoising_step:
|
| 30 |
+
timesteps = torch.cat((self.scheduler.timesteps.cpu(), torch.tensor([0], dtype=torch.float32)))
|
| 31 |
+
self.denoising_step_list = timesteps[1000 - self.denoising_step_list]
|
| 32 |
+
|
| 33 |
+
def inference(self, noise: torch.Tensor, text_prompts: List[str]) -> torch.Tensor:
|
| 34 |
+
"""
|
| 35 |
+
Perform inference on the given noise and text prompts.
|
| 36 |
+
Inputs:
|
| 37 |
+
noise (torch.Tensor): The input noise tensor of shape
|
| 38 |
+
(batch_size, num_frames, num_channels, height, width).
|
| 39 |
+
text_prompts (List[str]): The list of text prompts.
|
| 40 |
+
Outputs:
|
| 41 |
+
video (torch.Tensor): The generated video tensor of shape
|
| 42 |
+
(batch_size, num_frames, num_channels, height, width). It is normalized to be in the range [0, 1].
|
| 43 |
+
"""
|
| 44 |
+
conditional_dict = self.text_encoder(
|
| 45 |
+
text_prompts=text_prompts
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
# initial point
|
| 49 |
+
noisy_image_or_video = noise
|
| 50 |
+
|
| 51 |
+
# use the last n-1 timesteps to simulate the generator's input
|
| 52 |
+
for index, current_timestep in enumerate(self.denoising_step_list[:-1]):
|
| 53 |
+
_, pred_image_or_video = self.generator(
|
| 54 |
+
noisy_image_or_video=noisy_image_or_video,
|
| 55 |
+
conditional_dict=conditional_dict,
|
| 56 |
+
timestep=torch.ones(
|
| 57 |
+
noise.shape[:2], dtype=torch.long, device=noise.device) * current_timestep
|
| 58 |
+
) # [B, F, C, H, W]
|
| 59 |
+
|
| 60 |
+
next_timestep = self.denoising_step_list[index + 1] * torch.ones(
|
| 61 |
+
noise.shape[:2], dtype=torch.long, device=noise.device)
|
| 62 |
+
|
| 63 |
+
noisy_image_or_video = self.scheduler.add_noise(
|
| 64 |
+
pred_image_or_video.flatten(0, 1),
|
| 65 |
+
torch.randn_like(pred_image_or_video.flatten(0, 1)),
|
| 66 |
+
next_timestep.flatten(0, 1)
|
| 67 |
+
).unflatten(0, noise.shape[:2])
|
| 68 |
+
|
| 69 |
+
video = self.vae.decode_to_pixel(pred_image_or_video)
|
| 70 |
+
video = (video * 0.5 + 0.5).clamp(0, 1)
|
| 71 |
+
return video
|
pipeline/causal_diffusion_inference.py
ADDED
|
@@ -0,0 +1,342 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from tqdm import tqdm
|
| 2 |
+
from typing import List, Optional
|
| 3 |
+
import torch
|
| 4 |
+
|
| 5 |
+
from wan.utils.fm_solvers import FlowDPMSolverMultistepScheduler, get_sampling_sigmas, retrieve_timesteps
|
| 6 |
+
from wan.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler
|
| 7 |
+
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class CausalDiffusionInferencePipeline(torch.nn.Module):
|
| 11 |
+
def __init__(
|
| 12 |
+
self,
|
| 13 |
+
args,
|
| 14 |
+
device,
|
| 15 |
+
generator=None,
|
| 16 |
+
text_encoder=None,
|
| 17 |
+
vae=None
|
| 18 |
+
):
|
| 19 |
+
super().__init__()
|
| 20 |
+
# Step 1: Initialize all models
|
| 21 |
+
self.generator = WanDiffusionWrapper(
|
| 22 |
+
**getattr(args, "model_kwargs", {}), is_causal=True) if generator is None else generator
|
| 23 |
+
self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
|
| 24 |
+
self.vae = WanVAEWrapper() if vae is None else vae
|
| 25 |
+
|
| 26 |
+
# Step 2: Initialize scheduler
|
| 27 |
+
self.num_train_timesteps = args.num_train_timestep
|
| 28 |
+
self.sampling_steps = 50
|
| 29 |
+
self.sample_solver = 'unipc'
|
| 30 |
+
self.shift = args.timestep_shift
|
| 31 |
+
|
| 32 |
+
self.num_transformer_blocks = 30
|
| 33 |
+
self.frame_seq_length = 1560
|
| 34 |
+
|
| 35 |
+
self.kv_cache_pos = None
|
| 36 |
+
self.kv_cache_neg = None
|
| 37 |
+
self.crossattn_cache_pos = None
|
| 38 |
+
self.crossattn_cache_neg = None
|
| 39 |
+
self.args = args
|
| 40 |
+
self.num_frame_per_block = getattr(args, "num_frame_per_block", 1)
|
| 41 |
+
self.independent_first_frame = args.independent_first_frame
|
| 42 |
+
self.local_attn_size = self.generator.model.local_attn_size
|
| 43 |
+
|
| 44 |
+
print(f"KV inference with {self.num_frame_per_block} frames per block")
|
| 45 |
+
|
| 46 |
+
if self.num_frame_per_block > 1:
|
| 47 |
+
self.generator.model.num_frame_per_block = self.num_frame_per_block
|
| 48 |
+
|
| 49 |
+
def inference(
|
| 50 |
+
self,
|
| 51 |
+
noise: torch.Tensor,
|
| 52 |
+
text_prompts: List[str],
|
| 53 |
+
initial_latent: Optional[torch.Tensor] = None,
|
| 54 |
+
return_latents: bool = False,
|
| 55 |
+
start_frame_index: Optional[int] = 0
|
| 56 |
+
) -> torch.Tensor:
|
| 57 |
+
"""
|
| 58 |
+
Perform inference on the given noise and text prompts.
|
| 59 |
+
Inputs:
|
| 60 |
+
noise (torch.Tensor): The input noise tensor of shape
|
| 61 |
+
(batch_size, num_output_frames, num_channels, height, width).
|
| 62 |
+
text_prompts (List[str]): The list of text prompts.
|
| 63 |
+
initial_latent (torch.Tensor): The initial latent tensor of shape
|
| 64 |
+
(batch_size, num_input_frames, num_channels, height, width).
|
| 65 |
+
If num_input_frames is 1, perform image to video.
|
| 66 |
+
If num_input_frames is greater than 1, perform video extension.
|
| 67 |
+
return_latents (bool): Whether to return the latents.
|
| 68 |
+
start_frame_index (int): In long video generation, where does the current window start?
|
| 69 |
+
Outputs:
|
| 70 |
+
video (torch.Tensor): The generated video tensor of shape
|
| 71 |
+
(batch_size, num_frames, num_channels, height, width). It is normalized to be in the range [0, 1].
|
| 72 |
+
"""
|
| 73 |
+
batch_size, num_frames, num_channels, height, width = noise.shape
|
| 74 |
+
if not self.independent_first_frame or (self.independent_first_frame and initial_latent is not None):
|
| 75 |
+
# If the first frame is independent and the first frame is provided, then the number of frames in the
|
| 76 |
+
# noise should still be a multiple of num_frame_per_block
|
| 77 |
+
assert num_frames % self.num_frame_per_block == 0
|
| 78 |
+
num_blocks = num_frames // self.num_frame_per_block
|
| 79 |
+
elif self.independent_first_frame and initial_latent is None:
|
| 80 |
+
# Using a [1, 4, 4, 4, 4, 4] model to generate a video without image conditioning
|
| 81 |
+
assert (num_frames - 1) % self.num_frame_per_block == 0
|
| 82 |
+
num_blocks = (num_frames - 1) // self.num_frame_per_block
|
| 83 |
+
num_input_frames = initial_latent.shape[1] if initial_latent is not None else 0
|
| 84 |
+
num_output_frames = num_frames + num_input_frames # add the initial latent frames
|
| 85 |
+
conditional_dict = self.text_encoder(
|
| 86 |
+
text_prompts=text_prompts
|
| 87 |
+
)
|
| 88 |
+
unconditional_dict = self.text_encoder(
|
| 89 |
+
text_prompts=[self.args.negative_prompt] * len(text_prompts)
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
output = torch.zeros(
|
| 93 |
+
[batch_size, num_output_frames, num_channels, height, width],
|
| 94 |
+
device=noise.device,
|
| 95 |
+
dtype=noise.dtype
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
# Step 1: Initialize KV cache to all zeros
|
| 99 |
+
if self.kv_cache_pos is None:
|
| 100 |
+
self._initialize_kv_cache(
|
| 101 |
+
batch_size=batch_size,
|
| 102 |
+
dtype=noise.dtype,
|
| 103 |
+
device=noise.device
|
| 104 |
+
)
|
| 105 |
+
self._initialize_crossattn_cache(
|
| 106 |
+
batch_size=batch_size,
|
| 107 |
+
dtype=noise.dtype,
|
| 108 |
+
device=noise.device
|
| 109 |
+
)
|
| 110 |
+
else:
|
| 111 |
+
# reset cross attn cache
|
| 112 |
+
for block_index in range(self.num_transformer_blocks):
|
| 113 |
+
self.crossattn_cache_pos[block_index]["is_init"] = False
|
| 114 |
+
self.crossattn_cache_neg[block_index]["is_init"] = False
|
| 115 |
+
# reset kv cache
|
| 116 |
+
for block_index in range(len(self.kv_cache_pos)):
|
| 117 |
+
self.kv_cache_pos[block_index]["global_end_index"] = torch.tensor(
|
| 118 |
+
[0], dtype=torch.long, device=noise.device)
|
| 119 |
+
self.kv_cache_pos[block_index]["local_end_index"] = torch.tensor(
|
| 120 |
+
[0], dtype=torch.long, device=noise.device)
|
| 121 |
+
self.kv_cache_neg[block_index]["global_end_index"] = torch.tensor(
|
| 122 |
+
[0], dtype=torch.long, device=noise.device)
|
| 123 |
+
self.kv_cache_neg[block_index]["local_end_index"] = torch.tensor(
|
| 124 |
+
[0], dtype=torch.long, device=noise.device)
|
| 125 |
+
|
| 126 |
+
# Step 2: Cache context feature
|
| 127 |
+
current_start_frame = start_frame_index
|
| 128 |
+
cache_start_frame = 0
|
| 129 |
+
if initial_latent is not None:
|
| 130 |
+
timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
|
| 131 |
+
if self.independent_first_frame:
|
| 132 |
+
# Assume num_input_frames is 1 + self.num_frame_per_block * num_input_blocks
|
| 133 |
+
assert (num_input_frames - 1) % self.num_frame_per_block == 0
|
| 134 |
+
num_input_blocks = (num_input_frames - 1) // self.num_frame_per_block
|
| 135 |
+
output[:, :1] = initial_latent[:, :1]
|
| 136 |
+
self.generator(
|
| 137 |
+
noisy_image_or_video=initial_latent[:, :1],
|
| 138 |
+
conditional_dict=conditional_dict,
|
| 139 |
+
timestep=timestep * 0,
|
| 140 |
+
kv_cache=self.kv_cache_pos,
|
| 141 |
+
crossattn_cache=self.crossattn_cache_pos,
|
| 142 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 143 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 144 |
+
)
|
| 145 |
+
self.generator(
|
| 146 |
+
noisy_image_or_video=initial_latent[:, :1],
|
| 147 |
+
conditional_dict=unconditional_dict,
|
| 148 |
+
timestep=timestep * 0,
|
| 149 |
+
kv_cache=self.kv_cache_neg,
|
| 150 |
+
crossattn_cache=self.crossattn_cache_neg,
|
| 151 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 152 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 153 |
+
)
|
| 154 |
+
current_start_frame += 1
|
| 155 |
+
cache_start_frame += 1
|
| 156 |
+
else:
|
| 157 |
+
# Assume num_input_frames is self.num_frame_per_block * num_input_blocks
|
| 158 |
+
assert num_input_frames % self.num_frame_per_block == 0
|
| 159 |
+
num_input_blocks = num_input_frames // self.num_frame_per_block
|
| 160 |
+
|
| 161 |
+
for block_index in range(num_input_blocks):
|
| 162 |
+
current_ref_latents = \
|
| 163 |
+
initial_latent[:, cache_start_frame:cache_start_frame + self.num_frame_per_block]
|
| 164 |
+
output[:, cache_start_frame:cache_start_frame + self.num_frame_per_block] = current_ref_latents
|
| 165 |
+
self.generator(
|
| 166 |
+
noisy_image_or_video=current_ref_latents,
|
| 167 |
+
conditional_dict=conditional_dict,
|
| 168 |
+
timestep=timestep * 0,
|
| 169 |
+
kv_cache=self.kv_cache_pos,
|
| 170 |
+
crossattn_cache=self.crossattn_cache_pos,
|
| 171 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 172 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 173 |
+
)
|
| 174 |
+
self.generator(
|
| 175 |
+
noisy_image_or_video=current_ref_latents,
|
| 176 |
+
conditional_dict=unconditional_dict,
|
| 177 |
+
timestep=timestep * 0,
|
| 178 |
+
kv_cache=self.kv_cache_neg,
|
| 179 |
+
crossattn_cache=self.crossattn_cache_neg,
|
| 180 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 181 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 182 |
+
)
|
| 183 |
+
current_start_frame += self.num_frame_per_block
|
| 184 |
+
cache_start_frame += self.num_frame_per_block
|
| 185 |
+
|
| 186 |
+
# Step 3: Temporal denoising loop
|
| 187 |
+
all_num_frames = [self.num_frame_per_block] * num_blocks
|
| 188 |
+
if self.independent_first_frame and initial_latent is None:
|
| 189 |
+
all_num_frames = [1] + all_num_frames
|
| 190 |
+
for current_num_frames in all_num_frames:
|
| 191 |
+
noisy_input = noise[
|
| 192 |
+
:, cache_start_frame - num_input_frames:cache_start_frame + current_num_frames - num_input_frames]
|
| 193 |
+
latents = noisy_input
|
| 194 |
+
|
| 195 |
+
# Step 3.1: Spatial denoising loop
|
| 196 |
+
sample_scheduler = self._initialize_sample_scheduler(noise)
|
| 197 |
+
for _, t in enumerate(tqdm(sample_scheduler.timesteps)):
|
| 198 |
+
latent_model_input = latents
|
| 199 |
+
timestep = t * torch.ones(
|
| 200 |
+
[batch_size, current_num_frames], device=noise.device, dtype=torch.float32
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
flow_pred_cond, _ = self.generator(
|
| 204 |
+
noisy_image_or_video=latent_model_input,
|
| 205 |
+
conditional_dict=conditional_dict,
|
| 206 |
+
timestep=timestep,
|
| 207 |
+
kv_cache=self.kv_cache_pos,
|
| 208 |
+
crossattn_cache=self.crossattn_cache_pos,
|
| 209 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 210 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 211 |
+
)
|
| 212 |
+
flow_pred_uncond, _ = self.generator(
|
| 213 |
+
noisy_image_or_video=latent_model_input,
|
| 214 |
+
conditional_dict=unconditional_dict,
|
| 215 |
+
timestep=timestep,
|
| 216 |
+
kv_cache=self.kv_cache_neg,
|
| 217 |
+
crossattn_cache=self.crossattn_cache_neg,
|
| 218 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 219 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
flow_pred = flow_pred_uncond + self.args.guidance_scale * (
|
| 223 |
+
flow_pred_cond - flow_pred_uncond)
|
| 224 |
+
|
| 225 |
+
temp_x0 = sample_scheduler.step(
|
| 226 |
+
flow_pred,
|
| 227 |
+
t,
|
| 228 |
+
latents,
|
| 229 |
+
return_dict=False)[0]
|
| 230 |
+
latents = temp_x0
|
| 231 |
+
print(f"kv_cache['local_end_index']: {self.kv_cache_pos[0]['local_end_index']}")
|
| 232 |
+
print(f"kv_cache['global_end_index']: {self.kv_cache_pos[0]['global_end_index']}")
|
| 233 |
+
|
| 234 |
+
# Step 3.2: record the model's output
|
| 235 |
+
output[:, cache_start_frame:cache_start_frame + current_num_frames] = latents
|
| 236 |
+
|
| 237 |
+
# Step 3.3: rerun with timestep zero to update KV cache using clean context
|
| 238 |
+
self.generator(
|
| 239 |
+
noisy_image_or_video=latents,
|
| 240 |
+
conditional_dict=conditional_dict,
|
| 241 |
+
timestep=timestep * 0,
|
| 242 |
+
kv_cache=self.kv_cache_pos,
|
| 243 |
+
crossattn_cache=self.crossattn_cache_pos,
|
| 244 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 245 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 246 |
+
)
|
| 247 |
+
self.generator(
|
| 248 |
+
noisy_image_or_video=latents,
|
| 249 |
+
conditional_dict=unconditional_dict,
|
| 250 |
+
timestep=timestep * 0,
|
| 251 |
+
kv_cache=self.kv_cache_neg,
|
| 252 |
+
crossattn_cache=self.crossattn_cache_neg,
|
| 253 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 254 |
+
cache_start=cache_start_frame * self.frame_seq_length
|
| 255 |
+
)
|
| 256 |
+
|
| 257 |
+
# Step 3.4: update the start and end frame indices
|
| 258 |
+
current_start_frame += current_num_frames
|
| 259 |
+
cache_start_frame += current_num_frames
|
| 260 |
+
|
| 261 |
+
# Step 4: Decode the output
|
| 262 |
+
video = self.vae.decode_to_pixel(output)
|
| 263 |
+
video = (video * 0.5 + 0.5).clamp(0, 1)
|
| 264 |
+
|
| 265 |
+
if return_latents:
|
| 266 |
+
return video, output
|
| 267 |
+
else:
|
| 268 |
+
return video
|
| 269 |
+
|
| 270 |
+
def _initialize_kv_cache(self, batch_size, dtype, device):
|
| 271 |
+
"""
|
| 272 |
+
Initialize a Per-GPU KV cache for the Wan model.
|
| 273 |
+
"""
|
| 274 |
+
kv_cache_pos = []
|
| 275 |
+
kv_cache_neg = []
|
| 276 |
+
if self.local_attn_size != -1:
|
| 277 |
+
# Use the local attention size to compute the KV cache size
|
| 278 |
+
kv_cache_size = self.local_attn_size * self.frame_seq_length
|
| 279 |
+
else:
|
| 280 |
+
# Use the default KV cache size
|
| 281 |
+
kv_cache_size = 32760
|
| 282 |
+
|
| 283 |
+
for _ in range(self.num_transformer_blocks):
|
| 284 |
+
kv_cache_pos.append({
|
| 285 |
+
"k": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 286 |
+
"v": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 287 |
+
"global_end_index": torch.tensor([0], dtype=torch.long, device=device),
|
| 288 |
+
"local_end_index": torch.tensor([0], dtype=torch.long, device=device)
|
| 289 |
+
})
|
| 290 |
+
kv_cache_neg.append({
|
| 291 |
+
"k": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 292 |
+
"v": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 293 |
+
"global_end_index": torch.tensor([0], dtype=torch.long, device=device),
|
| 294 |
+
"local_end_index": torch.tensor([0], dtype=torch.long, device=device)
|
| 295 |
+
})
|
| 296 |
+
|
| 297 |
+
self.kv_cache_pos = kv_cache_pos # always store the clean cache
|
| 298 |
+
self.kv_cache_neg = kv_cache_neg # always store the clean cache
|
| 299 |
+
|
| 300 |
+
def _initialize_crossattn_cache(self, batch_size, dtype, device):
|
| 301 |
+
"""
|
| 302 |
+
Initialize a Per-GPU cross-attention cache for the Wan model.
|
| 303 |
+
"""
|
| 304 |
+
crossattn_cache_pos = []
|
| 305 |
+
crossattn_cache_neg = []
|
| 306 |
+
for _ in range(self.num_transformer_blocks):
|
| 307 |
+
crossattn_cache_pos.append({
|
| 308 |
+
"k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 309 |
+
"v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 310 |
+
"is_init": False
|
| 311 |
+
})
|
| 312 |
+
crossattn_cache_neg.append({
|
| 313 |
+
"k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 314 |
+
"v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 315 |
+
"is_init": False
|
| 316 |
+
})
|
| 317 |
+
|
| 318 |
+
self.crossattn_cache_pos = crossattn_cache_pos # always store the clean cache
|
| 319 |
+
self.crossattn_cache_neg = crossattn_cache_neg # always store the clean cache
|
| 320 |
+
|
| 321 |
+
def _initialize_sample_scheduler(self, noise):
|
| 322 |
+
if self.sample_solver == 'unipc':
|
| 323 |
+
sample_scheduler = FlowUniPCMultistepScheduler(
|
| 324 |
+
num_train_timesteps=self.num_train_timesteps,
|
| 325 |
+
shift=1,
|
| 326 |
+
use_dynamic_shifting=False)
|
| 327 |
+
sample_scheduler.set_timesteps(
|
| 328 |
+
self.sampling_steps, device=noise.device, shift=self.shift)
|
| 329 |
+
self.timesteps = sample_scheduler.timesteps
|
| 330 |
+
elif self.sample_solver == 'dpm++':
|
| 331 |
+
sample_scheduler = FlowDPMSolverMultistepScheduler(
|
| 332 |
+
num_train_timesteps=self.num_train_timesteps,
|
| 333 |
+
shift=1,
|
| 334 |
+
use_dynamic_shifting=False)
|
| 335 |
+
sampling_sigmas = get_sampling_sigmas(self.sampling_steps, self.shift)
|
| 336 |
+
self.timesteps, _ = retrieve_timesteps(
|
| 337 |
+
sample_scheduler,
|
| 338 |
+
device=noise.device,
|
| 339 |
+
sigmas=sampling_sigmas)
|
| 340 |
+
else:
|
| 341 |
+
raise NotImplementedError("Unsupported solver.")
|
| 342 |
+
return sample_scheduler
|
pipeline/causal_inference.py
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List, Optional
|
| 2 |
+
import torch
|
| 3 |
+
|
| 4 |
+
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
|
| 5 |
+
|
| 6 |
+
from demo_utils.memory import gpu, get_cuda_free_memory_gb, DynamicSwapInstaller, move_model_to_device_with_memory_preservation
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class CausalInferencePipeline(torch.nn.Module):
|
| 10 |
+
def __init__(
|
| 11 |
+
self,
|
| 12 |
+
args,
|
| 13 |
+
device,
|
| 14 |
+
generator=None,
|
| 15 |
+
text_encoder=None,
|
| 16 |
+
vae=None
|
| 17 |
+
):
|
| 18 |
+
super().__init__()
|
| 19 |
+
# Step 1: Initialize all models
|
| 20 |
+
self.generator = WanDiffusionWrapper(
|
| 21 |
+
**getattr(args, "model_kwargs", {}), is_causal=True) if generator is None else generator
|
| 22 |
+
self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
|
| 23 |
+
self.vae = WanVAEWrapper() if vae is None else vae
|
| 24 |
+
|
| 25 |
+
# Step 2: Initialize all causal hyperparmeters
|
| 26 |
+
self.scheduler = self.generator.get_scheduler()
|
| 27 |
+
self.denoising_step_list = torch.tensor(
|
| 28 |
+
args.denoising_step_list, dtype=torch.long)
|
| 29 |
+
if args.warp_denoising_step:
|
| 30 |
+
timesteps = torch.cat((self.scheduler.timesteps.cpu(), torch.tensor([0], dtype=torch.float32)))
|
| 31 |
+
self.denoising_step_list = timesteps[1000 - self.denoising_step_list]
|
| 32 |
+
|
| 33 |
+
self.num_transformer_blocks = 30
|
| 34 |
+
self.frame_seq_length = 1560
|
| 35 |
+
|
| 36 |
+
self.kv_cache1 = None
|
| 37 |
+
self.args = args
|
| 38 |
+
self.num_frame_per_block = getattr(args, "num_frame_per_block", 1)
|
| 39 |
+
self.independent_first_frame = args.independent_first_frame
|
| 40 |
+
self.local_attn_size = self.generator.model.local_attn_size
|
| 41 |
+
|
| 42 |
+
print(f"KV inference with {self.num_frame_per_block} frames per block")
|
| 43 |
+
|
| 44 |
+
if self.num_frame_per_block > 1:
|
| 45 |
+
self.generator.model.num_frame_per_block = self.num_frame_per_block
|
| 46 |
+
|
| 47 |
+
def inference(
|
| 48 |
+
self,
|
| 49 |
+
noise: torch.Tensor,
|
| 50 |
+
text_prompts: List[str],
|
| 51 |
+
initial_latent: Optional[torch.Tensor] = None,
|
| 52 |
+
return_latents: bool = False,
|
| 53 |
+
profile: bool = False,
|
| 54 |
+
low_memory: bool = False,
|
| 55 |
+
) -> torch.Tensor:
|
| 56 |
+
"""
|
| 57 |
+
Perform inference on the given noise and text prompts.
|
| 58 |
+
Inputs:
|
| 59 |
+
noise (torch.Tensor): The input noise tensor of shape
|
| 60 |
+
(batch_size, num_output_frames, num_channels, height, width).
|
| 61 |
+
text_prompts (List[str]): The list of text prompts.
|
| 62 |
+
initial_latent (torch.Tensor): The initial latent tensor of shape
|
| 63 |
+
(batch_size, num_input_frames, num_channels, height, width).
|
| 64 |
+
If num_input_frames is 1, perform image to video.
|
| 65 |
+
If num_input_frames is greater than 1, perform video extension.
|
| 66 |
+
return_latents (bool): Whether to return the latents.
|
| 67 |
+
Outputs:
|
| 68 |
+
video (torch.Tensor): The generated video tensor of shape
|
| 69 |
+
(batch_size, num_output_frames, num_channels, height, width).
|
| 70 |
+
It is normalized to be in the range [0, 1].
|
| 71 |
+
"""
|
| 72 |
+
batch_size, num_frames, num_channels, height, width = noise.shape
|
| 73 |
+
if not self.independent_first_frame or (self.independent_first_frame and initial_latent is not None):
|
| 74 |
+
# If the first frame is independent and the first frame is provided, then the number of frames in the
|
| 75 |
+
# noise should still be a multiple of num_frame_per_block
|
| 76 |
+
assert num_frames % self.num_frame_per_block == 0
|
| 77 |
+
num_blocks = num_frames // self.num_frame_per_block
|
| 78 |
+
else:
|
| 79 |
+
# Using a [1, 4, 4, 4, 4, 4, ...] model to generate a video without image conditioning
|
| 80 |
+
assert (num_frames - 1) % self.num_frame_per_block == 0
|
| 81 |
+
num_blocks = (num_frames - 1) // self.num_frame_per_block
|
| 82 |
+
num_input_frames = initial_latent.shape[1] if initial_latent is not None else 0
|
| 83 |
+
num_output_frames = num_frames + num_input_frames # add the initial latent frames
|
| 84 |
+
conditional_dict = self.text_encoder(
|
| 85 |
+
text_prompts=text_prompts
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
if low_memory:
|
| 89 |
+
gpu_memory_preservation = get_cuda_free_memory_gb(gpu) + 5
|
| 90 |
+
move_model_to_device_with_memory_preservation(self.text_encoder, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
|
| 91 |
+
|
| 92 |
+
output = torch.zeros(
|
| 93 |
+
[batch_size, num_output_frames, num_channels, height, width],
|
| 94 |
+
device=noise.device,
|
| 95 |
+
dtype=noise.dtype
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
# Set up profiling if requested
|
| 99 |
+
if profile:
|
| 100 |
+
init_start = torch.cuda.Event(enable_timing=True)
|
| 101 |
+
init_end = torch.cuda.Event(enable_timing=True)
|
| 102 |
+
diffusion_start = torch.cuda.Event(enable_timing=True)
|
| 103 |
+
diffusion_end = torch.cuda.Event(enable_timing=True)
|
| 104 |
+
vae_start = torch.cuda.Event(enable_timing=True)
|
| 105 |
+
vae_end = torch.cuda.Event(enable_timing=True)
|
| 106 |
+
block_times = []
|
| 107 |
+
block_start = torch.cuda.Event(enable_timing=True)
|
| 108 |
+
block_end = torch.cuda.Event(enable_timing=True)
|
| 109 |
+
init_start.record()
|
| 110 |
+
|
| 111 |
+
# Step 1: Initialize KV cache to all zeros
|
| 112 |
+
if self.kv_cache1 is None:
|
| 113 |
+
self._initialize_kv_cache(
|
| 114 |
+
batch_size=batch_size,
|
| 115 |
+
dtype=noise.dtype,
|
| 116 |
+
device=noise.device
|
| 117 |
+
)
|
| 118 |
+
self._initialize_crossattn_cache(
|
| 119 |
+
batch_size=batch_size,
|
| 120 |
+
dtype=noise.dtype,
|
| 121 |
+
device=noise.device
|
| 122 |
+
)
|
| 123 |
+
else:
|
| 124 |
+
# reset cross attn cache
|
| 125 |
+
for block_index in range(self.num_transformer_blocks):
|
| 126 |
+
self.crossattn_cache[block_index]["is_init"] = False
|
| 127 |
+
# reset kv cache
|
| 128 |
+
for block_index in range(len(self.kv_cache1)):
|
| 129 |
+
self.kv_cache1[block_index]["global_end_index"] = torch.tensor(
|
| 130 |
+
[0], dtype=torch.long, device=noise.device)
|
| 131 |
+
self.kv_cache1[block_index]["local_end_index"] = torch.tensor(
|
| 132 |
+
[0], dtype=torch.long, device=noise.device)
|
| 133 |
+
|
| 134 |
+
# Step 2: Cache context feature
|
| 135 |
+
current_start_frame = 0
|
| 136 |
+
if initial_latent is not None:
|
| 137 |
+
timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
|
| 138 |
+
if self.independent_first_frame:
|
| 139 |
+
# Assume num_input_frames is 1 + self.num_frame_per_block * num_input_blocks
|
| 140 |
+
assert (num_input_frames - 1) % self.num_frame_per_block == 0
|
| 141 |
+
num_input_blocks = (num_input_frames - 1) // self.num_frame_per_block
|
| 142 |
+
output[:, :1] = initial_latent[:, :1]
|
| 143 |
+
self.generator(
|
| 144 |
+
noisy_image_or_video=initial_latent[:, :1],
|
| 145 |
+
conditional_dict=conditional_dict,
|
| 146 |
+
timestep=timestep * 0,
|
| 147 |
+
kv_cache=self.kv_cache1,
|
| 148 |
+
crossattn_cache=self.crossattn_cache,
|
| 149 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 150 |
+
)
|
| 151 |
+
current_start_frame += 1
|
| 152 |
+
else:
|
| 153 |
+
# Assume num_input_frames is self.num_frame_per_block * num_input_blocks
|
| 154 |
+
assert num_input_frames % self.num_frame_per_block == 0
|
| 155 |
+
num_input_blocks = num_input_frames // self.num_frame_per_block
|
| 156 |
+
|
| 157 |
+
for _ in range(num_input_blocks):
|
| 158 |
+
current_ref_latents = \
|
| 159 |
+
initial_latent[:, current_start_frame:current_start_frame + self.num_frame_per_block]
|
| 160 |
+
output[:, current_start_frame:current_start_frame + self.num_frame_per_block] = current_ref_latents
|
| 161 |
+
self.generator(
|
| 162 |
+
noisy_image_or_video=current_ref_latents,
|
| 163 |
+
conditional_dict=conditional_dict,
|
| 164 |
+
timestep=timestep * 0,
|
| 165 |
+
kv_cache=self.kv_cache1,
|
| 166 |
+
crossattn_cache=self.crossattn_cache,
|
| 167 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 168 |
+
)
|
| 169 |
+
current_start_frame += self.num_frame_per_block
|
| 170 |
+
|
| 171 |
+
if profile:
|
| 172 |
+
init_end.record()
|
| 173 |
+
torch.cuda.synchronize()
|
| 174 |
+
diffusion_start.record()
|
| 175 |
+
|
| 176 |
+
# Step 3: Temporal denoising loop
|
| 177 |
+
all_num_frames = [self.num_frame_per_block] * num_blocks
|
| 178 |
+
if self.independent_first_frame and initial_latent is None:
|
| 179 |
+
all_num_frames = [1] + all_num_frames
|
| 180 |
+
for current_num_frames in all_num_frames:
|
| 181 |
+
if profile:
|
| 182 |
+
block_start.record()
|
| 183 |
+
|
| 184 |
+
noisy_input = noise[
|
| 185 |
+
:, current_start_frame - num_input_frames:current_start_frame + current_num_frames - num_input_frames]
|
| 186 |
+
|
| 187 |
+
# Step 3.1: Spatial denoising loop
|
| 188 |
+
for index, current_timestep in enumerate(self.denoising_step_list):
|
| 189 |
+
print(f"current_timestep: {current_timestep}")
|
| 190 |
+
# set current timestep
|
| 191 |
+
timestep = torch.ones(
|
| 192 |
+
[batch_size, current_num_frames],
|
| 193 |
+
device=noise.device,
|
| 194 |
+
dtype=torch.int64) * current_timestep
|
| 195 |
+
|
| 196 |
+
if index < len(self.denoising_step_list) - 1:
|
| 197 |
+
_, denoised_pred = self.generator(
|
| 198 |
+
noisy_image_or_video=noisy_input,
|
| 199 |
+
conditional_dict=conditional_dict,
|
| 200 |
+
timestep=timestep,
|
| 201 |
+
kv_cache=self.kv_cache1,
|
| 202 |
+
crossattn_cache=self.crossattn_cache,
|
| 203 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 204 |
+
)
|
| 205 |
+
next_timestep = self.denoising_step_list[index + 1]
|
| 206 |
+
noisy_input = self.scheduler.add_noise(
|
| 207 |
+
denoised_pred.flatten(0, 1),
|
| 208 |
+
torch.randn_like(denoised_pred.flatten(0, 1)),
|
| 209 |
+
next_timestep * torch.ones(
|
| 210 |
+
[batch_size * current_num_frames], device=noise.device, dtype=torch.long)
|
| 211 |
+
).unflatten(0, denoised_pred.shape[:2])
|
| 212 |
+
else:
|
| 213 |
+
# for getting real output
|
| 214 |
+
_, denoised_pred = self.generator(
|
| 215 |
+
noisy_image_or_video=noisy_input,
|
| 216 |
+
conditional_dict=conditional_dict,
|
| 217 |
+
timestep=timestep,
|
| 218 |
+
kv_cache=self.kv_cache1,
|
| 219 |
+
crossattn_cache=self.crossattn_cache,
|
| 220 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
# Step 3.2: record the model's output
|
| 224 |
+
output[:, current_start_frame:current_start_frame + current_num_frames] = denoised_pred
|
| 225 |
+
|
| 226 |
+
# Step 3.3: rerun with timestep zero to update KV cache using clean context
|
| 227 |
+
context_timestep = torch.ones_like(timestep) * self.args.context_noise
|
| 228 |
+
self.generator(
|
| 229 |
+
noisy_image_or_video=denoised_pred,
|
| 230 |
+
conditional_dict=conditional_dict,
|
| 231 |
+
timestep=context_timestep,
|
| 232 |
+
kv_cache=self.kv_cache1,
|
| 233 |
+
crossattn_cache=self.crossattn_cache,
|
| 234 |
+
current_start=current_start_frame * self.frame_seq_length,
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
if profile:
|
| 238 |
+
block_end.record()
|
| 239 |
+
torch.cuda.synchronize()
|
| 240 |
+
block_time = block_start.elapsed_time(block_end)
|
| 241 |
+
block_times.append(block_time)
|
| 242 |
+
|
| 243 |
+
# Step 3.4: update the start and end frame indices
|
| 244 |
+
current_start_frame += current_num_frames
|
| 245 |
+
|
| 246 |
+
if profile:
|
| 247 |
+
# End diffusion timing and synchronize CUDA
|
| 248 |
+
diffusion_end.record()
|
| 249 |
+
torch.cuda.synchronize()
|
| 250 |
+
diffusion_time = diffusion_start.elapsed_time(diffusion_end)
|
| 251 |
+
init_time = init_start.elapsed_time(init_end)
|
| 252 |
+
vae_start.record()
|
| 253 |
+
|
| 254 |
+
# Step 4: Decode the output
|
| 255 |
+
video = self.vae.decode_to_pixel(output, use_cache=False)
|
| 256 |
+
video = (video * 0.5 + 0.5).clamp(0, 1)
|
| 257 |
+
|
| 258 |
+
if profile:
|
| 259 |
+
# End VAE timing and synchronize CUDA
|
| 260 |
+
vae_end.record()
|
| 261 |
+
torch.cuda.synchronize()
|
| 262 |
+
vae_time = vae_start.elapsed_time(vae_end)
|
| 263 |
+
total_time = init_time + diffusion_time + vae_time
|
| 264 |
+
|
| 265 |
+
print("Profiling results:")
|
| 266 |
+
print(f" - Initialization/caching time: {init_time:.2f} ms ({100 * init_time / total_time:.2f}%)")
|
| 267 |
+
print(f" - Diffusion generation time: {diffusion_time:.2f} ms ({100 * diffusion_time / total_time:.2f}%)")
|
| 268 |
+
for i, block_time in enumerate(block_times):
|
| 269 |
+
print(f" - Block {i} generation time: {block_time:.2f} ms ({100 * block_time / diffusion_time:.2f}% of diffusion)")
|
| 270 |
+
print(f" - VAE decoding time: {vae_time:.2f} ms ({100 * vae_time / total_time:.2f}%)")
|
| 271 |
+
print(f" - Total time: {total_time:.2f} ms")
|
| 272 |
+
|
| 273 |
+
if return_latents:
|
| 274 |
+
return video, output
|
| 275 |
+
else:
|
| 276 |
+
return video
|
| 277 |
+
|
| 278 |
+
def _initialize_kv_cache(self, batch_size, dtype, device):
|
| 279 |
+
"""
|
| 280 |
+
Initialize a Per-GPU KV cache for the Wan model.
|
| 281 |
+
"""
|
| 282 |
+
kv_cache1 = []
|
| 283 |
+
if self.local_attn_size != -1:
|
| 284 |
+
# Use the local attention size to compute the KV cache size
|
| 285 |
+
kv_cache_size = self.local_attn_size * self.frame_seq_length
|
| 286 |
+
else:
|
| 287 |
+
# Use the default KV cache size
|
| 288 |
+
kv_cache_size = 32760
|
| 289 |
+
|
| 290 |
+
for _ in range(self.num_transformer_blocks):
|
| 291 |
+
kv_cache1.append({
|
| 292 |
+
"k": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 293 |
+
"v": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 294 |
+
"global_end_index": torch.tensor([0], dtype=torch.long, device=device),
|
| 295 |
+
"local_end_index": torch.tensor([0], dtype=torch.long, device=device)
|
| 296 |
+
})
|
| 297 |
+
|
| 298 |
+
self.kv_cache1 = kv_cache1 # always store the clean cache
|
| 299 |
+
|
| 300 |
+
def _initialize_crossattn_cache(self, batch_size, dtype, device):
|
| 301 |
+
"""
|
| 302 |
+
Initialize a Per-GPU cross-attention cache for the Wan model.
|
| 303 |
+
"""
|
| 304 |
+
crossattn_cache = []
|
| 305 |
+
|
| 306 |
+
for _ in range(self.num_transformer_blocks):
|
| 307 |
+
crossattn_cache.append({
|
| 308 |
+
"k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 309 |
+
"v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 310 |
+
"is_init": False
|
| 311 |
+
})
|
| 312 |
+
self.crossattn_cache = crossattn_cache
|
pipeline/self_forcing_training.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from utils.wan_wrapper import WanDiffusionWrapper
|
| 2 |
+
from utils.scheduler import SchedulerInterface
|
| 3 |
+
from typing import List, Optional
|
| 4 |
+
import torch
|
| 5 |
+
import torch.distributed as dist
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class SelfForcingTrainingPipeline:
|
| 9 |
+
def __init__(self,
|
| 10 |
+
denoising_step_list: List[int],
|
| 11 |
+
scheduler: SchedulerInterface,
|
| 12 |
+
generator: WanDiffusionWrapper,
|
| 13 |
+
num_frame_per_block=3,
|
| 14 |
+
independent_first_frame: bool = False,
|
| 15 |
+
same_step_across_blocks: bool = False,
|
| 16 |
+
last_step_only: bool = False,
|
| 17 |
+
num_max_frames: int = 21,
|
| 18 |
+
context_noise: int = 0,
|
| 19 |
+
**kwargs):
|
| 20 |
+
super().__init__()
|
| 21 |
+
self.scheduler = scheduler
|
| 22 |
+
self.generator = generator
|
| 23 |
+
self.denoising_step_list = denoising_step_list
|
| 24 |
+
if self.denoising_step_list[-1] == 0:
|
| 25 |
+
self.denoising_step_list = self.denoising_step_list[:-1] # remove the zero timestep for inference
|
| 26 |
+
|
| 27 |
+
# Wan specific hyperparameters
|
| 28 |
+
self.num_transformer_blocks = 30
|
| 29 |
+
self.frame_seq_length = 1560
|
| 30 |
+
self.num_frame_per_block = num_frame_per_block
|
| 31 |
+
self.context_noise = context_noise
|
| 32 |
+
self.i2v = False
|
| 33 |
+
|
| 34 |
+
self.kv_cache1 = None
|
| 35 |
+
self.kv_cache2 = None
|
| 36 |
+
self.independent_first_frame = independent_first_frame
|
| 37 |
+
self.same_step_across_blocks = same_step_across_blocks
|
| 38 |
+
self.last_step_only = last_step_only
|
| 39 |
+
self.kv_cache_size = num_max_frames * self.frame_seq_length
|
| 40 |
+
|
| 41 |
+
def generate_and_sync_list(self, num_blocks, num_denoising_steps, device):
|
| 42 |
+
rank = dist.get_rank() if dist.is_initialized() else 0
|
| 43 |
+
|
| 44 |
+
if rank == 0:
|
| 45 |
+
# Generate random indices
|
| 46 |
+
indices = torch.randint(
|
| 47 |
+
low=0,
|
| 48 |
+
high=num_denoising_steps,
|
| 49 |
+
size=(num_blocks,),
|
| 50 |
+
device=device
|
| 51 |
+
)
|
| 52 |
+
if self.last_step_only:
|
| 53 |
+
indices = torch.ones_like(indices) * (num_denoising_steps - 1)
|
| 54 |
+
else:
|
| 55 |
+
indices = torch.empty(num_blocks, dtype=torch.long, device=device)
|
| 56 |
+
|
| 57 |
+
dist.broadcast(indices, src=0) # Broadcast the random indices to all ranks
|
| 58 |
+
return indices.tolist()
|
| 59 |
+
|
| 60 |
+
def inference_with_trajectory(
|
| 61 |
+
self,
|
| 62 |
+
noise: torch.Tensor,
|
| 63 |
+
initial_latent: Optional[torch.Tensor] = None,
|
| 64 |
+
return_sim_step: bool = False,
|
| 65 |
+
**conditional_dict
|
| 66 |
+
) -> torch.Tensor:
|
| 67 |
+
batch_size, num_frames, num_channels, height, width = noise.shape
|
| 68 |
+
if not self.independent_first_frame or (self.independent_first_frame and initial_latent is not None):
|
| 69 |
+
# If the first frame is independent and the first frame is provided, then the number of frames in the
|
| 70 |
+
# noise should still be a multiple of num_frame_per_block
|
| 71 |
+
assert num_frames % self.num_frame_per_block == 0
|
| 72 |
+
num_blocks = num_frames // self.num_frame_per_block
|
| 73 |
+
else:
|
| 74 |
+
# Using a [1, 4, 4, 4, 4, 4, ...] model to generate a video without image conditioning
|
| 75 |
+
assert (num_frames - 1) % self.num_frame_per_block == 0
|
| 76 |
+
num_blocks = (num_frames - 1) // self.num_frame_per_block
|
| 77 |
+
num_input_frames = initial_latent.shape[1] if initial_latent is not None else 0
|
| 78 |
+
num_output_frames = num_frames + num_input_frames # add the initial latent frames
|
| 79 |
+
output = torch.zeros(
|
| 80 |
+
[batch_size, num_output_frames, num_channels, height, width],
|
| 81 |
+
device=noise.device,
|
| 82 |
+
dtype=noise.dtype
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
# Step 1: Initialize KV cache to all zeros
|
| 86 |
+
self._initialize_kv_cache(
|
| 87 |
+
batch_size=batch_size, dtype=noise.dtype, device=noise.device
|
| 88 |
+
)
|
| 89 |
+
self._initialize_crossattn_cache(
|
| 90 |
+
batch_size=batch_size, dtype=noise.dtype, device=noise.device
|
| 91 |
+
)
|
| 92 |
+
# if self.kv_cache1 is None:
|
| 93 |
+
# self._initialize_kv_cache(
|
| 94 |
+
# batch_size=batch_size,
|
| 95 |
+
# dtype=noise.dtype,
|
| 96 |
+
# device=noise.device,
|
| 97 |
+
# )
|
| 98 |
+
# self._initialize_crossattn_cache(
|
| 99 |
+
# batch_size=batch_size,
|
| 100 |
+
# dtype=noise.dtype,
|
| 101 |
+
# device=noise.device
|
| 102 |
+
# )
|
| 103 |
+
# else:
|
| 104 |
+
# # reset cross attn cache
|
| 105 |
+
# for block_index in range(self.num_transformer_blocks):
|
| 106 |
+
# self.crossattn_cache[block_index]["is_init"] = False
|
| 107 |
+
# # reset kv cache
|
| 108 |
+
# for block_index in range(len(self.kv_cache1)):
|
| 109 |
+
# self.kv_cache1[block_index]["global_end_index"] = torch.tensor(
|
| 110 |
+
# [0], dtype=torch.long, device=noise.device)
|
| 111 |
+
# self.kv_cache1[block_index]["local_end_index"] = torch.tensor(
|
| 112 |
+
# [0], dtype=torch.long, device=noise.device)
|
| 113 |
+
|
| 114 |
+
# Step 2: Cache context feature
|
| 115 |
+
current_start_frame = 0
|
| 116 |
+
if initial_latent is not None:
|
| 117 |
+
timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
|
| 118 |
+
# Assume num_input_frames is 1 + self.num_frame_per_block * num_input_blocks
|
| 119 |
+
output[:, :1] = initial_latent
|
| 120 |
+
with torch.no_grad():
|
| 121 |
+
self.generator(
|
| 122 |
+
noisy_image_or_video=initial_latent,
|
| 123 |
+
conditional_dict=conditional_dict,
|
| 124 |
+
timestep=timestep * 0,
|
| 125 |
+
kv_cache=self.kv_cache1,
|
| 126 |
+
crossattn_cache=self.crossattn_cache,
|
| 127 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 128 |
+
)
|
| 129 |
+
current_start_frame += 1
|
| 130 |
+
|
| 131 |
+
# Step 3: Temporal denoising loop
|
| 132 |
+
all_num_frames = [self.num_frame_per_block] * num_blocks
|
| 133 |
+
if self.independent_first_frame and initial_latent is None:
|
| 134 |
+
all_num_frames = [1] + all_num_frames
|
| 135 |
+
num_denoising_steps = len(self.denoising_step_list)
|
| 136 |
+
exit_flags = self.generate_and_sync_list(len(all_num_frames), num_denoising_steps, device=noise.device)
|
| 137 |
+
start_gradient_frame_index = num_output_frames - 21
|
| 138 |
+
|
| 139 |
+
# for block_index in range(num_blocks):
|
| 140 |
+
for block_index, current_num_frames in enumerate(all_num_frames):
|
| 141 |
+
noisy_input = noise[
|
| 142 |
+
:, current_start_frame - num_input_frames:current_start_frame + current_num_frames - num_input_frames]
|
| 143 |
+
|
| 144 |
+
# Step 3.1: Spatial denoising loop
|
| 145 |
+
for index, current_timestep in enumerate(self.denoising_step_list):
|
| 146 |
+
if self.same_step_across_blocks:
|
| 147 |
+
exit_flag = (index == exit_flags[0])
|
| 148 |
+
else:
|
| 149 |
+
exit_flag = (index == exit_flags[block_index]) # Only backprop at the randomly selected timestep (consistent across all ranks)
|
| 150 |
+
timestep = torch.ones(
|
| 151 |
+
[batch_size, current_num_frames],
|
| 152 |
+
device=noise.device,
|
| 153 |
+
dtype=torch.int64) * current_timestep
|
| 154 |
+
|
| 155 |
+
if not exit_flag:
|
| 156 |
+
with torch.no_grad():
|
| 157 |
+
_, denoised_pred = self.generator(
|
| 158 |
+
noisy_image_or_video=noisy_input,
|
| 159 |
+
conditional_dict=conditional_dict,
|
| 160 |
+
timestep=timestep,
|
| 161 |
+
kv_cache=self.kv_cache1,
|
| 162 |
+
crossattn_cache=self.crossattn_cache,
|
| 163 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 164 |
+
)
|
| 165 |
+
next_timestep = self.denoising_step_list[index + 1]
|
| 166 |
+
noisy_input = self.scheduler.add_noise(
|
| 167 |
+
denoised_pred.flatten(0, 1),
|
| 168 |
+
torch.randn_like(denoised_pred.flatten(0, 1)),
|
| 169 |
+
next_timestep * torch.ones(
|
| 170 |
+
[batch_size * current_num_frames], device=noise.device, dtype=torch.long)
|
| 171 |
+
).unflatten(0, denoised_pred.shape[:2])
|
| 172 |
+
else:
|
| 173 |
+
# for getting real output
|
| 174 |
+
# with torch.set_grad_enabled(current_start_frame >= start_gradient_frame_index):
|
| 175 |
+
if current_start_frame < start_gradient_frame_index:
|
| 176 |
+
with torch.no_grad():
|
| 177 |
+
_, denoised_pred = self.generator(
|
| 178 |
+
noisy_image_or_video=noisy_input,
|
| 179 |
+
conditional_dict=conditional_dict,
|
| 180 |
+
timestep=timestep,
|
| 181 |
+
kv_cache=self.kv_cache1,
|
| 182 |
+
crossattn_cache=self.crossattn_cache,
|
| 183 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 184 |
+
)
|
| 185 |
+
else:
|
| 186 |
+
_, denoised_pred = self.generator(
|
| 187 |
+
noisy_image_or_video=noisy_input,
|
| 188 |
+
conditional_dict=conditional_dict,
|
| 189 |
+
timestep=timestep,
|
| 190 |
+
kv_cache=self.kv_cache1,
|
| 191 |
+
crossattn_cache=self.crossattn_cache,
|
| 192 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 193 |
+
)
|
| 194 |
+
break
|
| 195 |
+
|
| 196 |
+
# Step 3.2: record the model's output
|
| 197 |
+
output[:, current_start_frame:current_start_frame + current_num_frames] = denoised_pred
|
| 198 |
+
|
| 199 |
+
# Step 3.3: rerun with timestep zero to update the cache
|
| 200 |
+
context_timestep = torch.ones_like(timestep) * self.context_noise
|
| 201 |
+
# add context noise
|
| 202 |
+
denoised_pred = self.scheduler.add_noise(
|
| 203 |
+
denoised_pred.flatten(0, 1),
|
| 204 |
+
torch.randn_like(denoised_pred.flatten(0, 1)),
|
| 205 |
+
context_timestep * torch.ones(
|
| 206 |
+
[batch_size * current_num_frames], device=noise.device, dtype=torch.long)
|
| 207 |
+
).unflatten(0, denoised_pred.shape[:2])
|
| 208 |
+
with torch.no_grad():
|
| 209 |
+
self.generator(
|
| 210 |
+
noisy_image_or_video=denoised_pred,
|
| 211 |
+
conditional_dict=conditional_dict,
|
| 212 |
+
timestep=context_timestep,
|
| 213 |
+
kv_cache=self.kv_cache1,
|
| 214 |
+
crossattn_cache=self.crossattn_cache,
|
| 215 |
+
current_start=current_start_frame * self.frame_seq_length
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
# Step 3.4: update the start and end frame indices
|
| 219 |
+
current_start_frame += current_num_frames
|
| 220 |
+
|
| 221 |
+
# Step 3.5: Return the denoised timestep
|
| 222 |
+
if not self.same_step_across_blocks:
|
| 223 |
+
denoised_timestep_from, denoised_timestep_to = None, None
|
| 224 |
+
elif exit_flags[0] == len(self.denoising_step_list) - 1:
|
| 225 |
+
denoised_timestep_to = 0
|
| 226 |
+
denoised_timestep_from = 1000 - torch.argmin(
|
| 227 |
+
(self.scheduler.timesteps.cuda() - self.denoising_step_list[exit_flags[0]].cuda()).abs(), dim=0).item()
|
| 228 |
+
else:
|
| 229 |
+
denoised_timestep_to = 1000 - torch.argmin(
|
| 230 |
+
(self.scheduler.timesteps.cuda() - self.denoising_step_list[exit_flags[0] + 1].cuda()).abs(), dim=0).item()
|
| 231 |
+
denoised_timestep_from = 1000 - torch.argmin(
|
| 232 |
+
(self.scheduler.timesteps.cuda() - self.denoising_step_list[exit_flags[0]].cuda()).abs(), dim=0).item()
|
| 233 |
+
|
| 234 |
+
if return_sim_step:
|
| 235 |
+
return output, denoised_timestep_from, denoised_timestep_to, exit_flags[0] + 1
|
| 236 |
+
|
| 237 |
+
return output, denoised_timestep_from, denoised_timestep_to
|
| 238 |
+
|
| 239 |
+
def _initialize_kv_cache(self, batch_size, dtype, device):
|
| 240 |
+
"""
|
| 241 |
+
Initialize a Per-GPU KV cache for the Wan model.
|
| 242 |
+
"""
|
| 243 |
+
kv_cache1 = []
|
| 244 |
+
|
| 245 |
+
for _ in range(self.num_transformer_blocks):
|
| 246 |
+
kv_cache1.append({
|
| 247 |
+
"k": torch.zeros([batch_size, self.kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 248 |
+
"v": torch.zeros([batch_size, self.kv_cache_size, 12, 128], dtype=dtype, device=device),
|
| 249 |
+
"global_end_index": torch.tensor([0], dtype=torch.long, device=device),
|
| 250 |
+
"local_end_index": torch.tensor([0], dtype=torch.long, device=device)
|
| 251 |
+
})
|
| 252 |
+
|
| 253 |
+
self.kv_cache1 = kv_cache1 # always store the clean cache
|
| 254 |
+
|
| 255 |
+
def _initialize_crossattn_cache(self, batch_size, dtype, device):
|
| 256 |
+
"""
|
| 257 |
+
Initialize a Per-GPU cross-attention cache for the Wan model.
|
| 258 |
+
"""
|
| 259 |
+
crossattn_cache = []
|
| 260 |
+
|
| 261 |
+
for _ in range(self.num_transformer_blocks):
|
| 262 |
+
crossattn_cache.append({
|
| 263 |
+
"k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 264 |
+
"v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
|
| 265 |
+
"is_init": False
|
| 266 |
+
})
|
| 267 |
+
self.crossattn_cache = crossattn_cache
|