Cccccz commited on
Commit
e3272d9
·
verified ·
1 Parent(s): 046649d

Upload pipeline source

Browse files
pipeline/__init__.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from .bidirectional_diffusion_inference import BidirectionalDiffusionInferencePipeline
2
+ from .bidirectional_inference import BidirectionalInferencePipeline
3
+ from .causal_diffusion_inference import CausalDiffusionInferencePipeline
4
+ from .causal_inference import CausalInferencePipeline
5
+ from .self_forcing_training import SelfForcingTrainingPipeline
6
+
7
+ __all__ = [
8
+ "BidirectionalDiffusionInferencePipeline",
9
+ "BidirectionalInferencePipeline",
10
+ "CausalDiffusionInferencePipeline",
11
+ "CausalInferencePipeline",
12
+ "SelfForcingTrainingPipeline"
13
+ ]
pipeline/bidirectional_diffusion_inference.py ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from tqdm import tqdm
2
+ from typing import List
3
+ import torch
4
+
5
+ from wan.utils.fm_solvers import FlowDPMSolverMultistepScheduler, get_sampling_sigmas, retrieve_timesteps
6
+ from wan.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler
7
+ from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
8
+
9
+
10
+ class BidirectionalDiffusionInferencePipeline(torch.nn.Module):
11
+ def __init__(
12
+ self,
13
+ args,
14
+ device,
15
+ generator=None,
16
+ text_encoder=None,
17
+ vae=None
18
+ ):
19
+ super().__init__()
20
+ # Step 1: Initialize all models
21
+ self.generator = WanDiffusionWrapper(
22
+ **getattr(args, "model_kwargs", {}), is_causal=False) if generator is None else generator
23
+ self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
24
+ self.vae = WanVAEWrapper() if vae is None else vae
25
+
26
+ # Step 2: Initialize scheduler
27
+ self.num_train_timesteps = args.num_train_timestep
28
+ self.sampling_steps = 50
29
+ self.sample_solver = 'unipc'
30
+ self.shift = 8.0
31
+
32
+ self.args = args
33
+
34
+ def inference(
35
+ self,
36
+ noise: torch.Tensor,
37
+ text_prompts: List[str],
38
+ return_latents=False
39
+ ) -> torch.Tensor:
40
+ """
41
+ Perform inference on the given noise and text prompts.
42
+ Inputs:
43
+ noise (torch.Tensor): The input noise tensor of shape
44
+ (batch_size, num_frames, num_channels, height, width).
45
+ text_prompts (List[str]): The list of text prompts.
46
+ Outputs:
47
+ video (torch.Tensor): The generated video tensor of shape
48
+ (batch_size, num_frames, num_channels, height, width). It is normalized to be in the range [0, 1].
49
+ """
50
+
51
+ conditional_dict = self.text_encoder(
52
+ text_prompts=text_prompts
53
+ )
54
+ unconditional_dict = self.text_encoder(
55
+ text_prompts=[self.args.negative_prompt] * len(text_prompts)
56
+ )
57
+
58
+ latents = noise
59
+
60
+ sample_scheduler = self._initialize_sample_scheduler(noise)
61
+ for _, t in enumerate(tqdm(sample_scheduler.timesteps)):
62
+ latent_model_input = latents
63
+ timestep = t * torch.ones([latents.shape[0], 21], device=noise.device, dtype=torch.float32)
64
+
65
+ flow_pred_cond, _ = self.generator(latent_model_input, conditional_dict, timestep)
66
+ flow_pred_uncond, _ = self.generator(latent_model_input, unconditional_dict, timestep)
67
+
68
+ flow_pred = flow_pred_uncond + self.args.guidance_scale * (
69
+ flow_pred_cond - flow_pred_uncond)
70
+
71
+ temp_x0 = sample_scheduler.step(
72
+ flow_pred.unsqueeze(0),
73
+ t,
74
+ latents.unsqueeze(0),
75
+ return_dict=False)[0]
76
+ latents = temp_x0.squeeze(0)
77
+
78
+ x0 = latents
79
+ video = self.vae.decode_to_pixel(x0)
80
+ video = (video * 0.5 + 0.5).clamp(0, 1)
81
+
82
+ del sample_scheduler
83
+
84
+ if return_latents:
85
+ return video, latents
86
+ else:
87
+ return video
88
+
89
+ def _initialize_sample_scheduler(self, noise):
90
+ if self.sample_solver == 'unipc':
91
+ sample_scheduler = FlowUniPCMultistepScheduler(
92
+ num_train_timesteps=self.num_train_timesteps,
93
+ shift=1,
94
+ use_dynamic_shifting=False)
95
+ sample_scheduler.set_timesteps(
96
+ self.sampling_steps, device=noise.device, shift=self.shift)
97
+ self.timesteps = sample_scheduler.timesteps
98
+ elif self.sample_solver == 'dpm++':
99
+ sample_scheduler = FlowDPMSolverMultistepScheduler(
100
+ num_train_timesteps=self.num_train_timesteps,
101
+ shift=1,
102
+ use_dynamic_shifting=False)
103
+ sampling_sigmas = get_sampling_sigmas(self.sampling_steps, self.shift)
104
+ self.timesteps, _ = retrieve_timesteps(
105
+ sample_scheduler,
106
+ device=noise.device,
107
+ sigmas=sampling_sigmas)
108
+ else:
109
+ raise NotImplementedError("Unsupported solver.")
110
+ return sample_scheduler
pipeline/bidirectional_inference.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List
2
+ import torch
3
+
4
+ from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
5
+
6
+
7
+ class BidirectionalInferencePipeline(torch.nn.Module):
8
+ def __init__(
9
+ self,
10
+ args,
11
+ device,
12
+ generator=None,
13
+ text_encoder=None,
14
+ vae=None
15
+ ):
16
+ super().__init__()
17
+ # Step 1: Initialize all models
18
+ self.generator = WanDiffusionWrapper(
19
+ **getattr(args, "model_kwargs", {}), is_causal=False) if generator is None else generator
20
+ self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
21
+ self.vae = WanVAEWrapper() if vae is None else vae
22
+
23
+ # Step 2: Initialize all bidirectional wan hyperparmeters
24
+ self.scheduler = self.generator.get_scheduler()
25
+ self.denoising_step_list = torch.tensor(
26
+ args.denoising_step_list, dtype=torch.long, device=device)
27
+ if self.denoising_step_list[-1] == 0:
28
+ self.denoising_step_list = self.denoising_step_list[:-1] # remove the zero timestep for inference
29
+ if args.warp_denoising_step:
30
+ timesteps = torch.cat((self.scheduler.timesteps.cpu(), torch.tensor([0], dtype=torch.float32)))
31
+ self.denoising_step_list = timesteps[1000 - self.denoising_step_list]
32
+
33
+ def inference(self, noise: torch.Tensor, text_prompts: List[str]) -> torch.Tensor:
34
+ """
35
+ Perform inference on the given noise and text prompts.
36
+ Inputs:
37
+ noise (torch.Tensor): The input noise tensor of shape
38
+ (batch_size, num_frames, num_channels, height, width).
39
+ text_prompts (List[str]): The list of text prompts.
40
+ Outputs:
41
+ video (torch.Tensor): The generated video tensor of shape
42
+ (batch_size, num_frames, num_channels, height, width). It is normalized to be in the range [0, 1].
43
+ """
44
+ conditional_dict = self.text_encoder(
45
+ text_prompts=text_prompts
46
+ )
47
+
48
+ # initial point
49
+ noisy_image_or_video = noise
50
+
51
+ # use the last n-1 timesteps to simulate the generator's input
52
+ for index, current_timestep in enumerate(self.denoising_step_list[:-1]):
53
+ _, pred_image_or_video = self.generator(
54
+ noisy_image_or_video=noisy_image_or_video,
55
+ conditional_dict=conditional_dict,
56
+ timestep=torch.ones(
57
+ noise.shape[:2], dtype=torch.long, device=noise.device) * current_timestep
58
+ ) # [B, F, C, H, W]
59
+
60
+ next_timestep = self.denoising_step_list[index + 1] * torch.ones(
61
+ noise.shape[:2], dtype=torch.long, device=noise.device)
62
+
63
+ noisy_image_or_video = self.scheduler.add_noise(
64
+ pred_image_or_video.flatten(0, 1),
65
+ torch.randn_like(pred_image_or_video.flatten(0, 1)),
66
+ next_timestep.flatten(0, 1)
67
+ ).unflatten(0, noise.shape[:2])
68
+
69
+ video = self.vae.decode_to_pixel(pred_image_or_video)
70
+ video = (video * 0.5 + 0.5).clamp(0, 1)
71
+ return video
pipeline/causal_diffusion_inference.py ADDED
@@ -0,0 +1,342 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from tqdm import tqdm
2
+ from typing import List, Optional
3
+ import torch
4
+
5
+ from wan.utils.fm_solvers import FlowDPMSolverMultistepScheduler, get_sampling_sigmas, retrieve_timesteps
6
+ from wan.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler
7
+ from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
8
+
9
+
10
+ class CausalDiffusionInferencePipeline(torch.nn.Module):
11
+ def __init__(
12
+ self,
13
+ args,
14
+ device,
15
+ generator=None,
16
+ text_encoder=None,
17
+ vae=None
18
+ ):
19
+ super().__init__()
20
+ # Step 1: Initialize all models
21
+ self.generator = WanDiffusionWrapper(
22
+ **getattr(args, "model_kwargs", {}), is_causal=True) if generator is None else generator
23
+ self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
24
+ self.vae = WanVAEWrapper() if vae is None else vae
25
+
26
+ # Step 2: Initialize scheduler
27
+ self.num_train_timesteps = args.num_train_timestep
28
+ self.sampling_steps = 50
29
+ self.sample_solver = 'unipc'
30
+ self.shift = args.timestep_shift
31
+
32
+ self.num_transformer_blocks = 30
33
+ self.frame_seq_length = 1560
34
+
35
+ self.kv_cache_pos = None
36
+ self.kv_cache_neg = None
37
+ self.crossattn_cache_pos = None
38
+ self.crossattn_cache_neg = None
39
+ self.args = args
40
+ self.num_frame_per_block = getattr(args, "num_frame_per_block", 1)
41
+ self.independent_first_frame = args.independent_first_frame
42
+ self.local_attn_size = self.generator.model.local_attn_size
43
+
44
+ print(f"KV inference with {self.num_frame_per_block} frames per block")
45
+
46
+ if self.num_frame_per_block > 1:
47
+ self.generator.model.num_frame_per_block = self.num_frame_per_block
48
+
49
+ def inference(
50
+ self,
51
+ noise: torch.Tensor,
52
+ text_prompts: List[str],
53
+ initial_latent: Optional[torch.Tensor] = None,
54
+ return_latents: bool = False,
55
+ start_frame_index: Optional[int] = 0
56
+ ) -> torch.Tensor:
57
+ """
58
+ Perform inference on the given noise and text prompts.
59
+ Inputs:
60
+ noise (torch.Tensor): The input noise tensor of shape
61
+ (batch_size, num_output_frames, num_channels, height, width).
62
+ text_prompts (List[str]): The list of text prompts.
63
+ initial_latent (torch.Tensor): The initial latent tensor of shape
64
+ (batch_size, num_input_frames, num_channels, height, width).
65
+ If num_input_frames is 1, perform image to video.
66
+ If num_input_frames is greater than 1, perform video extension.
67
+ return_latents (bool): Whether to return the latents.
68
+ start_frame_index (int): In long video generation, where does the current window start?
69
+ Outputs:
70
+ video (torch.Tensor): The generated video tensor of shape
71
+ (batch_size, num_frames, num_channels, height, width). It is normalized to be in the range [0, 1].
72
+ """
73
+ batch_size, num_frames, num_channels, height, width = noise.shape
74
+ if not self.independent_first_frame or (self.independent_first_frame and initial_latent is not None):
75
+ # If the first frame is independent and the first frame is provided, then the number of frames in the
76
+ # noise should still be a multiple of num_frame_per_block
77
+ assert num_frames % self.num_frame_per_block == 0
78
+ num_blocks = num_frames // self.num_frame_per_block
79
+ elif self.independent_first_frame and initial_latent is None:
80
+ # Using a [1, 4, 4, 4, 4, 4] model to generate a video without image conditioning
81
+ assert (num_frames - 1) % self.num_frame_per_block == 0
82
+ num_blocks = (num_frames - 1) // self.num_frame_per_block
83
+ num_input_frames = initial_latent.shape[1] if initial_latent is not None else 0
84
+ num_output_frames = num_frames + num_input_frames # add the initial latent frames
85
+ conditional_dict = self.text_encoder(
86
+ text_prompts=text_prompts
87
+ )
88
+ unconditional_dict = self.text_encoder(
89
+ text_prompts=[self.args.negative_prompt] * len(text_prompts)
90
+ )
91
+
92
+ output = torch.zeros(
93
+ [batch_size, num_output_frames, num_channels, height, width],
94
+ device=noise.device,
95
+ dtype=noise.dtype
96
+ )
97
+
98
+ # Step 1: Initialize KV cache to all zeros
99
+ if self.kv_cache_pos is None:
100
+ self._initialize_kv_cache(
101
+ batch_size=batch_size,
102
+ dtype=noise.dtype,
103
+ device=noise.device
104
+ )
105
+ self._initialize_crossattn_cache(
106
+ batch_size=batch_size,
107
+ dtype=noise.dtype,
108
+ device=noise.device
109
+ )
110
+ else:
111
+ # reset cross attn cache
112
+ for block_index in range(self.num_transformer_blocks):
113
+ self.crossattn_cache_pos[block_index]["is_init"] = False
114
+ self.crossattn_cache_neg[block_index]["is_init"] = False
115
+ # reset kv cache
116
+ for block_index in range(len(self.kv_cache_pos)):
117
+ self.kv_cache_pos[block_index]["global_end_index"] = torch.tensor(
118
+ [0], dtype=torch.long, device=noise.device)
119
+ self.kv_cache_pos[block_index]["local_end_index"] = torch.tensor(
120
+ [0], dtype=torch.long, device=noise.device)
121
+ self.kv_cache_neg[block_index]["global_end_index"] = torch.tensor(
122
+ [0], dtype=torch.long, device=noise.device)
123
+ self.kv_cache_neg[block_index]["local_end_index"] = torch.tensor(
124
+ [0], dtype=torch.long, device=noise.device)
125
+
126
+ # Step 2: Cache context feature
127
+ current_start_frame = start_frame_index
128
+ cache_start_frame = 0
129
+ if initial_latent is not None:
130
+ timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
131
+ if self.independent_first_frame:
132
+ # Assume num_input_frames is 1 + self.num_frame_per_block * num_input_blocks
133
+ assert (num_input_frames - 1) % self.num_frame_per_block == 0
134
+ num_input_blocks = (num_input_frames - 1) // self.num_frame_per_block
135
+ output[:, :1] = initial_latent[:, :1]
136
+ self.generator(
137
+ noisy_image_or_video=initial_latent[:, :1],
138
+ conditional_dict=conditional_dict,
139
+ timestep=timestep * 0,
140
+ kv_cache=self.kv_cache_pos,
141
+ crossattn_cache=self.crossattn_cache_pos,
142
+ current_start=current_start_frame * self.frame_seq_length,
143
+ cache_start=cache_start_frame * self.frame_seq_length
144
+ )
145
+ self.generator(
146
+ noisy_image_or_video=initial_latent[:, :1],
147
+ conditional_dict=unconditional_dict,
148
+ timestep=timestep * 0,
149
+ kv_cache=self.kv_cache_neg,
150
+ crossattn_cache=self.crossattn_cache_neg,
151
+ current_start=current_start_frame * self.frame_seq_length,
152
+ cache_start=cache_start_frame * self.frame_seq_length
153
+ )
154
+ current_start_frame += 1
155
+ cache_start_frame += 1
156
+ else:
157
+ # Assume num_input_frames is self.num_frame_per_block * num_input_blocks
158
+ assert num_input_frames % self.num_frame_per_block == 0
159
+ num_input_blocks = num_input_frames // self.num_frame_per_block
160
+
161
+ for block_index in range(num_input_blocks):
162
+ current_ref_latents = \
163
+ initial_latent[:, cache_start_frame:cache_start_frame + self.num_frame_per_block]
164
+ output[:, cache_start_frame:cache_start_frame + self.num_frame_per_block] = current_ref_latents
165
+ self.generator(
166
+ noisy_image_or_video=current_ref_latents,
167
+ conditional_dict=conditional_dict,
168
+ timestep=timestep * 0,
169
+ kv_cache=self.kv_cache_pos,
170
+ crossattn_cache=self.crossattn_cache_pos,
171
+ current_start=current_start_frame * self.frame_seq_length,
172
+ cache_start=cache_start_frame * self.frame_seq_length
173
+ )
174
+ self.generator(
175
+ noisy_image_or_video=current_ref_latents,
176
+ conditional_dict=unconditional_dict,
177
+ timestep=timestep * 0,
178
+ kv_cache=self.kv_cache_neg,
179
+ crossattn_cache=self.crossattn_cache_neg,
180
+ current_start=current_start_frame * self.frame_seq_length,
181
+ cache_start=cache_start_frame * self.frame_seq_length
182
+ )
183
+ current_start_frame += self.num_frame_per_block
184
+ cache_start_frame += self.num_frame_per_block
185
+
186
+ # Step 3: Temporal denoising loop
187
+ all_num_frames = [self.num_frame_per_block] * num_blocks
188
+ if self.independent_first_frame and initial_latent is None:
189
+ all_num_frames = [1] + all_num_frames
190
+ for current_num_frames in all_num_frames:
191
+ noisy_input = noise[
192
+ :, cache_start_frame - num_input_frames:cache_start_frame + current_num_frames - num_input_frames]
193
+ latents = noisy_input
194
+
195
+ # Step 3.1: Spatial denoising loop
196
+ sample_scheduler = self._initialize_sample_scheduler(noise)
197
+ for _, t in enumerate(tqdm(sample_scheduler.timesteps)):
198
+ latent_model_input = latents
199
+ timestep = t * torch.ones(
200
+ [batch_size, current_num_frames], device=noise.device, dtype=torch.float32
201
+ )
202
+
203
+ flow_pred_cond, _ = self.generator(
204
+ noisy_image_or_video=latent_model_input,
205
+ conditional_dict=conditional_dict,
206
+ timestep=timestep,
207
+ kv_cache=self.kv_cache_pos,
208
+ crossattn_cache=self.crossattn_cache_pos,
209
+ current_start=current_start_frame * self.frame_seq_length,
210
+ cache_start=cache_start_frame * self.frame_seq_length
211
+ )
212
+ flow_pred_uncond, _ = self.generator(
213
+ noisy_image_or_video=latent_model_input,
214
+ conditional_dict=unconditional_dict,
215
+ timestep=timestep,
216
+ kv_cache=self.kv_cache_neg,
217
+ crossattn_cache=self.crossattn_cache_neg,
218
+ current_start=current_start_frame * self.frame_seq_length,
219
+ cache_start=cache_start_frame * self.frame_seq_length
220
+ )
221
+
222
+ flow_pred = flow_pred_uncond + self.args.guidance_scale * (
223
+ flow_pred_cond - flow_pred_uncond)
224
+
225
+ temp_x0 = sample_scheduler.step(
226
+ flow_pred,
227
+ t,
228
+ latents,
229
+ return_dict=False)[0]
230
+ latents = temp_x0
231
+ print(f"kv_cache['local_end_index']: {self.kv_cache_pos[0]['local_end_index']}")
232
+ print(f"kv_cache['global_end_index']: {self.kv_cache_pos[0]['global_end_index']}")
233
+
234
+ # Step 3.2: record the model's output
235
+ output[:, cache_start_frame:cache_start_frame + current_num_frames] = latents
236
+
237
+ # Step 3.3: rerun with timestep zero to update KV cache using clean context
238
+ self.generator(
239
+ noisy_image_or_video=latents,
240
+ conditional_dict=conditional_dict,
241
+ timestep=timestep * 0,
242
+ kv_cache=self.kv_cache_pos,
243
+ crossattn_cache=self.crossattn_cache_pos,
244
+ current_start=current_start_frame * self.frame_seq_length,
245
+ cache_start=cache_start_frame * self.frame_seq_length
246
+ )
247
+ self.generator(
248
+ noisy_image_or_video=latents,
249
+ conditional_dict=unconditional_dict,
250
+ timestep=timestep * 0,
251
+ kv_cache=self.kv_cache_neg,
252
+ crossattn_cache=self.crossattn_cache_neg,
253
+ current_start=current_start_frame * self.frame_seq_length,
254
+ cache_start=cache_start_frame * self.frame_seq_length
255
+ )
256
+
257
+ # Step 3.4: update the start and end frame indices
258
+ current_start_frame += current_num_frames
259
+ cache_start_frame += current_num_frames
260
+
261
+ # Step 4: Decode the output
262
+ video = self.vae.decode_to_pixel(output)
263
+ video = (video * 0.5 + 0.5).clamp(0, 1)
264
+
265
+ if return_latents:
266
+ return video, output
267
+ else:
268
+ return video
269
+
270
+ def _initialize_kv_cache(self, batch_size, dtype, device):
271
+ """
272
+ Initialize a Per-GPU KV cache for the Wan model.
273
+ """
274
+ kv_cache_pos = []
275
+ kv_cache_neg = []
276
+ if self.local_attn_size != -1:
277
+ # Use the local attention size to compute the KV cache size
278
+ kv_cache_size = self.local_attn_size * self.frame_seq_length
279
+ else:
280
+ # Use the default KV cache size
281
+ kv_cache_size = 32760
282
+
283
+ for _ in range(self.num_transformer_blocks):
284
+ kv_cache_pos.append({
285
+ "k": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
286
+ "v": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
287
+ "global_end_index": torch.tensor([0], dtype=torch.long, device=device),
288
+ "local_end_index": torch.tensor([0], dtype=torch.long, device=device)
289
+ })
290
+ kv_cache_neg.append({
291
+ "k": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
292
+ "v": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
293
+ "global_end_index": torch.tensor([0], dtype=torch.long, device=device),
294
+ "local_end_index": torch.tensor([0], dtype=torch.long, device=device)
295
+ })
296
+
297
+ self.kv_cache_pos = kv_cache_pos # always store the clean cache
298
+ self.kv_cache_neg = kv_cache_neg # always store the clean cache
299
+
300
+ def _initialize_crossattn_cache(self, batch_size, dtype, device):
301
+ """
302
+ Initialize a Per-GPU cross-attention cache for the Wan model.
303
+ """
304
+ crossattn_cache_pos = []
305
+ crossattn_cache_neg = []
306
+ for _ in range(self.num_transformer_blocks):
307
+ crossattn_cache_pos.append({
308
+ "k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
309
+ "v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
310
+ "is_init": False
311
+ })
312
+ crossattn_cache_neg.append({
313
+ "k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
314
+ "v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
315
+ "is_init": False
316
+ })
317
+
318
+ self.crossattn_cache_pos = crossattn_cache_pos # always store the clean cache
319
+ self.crossattn_cache_neg = crossattn_cache_neg # always store the clean cache
320
+
321
+ def _initialize_sample_scheduler(self, noise):
322
+ if self.sample_solver == 'unipc':
323
+ sample_scheduler = FlowUniPCMultistepScheduler(
324
+ num_train_timesteps=self.num_train_timesteps,
325
+ shift=1,
326
+ use_dynamic_shifting=False)
327
+ sample_scheduler.set_timesteps(
328
+ self.sampling_steps, device=noise.device, shift=self.shift)
329
+ self.timesteps = sample_scheduler.timesteps
330
+ elif self.sample_solver == 'dpm++':
331
+ sample_scheduler = FlowDPMSolverMultistepScheduler(
332
+ num_train_timesteps=self.num_train_timesteps,
333
+ shift=1,
334
+ use_dynamic_shifting=False)
335
+ sampling_sigmas = get_sampling_sigmas(self.sampling_steps, self.shift)
336
+ self.timesteps, _ = retrieve_timesteps(
337
+ sample_scheduler,
338
+ device=noise.device,
339
+ sigmas=sampling_sigmas)
340
+ else:
341
+ raise NotImplementedError("Unsupported solver.")
342
+ return sample_scheduler
pipeline/causal_inference.py ADDED
@@ -0,0 +1,312 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List, Optional
2
+ import torch
3
+
4
+ from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder, WanVAEWrapper
5
+
6
+ from demo_utils.memory import gpu, get_cuda_free_memory_gb, DynamicSwapInstaller, move_model_to_device_with_memory_preservation
7
+
8
+
9
+ class CausalInferencePipeline(torch.nn.Module):
10
+ def __init__(
11
+ self,
12
+ args,
13
+ device,
14
+ generator=None,
15
+ text_encoder=None,
16
+ vae=None
17
+ ):
18
+ super().__init__()
19
+ # Step 1: Initialize all models
20
+ self.generator = WanDiffusionWrapper(
21
+ **getattr(args, "model_kwargs", {}), is_causal=True) if generator is None else generator
22
+ self.text_encoder = WanTextEncoder() if text_encoder is None else text_encoder
23
+ self.vae = WanVAEWrapper() if vae is None else vae
24
+
25
+ # Step 2: Initialize all causal hyperparmeters
26
+ self.scheduler = self.generator.get_scheduler()
27
+ self.denoising_step_list = torch.tensor(
28
+ args.denoising_step_list, dtype=torch.long)
29
+ if args.warp_denoising_step:
30
+ timesteps = torch.cat((self.scheduler.timesteps.cpu(), torch.tensor([0], dtype=torch.float32)))
31
+ self.denoising_step_list = timesteps[1000 - self.denoising_step_list]
32
+
33
+ self.num_transformer_blocks = 30
34
+ self.frame_seq_length = 1560
35
+
36
+ self.kv_cache1 = None
37
+ self.args = args
38
+ self.num_frame_per_block = getattr(args, "num_frame_per_block", 1)
39
+ self.independent_first_frame = args.independent_first_frame
40
+ self.local_attn_size = self.generator.model.local_attn_size
41
+
42
+ print(f"KV inference with {self.num_frame_per_block} frames per block")
43
+
44
+ if self.num_frame_per_block > 1:
45
+ self.generator.model.num_frame_per_block = self.num_frame_per_block
46
+
47
+ def inference(
48
+ self,
49
+ noise: torch.Tensor,
50
+ text_prompts: List[str],
51
+ initial_latent: Optional[torch.Tensor] = None,
52
+ return_latents: bool = False,
53
+ profile: bool = False,
54
+ low_memory: bool = False,
55
+ ) -> torch.Tensor:
56
+ """
57
+ Perform inference on the given noise and text prompts.
58
+ Inputs:
59
+ noise (torch.Tensor): The input noise tensor of shape
60
+ (batch_size, num_output_frames, num_channels, height, width).
61
+ text_prompts (List[str]): The list of text prompts.
62
+ initial_latent (torch.Tensor): The initial latent tensor of shape
63
+ (batch_size, num_input_frames, num_channels, height, width).
64
+ If num_input_frames is 1, perform image to video.
65
+ If num_input_frames is greater than 1, perform video extension.
66
+ return_latents (bool): Whether to return the latents.
67
+ Outputs:
68
+ video (torch.Tensor): The generated video tensor of shape
69
+ (batch_size, num_output_frames, num_channels, height, width).
70
+ It is normalized to be in the range [0, 1].
71
+ """
72
+ batch_size, num_frames, num_channels, height, width = noise.shape
73
+ if not self.independent_first_frame or (self.independent_first_frame and initial_latent is not None):
74
+ # If the first frame is independent and the first frame is provided, then the number of frames in the
75
+ # noise should still be a multiple of num_frame_per_block
76
+ assert num_frames % self.num_frame_per_block == 0
77
+ num_blocks = num_frames // self.num_frame_per_block
78
+ else:
79
+ # Using a [1, 4, 4, 4, 4, 4, ...] model to generate a video without image conditioning
80
+ assert (num_frames - 1) % self.num_frame_per_block == 0
81
+ num_blocks = (num_frames - 1) // self.num_frame_per_block
82
+ num_input_frames = initial_latent.shape[1] if initial_latent is not None else 0
83
+ num_output_frames = num_frames + num_input_frames # add the initial latent frames
84
+ conditional_dict = self.text_encoder(
85
+ text_prompts=text_prompts
86
+ )
87
+
88
+ if low_memory:
89
+ gpu_memory_preservation = get_cuda_free_memory_gb(gpu) + 5
90
+ move_model_to_device_with_memory_preservation(self.text_encoder, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
91
+
92
+ output = torch.zeros(
93
+ [batch_size, num_output_frames, num_channels, height, width],
94
+ device=noise.device,
95
+ dtype=noise.dtype
96
+ )
97
+
98
+ # Set up profiling if requested
99
+ if profile:
100
+ init_start = torch.cuda.Event(enable_timing=True)
101
+ init_end = torch.cuda.Event(enable_timing=True)
102
+ diffusion_start = torch.cuda.Event(enable_timing=True)
103
+ diffusion_end = torch.cuda.Event(enable_timing=True)
104
+ vae_start = torch.cuda.Event(enable_timing=True)
105
+ vae_end = torch.cuda.Event(enable_timing=True)
106
+ block_times = []
107
+ block_start = torch.cuda.Event(enable_timing=True)
108
+ block_end = torch.cuda.Event(enable_timing=True)
109
+ init_start.record()
110
+
111
+ # Step 1: Initialize KV cache to all zeros
112
+ if self.kv_cache1 is None:
113
+ self._initialize_kv_cache(
114
+ batch_size=batch_size,
115
+ dtype=noise.dtype,
116
+ device=noise.device
117
+ )
118
+ self._initialize_crossattn_cache(
119
+ batch_size=batch_size,
120
+ dtype=noise.dtype,
121
+ device=noise.device
122
+ )
123
+ else:
124
+ # reset cross attn cache
125
+ for block_index in range(self.num_transformer_blocks):
126
+ self.crossattn_cache[block_index]["is_init"] = False
127
+ # reset kv cache
128
+ for block_index in range(len(self.kv_cache1)):
129
+ self.kv_cache1[block_index]["global_end_index"] = torch.tensor(
130
+ [0], dtype=torch.long, device=noise.device)
131
+ self.kv_cache1[block_index]["local_end_index"] = torch.tensor(
132
+ [0], dtype=torch.long, device=noise.device)
133
+
134
+ # Step 2: Cache context feature
135
+ current_start_frame = 0
136
+ if initial_latent is not None:
137
+ timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
138
+ if self.independent_first_frame:
139
+ # Assume num_input_frames is 1 + self.num_frame_per_block * num_input_blocks
140
+ assert (num_input_frames - 1) % self.num_frame_per_block == 0
141
+ num_input_blocks = (num_input_frames - 1) // self.num_frame_per_block
142
+ output[:, :1] = initial_latent[:, :1]
143
+ self.generator(
144
+ noisy_image_or_video=initial_latent[:, :1],
145
+ conditional_dict=conditional_dict,
146
+ timestep=timestep * 0,
147
+ kv_cache=self.kv_cache1,
148
+ crossattn_cache=self.crossattn_cache,
149
+ current_start=current_start_frame * self.frame_seq_length,
150
+ )
151
+ current_start_frame += 1
152
+ else:
153
+ # Assume num_input_frames is self.num_frame_per_block * num_input_blocks
154
+ assert num_input_frames % self.num_frame_per_block == 0
155
+ num_input_blocks = num_input_frames // self.num_frame_per_block
156
+
157
+ for _ in range(num_input_blocks):
158
+ current_ref_latents = \
159
+ initial_latent[:, current_start_frame:current_start_frame + self.num_frame_per_block]
160
+ output[:, current_start_frame:current_start_frame + self.num_frame_per_block] = current_ref_latents
161
+ self.generator(
162
+ noisy_image_or_video=current_ref_latents,
163
+ conditional_dict=conditional_dict,
164
+ timestep=timestep * 0,
165
+ kv_cache=self.kv_cache1,
166
+ crossattn_cache=self.crossattn_cache,
167
+ current_start=current_start_frame * self.frame_seq_length,
168
+ )
169
+ current_start_frame += self.num_frame_per_block
170
+
171
+ if profile:
172
+ init_end.record()
173
+ torch.cuda.synchronize()
174
+ diffusion_start.record()
175
+
176
+ # Step 3: Temporal denoising loop
177
+ all_num_frames = [self.num_frame_per_block] * num_blocks
178
+ if self.independent_first_frame and initial_latent is None:
179
+ all_num_frames = [1] + all_num_frames
180
+ for current_num_frames in all_num_frames:
181
+ if profile:
182
+ block_start.record()
183
+
184
+ noisy_input = noise[
185
+ :, current_start_frame - num_input_frames:current_start_frame + current_num_frames - num_input_frames]
186
+
187
+ # Step 3.1: Spatial denoising loop
188
+ for index, current_timestep in enumerate(self.denoising_step_list):
189
+ print(f"current_timestep: {current_timestep}")
190
+ # set current timestep
191
+ timestep = torch.ones(
192
+ [batch_size, current_num_frames],
193
+ device=noise.device,
194
+ dtype=torch.int64) * current_timestep
195
+
196
+ if index < len(self.denoising_step_list) - 1:
197
+ _, denoised_pred = self.generator(
198
+ noisy_image_or_video=noisy_input,
199
+ conditional_dict=conditional_dict,
200
+ timestep=timestep,
201
+ kv_cache=self.kv_cache1,
202
+ crossattn_cache=self.crossattn_cache,
203
+ current_start=current_start_frame * self.frame_seq_length
204
+ )
205
+ next_timestep = self.denoising_step_list[index + 1]
206
+ noisy_input = self.scheduler.add_noise(
207
+ denoised_pred.flatten(0, 1),
208
+ torch.randn_like(denoised_pred.flatten(0, 1)),
209
+ next_timestep * torch.ones(
210
+ [batch_size * current_num_frames], device=noise.device, dtype=torch.long)
211
+ ).unflatten(0, denoised_pred.shape[:2])
212
+ else:
213
+ # for getting real output
214
+ _, denoised_pred = self.generator(
215
+ noisy_image_or_video=noisy_input,
216
+ conditional_dict=conditional_dict,
217
+ timestep=timestep,
218
+ kv_cache=self.kv_cache1,
219
+ crossattn_cache=self.crossattn_cache,
220
+ current_start=current_start_frame * self.frame_seq_length
221
+ )
222
+
223
+ # Step 3.2: record the model's output
224
+ output[:, current_start_frame:current_start_frame + current_num_frames] = denoised_pred
225
+
226
+ # Step 3.3: rerun with timestep zero to update KV cache using clean context
227
+ context_timestep = torch.ones_like(timestep) * self.args.context_noise
228
+ self.generator(
229
+ noisy_image_or_video=denoised_pred,
230
+ conditional_dict=conditional_dict,
231
+ timestep=context_timestep,
232
+ kv_cache=self.kv_cache1,
233
+ crossattn_cache=self.crossattn_cache,
234
+ current_start=current_start_frame * self.frame_seq_length,
235
+ )
236
+
237
+ if profile:
238
+ block_end.record()
239
+ torch.cuda.synchronize()
240
+ block_time = block_start.elapsed_time(block_end)
241
+ block_times.append(block_time)
242
+
243
+ # Step 3.4: update the start and end frame indices
244
+ current_start_frame += current_num_frames
245
+
246
+ if profile:
247
+ # End diffusion timing and synchronize CUDA
248
+ diffusion_end.record()
249
+ torch.cuda.synchronize()
250
+ diffusion_time = diffusion_start.elapsed_time(diffusion_end)
251
+ init_time = init_start.elapsed_time(init_end)
252
+ vae_start.record()
253
+
254
+ # Step 4: Decode the output
255
+ video = self.vae.decode_to_pixel(output, use_cache=False)
256
+ video = (video * 0.5 + 0.5).clamp(0, 1)
257
+
258
+ if profile:
259
+ # End VAE timing and synchronize CUDA
260
+ vae_end.record()
261
+ torch.cuda.synchronize()
262
+ vae_time = vae_start.elapsed_time(vae_end)
263
+ total_time = init_time + diffusion_time + vae_time
264
+
265
+ print("Profiling results:")
266
+ print(f" - Initialization/caching time: {init_time:.2f} ms ({100 * init_time / total_time:.2f}%)")
267
+ print(f" - Diffusion generation time: {diffusion_time:.2f} ms ({100 * diffusion_time / total_time:.2f}%)")
268
+ for i, block_time in enumerate(block_times):
269
+ print(f" - Block {i} generation time: {block_time:.2f} ms ({100 * block_time / diffusion_time:.2f}% of diffusion)")
270
+ print(f" - VAE decoding time: {vae_time:.2f} ms ({100 * vae_time / total_time:.2f}%)")
271
+ print(f" - Total time: {total_time:.2f} ms")
272
+
273
+ if return_latents:
274
+ return video, output
275
+ else:
276
+ return video
277
+
278
+ def _initialize_kv_cache(self, batch_size, dtype, device):
279
+ """
280
+ Initialize a Per-GPU KV cache for the Wan model.
281
+ """
282
+ kv_cache1 = []
283
+ if self.local_attn_size != -1:
284
+ # Use the local attention size to compute the KV cache size
285
+ kv_cache_size = self.local_attn_size * self.frame_seq_length
286
+ else:
287
+ # Use the default KV cache size
288
+ kv_cache_size = 32760
289
+
290
+ for _ in range(self.num_transformer_blocks):
291
+ kv_cache1.append({
292
+ "k": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
293
+ "v": torch.zeros([batch_size, kv_cache_size, 12, 128], dtype=dtype, device=device),
294
+ "global_end_index": torch.tensor([0], dtype=torch.long, device=device),
295
+ "local_end_index": torch.tensor([0], dtype=torch.long, device=device)
296
+ })
297
+
298
+ self.kv_cache1 = kv_cache1 # always store the clean cache
299
+
300
+ def _initialize_crossattn_cache(self, batch_size, dtype, device):
301
+ """
302
+ Initialize a Per-GPU cross-attention cache for the Wan model.
303
+ """
304
+ crossattn_cache = []
305
+
306
+ for _ in range(self.num_transformer_blocks):
307
+ crossattn_cache.append({
308
+ "k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
309
+ "v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
310
+ "is_init": False
311
+ })
312
+ self.crossattn_cache = crossattn_cache
pipeline/self_forcing_training.py ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from utils.wan_wrapper import WanDiffusionWrapper
2
+ from utils.scheduler import SchedulerInterface
3
+ from typing import List, Optional
4
+ import torch
5
+ import torch.distributed as dist
6
+
7
+
8
+ class SelfForcingTrainingPipeline:
9
+ def __init__(self,
10
+ denoising_step_list: List[int],
11
+ scheduler: SchedulerInterface,
12
+ generator: WanDiffusionWrapper,
13
+ num_frame_per_block=3,
14
+ independent_first_frame: bool = False,
15
+ same_step_across_blocks: bool = False,
16
+ last_step_only: bool = False,
17
+ num_max_frames: int = 21,
18
+ context_noise: int = 0,
19
+ **kwargs):
20
+ super().__init__()
21
+ self.scheduler = scheduler
22
+ self.generator = generator
23
+ self.denoising_step_list = denoising_step_list
24
+ if self.denoising_step_list[-1] == 0:
25
+ self.denoising_step_list = self.denoising_step_list[:-1] # remove the zero timestep for inference
26
+
27
+ # Wan specific hyperparameters
28
+ self.num_transformer_blocks = 30
29
+ self.frame_seq_length = 1560
30
+ self.num_frame_per_block = num_frame_per_block
31
+ self.context_noise = context_noise
32
+ self.i2v = False
33
+
34
+ self.kv_cache1 = None
35
+ self.kv_cache2 = None
36
+ self.independent_first_frame = independent_first_frame
37
+ self.same_step_across_blocks = same_step_across_blocks
38
+ self.last_step_only = last_step_only
39
+ self.kv_cache_size = num_max_frames * self.frame_seq_length
40
+
41
+ def generate_and_sync_list(self, num_blocks, num_denoising_steps, device):
42
+ rank = dist.get_rank() if dist.is_initialized() else 0
43
+
44
+ if rank == 0:
45
+ # Generate random indices
46
+ indices = torch.randint(
47
+ low=0,
48
+ high=num_denoising_steps,
49
+ size=(num_blocks,),
50
+ device=device
51
+ )
52
+ if self.last_step_only:
53
+ indices = torch.ones_like(indices) * (num_denoising_steps - 1)
54
+ else:
55
+ indices = torch.empty(num_blocks, dtype=torch.long, device=device)
56
+
57
+ dist.broadcast(indices, src=0) # Broadcast the random indices to all ranks
58
+ return indices.tolist()
59
+
60
+ def inference_with_trajectory(
61
+ self,
62
+ noise: torch.Tensor,
63
+ initial_latent: Optional[torch.Tensor] = None,
64
+ return_sim_step: bool = False,
65
+ **conditional_dict
66
+ ) -> torch.Tensor:
67
+ batch_size, num_frames, num_channels, height, width = noise.shape
68
+ if not self.independent_first_frame or (self.independent_first_frame and initial_latent is not None):
69
+ # If the first frame is independent and the first frame is provided, then the number of frames in the
70
+ # noise should still be a multiple of num_frame_per_block
71
+ assert num_frames % self.num_frame_per_block == 0
72
+ num_blocks = num_frames // self.num_frame_per_block
73
+ else:
74
+ # Using a [1, 4, 4, 4, 4, 4, ...] model to generate a video without image conditioning
75
+ assert (num_frames - 1) % self.num_frame_per_block == 0
76
+ num_blocks = (num_frames - 1) // self.num_frame_per_block
77
+ num_input_frames = initial_latent.shape[1] if initial_latent is not None else 0
78
+ num_output_frames = num_frames + num_input_frames # add the initial latent frames
79
+ output = torch.zeros(
80
+ [batch_size, num_output_frames, num_channels, height, width],
81
+ device=noise.device,
82
+ dtype=noise.dtype
83
+ )
84
+
85
+ # Step 1: Initialize KV cache to all zeros
86
+ self._initialize_kv_cache(
87
+ batch_size=batch_size, dtype=noise.dtype, device=noise.device
88
+ )
89
+ self._initialize_crossattn_cache(
90
+ batch_size=batch_size, dtype=noise.dtype, device=noise.device
91
+ )
92
+ # if self.kv_cache1 is None:
93
+ # self._initialize_kv_cache(
94
+ # batch_size=batch_size,
95
+ # dtype=noise.dtype,
96
+ # device=noise.device,
97
+ # )
98
+ # self._initialize_crossattn_cache(
99
+ # batch_size=batch_size,
100
+ # dtype=noise.dtype,
101
+ # device=noise.device
102
+ # )
103
+ # else:
104
+ # # reset cross attn cache
105
+ # for block_index in range(self.num_transformer_blocks):
106
+ # self.crossattn_cache[block_index]["is_init"] = False
107
+ # # reset kv cache
108
+ # for block_index in range(len(self.kv_cache1)):
109
+ # self.kv_cache1[block_index]["global_end_index"] = torch.tensor(
110
+ # [0], dtype=torch.long, device=noise.device)
111
+ # self.kv_cache1[block_index]["local_end_index"] = torch.tensor(
112
+ # [0], dtype=torch.long, device=noise.device)
113
+
114
+ # Step 2: Cache context feature
115
+ current_start_frame = 0
116
+ if initial_latent is not None:
117
+ timestep = torch.ones([batch_size, 1], device=noise.device, dtype=torch.int64) * 0
118
+ # Assume num_input_frames is 1 + self.num_frame_per_block * num_input_blocks
119
+ output[:, :1] = initial_latent
120
+ with torch.no_grad():
121
+ self.generator(
122
+ noisy_image_or_video=initial_latent,
123
+ conditional_dict=conditional_dict,
124
+ timestep=timestep * 0,
125
+ kv_cache=self.kv_cache1,
126
+ crossattn_cache=self.crossattn_cache,
127
+ current_start=current_start_frame * self.frame_seq_length
128
+ )
129
+ current_start_frame += 1
130
+
131
+ # Step 3: Temporal denoising loop
132
+ all_num_frames = [self.num_frame_per_block] * num_blocks
133
+ if self.independent_first_frame and initial_latent is None:
134
+ all_num_frames = [1] + all_num_frames
135
+ num_denoising_steps = len(self.denoising_step_list)
136
+ exit_flags = self.generate_and_sync_list(len(all_num_frames), num_denoising_steps, device=noise.device)
137
+ start_gradient_frame_index = num_output_frames - 21
138
+
139
+ # for block_index in range(num_blocks):
140
+ for block_index, current_num_frames in enumerate(all_num_frames):
141
+ noisy_input = noise[
142
+ :, current_start_frame - num_input_frames:current_start_frame + current_num_frames - num_input_frames]
143
+
144
+ # Step 3.1: Spatial denoising loop
145
+ for index, current_timestep in enumerate(self.denoising_step_list):
146
+ if self.same_step_across_blocks:
147
+ exit_flag = (index == exit_flags[0])
148
+ else:
149
+ exit_flag = (index == exit_flags[block_index]) # Only backprop at the randomly selected timestep (consistent across all ranks)
150
+ timestep = torch.ones(
151
+ [batch_size, current_num_frames],
152
+ device=noise.device,
153
+ dtype=torch.int64) * current_timestep
154
+
155
+ if not exit_flag:
156
+ with torch.no_grad():
157
+ _, denoised_pred = self.generator(
158
+ noisy_image_or_video=noisy_input,
159
+ conditional_dict=conditional_dict,
160
+ timestep=timestep,
161
+ kv_cache=self.kv_cache1,
162
+ crossattn_cache=self.crossattn_cache,
163
+ current_start=current_start_frame * self.frame_seq_length
164
+ )
165
+ next_timestep = self.denoising_step_list[index + 1]
166
+ noisy_input = self.scheduler.add_noise(
167
+ denoised_pred.flatten(0, 1),
168
+ torch.randn_like(denoised_pred.flatten(0, 1)),
169
+ next_timestep * torch.ones(
170
+ [batch_size * current_num_frames], device=noise.device, dtype=torch.long)
171
+ ).unflatten(0, denoised_pred.shape[:2])
172
+ else:
173
+ # for getting real output
174
+ # with torch.set_grad_enabled(current_start_frame >= start_gradient_frame_index):
175
+ if current_start_frame < start_gradient_frame_index:
176
+ with torch.no_grad():
177
+ _, denoised_pred = self.generator(
178
+ noisy_image_or_video=noisy_input,
179
+ conditional_dict=conditional_dict,
180
+ timestep=timestep,
181
+ kv_cache=self.kv_cache1,
182
+ crossattn_cache=self.crossattn_cache,
183
+ current_start=current_start_frame * self.frame_seq_length
184
+ )
185
+ else:
186
+ _, denoised_pred = self.generator(
187
+ noisy_image_or_video=noisy_input,
188
+ conditional_dict=conditional_dict,
189
+ timestep=timestep,
190
+ kv_cache=self.kv_cache1,
191
+ crossattn_cache=self.crossattn_cache,
192
+ current_start=current_start_frame * self.frame_seq_length
193
+ )
194
+ break
195
+
196
+ # Step 3.2: record the model's output
197
+ output[:, current_start_frame:current_start_frame + current_num_frames] = denoised_pred
198
+
199
+ # Step 3.3: rerun with timestep zero to update the cache
200
+ context_timestep = torch.ones_like(timestep) * self.context_noise
201
+ # add context noise
202
+ denoised_pred = self.scheduler.add_noise(
203
+ denoised_pred.flatten(0, 1),
204
+ torch.randn_like(denoised_pred.flatten(0, 1)),
205
+ context_timestep * torch.ones(
206
+ [batch_size * current_num_frames], device=noise.device, dtype=torch.long)
207
+ ).unflatten(0, denoised_pred.shape[:2])
208
+ with torch.no_grad():
209
+ self.generator(
210
+ noisy_image_or_video=denoised_pred,
211
+ conditional_dict=conditional_dict,
212
+ timestep=context_timestep,
213
+ kv_cache=self.kv_cache1,
214
+ crossattn_cache=self.crossattn_cache,
215
+ current_start=current_start_frame * self.frame_seq_length
216
+ )
217
+
218
+ # Step 3.4: update the start and end frame indices
219
+ current_start_frame += current_num_frames
220
+
221
+ # Step 3.5: Return the denoised timestep
222
+ if not self.same_step_across_blocks:
223
+ denoised_timestep_from, denoised_timestep_to = None, None
224
+ elif exit_flags[0] == len(self.denoising_step_list) - 1:
225
+ denoised_timestep_to = 0
226
+ denoised_timestep_from = 1000 - torch.argmin(
227
+ (self.scheduler.timesteps.cuda() - self.denoising_step_list[exit_flags[0]].cuda()).abs(), dim=0).item()
228
+ else:
229
+ denoised_timestep_to = 1000 - torch.argmin(
230
+ (self.scheduler.timesteps.cuda() - self.denoising_step_list[exit_flags[0] + 1].cuda()).abs(), dim=0).item()
231
+ denoised_timestep_from = 1000 - torch.argmin(
232
+ (self.scheduler.timesteps.cuda() - self.denoising_step_list[exit_flags[0]].cuda()).abs(), dim=0).item()
233
+
234
+ if return_sim_step:
235
+ return output, denoised_timestep_from, denoised_timestep_to, exit_flags[0] + 1
236
+
237
+ return output, denoised_timestep_from, denoised_timestep_to
238
+
239
+ def _initialize_kv_cache(self, batch_size, dtype, device):
240
+ """
241
+ Initialize a Per-GPU KV cache for the Wan model.
242
+ """
243
+ kv_cache1 = []
244
+
245
+ for _ in range(self.num_transformer_blocks):
246
+ kv_cache1.append({
247
+ "k": torch.zeros([batch_size, self.kv_cache_size, 12, 128], dtype=dtype, device=device),
248
+ "v": torch.zeros([batch_size, self.kv_cache_size, 12, 128], dtype=dtype, device=device),
249
+ "global_end_index": torch.tensor([0], dtype=torch.long, device=device),
250
+ "local_end_index": torch.tensor([0], dtype=torch.long, device=device)
251
+ })
252
+
253
+ self.kv_cache1 = kv_cache1 # always store the clean cache
254
+
255
+ def _initialize_crossattn_cache(self, batch_size, dtype, device):
256
+ """
257
+ Initialize a Per-GPU cross-attention cache for the Wan model.
258
+ """
259
+ crossattn_cache = []
260
+
261
+ for _ in range(self.num_transformer_blocks):
262
+ crossattn_cache.append({
263
+ "k": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
264
+ "v": torch.zeros([batch_size, 512, 12, 128], dtype=dtype, device=device),
265
+ "is_init": False
266
+ })
267
+ self.crossattn_cache = crossattn_cache