Fabrice-TIERCELIN commited on
Commit
4465f40
·
verified ·
1 Parent(s): e409fba
Files changed (1) hide show
  1. app.py +523 -523
app.py CHANGED
@@ -1,524 +1,524 @@
1
- from diffusers_helper.hf_login import login
2
-
3
- import os
4
-
5
- os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
-
7
- import gradio as gr
8
- import torch
9
- import traceback
10
- import einops
11
- import safetensors.torch as sf
12
- import numpy as np
13
- import math
14
- import random
15
- import spaces
16
-
17
- from PIL import Image
18
- from diffusers import AutoencoderKLHunyuanVideo
19
- from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
20
- from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
21
- from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
22
- from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
23
- from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
24
- from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
25
- from diffusers_helper.thread_utils import AsyncStream, async_run
26
- from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
27
- from transformers import SiglipImageProcessor, SiglipVisionModel
28
- from diffusers_helper.clip_vision import hf_clip_vision_encode
29
- from diffusers_helper.bucket_tools import find_nearest_bucket
30
-
31
-
32
- free_mem_gb = get_cuda_free_memory_gb(gpu)
33
- high_vram = free_mem_gb > 80
34
-
35
- print(f'Free VRAM {free_mem_gb} GB')
36
- print(f'High-VRAM Mode: {high_vram}')
37
-
38
- text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
39
- text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
40
- tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
41
- tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
42
- vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
43
-
44
- feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
45
- image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
46
-
47
- transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
48
-
49
- vae.eval()
50
- text_encoder.eval()
51
- text_encoder_2.eval()
52
- image_encoder.eval()
53
- transformer.eval()
54
-
55
- if not high_vram:
56
- vae.enable_slicing()
57
- vae.enable_tiling()
58
-
59
- transformer.high_quality_fp32_output_for_inference = True
60
- print('transformer.high_quality_fp32_output_for_inference = True')
61
-
62
- transformer.to(dtype=torch.bfloat16)
63
- vae.to(dtype=torch.float16)
64
- image_encoder.to(dtype=torch.float16)
65
- text_encoder.to(dtype=torch.float16)
66
- text_encoder_2.to(dtype=torch.float16)
67
-
68
- vae.requires_grad_(False)
69
- text_encoder.requires_grad_(False)
70
- text_encoder_2.requires_grad_(False)
71
- image_encoder.requires_grad_(False)
72
- transformer.requires_grad_(False)
73
-
74
- if not high_vram:
75
- # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
76
- DynamicSwapInstaller.install_model(transformer, device=gpu)
77
- DynamicSwapInstaller.install_model(text_encoder, device=gpu)
78
- else:
79
- text_encoder.to(gpu)
80
- text_encoder_2.to(gpu)
81
- image_encoder.to(gpu)
82
- vae.to(gpu)
83
- transformer.to(gpu)
84
-
85
- stream = AsyncStream()
86
-
87
- outputs_folder = './outputs/'
88
- os.makedirs(outputs_folder, exist_ok=True)
89
-
90
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
91
-
92
- def callback(d):
93
- preview = d['denoised']
94
- preview = vae_decode_fake(preview)
95
-
96
- preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
97
- preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
98
-
99
- if stream.input_queue.top() == 'end':
100
- stream.output_queue.push(('end', None))
101
- raise KeyboardInterrupt('User ends the task.')
102
-
103
- current_step = d['i'] + 1
104
- percentage = int(100.0 * current_step / steps)
105
- hint = f'Sampling {current_step}/{steps}'
106
- desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
107
- stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
108
- return
109
-
110
- @torch.no_grad()
111
- def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
112
- total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
113
- total_latent_sections = int(max(round(total_latent_sections), 1))
114
-
115
- job_id = generate_timestamp()
116
-
117
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
118
-
119
- try:
120
- # Clean GPU
121
- if not high_vram:
122
- unload_complete_models(
123
- text_encoder, text_encoder_2, image_encoder, vae, transformer
124
- )
125
-
126
- # Text encoding
127
-
128
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
129
-
130
- if not high_vram:
131
- fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
132
- load_model_as_complete(text_encoder_2, target_device=gpu)
133
-
134
- llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
135
-
136
- if cfg == 1:
137
- llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
138
- else:
139
- llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
140
-
141
- llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
142
- llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
143
-
144
- # Processing input image
145
-
146
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
147
-
148
- H, W, C = input_image.shape
149
- height, width = find_nearest_bucket(H, W, resolution=640)
150
- input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
151
-
152
- Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
153
-
154
- input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
155
- input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
156
-
157
- # VAE encoding
158
-
159
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
160
-
161
- if not high_vram:
162
- load_model_as_complete(vae, target_device=gpu)
163
-
164
- start_latent = vae_encode(input_image_pt, vae)
165
-
166
- # CLIP Vision
167
-
168
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
169
-
170
- if not high_vram:
171
- load_model_as_complete(image_encoder, target_device=gpu)
172
-
173
- image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
174
- image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
175
-
176
- # Dtype
177
-
178
- llama_vec = llama_vec.to(transformer.dtype)
179
- llama_vec_n = llama_vec_n.to(transformer.dtype)
180
- clip_l_pooler = clip_l_pooler.to(transformer.dtype)
181
- clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
182
- image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
183
-
184
- # Sampling
185
-
186
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
187
-
188
- rnd = torch.Generator("cpu").manual_seed(seed)
189
-
190
- history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
191
- history_pixels = None
192
-
193
- history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
194
- total_generated_latent_frames = 1
195
-
196
- for section_index in range(total_latent_sections):
197
- if stream.input_queue.top() == 'end':
198
- stream.output_queue.push(('end', None))
199
- return
200
-
201
- print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
202
-
203
- if not high_vram:
204
- unload_complete_models()
205
- move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
206
-
207
- if use_teacache:
208
- transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
209
- else:
210
- transformer.initialize_teacache(enable_teacache=False)
211
-
212
- indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
213
- clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
214
- clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
215
-
216
- clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
217
- clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
218
-
219
- generated_latents = sample_hunyuan(
220
- transformer=transformer,
221
- sampler='unipc',
222
- width=width,
223
- height=height,
224
- frames=latent_window_size * 4 - 3,
225
- real_guidance_scale=cfg,
226
- distilled_guidance_scale=gs,
227
- guidance_rescale=rs,
228
- # shift=3.0,
229
- num_inference_steps=steps,
230
- generator=rnd,
231
- prompt_embeds=llama_vec,
232
- prompt_embeds_mask=llama_attention_mask,
233
- prompt_poolers=clip_l_pooler,
234
- negative_prompt_embeds=llama_vec_n,
235
- negative_prompt_embeds_mask=llama_attention_mask_n,
236
- negative_prompt_poolers=clip_l_pooler_n,
237
- device=gpu,
238
- dtype=torch.bfloat16,
239
- image_embeddings=image_encoder_last_hidden_state,
240
- latent_indices=latent_indices,
241
- clean_latents=clean_latents,
242
- clean_latent_indices=clean_latent_indices,
243
- clean_latents_2x=clean_latents_2x,
244
- clean_latent_2x_indices=clean_latent_2x_indices,
245
- clean_latents_4x=clean_latents_4x,
246
- clean_latent_4x_indices=clean_latent_4x_indices,
247
- callback=callback,
248
- )
249
-
250
- total_generated_latent_frames += int(generated_latents.shape[2])
251
- history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
252
-
253
- if not high_vram:
254
- offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
255
- load_model_as_complete(vae, target_device=gpu)
256
-
257
- real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
258
-
259
- if history_pixels is None:
260
- history_pixels = vae_decode(real_history_latents, vae).cpu()
261
- else:
262
- section_latent_frames = latent_window_size * 2
263
- overlapped_frames = latent_window_size * 4 - 3
264
-
265
- current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
266
- history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
267
-
268
- if not high_vram:
269
- unload_complete_models()
270
-
271
- output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
272
-
273
- save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
274
-
275
- print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
276
-
277
- stream.output_queue.push(('file', output_filename))
278
- except:
279
- traceback.print_exc()
280
-
281
- if not high_vram:
282
- unload_complete_models(
283
- text_encoder, text_encoder_2, image_encoder, vae, transformer
284
- )
285
-
286
- stream.output_queue.push(('end', None))
287
- return
288
-
289
- def get_duration(input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
290
- global total_second_length_debug_value
291
-
292
- if total_second_length_debug_value is not None:
293
- return min(total_second_length_debug_value * 60, 600)
294
- return total_second_length * 60
295
-
296
-
297
- @spaces.GPU(duration=get_duration)
298
- def process(input_image, prompt,
299
- t2v=False,
300
- n_prompt="",
301
- randomize_seed=True,
302
- seed=31337,
303
- total_second_length=5,
304
- latent_window_size=9,
305
- steps=25,
306
- cfg=1.0,
307
- gs=10.0,
308
- rs=0.0,
309
- gpu_memory_preservation=6,
310
- use_teacache=True,
311
- mp4_crf=16
312
- ):
313
- global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
314
-
315
- if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
316
- print("Debug mode")
317
- input_image = input_image_debug_value
318
- prompt = prompt_debug_value
319
- total_second_length = total_second_length_debug_value
320
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
321
-
322
- if randomize_seed:
323
- seed = random.randint(0, np.iinfo(np.int32).max)
324
-
325
- # assert input_image is not None, 'No input image!'
326
- if t2v:
327
- default_height, default_width = 640, 640
328
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
329
- print("No input image provided. Using a blank white image.")
330
-
331
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
332
-
333
- stream = AsyncStream()
334
-
335
- async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
336
-
337
- output_filename = None
338
-
339
- while True:
340
- flag, data = stream.output_queue.next()
341
-
342
- if flag == 'file':
343
- output_filename = data
344
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
345
-
346
- if flag == 'progress':
347
- preview, desc, html = data
348
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
349
-
350
- if flag == 'end':
351
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
352
- break
353
-
354
-
355
- def end_process():
356
- stream.input_queue.push('end')
357
-
358
-
359
- css = make_progress_bar_css()
360
- block = gr.Blocks(css=css).queue()
361
- with block:
362
- gr.Markdown('# FramePack-F1')
363
- gr.Markdown(f"""### Video diffusion, but feels like image diffusion
364
- *FramePack F1 - a FramePack model that only predicts future frames from history frames*
365
- ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
366
- adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
367
- """)
368
- with gr.Row():
369
- with gr.Column():
370
- input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
371
- prompt = gr.Textbox(label="Prompt", value='')
372
- t2v = gr.Checkbox(label="do text-to-video", value=False)
373
-
374
- with gr.Row():
375
- start_button = gr.Button(value="Start Generation", variant="primary")
376
- end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
377
-
378
- total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
379
- with gr.Group():
380
- with gr.Accordion("Advanced settings", open=False):
381
- use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
382
-
383
- n_prompt = gr.Textbox(label="Negative Prompt", value="") # Not used
384
- randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different')
385
- seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True)
386
-
387
-
388
- latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
389
- steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
390
-
391
- cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
392
- gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
393
- rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
394
-
395
- gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
396
-
397
- mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
398
-
399
- with gr.Accordion("Debug", open=False):
400
- input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
401
- prompt_debug = gr.Textbox(label="Prompt Debug", value='')
402
- total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
403
-
404
- with gr.Column():
405
- preview_image = gr.Image(label="Next Latents", height=200)
406
- result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
407
- progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
408
- progress_bar = gr.HTML('', elem_classes='no-generating-animation')
409
-
410
- gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
411
-
412
- ips = [input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
413
- start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
414
- end_button.click(fn=end_process)
415
-
416
- with gr.Row(visible=False):
417
- gr.Examples(
418
- examples = [
419
- [
420
- "./img_examples/Example1.png", # input_image
421
- "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
422
- False, # t2v
423
- "", # n_prompt
424
- True, # randomize_seed
425
- 42, # seed
426
- 1, # total_second_length
427
- 9, # latent_window_size
428
- 25, # steps
429
- 1.0, # cfg
430
- 10.0, # gs
431
- 0.0, # rs
432
- 6, # gpu_memory_preservation
433
- True, # use_teacache
434
- 16 # mp4_crf
435
- ],
436
- [
437
- "./img_examples/Example1.png", # input_image
438
- "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
439
- False, # t2v
440
- "", # n_prompt
441
- True, # randomize_seed
442
- 42, # seed
443
- 1, # total_second_length
444
- 9, # latent_window_size
445
- 25, # steps
446
- 1.0, # cfg
447
- 10.0, # gs
448
- 0.0, # rs
449
- 6, # gpu_memory_preservation
450
- True, # use_teacache
451
- 16 # mp4_crf
452
- ],
453
- [
454
- "./img_examples/Example1.png", # input_image
455
- "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
456
- False, # t2v
457
- "", # n_prompt
458
- True, # randomize_seed
459
- 42, # seed
460
- 1, # total_second_length
461
- 9, # latent_window_size
462
- 25, # steps
463
- 1.0, # cfg
464
- 10.0, # gs
465
- 0.0, # rs
466
- 6, # gpu_memory_preservation
467
- True, # use_teacache
468
- 16 # mp4_crf
469
- ],
470
- [
471
- "./img_examples/Example1.png", # input_image
472
- "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
473
- False, # t2v
474
- "", # n_prompt
475
- True, # randomize_seed
476
- 42, # seed
477
- 1, # total_second_length
478
- 9, # latent_window_size
479
- 25, # steps
480
- 1.0, # cfg
481
- 10.0, # gs
482
- 0.0, # rs
483
- 6, # gpu_memory_preservation
484
- True, # use_teacache
485
- 16 # mp4_crf
486
- ],
487
- ],
488
- run_on_click = True,
489
- fn = process,
490
- inputs = ips,
491
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
492
- cache_examples = True,
493
- )
494
- gr.Markdown('## Guide')
495
- gr.Markdown("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
496
-
497
-
498
- def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
499
- global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
500
- input_image_debug_value = input_image_debug_data
501
- prompt_debug_value = prompt_debug_data
502
- total_second_length_debug_value = total_second_length_debug_data
503
- return []
504
-
505
- input_image_debug.upload(
506
- fn=handle_field_debug_change,
507
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
508
- outputs=[]
509
- )
510
-
511
- prompt_debug.change(
512
- fn=handle_field_debug_change,
513
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
514
- outputs=[]
515
- )
516
-
517
- total_second_length_debug.change(
518
- fn=handle_field_debug_change,
519
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
520
- outputs=[]
521
- )
522
-
523
-
524
  block.launch(mcp_server=False)
 
1
+ from diffusers_helper.hf_login import login
2
+
3
+ import os
4
+
5
+ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
+
7
+ import gradio as gr
8
+ import torch
9
+ import traceback
10
+ import einops
11
+ import safetensors.torch as sf
12
+ import numpy as np
13
+ import math
14
+ import random
15
+ import spaces
16
+
17
+ from PIL import Image
18
+ from diffusers import AutoencoderKLHunyuanVideo
19
+ from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
20
+ from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
21
+ from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
22
+ from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
23
+ from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
24
+ from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
25
+ from diffusers_helper.thread_utils import AsyncStream, async_run
26
+ from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
27
+ from transformers import SiglipImageProcessor, SiglipVisionModel
28
+ from diffusers_helper.clip_vision import hf_clip_vision_encode
29
+ from diffusers_helper.bucket_tools import find_nearest_bucket
30
+
31
+
32
+ free_mem_gb = get_cuda_free_memory_gb(gpu)
33
+ high_vram = free_mem_gb > 80
34
+
35
+ print(f'Free VRAM {free_mem_gb} GB')
36
+ print(f'High-VRAM Mode: {high_vram}')
37
+
38
+ text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
39
+ text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
40
+ tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
41
+ tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
42
+ vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
43
+
44
+ feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
45
+ image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
46
+
47
+ transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
48
+
49
+ vae.eval()
50
+ text_encoder.eval()
51
+ text_encoder_2.eval()
52
+ image_encoder.eval()
53
+ transformer.eval()
54
+
55
+ if not high_vram:
56
+ vae.enable_slicing()
57
+ vae.enable_tiling()
58
+
59
+ transformer.high_quality_fp32_output_for_inference = True
60
+ print('transformer.high_quality_fp32_output_for_inference = True')
61
+
62
+ transformer.to(dtype=torch.bfloat16)
63
+ vae.to(dtype=torch.float16)
64
+ image_encoder.to(dtype=torch.float16)
65
+ text_encoder.to(dtype=torch.float16)
66
+ text_encoder_2.to(dtype=torch.float16)
67
+
68
+ vae.requires_grad_(False)
69
+ text_encoder.requires_grad_(False)
70
+ text_encoder_2.requires_grad_(False)
71
+ image_encoder.requires_grad_(False)
72
+ transformer.requires_grad_(False)
73
+
74
+ if not high_vram:
75
+ # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
76
+ DynamicSwapInstaller.install_model(transformer, device=gpu)
77
+ DynamicSwapInstaller.install_model(text_encoder, device=gpu)
78
+ else:
79
+ text_encoder.to(gpu)
80
+ text_encoder_2.to(gpu)
81
+ image_encoder.to(gpu)
82
+ vae.to(gpu)
83
+ transformer.to(gpu)
84
+
85
+ stream = AsyncStream()
86
+
87
+ outputs_folder = './outputs/'
88
+ os.makedirs(outputs_folder, exist_ok=True)
89
+
90
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
91
+
92
+ @torch.no_grad()
93
+ def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
94
+ total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
95
+ total_latent_sections = int(max(round(total_latent_sections), 1))
96
+
97
+ job_id = generate_timestamp()
98
+
99
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
100
+
101
+ try:
102
+ # Clean GPU
103
+ if not high_vram:
104
+ unload_complete_models(
105
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
106
+ )
107
+
108
+ # Text encoding
109
+
110
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
111
+
112
+ if not high_vram:
113
+ fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
114
+ load_model_as_complete(text_encoder_2, target_device=gpu)
115
+
116
+ llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
117
+
118
+ if cfg == 1:
119
+ llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
120
+ else:
121
+ llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
122
+
123
+ llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
124
+ llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
125
+
126
+ # Processing input image
127
+
128
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
129
+
130
+ H, W, C = input_image.shape
131
+ height, width = find_nearest_bucket(H, W, resolution=640)
132
+ input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
133
+
134
+ Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
135
+
136
+ input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
137
+ input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
138
+
139
+ # VAE encoding
140
+
141
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
142
+
143
+ if not high_vram:
144
+ load_model_as_complete(vae, target_device=gpu)
145
+
146
+ start_latent = vae_encode(input_image_pt, vae)
147
+
148
+ # CLIP Vision
149
+
150
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
151
+
152
+ if not high_vram:
153
+ load_model_as_complete(image_encoder, target_device=gpu)
154
+
155
+ image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
156
+ image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
157
+
158
+ # Dtype
159
+
160
+ llama_vec = llama_vec.to(transformer.dtype)
161
+ llama_vec_n = llama_vec_n.to(transformer.dtype)
162
+ clip_l_pooler = clip_l_pooler.to(transformer.dtype)
163
+ clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
164
+ image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
165
+
166
+ # Sampling
167
+
168
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
169
+
170
+ rnd = torch.Generator("cpu").manual_seed(seed)
171
+
172
+ history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
173
+ history_pixels = None
174
+
175
+ history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
176
+ total_generated_latent_frames = 1
177
+
178
+ for section_index in range(total_latent_sections):
179
+ if stream.input_queue.top() == 'end':
180
+ stream.output_queue.push(('end', None))
181
+ return
182
+
183
+ print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
184
+
185
+ if not high_vram:
186
+ unload_complete_models()
187
+ move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
188
+
189
+ if use_teacache:
190
+ transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
191
+ else:
192
+ transformer.initialize_teacache(enable_teacache=False)
193
+
194
+ def callback(d):
195
+ preview = d['denoised']
196
+ preview = vae_decode_fake(preview)
197
+
198
+ preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
199
+ preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
200
+
201
+ if stream.input_queue.top() == 'end':
202
+ stream.output_queue.push(('end', None))
203
+ raise KeyboardInterrupt('User ends the task.')
204
+
205
+ current_step = d['i'] + 1
206
+ percentage = int(100.0 * current_step / steps)
207
+ hint = f'Sampling {current_step}/{steps}'
208
+ desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
209
+ stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
210
+ return
211
+
212
+ indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
213
+ clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
214
+ clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
215
+
216
+ clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
217
+ clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
218
+
219
+ generated_latents = sample_hunyuan(
220
+ transformer=transformer,
221
+ sampler='unipc',
222
+ width=width,
223
+ height=height,
224
+ frames=latent_window_size * 4 - 3,
225
+ real_guidance_scale=cfg,
226
+ distilled_guidance_scale=gs,
227
+ guidance_rescale=rs,
228
+ # shift=3.0,
229
+ num_inference_steps=steps,
230
+ generator=rnd,
231
+ prompt_embeds=llama_vec,
232
+ prompt_embeds_mask=llama_attention_mask,
233
+ prompt_poolers=clip_l_pooler,
234
+ negative_prompt_embeds=llama_vec_n,
235
+ negative_prompt_embeds_mask=llama_attention_mask_n,
236
+ negative_prompt_poolers=clip_l_pooler_n,
237
+ device=gpu,
238
+ dtype=torch.bfloat16,
239
+ image_embeddings=image_encoder_last_hidden_state,
240
+ latent_indices=latent_indices,
241
+ clean_latents=clean_latents,
242
+ clean_latent_indices=clean_latent_indices,
243
+ clean_latents_2x=clean_latents_2x,
244
+ clean_latent_2x_indices=clean_latent_2x_indices,
245
+ clean_latents_4x=clean_latents_4x,
246
+ clean_latent_4x_indices=clean_latent_4x_indices,
247
+ callback=callback,
248
+ )
249
+
250
+ total_generated_latent_frames += int(generated_latents.shape[2])
251
+ history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
252
+
253
+ if not high_vram:
254
+ offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
255
+ load_model_as_complete(vae, target_device=gpu)
256
+
257
+ real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
258
+
259
+ if history_pixels is None:
260
+ history_pixels = vae_decode(real_history_latents, vae).cpu()
261
+ else:
262
+ section_latent_frames = latent_window_size * 2
263
+ overlapped_frames = latent_window_size * 4 - 3
264
+
265
+ current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
266
+ history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
267
+
268
+ if not high_vram:
269
+ unload_complete_models()
270
+
271
+ output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
272
+
273
+ save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
274
+
275
+ print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
276
+
277
+ stream.output_queue.push(('file', output_filename))
278
+ except:
279
+ traceback.print_exc()
280
+
281
+ if not high_vram:
282
+ unload_complete_models(
283
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
284
+ )
285
+
286
+ stream.output_queue.push(('end', None))
287
+ return
288
+
289
+ def get_duration(input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
290
+ global total_second_length_debug_value
291
+
292
+ if total_second_length_debug_value is not None:
293
+ return min(total_second_length_debug_value * 60, 600)
294
+ return total_second_length * 60
295
+
296
+
297
+ @spaces.GPU(duration=get_duration)
298
+ def process(input_image, prompt,
299
+ t2v=False,
300
+ n_prompt="",
301
+ randomize_seed=True,
302
+ seed=31337,
303
+ total_second_length=5,
304
+ latent_window_size=9,
305
+ steps=25,
306
+ cfg=1.0,
307
+ gs=10.0,
308
+ rs=0.0,
309
+ gpu_memory_preservation=6,
310
+ use_teacache=True,
311
+ mp4_crf=16
312
+ ):
313
+ global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
314
+
315
+ if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
316
+ print("Debug mode")
317
+ input_image = input_image_debug_value
318
+ prompt = prompt_debug_value
319
+ total_second_length = total_second_length_debug_value
320
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
321
+
322
+ if randomize_seed:
323
+ seed = random.randint(0, np.iinfo(np.int32).max)
324
+
325
+ # assert input_image is not None, 'No input image!'
326
+ if t2v:
327
+ default_height, default_width = 640, 640
328
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
329
+ print("No input image provided. Using a blank white image.")
330
+
331
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
332
+
333
+ stream = AsyncStream()
334
+
335
+ async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
336
+
337
+ output_filename = None
338
+
339
+ while True:
340
+ flag, data = stream.output_queue.next()
341
+
342
+ if flag == 'file':
343
+ output_filename = data
344
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
345
+
346
+ if flag == 'progress':
347
+ preview, desc, html = data
348
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
349
+
350
+ if flag == 'end':
351
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
352
+ break
353
+
354
+
355
+ def end_process():
356
+ stream.input_queue.push('end')
357
+
358
+
359
+ css = make_progress_bar_css()
360
+ block = gr.Blocks(css=css).queue()
361
+ with block:
362
+ gr.Markdown('# FramePack-F1')
363
+ gr.Markdown(f"""### Video diffusion, but feels like image diffusion
364
+ *FramePack F1 - a FramePack model that only predicts future frames from history frames*
365
+ ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
366
+ adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
367
+ """)
368
+ with gr.Row():
369
+ with gr.Column():
370
+ input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
371
+ prompt = gr.Textbox(label="Prompt", value='')
372
+ t2v = gr.Checkbox(label="do text-to-video", value=False)
373
+
374
+ with gr.Row():
375
+ start_button = gr.Button(value="Start Generation", variant="primary")
376
+ end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
377
+
378
+ total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
379
+ with gr.Group():
380
+ with gr.Accordion("Advanced settings", open=False):
381
+ use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
382
+
383
+ n_prompt = gr.Textbox(label="Negative Prompt", value="") # Not used
384
+ randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different')
385
+ seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True)
386
+
387
+
388
+ latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
389
+ steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
390
+
391
+ cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
392
+ gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
393
+ rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
394
+
395
+ gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
396
+
397
+ mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
398
+
399
+ with gr.Accordion("Debug", open=False):
400
+ input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
401
+ prompt_debug = gr.Textbox(label="Prompt Debug", value='')
402
+ total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
403
+
404
+ with gr.Column():
405
+ preview_image = gr.Image(label="Next Latents", height=200)
406
+ result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
407
+ progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
408
+ progress_bar = gr.HTML('', elem_classes='no-generating-animation')
409
+
410
+ gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
411
+
412
+ ips = [input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
413
+ start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
414
+ end_button.click(fn=end_process)
415
+
416
+ with gr.Row(visible=False):
417
+ gr.Examples(
418
+ examples = [
419
+ [
420
+ "./img_examples/Example1.png", # input_image
421
+ "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
422
+ False, # t2v
423
+ "", # n_prompt
424
+ True, # randomize_seed
425
+ 42, # seed
426
+ 1, # total_second_length
427
+ 9, # latent_window_size
428
+ 25, # steps
429
+ 1.0, # cfg
430
+ 10.0, # gs
431
+ 0.0, # rs
432
+ 6, # gpu_memory_preservation
433
+ True, # use_teacache
434
+ 16 # mp4_crf
435
+ ],
436
+ [
437
+ "./img_examples/Example1.png", # input_image
438
+ "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
439
+ False, # t2v
440
+ "", # n_prompt
441
+ True, # randomize_seed
442
+ 42, # seed
443
+ 1, # total_second_length
444
+ 9, # latent_window_size
445
+ 25, # steps
446
+ 1.0, # cfg
447
+ 10.0, # gs
448
+ 0.0, # rs
449
+ 6, # gpu_memory_preservation
450
+ True, # use_teacache
451
+ 16 # mp4_crf
452
+ ],
453
+ [
454
+ "./img_examples/Example1.png", # input_image
455
+ "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
456
+ False, # t2v
457
+ "", # n_prompt
458
+ True, # randomize_seed
459
+ 42, # seed
460
+ 1, # total_second_length
461
+ 9, # latent_window_size
462
+ 25, # steps
463
+ 1.0, # cfg
464
+ 10.0, # gs
465
+ 0.0, # rs
466
+ 6, # gpu_memory_preservation
467
+ True, # use_teacache
468
+ 16 # mp4_crf
469
+ ],
470
+ [
471
+ "./img_examples/Example1.png", # input_image
472
+ "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
473
+ False, # t2v
474
+ "", # n_prompt
475
+ True, # randomize_seed
476
+ 42, # seed
477
+ 1, # total_second_length
478
+ 9, # latent_window_size
479
+ 25, # steps
480
+ 1.0, # cfg
481
+ 10.0, # gs
482
+ 0.0, # rs
483
+ 6, # gpu_memory_preservation
484
+ True, # use_teacache
485
+ 16 # mp4_crf
486
+ ],
487
+ ],
488
+ run_on_click = True,
489
+ fn = process,
490
+ inputs = ips,
491
+ outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
492
+ cache_examples = True,
493
+ )
494
+ gr.Markdown('## Guide')
495
+ gr.Markdown("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
496
+
497
+
498
+ def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
499
+ global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
500
+ input_image_debug_value = input_image_debug_data
501
+ prompt_debug_value = prompt_debug_data
502
+ total_second_length_debug_value = total_second_length_debug_data
503
+ return []
504
+
505
+ input_image_debug.upload(
506
+ fn=handle_field_debug_change,
507
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
508
+ outputs=[]
509
+ )
510
+
511
+ prompt_debug.change(
512
+ fn=handle_field_debug_change,
513
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
514
+ outputs=[]
515
+ )
516
+
517
+ total_second_length_debug.change(
518
+ fn=handle_field_debug_change,
519
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
520
+ outputs=[]
521
+ )
522
+
523
+
524
  block.launch(mcp_server=False)