Fabrice-TIERCELIN commited on
Commit
81c8c91
·
verified ·
1 Parent(s): c093547

A dolphin emerges from the water

Browse files
Files changed (1) hide show
  1. app.py +575 -527
app.py CHANGED
@@ -1,527 +1,575 @@
1
- from diffusers_helper.hf_login import login
2
-
3
- import os
4
-
5
- os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
-
7
- import gradio as gr
8
- import torch
9
- import traceback
10
- import einops
11
- import safetensors.torch as sf
12
- import numpy as np
13
- import math
14
- import spaces
15
-
16
- from PIL import Image
17
- from diffusers import AutoencoderKLHunyuanVideo
18
- from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
19
- from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
20
- from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
21
- from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
22
- from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
23
- from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
24
- from diffusers_helper.thread_utils import AsyncStream, async_run
25
- from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
26
- from transformers import SiglipImageProcessor, SiglipVisionModel
27
- from diffusers_helper.clip_vision import hf_clip_vision_encode
28
- from diffusers_helper.bucket_tools import find_nearest_bucket
29
-
30
-
31
- free_mem_gb = get_cuda_free_memory_gb(gpu)
32
- high_vram = free_mem_gb > 80
33
-
34
- print(f'Free VRAM {free_mem_gb} GB')
35
- print(f'High-VRAM Mode: {high_vram}')
36
-
37
- text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
38
- text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
39
- tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
40
- tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
41
- vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
42
-
43
- feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
44
- image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
45
-
46
- transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
47
-
48
- vae.eval()
49
- text_encoder.eval()
50
- text_encoder_2.eval()
51
- image_encoder.eval()
52
- transformer.eval()
53
-
54
- if not high_vram:
55
- vae.enable_slicing()
56
- vae.enable_tiling()
57
-
58
- transformer.high_quality_fp32_output_for_inference = True
59
- print('transformer.high_quality_fp32_output_for_inference = True')
60
-
61
- transformer.to(dtype=torch.bfloat16)
62
- vae.to(dtype=torch.float16)
63
- image_encoder.to(dtype=torch.float16)
64
- text_encoder.to(dtype=torch.float16)
65
- text_encoder_2.to(dtype=torch.float16)
66
-
67
- vae.requires_grad_(False)
68
- text_encoder.requires_grad_(False)
69
- text_encoder_2.requires_grad_(False)
70
- image_encoder.requires_grad_(False)
71
- transformer.requires_grad_(False)
72
-
73
- if not high_vram:
74
- # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
75
- DynamicSwapInstaller.install_model(transformer, device=gpu)
76
- DynamicSwapInstaller.install_model(text_encoder, device=gpu)
77
- else:
78
- text_encoder.to(gpu)
79
- text_encoder_2.to(gpu)
80
- image_encoder.to(gpu)
81
- vae.to(gpu)
82
- transformer.to(gpu)
83
-
84
- stream = AsyncStream()
85
-
86
- outputs_folder = './outputs/'
87
- os.makedirs(outputs_folder, exist_ok=True)
88
-
89
- examples = [
90
- ["img_examples/1.png", "The girl dances gracefully, with clear movements, full of charm.",],
91
- ["img_examples/2.jpg", "The man dances flamboyantly, swinging his hips and striking bold poses with dramatic flair."],
92
- ["img_examples/3.png", "The woman dances elegantly among the blossoms, spinning slowly with flowing sleeves and graceful hand movements."],
93
- ]
94
-
95
- input_image_debug_value = None
96
- prompt_debug_value = None
97
- total_second_length_debug_value = None
98
-
99
- def generate_examples(input_image, prompt):
100
-
101
- t2v=False
102
- n_prompt=""
103
- seed=31337
104
- total_second_length=5
105
- latent_window_size=9
106
- steps=25
107
- cfg=1.0
108
- gs=10.0
109
- rs=0.0
110
- gpu_memory_preservation=6
111
- use_teacache=True
112
- mp4_crf=16
113
-
114
- global stream
115
-
116
- # assert input_image is not None, 'No input image!'
117
- if t2v:
118
- default_height, default_width = 640, 640
119
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
120
- print("No input image provided. Using a blank white image.")
121
-
122
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
123
-
124
- stream = AsyncStream()
125
-
126
- async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
127
-
128
- output_filename = None
129
-
130
- while True:
131
- flag, data = stream.output_queue.next()
132
-
133
- if flag == 'file':
134
- output_filename = data
135
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
136
-
137
- if flag == 'progress':
138
- preview, desc, html = data
139
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
140
-
141
- if flag == 'end':
142
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
143
- break
144
-
145
-
146
-
147
- @torch.no_grad()
148
- def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
149
- total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
150
- total_latent_sections = int(max(round(total_latent_sections), 1))
151
-
152
- job_id = generate_timestamp()
153
-
154
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
155
-
156
- try:
157
- # Clean GPU
158
- if not high_vram:
159
- unload_complete_models(
160
- text_encoder, text_encoder_2, image_encoder, vae, transformer
161
- )
162
-
163
- # Text encoding
164
-
165
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
166
-
167
- if not high_vram:
168
- fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
169
- load_model_as_complete(text_encoder_2, target_device=gpu)
170
-
171
- llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
172
-
173
- if cfg == 1:
174
- llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
175
- else:
176
- llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
177
-
178
- llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
179
- llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
180
-
181
- # Processing input image
182
-
183
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
184
-
185
- H, W, C = input_image.shape
186
- height, width = find_nearest_bucket(H, W, resolution=640)
187
- input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
188
-
189
- Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
190
-
191
- input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
192
- input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
193
-
194
- # VAE encoding
195
-
196
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
197
-
198
- if not high_vram:
199
- load_model_as_complete(vae, target_device=gpu)
200
-
201
- start_latent = vae_encode(input_image_pt, vae)
202
-
203
- # CLIP Vision
204
-
205
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
206
-
207
- if not high_vram:
208
- load_model_as_complete(image_encoder, target_device=gpu)
209
-
210
- image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
211
- image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
212
-
213
- # Dtype
214
-
215
- llama_vec = llama_vec.to(transformer.dtype)
216
- llama_vec_n = llama_vec_n.to(transformer.dtype)
217
- clip_l_pooler = clip_l_pooler.to(transformer.dtype)
218
- clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
219
- image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
220
-
221
- # Sampling
222
-
223
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
224
-
225
- rnd = torch.Generator("cpu").manual_seed(seed)
226
-
227
- history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
228
- history_pixels = None
229
-
230
- history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
231
- total_generated_latent_frames = 1
232
-
233
- for section_index in range(total_latent_sections):
234
- if stream.input_queue.top() == 'end':
235
- stream.output_queue.push(('end', None))
236
- return
237
-
238
- print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
239
-
240
- if not high_vram:
241
- unload_complete_models()
242
- move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
243
-
244
- if use_teacache:
245
- transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
246
- else:
247
- transformer.initialize_teacache(enable_teacache=False)
248
-
249
- def callback(d):
250
- preview = d['denoised']
251
- preview = vae_decode_fake(preview)
252
-
253
- preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
254
- preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
255
-
256
- if stream.input_queue.top() == 'end':
257
- stream.output_queue.push(('end', None))
258
- raise KeyboardInterrupt('User ends the task.')
259
-
260
- current_step = d['i'] + 1
261
- percentage = int(100.0 * current_step / steps)
262
- hint = f'Sampling {current_step}/{steps}'
263
- desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
264
- stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
265
- return
266
-
267
- indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
268
- clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
269
- clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
270
-
271
- clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
272
- clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
273
-
274
- generated_latents = sample_hunyuan(
275
- transformer=transformer,
276
- sampler='unipc',
277
- width=width,
278
- height=height,
279
- frames=latent_window_size * 4 - 3,
280
- real_guidance_scale=cfg,
281
- distilled_guidance_scale=gs,
282
- guidance_rescale=rs,
283
- # shift=3.0,
284
- num_inference_steps=steps,
285
- generator=rnd,
286
- prompt_embeds=llama_vec,
287
- prompt_embeds_mask=llama_attention_mask,
288
- prompt_poolers=clip_l_pooler,
289
- negative_prompt_embeds=llama_vec_n,
290
- negative_prompt_embeds_mask=llama_attention_mask_n,
291
- negative_prompt_poolers=clip_l_pooler_n,
292
- device=gpu,
293
- dtype=torch.bfloat16,
294
- image_embeddings=image_encoder_last_hidden_state,
295
- latent_indices=latent_indices,
296
- clean_latents=clean_latents,
297
- clean_latent_indices=clean_latent_indices,
298
- clean_latents_2x=clean_latents_2x,
299
- clean_latent_2x_indices=clean_latent_2x_indices,
300
- clean_latents_4x=clean_latents_4x,
301
- clean_latent_4x_indices=clean_latent_4x_indices,
302
- callback=callback,
303
- )
304
-
305
- total_generated_latent_frames += int(generated_latents.shape[2])
306
- history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
307
-
308
- if not high_vram:
309
- offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
310
- load_model_as_complete(vae, target_device=gpu)
311
-
312
- real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
313
-
314
- if history_pixels is None:
315
- history_pixels = vae_decode(real_history_latents, vae).cpu()
316
- else:
317
- section_latent_frames = latent_window_size * 2
318
- overlapped_frames = latent_window_size * 4 - 3
319
-
320
- current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
321
- history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
322
-
323
- if not high_vram:
324
- unload_complete_models()
325
-
326
- output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
327
-
328
- save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
329
-
330
- print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
331
-
332
- stream.output_queue.push(('file', output_filename))
333
- except:
334
- traceback.print_exc()
335
-
336
- if not high_vram:
337
- unload_complete_models(
338
- text_encoder, text_encoder_2, image_encoder, vae, transformer
339
- )
340
-
341
- stream.output_queue.push(('end', None))
342
- return
343
-
344
- def get_duration(input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
345
- global total_second_length_debug_value
346
-
347
- if total_second_length_debug_value is not None:
348
- return total_second_length_debug_value * 60
349
- return total_second_length * 60
350
-
351
-
352
- @spaces.GPU(duration=get_duration)
353
- def process(input_image, prompt,
354
- t2v=False,
355
- n_prompt="",
356
- seed=31337,
357
- total_second_length=5,
358
- latent_window_size=9,
359
- steps=25,
360
- cfg=1.0,
361
- gs=10.0,
362
- rs=0.0,
363
- gpu_memory_preservation=6,
364
- use_teacache=True,
365
- mp4_crf=16
366
- ):
367
- global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
368
-
369
- if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
370
- input_image = input_image_debug_value
371
- prompt = prompt_debug_value
372
- total_second_length = total_second_length_debug_value
373
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
374
-
375
- # assert input_image is not None, 'No input image!'
376
- if t2v:
377
- default_height, default_width = 640, 640
378
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
379
- print("No input image provided. Using a blank white image.")
380
-
381
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
382
-
383
- stream = AsyncStream()
384
-
385
- async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
386
-
387
- output_filename = None
388
-
389
- while True:
390
- flag, data = stream.output_queue.next()
391
-
392
- if flag == 'file':
393
- output_filename = data
394
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
395
-
396
- if flag == 'progress':
397
- preview, desc, html = data
398
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
399
-
400
- if flag == 'end':
401
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
402
- break
403
-
404
-
405
- def end_process():
406
- stream.input_queue.push('end')
407
-
408
-
409
- css = make_progress_bar_css()
410
- block = gr.Blocks(css=css).queue()
411
- with block:
412
- gr.Markdown('# FramePack-F1')
413
- gr.Markdown(f"""### Video diffusion, but feels like image diffusion
414
- *FramePack F1 - a FramePack model that only predicts future frames from history frames*
415
- ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
416
- adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
417
- """)
418
- with gr.Row():
419
- with gr.Column():
420
- input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
421
- prompt = gr.Textbox(label="Prompt", value='')
422
- t2v = gr.Checkbox(label="do text-to-video", value=False)
423
-
424
- with gr.Row():
425
- start_button = gr.Button(value="Start Generation", variant="primary")
426
- end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
427
-
428
- total_second_length = gr.Slider(label="Total Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
429
- with gr.Group():
430
- with gr.Accordion("Advanced settings", open=False):
431
- use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
432
-
433
- n_prompt = gr.Textbox(label="Negative Prompt", value="") # Not used
434
- seed = gr.Number(label="Seed", value=31337, precision=0)
435
-
436
-
437
- latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
438
- steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
439
-
440
- cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
441
- gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
442
- rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
443
-
444
- gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
445
-
446
- mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
447
-
448
- with gr.Accordion("Debug", open=False):
449
- input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
450
- prompt_debug = gr.Textbox(label="Prompt Debug", value='')
451
- total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
452
-
453
- with gr.Column():
454
- preview_image = gr.Image(label="Next Latents", height=200)
455
- result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
456
- progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
457
- progress_bar = gr.HTML('', elem_classes='no-generating-animation')
458
-
459
- gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
460
-
461
- ips = [input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
462
- start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
463
- end_button.click(fn=end_process)
464
-
465
- # gr.Examples(
466
- # examples,
467
- # inputs=[input_image, prompt],
468
- # outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
469
- # fn=generate_examples,
470
- # cache_examples=True
471
- # )
472
-
473
- with gr.Row(visible=False):
474
- gr.Examples(
475
- examples = [
476
- [
477
- "./img_examples/Example1.png", # input_image
478
- "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
479
- False, # t2v
480
- "", # n_prompt
481
- 42, # seed
482
- 1, # total_second_length
483
- 9, # latent_window_size
484
- 25, # steps
485
- 1.0, # cfg
486
- 10.0, # gs
487
- 0.0, # rs
488
- 6, # gpu_memory_preservation
489
- True, # use_teacache
490
- 16 # mp4_crf
491
- ],
492
- ],
493
- run_on_click = True,
494
- fn = process,
495
- inputs = ips,
496
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
497
- cache_examples = True,
498
- )
499
-
500
-
501
- def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
502
- global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
503
- input_image_debug_value = input_image_debug_data
504
- prompt_debug_value = prompt_debug_data
505
- total_second_length_debug_value = total_second_length_debug_data
506
- return []
507
-
508
- input_image_debug.upload(
509
- fn=handle_field_debug_change,
510
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
511
- outputs=[]
512
- )
513
-
514
- prompt_debug.change(
515
- fn=handle_field_debug_change,
516
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
517
- outputs=[]
518
- )
519
-
520
- total_second_length_debug.change(
521
- fn=handle_field_debug_change,
522
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
523
- outputs=[]
524
- )
525
-
526
-
527
- block.launch(ssr_mode=False)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from diffusers_helper.hf_login import login
2
+
3
+ import os
4
+
5
+ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
+
7
+ import gradio as gr
8
+ import torch
9
+ import traceback
10
+ import einops
11
+ import safetensors.torch as sf
12
+ import numpy as np
13
+ import math
14
+ import spaces
15
+
16
+ from PIL import Image
17
+ from diffusers import AutoencoderKLHunyuanVideo
18
+ from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
19
+ from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
20
+ from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
21
+ from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
22
+ from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
23
+ from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
24
+ from diffusers_helper.thread_utils import AsyncStream, async_run
25
+ from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
26
+ from transformers import SiglipImageProcessor, SiglipVisionModel
27
+ from diffusers_helper.clip_vision import hf_clip_vision_encode
28
+ from diffusers_helper.bucket_tools import find_nearest_bucket
29
+
30
+
31
+ free_mem_gb = get_cuda_free_memory_gb(gpu)
32
+ high_vram = free_mem_gb > 80
33
+
34
+ print(f'Free VRAM {free_mem_gb} GB')
35
+ print(f'High-VRAM Mode: {high_vram}')
36
+
37
+ text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
38
+ text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
39
+ tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
40
+ tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
41
+ vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
42
+
43
+ feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
44
+ image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
45
+
46
+ transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
47
+
48
+ vae.eval()
49
+ text_encoder.eval()
50
+ text_encoder_2.eval()
51
+ image_encoder.eval()
52
+ transformer.eval()
53
+
54
+ if not high_vram:
55
+ vae.enable_slicing()
56
+ vae.enable_tiling()
57
+
58
+ transformer.high_quality_fp32_output_for_inference = True
59
+ print('transformer.high_quality_fp32_output_for_inference = True')
60
+
61
+ transformer.to(dtype=torch.bfloat16)
62
+ vae.to(dtype=torch.float16)
63
+ image_encoder.to(dtype=torch.float16)
64
+ text_encoder.to(dtype=torch.float16)
65
+ text_encoder_2.to(dtype=torch.float16)
66
+
67
+ vae.requires_grad_(False)
68
+ text_encoder.requires_grad_(False)
69
+ text_encoder_2.requires_grad_(False)
70
+ image_encoder.requires_grad_(False)
71
+ transformer.requires_grad_(False)
72
+
73
+ if not high_vram:
74
+ # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
75
+ DynamicSwapInstaller.install_model(transformer, device=gpu)
76
+ DynamicSwapInstaller.install_model(text_encoder, device=gpu)
77
+ else:
78
+ text_encoder.to(gpu)
79
+ text_encoder_2.to(gpu)
80
+ image_encoder.to(gpu)
81
+ vae.to(gpu)
82
+ transformer.to(gpu)
83
+
84
+ stream = AsyncStream()
85
+
86
+ outputs_folder = './outputs/'
87
+ os.makedirs(outputs_folder, exist_ok=True)
88
+
89
+ examples = [
90
+ ["img_examples/1.png", "The girl dances gracefully, with clear movements, full of charm.",],
91
+ ["img_examples/2.jpg", "The man dances flamboyantly, swinging his hips and striking bold poses with dramatic flair."],
92
+ ["img_examples/3.png", "The woman dances elegantly among the blossoms, spinning slowly with flowing sleeves and graceful hand movements."],
93
+ ]
94
+
95
+ input_image_debug_value = None
96
+ prompt_debug_value = None
97
+ total_second_length_debug_value = None
98
+
99
+ def generate_examples(input_image, prompt):
100
+
101
+ t2v=False
102
+ n_prompt=""
103
+ seed=31337
104
+ total_second_length=5
105
+ latent_window_size=9
106
+ steps=25
107
+ cfg=1.0
108
+ gs=10.0
109
+ rs=0.0
110
+ gpu_memory_preservation=6
111
+ use_teacache=True
112
+ mp4_crf=16
113
+
114
+ global stream
115
+
116
+ # assert input_image is not None, 'No input image!'
117
+ if t2v:
118
+ default_height, default_width = 640, 640
119
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
120
+ print("No input image provided. Using a blank white image.")
121
+
122
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
123
+
124
+ stream = AsyncStream()
125
+
126
+ async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
127
+
128
+ output_filename = None
129
+
130
+ while True:
131
+ flag, data = stream.output_queue.next()
132
+
133
+ if flag == 'file':
134
+ output_filename = data
135
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
136
+
137
+ if flag == 'progress':
138
+ preview, desc, html = data
139
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
140
+
141
+ if flag == 'end':
142
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
143
+ break
144
+
145
+
146
+
147
+ @torch.no_grad()
148
+ def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
149
+ total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
150
+ total_latent_sections = int(max(round(total_latent_sections), 1))
151
+
152
+ job_id = generate_timestamp()
153
+
154
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
155
+
156
+ try:
157
+ # Clean GPU
158
+ if not high_vram:
159
+ unload_complete_models(
160
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
161
+ )
162
+
163
+ # Text encoding
164
+
165
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
166
+
167
+ if not high_vram:
168
+ fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
169
+ load_model_as_complete(text_encoder_2, target_device=gpu)
170
+
171
+ llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
172
+
173
+ if cfg == 1:
174
+ llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
175
+ else:
176
+ llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
177
+
178
+ llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
179
+ llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
180
+
181
+ # Processing input image
182
+
183
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
184
+
185
+ H, W, C = input_image.shape
186
+ height, width = find_nearest_bucket(H, W, resolution=640)
187
+ input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
188
+
189
+ Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
190
+
191
+ input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
192
+ input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
193
+
194
+ # VAE encoding
195
+
196
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
197
+
198
+ if not high_vram:
199
+ load_model_as_complete(vae, target_device=gpu)
200
+
201
+ start_latent = vae_encode(input_image_pt, vae)
202
+
203
+ # CLIP Vision
204
+
205
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
206
+
207
+ if not high_vram:
208
+ load_model_as_complete(image_encoder, target_device=gpu)
209
+
210
+ image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
211
+ image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
212
+
213
+ # Dtype
214
+
215
+ llama_vec = llama_vec.to(transformer.dtype)
216
+ llama_vec_n = llama_vec_n.to(transformer.dtype)
217
+ clip_l_pooler = clip_l_pooler.to(transformer.dtype)
218
+ clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
219
+ image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
220
+
221
+ # Sampling
222
+
223
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
224
+
225
+ rnd = torch.Generator("cpu").manual_seed(seed)
226
+
227
+ history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
228
+ history_pixels = None
229
+
230
+ history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
231
+ total_generated_latent_frames = 1
232
+
233
+ for section_index in range(total_latent_sections):
234
+ if stream.input_queue.top() == 'end':
235
+ stream.output_queue.push(('end', None))
236
+ return
237
+
238
+ print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
239
+
240
+ if not high_vram:
241
+ unload_complete_models()
242
+ move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
243
+
244
+ if use_teacache:
245
+ transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
246
+ else:
247
+ transformer.initialize_teacache(enable_teacache=False)
248
+
249
+ def callback(d):
250
+ preview = d['denoised']
251
+ preview = vae_decode_fake(preview)
252
+
253
+ preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
254
+ preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
255
+
256
+ if stream.input_queue.top() == 'end':
257
+ stream.output_queue.push(('end', None))
258
+ raise KeyboardInterrupt('User ends the task.')
259
+
260
+ current_step = d['i'] + 1
261
+ percentage = int(100.0 * current_step / steps)
262
+ hint = f'Sampling {current_step}/{steps}'
263
+ desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
264
+ stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
265
+ return
266
+
267
+ indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
268
+ clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
269
+ clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
270
+
271
+ clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
272
+ clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
273
+
274
+ generated_latents = sample_hunyuan(
275
+ transformer=transformer,
276
+ sampler='unipc',
277
+ width=width,
278
+ height=height,
279
+ frames=latent_window_size * 4 - 3,
280
+ real_guidance_scale=cfg,
281
+ distilled_guidance_scale=gs,
282
+ guidance_rescale=rs,
283
+ # shift=3.0,
284
+ num_inference_steps=steps,
285
+ generator=rnd,
286
+ prompt_embeds=llama_vec,
287
+ prompt_embeds_mask=llama_attention_mask,
288
+ prompt_poolers=clip_l_pooler,
289
+ negative_prompt_embeds=llama_vec_n,
290
+ negative_prompt_embeds_mask=llama_attention_mask_n,
291
+ negative_prompt_poolers=clip_l_pooler_n,
292
+ device=gpu,
293
+ dtype=torch.bfloat16,
294
+ image_embeddings=image_encoder_last_hidden_state,
295
+ latent_indices=latent_indices,
296
+ clean_latents=clean_latents,
297
+ clean_latent_indices=clean_latent_indices,
298
+ clean_latents_2x=clean_latents_2x,
299
+ clean_latent_2x_indices=clean_latent_2x_indices,
300
+ clean_latents_4x=clean_latents_4x,
301
+ clean_latent_4x_indices=clean_latent_4x_indices,
302
+ callback=callback,
303
+ )
304
+
305
+ total_generated_latent_frames += int(generated_latents.shape[2])
306
+ history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
307
+
308
+ if not high_vram:
309
+ offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
310
+ load_model_as_complete(vae, target_device=gpu)
311
+
312
+ real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
313
+
314
+ if history_pixels is None:
315
+ history_pixels = vae_decode(real_history_latents, vae).cpu()
316
+ else:
317
+ section_latent_frames = latent_window_size * 2
318
+ overlapped_frames = latent_window_size * 4 - 3
319
+
320
+ current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
321
+ history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
322
+
323
+ if not high_vram:
324
+ unload_complete_models()
325
+
326
+ output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
327
+
328
+ save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
329
+
330
+ print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
331
+
332
+ stream.output_queue.push(('file', output_filename))
333
+ except:
334
+ traceback.print_exc()
335
+
336
+ if not high_vram:
337
+ unload_complete_models(
338
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
339
+ )
340
+
341
+ stream.output_queue.push(('end', None))
342
+ return
343
+
344
+ def get_duration(input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
345
+ global total_second_length_debug_value
346
+
347
+ if total_second_length_debug_value is not None:
348
+ return total_second_length_debug_value * 60
349
+ return total_second_length * 60
350
+
351
+
352
+ @spaces.GPU(duration=get_duration)
353
+ def process(input_image, prompt,
354
+ t2v=False,
355
+ n_prompt="",
356
+ seed=31337,
357
+ total_second_length=5,
358
+ latent_window_size=9,
359
+ steps=25,
360
+ cfg=1.0,
361
+ gs=10.0,
362
+ rs=0.0,
363
+ gpu_memory_preservation=6,
364
+ use_teacache=True,
365
+ mp4_crf=16
366
+ ):
367
+ global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
368
+
369
+ if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
370
+ input_image = input_image_debug_value
371
+ prompt = prompt_debug_value
372
+ total_second_length = total_second_length_debug_value
373
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
374
+
375
+ # assert input_image is not None, 'No input image!'
376
+ if t2v:
377
+ default_height, default_width = 640, 640
378
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
379
+ print("No input image provided. Using a blank white image.")
380
+
381
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
382
+
383
+ stream = AsyncStream()
384
+
385
+ async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
386
+
387
+ output_filename = None
388
+
389
+ while True:
390
+ flag, data = stream.output_queue.next()
391
+
392
+ if flag == 'file':
393
+ output_filename = data
394
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
395
+
396
+ if flag == 'progress':
397
+ preview, desc, html = data
398
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
399
+
400
+ if flag == 'end':
401
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
402
+ break
403
+
404
+
405
+ def end_process():
406
+ stream.input_queue.push('end')
407
+
408
+
409
+ css = make_progress_bar_css()
410
+ block = gr.Blocks(css=css).queue()
411
+ with block:
412
+ gr.Markdown('# FramePack-F1')
413
+ gr.Markdown(f"""### Video diffusion, but feels like image diffusion
414
+ *FramePack F1 - a FramePack model that only predicts future frames from history frames*
415
+ ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
416
+ adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
417
+ """)
418
+ with gr.Row():
419
+ with gr.Column():
420
+ input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
421
+ prompt = gr.Textbox(label="Prompt", value='')
422
+ t2v = gr.Checkbox(label="do text-to-video", value=False)
423
+
424
+ with gr.Row():
425
+ start_button = gr.Button(value="Start Generation", variant="primary")
426
+ end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
427
+
428
+ total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
429
+ with gr.Group():
430
+ with gr.Accordion("Advanced settings", open=False):
431
+ use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
432
+
433
+ n_prompt = gr.Textbox(label="Negative Prompt", value="") # Not used
434
+ seed = gr.Number(label="Seed", value=31337, precision=0)
435
+
436
+
437
+ latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
438
+ steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
439
+
440
+ cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
441
+ gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
442
+ rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
443
+
444
+ gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
445
+
446
+ mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
447
+
448
+ with gr.Accordion("Debug", open=False):
449
+ input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
450
+ prompt_debug = gr.Textbox(label="Prompt Debug", value='')
451
+ total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
452
+
453
+ with gr.Column():
454
+ preview_image = gr.Image(label="Next Latents", height=200)
455
+ result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
456
+ progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
457
+ progress_bar = gr.HTML('', elem_classes='no-generating-animation')
458
+
459
+ gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
460
+
461
+ ips = [input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
462
+ start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
463
+ end_button.click(fn=end_process)
464
+
465
+ # gr.Examples(
466
+ # examples,
467
+ # inputs=[input_image, prompt],
468
+ # outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
469
+ # fn=generate_examples,
470
+ # cache_examples=True
471
+ # )
472
+
473
+ with gr.Row(visible=False):
474
+ gr.Examples(
475
+ examples = [
476
+ [
477
+ "./img_examples/Example1.png", # input_image
478
+ "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
479
+ False, # t2v
480
+ "", # n_prompt
481
+ 42, # seed
482
+ 1, # total_second_length
483
+ 9, # latent_window_size
484
+ 25, # steps
485
+ 1.0, # cfg
486
+ 10.0, # gs
487
+ 0.0, # rs
488
+ 6, # gpu_memory_preservation
489
+ True, # use_teacache
490
+ 16 # mp4_crf
491
+ ],
492
+ [
493
+ "./img_examples/Example1.png", # input_image
494
+ "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
495
+ False, # t2v
496
+ "", # n_prompt
497
+ 42, # seed
498
+ 1, # total_second_length
499
+ 9, # latent_window_size
500
+ 25, # steps
501
+ 1.0, # cfg
502
+ 10.0, # gs
503
+ 0.0, # rs
504
+ 6, # gpu_memory_preservation
505
+ True, # use_teacache
506
+ 16 # mp4_crf
507
+ ],
508
+ [
509
+ "./img_examples/Example1.png", # input_image
510
+ "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
511
+ False, # t2v
512
+ "", # n_prompt
513
+ 42, # seed
514
+ 1, # total_second_length
515
+ 9, # latent_window_size
516
+ 25, # steps
517
+ 1.0, # cfg
518
+ 10.0, # gs
519
+ 0.0, # rs
520
+ 6, # gpu_memory_preservation
521
+ True, # use_teacache
522
+ 16 # mp4_crf
523
+ ],
524
+ [
525
+ "./img_examples/Example1.png", # input_image
526
+ "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
527
+ False, # t2v
528
+ "", # n_prompt
529
+ 42, # seed
530
+ 1, # total_second_length
531
+ 9, # latent_window_size
532
+ 25, # steps
533
+ 1.0, # cfg
534
+ 10.0, # gs
535
+ 0.0, # rs
536
+ 6, # gpu_memory_preservation
537
+ True, # use_teacache
538
+ 16 # mp4_crf
539
+ ],
540
+ ],
541
+ run_on_click = True,
542
+ fn = process,
543
+ inputs = ips,
544
+ outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
545
+ cache_examples = True,
546
+ )
547
+
548
+
549
+ def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
550
+ global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
551
+ input_image_debug_value = input_image_debug_data
552
+ prompt_debug_value = prompt_debug_data
553
+ total_second_length_debug_value = total_second_length_debug_data
554
+ return []
555
+
556
+ input_image_debug.upload(
557
+ fn=handle_field_debug_change,
558
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
559
+ outputs=[]
560
+ )
561
+
562
+ prompt_debug.change(
563
+ fn=handle_field_debug_change,
564
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
565
+ outputs=[]
566
+ )
567
+
568
+ total_second_length_debug.change(
569
+ fn=handle_field_debug_change,
570
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
571
+ outputs=[]
572
+ )
573
+
574
+
575
+ block.launch(share=True, mcp_server=True, ssr_mode=True)