Fabrice-TIERCELIN commited on
Commit
65a628f
·
verified ·
1 Parent(s): 1f6351a
Files changed (1) hide show
  1. app.py +514 -488
app.py CHANGED
@@ -1,488 +1,514 @@
1
- from diffusers_helper.hf_login import login
2
-
3
- import os
4
-
5
- os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
-
7
- import gradio as gr
8
- import torch
9
- import traceback
10
- import einops
11
- import safetensors.torch as sf
12
- import numpy as np
13
- import math
14
- import spaces
15
-
16
- from PIL import Image
17
- from diffusers import AutoencoderKLHunyuanVideo
18
- from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
19
- from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
20
- from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
21
- from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
22
- from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
23
- from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
24
- from diffusers_helper.thread_utils import AsyncStream, async_run
25
- from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
26
- from transformers import SiglipImageProcessor, SiglipVisionModel
27
- from diffusers_helper.clip_vision import hf_clip_vision_encode
28
- from diffusers_helper.bucket_tools import find_nearest_bucket
29
-
30
-
31
- free_mem_gb = get_cuda_free_memory_gb(gpu)
32
- high_vram = free_mem_gb > 60
33
-
34
- print(f'Free VRAM {free_mem_gb} GB')
35
- print(f'High-VRAM Mode: {high_vram}')
36
-
37
- text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
38
- text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
39
- tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
40
- tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
41
- vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
42
-
43
- feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
44
- image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
45
-
46
- transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
47
-
48
- vae.eval()
49
- text_encoder.eval()
50
- text_encoder_2.eval()
51
- image_encoder.eval()
52
- transformer.eval()
53
-
54
- if not high_vram:
55
- vae.enable_slicing()
56
- vae.enable_tiling()
57
-
58
- transformer.high_quality_fp32_output_for_inference = True
59
- print('transformer.high_quality_fp32_output_for_inference = True')
60
-
61
- transformer.to(dtype=torch.bfloat16)
62
- vae.to(dtype=torch.float16)
63
- image_encoder.to(dtype=torch.float16)
64
- text_encoder.to(dtype=torch.float16)
65
- text_encoder_2.to(dtype=torch.float16)
66
-
67
- vae.requires_grad_(False)
68
- text_encoder.requires_grad_(False)
69
- text_encoder_2.requires_grad_(False)
70
- image_encoder.requires_grad_(False)
71
- transformer.requires_grad_(False)
72
-
73
- if not high_vram:
74
- # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
75
- DynamicSwapInstaller.install_model(transformer, device=gpu)
76
- DynamicSwapInstaller.install_model(text_encoder, device=gpu)
77
- else:
78
- text_encoder.to(gpu)
79
- text_encoder_2.to(gpu)
80
- image_encoder.to(gpu)
81
- vae.to(gpu)
82
- transformer.to(gpu)
83
-
84
- stream = AsyncStream()
85
-
86
- outputs_folder = './outputs/'
87
- os.makedirs(outputs_folder, exist_ok=True)
88
-
89
- examples = [
90
- ["img_examples/1.png", "The girl dances gracefully, with clear movements, full of charm.",],
91
- ["img_examples/2.jpg", "The man dances flamboyantly, swinging his hips and striking bold poses with dramatic flair."],
92
- ["img_examples/3.png", "The woman dances elegantly among the blossoms, spinning slowly with flowing sleeves and graceful hand movements."],
93
- ]
94
-
95
- def generate_examples(input_image, prompt):
96
-
97
- t2v=False
98
- n_prompt=""
99
- seed=31337
100
- total_second_length=5
101
- latent_window_size=9
102
- steps=25
103
- cfg=1.0
104
- gs=10.0
105
- rs=0.0
106
- gpu_memory_preservation=6
107
- use_teacache=True
108
- mp4_crf=16
109
-
110
- global stream
111
-
112
- # assert input_image is not None, 'No input image!'
113
- if t2v:
114
- default_height, default_width = 640, 640
115
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
116
- print("No input image provided. Using a blank white image.")
117
-
118
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
119
-
120
- stream = AsyncStream()
121
-
122
- async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
123
-
124
- output_filename = None
125
-
126
- while True:
127
- flag, data = stream.output_queue.next()
128
-
129
- if flag == 'file':
130
- output_filename = data
131
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
132
-
133
- if flag == 'progress':
134
- preview, desc, html = data
135
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
136
-
137
- if flag == 'end':
138
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
139
- break
140
-
141
-
142
-
143
- @torch.no_grad()
144
- def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
145
- total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
146
- total_latent_sections = int(max(round(total_latent_sections), 1))
147
-
148
- job_id = generate_timestamp()
149
-
150
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
151
-
152
- try:
153
- # Clean GPU
154
- if not high_vram:
155
- unload_complete_models(
156
- text_encoder, text_encoder_2, image_encoder, vae, transformer
157
- )
158
-
159
- # Text encoding
160
-
161
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
162
-
163
- if not high_vram:
164
- fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
165
- load_model_as_complete(text_encoder_2, target_device=gpu)
166
-
167
- llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
168
-
169
- if cfg == 1:
170
- llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
171
- else:
172
- llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
173
-
174
- llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
175
- llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
176
-
177
- # Processing input image
178
-
179
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
180
-
181
- H, W, C = input_image.shape
182
- height, width = find_nearest_bucket(H, W, resolution=640)
183
- input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
184
-
185
- Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
186
-
187
- input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
188
- input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
189
-
190
- # VAE encoding
191
-
192
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
193
-
194
- if not high_vram:
195
- load_model_as_complete(vae, target_device=gpu)
196
-
197
- start_latent = vae_encode(input_image_pt, vae)
198
-
199
- # CLIP Vision
200
-
201
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
202
-
203
- if not high_vram:
204
- load_model_as_complete(image_encoder, target_device=gpu)
205
-
206
- image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
207
- image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
208
-
209
- # Dtype
210
-
211
- llama_vec = llama_vec.to(transformer.dtype)
212
- llama_vec_n = llama_vec_n.to(transformer.dtype)
213
- clip_l_pooler = clip_l_pooler.to(transformer.dtype)
214
- clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
215
- image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
216
-
217
- # Sampling
218
-
219
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
220
-
221
- rnd = torch.Generator("cpu").manual_seed(seed)
222
-
223
- history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
224
- history_pixels = None
225
-
226
- history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
227
- total_generated_latent_frames = 1
228
-
229
- for section_index in range(total_latent_sections):
230
- if stream.input_queue.top() == 'end':
231
- stream.output_queue.push(('end', None))
232
- return
233
-
234
- print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
235
-
236
- if not high_vram:
237
- unload_complete_models()
238
- move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
239
-
240
- if use_teacache:
241
- transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
242
- else:
243
- transformer.initialize_teacache(enable_teacache=False)
244
-
245
- def callback(d):
246
- preview = d['denoised']
247
- preview = vae_decode_fake(preview)
248
-
249
- preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
250
- preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
251
-
252
- if stream.input_queue.top() == 'end':
253
- stream.output_queue.push(('end', None))
254
- raise KeyboardInterrupt('User ends the task.')
255
-
256
- current_step = d['i'] + 1
257
- percentage = int(100.0 * current_step / steps)
258
- hint = f'Sampling {current_step}/{steps}'
259
- desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
260
- stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
261
- return
262
-
263
- indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
264
- clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
265
- clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
266
-
267
- clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
268
- clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
269
-
270
- generated_latents = sample_hunyuan(
271
- transformer=transformer,
272
- sampler='unipc',
273
- width=width,
274
- height=height,
275
- frames=latent_window_size * 4 - 3,
276
- real_guidance_scale=cfg,
277
- distilled_guidance_scale=gs,
278
- guidance_rescale=rs,
279
- # shift=3.0,
280
- num_inference_steps=steps,
281
- generator=rnd,
282
- prompt_embeds=llama_vec,
283
- prompt_embeds_mask=llama_attention_mask,
284
- prompt_poolers=clip_l_pooler,
285
- negative_prompt_embeds=llama_vec_n,
286
- negative_prompt_embeds_mask=llama_attention_mask_n,
287
- negative_prompt_poolers=clip_l_pooler_n,
288
- device=gpu,
289
- dtype=torch.bfloat16,
290
- image_embeddings=image_encoder_last_hidden_state,
291
- latent_indices=latent_indices,
292
- clean_latents=clean_latents,
293
- clean_latent_indices=clean_latent_indices,
294
- clean_latents_2x=clean_latents_2x,
295
- clean_latent_2x_indices=clean_latent_2x_indices,
296
- clean_latents_4x=clean_latents_4x,
297
- clean_latent_4x_indices=clean_latent_4x_indices,
298
- callback=callback,
299
- )
300
-
301
- total_generated_latent_frames += int(generated_latents.shape[2])
302
- history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
303
-
304
- if not high_vram:
305
- offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
306
- load_model_as_complete(vae, target_device=gpu)
307
-
308
- real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
309
-
310
- if history_pixels is None:
311
- history_pixels = vae_decode(real_history_latents, vae).cpu()
312
- else:
313
- section_latent_frames = latent_window_size * 2
314
- overlapped_frames = latent_window_size * 4 - 3
315
-
316
- current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
317
- history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
318
-
319
- if not high_vram:
320
- unload_complete_models()
321
-
322
- output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
323
-
324
- save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
325
-
326
- print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
327
-
328
- stream.output_queue.push(('file', output_filename))
329
- except:
330
- traceback.print_exc()
331
-
332
- if not high_vram:
333
- unload_complete_models(
334
- text_encoder, text_encoder_2, image_encoder, vae, transformer
335
- )
336
-
337
- stream.output_queue.push(('end', None))
338
- return
339
-
340
- def get_duration(input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
341
- return total_second_length * 60
342
-
343
- @spaces.GPU(duration=get_duration)
344
- def process(input_image, prompt,
345
- t2v=False,
346
- n_prompt="",
347
- seed=31337,
348
- total_second_length=5,
349
- latent_window_size=9,
350
- steps=25,
351
- cfg=1.0,
352
- gs=10.0,
353
- rs=0.0,
354
- gpu_memory_preservation=6,
355
- use_teacache=True,
356
- mp4_crf=16
357
- ):
358
- global stream
359
-
360
- # assert input_image is not None, 'No input image!'
361
- if t2v:
362
- default_height, default_width = 640, 640
363
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
364
- print("No input image provided. Using a blank white image.")
365
- else:
366
- composite_rgba_uint8 = input_image["composite"]
367
-
368
- # rgb_uint8 will be (H, W, 3), dtype uint8
369
- rgb_uint8 = composite_rgba_uint8[:, :, :3]
370
- # mask_uint8 will be (H, W), dtype uint8
371
- mask_uint8 = composite_rgba_uint8[:, :, 3]
372
-
373
- # Create background
374
- h, w = rgb_uint8.shape[:2]
375
- # White background, (H, W, 3), dtype uint8
376
- background_uint8 = np.full((h, w, 3), 255, dtype=np.uint8)
377
-
378
- # Normalize mask to range [0.0, 1.0].
379
- alpha_normalized_float32 = mask_uint8.astype(np.float32) / 255.0
380
-
381
- # Expand alpha to 3 channels to match RGB images for broadcasting.
382
- # alpha_mask_float32 will have shape (H, W, 3)
383
- alpha_mask_float32 = np.stack([alpha_normalized_float32] * 3, axis=2)
384
-
385
- # alpha blending
386
- blended_image_float32 = rgb_uint8.astype(np.float32) * alpha_mask_float32 + \
387
- background_uint8.astype(np.float32) * (1.0 - alpha_mask_float32)
388
-
389
- input_image = np.clip(blended_image_float32, 0, 255).astype(np.uint8)
390
-
391
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
392
-
393
- stream = AsyncStream()
394
-
395
- async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
396
-
397
- output_filename = None
398
-
399
- while True:
400
- flag, data = stream.output_queue.next()
401
-
402
- if flag == 'file':
403
- output_filename = data
404
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
405
-
406
- if flag == 'progress':
407
- preview, desc, html = data
408
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
409
-
410
- if flag == 'end':
411
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
412
- break
413
-
414
-
415
- def end_process():
416
- stream.input_queue.push('end')
417
-
418
-
419
- quick_prompts = [
420
- 'The girl dances gracefully, with clear movements, full of charm.',
421
- 'A character doing some simple body movements.',
422
- ]
423
- quick_prompts = [[x] for x in quick_prompts]
424
-
425
-
426
- css = make_progress_bar_css()
427
- block = gr.Blocks(css=css).queue()
428
- with block:
429
- gr.Markdown('# FramePack-F1')
430
- gr.Markdown(f"""### Video diffusion, but feels like image diffusion
431
- *FramePack F1 - a FramePack model that only predicts future frames from history frames*
432
- ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
433
- adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
434
- """)
435
- with gr.Row():
436
- with gr.Column():
437
- input_image = gr.ImageEditor(type="numpy", label="Image", height=320, brush=gr.Brush(colors=["#ffffff"]))
438
- prompt = gr.Textbox(label="Prompt", value='')
439
- t2v = gr.Checkbox(label="do text-to-video", value=False)
440
- example_quick_prompts = gr.Dataset(samples=quick_prompts, label='Quick List', samples_per_page=1000, components=[prompt])
441
- example_quick_prompts.click(lambda x: x[0], inputs=[example_quick_prompts], outputs=prompt, show_progress=False, queue=False)
442
-
443
- with gr.Row():
444
- start_button = gr.Button(value="Start Generation")
445
- end_button = gr.Button(value="End Generation", interactive=False)
446
-
447
- total_second_length = gr.Slider(label="Total Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
448
- with gr.Group():
449
- with gr.Accordion("Advanced settings", open=False):
450
- use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
451
-
452
- n_prompt = gr.Textbox(label="Negative Prompt", value="", visible=False) # Not used
453
- seed = gr.Number(label="Seed", value=31337, precision=0)
454
-
455
-
456
- latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1, visible=False) # Should not change
457
- steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
458
-
459
- cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01, visible=False) # Should not change
460
- gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended.')
461
- rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01, visible=False) # Should not change
462
-
463
- gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
464
-
465
- mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
466
-
467
- with gr.Column():
468
- preview_image = gr.Image(label="Next Latents", height=200, visible=False)
469
- result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
470
- progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
471
- progress_bar = gr.HTML('', elem_classes='no-generating-animation')
472
-
473
- gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
474
-
475
- ips = [input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
476
- start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
477
- end_button.click(fn=end_process)
478
-
479
- # gr.Examples(
480
- # examples,
481
- # inputs=[input_image, prompt],
482
- # outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
483
- # fn=generate_examples,
484
- # cache_examples=True
485
- # )
486
-
487
-
488
- block.launch(share=True, mcp_server=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from diffusers_helper.hf_login import login
2
+
3
+ import os
4
+
5
+ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
+
7
+ import gradio as gr
8
+ import torch
9
+ import traceback
10
+ import einops
11
+ import safetensors.torch as sf
12
+ import numpy as np
13
+ import math
14
+ import spaces
15
+
16
+ from PIL import Image
17
+ from diffusers import AutoencoderKLHunyuanVideo
18
+ from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
19
+ from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
20
+ from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
21
+ from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
22
+ from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
23
+ from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
24
+ from diffusers_helper.thread_utils import AsyncStream, async_run
25
+ from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
26
+ from transformers import SiglipImageProcessor, SiglipVisionModel
27
+ from diffusers_helper.clip_vision import hf_clip_vision_encode
28
+ from diffusers_helper.bucket_tools import find_nearest_bucket
29
+
30
+
31
+ free_mem_gb = get_cuda_free_memory_gb(gpu)
32
+ high_vram = free_mem_gb > 60
33
+
34
+ print(f'Free VRAM {free_mem_gb} GB')
35
+ print(f'High-VRAM Mode: {high_vram}')
36
+
37
+ text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
38
+ text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
39
+ tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
40
+ tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
41
+ vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
42
+
43
+ feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
44
+ image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
45
+
46
+ transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
47
+
48
+ vae.eval()
49
+ text_encoder.eval()
50
+ text_encoder_2.eval()
51
+ image_encoder.eval()
52
+ transformer.eval()
53
+
54
+ if not high_vram:
55
+ vae.enable_slicing()
56
+ vae.enable_tiling()
57
+
58
+ transformer.high_quality_fp32_output_for_inference = True
59
+ print('transformer.high_quality_fp32_output_for_inference = True')
60
+
61
+ transformer.to(dtype=torch.bfloat16)
62
+ vae.to(dtype=torch.float16)
63
+ image_encoder.to(dtype=torch.float16)
64
+ text_encoder.to(dtype=torch.float16)
65
+ text_encoder_2.to(dtype=torch.float16)
66
+
67
+ vae.requires_grad_(False)
68
+ text_encoder.requires_grad_(False)
69
+ text_encoder_2.requires_grad_(False)
70
+ image_encoder.requires_grad_(False)
71
+ transformer.requires_grad_(False)
72
+
73
+ if not high_vram:
74
+ # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
75
+ DynamicSwapInstaller.install_model(transformer, device=gpu)
76
+ DynamicSwapInstaller.install_model(text_encoder, device=gpu)
77
+ else:
78
+ text_encoder.to(gpu)
79
+ text_encoder_2.to(gpu)
80
+ image_encoder.to(gpu)
81
+ vae.to(gpu)
82
+ transformer.to(gpu)
83
+
84
+ stream = AsyncStream()
85
+
86
+ outputs_folder = './outputs/'
87
+ os.makedirs(outputs_folder, exist_ok=True)
88
+
89
+ examples = [
90
+ ["img_examples/1.png", "The girl dances gracefully, with clear movements, full of charm.",],
91
+ ["img_examples/2.jpg", "The man dances flamboyantly, swinging his hips and striking bold poses with dramatic flair."],
92
+ ["img_examples/3.png", "The woman dances elegantly among the blossoms, spinning slowly with flowing sleeves and graceful hand movements."],
93
+ ]
94
+
95
+ def generate_examples(input_image, prompt):
96
+
97
+ t2v=False
98
+ n_prompt=""
99
+ seed=31337
100
+ total_second_length=5
101
+ latent_window_size=9
102
+ steps=25
103
+ cfg=1.0
104
+ gs=10.0
105
+ rs=0.0
106
+ gpu_memory_preservation=6
107
+ use_teacache=True
108
+ mp4_crf=16
109
+
110
+ global stream
111
+
112
+ # assert input_image is not None, 'No input image!'
113
+ if t2v:
114
+ default_height, default_width = 640, 640
115
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
116
+ print("No input image provided. Using a blank white image.")
117
+
118
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
119
+
120
+ stream = AsyncStream()
121
+
122
+ async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
123
+
124
+ output_filename = None
125
+
126
+ while True:
127
+ flag, data = stream.output_queue.next()
128
+
129
+ if flag == 'file':
130
+ output_filename = data
131
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
132
+
133
+ if flag == 'progress':
134
+ preview, desc, html = data
135
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
136
+
137
+ if flag == 'end':
138
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
139
+ break
140
+
141
+
142
+
143
+ @torch.no_grad()
144
+ def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
145
+ total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
146
+ total_latent_sections = int(max(round(total_latent_sections), 1))
147
+
148
+ job_id = generate_timestamp()
149
+
150
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
151
+
152
+ try:
153
+ # Clean GPU
154
+ if not high_vram:
155
+ unload_complete_models(
156
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
157
+ )
158
+
159
+ # Text encoding
160
+
161
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
162
+
163
+ if not high_vram:
164
+ fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
165
+ load_model_as_complete(text_encoder_2, target_device=gpu)
166
+
167
+ llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
168
+
169
+ if cfg == 1:
170
+ llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
171
+ else:
172
+ llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
173
+
174
+ llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
175
+ llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
176
+
177
+ # Processing input image
178
+
179
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
180
+
181
+ H, W, C = input_image.shape
182
+ height, width = find_nearest_bucket(H, W, resolution=640)
183
+ input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
184
+
185
+ Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
186
+
187
+ input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
188
+ input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
189
+
190
+ # VAE encoding
191
+
192
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
193
+
194
+ if not high_vram:
195
+ load_model_as_complete(vae, target_device=gpu)
196
+
197
+ start_latent = vae_encode(input_image_pt, vae)
198
+
199
+ # CLIP Vision
200
+
201
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
202
+
203
+ if not high_vram:
204
+ load_model_as_complete(image_encoder, target_device=gpu)
205
+
206
+ image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
207
+ image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
208
+
209
+ # Dtype
210
+
211
+ llama_vec = llama_vec.to(transformer.dtype)
212
+ llama_vec_n = llama_vec_n.to(transformer.dtype)
213
+ clip_l_pooler = clip_l_pooler.to(transformer.dtype)
214
+ clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
215
+ image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
216
+
217
+ # Sampling
218
+
219
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
220
+
221
+ rnd = torch.Generator("cpu").manual_seed(seed)
222
+
223
+ history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
224
+ history_pixels = None
225
+
226
+ history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
227
+ total_generated_latent_frames = 1
228
+
229
+ for section_index in range(total_latent_sections):
230
+ if stream.input_queue.top() == 'end':
231
+ stream.output_queue.push(('end', None))
232
+ return
233
+
234
+ print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
235
+
236
+ if not high_vram:
237
+ unload_complete_models()
238
+ move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
239
+
240
+ if use_teacache:
241
+ transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
242
+ else:
243
+ transformer.initialize_teacache(enable_teacache=False)
244
+
245
+ def callback(d):
246
+ preview = d['denoised']
247
+ preview = vae_decode_fake(preview)
248
+
249
+ preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
250
+ preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
251
+
252
+ if stream.input_queue.top() == 'end':
253
+ stream.output_queue.push(('end', None))
254
+ raise KeyboardInterrupt('User ends the task.')
255
+
256
+ current_step = d['i'] + 1
257
+ percentage = int(100.0 * current_step / steps)
258
+ hint = f'Sampling {current_step}/{steps}'
259
+ desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
260
+ stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
261
+ return
262
+
263
+ indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
264
+ clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
265
+ clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
266
+
267
+ clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
268
+ clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
269
+
270
+ generated_latents = sample_hunyuan(
271
+ transformer=transformer,
272
+ sampler='unipc',
273
+ width=width,
274
+ height=height,
275
+ frames=latent_window_size * 4 - 3,
276
+ real_guidance_scale=cfg,
277
+ distilled_guidance_scale=gs,
278
+ guidance_rescale=rs,
279
+ # shift=3.0,
280
+ num_inference_steps=steps,
281
+ generator=rnd,
282
+ prompt_embeds=llama_vec,
283
+ prompt_embeds_mask=llama_attention_mask,
284
+ prompt_poolers=clip_l_pooler,
285
+ negative_prompt_embeds=llama_vec_n,
286
+ negative_prompt_embeds_mask=llama_attention_mask_n,
287
+ negative_prompt_poolers=clip_l_pooler_n,
288
+ device=gpu,
289
+ dtype=torch.bfloat16,
290
+ image_embeddings=image_encoder_last_hidden_state,
291
+ latent_indices=latent_indices,
292
+ clean_latents=clean_latents,
293
+ clean_latent_indices=clean_latent_indices,
294
+ clean_latents_2x=clean_latents_2x,
295
+ clean_latent_2x_indices=clean_latent_2x_indices,
296
+ clean_latents_4x=clean_latents_4x,
297
+ clean_latent_4x_indices=clean_latent_4x_indices,
298
+ callback=callback,
299
+ )
300
+
301
+ total_generated_latent_frames += int(generated_latents.shape[2])
302
+ history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
303
+
304
+ if not high_vram:
305
+ offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
306
+ load_model_as_complete(vae, target_device=gpu)
307
+
308
+ real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
309
+
310
+ if history_pixels is None:
311
+ history_pixels = vae_decode(real_history_latents, vae).cpu()
312
+ else:
313
+ section_latent_frames = latent_window_size * 2
314
+ overlapped_frames = latent_window_size * 4 - 3
315
+
316
+ current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
317
+ history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
318
+
319
+ if not high_vram:
320
+ unload_complete_models()
321
+
322
+ output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
323
+
324
+ save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
325
+
326
+ print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
327
+
328
+ stream.output_queue.push(('file', output_filename))
329
+ except:
330
+ traceback.print_exc()
331
+
332
+ if not high_vram:
333
+ unload_complete_models(
334
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
335
+ )
336
+
337
+ stream.output_queue.push(('end', None))
338
+ return
339
+
340
+ def get_duration(input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
341
+ return total_second_length * 60
342
+
343
+ @spaces.GPU(duration=get_duration)
344
+ def process(input_image, prompt,
345
+ t2v=False,
346
+ n_prompt="",
347
+ seed=31337,
348
+ total_second_length=5,
349
+ latent_window_size=9,
350
+ steps=25,
351
+ cfg=1.0,
352
+ gs=10.0,
353
+ rs=0.0,
354
+ gpu_memory_preservation=6,
355
+ use_teacache=True,
356
+ mp4_crf=16
357
+ ):
358
+ global stream
359
+
360
+ # assert input_image is not None, 'No input image!'
361
+ if t2v:
362
+ default_height, default_width = 640, 640
363
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
364
+ print("No input image provided. Using a blank white image.")
365
+ else:
366
+ composite_rgba_uint8 = input_image["composite"]
367
+
368
+ # rgb_uint8 will be (H, W, 3), dtype uint8
369
+ rgb_uint8 = composite_rgba_uint8[:, :, :3]
370
+ # mask_uint8 will be (H, W), dtype uint8
371
+ mask_uint8 = composite_rgba_uint8[:, :, 3]
372
+
373
+ # Create background
374
+ h, w = rgb_uint8.shape[:2]
375
+ # White background, (H, W, 3), dtype uint8
376
+ background_uint8 = np.full((h, w, 3), 255, dtype=np.uint8)
377
+
378
+ # Normalize mask to range [0.0, 1.0].
379
+ alpha_normalized_float32 = mask_uint8.astype(np.float32) / 255.0
380
+
381
+ # Expand alpha to 3 channels to match RGB images for broadcasting.
382
+ # alpha_mask_float32 will have shape (H, W, 3)
383
+ alpha_mask_float32 = np.stack([alpha_normalized_float32] * 3, axis=2)
384
+
385
+ # alpha blending
386
+ blended_image_float32 = rgb_uint8.astype(np.float32) * alpha_mask_float32 + \
387
+ background_uint8.astype(np.float32) * (1.0 - alpha_mask_float32)
388
+
389
+ input_image = np.clip(blended_image_float32, 0, 255).astype(np.uint8)
390
+
391
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
392
+
393
+ stream = AsyncStream()
394
+
395
+ async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
396
+
397
+ output_filename = None
398
+
399
+ while True:
400
+ flag, data = stream.output_queue.next()
401
+
402
+ if flag == 'file':
403
+ output_filename = data
404
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
405
+
406
+ if flag == 'progress':
407
+ preview, desc, html = data
408
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
409
+
410
+ if flag == 'end':
411
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
412
+ break
413
+
414
+
415
+ def end_process():
416
+ stream.input_queue.push('end')
417
+
418
+
419
+ quick_prompts = [
420
+ 'The girl dances gracefully, with clear movements, full of charm.',
421
+ 'A character doing some simple body movements.',
422
+ ]
423
+ quick_prompts = [[x] for x in quick_prompts]
424
+
425
+
426
+ css = make_progress_bar_css()
427
+ block = gr.Blocks(css=css).queue()
428
+ with block:
429
+ gr.Markdown('# FramePack-F1')
430
+ gr.Markdown(f"""### Video diffusion, but feels like image diffusion
431
+ *FramePack F1 - a FramePack model that only predicts future frames from history frames*
432
+ ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
433
+ adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
434
+ """)
435
+ with gr.Row():
436
+ with gr.Column():
437
+ input_image = gr.ImageEditor(type="numpy", label="Image", height=320, brush=gr.Brush(colors=["#ffffff"]))
438
+ prompt = gr.Textbox(label="Prompt", value='')
439
+ t2v = gr.Checkbox(label="do text-to-video", value=False)
440
+ example_quick_prompts = gr.Dataset(samples=quick_prompts, label='Quick List', samples_per_page=1000, components=[prompt])
441
+ example_quick_prompts.click(lambda x: x[0], inputs=[example_quick_prompts], outputs=prompt, show_progress=False, queue=False)
442
+
443
+ with gr.Row():
444
+ start_button = gr.Button(value="Start Generation")
445
+ end_button = gr.Button(value="End Generation", interactive=False)
446
+
447
+ total_second_length = gr.Slider(label="Total Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
448
+ with gr.Group():
449
+ with gr.Accordion("Advanced settings", open=False):
450
+ use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
451
+
452
+ n_prompt = gr.Textbox(label="Negative Prompt", value="", visible=False) # Not used
453
+ seed = gr.Number(label="Seed", value=31337, precision=0)
454
+
455
+
456
+ latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1, visible=False) # Should not change
457
+ steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
458
+
459
+ cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01, visible=False) # Should not change
460
+ gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended.')
461
+ rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01, visible=False) # Should not change
462
+
463
+ gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
464
+
465
+ mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
466
+
467
+ with gr.Column():
468
+ preview_image = gr.Image(label="Next Latents", height=200, visible=False)
469
+ result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
470
+ progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
471
+ progress_bar = gr.HTML('', elem_classes='no-generating-animation')
472
+
473
+ gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
474
+
475
+ ips = [input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
476
+ start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
477
+ end_button.click(fn=end_process)
478
+
479
+ # gr.Examples(
480
+ # examples,
481
+ # inputs=[input_image, prompt],
482
+ # outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
483
+ # fn=generate_examples,
484
+ # cache_examples=True
485
+ # )
486
+
487
+ gr.Examples(
488
+ examples = [
489
+ [
490
+ "./img_examples/Example1.png", # input_image
491
+ "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
492
+ False, # t2v
493
+ "", # n_prompt
494
+ 42, # seed
495
+ 1, # total_second_length
496
+ 9, # latent_window_size
497
+ 25, # steps
498
+ 1.0, # cfg
499
+ 10.0, # gs
500
+ 0.0, # rs
501
+ 6, # gpu_memory_preservation
502
+ True, # use_teacache
503
+ 16 # mp4_crf
504
+ ],
505
+ ],
506
+ run_on_click = True,
507
+ fn = process,
508
+ inputs = ips,
509
+ outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
510
+ cache_examples = True,
511
+ )
512
+
513
+
514
+ block.launch(share=True, mcp_server=True)