Fabrice-TIERCELIN commited on
Commit
766a3f2
·
verified ·
1 Parent(s): c419402

Good condition

Browse files
Files changed (1) hide show
  1. app.py +548 -549
app.py CHANGED
@@ -1,550 +1,549 @@
1
- from diffusers_helper.hf_login import login
2
-
3
- import os
4
-
5
- os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
-
7
- import gradio as gr
8
- import torch
9
- import traceback
10
- import einops
11
- import safetensors.torch as sf
12
- import numpy as np
13
- import math
14
- import random
15
- import spaces
16
-
17
- from PIL import Image
18
- from diffusers import AutoencoderKLHunyuanVideo
19
- from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
20
- from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
21
- from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
22
- from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
23
- from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
24
- from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
25
- from diffusers_helper.thread_utils import AsyncStream, async_run
26
- from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
27
- from transformers import SiglipImageProcessor, SiglipVisionModel
28
- from diffusers_helper.clip_vision import hf_clip_vision_encode
29
- from diffusers_helper.bucket_tools import find_nearest_bucket
30
-
31
-
32
- free_mem_gb = get_cuda_free_memory_gb(gpu)
33
- high_vram = free_mem_gb > 80
34
-
35
- print(f'Free VRAM {free_mem_gb} GB')
36
- print(f'High-VRAM Mode: {high_vram}')
37
-
38
- text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
39
- text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
40
- tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
41
- tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
42
- vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
43
-
44
- feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
45
- image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
46
-
47
- transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
48
-
49
- vae.eval()
50
- text_encoder.eval()
51
- text_encoder_2.eval()
52
- image_encoder.eval()
53
- transformer.eval()
54
-
55
- if not high_vram:
56
- vae.enable_slicing()
57
- vae.enable_tiling()
58
-
59
- transformer.high_quality_fp32_output_for_inference = True
60
- print('transformer.high_quality_fp32_output_for_inference = True')
61
-
62
- transformer.to(dtype=torch.bfloat16)
63
- vae.to(dtype=torch.float16)
64
- image_encoder.to(dtype=torch.float16)
65
- text_encoder.to(dtype=torch.float16)
66
- text_encoder_2.to(dtype=torch.float16)
67
-
68
- vae.requires_grad_(False)
69
- text_encoder.requires_grad_(False)
70
- text_encoder_2.requires_grad_(False)
71
- image_encoder.requires_grad_(False)
72
- transformer.requires_grad_(False)
73
-
74
- if not high_vram:
75
- # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
76
- DynamicSwapInstaller.install_model(transformer, device=gpu)
77
- DynamicSwapInstaller.install_model(text_encoder, device=gpu)
78
- else:
79
- text_encoder.to(gpu)
80
- text_encoder_2.to(gpu)
81
- image_encoder.to(gpu)
82
- vae.to(gpu)
83
- transformer.to(gpu)
84
-
85
- stream = AsyncStream()
86
-
87
- outputs_folder = './outputs/'
88
- os.makedirs(outputs_folder, exist_ok=True)
89
-
90
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
91
-
92
- @torch.no_grad()
93
- def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
94
- def encode_prompt(prompt, n_prompt):
95
- llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
96
-
97
- if cfg == 1:
98
- llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
99
- else:
100
- llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
101
-
102
- llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
103
- llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
104
-
105
- llama_vec = llama_vec.to(transformer.dtype)
106
- llama_vec_n = llama_vec_n.to(transformer.dtype)
107
- clip_l_pooler = clip_l_pooler.to(transformer.dtype)
108
- clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
109
- return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
110
-
111
- total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
112
- total_latent_sections = int(max(round(total_latent_sections), 1))
113
-
114
- job_id = generate_timestamp()
115
-
116
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
117
-
118
- try:
119
- # Clean GPU
120
- if not high_vram:
121
- unload_complete_models(
122
- text_encoder, text_encoder_2, image_encoder, vae, transformer
123
- )
124
-
125
- # Text encoding
126
-
127
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
128
-
129
- if not high_vram:
130
- fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
131
- load_model_as_complete(text_encoder_2, target_device=gpu)
132
-
133
- prompt_parameters = []
134
-
135
- for prompt_part in prompt.split(";"):
136
- prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
137
- #[llama_vec_part, clip_l_pooler_part, llama_vec_n_part, clip_l_pooler_n_part, llama_attention_mask_part, llama_attention_mask_n_part] = encode_prompt(prompt_part, n_prompt)
138
-
139
- #[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = [llama_vec_part, clip_l_pooler_part, llama_vec_n_part, clip_l_pooler_n_part, llama_attention_mask_part, llama_attention_mask_n_part]
140
-
141
-
142
-
143
- #[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = encode_prompt(prompt, n_prompt)
144
-
145
- # Processing input image
146
-
147
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
148
-
149
- H, W, C = input_image.shape
150
- height, width = find_nearest_bucket(H, W, resolution=640)
151
- input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
152
-
153
- Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
154
-
155
- input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
156
- input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
157
-
158
- # VAE encoding
159
-
160
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
161
-
162
- if not high_vram:
163
- load_model_as_complete(vae, target_device=gpu)
164
-
165
- start_latent = vae_encode(input_image_pt, vae)
166
-
167
- # CLIP Vision
168
-
169
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
170
-
171
- if not high_vram:
172
- load_model_as_complete(image_encoder, target_device=gpu)
173
-
174
- image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
175
- image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
176
-
177
- # Dtype
178
-
179
- image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
180
-
181
- # Sampling
182
-
183
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
184
-
185
- rnd = torch.Generator("cpu").manual_seed(seed)
186
-
187
- history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
188
- history_pixels = None
189
-
190
- history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
191
- total_generated_latent_frames = 1
192
-
193
- for section_index in range(total_latent_sections):
194
- if stream.input_queue.top() == 'end':
195
- stream.output_queue.push(('end', None))
196
- return
197
-
198
- print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
199
-
200
- if len(prompt_parameters) > 0:
201
- [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0)
202
-
203
- if not high_vram:
204
- unload_complete_models()
205
- move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
206
-
207
- if use_teacache:
208
- transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
209
- else:
210
- transformer.initialize_teacache(enable_teacache=False)
211
-
212
- def callback(d):
213
- preview = d['denoised']
214
- preview = vae_decode_fake(preview)
215
-
216
- preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
217
- preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
218
-
219
- if stream.input_queue.top() == 'end':
220
- stream.output_queue.push(('end', None))
221
- raise KeyboardInterrupt('User ends the task.')
222
-
223
- current_step = d['i'] + 1
224
- percentage = int(100.0 * current_step / steps)
225
- hint = f'Sampling {current_step}/{steps}'
226
- desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
227
- stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
228
- return
229
-
230
- indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
231
- clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
232
- clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
233
-
234
- clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
235
- clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
236
-
237
- generated_latents = sample_hunyuan(
238
- transformer=transformer,
239
- sampler='unipc',
240
- width=width,
241
- height=height,
242
- frames=latent_window_size * 4 - 3,
243
- real_guidance_scale=cfg,
244
- distilled_guidance_scale=gs,
245
- guidance_rescale=rs,
246
- # shift=3.0,
247
- num_inference_steps=steps,
248
- generator=rnd,
249
- prompt_embeds=llama_vec,
250
- prompt_embeds_mask=llama_attention_mask,
251
- prompt_poolers=clip_l_pooler,
252
- negative_prompt_embeds=llama_vec_n,
253
- negative_prompt_embeds_mask=llama_attention_mask_n,
254
- negative_prompt_poolers=clip_l_pooler_n,
255
- device=gpu,
256
- dtype=torch.bfloat16,
257
- image_embeddings=image_encoder_last_hidden_state,
258
- latent_indices=latent_indices,
259
- clean_latents=clean_latents,
260
- clean_latent_indices=clean_latent_indices,
261
- clean_latents_2x=clean_latents_2x,
262
- clean_latent_2x_indices=clean_latent_2x_indices,
263
- clean_latents_4x=clean_latents_4x,
264
- clean_latent_4x_indices=clean_latent_4x_indices,
265
- callback=callback,
266
- )
267
-
268
- total_generated_latent_frames += int(generated_latents.shape[2])
269
- history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
270
-
271
- if not high_vram:
272
- offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
273
- load_model_as_complete(vae, target_device=gpu)
274
-
275
- real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
276
-
277
- if history_pixels is None:
278
- history_pixels = vae_decode(real_history_latents, vae).cpu()
279
- else:
280
- section_latent_frames = latent_window_size * 2
281
- overlapped_frames = latent_window_size * 4 - 3
282
-
283
- current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
284
- history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
285
-
286
- if not high_vram:
287
- unload_complete_models()
288
-
289
- output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
290
-
291
- save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
292
-
293
- print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
294
-
295
- stream.output_queue.push(('file', output_filename))
296
- except:
297
- traceback.print_exc()
298
-
299
- if not high_vram:
300
- unload_complete_models(
301
- text_encoder, text_encoder_2, image_encoder, vae, transformer
302
- )
303
-
304
- stream.output_queue.push(('end', None))
305
- return
306
-
307
- def get_duration(input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
308
- global total_second_length_debug_value
309
-
310
- if total_second_length_debug_value is not None:
311
- return min(total_second_length_debug_value * 60, 600)
312
- return total_second_length * 60
313
-
314
-
315
- @spaces.GPU(duration=get_duration)
316
- def process(input_image, prompt,
317
- t2v=False,
318
- n_prompt="",
319
- randomize_seed=True,
320
- seed=31337,
321
- total_second_length=5,
322
- latent_window_size=9,
323
- steps=25,
324
- cfg=1.0,
325
- gs=10.0,
326
- rs=0.0,
327
- gpu_memory_preservation=6,
328
- use_teacache=True,
329
- mp4_crf=16
330
- ):
331
- global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
332
-
333
- if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
334
- print("Debug mode")
335
- input_image = input_image_debug_value
336
- prompt = prompt_debug_value
337
- total_second_length = total_second_length_debug_value
338
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
339
-
340
- if randomize_seed:
341
- seed = random.randint(0, np.iinfo(np.int32).max)
342
-
343
- # assert input_image is not None, 'No input image!'
344
- if t2v:
345
- default_height, default_width = 640, 640
346
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
347
- print("No input image provided. Using a blank white image.")
348
-
349
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
350
-
351
- stream = AsyncStream()
352
-
353
- async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
354
-
355
- output_filename = None
356
-
357
- while True:
358
- flag, data = stream.output_queue.next()
359
-
360
- if flag == 'file':
361
- output_filename = data
362
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
363
-
364
- if flag == 'progress':
365
- preview, desc, html = data
366
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
367
-
368
- if flag == 'end':
369
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
370
- break
371
-
372
-
373
- def end_process():
374
- stream.input_queue.push('end')
375
-
376
-
377
- css = make_progress_bar_css()
378
- block = gr.Blocks(css=css).queue()
379
- with block:
380
- if torch.cuda.device_count() == 0:
381
- with gr.Row():
382
- gr.HTML("""
383
- <p style="background-color: red;"><big><big><big><b>⚠️To use FramePack, <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR?duplicate=true">duplicate this space</a> and set a GPU with 30 GB VRAM.</b>
384
-
385
- You can't use FramePack directly here because this space runs on a CPU, which is not enough for FramePack. Please provide <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR/discussions/new">feedback</a> if you have issues.
386
- </big></big></big></p>
387
- """)
388
- gr.Markdown('# FramePack-F1')
389
- gr.Markdown(f"""### Video diffusion, but feels like image diffusion
390
- *FramePack F1 - a FramePack model that only predicts future frames from history frames*
391
- ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
392
- adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
393
- """)
394
- with gr.Row():
395
- with gr.Column():
396
- input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
397
- prompt = gr.Textbox(label="Prompt", value='')
398
- t2v = gr.Checkbox(label="do text-to-video", value=False)
399
-
400
- with gr.Row():
401
- start_button = gr.Button(value="Start Generation", variant="primary")
402
- end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
403
-
404
- total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
405
- with gr.Group():
406
- with gr.Accordion("Advanced settings", open=False):
407
- use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
408
-
409
- n_prompt = gr.Textbox(label="Negative Prompt", value="") # Not used
410
- randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different')
411
- seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True)
412
-
413
-
414
- latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
415
- steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
416
-
417
- cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
418
- gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
419
- rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
420
-
421
- gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
422
-
423
- mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
424
-
425
- with gr.Accordion("Debug", open=False):
426
- input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
427
- prompt_debug = gr.Textbox(label="Prompt Debug", value='')
428
- total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
429
-
430
- with gr.Column():
431
- preview_image = gr.Image(label="Next Latents", height=200, visible=False)
432
- result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
433
- progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
434
- progress_bar = gr.HTML('', elem_classes='no-generating-animation')
435
-
436
- gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
437
-
438
- ips = [input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
439
- start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
440
- end_button.click(fn=end_process)
441
-
442
- with gr.Row(visible=False):
443
- gr.Examples(
444
- examples = [
445
- [
446
- "./img_examples/Example1.png", # input_image
447
- "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
448
- False, # t2v
449
- "", # n_prompt
450
- True, # randomize_seed
451
- 42, # seed
452
- 1, # total_second_length
453
- 9, # latent_window_size
454
- 25, # steps
455
- 1.0, # cfg
456
- 10.0, # gs
457
- 0.0, # rs
458
- 6, # gpu_memory_preservation
459
- True, # use_teacache
460
- 16 # mp4_crf
461
- ],
462
- [
463
- "./img_examples/Example1.png", # input_image
464
- "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
465
- False, # t2v
466
- "", # n_prompt
467
- True, # randomize_seed
468
- 42, # seed
469
- 1, # total_second_length
470
- 9, # latent_window_size
471
- 25, # steps
472
- 1.0, # cfg
473
- 10.0, # gs
474
- 0.0, # rs
475
- 6, # gpu_memory_preservation
476
- True, # use_teacache
477
- 16 # mp4_crf
478
- ],
479
- [
480
- "./img_examples/Example1.png", # input_image
481
- "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
482
- False, # t2v
483
- "", # n_prompt
484
- True, # randomize_seed
485
- 42, # seed
486
- 1, # total_second_length
487
- 9, # latent_window_size
488
- 25, # steps
489
- 1.0, # cfg
490
- 10.0, # gs
491
- 0.0, # rs
492
- 6, # gpu_memory_preservation
493
- True, # use_teacache
494
- 16 # mp4_crf
495
- ],
496
- [
497
- "./img_examples/Example1.png", # input_image
498
- "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
499
- False, # t2v
500
- "", # n_prompt
501
- True, # randomize_seed
502
- 42, # seed
503
- 1, # total_second_length
504
- 9, # latent_window_size
505
- 25, # steps
506
- 1.0, # cfg
507
- 10.0, # gs
508
- 0.0, # rs
509
- 6, # gpu_memory_preservation
510
- True, # use_teacache
511
- 16 # mp4_crf
512
- ],
513
- ],
514
- run_on_click = True,
515
- fn = process,
516
- inputs = ips,
517
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
518
- cache_examples = True,
519
- )
520
- gr.Markdown('## Guide')
521
- gr.Markdown("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
522
-
523
-
524
- def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
525
- global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
526
- input_image_debug_value = input_image_debug_data
527
- prompt_debug_value = prompt_debug_data
528
- total_second_length_debug_value = total_second_length_debug_data
529
- return []
530
-
531
- input_image_debug.upload(
532
- fn=handle_field_debug_change,
533
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
534
- outputs=[]
535
- )
536
-
537
- prompt_debug.change(
538
- fn=handle_field_debug_change,
539
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
540
- outputs=[]
541
- )
542
-
543
- total_second_length_debug.change(
544
- fn=handle_field_debug_change,
545
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
546
- outputs=[]
547
- )
548
-
549
-
550
  block.launch(mcp_server=False)
 
1
+ from diffusers_helper.hf_login import login
2
+
3
+ import os
4
+
5
+ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
+
7
+ import gradio as gr
8
+ import torch
9
+ import traceback
10
+ import einops
11
+ import safetensors.torch as sf
12
+ import numpy as np
13
+ import math
14
+ import random
15
+ import spaces
16
+
17
+ from PIL import Image
18
+ from diffusers import AutoencoderKLHunyuanVideo
19
+ from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
20
+ from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
21
+ from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
22
+ from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
23
+ from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
24
+ from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
25
+ from diffusers_helper.thread_utils import AsyncStream, async_run
26
+ from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
27
+ from transformers import SiglipImageProcessor, SiglipVisionModel
28
+ from diffusers_helper.clip_vision import hf_clip_vision_encode
29
+ from diffusers_helper.bucket_tools import find_nearest_bucket
30
+
31
+
32
+ free_mem_gb = get_cuda_free_memory_gb(gpu)
33
+ high_vram = free_mem_gb > 80
34
+
35
+ print(f'Free VRAM {free_mem_gb} GB')
36
+ print(f'High-VRAM Mode: {high_vram}')
37
+
38
+ text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
39
+ text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
40
+ tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
41
+ tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
42
+ vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
43
+
44
+ feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
45
+ image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
46
+
47
+ transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
48
+
49
+ vae.eval()
50
+ text_encoder.eval()
51
+ text_encoder_2.eval()
52
+ image_encoder.eval()
53
+ transformer.eval()
54
+
55
+ if not high_vram:
56
+ vae.enable_slicing()
57
+ vae.enable_tiling()
58
+
59
+ transformer.high_quality_fp32_output_for_inference = True
60
+ print('transformer.high_quality_fp32_output_for_inference = True')
61
+
62
+ transformer.to(dtype=torch.bfloat16)
63
+ vae.to(dtype=torch.float16)
64
+ image_encoder.to(dtype=torch.float16)
65
+ text_encoder.to(dtype=torch.float16)
66
+ text_encoder_2.to(dtype=torch.float16)
67
+
68
+ vae.requires_grad_(False)
69
+ text_encoder.requires_grad_(False)
70
+ text_encoder_2.requires_grad_(False)
71
+ image_encoder.requires_grad_(False)
72
+ transformer.requires_grad_(False)
73
+
74
+ if not high_vram:
75
+ # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
76
+ DynamicSwapInstaller.install_model(transformer, device=gpu)
77
+ DynamicSwapInstaller.install_model(text_encoder, device=gpu)
78
+ else:
79
+ text_encoder.to(gpu)
80
+ text_encoder_2.to(gpu)
81
+ image_encoder.to(gpu)
82
+ vae.to(gpu)
83
+ transformer.to(gpu)
84
+
85
+ stream = AsyncStream()
86
+
87
+ outputs_folder = './outputs/'
88
+ os.makedirs(outputs_folder, exist_ok=True)
89
+
90
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
91
+
92
+ @torch.no_grad()
93
+ def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
94
+ def encode_prompt(prompt, n_prompt):
95
+ llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
96
+
97
+ if cfg == 1:
98
+ llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
99
+ else:
100
+ llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
101
+
102
+ llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
103
+ llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
104
+
105
+ llama_vec = llama_vec.to(transformer.dtype)
106
+ llama_vec_n = llama_vec_n.to(transformer.dtype)
107
+ clip_l_pooler = clip_l_pooler.to(transformer.dtype)
108
+ clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
109
+ return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
110
+
111
+ total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
112
+ total_latent_sections = int(max(round(total_latent_sections), 1))
113
+
114
+ job_id = generate_timestamp()
115
+
116
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
117
+
118
+ try:
119
+ # Clean GPU
120
+ if not high_vram:
121
+ unload_complete_models(
122
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
123
+ )
124
+
125
+ # Text encoding
126
+
127
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
128
+
129
+ if not high_vram:
130
+ fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
131
+ load_model_as_complete(text_encoder_2, target_device=gpu)
132
+
133
+ prompt_parameters = []
134
+
135
+ for prompt_part in prompt.split(";"):
136
+ prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
137
+ #[llama_vec_part, clip_l_pooler_part, llama_vec_n_part, clip_l_pooler_n_part, llama_attention_mask_part, llama_attention_mask_n_part] = encode_prompt(prompt_part, n_prompt)
138
+
139
+ #[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = [llama_vec_part, clip_l_pooler_part, llama_vec_n_part, clip_l_pooler_n_part, llama_attention_mask_part, llama_attention_mask_n_part]
140
+
141
+
142
+
143
+ #[llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = encode_prompt(prompt, n_prompt)
144
+
145
+ # Processing input image
146
+
147
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
148
+
149
+ H, W, C = input_image.shape
150
+ height, width = find_nearest_bucket(H, W, resolution=640)
151
+ input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
152
+
153
+ Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
154
+
155
+ input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
156
+ input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
157
+
158
+ # VAE encoding
159
+
160
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
161
+
162
+ if not high_vram:
163
+ load_model_as_complete(vae, target_device=gpu)
164
+
165
+ start_latent = vae_encode(input_image_pt, vae)
166
+
167
+ # CLIP Vision
168
+
169
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
170
+
171
+ if not high_vram:
172
+ load_model_as_complete(image_encoder, target_device=gpu)
173
+
174
+ image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
175
+ image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
176
+
177
+ # Dtype
178
+
179
+ image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
180
+
181
+ # Sampling
182
+
183
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
184
+
185
+ rnd = torch.Generator("cpu").manual_seed(seed)
186
+
187
+ history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
188
+ history_pixels = None
189
+
190
+ history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
191
+ total_generated_latent_frames = 1
192
+
193
+ for section_index in range(total_latent_sections):
194
+ if stream.input_queue.top() == 'end':
195
+ stream.output_queue.push(('end', None))
196
+ return
197
+
198
+ print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
199
+
200
+ if len(prompt_parameters) > 0:
201
+ [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0)
202
+
203
+ if not high_vram:
204
+ unload_complete_models()
205
+ move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
206
+
207
+ if use_teacache:
208
+ transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
209
+ else:
210
+ transformer.initialize_teacache(enable_teacache=False)
211
+
212
+ def callback(d):
213
+ preview = d['denoised']
214
+ preview = vae_decode_fake(preview)
215
+
216
+ preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
217
+ preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
218
+
219
+ if stream.input_queue.top() == 'end':
220
+ stream.output_queue.push(('end', None))
221
+ raise KeyboardInterrupt('User ends the task.')
222
+
223
+ current_step = d['i'] + 1
224
+ percentage = int(100.0 * current_step / steps)
225
+ hint = f'Sampling {current_step}/{steps}'
226
+ desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
227
+ stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
228
+ return
229
+
230
+ indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
231
+ clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
232
+ clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
233
+
234
+ clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
235
+ clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
236
+
237
+ generated_latents = sample_hunyuan(
238
+ transformer=transformer,
239
+ sampler='unipc',
240
+ width=width,
241
+ height=height,
242
+ frames=latent_window_size * 4 - 3,
243
+ real_guidance_scale=cfg,
244
+ distilled_guidance_scale=gs,
245
+ guidance_rescale=rs,
246
+ # shift=3.0,
247
+ num_inference_steps=steps,
248
+ generator=rnd,
249
+ prompt_embeds=llama_vec,
250
+ prompt_embeds_mask=llama_attention_mask,
251
+ prompt_poolers=clip_l_pooler,
252
+ negative_prompt_embeds=llama_vec_n,
253
+ negative_prompt_embeds_mask=llama_attention_mask_n,
254
+ negative_prompt_poolers=clip_l_pooler_n,
255
+ device=gpu,
256
+ dtype=torch.bfloat16,
257
+ image_embeddings=image_encoder_last_hidden_state,
258
+ latent_indices=latent_indices,
259
+ clean_latents=clean_latents,
260
+ clean_latent_indices=clean_latent_indices,
261
+ clean_latents_2x=clean_latents_2x,
262
+ clean_latent_2x_indices=clean_latent_2x_indices,
263
+ clean_latents_4x=clean_latents_4x,
264
+ clean_latent_4x_indices=clean_latent_4x_indices,
265
+ callback=callback,
266
+ )
267
+
268
+ total_generated_latent_frames += int(generated_latents.shape[2])
269
+ history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
270
+
271
+ if not high_vram:
272
+ offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
273
+ load_model_as_complete(vae, target_device=gpu)
274
+
275
+ real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
276
+
277
+ if history_pixels is None:
278
+ history_pixels = vae_decode(real_history_latents, vae).cpu()
279
+ else:
280
+ section_latent_frames = latent_window_size * 2
281
+ overlapped_frames = latent_window_size * 4 - 3
282
+
283
+ current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
284
+ history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
285
+
286
+ if not high_vram:
287
+ unload_complete_models()
288
+
289
+ output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
290
+
291
+ save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
292
+
293
+ print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
294
+
295
+ stream.output_queue.push(('file', output_filename))
296
+ except:
297
+ traceback.print_exc()
298
+
299
+ if not high_vram:
300
+ unload_complete_models(
301
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
302
+ )
303
+
304
+ stream.output_queue.push(('end', None))
305
+ return
306
+
307
+ def get_duration(input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
308
+ global total_second_length_debug_value
309
+
310
+ if total_second_length_debug_value is not None:
311
+ return min(total_second_length_debug_value * 60, 600)
312
+ return total_second_length * 60
313
+
314
+
315
+ @spaces.GPU(duration=get_duration)
316
+ def process(input_image, prompt,
317
+ t2v=False,
318
+ n_prompt="",
319
+ randomize_seed=True,
320
+ seed=31337,
321
+ total_second_length=5,
322
+ latent_window_size=9,
323
+ steps=25,
324
+ cfg=1.0,
325
+ gs=10.0,
326
+ rs=0.0,
327
+ gpu_memory_preservation=6,
328
+ use_teacache=True,
329
+ mp4_crf=16
330
+ ):
331
+ global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
332
+
333
+ if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
334
+ print("Debug mode")
335
+ input_image = input_image_debug_value
336
+ prompt = prompt_debug_value
337
+ total_second_length = total_second_length_debug_value
338
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
339
+
340
+ if randomize_seed:
341
+ seed = random.randint(0, np.iinfo(np.int32).max)
342
+
343
+ # assert input_image is not None, 'No input image!'
344
+ if t2v:
345
+ default_height, default_width = 640, 640
346
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
347
+ print("No input image provided. Using a blank white image.")
348
+
349
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
350
+
351
+ stream = AsyncStream()
352
+
353
+ async_run(worker, input_image, prompt, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
354
+
355
+ output_filename = None
356
+
357
+ while True:
358
+ flag, data = stream.output_queue.next()
359
+
360
+ if flag == 'file':
361
+ output_filename = data
362
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
363
+
364
+ if flag == 'progress':
365
+ preview, desc, html = data
366
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
367
+
368
+ if flag == 'end':
369
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
370
+ break
371
+
372
+
373
+ def end_process():
374
+ stream.input_queue.push('end')
375
+
376
+
377
+ css = make_progress_bar_css()
378
+ block = gr.Blocks(css=css).queue()
379
+ with block:
380
+ if torch.cuda.device_count() == 0:
381
+ with gr.Row():
382
+ gr.HTML("""
383
+ <p style="background-color: red;"><big><big><big><b>⚠️To use FramePack, <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR?duplicate=true">duplicate this space</a> and set a GPU with 30 GB VRAM.</b>
384
+
385
+ You can't use FramePack directly here because this space runs on a CPU, which is not enough for FramePack. Please provide <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR/discussions/new">feedback</a> if you have issues.
386
+ </big></big></big></p>
387
+ """)
388
+ gr.Markdown('# FramePack-F1')
389
+ gr.Markdown(f"""### Video diffusion, but feels like image diffusion
390
+ *FramePack F1 - a FramePack model that only predicts future frames from history frames*
391
+ ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
392
+ adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
393
+ """)
394
+ with gr.Row():
395
+ with gr.Column():
396
+ input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
397
+ prompt = gr.Textbox(label="Prompt", value='')
398
+ t2v = gr.Checkbox(label="do text-to-video", value=False)
399
+
400
+ with gr.Row():
401
+ start_button = gr.Button(value="Start Generation", variant="primary")
402
+ end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
403
+
404
+ total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
405
+ with gr.Accordion("Advanced settings", open=False):
406
+ use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
407
+
408
+ n_prompt = gr.Textbox(label="Negative Prompt", value="") # Not used
409
+ randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different')
410
+ seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True)
411
+
412
+
413
+ latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
414
+ steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
415
+
416
+ cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
417
+ gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
418
+ rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
419
+
420
+ gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
421
+
422
+ mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
423
+
424
+ with gr.Accordion("Debug", open=False):
425
+ input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
426
+ prompt_debug = gr.Textbox(label="Prompt Debug", value='')
427
+ total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
428
+
429
+ with gr.Column():
430
+ preview_image = gr.Image(label="Next Latents", height=200, visible=False)
431
+ result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
432
+ progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
433
+ progress_bar = gr.HTML('', elem_classes='no-generating-animation')
434
+
435
+ gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
436
+
437
+ ips = [input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
438
+ start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
439
+ end_button.click(fn=end_process)
440
+
441
+ with gr.Row(visible=False):
442
+ gr.Examples(
443
+ examples = [
444
+ [
445
+ "./img_examples/Example1.png", # input_image
446
+ "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
447
+ False, # t2v
448
+ "", # n_prompt
449
+ True, # randomize_seed
450
+ 42, # seed
451
+ 1, # total_second_length
452
+ 9, # latent_window_size
453
+ 25, # steps
454
+ 1.0, # cfg
455
+ 10.0, # gs
456
+ 0.0, # rs
457
+ 6, # gpu_memory_preservation
458
+ True, # use_teacache
459
+ 16 # mp4_crf
460
+ ],
461
+ [
462
+ "./img_examples/Example1.png", # input_image
463
+ "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
464
+ False, # t2v
465
+ "", # n_prompt
466
+ True, # randomize_seed
467
+ 42, # seed
468
+ 1, # total_second_length
469
+ 9, # latent_window_size
470
+ 25, # steps
471
+ 1.0, # cfg
472
+ 10.0, # gs
473
+ 0.0, # rs
474
+ 6, # gpu_memory_preservation
475
+ True, # use_teacache
476
+ 16 # mp4_crf
477
+ ],
478
+ [
479
+ "./img_examples/Example1.png", # input_image
480
+ "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
481
+ False, # t2v
482
+ "", # n_prompt
483
+ True, # randomize_seed
484
+ 42, # seed
485
+ 1, # total_second_length
486
+ 9, # latent_window_size
487
+ 25, # steps
488
+ 1.0, # cfg
489
+ 10.0, # gs
490
+ 0.0, # rs
491
+ 6, # gpu_memory_preservation
492
+ True, # use_teacache
493
+ 16 # mp4_crf
494
+ ],
495
+ [
496
+ "./img_examples/Example1.png", # input_image
497
+ "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
498
+ False, # t2v
499
+ "", # n_prompt
500
+ True, # randomize_seed
501
+ 42, # seed
502
+ 1, # total_second_length
503
+ 9, # latent_window_size
504
+ 25, # steps
505
+ 1.0, # cfg
506
+ 10.0, # gs
507
+ 0.0, # rs
508
+ 6, # gpu_memory_preservation
509
+ True, # use_teacache
510
+ 16 # mp4_crf
511
+ ],
512
+ ],
513
+ run_on_click = True,
514
+ fn = process,
515
+ inputs = ips,
516
+ outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
517
+ cache_examples = True,
518
+ )
519
+ gr.Markdown('## Guide')
520
+ gr.Markdown("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
521
+
522
+
523
+ def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
524
+ global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
525
+ input_image_debug_value = input_image_debug_data
526
+ prompt_debug_value = prompt_debug_data
527
+ total_second_length_debug_value = total_second_length_debug_data
528
+ return []
529
+
530
+ input_image_debug.upload(
531
+ fn=handle_field_debug_change,
532
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
533
+ outputs=[]
534
+ )
535
+
536
+ prompt_debug.change(
537
+ fn=handle_field_debug_change,
538
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
539
+ outputs=[]
540
+ )
541
+
542
+ total_second_length_debug.change(
543
+ fn=handle_field_debug_change,
544
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
545
+ outputs=[]
546
+ )
547
+
548
+
 
549
  block.launch(mcp_server=False)