Fabrice-TIERCELIN commited on
Commit
0fe9d2f
·
verified ·
1 Parent(s): 0cbd9f8
Files changed (1) hide show
  1. app.py +548 -547
app.py CHANGED
@@ -1,548 +1,549 @@
1
- from diffusers_helper.hf_login import login
2
-
3
- import os
4
-
5
- os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
-
7
- import gradio as gr
8
- import torch
9
- import traceback
10
- import einops
11
- import safetensors.torch as sf
12
- import numpy as np
13
- import math
14
- import random
15
- import spaces
16
-
17
- from PIL import Image
18
- from diffusers import AutoencoderKLHunyuanVideo
19
- from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
20
- from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
21
- from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
22
- from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
23
- from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
24
- from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
25
- from diffusers_helper.thread_utils import AsyncStream, async_run
26
- from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
27
- from transformers import SiglipImageProcessor, SiglipVisionModel
28
- from diffusers_helper.clip_vision import hf_clip_vision_encode
29
- from diffusers_helper.bucket_tools import find_nearest_bucket
30
-
31
-
32
- free_mem_gb = get_cuda_free_memory_gb(gpu)
33
- high_vram = free_mem_gb > 80
34
-
35
- print(f'Free VRAM {free_mem_gb} GB')
36
- print(f'High-VRAM Mode: {high_vram}')
37
-
38
- text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
39
- text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
40
- tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
41
- tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
42
- vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
43
-
44
- feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
45
- image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
46
-
47
- transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
48
-
49
- vae.eval()
50
- text_encoder.eval()
51
- text_encoder_2.eval()
52
- image_encoder.eval()
53
- transformer.eval()
54
-
55
- if not high_vram:
56
- vae.enable_slicing()
57
- vae.enable_tiling()
58
-
59
- transformer.high_quality_fp32_output_for_inference = True
60
- print('transformer.high_quality_fp32_output_for_inference = True')
61
-
62
- transformer.to(dtype=torch.bfloat16)
63
- vae.to(dtype=torch.float16)
64
- image_encoder.to(dtype=torch.float16)
65
- text_encoder.to(dtype=torch.float16)
66
- text_encoder_2.to(dtype=torch.float16)
67
-
68
- vae.requires_grad_(False)
69
- text_encoder.requires_grad_(False)
70
- text_encoder_2.requires_grad_(False)
71
- image_encoder.requires_grad_(False)
72
- transformer.requires_grad_(False)
73
-
74
- if not high_vram:
75
- # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
76
- DynamicSwapInstaller.install_model(transformer, device=gpu)
77
- DynamicSwapInstaller.install_model(text_encoder, device=gpu)
78
- else:
79
- text_encoder.to(gpu)
80
- text_encoder_2.to(gpu)
81
- image_encoder.to(gpu)
82
- vae.to(gpu)
83
- transformer.to(gpu)
84
-
85
- stream = AsyncStream()
86
-
87
- outputs_folder = './outputs/'
88
- os.makedirs(outputs_folder, exist_ok=True)
89
-
90
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
91
-
92
- @torch.no_grad()
93
- def worker(input_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
94
- def encode_prompt(prompt, n_prompt):
95
- llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
96
-
97
- if cfg == 1:
98
- llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
99
- else:
100
- llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
101
-
102
- llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
103
- llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
104
-
105
- llama_vec = llama_vec.to(transformer.dtype)
106
- llama_vec_n = llama_vec_n.to(transformer.dtype)
107
- clip_l_pooler = clip_l_pooler.to(transformer.dtype)
108
- clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
109
- return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
110
-
111
- total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
112
- total_latent_sections = int(max(round(total_latent_sections), 1))
113
-
114
- job_id = generate_timestamp()
115
-
116
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
117
-
118
- try:
119
- # Clean GPU
120
- if not high_vram:
121
- unload_complete_models(
122
- text_encoder, text_encoder_2, image_encoder, vae, transformer
123
- )
124
-
125
- # Text encoding
126
-
127
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
128
-
129
- if not high_vram:
130
- fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
131
- load_model_as_complete(text_encoder_2, target_device=gpu)
132
-
133
- prompt_parameters = []
134
-
135
- for prompt_part in prompts:
136
- prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
137
-
138
- # Processing input image
139
-
140
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
141
-
142
- H, W, C = input_image.shape
143
- height, width = find_nearest_bucket(H, W, resolution=640)
144
- input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
145
-
146
- Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
147
-
148
- input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
149
- input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
150
-
151
- # VAE encoding
152
-
153
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
154
-
155
- if not high_vram:
156
- load_model_as_complete(vae, target_device=gpu)
157
-
158
- start_latent = vae_encode(input_image_pt, vae)
159
-
160
- # CLIP Vision
161
-
162
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
163
-
164
- if not high_vram:
165
- load_model_as_complete(image_encoder, target_device=gpu)
166
-
167
- image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
168
- image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
169
-
170
- # Dtype
171
-
172
- image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
173
-
174
- # Sampling
175
-
176
- stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
177
-
178
- rnd = torch.Generator("cpu").manual_seed(seed)
179
-
180
- history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
181
- history_pixels = None
182
-
183
- history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
184
- total_generated_latent_frames = 1
185
-
186
- for section_index in range(total_latent_sections):
187
- if stream.input_queue.top() == 'end':
188
- stream.output_queue.push(('end', None))
189
- return
190
-
191
- print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
192
-
193
- if len(prompt_parameters) > 0:
194
- [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0)
195
-
196
- if not high_vram:
197
- unload_complete_models()
198
- move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
199
-
200
- if use_teacache:
201
- transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
202
- else:
203
- transformer.initialize_teacache(enable_teacache=False)
204
-
205
- def callback(d):
206
- preview = d['denoised']
207
- preview = vae_decode_fake(preview)
208
-
209
- preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
210
- preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
211
-
212
- if stream.input_queue.top() == 'end':
213
- stream.output_queue.push(('end', None))
214
- raise KeyboardInterrupt('User ends the task.')
215
-
216
- current_step = d['i'] + 1
217
- percentage = int(100.0 * current_step / steps)
218
- hint = f'Sampling {current_step}/{steps}'
219
- desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
220
- stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
221
- return
222
-
223
- indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
224
- clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
225
- clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
226
-
227
- clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
228
- clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
229
-
230
- generated_latents = sample_hunyuan(
231
- transformer=transformer,
232
- sampler='unipc',
233
- width=width,
234
- height=height,
235
- frames=latent_window_size * 4 - 3,
236
- real_guidance_scale=cfg,
237
- distilled_guidance_scale=gs,
238
- guidance_rescale=rs,
239
- # shift=3.0,
240
- num_inference_steps=steps,
241
- generator=rnd,
242
- prompt_embeds=llama_vec,
243
- prompt_embeds_mask=llama_attention_mask,
244
- prompt_poolers=clip_l_pooler,
245
- negative_prompt_embeds=llama_vec_n,
246
- negative_prompt_embeds_mask=llama_attention_mask_n,
247
- negative_prompt_poolers=clip_l_pooler_n,
248
- device=gpu,
249
- dtype=torch.bfloat16,
250
- image_embeddings=image_encoder_last_hidden_state,
251
- latent_indices=latent_indices,
252
- clean_latents=clean_latents,
253
- clean_latent_indices=clean_latent_indices,
254
- clean_latents_2x=clean_latents_2x,
255
- clean_latent_2x_indices=clean_latent_2x_indices,
256
- clean_latents_4x=clean_latents_4x,
257
- clean_latent_4x_indices=clean_latent_4x_indices,
258
- callback=callback,
259
- )
260
-
261
- total_generated_latent_frames += int(generated_latents.shape[2])
262
- history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
263
-
264
- if not high_vram:
265
- offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
266
- load_model_as_complete(vae, target_device=gpu)
267
-
268
- real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
269
-
270
- if history_pixels is None:
271
- history_pixels = vae_decode(real_history_latents, vae).cpu()
272
- else:
273
- section_latent_frames = latent_window_size * 2
274
- overlapped_frames = latent_window_size * 4 - 3
275
-
276
- current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
277
- history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
278
-
279
- if not high_vram:
280
- unload_complete_models()
281
-
282
- output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
283
-
284
- save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
285
-
286
- print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
287
-
288
- stream.output_queue.push(('file', output_filename))
289
- except:
290
- traceback.print_exc()
291
-
292
- if not high_vram:
293
- unload_complete_models(
294
- text_encoder, text_encoder_2, image_encoder, vae, transformer
295
- )
296
-
297
- stream.output_queue.push(('end', None))
298
- return
299
-
300
- def get_duration(input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
301
- global total_second_length_debug_value
302
-
303
- if total_second_length_debug_value is not None:
304
- return min(total_second_length_debug_value * 60, 600)
305
- return total_second_length * 60
306
-
307
-
308
- @spaces.GPU(duration=get_duration)
309
- def process(input_image, prompt,
310
- t2v=False,
311
- n_prompt="",
312
- randomize_seed=True,
313
- seed=31337,
314
- total_second_length=5,
315
- latent_window_size=9,
316
- steps=25,
317
- cfg=1.0,
318
- gs=10.0,
319
- rs=0.0,
320
- gpu_memory_preservation=6,
321
- use_teacache=True,
322
- mp4_crf=16
323
- ):
324
- global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
325
-
326
- if torch.cuda.device_count() == 0:
327
- gr.Warning('Set this space to GPU config to make it work.')
328
- return None, None, None, None, None, None
329
-
330
- if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
331
- print("Debug mode")
332
- input_image = input_image_debug_value
333
- prompt = prompt_debug_value
334
- total_second_length = total_second_length_debug_value
335
- input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
336
-
337
- if randomize_seed:
338
- seed = random.randint(0, np.iinfo(np.int32).max)
339
-
340
- prompts = prompt.split(";")
341
-
342
- # assert input_image is not None, 'No input image!'
343
- if t2v:
344
- default_height, default_width = 640, 640
345
- input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
346
- print("No input image provided. Using a blank white image.")
347
-
348
- yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
349
-
350
- stream = AsyncStream()
351
-
352
- async_run(worker, input_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
353
-
354
- output_filename = None
355
-
356
- while True:
357
- flag, data = stream.output_queue.next()
358
-
359
- if flag == 'file':
360
- output_filename = data
361
- yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
362
-
363
- if flag == 'progress':
364
- preview, desc, html = data
365
- yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
366
-
367
- if flag == 'end':
368
- yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
369
- break
370
-
371
-
372
- def end_process():
373
- stream.input_queue.push('end')
374
-
375
-
376
- css = make_progress_bar_css()
377
- block = gr.Blocks(css=css).queue()
378
- with block:
379
- if torch.cuda.device_count() == 0:
380
- with gr.Row():
381
- gr.HTML("""
382
- <p style="background-color: red;"><big><big><big><b>⚠️To use FramePack, <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR?duplicate=true">duplicate this space</a> and set a GPU with 30 GB VRAM.</b>
383
-
384
- You can't use FramePack directly here because this space runs on a CPU, which is not enough for FramePack. Please provide <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR/discussions/new">feedback</a> if you have issues.
385
- </big></big></big></p>
386
- """)
387
- gr.Markdown('# FramePack-F1')
388
- gr.Markdown(f"""### Video diffusion, but feels like image diffusion
389
- *FramePack F1 - a FramePack model that only predicts future frames from history frames*
390
- ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
391
- adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
392
- """)
393
- with gr.Row():
394
- with gr.Column():
395
- input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
396
- prompt = gr.Textbox(label="Prompt", value='')
397
- t2v = gr.Checkbox(label="do text-to-video", value=False)
398
-
399
- with gr.Row():
400
- start_button = gr.Button(value="Start Generation", variant="primary")
401
- end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
402
-
403
- total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
404
- with gr.Accordion("Advanced settings", open=False):
405
- use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
406
-
407
- n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, unrealistic position, blurred, blurry") # Not used
408
- randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different')
409
- seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True)
410
-
411
-
412
- latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
413
- steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
414
-
415
- cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
416
- gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
417
- rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
418
-
419
- gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
420
-
421
- mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
422
-
423
- with gr.Accordion("Debug", open=False):
424
- input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
425
- prompt_debug = gr.Textbox(label="Prompt Debug", value='')
426
- total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
427
-
428
- with gr.Column():
429
- preview_image = gr.Image(label="Next Latents", height=200, visible=False)
430
- result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
431
- progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
432
- progress_bar = gr.HTML('', elem_classes='no-generating-animation')
433
-
434
- gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
435
-
436
- ips = [input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
437
- start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
438
- end_button.click(fn=end_process)
439
-
440
- with gr.Row(visible=False):
441
- gr.Examples(
442
- examples = [
443
- [
444
- "./img_examples/Example1.png", # input_image
445
- "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
446
- False, # t2v
447
- "Missing arm, unrealistic position, blurred, blurry", # n_prompt
448
- True, # randomize_seed
449
- 42, # seed
450
- 1, # total_second_length
451
- 9, # latent_window_size
452
- 25, # steps
453
- 1.0, # cfg
454
- 10.0, # gs
455
- 0.0, # rs
456
- 6, # gpu_memory_preservation
457
- True, # use_teacache
458
- 16 # mp4_crf
459
- ],
460
- [
461
- "./img_examples/Example1.png", # input_image
462
- "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
463
- False, # t2v
464
- "Missing arm, unrealistic position, blurred, blurry", # n_prompt
465
- True, # randomize_seed
466
- 42, # seed
467
- 1, # total_second_length
468
- 9, # latent_window_size
469
- 25, # steps
470
- 1.0, # cfg
471
- 10.0, # gs
472
- 0.0, # rs
473
- 6, # gpu_memory_preservation
474
- True, # use_teacache
475
- 16 # mp4_crf
476
- ],
477
- [
478
- "./img_examples/Example1.png", # input_image
479
- "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
480
- False, # t2v
481
- "Missing arm, unrealistic position, blurred, blurry", # n_prompt
482
- True, # randomize_seed
483
- 42, # seed
484
- 1, # total_second_length
485
- 9, # latent_window_size
486
- 25, # steps
487
- 1.0, # cfg
488
- 10.0, # gs
489
- 0.0, # rs
490
- 6, # gpu_memory_preservation
491
- True, # use_teacache
492
- 16 # mp4_crf
493
- ],
494
- [
495
- "./img_examples/Example1.png", # input_image
496
- "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
497
- False, # t2v
498
- "Missing arm, unrealistic position, blurred, blurry", # n_prompt
499
- True, # randomize_seed
500
- 42, # seed
501
- 1, # total_second_length
502
- 9, # latent_window_size
503
- 25, # steps
504
- 1.0, # cfg
505
- 10.0, # gs
506
- 0.0, # rs
507
- 6, # gpu_memory_preservation
508
- True, # use_teacache
509
- 16 # mp4_crf
510
- ],
511
- ],
512
- run_on_click = True,
513
- fn = process,
514
- inputs = ips,
515
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
516
- cache_examples = True,
517
- )
518
- gr.Markdown('## Guide')
519
- gr.Markdown("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
520
-
521
-
522
- def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
523
- global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
524
- input_image_debug_value = input_image_debug_data
525
- prompt_debug_value = prompt_debug_data
526
- total_second_length_debug_value = total_second_length_debug_data
527
- return []
528
-
529
- input_image_debug.upload(
530
- fn=handle_field_debug_change,
531
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
532
- outputs=[]
533
- )
534
-
535
- prompt_debug.change(
536
- fn=handle_field_debug_change,
537
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
538
- outputs=[]
539
- )
540
-
541
- total_second_length_debug.change(
542
- fn=handle_field_debug_change,
543
- inputs=[input_image_debug, prompt_debug, total_second_length_debug],
544
- outputs=[]
545
- )
546
-
547
-
 
548
  block.launch(mcp_server=False)
 
1
+ from diffusers_helper.hf_login import login
2
+
3
+ import os
4
+
5
+ os.environ['HF_HOME'] = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')))
6
+
7
+ import gradio as gr
8
+ import torch
9
+ import traceback
10
+ import einops
11
+ import safetensors.torch as sf
12
+ import numpy as np
13
+ import math
14
+ import random
15
+ import spaces
16
+
17
+ from PIL import Image
18
+ from diffusers import AutoencoderKLHunyuanVideo
19
+ from transformers import LlamaModel, CLIPTextModel, LlamaTokenizerFast, CLIPTokenizer
20
+ from diffusers_helper.hunyuan import encode_prompt_conds, vae_decode, vae_encode, vae_decode_fake
21
+ from diffusers_helper.utils import save_bcthw_as_mp4, crop_or_pad_yield_mask, soft_append_bcthw, resize_and_center_crop, state_dict_weighted_merge, state_dict_offset_merge, generate_timestamp
22
+ from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked
23
+ from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan
24
+ from diffusers_helper.memory import cpu, gpu, get_cuda_free_memory_gb, move_model_to_device_with_memory_preservation, offload_model_from_device_for_memory_preservation, fake_diffusers_current_device, DynamicSwapInstaller, unload_complete_models, load_model_as_complete
25
+ from diffusers_helper.thread_utils import AsyncStream, async_run
26
+ from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html
27
+ from transformers import SiglipImageProcessor, SiglipVisionModel
28
+ from diffusers_helper.clip_vision import hf_clip_vision_encode
29
+ from diffusers_helper.bucket_tools import find_nearest_bucket
30
+
31
+
32
+ if torch.cuda.device_count() > 0:
33
+ free_mem_gb = get_cuda_free_memory_gb(gpu)
34
+ high_vram = free_mem_gb > 80
35
+
36
+ print(f'Free VRAM {free_mem_gb} GB')
37
+ print(f'High-VRAM Mode: {high_vram}')
38
+
39
+ text_encoder = LlamaModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder', torch_dtype=torch.float16).cpu()
40
+ text_encoder_2 = CLIPTextModel.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='text_encoder_2', torch_dtype=torch.float16).cpu()
41
+ tokenizer = LlamaTokenizerFast.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer')
42
+ tokenizer_2 = CLIPTokenizer.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='tokenizer_2')
43
+ vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanVideo", subfolder='vae', torch_dtype=torch.float16).cpu()
44
+
45
+ feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
46
+ image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
47
+
48
+ transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
49
+
50
+ vae.eval()
51
+ text_encoder.eval()
52
+ text_encoder_2.eval()
53
+ image_encoder.eval()
54
+ transformer.eval()
55
+
56
+ if not high_vram:
57
+ vae.enable_slicing()
58
+ vae.enable_tiling()
59
+
60
+ transformer.high_quality_fp32_output_for_inference = True
61
+ print('transformer.high_quality_fp32_output_for_inference = True')
62
+
63
+ transformer.to(dtype=torch.bfloat16)
64
+ vae.to(dtype=torch.float16)
65
+ image_encoder.to(dtype=torch.float16)
66
+ text_encoder.to(dtype=torch.float16)
67
+ text_encoder_2.to(dtype=torch.float16)
68
+
69
+ vae.requires_grad_(False)
70
+ text_encoder.requires_grad_(False)
71
+ text_encoder_2.requires_grad_(False)
72
+ image_encoder.requires_grad_(False)
73
+ transformer.requires_grad_(False)
74
+
75
+ if not high_vram:
76
+ # DynamicSwapInstaller is same as huggingface's enable_sequential_offload but 3x faster
77
+ DynamicSwapInstaller.install_model(transformer, device=gpu)
78
+ DynamicSwapInstaller.install_model(text_encoder, device=gpu)
79
+ else:
80
+ text_encoder.to(gpu)
81
+ text_encoder_2.to(gpu)
82
+ image_encoder.to(gpu)
83
+ vae.to(gpu)
84
+ transformer.to(gpu)
85
+
86
+ stream = AsyncStream()
87
+
88
+ outputs_folder = './outputs/'
89
+ os.makedirs(outputs_folder, exist_ok=True)
90
+
91
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
92
+
93
+ @torch.no_grad()
94
+ def worker(input_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
95
+ def encode_prompt(prompt, n_prompt):
96
+ llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
97
+
98
+ if cfg == 1:
99
+ llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler)
100
+ else:
101
+ llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2)
102
+
103
+ llama_vec, llama_attention_mask = crop_or_pad_yield_mask(llama_vec, length=512)
104
+ llama_vec_n, llama_attention_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512)
105
+
106
+ llama_vec = llama_vec.to(transformer.dtype)
107
+ llama_vec_n = llama_vec_n.to(transformer.dtype)
108
+ clip_l_pooler = clip_l_pooler.to(transformer.dtype)
109
+ clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype)
110
+ return [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n]
111
+
112
+ total_latent_sections = (total_second_length * 30) / (latent_window_size * 4)
113
+ total_latent_sections = int(max(round(total_latent_sections), 1))
114
+
115
+ job_id = generate_timestamp()
116
+
117
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...'))))
118
+
119
+ try:
120
+ # Clean GPU
121
+ if not high_vram:
122
+ unload_complete_models(
123
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
124
+ )
125
+
126
+ # Text encoding
127
+
128
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...'))))
129
+
130
+ if not high_vram:
131
+ fake_diffusers_current_device(text_encoder, gpu) # since we only encode one text - that is one model move and one encode, offload is same time consumption since it is also one load and one encode.
132
+ load_model_as_complete(text_encoder_2, target_device=gpu)
133
+
134
+ prompt_parameters = []
135
+
136
+ for prompt_part in prompts:
137
+ prompt_parameters.append(encode_prompt(prompt_part, n_prompt))
138
+
139
+ # Processing input image
140
+
141
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...'))))
142
+
143
+ H, W, C = input_image.shape
144
+ height, width = find_nearest_bucket(H, W, resolution=640)
145
+ input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height)
146
+
147
+ Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png'))
148
+
149
+ input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1
150
+ input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None]
151
+
152
+ # VAE encoding
153
+
154
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...'))))
155
+
156
+ if not high_vram:
157
+ load_model_as_complete(vae, target_device=gpu)
158
+
159
+ start_latent = vae_encode(input_image_pt, vae)
160
+
161
+ # CLIP Vision
162
+
163
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...'))))
164
+
165
+ if not high_vram:
166
+ load_model_as_complete(image_encoder, target_device=gpu)
167
+
168
+ image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder)
169
+ image_encoder_last_hidden_state = image_encoder_output.last_hidden_state
170
+
171
+ # Dtype
172
+
173
+ image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype)
174
+
175
+ # Sampling
176
+
177
+ stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...'))))
178
+
179
+ rnd = torch.Generator("cpu").manual_seed(seed)
180
+
181
+ history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu()
182
+ history_pixels = None
183
+
184
+ history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2)
185
+ total_generated_latent_frames = 1
186
+
187
+ for section_index in range(total_latent_sections):
188
+ if stream.input_queue.top() == 'end':
189
+ stream.output_queue.push(('end', None))
190
+ return
191
+
192
+ print(f'section_index = {section_index}, total_latent_sections = {total_latent_sections}')
193
+
194
+ if len(prompt_parameters) > 0:
195
+ [llama_vec, clip_l_pooler, llama_vec_n, clip_l_pooler_n, llama_attention_mask, llama_attention_mask_n] = prompt_parameters.pop(0)
196
+
197
+ if not high_vram:
198
+ unload_complete_models()
199
+ move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation)
200
+
201
+ if use_teacache:
202
+ transformer.initialize_teacache(enable_teacache=True, num_steps=steps)
203
+ else:
204
+ transformer.initialize_teacache(enable_teacache=False)
205
+
206
+ def callback(d):
207
+ preview = d['denoised']
208
+ preview = vae_decode_fake(preview)
209
+
210
+ preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8)
211
+ preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c')
212
+
213
+ if stream.input_queue.top() == 'end':
214
+ stream.output_queue.push(('end', None))
215
+ raise KeyboardInterrupt('User ends the task.')
216
+
217
+ current_step = d['i'] + 1
218
+ percentage = int(100.0 * current_step / steps)
219
+ hint = f'Sampling {current_step}/{steps}'
220
+ desc = f'Total generated frames: {int(max(0, total_generated_latent_frames * 4 - 3))}, Video length: {max(0, (total_generated_latent_frames * 4 - 3) / 30) :.2f} seconds (FPS-30). The video is being extended now ...'
221
+ stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint))))
222
+ return
223
+
224
+ indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0)
225
+ clean_latent_indices_start, clean_latent_4x_indices, clean_latent_2x_indices, clean_latent_1x_indices, latent_indices = indices.split([1, 16, 2, 1, latent_window_size], dim=1)
226
+ clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1)
227
+
228
+ clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -sum([16, 2, 1]):, :, :].split([16, 2, 1], dim=2)
229
+ clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2)
230
+
231
+ generated_latents = sample_hunyuan(
232
+ transformer=transformer,
233
+ sampler='unipc',
234
+ width=width,
235
+ height=height,
236
+ frames=latent_window_size * 4 - 3,
237
+ real_guidance_scale=cfg,
238
+ distilled_guidance_scale=gs,
239
+ guidance_rescale=rs,
240
+ # shift=3.0,
241
+ num_inference_steps=steps,
242
+ generator=rnd,
243
+ prompt_embeds=llama_vec,
244
+ prompt_embeds_mask=llama_attention_mask,
245
+ prompt_poolers=clip_l_pooler,
246
+ negative_prompt_embeds=llama_vec_n,
247
+ negative_prompt_embeds_mask=llama_attention_mask_n,
248
+ negative_prompt_poolers=clip_l_pooler_n,
249
+ device=gpu,
250
+ dtype=torch.bfloat16,
251
+ image_embeddings=image_encoder_last_hidden_state,
252
+ latent_indices=latent_indices,
253
+ clean_latents=clean_latents,
254
+ clean_latent_indices=clean_latent_indices,
255
+ clean_latents_2x=clean_latents_2x,
256
+ clean_latent_2x_indices=clean_latent_2x_indices,
257
+ clean_latents_4x=clean_latents_4x,
258
+ clean_latent_4x_indices=clean_latent_4x_indices,
259
+ callback=callback,
260
+ )
261
+
262
+ total_generated_latent_frames += int(generated_latents.shape[2])
263
+ history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2)
264
+
265
+ if not high_vram:
266
+ offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8)
267
+ load_model_as_complete(vae, target_device=gpu)
268
+
269
+ real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :]
270
+
271
+ if history_pixels is None:
272
+ history_pixels = vae_decode(real_history_latents, vae).cpu()
273
+ else:
274
+ section_latent_frames = latent_window_size * 2
275
+ overlapped_frames = latent_window_size * 4 - 3
276
+
277
+ current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu()
278
+ history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames)
279
+
280
+ if not high_vram:
281
+ unload_complete_models()
282
+
283
+ output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4')
284
+
285
+ save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=mp4_crf)
286
+
287
+ print(f'Decoded. Current latent shape {real_history_latents.shape}; pixel shape {history_pixels.shape}')
288
+
289
+ stream.output_queue.push(('file', output_filename))
290
+ except:
291
+ traceback.print_exc()
292
+
293
+ if not high_vram:
294
+ unload_complete_models(
295
+ text_encoder, text_encoder_2, image_encoder, vae, transformer
296
+ )
297
+
298
+ stream.output_queue.push(('end', None))
299
+ return
300
+
301
+ def get_duration(input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
302
+ global total_second_length_debug_value
303
+
304
+ if total_second_length_debug_value is not None:
305
+ return min(total_second_length_debug_value * 60, 600)
306
+ return total_second_length * 60
307
+
308
+
309
+ @spaces.GPU(duration=get_duration)
310
+ def process(input_image, prompt,
311
+ t2v=False,
312
+ n_prompt="",
313
+ randomize_seed=True,
314
+ seed=31337,
315
+ total_second_length=5,
316
+ latent_window_size=9,
317
+ steps=25,
318
+ cfg=1.0,
319
+ gs=10.0,
320
+ rs=0.0,
321
+ gpu_memory_preservation=6,
322
+ use_teacache=True,
323
+ mp4_crf=16
324
+ ):
325
+ global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
326
+
327
+ if torch.cuda.device_count() == 0:
328
+ gr.Warning('Set this space to GPU config to make it work.')
329
+ return None, None, None, None, None, None
330
+
331
+ if input_image_debug_value is not None or prompt_debug_value is not None or total_second_length_debug_value is not None:
332
+ print("Debug mode")
333
+ input_image = input_image_debug_value
334
+ prompt = prompt_debug_value
335
+ total_second_length = total_second_length_debug_value
336
+ input_image_debug_value = prompt_debug_value = total_second_length_debug_value = None
337
+
338
+ if randomize_seed:
339
+ seed = random.randint(0, np.iinfo(np.int32).max)
340
+
341
+ prompts = prompt.split(";")
342
+
343
+ # assert input_image is not None, 'No input image!'
344
+ if t2v:
345
+ default_height, default_width = 640, 640
346
+ input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
347
+ print("No input image provided. Using a blank white image.")
348
+
349
+ yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
350
+
351
+ stream = AsyncStream()
352
+
353
+ async_run(worker, input_image, prompts, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf)
354
+
355
+ output_filename = None
356
+
357
+ while True:
358
+ flag, data = stream.output_queue.next()
359
+
360
+ if flag == 'file':
361
+ output_filename = data
362
+ yield output_filename, gr.update(), gr.update(), gr.update(), gr.update(interactive=False), gr.update(interactive=True)
363
+
364
+ if flag == 'progress':
365
+ preview, desc, html = data
366
+ yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True)
367
+
368
+ if flag == 'end':
369
+ yield output_filename, gr.update(visible=False), gr.update(), '', gr.update(interactive=True), gr.update(interactive=False)
370
+ break
371
+
372
+
373
+ def end_process():
374
+ stream.input_queue.push('end')
375
+
376
+
377
+ css = make_progress_bar_css()
378
+ block = gr.Blocks(css=css).queue()
379
+ with block:
380
+ if torch.cuda.device_count() == 0:
381
+ with gr.Row():
382
+ gr.HTML("""
383
+ <p style="background-color: red;"><big><big><big><b>⚠️To use FramePack, <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR?duplicate=true">duplicate this space</a> and set a GPU with 30 GB VRAM.</b>
384
+
385
+ You can't use FramePack directly here because this space runs on a CPU, which is not enough for FramePack. Please provide <a href="https://huggingface.co/spaces/Fabrice-TIERCELIN/SUPIR/discussions/new">feedback</a> if you have issues.
386
+ </big></big></big></p>
387
+ """)
388
+ gr.Markdown('# FramePack-F1')
389
+ gr.Markdown(f"""### Video diffusion, but feels like image diffusion
390
+ *FramePack F1 - a FramePack model that only predicts future frames from history frames*
391
+ ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
392
+ adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
393
+ """)
394
+ with gr.Row():
395
+ with gr.Column():
396
+ input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
397
+ prompt = gr.Textbox(label="Prompt", value='')
398
+ t2v = gr.Checkbox(label="do text-to-video", value=False)
399
+
400
+ with gr.Row():
401
+ start_button = gr.Button(value="Start Generation", variant="primary")
402
+ end_button = gr.Button(value="End Generation", variant="stop", interactive=False)
403
+
404
+ total_second_length = gr.Slider(label="Generated Video Length (Seconds)", minimum=1, maximum=5, value=2, step=0.1)
405
+ with gr.Accordion("Advanced settings", open=False):
406
+ use_teacache = gr.Checkbox(label='Use TeaCache', value=True, info='Faster speed, but often makes hands and fingers slightly worse.')
407
+
408
+ n_prompt = gr.Textbox(label="Negative Prompt", value="Missing arm, unrealistic position, blurred, blurry") # Not used
409
+ randomize_seed = gr.Checkbox(label='Randomize seed', value=True, info='If checked, the seed is always different')
410
+ seed = gr.Slider(label="Seed", minimum=0, maximum=np.iinfo(np.int32).max, step=1, randomize=True)
411
+
412
+
413
+ latent_window_size = gr.Slider(label="Latent Window Size", minimum=1, maximum=33, value=9, step=1) # Should not change
414
+ steps = gr.Slider(label="Steps", minimum=1, maximum=100, value=25, step=1, info='Changing this value is not recommended.')
415
+
416
+ cfg = gr.Slider(label="CFG Scale", minimum=1.0, maximum=32.0, value=1.0, step=0.01) # Should not change
417
+ gs = gr.Slider(label="Distilled CFG Scale", minimum=1.0, maximum=32.0, value=10.0, step=0.01, info='Changing this value is not recommended; 3=blurred motions& & unsharped; 10 focus motion')
418
+ rs = gr.Slider(label="CFG Re-Scale", minimum=0.0, maximum=1.0, value=0.0, step=0.01) # Should not change
419
+
420
+ gpu_memory_preservation = gr.Slider(label="GPU Inference Preserved Memory (GB) (larger means slower)", minimum=6, maximum=128, value=6, step=0.1, info="Set this number to a larger value if you encounter OOM. Larger value causes slower speed.")
421
+
422
+ mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
423
+
424
+ with gr.Accordion("Debug", open=False):
425
+ input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
426
+ prompt_debug = gr.Textbox(label="Prompt Debug", value='')
427
+ total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
428
+
429
+ with gr.Column():
430
+ preview_image = gr.Image(label="Next Latents", height=200, visible=False)
431
+ result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
432
+ progress_desc = gr.Markdown('', elem_classes='no-generating-animation')
433
+ progress_bar = gr.HTML('', elem_classes='no-generating-animation')
434
+
435
+ gr.HTML('<div style="text-align:center; margin-top:20px;">Share your results and find ideas at the <a href="https://x.com/search?q=framepack&f=live" target="_blank">FramePack Twitter (X) thread</a></div>')
436
+
437
+ ips = [input_image, prompt, t2v, n_prompt, randomize_seed, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf]
438
+ start_button.click(fn=process, inputs=ips, outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, end_button])
439
+ end_button.click(fn=end_process)
440
+
441
+ with gr.Row(visible=False):
442
+ gr.Examples(
443
+ examples = [
444
+ [
445
+ "./img_examples/Example1.png", # input_image
446
+ "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
447
+ False, # t2v
448
+ "Missing arm, unrealistic position, blurred, blurry", # n_prompt
449
+ True, # randomize_seed
450
+ 42, # seed
451
+ 1, # total_second_length
452
+ 9, # latent_window_size
453
+ 25, # steps
454
+ 1.0, # cfg
455
+ 10.0, # gs
456
+ 0.0, # rs
457
+ 6, # gpu_memory_preservation
458
+ True, # use_teacache
459
+ 16 # mp4_crf
460
+ ],
461
+ [
462
+ "./img_examples/Example1.png", # input_image
463
+ "A dolphin emerges from the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
464
+ False, # t2v
465
+ "Missing arm, unrealistic position, blurred, blurry", # n_prompt
466
+ True, # randomize_seed
467
+ 42, # seed
468
+ 1, # total_second_length
469
+ 9, # latent_window_size
470
+ 25, # steps
471
+ 1.0, # cfg
472
+ 10.0, # gs
473
+ 0.0, # rs
474
+ 6, # gpu_memory_preservation
475
+ True, # use_teacache
476
+ 16 # mp4_crf
477
+ ],
478
+ [
479
+ "./img_examples/Example1.png", # input_image
480
+ "We are sinking, photorealistic, realistic, intricate details, 8k, insanely detailed",
481
+ False, # t2v
482
+ "Missing arm, unrealistic position, blurred, blurry", # n_prompt
483
+ True, # randomize_seed
484
+ 42, # seed
485
+ 1, # total_second_length
486
+ 9, # latent_window_size
487
+ 25, # steps
488
+ 1.0, # cfg
489
+ 10.0, # gs
490
+ 0.0, # rs
491
+ 6, # gpu_memory_preservation
492
+ True, # use_teacache
493
+ 16 # mp4_crf
494
+ ],
495
+ [
496
+ "./img_examples/Example1.png", # input_image
497
+ "A boat is passing, photorealistic, realistic, intricate details, 8k, insanely detailed",
498
+ False, # t2v
499
+ "Missing arm, unrealistic position, blurred, blurry", # n_prompt
500
+ True, # randomize_seed
501
+ 42, # seed
502
+ 1, # total_second_length
503
+ 9, # latent_window_size
504
+ 25, # steps
505
+ 1.0, # cfg
506
+ 10.0, # gs
507
+ 0.0, # rs
508
+ 6, # gpu_memory_preservation
509
+ True, # use_teacache
510
+ 16 # mp4_crf
511
+ ],
512
+ ],
513
+ run_on_click = True,
514
+ fn = process,
515
+ inputs = ips,
516
+ outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
517
+ cache_examples = True,
518
+ )
519
+ gr.Markdown('## Guide')
520
+ gr.Markdown("I discourage to use the Text-to-Video feature. You should rather generate an image with Flux and use Image-to-Video. You will save time.")
521
+
522
+
523
+ def handle_field_debug_change(input_image_debug_data, prompt_debug_data, total_second_length_debug_data):
524
+ global input_image_debug_value, prompt_debug_value, total_second_length_debug_value
525
+ input_image_debug_value = input_image_debug_data
526
+ prompt_debug_value = prompt_debug_data
527
+ total_second_length_debug_value = total_second_length_debug_data
528
+ return []
529
+
530
+ input_image_debug.upload(
531
+ fn=handle_field_debug_change,
532
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
533
+ outputs=[]
534
+ )
535
+
536
+ prompt_debug.change(
537
+ fn=handle_field_debug_change,
538
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
539
+ outputs=[]
540
+ )
541
+
542
+ total_second_length_debug.change(
543
+ fn=handle_field_debug_change,
544
+ inputs=[input_image_debug, prompt_debug, total_second_length_debug],
545
+ outputs=[]
546
+ )
547
+
548
+
549
  block.launch(mcp_server=False)