Fabrice-TIERCELIN commited on
Commit
1f6351a
·
verified ·
1 Parent(s): a123e4f

Upload 3 files

Browse files
Files changed (3) hide show
  1. README.md +5 -5
  2. app.py +34 -95
  3. requirements.txt +1 -6
README.md CHANGED
@@ -1,14 +1,14 @@
1
  ---
2
- title: FramePack F1 + V2V + EF
3
- emoji: 👽
4
  colorFrom: pink
5
  colorTo: gray
6
  sdk: gradio
7
- sdk_version: 5.29.0
8
  app_file: app.py
9
- pinned: true
10
  license: apache-2.0
11
  short_description: fast video generation from images & text
12
  ---
13
-
14
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
1
  ---
2
+ title: FramePack F1
3
+ emoji: 📹⚡️
4
  colorFrom: pink
5
  colorTo: gray
6
  sdk: gradio
7
+ sdk_version: 5.30.0
8
  app_file: app.py
9
+ pinned: false
10
  license: apache-2.0
11
  short_description: fast video generation from images & text
12
  ---
13
+ paper: arxiv:2504.12626
14
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py CHANGED
@@ -29,7 +29,7 @@ from diffusers_helper.bucket_tools import find_nearest_bucket
29
 
30
 
31
  free_mem_gb = get_cuda_free_memory_gb(gpu)
32
- high_vram = free_mem_gb > 80
33
 
34
  print(f'Free VRAM {free_mem_gb} GB')
35
  print(f'High-VRAM Mode: {high_vram}')
@@ -43,9 +43,6 @@ vae = AutoencoderKLHunyuanVideo.from_pretrained("hunyuanvideo-community/HunyuanV
43
  feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
44
  image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
45
 
46
- # quant_config = DiffusersBitsAndBytesConfig(load_in_8bit=True)
47
- # transformer = HunyuanVideoTransformer3DModelPacked.from_single_file("https://huggingface.co/sirolim/FramePack_F1_I2V_FP8/resolve/main/FramePack_F1_I2V_HY_fp8_e4m3fn.safetensors", torch_dtype=torch.bfloat16)
48
- # transformer = HunyuanVideoTransformer3DModelPacked.from_single_file('sirolim/FramePack_F1_I2V_FP8', "FramePack_F1_I2V_HY_fp8_e4m3fn.safetensors", use_safetensors=True, torch_dtype=torch.bfloat16).cpu()
49
  transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
50
 
51
  vae.eval()
@@ -95,10 +92,6 @@ examples = [
95
  ["img_examples/3.png", "The woman dances elegantly among the blossoms, spinning slowly with flowing sleeves and graceful hand movements."],
96
  ]
97
 
98
- input_image_debug_value = None
99
- prompt_debug_value = None
100
- total_second_length_debug_value = None
101
-
102
  def generate_examples(input_image, prompt):
103
 
104
  t2v=False
@@ -345,10 +338,6 @@ def worker(input_image, prompt, n_prompt, seed, total_second_length, latent_wind
345
  return
346
 
347
  def get_duration(input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
348
- global total_second_length_debug_value
349
-
350
- if total_second_length_debug_value is not None:
351
- return total_second_length_debug_value * 60
352
  return total_second_length * 60
353
 
354
  @spaces.GPU(duration=get_duration)
@@ -366,25 +355,38 @@ def process(input_image, prompt,
366
  use_teacache=True,
367
  mp4_crf=16
368
  ):
369
- global stream, input_image_debug_value, prompt_debug_value, total_second_length_debug_value
370
-
371
- if input_image_debug_value is not None:
372
- input_image = input_image_debug_value
373
- input_image_debug_value = None
374
-
375
- if prompt_debug_value is not None:
376
- prompt = prompt_debug_value
377
- prompt_debug_value = None
378
-
379
- if total_second_length_debug_value is not None:
380
- total_second_length = total_second_length_debug_value
381
- total_second_length_debug_value = None
382
 
383
  # assert input_image is not None, 'No input image!'
384
  if t2v:
385
  default_height, default_width = 640, 640
386
  input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
387
  print("No input image provided. Using a blank white image.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
388
 
389
  yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
390
 
@@ -424,12 +426,15 @@ quick_prompts = [[x] for x in quick_prompts]
424
  css = make_progress_bar_css()
425
  block = gr.Blocks(css=css).queue()
426
  with block:
427
- gr.Markdown('# FramePack Essentials | Experimentation in Progress')
428
- gr.Markdown(f"""### Space is constantly being tinkered with, expect downtime and errors.
 
 
 
429
  """)
430
  with gr.Row():
431
  with gr.Column():
432
- input_image = gr.Image(sources='upload', type="numpy", label="Image", height=320)
433
  prompt = gr.Textbox(label="Prompt", value='')
434
  t2v = gr.Checkbox(label="do text-to-video", value=False)
435
  example_quick_prompts = gr.Dataset(samples=quick_prompts, label='Quick List', samples_per_page=1000, components=[prompt])
@@ -459,11 +464,6 @@ with block:
459
 
460
  mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
461
 
462
- with gr.Accordion("Debug", open=False):
463
- input_image_debug = gr.Image(type="numpy", label="Image Debug", height=320)
464
- prompt_debug = gr.Textbox(label="Prompt Debug", value='')
465
- total_second_length_debug = gr.Slider(label="Additional Video Length to Generate (Seconds) Debug", minimum=1, maximum=120, value=5, step=0.1)
466
-
467
  with gr.Column():
468
  preview_image = gr.Image(label="Next Latents", height=200, visible=False)
469
  result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
@@ -484,66 +484,5 @@ with block:
484
  # cache_examples=True
485
  # )
486
 
487
- with gr.Row(visible=False):
488
- gr.Examples(
489
- examples = [
490
- [
491
- "./img_examples/Example1.png", # input_image
492
- "View of the sea as far as the eye can see, from the seaside, a piece of land is barely visible on the horizon at the middle, the sky is radiant, reflections of the sun in the water, photorealistic, realistic, intricate details, 8k, insanely detailed",
493
- False, # t2v
494
- "", # n_prompt
495
- 42, # seed
496
- 1, # total_second_length
497
- 9, # latent_window_size
498
- 25, # steps
499
- 1.0, # cfg
500
- 3.0, # gs
501
- 0.0, # rs
502
- 6, # gpu_memory_preservation
503
- True, # use_teacache
504
- 16 # mp4_crf
505
- ],
506
- ],
507
- run_on_click = True,
508
- fn = process,
509
- inputs = ips,
510
- outputs = [result_video, preview_image, progress_desc, progress_bar, start_button, end_button],
511
- cache_examples = True,
512
- )
513
-
514
-
515
- def handle_input_image_debug_upload(input):
516
- global input_image_debug_value
517
- input_image_debug_value = input
518
- return []
519
-
520
- def handle_prompt_debug_change(input):
521
- global prompt_debug_value
522
- prompt_debug_value = input
523
- return []
524
-
525
- def handle_total_second_length_debug_change(input):
526
- global total_second_length_debug_value
527
- total_second_length_debug_value = input
528
- return []
529
-
530
- input_image_debug.upload(
531
- fn=handle_input_image_debug_upload,
532
- inputs=[input_image_debug],
533
- outputs=[]
534
- )
535
-
536
- prompt_debug.change(
537
- fn=handle_prompt_debug_change,
538
- inputs=[prompt_debug],
539
- outputs=[]
540
- )
541
-
542
- total_second_length_debug.change(
543
- fn=handle_total_second_length_debug_change,
544
- inputs=[total_second_length_debug],
545
- outputs=[]
546
- )
547
-
548
 
549
- block.launch(ssr_mode=False)
 
29
 
30
 
31
  free_mem_gb = get_cuda_free_memory_gb(gpu)
32
+ high_vram = free_mem_gb > 60
33
 
34
  print(f'Free VRAM {free_mem_gb} GB')
35
  print(f'High-VRAM Mode: {high_vram}')
 
43
  feature_extractor = SiglipImageProcessor.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='feature_extractor')
44
  image_encoder = SiglipVisionModel.from_pretrained("lllyasviel/flux_redux_bfl", subfolder='image_encoder', torch_dtype=torch.float16).cpu()
45
 
 
 
 
46
  transformer = HunyuanVideoTransformer3DModelPacked.from_pretrained('lllyasviel/FramePack_F1_I2V_HY_20250503', torch_dtype=torch.bfloat16).cpu()
47
 
48
  vae.eval()
 
92
  ["img_examples/3.png", "The woman dances elegantly among the blossoms, spinning slowly with flowing sleeves and graceful hand movements."],
93
  ]
94
 
 
 
 
 
95
  def generate_examples(input_image, prompt):
96
 
97
  t2v=False
 
338
  return
339
 
340
  def get_duration(input_image, prompt, t2v, n_prompt, seed, total_second_length, latent_window_size, steps, cfg, gs, rs, gpu_memory_preservation, use_teacache, mp4_crf):
 
 
 
 
341
  return total_second_length * 60
342
 
343
  @spaces.GPU(duration=get_duration)
 
355
  use_teacache=True,
356
  mp4_crf=16
357
  ):
358
+ global stream
 
 
 
 
 
 
 
 
 
 
 
 
359
 
360
  # assert input_image is not None, 'No input image!'
361
  if t2v:
362
  default_height, default_width = 640, 640
363
  input_image = np.ones((default_height, default_width, 3), dtype=np.uint8) * 255
364
  print("No input image provided. Using a blank white image.")
365
+ else:
366
+ composite_rgba_uint8 = input_image["composite"]
367
+
368
+ # rgb_uint8 will be (H, W, 3), dtype uint8
369
+ rgb_uint8 = composite_rgba_uint8[:, :, :3]
370
+ # mask_uint8 will be (H, W), dtype uint8
371
+ mask_uint8 = composite_rgba_uint8[:, :, 3]
372
+
373
+ # Create background
374
+ h, w = rgb_uint8.shape[:2]
375
+ # White background, (H, W, 3), dtype uint8
376
+ background_uint8 = np.full((h, w, 3), 255, dtype=np.uint8)
377
+
378
+ # Normalize mask to range [0.0, 1.0].
379
+ alpha_normalized_float32 = mask_uint8.astype(np.float32) / 255.0
380
+
381
+ # Expand alpha to 3 channels to match RGB images for broadcasting.
382
+ # alpha_mask_float32 will have shape (H, W, 3)
383
+ alpha_mask_float32 = np.stack([alpha_normalized_float32] * 3, axis=2)
384
+
385
+ # alpha blending
386
+ blended_image_float32 = rgb_uint8.astype(np.float32) * alpha_mask_float32 + \
387
+ background_uint8.astype(np.float32) * (1.0 - alpha_mask_float32)
388
+
389
+ input_image = np.clip(blended_image_float32, 0, 255).astype(np.uint8)
390
 
391
  yield None, None, '', '', gr.update(interactive=False), gr.update(interactive=True)
392
 
 
426
  css = make_progress_bar_css()
427
  block = gr.Blocks(css=css).queue()
428
  with block:
429
+ gr.Markdown('# FramePack-F1')
430
+ gr.Markdown(f"""### Video diffusion, but feels like image diffusion
431
+ *FramePack F1 - a FramePack model that only predicts future frames from history frames*
432
+ ### *beta* FramePack Fill 🖋️- draw a mask over the input image to inpaint the video output
433
+ adapted from the officical code repo [FramePack](https://github.com/lllyasviel/FramePack) by [lllyasviel](lllyasviel/FramePack_F1_I2V_HY_20250503) and [FramePack Studio](https://github.com/colinurbs/FramePack-Studio) 🙌🏻
434
  """)
435
  with gr.Row():
436
  with gr.Column():
437
+ input_image = gr.ImageEditor(type="numpy", label="Image", height=320, brush=gr.Brush(colors=["#ffffff"]))
438
  prompt = gr.Textbox(label="Prompt", value='')
439
  t2v = gr.Checkbox(label="do text-to-video", value=False)
440
  example_quick_prompts = gr.Dataset(samples=quick_prompts, label='Quick List', samples_per_page=1000, components=[prompt])
 
464
 
465
  mp4_crf = gr.Slider(label="MP4 Compression", minimum=0, maximum=100, value=16, step=1, info="Lower means better quality. 0 is uncompressed. Change to 16 if you get black outputs. ")
466
 
 
 
 
 
 
467
  with gr.Column():
468
  preview_image = gr.Image(label="Next Latents", height=200, visible=False)
469
  result_video = gr.Video(label="Finished Frames", autoplay=True, show_share_button=False, height=512, loop=True)
 
484
  # cache_examples=True
485
  # )
486
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
487
 
488
+ block.launch(share=True, mcp_server=True)
requirements.txt CHANGED
@@ -15,9 +15,4 @@ einops
15
  opencv-contrib-python
16
  safetensors
17
  huggingface_hub
18
- spaces
19
- decord
20
- imageio_ffmpeg
21
- sageattention
22
- xformers
23
- bitsandbytes
 
15
  opencv-contrib-python
16
  safetensors
17
  huggingface_hub
18
+ spaces