Spaces:
Sleeping
Sleeping
Download app.py from emmabiz86/multitalk-test: direct link, hf CLI and curl.
- Browser
- Download file 7.33 kB
-
https://huggingface.co/spaces/emmabiz86/multitalk-test/resolve/main/app.py
- Command line
-
hf download hf://spaces/emmabiz86/multitalk-test/app.py
-
curl -L -o app.py https://huggingface.co/spaces/emmabiz86/multitalk-test/resolve/main/app.py
7.33 kB
| import os | |
| import sys | |
| import json | |
| import shutil | |
| import tempfile | |
| from datetime import datetime | |
| import torch | |
| import gradio as gr | |
| import spaces | |
| from huggingface_hub import snapshot_download | |
| # --------------------------------------------------------------------------- | |
| # ZeroGPU port of the fffiloni/Meigen-MultiTalk demo. | |
| # Differences from the original: | |
| # - Weights are downloaded at startup on CPU (ZeroGPU attaches a GPU only | |
| # inside a @spaces.GPU function, so the original "download if GPU present" | |
| # logic never ran and its gpu_name lookup crashed without a GPU). | |
| # - Inference runs IN-PROCESS via generate_multitalk.generate() inside a | |
| # @spaces.GPU function (a subprocess would not see the ZeroGPU device). | |
| # - No flash-attn wheel: wan/modules/attention.py falls back to | |
| # scaled_dot_product_attention when flash-attn is unavailable. | |
| # - No 5-second audio trim and no shared-UI restrictions. | |
| # --------------------------------------------------------------------------- | |
| WEIGHTS_DIR = "weights" | |
| BASE_MODEL_DIR = os.path.join(WEIGHTS_DIR, "Wan2.1-I2V-14B-480P") | |
| WAV2VEC_DIR = os.path.join(WEIGHTS_DIR, "chinese-wav2vec2-base") | |
| MULTITALK_DIR = os.path.join(WEIGHTS_DIR, "MeiGen-MultiTalk") | |
| def ensure_weights(): | |
| """Download the three model repos and lay the MultiTalk weights over the | |
| Wan base model directory, exactly as the original app did. | |
| ZeroGPU disk is capped at 150 GB and the three full repos total about | |
| 165 GB, which got the first run evicted ("storage limit exceeded"). | |
| The MeiGen-MultiTalk repo is 80.7 GB but only two of its files are used | |
| here (multitalk.safetensors and the safetensors index); the quant_models | |
| variants are never loaded by this app. Downloading only those two files | |
| brings the total footprint to about 94 GB. The two files are MOVED into | |
| the base model dir and the staging dir is removed afterwards.""" | |
| snapshot_download( | |
| repo_id="Wan-AI/Wan2.1-I2V-14B-480P", | |
| local_dir=BASE_MODEL_DIR, | |
| ) | |
| snapshot_download( | |
| repo_id="TencentGameMate/chinese-wav2vec2-base", | |
| local_dir=WAV2VEC_DIR, | |
| ) | |
| snapshot_download( | |
| repo_id="MeiGen-AI/MeiGen-MultiTalk", | |
| local_dir=MULTITALK_DIR, | |
| allow_patterns=[ | |
| "multitalk.safetensors", | |
| "diffusion_pytorch_model.safetensors.index.json", | |
| ], | |
| ) | |
| for d in (BASE_MODEL_DIR, WAV2VEC_DIR, MULTITALK_DIR): | |
| shutil.rmtree(os.path.join(d, ".cache"), ignore_errors=True) | |
| original_index = os.path.join( | |
| BASE_MODEL_DIR, "diffusion_pytorch_model.safetensors.index.json") | |
| backup_index = original_index + "_old" | |
| if os.path.exists(original_index) and not os.path.exists(backup_index): | |
| os.rename(original_index, backup_index) | |
| print("Renamed original index file to .json_old") | |
| os.replace( | |
| os.path.join(MULTITALK_DIR, | |
| "diffusion_pytorch_model.safetensors.index.json"), | |
| original_index, | |
| ) | |
| os.replace( | |
| os.path.join(MULTITALK_DIR, "multitalk.safetensors"), | |
| os.path.join(BASE_MODEL_DIR, "multitalk.safetensors"), | |
| ) | |
| shutil.rmtree(MULTITALK_DIR, ignore_errors=True) | |
| print("Weights ready.") | |
| ensure_weights() | |
| def create_temp_input_json(prompt, cond_image_path, spk1, spk2): | |
| if spk2 is None: | |
| data = { | |
| "prompt": prompt, | |
| "cond_image": cond_image_path, | |
| "cond_audio": {"person1": spk1}, | |
| } | |
| else: | |
| data = { | |
| "prompt": prompt, | |
| "cond_image": cond_image_path, | |
| "audio_type": "para", | |
| "cond_audio": {"person1": spk1, "person2": spk2}, | |
| } | |
| tmp = tempfile.NamedTemporaryFile( | |
| delete=False, suffix=".json", mode="w", encoding="utf-8") | |
| json.dump(data, tmp, indent=4) | |
| tmp.close() | |
| return tmp.name | |
| def _persistent_params_for_gpu(): | |
| """Mirror the original app's per-GPU table, decided at call time when a | |
| GPU is actually attached.""" | |
| try: | |
| total = torch.cuda.get_device_properties(0).total_memory | |
| except Exception: | |
| return 0 | |
| if total >= 60 * 10**9: | |
| return 22_000_000_000 | |
| if total >= 30 * 10**9: | |
| return 11_000_000_000 | |
| if total >= 20 * 10**9: | |
| return 5_000_000_000 | |
| return 0 | |
| def run_generation(prompt, cond_image_path, spk1, spk2, sample_steps): | |
| import generate_multitalk | |
| timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f") | |
| result_name = f"multitalk_result_{timestamp}" | |
| input_json_path = create_temp_input_json( | |
| prompt, cond_image_path, spk1, spk2) | |
| argv = [ | |
| "generate_multitalk.py", | |
| "--ckpt_dir", BASE_MODEL_DIR, | |
| "--wav2vec_dir", WAV2VEC_DIR, | |
| "--input_json", input_json_path, | |
| "--sample_steps", str(int(sample_steps)), | |
| "--mode", "streaming", | |
| "--use_teacache", | |
| "--save_file", result_name, | |
| "--num_persistent_param_in_dit", str(_persistent_params_for_gpu()), | |
| ] | |
| old_argv = sys.argv | |
| try: | |
| sys.argv = argv | |
| args = generate_multitalk._parse_args() | |
| generate_multitalk.generate(args) | |
| finally: | |
| sys.argv = old_argv | |
| try: | |
| os.remove(input_json_path) | |
| except OSError: | |
| pass | |
| out = result_name + ".mp4" | |
| if not os.path.exists(out): | |
| raise RuntimeError("Generation finished but no output mp4 was found.") | |
| return out | |
| def infer(prompt, cond_image_path, spk1, spk2, sample_steps): | |
| if cond_image_path is None: | |
| raise gr.Error("Please provide a conditioning image.") | |
| if spk1 is None: | |
| raise gr.Error("Please provide audio for speaker 1.") | |
| return run_generation(prompt, cond_image_path, spk1, spk2, sample_steps) | |
| with gr.Blocks(title="MultiTalk (ZeroGPU)") as demo: | |
| gr.Markdown("## Meigen MultiTalk Inference (ZeroGPU port)") | |
| gr.Markdown( | |
| "Audio-Driven Multi-Person Conversational Video Generation. " | |
| "Speaker 1 is the person on the left of the image, speaker 2 the " | |
| "person on the right. Each audio track should contain that speaker's " | |
| "lines with silence while the other speaks (parallel tracks).") | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| prompt_input = gr.Textbox( | |
| label="Text Prompt", placeholder="Describe the scene...") | |
| image_input = gr.Image( | |
| type="filepath", label="Conditioning Image") | |
| audio_input_spk1 = gr.Audio( | |
| type="filepath", label="Audio for speaker 1 (.wav)") | |
| audio_input_spk2 = gr.Audio( | |
| type="filepath", | |
| label="Audio for speaker 2 (.wav) (Optional)") | |
| with gr.Accordion("Advanced settings", open=False): | |
| sample_steps = gr.Slider( | |
| label="sample steps", value=12, minimum=2, maximum=25, | |
| step=1) | |
| submit_btn = gr.Button("Generate") | |
| with gr.Column(scale=3): | |
| output_video = gr.Video(label="Generated Video", interactive=False) | |
| submit_btn.click( | |
| fn=infer, | |
| inputs=[prompt_input, image_input, audio_input_spk1, | |
| audio_input_spk2, sample_steps], | |
| outputs=output_video, | |
| ) | |
| demo.queue(max_size=2).launch(ssr_mode=False, show_error=True) | |