import os import sys import json import shutil import tempfile from datetime import datetime import torch import gradio as gr import spaces from huggingface_hub import snapshot_download # --------------------------------------------------------------------------- # ZeroGPU port of the fffiloni/Meigen-MultiTalk demo. # Differences from the original: # - Weights are downloaded at startup on CPU (ZeroGPU attaches a GPU only # inside a @spaces.GPU function, so the original "download if GPU present" # logic never ran and its gpu_name lookup crashed without a GPU). # - Inference runs IN-PROCESS via generate_multitalk.generate() inside a # @spaces.GPU function (a subprocess would not see the ZeroGPU device). # - No flash-attn wheel: wan/modules/attention.py falls back to # scaled_dot_product_attention when flash-attn is unavailable. # - No 5-second audio trim and no shared-UI restrictions. # --------------------------------------------------------------------------- WEIGHTS_DIR = "weights" BASE_MODEL_DIR = os.path.join(WEIGHTS_DIR, "Wan2.1-I2V-14B-480P") WAV2VEC_DIR = os.path.join(WEIGHTS_DIR, "chinese-wav2vec2-base") MULTITALK_DIR = os.path.join(WEIGHTS_DIR, "MeiGen-MultiTalk") def ensure_weights(): """Download the three model repos and lay the MultiTalk weights over the Wan base model directory, exactly as the original app did. ZeroGPU disk is capped at 150 GB and the three full repos total about 165 GB, which got the first run evicted ("storage limit exceeded"). The MeiGen-MultiTalk repo is 80.7 GB but only two of its files are used here (multitalk.safetensors and the safetensors index); the quant_models variants are never loaded by this app. Downloading only those two files brings the total footprint to about 94 GB. The two files are MOVED into the base model dir and the staging dir is removed afterwards.""" snapshot_download( repo_id="Wan-AI/Wan2.1-I2V-14B-480P", local_dir=BASE_MODEL_DIR, ) snapshot_download( repo_id="TencentGameMate/chinese-wav2vec2-base", local_dir=WAV2VEC_DIR, ) snapshot_download( repo_id="MeiGen-AI/MeiGen-MultiTalk", local_dir=MULTITALK_DIR, allow_patterns=[ "multitalk.safetensors", "diffusion_pytorch_model.safetensors.index.json", ], ) for d in (BASE_MODEL_DIR, WAV2VEC_DIR, MULTITALK_DIR): shutil.rmtree(os.path.join(d, ".cache"), ignore_errors=True) original_index = os.path.join( BASE_MODEL_DIR, "diffusion_pytorch_model.safetensors.index.json") backup_index = original_index + "_old" if os.path.exists(original_index) and not os.path.exists(backup_index): os.rename(original_index, backup_index) print("Renamed original index file to .json_old") os.replace( os.path.join(MULTITALK_DIR, "diffusion_pytorch_model.safetensors.index.json"), original_index, ) os.replace( os.path.join(MULTITALK_DIR, "multitalk.safetensors"), os.path.join(BASE_MODEL_DIR, "multitalk.safetensors"), ) shutil.rmtree(MULTITALK_DIR, ignore_errors=True) print("Weights ready.") ensure_weights() def create_temp_input_json(prompt, cond_image_path, spk1, spk2): if spk2 is None: data = { "prompt": prompt, "cond_image": cond_image_path, "cond_audio": {"person1": spk1}, } else: data = { "prompt": prompt, "cond_image": cond_image_path, "audio_type": "para", "cond_audio": {"person1": spk1, "person2": spk2}, } tmp = tempfile.NamedTemporaryFile( delete=False, suffix=".json", mode="w", encoding="utf-8") json.dump(data, tmp, indent=4) tmp.close() return tmp.name def _persistent_params_for_gpu(): """Mirror the original app's per-GPU table, decided at call time when a GPU is actually attached.""" try: total = torch.cuda.get_device_properties(0).total_memory except Exception: return 0 if total >= 60 * 10**9: return 22_000_000_000 if total >= 30 * 10**9: return 11_000_000_000 if total >= 20 * 10**9: return 5_000_000_000 return 0 @spaces.GPU(duration=180) def run_generation(prompt, cond_image_path, spk1, spk2, sample_steps): import generate_multitalk timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f") result_name = f"multitalk_result_{timestamp}" input_json_path = create_temp_input_json( prompt, cond_image_path, spk1, spk2) argv = [ "generate_multitalk.py", "--ckpt_dir", BASE_MODEL_DIR, "--wav2vec_dir", WAV2VEC_DIR, "--input_json", input_json_path, "--sample_steps", str(int(sample_steps)), "--mode", "streaming", "--use_teacache", "--save_file", result_name, "--num_persistent_param_in_dit", str(_persistent_params_for_gpu()), ] old_argv = sys.argv try: sys.argv = argv args = generate_multitalk._parse_args() generate_multitalk.generate(args) finally: sys.argv = old_argv try: os.remove(input_json_path) except OSError: pass out = result_name + ".mp4" if not os.path.exists(out): raise RuntimeError("Generation finished but no output mp4 was found.") return out def infer(prompt, cond_image_path, spk1, spk2, sample_steps): if cond_image_path is None: raise gr.Error("Please provide a conditioning image.") if spk1 is None: raise gr.Error("Please provide audio for speaker 1.") return run_generation(prompt, cond_image_path, spk1, spk2, sample_steps) with gr.Blocks(title="MultiTalk (ZeroGPU)") as demo: gr.Markdown("## Meigen MultiTalk Inference (ZeroGPU port)") gr.Markdown( "Audio-Driven Multi-Person Conversational Video Generation. " "Speaker 1 is the person on the left of the image, speaker 2 the " "person on the right. Each audio track should contain that speaker's " "lines with silence while the other speaks (parallel tracks).") with gr.Row(): with gr.Column(scale=1): prompt_input = gr.Textbox( label="Text Prompt", placeholder="Describe the scene...") image_input = gr.Image( type="filepath", label="Conditioning Image") audio_input_spk1 = gr.Audio( type="filepath", label="Audio for speaker 1 (.wav)") audio_input_spk2 = gr.Audio( type="filepath", label="Audio for speaker 2 (.wav) (Optional)") with gr.Accordion("Advanced settings", open=False): sample_steps = gr.Slider( label="sample steps", value=12, minimum=2, maximum=25, step=1) submit_btn = gr.Button("Generate") with gr.Column(scale=3): output_video = gr.Video(label="Generated Video", interactive=False) submit_btn.click( fn=infer, inputs=[prompt_input, image_input, audio_input_spk1, audio_input_spk2, sample_steps], outputs=output_video, ) demo.queue(max_size=2).launch(ssr_mode=False, show_error=True)