multitalk-test / app.py
emmabiz86's picture
Lower GPU duration request to 180s to fit free ZeroGPU cap
8a5c76f verified
Raw History Blame Contribute Delete
7.33 kB
import os
import sys
import json
import shutil
import tempfile
from datetime import datetime
import torch
import gradio as gr
import spaces
from huggingface_hub import snapshot_download
# ---------------------------------------------------------------------------
# ZeroGPU port of the fffiloni/Meigen-MultiTalk demo.
# Differences from the original:
# - Weights are downloaded at startup on CPU (ZeroGPU attaches a GPU only
# inside a @spaces.GPU function, so the original "download if GPU present"
# logic never ran and its gpu_name lookup crashed without a GPU).
# - Inference runs IN-PROCESS via generate_multitalk.generate() inside a
# @spaces.GPU function (a subprocess would not see the ZeroGPU device).
# - No flash-attn wheel: wan/modules/attention.py falls back to
# scaled_dot_product_attention when flash-attn is unavailable.
# - No 5-second audio trim and no shared-UI restrictions.
# ---------------------------------------------------------------------------
WEIGHTS_DIR = "weights"
BASE_MODEL_DIR = os.path.join(WEIGHTS_DIR, "Wan2.1-I2V-14B-480P")
WAV2VEC_DIR = os.path.join(WEIGHTS_DIR, "chinese-wav2vec2-base")
MULTITALK_DIR = os.path.join(WEIGHTS_DIR, "MeiGen-MultiTalk")
def ensure_weights():
"""Download the three model repos and lay the MultiTalk weights over the
Wan base model directory, exactly as the original app did.
ZeroGPU disk is capped at 150 GB and the three full repos total about
165 GB, which got the first run evicted ("storage limit exceeded").
The MeiGen-MultiTalk repo is 80.7 GB but only two of its files are used
here (multitalk.safetensors and the safetensors index); the quant_models
variants are never loaded by this app. Downloading only those two files
brings the total footprint to about 94 GB. The two files are MOVED into
the base model dir and the staging dir is removed afterwards."""
snapshot_download(
repo_id="Wan-AI/Wan2.1-I2V-14B-480P",
local_dir=BASE_MODEL_DIR,
)
snapshot_download(
repo_id="TencentGameMate/chinese-wav2vec2-base",
local_dir=WAV2VEC_DIR,
)
snapshot_download(
repo_id="MeiGen-AI/MeiGen-MultiTalk",
local_dir=MULTITALK_DIR,
allow_patterns=[
"multitalk.safetensors",
"diffusion_pytorch_model.safetensors.index.json",
],
)
for d in (BASE_MODEL_DIR, WAV2VEC_DIR, MULTITALK_DIR):
shutil.rmtree(os.path.join(d, ".cache"), ignore_errors=True)
original_index = os.path.join(
BASE_MODEL_DIR, "diffusion_pytorch_model.safetensors.index.json")
backup_index = original_index + "_old"
if os.path.exists(original_index) and not os.path.exists(backup_index):
os.rename(original_index, backup_index)
print("Renamed original index file to .json_old")
os.replace(
os.path.join(MULTITALK_DIR,
"diffusion_pytorch_model.safetensors.index.json"),
original_index,
)
os.replace(
os.path.join(MULTITALK_DIR, "multitalk.safetensors"),
os.path.join(BASE_MODEL_DIR, "multitalk.safetensors"),
)
shutil.rmtree(MULTITALK_DIR, ignore_errors=True)
print("Weights ready.")
ensure_weights()
def create_temp_input_json(prompt, cond_image_path, spk1, spk2):
if spk2 is None:
data = {
"prompt": prompt,
"cond_image": cond_image_path,
"cond_audio": {"person1": spk1},
}
else:
data = {
"prompt": prompt,
"cond_image": cond_image_path,
"audio_type": "para",
"cond_audio": {"person1": spk1, "person2": spk2},
}
tmp = tempfile.NamedTemporaryFile(
delete=False, suffix=".json", mode="w", encoding="utf-8")
json.dump(data, tmp, indent=4)
tmp.close()
return tmp.name
def _persistent_params_for_gpu():
"""Mirror the original app's per-GPU table, decided at call time when a
GPU is actually attached."""
try:
total = torch.cuda.get_device_properties(0).total_memory
except Exception:
return 0
if total >= 60 * 10**9:
return 22_000_000_000
if total >= 30 * 10**9:
return 11_000_000_000
if total >= 20 * 10**9:
return 5_000_000_000
return 0
@spaces.GPU(duration=180)
def run_generation(prompt, cond_image_path, spk1, spk2, sample_steps):
import generate_multitalk
timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f")
result_name = f"multitalk_result_{timestamp}"
input_json_path = create_temp_input_json(
prompt, cond_image_path, spk1, spk2)
argv = [
"generate_multitalk.py",
"--ckpt_dir", BASE_MODEL_DIR,
"--wav2vec_dir", WAV2VEC_DIR,
"--input_json", input_json_path,
"--sample_steps", str(int(sample_steps)),
"--mode", "streaming",
"--use_teacache",
"--save_file", result_name,
"--num_persistent_param_in_dit", str(_persistent_params_for_gpu()),
]
old_argv = sys.argv
try:
sys.argv = argv
args = generate_multitalk._parse_args()
generate_multitalk.generate(args)
finally:
sys.argv = old_argv
try:
os.remove(input_json_path)
except OSError:
pass
out = result_name + ".mp4"
if not os.path.exists(out):
raise RuntimeError("Generation finished but no output mp4 was found.")
return out
def infer(prompt, cond_image_path, spk1, spk2, sample_steps):
if cond_image_path is None:
raise gr.Error("Please provide a conditioning image.")
if spk1 is None:
raise gr.Error("Please provide audio for speaker 1.")
return run_generation(prompt, cond_image_path, spk1, spk2, sample_steps)
with gr.Blocks(title="MultiTalk (ZeroGPU)") as demo:
gr.Markdown("## Meigen MultiTalk Inference (ZeroGPU port)")
gr.Markdown(
"Audio-Driven Multi-Person Conversational Video Generation. "
"Speaker 1 is the person on the left of the image, speaker 2 the "
"person on the right. Each audio track should contain that speaker's "
"lines with silence while the other speaks (parallel tracks).")
with gr.Row():
with gr.Column(scale=1):
prompt_input = gr.Textbox(
label="Text Prompt", placeholder="Describe the scene...")
image_input = gr.Image(
type="filepath", label="Conditioning Image")
audio_input_spk1 = gr.Audio(
type="filepath", label="Audio for speaker 1 (.wav)")
audio_input_spk2 = gr.Audio(
type="filepath",
label="Audio for speaker 2 (.wav) (Optional)")
with gr.Accordion("Advanced settings", open=False):
sample_steps = gr.Slider(
label="sample steps", value=12, minimum=2, maximum=25,
step=1)
submit_btn = gr.Button("Generate")
with gr.Column(scale=3):
output_video = gr.Video(label="Generated Video", interactive=False)
submit_btn.click(
fn=infer,
inputs=[prompt_input, image_input, audio_input_spk1,
audio_input_spk2, sample_steps],
outputs=output_video,
)
demo.queue(max_size=2).launch(ssr_mode=False, show_error=True)