infinitetalk / app.py
Daniel
Optimize ZeroGPU duration to 300s, set recommended 15 steps, and remove torch.compile lag
d05af4e
Raw History Blame Contribute Delete
13.9 kB
"""
InfiniteTalk - Talking Video Generator
Gradio Space for Hugging Face (ZeroGPU Compatible)
Model: https://huggingface.co/MeiGen-AI/InfiniteTalk
"""
import os
import sys
# ZeroGPU: import spaces before any CUDA or torch imports
try:
import spaces
except ImportError:
class spaces:
@staticmethod
def GPU(duration=180):
def decorator(f):
return f
return decorator
os.environ["TORCHVISION_DISABLE_META_REGISTRATIONS"] = "1"
os.environ["TORCH_LOGS"] = "-all"
import random
import logging
import warnings
from pathlib import Path
import gradio as gr
import torch
import numpy as np
# Defensive patch for gradio_client json schema typing bug
try:
import gradio_client.utils as client_utils
_orig_get_type = client_utils.get_type
def _safe_get_type(schema):
if not isinstance(schema, dict):
return "Any"
return _orig_get_type(schema)
client_utils.get_type = _safe_get_type
except Exception:
pass
from PIL import Image
from einops import rearrange
import librosa
import soundfile as sf
import pyloudnorm as pyln
# Suppress warnings
warnings.filterwarnings("ignore")
# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Add current directory to path
sys.path.insert(0, str(Path(__file__).parent))
# Lazy model manager and GPU cleanup to prevent premature CUDA driver init in ZeroGPU main process
_model_manager = None
def get_model_manager():
global _model_manager
if _model_manager is None:
from utils.model_loader import ModelManager
_model_manager = ModelManager()
return _model_manager
def cleanup_gpu():
import gc
gc.collect()
if torch.cuda.is_available():
try:
torch.cuda.empty_cache()
torch.cuda.synchronize()
except Exception:
pass
def loudness_norm(audio_array, sr=16000, lufs=-20.0):
"""Normalize audio loudness using pyloudnorm"""
try:
meter = pyln.Meter(sr)
loudness = meter.integrated_loudness(audio_array)
if abs(loudness) > 100:
return audio_array
normalized_audio = pyln.normalize.loudness(audio_array, loudness, lufs)
return normalized_audio
except Exception as e:
logger.warning(f"Loudness normalization fallback: {e}")
return audio_array
def process_audio(audio_path, target_sr=16000):
"""Process audio file for InfiniteTalk"""
try:
audio, sr = librosa.load(audio_path, sr=target_sr)
audio = loudness_norm(audio, sr)
if len(audio.shape) > 1:
audio = np.mean(audio, axis=1)
return audio, sr
except Exception as e:
logger.error(f"Error processing audio: {e}")
raise gr.Error(f"Fallo al procesar audio: {str(e)}")
def validate_inputs(media_file, audio_file, resolution, steps):
"""Validate user inputs"""
errors = []
if media_file is None:
errors.append("Por favor sube una imagen o un video.")
if audio_file is None:
errors.append("Por favor sube un archivo de audio.")
if resolution not in ["480p", "720p"]:
errors.append("Resoluci贸n no soportada.")
if not (5 <= steps <= 50):
errors.append("Los pasos deben estar entre 5 y 50.")
if errors:
raise gr.Error(" | ".join(errors))
@spaces.GPU(duration=300)
def generate_video(
image_or_video,
audio_file,
resolution="480p",
steps=15,
audio_guide_scale=3.0,
seed=-1,
progress=gr.Progress(track_tqdm=True),
):
"""
Generate talking video with InfiniteTalk on ZeroGPU
"""
validate_inputs(image_or_video, audio_file, resolution, steps)
try:
from wan.utils.multitalk_utils import save_video_ffmpeg
progress(0.05, desc="Normalizando audio...")
audio, sr = process_audio(audio_file)
audio_duration = len(audio) / sr
logger.info(f"Audio duration: {audio_duration:.2f}s")
progress(0.15, desc="Cargando modelos InfiniteTalk (FP8 ultra-eficiente)...")
size = f"infinitetalk-{resolution.replace('p', '')}"
device = "cuda" if torch.cuda.is_available() else "cpu"
model_manager = get_model_manager()
# Load pipeline and audio encoder
wan_pipeline = model_manager.load_wan_model(size=size, device=device, quant="fp8")
audio_encoder, feature_extractor = model_manager.load_audio_encoder(device=device)
progress(0.35, desc="Extrayendo embeddings de audio...")
video_length = audio_duration * 25 # 25 FPS
audio_feature = np.squeeze(
feature_extractor(audio, sampling_rate=sr).input_values
)
audio_feature = torch.from_numpy(audio_feature).float().to(device=device)
audio_feature = audio_feature.unsqueeze(0)
with torch.no_grad():
embeddings = audio_encoder(
audio_feature, seq_len=int(video_length), output_hidden_states=True
)
if embeddings is None or getattr(embeddings, "hidden_states", None) is None:
raise gr.Error("Error al extraer embeddings de audio.")
audio_embeddings = torch.stack(embeddings.hidden_states[1:], dim=1).squeeze(0)
audio_embeddings = rearrange(audio_embeddings, "b s d -> s b d")
audio_embeddings = audio_embeddings.cpu().detach()
progress(0.45, desc=f"Generando video con InfiniteTalk ({resolution}, {steps} pasos)...")
if seed == -1:
seed = random.randint(0, 99999999)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
temp_dir = Path("/tmp/infinitetalk_work")
temp_dir.mkdir(parents=True, exist_ok=True)
emb_path = str(temp_dir / f"emb_{seed}.pt")
audio_wav_path = str(temp_dir / f"audio_{seed}.wav")
output_path = str(temp_dir / f"infinitetalk_{seed}.mp4")
torch.save(audio_embeddings, emb_path)
sf.write(audio_wav_path, audio, sr)
input_clip = {
"prompt": "",
"cond_video": image_or_video,
"cond_audio": {"person1": emb_path},
"video_audio": audio_wav_path,
}
sample_shift = 7 if resolution == "480p" else 11
with torch.no_grad():
video_tensor = wan_pipeline.generate_infinitetalk(
input_clip,
size_buckget=size,
motion_frame=9,
frame_num=81,
shift=sample_shift,
sampling_steps=int(steps),
text_guide_scale=1.0,
audio_guide_scale=float(audio_guide_scale),
seed=int(seed),
offload_model=True,
max_frames_num=81,
color_correction_strength=1.0,
)
save_video_ffmpeg(
video_tensor,
output_path.replace(".mp4", ""),
[audio_wav_path],
high_quality_save=False,
)
progress(0.95, desc="Finalizando video...")
cleanup_gpu()
progress(1.0, desc="隆Completado con 茅xito!")
return output_path
except Exception as e:
logger.error(f"Error generating video: {e}")
cleanup_gpu()
import traceback
traceback.print_exc()
raise gr.Error(f"Error durante la generaci贸n: {str(e)}")
def create_interface():
theme = gr.themes.Soft(
primary_hue="blue",
secondary_hue="purple",
)
with gr.Blocks(title="InfiniteTalk - Audio-Driven Video Generation", theme=theme) as demo:
gr.Markdown(
"""
# 馃幀 InfiniteTalk: Audio-driven Video Generation & Video Dubbing
**[InfiniteTalk](https://huggingface.co/MeiGen-AI/InfiniteTalk)** (MeiGen-AI) es un modelo avanzado de generaci贸n de video conducido por audio basado en Wan2.1.
Sincroniza no solo los **labios** con extrema precisi贸n, sino tambi茅n la **cabeza, postura corporal y expresiones faciales**, permitiendo dubbing de videos y animaci贸n de retratos de longitud ilimitada.
"""
)
with gr.Tabs():
with gr.Tab("馃摳 Imagen a Video (Image-to-Video)"):
gr.Markdown("Anima una imagen o retrato a partir de una pista de audio.")
with gr.Row():
with gr.Column():
img_input = gr.Image(
type="filepath",
label="Retrato de entrada (PNG o JPG con rostro visible)",
)
audio_input_i2v = gr.Audio(
type="filepath",
label="Audio de sincronizaci贸n (WAV/MP3)",
)
with gr.Accordion("鈿欙笍 Configuraci贸n Avanzada", open=False):
res_i2v = gr.Radio(
choices=["480p", "720p"],
value="480p",
label="Resoluci贸n",
)
steps_i2v = gr.Slider(
minimum=10,
maximum=50,
value=15,
step=1,
label="Pasos de difusi贸n (Steps) [15 recomendados para ZeroGPU]",
)
scale_i2v = gr.Slider(
minimum=1.0,
maximum=6.0,
value=3.0,
step=0.5,
label="Audio Guide Scale",
)
seed_i2v = gr.Number(value=42, label="Semilla (Seed)", precision=0)
btn_i2v = gr.Button("馃殌 Generar Video Parlante", variant="primary")
with gr.Column():
output_i2v = gr.Video(label="馃幀 Video Generado", autoplay=True)
gr.Examples(
examples=[
["examples/single/ref_image.png", "examples/single/1.wav", "480p", 20, 3.0, 42],
],
inputs=[img_input, audio_input_i2v, res_i2v, steps_i2v, scale_i2v, seed_i2v],
)
btn_i2v.click(
fn=generate_video,
inputs=[img_input, audio_input_i2v, res_i2v, steps_i2v, scale_i2v, seed_i2v],
outputs=[output_i2v],
)
with gr.Tab("馃帴 Doblaje de Video (Video Dubbing)"):
gr.Markdown("Redobla un video existente con una nueva voz, sincronizando labios y postura corporal.")
with gr.Row():
with gr.Column():
vid_input = gr.Video(label="Video original de entrada (MP4)")
audio_input_v2v = gr.Audio(
type="filepath",
label="Nueva pista de audio (WAV/MP3)",
)
with gr.Accordion("鈿欙笍 Configuraci贸n Avanzada", open=False):
res_v2v = gr.Radio(
choices=["480p", "720p"],
value="480p",
label="Resoluci贸n",
)
steps_v2v = gr.Slider(
minimum=10,
maximum=50,
value=15,
step=1,
label="Pasos de difusi贸n (Steps) [15 recomendados para ZeroGPU]",
)
scale_v2v = gr.Slider(
minimum=1.0,
maximum=6.0,
value=3.0,
step=0.5,
label="Audio Guide Scale",
)
seed_v2v = gr.Number(value=42, label="Semilla (Seed)", precision=0)
btn_v2v = gr.Button("馃殌 Redoblar Video", variant="primary")
with gr.Column():
output_v2v = gr.Video(label="馃幀 Video Doblado", autoplay=True)
gr.Examples(
examples=[
["examples/single/ref_video.mp4", "examples/single/1.wav", "480p", 20, 3.0, 42],
],
inputs=[vid_input, audio_input_v2v, res_v2v, steps_v2v, scale_v2v, seed_v2v],
)
btn_v2v.click(
fn=generate_video,
inputs=[vid_input, audio_input_v2v, res_v2v, steps_v2v, scale_v2v, seed_v2v],
outputs=[output_v2v],
)
gr.Markdown(
"""
---
### 鈩癸笍 Informaci贸n & Cr茅ditos
- **Modelo oficial:** [MeiGen-AI/InfiniteTalk](https://huggingface.co/MeiGen-AI/InfiniteTalk)
- **Arquitectura:** DiT basado en Wan2.1 con condicionamiento ac煤stico Wav2Vec2 y cuantizaci贸n FP8 para inferencia 谩gil en ZeroGPU.
- **Licencia:** Apache 2.0
"""
)
return demo
if __name__ == "__main__":
# Pre-download weights on CPU startup so ZeroGPU time is not wasted downloading
try:
logger.info("Checking and pre-downloading model weights on CPU startup...")
get_model_manager().prepare_all()
except Exception as e:
logger.warning(f"Startup model download failed or deferred: {e}")
demo = create_interface()
demo.queue().launch(show_error=True, ssr_mode=False)