""" InfiniteTalk - Talking Video Generator Gradio Space for Hugging Face (ZeroGPU Compatible) Model: https://huggingface.co/MeiGen-AI/InfiniteTalk """ import os import sys # ZeroGPU: import spaces before any CUDA or torch imports try: import spaces except ImportError: class spaces: @staticmethod def GPU(duration=180): def decorator(f): return f return decorator os.environ["TORCHVISION_DISABLE_META_REGISTRATIONS"] = "1" os.environ["TORCH_LOGS"] = "-all" import random import logging import warnings from pathlib import Path import gradio as gr import torch import numpy as np # Defensive patch for gradio_client json schema typing bug try: import gradio_client.utils as client_utils _orig_get_type = client_utils.get_type def _safe_get_type(schema): if not isinstance(schema, dict): return "Any" return _orig_get_type(schema) client_utils.get_type = _safe_get_type except Exception: pass from PIL import Image from einops import rearrange import librosa import soundfile as sf import pyloudnorm as pyln # Suppress warnings warnings.filterwarnings("ignore") # Setup logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Add current directory to path sys.path.insert(0, str(Path(__file__).parent)) # Lazy model manager and GPU cleanup to prevent premature CUDA driver init in ZeroGPU main process _model_manager = None def get_model_manager(): global _model_manager if _model_manager is None: from utils.model_loader import ModelManager _model_manager = ModelManager() return _model_manager def cleanup_gpu(): import gc gc.collect() if torch.cuda.is_available(): try: torch.cuda.empty_cache() torch.cuda.synchronize() except Exception: pass def loudness_norm(audio_array, sr=16000, lufs=-20.0): """Normalize audio loudness using pyloudnorm""" try: meter = pyln.Meter(sr) loudness = meter.integrated_loudness(audio_array) if abs(loudness) > 100: return audio_array normalized_audio = pyln.normalize.loudness(audio_array, loudness, lufs) return normalized_audio except Exception as e: logger.warning(f"Loudness normalization fallback: {e}") return audio_array def process_audio(audio_path, target_sr=16000): """Process audio file for InfiniteTalk""" try: audio, sr = librosa.load(audio_path, sr=target_sr) audio = loudness_norm(audio, sr) if len(audio.shape) > 1: audio = np.mean(audio, axis=1) return audio, sr except Exception as e: logger.error(f"Error processing audio: {e}") raise gr.Error(f"Fallo al procesar audio: {str(e)}") def validate_inputs(media_file, audio_file, resolution, steps): """Validate user inputs""" errors = [] if media_file is None: errors.append("Por favor sube una imagen o un video.") if audio_file is None: errors.append("Por favor sube un archivo de audio.") if resolution not in ["480p", "720p"]: errors.append("Resoluci贸n no soportada.") if not (5 <= steps <= 50): errors.append("Los pasos deben estar entre 5 y 50.") if errors: raise gr.Error(" | ".join(errors)) @spaces.GPU(duration=300) def generate_video( image_or_video, audio_file, resolution="480p", steps=15, audio_guide_scale=3.0, seed=-1, progress=gr.Progress(track_tqdm=True), ): """ Generate talking video with InfiniteTalk on ZeroGPU """ validate_inputs(image_or_video, audio_file, resolution, steps) try: from wan.utils.multitalk_utils import save_video_ffmpeg progress(0.05, desc="Normalizando audio...") audio, sr = process_audio(audio_file) audio_duration = len(audio) / sr logger.info(f"Audio duration: {audio_duration:.2f}s") progress(0.15, desc="Cargando modelos InfiniteTalk (FP8 ultra-eficiente)...") size = f"infinitetalk-{resolution.replace('p', '')}" device = "cuda" if torch.cuda.is_available() else "cpu" model_manager = get_model_manager() # Load pipeline and audio encoder wan_pipeline = model_manager.load_wan_model(size=size, device=device, quant="fp8") audio_encoder, feature_extractor = model_manager.load_audio_encoder(device=device) progress(0.35, desc="Extrayendo embeddings de audio...") video_length = audio_duration * 25 # 25 FPS audio_feature = np.squeeze( feature_extractor(audio, sampling_rate=sr).input_values ) audio_feature = torch.from_numpy(audio_feature).float().to(device=device) audio_feature = audio_feature.unsqueeze(0) with torch.no_grad(): embeddings = audio_encoder( audio_feature, seq_len=int(video_length), output_hidden_states=True ) if embeddings is None or getattr(embeddings, "hidden_states", None) is None: raise gr.Error("Error al extraer embeddings de audio.") audio_embeddings = torch.stack(embeddings.hidden_states[1:], dim=1).squeeze(0) audio_embeddings = rearrange(audio_embeddings, "b s d -> s b d") audio_embeddings = audio_embeddings.cpu().detach() progress(0.45, desc=f"Generando video con InfiniteTalk ({resolution}, {steps} pasos)...") if seed == -1: seed = random.randint(0, 99999999) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed(seed) temp_dir = Path("/tmp/infinitetalk_work") temp_dir.mkdir(parents=True, exist_ok=True) emb_path = str(temp_dir / f"emb_{seed}.pt") audio_wav_path = str(temp_dir / f"audio_{seed}.wav") output_path = str(temp_dir / f"infinitetalk_{seed}.mp4") torch.save(audio_embeddings, emb_path) sf.write(audio_wav_path, audio, sr) input_clip = { "prompt": "", "cond_video": image_or_video, "cond_audio": {"person1": emb_path}, "video_audio": audio_wav_path, } sample_shift = 7 if resolution == "480p" else 11 with torch.no_grad(): video_tensor = wan_pipeline.generate_infinitetalk( input_clip, size_buckget=size, motion_frame=9, frame_num=81, shift=sample_shift, sampling_steps=int(steps), text_guide_scale=1.0, audio_guide_scale=float(audio_guide_scale), seed=int(seed), offload_model=True, max_frames_num=81, color_correction_strength=1.0, ) save_video_ffmpeg( video_tensor, output_path.replace(".mp4", ""), [audio_wav_path], high_quality_save=False, ) progress(0.95, desc="Finalizando video...") cleanup_gpu() progress(1.0, desc="隆Completado con 茅xito!") return output_path except Exception as e: logger.error(f"Error generating video: {e}") cleanup_gpu() import traceback traceback.print_exc() raise gr.Error(f"Error durante la generaci贸n: {str(e)}") def create_interface(): theme = gr.themes.Soft( primary_hue="blue", secondary_hue="purple", ) with gr.Blocks(title="InfiniteTalk - Audio-Driven Video Generation", theme=theme) as demo: gr.Markdown( """ # 馃幀 InfiniteTalk: Audio-driven Video Generation & Video Dubbing **[InfiniteTalk](https://huggingface.co/MeiGen-AI/InfiniteTalk)** (MeiGen-AI) es un modelo avanzado de generaci贸n de video conducido por audio basado en Wan2.1. Sincroniza no solo los **labios** con extrema precisi贸n, sino tambi茅n la **cabeza, postura corporal y expresiones faciales**, permitiendo dubbing de videos y animaci贸n de retratos de longitud ilimitada. """ ) with gr.Tabs(): with gr.Tab("馃摳 Imagen a Video (Image-to-Video)"): gr.Markdown("Anima una imagen o retrato a partir de una pista de audio.") with gr.Row(): with gr.Column(): img_input = gr.Image( type="filepath", label="Retrato de entrada (PNG o JPG con rostro visible)", ) audio_input_i2v = gr.Audio( type="filepath", label="Audio de sincronizaci贸n (WAV/MP3)", ) with gr.Accordion("鈿欙笍 Configuraci贸n Avanzada", open=False): res_i2v = gr.Radio( choices=["480p", "720p"], value="480p", label="Resoluci贸n", ) steps_i2v = gr.Slider( minimum=10, maximum=50, value=15, step=1, label="Pasos de difusi贸n (Steps) [15 recomendados para ZeroGPU]", ) scale_i2v = gr.Slider( minimum=1.0, maximum=6.0, value=3.0, step=0.5, label="Audio Guide Scale", ) seed_i2v = gr.Number(value=42, label="Semilla (Seed)", precision=0) btn_i2v = gr.Button("馃殌 Generar Video Parlante", variant="primary") with gr.Column(): output_i2v = gr.Video(label="馃幀 Video Generado", autoplay=True) gr.Examples( examples=[ ["examples/single/ref_image.png", "examples/single/1.wav", "480p", 20, 3.0, 42], ], inputs=[img_input, audio_input_i2v, res_i2v, steps_i2v, scale_i2v, seed_i2v], ) btn_i2v.click( fn=generate_video, inputs=[img_input, audio_input_i2v, res_i2v, steps_i2v, scale_i2v, seed_i2v], outputs=[output_i2v], ) with gr.Tab("馃帴 Doblaje de Video (Video Dubbing)"): gr.Markdown("Redobla un video existente con una nueva voz, sincronizando labios y postura corporal.") with gr.Row(): with gr.Column(): vid_input = gr.Video(label="Video original de entrada (MP4)") audio_input_v2v = gr.Audio( type="filepath", label="Nueva pista de audio (WAV/MP3)", ) with gr.Accordion("鈿欙笍 Configuraci贸n Avanzada", open=False): res_v2v = gr.Radio( choices=["480p", "720p"], value="480p", label="Resoluci贸n", ) steps_v2v = gr.Slider( minimum=10, maximum=50, value=15, step=1, label="Pasos de difusi贸n (Steps) [15 recomendados para ZeroGPU]", ) scale_v2v = gr.Slider( minimum=1.0, maximum=6.0, value=3.0, step=0.5, label="Audio Guide Scale", ) seed_v2v = gr.Number(value=42, label="Semilla (Seed)", precision=0) btn_v2v = gr.Button("馃殌 Redoblar Video", variant="primary") with gr.Column(): output_v2v = gr.Video(label="馃幀 Video Doblado", autoplay=True) gr.Examples( examples=[ ["examples/single/ref_video.mp4", "examples/single/1.wav", "480p", 20, 3.0, 42], ], inputs=[vid_input, audio_input_v2v, res_v2v, steps_v2v, scale_v2v, seed_v2v], ) btn_v2v.click( fn=generate_video, inputs=[vid_input, audio_input_v2v, res_v2v, steps_v2v, scale_v2v, seed_v2v], outputs=[output_v2v], ) gr.Markdown( """ --- ### 鈩癸笍 Informaci贸n & Cr茅ditos - **Modelo oficial:** [MeiGen-AI/InfiniteTalk](https://huggingface.co/MeiGen-AI/InfiniteTalk) - **Arquitectura:** DiT basado en Wan2.1 con condicionamiento ac煤stico Wav2Vec2 y cuantizaci贸n FP8 para inferencia 谩gil en ZeroGPU. - **Licencia:** Apache 2.0 """ ) return demo if __name__ == "__main__": # Pre-download weights on CPU startup so ZeroGPU time is not wasted downloading try: logger.info("Checking and pre-downloading model weights on CPU startup...") get_model_manager().prepare_all() except Exception as e: logger.warning(f"Startup model download failed or deferred: {e}") demo = create_interface() demo.queue().launch(show_error=True, ssr_mode=False)