Spaces:
Running on Zero
Running on Zero
Daniel
Optimize ZeroGPU duration to 300s, set recommended 15 steps, and remove torch.compile lag
d05af4e Download app.py from avila110/infinitetalk: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/spaces/avila110/infinitetalk/resolve/main/app.py
- Command line
-
hf download hf://spaces/avila110/infinitetalk/app.py
-
curl -L -o app.py https://huggingface.co/spaces/avila110/infinitetalk/resolve/main/app.py
13.9 kB
| """ | |
| InfiniteTalk - Talking Video Generator | |
| Gradio Space for Hugging Face (ZeroGPU Compatible) | |
| Model: https://huggingface.co/MeiGen-AI/InfiniteTalk | |
| """ | |
| import os | |
| import sys | |
| # ZeroGPU: import spaces before any CUDA or torch imports | |
| try: | |
| import spaces | |
| except ImportError: | |
| class spaces: | |
| def GPU(duration=180): | |
| def decorator(f): | |
| return f | |
| return decorator | |
| os.environ["TORCHVISION_DISABLE_META_REGISTRATIONS"] = "1" | |
| os.environ["TORCH_LOGS"] = "-all" | |
| import random | |
| import logging | |
| import warnings | |
| from pathlib import Path | |
| import gradio as gr | |
| import torch | |
| import numpy as np | |
| # Defensive patch for gradio_client json schema typing bug | |
| try: | |
| import gradio_client.utils as client_utils | |
| _orig_get_type = client_utils.get_type | |
| def _safe_get_type(schema): | |
| if not isinstance(schema, dict): | |
| return "Any" | |
| return _orig_get_type(schema) | |
| client_utils.get_type = _safe_get_type | |
| except Exception: | |
| pass | |
| from PIL import Image | |
| from einops import rearrange | |
| import librosa | |
| import soundfile as sf | |
| import pyloudnorm as pyln | |
| # Suppress warnings | |
| warnings.filterwarnings("ignore") | |
| # Setup logging | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| # Add current directory to path | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| # Lazy model manager and GPU cleanup to prevent premature CUDA driver init in ZeroGPU main process | |
| _model_manager = None | |
| def get_model_manager(): | |
| global _model_manager | |
| if _model_manager is None: | |
| from utils.model_loader import ModelManager | |
| _model_manager = ModelManager() | |
| return _model_manager | |
| def cleanup_gpu(): | |
| import gc | |
| gc.collect() | |
| if torch.cuda.is_available(): | |
| try: | |
| torch.cuda.empty_cache() | |
| torch.cuda.synchronize() | |
| except Exception: | |
| pass | |
| def loudness_norm(audio_array, sr=16000, lufs=-20.0): | |
| """Normalize audio loudness using pyloudnorm""" | |
| try: | |
| meter = pyln.Meter(sr) | |
| loudness = meter.integrated_loudness(audio_array) | |
| if abs(loudness) > 100: | |
| return audio_array | |
| normalized_audio = pyln.normalize.loudness(audio_array, loudness, lufs) | |
| return normalized_audio | |
| except Exception as e: | |
| logger.warning(f"Loudness normalization fallback: {e}") | |
| return audio_array | |
| def process_audio(audio_path, target_sr=16000): | |
| """Process audio file for InfiniteTalk""" | |
| try: | |
| audio, sr = librosa.load(audio_path, sr=target_sr) | |
| audio = loudness_norm(audio, sr) | |
| if len(audio.shape) > 1: | |
| audio = np.mean(audio, axis=1) | |
| return audio, sr | |
| except Exception as e: | |
| logger.error(f"Error processing audio: {e}") | |
| raise gr.Error(f"Fallo al procesar audio: {str(e)}") | |
| def validate_inputs(media_file, audio_file, resolution, steps): | |
| """Validate user inputs""" | |
| errors = [] | |
| if media_file is None: | |
| errors.append("Por favor sube una imagen o un video.") | |
| if audio_file is None: | |
| errors.append("Por favor sube un archivo de audio.") | |
| if resolution not in ["480p", "720p"]: | |
| errors.append("Resoluci贸n no soportada.") | |
| if not (5 <= steps <= 50): | |
| errors.append("Los pasos deben estar entre 5 y 50.") | |
| if errors: | |
| raise gr.Error(" | ".join(errors)) | |
| def generate_video( | |
| image_or_video, | |
| audio_file, | |
| resolution="480p", | |
| steps=15, | |
| audio_guide_scale=3.0, | |
| seed=-1, | |
| progress=gr.Progress(track_tqdm=True), | |
| ): | |
| """ | |
| Generate talking video with InfiniteTalk on ZeroGPU | |
| """ | |
| validate_inputs(image_or_video, audio_file, resolution, steps) | |
| try: | |
| from wan.utils.multitalk_utils import save_video_ffmpeg | |
| progress(0.05, desc="Normalizando audio...") | |
| audio, sr = process_audio(audio_file) | |
| audio_duration = len(audio) / sr | |
| logger.info(f"Audio duration: {audio_duration:.2f}s") | |
| progress(0.15, desc="Cargando modelos InfiniteTalk (FP8 ultra-eficiente)...") | |
| size = f"infinitetalk-{resolution.replace('p', '')}" | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model_manager = get_model_manager() | |
| # Load pipeline and audio encoder | |
| wan_pipeline = model_manager.load_wan_model(size=size, device=device, quant="fp8") | |
| audio_encoder, feature_extractor = model_manager.load_audio_encoder(device=device) | |
| progress(0.35, desc="Extrayendo embeddings de audio...") | |
| video_length = audio_duration * 25 # 25 FPS | |
| audio_feature = np.squeeze( | |
| feature_extractor(audio, sampling_rate=sr).input_values | |
| ) | |
| audio_feature = torch.from_numpy(audio_feature).float().to(device=device) | |
| audio_feature = audio_feature.unsqueeze(0) | |
| with torch.no_grad(): | |
| embeddings = audio_encoder( | |
| audio_feature, seq_len=int(video_length), output_hidden_states=True | |
| ) | |
| if embeddings is None or getattr(embeddings, "hidden_states", None) is None: | |
| raise gr.Error("Error al extraer embeddings de audio.") | |
| audio_embeddings = torch.stack(embeddings.hidden_states[1:], dim=1).squeeze(0) | |
| audio_embeddings = rearrange(audio_embeddings, "b s d -> s b d") | |
| audio_embeddings = audio_embeddings.cpu().detach() | |
| progress(0.45, desc=f"Generando video con InfiniteTalk ({resolution}, {steps} pasos)...") | |
| if seed == -1: | |
| seed = random.randint(0, 99999999) | |
| torch.manual_seed(seed) | |
| if torch.cuda.is_available(): | |
| torch.cuda.manual_seed(seed) | |
| temp_dir = Path("/tmp/infinitetalk_work") | |
| temp_dir.mkdir(parents=True, exist_ok=True) | |
| emb_path = str(temp_dir / f"emb_{seed}.pt") | |
| audio_wav_path = str(temp_dir / f"audio_{seed}.wav") | |
| output_path = str(temp_dir / f"infinitetalk_{seed}.mp4") | |
| torch.save(audio_embeddings, emb_path) | |
| sf.write(audio_wav_path, audio, sr) | |
| input_clip = { | |
| "prompt": "", | |
| "cond_video": image_or_video, | |
| "cond_audio": {"person1": emb_path}, | |
| "video_audio": audio_wav_path, | |
| } | |
| sample_shift = 7 if resolution == "480p" else 11 | |
| with torch.no_grad(): | |
| video_tensor = wan_pipeline.generate_infinitetalk( | |
| input_clip, | |
| size_buckget=size, | |
| motion_frame=9, | |
| frame_num=81, | |
| shift=sample_shift, | |
| sampling_steps=int(steps), | |
| text_guide_scale=1.0, | |
| audio_guide_scale=float(audio_guide_scale), | |
| seed=int(seed), | |
| offload_model=True, | |
| max_frames_num=81, | |
| color_correction_strength=1.0, | |
| ) | |
| save_video_ffmpeg( | |
| video_tensor, | |
| output_path.replace(".mp4", ""), | |
| [audio_wav_path], | |
| high_quality_save=False, | |
| ) | |
| progress(0.95, desc="Finalizando video...") | |
| cleanup_gpu() | |
| progress(1.0, desc="隆Completado con 茅xito!") | |
| return output_path | |
| except Exception as e: | |
| logger.error(f"Error generating video: {e}") | |
| cleanup_gpu() | |
| import traceback | |
| traceback.print_exc() | |
| raise gr.Error(f"Error durante la generaci贸n: {str(e)}") | |
| def create_interface(): | |
| theme = gr.themes.Soft( | |
| primary_hue="blue", | |
| secondary_hue="purple", | |
| ) | |
| with gr.Blocks(title="InfiniteTalk - Audio-Driven Video Generation", theme=theme) as demo: | |
| gr.Markdown( | |
| """ | |
| # 馃幀 InfiniteTalk: Audio-driven Video Generation & Video Dubbing | |
| **[InfiniteTalk](https://huggingface.co/MeiGen-AI/InfiniteTalk)** (MeiGen-AI) es un modelo avanzado de generaci贸n de video conducido por audio basado en Wan2.1. | |
| Sincroniza no solo los **labios** con extrema precisi贸n, sino tambi茅n la **cabeza, postura corporal y expresiones faciales**, permitiendo dubbing de videos y animaci贸n de retratos de longitud ilimitada. | |
| """ | |
| ) | |
| with gr.Tabs(): | |
| with gr.Tab("馃摳 Imagen a Video (Image-to-Video)"): | |
| gr.Markdown("Anima una imagen o retrato a partir de una pista de audio.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| img_input = gr.Image( | |
| type="filepath", | |
| label="Retrato de entrada (PNG o JPG con rostro visible)", | |
| ) | |
| audio_input_i2v = gr.Audio( | |
| type="filepath", | |
| label="Audio de sincronizaci贸n (WAV/MP3)", | |
| ) | |
| with gr.Accordion("鈿欙笍 Configuraci贸n Avanzada", open=False): | |
| res_i2v = gr.Radio( | |
| choices=["480p", "720p"], | |
| value="480p", | |
| label="Resoluci贸n", | |
| ) | |
| steps_i2v = gr.Slider( | |
| minimum=10, | |
| maximum=50, | |
| value=15, | |
| step=1, | |
| label="Pasos de difusi贸n (Steps) [15 recomendados para ZeroGPU]", | |
| ) | |
| scale_i2v = gr.Slider( | |
| minimum=1.0, | |
| maximum=6.0, | |
| value=3.0, | |
| step=0.5, | |
| label="Audio Guide Scale", | |
| ) | |
| seed_i2v = gr.Number(value=42, label="Semilla (Seed)", precision=0) | |
| btn_i2v = gr.Button("馃殌 Generar Video Parlante", variant="primary") | |
| with gr.Column(): | |
| output_i2v = gr.Video(label="馃幀 Video Generado", autoplay=True) | |
| gr.Examples( | |
| examples=[ | |
| ["examples/single/ref_image.png", "examples/single/1.wav", "480p", 20, 3.0, 42], | |
| ], | |
| inputs=[img_input, audio_input_i2v, res_i2v, steps_i2v, scale_i2v, seed_i2v], | |
| ) | |
| btn_i2v.click( | |
| fn=generate_video, | |
| inputs=[img_input, audio_input_i2v, res_i2v, steps_i2v, scale_i2v, seed_i2v], | |
| outputs=[output_i2v], | |
| ) | |
| with gr.Tab("馃帴 Doblaje de Video (Video Dubbing)"): | |
| gr.Markdown("Redobla un video existente con una nueva voz, sincronizando labios y postura corporal.") | |
| with gr.Row(): | |
| with gr.Column(): | |
| vid_input = gr.Video(label="Video original de entrada (MP4)") | |
| audio_input_v2v = gr.Audio( | |
| type="filepath", | |
| label="Nueva pista de audio (WAV/MP3)", | |
| ) | |
| with gr.Accordion("鈿欙笍 Configuraci贸n Avanzada", open=False): | |
| res_v2v = gr.Radio( | |
| choices=["480p", "720p"], | |
| value="480p", | |
| label="Resoluci贸n", | |
| ) | |
| steps_v2v = gr.Slider( | |
| minimum=10, | |
| maximum=50, | |
| value=15, | |
| step=1, | |
| label="Pasos de difusi贸n (Steps) [15 recomendados para ZeroGPU]", | |
| ) | |
| scale_v2v = gr.Slider( | |
| minimum=1.0, | |
| maximum=6.0, | |
| value=3.0, | |
| step=0.5, | |
| label="Audio Guide Scale", | |
| ) | |
| seed_v2v = gr.Number(value=42, label="Semilla (Seed)", precision=0) | |
| btn_v2v = gr.Button("馃殌 Redoblar Video", variant="primary") | |
| with gr.Column(): | |
| output_v2v = gr.Video(label="馃幀 Video Doblado", autoplay=True) | |
| gr.Examples( | |
| examples=[ | |
| ["examples/single/ref_video.mp4", "examples/single/1.wav", "480p", 20, 3.0, 42], | |
| ], | |
| inputs=[vid_input, audio_input_v2v, res_v2v, steps_v2v, scale_v2v, seed_v2v], | |
| ) | |
| btn_v2v.click( | |
| fn=generate_video, | |
| inputs=[vid_input, audio_input_v2v, res_v2v, steps_v2v, scale_v2v, seed_v2v], | |
| outputs=[output_v2v], | |
| ) | |
| gr.Markdown( | |
| """ | |
| --- | |
| ### 鈩癸笍 Informaci贸n & Cr茅ditos | |
| - **Modelo oficial:** [MeiGen-AI/InfiniteTalk](https://huggingface.co/MeiGen-AI/InfiniteTalk) | |
| - **Arquitectura:** DiT basado en Wan2.1 con condicionamiento ac煤stico Wav2Vec2 y cuantizaci贸n FP8 para inferencia 谩gil en ZeroGPU. | |
| - **Licencia:** Apache 2.0 | |
| """ | |
| ) | |
| return demo | |
| if __name__ == "__main__": | |
| # Pre-download weights on CPU startup so ZeroGPU time is not wasted downloading | |
| try: | |
| logger.info("Checking and pre-downloading model weights on CPU startup...") | |
| get_model_manager().prepare_all() | |
| except Exception as e: | |
| logger.warning(f"Startup model download failed or deferred: {e}") | |
| demo = create_interface() | |
| demo.queue().launch(show_error=True, ssr_mode=False) | |