""" Model Manager for InfiniteTalk Handles lazy downloading, caching, and pipeline instantiation from HuggingFace Hub """ import os import json import logging from pathlib import Path import torch from huggingface_hub import hf_hub_download, snapshot_download logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class ModelManager: """Manages InfiniteTalk model downloading and pipeline initialization""" def __init__(self, weights_dir=None): if weights_dir is None: weights_dir = os.environ.get("WEIGHTS_DIR", "./weights") self.weights_dir = Path(weights_dir) self.weights_dir.mkdir(parents=True, exist_ok=True) self.wan_dir = self.weights_dir / "Wan2.1-I2V-14B-480P" self.infinitetalk_dir = self.weights_dir / "InfiniteTalk" self.wav2vec_dir = self.weights_dir / "chinese-wav2vec2-base" self.wan_dir.mkdir(parents=True, exist_ok=True) self.infinitetalk_dir.mkdir(parents=True, exist_ok=True) (self.infinitetalk_dir / "quant_models").mkdir(parents=True, exist_ok=True) self.wav2vec_dir.mkdir(parents=True, exist_ok=True) self.models = {} def prepare_wav2vec(self): """Ensure chinese-wav2vec2-base weights and configs are present""" for filename in ["config.json", "preprocessor_config.json", "pytorch_model.bin"]: dest = self.wav2vec_dir / filename if not dest.exists(): logger.info(f"Downloading {filename} for chinese-wav2vec2-base...") try: hf_hub_download( repo_id="TencentGameMate/chinese-wav2vec2-base", filename=filename, local_dir=str(self.wav2vec_dir), ) except Exception as e: logger.warning(f"Download of {filename} failed: {e}") return str(self.wav2vec_dir) def prepare_wan_base(self): """Ensure Wan2.1 VAE, CLIP, and config.json are present""" files = [ "config.json", "Wan2.1_VAE.pth", "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", ] for filename in files: dest = self.wan_dir / filename if not dest.exists(): logger.info(f"Downloading {filename} from Wan-AI/Wan2.1-I2V-14B-480P...") hf_hub_download( repo_id="Wan-AI/Wan2.1-I2V-14B-480P", filename=filename, local_dir=str(self.wan_dir), ) return str(self.wan_dir) def prepare_infinitetalk_fp8(self): """Ensure InfiniteTalk FP8 models and maps are downloaded""" files = [ "quant_models/infinitetalk_single_fp8.safetensors", "quant_models/infinitetalk_single_fp8.json", "quant_models/t5_fp8.safetensors", "quant_models/t5_map_fp8.json", ] for filename in files: dest = self.infinitetalk_dir / filename if not dest.exists(): logger.info(f"Downloading {filename} from MeiGen-AI/InfiniteTalk...") hf_hub_download( repo_id="MeiGen-AI/InfiniteTalk", filename=filename, local_dir=str(self.infinitetalk_dir), ) return str(self.infinitetalk_dir / "quant_models" / "infinitetalk_single_fp8.safetensors") def prepare_all(self): """Pre-download all required model weights and tokenizers from HF Hub to disk (pure CPU/HTTP)""" logger.info("Pre-downloading model weights to disk (if not already cached)...") self.prepare_wav2vec() self.prepare_wan_base() self.prepare_infinitetalk_fp8() try: from transformers import AutoTokenizer logger.info("Pre-caching tokenizers...") AutoTokenizer.from_pretrained("google/umt5-xxl") AutoTokenizer.from_pretrained("xlm-roberta-large") except Exception as e: logger.warning(f"Tokenizer pre-cache notice: {e}") logger.info("All model weights and tokenizers verified on disk.") def load_wan_model(self, size="infinitetalk-480", device="cuda", quant="fp8"): """Instantiate InfiniteTalk pipeline with FP8 quantization for ZeroGPU efficiency""" if "wan_pipeline" not in self.models: import wan from wan.configs import WAN_CONFIGS self.prepare_wan_base() quant_file = self.prepare_infinitetalk_fp8() task = "infinitetalk-14B" cfg = WAN_CONFIGS[task] logger.info("Initializing InfiniteTalk pipeline...") pipeline = wan.InfiniteTalkPipeline( config=cfg, checkpoint_dir=str(self.wan_dir), quant_dir=quant_file, device_id=device, rank=0, t5_fsdp=False, dit_fsdp=False, use_usp=False, t5_cpu=False, lora_dir=None, lora_scales=None, quant=quant, dit_path=None, infinitetalk_dir=quant_file, ) # Native Wan2.1 offload_model handles T5/CLIP/DiT/VAE swapping cleanly. # Do NOT enable layer-by-layer CPU offload (num_persistent_param_in_dit=0) on ZeroGPU # as it causes extreme PCIe bottleneck (~230s/step). if os.environ.get("ENABLE_VRAM_MANAGEMENT", "0") == "1" and hasattr(pipeline, "enable_vram_management"): pipeline.enable_vram_management() self.models["wan_pipeline"] = pipeline logger.info("InfiniteTalk pipeline initialized successfully.") return self.models["wan_pipeline"] def load_audio_encoder(self, device="cuda"): """Load Wav2Vec2 audio encoder model and feature extractor""" if "audio_encoder" not in self.models: from transformers import Wav2Vec2FeatureExtractor from src.audio_analysis.wav2vec2 import Wav2Vec2Model wav2vec_path = self.prepare_wav2vec() logger.info(f"Loading Wav2Vec2 audio encoder from {wav2vec_path}...") feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(wav2vec_path) audio_encoder = Wav2Vec2Model.from_pretrained(wav2vec_path, attn_implementation="eager") if hasattr(audio_encoder, "to"): audio_encoder = audio_encoder.to(device) audio_encoder.eval() self.models["audio_encoder"] = (audio_encoder, feature_extractor) logger.info("Wav2Vec2 audio encoder loaded successfully.") return self.models["audio_encoder"]