Spaces:
Running on Zero
Running on Zero
Download utils/model_loader.py from avila110/infinitetalk: direct link, hf CLI and curl.
- Browser
- Download file 6.81 kB
-
https://huggingface.co/spaces/avila110/infinitetalk/resolve/main/utils/model_loader.py
- Command line
-
hf download hf://spaces/avila110/infinitetalk/utils/model_loader.py
-
curl -L -o model_loader.py https://huggingface.co/spaces/avila110/infinitetalk/resolve/main/utils/model_loader.py
6.81 kB
| """ | |
| Model Manager for InfiniteTalk | |
| Handles lazy downloading, caching, and pipeline instantiation from HuggingFace Hub | |
| """ | |
| import os | |
| import json | |
| import logging | |
| from pathlib import Path | |
| import torch | |
| from huggingface_hub import hf_hub_download, snapshot_download | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| class ModelManager: | |
| """Manages InfiniteTalk model downloading and pipeline initialization""" | |
| def __init__(self, weights_dir=None): | |
| if weights_dir is None: | |
| weights_dir = os.environ.get("WEIGHTS_DIR", "./weights") | |
| self.weights_dir = Path(weights_dir) | |
| self.weights_dir.mkdir(parents=True, exist_ok=True) | |
| self.wan_dir = self.weights_dir / "Wan2.1-I2V-14B-480P" | |
| self.infinitetalk_dir = self.weights_dir / "InfiniteTalk" | |
| self.wav2vec_dir = self.weights_dir / "chinese-wav2vec2-base" | |
| self.wan_dir.mkdir(parents=True, exist_ok=True) | |
| self.infinitetalk_dir.mkdir(parents=True, exist_ok=True) | |
| (self.infinitetalk_dir / "quant_models").mkdir(parents=True, exist_ok=True) | |
| self.wav2vec_dir.mkdir(parents=True, exist_ok=True) | |
| self.models = {} | |
| def prepare_wav2vec(self): | |
| """Ensure chinese-wav2vec2-base weights and configs are present""" | |
| for filename in ["config.json", "preprocessor_config.json", "pytorch_model.bin"]: | |
| dest = self.wav2vec_dir / filename | |
| if not dest.exists(): | |
| logger.info(f"Downloading {filename} for chinese-wav2vec2-base...") | |
| try: | |
| hf_hub_download( | |
| repo_id="TencentGameMate/chinese-wav2vec2-base", | |
| filename=filename, | |
| local_dir=str(self.wav2vec_dir), | |
| ) | |
| except Exception as e: | |
| logger.warning(f"Download of {filename} failed: {e}") | |
| return str(self.wav2vec_dir) | |
| def prepare_wan_base(self): | |
| """Ensure Wan2.1 VAE, CLIP, and config.json are present""" | |
| files = [ | |
| "config.json", | |
| "Wan2.1_VAE.pth", | |
| "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", | |
| ] | |
| for filename in files: | |
| dest = self.wan_dir / filename | |
| if not dest.exists(): | |
| logger.info(f"Downloading {filename} from Wan-AI/Wan2.1-I2V-14B-480P...") | |
| hf_hub_download( | |
| repo_id="Wan-AI/Wan2.1-I2V-14B-480P", | |
| filename=filename, | |
| local_dir=str(self.wan_dir), | |
| ) | |
| return str(self.wan_dir) | |
| def prepare_infinitetalk_fp8(self): | |
| """Ensure InfiniteTalk FP8 models and maps are downloaded""" | |
| files = [ | |
| "quant_models/infinitetalk_single_fp8.safetensors", | |
| "quant_models/infinitetalk_single_fp8.json", | |
| "quant_models/t5_fp8.safetensors", | |
| "quant_models/t5_map_fp8.json", | |
| ] | |
| for filename in files: | |
| dest = self.infinitetalk_dir / filename | |
| if not dest.exists(): | |
| logger.info(f"Downloading {filename} from MeiGen-AI/InfiniteTalk...") | |
| hf_hub_download( | |
| repo_id="MeiGen-AI/InfiniteTalk", | |
| filename=filename, | |
| local_dir=str(self.infinitetalk_dir), | |
| ) | |
| return str(self.infinitetalk_dir / "quant_models" / "infinitetalk_single_fp8.safetensors") | |
| def prepare_all(self): | |
| """Pre-download all required model weights and tokenizers from HF Hub to disk (pure CPU/HTTP)""" | |
| logger.info("Pre-downloading model weights to disk (if not already cached)...") | |
| self.prepare_wav2vec() | |
| self.prepare_wan_base() | |
| self.prepare_infinitetalk_fp8() | |
| try: | |
| from transformers import AutoTokenizer | |
| logger.info("Pre-caching tokenizers...") | |
| AutoTokenizer.from_pretrained("google/umt5-xxl") | |
| AutoTokenizer.from_pretrained("xlm-roberta-large") | |
| except Exception as e: | |
| logger.warning(f"Tokenizer pre-cache notice: {e}") | |
| logger.info("All model weights and tokenizers verified on disk.") | |
| def load_wan_model(self, size="infinitetalk-480", device="cuda", quant="fp8"): | |
| """Instantiate InfiniteTalk pipeline with FP8 quantization for ZeroGPU efficiency""" | |
| if "wan_pipeline" not in self.models: | |
| import wan | |
| from wan.configs import WAN_CONFIGS | |
| self.prepare_wan_base() | |
| quant_file = self.prepare_infinitetalk_fp8() | |
| task = "infinitetalk-14B" | |
| cfg = WAN_CONFIGS[task] | |
| logger.info("Initializing InfiniteTalk pipeline...") | |
| pipeline = wan.InfiniteTalkPipeline( | |
| config=cfg, | |
| checkpoint_dir=str(self.wan_dir), | |
| quant_dir=quant_file, | |
| device_id=device, | |
| rank=0, | |
| t5_fsdp=False, | |
| dit_fsdp=False, | |
| use_usp=False, | |
| t5_cpu=False, | |
| lora_dir=None, | |
| lora_scales=None, | |
| quant=quant, | |
| dit_path=None, | |
| infinitetalk_dir=quant_file, | |
| ) | |
| # Native Wan2.1 offload_model handles T5/CLIP/DiT/VAE swapping cleanly. | |
| # Do NOT enable layer-by-layer CPU offload (num_persistent_param_in_dit=0) on ZeroGPU | |
| # as it causes extreme PCIe bottleneck (~230s/step). | |
| if os.environ.get("ENABLE_VRAM_MANAGEMENT", "0") == "1" and hasattr(pipeline, "enable_vram_management"): | |
| pipeline.enable_vram_management() | |
| self.models["wan_pipeline"] = pipeline | |
| logger.info("InfiniteTalk pipeline initialized successfully.") | |
| return self.models["wan_pipeline"] | |
| def load_audio_encoder(self, device="cuda"): | |
| """Load Wav2Vec2 audio encoder model and feature extractor""" | |
| if "audio_encoder" not in self.models: | |
| from transformers import Wav2Vec2FeatureExtractor | |
| from src.audio_analysis.wav2vec2 import Wav2Vec2Model | |
| wav2vec_path = self.prepare_wav2vec() | |
| logger.info(f"Loading Wav2Vec2 audio encoder from {wav2vec_path}...") | |
| feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(wav2vec_path) | |
| audio_encoder = Wav2Vec2Model.from_pretrained(wav2vec_path, attn_implementation="eager") | |
| if hasattr(audio_encoder, "to"): | |
| audio_encoder = audio_encoder.to(device) | |
| audio_encoder.eval() | |
| self.models["audio_encoder"] = (audio_encoder, feature_extractor) | |
| logger.info("Wav2Vec2 audio encoder loaded successfully.") | |
| return self.models["audio_encoder"] | |