infinitetalk / utils /model_loader.py
Daniel
Disable layer-by-layer VRAM offloading to run DiT natively on GPU
5045d7c
Raw History Blame Contribute Delete
6.81 kB
"""
Model Manager for InfiniteTalk
Handles lazy downloading, caching, and pipeline instantiation from HuggingFace Hub
"""
import os
import json
import logging
from pathlib import Path
import torch
from huggingface_hub import hf_hub_download, snapshot_download
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class ModelManager:
"""Manages InfiniteTalk model downloading and pipeline initialization"""
def __init__(self, weights_dir=None):
if weights_dir is None:
weights_dir = os.environ.get("WEIGHTS_DIR", "./weights")
self.weights_dir = Path(weights_dir)
self.weights_dir.mkdir(parents=True, exist_ok=True)
self.wan_dir = self.weights_dir / "Wan2.1-I2V-14B-480P"
self.infinitetalk_dir = self.weights_dir / "InfiniteTalk"
self.wav2vec_dir = self.weights_dir / "chinese-wav2vec2-base"
self.wan_dir.mkdir(parents=True, exist_ok=True)
self.infinitetalk_dir.mkdir(parents=True, exist_ok=True)
(self.infinitetalk_dir / "quant_models").mkdir(parents=True, exist_ok=True)
self.wav2vec_dir.mkdir(parents=True, exist_ok=True)
self.models = {}
def prepare_wav2vec(self):
"""Ensure chinese-wav2vec2-base weights and configs are present"""
for filename in ["config.json", "preprocessor_config.json", "pytorch_model.bin"]:
dest = self.wav2vec_dir / filename
if not dest.exists():
logger.info(f"Downloading {filename} for chinese-wav2vec2-base...")
try:
hf_hub_download(
repo_id="TencentGameMate/chinese-wav2vec2-base",
filename=filename,
local_dir=str(self.wav2vec_dir),
)
except Exception as e:
logger.warning(f"Download of {filename} failed: {e}")
return str(self.wav2vec_dir)
def prepare_wan_base(self):
"""Ensure Wan2.1 VAE, CLIP, and config.json are present"""
files = [
"config.json",
"Wan2.1_VAE.pth",
"models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth",
]
for filename in files:
dest = self.wan_dir / filename
if not dest.exists():
logger.info(f"Downloading {filename} from Wan-AI/Wan2.1-I2V-14B-480P...")
hf_hub_download(
repo_id="Wan-AI/Wan2.1-I2V-14B-480P",
filename=filename,
local_dir=str(self.wan_dir),
)
return str(self.wan_dir)
def prepare_infinitetalk_fp8(self):
"""Ensure InfiniteTalk FP8 models and maps are downloaded"""
files = [
"quant_models/infinitetalk_single_fp8.safetensors",
"quant_models/infinitetalk_single_fp8.json",
"quant_models/t5_fp8.safetensors",
"quant_models/t5_map_fp8.json",
]
for filename in files:
dest = self.infinitetalk_dir / filename
if not dest.exists():
logger.info(f"Downloading {filename} from MeiGen-AI/InfiniteTalk...")
hf_hub_download(
repo_id="MeiGen-AI/InfiniteTalk",
filename=filename,
local_dir=str(self.infinitetalk_dir),
)
return str(self.infinitetalk_dir / "quant_models" / "infinitetalk_single_fp8.safetensors")
def prepare_all(self):
"""Pre-download all required model weights and tokenizers from HF Hub to disk (pure CPU/HTTP)"""
logger.info("Pre-downloading model weights to disk (if not already cached)...")
self.prepare_wav2vec()
self.prepare_wan_base()
self.prepare_infinitetalk_fp8()
try:
from transformers import AutoTokenizer
logger.info("Pre-caching tokenizers...")
AutoTokenizer.from_pretrained("google/umt5-xxl")
AutoTokenizer.from_pretrained("xlm-roberta-large")
except Exception as e:
logger.warning(f"Tokenizer pre-cache notice: {e}")
logger.info("All model weights and tokenizers verified on disk.")
def load_wan_model(self, size="infinitetalk-480", device="cuda", quant="fp8"):
"""Instantiate InfiniteTalk pipeline with FP8 quantization for ZeroGPU efficiency"""
if "wan_pipeline" not in self.models:
import wan
from wan.configs import WAN_CONFIGS
self.prepare_wan_base()
quant_file = self.prepare_infinitetalk_fp8()
task = "infinitetalk-14B"
cfg = WAN_CONFIGS[task]
logger.info("Initializing InfiniteTalk pipeline...")
pipeline = wan.InfiniteTalkPipeline(
config=cfg,
checkpoint_dir=str(self.wan_dir),
quant_dir=quant_file,
device_id=device,
rank=0,
t5_fsdp=False,
dit_fsdp=False,
use_usp=False,
t5_cpu=False,
lora_dir=None,
lora_scales=None,
quant=quant,
dit_path=None,
infinitetalk_dir=quant_file,
)
# Native Wan2.1 offload_model handles T5/CLIP/DiT/VAE swapping cleanly.
# Do NOT enable layer-by-layer CPU offload (num_persistent_param_in_dit=0) on ZeroGPU
# as it causes extreme PCIe bottleneck (~230s/step).
if os.environ.get("ENABLE_VRAM_MANAGEMENT", "0") == "1" and hasattr(pipeline, "enable_vram_management"):
pipeline.enable_vram_management()
self.models["wan_pipeline"] = pipeline
logger.info("InfiniteTalk pipeline initialized successfully.")
return self.models["wan_pipeline"]
def load_audio_encoder(self, device="cuda"):
"""Load Wav2Vec2 audio encoder model and feature extractor"""
if "audio_encoder" not in self.models:
from transformers import Wav2Vec2FeatureExtractor
from src.audio_analysis.wav2vec2 import Wav2Vec2Model
wav2vec_path = self.prepare_wav2vec()
logger.info(f"Loading Wav2Vec2 audio encoder from {wav2vec_path}...")
feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(wav2vec_path)
audio_encoder = Wav2Vec2Model.from_pretrained(wav2vec_path, attn_implementation="eager")
if hasattr(audio_encoder, "to"):
audio_encoder = audio_encoder.to(device)
audio_encoder.eval()
self.models["audio_encoder"] = (audio_encoder, feature_extractor)
logger.info("Wav2Vec2 audio encoder loaded successfully.")
return self.models["audio_encoder"]