Spaces:
Running on Zero
Running on Zero
File size: 6,809 Bytes
13c793c e579ffb ca5c28f e579ffb ca5c28f e579ffb 13c793c 5045d7c 13c793c b7c2342 13c793c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | """
Model Manager for InfiniteTalk
Handles lazy downloading, caching, and pipeline instantiation from HuggingFace Hub
"""
import os
import json
import logging
from pathlib import Path
import torch
from huggingface_hub import hf_hub_download, snapshot_download
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class ModelManager:
"""Manages InfiniteTalk model downloading and pipeline initialization"""
def __init__(self, weights_dir=None):
if weights_dir is None:
weights_dir = os.environ.get("WEIGHTS_DIR", "./weights")
self.weights_dir = Path(weights_dir)
self.weights_dir.mkdir(parents=True, exist_ok=True)
self.wan_dir = self.weights_dir / "Wan2.1-I2V-14B-480P"
self.infinitetalk_dir = self.weights_dir / "InfiniteTalk"
self.wav2vec_dir = self.weights_dir / "chinese-wav2vec2-base"
self.wan_dir.mkdir(parents=True, exist_ok=True)
self.infinitetalk_dir.mkdir(parents=True, exist_ok=True)
(self.infinitetalk_dir / "quant_models").mkdir(parents=True, exist_ok=True)
self.wav2vec_dir.mkdir(parents=True, exist_ok=True)
self.models = {}
def prepare_wav2vec(self):
"""Ensure chinese-wav2vec2-base weights and configs are present"""
for filename in ["config.json", "preprocessor_config.json", "pytorch_model.bin"]:
dest = self.wav2vec_dir / filename
if not dest.exists():
logger.info(f"Downloading {filename} for chinese-wav2vec2-base...")
try:
hf_hub_download(
repo_id="TencentGameMate/chinese-wav2vec2-base",
filename=filename,
local_dir=str(self.wav2vec_dir),
)
except Exception as e:
logger.warning(f"Download of {filename} failed: {e}")
return str(self.wav2vec_dir)
def prepare_wan_base(self):
"""Ensure Wan2.1 VAE, CLIP, and config.json are present"""
files = [
"config.json",
"Wan2.1_VAE.pth",
"models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth",
]
for filename in files:
dest = self.wan_dir / filename
if not dest.exists():
logger.info(f"Downloading {filename} from Wan-AI/Wan2.1-I2V-14B-480P...")
hf_hub_download(
repo_id="Wan-AI/Wan2.1-I2V-14B-480P",
filename=filename,
local_dir=str(self.wan_dir),
)
return str(self.wan_dir)
def prepare_infinitetalk_fp8(self):
"""Ensure InfiniteTalk FP8 models and maps are downloaded"""
files = [
"quant_models/infinitetalk_single_fp8.safetensors",
"quant_models/infinitetalk_single_fp8.json",
"quant_models/t5_fp8.safetensors",
"quant_models/t5_map_fp8.json",
]
for filename in files:
dest = self.infinitetalk_dir / filename
if not dest.exists():
logger.info(f"Downloading {filename} from MeiGen-AI/InfiniteTalk...")
hf_hub_download(
repo_id="MeiGen-AI/InfiniteTalk",
filename=filename,
local_dir=str(self.infinitetalk_dir),
)
return str(self.infinitetalk_dir / "quant_models" / "infinitetalk_single_fp8.safetensors")
def prepare_all(self):
"""Pre-download all required model weights and tokenizers from HF Hub to disk (pure CPU/HTTP)"""
logger.info("Pre-downloading model weights to disk (if not already cached)...")
self.prepare_wav2vec()
self.prepare_wan_base()
self.prepare_infinitetalk_fp8()
try:
from transformers import AutoTokenizer
logger.info("Pre-caching tokenizers...")
AutoTokenizer.from_pretrained("google/umt5-xxl")
AutoTokenizer.from_pretrained("xlm-roberta-large")
except Exception as e:
logger.warning(f"Tokenizer pre-cache notice: {e}")
logger.info("All model weights and tokenizers verified on disk.")
def load_wan_model(self, size="infinitetalk-480", device="cuda", quant="fp8"):
"""Instantiate InfiniteTalk pipeline with FP8 quantization for ZeroGPU efficiency"""
if "wan_pipeline" not in self.models:
import wan
from wan.configs import WAN_CONFIGS
self.prepare_wan_base()
quant_file = self.prepare_infinitetalk_fp8()
task = "infinitetalk-14B"
cfg = WAN_CONFIGS[task]
logger.info("Initializing InfiniteTalk pipeline...")
pipeline = wan.InfiniteTalkPipeline(
config=cfg,
checkpoint_dir=str(self.wan_dir),
quant_dir=quant_file,
device_id=device,
rank=0,
t5_fsdp=False,
dit_fsdp=False,
use_usp=False,
t5_cpu=False,
lora_dir=None,
lora_scales=None,
quant=quant,
dit_path=None,
infinitetalk_dir=quant_file,
)
# Native Wan2.1 offload_model handles T5/CLIP/DiT/VAE swapping cleanly.
# Do NOT enable layer-by-layer CPU offload (num_persistent_param_in_dit=0) on ZeroGPU
# as it causes extreme PCIe bottleneck (~230s/step).
if os.environ.get("ENABLE_VRAM_MANAGEMENT", "0") == "1" and hasattr(pipeline, "enable_vram_management"):
pipeline.enable_vram_management()
self.models["wan_pipeline"] = pipeline
logger.info("InfiniteTalk pipeline initialized successfully.")
return self.models["wan_pipeline"]
def load_audio_encoder(self, device="cuda"):
"""Load Wav2Vec2 audio encoder model and feature extractor"""
if "audio_encoder" not in self.models:
from transformers import Wav2Vec2FeatureExtractor
from src.audio_analysis.wav2vec2 import Wav2Vec2Model
wav2vec_path = self.prepare_wav2vec()
logger.info(f"Loading Wav2Vec2 audio encoder from {wav2vec_path}...")
feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(wav2vec_path)
audio_encoder = Wav2Vec2Model.from_pretrained(wav2vec_path, attn_implementation="eager")
if hasattr(audio_encoder, "to"):
audio_encoder = audio_encoder.to(device)
audio_encoder.eval()
self.models["audio_encoder"] = (audio_encoder, feature_extractor)
logger.info("Wav2Vec2 audio encoder loaded successfully.")
return self.models["audio_encoder"]
|