File size: 6,809 Bytes
13c793c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
e579ffb
ca5c28f
e579ffb
 
 
 
ca5c28f
 
 
 
 
 
 
 
e579ffb
13c793c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5045d7c
 
 
 
 
13c793c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
b7c2342
13c793c
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
"""
Model Manager for InfiniteTalk
Handles lazy downloading, caching, and pipeline instantiation from HuggingFace Hub
"""

import os
import json
import logging
from pathlib import Path
import torch
from huggingface_hub import hf_hub_download, snapshot_download

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


class ModelManager:
    """Manages InfiniteTalk model downloading and pipeline initialization"""

    def __init__(self, weights_dir=None):
        if weights_dir is None:
            weights_dir = os.environ.get("WEIGHTS_DIR", "./weights")
        self.weights_dir = Path(weights_dir)
        self.weights_dir.mkdir(parents=True, exist_ok=True)

        self.wan_dir = self.weights_dir / "Wan2.1-I2V-14B-480P"
        self.infinitetalk_dir = self.weights_dir / "InfiniteTalk"
        self.wav2vec_dir = self.weights_dir / "chinese-wav2vec2-base"

        self.wan_dir.mkdir(parents=True, exist_ok=True)
        self.infinitetalk_dir.mkdir(parents=True, exist_ok=True)
        (self.infinitetalk_dir / "quant_models").mkdir(parents=True, exist_ok=True)
        self.wav2vec_dir.mkdir(parents=True, exist_ok=True)

        self.models = {}

    def prepare_wav2vec(self):
        """Ensure chinese-wav2vec2-base weights and configs are present"""
        for filename in ["config.json", "preprocessor_config.json", "pytorch_model.bin"]:
            dest = self.wav2vec_dir / filename
            if not dest.exists():
                logger.info(f"Downloading {filename} for chinese-wav2vec2-base...")
                try:
                    hf_hub_download(
                        repo_id="TencentGameMate/chinese-wav2vec2-base",
                        filename=filename,
                        local_dir=str(self.wav2vec_dir),
                    )
                except Exception as e:
                    logger.warning(f"Download of {filename} failed: {e}")
        return str(self.wav2vec_dir)

    def prepare_wan_base(self):
        """Ensure Wan2.1 VAE, CLIP, and config.json are present"""
        files = [
            "config.json",
            "Wan2.1_VAE.pth",
            "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth",
        ]
        for filename in files:
            dest = self.wan_dir / filename
            if not dest.exists():
                logger.info(f"Downloading {filename} from Wan-AI/Wan2.1-I2V-14B-480P...")
                hf_hub_download(
                    repo_id="Wan-AI/Wan2.1-I2V-14B-480P",
                    filename=filename,
                    local_dir=str(self.wan_dir),
                )
        return str(self.wan_dir)

    def prepare_infinitetalk_fp8(self):
        """Ensure InfiniteTalk FP8 models and maps are downloaded"""
        files = [
            "quant_models/infinitetalk_single_fp8.safetensors",
            "quant_models/infinitetalk_single_fp8.json",
            "quant_models/t5_fp8.safetensors",
            "quant_models/t5_map_fp8.json",
        ]
        for filename in files:
            dest = self.infinitetalk_dir / filename
            if not dest.exists():
                logger.info(f"Downloading {filename} from MeiGen-AI/InfiniteTalk...")
                hf_hub_download(
                    repo_id="MeiGen-AI/InfiniteTalk",
                    filename=filename,
                    local_dir=str(self.infinitetalk_dir),
                )
        return str(self.infinitetalk_dir / "quant_models" / "infinitetalk_single_fp8.safetensors")

    def prepare_all(self):
        """Pre-download all required model weights and tokenizers from HF Hub to disk (pure CPU/HTTP)"""
        logger.info("Pre-downloading model weights to disk (if not already cached)...")
        self.prepare_wav2vec()
        self.prepare_wan_base()
        self.prepare_infinitetalk_fp8()
        try:
            from transformers import AutoTokenizer
            logger.info("Pre-caching tokenizers...")
            AutoTokenizer.from_pretrained("google/umt5-xxl")
            AutoTokenizer.from_pretrained("xlm-roberta-large")
        except Exception as e:
            logger.warning(f"Tokenizer pre-cache notice: {e}")
        logger.info("All model weights and tokenizers verified on disk.")

    def load_wan_model(self, size="infinitetalk-480", device="cuda", quant="fp8"):
        """Instantiate InfiniteTalk pipeline with FP8 quantization for ZeroGPU efficiency"""
        if "wan_pipeline" not in self.models:
            import wan
            from wan.configs import WAN_CONFIGS

            self.prepare_wan_base()
            quant_file = self.prepare_infinitetalk_fp8()

            task = "infinitetalk-14B"
            cfg = WAN_CONFIGS[task]

            logger.info("Initializing InfiniteTalk pipeline...")
            pipeline = wan.InfiniteTalkPipeline(
                config=cfg,
                checkpoint_dir=str(self.wan_dir),
                quant_dir=quant_file,
                device_id=device,
                rank=0,
                t5_fsdp=False,
                dit_fsdp=False,
                use_usp=False,
                t5_cpu=False,
                lora_dir=None,
                lora_scales=None,
                quant=quant,
                dit_path=None,
                infinitetalk_dir=quant_file,
            )

            # Native Wan2.1 offload_model handles T5/CLIP/DiT/VAE swapping cleanly.
            # Do NOT enable layer-by-layer CPU offload (num_persistent_param_in_dit=0) on ZeroGPU
            # as it causes extreme PCIe bottleneck (~230s/step).
            if os.environ.get("ENABLE_VRAM_MANAGEMENT", "0") == "1" and hasattr(pipeline, "enable_vram_management"):
                pipeline.enable_vram_management()

            self.models["wan_pipeline"] = pipeline
            logger.info("InfiniteTalk pipeline initialized successfully.")

        return self.models["wan_pipeline"]

    def load_audio_encoder(self, device="cuda"):
        """Load Wav2Vec2 audio encoder model and feature extractor"""
        if "audio_encoder" not in self.models:
            from transformers import Wav2Vec2FeatureExtractor
            from src.audio_analysis.wav2vec2 import Wav2Vec2Model

            wav2vec_path = self.prepare_wav2vec()
            logger.info(f"Loading Wav2Vec2 audio encoder from {wav2vec_path}...")

            feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(wav2vec_path)
            audio_encoder = Wav2Vec2Model.from_pretrained(wav2vec_path, attn_implementation="eager")
            if hasattr(audio_encoder, "to"):
                audio_encoder = audio_encoder.to(device)
            audio_encoder.eval()

            self.models["audio_encoder"] = (audio_encoder, feature_extractor)
            logger.info("Wav2Vec2 audio encoder loaded successfully.")

        return self.models["audio_encoder"]