File size: 7,195 Bytes
6c7d2b8
 
 
 
 
 
 
 
 
c0b9b08
6c7d2b8
 
 
 
 
c0b9b08
 
 
 
 
6c7d2b8
 
c0b9b08
5801d6e
6c7d2b8
c0b9b08
6c7d2b8
 
c0b9b08
 
 
6c7d2b8
c0b9b08
 
6c7d2b8
c0b9b08
 
 
 
 
 
6c7d2b8
c0b9b08
6c7d2b8
 
 
c0b9b08
 
6c7d2b8
c0b9b08
 
 
 
6c7d2b8
c0b9b08
 
6c7d2b8
 
5801d6e
6c7d2b8
 
5801d6e
 
 
 
 
 
 
 
6c7d2b8
c0b9b08
 
 
 
 
 
 
 
 
59e881b
c0b9b08
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
59e881b
c0b9b08
 
 
 
 
 
 
 
6c7d2b8
c0b9b08
 
 
6c7d2b8
 
c0b9b08
6c7d2b8
 
 
 
 
 
 
 
 
 
 
 
59e881b
6c7d2b8
59e881b
6c7d2b8
 
c0b9b08
6c7d2b8
 
 
 
 
 
5801d6e
6c7d2b8
 
59e881b
c0b9b08
59e881b
6c7d2b8
 
c0b9b08
6c7d2b8
59e881b
6c7d2b8
c0b9b08
59e881b
c0b9b08
 
 
 
 
 
 
6c7d2b8
 
 
5801d6e
59e881b
 
c0b9b08
59e881b
c0b9b08
59e881b
c0b9b08
 
59e881b
c0b9b08
 
59e881b
c0b9b08
 
59e881b
c0b9b08
 
59e881b
c0b9b08
 
59e881b
c0b9b08
 
59e881b
6c7d2b8
5801d6e
59e881b
 
6c7d2b8
 
 
 
c0b9b08
 
 
 
 
 
 
 
 
59e881b
6c7d2b8
 
 
c0b9b08
6c7d2b8
c0b9b08
6c7d2b8
c0b9b08
 
 
 
59e881b
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
"""
Hallo2: Long-Duration and High-Resolution Audio-driven Portrait Image Animation
Gradio App for HuggingFace Spaces
"""

import os
import sys
import argparse
import tempfile
import traceback
from pathlib import Path

import gradio as gr
import torch

# ---------------------------------------------------------------------------
# Environment
# ---------------------------------------------------------------------------
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, ROOT_DIR)

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Running on device: {DEVICE}")

# ---------------------------------------------------------------------------
# Model Download
# ---------------------------------------------------------------------------
def download_pretrained_models():
    """
    Download Hallo2 pretrained models if not already present.
    """
    pretrained_dir = "./pretrained_models"
    os.makedirs(pretrained_dir, exist_ok=True)

    try:
        marker_file = os.path.join(pretrained_dir, ".download_complete")
        if os.path.exists(marker_file):
            print("Pretrained models already present.")
            return

        print("Downloading Hallo2 pretrained models...")
        from huggingface_hub import snapshot_download

        snapshot_download(
            repo_id="fudan-generative-ai/hallo2",
            local_dir=pretrained_dir,
            local_dir_use_symlinks=False,
            resume_download=True,
        )

        with open(marker_file, "w") as f:
            f.write("ok")
        print("Model download completed.")
    except Exception as e:
        print(f"[WARNING] Model download failed:")
        print(str(e))

# ---------------------------------------------------------------------------
# Inference
# ---------------------------------------------------------------------------
def run_inference(
    source_image_path,
    driving_audio_path,
    pose_weight,
    face_weight,
    lip_weight,
    face_expand_ratio,
    inference_steps,
    cfg_scale,
):
    try:
        from omegaconf import OmegaConf
        from scripts.inference_long import inference_process
        from hallo.utils.util import merge_videos

        output_dir = tempfile.mkdtemp(prefix="hallo2_out_")
        config_path = "configs/inference/long.yaml"

        if not os.path.exists(config_path):
            raise FileNotFoundError(f"Config not found: {config_path}")

        cfg = OmegaConf.load(config_path)
        cfg.inference_steps = int(inference_steps)
        cfg.cfg_scale = float(cfg_scale)
        cfg.save_path = output_dir
        cfg.source_image = source_image_path
        cfg.driving_audio = driving_audio_path
        cfg.pose_weight = float(pose_weight)
        cfg.face_weight = float(face_weight)
        cfg.lip_weight = float(lip_weight)
        cfg.face_expand_ratio = float(face_expand_ratio)

        temp_cfg = os.path.join(output_dir, "temp_config.yaml")
        OmegaConf.save(cfg, temp_cfg)

        args = argparse.Namespace(
            config=temp_cfg,
            source_image=source_image_path,
            driving_audio=driving_audio_path,
            pose_weight=float(pose_weight),
            face_weight=float(face_weight),
            lip_weight=float(lip_weight),
            face_expand_ratio=float(face_expand_ratio),
            audio_ckpt_dir=None,
        )

        seg_path = inference_process(args)

        merged_path = os.path.join(Path(seg_path).parent, "merge_video.mp4")
        merge_videos(seg_path, merged_path)

        if os.path.exists(merged_path):
            return merged_path

        mp4_files = list(Path(seg_path).glob("*.mp4"))
        if mp4_files:
            return str(mp4_files[0])

        raise RuntimeError("No output video generated.")
    except Exception:
        raise RuntimeError(traceback.format_exc())

# ---------------------------------------------------------------------------
# Gradio Inference Wrapper
# ---------------------------------------------------------------------------
def infer(
    source_image,
    driving_audio,
    pose_weight,
    face_weight,
    lip_weight,
    face_expand_ratio,
    inference_steps,
    cfg_scale,
):
    if source_image is None:
        return None, "Please upload a portrait image."
    if driving_audio is None:
        return None, "Please upload a WAV audio file."

    try:
        video_path = run_inference(
            source_image_path=source_image,
            driving_audio_path=driving_audio,
            pose_weight=pose_weight,
            face_weight=face_weight,
            lip_weight=lip_weight,
            face_expand_ratio=face_expand_ratio,
            inference_steps=inference_steps,
            cfg_scale=cfg_scale,
        )
        return video_path, "Video generated successfully."
    except Exception as e:
        return None, f"Error:\n{str(e)}"

# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
with gr.Blocks(title="Hallo2 Audio Driven Portrait Animation") as demo:
    gr.Markdown(
        """
# Hallo2

Upload:
- Portrait image
- WAV audio

Generate realistic talking-head animation.
"""
    )

    with gr.Row():
        with gr.Column():
            source_image = gr.Image(label="Portrait Image", type="filepath")
            driving_audio = gr.Audio(label="Driving Audio", type="filepath")

            with gr.Accordion("Advanced Settings", open=False):
                pose_weight = gr.Slider(
                    0.0, 2.0, value=1.0, step=0.1, label="Pose Weight"
                )
                face_weight = gr.Slider(
                    0.0, 2.0, value=1.0, step=0.1, label="Face Weight"
                )
                lip_weight = gr.Slider(
                    0.0, 2.0, value=1.0, step=0.1, label="Lip Weight"
                )
                face_expand_ratio = gr.Slider(
                    1.0, 2.0, value=1.2, step=0.05, label="Face Expand Ratio"
                )
                inference_steps = gr.Slider(
                    10, 100, value=40, step=5, label="Inference Steps"
                )
                cfg_scale = gr.Slider(
                    1.0, 10.0, value=3.5, step=0.5, label="CFG Scale"
                )

            run_btn = gr.Button("Generate Video", variant="primary")

        with gr.Column():
            output_video = gr.Video(label="Generated Video")
            status_box = gr.Textbox(label="Status", lines=8, interactive=False)

    run_btn.click(
        fn=infer,
        inputs=[
            source_image,
            driving_audio,
            pose_weight,
            face_weight,
            lip_weight,
            face_expand_ratio,
            inference_steps,
            cfg_scale,
        ],
        outputs=[output_video, status_box],
    )

# ---------------------------------------------------------------------------
# Startup
# ---------------------------------------------------------------------------
try:
    download_pretrained_models()
except Exception as e:
    print(f"Startup warning: {e}")

if __name__ == "__main__":
    demo.queue(max_size=5).launch()