hallo2-final2 / app.py
PULKIT52's picture
Update app.py
59e881b verified
Raw History Blame Contribute Delete
7.2 kB
"""
Hallo2: Long-Duration and High-Resolution Audio-driven Portrait Image Animation
Gradio App for HuggingFace Spaces
"""
import os
import sys
import argparse
import tempfile
import traceback
from pathlib import Path
import gradio as gr
import torch
# ---------------------------------------------------------------------------
# Environment
# ---------------------------------------------------------------------------
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, ROOT_DIR)
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Running on device: {DEVICE}")
# ---------------------------------------------------------------------------
# Model Download
# ---------------------------------------------------------------------------
def download_pretrained_models():
"""
Download Hallo2 pretrained models if not already present.
"""
pretrained_dir = "./pretrained_models"
os.makedirs(pretrained_dir, exist_ok=True)
try:
marker_file = os.path.join(pretrained_dir, ".download_complete")
if os.path.exists(marker_file):
print("Pretrained models already present.")
return
print("Downloading Hallo2 pretrained models...")
from huggingface_hub import snapshot_download
snapshot_download(
repo_id="fudan-generative-ai/hallo2",
local_dir=pretrained_dir,
local_dir_use_symlinks=False,
resume_download=True,
)
with open(marker_file, "w") as f:
f.write("ok")
print("Model download completed.")
except Exception as e:
print(f"[WARNING] Model download failed:")
print(str(e))
# ---------------------------------------------------------------------------
# Inference
# ---------------------------------------------------------------------------
def run_inference(
source_image_path,
driving_audio_path,
pose_weight,
face_weight,
lip_weight,
face_expand_ratio,
inference_steps,
cfg_scale,
):
try:
from omegaconf import OmegaConf
from scripts.inference_long import inference_process
from hallo.utils.util import merge_videos
output_dir = tempfile.mkdtemp(prefix="hallo2_out_")
config_path = "configs/inference/long.yaml"
if not os.path.exists(config_path):
raise FileNotFoundError(f"Config not found: {config_path}")
cfg = OmegaConf.load(config_path)
cfg.inference_steps = int(inference_steps)
cfg.cfg_scale = float(cfg_scale)
cfg.save_path = output_dir
cfg.source_image = source_image_path
cfg.driving_audio = driving_audio_path
cfg.pose_weight = float(pose_weight)
cfg.face_weight = float(face_weight)
cfg.lip_weight = float(lip_weight)
cfg.face_expand_ratio = float(face_expand_ratio)
temp_cfg = os.path.join(output_dir, "temp_config.yaml")
OmegaConf.save(cfg, temp_cfg)
args = argparse.Namespace(
config=temp_cfg,
source_image=source_image_path,
driving_audio=driving_audio_path,
pose_weight=float(pose_weight),
face_weight=float(face_weight),
lip_weight=float(lip_weight),
face_expand_ratio=float(face_expand_ratio),
audio_ckpt_dir=None,
)
seg_path = inference_process(args)
merged_path = os.path.join(Path(seg_path).parent, "merge_video.mp4")
merge_videos(seg_path, merged_path)
if os.path.exists(merged_path):
return merged_path
mp4_files = list(Path(seg_path).glob("*.mp4"))
if mp4_files:
return str(mp4_files[0])
raise RuntimeError("No output video generated.")
except Exception:
raise RuntimeError(traceback.format_exc())
# ---------------------------------------------------------------------------
# Gradio Inference Wrapper
# ---------------------------------------------------------------------------
def infer(
source_image,
driving_audio,
pose_weight,
face_weight,
lip_weight,
face_expand_ratio,
inference_steps,
cfg_scale,
):
if source_image is None:
return None, "Please upload a portrait image."
if driving_audio is None:
return None, "Please upload a WAV audio file."
try:
video_path = run_inference(
source_image_path=source_image,
driving_audio_path=driving_audio,
pose_weight=pose_weight,
face_weight=face_weight,
lip_weight=lip_weight,
face_expand_ratio=face_expand_ratio,
inference_steps=inference_steps,
cfg_scale=cfg_scale,
)
return video_path, "Video generated successfully."
except Exception as e:
return None, f"Error:\n{str(e)}"
# ---------------------------------------------------------------------------
# UI
# ---------------------------------------------------------------------------
with gr.Blocks(title="Hallo2 Audio Driven Portrait Animation") as demo:
gr.Markdown(
"""
# Hallo2
Upload:
- Portrait image
- WAV audio
Generate realistic talking-head animation.
"""
)
with gr.Row():
with gr.Column():
source_image = gr.Image(label="Portrait Image", type="filepath")
driving_audio = gr.Audio(label="Driving Audio", type="filepath")
with gr.Accordion("Advanced Settings", open=False):
pose_weight = gr.Slider(
0.0, 2.0, value=1.0, step=0.1, label="Pose Weight"
)
face_weight = gr.Slider(
0.0, 2.0, value=1.0, step=0.1, label="Face Weight"
)
lip_weight = gr.Slider(
0.0, 2.0, value=1.0, step=0.1, label="Lip Weight"
)
face_expand_ratio = gr.Slider(
1.0, 2.0, value=1.2, step=0.05, label="Face Expand Ratio"
)
inference_steps = gr.Slider(
10, 100, value=40, step=5, label="Inference Steps"
)
cfg_scale = gr.Slider(
1.0, 10.0, value=3.5, step=0.5, label="CFG Scale"
)
run_btn = gr.Button("Generate Video", variant="primary")
with gr.Column():
output_video = gr.Video(label="Generated Video")
status_box = gr.Textbox(label="Status", lines=8, interactive=False)
run_btn.click(
fn=infer,
inputs=[
source_image,
driving_audio,
pose_weight,
face_weight,
lip_weight,
face_expand_ratio,
inference_steps,
cfg_scale,
],
outputs=[output_video, status_box],
)
# ---------------------------------------------------------------------------
# Startup
# ---------------------------------------------------------------------------
try:
download_pretrained_models()
except Exception as e:
print(f"Startup warning: {e}")
if __name__ == "__main__":
demo.queue(max_size=5).launch()