joeWorld's picture
Tune GPU duration from measured runs; clearer remix note
0c667ca verified
Raw History Blame Contribute Delete
19 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must come before torch)
import glob # noqa: E402
import hashlib # noqa: E402
import json # noqa: E402
import tempfile # noqa: E402
import time # noqa: E402
from pathlib import Path # noqa: E402
import gradio as gr # noqa: E402
import imageio.v2 as imageio # noqa: E402
import numpy as np # noqa: E402
import torch # noqa: E402
from huggingface_hub import snapshot_download # noqa: E402
from PIL import Image, ImageFilter # noqa: E402
VDA_REPO = "SimonGiebenhain/video-diffusion-avatars"
ANIMATE_REPO = "Wan-AI/Wan2.2-Animate-14B"
WIDTH, HEIGHT = 288, 512 # the resolution that fits a ZeroGPU request (576x1024 takes ~12 min on an A100)
FPS = 24
DELETE_BASE_AFTER_LOAD = False # ZeroGPU removes the mmap-backed source files itself after packing
FAKE = os.environ.get("VDA_FAKE") == "1" # local UI testing without the model
# --------------------------------------------------------------------------------------------
# Examples: 17 precomputed full-model conditionings (proxy render + skeleton + expression codes)
MOTIONS = { # name -> (expression, camera, group)
"teaser_swimmer": ("Disgust → happy", "over the top + gentle zoom", "Expressions + camera"),
"teaser_hijab": ("Disgust → happy", "over the top + gentle zoom", "Expressions + camera"),
"teaser_gele": ("Tongue out", "over the top + zoom", "Expressions + camera"),
"teaser_cartoon": ("Tongue out", "over the top + zoom", "Expressions + camera"),
"teaser_greenhair": ("Shout → laugh", "over the top + gentle zoom", "Expressions + camera"),
"teaser_grandma": ("Jaw movements", "over the top + zoom", "Expressions + camera"),
"control_expr_swimmer": ("Surprise → fear", "over the top + gentle zoom", "Expressions + camera"),
"control_expr_blinds": ("Jaw movements", "above → profile → below", "Expressions + camera"),
"control_expr_gele": ("Cheeks and nose", "above → profile → below", "Expressions + camera"),
"teaser_vangogh": ("Photo's own expression", "above → profile → below", "Camera only"),
"control_camera_tortoise": ("Photo's own expression", "over the top + zoom", "Camera only"),
"control_camera_curlyboy": ("Photo's own expression", "below → profile → above", "Camera only"),
"control_camera_refraction": ("Photo's own expression", "over the top + zoom", "Camera only"),
"control_neck_neon_earring": ("Head nods yes / no", "static camera", "Head pose only"),
"control_neck_red_hair_teen": ("Head nods yes / no", "static camera", "Head pose only"),
"control_neck_straw_hat": ("Head rolls", "static camera", "Head pose only"),
"control_neck_beanie_locs": ("Head turns", "static camera", "Head pose only"),
}
EXTRA_PORTRAITS = ["distilled_cgrandma", "distilled_frida", "distilled_marble", "distilled_redbangs",
"distilled_watchmaker"] # long-video examples: only their reference image is used here
t_start = time.time()
vda_dir = Path(snapshot_download(
VDA_REPO,
allow_patterns=([] if FAKE else ["vda_full.safetensors", "xnemo_disentangler.pt", "text_emb_cache.pt"])
+ ["config.json"]
+ [f"examples/{n}/*" for n in MOTIONS]
+ [f"examples/{n}/{f}" for n in EXTRA_PORTRAITS for f in ("ref.png", "meta.json")],
))
EX = vda_dir / "examples"
os.environ.setdefault("VDA_WEIGHTS", str(vda_dir))
def _meta(name):
return json.loads((EX / name / "meta.json").read_text())
# Unique portraits (several examples share a reference image).
PORTRAITS = {} # key -> {"path", "subject", "examples"}
_by_hash = {}
for name in list(MOTIONS) + EXTRA_PORTRAITS:
p = EX / name / "ref.png"
h = hashlib.md5(p.read_bytes()).hexdigest()
subject = _meta(name)["subject"]
if h in _by_hash:
PORTRAITS[_by_hash[h]]["examples"].append(name)
continue
key = subject[0].upper() + subject[1:]
_by_hash[h] = key
PORTRAITS[key] = {"path": str(p), "subject": subject, "examples": [name]}
PORTRAIT_OF = {n: k for k, v in PORTRAITS.items() for n in v["examples"]} # example -> its own portrait
def motion_label(name):
expr, cam, _ = MOTIONS[name]
return f"{expr} · {cam} ({PORTRAITS[PORTRAIT_OF[name]]['subject']})"
MOTION_BY_LABEL = {motion_label(n): n for n in MOTIONS}
GROUPED_LABELS = [motion_label(n) for g in ("Expressions + camera", "Camera only", "Head pose only")
for n in MOTIONS if MOTIONS[n][2] == g]
# --------------------------------------------------------------------------------------------
# Model: WAN2.2-Animate-14B + the VDA full adapter, built on CPU and moved to "cuda" (ZeroGPU packs it)
def load_pipeline():
"""vda.model.load_pipeline, but with explicit file paths: the original Wan2.1 VAE / CLIP .pth files from the
Hugging Face Wan2.2-Animate-14B repo hash-match DiffSynth's registry, so DiffSynth's redirect to its
ModelScope-converted safetensors is not needed."""
from vda.model import DEFAULT_PROMPT, _install_text_cache
from vda.third_party.diffsynth.core import load_state_dict
from vda.third_party.diffsynth.models.wan_video_animate_adapter_xnemo import (
WanAnimateAdapterXNemo, build_translator_head)
from vda.third_party.diffsynth.pipelines.wan_video import ModelConfig, WanVideoPipeline
base = Path(snapshot_download(ANIMATE_REPO, allow_patterns=[
"diffusion_pytorch_model*.safetensors", "config.json", "Wan2.1_VAE.pth",
"models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"]))
shards = sorted(str(p) for p in base.glob("diffusion_pytorch_model-*.safetensors"))
print(f"[space] base files ready after {time.time() - t_start:.0f}s", flush=True)
pipe = WanVideoPipeline.from_pretrained(
torch_dtype=torch.bfloat16, device="cpu",
model_configs=[ModelConfig(path=shards), ModelConfig(path=str(base / "Wan2.1_VAE.pth")),
ModelConfig(path=str(base / "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"))],
tokenizer_config=None, # prompts come from the cached embeddings below
)
prompt = DEFAULT_PROMPT
_install_text_cache(pipe, str(vda_dir / "text_emb_cache.pt"), [prompt])
# Same steps as vda.model.load_pipeline (order matters: new modules exist before the checkpoint loads).
ad = pipe.animate_adapter
ad.__class__ = WanAnimateAdapterXNemo
ad.translator = build_translator_head(str(vda_dir / "xnemo_disentangler.pt"), "cpu")
ppe = ad.pose_patch_embedding
ge = torch.nn.Conv3d(ppe.in_channels, ppe.out_channels, kernel_size=ppe.kernel_size, stride=ppe.stride)
torch.nn.init.zeros_(ge.weight)
torch.nn.init.zeros_(ge.bias)
ad.graft_patch_embedding = ge.to(dtype=ppe.weight.dtype)
trained = load_state_dict(str(vda_dir / "vda_full.safetensors"), torch_dtype=torch.float32, device="cpu")
trained = {k: v for k, v in trained.items() if not k.startswith("gates.")}
_, unexpected = ad.load_state_dict(trained, strict=False)
if unexpected:
raise RuntimeError(f"unexpected adapter keys: {unexpected[:5]}")
need = [k for k, _ in ad.named_parameters()
if k.startswith(("translator.f.", "graft_patch_embedding.", "face_adapter.", "face_encoder."))]
lost = [k for k in need if k not in trained]
if lost:
raise RuntimeError(f"adapter is missing {len(lost)} tensors, e.g. {lost[:3]}")
pipe.vda_prompts = (prompt, prompt)
pipe.to("cuda")
pipe.eval()
print(f"[space] pipeline ready after {time.time() - t_start:.0f}s", flush=True)
if DELETE_BASE_AFTER_LOAD:
for f in shards + [str(base / "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"),
str(vda_dir / "vda_full.safetensors")]:
for p in {os.path.realpath(f), f}:
try:
os.remove(p)
except OSError:
pass
return pipe
pipe = None if FAKE else load_pipeline()
from vda.generate import FULL, ID_BLUR_SIGMA, _oot, full_num_frames, load_frames, load_ref # noqa: E402
# --------------------------------------------------------------------------------------------
# Generation
SEC_PER_STEP = 3.6 # 2 DiT passes per step at 288x512; measured ~3.2 s on a ZeroGPU xlarge slice (20 steps: 71 s)
SETUP_S = 20 # VAE encode of the reference / skeleton / proxy streams, CLIP, decode
def _duration(motion, portrait_path, steps, seed, *args, **kwargs):
return int(min(300, SETUP_S + int(steps) * SEC_PER_STEP))
@spaces.GPU(size="xlarge", duration=_duration)
def _generate(motion: str, portrait_path: str, steps: int, seed: int):
t0 = time.time()
cond = EX / motion
size = (WIDTH, HEIGHT)
num_frames = full_num_frames(str(cond / "pose_skel.mp4"))
n = num_frames - 4
if FAKE:
frames = np.stack([np.asarray(f) for f in load_frames(str(cond / "pose_proxy.mp4"), size, n)])
return frames, 0.0
pose = load_frames(str(cond / "pose_skel.mp4"), size, n)
proxy = load_frames(str(cond / "pose_proxy.mp4"), size, n)
codes = torch.from_numpy(np.load(cond / "codes.npy")).float()
ref = load_ref(portrait_path, size)
prompt, negative = pipe.vda_prompts
video = pipe(prompt=prompt, negative_prompt=negative, input_image=ref,
animate_pose_video=pose, animate_graft_video=proxy, expr_codes=codes,
height=HEIGHT, width=WIDTH, num_frames=num_frames,
id_weak_image=ref.filter(ImageFilter.GaussianBlur(ID_BLUR_SIGMA)),
sampler="ode", seed=int(seed), tiled=True, **_oot(size, n),
**{**FULL, "num_inference_steps": int(steps)})
frames = np.stack([np.asarray(f.convert("RGB")) for f in video]).astype(np.uint8)
gpu_s = time.time() - t0
print(f"[space] {motion} x {Path(portrait_path).parent.name}: {len(frames)} frames, {steps} steps, "
f"{gpu_s:.1f}s, peak {torch.cuda.max_memory_allocated() / 2**30:.1f} GiB", flush=True)
return frames, round(gpu_s, 1)
def _write_mp4(frames, path):
with imageio.get_writer(path, fps=FPS, codec="libx264", quality=8, macro_block_size=16,
ffmpeg_params=["-pix_fmt", "yuv420p"]) as w:
for f in frames:
w.append_data(f)
def animate(motion: str, portrait: str, steps: int = 20, seed: int = 42, progress=gr.Progress(track_tqdm=True)):
"""Animate a portrait with Video Diffusion Avatars (expression- and camera-controllable, WAN2.2-Animate-14B).
Args:
motion: The motion & camera path, as one of the labels listed in the UI dropdown
(e.g. "Disgust → happy · over the top + gentle zoom (swimmer with goggles)").
portrait: The portrait to animate, as one of the portrait names listed in the UI
(e.g. "Swimmer with goggles"). Each motion was made for one portrait; others work but may drift.
steps: Denoising steps (the model was trained with 20; 10-12 is a faster preview).
seed: Random seed.
Returns:
The generated 77-frame (3.2 s) video at 288x512, a side-by-side comparison
(portrait | driving proxy render | result), and a status line.
"""
if motion not in MOTION_BY_LABEL:
raise gr.Error("Pick a motion from the list.")
if portrait not in PORTRAITS:
raise gr.Error("Pick a portrait from the gallery.")
name = MOTION_BY_LABEL[motion]
p = PORTRAITS[portrait]
t0 = time.time()
frames, gpu_s = _generate(name, p["path"], int(steps), int(seed))
out_dir = tempfile.mkdtemp(prefix="vda-")
out = os.path.join(out_dir, "vda.mp4")
_write_mp4(frames, out)
size = (WIDTH, HEIGHT)
proxy = load_frames(str(EX / name / "pose_proxy.mp4"), size, len(frames))
ref = np.asarray(Image.open(p["path"]).convert("RGB").resize(size))
sep = np.full((HEIGHT, 8, 3), 255, np.uint8) # 3 x 288 + 2 x 8 = 880 = 55 x 16 (codec-friendly)
comp = [np.concatenate([ref, sep, np.asarray(proxy[min(i, len(proxy) - 1)]), sep, f], axis=1)
for i, f in enumerate(frames)]
comp = [c[:, : c.shape[1] // 16 * 16] for c in comp]
comp_path = os.path.join(out_dir, "vda_comparison.mp4")
_write_mp4(comp, comp_path)
paired = PORTRAIT_OF[name] == portrait
status = (f"{len(frames)} frames at {WIDTH}×{HEIGHT} · {int(steps)} steps · seed {int(seed)} · "
f"GPU {gpu_s:.0f} s · total {time.time() - t0:.0f} s"
+ ("" if paired else " · remix: this motion's head proxy was fitted to another portrait"))
return out, comp_path, status
# --------------------------------------------------------------------------------------------
# UI
PORTRAIT_KEYS = list(PORTRAITS)
GALLERY = [(PORTRAITS[k]["path"], k) for k in PORTRAIT_KEYS]
DEFAULT_MOTION = motion_label("teaser_swimmer")
DEFAULT_PORTRAIT = PORTRAIT_OF["teaser_swimmer"]
CSS = """
#hdr {text-align:center; margin: 6px 0 4px}
#hdr h1 {font-size: 2rem; margin: 0 0 6px}
#hdr p {margin: 3px auto; max-width: 860px; opacity: .85}
#hdr .links a {margin: 0 7px; font-weight: 600}
#pair {font-size: .9rem; min-height: 1.5em}
#status {font-size: .85rem; opacity: .8}
footer {display:none !important}
"""
HEADER = """
<div id="hdr">
<h1>🎭 Video Diffusion Avatars</h1>
<p>Expression- and camera-controllable portrait animation on WAN2.2-Animate-14B. Pick a motion and camera path,
pick a portrait, and the model renders that person performing the motion from the moving camera.</p>
<p class="links"><a href="https://arxiv.org/abs/2610.23772" target="_blank">Paper</a>·
<a href="https://simongiebenhain.github.io/video-diffusion-avatars/" target="_blank">Project page</a>·
<a href="https://github.com/SimonGiebenhain/video-diffusion-avatars" target="_blank">Code</a>·
<a href="https://huggingface.co/SimonGiebenhain/video-diffusion-avatars" target="_blank">Weights</a></p>
</div>
"""
def pairing_note(motion, portrait):
name = MOTION_BY_LABEL.get(motion)
if not name or portrait not in PORTRAITS:
return ""
if PORTRAIT_OF[name] == portrait:
return "✅ **Original pairing** — the driving proxy was fitted to this portrait's head shape."
return ("🔀 **Remix** — this motion was made for *" + PORTRAITS[PORTRAIT_OF[name]]["subject"]
+ "*: its head proxy has that person's shape and its expression codes carry that person's expression "
"(for the camera-only and head-pose motions, their photo's expression). Head shape and expression can "
"drift toward the original. Re-pick the motion to restore its own portrait.")
def on_motion(motion):
name = MOTION_BY_LABEL[motion]
portrait = PORTRAIT_OF[name]
return (portrait, gr.update(selected_index=PORTRAIT_KEYS.index(portrait)),
str(EX / name / "pose_proxy.mp4"), pairing_note(motion, portrait))
def on_gallery(motion, evt: gr.SelectData):
portrait = PORTRAIT_KEYS[evt.index]
return portrait, pairing_note(motion, portrait)
with gr.Blocks(title="Video Diffusion Avatars") as demo:
gr.HTML(HEADER)
with gr.Row(equal_height=False):
with gr.Column(scale=5):
motion = gr.Dropdown(GROUPED_LABELS, value=DEFAULT_MOTION, label="1 · Motion & camera path",
info="Expression driver · camera — in brackets, the portrait it was made for")
portrait = gr.Dropdown(PORTRAIT_KEYS, value=DEFAULT_PORTRAIT, label="2 · Portrait",
info="Or click one below")
gallery = gr.Gallery(GALLERY, columns=5, height=330, object_fit="cover", allow_preview=False,
selected_index=PORTRAIT_KEYS.index(DEFAULT_PORTRAIT), show_label=False)
pair = gr.Markdown(pairing_note(DEFAULT_MOTION, DEFAULT_PORTRAIT), elem_id="pair")
with gr.Accordion("Settings", open=False):
steps = gr.Slider(8, 20, value=20, step=1, label="Denoising steps",
info="Trained with 20. Fewer = faster preview, softer detail.")
seed = gr.Number(42, label="Seed", precision=0)
run = gr.Button("Animate (≈75 s on GPU)", variant="primary")
with gr.Column(scale=6):
with gr.Row():
driving = gr.Video(str(EX / "teaser_swimmer" / "pose_proxy.mp4"), label="Driving proxy (motion + camera)",
autoplay=True, loop=True, height=420, interactive=False)
result = gr.Video(label="Result", autoplay=True, loop=True, height=420, interactive=False)
comparison = gr.Video(label="Portrait · driving proxy · result", autoplay=True, loop=True,
interactive=False)
status = gr.Markdown("", elem_id="status")
gr.Examples(
examples=[[motion_label(n), PORTRAIT_OF[n]] for n in
("teaser_swimmer", "teaser_cartoon", "teaser_vangogh", "control_neck_straw_hat",
"control_expr_blinds", "teaser_grandma")],
inputs=[motion, portrait],
outputs=[result, comparison, status],
fn=lambda m, p: animate(m, p, 20, 42),
cache_examples=True,
cache_mode="lazy",
label="Original pairings (cached after the first run)",
)
gr.Markdown(
"**How it works.** Each motion is a precomputed conditioning from the authors' examples: a *proxy render* "
"(a FLAME head with the portrait's fitted shape and the driver's expression, on a body, seen from the target "
"camera), a 2D skeleton and X-NeMo expression codes. The VDA adapter turns WAN2.2-Animate-14B into a "
"controllable avatar: 20 Euler steps with joint identity + expression guidance, 77 frames (3.2 s). "
"This demo renders at 288×512 to fit a ZeroGPU request; the "
"[code](https://github.com/SimonGiebenhain/video-diffusion-avatars) also does 576×1024 / 720×1280, your own "
"photos and driving videos (needs a FLAME licence), and 41-second videos with the distilled model.\n\n"
"**Licence.** VDA code and weights: CC BY-NC-SA 4.0 (non-commercial). WAN2.2-Animate-14B: Apache-2.0. "
"Driving motions derive from NeRSemble (its terms of use apply). Outputs are AI-generated video."
)
motion.change(on_motion, motion, [portrait, gallery, driving, pair], api_visibility="private")
gallery.select(on_gallery, motion, [portrait, pair], api_visibility="private")
portrait.input(lambda m, p: (gr.update(selected_index=PORTRAIT_KEYS.index(p)), pairing_note(m, p)),
[motion, portrait], [gallery, pair], api_visibility="private")
run.click(animate, [motion, portrait, steps, seed], [result, comparison, status], api_name="animate",
concurrency_limit=2)
print(f"[space] UI built after {time.time() - t_start:.0f}s", flush=True)
if __name__ == "__main__":
demo.queue().launch(theme=gr.themes.Soft(primary_hue="violet"), css=CSS, mcp_server=True)