import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must come before torch) import glob # noqa: E402 import hashlib # noqa: E402 import json # noqa: E402 import tempfile # noqa: E402 import time # noqa: E402 from pathlib import Path # noqa: E402 import gradio as gr # noqa: E402 import imageio.v2 as imageio # noqa: E402 import numpy as np # noqa: E402 import torch # noqa: E402 from huggingface_hub import snapshot_download # noqa: E402 from PIL import Image, ImageFilter # noqa: E402 VDA_REPO = "SimonGiebenhain/video-diffusion-avatars" ANIMATE_REPO = "Wan-AI/Wan2.2-Animate-14B" WIDTH, HEIGHT = 288, 512 # the resolution that fits a ZeroGPU request (576x1024 takes ~12 min on an A100) FPS = 24 DELETE_BASE_AFTER_LOAD = False # ZeroGPU removes the mmap-backed source files itself after packing FAKE = os.environ.get("VDA_FAKE") == "1" # local UI testing without the model # -------------------------------------------------------------------------------------------- # Examples: 17 precomputed full-model conditionings (proxy render + skeleton + expression codes) MOTIONS = { # name -> (expression, camera, group) "teaser_swimmer": ("Disgust → happy", "over the top + gentle zoom", "Expressions + camera"), "teaser_hijab": ("Disgust → happy", "over the top + gentle zoom", "Expressions + camera"), "teaser_gele": ("Tongue out", "over the top + zoom", "Expressions + camera"), "teaser_cartoon": ("Tongue out", "over the top + zoom", "Expressions + camera"), "teaser_greenhair": ("Shout → laugh", "over the top + gentle zoom", "Expressions + camera"), "teaser_grandma": ("Jaw movements", "over the top + zoom", "Expressions + camera"), "control_expr_swimmer": ("Surprise → fear", "over the top + gentle zoom", "Expressions + camera"), "control_expr_blinds": ("Jaw movements", "above → profile → below", "Expressions + camera"), "control_expr_gele": ("Cheeks and nose", "above → profile → below", "Expressions + camera"), "teaser_vangogh": ("Photo's own expression", "above → profile → below", "Camera only"), "control_camera_tortoise": ("Photo's own expression", "over the top + zoom", "Camera only"), "control_camera_curlyboy": ("Photo's own expression", "below → profile → above", "Camera only"), "control_camera_refraction": ("Photo's own expression", "over the top + zoom", "Camera only"), "control_neck_neon_earring": ("Head nods yes / no", "static camera", "Head pose only"), "control_neck_red_hair_teen": ("Head nods yes / no", "static camera", "Head pose only"), "control_neck_straw_hat": ("Head rolls", "static camera", "Head pose only"), "control_neck_beanie_locs": ("Head turns", "static camera", "Head pose only"), } EXTRA_PORTRAITS = ["distilled_cgrandma", "distilled_frida", "distilled_marble", "distilled_redbangs", "distilled_watchmaker"] # long-video examples: only their reference image is used here t_start = time.time() vda_dir = Path(snapshot_download( VDA_REPO, allow_patterns=([] if FAKE else ["vda_full.safetensors", "xnemo_disentangler.pt", "text_emb_cache.pt"]) + ["config.json"] + [f"examples/{n}/*" for n in MOTIONS] + [f"examples/{n}/{f}" for n in EXTRA_PORTRAITS for f in ("ref.png", "meta.json")], )) EX = vda_dir / "examples" os.environ.setdefault("VDA_WEIGHTS", str(vda_dir)) def _meta(name): return json.loads((EX / name / "meta.json").read_text()) # Unique portraits (several examples share a reference image). PORTRAITS = {} # key -> {"path", "subject", "examples"} _by_hash = {} for name in list(MOTIONS) + EXTRA_PORTRAITS: p = EX / name / "ref.png" h = hashlib.md5(p.read_bytes()).hexdigest() subject = _meta(name)["subject"] if h in _by_hash: PORTRAITS[_by_hash[h]]["examples"].append(name) continue key = subject[0].upper() + subject[1:] _by_hash[h] = key PORTRAITS[key] = {"path": str(p), "subject": subject, "examples": [name]} PORTRAIT_OF = {n: k for k, v in PORTRAITS.items() for n in v["examples"]} # example -> its own portrait def motion_label(name): expr, cam, _ = MOTIONS[name] return f"{expr} · {cam} ({PORTRAITS[PORTRAIT_OF[name]]['subject']})" MOTION_BY_LABEL = {motion_label(n): n for n in MOTIONS} GROUPED_LABELS = [motion_label(n) for g in ("Expressions + camera", "Camera only", "Head pose only") for n in MOTIONS if MOTIONS[n][2] == g] # -------------------------------------------------------------------------------------------- # Model: WAN2.2-Animate-14B + the VDA full adapter, built on CPU and moved to "cuda" (ZeroGPU packs it) def load_pipeline(): """vda.model.load_pipeline, but with explicit file paths: the original Wan2.1 VAE / CLIP .pth files from the Hugging Face Wan2.2-Animate-14B repo hash-match DiffSynth's registry, so DiffSynth's redirect to its ModelScope-converted safetensors is not needed.""" from vda.model import DEFAULT_PROMPT, _install_text_cache from vda.third_party.diffsynth.core import load_state_dict from vda.third_party.diffsynth.models.wan_video_animate_adapter_xnemo import ( WanAnimateAdapterXNemo, build_translator_head) from vda.third_party.diffsynth.pipelines.wan_video import ModelConfig, WanVideoPipeline base = Path(snapshot_download(ANIMATE_REPO, allow_patterns=[ "diffusion_pytorch_model*.safetensors", "config.json", "Wan2.1_VAE.pth", "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"])) shards = sorted(str(p) for p in base.glob("diffusion_pytorch_model-*.safetensors")) print(f"[space] base files ready after {time.time() - t_start:.0f}s", flush=True) pipe = WanVideoPipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cpu", model_configs=[ModelConfig(path=shards), ModelConfig(path=str(base / "Wan2.1_VAE.pth")), ModelConfig(path=str(base / "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"))], tokenizer_config=None, # prompts come from the cached embeddings below ) prompt = DEFAULT_PROMPT _install_text_cache(pipe, str(vda_dir / "text_emb_cache.pt"), [prompt]) # Same steps as vda.model.load_pipeline (order matters: new modules exist before the checkpoint loads). ad = pipe.animate_adapter ad.__class__ = WanAnimateAdapterXNemo ad.translator = build_translator_head(str(vda_dir / "xnemo_disentangler.pt"), "cpu") ppe = ad.pose_patch_embedding ge = torch.nn.Conv3d(ppe.in_channels, ppe.out_channels, kernel_size=ppe.kernel_size, stride=ppe.stride) torch.nn.init.zeros_(ge.weight) torch.nn.init.zeros_(ge.bias) ad.graft_patch_embedding = ge.to(dtype=ppe.weight.dtype) trained = load_state_dict(str(vda_dir / "vda_full.safetensors"), torch_dtype=torch.float32, device="cpu") trained = {k: v for k, v in trained.items() if not k.startswith("gates.")} _, unexpected = ad.load_state_dict(trained, strict=False) if unexpected: raise RuntimeError(f"unexpected adapter keys: {unexpected[:5]}") need = [k for k, _ in ad.named_parameters() if k.startswith(("translator.f.", "graft_patch_embedding.", "face_adapter.", "face_encoder."))] lost = [k for k in need if k not in trained] if lost: raise RuntimeError(f"adapter is missing {len(lost)} tensors, e.g. {lost[:3]}") pipe.vda_prompts = (prompt, prompt) pipe.to("cuda") pipe.eval() print(f"[space] pipeline ready after {time.time() - t_start:.0f}s", flush=True) if DELETE_BASE_AFTER_LOAD: for f in shards + [str(base / "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"), str(vda_dir / "vda_full.safetensors")]: for p in {os.path.realpath(f), f}: try: os.remove(p) except OSError: pass return pipe pipe = None if FAKE else load_pipeline() from vda.generate import FULL, ID_BLUR_SIGMA, _oot, full_num_frames, load_frames, load_ref # noqa: E402 # -------------------------------------------------------------------------------------------- # Generation SEC_PER_STEP = 3.6 # 2 DiT passes per step at 288x512; measured ~3.2 s on a ZeroGPU xlarge slice (20 steps: 71 s) SETUP_S = 20 # VAE encode of the reference / skeleton / proxy streams, CLIP, decode def _duration(motion, portrait_path, steps, seed, *args, **kwargs): return int(min(300, SETUP_S + int(steps) * SEC_PER_STEP)) @spaces.GPU(size="xlarge", duration=_duration) def _generate(motion: str, portrait_path: str, steps: int, seed: int): t0 = time.time() cond = EX / motion size = (WIDTH, HEIGHT) num_frames = full_num_frames(str(cond / "pose_skel.mp4")) n = num_frames - 4 if FAKE: frames = np.stack([np.asarray(f) for f in load_frames(str(cond / "pose_proxy.mp4"), size, n)]) return frames, 0.0 pose = load_frames(str(cond / "pose_skel.mp4"), size, n) proxy = load_frames(str(cond / "pose_proxy.mp4"), size, n) codes = torch.from_numpy(np.load(cond / "codes.npy")).float() ref = load_ref(portrait_path, size) prompt, negative = pipe.vda_prompts video = pipe(prompt=prompt, negative_prompt=negative, input_image=ref, animate_pose_video=pose, animate_graft_video=proxy, expr_codes=codes, height=HEIGHT, width=WIDTH, num_frames=num_frames, id_weak_image=ref.filter(ImageFilter.GaussianBlur(ID_BLUR_SIGMA)), sampler="ode", seed=int(seed), tiled=True, **_oot(size, n), **{**FULL, "num_inference_steps": int(steps)}) frames = np.stack([np.asarray(f.convert("RGB")) for f in video]).astype(np.uint8) gpu_s = time.time() - t0 print(f"[space] {motion} x {Path(portrait_path).parent.name}: {len(frames)} frames, {steps} steps, " f"{gpu_s:.1f}s, peak {torch.cuda.max_memory_allocated() / 2**30:.1f} GiB", flush=True) return frames, round(gpu_s, 1) def _write_mp4(frames, path): with imageio.get_writer(path, fps=FPS, codec="libx264", quality=8, macro_block_size=16, ffmpeg_params=["-pix_fmt", "yuv420p"]) as w: for f in frames: w.append_data(f) def animate(motion: str, portrait: str, steps: int = 20, seed: int = 42, progress=gr.Progress(track_tqdm=True)): """Animate a portrait with Video Diffusion Avatars (expression- and camera-controllable, WAN2.2-Animate-14B). Args: motion: The motion & camera path, as one of the labels listed in the UI dropdown (e.g. "Disgust → happy · over the top + gentle zoom (swimmer with goggles)"). portrait: The portrait to animate, as one of the portrait names listed in the UI (e.g. "Swimmer with goggles"). Each motion was made for one portrait; others work but may drift. steps: Denoising steps (the model was trained with 20; 10-12 is a faster preview). seed: Random seed. Returns: The generated 77-frame (3.2 s) video at 288x512, a side-by-side comparison (portrait | driving proxy render | result), and a status line. """ if motion not in MOTION_BY_LABEL: raise gr.Error("Pick a motion from the list.") if portrait not in PORTRAITS: raise gr.Error("Pick a portrait from the gallery.") name = MOTION_BY_LABEL[motion] p = PORTRAITS[portrait] t0 = time.time() frames, gpu_s = _generate(name, p["path"], int(steps), int(seed)) out_dir = tempfile.mkdtemp(prefix="vda-") out = os.path.join(out_dir, "vda.mp4") _write_mp4(frames, out) size = (WIDTH, HEIGHT) proxy = load_frames(str(EX / name / "pose_proxy.mp4"), size, len(frames)) ref = np.asarray(Image.open(p["path"]).convert("RGB").resize(size)) sep = np.full((HEIGHT, 8, 3), 255, np.uint8) # 3 x 288 + 2 x 8 = 880 = 55 x 16 (codec-friendly) comp = [np.concatenate([ref, sep, np.asarray(proxy[min(i, len(proxy) - 1)]), sep, f], axis=1) for i, f in enumerate(frames)] comp = [c[:, : c.shape[1] // 16 * 16] for c in comp] comp_path = os.path.join(out_dir, "vda_comparison.mp4") _write_mp4(comp, comp_path) paired = PORTRAIT_OF[name] == portrait status = (f"{len(frames)} frames at {WIDTH}×{HEIGHT} · {int(steps)} steps · seed {int(seed)} · " f"GPU {gpu_s:.0f} s · total {time.time() - t0:.0f} s" + ("" if paired else " · remix: this motion's head proxy was fitted to another portrait")) return out, comp_path, status # -------------------------------------------------------------------------------------------- # UI PORTRAIT_KEYS = list(PORTRAITS) GALLERY = [(PORTRAITS[k]["path"], k) for k in PORTRAIT_KEYS] DEFAULT_MOTION = motion_label("teaser_swimmer") DEFAULT_PORTRAIT = PORTRAIT_OF["teaser_swimmer"] CSS = """ #hdr {text-align:center; margin: 6px 0 4px} #hdr h1 {font-size: 2rem; margin: 0 0 6px} #hdr p {margin: 3px auto; max-width: 860px; opacity: .85} #hdr .links a {margin: 0 7px; font-weight: 600} #pair {font-size: .9rem; min-height: 1.5em} #status {font-size: .85rem; opacity: .8} footer {display:none !important} """ HEADER = """
Expression- and camera-controllable portrait animation on WAN2.2-Animate-14B. Pick a motion and camera path, pick a portrait, and the model renders that person performing the motion from the moving camera.