Spaces:
Running on Zero
Running on Zero
Download app.py from joeWorld/video-diffusion-avatars: direct link, hf CLI and curl.
- Browser
- Download file 19 kB
-
https://huggingface.co/spaces/joeWorld/video-diffusion-avatars/resolve/main/app.py
- Command line
-
hf download hf://spaces/joeWorld/video-diffusion-avatars/app.py
-
curl -L -o app.py https://huggingface.co/spaces/joeWorld/video-diffusion-avatars/resolve/main/app.py
19 kB
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 (must come before torch) | |
| import glob # noqa: E402 | |
| import hashlib # noqa: E402 | |
| import json # noqa: E402 | |
| import tempfile # noqa: E402 | |
| import time # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import gradio as gr # noqa: E402 | |
| import imageio.v2 as imageio # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import torch # noqa: E402 | |
| from huggingface_hub import snapshot_download # noqa: E402 | |
| from PIL import Image, ImageFilter # noqa: E402 | |
| VDA_REPO = "SimonGiebenhain/video-diffusion-avatars" | |
| ANIMATE_REPO = "Wan-AI/Wan2.2-Animate-14B" | |
| WIDTH, HEIGHT = 288, 512 # the resolution that fits a ZeroGPU request (576x1024 takes ~12 min on an A100) | |
| FPS = 24 | |
| DELETE_BASE_AFTER_LOAD = False # ZeroGPU removes the mmap-backed source files itself after packing | |
| FAKE = os.environ.get("VDA_FAKE") == "1" # local UI testing without the model | |
| # -------------------------------------------------------------------------------------------- | |
| # Examples: 17 precomputed full-model conditionings (proxy render + skeleton + expression codes) | |
| MOTIONS = { # name -> (expression, camera, group) | |
| "teaser_swimmer": ("Disgust → happy", "over the top + gentle zoom", "Expressions + camera"), | |
| "teaser_hijab": ("Disgust → happy", "over the top + gentle zoom", "Expressions + camera"), | |
| "teaser_gele": ("Tongue out", "over the top + zoom", "Expressions + camera"), | |
| "teaser_cartoon": ("Tongue out", "over the top + zoom", "Expressions + camera"), | |
| "teaser_greenhair": ("Shout → laugh", "over the top + gentle zoom", "Expressions + camera"), | |
| "teaser_grandma": ("Jaw movements", "over the top + zoom", "Expressions + camera"), | |
| "control_expr_swimmer": ("Surprise → fear", "over the top + gentle zoom", "Expressions + camera"), | |
| "control_expr_blinds": ("Jaw movements", "above → profile → below", "Expressions + camera"), | |
| "control_expr_gele": ("Cheeks and nose", "above → profile → below", "Expressions + camera"), | |
| "teaser_vangogh": ("Photo's own expression", "above → profile → below", "Camera only"), | |
| "control_camera_tortoise": ("Photo's own expression", "over the top + zoom", "Camera only"), | |
| "control_camera_curlyboy": ("Photo's own expression", "below → profile → above", "Camera only"), | |
| "control_camera_refraction": ("Photo's own expression", "over the top + zoom", "Camera only"), | |
| "control_neck_neon_earring": ("Head nods yes / no", "static camera", "Head pose only"), | |
| "control_neck_red_hair_teen": ("Head nods yes / no", "static camera", "Head pose only"), | |
| "control_neck_straw_hat": ("Head rolls", "static camera", "Head pose only"), | |
| "control_neck_beanie_locs": ("Head turns", "static camera", "Head pose only"), | |
| } | |
| EXTRA_PORTRAITS = ["distilled_cgrandma", "distilled_frida", "distilled_marble", "distilled_redbangs", | |
| "distilled_watchmaker"] # long-video examples: only their reference image is used here | |
| t_start = time.time() | |
| vda_dir = Path(snapshot_download( | |
| VDA_REPO, | |
| allow_patterns=([] if FAKE else ["vda_full.safetensors", "xnemo_disentangler.pt", "text_emb_cache.pt"]) | |
| + ["config.json"] | |
| + [f"examples/{n}/*" for n in MOTIONS] | |
| + [f"examples/{n}/{f}" for n in EXTRA_PORTRAITS for f in ("ref.png", "meta.json")], | |
| )) | |
| EX = vda_dir / "examples" | |
| os.environ.setdefault("VDA_WEIGHTS", str(vda_dir)) | |
| def _meta(name): | |
| return json.loads((EX / name / "meta.json").read_text()) | |
| # Unique portraits (several examples share a reference image). | |
| PORTRAITS = {} # key -> {"path", "subject", "examples"} | |
| _by_hash = {} | |
| for name in list(MOTIONS) + EXTRA_PORTRAITS: | |
| p = EX / name / "ref.png" | |
| h = hashlib.md5(p.read_bytes()).hexdigest() | |
| subject = _meta(name)["subject"] | |
| if h in _by_hash: | |
| PORTRAITS[_by_hash[h]]["examples"].append(name) | |
| continue | |
| key = subject[0].upper() + subject[1:] | |
| _by_hash[h] = key | |
| PORTRAITS[key] = {"path": str(p), "subject": subject, "examples": [name]} | |
| PORTRAIT_OF = {n: k for k, v in PORTRAITS.items() for n in v["examples"]} # example -> its own portrait | |
| def motion_label(name): | |
| expr, cam, _ = MOTIONS[name] | |
| return f"{expr} · {cam} ({PORTRAITS[PORTRAIT_OF[name]]['subject']})" | |
| MOTION_BY_LABEL = {motion_label(n): n for n in MOTIONS} | |
| GROUPED_LABELS = [motion_label(n) for g in ("Expressions + camera", "Camera only", "Head pose only") | |
| for n in MOTIONS if MOTIONS[n][2] == g] | |
| # -------------------------------------------------------------------------------------------- | |
| # Model: WAN2.2-Animate-14B + the VDA full adapter, built on CPU and moved to "cuda" (ZeroGPU packs it) | |
| def load_pipeline(): | |
| """vda.model.load_pipeline, but with explicit file paths: the original Wan2.1 VAE / CLIP .pth files from the | |
| Hugging Face Wan2.2-Animate-14B repo hash-match DiffSynth's registry, so DiffSynth's redirect to its | |
| ModelScope-converted safetensors is not needed.""" | |
| from vda.model import DEFAULT_PROMPT, _install_text_cache | |
| from vda.third_party.diffsynth.core import load_state_dict | |
| from vda.third_party.diffsynth.models.wan_video_animate_adapter_xnemo import ( | |
| WanAnimateAdapterXNemo, build_translator_head) | |
| from vda.third_party.diffsynth.pipelines.wan_video import ModelConfig, WanVideoPipeline | |
| base = Path(snapshot_download(ANIMATE_REPO, allow_patterns=[ | |
| "diffusion_pytorch_model*.safetensors", "config.json", "Wan2.1_VAE.pth", | |
| "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"])) | |
| shards = sorted(str(p) for p in base.glob("diffusion_pytorch_model-*.safetensors")) | |
| print(f"[space] base files ready after {time.time() - t_start:.0f}s", flush=True) | |
| pipe = WanVideoPipeline.from_pretrained( | |
| torch_dtype=torch.bfloat16, device="cpu", | |
| model_configs=[ModelConfig(path=shards), ModelConfig(path=str(base / "Wan2.1_VAE.pth")), | |
| ModelConfig(path=str(base / "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"))], | |
| tokenizer_config=None, # prompts come from the cached embeddings below | |
| ) | |
| prompt = DEFAULT_PROMPT | |
| _install_text_cache(pipe, str(vda_dir / "text_emb_cache.pt"), [prompt]) | |
| # Same steps as vda.model.load_pipeline (order matters: new modules exist before the checkpoint loads). | |
| ad = pipe.animate_adapter | |
| ad.__class__ = WanAnimateAdapterXNemo | |
| ad.translator = build_translator_head(str(vda_dir / "xnemo_disentangler.pt"), "cpu") | |
| ppe = ad.pose_patch_embedding | |
| ge = torch.nn.Conv3d(ppe.in_channels, ppe.out_channels, kernel_size=ppe.kernel_size, stride=ppe.stride) | |
| torch.nn.init.zeros_(ge.weight) | |
| torch.nn.init.zeros_(ge.bias) | |
| ad.graft_patch_embedding = ge.to(dtype=ppe.weight.dtype) | |
| trained = load_state_dict(str(vda_dir / "vda_full.safetensors"), torch_dtype=torch.float32, device="cpu") | |
| trained = {k: v for k, v in trained.items() if not k.startswith("gates.")} | |
| _, unexpected = ad.load_state_dict(trained, strict=False) | |
| if unexpected: | |
| raise RuntimeError(f"unexpected adapter keys: {unexpected[:5]}") | |
| need = [k for k, _ in ad.named_parameters() | |
| if k.startswith(("translator.f.", "graft_patch_embedding.", "face_adapter.", "face_encoder."))] | |
| lost = [k for k in need if k not in trained] | |
| if lost: | |
| raise RuntimeError(f"adapter is missing {len(lost)} tensors, e.g. {lost[:3]}") | |
| pipe.vda_prompts = (prompt, prompt) | |
| pipe.to("cuda") | |
| pipe.eval() | |
| print(f"[space] pipeline ready after {time.time() - t_start:.0f}s", flush=True) | |
| if DELETE_BASE_AFTER_LOAD: | |
| for f in shards + [str(base / "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"), | |
| str(vda_dir / "vda_full.safetensors")]: | |
| for p in {os.path.realpath(f), f}: | |
| try: | |
| os.remove(p) | |
| except OSError: | |
| pass | |
| return pipe | |
| pipe = None if FAKE else load_pipeline() | |
| from vda.generate import FULL, ID_BLUR_SIGMA, _oot, full_num_frames, load_frames, load_ref # noqa: E402 | |
| # -------------------------------------------------------------------------------------------- | |
| # Generation | |
| SEC_PER_STEP = 3.6 # 2 DiT passes per step at 288x512; measured ~3.2 s on a ZeroGPU xlarge slice (20 steps: 71 s) | |
| SETUP_S = 20 # VAE encode of the reference / skeleton / proxy streams, CLIP, decode | |
| def _duration(motion, portrait_path, steps, seed, *args, **kwargs): | |
| return int(min(300, SETUP_S + int(steps) * SEC_PER_STEP)) | |
| def _generate(motion: str, portrait_path: str, steps: int, seed: int): | |
| t0 = time.time() | |
| cond = EX / motion | |
| size = (WIDTH, HEIGHT) | |
| num_frames = full_num_frames(str(cond / "pose_skel.mp4")) | |
| n = num_frames - 4 | |
| if FAKE: | |
| frames = np.stack([np.asarray(f) for f in load_frames(str(cond / "pose_proxy.mp4"), size, n)]) | |
| return frames, 0.0 | |
| pose = load_frames(str(cond / "pose_skel.mp4"), size, n) | |
| proxy = load_frames(str(cond / "pose_proxy.mp4"), size, n) | |
| codes = torch.from_numpy(np.load(cond / "codes.npy")).float() | |
| ref = load_ref(portrait_path, size) | |
| prompt, negative = pipe.vda_prompts | |
| video = pipe(prompt=prompt, negative_prompt=negative, input_image=ref, | |
| animate_pose_video=pose, animate_graft_video=proxy, expr_codes=codes, | |
| height=HEIGHT, width=WIDTH, num_frames=num_frames, | |
| id_weak_image=ref.filter(ImageFilter.GaussianBlur(ID_BLUR_SIGMA)), | |
| sampler="ode", seed=int(seed), tiled=True, **_oot(size, n), | |
| **{**FULL, "num_inference_steps": int(steps)}) | |
| frames = np.stack([np.asarray(f.convert("RGB")) for f in video]).astype(np.uint8) | |
| gpu_s = time.time() - t0 | |
| print(f"[space] {motion} x {Path(portrait_path).parent.name}: {len(frames)} frames, {steps} steps, " | |
| f"{gpu_s:.1f}s, peak {torch.cuda.max_memory_allocated() / 2**30:.1f} GiB", flush=True) | |
| return frames, round(gpu_s, 1) | |
| def _write_mp4(frames, path): | |
| with imageio.get_writer(path, fps=FPS, codec="libx264", quality=8, macro_block_size=16, | |
| ffmpeg_params=["-pix_fmt", "yuv420p"]) as w: | |
| for f in frames: | |
| w.append_data(f) | |
| def animate(motion: str, portrait: str, steps: int = 20, seed: int = 42, progress=gr.Progress(track_tqdm=True)): | |
| """Animate a portrait with Video Diffusion Avatars (expression- and camera-controllable, WAN2.2-Animate-14B). | |
| Args: | |
| motion: The motion & camera path, as one of the labels listed in the UI dropdown | |
| (e.g. "Disgust → happy · over the top + gentle zoom (swimmer with goggles)"). | |
| portrait: The portrait to animate, as one of the portrait names listed in the UI | |
| (e.g. "Swimmer with goggles"). Each motion was made for one portrait; others work but may drift. | |
| steps: Denoising steps (the model was trained with 20; 10-12 is a faster preview). | |
| seed: Random seed. | |
| Returns: | |
| The generated 77-frame (3.2 s) video at 288x512, a side-by-side comparison | |
| (portrait | driving proxy render | result), and a status line. | |
| """ | |
| if motion not in MOTION_BY_LABEL: | |
| raise gr.Error("Pick a motion from the list.") | |
| if portrait not in PORTRAITS: | |
| raise gr.Error("Pick a portrait from the gallery.") | |
| name = MOTION_BY_LABEL[motion] | |
| p = PORTRAITS[portrait] | |
| t0 = time.time() | |
| frames, gpu_s = _generate(name, p["path"], int(steps), int(seed)) | |
| out_dir = tempfile.mkdtemp(prefix="vda-") | |
| out = os.path.join(out_dir, "vda.mp4") | |
| _write_mp4(frames, out) | |
| size = (WIDTH, HEIGHT) | |
| proxy = load_frames(str(EX / name / "pose_proxy.mp4"), size, len(frames)) | |
| ref = np.asarray(Image.open(p["path"]).convert("RGB").resize(size)) | |
| sep = np.full((HEIGHT, 8, 3), 255, np.uint8) # 3 x 288 + 2 x 8 = 880 = 55 x 16 (codec-friendly) | |
| comp = [np.concatenate([ref, sep, np.asarray(proxy[min(i, len(proxy) - 1)]), sep, f], axis=1) | |
| for i, f in enumerate(frames)] | |
| comp = [c[:, : c.shape[1] // 16 * 16] for c in comp] | |
| comp_path = os.path.join(out_dir, "vda_comparison.mp4") | |
| _write_mp4(comp, comp_path) | |
| paired = PORTRAIT_OF[name] == portrait | |
| status = (f"{len(frames)} frames at {WIDTH}×{HEIGHT} · {int(steps)} steps · seed {int(seed)} · " | |
| f"GPU {gpu_s:.0f} s · total {time.time() - t0:.0f} s" | |
| + ("" if paired else " · remix: this motion's head proxy was fitted to another portrait")) | |
| return out, comp_path, status | |
| # -------------------------------------------------------------------------------------------- | |
| # UI | |
| PORTRAIT_KEYS = list(PORTRAITS) | |
| GALLERY = [(PORTRAITS[k]["path"], k) for k in PORTRAIT_KEYS] | |
| DEFAULT_MOTION = motion_label("teaser_swimmer") | |
| DEFAULT_PORTRAIT = PORTRAIT_OF["teaser_swimmer"] | |
| CSS = """ | |
| #hdr {text-align:center; margin: 6px 0 4px} | |
| #hdr h1 {font-size: 2rem; margin: 0 0 6px} | |
| #hdr p {margin: 3px auto; max-width: 860px; opacity: .85} | |
| #hdr .links a {margin: 0 7px; font-weight: 600} | |
| #pair {font-size: .9rem; min-height: 1.5em} | |
| #status {font-size: .85rem; opacity: .8} | |
| footer {display:none !important} | |
| """ | |
| HEADER = """ | |
| <div id="hdr"> | |
| <h1>🎭 Video Diffusion Avatars</h1> | |
| <p>Expression- and camera-controllable portrait animation on WAN2.2-Animate-14B. Pick a motion and camera path, | |
| pick a portrait, and the model renders that person performing the motion from the moving camera.</p> | |
| <p class="links"><a href="https://arxiv.org/abs/2610.23772" target="_blank">Paper</a>· | |
| <a href="https://simongiebenhain.github.io/video-diffusion-avatars/" target="_blank">Project page</a>· | |
| <a href="https://github.com/SimonGiebenhain/video-diffusion-avatars" target="_blank">Code</a>· | |
| <a href="https://huggingface.co/SimonGiebenhain/video-diffusion-avatars" target="_blank">Weights</a></p> | |
| </div> | |
| """ | |
| def pairing_note(motion, portrait): | |
| name = MOTION_BY_LABEL.get(motion) | |
| if not name or portrait not in PORTRAITS: | |
| return "" | |
| if PORTRAIT_OF[name] == portrait: | |
| return "✅ **Original pairing** — the driving proxy was fitted to this portrait's head shape." | |
| return ("🔀 **Remix** — this motion was made for *" + PORTRAITS[PORTRAIT_OF[name]]["subject"] | |
| + "*: its head proxy has that person's shape and its expression codes carry that person's expression " | |
| "(for the camera-only and head-pose motions, their photo's expression). Head shape and expression can " | |
| "drift toward the original. Re-pick the motion to restore its own portrait.") | |
| def on_motion(motion): | |
| name = MOTION_BY_LABEL[motion] | |
| portrait = PORTRAIT_OF[name] | |
| return (portrait, gr.update(selected_index=PORTRAIT_KEYS.index(portrait)), | |
| str(EX / name / "pose_proxy.mp4"), pairing_note(motion, portrait)) | |
| def on_gallery(motion, evt: gr.SelectData): | |
| portrait = PORTRAIT_KEYS[evt.index] | |
| return portrait, pairing_note(motion, portrait) | |
| with gr.Blocks(title="Video Diffusion Avatars") as demo: | |
| gr.HTML(HEADER) | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=5): | |
| motion = gr.Dropdown(GROUPED_LABELS, value=DEFAULT_MOTION, label="1 · Motion & camera path", | |
| info="Expression driver · camera — in brackets, the portrait it was made for") | |
| portrait = gr.Dropdown(PORTRAIT_KEYS, value=DEFAULT_PORTRAIT, label="2 · Portrait", | |
| info="Or click one below") | |
| gallery = gr.Gallery(GALLERY, columns=5, height=330, object_fit="cover", allow_preview=False, | |
| selected_index=PORTRAIT_KEYS.index(DEFAULT_PORTRAIT), show_label=False) | |
| pair = gr.Markdown(pairing_note(DEFAULT_MOTION, DEFAULT_PORTRAIT), elem_id="pair") | |
| with gr.Accordion("Settings", open=False): | |
| steps = gr.Slider(8, 20, value=20, step=1, label="Denoising steps", | |
| info="Trained with 20. Fewer = faster preview, softer detail.") | |
| seed = gr.Number(42, label="Seed", precision=0) | |
| run = gr.Button("Animate (≈75 s on GPU)", variant="primary") | |
| with gr.Column(scale=6): | |
| with gr.Row(): | |
| driving = gr.Video(str(EX / "teaser_swimmer" / "pose_proxy.mp4"), label="Driving proxy (motion + camera)", | |
| autoplay=True, loop=True, height=420, interactive=False) | |
| result = gr.Video(label="Result", autoplay=True, loop=True, height=420, interactive=False) | |
| comparison = gr.Video(label="Portrait · driving proxy · result", autoplay=True, loop=True, | |
| interactive=False) | |
| status = gr.Markdown("", elem_id="status") | |
| gr.Examples( | |
| examples=[[motion_label(n), PORTRAIT_OF[n]] for n in | |
| ("teaser_swimmer", "teaser_cartoon", "teaser_vangogh", "control_neck_straw_hat", | |
| "control_expr_blinds", "teaser_grandma")], | |
| inputs=[motion, portrait], | |
| outputs=[result, comparison, status], | |
| fn=lambda m, p: animate(m, p, 20, 42), | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| label="Original pairings (cached after the first run)", | |
| ) | |
| gr.Markdown( | |
| "**How it works.** Each motion is a precomputed conditioning from the authors' examples: a *proxy render* " | |
| "(a FLAME head with the portrait's fitted shape and the driver's expression, on a body, seen from the target " | |
| "camera), a 2D skeleton and X-NeMo expression codes. The VDA adapter turns WAN2.2-Animate-14B into a " | |
| "controllable avatar: 20 Euler steps with joint identity + expression guidance, 77 frames (3.2 s). " | |
| "This demo renders at 288×512 to fit a ZeroGPU request; the " | |
| "[code](https://github.com/SimonGiebenhain/video-diffusion-avatars) also does 576×1024 / 720×1280, your own " | |
| "photos and driving videos (needs a FLAME licence), and 41-second videos with the distilled model.\n\n" | |
| "**Licence.** VDA code and weights: CC BY-NC-SA 4.0 (non-commercial). WAN2.2-Animate-14B: Apache-2.0. " | |
| "Driving motions derive from NeRSemble (its terms of use apply). Outputs are AI-generated video." | |
| ) | |
| motion.change(on_motion, motion, [portrait, gallery, driving, pair], api_visibility="private") | |
| gallery.select(on_gallery, motion, [portrait, pair], api_visibility="private") | |
| portrait.input(lambda m, p: (gr.update(selected_index=PORTRAIT_KEYS.index(p)), pairing_note(m, p)), | |
| [motion, portrait], [gallery, pair], api_visibility="private") | |
| run.click(animate, [motion, portrait, steps, seed], [result, comparison, status], api_name="animate", | |
| concurrency_limit=2) | |
| print(f"[space] UI built after {time.time() - t_start:.0f}s", flush=True) | |
| if __name__ == "__main__": | |
| demo.queue().launch(theme=gr.themes.Soft(primary_hue="violet"), css=CSS, mcp_server=True) | |