wan2-2-animate / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
e357318 verified
Raw History Blame Contribute Delete
16.1 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede torch / CUDA imports)
import torch # noqa: E402
import math # noqa: E402
import subprocess # noqa: E402
import tempfile # noqa: E402
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
from PIL import Image # noqa: E402
from huggingface_hub import snapshot_download # noqa: E402
from diffusers import WanAnimate2DistilledModularPipeline # noqa: E402
from diffusers.utils import export_to_video, load_image, load_video # noqa: E402
# ---------------------------------------------------------------------------
# ZeroGPU cannot host torch.compile / dynamo inside the forked GPU worker.
# The Wan-Animate-2 transformer builds its in-context BlockMask with
# `flex_attention.create_block_mask(..., _compile=True)`, which triggers a
# dynamo compile and aborts on ZeroGPU. Force the eager (`_compile=False`)
# path so the mask is built without compilation.
# ---------------------------------------------------------------------------
import torch.nn.attention.flex_attention as _flex_mod # noqa: E402
_orig_create_block_mask = _flex_mod.create_block_mask
def _create_block_mask_no_compile(*args, **kwargs):
kwargs["_compile"] = False
return _orig_create_block_mask(*args, **kwargs)
_flex_mod.create_block_mask = _create_block_mask_no_compile
# Also patch the reference already imported into the transformer module.
try:
import diffusers.models.transformers.transformer_wan_animate_2 as _wa2 # noqa: E402
_wa2.create_block_mask = _create_block_mask_no_compile
except Exception as _e: # pragma: no cover
print(f"[patch] could not patch transformer_wan_animate_2.create_block_mask: {_e!r}")
# ---------------------------------------------------------------------------
# The Wan-Animate-2 in-context self-attention runs on the FLEX backend with a
# custom `BlockMask`. On ZeroGPU we cannot `torch.compile` flex_attention (the
# forked GPU worker has no dynamo daemon), and the eager flex fallback
# decomposes into a dense `Q@K^T` (math_attention) that OOMs the allocator on
# a 14B video DiT. Instead we replace the FLEX backend with a memory-efficient
# SDPA path: materialise the BlockMask's boolean mask ONCE (Q*KV bytes,
# cheap) and hand it to `scaled_dot_product_attention`, whose mem-efficient
# kernel never builds the full scores matrix. Same masking semantics, no
# compile.
# ---------------------------------------------------------------------------
from torch.nn.attention.flex_attention import BlockMask as _BlockMask # noqa: E402
import torch.nn.functional as _F # noqa: E402
import diffusers.models.attention_dispatch as _attn_dispatch # noqa: E402
_dense_mask_cache = {}
def _blockmask_to_dense_bool(block_mask, seq_len_q, seq_len_kv, device):
key = (id(block_mask), seq_len_q, seq_len_kv)
cached = _dense_mask_cache.get(key)
if cached is not None:
return cached
mask_mod = block_mask.mask_mod
zero = torch.zeros((), dtype=torch.long, device=device)
kv_idx = torch.arange(seq_len_kv, device=device)
# Build the [Q, KV] boolean mask row-block by row-block so we never hold
# multiple Q*KV int64 temporaries at once (the mask_mod uses torch.where on
# int64 index grids, which spikes memory badly at high resolution).
dense = torch.empty((seq_len_q, seq_len_kv), dtype=torch.bool, device=device)
q_chunk = max(1, min(seq_len_q, 512))
for q0 in range(0, seq_len_q, q_chunk):
q1 = min(seq_len_q, q0 + q_chunk)
rows = q1 - q0
qg = torch.arange(q0, q1, device=device).view(rows, 1).expand(rows, seq_len_kv)
kg = kv_idx.view(1, seq_len_kv).expand(rows, seq_len_kv)
dense[q0:q1] = mask_mod(zero, zero, qg, kg).to(torch.bool)
dense = dense.view(1, 1, seq_len_q, seq_len_kv)
if len(_dense_mask_cache) > 4:
_dense_mask_cache.clear()
_dense_mask_cache[key] = dense
return dense
def _flex_backend_sdpa(
query, key, value, attn_mask=None, is_causal=False, scale=None,
enable_gqa=False, return_lse=False, _parallel_config=None,
):
# query/key/value: [B, seq, H, D] -> SDPA wants [B, H, seq, D]
seq_len_q = query.shape[1]
seq_len_kv = key.shape[1]
q = query.permute(0, 2, 1, 3)
k = key.permute(0, 2, 1, 3)
v = value.permute(0, 2, 1, 3)
sdpa_mask = None
if isinstance(attn_mask, _BlockMask):
sdpa_mask = _blockmask_to_dense_bool(attn_mask, seq_len_q, seq_len_kv, query.device)
elif attn_mask is not None and torch.is_tensor(attn_mask):
sdpa_mask = attn_mask
out = _F.scaled_dot_product_attention(
q, k, v, attn_mask=sdpa_mask, is_causal=is_causal, scale=scale, enable_gqa=enable_gqa,
)
out = out.permute(0, 2, 1, 3)
if return_lse:
return out, None
return out
# Re-register the FLEX backend to point at the SDPA-based implementation.
_attn_dispatch._native_flex_attention = _flex_backend_sdpa
try:
_reg = _attn_dispatch._AttentionBackendRegistry
_reg._backends[_attn_dispatch.AttentionBackendName.FLEX] = _flex_backend_sdpa
print("[patch] FLEX backend replaced with memory-efficient SDPA path")
except Exception as _e: # pragma: no cover
print(f"[patch] could not re-register FLEX backend: {_e!r}")
MODEL_ID = "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"
DEFAULT_PROMPT_REF = "人物动作的参考视频"
# ---------------------------------------------------------------------------
# Load the pipeline at module scope, eagerly to CUDA (ZeroGPU packs it).
#
# The repo's `modular_model_index.json` pins `revision: "refs/pr/1"` on the
# transformer and scheduler, a PR revision that no longer exists on the Hub.
# `load_components(revision=None)` overrides the stale spec value so the
# components load from `main`; otherwise load_components would silently skip
# the failed component and crash later.
# ---------------------------------------------------------------------------
pipe = WanAnimate2DistilledModularPipeline.from_pretrained(MODEL_ID)
pipe.load_components(dtype=torch.bfloat16, revision=None)
pipe.to("cuda")
# ---------------------------------------------------------------------------
# AoTI (Ahead-of-Time Inductor) acceleration.
#
# The precompiled artifact covers the repeated block's feed-forward sub-graph
# (`block.ffn`), which is a clean tensor->tensor graph (Linear->GELU->Linear)
# and is architecturally identical across all 40 blocks. One compiled graph
# (with a dynamic token-count axis, per-block weights streamed in as runtime
# inputs) patches every block. Falls back to eager on any error so the demo
# keeps running regardless.
# ---------------------------------------------------------------------------
AOTI_REPO = "multimodalart/Wan2.2-Animate-2-14B-Distilled-aoti"
AOTI_FFN_DIR = "WanAnimate2TransformerBlockFFN"
try:
from pathlib import Path as _Path
from spaces.zero.torch.aoti import aoti_load_from_module_dir
_aoti_root = _Path(snapshot_download(AOTI_REPO, allow_patterns=[f"{AOTI_FFN_DIR}/*"]))
_ffn_pkg_dir = _aoti_root / AOTI_FFN_DIR
if (_ffn_pkg_dir / "package.pt2").exists():
_ffns = [blk.ffn for blk in pipe.transformer.blocks]
aoti_load_from_module_dir(_ffns, _ffn_pkg_dir)
print(f"[aoti] patched {len(_ffns)} block.ffn modules from {AOTI_REPO}/{AOTI_FFN_DIR}")
else:
print(f"[aoti] {AOTI_FFN_DIR}/package.pt2 not found in {AOTI_REPO}; running eager")
except Exception as _e: # pragma: no cover
print(f"[aoti] AoTI load failed ({_e!r}); running eager")
def _trim_video(src_path: str, max_seconds: float, target_fps: int = 24) -> str:
"""Trim the driving video to the first `max_seconds` at `target_fps`.
The pipeline animates the whole clip in 81-frame segments, so a long clip
is very slow. Trimming keeps a single-request demo tractable. Falls back
to the original path if ffmpeg is unavailable.
"""
out = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
try:
subprocess.run(
[
"ffmpeg", "-y", "-i", src_path,
"-t", str(max_seconds),
"-r", str(target_fps),
"-an",
"-c:v", "libx264", "-pix_fmt", "yuv420p",
out,
],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
return out
except Exception as e:
print(f"[trim] ffmpeg failed ({e!r}); using original video")
return src_path
def _estimate_duration(image, driving_video, prompt, max_seconds=2.0, height=480,
width=384, num_inference_steps=6, *args, **kwargs):
# The in-context self-attention runs on a memory-efficient (but not
# compiled) SDPA path whose cost grows ~quadratically with the token count
# (h*w), and linearly with steps and segments. Anchored on live
# measurements of this exact Space:
# 320x480, 4 steps, 1 seg -> ~23 s
# 640x480, 8 steps, 1 seg -> ~237 s
try:
segments = max(1, int(math.ceil(float(max_seconds) / 3.4)))
except Exception:
segments = 1
try:
pixels = float(height) * float(width)
steps = max(1, int(num_inference_steps))
except Exception:
pixels, steps = 480 * 384, 6
P0 = 320.0 * 480.0
per_step = 3.25 * (pixels / P0) ** 3.1 # seconds per step
per_segment = steps * per_step # dominant per-segment cost
total = 10.0 + segments * per_segment # + model/setup overhead
return int(min(1500, total * 1.2 + 15))
@spaces.GPU(duration=_estimate_duration, size="xlarge")
def animate(
image,
driving_video,
prompt,
max_seconds: float = 2.0,
height: int = 480,
width: int = 384,
num_inference_steps: int = 6,
seed: int = 0,
prompt_ref: str = DEFAULT_PROMPT_REF,
progress=gr.Progress(track_tqdm=True),
):
"""Animate a reference character image with the motion from a driving video.
Args:
image: Reference character image (the person / character to animate).
driving_video: Driving video whose motion is transferred to the character.
prompt: Text description of the character appearance and background.
max_seconds: How many seconds of the driving video to animate.
height: Together with width, the target area of the generated video;
the aspect ratio comes from the reference image.
width: See height.
num_inference_steps: Denoising steps per segment.
seed: Random seed.
prompt_ref: Reference prompt for the driving-video context.
Returns:
Path to the generated animated video (mp4).
"""
if image is None:
raise gr.Error("Please provide a reference character image.")
if driving_video is None:
raise gr.Error("Please provide a driving video.")
if isinstance(image, str):
image = load_image(image)
if not isinstance(image, Image.Image):
image = Image.fromarray(np.asarray(image))
image = image.convert("RGB")
# The pipeline resolves the actual frame size from the target AREA
# (height x width) and the image's aspect ratio, snapping to multiples of
# the VAE scale x patch size (16). Pre-snap so the estimate matches.
height = int(height) - (int(height) % 16)
width = int(width) - (int(width) % 16)
trimmed = _trim_video(driving_video, float(max_seconds), target_fps=24)
driving_frames, driving_fps = load_video(trimmed, return_fps=True)
generator = torch.Generator(device="cuda").manual_seed(int(seed))
try:
output = pipe(
prompt=prompt or "",
image=image,
driving_video=driving_frames,
driving_video_fps=driving_fps,
height=height,
width=width,
fps=24,
num_inference_steps=int(num_inference_steps),
prompt_ref=prompt_ref or DEFAULT_PROMPT_REF,
generator=generator,
output_type="np",
output="videos",
)
except Exception as exc: # surface the real traceback to the client/logs
import traceback
tb = traceback.format_exc()
print("[animate] inference failed:\n" + tb, flush=True)
raise gr.Error(f"{type(exc).__name__}: {exc}\n{tb[-1500:]}")
frames = output[0]
out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
export_to_video(frames, out_path, fps=24)
return out_path
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks() as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"""
# Wan2.2-Animate-2-14B (Distilled)
Animate a **reference character image** with the motion from a
**driving video**. Powered by
[`Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers`](https://huggingface.co/Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers)
via 🧨 diffusers. The distilled variant samples in a few steps
without classifier-free guidance, which keeps it fast enough for an
interactive demo.
"""
)
with gr.Row():
with gr.Column():
image = gr.Image(label="Reference character image", type="filepath", height=340)
driving_video = gr.Video(label="Driving video (motion source)", height=340)
prompt = gr.Textbox(
label="Prompt",
placeholder="Describe the character appearance and background…",
value="a person",
lines=2,
)
run = gr.Button("Animate", variant="primary")
with gr.Column():
output_video = gr.Video(label="Animated result", height=520)
with gr.Accordion("Advanced settings", open=False):
max_seconds = gr.Slider(
label="Seconds of driving video to animate",
minimum=1.0, maximum=5.0, step=0.5, value=2.0,
)
with gr.Row():
height = gr.Slider(label="Height", minimum=320, maximum=640, step=16, value=480)
width = gr.Slider(label="Width", minimum=320, maximum=640, step=16, value=384)
with gr.Row():
num_inference_steps = gr.Slider(
label="Inference steps", minimum=4, maximum=12, step=1, value=6,
)
seed = gr.Number(label="Seed", value=0, precision=0)
prompt_ref = gr.Textbox(
label="Reference prompt (driving-video context)",
value=DEFAULT_PROMPT_REF, lines=1,
)
gr.Examples(
examples=[
[
"examples/cat_reference.jpg",
"examples/cat_driving.mp4",
"人物外观描述:一只银灰色虎斑纹的小猫,拥有圆润的脸庞、竖立的耳朵和巨大的圆形眼睛。它身穿一套深蓝色的制服套装,包括一件带有金色纽扣的西装外套和一条百褶裙。外套里面搭配着白色衬衫,领口处系着一个红色的蝴蝶结,袖口露出白色的衬衫边缘。背景描述:背景为纯白色,光线均匀明亮,无其他杂物或装饰。",
],
],
inputs=[image, driving_video, prompt],
outputs=output_video,
fn=animate,
cache_examples=True,
cache_mode="lazy",
)
run.click(
fn=animate,
inputs=[
image, driving_video, prompt, max_seconds, height, width,
num_inference_steps, seed, prompt_ref,
],
outputs=output_video,
api_name="animate",
)
if __name__ == "__main__":
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)