mfn
Initial Space demo
c94e874
Raw History Blame Contribute Delete
11.6 kB
# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.
# Modifications Copyright (C) 2026 Apple Inc. All Rights Reserved.
# Licensed under the Apache License, Version 2.0; see ACKNOWLEDGEMENTS.
# Modified for FastFrames (shared layers and the bidirectional Wan2.1 backbone); the frame-offset
# RoPE ``causal_rope_apply`` is adapted from Self-Forcing (Apache-2.0).
import math
from dataclasses import dataclass
import torch
import torch.nn as nn
from .attention import attention, cross_attention
__all__ = [
"WanConfig",
"WAN_CONFIGS",
"WanModel",
"WanRMSNorm",
"WanLayerNorm",
"WanCrossAttention",
"sinusoidal_embedding_1d",
"rope_params",
"rope_apply",
"causal_rope_apply",
]
def sinusoidal_embedding_1d(dim, position):
assert dim % 2 == 0
half = dim // 2
position = position.type(torch.float64)
sinusoid = torch.outer(
position, torch.pow(10000, -torch.arange(half).to(position).div(half)))
return torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1)
def rope_params(max_seq_len, dim, theta=10000):
assert dim % 2 == 0
freqs = torch.outer(
torch.arange(max_seq_len),
1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float64).div(dim)))
return torch.polar(torch.ones_like(freqs), freqs)
def causal_rope_apply(x, grid_sizes, freqs, start_frame=0):
"""3-D RoPE on ``[B, L, H, D]`` tokens whose first frame is ``start_frame``."""
n, c = x.size(2), x.size(3) // 2
freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1)
output = []
for i, (f, h, w) in enumerate(grid_sizes.tolist()):
seq_len = f * h * w
x_i = torch.view_as_complex(
x[i, :seq_len].to(torch.float64).reshape(seq_len, n, -1, 2))
freqs_i = torch.cat([
freqs[0][start_frame:start_frame + f].view(f, 1, 1, -1).expand(f, h, w, -1),
freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),
freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1),
], dim=-1).reshape(seq_len, 1, -1)
x_i = torch.view_as_real(x_i * freqs_i).flatten(2)
output.append(torch.cat([x_i, x[i, seq_len:]]))
return torch.stack(output).type_as(x)
def rope_apply(x, grid_sizes, freqs):
return causal_rope_apply(x, grid_sizes, freqs, start_frame=0)
class WanRMSNorm(nn.Module):
def __init__(self, dim, eps=1e-5):
super().__init__()
self.dim = dim
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x):
return self._norm(x.float()).type_as(x) * self.weight
def _norm(self, x):
return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
class WanLayerNorm(nn.LayerNorm):
def __init__(self, dim, eps=1e-6, elementwise_affine=False):
super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps)
def forward(self, x):
return super().forward(x).type_as(x)
class WanSelfAttention(nn.Module):
def __init__(self, dim, num_heads, qk_norm=True, eps=1e-6):
assert dim % num_heads == 0
super().__init__()
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.q = nn.Linear(dim, dim)
self.k = nn.Linear(dim, dim)
self.v = nn.Linear(dim, dim)
self.o = nn.Linear(dim, dim)
self.norm_q = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity()
self.norm_k = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity()
def qkv(self, x):
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
q = self.norm_q(self.q(x)).view(b, s, n, d)
k = self.norm_k(self.k(x)).view(b, s, n, d)
v = self.v(x).view(b, s, n, d)
return q, k, v
def forward(self, x, grid_sizes, freqs):
q, k, v = self.qkv(x)
x = attention(rope_apply(q, grid_sizes, freqs), rope_apply(k, grid_sizes, freqs), v)
return self.o(x.flatten(2))
class WanCrossAttention(WanSelfAttention):
"""Text cross-attention. ``cache`` optionally holds the projected text K/V."""
def text_kv(self, context):
b, n, d = context.size(0), self.num_heads, self.head_dim
k = self.norm_k(self.k(context)).view(b, -1, n, d)
v = self.v(context).view(b, -1, n, d)
return k, v
def forward(self, x, context, cache=None):
b, n, d = x.size(0), self.num_heads, self.head_dim
q = self.norm_q(self.q(x)).view(b, -1, n, d)
k, v = (cache["k"], cache["v"]) if cache is not None else self.text_kv(context)
return self.o(cross_attention(q, k, v).flatten(2))
class WanAttentionBlock(nn.Module):
def __init__(self, dim, ffn_dim, num_heads, qk_norm=True, cross_attn_norm=False, eps=1e-6):
super().__init__()
self.dim = dim
self.norm1 = WanLayerNorm(dim, eps)
self.self_attn = WanSelfAttention(dim, num_heads, qk_norm, eps)
self.norm3 = WanLayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity()
self.cross_attn = WanCrossAttention(dim, num_heads, qk_norm, eps)
self.norm2 = WanLayerNorm(dim, eps)
self.ffn = nn.Sequential(
nn.Linear(dim, ffn_dim), nn.GELU(approximate="tanh"), nn.Linear(ffn_dim, dim))
self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
def forward(self, x, e, grid_sizes, freqs, context):
e = (self.modulation + e).chunk(6, dim=1)
y = self.self_attn(self.norm1(x) * (1 + e[1]) + e[0], grid_sizes, freqs)
x = x + y * e[2]
x = x + self.cross_attn(self.norm3(x), context)
y = self.ffn(self.norm2(x) * (1 + e[4]) + e[3])
return x + y * e[5]
class Head(nn.Module):
def __init__(self, dim, out_dim, patch_size, eps=1e-6):
super().__init__()
self.dim = dim
self.out_dim = out_dim
self.patch_size = patch_size
self.norm = WanLayerNorm(dim, eps)
self.head = nn.Linear(dim, math.prod(patch_size) * out_dim)
self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
def forward(self, x, e):
e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1)
return self.head(self.norm(x) * (1 + e[1]) + e[0])
def unpatchify(x, grid_sizes, patch_size, out_dim):
"""``[B, L, prod(patch) * C]`` tokens -> list of ``[C, F, H, W]`` videos."""
out = []
for u, v in zip(x, grid_sizes.tolist()):
u = u[:math.prod(v)].view(*v, *patch_size, out_dim)
u = torch.einsum("fhwpqrc->cfphqwr", u)
out.append(u.reshape(out_dim, *[i * j for i, j in zip(v, patch_size)]))
return out
def init_wan_weights(model):
for m in model.modules():
if isinstance(m, nn.Linear):
nn.init.xavier_uniform_(m.weight)
if m.bias is not None:
nn.init.zeros_(m.bias)
nn.init.xavier_uniform_(model.patch_embedding.weight.flatten(1))
for m in model.text_embedding.modules():
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, std=.02)
for m in model.time_embedding.modules():
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, std=.02)
nn.init.zeros_(model.head.head.weight)
@dataclass(frozen=True)
class WanConfig:
dim: int
ffn_dim: int
num_heads: int
num_layers: int
patch_size: tuple = (1, 2, 2)
text_len: int = 512
in_dim: int = 16
freq_dim: int = 256
text_dim: int = 4096
out_dim: int = 16
qk_norm: bool = True
cross_attn_norm: bool = True
eps: float = 1e-6
WAN_CONFIGS = {
"Wan2.1-T2V-1.3B": WanConfig(dim=1536, ffn_dim=8960, num_heads=12, num_layers=30),
"Wan2.1-T2V-14B": WanConfig(dim=5120, ffn_dim=13824, num_heads=40, num_layers=40),
}
class WanBackbone(nn.Module):
"""Embeddings and hyper-parameters shared by the bidirectional and causal models."""
block_class = None
head_class = Head
def __init__(self, config: WanConfig):
super().__init__()
self.config = config
dim = config.dim
self.patch_size = config.patch_size
self.text_len = config.text_len
self.dim = dim
self.freq_dim = config.freq_dim
self.out_dim = config.out_dim
self.num_heads = config.num_heads
self.num_layers = config.num_layers
self.patch_embedding = nn.Conv3d(
config.in_dim, dim, kernel_size=config.patch_size, stride=config.patch_size)
self.text_embedding = nn.Sequential(
nn.Linear(config.text_dim, dim), nn.GELU(approximate="tanh"), nn.Linear(dim, dim))
self.time_embedding = nn.Sequential(
nn.Linear(config.freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim))
self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * 6))
self.blocks = nn.ModuleList([
self.block_class(dim, config.ffn_dim, config.num_heads, config.qk_norm,
config.cross_attn_norm, config.eps)
for _ in range(config.num_layers)
])
self.head = self.head_class(dim, config.out_dim, config.patch_size, config.eps)
self.freqs = self.rope_table()
init_wan_weights(self)
self.gradient_checkpointing = False
def rope_table(self):
"""Complex128 RoPE table; deliberately not a buffer so ``.to(dtype)`` keeps it."""
d = self.dim // self.num_heads
with torch.device("cpu"):
return torch.cat([
rope_params(1024, d - 4 * (d // 6)),
rope_params(1024, 2 * (d // 6)),
rope_params(1024, 2 * (d // 6)),
], dim=1)
def enable_gradient_checkpointing(self):
self.gradient_checkpointing = True
def embed_text(self, prompt_embeds):
"""Zero-pad T5 embeddings to ``text_len`` tokens and project them."""
return self.text_embedding(torch.stack([
torch.cat([u, u.new_zeros(self.text_len - u.size(0), u.size(1))])
for u in prompt_embeds
]))
def unpatchify(self, x, grid_sizes):
return unpatchify(x, grid_sizes, self.patch_size, self.out_dim)
def _run_block(self, block, *args, **kwargs):
if torch.is_grad_enabled() and self.gradient_checkpointing:
return torch.utils.checkpoint.checkpoint(block, *args, use_reentrant=False, **kwargs)
return block(*args, **kwargs)
class WanModel(WanBackbone):
"""Bidirectional Wan2.1 T2V backbone (teacher / score networks during training)."""
block_class = WanAttentionBlock
def forward(self, x, t, context, seq_len):
"""x: ``[B, C, F, H, W]``; t: ``[B]``; context: T5 embeddings ``[B, L, 4096]``."""
device = self.patch_embedding.weight.device
if self.freqs.device != device:
self.freqs = self.freqs.to(device)
x = [self.patch_embedding(u.unsqueeze(0)) for u in x]
grid_sizes = torch.stack([torch.tensor(u.shape[2:], dtype=torch.long) for u in x])
x = [u.flatten(2).transpose(1, 2) for u in x]
assert max(u.size(1) for u in x) <= seq_len
x = torch.cat([torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], dim=1) for u in x])
e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t).type_as(x))
e0 = self.time_projection(e).unflatten(1, (6, self.dim))
context = self.embed_text(context)
for block in self.blocks:
x = self._run_block(block, x, e0, grid_sizes, self.freqs, context)
x = self.head(x, e)
return torch.stack(self.unpatchify(x, grid_sizes))