Spaces:
Running on Zero
Running on Zero
Download fastframes/wan/model.py from mrfakename/fastframes-l: direct link, hf CLI and curl.
- Browser
- Download file 11.6 kB
-
https://huggingface.co/spaces/mrfakename/fastframes-l/resolve/main/fastframes/wan/model.py
- Command line
-
hf download hf://spaces/mrfakename/fastframes-l/fastframes/wan/model.py
-
curl -L -o model.py https://huggingface.co/spaces/mrfakename/fastframes-l/resolve/main/fastframes/wan/model.py
11.6 kB
| # Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. | |
| # Modifications Copyright (C) 2026 Apple Inc. All Rights Reserved. | |
| # Licensed under the Apache License, Version 2.0; see ACKNOWLEDGEMENTS. | |
| # Modified for FastFrames (shared layers and the bidirectional Wan2.1 backbone); the frame-offset | |
| # RoPE ``causal_rope_apply`` is adapted from Self-Forcing (Apache-2.0). | |
| import math | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| from .attention import attention, cross_attention | |
| __all__ = [ | |
| "WanConfig", | |
| "WAN_CONFIGS", | |
| "WanModel", | |
| "WanRMSNorm", | |
| "WanLayerNorm", | |
| "WanCrossAttention", | |
| "sinusoidal_embedding_1d", | |
| "rope_params", | |
| "rope_apply", | |
| "causal_rope_apply", | |
| ] | |
| def sinusoidal_embedding_1d(dim, position): | |
| assert dim % 2 == 0 | |
| half = dim // 2 | |
| position = position.type(torch.float64) | |
| sinusoid = torch.outer( | |
| position, torch.pow(10000, -torch.arange(half).to(position).div(half))) | |
| return torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) | |
| def rope_params(max_seq_len, dim, theta=10000): | |
| assert dim % 2 == 0 | |
| freqs = torch.outer( | |
| torch.arange(max_seq_len), | |
| 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float64).div(dim))) | |
| return torch.polar(torch.ones_like(freqs), freqs) | |
| def causal_rope_apply(x, grid_sizes, freqs, start_frame=0): | |
| """3-D RoPE on ``[B, L, H, D]`` tokens whose first frame is ``start_frame``.""" | |
| n, c = x.size(2), x.size(3) // 2 | |
| freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) | |
| output = [] | |
| for i, (f, h, w) in enumerate(grid_sizes.tolist()): | |
| seq_len = f * h * w | |
| x_i = torch.view_as_complex( | |
| x[i, :seq_len].to(torch.float64).reshape(seq_len, n, -1, 2)) | |
| freqs_i = torch.cat([ | |
| freqs[0][start_frame:start_frame + f].view(f, 1, 1, -1).expand(f, h, w, -1), | |
| freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1), | |
| freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1), | |
| ], dim=-1).reshape(seq_len, 1, -1) | |
| x_i = torch.view_as_real(x_i * freqs_i).flatten(2) | |
| output.append(torch.cat([x_i, x[i, seq_len:]])) | |
| return torch.stack(output).type_as(x) | |
| def rope_apply(x, grid_sizes, freqs): | |
| return causal_rope_apply(x, grid_sizes, freqs, start_frame=0) | |
| class WanRMSNorm(nn.Module): | |
| def __init__(self, dim, eps=1e-5): | |
| super().__init__() | |
| self.dim = dim | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| def forward(self, x): | |
| return self._norm(x.float()).type_as(x) * self.weight | |
| def _norm(self, x): | |
| return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) | |
| class WanLayerNorm(nn.LayerNorm): | |
| def __init__(self, dim, eps=1e-6, elementwise_affine=False): | |
| super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) | |
| def forward(self, x): | |
| return super().forward(x).type_as(x) | |
| class WanSelfAttention(nn.Module): | |
| def __init__(self, dim, num_heads, qk_norm=True, eps=1e-6): | |
| assert dim % num_heads == 0 | |
| super().__init__() | |
| self.dim = dim | |
| self.num_heads = num_heads | |
| self.head_dim = dim // num_heads | |
| self.q = nn.Linear(dim, dim) | |
| self.k = nn.Linear(dim, dim) | |
| self.v = nn.Linear(dim, dim) | |
| self.o = nn.Linear(dim, dim) | |
| self.norm_q = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity() | |
| self.norm_k = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity() | |
| def qkv(self, x): | |
| b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim | |
| q = self.norm_q(self.q(x)).view(b, s, n, d) | |
| k = self.norm_k(self.k(x)).view(b, s, n, d) | |
| v = self.v(x).view(b, s, n, d) | |
| return q, k, v | |
| def forward(self, x, grid_sizes, freqs): | |
| q, k, v = self.qkv(x) | |
| x = attention(rope_apply(q, grid_sizes, freqs), rope_apply(k, grid_sizes, freqs), v) | |
| return self.o(x.flatten(2)) | |
| class WanCrossAttention(WanSelfAttention): | |
| """Text cross-attention. ``cache`` optionally holds the projected text K/V.""" | |
| def text_kv(self, context): | |
| b, n, d = context.size(0), self.num_heads, self.head_dim | |
| k = self.norm_k(self.k(context)).view(b, -1, n, d) | |
| v = self.v(context).view(b, -1, n, d) | |
| return k, v | |
| def forward(self, x, context, cache=None): | |
| b, n, d = x.size(0), self.num_heads, self.head_dim | |
| q = self.norm_q(self.q(x)).view(b, -1, n, d) | |
| k, v = (cache["k"], cache["v"]) if cache is not None else self.text_kv(context) | |
| return self.o(cross_attention(q, k, v).flatten(2)) | |
| class WanAttentionBlock(nn.Module): | |
| def __init__(self, dim, ffn_dim, num_heads, qk_norm=True, cross_attn_norm=False, eps=1e-6): | |
| super().__init__() | |
| self.dim = dim | |
| self.norm1 = WanLayerNorm(dim, eps) | |
| self.self_attn = WanSelfAttention(dim, num_heads, qk_norm, eps) | |
| self.norm3 = WanLayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity() | |
| self.cross_attn = WanCrossAttention(dim, num_heads, qk_norm, eps) | |
| self.norm2 = WanLayerNorm(dim, eps) | |
| self.ffn = nn.Sequential( | |
| nn.Linear(dim, ffn_dim), nn.GELU(approximate="tanh"), nn.Linear(ffn_dim, dim)) | |
| self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) | |
| def forward(self, x, e, grid_sizes, freqs, context): | |
| e = (self.modulation + e).chunk(6, dim=1) | |
| y = self.self_attn(self.norm1(x) * (1 + e[1]) + e[0], grid_sizes, freqs) | |
| x = x + y * e[2] | |
| x = x + self.cross_attn(self.norm3(x), context) | |
| y = self.ffn(self.norm2(x) * (1 + e[4]) + e[3]) | |
| return x + y * e[5] | |
| class Head(nn.Module): | |
| def __init__(self, dim, out_dim, patch_size, eps=1e-6): | |
| super().__init__() | |
| self.dim = dim | |
| self.out_dim = out_dim | |
| self.patch_size = patch_size | |
| self.norm = WanLayerNorm(dim, eps) | |
| self.head = nn.Linear(dim, math.prod(patch_size) * out_dim) | |
| self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) | |
| def forward(self, x, e): | |
| e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1) | |
| return self.head(self.norm(x) * (1 + e[1]) + e[0]) | |
| def unpatchify(x, grid_sizes, patch_size, out_dim): | |
| """``[B, L, prod(patch) * C]`` tokens -> list of ``[C, F, H, W]`` videos.""" | |
| out = [] | |
| for u, v in zip(x, grid_sizes.tolist()): | |
| u = u[:math.prod(v)].view(*v, *patch_size, out_dim) | |
| u = torch.einsum("fhwpqrc->cfphqwr", u) | |
| out.append(u.reshape(out_dim, *[i * j for i, j in zip(v, patch_size)])) | |
| return out | |
| def init_wan_weights(model): | |
| for m in model.modules(): | |
| if isinstance(m, nn.Linear): | |
| nn.init.xavier_uniform_(m.weight) | |
| if m.bias is not None: | |
| nn.init.zeros_(m.bias) | |
| nn.init.xavier_uniform_(model.patch_embedding.weight.flatten(1)) | |
| for m in model.text_embedding.modules(): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, std=.02) | |
| for m in model.time_embedding.modules(): | |
| if isinstance(m, nn.Linear): | |
| nn.init.normal_(m.weight, std=.02) | |
| nn.init.zeros_(model.head.head.weight) | |
| class WanConfig: | |
| dim: int | |
| ffn_dim: int | |
| num_heads: int | |
| num_layers: int | |
| patch_size: tuple = (1, 2, 2) | |
| text_len: int = 512 | |
| in_dim: int = 16 | |
| freq_dim: int = 256 | |
| text_dim: int = 4096 | |
| out_dim: int = 16 | |
| qk_norm: bool = True | |
| cross_attn_norm: bool = True | |
| eps: float = 1e-6 | |
| WAN_CONFIGS = { | |
| "Wan2.1-T2V-1.3B": WanConfig(dim=1536, ffn_dim=8960, num_heads=12, num_layers=30), | |
| "Wan2.1-T2V-14B": WanConfig(dim=5120, ffn_dim=13824, num_heads=40, num_layers=40), | |
| } | |
| class WanBackbone(nn.Module): | |
| """Embeddings and hyper-parameters shared by the bidirectional and causal models.""" | |
| block_class = None | |
| head_class = Head | |
| def __init__(self, config: WanConfig): | |
| super().__init__() | |
| self.config = config | |
| dim = config.dim | |
| self.patch_size = config.patch_size | |
| self.text_len = config.text_len | |
| self.dim = dim | |
| self.freq_dim = config.freq_dim | |
| self.out_dim = config.out_dim | |
| self.num_heads = config.num_heads | |
| self.num_layers = config.num_layers | |
| self.patch_embedding = nn.Conv3d( | |
| config.in_dim, dim, kernel_size=config.patch_size, stride=config.patch_size) | |
| self.text_embedding = nn.Sequential( | |
| nn.Linear(config.text_dim, dim), nn.GELU(approximate="tanh"), nn.Linear(dim, dim)) | |
| self.time_embedding = nn.Sequential( | |
| nn.Linear(config.freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim)) | |
| self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * 6)) | |
| self.blocks = nn.ModuleList([ | |
| self.block_class(dim, config.ffn_dim, config.num_heads, config.qk_norm, | |
| config.cross_attn_norm, config.eps) | |
| for _ in range(config.num_layers) | |
| ]) | |
| self.head = self.head_class(dim, config.out_dim, config.patch_size, config.eps) | |
| self.freqs = self.rope_table() | |
| init_wan_weights(self) | |
| self.gradient_checkpointing = False | |
| def rope_table(self): | |
| """Complex128 RoPE table; deliberately not a buffer so ``.to(dtype)`` keeps it.""" | |
| d = self.dim // self.num_heads | |
| with torch.device("cpu"): | |
| return torch.cat([ | |
| rope_params(1024, d - 4 * (d // 6)), | |
| rope_params(1024, 2 * (d // 6)), | |
| rope_params(1024, 2 * (d // 6)), | |
| ], dim=1) | |
| def enable_gradient_checkpointing(self): | |
| self.gradient_checkpointing = True | |
| def embed_text(self, prompt_embeds): | |
| """Zero-pad T5 embeddings to ``text_len`` tokens and project them.""" | |
| return self.text_embedding(torch.stack([ | |
| torch.cat([u, u.new_zeros(self.text_len - u.size(0), u.size(1))]) | |
| for u in prompt_embeds | |
| ])) | |
| def unpatchify(self, x, grid_sizes): | |
| return unpatchify(x, grid_sizes, self.patch_size, self.out_dim) | |
| def _run_block(self, block, *args, **kwargs): | |
| if torch.is_grad_enabled() and self.gradient_checkpointing: | |
| return torch.utils.checkpoint.checkpoint(block, *args, use_reentrant=False, **kwargs) | |
| return block(*args, **kwargs) | |
| class WanModel(WanBackbone): | |
| """Bidirectional Wan2.1 T2V backbone (teacher / score networks during training).""" | |
| block_class = WanAttentionBlock | |
| def forward(self, x, t, context, seq_len): | |
| """x: ``[B, C, F, H, W]``; t: ``[B]``; context: T5 embeddings ``[B, L, 4096]``.""" | |
| device = self.patch_embedding.weight.device | |
| if self.freqs.device != device: | |
| self.freqs = self.freqs.to(device) | |
| x = [self.patch_embedding(u.unsqueeze(0)) for u in x] | |
| grid_sizes = torch.stack([torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) | |
| x = [u.flatten(2).transpose(1, 2) for u in x] | |
| assert max(u.size(1) for u in x) <= seq_len | |
| x = torch.cat([torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], dim=1) for u in x]) | |
| e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t).type_as(x)) | |
| e0 = self.time_projection(e).unflatten(1, (6, self.dim)) | |
| context = self.embed_text(context) | |
| for block in self.blocks: | |
| x = self._run_block(block, x, e0, grid_sizes, self.freqs, context) | |
| x = self.head(x, e) | |
| return torch.stack(self.unpatchify(x, grid_sizes)) | |