"""Frame-wise causal Wan2.1 DiT used as the FastFrames generator. This is the inference path in ``CausalWanModel.forward_frame``: one latent frame attends to its own tokens and to the KV cache of earlier frames. The pre-head hidden state is what the pixel decoder reads. RoPE follows the complex reference (FastFrames-S sets ``fast_ops=False``) but the multiply is float32 so it can run on the GPU. """ from __future__ import annotations import math import mlx.core as mx import mlx.nn as nn from .config import WanConfig from .ops import gelu_tanh, layer_norm, rms_norm, sdpa, silu, sinusoidal_embedding_1d from .rope import RopeTables def _zeros(shape): return mx.zeros(shape) class _RMS(nn.Module): def __init__(self, dim, eps): super().__init__() self.eps = eps self.weight = mx.ones((dim,)) def __call__(self, x): return rms_norm(x, self.weight, self.eps) class _AffineLN(nn.Module): def __init__(self, dim, eps): super().__init__() self.eps = eps self.weight = mx.ones((dim,)) self.bias = mx.zeros((dim,)) def __call__(self, x): return layer_norm(x, self.eps, self.weight, self.bias) class SelfAttention(nn.Module): def __init__(self, dim, num_heads, eps): super().__init__() self.num_heads = num_heads self.head_dim = dim // num_heads self.q = nn.Linear(dim, dim) self.k = nn.Linear(dim, dim) self.v = nn.Linear(dim, dim) self.o = nn.Linear(dim, dim) self.norm_q = _RMS(dim, eps) self.norm_k = _RMS(dim, eps) def qkv(self, x): b, s, _ = x.shape q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim) k = self.norm_k(self.k(x)).reshape(b, s, self.num_heads, self.head_dim) v = self.v(x).reshape(b, s, self.num_heads, self.head_dim) return q, k, v def text_kv(self, context): b = context.shape[0] k = self.norm_k(self.k(context)).reshape(b, -1, self.num_heads, self.head_dim) v = self.v(context).reshape(b, -1, self.num_heads, self.head_dim) return k, v class CrossAttention(SelfAttention): def __call__(self, x, cache): b, s, _ = x.shape q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim) k, v = cache out = sdpa(q, k, v).reshape(b, s, -1) return self.o(out) class FeedForward(nn.Module): def __init__(self, dim, ffn_dim): super().__init__() self.fc1 = nn.Linear(dim, ffn_dim) self.fc2 = nn.Linear(ffn_dim, dim) def __call__(self, x): return self.fc2(gelu_tanh(self.fc1(x))) class CausalBlock(nn.Module): def __init__(self, dim, ffn_dim, num_heads, eps): super().__init__() self.eps = eps self.self_attn = SelfAttention(dim, num_heads, eps) self.cross_attn = CrossAttention(dim, num_heads, eps) self.norm3 = _AffineLN(dim, eps) self.ffn = FeedForward(dim, ffn_dim) self.modulation = _zeros((1, 6, dim)) def forward_frame(self, x, e, grid, rope: RopeTables, cross_kv, kv_cache, frame: int): """``x`` is ``[B, tokens, dim]``. ``e`` is ``[B, 1, 6, dim]``.""" tokens = x.shape[1] frames, height, width = grid mod = self.modulation.reshape(1, 1, 6, -1) + e parts = mx.split(mod, 6, axis=2) e0, e1, e2, e3, e4, e5 = [p.reshape(p.shape[0], p.shape[1], 1, p.shape[-1]) for p in parts] h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) h = (h * (1.0 + e1) + e0).reshape(x.shape[0], tokens, -1) q, k, v = self.self_attn.qkv(h) q = rope.apply(q, frames, height, width, start_frame=frame) k = rope.apply(k, frames, height, width, start_frame=frame) kv_cache.write(frame, k, v) k_pre, v_pre = kv_cache.prefix(frame) y = self.self_attn.o(sdpa(q, k_pre, v_pre).reshape(x.shape[0], tokens, -1)) x = x + (y.reshape(x.shape[0], 1, tokens, -1) * e2).reshape(x.shape[0], tokens, -1) x = x + self.cross_attn(self.norm3(x), cross_kv) y = self.ffn((layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) * (1.0 + e4) + e3).reshape(x.shape[0], tokens, -1)) y = (y.reshape(x.shape[0], 1, tokens, -1) * e5).reshape(x.shape[0], tokens, -1) return x + y class CausalHead(nn.Module): def __init__(self, dim, out_dim, patch_size, eps): super().__init__() self.eps = eps self.patch_size = patch_size self.out_dim = out_dim self.head = nn.Linear(dim, math.prod(patch_size) * out_dim) self.modulation = _zeros((1, 2, dim)) def __call__(self, x, e): """``x`` ``[B, tokens, dim]``, ``e`` ``[B, 1, 1, dim]``.""" tokens = x.shape[1] mod = self.modulation.reshape(1, 1, 2, -1) + e shift, scale = mx.split(mod, 2, axis=2) h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) h = (h * (1.0 + scale) + shift).reshape(x.shape[0], tokens, -1) return self.head(h) def unpatchify(tokens, grid, patch_size, out_dim): """``[B, tokens, prod(patch)*C]`` -> ``[B, C, F, H, W]`` in Wan layout.""" b = tokens.shape[0] f, h, w = grid pt, ph, pw = patch_size u = tokens.reshape(b, f, h, w, pt, ph, pw, out_dim) u = mx.transpose(u, (0, 7, 1, 4, 2, 5, 3, 6)) return u.reshape(b, out_dim, f * pt, h * ph, w * pw) class PatchEmbed(nn.Module): def __init__(self, in_dim, dim, patch_size): super().__init__() self.patch_size = tuple(patch_size) pt, ph, pw = self.patch_size # MLX conv weight is [out, kd, kh, kw, in]. self.weight = _zeros((dim, pt, ph, pw, in_dim)) self.bias = _zeros((dim,)) def __call__(self, latent): """``latent`` is ``[B, 1, C, H, W]`` -> ``[B, tokens, dim]``.""" from .ops import conv3d_ncdhw x = mx.transpose(latent, (0, 2, 1, 3, 4)) y = conv3d_ncdhw(x, self.weight, self.bias, stride=self.patch_size) y = mx.transpose(y, (0, 2, 3, 4, 1)) return y.reshape(y.shape[0], -1, y.shape[-1]) class KVCache: """Per-layer list of per-frame ``(k, v)`` with shape ``[B, tokens, heads, dim]``.""" def __init__(self, num_frames: int): self.num_frames = num_frames self.slots = [None] * num_frames def write(self, frame, k, v): mx.eval(k, v) self.slots[frame] = (k, v) def prefix(self, frame): ks = [self.slots[i][0] for i in range(frame + 1)] vs = [self.slots[i][1] for i in range(frame + 1)] if len(ks) == 1: return ks[0], vs[0] return mx.concatenate(ks, axis=1), mx.concatenate(vs, axis=1) def slot(self, frame): return self.slots[frame] class LayerCache: def __init__(self, num_layers, num_frames): self.layers = [KVCache(num_frames) for _ in range(num_layers)] def __getitem__(self, index): return self.layers[index] class CausalWanModel(nn.Module): def __init__(self, config: WanConfig = WanConfig()): super().__init__() self.config = config dim = config.dim self.dim = dim self.num_heads = config.num_heads self.num_layers = config.num_layers self.patch_size = config.patch_size self.text_len = config.text_len self.freq_dim = config.freq_dim self.out_dim = config.out_dim self.eps = config.eps self.patch_embedding = PatchEmbed(config.in_dim, dim, config.patch_size) self.text_embed_fc1 = nn.Linear(config.text_dim, dim) self.text_embed_fc2 = nn.Linear(dim, dim) self.time_embed_fc1 = nn.Linear(config.freq_dim, dim) self.time_embed_fc2 = nn.Linear(dim, dim) self.time_proj_fc = nn.Linear(dim, dim * 6) self.blocks = [ CausalBlock(dim, config.ffn_dim, config.num_heads, config.eps) for _ in range(config.num_layers) ] self.head = CausalHead(dim, config.out_dim, config.patch_size, config.eps) # Plain object so the cos/sin tables are not registered as parameters. self.rope = RopeTables(config.head_dim) def embed_text(self, prompt_embeds): if isinstance(prompt_embeds, (list, tuple)): rows = [] for row in prompt_embeds: if row.shape[0] < self.text_len: pad = mx.zeros((self.text_len - row.shape[0], row.shape[-1]), dtype=row.dtype) row = mx.concatenate([row, pad], axis=0) rows.append(row[: self.text_len]) prompt_embeds = mx.stack(rows, axis=0) elif prompt_embeds.shape[1] < self.text_len: pad = mx.zeros( (prompt_embeds.shape[0], self.text_len - prompt_embeds.shape[1], prompt_embeds.shape[2]), dtype=prompt_embeds.dtype, ) prompt_embeds = mx.concatenate([prompt_embeds, pad], axis=1) elif prompt_embeds.shape[1] > self.text_len: prompt_embeds = prompt_embeds[:, : self.text_len] return self.text_embed_fc2(gelu_tanh(self.text_embed_fc1(prompt_embeds))) def time_conditioning(self, t, dtype): """``t`` is ``[B, F]`` -> ``(e0 [B,F,6,C], e_head [B,F,1,C])``.""" emb = sinusoidal_embedding_1d(self.freq_dim, t.reshape(-1)).astype(dtype) e = self.time_embed_fc2(silu(self.time_embed_fc1(emb))) e0 = self.time_proj_fc(silu(e)).reshape(*t.shape, 6, self.dim) e_head = e.reshape(*t.shape, 1, self.dim) return e0, e_head def text_cache(self, prompt_embeds): context = self.embed_text(prompt_embeds) caches = [] for block in self.blocks: k, v = block.cross_attn.text_kv(context) mx.eval(k, v) caches.append((k, v)) return caches def forward_frame(self, latent, e0, e_head, text_cache, kv_cache: LayerCache, frame: int): """Denoise ``latent`` ``[B, 1, 16, H, W]``. Returns flow ``[B, 1, 16, H, W]`` and pre-head states ``[B, tokens, dim]``. """ _, _, _, height, width = latent.shape x = self.patch_embedding(latent) grid = (1, height // self.patch_size[1], width // self.patch_size[2]) for index, block in enumerate(self.blocks): x = block.forward_frame(x, e0, grid, self.rope, text_cache[index], kv_cache[index], frame) mx.eval(x) hidden = x tokens = self.head(x, e_head) flow = unpatchify(tokens, grid, self.patch_size, self.out_dim) flow = mx.transpose(flow, (0, 2, 1, 3, 4)) return flow, hidden