Text-to-Video
MLX
apple-silicon
pratikgx's picture
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
e87ef2c verified
Raw History Blame Contribute Delete
10.7 kB
"""Frame-wise causal Wan2.1 DiT used as the FastFrames generator.
This is the inference path in ``CausalWanModel.forward_frame``: one latent
frame attends to its own tokens and to the KV cache of earlier frames. The
pre-head hidden state is what the pixel decoder reads. RoPE follows the
complex reference (FastFrames-S sets ``fast_ops=False``) but the multiply is
float32 so it can run on the GPU.
"""
from __future__ import annotations
import math
import mlx.core as mx
import mlx.nn as nn
from .config import WanConfig
from .ops import gelu_tanh, layer_norm, rms_norm, sdpa, silu, sinusoidal_embedding_1d
from .rope import RopeTables
def _zeros(shape):
return mx.zeros(shape)
class _RMS(nn.Module):
def __init__(self, dim, eps):
super().__init__()
self.eps = eps
self.weight = mx.ones((dim,))
def __call__(self, x):
return rms_norm(x, self.weight, self.eps)
class _AffineLN(nn.Module):
def __init__(self, dim, eps):
super().__init__()
self.eps = eps
self.weight = mx.ones((dim,))
self.bias = mx.zeros((dim,))
def __call__(self, x):
return layer_norm(x, self.eps, self.weight, self.bias)
class SelfAttention(nn.Module):
def __init__(self, dim, num_heads, eps):
super().__init__()
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.q = nn.Linear(dim, dim)
self.k = nn.Linear(dim, dim)
self.v = nn.Linear(dim, dim)
self.o = nn.Linear(dim, dim)
self.norm_q = _RMS(dim, eps)
self.norm_k = _RMS(dim, eps)
def qkv(self, x):
b, s, _ = x.shape
q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim)
k = self.norm_k(self.k(x)).reshape(b, s, self.num_heads, self.head_dim)
v = self.v(x).reshape(b, s, self.num_heads, self.head_dim)
return q, k, v
def text_kv(self, context):
b = context.shape[0]
k = self.norm_k(self.k(context)).reshape(b, -1, self.num_heads, self.head_dim)
v = self.v(context).reshape(b, -1, self.num_heads, self.head_dim)
return k, v
class CrossAttention(SelfAttention):
def __call__(self, x, cache):
b, s, _ = x.shape
q = self.norm_q(self.q(x)).reshape(b, s, self.num_heads, self.head_dim)
k, v = cache
out = sdpa(q, k, v).reshape(b, s, -1)
return self.o(out)
class FeedForward(nn.Module):
def __init__(self, dim, ffn_dim):
super().__init__()
self.fc1 = nn.Linear(dim, ffn_dim)
self.fc2 = nn.Linear(ffn_dim, dim)
def __call__(self, x):
return self.fc2(gelu_tanh(self.fc1(x)))
class CausalBlock(nn.Module):
def __init__(self, dim, ffn_dim, num_heads, eps):
super().__init__()
self.eps = eps
self.self_attn = SelfAttention(dim, num_heads, eps)
self.cross_attn = CrossAttention(dim, num_heads, eps)
self.norm3 = _AffineLN(dim, eps)
self.ffn = FeedForward(dim, ffn_dim)
self.modulation = _zeros((1, 6, dim))
def forward_frame(self, x, e, grid, rope: RopeTables, cross_kv, kv_cache, frame: int):
"""``x`` is ``[B, tokens, dim]``. ``e`` is ``[B, 1, 6, dim]``."""
tokens = x.shape[1]
frames, height, width = grid
mod = self.modulation.reshape(1, 1, 6, -1) + e
parts = mx.split(mod, 6, axis=2)
e0, e1, e2, e3, e4, e5 = [p.reshape(p.shape[0], p.shape[1], 1, p.shape[-1]) for p in parts]
h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1)
h = (h * (1.0 + e1) + e0).reshape(x.shape[0], tokens, -1)
q, k, v = self.self_attn.qkv(h)
q = rope.apply(q, frames, height, width, start_frame=frame)
k = rope.apply(k, frames, height, width, start_frame=frame)
kv_cache.write(frame, k, v)
k_pre, v_pre = kv_cache.prefix(frame)
y = self.self_attn.o(sdpa(q, k_pre, v_pre).reshape(x.shape[0], tokens, -1))
x = x + (y.reshape(x.shape[0], 1, tokens, -1) * e2).reshape(x.shape[0], tokens, -1)
x = x + self.cross_attn(self.norm3(x), cross_kv)
y = self.ffn((layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1) * (1.0 + e4) + e3).reshape(x.shape[0], tokens, -1))
y = (y.reshape(x.shape[0], 1, tokens, -1) * e5).reshape(x.shape[0], tokens, -1)
return x + y
class CausalHead(nn.Module):
def __init__(self, dim, out_dim, patch_size, eps):
super().__init__()
self.eps = eps
self.patch_size = patch_size
self.out_dim = out_dim
self.head = nn.Linear(dim, math.prod(patch_size) * out_dim)
self.modulation = _zeros((1, 2, dim))
def __call__(self, x, e):
"""``x`` ``[B, tokens, dim]``, ``e`` ``[B, 1, 1, dim]``."""
tokens = x.shape[1]
mod = self.modulation.reshape(1, 1, 2, -1) + e
shift, scale = mx.split(mod, 2, axis=2)
h = layer_norm(x, self.eps).reshape(x.shape[0], 1, tokens, -1)
h = (h * (1.0 + scale) + shift).reshape(x.shape[0], tokens, -1)
return self.head(h)
def unpatchify(tokens, grid, patch_size, out_dim):
"""``[B, tokens, prod(patch)*C]`` -> ``[B, C, F, H, W]`` in Wan layout."""
b = tokens.shape[0]
f, h, w = grid
pt, ph, pw = patch_size
u = tokens.reshape(b, f, h, w, pt, ph, pw, out_dim)
u = mx.transpose(u, (0, 7, 1, 4, 2, 5, 3, 6))
return u.reshape(b, out_dim, f * pt, h * ph, w * pw)
class PatchEmbed(nn.Module):
def __init__(self, in_dim, dim, patch_size):
super().__init__()
self.patch_size = tuple(patch_size)
pt, ph, pw = self.patch_size
# MLX conv weight is [out, kd, kh, kw, in].
self.weight = _zeros((dim, pt, ph, pw, in_dim))
self.bias = _zeros((dim,))
def __call__(self, latent):
"""``latent`` is ``[B, 1, C, H, W]`` -> ``[B, tokens, dim]``."""
from .ops import conv3d_ncdhw
x = mx.transpose(latent, (0, 2, 1, 3, 4))
y = conv3d_ncdhw(x, self.weight, self.bias, stride=self.patch_size)
y = mx.transpose(y, (0, 2, 3, 4, 1))
return y.reshape(y.shape[0], -1, y.shape[-1])
class KVCache:
"""Per-layer list of per-frame ``(k, v)`` with shape ``[B, tokens, heads, dim]``."""
def __init__(self, num_frames: int):
self.num_frames = num_frames
self.slots = [None] * num_frames
def write(self, frame, k, v):
mx.eval(k, v)
self.slots[frame] = (k, v)
def prefix(self, frame):
ks = [self.slots[i][0] for i in range(frame + 1)]
vs = [self.slots[i][1] for i in range(frame + 1)]
if len(ks) == 1:
return ks[0], vs[0]
return mx.concatenate(ks, axis=1), mx.concatenate(vs, axis=1)
def slot(self, frame):
return self.slots[frame]
class LayerCache:
def __init__(self, num_layers, num_frames):
self.layers = [KVCache(num_frames) for _ in range(num_layers)]
def __getitem__(self, index):
return self.layers[index]
class CausalWanModel(nn.Module):
def __init__(self, config: WanConfig = WanConfig()):
super().__init__()
self.config = config
dim = config.dim
self.dim = dim
self.num_heads = config.num_heads
self.num_layers = config.num_layers
self.patch_size = config.patch_size
self.text_len = config.text_len
self.freq_dim = config.freq_dim
self.out_dim = config.out_dim
self.eps = config.eps
self.patch_embedding = PatchEmbed(config.in_dim, dim, config.patch_size)
self.text_embed_fc1 = nn.Linear(config.text_dim, dim)
self.text_embed_fc2 = nn.Linear(dim, dim)
self.time_embed_fc1 = nn.Linear(config.freq_dim, dim)
self.time_embed_fc2 = nn.Linear(dim, dim)
self.time_proj_fc = nn.Linear(dim, dim * 6)
self.blocks = [
CausalBlock(dim, config.ffn_dim, config.num_heads, config.eps) for _ in range(config.num_layers)
]
self.head = CausalHead(dim, config.out_dim, config.patch_size, config.eps)
# Plain object so the cos/sin tables are not registered as parameters.
self.rope = RopeTables(config.head_dim)
def embed_text(self, prompt_embeds):
if isinstance(prompt_embeds, (list, tuple)):
rows = []
for row in prompt_embeds:
if row.shape[0] < self.text_len:
pad = mx.zeros((self.text_len - row.shape[0], row.shape[-1]), dtype=row.dtype)
row = mx.concatenate([row, pad], axis=0)
rows.append(row[: self.text_len])
prompt_embeds = mx.stack(rows, axis=0)
elif prompt_embeds.shape[1] < self.text_len:
pad = mx.zeros(
(prompt_embeds.shape[0], self.text_len - prompt_embeds.shape[1], prompt_embeds.shape[2]),
dtype=prompt_embeds.dtype,
)
prompt_embeds = mx.concatenate([prompt_embeds, pad], axis=1)
elif prompt_embeds.shape[1] > self.text_len:
prompt_embeds = prompt_embeds[:, : self.text_len]
return self.text_embed_fc2(gelu_tanh(self.text_embed_fc1(prompt_embeds)))
def time_conditioning(self, t, dtype):
"""``t`` is ``[B, F]`` -> ``(e0 [B,F,6,C], e_head [B,F,1,C])``."""
emb = sinusoidal_embedding_1d(self.freq_dim, t.reshape(-1)).astype(dtype)
e = self.time_embed_fc2(silu(self.time_embed_fc1(emb)))
e0 = self.time_proj_fc(silu(e)).reshape(*t.shape, 6, self.dim)
e_head = e.reshape(*t.shape, 1, self.dim)
return e0, e_head
def text_cache(self, prompt_embeds):
context = self.embed_text(prompt_embeds)
caches = []
for block in self.blocks:
k, v = block.cross_attn.text_kv(context)
mx.eval(k, v)
caches.append((k, v))
return caches
def forward_frame(self, latent, e0, e_head, text_cache, kv_cache: LayerCache, frame: int):
"""Denoise ``latent`` ``[B, 1, 16, H, W]``.
Returns flow ``[B, 1, 16, H, W]`` and pre-head states ``[B, tokens, dim]``.
"""
_, _, _, height, width = latent.shape
x = self.patch_embedding(latent)
grid = (1, height // self.patch_size[1], width // self.patch_size[2])
for index, block in enumerate(self.blocks):
x = block.forward_frame(x, e0, grid, self.rope, text_cache[index], kv_cache[index], frame)
mx.eval(x)
hidden = x
tokens = self.head(x, e_head)
flow = unpatchify(tokens, grid, self.patch_size, self.out_dim)
flow = mx.transpose(flow, (0, 2, 1, 3, 4))
return flow, hidden