Text-to-Video
MLX
apple-silicon
pratikgx's picture
Add unofficial MLX port of Apple FastFrames-S (code only; official weights loaded at runtime)
e87ef2c verified
Raw History Blame Contribute Delete
7.83 kB
# Adapted from ml-explore/mlx-examples video/wan2.1 (MIT).
# Upstream copyright: Copyright © 2026 Apple Inc.
# Copyright © 2026 Apple Inc.
"""
Building blocks for Wan2.1 VAE.
All layers use channels-last format (NTHWC) as required by MLX.
"""
import mlx.core as mx
import mlx.nn as nn
# Temporal cache depth: 2 frames for causal conv with kernel_size=3 along time.
CACHE_T = 2
def _normalize_tuple(value, n):
if isinstance(value, int):
return (value,) * n
return tuple(value)
def create_cache_entry(x, existing_cache=None):
"""Build temporal cache from the last CACHE_T frames of x, merging with existing cache."""
t = x.shape[1]
if t >= CACHE_T:
return x[:, -CACHE_T:, :, :, :]
else:
cache_x = x[:, -t:, :, :, :]
if existing_cache is not None:
old_frames = existing_cache[:, -(CACHE_T - t) :, :, :, :]
return mx.concatenate([old_frames, cache_x], axis=1)
else:
pad_t = CACHE_T - t
zeros = mx.zeros((x.shape[0], pad_t, *x.shape[2:]), dtype=x.dtype)
return mx.concatenate([zeros, cache_x], axis=1)
class CausalConv3d(nn.Module):
def __init__(
self, in_channels, out_channels, kernel_size, stride=1, padding=0, bias=True
):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.kernel_size = _normalize_tuple(kernel_size, 3)
self.stride = _normalize_tuple(stride, 3)
self.padding = _normalize_tuple(padding, 3)
self._temporal_pad = self.padding[0] * 2
self._spatial_pad_h = self.padding[1]
self._spatial_pad_w = self.padding[2]
scale = (
1.0
/ (
in_channels
* self.kernel_size[0]
* self.kernel_size[1]
* self.kernel_size[2]
)
** 0.5
)
self.weight = mx.random.uniform(
low=-scale,
high=scale,
shape=(out_channels, *self.kernel_size, in_channels),
)
if bias:
self.bias = mx.zeros((out_channels,))
def __call__(self, x, cache_x=None):
# Causal temporal padding (left-only), then symmetric spatial padding, then conv with padding=0.
temporal_pad = self._temporal_pad
if cache_x is not None and self._temporal_pad > 0:
x = mx.concatenate([cache_x, x], axis=1)
temporal_pad = max(0, self._temporal_pad - cache_x.shape[1])
if temporal_pad > 0:
x = mx.pad(x, [(0, 0), (temporal_pad, 0), (0, 0), (0, 0), (0, 0)])
if self._spatial_pad_h > 0 or self._spatial_pad_w > 0:
x = mx.pad(
x,
[
(0, 0),
(0, 0),
(self._spatial_pad_h, self._spatial_pad_h),
(self._spatial_pad_w, self._spatial_pad_w),
(0, 0),
],
)
y = mx.conv3d(x, self.weight, stride=self.stride, padding=0)
if "bias" in self:
y = y + self.bias
return y
class Resample(nn.Module):
def __init__(self, dim, mode):
assert mode in (
"upsample2d",
"upsample3d",
"downsample2d",
"downsample3d",
)
super().__init__()
self.dim = dim
self.mode = mode
if mode == "upsample2d":
self.upsample = nn.Upsample(scale_factor=(2.0, 2.0), mode="nearest")
self.conv = nn.Conv2d(
dim, dim // 2, kernel_size=3, stride=1, padding=0, bias=True
)
elif mode == "upsample3d":
self.upsample = nn.Upsample(scale_factor=(2.0, 2.0), mode="nearest")
self.conv = nn.Conv2d(
dim, dim // 2, kernel_size=3, stride=1, padding=0, bias=True
)
self.time_conv = CausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0))
elif mode in ("downsample2d", "downsample3d"):
self.conv = nn.Conv2d(
dim, dim, kernel_size=3, stride=2, padding=0, bias=True
)
if mode == "downsample3d":
self.time_conv = CausalConv3d(
dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)
)
def __call__(self, x, cache=None):
b, t, h, w, c = x.shape
new_cache = None
if self.mode == "upsample3d":
if cache is None:
new_cache = mx.zeros((b, CACHE_T, h, w, c), dtype=x.dtype)
else:
cache_input = x
x = self.time_conv(x, cache)
new_cache = create_cache_entry(cache_input, cache)
x = x.reshape(b, t, h, w, 2, c)
x = x.transpose(0, 1, 4, 2, 3, 5)
x = x.reshape(b, t * 2, h, w, c)
t_out = x.shape[1]
c_out = x.shape[4]
x = x.reshape(b * t_out, x.shape[2], x.shape[3], c_out)
if self.mode in ("upsample2d", "upsample3d"):
x = self.upsample(x)
x = mx.pad(x, [(0, 0), (1, 1), (1, 1), (0, 0)])
x = self.conv(x)
elif self.mode in ("downsample2d", "downsample3d"):
x = mx.pad(x, [(0, 0), (0, 1), (0, 1), (0, 0)])
x = self.conv(x)
x = x.reshape(b, t_out, x.shape[1], x.shape[2], x.shape[3])
if self.mode == "downsample3d":
if cache is None:
new_cache = x
else:
x_with_cache = mx.concatenate([cache[:, -1:, :, :, :], x], axis=1)
new_cache = x[:, -1:, :, :, :]
x = self.time_conv(x_with_cache, None)
return x, new_cache
class ResidualBlock(nn.Module):
def __init__(self, in_dim, out_dim):
super().__init__()
self.in_dim = in_dim
self.out_dim = out_dim
self.norm1 = nn.RMSNorm(in_dim, eps=1e-12)
self.conv1 = CausalConv3d(in_dim, out_dim, 3, padding=1)
self.norm2 = nn.RMSNorm(out_dim, eps=1e-12)
self.conv2 = CausalConv3d(out_dim, out_dim, 3, padding=1)
if in_dim != out_dim:
self.shortcut = CausalConv3d(in_dim, out_dim, 1)
else:
self.shortcut = None
def __call__(self, x, cache1, cache2):
if self.shortcut is not None:
h = self.shortcut(x)
else:
h = x
residual = self.norm1(x)
residual = nn.silu(residual)
cache_input = residual
residual = self.conv1(residual, cache1)
new_cache1 = create_cache_entry(cache_input, cache1)
residual = self.norm2(residual)
residual = nn.silu(residual)
cache_input = residual
residual = self.conv2(residual, cache2)
new_cache2 = create_cache_entry(cache_input, cache2)
return h + residual, new_cache1, new_cache2
class AttentionBlock(nn.Module):
def __init__(self, dim):
super().__init__()
self.dim = dim
self.norm = nn.RMSNorm(dim, eps=1e-12)
self.to_qkv = nn.Linear(dim, dim * 3)
self.proj = nn.Linear(dim, dim)
def __call__(self, x):
identity = x
b, t, h, w, c = x.shape
x = x.reshape(b * t, h, w, c)
x = self.norm(x)
qkv = self.to_qkv(x)
qkv = qkv.reshape(b * t, h * w, 3, c)
q, k, v = qkv[:, :, 0, :], qkv[:, :, 1, :], qkv[:, :, 2, :]
q = q.reshape(b * t, 1, h * w, c)
k = k.reshape(b * t, 1, h * w, c)
v = v.reshape(b * t, 1, h * w, c)
scale = c**-0.5
attn = mx.fast.scaled_dot_product_attention(q, k, v, scale=scale)
attn = attn.squeeze(1).reshape(b * t, h, w, c)
out = self.proj(attn)
out = out.reshape(b, t, h, w, c)
return out + identity