"""Causal ConvNeXt pixel decoder (FastFrames section 3.2). Reads the DiT's pre-head states and the head's time modulation and emits RGB. The released FastFrames-S weights are float32. A latent clip of ``F`` frames on the ``(F, 30, 52)`` grid becomes ``4F`` RGB frames at 480x832; the pipeline keeps the last 81 (drops the three frames of causal warmup). """ from __future__ import annotations import math import mlx.core as mx import mlx.nn as nn from .dit import unpatchify from .ops import conv3d_ncdhw, depthwise_conv3d_ncdhw, gelu_erf, layer_norm class CausalConv3d(nn.Module): """Left-only temporal pad, symmetric spatial pad, then conv.""" def __init__(self, in_channels, out_channels, kernel_size, groups=1): super().__init__() if isinstance(kernel_size, int): kernel_size = (kernel_size, kernel_size, kernel_size) self.kernel = tuple(kernel_size) self.groups = groups self.in_channels = in_channels ic = in_channels // groups self.weight = mx.zeros((out_channels, *self.kernel, ic)) self.bias = mx.zeros((out_channels,)) def __call__(self, x): pt, ph, pw = (k // 2 for k in self.kernel) if pt or ph or pw: x = mx.pad(x, [(0, 0), (0, 0), (2 * pt, 0), (ph, ph), (pw, pw)]) if self.groups == self.in_channels and self.groups > 1: return depthwise_conv3d_ncdhw(x, self.weight, self.bias) return conv3d_ncdhw(x, self.weight, self.bias) class ChannelLayerNorm(nn.Module): def __init__(self, channels, eps=1e-6): super().__init__() self.eps = eps self.weight = mx.ones((channels,)) self.bias = mx.zeros((channels,)) def __call__(self, x): y = mx.transpose(x, (0, 2, 3, 4, 1)) y = layer_norm(y, self.eps, self.weight, self.bias) return mx.transpose(y, (0, 4, 1, 2, 3)) class CausalResBlock(nn.Module): def __init__(self, dim, ffn_mult=4): super().__init__() self.dw = CausalConv3d(dim, dim, 3, groups=dim) self.norm = ChannelLayerNorm(dim) self.pw1 = CausalConv3d(dim, dim * ffn_mult, 1) self.pw2 = CausalConv3d(dim * ffn_mult, dim, 1) def __call__(self, x): h = self.dw(x) h = self.norm(h) h = self.pw1(h) h = gelu_erf(h) h = self.pw2(h) return x + h class PixelDecoder(nn.Module): def __init__( self, dim=1536, pixel_patch_size=(4, 16, 16), out_dim=3, n_low_blocks=2, n_high_blocks=1, ffn_mult=4, eps=1e-6, ): super().__init__() self.dim = dim self.out_dim = out_dim self.patch_size = tuple(pixel_patch_size) self.eps = eps self.modulation = mx.zeros((1, 2, dim)) self.entry_norm = ChannelLayerNorm(dim, eps) self.entry_pw = CausalConv3d(dim, dim, 1) self.conv_low = [CausalResBlock(dim, ffn_mult) for _ in range(n_low_blocks)] self.conv_high = [CausalResBlock(dim, ffn_mult) for _ in range(n_high_blocks)] self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * out_dim) def __call__(self, x, e_mod, grid): """``x`` ``[B, F*L, C]``, ``e_mod`` ``[B, F, 1, C]``, ``grid`` ``(F, H, W)``.""" num_frames = e_mod.shape[1] frame_tokens = x.shape[1] // num_frames mod = self.modulation.reshape(1, 1, 2, -1) + e_mod shift, scale = mx.split(mod, 2, axis=2) h = layer_norm(x, self.eps).reshape(x.shape[0], num_frames, frame_tokens, self.dim) h = h * (1.0 + scale) + shift f, gh, gw = grid value = h.reshape(x.shape[0], f, gh, gw, self.dim) value = mx.transpose(value, (0, 4, 1, 2, 3)) value = self.entry_pw(self.entry_norm(value)) for block in self.conv_low: value = block(value) for block in self.conv_high: value = block(value) b, _, f, gh, gw = value.shape tokens = mx.transpose(value, (0, 2, 3, 4, 1)).reshape(b, f, gh * gw, self.dim) tokens = self.fc_out(tokens).reshape(b, f * gh * gw, -1) pixels = unpatchify(tokens, (f, gh, gw), self.patch_size, self.out_dim) return mx.transpose(pixels, (0, 2, 1, 3, 4)) def align_frames(pixels, num_frames: int): """Keep the last ``num_frames`` frames (84 -> 81 for the released recipe).""" if pixels.shape[1] < num_frames: raise ValueError(f"decoder produced {pixels.shape[1]} frames, need at least {num_frames}") return pixels[:, pixels.shape[1] - num_frames :]