Instructions to use pratikgx/FastFrames-S-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/FastFrames-S-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/FastFrames-S-MLX --local-dir FastFrames-S-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
Download fastframes_mlx/pixel_decoder.py from pratikgx/FastFrames-S-MLX: direct link, hf CLI and curl.
- Browser
- Download file 4.58 kB
-
https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/pixel_decoder.py
- Command line
-
hf download hf://pratikgx/FastFrames-S-MLX/fastframes_mlx/pixel_decoder.py
-
curl -L -o pixel_decoder.py https://huggingface.co/pratikgx/FastFrames-S-MLX/resolve/main/fastframes_mlx/pixel_decoder.py
4.58 kB
| """Causal ConvNeXt pixel decoder (FastFrames section 3.2). | |
| Reads the DiT's pre-head states and the head's time modulation and emits RGB. | |
| The released FastFrames-S weights are float32. A latent clip of ``F`` frames on | |
| the ``(F, 30, 52)`` grid becomes ``4F`` RGB frames at 480x832; the pipeline | |
| keeps the last 81 (drops the three frames of causal warmup). | |
| """ | |
| from __future__ import annotations | |
| import math | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| from .dit import unpatchify | |
| from .ops import conv3d_ncdhw, depthwise_conv3d_ncdhw, gelu_erf, layer_norm | |
| class CausalConv3d(nn.Module): | |
| """Left-only temporal pad, symmetric spatial pad, then conv.""" | |
| def __init__(self, in_channels, out_channels, kernel_size, groups=1): | |
| super().__init__() | |
| if isinstance(kernel_size, int): | |
| kernel_size = (kernel_size, kernel_size, kernel_size) | |
| self.kernel = tuple(kernel_size) | |
| self.groups = groups | |
| self.in_channels = in_channels | |
| ic = in_channels // groups | |
| self.weight = mx.zeros((out_channels, *self.kernel, ic)) | |
| self.bias = mx.zeros((out_channels,)) | |
| def __call__(self, x): | |
| pt, ph, pw = (k // 2 for k in self.kernel) | |
| if pt or ph or pw: | |
| x = mx.pad(x, [(0, 0), (0, 0), (2 * pt, 0), (ph, ph), (pw, pw)]) | |
| if self.groups == self.in_channels and self.groups > 1: | |
| return depthwise_conv3d_ncdhw(x, self.weight, self.bias) | |
| return conv3d_ncdhw(x, self.weight, self.bias) | |
| class ChannelLayerNorm(nn.Module): | |
| def __init__(self, channels, eps=1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = mx.ones((channels,)) | |
| self.bias = mx.zeros((channels,)) | |
| def __call__(self, x): | |
| y = mx.transpose(x, (0, 2, 3, 4, 1)) | |
| y = layer_norm(y, self.eps, self.weight, self.bias) | |
| return mx.transpose(y, (0, 4, 1, 2, 3)) | |
| class CausalResBlock(nn.Module): | |
| def __init__(self, dim, ffn_mult=4): | |
| super().__init__() | |
| self.dw = CausalConv3d(dim, dim, 3, groups=dim) | |
| self.norm = ChannelLayerNorm(dim) | |
| self.pw1 = CausalConv3d(dim, dim * ffn_mult, 1) | |
| self.pw2 = CausalConv3d(dim * ffn_mult, dim, 1) | |
| def __call__(self, x): | |
| h = self.dw(x) | |
| h = self.norm(h) | |
| h = self.pw1(h) | |
| h = gelu_erf(h) | |
| h = self.pw2(h) | |
| return x + h | |
| class PixelDecoder(nn.Module): | |
| def __init__( | |
| self, | |
| dim=1536, | |
| pixel_patch_size=(4, 16, 16), | |
| out_dim=3, | |
| n_low_blocks=2, | |
| n_high_blocks=1, | |
| ffn_mult=4, | |
| eps=1e-6, | |
| ): | |
| super().__init__() | |
| self.dim = dim | |
| self.out_dim = out_dim | |
| self.patch_size = tuple(pixel_patch_size) | |
| self.eps = eps | |
| self.modulation = mx.zeros((1, 2, dim)) | |
| self.entry_norm = ChannelLayerNorm(dim, eps) | |
| self.entry_pw = CausalConv3d(dim, dim, 1) | |
| self.conv_low = [CausalResBlock(dim, ffn_mult) for _ in range(n_low_blocks)] | |
| self.conv_high = [CausalResBlock(dim, ffn_mult) for _ in range(n_high_blocks)] | |
| self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * out_dim) | |
| def __call__(self, x, e_mod, grid): | |
| """``x`` ``[B, F*L, C]``, ``e_mod`` ``[B, F, 1, C]``, ``grid`` ``(F, H, W)``.""" | |
| num_frames = e_mod.shape[1] | |
| frame_tokens = x.shape[1] // num_frames | |
| mod = self.modulation.reshape(1, 1, 2, -1) + e_mod | |
| shift, scale = mx.split(mod, 2, axis=2) | |
| h = layer_norm(x, self.eps).reshape(x.shape[0], num_frames, frame_tokens, self.dim) | |
| h = h * (1.0 + scale) + shift | |
| f, gh, gw = grid | |
| value = h.reshape(x.shape[0], f, gh, gw, self.dim) | |
| value = mx.transpose(value, (0, 4, 1, 2, 3)) | |
| value = self.entry_pw(self.entry_norm(value)) | |
| for block in self.conv_low: | |
| value = block(value) | |
| for block in self.conv_high: | |
| value = block(value) | |
| b, _, f, gh, gw = value.shape | |
| tokens = mx.transpose(value, (0, 2, 3, 4, 1)).reshape(b, f, gh * gw, self.dim) | |
| tokens = self.fc_out(tokens).reshape(b, f * gh * gw, -1) | |
| pixels = unpatchify(tokens, (f, gh, gw), self.patch_size, self.out_dim) | |
| return mx.transpose(pixels, (0, 2, 1, 3, 4)) | |
| def align_frames(pixels, num_frames: int): | |
| """Keep the last ``num_frames`` frames (84 -> 81 for the released recipe).""" | |
| if pixels.shape[1] < num_frames: | |
| raise ValueError(f"decoder produced {pixels.shape[1]} frames, need at least {num_frames}") | |
| return pixels[:, pixels.shape[1] - num_frames :] | |