Instructions to use pratikgx/FastFrames-S-MLX with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use pratikgx/FastFrames-S-MLX with MLX:
# Download the model from the Hub pip install huggingface_hub[hf_xet] hf download pratikgx/FastFrames-S-MLX --local-dir FastFrames-S-MLX
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Atomic Chat
File size: 4,577 Bytes
e87ef2c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | """Causal ConvNeXt pixel decoder (FastFrames section 3.2).
Reads the DiT's pre-head states and the head's time modulation and emits RGB.
The released FastFrames-S weights are float32. A latent clip of ``F`` frames on
the ``(F, 30, 52)`` grid becomes ``4F`` RGB frames at 480x832; the pipeline
keeps the last 81 (drops the three frames of causal warmup).
"""
from __future__ import annotations
import math
import mlx.core as mx
import mlx.nn as nn
from .dit import unpatchify
from .ops import conv3d_ncdhw, depthwise_conv3d_ncdhw, gelu_erf, layer_norm
class CausalConv3d(nn.Module):
"""Left-only temporal pad, symmetric spatial pad, then conv."""
def __init__(self, in_channels, out_channels, kernel_size, groups=1):
super().__init__()
if isinstance(kernel_size, int):
kernel_size = (kernel_size, kernel_size, kernel_size)
self.kernel = tuple(kernel_size)
self.groups = groups
self.in_channels = in_channels
ic = in_channels // groups
self.weight = mx.zeros((out_channels, *self.kernel, ic))
self.bias = mx.zeros((out_channels,))
def __call__(self, x):
pt, ph, pw = (k // 2 for k in self.kernel)
if pt or ph or pw:
x = mx.pad(x, [(0, 0), (0, 0), (2 * pt, 0), (ph, ph), (pw, pw)])
if self.groups == self.in_channels and self.groups > 1:
return depthwise_conv3d_ncdhw(x, self.weight, self.bias)
return conv3d_ncdhw(x, self.weight, self.bias)
class ChannelLayerNorm(nn.Module):
def __init__(self, channels, eps=1e-6):
super().__init__()
self.eps = eps
self.weight = mx.ones((channels,))
self.bias = mx.zeros((channels,))
def __call__(self, x):
y = mx.transpose(x, (0, 2, 3, 4, 1))
y = layer_norm(y, self.eps, self.weight, self.bias)
return mx.transpose(y, (0, 4, 1, 2, 3))
class CausalResBlock(nn.Module):
def __init__(self, dim, ffn_mult=4):
super().__init__()
self.dw = CausalConv3d(dim, dim, 3, groups=dim)
self.norm = ChannelLayerNorm(dim)
self.pw1 = CausalConv3d(dim, dim * ffn_mult, 1)
self.pw2 = CausalConv3d(dim * ffn_mult, dim, 1)
def __call__(self, x):
h = self.dw(x)
h = self.norm(h)
h = self.pw1(h)
h = gelu_erf(h)
h = self.pw2(h)
return x + h
class PixelDecoder(nn.Module):
def __init__(
self,
dim=1536,
pixel_patch_size=(4, 16, 16),
out_dim=3,
n_low_blocks=2,
n_high_blocks=1,
ffn_mult=4,
eps=1e-6,
):
super().__init__()
self.dim = dim
self.out_dim = out_dim
self.patch_size = tuple(pixel_patch_size)
self.eps = eps
self.modulation = mx.zeros((1, 2, dim))
self.entry_norm = ChannelLayerNorm(dim, eps)
self.entry_pw = CausalConv3d(dim, dim, 1)
self.conv_low = [CausalResBlock(dim, ffn_mult) for _ in range(n_low_blocks)]
self.conv_high = [CausalResBlock(dim, ffn_mult) for _ in range(n_high_blocks)]
self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * out_dim)
def __call__(self, x, e_mod, grid):
"""``x`` ``[B, F*L, C]``, ``e_mod`` ``[B, F, 1, C]``, ``grid`` ``(F, H, W)``."""
num_frames = e_mod.shape[1]
frame_tokens = x.shape[1] // num_frames
mod = self.modulation.reshape(1, 1, 2, -1) + e_mod
shift, scale = mx.split(mod, 2, axis=2)
h = layer_norm(x, self.eps).reshape(x.shape[0], num_frames, frame_tokens, self.dim)
h = h * (1.0 + scale) + shift
f, gh, gw = grid
value = h.reshape(x.shape[0], f, gh, gw, self.dim)
value = mx.transpose(value, (0, 4, 1, 2, 3))
value = self.entry_pw(self.entry_norm(value))
for block in self.conv_low:
value = block(value)
for block in self.conv_high:
value = block(value)
b, _, f, gh, gw = value.shape
tokens = mx.transpose(value, (0, 2, 3, 4, 1)).reshape(b, f, gh * gw, self.dim)
tokens = self.fc_out(tokens).reshape(b, f * gh * gw, -1)
pixels = unpatchify(tokens, (f, gh, gw), self.patch_size, self.out_dim)
return mx.transpose(pixels, (0, 2, 1, 3, 4))
def align_frames(pixels, num_frames: int):
"""Keep the last ``num_frames`` frames (84 -> 81 for the released recipe)."""
if pixels.shape[1] < num_frames:
raise ValueError(f"decoder produced {pixels.shape[1]} frames, need at least {num_frames}")
return pixels[:, pixels.shape[1] - num_frames :]
|