Text-to-Video
MLX
apple-silicon
File size: 4,577 Bytes
e87ef2c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
"""Causal ConvNeXt pixel decoder (FastFrames section 3.2).

Reads the DiT's pre-head states and the head's time modulation and emits RGB.
The released FastFrames-S weights are float32. A latent clip of ``F`` frames on
the ``(F, 30, 52)`` grid becomes ``4F`` RGB frames at 480x832; the pipeline
keeps the last 81 (drops the three frames of causal warmup).
"""

from __future__ import annotations

import math

import mlx.core as mx
import mlx.nn as nn

from .dit import unpatchify
from .ops import conv3d_ncdhw, depthwise_conv3d_ncdhw, gelu_erf, layer_norm


class CausalConv3d(nn.Module):
    """Left-only temporal pad, symmetric spatial pad, then conv."""

    def __init__(self, in_channels, out_channels, kernel_size, groups=1):
        super().__init__()
        if isinstance(kernel_size, int):
            kernel_size = (kernel_size, kernel_size, kernel_size)
        self.kernel = tuple(kernel_size)
        self.groups = groups
        self.in_channels = in_channels
        ic = in_channels // groups
        self.weight = mx.zeros((out_channels, *self.kernel, ic))
        self.bias = mx.zeros((out_channels,))

    def __call__(self, x):
        pt, ph, pw = (k // 2 for k in self.kernel)
        if pt or ph or pw:
            x = mx.pad(x, [(0, 0), (0, 0), (2 * pt, 0), (ph, ph), (pw, pw)])
        if self.groups == self.in_channels and self.groups > 1:
            return depthwise_conv3d_ncdhw(x, self.weight, self.bias)
        return conv3d_ncdhw(x, self.weight, self.bias)


class ChannelLayerNorm(nn.Module):
    def __init__(self, channels, eps=1e-6):
        super().__init__()
        self.eps = eps
        self.weight = mx.ones((channels,))
        self.bias = mx.zeros((channels,))

    def __call__(self, x):
        y = mx.transpose(x, (0, 2, 3, 4, 1))
        y = layer_norm(y, self.eps, self.weight, self.bias)
        return mx.transpose(y, (0, 4, 1, 2, 3))


class CausalResBlock(nn.Module):
    def __init__(self, dim, ffn_mult=4):
        super().__init__()
        self.dw = CausalConv3d(dim, dim, 3, groups=dim)
        self.norm = ChannelLayerNorm(dim)
        self.pw1 = CausalConv3d(dim, dim * ffn_mult, 1)
        self.pw2 = CausalConv3d(dim * ffn_mult, dim, 1)

    def __call__(self, x):
        h = self.dw(x)
        h = self.norm(h)
        h = self.pw1(h)
        h = gelu_erf(h)
        h = self.pw2(h)
        return x + h


class PixelDecoder(nn.Module):
    def __init__(
        self,
        dim=1536,
        pixel_patch_size=(4, 16, 16),
        out_dim=3,
        n_low_blocks=2,
        n_high_blocks=1,
        ffn_mult=4,
        eps=1e-6,
    ):
        super().__init__()
        self.dim = dim
        self.out_dim = out_dim
        self.patch_size = tuple(pixel_patch_size)
        self.eps = eps
        self.modulation = mx.zeros((1, 2, dim))
        self.entry_norm = ChannelLayerNorm(dim, eps)
        self.entry_pw = CausalConv3d(dim, dim, 1)
        self.conv_low = [CausalResBlock(dim, ffn_mult) for _ in range(n_low_blocks)]
        self.conv_high = [CausalResBlock(dim, ffn_mult) for _ in range(n_high_blocks)]
        self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * out_dim)

    def __call__(self, x, e_mod, grid):
        """``x`` ``[B, F*L, C]``, ``e_mod`` ``[B, F, 1, C]``, ``grid`` ``(F, H, W)``."""
        num_frames = e_mod.shape[1]
        frame_tokens = x.shape[1] // num_frames
        mod = self.modulation.reshape(1, 1, 2, -1) + e_mod
        shift, scale = mx.split(mod, 2, axis=2)
        h = layer_norm(x, self.eps).reshape(x.shape[0], num_frames, frame_tokens, self.dim)
        h = h * (1.0 + scale) + shift
        f, gh, gw = grid
        value = h.reshape(x.shape[0], f, gh, gw, self.dim)
        value = mx.transpose(value, (0, 4, 1, 2, 3))
        value = self.entry_pw(self.entry_norm(value))
        for block in self.conv_low:
            value = block(value)
        for block in self.conv_high:
            value = block(value)
        b, _, f, gh, gw = value.shape
        tokens = mx.transpose(value, (0, 2, 3, 4, 1)).reshape(b, f, gh * gw, self.dim)
        tokens = self.fc_out(tokens).reshape(b, f * gh * gw, -1)
        pixels = unpatchify(tokens, (f, gh, gw), self.patch_size, self.out_dim)
        return mx.transpose(pixels, (0, 2, 1, 3, 4))


def align_frames(pixels, num_frames: int):
    """Keep the last ``num_frames`` frames (84 -> 81 for the released recipe)."""
    if pixels.shape[1] < num_frames:
        raise ValueError(f"decoder produced {pixels.shape[1]} frames, need at least {num_frames}")
    return pixels[:, pixels.shape[1] - num_frames :]