File size: 6,511 Bytes
c94e874
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
#
# For licensing see accompanying LICENSE file.
# Copyright (C) 2026 Apple Inc. All Rights Reserved.
#
"""Pixel decoders (Sec. 3.2 of the paper).

The decoder reads the DiT's pre-head hidden states of the frame's final
denoising step (plus the head's time modulation) and emits RGB directly,
replacing the VAE decoder. ``PixelDecoder`` is the causal ConvNeXt decoder of
FastFrames. It maps ``[B, F*H*W, dim]`` tokens on the ``(F, 30, 52)`` DiT grid to
``[B, 4F, 3, 480, 832]`` pixels in ``[-1, 1]``; callers drop the first three
frames (``align_frames``) to obtain the 81 output frames.
"""
from __future__ import annotations

import math
from typing import Tuple

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.checkpoint

from ..wan.model import WanLayerNorm

__all__ = ["PixelDecoder", "BridgedPixelDecoder", "align_frames"]


def unpatchify_tokens(tokens, grid_sizes, patch_size, out_dim):
    """``[B, F, H*W, prod(patch)*C]`` -> ``[B, F*p_t, C, H*p_h, W*p_w]`` (Wan layout)."""
    p_t, p_h, p_w = patch_size
    out = []
    for item, (f, h, w) in zip(tokens, grid_sizes.tolist()):
        u = torch.einsum("fhwpqrc->cfphqwr", item.reshape(f, h, w, p_t, p_h, p_w, out_dim))
        out.append(u.reshape(out_dim, f * p_t, h * p_h, w * p_w))
    return torch.stack(out).permute(0, 2, 1, 3, 4).contiguous()


def align_frames(pixels: torch.Tensor, num_frames: int = 81) -> torch.Tensor:
    """Drop leading frames so that ``pixels`` has ``num_frames`` frames (84 -> 81)."""
    if pixels.shape[1] < num_frames:
        raise ValueError(f"decoder produced {pixels.shape[1]} < {num_frames} frames")
    return pixels[:, pixels.shape[1] - num_frames:]


class _CausalConv3d(nn.Conv3d):
    """Conv3d with left-only temporal padding and symmetric spatial padding."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        pad_t, pad_h, pad_w = self.padding
        self._pad = (pad_w, pad_w, pad_h, pad_h, 2 * pad_t, 0)
        self.padding = (0, 0, 0)

    def forward(self, x):
        return super().forward(F.pad(x, self._pad))


class _ChannelLayerNorm3d(nn.Module):
    """LayerNorm over the channels of a ``[B, C, T, H, W]`` tensor."""

    def __init__(self, channels: int, eps: float = 1e-6):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(channels))
        self.bias = nn.Parameter(torch.zeros(channels))
        self.eps = float(eps)

    def forward(self, x):
        x = x.movedim(1, -1)
        x = F.layer_norm(x, x.shape[-1:], self.weight, self.bias, self.eps)
        return x.movedim(-1, 1).contiguous()


class _CausalResDWBlock3d(nn.Module):
    """Causal depthwise conv -> LN -> pointwise MLP, zero-initialised residual."""

    def __init__(self, dim: int, kernel: Tuple[int, int, int] = (3, 3, 3), ffn_mult: int = 4):
        super().__init__()
        self.dw = _CausalConv3d(dim, dim, kernel_size=kernel, padding=tuple(k // 2 for k in kernel), groups=dim)
        self.norm = _ChannelLayerNorm3d(dim)
        self.pw1 = nn.Conv3d(dim, dim * ffn_mult, kernel_size=1)
        self.act = nn.GELU()
        self.pw2 = nn.Conv3d(dim * ffn_mult, dim, kernel_size=1)
        nn.init.zeros_(self.pw2.weight)
        nn.init.zeros_(self.pw2.bias)

    def forward(self, x):
        return x + self.pw2(self.act(self.pw1(self.norm(self.dw(x)))))


class PixelDecoder(nn.Module):
    """AdaLN -> channel LN + 1x1 mixer -> causal ConvNeXt blocks -> linear unpatchify."""

    def __init__(self, dim: int = 1536, pixel_patch_size: Tuple[int, int, int] = (4, 16, 16),
                 out_dim: int = 3, n_low_blocks: int = 2, n_high_blocks: int = 1,
                 ffn_mult: int = 4, eps: float = 1e-6):
        super().__init__()
        self.dim, self.out_dim = int(dim), int(out_dim)
        self.patch_size = tuple(pixel_patch_size)
        self.gradient_checkpointing = False
        self.norm = WanLayerNorm(dim, eps)
        self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
        self.entry_norm = _ChannelLayerNorm3d(dim)
        self.entry_pw = nn.Conv3d(dim, dim, kernel_size=1)
        self.conv_low = nn.Sequential(*[_CausalResDWBlock3d(dim, ffn_mult=ffn_mult) for _ in range(n_low_blocks)])
        self.conv_high = nn.Sequential(*[_CausalResDWBlock3d(dim, ffn_mult=ffn_mult) for _ in range(n_high_blocks)])
        self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * self.out_dim)
        nn.init.zeros_(self.fc_out.weight)
        nn.init.zeros_(self.fc_out.bias)

    def _run_stack(self, stack, value):
        if self.gradient_checkpointing and torch.is_grad_enabled():
            for block in stack:
                value = torch.utils.checkpoint.checkpoint(block, value, use_reentrant=False)
            return value
        return stack(value)

    def forward(self, x, e_mod, grid_sizes):
        """x: ``[B, F*L, dim]``; e_mod: ``[B, F, 1, dim]``; grid_sizes: ``[B, 3]`` token grid."""
        num_frames, frame_tokens = e_mod.shape[1], x.shape[1] // e_mod.shape[1]
        shift, scale = (self.modulation.unsqueeze(1) + e_mod).chunk(2, dim=2)
        x = self.norm(x).unflatten(1, (num_frames, frame_tokens)) * (1 + scale) + shift
        f, h, w = grid_sizes[0].tolist()
        value = x.reshape(x.shape[0], f, h, w, self.dim).permute(0, 4, 1, 2, 3).contiguous()
        value = self.entry_pw(self.entry_norm(value))
        value = self._run_stack(self.conv_low, value)
        value = self._run_stack(self.conv_high, value)
        b, c, f, h, w = value.shape
        tokens = self.fc_out(value.permute(0, 2, 3, 4, 1).contiguous().view(b, f, h * w, c))
        grid = torch.tensor([[f, h, w]] * b, dtype=torch.long, device=tokens.device)
        return unpatchify_tokens(tokens, grid, self.patch_size, self.out_dim)


class BridgedPixelDecoder(nn.Module):
    """A bias-free projection of hidden states and modulation into a narrower decoder.

    FF-L uses it to decode Wan2.1-14B features (5120) with the 1536-wide decoder.
    """

    def __init__(self, in_dim: int = 5120, dim: int = 1536, bridge_seed: int = 20260821):
        super().__init__()
        self.bridge = nn.Linear(in_dim, dim, bias=False)
        with torch.random.fork_rng(devices=[]):
            torch.manual_seed(int(bridge_seed))
            nn.init.normal_(self.bridge.weight, std=in_dim**-0.5)
        self.pixel_head = PixelDecoder(dim)

    def forward(self, x, e_mod, grid_sizes):
        return self.pixel_head(self.bridge(x), self.bridge(e_mod), grid_sizes)