Spaces:
Running on Zero
Running on Zero
File size: 6,511 Bytes
c94e874 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | #
# For licensing see accompanying LICENSE file.
# Copyright (C) 2026 Apple Inc. All Rights Reserved.
#
"""Pixel decoders (Sec. 3.2 of the paper).
The decoder reads the DiT's pre-head hidden states of the frame's final
denoising step (plus the head's time modulation) and emits RGB directly,
replacing the VAE decoder. ``PixelDecoder`` is the causal ConvNeXt decoder of
FastFrames. It maps ``[B, F*H*W, dim]`` tokens on the ``(F, 30, 52)`` DiT grid to
``[B, 4F, 3, 480, 832]`` pixels in ``[-1, 1]``; callers drop the first three
frames (``align_frames``) to obtain the 81 output frames.
"""
from __future__ import annotations
import math
from typing import Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.checkpoint
from ..wan.model import WanLayerNorm
__all__ = ["PixelDecoder", "BridgedPixelDecoder", "align_frames"]
def unpatchify_tokens(tokens, grid_sizes, patch_size, out_dim):
"""``[B, F, H*W, prod(patch)*C]`` -> ``[B, F*p_t, C, H*p_h, W*p_w]`` (Wan layout)."""
p_t, p_h, p_w = patch_size
out = []
for item, (f, h, w) in zip(tokens, grid_sizes.tolist()):
u = torch.einsum("fhwpqrc->cfphqwr", item.reshape(f, h, w, p_t, p_h, p_w, out_dim))
out.append(u.reshape(out_dim, f * p_t, h * p_h, w * p_w))
return torch.stack(out).permute(0, 2, 1, 3, 4).contiguous()
def align_frames(pixels: torch.Tensor, num_frames: int = 81) -> torch.Tensor:
"""Drop leading frames so that ``pixels`` has ``num_frames`` frames (84 -> 81)."""
if pixels.shape[1] < num_frames:
raise ValueError(f"decoder produced {pixels.shape[1]} < {num_frames} frames")
return pixels[:, pixels.shape[1] - num_frames:]
class _CausalConv3d(nn.Conv3d):
"""Conv3d with left-only temporal padding and symmetric spatial padding."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
pad_t, pad_h, pad_w = self.padding
self._pad = (pad_w, pad_w, pad_h, pad_h, 2 * pad_t, 0)
self.padding = (0, 0, 0)
def forward(self, x):
return super().forward(F.pad(x, self._pad))
class _ChannelLayerNorm3d(nn.Module):
"""LayerNorm over the channels of a ``[B, C, T, H, W]`` tensor."""
def __init__(self, channels: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(channels))
self.bias = nn.Parameter(torch.zeros(channels))
self.eps = float(eps)
def forward(self, x):
x = x.movedim(1, -1)
x = F.layer_norm(x, x.shape[-1:], self.weight, self.bias, self.eps)
return x.movedim(-1, 1).contiguous()
class _CausalResDWBlock3d(nn.Module):
"""Causal depthwise conv -> LN -> pointwise MLP, zero-initialised residual."""
def __init__(self, dim: int, kernel: Tuple[int, int, int] = (3, 3, 3), ffn_mult: int = 4):
super().__init__()
self.dw = _CausalConv3d(dim, dim, kernel_size=kernel, padding=tuple(k // 2 for k in kernel), groups=dim)
self.norm = _ChannelLayerNorm3d(dim)
self.pw1 = nn.Conv3d(dim, dim * ffn_mult, kernel_size=1)
self.act = nn.GELU()
self.pw2 = nn.Conv3d(dim * ffn_mult, dim, kernel_size=1)
nn.init.zeros_(self.pw2.weight)
nn.init.zeros_(self.pw2.bias)
def forward(self, x):
return x + self.pw2(self.act(self.pw1(self.norm(self.dw(x)))))
class PixelDecoder(nn.Module):
"""AdaLN -> channel LN + 1x1 mixer -> causal ConvNeXt blocks -> linear unpatchify."""
def __init__(self, dim: int = 1536, pixel_patch_size: Tuple[int, int, int] = (4, 16, 16),
out_dim: int = 3, n_low_blocks: int = 2, n_high_blocks: int = 1,
ffn_mult: int = 4, eps: float = 1e-6):
super().__init__()
self.dim, self.out_dim = int(dim), int(out_dim)
self.patch_size = tuple(pixel_patch_size)
self.gradient_checkpointing = False
self.norm = WanLayerNorm(dim, eps)
self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
self.entry_norm = _ChannelLayerNorm3d(dim)
self.entry_pw = nn.Conv3d(dim, dim, kernel_size=1)
self.conv_low = nn.Sequential(*[_CausalResDWBlock3d(dim, ffn_mult=ffn_mult) for _ in range(n_low_blocks)])
self.conv_high = nn.Sequential(*[_CausalResDWBlock3d(dim, ffn_mult=ffn_mult) for _ in range(n_high_blocks)])
self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * self.out_dim)
nn.init.zeros_(self.fc_out.weight)
nn.init.zeros_(self.fc_out.bias)
def _run_stack(self, stack, value):
if self.gradient_checkpointing and torch.is_grad_enabled():
for block in stack:
value = torch.utils.checkpoint.checkpoint(block, value, use_reentrant=False)
return value
return stack(value)
def forward(self, x, e_mod, grid_sizes):
"""x: ``[B, F*L, dim]``; e_mod: ``[B, F, 1, dim]``; grid_sizes: ``[B, 3]`` token grid."""
num_frames, frame_tokens = e_mod.shape[1], x.shape[1] // e_mod.shape[1]
shift, scale = (self.modulation.unsqueeze(1) + e_mod).chunk(2, dim=2)
x = self.norm(x).unflatten(1, (num_frames, frame_tokens)) * (1 + scale) + shift
f, h, w = grid_sizes[0].tolist()
value = x.reshape(x.shape[0], f, h, w, self.dim).permute(0, 4, 1, 2, 3).contiguous()
value = self.entry_pw(self.entry_norm(value))
value = self._run_stack(self.conv_low, value)
value = self._run_stack(self.conv_high, value)
b, c, f, h, w = value.shape
tokens = self.fc_out(value.permute(0, 2, 3, 4, 1).contiguous().view(b, f, h * w, c))
grid = torch.tensor([[f, h, w]] * b, dtype=torch.long, device=tokens.device)
return unpatchify_tokens(tokens, grid, self.patch_size, self.out_dim)
class BridgedPixelDecoder(nn.Module):
"""A bias-free projection of hidden states and modulation into a narrower decoder.
FF-L uses it to decode Wan2.1-14B features (5120) with the 1536-wide decoder.
"""
def __init__(self, in_dim: int = 5120, dim: int = 1536, bridge_seed: int = 20260821):
super().__init__()
self.bridge = nn.Linear(in_dim, dim, bias=False)
with torch.random.fork_rng(devices=[]):
torch.manual_seed(int(bridge_seed))
nn.init.normal_(self.bridge.weight, std=in_dim**-0.5)
self.pixel_head = PixelDecoder(dim)
def forward(self, x, e_mod, grid_sizes):
return self.pixel_head(self.bridge(x), self.bridge(e_mod), grid_sizes)
|