Spaces:
Running on Zero
Running on Zero
Download fastframes/models/pixel_decoder.py from mrfakename/fastframes-l: direct link, hf CLI and curl.
- Browser
- Download file 6.51 kB
-
https://huggingface.co/spaces/mrfakename/fastframes-l/resolve/main/fastframes/models/pixel_decoder.py
- Command line
-
hf download hf://spaces/mrfakename/fastframes-l/fastframes/models/pixel_decoder.py
-
curl -L -o pixel_decoder.py https://huggingface.co/spaces/mrfakename/fastframes-l/resolve/main/fastframes/models/pixel_decoder.py
6.51 kB
| # | |
| # For licensing see accompanying LICENSE file. | |
| # Copyright (C) 2026 Apple Inc. All Rights Reserved. | |
| # | |
| """Pixel decoders (Sec. 3.2 of the paper). | |
| The decoder reads the DiT's pre-head hidden states of the frame's final | |
| denoising step (plus the head's time modulation) and emits RGB directly, | |
| replacing the VAE decoder. ``PixelDecoder`` is the causal ConvNeXt decoder of | |
| FastFrames. It maps ``[B, F*H*W, dim]`` tokens on the ``(F, 30, 52)`` DiT grid to | |
| ``[B, 4F, 3, 480, 832]`` pixels in ``[-1, 1]``; callers drop the first three | |
| frames (``align_frames``) to obtain the 81 output frames. | |
| """ | |
| from __future__ import annotations | |
| import math | |
| from typing import Tuple | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torch.utils.checkpoint | |
| from ..wan.model import WanLayerNorm | |
| __all__ = ["PixelDecoder", "BridgedPixelDecoder", "align_frames"] | |
| def unpatchify_tokens(tokens, grid_sizes, patch_size, out_dim): | |
| """``[B, F, H*W, prod(patch)*C]`` -> ``[B, F*p_t, C, H*p_h, W*p_w]`` (Wan layout).""" | |
| p_t, p_h, p_w = patch_size | |
| out = [] | |
| for item, (f, h, w) in zip(tokens, grid_sizes.tolist()): | |
| u = torch.einsum("fhwpqrc->cfphqwr", item.reshape(f, h, w, p_t, p_h, p_w, out_dim)) | |
| out.append(u.reshape(out_dim, f * p_t, h * p_h, w * p_w)) | |
| return torch.stack(out).permute(0, 2, 1, 3, 4).contiguous() | |
| def align_frames(pixels: torch.Tensor, num_frames: int = 81) -> torch.Tensor: | |
| """Drop leading frames so that ``pixels`` has ``num_frames`` frames (84 -> 81).""" | |
| if pixels.shape[1] < num_frames: | |
| raise ValueError(f"decoder produced {pixels.shape[1]} < {num_frames} frames") | |
| return pixels[:, pixels.shape[1] - num_frames:] | |
| class _CausalConv3d(nn.Conv3d): | |
| """Conv3d with left-only temporal padding and symmetric spatial padding.""" | |
| def __init__(self, *args, **kwargs): | |
| super().__init__(*args, **kwargs) | |
| pad_t, pad_h, pad_w = self.padding | |
| self._pad = (pad_w, pad_w, pad_h, pad_h, 2 * pad_t, 0) | |
| self.padding = (0, 0, 0) | |
| def forward(self, x): | |
| return super().forward(F.pad(x, self._pad)) | |
| class _ChannelLayerNorm3d(nn.Module): | |
| """LayerNorm over the channels of a ``[B, C, T, H, W]`` tensor.""" | |
| def __init__(self, channels: int, eps: float = 1e-6): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(channels)) | |
| self.bias = nn.Parameter(torch.zeros(channels)) | |
| self.eps = float(eps) | |
| def forward(self, x): | |
| x = x.movedim(1, -1) | |
| x = F.layer_norm(x, x.shape[-1:], self.weight, self.bias, self.eps) | |
| return x.movedim(-1, 1).contiguous() | |
| class _CausalResDWBlock3d(nn.Module): | |
| """Causal depthwise conv -> LN -> pointwise MLP, zero-initialised residual.""" | |
| def __init__(self, dim: int, kernel: Tuple[int, int, int] = (3, 3, 3), ffn_mult: int = 4): | |
| super().__init__() | |
| self.dw = _CausalConv3d(dim, dim, kernel_size=kernel, padding=tuple(k // 2 for k in kernel), groups=dim) | |
| self.norm = _ChannelLayerNorm3d(dim) | |
| self.pw1 = nn.Conv3d(dim, dim * ffn_mult, kernel_size=1) | |
| self.act = nn.GELU() | |
| self.pw2 = nn.Conv3d(dim * ffn_mult, dim, kernel_size=1) | |
| nn.init.zeros_(self.pw2.weight) | |
| nn.init.zeros_(self.pw2.bias) | |
| def forward(self, x): | |
| return x + self.pw2(self.act(self.pw1(self.norm(self.dw(x))))) | |
| class PixelDecoder(nn.Module): | |
| """AdaLN -> channel LN + 1x1 mixer -> causal ConvNeXt blocks -> linear unpatchify.""" | |
| def __init__(self, dim: int = 1536, pixel_patch_size: Tuple[int, int, int] = (4, 16, 16), | |
| out_dim: int = 3, n_low_blocks: int = 2, n_high_blocks: int = 1, | |
| ffn_mult: int = 4, eps: float = 1e-6): | |
| super().__init__() | |
| self.dim, self.out_dim = int(dim), int(out_dim) | |
| self.patch_size = tuple(pixel_patch_size) | |
| self.gradient_checkpointing = False | |
| self.norm = WanLayerNorm(dim, eps) | |
| self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) | |
| self.entry_norm = _ChannelLayerNorm3d(dim) | |
| self.entry_pw = nn.Conv3d(dim, dim, kernel_size=1) | |
| self.conv_low = nn.Sequential(*[_CausalResDWBlock3d(dim, ffn_mult=ffn_mult) for _ in range(n_low_blocks)]) | |
| self.conv_high = nn.Sequential(*[_CausalResDWBlock3d(dim, ffn_mult=ffn_mult) for _ in range(n_high_blocks)]) | |
| self.fc_out = nn.Linear(dim, math.prod(self.patch_size) * self.out_dim) | |
| nn.init.zeros_(self.fc_out.weight) | |
| nn.init.zeros_(self.fc_out.bias) | |
| def _run_stack(self, stack, value): | |
| if self.gradient_checkpointing and torch.is_grad_enabled(): | |
| for block in stack: | |
| value = torch.utils.checkpoint.checkpoint(block, value, use_reentrant=False) | |
| return value | |
| return stack(value) | |
| def forward(self, x, e_mod, grid_sizes): | |
| """x: ``[B, F*L, dim]``; e_mod: ``[B, F, 1, dim]``; grid_sizes: ``[B, 3]`` token grid.""" | |
| num_frames, frame_tokens = e_mod.shape[1], x.shape[1] // e_mod.shape[1] | |
| shift, scale = (self.modulation.unsqueeze(1) + e_mod).chunk(2, dim=2) | |
| x = self.norm(x).unflatten(1, (num_frames, frame_tokens)) * (1 + scale) + shift | |
| f, h, w = grid_sizes[0].tolist() | |
| value = x.reshape(x.shape[0], f, h, w, self.dim).permute(0, 4, 1, 2, 3).contiguous() | |
| value = self.entry_pw(self.entry_norm(value)) | |
| value = self._run_stack(self.conv_low, value) | |
| value = self._run_stack(self.conv_high, value) | |
| b, c, f, h, w = value.shape | |
| tokens = self.fc_out(value.permute(0, 2, 3, 4, 1).contiguous().view(b, f, h * w, c)) | |
| grid = torch.tensor([[f, h, w]] * b, dtype=torch.long, device=tokens.device) | |
| return unpatchify_tokens(tokens, grid, self.patch_size, self.out_dim) | |
| class BridgedPixelDecoder(nn.Module): | |
| """A bias-free projection of hidden states and modulation into a narrower decoder. | |
| FF-L uses it to decode Wan2.1-14B features (5120) with the 1536-wide decoder. | |
| """ | |
| def __init__(self, in_dim: int = 5120, dim: int = 1536, bridge_seed: int = 20260821): | |
| super().__init__() | |
| self.bridge = nn.Linear(in_dim, dim, bias=False) | |
| with torch.random.fork_rng(devices=[]): | |
| torch.manual_seed(int(bridge_seed)) | |
| nn.init.normal_(self.bridge.weight, std=in_dim**-0.5) | |
| self.pixel_head = PixelDecoder(dim) | |
| def forward(self, x, e_mod, grid_sizes): | |
| return self.pixel_head(self.bridge(x), self.bridge(e_mod), grid_sizes) | |