gta / sae.py
abhisheksensharma's picture
Upload folder using huggingface_hub
ae4e95d verified
Raw History Blame Contribute Delete
30.3 kB
"""sae.py -- the autoencoder.
encoder: RGB -> (z0, z_rest)
decoder: (z0, z_rest) -> RGB
Two latents on the same /16 grid, 48 channels each, every token on the RMS-1 sphere:
z0 : [B, 48, 1, H/16, W/16] the first frame, by itself (an image)
z_rest : [B, 48, k, H/16, W/16] the video body, one latent frame per 4 RGB frames
A clip is 1 + 4k RGB frames. Frame 0 never mixes with anything in time, so a single image is a legal clip
and the seed of a stream.
Both sides are built from one schedule format, coarse -> fine:
(scale_vs_RGB, kind, n_res, channels)
kind = "2d" | "3d" | "st"
2d -- per-frame residual blocks; spatial hop attached to this scale
3d -- causal residual blocks; spatial hop attached to this scale
st -- causal residual blocks; 2x in space and 2x in time
The encoder walks it fine -> coarse (RGB in, latent out), the decoder coarse -> fine. The decoder does both
time doublings first, at /16, and then only spatial hops.
Every temporal op is causal, so there are two ways to run each side and they give the same result.
Parallel, the whole clip in one call:
encode_parallel(1 + 4k RGB frames) -> (z0, z_rest with k frames)
decode_parallel(z0, z_rest) -> 1 + 4k RGB frames
Autoregressive, one chunk per call, each call taking the cache the previous one returned:
encode_bootstrap(1 RGB frame) -> (z0, cache)
encode_steady(4 RGB frames, cache) -> (1 z_rest frame, cache)
decode_bootstrap(z0) -> (1 RGB frame, cache)
decode_steady(1 z_rest frame, cache) -> (4 RGB frames, cache)
The cache is a list over the causal blocks, each entry the tail tensors that block needs from the past.
The server uses the autoregressive calls: encode_bootstrap for the start image, decode_* for every frame.
"""
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.nn as nn
import torch.nn.functional as F
# Every temporal conv has kernel 3, so one output frame reads the 2 frames before it.
TAIL_FRAMES = 2
Cache = list # list[list[torch.Tensor | None]]
def _causal_kind(kind: str) -> bool:
return kind in ("3d", "st")
def _st_kind(kind: str) -> bool:
return kind == "st"
def _tail(x: torch.Tensor, frames: int) -> torch.Tensor:
"""The last ``frames`` frames as their own channels-last tensor.
A slice keeps the strides of the tensor it was cut from. These tails cross torch.compile calls, so the
layout has to be the same every time. clone() rather than contiguous(): it also fixes the strides when a
size-1 dimension makes PyTorch call a view contiguous.
"""
return x[:, :, -frames:].clone(memory_format=torch.channels_last_3d)
def _cl3d(x: torch.Tensor) -> torch.Tensor:
"""Conv3d activations as NDHWC."""
return x.contiguous(memory_format=torch.channels_last_3d)
# ---------------------------------------------------------------------------
# config
# ---------------------------------------------------------------------------
@dataclass
class SAEConfig:
patch: int = 16
out_dims_z0: int = 48
out_dims_z_rest: int = 48
rgb_stages_encoder: tuple[tuple[int, str, int, int], ...] = (
(16, "st", 5, 512),
(8, "st", 2, 384),
(4, "3d", 2, 192),
(2, "2d", 2, 96),
)
rgb_stages_decoder: tuple[tuple[int, str, int, int], ...] = (
(16, "st", 6, 512),
(8, "st", 2, 384),
(4, "3d", 2, 192),
(2, "2d", 2, 96),
)
@property
def width(self) -> int:
"""Channel width at the latent grid."""
return int(self.rgb_stages_encoder[0][3])
@property
def t_compress(self) -> int:
"""RGB frames per z_rest frame: 2 per "st" stage."""
return 2 ** sum(1 for s in self.rgb_stages_encoder if _st_kind(s[1]))
def _frame_groups(n_frames: int, group: int) -> list[slice]:
"""The clip layout as slices: frame 0 by itself, then groups of ``group`` frames. The parallel calls run
their per-frame parts on these groups, the same batches the autoregressive calls see."""
return [slice(0, 1)] + [slice(i, i + group) for i in range(1, n_frames, group)]
# ---------------------------------------------------------------------------
# norms
# ---------------------------------------------------------------------------
class RMSNorm3d(nn.Module):
"""Channel-only RMS norm on [B, C, T, H, W]. Nothing is reduced over time or space, so a streamed chunk
is normalized exactly like the whole clip."""
def __init__(self, dim: int, eps: float = 1e-4):
super().__init__()
self.eps = eps
self.gamma = nn.Parameter(torch.ones(1, dim, 1, 1, 1))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x * torch.rsqrt(x.pow(2).mean(1, keepdim=True) + self.eps) * self.gamma
class RMSNorm2d(nn.Module):
"""The same norm on [N, C, H, W]."""
def __init__(self, dim: int, eps: float = 1e-4):
super().__init__()
self.eps = eps
self.gamma = nn.Parameter(torch.ones(1, dim, 1, 1))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x * torch.rsqrt(x.pow(2).mean(1, keepdim=True) + self.eps) * self.gamma
class SphereNorm3d(nn.Module):
"""Put every token's channel vector at RMS 1. No parameters."""
def __init__(self, eps: float = 1e-6):
super().__init__()
self.eps = eps
def forward(self, z: torch.Tensor) -> torch.Tensor:
rms = z.float().pow(2).mean(dim=1, keepdim=True).add(self.eps).sqrt()
return z / rms.to(z.dtype)
# ---------------------------------------------------------------------------
# causal conv
# ---------------------------------------------------------------------------
class CausalConv3d(nn.Conv3d):
"""Conv3d that never reads a future frame: all of the temporal padding goes on the past side.
Streaming: the frames the kernel needs from before this chunk come in as ``tail`` (what the previous call
returned). With no tail the past is zeros, which is the start of a clip. Carrying a short tail of past
frames through a causal conv is the streaming scheme of the Wan2.1 VAE (Apache-2.0).
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
kt = self.kernel_size[0]
pt, ph, pw = self.padding
assert 2 * pt == kt - 1, f"needs 'same' temporal padding, got kernel {kt} pad {pt}"
self.past = kt - 1 # frames of history one output frame reads
self._space_pad = (pw, pw, ph, ph)
self.padding = (0, 0, 0)
def _conv(self, x: torch.Tensor, tail: torch.Tensor | None):
seen = x if tail is None else torch.cat([tail, x], dim=2)
missing = self.past - (0 if tail is None else tail.shape[2])
y = super().forward(_cl3d(F.pad(seen, (*self._space_pad, missing, 0))))
return _cl3d(y), seen
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self._conv(x, None)[0]
def stream(self, x: torch.Tensor, tail: torch.Tensor | None):
"""-> (y, the tail the next call needs)."""
y, seen = self._conv(x, tail)
return y, _tail(seen, TAIL_FRAMES)
# ---------------------------------------------------------------------------
# time and space hops
# ---------------------------------------------------------------------------
def _match_channels(x: torch.Tensor, cout: int) -> torch.Tensor:
"""Channel adapter with no parameters, for the bypass around a hop: tile channels to widen, mean
channel groups to narrow, tile-and-crop / crop when the widths do not divide."""
c = x.shape[1]
if cout == c:
return x
if cout % c == 0:
return x.repeat_interleave(cout // c, dim=1)
if c % cout == 0:
b, _, t, h, w = x.shape
return x.reshape(b, cout, c // cout, t, h, w).mean(2)
if cout > c:
reps = (cout + c - 1) // c
return x.repeat_interleave(reps, dim=1)[:, :cout]
return x[:, :cout]
class TemporalDown(nn.Module):
"""2x in time: every pair of new frames becomes one. The conv output is summed with the plain mean of
the pair. The first frame of a stream skips this (bootstrap): it passes through and becomes the carry."""
def __init__(self, c: int):
super().__init__()
self.conv = nn.Conv3d(c, c, (3, 1, 1), stride=(2, 1, 1)) # no padding: the carry is the past
@staticmethod
def _pair_mean(frames: torch.Tensor) -> torch.Tensor:
return F.avg_pool3d(frames, kernel_size=(2, 1, 1), stride=(2, 1, 1))
def forward(self, x: torch.Tensor) -> torch.Tensor: # parallel: [.., 1 + 2k, ..] -> [.., 1 + k, ..]
if x.shape[2] < 3:
return x[:, :, :1]
return _cl3d(torch.cat([x[:, :, :1], self.conv(_cl3d(x)) + self._pair_mean(x[:, :, 1:])], dim=2))
def bootstrap(self, x: torch.Tensor):
return x, _tail(x, 1)
def steady(self, x: torch.Tensor, carry: torch.Tensor):
inp = _cl3d(torch.cat([carry, x], dim=2)) # x = an even number of new frames
y = _cl3d(self.conv(inp) + self._pair_mean(x))
return y, _tail(inp, 1)
class Downsample3D(nn.Module):
"""2x in space, then 2x in time. Each step is a conv summed with a pooling bypass (block mean in space,
pair mean in time). The two steps are separate so the first frame can skip the temporal one."""
def __init__(self, cin: int, cout: int):
super().__init__()
self.cout = cout
self.spatial = CausalConv3d(cin, cout, kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1))
self.tdown = TemporalDown(cout)
def _spatial(self, x: torch.Tensor) -> torch.Tensor:
s = F.avg_pool3d(x, kernel_size=(1, 2, 2), stride=(1, 2, 2))
return _cl3d(self.spatial(x) + _match_channels(s, self.cout))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.tdown(self._spatial(x))
def bootstrap(self, x: torch.Tensor) -> tuple[torch.Tensor, list]:
y, carry = self.tdown.bootstrap(self._spatial(x))
return y, [carry]
def steady(self, x: torch.Tensor, cache: list) -> tuple[torch.Tensor, list]:
y, carry = self.tdown.steady(self._spatial(x), cache[0])
return y, [carry]
class TemporalUp(nn.Module):
"""2x in time: the conv emits two channel groups per frame, which become two consecutive frames, summed
with the input frame repeated twice. Output frames 2t and 2t+1 read input frames <= t only. The first
frame of a stream skips this (bootstrap), so one latent image stays one frame."""
def __init__(self, c: int):
super().__init__()
self.conv = CausalConv3d(c, c * 2, kernel_size=(3, 1, 1), padding=(1, 0, 0))
@staticmethod
def _interleave(y: torch.Tensor) -> torch.Tensor:
"""[b, 2c, t, h, w] -> [b, c, 2t, h, w]."""
b, c2, t, h, w = y.shape
c = c2 // 2
y = y.reshape(b, 2, c, t, h, w)
y = torch.stack((y[:, 0], y[:, 1]), dim=3)
return _cl3d(y.reshape(b, c, t * 2, h, w))
def forward(self, x: torch.Tensor) -> torch.Tensor: # parallel: [.., 1 + k, ..] -> [.., 1 + 2k, ..]
if x.shape[2] < 2:
return x
body = x[:, :, 1:]
rest = self._interleave(self.conv(body)) + body.repeat_interleave(2, dim=2)
return _cl3d(torch.cat([x[:, :, :1], rest], dim=2))
def bootstrap(self, x: torch.Tensor):
return x, None
def steady(self, x: torch.Tensor, tail):
y, tail = self.conv.stream(x, tail)
return _cl3d(self._interleave(y) + x.repeat_interleave(2, dim=2)), tail
class UpsampleTime(nn.Module):
"""TemporalUp, then a per-frame 3x3 conv summed with its input. No change in space."""
def __init__(self, c: int):
super().__init__()
self.tup = TemporalUp(c)
self.up = CausalConv3d(c, c, kernel_size=(1, 3, 3), padding=(0, 1, 1))
def _up(self, x: torch.Tensor) -> torch.Tensor:
return _cl3d(self.up(x) + x)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self._up(self.tup(x))
def bootstrap(self, x: torch.Tensor) -> tuple[torch.Tensor, list]:
x, tail = self.tup.bootstrap(x)
return self._up(x), [tail]
def steady(self, x: torch.Tensor, cache: list) -> tuple[torch.Tensor, list]:
x, tail = self.tup.steady(x, cache[0])
return self._up(x), [tail]
class As5D(nn.Module):
"""Run a per-frame 2-D module on [B, C, T, H, W] by folding T into the batch."""
def __init__(self, mod: nn.Module):
super().__init__()
self.mod = mod
def forward(self, x: torch.Tensor) -> torch.Tensor:
B, C, T, h, w = x.shape
y = self.mod(x.permute(0, 2, 1, 3, 4).reshape(B * T, C, h, w).contiguous(memory_format=torch.channels_last))
_, Cout, H, W = y.shape
return y.reshape(B, T, Cout, H, W).permute(0, 2, 1, 3, 4).contiguous(memory_format=torch.channels_last_3d)
class PixelShuffleUp(nn.Module):
"""Decoder spatial hop: conv at the low resolution -> PixelShuffle(r) -> SiLU."""
def __init__(self, cin: int, cout: int, r: int):
super().__init__()
self.r = r
self.conv = nn.Conv2d(cin, cout * r * r, 3, padding=1)
self.shuffle = nn.PixelShuffle(r)
self.act = nn.SiLU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.act(self.shuffle(self.conv(x)))
class PixelUnshuffleDown(nn.Module):
"""Encoder spatial hop: PixelUnshuffle(r) -> conv -> SiLU."""
def __init__(self, cin: int, cout: int, r: int):
super().__init__()
self.r = r
self.unshuffle = nn.PixelUnshuffle(r)
self.conv = nn.Conv2d(cin * r * r, cout, 3, padding=1)
self.act = nn.SiLU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.act(self.conv(self.unshuffle(x)))
class SubPixelToRGB(nn.Module):
"""Last decoder hop: conv -> PixelShuffle(r) -> RGB at full resolution. No activation."""
def __init__(self, cin: int, r: int):
super().__init__()
self.r = r
self.conv = nn.Conv2d(cin, 3 * r * r, kernel_size=3, padding=1)
self.shuffle = nn.PixelShuffle(r)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.shuffle(self.conv(x))
class SubPixelFromRGB(nn.Module):
"""First encoder hop: RGB at full resolution -> PixelUnshuffle(r) -> conv. No activation."""
def __init__(self, cout: int, r: int):
super().__init__()
self.r = r
self.unshuffle = nn.PixelUnshuffle(r)
self.conv = nn.Conv2d(3 * r * r, cout, kernel_size=3, padding=1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.conv(self.unshuffle(x))
# ---------------------------------------------------------------------------
# residual blocks
# ---------------------------------------------------------------------------
class ResBlock(nn.Module):
"""Pre-norm causal 3x3x3 residual block. Cache = the tails of its two convs."""
def __init__(self, c: int):
super().__init__()
self.norm1 = RMSNorm3d(c)
self.conv1 = CausalConv3d(c, c, kernel_size=3, padding=1)
self.norm2 = RMSNorm3d(c)
self.conv2 = CausalConv3d(c, c, kernel_size=3, padding=1)
self.act = nn.GELU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
h = self.conv1(self.act(self.norm1(x)))
h = self.conv2(self.act(self.norm2(h)))
return _cl3d(x + h)
def steady(self, x: torch.Tensor, cache: list) -> tuple[torch.Tensor, list]:
h, tail1 = self.conv1.stream(self.act(self.norm1(x)), cache[0])
h, tail2 = self.conv2.stream(self.act(self.norm2(h)), cache[1])
return _cl3d(x + h), [tail1, tail2]
def bootstrap(self, x: torch.Tensor) -> tuple[torch.Tensor, list]:
return self.steady(x, [None, None])
class ResBlock2D(nn.Module):
"""Pre-norm per-frame 3x3 residual block. No state."""
def __init__(self, c: int):
super().__init__()
self.norm1 = RMSNorm2d(c)
self.conv1 = nn.Conv2d(c, c, kernel_size=3, padding=1)
self.norm2 = RMSNorm2d(c)
self.conv2 = nn.Conv2d(c, c, kernel_size=3, padding=1)
self.act = nn.GELU()
def forward(self, x: torch.Tensor) -> torch.Tensor:
h = self.conv1(self.act(self.norm1(x)))
h = self.conv2(self.act(self.norm2(h)))
return x + h
# ---------------------------------------------------------------------------
# the pixel path between RGB and the latent grid
# ---------------------------------------------------------------------------
class RgbStages(nn.Module):
"""One schedule, built as the encoder (RGB -> latent grid) or as the decoder (latent grid -> RGB).
blocks_3d: the causal part (residual blocks and the hops that touch time); it carries the stream cache.
blocks_2d: the per-frame part; no state.
Encoder order: blocks_2d then blocks_3d. Decoder order: blocks_3d then blocks_2d.
"""
def __init__(self, stages: tuple[tuple[int, str, int, int], ...], *, encode: bool):
super().__init__()
self.encode = encode
b3d: list[nn.Module] = []
b2d: list[nn.Module] = []
if encode:
# Open from RGB, walk fine -> coarse.
b2d.append(SubPixelFromRGB(stages[-1][3], stages[-1][0]))
for i in range(len(stages) - 1, -1, -1):
scale, kind, n_res, c = stages[i]
target = b3d if _causal_kind(kind) else b2d
for _ in range(n_res):
target.append(ResBlock(c) if _causal_kind(kind) else ResBlock2D(c))
if i == 0:
continue
nscale, nkind, _, nc = stages[i - 1]
if _st_kind(nkind):
b3d.append(Downsample3D(c, nc))
else:
down = PixelUnshuffleDown(c, nc, nscale // scale)
if _causal_kind(nkind) and _causal_kind(kind):
b3d.append(As5D(down))
else:
b2d.append(down)
else:
# Residual blocks at the latent grid, then every time doubling, then spatial hops only.
scale0, _kind0, n_res0, c0 = stages[0]
for _ in range(n_res0):
b3d.append(ResBlock(c0))
for _ in range(sum(1 for s in stages if _st_kind(s[1]))):
b3d.append(UpsampleTime(c0))
prev_scale, prev_ch = scale0, c0
for scale, kind, n_res, c in stages[1:]:
up = PixelShuffleUp(prev_ch, c, prev_scale // scale)
if _causal_kind(kind):
b3d.append(As5D(up))
else:
b2d.append(up)
target = b3d if _causal_kind(kind) else b2d
for _ in range(n_res):
target.append(ResBlock(c) if _causal_kind(kind) else ResBlock2D(c))
prev_scale, prev_ch = scale, c
b2d.append(SubPixelToRGB(prev_ch, prev_scale))
self.blocks_3d = nn.ModuleList(b3d)
self.blocks_2d = nn.ModuleList(b2d)
def forward_3d(self, x: torch.Tensor, group: int) -> torch.Tensor:
"""Parallel: the causal part on a whole clip. Past frames come from the clip itself, zeros before it.
A per-frame hop (As5D) runs on the clip's frame groups."""
for blk in self.blocks_3d:
if isinstance(blk, As5D):
parts = [blk(x[:, :, g]) for g in _frame_groups(x.shape[2], group)]
x = parts[0] if len(parts) == 1 else _cl3d(torch.cat(parts, dim=2))
else:
x = blk(x)
return x
def stream_3d(self, x: torch.Tensor, cache: Cache | None, bootstrap: bool) -> tuple[torch.Tensor, Cache]:
"""Autoregressive: the causal part on one chunk. ``bootstrap``: the first frame (no cache in)."""
new_cache: Cache = []
ci = 0
for blk in self.blocks_3d:
if isinstance(blk, (ResBlock, Downsample3D, UpsampleTime)):
if bootstrap:
x, c = blk.bootstrap(x)
else:
assert cache is not None
x, c = blk.steady(x, cache[ci])
ci += 1
new_cache.append(c)
else:
x = blk(x)
return x, new_cache
def encode_2d(self, video: torch.Tensor) -> torch.Tensor:
"""RGB [B, T, 3, H, W] -> hidden [B, C, T, h, w], frame by frame."""
B, T, C, H, W = video.shape
x = video.reshape(B * T, C, H, W).contiguous(memory_format=torch.channels_last)
for blk in self.blocks_2d:
x = blk(x)
_, D, h, w = x.shape
return x.reshape(B, T, D, h, w).permute(0, 2, 1, 3, 4).contiguous(memory_format=torch.channels_last_3d)
def decode_2d(self, hidden: torch.Tensor) -> torch.Tensor:
"""hidden [B, C, T, h, w] -> RGB [B, T, 3, H, W], frame by frame."""
B, C, T, h, w = hidden.shape
x = hidden.permute(0, 2, 1, 3, 4).reshape(B * T, C, h, w).contiguous(memory_format=torch.channels_last)
for blk in self.blocks_2d:
x = blk(x)
return x.reshape(B, T, 3, x.shape[-2], x.shape[-1])
# ---------------------------------------------------------------------------
# encoder
# ---------------------------------------------------------------------------
class BottleneckHead(nn.Module):
"""1x1x1 linear read-out to the latent channels, then onto the sphere."""
def __init__(self, cin: int, out_dims: int):
super().__init__()
self.out_dims = out_dims
self.proj = nn.Conv3d(cin, out_dims, kernel_size=1)
self.sphere_norm = SphereNorm3d()
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.sphere_norm(_cl3d(self.proj(_cl3d(x))))
class ShakeAE(nn.Module):
"""The encoder. RGB [B, T, 3, H, W] (H, W multiples of 16) -> z0 or z_rest.
One trunk, two read-outs: ``head_z0`` for the first frame, ``head_rest`` for the body.
"""
def __init__(self, cfg: SAEConfig | None = None):
super().__init__()
self.cfg = cfg or SAEConfig()
self.rgb = RgbStages(self.cfg.rgb_stages_encoder, encode=True)
self.head_z0 = BottleneckHead(self.cfg.width, self.cfg.out_dims_z0)
self.head_rest = BottleneckHead(self.cfg.width, self.cfg.out_dims_z_rest)
@property
def t_compress(self) -> int:
"""RGB frames per z_rest frame."""
return self.cfg.t_compress
# ---- parallel: the whole clip in one call ---------------------------------------------------
def encode_parallel(self, video: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""RGB [B, 1 + 4k, 3, H, W] -> (z0 [B, C0, 1, h, w], z_rest [B, Cr, k, h, w]). k = 0 is one image.
Same weights and the same result as encode_bootstrap followed by k encode_steady calls.
"""
t = self.t_compress
assert (video.shape[1] - 1) % t == 0, f"T={video.shape[1]} must be 1 + {t}k"
x = _cl3d(torch.cat([self.rgb.encode_2d(video[:, g]) for g in _frame_groups(video.shape[1], t)], dim=2))
x = self.rgb.forward_3d(x, t)
z0 = self.head_z0(x[:, :, :1])
if x.shape[2] > 1:
return z0, self.head_rest(x[:, :, 1:])
return z0, video.new_zeros(video.shape[0], self.cfg.out_dims_z_rest, 0, *z0.shape[-2:])
# ---- autoregressive: one chunk per call -----------------------------------------------------
def encode_bootstrap(self, video_chunk: torch.Tensor) -> tuple[torch.Tensor, Cache]:
"""The first frame: 1 RGB frame -> z0 [B, C0, 1, h, w]."""
assert video_chunk.shape[1] == 1, f"bootstrap expects T=1, got T={video_chunk.shape[1]}"
x, cache = self.rgb.stream_3d(self.rgb.encode_2d(video_chunk), None, bootstrap=True)
return self.head_z0(x), cache
def encode_steady(self, video_chunk: torch.Tensor, cache: Cache) -> tuple[torch.Tensor, Cache]:
"""A body chunk: 4 RGB frames -> 1 z_rest frame [B, Cr, 1, h, w]."""
assert video_chunk.shape[1] == self.t_compress, (
f"steady expects T={self.t_compress}, got T={video_chunk.shape[1]}")
x, cache = self.rgb.stream_3d(self.rgb.encode_2d(video_chunk), cache, bootstrap=False)
return self.head_rest(x), cache
# ---------------------------------------------------------------------------
# decoder
# ---------------------------------------------------------------------------
class Decoder(nn.Module):
"""The decoder. z0 and z_rest enter the same trunk through their own 1x1x1 projection."""
def __init__(self, cfg: SAEConfig | None = None):
super().__init__()
self.cfg = cfg or SAEConfig()
self.proj_in_z0 = nn.Conv3d(self.cfg.out_dims_z0, self.cfg.width, kernel_size=1)
self.proj_in_rest = nn.Conv3d(self.cfg.out_dims_z_rest, self.cfg.width, kernel_size=1)
self.rgb = RgbStages(self.cfg.rgb_stages_decoder, encode=False)
@property
def t_compress(self) -> int:
"""RGB frames per z_rest frame."""
return self.cfg.t_compress
# ---- parallel: the whole clip in one call ---------------------------------------------------
def decode_parallel(self, z0: torch.Tensor, z_rest: torch.Tensor) -> torch.Tensor:
"""(z0 [B, C0, 1, h, w], z_rest [B, Cr, k, h, w]) -> RGB [B, 1 + 4k, 3, H, W]. k = 0 is one image.
Same weights and the same result as decode_bootstrap followed by k decode_steady calls.
"""
assert z0.shape[2] == 1 and z0.shape[1] == self.cfg.out_dims_z0, tuple(z0.shape)
x = _cl3d(self.proj_in_z0(z0))
if z_rest.shape[2] > 0:
assert z_rest.shape[1] == self.cfg.out_dims_z_rest, tuple(z_rest.shape)
x = _cl3d(torch.cat([x, _cl3d(self.proj_in_rest(z_rest))], dim=2))
t = self.t_compress
x = self.rgb.forward_3d(x, t)
return torch.cat([self.rgb.decode_2d(x[:, :, g]) for g in _frame_groups(x.shape[2], t)], dim=1)
# ---- autoregressive: one chunk per call -----------------------------------------------------
def decode_bootstrap(self, z_chunk: torch.Tensor) -> tuple[torch.Tensor, Cache]:
"""z0 [B, C0, 1, h, w] -> 1 RGB frame [B, 1, 3, H, W]."""
assert z_chunk.shape[2] == 1, f"bootstrap expects 1 latent frame, got T={z_chunk.shape[2]}"
assert z_chunk.shape[1] == self.cfg.out_dims_z0, (
f"z0 channels={z_chunk.shape[1]} != out_dims_z0={self.cfg.out_dims_z0}")
h, cache = self.rgb.stream_3d(_cl3d(self.proj_in_z0(z_chunk)), None, bootstrap=True)
return self.rgb.decode_2d(h), cache
def decode_steady(self, z_chunk: torch.Tensor, cache: Cache) -> tuple[torch.Tensor, Cache]:
"""One z_rest frame [B, Cr, 1, h, w] -> 4 RGB frames [B, 4, 3, H, W]."""
assert z_chunk.shape[2] == 1, f"steady expects 1 latent frame, got T={z_chunk.shape[2]}"
assert z_chunk.shape[1] == self.cfg.out_dims_z_rest, (
f"z_rest channels={z_chunk.shape[1]} != out_dims_z_rest={self.cfg.out_dims_z_rest}")
h, cache = self.rgb.stream_3d(_cl3d(self.proj_in_rest(z_chunk)), cache, bootstrap=False)
return self.rgb.decode_2d(h), cache
# ---------------------------------------------------------------------------
# memory format
# ---------------------------------------------------------------------------
def _apply_channels_last(module: nn.Module) -> tuple[int, int]:
"""Conv3d weights -> channels_last_3d, Conv2d weights -> channels_last. Returns the two counts."""
n3 = n2 = 0
for m in module.modules():
if isinstance(m, nn.Conv3d):
m.weight.data = m.weight.data.contiguous(memory_format=torch.channels_last_3d)
n3 += 1
elif isinstance(m, nn.Conv2d):
m.weight.data = m.weight.data.contiguous(memory_format=torch.channels_last)
n2 += 1
return n3, n2
def _assert_channels_last(module: nn.Module, *, tag: str) -> None:
bad: list[str] = []
for name, m in module.named_modules():
if isinstance(m, nn.Conv3d) and not m.weight.is_contiguous(memory_format=torch.channels_last_3d):
bad.append(f"{name} (Conv3d not channels_last_3d)")
elif isinstance(m, nn.Conv2d) and not m.weight.is_contiguous(memory_format=torch.channels_last):
bad.append(f"{name} (Conv2d not channels_last)")
assert not bad, f"[{tag}] conv weights not channels-last: {bad[:8]}"
# ---------------------------------------------------------------------------
# The space the model works in. The encoder's latent sits on the sphere; the model sees it standardized
# per channel. ``stats`` is a dict with 'mean' [C] and 'std' [C], or a (mean, std) pair. Both directions
# share _diag_stats_like, so they are exact inverses.
# ---------------------------------------------------------------------------
def _diag_stats_like(stats, z: torch.Tensor, *, eps: float = 1e-6):
"""Per-channel (mean, std) shaped to broadcast against ``z`` along dim 1, on z's device and dtype."""
if isinstance(stats, dict):
mean, std = stats["mean"], stats["std"]
else:
mean, std = stats
mean = torch.as_tensor(mean, device=z.device, dtype=z.dtype)
std = torch.as_tensor(std, device=z.device, dtype=z.dtype).clamp_min(eps)
assert mean.numel() == z.shape[1] == std.numel(), (
f"stats channels (mean {mean.numel()}, std {std.numel()}) != latent channels {z.shape[1]}")
view = [1, z.shape[1]] + [1] * (z.ndim - 2)
return mean.reshape(view), std.reshape(view)
def training_latent_to_diag_gaussian_latent(z: torch.Tensor, stats, *, eps: float = 1e-6) -> torch.Tensor:
"""Encoder latent -> the model's space: (z - mean) / std per channel. ``z``: [B, C, ...]."""
mean, std = _diag_stats_like(stats, z, eps=eps)
return (z - mean) / std
def diag_gaussian_latent_to_training_latent(z: torch.Tensor, stats, *, eps: float = 1e-6) -> torch.Tensor:
"""The model's space -> encoder latent: z * std + mean per channel. Apply before decoding."""
mean, std = _diag_stats_like(stats, z, eps=eps)
return z * std + mean