Download sae.py from lucidml/gta: direct link, hf CLI and curl.
- Browser
- Download file 30.3 kB
-
https://huggingface.co/lucidml/gta/resolve/main/sae.py
- Command line
-
hf download hf://lucidml/gta/sae.py
-
curl -L -o sae.py https://huggingface.co/lucidml/gta/resolve/main/sae.py
30.3 kB
| """sae.py -- the autoencoder. | |
| encoder: RGB -> (z0, z_rest) | |
| decoder: (z0, z_rest) -> RGB | |
| Two latents on the same /16 grid, 48 channels each, every token on the RMS-1 sphere: | |
| z0 : [B, 48, 1, H/16, W/16] the first frame, by itself (an image) | |
| z_rest : [B, 48, k, H/16, W/16] the video body, one latent frame per 4 RGB frames | |
| A clip is 1 + 4k RGB frames. Frame 0 never mixes with anything in time, so a single image is a legal clip | |
| and the seed of a stream. | |
| Both sides are built from one schedule format, coarse -> fine: | |
| (scale_vs_RGB, kind, n_res, channels) | |
| kind = "2d" | "3d" | "st" | |
| 2d -- per-frame residual blocks; spatial hop attached to this scale | |
| 3d -- causal residual blocks; spatial hop attached to this scale | |
| st -- causal residual blocks; 2x in space and 2x in time | |
| The encoder walks it fine -> coarse (RGB in, latent out), the decoder coarse -> fine. The decoder does both | |
| time doublings first, at /16, and then only spatial hops. | |
| Every temporal op is causal, so there are two ways to run each side and they give the same result. | |
| Parallel, the whole clip in one call: | |
| encode_parallel(1 + 4k RGB frames) -> (z0, z_rest with k frames) | |
| decode_parallel(z0, z_rest) -> 1 + 4k RGB frames | |
| Autoregressive, one chunk per call, each call taking the cache the previous one returned: | |
| encode_bootstrap(1 RGB frame) -> (z0, cache) | |
| encode_steady(4 RGB frames, cache) -> (1 z_rest frame, cache) | |
| decode_bootstrap(z0) -> (1 RGB frame, cache) | |
| decode_steady(1 z_rest frame, cache) -> (4 RGB frames, cache) | |
| The cache is a list over the causal blocks, each entry the tail tensors that block needs from the past. | |
| The server uses the autoregressive calls: encode_bootstrap for the start image, decode_* for every frame. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| # Every temporal conv has kernel 3, so one output frame reads the 2 frames before it. | |
| TAIL_FRAMES = 2 | |
| Cache = list # list[list[torch.Tensor | None]] | |
| def _causal_kind(kind: str) -> bool: | |
| return kind in ("3d", "st") | |
| def _st_kind(kind: str) -> bool: | |
| return kind == "st" | |
| def _tail(x: torch.Tensor, frames: int) -> torch.Tensor: | |
| """The last ``frames`` frames as their own channels-last tensor. | |
| A slice keeps the strides of the tensor it was cut from. These tails cross torch.compile calls, so the | |
| layout has to be the same every time. clone() rather than contiguous(): it also fixes the strides when a | |
| size-1 dimension makes PyTorch call a view contiguous. | |
| """ | |
| return x[:, :, -frames:].clone(memory_format=torch.channels_last_3d) | |
| def _cl3d(x: torch.Tensor) -> torch.Tensor: | |
| """Conv3d activations as NDHWC.""" | |
| return x.contiguous(memory_format=torch.channels_last_3d) | |
| # --------------------------------------------------------------------------- | |
| # config | |
| # --------------------------------------------------------------------------- | |
| class SAEConfig: | |
| patch: int = 16 | |
| out_dims_z0: int = 48 | |
| out_dims_z_rest: int = 48 | |
| rgb_stages_encoder: tuple[tuple[int, str, int, int], ...] = ( | |
| (16, "st", 5, 512), | |
| (8, "st", 2, 384), | |
| (4, "3d", 2, 192), | |
| (2, "2d", 2, 96), | |
| ) | |
| rgb_stages_decoder: tuple[tuple[int, str, int, int], ...] = ( | |
| (16, "st", 6, 512), | |
| (8, "st", 2, 384), | |
| (4, "3d", 2, 192), | |
| (2, "2d", 2, 96), | |
| ) | |
| def width(self) -> int: | |
| """Channel width at the latent grid.""" | |
| return int(self.rgb_stages_encoder[0][3]) | |
| def t_compress(self) -> int: | |
| """RGB frames per z_rest frame: 2 per "st" stage.""" | |
| return 2 ** sum(1 for s in self.rgb_stages_encoder if _st_kind(s[1])) | |
| def _frame_groups(n_frames: int, group: int) -> list[slice]: | |
| """The clip layout as slices: frame 0 by itself, then groups of ``group`` frames. The parallel calls run | |
| their per-frame parts on these groups, the same batches the autoregressive calls see.""" | |
| return [slice(0, 1)] + [slice(i, i + group) for i in range(1, n_frames, group)] | |
| # --------------------------------------------------------------------------- | |
| # norms | |
| # --------------------------------------------------------------------------- | |
| class RMSNorm3d(nn.Module): | |
| """Channel-only RMS norm on [B, C, T, H, W]. Nothing is reduced over time or space, so a streamed chunk | |
| is normalized exactly like the whole clip.""" | |
| def __init__(self, dim: int, eps: float = 1e-4): | |
| super().__init__() | |
| self.eps = eps | |
| self.gamma = nn.Parameter(torch.ones(1, dim, 1, 1, 1)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return x * torch.rsqrt(x.pow(2).mean(1, keepdim=True) + self.eps) * self.gamma | |
| class RMSNorm2d(nn.Module): | |
| """The same norm on [N, C, H, W].""" | |
| def __init__(self, dim: int, eps: float = 1e-4): | |
| super().__init__() | |
| self.eps = eps | |
| self.gamma = nn.Parameter(torch.ones(1, dim, 1, 1)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return x * torch.rsqrt(x.pow(2).mean(1, keepdim=True) + self.eps) * self.gamma | |
| class SphereNorm3d(nn.Module): | |
| """Put every token's channel vector at RMS 1. No parameters.""" | |
| def __init__(self, eps: float = 1e-6): | |
| super().__init__() | |
| self.eps = eps | |
| def forward(self, z: torch.Tensor) -> torch.Tensor: | |
| rms = z.float().pow(2).mean(dim=1, keepdim=True).add(self.eps).sqrt() | |
| return z / rms.to(z.dtype) | |
| # --------------------------------------------------------------------------- | |
| # causal conv | |
| # --------------------------------------------------------------------------- | |
| class CausalConv3d(nn.Conv3d): | |
| """Conv3d that never reads a future frame: all of the temporal padding goes on the past side. | |
| Streaming: the frames the kernel needs from before this chunk come in as ``tail`` (what the previous call | |
| returned). With no tail the past is zeros, which is the start of a clip. Carrying a short tail of past | |
| frames through a causal conv is the streaming scheme of the Wan2.1 VAE (Apache-2.0). | |
| """ | |
| def __init__(self, *args, **kwargs): | |
| super().__init__(*args, **kwargs) | |
| kt = self.kernel_size[0] | |
| pt, ph, pw = self.padding | |
| assert 2 * pt == kt - 1, f"needs 'same' temporal padding, got kernel {kt} pad {pt}" | |
| self.past = kt - 1 # frames of history one output frame reads | |
| self._space_pad = (pw, pw, ph, ph) | |
| self.padding = (0, 0, 0) | |
| def _conv(self, x: torch.Tensor, tail: torch.Tensor | None): | |
| seen = x if tail is None else torch.cat([tail, x], dim=2) | |
| missing = self.past - (0 if tail is None else tail.shape[2]) | |
| y = super().forward(_cl3d(F.pad(seen, (*self._space_pad, missing, 0)))) | |
| return _cl3d(y), seen | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self._conv(x, None)[0] | |
| def stream(self, x: torch.Tensor, tail: torch.Tensor | None): | |
| """-> (y, the tail the next call needs).""" | |
| y, seen = self._conv(x, tail) | |
| return y, _tail(seen, TAIL_FRAMES) | |
| # --------------------------------------------------------------------------- | |
| # time and space hops | |
| # --------------------------------------------------------------------------- | |
| def _match_channels(x: torch.Tensor, cout: int) -> torch.Tensor: | |
| """Channel adapter with no parameters, for the bypass around a hop: tile channels to widen, mean | |
| channel groups to narrow, tile-and-crop / crop when the widths do not divide.""" | |
| c = x.shape[1] | |
| if cout == c: | |
| return x | |
| if cout % c == 0: | |
| return x.repeat_interleave(cout // c, dim=1) | |
| if c % cout == 0: | |
| b, _, t, h, w = x.shape | |
| return x.reshape(b, cout, c // cout, t, h, w).mean(2) | |
| if cout > c: | |
| reps = (cout + c - 1) // c | |
| return x.repeat_interleave(reps, dim=1)[:, :cout] | |
| return x[:, :cout] | |
| class TemporalDown(nn.Module): | |
| """2x in time: every pair of new frames becomes one. The conv output is summed with the plain mean of | |
| the pair. The first frame of a stream skips this (bootstrap): it passes through and becomes the carry.""" | |
| def __init__(self, c: int): | |
| super().__init__() | |
| self.conv = nn.Conv3d(c, c, (3, 1, 1), stride=(2, 1, 1)) # no padding: the carry is the past | |
| def _pair_mean(frames: torch.Tensor) -> torch.Tensor: | |
| return F.avg_pool3d(frames, kernel_size=(2, 1, 1), stride=(2, 1, 1)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: # parallel: [.., 1 + 2k, ..] -> [.., 1 + k, ..] | |
| if x.shape[2] < 3: | |
| return x[:, :, :1] | |
| return _cl3d(torch.cat([x[:, :, :1], self.conv(_cl3d(x)) + self._pair_mean(x[:, :, 1:])], dim=2)) | |
| def bootstrap(self, x: torch.Tensor): | |
| return x, _tail(x, 1) | |
| def steady(self, x: torch.Tensor, carry: torch.Tensor): | |
| inp = _cl3d(torch.cat([carry, x], dim=2)) # x = an even number of new frames | |
| y = _cl3d(self.conv(inp) + self._pair_mean(x)) | |
| return y, _tail(inp, 1) | |
| class Downsample3D(nn.Module): | |
| """2x in space, then 2x in time. Each step is a conv summed with a pooling bypass (block mean in space, | |
| pair mean in time). The two steps are separate so the first frame can skip the temporal one.""" | |
| def __init__(self, cin: int, cout: int): | |
| super().__init__() | |
| self.cout = cout | |
| self.spatial = CausalConv3d(cin, cout, kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1)) | |
| self.tdown = TemporalDown(cout) | |
| def _spatial(self, x: torch.Tensor) -> torch.Tensor: | |
| s = F.avg_pool3d(x, kernel_size=(1, 2, 2), stride=(1, 2, 2)) | |
| return _cl3d(self.spatial(x) + _match_channels(s, self.cout)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.tdown(self._spatial(x)) | |
| def bootstrap(self, x: torch.Tensor) -> tuple[torch.Tensor, list]: | |
| y, carry = self.tdown.bootstrap(self._spatial(x)) | |
| return y, [carry] | |
| def steady(self, x: torch.Tensor, cache: list) -> tuple[torch.Tensor, list]: | |
| y, carry = self.tdown.steady(self._spatial(x), cache[0]) | |
| return y, [carry] | |
| class TemporalUp(nn.Module): | |
| """2x in time: the conv emits two channel groups per frame, which become two consecutive frames, summed | |
| with the input frame repeated twice. Output frames 2t and 2t+1 read input frames <= t only. The first | |
| frame of a stream skips this (bootstrap), so one latent image stays one frame.""" | |
| def __init__(self, c: int): | |
| super().__init__() | |
| self.conv = CausalConv3d(c, c * 2, kernel_size=(3, 1, 1), padding=(1, 0, 0)) | |
| def _interleave(y: torch.Tensor) -> torch.Tensor: | |
| """[b, 2c, t, h, w] -> [b, c, 2t, h, w].""" | |
| b, c2, t, h, w = y.shape | |
| c = c2 // 2 | |
| y = y.reshape(b, 2, c, t, h, w) | |
| y = torch.stack((y[:, 0], y[:, 1]), dim=3) | |
| return _cl3d(y.reshape(b, c, t * 2, h, w)) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: # parallel: [.., 1 + k, ..] -> [.., 1 + 2k, ..] | |
| if x.shape[2] < 2: | |
| return x | |
| body = x[:, :, 1:] | |
| rest = self._interleave(self.conv(body)) + body.repeat_interleave(2, dim=2) | |
| return _cl3d(torch.cat([x[:, :, :1], rest], dim=2)) | |
| def bootstrap(self, x: torch.Tensor): | |
| return x, None | |
| def steady(self, x: torch.Tensor, tail): | |
| y, tail = self.conv.stream(x, tail) | |
| return _cl3d(self._interleave(y) + x.repeat_interleave(2, dim=2)), tail | |
| class UpsampleTime(nn.Module): | |
| """TemporalUp, then a per-frame 3x3 conv summed with its input. No change in space.""" | |
| def __init__(self, c: int): | |
| super().__init__() | |
| self.tup = TemporalUp(c) | |
| self.up = CausalConv3d(c, c, kernel_size=(1, 3, 3), padding=(0, 1, 1)) | |
| def _up(self, x: torch.Tensor) -> torch.Tensor: | |
| return _cl3d(self.up(x) + x) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self._up(self.tup(x)) | |
| def bootstrap(self, x: torch.Tensor) -> tuple[torch.Tensor, list]: | |
| x, tail = self.tup.bootstrap(x) | |
| return self._up(x), [tail] | |
| def steady(self, x: torch.Tensor, cache: list) -> tuple[torch.Tensor, list]: | |
| x, tail = self.tup.steady(x, cache[0]) | |
| return self._up(x), [tail] | |
| class As5D(nn.Module): | |
| """Run a per-frame 2-D module on [B, C, T, H, W] by folding T into the batch.""" | |
| def __init__(self, mod: nn.Module): | |
| super().__init__() | |
| self.mod = mod | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| B, C, T, h, w = x.shape | |
| y = self.mod(x.permute(0, 2, 1, 3, 4).reshape(B * T, C, h, w).contiguous(memory_format=torch.channels_last)) | |
| _, Cout, H, W = y.shape | |
| return y.reshape(B, T, Cout, H, W).permute(0, 2, 1, 3, 4).contiguous(memory_format=torch.channels_last_3d) | |
| class PixelShuffleUp(nn.Module): | |
| """Decoder spatial hop: conv at the low resolution -> PixelShuffle(r) -> SiLU.""" | |
| def __init__(self, cin: int, cout: int, r: int): | |
| super().__init__() | |
| self.r = r | |
| self.conv = nn.Conv2d(cin, cout * r * r, 3, padding=1) | |
| self.shuffle = nn.PixelShuffle(r) | |
| self.act = nn.SiLU() | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.act(self.shuffle(self.conv(x))) | |
| class PixelUnshuffleDown(nn.Module): | |
| """Encoder spatial hop: PixelUnshuffle(r) -> conv -> SiLU.""" | |
| def __init__(self, cin: int, cout: int, r: int): | |
| super().__init__() | |
| self.r = r | |
| self.unshuffle = nn.PixelUnshuffle(r) | |
| self.conv = nn.Conv2d(cin * r * r, cout, 3, padding=1) | |
| self.act = nn.SiLU() | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.act(self.conv(self.unshuffle(x))) | |
| class SubPixelToRGB(nn.Module): | |
| """Last decoder hop: conv -> PixelShuffle(r) -> RGB at full resolution. No activation.""" | |
| def __init__(self, cin: int, r: int): | |
| super().__init__() | |
| self.r = r | |
| self.conv = nn.Conv2d(cin, 3 * r * r, kernel_size=3, padding=1) | |
| self.shuffle = nn.PixelShuffle(r) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.shuffle(self.conv(x)) | |
| class SubPixelFromRGB(nn.Module): | |
| """First encoder hop: RGB at full resolution -> PixelUnshuffle(r) -> conv. No activation.""" | |
| def __init__(self, cout: int, r: int): | |
| super().__init__() | |
| self.r = r | |
| self.unshuffle = nn.PixelUnshuffle(r) | |
| self.conv = nn.Conv2d(3 * r * r, cout, kernel_size=3, padding=1) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.conv(self.unshuffle(x)) | |
| # --------------------------------------------------------------------------- | |
| # residual blocks | |
| # --------------------------------------------------------------------------- | |
| class ResBlock(nn.Module): | |
| """Pre-norm causal 3x3x3 residual block. Cache = the tails of its two convs.""" | |
| def __init__(self, c: int): | |
| super().__init__() | |
| self.norm1 = RMSNorm3d(c) | |
| self.conv1 = CausalConv3d(c, c, kernel_size=3, padding=1) | |
| self.norm2 = RMSNorm3d(c) | |
| self.conv2 = CausalConv3d(c, c, kernel_size=3, padding=1) | |
| self.act = nn.GELU() | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| h = self.conv1(self.act(self.norm1(x))) | |
| h = self.conv2(self.act(self.norm2(h))) | |
| return _cl3d(x + h) | |
| def steady(self, x: torch.Tensor, cache: list) -> tuple[torch.Tensor, list]: | |
| h, tail1 = self.conv1.stream(self.act(self.norm1(x)), cache[0]) | |
| h, tail2 = self.conv2.stream(self.act(self.norm2(h)), cache[1]) | |
| return _cl3d(x + h), [tail1, tail2] | |
| def bootstrap(self, x: torch.Tensor) -> tuple[torch.Tensor, list]: | |
| return self.steady(x, [None, None]) | |
| class ResBlock2D(nn.Module): | |
| """Pre-norm per-frame 3x3 residual block. No state.""" | |
| def __init__(self, c: int): | |
| super().__init__() | |
| self.norm1 = RMSNorm2d(c) | |
| self.conv1 = nn.Conv2d(c, c, kernel_size=3, padding=1) | |
| self.norm2 = RMSNorm2d(c) | |
| self.conv2 = nn.Conv2d(c, c, kernel_size=3, padding=1) | |
| self.act = nn.GELU() | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| h = self.conv1(self.act(self.norm1(x))) | |
| h = self.conv2(self.act(self.norm2(h))) | |
| return x + h | |
| # --------------------------------------------------------------------------- | |
| # the pixel path between RGB and the latent grid | |
| # --------------------------------------------------------------------------- | |
| class RgbStages(nn.Module): | |
| """One schedule, built as the encoder (RGB -> latent grid) or as the decoder (latent grid -> RGB). | |
| blocks_3d: the causal part (residual blocks and the hops that touch time); it carries the stream cache. | |
| blocks_2d: the per-frame part; no state. | |
| Encoder order: blocks_2d then blocks_3d. Decoder order: blocks_3d then blocks_2d. | |
| """ | |
| def __init__(self, stages: tuple[tuple[int, str, int, int], ...], *, encode: bool): | |
| super().__init__() | |
| self.encode = encode | |
| b3d: list[nn.Module] = [] | |
| b2d: list[nn.Module] = [] | |
| if encode: | |
| # Open from RGB, walk fine -> coarse. | |
| b2d.append(SubPixelFromRGB(stages[-1][3], stages[-1][0])) | |
| for i in range(len(stages) - 1, -1, -1): | |
| scale, kind, n_res, c = stages[i] | |
| target = b3d if _causal_kind(kind) else b2d | |
| for _ in range(n_res): | |
| target.append(ResBlock(c) if _causal_kind(kind) else ResBlock2D(c)) | |
| if i == 0: | |
| continue | |
| nscale, nkind, _, nc = stages[i - 1] | |
| if _st_kind(nkind): | |
| b3d.append(Downsample3D(c, nc)) | |
| else: | |
| down = PixelUnshuffleDown(c, nc, nscale // scale) | |
| if _causal_kind(nkind) and _causal_kind(kind): | |
| b3d.append(As5D(down)) | |
| else: | |
| b2d.append(down) | |
| else: | |
| # Residual blocks at the latent grid, then every time doubling, then spatial hops only. | |
| scale0, _kind0, n_res0, c0 = stages[0] | |
| for _ in range(n_res0): | |
| b3d.append(ResBlock(c0)) | |
| for _ in range(sum(1 for s in stages if _st_kind(s[1]))): | |
| b3d.append(UpsampleTime(c0)) | |
| prev_scale, prev_ch = scale0, c0 | |
| for scale, kind, n_res, c in stages[1:]: | |
| up = PixelShuffleUp(prev_ch, c, prev_scale // scale) | |
| if _causal_kind(kind): | |
| b3d.append(As5D(up)) | |
| else: | |
| b2d.append(up) | |
| target = b3d if _causal_kind(kind) else b2d | |
| for _ in range(n_res): | |
| target.append(ResBlock(c) if _causal_kind(kind) else ResBlock2D(c)) | |
| prev_scale, prev_ch = scale, c | |
| b2d.append(SubPixelToRGB(prev_ch, prev_scale)) | |
| self.blocks_3d = nn.ModuleList(b3d) | |
| self.blocks_2d = nn.ModuleList(b2d) | |
| def forward_3d(self, x: torch.Tensor, group: int) -> torch.Tensor: | |
| """Parallel: the causal part on a whole clip. Past frames come from the clip itself, zeros before it. | |
| A per-frame hop (As5D) runs on the clip's frame groups.""" | |
| for blk in self.blocks_3d: | |
| if isinstance(blk, As5D): | |
| parts = [blk(x[:, :, g]) for g in _frame_groups(x.shape[2], group)] | |
| x = parts[0] if len(parts) == 1 else _cl3d(torch.cat(parts, dim=2)) | |
| else: | |
| x = blk(x) | |
| return x | |
| def stream_3d(self, x: torch.Tensor, cache: Cache | None, bootstrap: bool) -> tuple[torch.Tensor, Cache]: | |
| """Autoregressive: the causal part on one chunk. ``bootstrap``: the first frame (no cache in).""" | |
| new_cache: Cache = [] | |
| ci = 0 | |
| for blk in self.blocks_3d: | |
| if isinstance(blk, (ResBlock, Downsample3D, UpsampleTime)): | |
| if bootstrap: | |
| x, c = blk.bootstrap(x) | |
| else: | |
| assert cache is not None | |
| x, c = blk.steady(x, cache[ci]) | |
| ci += 1 | |
| new_cache.append(c) | |
| else: | |
| x = blk(x) | |
| return x, new_cache | |
| def encode_2d(self, video: torch.Tensor) -> torch.Tensor: | |
| """RGB [B, T, 3, H, W] -> hidden [B, C, T, h, w], frame by frame.""" | |
| B, T, C, H, W = video.shape | |
| x = video.reshape(B * T, C, H, W).contiguous(memory_format=torch.channels_last) | |
| for blk in self.blocks_2d: | |
| x = blk(x) | |
| _, D, h, w = x.shape | |
| return x.reshape(B, T, D, h, w).permute(0, 2, 1, 3, 4).contiguous(memory_format=torch.channels_last_3d) | |
| def decode_2d(self, hidden: torch.Tensor) -> torch.Tensor: | |
| """hidden [B, C, T, h, w] -> RGB [B, T, 3, H, W], frame by frame.""" | |
| B, C, T, h, w = hidden.shape | |
| x = hidden.permute(0, 2, 1, 3, 4).reshape(B * T, C, h, w).contiguous(memory_format=torch.channels_last) | |
| for blk in self.blocks_2d: | |
| x = blk(x) | |
| return x.reshape(B, T, 3, x.shape[-2], x.shape[-1]) | |
| # --------------------------------------------------------------------------- | |
| # encoder | |
| # --------------------------------------------------------------------------- | |
| class BottleneckHead(nn.Module): | |
| """1x1x1 linear read-out to the latent channels, then onto the sphere.""" | |
| def __init__(self, cin: int, out_dims: int): | |
| super().__init__() | |
| self.out_dims = out_dims | |
| self.proj = nn.Conv3d(cin, out_dims, kernel_size=1) | |
| self.sphere_norm = SphereNorm3d() | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.sphere_norm(_cl3d(self.proj(_cl3d(x)))) | |
| class ShakeAE(nn.Module): | |
| """The encoder. RGB [B, T, 3, H, W] (H, W multiples of 16) -> z0 or z_rest. | |
| One trunk, two read-outs: ``head_z0`` for the first frame, ``head_rest`` for the body. | |
| """ | |
| def __init__(self, cfg: SAEConfig | None = None): | |
| super().__init__() | |
| self.cfg = cfg or SAEConfig() | |
| self.rgb = RgbStages(self.cfg.rgb_stages_encoder, encode=True) | |
| self.head_z0 = BottleneckHead(self.cfg.width, self.cfg.out_dims_z0) | |
| self.head_rest = BottleneckHead(self.cfg.width, self.cfg.out_dims_z_rest) | |
| def t_compress(self) -> int: | |
| """RGB frames per z_rest frame.""" | |
| return self.cfg.t_compress | |
| # ---- parallel: the whole clip in one call --------------------------------------------------- | |
| def encode_parallel(self, video: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| """RGB [B, 1 + 4k, 3, H, W] -> (z0 [B, C0, 1, h, w], z_rest [B, Cr, k, h, w]). k = 0 is one image. | |
| Same weights and the same result as encode_bootstrap followed by k encode_steady calls. | |
| """ | |
| t = self.t_compress | |
| assert (video.shape[1] - 1) % t == 0, f"T={video.shape[1]} must be 1 + {t}k" | |
| x = _cl3d(torch.cat([self.rgb.encode_2d(video[:, g]) for g in _frame_groups(video.shape[1], t)], dim=2)) | |
| x = self.rgb.forward_3d(x, t) | |
| z0 = self.head_z0(x[:, :, :1]) | |
| if x.shape[2] > 1: | |
| return z0, self.head_rest(x[:, :, 1:]) | |
| return z0, video.new_zeros(video.shape[0], self.cfg.out_dims_z_rest, 0, *z0.shape[-2:]) | |
| # ---- autoregressive: one chunk per call ----------------------------------------------------- | |
| def encode_bootstrap(self, video_chunk: torch.Tensor) -> tuple[torch.Tensor, Cache]: | |
| """The first frame: 1 RGB frame -> z0 [B, C0, 1, h, w].""" | |
| assert video_chunk.shape[1] == 1, f"bootstrap expects T=1, got T={video_chunk.shape[1]}" | |
| x, cache = self.rgb.stream_3d(self.rgb.encode_2d(video_chunk), None, bootstrap=True) | |
| return self.head_z0(x), cache | |
| def encode_steady(self, video_chunk: torch.Tensor, cache: Cache) -> tuple[torch.Tensor, Cache]: | |
| """A body chunk: 4 RGB frames -> 1 z_rest frame [B, Cr, 1, h, w].""" | |
| assert video_chunk.shape[1] == self.t_compress, ( | |
| f"steady expects T={self.t_compress}, got T={video_chunk.shape[1]}") | |
| x, cache = self.rgb.stream_3d(self.rgb.encode_2d(video_chunk), cache, bootstrap=False) | |
| return self.head_rest(x), cache | |
| # --------------------------------------------------------------------------- | |
| # decoder | |
| # --------------------------------------------------------------------------- | |
| class Decoder(nn.Module): | |
| """The decoder. z0 and z_rest enter the same trunk through their own 1x1x1 projection.""" | |
| def __init__(self, cfg: SAEConfig | None = None): | |
| super().__init__() | |
| self.cfg = cfg or SAEConfig() | |
| self.proj_in_z0 = nn.Conv3d(self.cfg.out_dims_z0, self.cfg.width, kernel_size=1) | |
| self.proj_in_rest = nn.Conv3d(self.cfg.out_dims_z_rest, self.cfg.width, kernel_size=1) | |
| self.rgb = RgbStages(self.cfg.rgb_stages_decoder, encode=False) | |
| def t_compress(self) -> int: | |
| """RGB frames per z_rest frame.""" | |
| return self.cfg.t_compress | |
| # ---- parallel: the whole clip in one call --------------------------------------------------- | |
| def decode_parallel(self, z0: torch.Tensor, z_rest: torch.Tensor) -> torch.Tensor: | |
| """(z0 [B, C0, 1, h, w], z_rest [B, Cr, k, h, w]) -> RGB [B, 1 + 4k, 3, H, W]. k = 0 is one image. | |
| Same weights and the same result as decode_bootstrap followed by k decode_steady calls. | |
| """ | |
| assert z0.shape[2] == 1 and z0.shape[1] == self.cfg.out_dims_z0, tuple(z0.shape) | |
| x = _cl3d(self.proj_in_z0(z0)) | |
| if z_rest.shape[2] > 0: | |
| assert z_rest.shape[1] == self.cfg.out_dims_z_rest, tuple(z_rest.shape) | |
| x = _cl3d(torch.cat([x, _cl3d(self.proj_in_rest(z_rest))], dim=2)) | |
| t = self.t_compress | |
| x = self.rgb.forward_3d(x, t) | |
| return torch.cat([self.rgb.decode_2d(x[:, :, g]) for g in _frame_groups(x.shape[2], t)], dim=1) | |
| # ---- autoregressive: one chunk per call ----------------------------------------------------- | |
| def decode_bootstrap(self, z_chunk: torch.Tensor) -> tuple[torch.Tensor, Cache]: | |
| """z0 [B, C0, 1, h, w] -> 1 RGB frame [B, 1, 3, H, W].""" | |
| assert z_chunk.shape[2] == 1, f"bootstrap expects 1 latent frame, got T={z_chunk.shape[2]}" | |
| assert z_chunk.shape[1] == self.cfg.out_dims_z0, ( | |
| f"z0 channels={z_chunk.shape[1]} != out_dims_z0={self.cfg.out_dims_z0}") | |
| h, cache = self.rgb.stream_3d(_cl3d(self.proj_in_z0(z_chunk)), None, bootstrap=True) | |
| return self.rgb.decode_2d(h), cache | |
| def decode_steady(self, z_chunk: torch.Tensor, cache: Cache) -> tuple[torch.Tensor, Cache]: | |
| """One z_rest frame [B, Cr, 1, h, w] -> 4 RGB frames [B, 4, 3, H, W].""" | |
| assert z_chunk.shape[2] == 1, f"steady expects 1 latent frame, got T={z_chunk.shape[2]}" | |
| assert z_chunk.shape[1] == self.cfg.out_dims_z_rest, ( | |
| f"z_rest channels={z_chunk.shape[1]} != out_dims_z_rest={self.cfg.out_dims_z_rest}") | |
| h, cache = self.rgb.stream_3d(_cl3d(self.proj_in_rest(z_chunk)), cache, bootstrap=False) | |
| return self.rgb.decode_2d(h), cache | |
| # --------------------------------------------------------------------------- | |
| # memory format | |
| # --------------------------------------------------------------------------- | |
| def _apply_channels_last(module: nn.Module) -> tuple[int, int]: | |
| """Conv3d weights -> channels_last_3d, Conv2d weights -> channels_last. Returns the two counts.""" | |
| n3 = n2 = 0 | |
| for m in module.modules(): | |
| if isinstance(m, nn.Conv3d): | |
| m.weight.data = m.weight.data.contiguous(memory_format=torch.channels_last_3d) | |
| n3 += 1 | |
| elif isinstance(m, nn.Conv2d): | |
| m.weight.data = m.weight.data.contiguous(memory_format=torch.channels_last) | |
| n2 += 1 | |
| return n3, n2 | |
| def _assert_channels_last(module: nn.Module, *, tag: str) -> None: | |
| bad: list[str] = [] | |
| for name, m in module.named_modules(): | |
| if isinstance(m, nn.Conv3d) and not m.weight.is_contiguous(memory_format=torch.channels_last_3d): | |
| bad.append(f"{name} (Conv3d not channels_last_3d)") | |
| elif isinstance(m, nn.Conv2d) and not m.weight.is_contiguous(memory_format=torch.channels_last): | |
| bad.append(f"{name} (Conv2d not channels_last)") | |
| assert not bad, f"[{tag}] conv weights not channels-last: {bad[:8]}" | |
| # --------------------------------------------------------------------------- | |
| # The space the model works in. The encoder's latent sits on the sphere; the model sees it standardized | |
| # per channel. ``stats`` is a dict with 'mean' [C] and 'std' [C], or a (mean, std) pair. Both directions | |
| # share _diag_stats_like, so they are exact inverses. | |
| # --------------------------------------------------------------------------- | |
| def _diag_stats_like(stats, z: torch.Tensor, *, eps: float = 1e-6): | |
| """Per-channel (mean, std) shaped to broadcast against ``z`` along dim 1, on z's device and dtype.""" | |
| if isinstance(stats, dict): | |
| mean, std = stats["mean"], stats["std"] | |
| else: | |
| mean, std = stats | |
| mean = torch.as_tensor(mean, device=z.device, dtype=z.dtype) | |
| std = torch.as_tensor(std, device=z.device, dtype=z.dtype).clamp_min(eps) | |
| assert mean.numel() == z.shape[1] == std.numel(), ( | |
| f"stats channels (mean {mean.numel()}, std {std.numel()}) != latent channels {z.shape[1]}") | |
| view = [1, z.shape[1]] + [1] * (z.ndim - 2) | |
| return mean.reshape(view), std.reshape(view) | |
| def training_latent_to_diag_gaussian_latent(z: torch.Tensor, stats, *, eps: float = 1e-6) -> torch.Tensor: | |
| """Encoder latent -> the model's space: (z - mean) / std per channel. ``z``: [B, C, ...].""" | |
| mean, std = _diag_stats_like(stats, z, eps=eps) | |
| return (z - mean) / std | |
| def diag_gaussian_latent_to_training_latent(z: torch.Tensor, stats, *, eps: float = 1e-6) -> torch.Tensor: | |
| """The model's space -> encoder latent: z * std + mean per channel. Apply before decoding.""" | |
| mean, std = _diag_stats_like(stats, z, eps=eps) | |
| return z * std + mean | |