Download summaryformer.py from lucidml/gta: direct link, hf CLI and curl.
- Browser
- Download file 28.7 kB
-
https://huggingface.co/lucidml/gta/resolve/main/summaryformer.py
- Command line
-
hf download hf://lucidml/gta/summaryformer.py
-
curl -L -o summaryformer.py https://huggingface.co/lucidml/gta/resolve/main/summaryformer.py
28.7 kB
| """The world model: one transformer trunk over one stream of frames, two token kinds per frame. | |
| record(t) = [ raw(t) : H*W tokens, embed(x_t) ] ++ [ sum(t) : X tokens, the learned init S ] | |
| ``raw`` tokens are the frame's latent grid at full resolution. ``sum`` tokens are X learned, grid-free queries | |
| appended to every frame; after the trunk they hold a compact summary of that frame. Memory is ONE key/value buffer | |
| per block (``SummaryKVCache``): the last Y frames stay as whole records, older frames inside the window W survive | |
| only as their X summary tokens, and frame 0 (``z0``, the seed image) is never evicted. | |
| Generation is autoregressive, one frame per call, with no attention mask: a frame attends whatever the buffer | |
| holds. ``_forward_kv_cached_z0`` runs the seed frame and ``_forward_kv_cached_z_rest`` one body frame. The seed | |
| frame and the body frames each have their own token embed and their own output head (``embed_z0`` / ``head_z0`` | |
| vs ``embed_rest`` / ``head_rest``): the seed is a still image, the body is video. The public surface is | |
| ``SummaryformerWrapper``. | |
| """ | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| from gta_streaming_kv_cache import SummaryKVCache, SummaryKVPlan | |
| from summaryformer_utils.latent_head import LatentHead | |
| from summaryformer_utils.rope import ( | |
| RopeFactors, | |
| RopeTable, | |
| SpatialBases, | |
| build_rope_table, | |
| build_summary_rope_factors, | |
| build_visual_rope_factors, | |
| clear_rope_spatial_base_cache, | |
| get_rope_spatial_bases_cached, | |
| spatial_bases_from_complex, | |
| validate_level_planes_dead, | |
| ) | |
| # Block classes + the attention-kernel switch live in transformer_block.py; re-exported so | |
| # `from summaryformer import SelfAttention, set_streaming_attention_backend` works. | |
| from summaryformer_utils.transformer_block import ( # noqa: F401 | |
| FILM_SLOTS, | |
| FrameConditioning, | |
| SelfAttention, | |
| TransformerBlock, | |
| get_streaming_attention_backend, | |
| set_streaming_attention_backend, | |
| ) | |
| from summaryformer_utils.z0_focus_mask import apply_z0_focus_region_embedding, coerce_z0_region | |
| # The per-token additive "hold THIS subject" embedding on z0: ``z0_focus_region`` (Bool [B,hs,ws]) ADDS this | |
| # embedding to the z0 tokens where True. A checkpoint may lack it (zeros == no marker). | |
| Z0_FOCUS_REGION_EMBED_NAME = "z0_focus_embedding" | |
| Z0_FOCUS_REGION_EMBED_ALLOWED_MISSING = (Z0_FOCUS_REGION_EMBED_NAME, f"model.{Z0_FOCUS_REGION_EMBED_NAME}") | |
| __all__ = [ | |
| "Summaryformer", | |
| "SelfAttention", | |
| "TransformerBlock", | |
| "set_streaming_attention_backend", | |
| "get_streaming_attention_backend", | |
| ] | |
| class Summaryformer(nn.Module): | |
| """One trunk, one frame per forward, one KV buffer per block. | |
| ``summary_tokens`` (X) is the number of learned summary tokens appended to every frame. | |
| ``raw_window_frames`` (Y) is how many past frames stay in memory as whole records. | |
| ``summary_window_frames`` (W) is the total memory span in frames, raw window included (``W >= Y``). | |
| """ | |
| # Autoencoder latent width. The seed frame and the body frames share it and share the /16 spatial grid; | |
| # one body latent frame is 4 RGB frames. | |
| LATENT_CHANNELS = 48 | |
| # Width of the sinusoidal features fed to the timestep MLP. | |
| TIMESTEP_FEATURES = 256 | |
| # Action sentinel: a frame whose action vector contains this value has no action conditioning. | |
| NO_ACTION = -69 | |
| def __init__( | |
| self, | |
| dim: int = 1280, | |
| ffn_dim: int = 5120, | |
| num_heads: int = 20, | |
| num_layers: int = 28, | |
| *, | |
| summary_tokens: int, | |
| raw_window_frames: int, | |
| summary_window_frames: int, | |
| ): | |
| super().__init__() | |
| self.dim = int(dim) | |
| self.ffn_dim = int(ffn_dim) | |
| self.num_heads = int(num_heads) | |
| self.num_layers = int(num_layers) | |
| self.summary_tokens = int(summary_tokens) | |
| self.raw_window_frames = int(raw_window_frames) | |
| self.summary_window_frames = int(summary_window_frames) | |
| if self.num_layers < 1: | |
| raise ValueError(f"num_layers must be >= 1; got {self.num_layers}.") | |
| if self.dim % self.num_heads != 0 or (self.dim // self.num_heads) % 2 != 0: | |
| raise ValueError(f"dim={self.dim} must split into {self.num_heads} heads of even width.") | |
| if self.summary_tokens < 1: | |
| raise ValueError(f"summary_tokens must be >= 1; got {self.summary_tokens}.") | |
| if self.raw_window_frames < 0: | |
| raise ValueError(f"raw_window_frames must be >= 0; got {self.raw_window_frames}.") | |
| if self.summary_window_frames < 1: | |
| raise ValueError(f"summary_window_frames must be >= 1; got {self.summary_window_frames}.") | |
| dim, channels = self.dim, self.LATENT_CHANNELS | |
| # Timestep -> embedding (also read by the output heads) -> the FiLM table every block reads | |
| # ([FILM_SLOTS, dim]; transformer_block.py says which row does what). | |
| self.timestep_mlp = nn.Sequential(nn.Linear(self.TIMESTEP_FEATURES, dim), nn.SiLU(), nn.Linear(dim, dim)) | |
| self.timestep_to_film = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * FILM_SLOTS)) | |
| self.blocks = nn.ModuleList( | |
| [TransformerBlock(dim, ffn_dim, num_heads) for _ in range(self.num_layers)] | |
| ) | |
| # Seed frame (z0) and body frames (z_rest): separate per-token embeds and separate read-outs. | |
| # One token per latent cell, so the embed is a plain linear on the cell's channels. | |
| self.embed_z0 = nn.Linear(channels, dim) | |
| self.embed_rest = nn.Linear(channels, dim) | |
| self.head_z0 = LatentHead(dim, channels) | |
| self.head_rest = LatentHead(dim, channels) | |
| # ADDED to the z0 tokens inside `z0_focus_region` (Bool [B,hs,ws]): "this is the subject to hold". | |
| # Zeros add nothing, so a model that is never handed a region is unchanged. | |
| self.z0_focus_embedding = nn.Parameter(torch.zeros(dim)) | |
| # The learned summary init: X grid-free tokens, appended to every frame as sum(t). | |
| self.S = nn.Parameter(torch.zeros(self.summary_tokens, dim)) | |
| # Per-frame conditioning, each projected to its own FiLM table [FILM_SLOTS, dim] that every block adds | |
| # through its own learned gate: the action, the grid's aspect ratio / size (separate MLPs for the seed | |
| # frame and the body), and the frame rate. | |
| self.action_proj = self._to_film_mlp(5, dim) | |
| self.aspect_ratio_proj = self._to_film_mlp(2, dim) | |
| self.aspect_ratio_proj_z0 = self._to_film_mlp(2, dim) | |
| self.fps_embedder_mlp = nn.Sequential(nn.Linear(1, dim), nn.SiLU(), nn.Linear(dim, dim)) | |
| self.fps_embedder_outs = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * FILM_SLOTS)) | |
| # Frame rate is encoded relative to this anchor; fps == anchor is the neutral input (0.0). | |
| self.fps_anchor = 12.0 | |
| # RoPE table. Plain tensors on purpose, not registered buffers: a buffer follows `module.to(dtype=...)`, | |
| # and the table must stay complex / fp32 whatever dtype the weights are cast to. `_apply` moves it | |
| # between devices. | |
| self.set_rope_table(build_rope_table(dim // self.num_heads)) | |
| # Summary tokens carry their kind on two dead temporal planes (rope.LEVEL_PLANES). Fail at construction | |
| # if the table ever makes those planes rotate for raw tokens. | |
| validate_level_planes_dead(freqs=self.rope_table) | |
| def _to_film_mlp(in_dim: int, dim: int) -> nn.Module: | |
| return nn.Sequential( | |
| nn.Linear(in_dim, dim), nn.GELU(approximate='tanh'), | |
| nn.Linear(dim, dim * FILM_SLOTS), | |
| ) | |
| # -- RoPE table --------------------------------------------------------------------------------------- | |
| def set_rope_table(self, table: torch.Tensor) -> None: | |
| """Install the complex RoPE table ``[rows, head_dim / 2]`` and its fp32 cos / sin copies. | |
| Drops the spatial-base memo: the bases are slices of the table. | |
| """ | |
| self.rope_table = table | |
| self.rope_cos = table.real.to(torch.float32) | |
| self.rope_sin = table.imag.to(torch.float32) | |
| # (h, w) -> raw SpatialBases. The dynamo-DISABLED base builder is a graph break; a hit here keeps | |
| # `_record_rope` inside ONE compiled graph. | |
| self._stage_spatial_memo: dict[tuple[int, int], SpatialBases] = {} | |
| def _apply(self, fn, *args, **kwargs): | |
| """Device moves carry the RoPE table along (it is not a buffer; see ``__init__``). | |
| PyTorch calls ``_apply`` for every ``.to()`` / ``.cuda()``, also when it is called on a parent module. | |
| The table keeps its own dtype: complex128 is narrowed to complex64, and a ``dtype=`` cast of the weights | |
| never reaches it. | |
| """ | |
| out = super()._apply(fn, *args, **kwargs) | |
| device = self.embed_rest.weight.device | |
| table = self.rope_table | |
| dtype = torch.complex64 if table.dtype == torch.complex128 else table.dtype | |
| if table.device != device or table.dtype != dtype: | |
| table = table.to(device=device, dtype=dtype) | |
| self.rope_table = table | |
| self.rope_cos = table.real.to(device=device, dtype=torch.float32) | |
| self.rope_sin = table.imag.to(device=device, dtype=torch.float32) | |
| # The spatial-base cache is keyed by device and table storage: drop this device's entries and the memo. | |
| clear_rope_spatial_base_cache(device=device) | |
| self._stage_spatial_memo = {} | |
| return out | |
| def _get_rope_spatial_bases( | |
| self, | |
| *, | |
| h_p: int, | |
| w_p: int, | |
| ) -> tuple[torch.Tensor, torch.Tensor]: | |
| """ | |
| Get cached spatial RoPE bases on the model device: | |
| - h_freqs_base: [1,1,H,1,Dh] | |
| - w_freqs_base: [1,1,1,W,Dw] | |
| """ | |
| if int(h_p) <= 0 or int(w_p) <= 0: | |
| raise RuntimeError( | |
| "RoPE spatial bases require positive runtime h_p/w_p." | |
| ) | |
| device = self.embed_rest.weight.device | |
| kw = dict( | |
| freqs=self.rope_table, h_p=int(h_p), w_p=int(w_p), | |
| dim=int(self.dim), num_heads=int(self.num_heads), device=device, | |
| ) | |
| h_base, w_base = get_rope_spatial_bases_cached(**kw) | |
| # A base first built under inference_mode is an inference tensor; rebuild it as a normal one. | |
| if bool(h_base.is_inference()) or bool(w_base.is_inference()): | |
| clear_rope_spatial_base_cache(device=device) | |
| with torch.inference_mode(False): | |
| h_base, w_base = get_rope_spatial_bases_cached(**kw) | |
| return h_base, w_base | |
| def _memo_stage_spatial_bases(self, h_p: int, w_p: int) -> SpatialBases: | |
| """Slow path of `_stage_spatial_bases`: integer-lattice bases at the actual grid.""" | |
| key = (int(h_p), int(w_p)) | |
| raw_spatial = spatial_bases_from_complex(*self._get_rope_spatial_bases(h_p=key[0], w_p=key[1])) | |
| validate_level_planes_dead(freqs=self.rope_table) | |
| if not any(t.is_inference() for t in raw_spatial): | |
| self._stage_spatial_memo[key] = raw_spatial | |
| return raw_spatial | |
| def _stage_spatial_bases(self, h_p: int, w_p: int) -> SpatialBases: | |
| hit = self._stage_spatial_memo.get((int(h_p), int(w_p))) | |
| if hit is not None: | |
| return hit | |
| return self._memo_stage_spatial_bases(int(h_p), int(w_p)) | |
| def _record_rope(self, *, frame_ids: torch.Tensor, h_p: int, w_p: int, n_raw: int) -> RopeFactors: | |
| """RoPE for ONE frame's record ``[raw (h_p·w_p) | X summaries]`` at temporal id(s) ``frame_ids [B, 1]``. | |
| raw : the frame's temporal id + the integer H/W lattice of the grid | |
| summary : the same temporal id, no spatial position, + the level-plane marker | |
| """ | |
| table = RopeTable(cos=self.rope_cos, sin=self.rope_sin) | |
| raw = build_visual_rope_factors( | |
| frame_ids=frame_ids, spatial=self._stage_spatial_bases(int(h_p), int(w_p)), table=table, | |
| ) | |
| summary = build_summary_rope_factors( | |
| frame_ids=frame_ids, summary_tokens=int(self.summary_tokens), table=table, | |
| ) | |
| if int(raw.cos.size(1)) != int(n_raw) or int(summary.cos.size(1)) != int(self.summary_tokens): | |
| raise RuntimeError( | |
| f"record RoPE tape mismatch: raw {int(raw.cos.size(1))} vs {int(n_raw)}; " | |
| f"summary {int(summary.cos.size(1))} vs X={int(self.summary_tokens)}." | |
| ) | |
| return RopeFactors( | |
| cos=torch.cat([raw.cos, summary.cos], dim=1), | |
| sin=torch.cat([raw.sin, summary.sin], dim=1), | |
| ) | |
| # -- conditioning -> FiLM tables ------------------------------------------------------------------------ | |
| def _timestep_features(self, t: torch.Tensor) -> torch.Tensor: | |
| """``t [N]`` -> ``[N, TIMESTEP_FEATURES]``: cos | sin of ``t * 10000^(-i / half)``, in float64.""" | |
| half = self.TIMESTEP_FEATURES // 2 | |
| t = t.to(torch.float64) | |
| rates = torch.pow(10000, -(torch.arange(half, device=t.device, dtype=torch.float64) / half)) | |
| angles = t.unsqueeze(1) * rates.unsqueeze(0) | |
| return torch.cat([torch.cos(angles), torch.sin(angles)], dim=1) | |
| def _timestep_film(self, t: torch.Tensor, *, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]: | |
| """One frame's timestep ``t`` (``[B]`` or ``[B, 1]``) -> ``(t_emb [B, dim], film [B, FILM_SLOTS, dim])``. | |
| ``t_emb`` goes to the output head; ``film`` is the FiLM table every block reads. The two MLPs always run | |
| in fp32, whatever dtype the trunk is in. | |
| """ | |
| with torch.autocast(t.device.type, enabled=False): | |
| t_emb = self.timestep_mlp(self._timestep_features(t.flatten()).to(torch.float32)) | |
| film = self.timestep_to_film(t_emb) | |
| return t_emb.to(dtype=dtype), film.to(dtype=dtype).unflatten(1, (FILM_SLOTS, self.dim)) | |
| def _action_film(self, action: torch.Tensor | None, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor | None: | |
| """One frame's action ``[B, 1, 5]`` -> FiLM table ``[B, FILM_SLOTS, dim]`` (None when no action is given). | |
| A sample whose action contains ``NO_ACTION`` gets an all-zero table: no action conditioning for it. | |
| """ | |
| if action is None: | |
| return None | |
| if not (isinstance(action, torch.Tensor) and action.dim() == 3 and tuple(action.shape[1:]) == (1, 5)): | |
| raise TypeError( | |
| f"action_context must be a Tensor shaped [B, 1, 5] (one frame); got {getattr(action, 'shape', None)}." | |
| ) | |
| with torch.autocast(device.type, enabled=False): | |
| film = self.action_proj(action.to(torch.float32)) | |
| film = film.to(dtype=dtype, device=device).view(int(action.size(0)), FILM_SLOTS, self.dim) | |
| has_action = ~(action == self.NO_ACTION).any(dim=-1) # [B, 1] | |
| return film * has_action.to(dtype=dtype, device=device).unsqueeze(-1) | |
| def _grid_film(self, proj: nn.Module, *, batch: int, h_p: int, w_p: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: | |
| """The frame's grid shape -> FiLM table ``[B, FILM_SLOTS, dim]`` through ``proj``. | |
| The input is ``[log(W/H), log(H*W)]`` in tokens. ``proj`` is ``aspect_ratio_proj`` for body frames and | |
| ``aspect_ratio_proj_z0`` for the seed frame. | |
| """ | |
| b, h, w = int(batch), int(h_p), int(w_p) | |
| if b <= 0 or h <= 0 or w <= 0: | |
| raise RuntimeError(f"Invalid grid for conditioning: B={b}, H_p={h}, W_p={w}.") | |
| geom = torch.tensor( | |
| [math.log(float(w) / float(h)), math.log(float(h * w))], | |
| device=device, | |
| dtype=torch.float32, | |
| ).view(1, 1, 2).expand(b, 1, 2) | |
| with torch.autocast(device.type, enabled=False): | |
| film = proj(geom) | |
| return film.to(dtype=dtype, device=device).view(b, FILM_SLOTS, self.dim) | |
| def _fps_film(self, fps: int | float | torch.Tensor, *, batch: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: | |
| """Frame rate -> FiLM table ``[B, FILM_SLOTS, dim]``. | |
| The input is ``log(fps / fps_anchor)``: 0 at the anchor, symmetric for multiplicative changes (6 vs 24 | |
| around 12). ``fps`` is a number or a tensor (scalar, ``[1]``, ``[B]`` or ``[B, 1]``). A NaN entry means | |
| that sample has no clock: its table is exactly zero. NaN is deliberate, it cannot be mistaken for a real | |
| frame rate. | |
| """ | |
| b = int(batch) | |
| if isinstance(fps, torch.Tensor): | |
| fps_t = fps.to(device=device, dtype=torch.float32) | |
| if fps_t.dim() > 2 or (fps_t.dim() == 2 and int(fps_t.size(1)) != 1) or int(fps_t.numel()) not in (1, b): | |
| raise RuntimeError( | |
| f"fps tensor must be scalar, [1], [B], or [B,1]; got shape={tuple(fps_t.shape)} for B={b}." | |
| ) | |
| fps_t = fps_t.reshape(-1).expand(b) | |
| else: | |
| fps_t = torch.full((b,), float(fps), device=device, dtype=torch.float32) | |
| anchor = float(self.fps_anchor) | |
| has_fps = ~torch.isnan(fps_t) # [B] | |
| # Device-side assert: a Python `if torch.any(...)` here would be a host sync in every forward and a | |
| # dynamo graph break. | |
| torch._assert_async( | |
| ~torch.any(has_fps & (fps_t <= 0)), | |
| "fps must be positive (or NaN = no fps) for fps conditioning.", | |
| ) | |
| # NaN samples run the MLP on the anchor (finite, no NaN propagation) and are zeroed after. | |
| rel = torch.log(torch.where(has_fps, fps_t, torch.full_like(fps_t, anchor)) / anchor) | |
| with torch.autocast(device.type, enabled=False): | |
| film = self.fps_embedder_outs(self.fps_embedder_mlp(rel.view(b, 1, 1))) | |
| film = film.to(dtype=dtype, device=device).view(b, FILM_SLOTS, self.dim) | |
| return film * has_fps.to(film.dtype).view(b, 1, 1) | |
| # -- tokens in, tokens out -------------------------------------------------------------------------------- | |
| def _embed_frame(self, x: torch.Tensor, embed: nn.Module, *, what: str) -> tuple[torch.Tensor, int, int]: | |
| """One frame of latents ``[B, C, 1, H, W]`` -> raw tokens ``[B, H*W, dim]`` (row-major) and ``(H, W)``.""" | |
| c = int(self.LATENT_CHANNELS) | |
| if not (isinstance(x, torch.Tensor) and x.dim() == 5 and int(x.size(1)) == c and int(x.size(2)) == 1): | |
| raise TypeError( | |
| f"{what} must be Tensor[B, C={c}, 1, H, W] (one frame); got shape={getattr(x, 'shape', None)}." | |
| ) | |
| h_p, w_p = int(x.shape[3]), int(x.shape[4]) | |
| cells = x.to(device=embed.weight.device, dtype=embed.weight.dtype).flatten(2).transpose(1, 2) # [B, H*W, C] | |
| return embed(cells), h_p, w_p | |
| def _append_summary_tokens(self, x_raw: torch.Tensor) -> torch.Tensor: | |
| """``[B, raw | X]``: the learned init ``S`` appended to one frame's raw tokens.""" | |
| B = int(x_raw.size(0)) | |
| x_sum = self.S.to(device=x_raw.device, dtype=x_raw.dtype).expand(B, int(self.summary_tokens), int(self.dim)) | |
| return torch.cat([x_raw, x_sum], dim=1) | |
| def _decode_frame(self, raw_tokens: torch.Tensor, head: nn.Module, t_emb: torch.Tensor, h_p: int, w_p: int) -> torch.Tensor: | |
| """A frame's raw tokens ``[B, H*W, dim]`` -> latents ``[B, C, 1, H, W]``. Summary tokens are not decoded.""" | |
| out = head(raw_tokens.contiguous(), t_emb) | |
| B, c = int(out.size(0)), int(self.LATENT_CHANNELS) | |
| return out.view(B, int(h_p), int(w_p), c).permute(0, 3, 1, 2).unsqueeze(2).contiguous() | |
| def _resolve_visual_kv_cache(self, kv_cache) -> SummaryKVCache: | |
| """A forward needs a built ``SummaryKVCache`` whose geometry is this model's.""" | |
| if not isinstance(kv_cache, SummaryKVCache): | |
| raise TypeError( | |
| "A forward needs a SummaryKVCache (use wrapper.new_visual_kv_cache())." | |
| ) | |
| if kv_cache.layers is None: | |
| raise RuntimeError("SummaryKVCache.build(...) must run before any KV forward.") | |
| if int(len(kv_cache.layers)) != int(self.num_layers): | |
| raise RuntimeError( | |
| f"KV cache .layers has {int(len(kv_cache.layers))} slots; " | |
| f"model num_layers={int(self.num_layers)}." | |
| ) | |
| if ( | |
| int(kv_cache.summary_tokens) != int(self.summary_tokens) | |
| or int(kv_cache.raw_window_frames) != int(self.raw_window_frames) | |
| or int(kv_cache.summary_window_frames) != int(self.summary_window_frames) | |
| ): | |
| raise RuntimeError( | |
| "KV cache geometry (X, Y, W)=" | |
| f"({kv_cache.summary_tokens}, {kv_cache.raw_window_frames}, {kv_cache.summary_window_frames}) " | |
| f"!= model ({self.summary_tokens}, {self.raw_window_frames}, {self.summary_window_frames})." | |
| ) | |
| return kv_cache | |
| def _run_blocks( | |
| self, | |
| x: torch.Tensor, | |
| cond: FrameConditioning, | |
| *, | |
| kv_cache: SummaryKVCache, | |
| kv_plan: SummaryKVPlan, | |
| visual_rope: RopeFactors, | |
| ) -> torch.Tensor: | |
| """All trunk blocks over ONE frame's record ``x [B, REC, dim]`` against the one-buffer cache.""" | |
| if int(x.size(1)) != int(kv_plan.query_tokens): | |
| raise RuntimeError( | |
| f"KV plan ({kv_plan.kind}) record width {int(kv_plan.query_tokens)} != x L={int(x.size(1))}." | |
| ) | |
| if int(visual_rope.cos.size(1)) != int(x.size(1)): | |
| raise RuntimeError( | |
| f"RoPE tape L={int(visual_rope.cos.size(1))} != x L={int(x.size(1))}." | |
| ) | |
| for block_index in range(int(self.num_layers)): | |
| x = self.blocks[block_index]( | |
| x, cond, kv_cache=kv_cache.layers[block_index], kv_plan=kv_plan, visual_rope=visual_rope, | |
| ) | |
| return x | |
| # -- the two forwards ------------------------------------------------------------------------------------- | |
| def _forward_kv_cached_z_rest( | |
| self, | |
| x_rest: torch.Tensor, | |
| t_rest: torch.Tensor, | |
| kv_cache=None, | |
| action_context: torch.Tensor | None = None, | |
| fps: int | float | torch.Tensor | None = None, | |
| absolute_start_frames: int | torch.Tensor | None = None, | |
| ) -> torch.Tensor: | |
| """ONE body frame through the one-buffer cache: record(t) = ``[embed_rest(x_t) | S]``. | |
| The first call of a frame runs ``begin_frame`` (donation of the slot about to be overwritten; later | |
| denoising passes of the same frame are a no-op). Every call writes the whole record at that RAW slot | |
| and attends ``buffer[0 : end]``. The caller commits after the clean (t=0) pass with | |
| ``kv_cache.commit_current_frame()`` (cursor advance, no copy). | |
| ``x_rest [B, C, 1, H, W]``, ``t_rest [B, 1]``, ``absolute_start_frames``: this frame's temporal id | |
| (>= 1; z0 is 0), int or 0-d int64 device tensor (compiled streaming form, which also switches the plan | |
| to device ends). Returns the v-pred ``[B, C, 1, H, W]``. | |
| """ | |
| cache = self._resolve_visual_kv_cache(kv_cache) | |
| if absolute_start_frames is None: | |
| raise TypeError("_forward_kv_cached_z_rest requires absolute_start_frames (this frame's temporal id >= 1).") | |
| x_raw, h_p, w_p = self._embed_frame(x_rest, self.embed_rest, what="x_rest") | |
| B, tf = int(x_raw.size(0)), int(x_raw.size(1)) | |
| device = x_raw.device | |
| if not (isinstance(t_rest, torch.Tensor) and tuple(t_rest.shape) == (B, 1)): | |
| raise TypeError(f"KV body forward expects t_rest Tensor[B={B}, 1]; got {getattr(t_rest, 'shape', None)}.") | |
| if tf != int(cache.frame_tokens): | |
| raise RuntimeError( | |
| f"KV cache was built for frame_tokens={int(cache.frame_tokens)} but this frame has {tf} " | |
| f"(grid {h_p}x{w_p}); rebuild the cache for this bucket." | |
| ) | |
| x = self._append_summary_tokens(x_raw) | |
| t_emb, time_film = self._timestep_film(t_rest, dtype=x.dtype) | |
| cond = FrameConditioning( | |
| time=time_film, | |
| action=self._action_film(action_context, device=device, dtype=x.dtype), | |
| grid=self._grid_film(self.aspect_ratio_proj, batch=B, h_p=h_p, w_p=w_p, device=device, dtype=x.dtype), | |
| fps=self._fps_film(fps, batch=B, device=device, dtype=x.dtype) if fps is not None else None, | |
| ) | |
| # Donation once per frame, before the write. plan_body calls begin_frame too; the second call is a no-op. | |
| cache.begin_frame() | |
| if isinstance(absolute_start_frames, torch.Tensor): | |
| if absolute_start_frames.dim() != 0: | |
| raise TypeError("absolute_start_frames tensor must be 0-d (one id for the whole batch).") | |
| frame_ids = absolute_start_frames.to(device=device, dtype=torch.long).view(1, 1).expand(B, 1) | |
| plan = cache.plan_body(device_ends=True) | |
| else: | |
| if int(absolute_start_frames) < 1: | |
| raise ValueError(f"body frame id must be >= 1 (0 is z0); got {int(absolute_start_frames)}.") | |
| frame_ids = torch.full((B, 1), int(absolute_start_frames), dtype=torch.long, device=device) | |
| plan = cache.plan_body(device_ends=False) | |
| visual_rope = self._record_rope(frame_ids=frame_ids, h_p=h_p, w_p=w_p, n_raw=tf) | |
| x = self._run_blocks(x, cond, kv_cache=cache, kv_plan=plan, visual_rope=visual_rope) | |
| return self._decode_frame(x[:, :tf], self.head_rest, t_emb, h_p, w_p) | |
| def _forward_kv_cached_z0( | |
| self, | |
| x_z0: torch.Tensor, | |
| t_z0: torch.Tensor, | |
| *, | |
| kv_cache, | |
| action_context: torch.Tensor | None = None, | |
| fps: int | float | torch.Tensor | None = None, | |
| z0_focus_region: torch.Tensor | None = None, | |
| ) -> torch.Tensor: | |
| """The seed frame: record(0) = ``[embed_z0(x_z0) | S]`` at temporal id 0. | |
| Every call writes record(0) into the Z0 slot and every query attends ``[0 : REC_z0]`` (raw(0) and sum(0) | |
| share that key set). The caller marks the slot valid after the clean pass with ``kv_cache.note_z0_written()``. | |
| ``action_context`` / ``fps`` are accepted for signature symmetry with the body pass and ignored (z0 has no | |
| action / fps FiLM). ``z0_focus_region`` (Bool ``[B, H, W]``, None = off) adds the focus embedding to the | |
| marked tokens. ``x_z0 [B, C, H, W]`` or ``[B, C, 1, H, W]``, ``t_z0 [B]``. Returns the v-pred | |
| ``[B, C, 1, H, W]``. | |
| """ | |
| cache = self._resolve_visual_kv_cache(kv_cache) | |
| if not cache.has_sink: | |
| raise RuntimeError("KV z0 forward on a cache built with z0_tokens=0 (no Z0 slot).") | |
| z0_lat = x_z0.unsqueeze(2) if (isinstance(x_z0, torch.Tensor) and x_z0.dim() == 4) else x_z0 | |
| x_raw, z0_h_p, z0_w_p = self._embed_frame(z0_lat, self.embed_z0, what="x_z0") | |
| B, z0_L = int(x_raw.size(0)), int(x_raw.size(1)) | |
| device = x_raw.device | |
| if not (isinstance(t_z0, torch.Tensor) and tuple(t_z0.shape) == (B,)): | |
| raise TypeError(f"KV z0 forward expects t_z0 Tensor[B={B}]; got {getattr(t_z0, 'shape', None)}.") | |
| if z0_L != int(cache.z0_tokens): | |
| raise RuntimeError( | |
| f"KV cache was built for z0_tokens={int(cache.z0_tokens)} but this z0 has {z0_L} tokens " | |
| f"(grid {z0_h_p}x{z0_w_p}); rebuild the cache for this bucket." | |
| ) | |
| if z0_focus_region is not None: | |
| region = coerce_z0_region(z0_focus_region, batch=B, height=z0_h_p, width=z0_w_p, device=x_raw.device) | |
| x_raw = apply_z0_focus_region_embedding(x_raw, region, self.z0_focus_embedding) | |
| x = self._append_summary_tokens(x_raw) | |
| t_emb, time_film = self._timestep_film(t_z0, dtype=x.dtype) | |
| cond = FrameConditioning( | |
| time=time_film, | |
| grid=self._grid_film(self.aspect_ratio_proj_z0, batch=B, h_p=z0_h_p, w_p=z0_w_p, device=device, dtype=x.dtype), | |
| seed=True, | |
| ) | |
| frame_ids = torch.zeros((B, 1), dtype=torch.long, device=device) | |
| visual_rope = self._record_rope(frame_ids=frame_ids, h_p=z0_h_p, w_p=z0_w_p, n_raw=z0_L) | |
| x = self._run_blocks(x, cond, kv_cache=cache, kv_plan=cache.plan_z0(), visual_rope=visual_rope) | |
| return self._decode_frame(x[:, :z0_L], self.head_z0, t_emb, z0_h_p, z0_w_p) | |
| def forward(self, *args, **kwargs): | |
| raise TypeError( | |
| "Summaryformer has no generic forward. Use _forward_kv_cached_z0 / _forward_kv_cached_z_rest " | |
| "via SummaryformerWrapper (forward_z0 / forward_body)." | |
| ) | |