"""The world model: one transformer trunk over one stream of frames, two token kinds per frame. record(t) = [ raw(t) : H*W tokens, embed(x_t) ] ++ [ sum(t) : X tokens, the learned init S ] ``raw`` tokens are the frame's latent grid at full resolution. ``sum`` tokens are X learned, grid-free queries appended to every frame; after the trunk they hold a compact summary of that frame. Memory is ONE key/value buffer per block (``SummaryKVCache``): the last Y frames stay as whole records, older frames inside the window W survive only as their X summary tokens, and frame 0 (``z0``, the seed image) is never evicted. Generation is autoregressive, one frame per call, with no attention mask: a frame attends whatever the buffer holds. ``_forward_kv_cached_z0`` runs the seed frame and ``_forward_kv_cached_z_rest`` one body frame. The seed frame and the body frames each have their own token embed and their own output head (``embed_z0`` / ``head_z0`` vs ``embed_rest`` / ``head_rest``): the seed is a still image, the body is video. The public surface is ``SummaryformerWrapper``. """ import math import torch import torch.nn as nn from gta_streaming_kv_cache import SummaryKVCache, SummaryKVPlan from summaryformer_utils.latent_head import LatentHead from summaryformer_utils.rope import ( RopeFactors, RopeTable, SpatialBases, build_rope_table, build_summary_rope_factors, build_visual_rope_factors, clear_rope_spatial_base_cache, get_rope_spatial_bases_cached, spatial_bases_from_complex, validate_level_planes_dead, ) # Block classes + the attention-kernel switch live in transformer_block.py; re-exported so # `from summaryformer import SelfAttention, set_streaming_attention_backend` works. from summaryformer_utils.transformer_block import ( # noqa: F401 FILM_SLOTS, FrameConditioning, SelfAttention, TransformerBlock, get_streaming_attention_backend, set_streaming_attention_backend, ) from summaryformer_utils.z0_focus_mask import apply_z0_focus_region_embedding, coerce_z0_region # The per-token additive "hold THIS subject" embedding on z0: ``z0_focus_region`` (Bool [B,hs,ws]) ADDS this # embedding to the z0 tokens where True. A checkpoint may lack it (zeros == no marker). Z0_FOCUS_REGION_EMBED_NAME = "z0_focus_embedding" Z0_FOCUS_REGION_EMBED_ALLOWED_MISSING = (Z0_FOCUS_REGION_EMBED_NAME, f"model.{Z0_FOCUS_REGION_EMBED_NAME}") __all__ = [ "Summaryformer", "SelfAttention", "TransformerBlock", "set_streaming_attention_backend", "get_streaming_attention_backend", ] class Summaryformer(nn.Module): """One trunk, one frame per forward, one KV buffer per block. ``summary_tokens`` (X) is the number of learned summary tokens appended to every frame. ``raw_window_frames`` (Y) is how many past frames stay in memory as whole records. ``summary_window_frames`` (W) is the total memory span in frames, raw window included (``W >= Y``). """ # Autoencoder latent width. The seed frame and the body frames share it and share the /16 spatial grid; # one body latent frame is 4 RGB frames. LATENT_CHANNELS = 48 # Width of the sinusoidal features fed to the timestep MLP. TIMESTEP_FEATURES = 256 # Action sentinel: a frame whose action vector contains this value has no action conditioning. NO_ACTION = -69 def __init__( self, dim: int = 1280, ffn_dim: int = 5120, num_heads: int = 20, num_layers: int = 28, *, summary_tokens: int, raw_window_frames: int, summary_window_frames: int, ): super().__init__() self.dim = int(dim) self.ffn_dim = int(ffn_dim) self.num_heads = int(num_heads) self.num_layers = int(num_layers) self.summary_tokens = int(summary_tokens) self.raw_window_frames = int(raw_window_frames) self.summary_window_frames = int(summary_window_frames) if self.num_layers < 1: raise ValueError(f"num_layers must be >= 1; got {self.num_layers}.") if self.dim % self.num_heads != 0 or (self.dim // self.num_heads) % 2 != 0: raise ValueError(f"dim={self.dim} must split into {self.num_heads} heads of even width.") if self.summary_tokens < 1: raise ValueError(f"summary_tokens must be >= 1; got {self.summary_tokens}.") if self.raw_window_frames < 0: raise ValueError(f"raw_window_frames must be >= 0; got {self.raw_window_frames}.") if self.summary_window_frames < 1: raise ValueError(f"summary_window_frames must be >= 1; got {self.summary_window_frames}.") dim, channels = self.dim, self.LATENT_CHANNELS # Timestep -> embedding (also read by the output heads) -> the FiLM table every block reads # ([FILM_SLOTS, dim]; transformer_block.py says which row does what). self.timestep_mlp = nn.Sequential(nn.Linear(self.TIMESTEP_FEATURES, dim), nn.SiLU(), nn.Linear(dim, dim)) self.timestep_to_film = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * FILM_SLOTS)) self.blocks = nn.ModuleList( [TransformerBlock(dim, ffn_dim, num_heads) for _ in range(self.num_layers)] ) # Seed frame (z0) and body frames (z_rest): separate per-token embeds and separate read-outs. # One token per latent cell, so the embed is a plain linear on the cell's channels. self.embed_z0 = nn.Linear(channels, dim) self.embed_rest = nn.Linear(channels, dim) self.head_z0 = LatentHead(dim, channels) self.head_rest = LatentHead(dim, channels) # ADDED to the z0 tokens inside `z0_focus_region` (Bool [B,hs,ws]): "this is the subject to hold". # Zeros add nothing, so a model that is never handed a region is unchanged. self.z0_focus_embedding = nn.Parameter(torch.zeros(dim)) # The learned summary init: X grid-free tokens, appended to every frame as sum(t). self.S = nn.Parameter(torch.zeros(self.summary_tokens, dim)) # Per-frame conditioning, each projected to its own FiLM table [FILM_SLOTS, dim] that every block adds # through its own learned gate: the action, the grid's aspect ratio / size (separate MLPs for the seed # frame and the body), and the frame rate. self.action_proj = self._to_film_mlp(5, dim) self.aspect_ratio_proj = self._to_film_mlp(2, dim) self.aspect_ratio_proj_z0 = self._to_film_mlp(2, dim) self.fps_embedder_mlp = nn.Sequential(nn.Linear(1, dim), nn.SiLU(), nn.Linear(dim, dim)) self.fps_embedder_outs = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * FILM_SLOTS)) # Frame rate is encoded relative to this anchor; fps == anchor is the neutral input (0.0). self.fps_anchor = 12.0 # RoPE table. Plain tensors on purpose, not registered buffers: a buffer follows `module.to(dtype=...)`, # and the table must stay complex / fp32 whatever dtype the weights are cast to. `_apply` moves it # between devices. self.set_rope_table(build_rope_table(dim // self.num_heads)) # Summary tokens carry their kind on two dead temporal planes (rope.LEVEL_PLANES). Fail at construction # if the table ever makes those planes rotate for raw tokens. validate_level_planes_dead(freqs=self.rope_table) @staticmethod def _to_film_mlp(in_dim: int, dim: int) -> nn.Module: return nn.Sequential( nn.Linear(in_dim, dim), nn.GELU(approximate='tanh'), nn.Linear(dim, dim * FILM_SLOTS), ) # -- RoPE table --------------------------------------------------------------------------------------- def set_rope_table(self, table: torch.Tensor) -> None: """Install the complex RoPE table ``[rows, head_dim / 2]`` and its fp32 cos / sin copies. Drops the spatial-base memo: the bases are slices of the table. """ self.rope_table = table self.rope_cos = table.real.to(torch.float32) self.rope_sin = table.imag.to(torch.float32) # (h, w) -> raw SpatialBases. The dynamo-DISABLED base builder is a graph break; a hit here keeps # `_record_rope` inside ONE compiled graph. self._stage_spatial_memo: dict[tuple[int, int], SpatialBases] = {} def _apply(self, fn, *args, **kwargs): """Device moves carry the RoPE table along (it is not a buffer; see ``__init__``). PyTorch calls ``_apply`` for every ``.to()`` / ``.cuda()``, also when it is called on a parent module. The table keeps its own dtype: complex128 is narrowed to complex64, and a ``dtype=`` cast of the weights never reaches it. """ out = super()._apply(fn, *args, **kwargs) device = self.embed_rest.weight.device table = self.rope_table dtype = torch.complex64 if table.dtype == torch.complex128 else table.dtype if table.device != device or table.dtype != dtype: table = table.to(device=device, dtype=dtype) self.rope_table = table self.rope_cos = table.real.to(device=device, dtype=torch.float32) self.rope_sin = table.imag.to(device=device, dtype=torch.float32) # The spatial-base cache is keyed by device and table storage: drop this device's entries and the memo. clear_rope_spatial_base_cache(device=device) self._stage_spatial_memo = {} return out @torch._dynamo.disable def _get_rope_spatial_bases( self, *, h_p: int, w_p: int, ) -> tuple[torch.Tensor, torch.Tensor]: """ Get cached spatial RoPE bases on the model device: - h_freqs_base: [1,1,H,1,Dh] - w_freqs_base: [1,1,1,W,Dw] """ if int(h_p) <= 0 or int(w_p) <= 0: raise RuntimeError( "RoPE spatial bases require positive runtime h_p/w_p." ) device = self.embed_rest.weight.device kw = dict( freqs=self.rope_table, h_p=int(h_p), w_p=int(w_p), dim=int(self.dim), num_heads=int(self.num_heads), device=device, ) h_base, w_base = get_rope_spatial_bases_cached(**kw) # A base first built under inference_mode is an inference tensor; rebuild it as a normal one. if bool(h_base.is_inference()) or bool(w_base.is_inference()): clear_rope_spatial_base_cache(device=device) with torch.inference_mode(False): h_base, w_base = get_rope_spatial_bases_cached(**kw) return h_base, w_base @torch._dynamo.disable def _memo_stage_spatial_bases(self, h_p: int, w_p: int) -> SpatialBases: """Slow path of `_stage_spatial_bases`: integer-lattice bases at the actual grid.""" key = (int(h_p), int(w_p)) raw_spatial = spatial_bases_from_complex(*self._get_rope_spatial_bases(h_p=key[0], w_p=key[1])) validate_level_planes_dead(freqs=self.rope_table) if not any(t.is_inference() for t in raw_spatial): self._stage_spatial_memo[key] = raw_spatial return raw_spatial def _stage_spatial_bases(self, h_p: int, w_p: int) -> SpatialBases: hit = self._stage_spatial_memo.get((int(h_p), int(w_p))) if hit is not None: return hit return self._memo_stage_spatial_bases(int(h_p), int(w_p)) def _record_rope(self, *, frame_ids: torch.Tensor, h_p: int, w_p: int, n_raw: int) -> RopeFactors: """RoPE for ONE frame's record ``[raw (h_p·w_p) | X summaries]`` at temporal id(s) ``frame_ids [B, 1]``. raw : the frame's temporal id + the integer H/W lattice of the grid summary : the same temporal id, no spatial position, + the level-plane marker """ table = RopeTable(cos=self.rope_cos, sin=self.rope_sin) raw = build_visual_rope_factors( frame_ids=frame_ids, spatial=self._stage_spatial_bases(int(h_p), int(w_p)), table=table, ) summary = build_summary_rope_factors( frame_ids=frame_ids, summary_tokens=int(self.summary_tokens), table=table, ) if int(raw.cos.size(1)) != int(n_raw) or int(summary.cos.size(1)) != int(self.summary_tokens): raise RuntimeError( f"record RoPE tape mismatch: raw {int(raw.cos.size(1))} vs {int(n_raw)}; " f"summary {int(summary.cos.size(1))} vs X={int(self.summary_tokens)}." ) return RopeFactors( cos=torch.cat([raw.cos, summary.cos], dim=1), sin=torch.cat([raw.sin, summary.sin], dim=1), ) # -- conditioning -> FiLM tables ------------------------------------------------------------------------ def _timestep_features(self, t: torch.Tensor) -> torch.Tensor: """``t [N]`` -> ``[N, TIMESTEP_FEATURES]``: cos | sin of ``t * 10000^(-i / half)``, in float64.""" half = self.TIMESTEP_FEATURES // 2 t = t.to(torch.float64) rates = torch.pow(10000, -(torch.arange(half, device=t.device, dtype=torch.float64) / half)) angles = t.unsqueeze(1) * rates.unsqueeze(0) return torch.cat([torch.cos(angles), torch.sin(angles)], dim=1) def _timestep_film(self, t: torch.Tensor, *, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]: """One frame's timestep ``t`` (``[B]`` or ``[B, 1]``) -> ``(t_emb [B, dim], film [B, FILM_SLOTS, dim])``. ``t_emb`` goes to the output head; ``film`` is the FiLM table every block reads. The two MLPs always run in fp32, whatever dtype the trunk is in. """ with torch.autocast(t.device.type, enabled=False): t_emb = self.timestep_mlp(self._timestep_features(t.flatten()).to(torch.float32)) film = self.timestep_to_film(t_emb) return t_emb.to(dtype=dtype), film.to(dtype=dtype).unflatten(1, (FILM_SLOTS, self.dim)) def _action_film(self, action: torch.Tensor | None, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor | None: """One frame's action ``[B, 1, 5]`` -> FiLM table ``[B, FILM_SLOTS, dim]`` (None when no action is given). A sample whose action contains ``NO_ACTION`` gets an all-zero table: no action conditioning for it. """ if action is None: return None if not (isinstance(action, torch.Tensor) and action.dim() == 3 and tuple(action.shape[1:]) == (1, 5)): raise TypeError( f"action_context must be a Tensor shaped [B, 1, 5] (one frame); got {getattr(action, 'shape', None)}." ) with torch.autocast(device.type, enabled=False): film = self.action_proj(action.to(torch.float32)) film = film.to(dtype=dtype, device=device).view(int(action.size(0)), FILM_SLOTS, self.dim) has_action = ~(action == self.NO_ACTION).any(dim=-1) # [B, 1] return film * has_action.to(dtype=dtype, device=device).unsqueeze(-1) def _grid_film(self, proj: nn.Module, *, batch: int, h_p: int, w_p: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: """The frame's grid shape -> FiLM table ``[B, FILM_SLOTS, dim]`` through ``proj``. The input is ``[log(W/H), log(H*W)]`` in tokens. ``proj`` is ``aspect_ratio_proj`` for body frames and ``aspect_ratio_proj_z0`` for the seed frame. """ b, h, w = int(batch), int(h_p), int(w_p) if b <= 0 or h <= 0 or w <= 0: raise RuntimeError(f"Invalid grid for conditioning: B={b}, H_p={h}, W_p={w}.") geom = torch.tensor( [math.log(float(w) / float(h)), math.log(float(h * w))], device=device, dtype=torch.float32, ).view(1, 1, 2).expand(b, 1, 2) with torch.autocast(device.type, enabled=False): film = proj(geom) return film.to(dtype=dtype, device=device).view(b, FILM_SLOTS, self.dim) def _fps_film(self, fps: int | float | torch.Tensor, *, batch: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor: """Frame rate -> FiLM table ``[B, FILM_SLOTS, dim]``. The input is ``log(fps / fps_anchor)``: 0 at the anchor, symmetric for multiplicative changes (6 vs 24 around 12). ``fps`` is a number or a tensor (scalar, ``[1]``, ``[B]`` or ``[B, 1]``). A NaN entry means that sample has no clock: its table is exactly zero. NaN is deliberate, it cannot be mistaken for a real frame rate. """ b = int(batch) if isinstance(fps, torch.Tensor): fps_t = fps.to(device=device, dtype=torch.float32) if fps_t.dim() > 2 or (fps_t.dim() == 2 and int(fps_t.size(1)) != 1) or int(fps_t.numel()) not in (1, b): raise RuntimeError( f"fps tensor must be scalar, [1], [B], or [B,1]; got shape={tuple(fps_t.shape)} for B={b}." ) fps_t = fps_t.reshape(-1).expand(b) else: fps_t = torch.full((b,), float(fps), device=device, dtype=torch.float32) anchor = float(self.fps_anchor) has_fps = ~torch.isnan(fps_t) # [B] # Device-side assert: a Python `if torch.any(...)` here would be a host sync in every forward and a # dynamo graph break. torch._assert_async( ~torch.any(has_fps & (fps_t <= 0)), "fps must be positive (or NaN = no fps) for fps conditioning.", ) # NaN samples run the MLP on the anchor (finite, no NaN propagation) and are zeroed after. rel = torch.log(torch.where(has_fps, fps_t, torch.full_like(fps_t, anchor)) / anchor) with torch.autocast(device.type, enabled=False): film = self.fps_embedder_outs(self.fps_embedder_mlp(rel.view(b, 1, 1))) film = film.to(dtype=dtype, device=device).view(b, FILM_SLOTS, self.dim) return film * has_fps.to(film.dtype).view(b, 1, 1) # -- tokens in, tokens out -------------------------------------------------------------------------------- def _embed_frame(self, x: torch.Tensor, embed: nn.Module, *, what: str) -> tuple[torch.Tensor, int, int]: """One frame of latents ``[B, C, 1, H, W]`` -> raw tokens ``[B, H*W, dim]`` (row-major) and ``(H, W)``.""" c = int(self.LATENT_CHANNELS) if not (isinstance(x, torch.Tensor) and x.dim() == 5 and int(x.size(1)) == c and int(x.size(2)) == 1): raise TypeError( f"{what} must be Tensor[B, C={c}, 1, H, W] (one frame); got shape={getattr(x, 'shape', None)}." ) h_p, w_p = int(x.shape[3]), int(x.shape[4]) cells = x.to(device=embed.weight.device, dtype=embed.weight.dtype).flatten(2).transpose(1, 2) # [B, H*W, C] return embed(cells), h_p, w_p def _append_summary_tokens(self, x_raw: torch.Tensor) -> torch.Tensor: """``[B, raw | X]``: the learned init ``S`` appended to one frame's raw tokens.""" B = int(x_raw.size(0)) x_sum = self.S.to(device=x_raw.device, dtype=x_raw.dtype).expand(B, int(self.summary_tokens), int(self.dim)) return torch.cat([x_raw, x_sum], dim=1) def _decode_frame(self, raw_tokens: torch.Tensor, head: nn.Module, t_emb: torch.Tensor, h_p: int, w_p: int) -> torch.Tensor: """A frame's raw tokens ``[B, H*W, dim]`` -> latents ``[B, C, 1, H, W]``. Summary tokens are not decoded.""" out = head(raw_tokens.contiguous(), t_emb) B, c = int(out.size(0)), int(self.LATENT_CHANNELS) return out.view(B, int(h_p), int(w_p), c).permute(0, 3, 1, 2).unsqueeze(2).contiguous() def _resolve_visual_kv_cache(self, kv_cache) -> SummaryKVCache: """A forward needs a built ``SummaryKVCache`` whose geometry is this model's.""" if not isinstance(kv_cache, SummaryKVCache): raise TypeError( "A forward needs a SummaryKVCache (use wrapper.new_visual_kv_cache())." ) if kv_cache.layers is None: raise RuntimeError("SummaryKVCache.build(...) must run before any KV forward.") if int(len(kv_cache.layers)) != int(self.num_layers): raise RuntimeError( f"KV cache .layers has {int(len(kv_cache.layers))} slots; " f"model num_layers={int(self.num_layers)}." ) if ( int(kv_cache.summary_tokens) != int(self.summary_tokens) or int(kv_cache.raw_window_frames) != int(self.raw_window_frames) or int(kv_cache.summary_window_frames) != int(self.summary_window_frames) ): raise RuntimeError( "KV cache geometry (X, Y, W)=" f"({kv_cache.summary_tokens}, {kv_cache.raw_window_frames}, {kv_cache.summary_window_frames}) " f"!= model ({self.summary_tokens}, {self.raw_window_frames}, {self.summary_window_frames})." ) return kv_cache def _run_blocks( self, x: torch.Tensor, cond: FrameConditioning, *, kv_cache: SummaryKVCache, kv_plan: SummaryKVPlan, visual_rope: RopeFactors, ) -> torch.Tensor: """All trunk blocks over ONE frame's record ``x [B, REC, dim]`` against the one-buffer cache.""" if int(x.size(1)) != int(kv_plan.query_tokens): raise RuntimeError( f"KV plan ({kv_plan.kind}) record width {int(kv_plan.query_tokens)} != x L={int(x.size(1))}." ) if int(visual_rope.cos.size(1)) != int(x.size(1)): raise RuntimeError( f"RoPE tape L={int(visual_rope.cos.size(1))} != x L={int(x.size(1))}." ) for block_index in range(int(self.num_layers)): x = self.blocks[block_index]( x, cond, kv_cache=kv_cache.layers[block_index], kv_plan=kv_plan, visual_rope=visual_rope, ) return x # -- the two forwards ------------------------------------------------------------------------------------- def _forward_kv_cached_z_rest( self, x_rest: torch.Tensor, t_rest: torch.Tensor, kv_cache=None, action_context: torch.Tensor | None = None, fps: int | float | torch.Tensor | None = None, absolute_start_frames: int | torch.Tensor | None = None, ) -> torch.Tensor: """ONE body frame through the one-buffer cache: record(t) = ``[embed_rest(x_t) | S]``. The first call of a frame runs ``begin_frame`` (donation of the slot about to be overwritten; later denoising passes of the same frame are a no-op). Every call writes the whole record at that RAW slot and attends ``buffer[0 : end]``. The caller commits after the clean (t=0) pass with ``kv_cache.commit_current_frame()`` (cursor advance, no copy). ``x_rest [B, C, 1, H, W]``, ``t_rest [B, 1]``, ``absolute_start_frames``: this frame's temporal id (>= 1; z0 is 0), int or 0-d int64 device tensor (compiled streaming form, which also switches the plan to device ends). Returns the v-pred ``[B, C, 1, H, W]``. """ cache = self._resolve_visual_kv_cache(kv_cache) if absolute_start_frames is None: raise TypeError("_forward_kv_cached_z_rest requires absolute_start_frames (this frame's temporal id >= 1).") x_raw, h_p, w_p = self._embed_frame(x_rest, self.embed_rest, what="x_rest") B, tf = int(x_raw.size(0)), int(x_raw.size(1)) device = x_raw.device if not (isinstance(t_rest, torch.Tensor) and tuple(t_rest.shape) == (B, 1)): raise TypeError(f"KV body forward expects t_rest Tensor[B={B}, 1]; got {getattr(t_rest, 'shape', None)}.") if tf != int(cache.frame_tokens): raise RuntimeError( f"KV cache was built for frame_tokens={int(cache.frame_tokens)} but this frame has {tf} " f"(grid {h_p}x{w_p}); rebuild the cache for this bucket." ) x = self._append_summary_tokens(x_raw) t_emb, time_film = self._timestep_film(t_rest, dtype=x.dtype) cond = FrameConditioning( time=time_film, action=self._action_film(action_context, device=device, dtype=x.dtype), grid=self._grid_film(self.aspect_ratio_proj, batch=B, h_p=h_p, w_p=w_p, device=device, dtype=x.dtype), fps=self._fps_film(fps, batch=B, device=device, dtype=x.dtype) if fps is not None else None, ) # Donation once per frame, before the write. plan_body calls begin_frame too; the second call is a no-op. cache.begin_frame() if isinstance(absolute_start_frames, torch.Tensor): if absolute_start_frames.dim() != 0: raise TypeError("absolute_start_frames tensor must be 0-d (one id for the whole batch).") frame_ids = absolute_start_frames.to(device=device, dtype=torch.long).view(1, 1).expand(B, 1) plan = cache.plan_body(device_ends=True) else: if int(absolute_start_frames) < 1: raise ValueError(f"body frame id must be >= 1 (0 is z0); got {int(absolute_start_frames)}.") frame_ids = torch.full((B, 1), int(absolute_start_frames), dtype=torch.long, device=device) plan = cache.plan_body(device_ends=False) visual_rope = self._record_rope(frame_ids=frame_ids, h_p=h_p, w_p=w_p, n_raw=tf) x = self._run_blocks(x, cond, kv_cache=cache, kv_plan=plan, visual_rope=visual_rope) return self._decode_frame(x[:, :tf], self.head_rest, t_emb, h_p, w_p) def _forward_kv_cached_z0( self, x_z0: torch.Tensor, t_z0: torch.Tensor, *, kv_cache, action_context: torch.Tensor | None = None, fps: int | float | torch.Tensor | None = None, z0_focus_region: torch.Tensor | None = None, ) -> torch.Tensor: """The seed frame: record(0) = ``[embed_z0(x_z0) | S]`` at temporal id 0. Every call writes record(0) into the Z0 slot and every query attends ``[0 : REC_z0]`` (raw(0) and sum(0) share that key set). The caller marks the slot valid after the clean pass with ``kv_cache.note_z0_written()``. ``action_context`` / ``fps`` are accepted for signature symmetry with the body pass and ignored (z0 has no action / fps FiLM). ``z0_focus_region`` (Bool ``[B, H, W]``, None = off) adds the focus embedding to the marked tokens. ``x_z0 [B, C, H, W]`` or ``[B, C, 1, H, W]``, ``t_z0 [B]``. Returns the v-pred ``[B, C, 1, H, W]``. """ cache = self._resolve_visual_kv_cache(kv_cache) if not cache.has_sink: raise RuntimeError("KV z0 forward on a cache built with z0_tokens=0 (no Z0 slot).") z0_lat = x_z0.unsqueeze(2) if (isinstance(x_z0, torch.Tensor) and x_z0.dim() == 4) else x_z0 x_raw, z0_h_p, z0_w_p = self._embed_frame(z0_lat, self.embed_z0, what="x_z0") B, z0_L = int(x_raw.size(0)), int(x_raw.size(1)) device = x_raw.device if not (isinstance(t_z0, torch.Tensor) and tuple(t_z0.shape) == (B,)): raise TypeError(f"KV z0 forward expects t_z0 Tensor[B={B}]; got {getattr(t_z0, 'shape', None)}.") if z0_L != int(cache.z0_tokens): raise RuntimeError( f"KV cache was built for z0_tokens={int(cache.z0_tokens)} but this z0 has {z0_L} tokens " f"(grid {z0_h_p}x{z0_w_p}); rebuild the cache for this bucket." ) if z0_focus_region is not None: region = coerce_z0_region(z0_focus_region, batch=B, height=z0_h_p, width=z0_w_p, device=x_raw.device) x_raw = apply_z0_focus_region_embedding(x_raw, region, self.z0_focus_embedding) x = self._append_summary_tokens(x_raw) t_emb, time_film = self._timestep_film(t_z0, dtype=x.dtype) cond = FrameConditioning( time=time_film, grid=self._grid_film(self.aspect_ratio_proj_z0, batch=B, h_p=z0_h_p, w_p=z0_w_p, device=device, dtype=x.dtype), seed=True, ) frame_ids = torch.zeros((B, 1), dtype=torch.long, device=device) visual_rope = self._record_rope(frame_ids=frame_ids, h_p=z0_h_p, w_p=z0_w_p, n_raw=z0_L) x = self._run_blocks(x, cond, kv_cache=cache, kv_plan=cache.plan_z0(), visual_rope=visual_rope) return self._decode_frame(x[:, :z0_L], self.head_z0, t_emb, z0_h_p, z0_w_p) def forward(self, *args, **kwargs): raise TypeError( "Summaryformer has no generic forward. Use _forward_kv_cached_z0 / _forward_kv_cached_z_rest " "via SummaryformerWrapper (forward_z0 / forward_body)." )