gta / summaryformer.py
abhisheksensharma's picture
Upload folder using huggingface_hub
ae4e95d verified
Raw History Blame Contribute Delete
28.7 kB
"""The world model: one transformer trunk over one stream of frames, two token kinds per frame.
record(t) = [ raw(t) : H*W tokens, embed(x_t) ] ++ [ sum(t) : X tokens, the learned init S ]
``raw`` tokens are the frame's latent grid at full resolution. ``sum`` tokens are X learned, grid-free queries
appended to every frame; after the trunk they hold a compact summary of that frame. Memory is ONE key/value buffer
per block (``SummaryKVCache``): the last Y frames stay as whole records, older frames inside the window W survive
only as their X summary tokens, and frame 0 (``z0``, the seed image) is never evicted.
Generation is autoregressive, one frame per call, with no attention mask: a frame attends whatever the buffer
holds. ``_forward_kv_cached_z0`` runs the seed frame and ``_forward_kv_cached_z_rest`` one body frame. The seed
frame and the body frames each have their own token embed and their own output head (``embed_z0`` / ``head_z0``
vs ``embed_rest`` / ``head_rest``): the seed is a still image, the body is video. The public surface is
``SummaryformerWrapper``.
"""
import math
import torch
import torch.nn as nn
from gta_streaming_kv_cache import SummaryKVCache, SummaryKVPlan
from summaryformer_utils.latent_head import LatentHead
from summaryformer_utils.rope import (
RopeFactors,
RopeTable,
SpatialBases,
build_rope_table,
build_summary_rope_factors,
build_visual_rope_factors,
clear_rope_spatial_base_cache,
get_rope_spatial_bases_cached,
spatial_bases_from_complex,
validate_level_planes_dead,
)
# Block classes + the attention-kernel switch live in transformer_block.py; re-exported so
# `from summaryformer import SelfAttention, set_streaming_attention_backend` works.
from summaryformer_utils.transformer_block import ( # noqa: F401
FILM_SLOTS,
FrameConditioning,
SelfAttention,
TransformerBlock,
get_streaming_attention_backend,
set_streaming_attention_backend,
)
from summaryformer_utils.z0_focus_mask import apply_z0_focus_region_embedding, coerce_z0_region
# The per-token additive "hold THIS subject" embedding on z0: ``z0_focus_region`` (Bool [B,hs,ws]) ADDS this
# embedding to the z0 tokens where True. A checkpoint may lack it (zeros == no marker).
Z0_FOCUS_REGION_EMBED_NAME = "z0_focus_embedding"
Z0_FOCUS_REGION_EMBED_ALLOWED_MISSING = (Z0_FOCUS_REGION_EMBED_NAME, f"model.{Z0_FOCUS_REGION_EMBED_NAME}")
__all__ = [
"Summaryformer",
"SelfAttention",
"TransformerBlock",
"set_streaming_attention_backend",
"get_streaming_attention_backend",
]
class Summaryformer(nn.Module):
"""One trunk, one frame per forward, one KV buffer per block.
``summary_tokens`` (X) is the number of learned summary tokens appended to every frame.
``raw_window_frames`` (Y) is how many past frames stay in memory as whole records.
``summary_window_frames`` (W) is the total memory span in frames, raw window included (``W >= Y``).
"""
# Autoencoder latent width. The seed frame and the body frames share it and share the /16 spatial grid;
# one body latent frame is 4 RGB frames.
LATENT_CHANNELS = 48
# Width of the sinusoidal features fed to the timestep MLP.
TIMESTEP_FEATURES = 256
# Action sentinel: a frame whose action vector contains this value has no action conditioning.
NO_ACTION = -69
def __init__(
self,
dim: int = 1280,
ffn_dim: int = 5120,
num_heads: int = 20,
num_layers: int = 28,
*,
summary_tokens: int,
raw_window_frames: int,
summary_window_frames: int,
):
super().__init__()
self.dim = int(dim)
self.ffn_dim = int(ffn_dim)
self.num_heads = int(num_heads)
self.num_layers = int(num_layers)
self.summary_tokens = int(summary_tokens)
self.raw_window_frames = int(raw_window_frames)
self.summary_window_frames = int(summary_window_frames)
if self.num_layers < 1:
raise ValueError(f"num_layers must be >= 1; got {self.num_layers}.")
if self.dim % self.num_heads != 0 or (self.dim // self.num_heads) % 2 != 0:
raise ValueError(f"dim={self.dim} must split into {self.num_heads} heads of even width.")
if self.summary_tokens < 1:
raise ValueError(f"summary_tokens must be >= 1; got {self.summary_tokens}.")
if self.raw_window_frames < 0:
raise ValueError(f"raw_window_frames must be >= 0; got {self.raw_window_frames}.")
if self.summary_window_frames < 1:
raise ValueError(f"summary_window_frames must be >= 1; got {self.summary_window_frames}.")
dim, channels = self.dim, self.LATENT_CHANNELS
# Timestep -> embedding (also read by the output heads) -> the FiLM table every block reads
# ([FILM_SLOTS, dim]; transformer_block.py says which row does what).
self.timestep_mlp = nn.Sequential(nn.Linear(self.TIMESTEP_FEATURES, dim), nn.SiLU(), nn.Linear(dim, dim))
self.timestep_to_film = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * FILM_SLOTS))
self.blocks = nn.ModuleList(
[TransformerBlock(dim, ffn_dim, num_heads) for _ in range(self.num_layers)]
)
# Seed frame (z0) and body frames (z_rest): separate per-token embeds and separate read-outs.
# One token per latent cell, so the embed is a plain linear on the cell's channels.
self.embed_z0 = nn.Linear(channels, dim)
self.embed_rest = nn.Linear(channels, dim)
self.head_z0 = LatentHead(dim, channels)
self.head_rest = LatentHead(dim, channels)
# ADDED to the z0 tokens inside `z0_focus_region` (Bool [B,hs,ws]): "this is the subject to hold".
# Zeros add nothing, so a model that is never handed a region is unchanged.
self.z0_focus_embedding = nn.Parameter(torch.zeros(dim))
# The learned summary init: X grid-free tokens, appended to every frame as sum(t).
self.S = nn.Parameter(torch.zeros(self.summary_tokens, dim))
# Per-frame conditioning, each projected to its own FiLM table [FILM_SLOTS, dim] that every block adds
# through its own learned gate: the action, the grid's aspect ratio / size (separate MLPs for the seed
# frame and the body), and the frame rate.
self.action_proj = self._to_film_mlp(5, dim)
self.aspect_ratio_proj = self._to_film_mlp(2, dim)
self.aspect_ratio_proj_z0 = self._to_film_mlp(2, dim)
self.fps_embedder_mlp = nn.Sequential(nn.Linear(1, dim), nn.SiLU(), nn.Linear(dim, dim))
self.fps_embedder_outs = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * FILM_SLOTS))
# Frame rate is encoded relative to this anchor; fps == anchor is the neutral input (0.0).
self.fps_anchor = 12.0
# RoPE table. Plain tensors on purpose, not registered buffers: a buffer follows `module.to(dtype=...)`,
# and the table must stay complex / fp32 whatever dtype the weights are cast to. `_apply` moves it
# between devices.
self.set_rope_table(build_rope_table(dim // self.num_heads))
# Summary tokens carry their kind on two dead temporal planes (rope.LEVEL_PLANES). Fail at construction
# if the table ever makes those planes rotate for raw tokens.
validate_level_planes_dead(freqs=self.rope_table)
@staticmethod
def _to_film_mlp(in_dim: int, dim: int) -> nn.Module:
return nn.Sequential(
nn.Linear(in_dim, dim), nn.GELU(approximate='tanh'),
nn.Linear(dim, dim * FILM_SLOTS),
)
# -- RoPE table ---------------------------------------------------------------------------------------
def set_rope_table(self, table: torch.Tensor) -> None:
"""Install the complex RoPE table ``[rows, head_dim / 2]`` and its fp32 cos / sin copies.
Drops the spatial-base memo: the bases are slices of the table.
"""
self.rope_table = table
self.rope_cos = table.real.to(torch.float32)
self.rope_sin = table.imag.to(torch.float32)
# (h, w) -> raw SpatialBases. The dynamo-DISABLED base builder is a graph break; a hit here keeps
# `_record_rope` inside ONE compiled graph.
self._stage_spatial_memo: dict[tuple[int, int], SpatialBases] = {}
def _apply(self, fn, *args, **kwargs):
"""Device moves carry the RoPE table along (it is not a buffer; see ``__init__``).
PyTorch calls ``_apply`` for every ``.to()`` / ``.cuda()``, also when it is called on a parent module.
The table keeps its own dtype: complex128 is narrowed to complex64, and a ``dtype=`` cast of the weights
never reaches it.
"""
out = super()._apply(fn, *args, **kwargs)
device = self.embed_rest.weight.device
table = self.rope_table
dtype = torch.complex64 if table.dtype == torch.complex128 else table.dtype
if table.device != device or table.dtype != dtype:
table = table.to(device=device, dtype=dtype)
self.rope_table = table
self.rope_cos = table.real.to(device=device, dtype=torch.float32)
self.rope_sin = table.imag.to(device=device, dtype=torch.float32)
# The spatial-base cache is keyed by device and table storage: drop this device's entries and the memo.
clear_rope_spatial_base_cache(device=device)
self._stage_spatial_memo = {}
return out
@torch._dynamo.disable
def _get_rope_spatial_bases(
self,
*,
h_p: int,
w_p: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Get cached spatial RoPE bases on the model device:
- h_freqs_base: [1,1,H,1,Dh]
- w_freqs_base: [1,1,1,W,Dw]
"""
if int(h_p) <= 0 or int(w_p) <= 0:
raise RuntimeError(
"RoPE spatial bases require positive runtime h_p/w_p."
)
device = self.embed_rest.weight.device
kw = dict(
freqs=self.rope_table, h_p=int(h_p), w_p=int(w_p),
dim=int(self.dim), num_heads=int(self.num_heads), device=device,
)
h_base, w_base = get_rope_spatial_bases_cached(**kw)
# A base first built under inference_mode is an inference tensor; rebuild it as a normal one.
if bool(h_base.is_inference()) or bool(w_base.is_inference()):
clear_rope_spatial_base_cache(device=device)
with torch.inference_mode(False):
h_base, w_base = get_rope_spatial_bases_cached(**kw)
return h_base, w_base
@torch._dynamo.disable
def _memo_stage_spatial_bases(self, h_p: int, w_p: int) -> SpatialBases:
"""Slow path of `_stage_spatial_bases`: integer-lattice bases at the actual grid."""
key = (int(h_p), int(w_p))
raw_spatial = spatial_bases_from_complex(*self._get_rope_spatial_bases(h_p=key[0], w_p=key[1]))
validate_level_planes_dead(freqs=self.rope_table)
if not any(t.is_inference() for t in raw_spatial):
self._stage_spatial_memo[key] = raw_spatial
return raw_spatial
def _stage_spatial_bases(self, h_p: int, w_p: int) -> SpatialBases:
hit = self._stage_spatial_memo.get((int(h_p), int(w_p)))
if hit is not None:
return hit
return self._memo_stage_spatial_bases(int(h_p), int(w_p))
def _record_rope(self, *, frame_ids: torch.Tensor, h_p: int, w_p: int, n_raw: int) -> RopeFactors:
"""RoPE for ONE frame's record ``[raw (h_p·w_p) | X summaries]`` at temporal id(s) ``frame_ids [B, 1]``.
raw : the frame's temporal id + the integer H/W lattice of the grid
summary : the same temporal id, no spatial position, + the level-plane marker
"""
table = RopeTable(cos=self.rope_cos, sin=self.rope_sin)
raw = build_visual_rope_factors(
frame_ids=frame_ids, spatial=self._stage_spatial_bases(int(h_p), int(w_p)), table=table,
)
summary = build_summary_rope_factors(
frame_ids=frame_ids, summary_tokens=int(self.summary_tokens), table=table,
)
if int(raw.cos.size(1)) != int(n_raw) or int(summary.cos.size(1)) != int(self.summary_tokens):
raise RuntimeError(
f"record RoPE tape mismatch: raw {int(raw.cos.size(1))} vs {int(n_raw)}; "
f"summary {int(summary.cos.size(1))} vs X={int(self.summary_tokens)}."
)
return RopeFactors(
cos=torch.cat([raw.cos, summary.cos], dim=1),
sin=torch.cat([raw.sin, summary.sin], dim=1),
)
# -- conditioning -> FiLM tables ------------------------------------------------------------------------
def _timestep_features(self, t: torch.Tensor) -> torch.Tensor:
"""``t [N]`` -> ``[N, TIMESTEP_FEATURES]``: cos | sin of ``t * 10000^(-i / half)``, in float64."""
half = self.TIMESTEP_FEATURES // 2
t = t.to(torch.float64)
rates = torch.pow(10000, -(torch.arange(half, device=t.device, dtype=torch.float64) / half))
angles = t.unsqueeze(1) * rates.unsqueeze(0)
return torch.cat([torch.cos(angles), torch.sin(angles)], dim=1)
def _timestep_film(self, t: torch.Tensor, *, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor]:
"""One frame's timestep ``t`` (``[B]`` or ``[B, 1]``) -> ``(t_emb [B, dim], film [B, FILM_SLOTS, dim])``.
``t_emb`` goes to the output head; ``film`` is the FiLM table every block reads. The two MLPs always run
in fp32, whatever dtype the trunk is in.
"""
with torch.autocast(t.device.type, enabled=False):
t_emb = self.timestep_mlp(self._timestep_features(t.flatten()).to(torch.float32))
film = self.timestep_to_film(t_emb)
return t_emb.to(dtype=dtype), film.to(dtype=dtype).unflatten(1, (FILM_SLOTS, self.dim))
def _action_film(self, action: torch.Tensor | None, *, device: torch.device, dtype: torch.dtype) -> torch.Tensor | None:
"""One frame's action ``[B, 1, 5]`` -> FiLM table ``[B, FILM_SLOTS, dim]`` (None when no action is given).
A sample whose action contains ``NO_ACTION`` gets an all-zero table: no action conditioning for it.
"""
if action is None:
return None
if not (isinstance(action, torch.Tensor) and action.dim() == 3 and tuple(action.shape[1:]) == (1, 5)):
raise TypeError(
f"action_context must be a Tensor shaped [B, 1, 5] (one frame); got {getattr(action, 'shape', None)}."
)
with torch.autocast(device.type, enabled=False):
film = self.action_proj(action.to(torch.float32))
film = film.to(dtype=dtype, device=device).view(int(action.size(0)), FILM_SLOTS, self.dim)
has_action = ~(action == self.NO_ACTION).any(dim=-1) # [B, 1]
return film * has_action.to(dtype=dtype, device=device).unsqueeze(-1)
def _grid_film(self, proj: nn.Module, *, batch: int, h_p: int, w_p: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
"""The frame's grid shape -> FiLM table ``[B, FILM_SLOTS, dim]`` through ``proj``.
The input is ``[log(W/H), log(H*W)]`` in tokens. ``proj`` is ``aspect_ratio_proj`` for body frames and
``aspect_ratio_proj_z0`` for the seed frame.
"""
b, h, w = int(batch), int(h_p), int(w_p)
if b <= 0 or h <= 0 or w <= 0:
raise RuntimeError(f"Invalid grid for conditioning: B={b}, H_p={h}, W_p={w}.")
geom = torch.tensor(
[math.log(float(w) / float(h)), math.log(float(h * w))],
device=device,
dtype=torch.float32,
).view(1, 1, 2).expand(b, 1, 2)
with torch.autocast(device.type, enabled=False):
film = proj(geom)
return film.to(dtype=dtype, device=device).view(b, FILM_SLOTS, self.dim)
def _fps_film(self, fps: int | float | torch.Tensor, *, batch: int, device: torch.device, dtype: torch.dtype) -> torch.Tensor:
"""Frame rate -> FiLM table ``[B, FILM_SLOTS, dim]``.
The input is ``log(fps / fps_anchor)``: 0 at the anchor, symmetric for multiplicative changes (6 vs 24
around 12). ``fps`` is a number or a tensor (scalar, ``[1]``, ``[B]`` or ``[B, 1]``). A NaN entry means
that sample has no clock: its table is exactly zero. NaN is deliberate, it cannot be mistaken for a real
frame rate.
"""
b = int(batch)
if isinstance(fps, torch.Tensor):
fps_t = fps.to(device=device, dtype=torch.float32)
if fps_t.dim() > 2 or (fps_t.dim() == 2 and int(fps_t.size(1)) != 1) or int(fps_t.numel()) not in (1, b):
raise RuntimeError(
f"fps tensor must be scalar, [1], [B], or [B,1]; got shape={tuple(fps_t.shape)} for B={b}."
)
fps_t = fps_t.reshape(-1).expand(b)
else:
fps_t = torch.full((b,), float(fps), device=device, dtype=torch.float32)
anchor = float(self.fps_anchor)
has_fps = ~torch.isnan(fps_t) # [B]
# Device-side assert: a Python `if torch.any(...)` here would be a host sync in every forward and a
# dynamo graph break.
torch._assert_async(
~torch.any(has_fps & (fps_t <= 0)),
"fps must be positive (or NaN = no fps) for fps conditioning.",
)
# NaN samples run the MLP on the anchor (finite, no NaN propagation) and are zeroed after.
rel = torch.log(torch.where(has_fps, fps_t, torch.full_like(fps_t, anchor)) / anchor)
with torch.autocast(device.type, enabled=False):
film = self.fps_embedder_outs(self.fps_embedder_mlp(rel.view(b, 1, 1)))
film = film.to(dtype=dtype, device=device).view(b, FILM_SLOTS, self.dim)
return film * has_fps.to(film.dtype).view(b, 1, 1)
# -- tokens in, tokens out --------------------------------------------------------------------------------
def _embed_frame(self, x: torch.Tensor, embed: nn.Module, *, what: str) -> tuple[torch.Tensor, int, int]:
"""One frame of latents ``[B, C, 1, H, W]`` -> raw tokens ``[B, H*W, dim]`` (row-major) and ``(H, W)``."""
c = int(self.LATENT_CHANNELS)
if not (isinstance(x, torch.Tensor) and x.dim() == 5 and int(x.size(1)) == c and int(x.size(2)) == 1):
raise TypeError(
f"{what} must be Tensor[B, C={c}, 1, H, W] (one frame); got shape={getattr(x, 'shape', None)}."
)
h_p, w_p = int(x.shape[3]), int(x.shape[4])
cells = x.to(device=embed.weight.device, dtype=embed.weight.dtype).flatten(2).transpose(1, 2) # [B, H*W, C]
return embed(cells), h_p, w_p
def _append_summary_tokens(self, x_raw: torch.Tensor) -> torch.Tensor:
"""``[B, raw | X]``: the learned init ``S`` appended to one frame's raw tokens."""
B = int(x_raw.size(0))
x_sum = self.S.to(device=x_raw.device, dtype=x_raw.dtype).expand(B, int(self.summary_tokens), int(self.dim))
return torch.cat([x_raw, x_sum], dim=1)
def _decode_frame(self, raw_tokens: torch.Tensor, head: nn.Module, t_emb: torch.Tensor, h_p: int, w_p: int) -> torch.Tensor:
"""A frame's raw tokens ``[B, H*W, dim]`` -> latents ``[B, C, 1, H, W]``. Summary tokens are not decoded."""
out = head(raw_tokens.contiguous(), t_emb)
B, c = int(out.size(0)), int(self.LATENT_CHANNELS)
return out.view(B, int(h_p), int(w_p), c).permute(0, 3, 1, 2).unsqueeze(2).contiguous()
def _resolve_visual_kv_cache(self, kv_cache) -> SummaryKVCache:
"""A forward needs a built ``SummaryKVCache`` whose geometry is this model's."""
if not isinstance(kv_cache, SummaryKVCache):
raise TypeError(
"A forward needs a SummaryKVCache (use wrapper.new_visual_kv_cache())."
)
if kv_cache.layers is None:
raise RuntimeError("SummaryKVCache.build(...) must run before any KV forward.")
if int(len(kv_cache.layers)) != int(self.num_layers):
raise RuntimeError(
f"KV cache .layers has {int(len(kv_cache.layers))} slots; "
f"model num_layers={int(self.num_layers)}."
)
if (
int(kv_cache.summary_tokens) != int(self.summary_tokens)
or int(kv_cache.raw_window_frames) != int(self.raw_window_frames)
or int(kv_cache.summary_window_frames) != int(self.summary_window_frames)
):
raise RuntimeError(
"KV cache geometry (X, Y, W)="
f"({kv_cache.summary_tokens}, {kv_cache.raw_window_frames}, {kv_cache.summary_window_frames}) "
f"!= model ({self.summary_tokens}, {self.raw_window_frames}, {self.summary_window_frames})."
)
return kv_cache
def _run_blocks(
self,
x: torch.Tensor,
cond: FrameConditioning,
*,
kv_cache: SummaryKVCache,
kv_plan: SummaryKVPlan,
visual_rope: RopeFactors,
) -> torch.Tensor:
"""All trunk blocks over ONE frame's record ``x [B, REC, dim]`` against the one-buffer cache."""
if int(x.size(1)) != int(kv_plan.query_tokens):
raise RuntimeError(
f"KV plan ({kv_plan.kind}) record width {int(kv_plan.query_tokens)} != x L={int(x.size(1))}."
)
if int(visual_rope.cos.size(1)) != int(x.size(1)):
raise RuntimeError(
f"RoPE tape L={int(visual_rope.cos.size(1))} != x L={int(x.size(1))}."
)
for block_index in range(int(self.num_layers)):
x = self.blocks[block_index](
x, cond, kv_cache=kv_cache.layers[block_index], kv_plan=kv_plan, visual_rope=visual_rope,
)
return x
# -- the two forwards -------------------------------------------------------------------------------------
def _forward_kv_cached_z_rest(
self,
x_rest: torch.Tensor,
t_rest: torch.Tensor,
kv_cache=None,
action_context: torch.Tensor | None = None,
fps: int | float | torch.Tensor | None = None,
absolute_start_frames: int | torch.Tensor | None = None,
) -> torch.Tensor:
"""ONE body frame through the one-buffer cache: record(t) = ``[embed_rest(x_t) | S]``.
The first call of a frame runs ``begin_frame`` (donation of the slot about to be overwritten; later
denoising passes of the same frame are a no-op). Every call writes the whole record at that RAW slot
and attends ``buffer[0 : end]``. The caller commits after the clean (t=0) pass with
``kv_cache.commit_current_frame()`` (cursor advance, no copy).
``x_rest [B, C, 1, H, W]``, ``t_rest [B, 1]``, ``absolute_start_frames``: this frame's temporal id
(>= 1; z0 is 0), int or 0-d int64 device tensor (compiled streaming form, which also switches the plan
to device ends). Returns the v-pred ``[B, C, 1, H, W]``.
"""
cache = self._resolve_visual_kv_cache(kv_cache)
if absolute_start_frames is None:
raise TypeError("_forward_kv_cached_z_rest requires absolute_start_frames (this frame's temporal id >= 1).")
x_raw, h_p, w_p = self._embed_frame(x_rest, self.embed_rest, what="x_rest")
B, tf = int(x_raw.size(0)), int(x_raw.size(1))
device = x_raw.device
if not (isinstance(t_rest, torch.Tensor) and tuple(t_rest.shape) == (B, 1)):
raise TypeError(f"KV body forward expects t_rest Tensor[B={B}, 1]; got {getattr(t_rest, 'shape', None)}.")
if tf != int(cache.frame_tokens):
raise RuntimeError(
f"KV cache was built for frame_tokens={int(cache.frame_tokens)} but this frame has {tf} "
f"(grid {h_p}x{w_p}); rebuild the cache for this bucket."
)
x = self._append_summary_tokens(x_raw)
t_emb, time_film = self._timestep_film(t_rest, dtype=x.dtype)
cond = FrameConditioning(
time=time_film,
action=self._action_film(action_context, device=device, dtype=x.dtype),
grid=self._grid_film(self.aspect_ratio_proj, batch=B, h_p=h_p, w_p=w_p, device=device, dtype=x.dtype),
fps=self._fps_film(fps, batch=B, device=device, dtype=x.dtype) if fps is not None else None,
)
# Donation once per frame, before the write. plan_body calls begin_frame too; the second call is a no-op.
cache.begin_frame()
if isinstance(absolute_start_frames, torch.Tensor):
if absolute_start_frames.dim() != 0:
raise TypeError("absolute_start_frames tensor must be 0-d (one id for the whole batch).")
frame_ids = absolute_start_frames.to(device=device, dtype=torch.long).view(1, 1).expand(B, 1)
plan = cache.plan_body(device_ends=True)
else:
if int(absolute_start_frames) < 1:
raise ValueError(f"body frame id must be >= 1 (0 is z0); got {int(absolute_start_frames)}.")
frame_ids = torch.full((B, 1), int(absolute_start_frames), dtype=torch.long, device=device)
plan = cache.plan_body(device_ends=False)
visual_rope = self._record_rope(frame_ids=frame_ids, h_p=h_p, w_p=w_p, n_raw=tf)
x = self._run_blocks(x, cond, kv_cache=cache, kv_plan=plan, visual_rope=visual_rope)
return self._decode_frame(x[:, :tf], self.head_rest, t_emb, h_p, w_p)
def _forward_kv_cached_z0(
self,
x_z0: torch.Tensor,
t_z0: torch.Tensor,
*,
kv_cache,
action_context: torch.Tensor | None = None,
fps: int | float | torch.Tensor | None = None,
z0_focus_region: torch.Tensor | None = None,
) -> torch.Tensor:
"""The seed frame: record(0) = ``[embed_z0(x_z0) | S]`` at temporal id 0.
Every call writes record(0) into the Z0 slot and every query attends ``[0 : REC_z0]`` (raw(0) and sum(0)
share that key set). The caller marks the slot valid after the clean pass with ``kv_cache.note_z0_written()``.
``action_context`` / ``fps`` are accepted for signature symmetry with the body pass and ignored (z0 has no
action / fps FiLM). ``z0_focus_region`` (Bool ``[B, H, W]``, None = off) adds the focus embedding to the
marked tokens. ``x_z0 [B, C, H, W]`` or ``[B, C, 1, H, W]``, ``t_z0 [B]``. Returns the v-pred
``[B, C, 1, H, W]``.
"""
cache = self._resolve_visual_kv_cache(kv_cache)
if not cache.has_sink:
raise RuntimeError("KV z0 forward on a cache built with z0_tokens=0 (no Z0 slot).")
z0_lat = x_z0.unsqueeze(2) if (isinstance(x_z0, torch.Tensor) and x_z0.dim() == 4) else x_z0
x_raw, z0_h_p, z0_w_p = self._embed_frame(z0_lat, self.embed_z0, what="x_z0")
B, z0_L = int(x_raw.size(0)), int(x_raw.size(1))
device = x_raw.device
if not (isinstance(t_z0, torch.Tensor) and tuple(t_z0.shape) == (B,)):
raise TypeError(f"KV z0 forward expects t_z0 Tensor[B={B}]; got {getattr(t_z0, 'shape', None)}.")
if z0_L != int(cache.z0_tokens):
raise RuntimeError(
f"KV cache was built for z0_tokens={int(cache.z0_tokens)} but this z0 has {z0_L} tokens "
f"(grid {z0_h_p}x{z0_w_p}); rebuild the cache for this bucket."
)
if z0_focus_region is not None:
region = coerce_z0_region(z0_focus_region, batch=B, height=z0_h_p, width=z0_w_p, device=x_raw.device)
x_raw = apply_z0_focus_region_embedding(x_raw, region, self.z0_focus_embedding)
x = self._append_summary_tokens(x_raw)
t_emb, time_film = self._timestep_film(t_z0, dtype=x.dtype)
cond = FrameConditioning(
time=time_film,
grid=self._grid_film(self.aspect_ratio_proj_z0, batch=B, h_p=z0_h_p, w_p=z0_w_p, device=device, dtype=x.dtype),
seed=True,
)
frame_ids = torch.zeros((B, 1), dtype=torch.long, device=device)
visual_rope = self._record_rope(frame_ids=frame_ids, h_p=z0_h_p, w_p=z0_w_p, n_raw=z0_L)
x = self._run_blocks(x, cond, kv_cache=cache, kv_plan=cache.plan_z0(), visual_rope=visual_rope)
return self._decode_frame(x[:, :z0_L], self.head_z0, t_emb, z0_h_p, z0_w_p)
def forward(self, *args, **kwargs):
raise TypeError(
"Summaryformer has no generic forward. Use _forward_kv_cached_z0 / _forward_kv_cached_z_rest "
"via SummaryformerWrapper (forward_z0 / forward_body)."
)