"""Summaryformer visual KV cache: one buffer, two regions, two cursors, one dense prefix read. One ``SummaryKVCache`` per session, one entry per trunk layer. Per layer, ONE preallocated ``[B, cap, H, D]`` K and the matching V (flash layout):: REC = Tf + X one RECORD = raw(t) ++ sum(t) REC_z0 = Z + X (0 when z0_tokens=0) z0's record; no Z0 region and no sink when Z = 0 | Z0 : REC_z0 | RAW : (Y+1) slots x REC, rotating | SUM : (W-Y) slots x X, rotating | read = [0 : end] cap = REC_z0 + (Y+1)*REC + (W-Y)*X end = REC_z0 + n_raw*REC + filled_s*X * ``Z0`` holds frame 0's whole record (the sink). Written by the z0 pass, never evicted. ``streaming_sink`` rotates this ONE slot's K (RoPE, temporal band only). ``z0_tokens=0`` at ``build`` removes the region. * ``RAW`` holds the CURRENT record and the Y history records. There is no separate tail. Every denoising pass of frame t writes ``record(t)`` at ``raw_slot_start(cursor_r)`` (constant within the frame). Commit is ``cursor_r += 1 (mod Y+1)``, ``filled_r = min(filled_r+1, Y+1)``. No copy on commit. * ``SUM`` is fed by eviction. ``begin_frame`` — before the first write of a frame — if ``cursor_r`` holds a committed record (``filled_r == Y+1``) and ``W > Y``, copies that slot's X summary rows (``[Tf : Tf+X]``) into SUM slot ``cursor_s``, then ``cursor_s += 1 (mod W-Y)``, ``filled_s = min(filled_s+1, W-Y)``. When ``W == Y`` there is no SUM region and that donation is skipped (no copy, cursor stays). One X-row copy per layer per frame otherwise. Nothing is evicted before RAW is full, so the live prefix is contiguous through warm-up: ``Z0 ++ RAW[0 : n_raw*REC] ++ SUM[0 : filled_s*X]``. ``n_raw`` counts RAW slots that hold valid keys, including the slot being written this frame. * ``end`` is a Python int on the eager plan and one persistent 0-d int64 device tensor (``end_dev``, updated in place) for a compiled forward. Every query of the frame — the raw rows and the X summary rows — reads that same prefix: one ``attend_buffer_prefix`` per layer, ``Lq = REC``. * Slot order inside a region does not matter (K carries absolute-id RoPE). No duplicate keys: a frame inside the raw window is its whole record, a frame beyond it is its X summary rows, frame 0 is the sink record. Lifecycle (K denoising passes per frame, then the caller commits):: z0 : plan = cache.plan_z0() each pass writes record(0) into Z0 and reads [0 : REC_z0] after the clean z0 warm the caller calls cache.note_z0_written() body : cache.begin_frame() donation, once, before the first write (idempotent until commit) plan = cache.plan_body() write offset + end; also calls begin_frame each pass writes the whole record at plan.write and reads [0 : end] "recache": one more t=0 pass rewrites that same RAW slot with the clean record, then commit "last_step_writes": commit the slot the last pass already wrote (noisy x_t); no extra pass commit_current_frame() advances cursor_r. No copy. ``plan_body()`` returns ``end`` as a Python int. ``plan_body(device_ends=True)`` returns ``end_dev``. The write offset is always an int (the slot does not move during the frame). Recency bias is OFF unless ``enable_recency`` is called. The hook is the existing per-key additive bias over this one buffer (zeros on Z0 and RAW, values on SUM rows). No new kernel: the block passes it on the single prefix read. ``begin_frame`` refreshes Δ after the donation updates the slot→frame map (a warm-up frame with nothing donated still refreshes; the SUM rows stay 0). No mask. No ``torch.cat`` on this path: donation and the block's write are slice copies into storage allocated in ``build``. """ from __future__ import annotations from typing import NamedTuple, Optional import torch from summaryformer_utils.recency_bias import RECENCY_BIAS_KEY, RecencyBiasConfig, RecencyBiasState class SummaryKVPlan(NamedTuple): """What ONE KV-cached forward of ONE frame does to / reads from the buffer. Built by ``SummaryKVCache.plan_z0`` / ``plan_body``. Consumed by ``SelfAttention._attend_with_cache``. One write offset, one end, ``query_tokens`` queries (the whole record). """ kind: str # "z0" | "body" raw_tokens: int # raw rows in this frame's record (Z for the z0 pass, Tf for a body pass) summary_tokens: int # X write: int # row where the WHOLE record [raw | sum] is written end: "int | torch.Tensor" # every query attends buffer[0 : end] # Plain FIELDS, not @property: dynamo traces NamedTuple fields as constants but turns a property on a # NamedTuple into an opaque GetAttrVariable, so `int(kv_plan.query_tokens)` graph-broke once per block in the # compiled body. plan_z0 / plan_body fill them. query_tokens: int # raw_tokens + summary_tokens (the record width every block must receive) ends_are_device: bool # isinstance(end, torch.Tensor): the compiled streaming form class SummaryKVCache(torch.nn.Module): """One-buffer visual KV cache for the Summaryformer trunk (module docstring has the layout and the lifecycle). Geometry comes from the MODEL (``summary_tokens`` = X, ``raw_window_frames`` = Y, ``summary_window_frames`` = W, ``num_layers``, heads, head_dim). ``W`` is the total memory span and must be ``>= Y`` (``W == Y`` is an empty SUM region). ``build`` takes the only per-bucket number, ``frame_tokens`` = Tf (and ``z0_tokens``, default Tf). """ def __init__(self, model) -> None: super().__init__() for attr in ("summary_tokens", "raw_window_frames", "summary_window_frames", "num_layers", "num_heads", "dim"): if not hasattr(model, attr): raise TypeError(f"SummaryKVCache(model): model lacks `{attr}` (expected Summaryformer).") self.summary_tokens = int(model.summary_tokens) # X self.raw_window_frames = int(model.raw_window_frames) # Y self.summary_window_frames = int(model.summary_window_frames) # W, total span, raw window included self.num_layers = int(model.num_layers) self.num_heads = int(model.num_heads) self.head_dim = int(model.dim) // int(model.num_heads) if self.summary_tokens < 1: raise ValueError(f"summary_tokens must be >= 1; got {self.summary_tokens}.") if self.raw_window_frames < 0 or self.summary_window_frames < 0: raise ValueError( f"raw_window_frames / summary_window_frames must be >= 0; got " f"{self.raw_window_frames} / {self.summary_window_frames}." ) if self.summary_window_frames < self.raw_window_frames: raise ValueError( f"summary_window_frames={self.summary_window_frames} < raw_window_frames={self.raw_window_frames}: " "W is the total memory span, raw window included." ) self.frame_tokens: Optional[int] = None # Tf self.z0_tokens: Optional[int] = None # Z (0 = no sink) self.batch: Optional[int] = None self.capacity: Optional[int] = None # Per-layer [{"k","v"[, RECENCY_BIAS_KEY]}]. Allocated in build(). self.layers: Optional[list] = None self.cursor_r = 0 self.cursor_s = 0 self.filled_r = 0 # committed records currently sitting in RAW (0 .. Y+1) self.filled_s = 0 # summaries currently sitting in SUM (0 .. W-Y) self.n_committed = 0 # body frames committed this session (recency Δ; not a slot count) self.z0_written = False self._body_open = False # True after begin_frame, False after commit self.end_dev: Optional[torch.Tensor] = None self.recency: Optional[RecencyBiasState] = None # ------------------------------------------------------------------ # # Geometry. # ------------------------------------------------------------------ # def _require_built(self) -> None: if self.layers is None or self.frame_tokens is None or self.z0_tokens is None: raise RuntimeError("SummaryKVCache.build(...) must be called before use.") @property def record_tokens(self) -> int: """REC = Tf + X.""" self._require_built() return int(self.frame_tokens) + int(self.summary_tokens) @property def z0_record_tokens(self) -> int: """REC_z0 = Z + X, or 0 when built without a sink.""" self._require_built() return int(self.z0_tokens) + int(self.summary_tokens) if int(self.z0_tokens) > 0 else 0 @property def has_sink(self) -> bool: self._require_built() return int(self.z0_tokens) > 0 @property def n_raw_slots(self) -> int: """Y+1: the current record plus Y history records.""" return int(self.raw_window_frames) + 1 @property def n_sum_slots(self) -> int: """W−Y SUM-region slots. 0 when W == Y (nothing is evicted into SUM).""" return int(self.summary_window_frames) - int(self.raw_window_frames) @property def raw_region_start(self) -> int: self._require_built() return int(self.z0_record_tokens) @property def sum_region_start(self) -> int: self._require_built() return int(self.raw_region_start + self.n_raw_slots * self.record_tokens) def z0_span(self) -> tuple[int, int]: """``[start, end)`` of the one Z0 slot (empty when built without a sink).""" self._require_built() return 0, int(self.z0_record_tokens) def raw_slot_start(self, slot: int) -> int: slot = int(slot) if not 0 <= slot < self.n_raw_slots: raise IndexError(f"raw slot {slot} outside [0, {self.n_raw_slots}).") return int(self.raw_region_start + slot * self.record_tokens) def sum_slot_start(self, slot: int) -> int: slot = int(slot) if self.n_sum_slots <= 0 or not 0 <= slot < self.n_sum_slots: raise IndexError(f"summary slot {slot} outside [0, {self.n_sum_slots}).") return int(self.sum_region_start + slot * int(self.summary_tokens)) # ------------------------------------------------------------------ # # The one attended length. # ------------------------------------------------------------------ # def n_raw_live(self) -> int: """RAW slots that hold valid keys. Includes the slot being written while a body frame is open.""" self._require_built() if self._body_open: if int(self.filled_r) < self.n_raw_slots: return int(self.filled_r) + 1 return self.n_raw_slots return int(self.filled_r) def end_body(self) -> int: """``REC_z0 + n_raw*REC + filled_s*X``. During an open frame this is the prefix every query reads.""" self._require_built() return int( self.z0_record_tokens + self.n_raw_live() * self.record_tokens + int(self.filled_s) * int(self.summary_tokens) ) def _sync_end_dev(self) -> None: if isinstance(self.end_dev, torch.Tensor): self.end_dev.fill_(int(self.end_body())) def device_scalars(self) -> tuple[torch.Tensor, ...]: """``(end_dev,)``: the persistent 0-d device tensor (mark it static for cudagraphs).""" self._require_built() if not isinstance(self.end_dev, torch.Tensor): raise RuntimeError("SummaryKVCache end scalar missing: build() allocates it.") return (self.end_dev,) # ------------------------------------------------------------------ # # Plans. # ------------------------------------------------------------------ # def plan_z0(self) -> SummaryKVPlan: """Return the z0 plan: write offset 0 (the Z0 slot), every query attends ``[0 : REC_z0]``. Does not write K/V and does not set ``z0_written`` (the caller calls ``note_z0_written`` after the clean warm). Refuses a cache built with ``z0_tokens=0``. """ self._require_built() if not self.has_sink: raise RuntimeError("plan_z0 on a cache built with z0_tokens=0 (no sink).") rec0 = int(self.z0_record_tokens) return SummaryKVPlan( kind="z0", raw_tokens=int(self.z0_tokens), summary_tokens=int(self.summary_tokens), write=0, end=rec0, query_tokens=int(self.z0_tokens) + int(self.summary_tokens), ends_are_device=False, ) def plan_body(self, *, device_ends: bool = False) -> SummaryKVPlan: """Open the frame (``begin_frame``, once) and return the body plan. This method does not write K/V. The write offset is the current RAW slot; the block writes ``record(t)`` there. Every query reads ``[0 : end]``. ``device_ends=True`` hands out ``end_dev`` (compiled streaming; ``begin_frame`` has already filled it). Refuses to run before the sink is written when the cache has one. """ self.begin_frame() end = self.end_dev if device_ends else self.end_body() return SummaryKVPlan( kind="body", raw_tokens=int(self.frame_tokens), summary_tokens=int(self.summary_tokens), write=self.raw_slot_start(int(self.cursor_r)), end=end, query_tokens=int(self.frame_tokens) + int(self.summary_tokens), ends_are_device=bool(device_ends), ) # ------------------------------------------------------------------ # # Lifecycle: z0 warm -> begin_frame donation -> body passes -> commit (cursor advance). # ------------------------------------------------------------------ # def note_z0_written(self) -> None: """Mark the clean z0 record present in Z0. Call after the clean z0 warm, before any body frame. A second call only leaves the flag set. Refuses a cache built with ``z0_tokens=0``. """ self._require_built() if not self.has_sink: raise RuntimeError("note_z0_written on a cache built with z0_tokens=0.") self.z0_written = True def begin_frame(self) -> None: """Donate the RAW slot about to be overwritten, once, before this frame's first write. Idempotent until ``commit_current_frame``: a second call returns before any copy or cursor move. When ``filled_r == Y+1`` the slot at ``cursor_r`` holds a committed record. If ``W > Y`` its X summary rows are copied into SUM slot ``cursor_s`` and the SUM cursor advances; when ``W == Y`` that donation is skipped entirely (no copy, cursor stays). Then ``end_dev`` is the length of the prefix this frame's passes will read, including the slot about to be written and, when a donation happened, the new summary. """ self._require_built() if self._body_open: return if self.has_sink and not self.z0_written: raise RuntimeError( "begin_frame before the sink exists: run the z0 pass and call note_z0_written() first " "(or build with z0_tokens=0 for a sink-less stream)." ) if int(self.filled_r) == self.n_raw_slots and self.n_sum_slots > 0: tf = int(self.frame_tokens) x = int(self.summary_tokens) rs = self.raw_slot_start(int(self.cursor_r)) slot_s = int(self.cursor_s) ss = self.sum_slot_start(slot_s) for layer in self.layers: layer["k"][:, ss:ss + x].copy_(layer["k"][:, rs + tf:rs + tf + x]) layer["v"][:, ss:ss + x].copy_(layer["v"][:, rs + tf:rs + tf + x]) if self.recency is not None: # 0-based index of the body frame leaving RAW. Query frame is n_committed (not yet incremented). self.recency.slot_frame[slot_s] = int(self.n_committed) - self.n_raw_slots self.filled_s = min(int(self.filled_s) + 1, self.n_sum_slots) self.cursor_s = int((slot_s + 1) % self.n_sum_slots) self._body_open = True if self.recency is not None: self.recency.n_committed = int(self.n_committed) self.recency.refresh() self._sync_end_dev() def commit_current_frame(self) -> None: """Advance the RAW cursor. No copy: the passes already wrote the record into its slot. ``cursor_r = (cursor_r + 1) mod (Y+1)``, ``filled_r = min(filled_r+1, Y+1)``, ``n_committed += 1``, and the frame closes (``_body_open`` false), so a second commit raises until the next ``begin_frame``. The frame must already be open (``begin_frame`` / ``plan_body``) so a commit cannot skip the donation. """ self._require_built() if not self._body_open: raise RuntimeError( "commit_current_frame before begin_frame: open the frame (plan_body / begin_frame) so an " "evicted slot can donate its summary rows before the cursor moves." ) self.cursor_r = int((int(self.cursor_r) + 1) % self.n_raw_slots) self.filled_r = min(int(self.filled_r) + 1, self.n_raw_slots) self.n_committed = int(self.n_committed) + 1 self._body_open = False self._sync_end_dev() # ------------------------------------------------------------------ # # Recency hook (OFF unless enable_recency is called). Existing kernels, no new one. # ------------------------------------------------------------------ # def enable_recency(self, cfg) -> Optional[RecencyBiasState]: """Per-key additive logit bias over the one buffer. Zeros on Z0 and RAW; values on SUM rows. ``cfg``: RecencyBiasConfig | dict | None (None disables). Call AFTER ``build()``; the config survives ``reset()``. Fails if ``decay_end`` exceeds ``W + 1`` (a summary leaves SUM at Δ = W+1). Refuses when a body frame is open or any body frame has been committed, or the SUM cursor has moved; a z0 sink already written is allowed (SUM is still empty). The block adds ``layer["recency_bias"]`` on the single prefix read; the flex kernel takes it, the flash kvcache kernel does not (a biased read is always flex). """ self._require_built() cfg = RecencyBiasConfig.from_dict(cfg) if cfg is None: self.disable_recency() return None w = int(self.summary_window_frames) if int(cfg.decay_end) > w + 1: raise RuntimeError( f"recency.decay_end={cfg.decay_end} > W + 1 = {w + 1}: a summary is evicted from SUM " f"(Δ={w + 1}) before the bias reaches its floor; set decay_end <= {w + 1}" ) if int(self.filled_s) != 0 or int(self.cursor_s) != 0 or self._body_open or int(self.n_committed) != 0: raise RuntimeError("enable_recency must be called before any body frame is opened or committed and before any SUM donation (a written z0 is fine)") buf = self.layers[0]["k"] cap, device = int(buf.size(1)), buf.device reuse = None if ( self.recency is not None and tuple(self.recency.token_bias.shape) == (cap,) and self.recency.token_bias.device == device ): reuse = self.recency.token_bias self.recency = RecencyBiasState( cfg, ring_frames=int(self.n_sum_slots), ring_start=int(self.sum_region_start), frame_tokens=int(self.summary_tokens), capacity=cap, device=device, token_bias=reuse, ) self._attach_recency() return self.recency def disable_recency(self) -> None: self.recency = None if isinstance(self.layers, list): for layer in self.layers: layer.pop(RECENCY_BIAS_KEY, None) def _attach_recency(self) -> None: if self.recency is None or not isinstance(self.layers, list): return for layer in self.layers: layer[RECENCY_BIAS_KEY] = self.recency.token_bias # ------------------------------------------------------------------ # # Allocation / reset. # ------------------------------------------------------------------ # def build( self, *, batch: int, frame_tokens: int, dtype: torch.dtype, device: torch.device, z0_tokens: int | None = None, allow_oversize_reuse: bool = False, ) -> None: """Allocate (or reuse) the one K/V buffer per layer and ``end_dev``; zero them; reset state. Args: frame_tokens: Tf, tokens per body frame at this bucket. z0_tokens: Z, tokens of the z0 frame (default Tf). 0 = no Z0 region, no sink. allow_oversize_reuse: also reuse a buffer whose row axis is LONGER than ``cap`` (same B/H/D/dtype/ device). Every read is ``[0:end]`` and every write is at a geometry offset. Reuses existing storage when shape/dtype/device match so addresses stay stable (cudagraphs). """ frame_tokens = int(frame_tokens) z0_tokens = frame_tokens if z0_tokens is None else int(z0_tokens) if frame_tokens <= 0: raise ValueError(f"frame_tokens must be > 0; got {frame_tokens}.") if z0_tokens < 0: raise ValueError(f"z0_tokens must be >= 0; got {z0_tokens}.") x = int(self.summary_tokens) rec = frame_tokens + x rec0 = (z0_tokens + x) if z0_tokens > 0 else 0 n_raw = int(self.raw_window_frames) + 1 n_sum = int(self.summary_window_frames) - int(self.raw_window_frames) cap = int(rec0 + n_raw * rec + n_sum * x) b, h, d = int(batch), int(self.num_heads), int(self.head_dim) device = torch.device(device) def _reusable(have) -> bool: return ( isinstance(have, torch.Tensor) and have.dtype == dtype and have.device == device and int(have.dim()) == 4 and (int(have.size(0)), int(have.size(2)), int(have.size(3))) == (b, h, d) and (int(have.size(1)) == cap or (bool(allow_oversize_reuse) and int(have.size(1)) > cap)) ) have_layers = ( isinstance(self.layers, list) and len(self.layers) == int(self.num_layers) and all(isinstance(layer, dict) for layer in self.layers) ) can_reuse = have_layers and _reusable(self.layers[0].get("k")) and _reusable(self.layers[0].get("v")) self.frame_tokens = frame_tokens self.z0_tokens = z0_tokens self.batch = b self.capacity = cap if can_reuse: for layer in self.layers: layer["k"].zero_() layer["v"].zero_() else: self.layers = [ { "k": torch.zeros((b, cap, h, d), dtype=dtype, device=device), "v": torch.zeros((b, cap, h, d), dtype=dtype, device=device), } for _ in range(int(self.num_layers)) ] if not (isinstance(self.end_dev, torch.Tensor) and self.end_dev.device == device): self.end_dev = torch.zeros((), dtype=torch.long, device=device) self.reset_state() if self.recency is not None: self.enable_recency(self.recency.cfg) def reset_state(self) -> None: """Reset the mutable stream counters only (NOT the buffers).""" self.cursor_r = 0 self.cursor_s = 0 self.filled_r = 0 self.filled_s = 0 self.n_committed = 0 self.z0_written = False self._body_open = False if self.layers is not None and self.frame_tokens is not None and self.z0_tokens is not None: self._sync_end_dev() if self.recency is not None: self.recency.reset() def reset(self) -> None: """Zero the buffers in place and reset counters. Keeps addresses stable.""" if isinstance(self.layers, list): for layer in self.layers: layer["k"].zero_() layer["v"].zero_() self.reset_state() def release(self) -> None: """Drop the storages. The next ``build()`` allocates again.""" self.disable_recency() if isinstance(self.layers, list): for layer in self.layers: layer.clear() self.layers = None self.end_dev = None self.capacity = None def describe(self) -> str: self._require_built() return ( f"SummaryKVCache(L={self.num_layers} B={self.batch} H={self.num_heads} D={self.head_dim} " f"Tf={self.frame_tokens} Z={self.z0_tokens} X={self.summary_tokens} Y={self.raw_window_frames} " f"W={self.summary_window_frames} REC={self.record_tokens} cap={self.capacity} " f"| Z0 0:{self.raw_region_start} RAW {self.raw_region_start}:{self.sum_region_start} " f"({self.n_raw_slots}x{self.record_tokens}) SUM {self.sum_region_start}: " f"({self.n_sum_slots}x{self.summary_tokens}) | cursor_r={self.cursor_r} filled_r={self.filled_r} " f"cursor_s={self.cursor_s} filled_s={self.filled_s} n_raw_live={self.n_raw_live()} " f"end={self.end_body()} open={self._body_open} z0_written={self.z0_written})" )