gta / gta_streaming_kv_cache.py
abhisheksensharma's picture
Upload folder using huggingface_hub
ae4e95d verified
Raw History Blame Contribute Delete
25.7 kB
"""Summaryformer visual KV cache: one buffer, two regions, two cursors, one dense prefix read.
One ``SummaryKVCache`` per session, one entry per trunk layer.
Per layer, ONE preallocated ``[B, cap, H, D]`` K and the matching V (flash layout)::
REC = Tf + X one RECORD = raw(t) ++ sum(t)
REC_z0 = Z + X (0 when z0_tokens=0) z0's record; no Z0 region and no sink when Z = 0
| Z0 : REC_z0 | RAW : (Y+1) slots x REC, rotating | SUM : (W-Y) slots x X, rotating |
read = [0 : end]
cap = REC_z0 + (Y+1)*REC + (W-Y)*X
end = REC_z0 + n_raw*REC + filled_s*X
* ``Z0`` holds frame 0's whole record (the sink). Written by the z0 pass, never evicted. ``streaming_sink``
rotates this ONE slot's K (RoPE, temporal band only). ``z0_tokens=0`` at ``build`` removes the region.
* ``RAW`` holds the CURRENT record and the Y history records. There is no separate tail. Every denoising
pass of frame t writes ``record(t)`` at ``raw_slot_start(cursor_r)`` (constant within the frame).
Commit is ``cursor_r += 1 (mod Y+1)``, ``filled_r = min(filled_r+1, Y+1)``. No copy on commit.
* ``SUM`` is fed by eviction. ``begin_frame`` — before the first write of a frame — if ``cursor_r`` holds a
committed record (``filled_r == Y+1``) and ``W > Y``, copies that slot's X summary rows (``[Tf : Tf+X]``)
into SUM slot ``cursor_s``, then ``cursor_s += 1 (mod W-Y)``, ``filled_s = min(filled_s+1, W-Y)``. When
``W == Y`` there is no SUM region and that donation is skipped (no copy, cursor stays). One X-row copy per
layer per frame otherwise. Nothing is evicted before RAW is full, so the live prefix is contiguous through warm-up:
``Z0 ++ RAW[0 : n_raw*REC] ++ SUM[0 : filled_s*X]``. ``n_raw`` counts RAW slots that hold valid keys,
including the slot being written this frame.
* ``end`` is a Python int on the eager plan and one persistent 0-d int64 device tensor (``end_dev``, updated
in place) for a compiled forward. Every query of the frame — the raw rows and the X summary rows — reads
that same prefix: one ``attend_buffer_prefix`` per layer, ``Lq = REC``.
* Slot order inside a region does not matter (K carries absolute-id RoPE). No duplicate keys: a frame inside
the raw window is its whole record, a frame beyond it is its X summary rows, frame 0 is the sink record.
Lifecycle (K denoising passes per frame, then the caller commits)::
z0 : plan = cache.plan_z0()
each pass writes record(0) into Z0 and reads [0 : REC_z0]
after the clean z0 warm the caller calls cache.note_z0_written()
body : cache.begin_frame() donation, once, before the first write (idempotent until commit)
plan = cache.plan_body() write offset + end; also calls begin_frame
each pass writes the whole record at plan.write and reads [0 : end]
"recache": one more t=0 pass rewrites that same RAW slot with the clean record, then commit
"last_step_writes": commit the slot the last pass already wrote (noisy x_t); no extra pass
commit_current_frame() advances cursor_r. No copy.
``plan_body()`` returns ``end`` as a Python int. ``plan_body(device_ends=True)`` returns ``end_dev``.
The write offset is always an int (the slot does not move during the frame).
Recency bias is OFF unless ``enable_recency`` is called. The hook is the existing per-key additive bias
over this one buffer (zeros on Z0 and RAW, values on SUM rows). No new kernel: the block passes it on the
single prefix read. ``begin_frame`` refreshes Δ after the donation updates the slot→frame map (a warm-up
frame with nothing donated still refreshes; the SUM rows stay 0).
No mask. No ``torch.cat`` on this path: donation and the block's write are slice copies into
storage allocated in ``build``.
"""
from __future__ import annotations
from typing import NamedTuple, Optional
import torch
from summaryformer_utils.recency_bias import RECENCY_BIAS_KEY, RecencyBiasConfig, RecencyBiasState
class SummaryKVPlan(NamedTuple):
"""What ONE KV-cached forward of ONE frame does to / reads from the buffer.
Built by ``SummaryKVCache.plan_z0`` / ``plan_body``. Consumed by ``SelfAttention._attend_with_cache``.
One write offset, one end, ``query_tokens`` queries (the whole record).
"""
kind: str # "z0" | "body"
raw_tokens: int # raw rows in this frame's record (Z for the z0 pass, Tf for a body pass)
summary_tokens: int # X
write: int # row where the WHOLE record [raw | sum] is written
end: "int | torch.Tensor" # every query attends buffer[0 : end]
# Plain FIELDS, not @property: dynamo traces NamedTuple fields as constants but turns a property on a
# NamedTuple into an opaque GetAttrVariable, so `int(kv_plan.query_tokens)` graph-broke once per block in the
# compiled body. plan_z0 / plan_body fill them.
query_tokens: int # raw_tokens + summary_tokens (the record width every block must receive)
ends_are_device: bool # isinstance(end, torch.Tensor): the compiled streaming form
class SummaryKVCache(torch.nn.Module):
"""One-buffer visual KV cache for the Summaryformer trunk (module docstring has the layout and the lifecycle).
Geometry comes from the MODEL (``summary_tokens`` = X, ``raw_window_frames`` = Y, ``summary_window_frames`` = W,
``num_layers``, heads, head_dim). ``W`` is the total memory span and must be ``>= Y`` (``W == Y`` is an empty
SUM region). ``build`` takes the only per-bucket number, ``frame_tokens`` = Tf (and ``z0_tokens``, default Tf).
"""
def __init__(self, model) -> None:
super().__init__()
for attr in ("summary_tokens", "raw_window_frames", "summary_window_frames", "num_layers", "num_heads", "dim"):
if not hasattr(model, attr):
raise TypeError(f"SummaryKVCache(model): model lacks `{attr}` (expected Summaryformer).")
self.summary_tokens = int(model.summary_tokens) # X
self.raw_window_frames = int(model.raw_window_frames) # Y
self.summary_window_frames = int(model.summary_window_frames) # W, total span, raw window included
self.num_layers = int(model.num_layers)
self.num_heads = int(model.num_heads)
self.head_dim = int(model.dim) // int(model.num_heads)
if self.summary_tokens < 1:
raise ValueError(f"summary_tokens must be >= 1; got {self.summary_tokens}.")
if self.raw_window_frames < 0 or self.summary_window_frames < 0:
raise ValueError(
f"raw_window_frames / summary_window_frames must be >= 0; got "
f"{self.raw_window_frames} / {self.summary_window_frames}."
)
if self.summary_window_frames < self.raw_window_frames:
raise ValueError(
f"summary_window_frames={self.summary_window_frames} < raw_window_frames={self.raw_window_frames}: "
"W is the total memory span, raw window included."
)
self.frame_tokens: Optional[int] = None # Tf
self.z0_tokens: Optional[int] = None # Z (0 = no sink)
self.batch: Optional[int] = None
self.capacity: Optional[int] = None
# Per-layer [{"k","v"[, RECENCY_BIAS_KEY]}]. Allocated in build().
self.layers: Optional[list] = None
self.cursor_r = 0
self.cursor_s = 0
self.filled_r = 0 # committed records currently sitting in RAW (0 .. Y+1)
self.filled_s = 0 # summaries currently sitting in SUM (0 .. W-Y)
self.n_committed = 0 # body frames committed this session (recency Δ; not a slot count)
self.z0_written = False
self._body_open = False # True after begin_frame, False after commit
self.end_dev: Optional[torch.Tensor] = None
self.recency: Optional[RecencyBiasState] = None
# ------------------------------------------------------------------ #
# Geometry.
# ------------------------------------------------------------------ #
def _require_built(self) -> None:
if self.layers is None or self.frame_tokens is None or self.z0_tokens is None:
raise RuntimeError("SummaryKVCache.build(...) must be called before use.")
@property
def record_tokens(self) -> int:
"""REC = Tf + X."""
self._require_built()
return int(self.frame_tokens) + int(self.summary_tokens)
@property
def z0_record_tokens(self) -> int:
"""REC_z0 = Z + X, or 0 when built without a sink."""
self._require_built()
return int(self.z0_tokens) + int(self.summary_tokens) if int(self.z0_tokens) > 0 else 0
@property
def has_sink(self) -> bool:
self._require_built()
return int(self.z0_tokens) > 0
@property
def n_raw_slots(self) -> int:
"""Y+1: the current record plus Y history records."""
return int(self.raw_window_frames) + 1
@property
def n_sum_slots(self) -> int:
"""W−Y SUM-region slots. 0 when W == Y (nothing is evicted into SUM)."""
return int(self.summary_window_frames) - int(self.raw_window_frames)
@property
def raw_region_start(self) -> int:
self._require_built()
return int(self.z0_record_tokens)
@property
def sum_region_start(self) -> int:
self._require_built()
return int(self.raw_region_start + self.n_raw_slots * self.record_tokens)
def z0_span(self) -> tuple[int, int]:
"""``[start, end)`` of the one Z0 slot (empty when built without a sink)."""
self._require_built()
return 0, int(self.z0_record_tokens)
def raw_slot_start(self, slot: int) -> int:
slot = int(slot)
if not 0 <= slot < self.n_raw_slots:
raise IndexError(f"raw slot {slot} outside [0, {self.n_raw_slots}).")
return int(self.raw_region_start + slot * self.record_tokens)
def sum_slot_start(self, slot: int) -> int:
slot = int(slot)
if self.n_sum_slots <= 0 or not 0 <= slot < self.n_sum_slots:
raise IndexError(f"summary slot {slot} outside [0, {self.n_sum_slots}).")
return int(self.sum_region_start + slot * int(self.summary_tokens))
# ------------------------------------------------------------------ #
# The one attended length.
# ------------------------------------------------------------------ #
def n_raw_live(self) -> int:
"""RAW slots that hold valid keys. Includes the slot being written while a body frame is open."""
self._require_built()
if self._body_open:
if int(self.filled_r) < self.n_raw_slots:
return int(self.filled_r) + 1
return self.n_raw_slots
return int(self.filled_r)
def end_body(self) -> int:
"""``REC_z0 + n_raw*REC + filled_s*X``. During an open frame this is the prefix every query reads."""
self._require_built()
return int(
self.z0_record_tokens
+ self.n_raw_live() * self.record_tokens
+ int(self.filled_s) * int(self.summary_tokens)
)
def _sync_end_dev(self) -> None:
if isinstance(self.end_dev, torch.Tensor):
self.end_dev.fill_(int(self.end_body()))
def device_scalars(self) -> tuple[torch.Tensor, ...]:
"""``(end_dev,)``: the persistent 0-d device tensor (mark it static for cudagraphs)."""
self._require_built()
if not isinstance(self.end_dev, torch.Tensor):
raise RuntimeError("SummaryKVCache end scalar missing: build() allocates it.")
return (self.end_dev,)
# ------------------------------------------------------------------ #
# Plans.
# ------------------------------------------------------------------ #
def plan_z0(self) -> SummaryKVPlan:
"""Return the z0 plan: write offset 0 (the Z0 slot), every query attends ``[0 : REC_z0]``.
Does not write K/V and does not set ``z0_written`` (the caller calls ``note_z0_written``
after the clean warm). Refuses a cache built with ``z0_tokens=0``.
"""
self._require_built()
if not self.has_sink:
raise RuntimeError("plan_z0 on a cache built with z0_tokens=0 (no sink).")
rec0 = int(self.z0_record_tokens)
return SummaryKVPlan(
kind="z0",
raw_tokens=int(self.z0_tokens),
summary_tokens=int(self.summary_tokens),
write=0,
end=rec0,
query_tokens=int(self.z0_tokens) + int(self.summary_tokens),
ends_are_device=False,
)
def plan_body(self, *, device_ends: bool = False) -> SummaryKVPlan:
"""Open the frame (``begin_frame``, once) and return the body plan. This method does not write K/V.
The write offset is the current RAW slot; the block writes ``record(t)`` there. Every query reads
``[0 : end]``. ``device_ends=True`` hands out ``end_dev`` (compiled streaming; ``begin_frame`` has
already filled it). Refuses to run before the sink is written when the cache has one.
"""
self.begin_frame()
end = self.end_dev if device_ends else self.end_body()
return SummaryKVPlan(
kind="body",
raw_tokens=int(self.frame_tokens),
summary_tokens=int(self.summary_tokens),
write=self.raw_slot_start(int(self.cursor_r)),
end=end,
query_tokens=int(self.frame_tokens) + int(self.summary_tokens),
ends_are_device=bool(device_ends),
)
# ------------------------------------------------------------------ #
# Lifecycle: z0 warm -> begin_frame donation -> body passes -> commit (cursor advance).
# ------------------------------------------------------------------ #
def note_z0_written(self) -> None:
"""Mark the clean z0 record present in Z0. Call after the clean z0 warm, before any body frame.
A second call only leaves the flag set. Refuses a cache built with ``z0_tokens=0``.
"""
self._require_built()
if not self.has_sink:
raise RuntimeError("note_z0_written on a cache built with z0_tokens=0.")
self.z0_written = True
def begin_frame(self) -> None:
"""Donate the RAW slot about to be overwritten, once, before this frame's first write.
Idempotent until ``commit_current_frame``: a second call returns before any copy or cursor move.
When ``filled_r == Y+1`` the slot at ``cursor_r`` holds a committed record. If ``W > Y`` its X summary
rows are copied into SUM slot ``cursor_s`` and the SUM cursor advances; when ``W == Y`` that donation
is skipped entirely (no copy, cursor stays). Then ``end_dev`` is the length of the prefix this frame's
passes will read, including the slot about to be written and, when a donation happened, the new summary.
"""
self._require_built()
if self._body_open:
return
if self.has_sink and not self.z0_written:
raise RuntimeError(
"begin_frame before the sink exists: run the z0 pass and call note_z0_written() first "
"(or build with z0_tokens=0 for a sink-less stream)."
)
if int(self.filled_r) == self.n_raw_slots and self.n_sum_slots > 0:
tf = int(self.frame_tokens)
x = int(self.summary_tokens)
rs = self.raw_slot_start(int(self.cursor_r))
slot_s = int(self.cursor_s)
ss = self.sum_slot_start(slot_s)
for layer in self.layers:
layer["k"][:, ss:ss + x].copy_(layer["k"][:, rs + tf:rs + tf + x])
layer["v"][:, ss:ss + x].copy_(layer["v"][:, rs + tf:rs + tf + x])
if self.recency is not None:
# 0-based index of the body frame leaving RAW. Query frame is n_committed (not yet incremented).
self.recency.slot_frame[slot_s] = int(self.n_committed) - self.n_raw_slots
self.filled_s = min(int(self.filled_s) + 1, self.n_sum_slots)
self.cursor_s = int((slot_s + 1) % self.n_sum_slots)
self._body_open = True
if self.recency is not None:
self.recency.n_committed = int(self.n_committed)
self.recency.refresh()
self._sync_end_dev()
def commit_current_frame(self) -> None:
"""Advance the RAW cursor. No copy: the passes already wrote the record into its slot.
``cursor_r = (cursor_r + 1) mod (Y+1)``, ``filled_r = min(filled_r+1, Y+1)``, ``n_committed += 1``,
and the frame closes (``_body_open`` false), so a second commit raises until the next ``begin_frame``.
The frame must already be open (``begin_frame`` / ``plan_body``) so a commit cannot skip the donation.
"""
self._require_built()
if not self._body_open:
raise RuntimeError(
"commit_current_frame before begin_frame: open the frame (plan_body / begin_frame) so an "
"evicted slot can donate its summary rows before the cursor moves."
)
self.cursor_r = int((int(self.cursor_r) + 1) % self.n_raw_slots)
self.filled_r = min(int(self.filled_r) + 1, self.n_raw_slots)
self.n_committed = int(self.n_committed) + 1
self._body_open = False
self._sync_end_dev()
# ------------------------------------------------------------------ #
# Recency hook (OFF unless enable_recency is called). Existing kernels, no new one.
# ------------------------------------------------------------------ #
def enable_recency(self, cfg) -> Optional[RecencyBiasState]:
"""Per-key additive logit bias over the one buffer. Zeros on Z0 and RAW; values on SUM rows.
``cfg``: RecencyBiasConfig | dict | None (None disables). Call AFTER ``build()``; the config survives
``reset()``. Fails if ``decay_end`` exceeds ``W + 1`` (a summary leaves SUM at Δ = W+1). Refuses when
a body frame is open or any body frame has been committed, or the SUM cursor has moved; a z0 sink
already written is allowed (SUM is still empty). The block adds ``layer["recency_bias"]`` on the single
prefix read; the flex kernel takes it, the flash kvcache kernel does not (a biased read is always flex).
"""
self._require_built()
cfg = RecencyBiasConfig.from_dict(cfg)
if cfg is None:
self.disable_recency()
return None
w = int(self.summary_window_frames)
if int(cfg.decay_end) > w + 1:
raise RuntimeError(
f"recency.decay_end={cfg.decay_end} > W + 1 = {w + 1}: a summary is evicted from SUM "
f"(Δ={w + 1}) before the bias reaches its floor; set decay_end <= {w + 1}"
)
if int(self.filled_s) != 0 or int(self.cursor_s) != 0 or self._body_open or int(self.n_committed) != 0:
raise RuntimeError("enable_recency must be called before any body frame is opened or committed and before any SUM donation (a written z0 is fine)")
buf = self.layers[0]["k"]
cap, device = int(buf.size(1)), buf.device
reuse = None
if (
self.recency is not None
and tuple(self.recency.token_bias.shape) == (cap,)
and self.recency.token_bias.device == device
):
reuse = self.recency.token_bias
self.recency = RecencyBiasState(
cfg,
ring_frames=int(self.n_sum_slots),
ring_start=int(self.sum_region_start),
frame_tokens=int(self.summary_tokens),
capacity=cap,
device=device,
token_bias=reuse,
)
self._attach_recency()
return self.recency
def disable_recency(self) -> None:
self.recency = None
if isinstance(self.layers, list):
for layer in self.layers:
layer.pop(RECENCY_BIAS_KEY, None)
def _attach_recency(self) -> None:
if self.recency is None or not isinstance(self.layers, list):
return
for layer in self.layers:
layer[RECENCY_BIAS_KEY] = self.recency.token_bias
# ------------------------------------------------------------------ #
# Allocation / reset.
# ------------------------------------------------------------------ #
def build(
self,
*,
batch: int,
frame_tokens: int,
dtype: torch.dtype,
device: torch.device,
z0_tokens: int | None = None,
allow_oversize_reuse: bool = False,
) -> None:
"""Allocate (or reuse) the one K/V buffer per layer and ``end_dev``; zero them; reset state.
Args:
frame_tokens: Tf, tokens per body frame at this bucket.
z0_tokens: Z, tokens of the z0 frame (default Tf). 0 = no Z0 region, no sink.
allow_oversize_reuse: also reuse a buffer whose row axis is LONGER than ``cap`` (same B/H/D/dtype/
device). Every read is ``[0:end]`` and every write is at a geometry offset.
Reuses existing storage when shape/dtype/device match so addresses stay stable (cudagraphs).
"""
frame_tokens = int(frame_tokens)
z0_tokens = frame_tokens if z0_tokens is None else int(z0_tokens)
if frame_tokens <= 0:
raise ValueError(f"frame_tokens must be > 0; got {frame_tokens}.")
if z0_tokens < 0:
raise ValueError(f"z0_tokens must be >= 0; got {z0_tokens}.")
x = int(self.summary_tokens)
rec = frame_tokens + x
rec0 = (z0_tokens + x) if z0_tokens > 0 else 0
n_raw = int(self.raw_window_frames) + 1
n_sum = int(self.summary_window_frames) - int(self.raw_window_frames)
cap = int(rec0 + n_raw * rec + n_sum * x)
b, h, d = int(batch), int(self.num_heads), int(self.head_dim)
device = torch.device(device)
def _reusable(have) -> bool:
return (
isinstance(have, torch.Tensor)
and have.dtype == dtype
and have.device == device
and int(have.dim()) == 4
and (int(have.size(0)), int(have.size(2)), int(have.size(3))) == (b, h, d)
and (int(have.size(1)) == cap or (bool(allow_oversize_reuse) and int(have.size(1)) > cap))
)
have_layers = (
isinstance(self.layers, list)
and len(self.layers) == int(self.num_layers)
and all(isinstance(layer, dict) for layer in self.layers)
)
can_reuse = have_layers and _reusable(self.layers[0].get("k")) and _reusable(self.layers[0].get("v"))
self.frame_tokens = frame_tokens
self.z0_tokens = z0_tokens
self.batch = b
self.capacity = cap
if can_reuse:
for layer in self.layers:
layer["k"].zero_()
layer["v"].zero_()
else:
self.layers = [
{
"k": torch.zeros((b, cap, h, d), dtype=dtype, device=device),
"v": torch.zeros((b, cap, h, d), dtype=dtype, device=device),
}
for _ in range(int(self.num_layers))
]
if not (isinstance(self.end_dev, torch.Tensor) and self.end_dev.device == device):
self.end_dev = torch.zeros((), dtype=torch.long, device=device)
self.reset_state()
if self.recency is not None:
self.enable_recency(self.recency.cfg)
def reset_state(self) -> None:
"""Reset the mutable stream counters only (NOT the buffers)."""
self.cursor_r = 0
self.cursor_s = 0
self.filled_r = 0
self.filled_s = 0
self.n_committed = 0
self.z0_written = False
self._body_open = False
if self.layers is not None and self.frame_tokens is not None and self.z0_tokens is not None:
self._sync_end_dev()
if self.recency is not None:
self.recency.reset()
def reset(self) -> None:
"""Zero the buffers in place and reset counters. Keeps addresses stable."""
if isinstance(self.layers, list):
for layer in self.layers:
layer["k"].zero_()
layer["v"].zero_()
self.reset_state()
def release(self) -> None:
"""Drop the storages. The next ``build()`` allocates again."""
self.disable_recency()
if isinstance(self.layers, list):
for layer in self.layers:
layer.clear()
self.layers = None
self.end_dev = None
self.capacity = None
def describe(self) -> str:
self._require_built()
return (
f"SummaryKVCache(L={self.num_layers} B={self.batch} H={self.num_heads} D={self.head_dim} "
f"Tf={self.frame_tokens} Z={self.z0_tokens} X={self.summary_tokens} Y={self.raw_window_frames} "
f"W={self.summary_window_frames} REC={self.record_tokens} cap={self.capacity} "
f"| Z0 0:{self.raw_region_start} RAW {self.raw_region_start}:{self.sum_region_start} "
f"({self.n_raw_slots}x{self.record_tokens}) SUM {self.sum_region_start}: "
f"({self.n_sum_slots}x{self.summary_tokens}) | cursor_r={self.cursor_r} filled_r={self.filled_r} "
f"cursor_s={self.cursor_s} filled_s={self.filled_s} n_raw_live={self.n_raw_live()} "
f"end={self.end_body()} open={self._body_open} z0_written={self.z0_written})"
)