Download gta_streaming_kv_cache.py from lucidml/gta: direct link, hf CLI and curl.
- Browser
- Download file 25.7 kB
-
https://huggingface.co/lucidml/gta/resolve/main/gta_streaming_kv_cache.py
- Command line
-
hf download hf://lucidml/gta/gta_streaming_kv_cache.py
-
curl -L -o gta_streaming_kv_cache.py https://huggingface.co/lucidml/gta/resolve/main/gta_streaming_kv_cache.py
25.7 kB
| """Summaryformer visual KV cache: one buffer, two regions, two cursors, one dense prefix read. | |
| One ``SummaryKVCache`` per session, one entry per trunk layer. | |
| Per layer, ONE preallocated ``[B, cap, H, D]`` K and the matching V (flash layout):: | |
| REC = Tf + X one RECORD = raw(t) ++ sum(t) | |
| REC_z0 = Z + X (0 when z0_tokens=0) z0's record; no Z0 region and no sink when Z = 0 | |
| | Z0 : REC_z0 | RAW : (Y+1) slots x REC, rotating | SUM : (W-Y) slots x X, rotating | | |
| read = [0 : end] | |
| cap = REC_z0 + (Y+1)*REC + (W-Y)*X | |
| end = REC_z0 + n_raw*REC + filled_s*X | |
| * ``Z0`` holds frame 0's whole record (the sink). Written by the z0 pass, never evicted. ``streaming_sink`` | |
| rotates this ONE slot's K (RoPE, temporal band only). ``z0_tokens=0`` at ``build`` removes the region. | |
| * ``RAW`` holds the CURRENT record and the Y history records. There is no separate tail. Every denoising | |
| pass of frame t writes ``record(t)`` at ``raw_slot_start(cursor_r)`` (constant within the frame). | |
| Commit is ``cursor_r += 1 (mod Y+1)``, ``filled_r = min(filled_r+1, Y+1)``. No copy on commit. | |
| * ``SUM`` is fed by eviction. ``begin_frame`` — before the first write of a frame — if ``cursor_r`` holds a | |
| committed record (``filled_r == Y+1``) and ``W > Y``, copies that slot's X summary rows (``[Tf : Tf+X]``) | |
| into SUM slot ``cursor_s``, then ``cursor_s += 1 (mod W-Y)``, ``filled_s = min(filled_s+1, W-Y)``. When | |
| ``W == Y`` there is no SUM region and that donation is skipped (no copy, cursor stays). One X-row copy per | |
| layer per frame otherwise. Nothing is evicted before RAW is full, so the live prefix is contiguous through warm-up: | |
| ``Z0 ++ RAW[0 : n_raw*REC] ++ SUM[0 : filled_s*X]``. ``n_raw`` counts RAW slots that hold valid keys, | |
| including the slot being written this frame. | |
| * ``end`` is a Python int on the eager plan and one persistent 0-d int64 device tensor (``end_dev``, updated | |
| in place) for a compiled forward. Every query of the frame — the raw rows and the X summary rows — reads | |
| that same prefix: one ``attend_buffer_prefix`` per layer, ``Lq = REC``. | |
| * Slot order inside a region does not matter (K carries absolute-id RoPE). No duplicate keys: a frame inside | |
| the raw window is its whole record, a frame beyond it is its X summary rows, frame 0 is the sink record. | |
| Lifecycle (K denoising passes per frame, then the caller commits):: | |
| z0 : plan = cache.plan_z0() | |
| each pass writes record(0) into Z0 and reads [0 : REC_z0] | |
| after the clean z0 warm the caller calls cache.note_z0_written() | |
| body : cache.begin_frame() donation, once, before the first write (idempotent until commit) | |
| plan = cache.plan_body() write offset + end; also calls begin_frame | |
| each pass writes the whole record at plan.write and reads [0 : end] | |
| "recache": one more t=0 pass rewrites that same RAW slot with the clean record, then commit | |
| "last_step_writes": commit the slot the last pass already wrote (noisy x_t); no extra pass | |
| commit_current_frame() advances cursor_r. No copy. | |
| ``plan_body()`` returns ``end`` as a Python int. ``plan_body(device_ends=True)`` returns ``end_dev``. | |
| The write offset is always an int (the slot does not move during the frame). | |
| Recency bias is OFF unless ``enable_recency`` is called. The hook is the existing per-key additive bias | |
| over this one buffer (zeros on Z0 and RAW, values on SUM rows). No new kernel: the block passes it on the | |
| single prefix read. ``begin_frame`` refreshes Δ after the donation updates the slot→frame map (a warm-up | |
| frame with nothing donated still refreshes; the SUM rows stay 0). | |
| No mask. No ``torch.cat`` on this path: donation and the block's write are slice copies into | |
| storage allocated in ``build``. | |
| """ | |
| from __future__ import annotations | |
| from typing import NamedTuple, Optional | |
| import torch | |
| from summaryformer_utils.recency_bias import RECENCY_BIAS_KEY, RecencyBiasConfig, RecencyBiasState | |
| class SummaryKVPlan(NamedTuple): | |
| """What ONE KV-cached forward of ONE frame does to / reads from the buffer. | |
| Built by ``SummaryKVCache.plan_z0`` / ``plan_body``. Consumed by ``SelfAttention._attend_with_cache``. | |
| One write offset, one end, ``query_tokens`` queries (the whole record). | |
| """ | |
| kind: str # "z0" | "body" | |
| raw_tokens: int # raw rows in this frame's record (Z for the z0 pass, Tf for a body pass) | |
| summary_tokens: int # X | |
| write: int # row where the WHOLE record [raw | sum] is written | |
| end: "int | torch.Tensor" # every query attends buffer[0 : end] | |
| # Plain FIELDS, not @property: dynamo traces NamedTuple fields as constants but turns a property on a | |
| # NamedTuple into an opaque GetAttrVariable, so `int(kv_plan.query_tokens)` graph-broke once per block in the | |
| # compiled body. plan_z0 / plan_body fill them. | |
| query_tokens: int # raw_tokens + summary_tokens (the record width every block must receive) | |
| ends_are_device: bool # isinstance(end, torch.Tensor): the compiled streaming form | |
| class SummaryKVCache(torch.nn.Module): | |
| """One-buffer visual KV cache for the Summaryformer trunk (module docstring has the layout and the lifecycle). | |
| Geometry comes from the MODEL (``summary_tokens`` = X, ``raw_window_frames`` = Y, ``summary_window_frames`` = W, | |
| ``num_layers``, heads, head_dim). ``W`` is the total memory span and must be ``>= Y`` (``W == Y`` is an empty | |
| SUM region). ``build`` takes the only per-bucket number, ``frame_tokens`` = Tf (and ``z0_tokens``, default Tf). | |
| """ | |
| def __init__(self, model) -> None: | |
| super().__init__() | |
| for attr in ("summary_tokens", "raw_window_frames", "summary_window_frames", "num_layers", "num_heads", "dim"): | |
| if not hasattr(model, attr): | |
| raise TypeError(f"SummaryKVCache(model): model lacks `{attr}` (expected Summaryformer).") | |
| self.summary_tokens = int(model.summary_tokens) # X | |
| self.raw_window_frames = int(model.raw_window_frames) # Y | |
| self.summary_window_frames = int(model.summary_window_frames) # W, total span, raw window included | |
| self.num_layers = int(model.num_layers) | |
| self.num_heads = int(model.num_heads) | |
| self.head_dim = int(model.dim) // int(model.num_heads) | |
| if self.summary_tokens < 1: | |
| raise ValueError(f"summary_tokens must be >= 1; got {self.summary_tokens}.") | |
| if self.raw_window_frames < 0 or self.summary_window_frames < 0: | |
| raise ValueError( | |
| f"raw_window_frames / summary_window_frames must be >= 0; got " | |
| f"{self.raw_window_frames} / {self.summary_window_frames}." | |
| ) | |
| if self.summary_window_frames < self.raw_window_frames: | |
| raise ValueError( | |
| f"summary_window_frames={self.summary_window_frames} < raw_window_frames={self.raw_window_frames}: " | |
| "W is the total memory span, raw window included." | |
| ) | |
| self.frame_tokens: Optional[int] = None # Tf | |
| self.z0_tokens: Optional[int] = None # Z (0 = no sink) | |
| self.batch: Optional[int] = None | |
| self.capacity: Optional[int] = None | |
| # Per-layer [{"k","v"[, RECENCY_BIAS_KEY]}]. Allocated in build(). | |
| self.layers: Optional[list] = None | |
| self.cursor_r = 0 | |
| self.cursor_s = 0 | |
| self.filled_r = 0 # committed records currently sitting in RAW (0 .. Y+1) | |
| self.filled_s = 0 # summaries currently sitting in SUM (0 .. W-Y) | |
| self.n_committed = 0 # body frames committed this session (recency Δ; not a slot count) | |
| self.z0_written = False | |
| self._body_open = False # True after begin_frame, False after commit | |
| self.end_dev: Optional[torch.Tensor] = None | |
| self.recency: Optional[RecencyBiasState] = None | |
| # ------------------------------------------------------------------ # | |
| # Geometry. | |
| # ------------------------------------------------------------------ # | |
| def _require_built(self) -> None: | |
| if self.layers is None or self.frame_tokens is None or self.z0_tokens is None: | |
| raise RuntimeError("SummaryKVCache.build(...) must be called before use.") | |
| def record_tokens(self) -> int: | |
| """REC = Tf + X.""" | |
| self._require_built() | |
| return int(self.frame_tokens) + int(self.summary_tokens) | |
| def z0_record_tokens(self) -> int: | |
| """REC_z0 = Z + X, or 0 when built without a sink.""" | |
| self._require_built() | |
| return int(self.z0_tokens) + int(self.summary_tokens) if int(self.z0_tokens) > 0 else 0 | |
| def has_sink(self) -> bool: | |
| self._require_built() | |
| return int(self.z0_tokens) > 0 | |
| def n_raw_slots(self) -> int: | |
| """Y+1: the current record plus Y history records.""" | |
| return int(self.raw_window_frames) + 1 | |
| def n_sum_slots(self) -> int: | |
| """W−Y SUM-region slots. 0 when W == Y (nothing is evicted into SUM).""" | |
| return int(self.summary_window_frames) - int(self.raw_window_frames) | |
| def raw_region_start(self) -> int: | |
| self._require_built() | |
| return int(self.z0_record_tokens) | |
| def sum_region_start(self) -> int: | |
| self._require_built() | |
| return int(self.raw_region_start + self.n_raw_slots * self.record_tokens) | |
| def z0_span(self) -> tuple[int, int]: | |
| """``[start, end)`` of the one Z0 slot (empty when built without a sink).""" | |
| self._require_built() | |
| return 0, int(self.z0_record_tokens) | |
| def raw_slot_start(self, slot: int) -> int: | |
| slot = int(slot) | |
| if not 0 <= slot < self.n_raw_slots: | |
| raise IndexError(f"raw slot {slot} outside [0, {self.n_raw_slots}).") | |
| return int(self.raw_region_start + slot * self.record_tokens) | |
| def sum_slot_start(self, slot: int) -> int: | |
| slot = int(slot) | |
| if self.n_sum_slots <= 0 or not 0 <= slot < self.n_sum_slots: | |
| raise IndexError(f"summary slot {slot} outside [0, {self.n_sum_slots}).") | |
| return int(self.sum_region_start + slot * int(self.summary_tokens)) | |
| # ------------------------------------------------------------------ # | |
| # The one attended length. | |
| # ------------------------------------------------------------------ # | |
| def n_raw_live(self) -> int: | |
| """RAW slots that hold valid keys. Includes the slot being written while a body frame is open.""" | |
| self._require_built() | |
| if self._body_open: | |
| if int(self.filled_r) < self.n_raw_slots: | |
| return int(self.filled_r) + 1 | |
| return self.n_raw_slots | |
| return int(self.filled_r) | |
| def end_body(self) -> int: | |
| """``REC_z0 + n_raw*REC + filled_s*X``. During an open frame this is the prefix every query reads.""" | |
| self._require_built() | |
| return int( | |
| self.z0_record_tokens | |
| + self.n_raw_live() * self.record_tokens | |
| + int(self.filled_s) * int(self.summary_tokens) | |
| ) | |
| def _sync_end_dev(self) -> None: | |
| if isinstance(self.end_dev, torch.Tensor): | |
| self.end_dev.fill_(int(self.end_body())) | |
| def device_scalars(self) -> tuple[torch.Tensor, ...]: | |
| """``(end_dev,)``: the persistent 0-d device tensor (mark it static for cudagraphs).""" | |
| self._require_built() | |
| if not isinstance(self.end_dev, torch.Tensor): | |
| raise RuntimeError("SummaryKVCache end scalar missing: build() allocates it.") | |
| return (self.end_dev,) | |
| # ------------------------------------------------------------------ # | |
| # Plans. | |
| # ------------------------------------------------------------------ # | |
| def plan_z0(self) -> SummaryKVPlan: | |
| """Return the z0 plan: write offset 0 (the Z0 slot), every query attends ``[0 : REC_z0]``. | |
| Does not write K/V and does not set ``z0_written`` (the caller calls ``note_z0_written`` | |
| after the clean warm). Refuses a cache built with ``z0_tokens=0``. | |
| """ | |
| self._require_built() | |
| if not self.has_sink: | |
| raise RuntimeError("plan_z0 on a cache built with z0_tokens=0 (no sink).") | |
| rec0 = int(self.z0_record_tokens) | |
| return SummaryKVPlan( | |
| kind="z0", | |
| raw_tokens=int(self.z0_tokens), | |
| summary_tokens=int(self.summary_tokens), | |
| write=0, | |
| end=rec0, | |
| query_tokens=int(self.z0_tokens) + int(self.summary_tokens), | |
| ends_are_device=False, | |
| ) | |
| def plan_body(self, *, device_ends: bool = False) -> SummaryKVPlan: | |
| """Open the frame (``begin_frame``, once) and return the body plan. This method does not write K/V. | |
| The write offset is the current RAW slot; the block writes ``record(t)`` there. Every query reads | |
| ``[0 : end]``. ``device_ends=True`` hands out ``end_dev`` (compiled streaming; ``begin_frame`` has | |
| already filled it). Refuses to run before the sink is written when the cache has one. | |
| """ | |
| self.begin_frame() | |
| end = self.end_dev if device_ends else self.end_body() | |
| return SummaryKVPlan( | |
| kind="body", | |
| raw_tokens=int(self.frame_tokens), | |
| summary_tokens=int(self.summary_tokens), | |
| write=self.raw_slot_start(int(self.cursor_r)), | |
| end=end, | |
| query_tokens=int(self.frame_tokens) + int(self.summary_tokens), | |
| ends_are_device=bool(device_ends), | |
| ) | |
| # ------------------------------------------------------------------ # | |
| # Lifecycle: z0 warm -> begin_frame donation -> body passes -> commit (cursor advance). | |
| # ------------------------------------------------------------------ # | |
| def note_z0_written(self) -> None: | |
| """Mark the clean z0 record present in Z0. Call after the clean z0 warm, before any body frame. | |
| A second call only leaves the flag set. Refuses a cache built with ``z0_tokens=0``. | |
| """ | |
| self._require_built() | |
| if not self.has_sink: | |
| raise RuntimeError("note_z0_written on a cache built with z0_tokens=0.") | |
| self.z0_written = True | |
| def begin_frame(self) -> None: | |
| """Donate the RAW slot about to be overwritten, once, before this frame's first write. | |
| Idempotent until ``commit_current_frame``: a second call returns before any copy or cursor move. | |
| When ``filled_r == Y+1`` the slot at ``cursor_r`` holds a committed record. If ``W > Y`` its X summary | |
| rows are copied into SUM slot ``cursor_s`` and the SUM cursor advances; when ``W == Y`` that donation | |
| is skipped entirely (no copy, cursor stays). Then ``end_dev`` is the length of the prefix this frame's | |
| passes will read, including the slot about to be written and, when a donation happened, the new summary. | |
| """ | |
| self._require_built() | |
| if self._body_open: | |
| return | |
| if self.has_sink and not self.z0_written: | |
| raise RuntimeError( | |
| "begin_frame before the sink exists: run the z0 pass and call note_z0_written() first " | |
| "(or build with z0_tokens=0 for a sink-less stream)." | |
| ) | |
| if int(self.filled_r) == self.n_raw_slots and self.n_sum_slots > 0: | |
| tf = int(self.frame_tokens) | |
| x = int(self.summary_tokens) | |
| rs = self.raw_slot_start(int(self.cursor_r)) | |
| slot_s = int(self.cursor_s) | |
| ss = self.sum_slot_start(slot_s) | |
| for layer in self.layers: | |
| layer["k"][:, ss:ss + x].copy_(layer["k"][:, rs + tf:rs + tf + x]) | |
| layer["v"][:, ss:ss + x].copy_(layer["v"][:, rs + tf:rs + tf + x]) | |
| if self.recency is not None: | |
| # 0-based index of the body frame leaving RAW. Query frame is n_committed (not yet incremented). | |
| self.recency.slot_frame[slot_s] = int(self.n_committed) - self.n_raw_slots | |
| self.filled_s = min(int(self.filled_s) + 1, self.n_sum_slots) | |
| self.cursor_s = int((slot_s + 1) % self.n_sum_slots) | |
| self._body_open = True | |
| if self.recency is not None: | |
| self.recency.n_committed = int(self.n_committed) | |
| self.recency.refresh() | |
| self._sync_end_dev() | |
| def commit_current_frame(self) -> None: | |
| """Advance the RAW cursor. No copy: the passes already wrote the record into its slot. | |
| ``cursor_r = (cursor_r + 1) mod (Y+1)``, ``filled_r = min(filled_r+1, Y+1)``, ``n_committed += 1``, | |
| and the frame closes (``_body_open`` false), so a second commit raises until the next ``begin_frame``. | |
| The frame must already be open (``begin_frame`` / ``plan_body``) so a commit cannot skip the donation. | |
| """ | |
| self._require_built() | |
| if not self._body_open: | |
| raise RuntimeError( | |
| "commit_current_frame before begin_frame: open the frame (plan_body / begin_frame) so an " | |
| "evicted slot can donate its summary rows before the cursor moves." | |
| ) | |
| self.cursor_r = int((int(self.cursor_r) + 1) % self.n_raw_slots) | |
| self.filled_r = min(int(self.filled_r) + 1, self.n_raw_slots) | |
| self.n_committed = int(self.n_committed) + 1 | |
| self._body_open = False | |
| self._sync_end_dev() | |
| # ------------------------------------------------------------------ # | |
| # Recency hook (OFF unless enable_recency is called). Existing kernels, no new one. | |
| # ------------------------------------------------------------------ # | |
| def enable_recency(self, cfg) -> Optional[RecencyBiasState]: | |
| """Per-key additive logit bias over the one buffer. Zeros on Z0 and RAW; values on SUM rows. | |
| ``cfg``: RecencyBiasConfig | dict | None (None disables). Call AFTER ``build()``; the config survives | |
| ``reset()``. Fails if ``decay_end`` exceeds ``W + 1`` (a summary leaves SUM at Δ = W+1). Refuses when | |
| a body frame is open or any body frame has been committed, or the SUM cursor has moved; a z0 sink | |
| already written is allowed (SUM is still empty). The block adds ``layer["recency_bias"]`` on the single | |
| prefix read; the flex kernel takes it, the flash kvcache kernel does not (a biased read is always flex). | |
| """ | |
| self._require_built() | |
| cfg = RecencyBiasConfig.from_dict(cfg) | |
| if cfg is None: | |
| self.disable_recency() | |
| return None | |
| w = int(self.summary_window_frames) | |
| if int(cfg.decay_end) > w + 1: | |
| raise RuntimeError( | |
| f"recency.decay_end={cfg.decay_end} > W + 1 = {w + 1}: a summary is evicted from SUM " | |
| f"(Δ={w + 1}) before the bias reaches its floor; set decay_end <= {w + 1}" | |
| ) | |
| if int(self.filled_s) != 0 or int(self.cursor_s) != 0 or self._body_open or int(self.n_committed) != 0: | |
| raise RuntimeError("enable_recency must be called before any body frame is opened or committed and before any SUM donation (a written z0 is fine)") | |
| buf = self.layers[0]["k"] | |
| cap, device = int(buf.size(1)), buf.device | |
| reuse = None | |
| if ( | |
| self.recency is not None | |
| and tuple(self.recency.token_bias.shape) == (cap,) | |
| and self.recency.token_bias.device == device | |
| ): | |
| reuse = self.recency.token_bias | |
| self.recency = RecencyBiasState( | |
| cfg, | |
| ring_frames=int(self.n_sum_slots), | |
| ring_start=int(self.sum_region_start), | |
| frame_tokens=int(self.summary_tokens), | |
| capacity=cap, | |
| device=device, | |
| token_bias=reuse, | |
| ) | |
| self._attach_recency() | |
| return self.recency | |
| def disable_recency(self) -> None: | |
| self.recency = None | |
| if isinstance(self.layers, list): | |
| for layer in self.layers: | |
| layer.pop(RECENCY_BIAS_KEY, None) | |
| def _attach_recency(self) -> None: | |
| if self.recency is None or not isinstance(self.layers, list): | |
| return | |
| for layer in self.layers: | |
| layer[RECENCY_BIAS_KEY] = self.recency.token_bias | |
| # ------------------------------------------------------------------ # | |
| # Allocation / reset. | |
| # ------------------------------------------------------------------ # | |
| def build( | |
| self, | |
| *, | |
| batch: int, | |
| frame_tokens: int, | |
| dtype: torch.dtype, | |
| device: torch.device, | |
| z0_tokens: int | None = None, | |
| allow_oversize_reuse: bool = False, | |
| ) -> None: | |
| """Allocate (or reuse) the one K/V buffer per layer and ``end_dev``; zero them; reset state. | |
| Args: | |
| frame_tokens: Tf, tokens per body frame at this bucket. | |
| z0_tokens: Z, tokens of the z0 frame (default Tf). 0 = no Z0 region, no sink. | |
| allow_oversize_reuse: also reuse a buffer whose row axis is LONGER than ``cap`` (same B/H/D/dtype/ | |
| device). Every read is ``[0:end]`` and every write is at a geometry offset. | |
| Reuses existing storage when shape/dtype/device match so addresses stay stable (cudagraphs). | |
| """ | |
| frame_tokens = int(frame_tokens) | |
| z0_tokens = frame_tokens if z0_tokens is None else int(z0_tokens) | |
| if frame_tokens <= 0: | |
| raise ValueError(f"frame_tokens must be > 0; got {frame_tokens}.") | |
| if z0_tokens < 0: | |
| raise ValueError(f"z0_tokens must be >= 0; got {z0_tokens}.") | |
| x = int(self.summary_tokens) | |
| rec = frame_tokens + x | |
| rec0 = (z0_tokens + x) if z0_tokens > 0 else 0 | |
| n_raw = int(self.raw_window_frames) + 1 | |
| n_sum = int(self.summary_window_frames) - int(self.raw_window_frames) | |
| cap = int(rec0 + n_raw * rec + n_sum * x) | |
| b, h, d = int(batch), int(self.num_heads), int(self.head_dim) | |
| device = torch.device(device) | |
| def _reusable(have) -> bool: | |
| return ( | |
| isinstance(have, torch.Tensor) | |
| and have.dtype == dtype | |
| and have.device == device | |
| and int(have.dim()) == 4 | |
| and (int(have.size(0)), int(have.size(2)), int(have.size(3))) == (b, h, d) | |
| and (int(have.size(1)) == cap or (bool(allow_oversize_reuse) and int(have.size(1)) > cap)) | |
| ) | |
| have_layers = ( | |
| isinstance(self.layers, list) | |
| and len(self.layers) == int(self.num_layers) | |
| and all(isinstance(layer, dict) for layer in self.layers) | |
| ) | |
| can_reuse = have_layers and _reusable(self.layers[0].get("k")) and _reusable(self.layers[0].get("v")) | |
| self.frame_tokens = frame_tokens | |
| self.z0_tokens = z0_tokens | |
| self.batch = b | |
| self.capacity = cap | |
| if can_reuse: | |
| for layer in self.layers: | |
| layer["k"].zero_() | |
| layer["v"].zero_() | |
| else: | |
| self.layers = [ | |
| { | |
| "k": torch.zeros((b, cap, h, d), dtype=dtype, device=device), | |
| "v": torch.zeros((b, cap, h, d), dtype=dtype, device=device), | |
| } | |
| for _ in range(int(self.num_layers)) | |
| ] | |
| if not (isinstance(self.end_dev, torch.Tensor) and self.end_dev.device == device): | |
| self.end_dev = torch.zeros((), dtype=torch.long, device=device) | |
| self.reset_state() | |
| if self.recency is not None: | |
| self.enable_recency(self.recency.cfg) | |
| def reset_state(self) -> None: | |
| """Reset the mutable stream counters only (NOT the buffers).""" | |
| self.cursor_r = 0 | |
| self.cursor_s = 0 | |
| self.filled_r = 0 | |
| self.filled_s = 0 | |
| self.n_committed = 0 | |
| self.z0_written = False | |
| self._body_open = False | |
| if self.layers is not None and self.frame_tokens is not None and self.z0_tokens is not None: | |
| self._sync_end_dev() | |
| if self.recency is not None: | |
| self.recency.reset() | |
| def reset(self) -> None: | |
| """Zero the buffers in place and reset counters. Keeps addresses stable.""" | |
| if isinstance(self.layers, list): | |
| for layer in self.layers: | |
| layer["k"].zero_() | |
| layer["v"].zero_() | |
| self.reset_state() | |
| def release(self) -> None: | |
| """Drop the storages. The next ``build()`` allocates again.""" | |
| self.disable_recency() | |
| if isinstance(self.layers, list): | |
| for layer in self.layers: | |
| layer.clear() | |
| self.layers = None | |
| self.end_dev = None | |
| self.capacity = None | |
| def describe(self) -> str: | |
| self._require_built() | |
| return ( | |
| f"SummaryKVCache(L={self.num_layers} B={self.batch} H={self.num_heads} D={self.head_dim} " | |
| f"Tf={self.frame_tokens} Z={self.z0_tokens} X={self.summary_tokens} Y={self.raw_window_frames} " | |
| f"W={self.summary_window_frames} REC={self.record_tokens} cap={self.capacity} " | |
| f"| Z0 0:{self.raw_region_start} RAW {self.raw_region_start}:{self.sum_region_start} " | |
| f"({self.n_raw_slots}x{self.record_tokens}) SUM {self.sum_region_start}: " | |
| f"({self.n_sum_slots}x{self.summary_tokens}) | cursor_r={self.cursor_r} filled_r={self.filled_r} " | |
| f"cursor_s={self.cursor_s} filled_s={self.filled_s} n_raw_live={self.n_raw_live()} " | |
| f"end={self.end_body()} open={self._body_open} z0_written={self.z0_written})" | |
| ) | |