clairic / clairic_runtime /gdn_pool.py
areneau's picture
Update Clairic Transformers runtime
524fd9d verified
Raw History Blame Contribute Delete
4.65 kB
"""Engine-owned native linear-cache storage; dense caches remain fresh native."""
from __future__ import annotations
from typing import Any
import torch
class NativeStatePool:
"""One maxB allocation per linear layer, attached to fresh DynamicCache objects.
The caller serializes requests on its model worker and must not reuse a cache
after its request ends. The initialized flags prevent native prefill from
allocating duplicate state; has_previous_state=False retains original prefill.
"""
def __init__(self, backbone: Any, max_batch: int) -> None:
if max_batch < 1:
raise ValueError("Positive maximum batch required")
self.max_batch, self.closed = max_batch, False
self.language = backbone.model.language_model
self.states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
for index, layer in enumerate(self.language.layers):
if layer.layer_type != "linear_attention":
continue
gdn = layer.linear_attn
weight = gdn.in_proj_qkv.weight
conv = torch.zeros(max_batch, gdn.conv_dim, gdn.conv_kernel_size,
dtype=weight.dtype, device=weight.device)
source = torch.zeros(max_batch, gdn.num_v_heads, gdn.head_v_dim, gdn.head_k_dim,
dtype=torch.float32, device=weight.device)
self.states[index] = (conv, source.transpose(-1, -2))
if not self.states:
raise ValueError("Expected native linear-attention layers")
def attach(self, cache: Any, batch: int, *, previous: bool = False) -> Any:
if self.closed or not 1 <= batch <= self.max_batch:
raise RuntimeError("Closed pool or batch outside owned state")
if len(cache.layers) != len(self.language.layers):
raise ValueError("Cache layer count differs from language model")
for index, (conv, recurrent) in self.states.items():
layer = cache.layers[index]
# Attach to empty native linear layers only; replacing live storage
# would silently discard state or retain an unaccounted allocation.
if layer.is_conv_states_initialized or layer.is_recurrent_states_initialized:
raise ValueError("Fresh uninitialized native cache required")
layer.conv_states = conv[:batch]
layer.recurrent_states = recurrent[:batch]
layer.is_conv_states_initialized = True
layer.is_recurrent_states_initialized = True
layer.has_previous_state = previous
layer.dtype, layer.device = conv.dtype, conv.device
layer.max_batch_size, layer.conv_kernel_size = batch, conv.shape[-1]
return cache
def new_cache(self, batch: int, *, previous: bool = False) -> Any:
if self.closed:
raise RuntimeError("Closed pool")
from transformers.cache_utils import DynamicCache
return self.attach(DynamicCache(config=self.language.config), batch, previous=previous)
def owns(self, cache: Any, batch: int) -> bool:
if self.closed or not 1 <= batch <= self.max_batch or cache is None:
return False
if len(cache.layers) != len(self.language.layers):
return False
for index, (conv, recurrent) in self.states.items():
layer = cache.layers[index]
for actual, owned in ((layer.conv_states, conv), (layer.recurrent_states, recurrent)):
if (actual is None or actual.data_ptr() != owned.data_ptr()
or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
or actual.dtype != owned.dtype or actual.device != owned.device
or actual.storage_offset() != owned.storage_offset()
or tuple(actual.shape) != (batch, *owned.shape[1:])
or tuple(actual.stride()) != tuple(owned.stride())):
return False
if not (layer.is_conv_states_initialized and layer.is_recurrent_states_initialized):
return False
return True
@property
def storage_bytes(self) -> int:
return sum(value.untyped_storage().nbytes() for states in self.states.values() for value in states)
def zero(self) -> None:
if self.closed:
raise RuntimeError("Closed pool")
for conv, recurrent in self.states.values():
conv.zero_()
recurrent.zero_()
def close(self) -> None:
self.states.clear()
self.language = None
self.closed = True