Spark-engram-4B-V2 / memory_cache.py
win10's picture
Add files using upload-large-folder tool
68860fb verified
Raw History Blame Contribute Delete
7.57 kB
"""Bound sliding storage and reversible query continuations without prefix copy."""
import torch
from transformers.cache_utils import DynamicCache,DynamicLayer,DynamicSlidingWindowLayer
class AppendFull(DynamicLayer):
"""Exact full KV with a fixed backing capacity and only the filled prefix exposed.
No repeated concatenation or full-prefix copies. Allocation is KV state,
never sequence-wide activations; unwritten rows are invisible to attention.
"""
def __init__(self, max_cache_len):
super().__init__()
self.capacity = int(max_cache_len)
self.length = 0
def lazy_initialization(self, key_states, value_states):
self.dtype, self.device = key_states.dtype, key_states.device
shape = (*key_states.shape[:-2], self.capacity, key_states.shape[-1])
self.keys = torch.empty(shape, dtype=self.dtype, device=self.device)
self.values = torch.empty(shape, dtype=value_states.dtype, device=value_states.device)
self.is_initialized = True
def update(self, key_states, value_states, *args, **kwargs):
if torch.is_grad_enabled():
raise RuntimeError('AppendFull is an inference-only KV cache')
if not self.is_initialized:
self.lazy_initialization(key_states, value_states)
stop = self.length + key_states.shape[-2]
if stop > self.capacity:
raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed')
self.keys[..., self.length:stop, :].copy_(key_states)
self.values[..., self.length:stop, :].copy_(value_states)
self.length = stop
return self.keys[..., :stop, :], self.values[..., :stop, :]
def get_seq_length(self):
return self.length
def get_mask_sizes(self, query_length):
return self.length + query_length, 0
def get_max_cache_shape(self):
return self.capacity
def crop(self, max_length):
if max_length < 0:
max_length = max(0, self.length + max_length)
self.length = min(self.length, max_length)
@torch.inference_mode()
def reset(self):
super().reset()
self.length = 0
class CompactSliding(DynamicSlidingWindowLayer):
def update(self,key_states,value_states,*args,**kwargs):
key,value=super().update(key_states,value_states,*args,**kwargs)
# A narrow view otherwise pins the entire 1M input storage per layer.
self.keys=self.keys.clone();self.values=self.values.clone()
return key,value
class HostAppendFull(AppendFull):
"""Exact pageable-host full KV; stage one layer's filled prefix on GPU.
Sliding KV stays on GPU. This trades PCIe traffic for GPU headroom without
dropping, compressing, or windowing any full-attention keys.
"""
def lazy_initialization(self,key_states,value_states):
self.dtype,self.compute_device=key_states.dtype,key_states.device
self.device=torch.device('cpu')
shape=(*key_states.shape[:-2],self.capacity,key_states.shape[-1])
self.keys=torch.empty(shape,dtype=key_states.dtype,device='cpu')
self.values=torch.empty(shape,dtype=value_states.dtype,device='cpu')
self.is_initialized=True
def update(self,key_states,value_states,*args,**kwargs):
if torch.is_grad_enabled():raise RuntimeError('HostAppendFull is inference-only')
if not self.is_initialized:self.lazy_initialization(key_states,value_states)
stop=self.length+key_states.shape[-2]
if stop>self.capacity:raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed')
self.keys[...,self.length:stop,:].copy_(key_states)
self.values[...,self.length:stop,:].copy_(value_states)
self.length=stop
return self.keys[...,:stop,:].to(self.compute_device),self.values[...,:stop,:].to(self.compute_device)
class SequenceHostAppendFull(HostAppendFull):
"""Sequence-major host backing keeps a single-session prefix dense in memory."""
def lazy_initialization(self,key_states,value_states):
self.dtype,self.compute_device=key_states.dtype,key_states.device
self.device=torch.device('cpu')
batch,heads,_,dim=key_states.shape
self.keys=torch.empty((batch,self.capacity,heads,dim),dtype=key_states.dtype,device='cpu').transpose(1,2)
self.values=torch.empty((batch,self.capacity,heads,dim),dtype=value_states.dtype,device='cpu').transpose(1,2)
self.is_initialized=True
class RewindableCache(DynamicCache):
def __init__(self,config,max_cache_len=None,full_device='cuda'):
super().__init__(config=config)
if full_device not in ['cuda','cpu','cpu_sequence_major']:raise ValueError('unknown full_device')
for i,layer in enumerate(self.layers):
if getattr(layer,'is_sliding',False):self.layers[i]=CompactSliding(sliding_window=config.sliding_window)
elif max_cache_len is not None:self.layers[i]={'cuda':AppendFull,'cpu':HostAppendFull,'cpu_sequence_major':SequenceHostAppendFull}[full_device](max_cache_len)
@torch.inference_mode()
def reset(self):
super().reset()
for name in ['_engram_forward','_spark_memory_scan','_engram_attention_mask',
'_engram_plug_session','_engram_plug_epoch']:
if hasattr(self,name):delattr(self,name)
def storage_metrics(self):
full = sliding = used = 0
devices={}
for layer in self.layers:
if not layer.is_initialized:
continue
size = layer.keys.untyped_storage().nbytes() + layer.values.untyped_storage().nbytes()
device=str(layer.keys.device);devices[device]=devices.get(device,0)+size
if getattr(layer, 'is_sliding', False):
sliding += size
used += layer.keys.numel() * layer.keys.element_size() + layer.values.numel() * layer.values.element_size()
else:
full += size
used += 2 * layer.keys.shape[0] * layer.keys.shape[1] * layer.get_seq_length() * layer.keys.shape[-1] * layer.keys.element_size()
return dict(full_storage_bytes=full, sliding_storage_bytes=sliding, used_bytes=used,
total_storage_bytes=full + sliding,storage_by_device=devices)
def bookmark(self):
memory=getattr(self,'_engram_forward',None)
return dict(layers=[dict(length=l.get_seq_length(),keys=l.keys.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None,values=l.values.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None) for l in self.layers],memory=[s.fork() for s in memory] if memory else None,scan=getattr(self,'_spark_memory_scan',None),mask=getattr(self,'_engram_attention_mask',None))
def rewind(self,point):
for layer,saved in zip(self.layers,point['layers']):
if getattr(layer,'is_sliding',False):
if saved['keys'] is None:
layer.reset();layer.cumulative_length=0
else:
layer.keys=saved['keys'].clone();layer.values=saved['values'].clone();layer.cumulative_length=saved['length']
else:layer.crop(saved['length'])
if point['memory'] is not None:self._engram_forward=[s.fork() for s in point['memory']]
elif hasattr(self,'_engram_forward'):del self._engram_forward
if point['scan'] is not None:self._spark_memory_scan=point['scan']
if point['mask'] is not None:self._engram_attention_mask=point['mask']