"""Bound sliding storage and reversible query continuations without prefix copy.""" import torch from transformers.cache_utils import DynamicCache,DynamicLayer,DynamicSlidingWindowLayer class AppendFull(DynamicLayer): """Exact full KV with a fixed backing capacity and only the filled prefix exposed. No repeated concatenation or full-prefix copies. Allocation is KV state, never sequence-wide activations; unwritten rows are invisible to attention. """ def __init__(self, max_cache_len): super().__init__() self.capacity = int(max_cache_len) self.length = 0 def lazy_initialization(self, key_states, value_states): self.dtype, self.device = key_states.dtype, key_states.device shape = (*key_states.shape[:-2], self.capacity, key_states.shape[-1]) self.keys = torch.empty(shape, dtype=self.dtype, device=self.device) self.values = torch.empty(shape, dtype=value_states.dtype, device=value_states.device) self.is_initialized = True def update(self, key_states, value_states, *args, **kwargs): if torch.is_grad_enabled(): raise RuntimeError('AppendFull is an inference-only KV cache') if not self.is_initialized: self.lazy_initialization(key_states, value_states) stop = self.length + key_states.shape[-2] if stop > self.capacity: raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed') self.keys[..., self.length:stop, :].copy_(key_states) self.values[..., self.length:stop, :].copy_(value_states) self.length = stop return self.keys[..., :stop, :], self.values[..., :stop, :] def get_seq_length(self): return self.length def get_mask_sizes(self, query_length): return self.length + query_length, 0 def get_max_cache_shape(self): return self.capacity def crop(self, max_length): if max_length < 0: max_length = max(0, self.length + max_length) self.length = min(self.length, max_length) @torch.inference_mode() def reset(self): super().reset() self.length = 0 class CompactSliding(DynamicSlidingWindowLayer): def update(self,key_states,value_states,*args,**kwargs): key,value=super().update(key_states,value_states,*args,**kwargs) # A narrow view otherwise pins the entire 1M input storage per layer. self.keys=self.keys.clone();self.values=self.values.clone() return key,value class HostAppendFull(AppendFull): """Exact pageable-host full KV; stage one layer's filled prefix on GPU. Sliding KV stays on GPU. This trades PCIe traffic for GPU headroom without dropping, compressing, or windowing any full-attention keys. """ def lazy_initialization(self,key_states,value_states): self.dtype,self.compute_device=key_states.dtype,key_states.device self.device=torch.device('cpu') shape=(*key_states.shape[:-2],self.capacity,key_states.shape[-1]) self.keys=torch.empty(shape,dtype=key_states.dtype,device='cpu') self.values=torch.empty(shape,dtype=value_states.dtype,device='cpu') self.is_initialized=True def update(self,key_states,value_states,*args,**kwargs): if torch.is_grad_enabled():raise RuntimeError('HostAppendFull is inference-only') if not self.is_initialized:self.lazy_initialization(key_states,value_states) stop=self.length+key_states.shape[-2] if stop>self.capacity:raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed') self.keys[...,self.length:stop,:].copy_(key_states) self.values[...,self.length:stop,:].copy_(value_states) self.length=stop return self.keys[...,:stop,:].to(self.compute_device),self.values[...,:stop,:].to(self.compute_device) class SequenceHostAppendFull(HostAppendFull): """Sequence-major host backing keeps a single-session prefix dense in memory.""" def lazy_initialization(self,key_states,value_states): self.dtype,self.compute_device=key_states.dtype,key_states.device self.device=torch.device('cpu') batch,heads,_,dim=key_states.shape self.keys=torch.empty((batch,self.capacity,heads,dim),dtype=key_states.dtype,device='cpu').transpose(1,2) self.values=torch.empty((batch,self.capacity,heads,dim),dtype=value_states.dtype,device='cpu').transpose(1,2) self.is_initialized=True class RewindableCache(DynamicCache): def __init__(self,config,max_cache_len=None,full_device='cuda'): super().__init__(config=config) if full_device not in ['cuda','cpu','cpu_sequence_major']:raise ValueError('unknown full_device') for i,layer in enumerate(self.layers): if getattr(layer,'is_sliding',False):self.layers[i]=CompactSliding(sliding_window=config.sliding_window) elif max_cache_len is not None:self.layers[i]={'cuda':AppendFull,'cpu':HostAppendFull,'cpu_sequence_major':SequenceHostAppendFull}[full_device](max_cache_len) @torch.inference_mode() def reset(self): super().reset() for name in ['_engram_forward','_spark_memory_scan','_engram_attention_mask', '_engram_plug_session','_engram_plug_epoch']: if hasattr(self,name):delattr(self,name) def storage_metrics(self): full = sliding = used = 0 devices={} for layer in self.layers: if not layer.is_initialized: continue size = layer.keys.untyped_storage().nbytes() + layer.values.untyped_storage().nbytes() device=str(layer.keys.device);devices[device]=devices.get(device,0)+size if getattr(layer, 'is_sliding', False): sliding += size used += layer.keys.numel() * layer.keys.element_size() + layer.values.numel() * layer.values.element_size() else: full += size used += 2 * layer.keys.shape[0] * layer.keys.shape[1] * layer.get_seq_length() * layer.keys.shape[-1] * layer.keys.element_size() return dict(full_storage_bytes=full, sliding_storage_bytes=sliding, used_bytes=used, total_storage_bytes=full + sliding,storage_by_device=devices) def bookmark(self): memory=getattr(self,'_engram_forward',None) return dict(layers=[dict(length=l.get_seq_length(),keys=l.keys.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None,values=l.values.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None) for l in self.layers],memory=[s.fork() for s in memory] if memory else None,scan=getattr(self,'_spark_memory_scan',None),mask=getattr(self,'_engram_attention_mask',None)) def rewind(self,point): for layer,saved in zip(self.layers,point['layers']): if getattr(layer,'is_sliding',False): if saved['keys'] is None: layer.reset();layer.cumulative_length=0 else: layer.keys=saved['keys'].clone();layer.values=saved['values'].clone();layer.cumulative_length=saved['length'] else:layer.crop(saved['length']) if point['memory'] is not None:self._engram_forward=[s.fork() for s in point['memory']] elif hasattr(self,'_engram_forward'):del self._engram_forward if point['scan'] is not None:self._spark_memory_scan=point['scan'] if point['mask'] is not None:self._engram_attention_mask=point['mask']