Download memory_cache.py from win10/Spark-engram-4B-V2: direct link, hf CLI and curl.
- Browser
- Download file 7.57 kB
-
https://huggingface.co/win10/Spark-engram-4B-V2/resolve/main/memory_cache.py
- Command line
-
hf download hf://win10/Spark-engram-4B-V2/memory_cache.py
-
curl -L -o memory_cache.py https://huggingface.co/win10/Spark-engram-4B-V2/resolve/main/memory_cache.py
7.57 kB
| """Bound sliding storage and reversible query continuations without prefix copy.""" | |
| import torch | |
| from transformers.cache_utils import DynamicCache,DynamicLayer,DynamicSlidingWindowLayer | |
| class AppendFull(DynamicLayer): | |
| """Exact full KV with a fixed backing capacity and only the filled prefix exposed. | |
| No repeated concatenation or full-prefix copies. Allocation is KV state, | |
| never sequence-wide activations; unwritten rows are invisible to attention. | |
| """ | |
| def __init__(self, max_cache_len): | |
| super().__init__() | |
| self.capacity = int(max_cache_len) | |
| self.length = 0 | |
| def lazy_initialization(self, key_states, value_states): | |
| self.dtype, self.device = key_states.dtype, key_states.device | |
| shape = (*key_states.shape[:-2], self.capacity, key_states.shape[-1]) | |
| self.keys = torch.empty(shape, dtype=self.dtype, device=self.device) | |
| self.values = torch.empty(shape, dtype=value_states.dtype, device=value_states.device) | |
| self.is_initialized = True | |
| def update(self, key_states, value_states, *args, **kwargs): | |
| if torch.is_grad_enabled(): | |
| raise RuntimeError('AppendFull is an inference-only KV cache') | |
| if not self.is_initialized: | |
| self.lazy_initialization(key_states, value_states) | |
| stop = self.length + key_states.shape[-2] | |
| if stop > self.capacity: | |
| raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed') | |
| self.keys[..., self.length:stop, :].copy_(key_states) | |
| self.values[..., self.length:stop, :].copy_(value_states) | |
| self.length = stop | |
| return self.keys[..., :stop, :], self.values[..., :stop, :] | |
| def get_seq_length(self): | |
| return self.length | |
| def get_mask_sizes(self, query_length): | |
| return self.length + query_length, 0 | |
| def get_max_cache_shape(self): | |
| return self.capacity | |
| def crop(self, max_length): | |
| if max_length < 0: | |
| max_length = max(0, self.length + max_length) | |
| self.length = min(self.length, max_length) | |
| def reset(self): | |
| super().reset() | |
| self.length = 0 | |
| class CompactSliding(DynamicSlidingWindowLayer): | |
| def update(self,key_states,value_states,*args,**kwargs): | |
| key,value=super().update(key_states,value_states,*args,**kwargs) | |
| # A narrow view otherwise pins the entire 1M input storage per layer. | |
| self.keys=self.keys.clone();self.values=self.values.clone() | |
| return key,value | |
| class HostAppendFull(AppendFull): | |
| """Exact pageable-host full KV; stage one layer's filled prefix on GPU. | |
| Sliding KV stays on GPU. This trades PCIe traffic for GPU headroom without | |
| dropping, compressing, or windowing any full-attention keys. | |
| """ | |
| def lazy_initialization(self,key_states,value_states): | |
| self.dtype,self.compute_device=key_states.dtype,key_states.device | |
| self.device=torch.device('cpu') | |
| shape=(*key_states.shape[:-2],self.capacity,key_states.shape[-1]) | |
| self.keys=torch.empty(shape,dtype=key_states.dtype,device='cpu') | |
| self.values=torch.empty(shape,dtype=value_states.dtype,device='cpu') | |
| self.is_initialized=True | |
| def update(self,key_states,value_states,*args,**kwargs): | |
| if torch.is_grad_enabled():raise RuntimeError('HostAppendFull is inference-only') | |
| if not self.is_initialized:self.lazy_initialization(key_states,value_states) | |
| stop=self.length+key_states.shape[-2] | |
| if stop>self.capacity:raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed') | |
| self.keys[...,self.length:stop,:].copy_(key_states) | |
| self.values[...,self.length:stop,:].copy_(value_states) | |
| self.length=stop | |
| return self.keys[...,:stop,:].to(self.compute_device),self.values[...,:stop,:].to(self.compute_device) | |
| class SequenceHostAppendFull(HostAppendFull): | |
| """Sequence-major host backing keeps a single-session prefix dense in memory.""" | |
| def lazy_initialization(self,key_states,value_states): | |
| self.dtype,self.compute_device=key_states.dtype,key_states.device | |
| self.device=torch.device('cpu') | |
| batch,heads,_,dim=key_states.shape | |
| self.keys=torch.empty((batch,self.capacity,heads,dim),dtype=key_states.dtype,device='cpu').transpose(1,2) | |
| self.values=torch.empty((batch,self.capacity,heads,dim),dtype=value_states.dtype,device='cpu').transpose(1,2) | |
| self.is_initialized=True | |
| class RewindableCache(DynamicCache): | |
| def __init__(self,config,max_cache_len=None,full_device='cuda'): | |
| super().__init__(config=config) | |
| if full_device not in ['cuda','cpu','cpu_sequence_major']:raise ValueError('unknown full_device') | |
| for i,layer in enumerate(self.layers): | |
| if getattr(layer,'is_sliding',False):self.layers[i]=CompactSliding(sliding_window=config.sliding_window) | |
| elif max_cache_len is not None:self.layers[i]={'cuda':AppendFull,'cpu':HostAppendFull,'cpu_sequence_major':SequenceHostAppendFull}[full_device](max_cache_len) | |
| def reset(self): | |
| super().reset() | |
| for name in ['_engram_forward','_spark_memory_scan','_engram_attention_mask', | |
| '_engram_plug_session','_engram_plug_epoch']: | |
| if hasattr(self,name):delattr(self,name) | |
| def storage_metrics(self): | |
| full = sliding = used = 0 | |
| devices={} | |
| for layer in self.layers: | |
| if not layer.is_initialized: | |
| continue | |
| size = layer.keys.untyped_storage().nbytes() + layer.values.untyped_storage().nbytes() | |
| device=str(layer.keys.device);devices[device]=devices.get(device,0)+size | |
| if getattr(layer, 'is_sliding', False): | |
| sliding += size | |
| used += layer.keys.numel() * layer.keys.element_size() + layer.values.numel() * layer.values.element_size() | |
| else: | |
| full += size | |
| used += 2 * layer.keys.shape[0] * layer.keys.shape[1] * layer.get_seq_length() * layer.keys.shape[-1] * layer.keys.element_size() | |
| return dict(full_storage_bytes=full, sliding_storage_bytes=sliding, used_bytes=used, | |
| total_storage_bytes=full + sliding,storage_by_device=devices) | |
| def bookmark(self): | |
| memory=getattr(self,'_engram_forward',None) | |
| return dict(layers=[dict(length=l.get_seq_length(),keys=l.keys.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None,values=l.values.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None) for l in self.layers],memory=[s.fork() for s in memory] if memory else None,scan=getattr(self,'_spark_memory_scan',None),mask=getattr(self,'_engram_attention_mask',None)) | |
| def rewind(self,point): | |
| for layer,saved in zip(self.layers,point['layers']): | |
| if getattr(layer,'is_sliding',False): | |
| if saved['keys'] is None: | |
| layer.reset();layer.cumulative_length=0 | |
| else: | |
| layer.keys=saved['keys'].clone();layer.values=saved['values'].clone();layer.cumulative_length=saved['length'] | |
| else:layer.crop(saved['length']) | |
| if point['memory'] is not None:self._engram_forward=[s.fork() for s in point['memory']] | |
| elif hasattr(self,'_engram_forward'):del self._engram_forward | |
| if point['scan'] is not None:self._spark_memory_scan=point['scan'] | |
| if point['mask'] is not None:self._engram_attention_mask=point['mask'] | |