File size: 7,570 Bytes
68860fb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 | """Bound sliding storage and reversible query continuations without prefix copy."""
import torch
from transformers.cache_utils import DynamicCache,DynamicLayer,DynamicSlidingWindowLayer
class AppendFull(DynamicLayer):
"""Exact full KV with a fixed backing capacity and only the filled prefix exposed.
No repeated concatenation or full-prefix copies. Allocation is KV state,
never sequence-wide activations; unwritten rows are invisible to attention.
"""
def __init__(self, max_cache_len):
super().__init__()
self.capacity = int(max_cache_len)
self.length = 0
def lazy_initialization(self, key_states, value_states):
self.dtype, self.device = key_states.dtype, key_states.device
shape = (*key_states.shape[:-2], self.capacity, key_states.shape[-1])
self.keys = torch.empty(shape, dtype=self.dtype, device=self.device)
self.values = torch.empty(shape, dtype=value_states.dtype, device=value_states.device)
self.is_initialized = True
def update(self, key_states, value_states, *args, **kwargs):
if torch.is_grad_enabled():
raise RuntimeError('AppendFull is an inference-only KV cache')
if not self.is_initialized:
self.lazy_initialization(key_states, value_states)
stop = self.length + key_states.shape[-2]
if stop > self.capacity:
raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed')
self.keys[..., self.length:stop, :].copy_(key_states)
self.values[..., self.length:stop, :].copy_(value_states)
self.length = stop
return self.keys[..., :stop, :], self.values[..., :stop, :]
def get_seq_length(self):
return self.length
def get_mask_sizes(self, query_length):
return self.length + query_length, 0
def get_max_cache_shape(self):
return self.capacity
def crop(self, max_length):
if max_length < 0:
max_length = max(0, self.length + max_length)
self.length = min(self.length, max_length)
@torch.inference_mode()
def reset(self):
super().reset()
self.length = 0
class CompactSliding(DynamicSlidingWindowLayer):
def update(self,key_states,value_states,*args,**kwargs):
key,value=super().update(key_states,value_states,*args,**kwargs)
# A narrow view otherwise pins the entire 1M input storage per layer.
self.keys=self.keys.clone();self.values=self.values.clone()
return key,value
class HostAppendFull(AppendFull):
"""Exact pageable-host full KV; stage one layer's filled prefix on GPU.
Sliding KV stays on GPU. This trades PCIe traffic for GPU headroom without
dropping, compressing, or windowing any full-attention keys.
"""
def lazy_initialization(self,key_states,value_states):
self.dtype,self.compute_device=key_states.dtype,key_states.device
self.device=torch.device('cpu')
shape=(*key_states.shape[:-2],self.capacity,key_states.shape[-1])
self.keys=torch.empty(shape,dtype=key_states.dtype,device='cpu')
self.values=torch.empty(shape,dtype=value_states.dtype,device='cpu')
self.is_initialized=True
def update(self,key_states,value_states,*args,**kwargs):
if torch.is_grad_enabled():raise RuntimeError('HostAppendFull is inference-only')
if not self.is_initialized:self.lazy_initialization(key_states,value_states)
stop=self.length+key_states.shape[-2]
if stop>self.capacity:raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed')
self.keys[...,self.length:stop,:].copy_(key_states)
self.values[...,self.length:stop,:].copy_(value_states)
self.length=stop
return self.keys[...,:stop,:].to(self.compute_device),self.values[...,:stop,:].to(self.compute_device)
class SequenceHostAppendFull(HostAppendFull):
"""Sequence-major host backing keeps a single-session prefix dense in memory."""
def lazy_initialization(self,key_states,value_states):
self.dtype,self.compute_device=key_states.dtype,key_states.device
self.device=torch.device('cpu')
batch,heads,_,dim=key_states.shape
self.keys=torch.empty((batch,self.capacity,heads,dim),dtype=key_states.dtype,device='cpu').transpose(1,2)
self.values=torch.empty((batch,self.capacity,heads,dim),dtype=value_states.dtype,device='cpu').transpose(1,2)
self.is_initialized=True
class RewindableCache(DynamicCache):
def __init__(self,config,max_cache_len=None,full_device='cuda'):
super().__init__(config=config)
if full_device not in ['cuda','cpu','cpu_sequence_major']:raise ValueError('unknown full_device')
for i,layer in enumerate(self.layers):
if getattr(layer,'is_sliding',False):self.layers[i]=CompactSliding(sliding_window=config.sliding_window)
elif max_cache_len is not None:self.layers[i]={'cuda':AppendFull,'cpu':HostAppendFull,'cpu_sequence_major':SequenceHostAppendFull}[full_device](max_cache_len)
@torch.inference_mode()
def reset(self):
super().reset()
for name in ['_engram_forward','_spark_memory_scan','_engram_attention_mask',
'_engram_plug_session','_engram_plug_epoch']:
if hasattr(self,name):delattr(self,name)
def storage_metrics(self):
full = sliding = used = 0
devices={}
for layer in self.layers:
if not layer.is_initialized:
continue
size = layer.keys.untyped_storage().nbytes() + layer.values.untyped_storage().nbytes()
device=str(layer.keys.device);devices[device]=devices.get(device,0)+size
if getattr(layer, 'is_sliding', False):
sliding += size
used += layer.keys.numel() * layer.keys.element_size() + layer.values.numel() * layer.values.element_size()
else:
full += size
used += 2 * layer.keys.shape[0] * layer.keys.shape[1] * layer.get_seq_length() * layer.keys.shape[-1] * layer.keys.element_size()
return dict(full_storage_bytes=full, sliding_storage_bytes=sliding, used_bytes=used,
total_storage_bytes=full + sliding,storage_by_device=devices)
def bookmark(self):
memory=getattr(self,'_engram_forward',None)
return dict(layers=[dict(length=l.get_seq_length(),keys=l.keys.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None,values=l.values.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None) for l in self.layers],memory=[s.fork() for s in memory] if memory else None,scan=getattr(self,'_spark_memory_scan',None),mask=getattr(self,'_engram_attention_mask',None))
def rewind(self,point):
for layer,saved in zip(self.layers,point['layers']):
if getattr(layer,'is_sliding',False):
if saved['keys'] is None:
layer.reset();layer.cumulative_length=0
else:
layer.keys=saved['keys'].clone();layer.values=saved['values'].clone();layer.cumulative_length=saved['length']
else:layer.crop(saved['length'])
if point['memory'] is not None:self._engram_forward=[s.fork() for s in point['memory']]
elif hasattr(self,'_engram_forward'):del self._engram_forward
if point['scan'] is not None:self._spark_memory_scan=point['scan']
if point['mask'] is not None:self._engram_attention_mask=point['mask']
|