File size: 7,570 Bytes
68860fb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
"""Bound sliding storage and reversible query continuations without prefix copy."""
import torch
from transformers.cache_utils import DynamicCache,DynamicLayer,DynamicSlidingWindowLayer


class AppendFull(DynamicLayer):
    """Exact full KV with a fixed backing capacity and only the filled prefix exposed.

    No repeated concatenation or full-prefix copies. Allocation is KV state,
    never sequence-wide activations; unwritten rows are invisible to attention.
    """
    def __init__(self, max_cache_len):
        super().__init__()
        self.capacity = int(max_cache_len)
        self.length = 0

    def lazy_initialization(self, key_states, value_states):
        self.dtype, self.device = key_states.dtype, key_states.device
        shape = (*key_states.shape[:-2], self.capacity, key_states.shape[-1])
        self.keys = torch.empty(shape, dtype=self.dtype, device=self.device)
        self.values = torch.empty(shape, dtype=value_states.dtype, device=value_states.device)
        self.is_initialized = True

    def update(self, key_states, value_states, *args, **kwargs):
        if torch.is_grad_enabled():
            raise RuntimeError('AppendFull is an inference-only KV cache')
        if not self.is_initialized:
            self.lazy_initialization(key_states, value_states)
        stop = self.length + key_states.shape[-2]
        if stop > self.capacity:
            raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed')
        self.keys[..., self.length:stop, :].copy_(key_states)
        self.values[..., self.length:stop, :].copy_(value_states)
        self.length = stop
        return self.keys[..., :stop, :], self.values[..., :stop, :]

    def get_seq_length(self):
        return self.length

    def get_mask_sizes(self, query_length):
        return self.length + query_length, 0

    def get_max_cache_shape(self):
        return self.capacity

    def crop(self, max_length):
        if max_length < 0:
            max_length = max(0, self.length + max_length)
        self.length = min(self.length, max_length)

    @torch.inference_mode()
    def reset(self):
        super().reset()
        self.length = 0

class CompactSliding(DynamicSlidingWindowLayer):
    def update(self,key_states,value_states,*args,**kwargs):
        key,value=super().update(key_states,value_states,*args,**kwargs)
        # A narrow view otherwise pins the entire 1M input storage per layer.
        self.keys=self.keys.clone();self.values=self.values.clone()
        return key,value


class HostAppendFull(AppendFull):
    """Exact pageable-host full KV; stage one layer's filled prefix on GPU.

    Sliding KV stays on GPU. This trades PCIe traffic for GPU headroom without
    dropping, compressing, or windowing any full-attention keys.
    """
    def lazy_initialization(self,key_states,value_states):
        self.dtype,self.compute_device=key_states.dtype,key_states.device
        self.device=torch.device('cpu')
        shape=(*key_states.shape[:-2],self.capacity,key_states.shape[-1])
        self.keys=torch.empty(shape,dtype=key_states.dtype,device='cpu')
        self.values=torch.empty(shape,dtype=value_states.dtype,device='cpu')
        self.is_initialized=True

    def update(self,key_states,value_states,*args,**kwargs):
        if torch.is_grad_enabled():raise RuntimeError('HostAppendFull is inference-only')
        if not self.is_initialized:self.lazy_initialization(key_states,value_states)
        stop=self.length+key_states.shape[-2]
        if stop>self.capacity:raise ValueError(f'KV capacity exceeded: {stop} > {self.capacity}; no truncation performed')
        self.keys[...,self.length:stop,:].copy_(key_states)
        self.values[...,self.length:stop,:].copy_(value_states)
        self.length=stop
        return self.keys[...,:stop,:].to(self.compute_device),self.values[...,:stop,:].to(self.compute_device)


class SequenceHostAppendFull(HostAppendFull):
    """Sequence-major host backing keeps a single-session prefix dense in memory."""
    def lazy_initialization(self,key_states,value_states):
        self.dtype,self.compute_device=key_states.dtype,key_states.device
        self.device=torch.device('cpu')
        batch,heads,_,dim=key_states.shape
        self.keys=torch.empty((batch,self.capacity,heads,dim),dtype=key_states.dtype,device='cpu').transpose(1,2)
        self.values=torch.empty((batch,self.capacity,heads,dim),dtype=value_states.dtype,device='cpu').transpose(1,2)
        self.is_initialized=True

class RewindableCache(DynamicCache):
    def __init__(self,config,max_cache_len=None,full_device='cuda'):
        super().__init__(config=config)
        if full_device not in ['cuda','cpu','cpu_sequence_major']:raise ValueError('unknown full_device')
        for i,layer in enumerate(self.layers):
            if getattr(layer,'is_sliding',False):self.layers[i]=CompactSliding(sliding_window=config.sliding_window)
            elif max_cache_len is not None:self.layers[i]={'cuda':AppendFull,'cpu':HostAppendFull,'cpu_sequence_major':SequenceHostAppendFull}[full_device](max_cache_len)

    @torch.inference_mode()
    def reset(self):
        super().reset()
        for name in ['_engram_forward','_spark_memory_scan','_engram_attention_mask',
                     '_engram_plug_session','_engram_plug_epoch']:
            if hasattr(self,name):delattr(self,name)

    def storage_metrics(self):
        full = sliding = used = 0
        devices={}
        for layer in self.layers:
            if not layer.is_initialized:
                continue
            size = layer.keys.untyped_storage().nbytes() + layer.values.untyped_storage().nbytes()
            device=str(layer.keys.device);devices[device]=devices.get(device,0)+size
            if getattr(layer, 'is_sliding', False):
                sliding += size
                used += layer.keys.numel() * layer.keys.element_size() + layer.values.numel() * layer.values.element_size()
            else:
                full += size
                used += 2 * layer.keys.shape[0] * layer.keys.shape[1] * layer.get_seq_length() * layer.keys.shape[-1] * layer.keys.element_size()
        return dict(full_storage_bytes=full, sliding_storage_bytes=sliding, used_bytes=used,
                    total_storage_bytes=full + sliding,storage_by_device=devices)
    def bookmark(self):
        memory=getattr(self,'_engram_forward',None)
        return dict(layers=[dict(length=l.get_seq_length(),keys=l.keys.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None,values=l.values.clone() if getattr(l,'is_sliding',False) and l.is_initialized else None) for l in self.layers],memory=[s.fork() for s in memory] if memory else None,scan=getattr(self,'_spark_memory_scan',None),mask=getattr(self,'_engram_attention_mask',None))
    def rewind(self,point):
        for layer,saved in zip(self.layers,point['layers']):
            if getattr(layer,'is_sliding',False):
                if saved['keys'] is None:
                    layer.reset();layer.cumulative_length=0
                else:
                    layer.keys=saved['keys'].clone();layer.values=saved['values'].clone();layer.cumulative_length=saved['length']
            else:layer.crop(saved['length'])
        if point['memory'] is not None:self._engram_forward=[s.fork() for s in point['memory']]
        elif hasattr(self,'_engram_forward'):del self._engram_forward
        if point['scan'] is not None:self._spark_memory_scan=point['scan']
        if point['mask'] is not None:self._engram_attention_mask=point['mask']