Text Generation
English
transformer
abhinavv3 commited on
Commit
6274fff
·
1 Parent(s): 174b26d

xl_attention completed

Browse files
Files changed (1) hide show
  1. model_core/attention.py +70 -2
model_core/attention.py CHANGED
@@ -10,10 +10,10 @@ import inspect
10
  import os
11
 
12
  class RotaryPositionalEncoding(nn.Module):
13
- def __init__(self, dim, max_seq_len=1024, base=10000):
14
  super().__init__()
15
  assert dim % 2 == 0
16
-
17
  self.dim = dim
18
  self.max_seq_len = max_seq_len
19
  self.base = base
@@ -37,6 +37,7 @@ class RotaryPositionalEncoding(nn.Module):
37
  def apply_rotary_pos_emb(self, q, k, seq_len):
38
  assert q.shape[-1] == self.dim, f"Expected q.shape[-1] == {self.dim}, got {q.shape[-1]}"
39
  assert k.shape[-1] == self.dim, f"Expected k.shape[-1] == {self.dim}, got {k.shape[-1]}"
 
40
 
41
  device = q.device
42
  cos, sin = self._get_freqs(seq_len, device) # both [seq_len, dim//2]
@@ -107,6 +108,73 @@ class KNN():
107
  os.remove(self.db_filepath)
108
  except:
109
  pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
 
111
 
112
 
 
10
  import os
11
 
12
  class RotaryPositionalEncoding(nn.Module):
13
+ def __init__(self, dim, max_seq_len=2048, base=10000):
14
  super().__init__()
15
  assert dim % 2 == 0
16
+
17
  self.dim = dim
18
  self.max_seq_len = max_seq_len
19
  self.base = base
 
37
  def apply_rotary_pos_emb(self, q, k, seq_len):
38
  assert q.shape[-1] == self.dim, f"Expected q.shape[-1] == {self.dim}, got {q.shape[-1]}"
39
  assert k.shape[-1] == self.dim, f"Expected k.shape[-1] == {self.dim}, got {k.shape[-1]}"
40
+ assert seq_len <= self.max_seq_len, f"seq_len {seq_len} exceeds max_seq_len {self.max_seq_len}"
41
 
42
  device = q.device
43
  cos, sin = self._get_freqs(seq_len, device) # both [seq_len, dim//2]
 
108
  os.remove(self.db_filepath)
109
  except:
110
  pass
111
+
112
+ class XLAttention(nn.Module):
113
+ def __init__(self, config):
114
+ super().__init__()
115
+ assert config.n_embd % config.n_head == 0
116
+ self.n_head = config.n_head
117
+ self.n_embd = config.n_embd
118
+ self.head_dim = config.n_embd // config.n_head
119
+ self.dropout = nn.Dropout(config.dropout if hasattr(config, 'dropout') else 0.0)
120
+ self.scale = self.head_dim ** -0.5
121
+
122
+ self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
123
+ self.c_proj = nn.Linear(config.n_embd, config.n_embd)
124
+ self.c_proj.MEMGPT_SCALE_INIT = 1
125
+
126
+ self.rope = RotaryPositionalEncoding(self.head_dim, config.block_size)
127
+
128
+ def forward(self, x, xl_memory=None):
129
+ B, T, C = x.size()
130
+
131
+ qkv = self.c_attn(x) # (B,T,3C)
132
+ q, k, v = qkv.split(self.n_embd, dim=2) # (B,T,C)
133
+
134
+ # Handle XL memory
135
+ if xl_memory is not None:
136
+ k_xl, v_xl = xl_memory.unbind(dim=-2)
137
+ k = torch.cat((k_xl, k), dim=1)
138
+ v = torch.cat((v_xl, v), dim=1)
139
+ xl_seq_len = k_xl.shape[1]
140
+
141
+ # Reshape for multi-head attention
142
+ q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2) # (B, nh, T, hs)
143
+ k = k.view(B, -1, self.n_head, self.head_dim).transpose(1, 2) # (B, nh, T+xl, hs)
144
+ v = v.view(B, -1, self.n_head, self.head_dim).transpose(1, 2) # (B, nh, T+xl, hs)
145
+
146
+ # Apply rotary positional encoding
147
+ seq_len = k.shape[2]
148
+ q, k = self.rope.apply_rotary_pos_emb(q, k, seq_len)
149
+
150
+ # Attention computation
151
+ att = (q @ k.transpose(-2, -1)) * self.scale
152
+
153
+ # Causal mask
154
+ mask = torch.tril(torch.ones(T, seq_len, device=x.device, dtype=torch.bool))
155
+ att = att.masked_fill(~mask, float('-inf'))
156
+
157
+ att = F.softmax(att, dim=-1)
158
+ att = self.dropout(att)
159
+
160
+ y = att @ v # (B, nh, T, hs)
161
+ y = y.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)
162
+
163
+ y = self.c_proj(y)
164
+
165
+ # Prepare new XL memories
166
+ k = k.transpose(1, 2).contiguous().view(B, -1, C)
167
+ v = v.transpose(1, 2).contiguous().view(B, -1, C)
168
+ kv_memories = torch.stack((k, v), dim=-2)
169
+
170
+ if xl_memory is not None:
171
+ current_kv = kv_memories[:, -xl_seq_len:]
172
+ else:
173
+ current_kv = kv_memories
174
+
175
+ return y, current_kv
176
+
177
+
178
 
179
 
180