xl_attention completed
Browse files- model_core/attention.py +70 -2
model_core/attention.py
CHANGED
|
@@ -10,10 +10,10 @@ import inspect
|
|
| 10 |
import os
|
| 11 |
|
| 12 |
class RotaryPositionalEncoding(nn.Module):
|
| 13 |
-
def __init__(self, dim, max_seq_len=
|
| 14 |
super().__init__()
|
| 15 |
assert dim % 2 == 0
|
| 16 |
-
|
| 17 |
self.dim = dim
|
| 18 |
self.max_seq_len = max_seq_len
|
| 19 |
self.base = base
|
|
@@ -37,6 +37,7 @@ class RotaryPositionalEncoding(nn.Module):
|
|
| 37 |
def apply_rotary_pos_emb(self, q, k, seq_len):
|
| 38 |
assert q.shape[-1] == self.dim, f"Expected q.shape[-1] == {self.dim}, got {q.shape[-1]}"
|
| 39 |
assert k.shape[-1] == self.dim, f"Expected k.shape[-1] == {self.dim}, got {k.shape[-1]}"
|
|
|
|
| 40 |
|
| 41 |
device = q.device
|
| 42 |
cos, sin = self._get_freqs(seq_len, device) # both [seq_len, dim//2]
|
|
@@ -107,6 +108,73 @@ class KNN():
|
|
| 107 |
os.remove(self.db_filepath)
|
| 108 |
except:
|
| 109 |
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
|
| 112 |
|
|
|
|
| 10 |
import os
|
| 11 |
|
| 12 |
class RotaryPositionalEncoding(nn.Module):
|
| 13 |
+
def __init__(self, dim, max_seq_len=2048, base=10000):
|
| 14 |
super().__init__()
|
| 15 |
assert dim % 2 == 0
|
| 16 |
+
|
| 17 |
self.dim = dim
|
| 18 |
self.max_seq_len = max_seq_len
|
| 19 |
self.base = base
|
|
|
|
| 37 |
def apply_rotary_pos_emb(self, q, k, seq_len):
|
| 38 |
assert q.shape[-1] == self.dim, f"Expected q.shape[-1] == {self.dim}, got {q.shape[-1]}"
|
| 39 |
assert k.shape[-1] == self.dim, f"Expected k.shape[-1] == {self.dim}, got {k.shape[-1]}"
|
| 40 |
+
assert seq_len <= self.max_seq_len, f"seq_len {seq_len} exceeds max_seq_len {self.max_seq_len}"
|
| 41 |
|
| 42 |
device = q.device
|
| 43 |
cos, sin = self._get_freqs(seq_len, device) # both [seq_len, dim//2]
|
|
|
|
| 108 |
os.remove(self.db_filepath)
|
| 109 |
except:
|
| 110 |
pass
|
| 111 |
+
|
| 112 |
+
class XLAttention(nn.Module):
|
| 113 |
+
def __init__(self, config):
|
| 114 |
+
super().__init__()
|
| 115 |
+
assert config.n_embd % config.n_head == 0
|
| 116 |
+
self.n_head = config.n_head
|
| 117 |
+
self.n_embd = config.n_embd
|
| 118 |
+
self.head_dim = config.n_embd // config.n_head
|
| 119 |
+
self.dropout = nn.Dropout(config.dropout if hasattr(config, 'dropout') else 0.0)
|
| 120 |
+
self.scale = self.head_dim ** -0.5
|
| 121 |
+
|
| 122 |
+
self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
|
| 123 |
+
self.c_proj = nn.Linear(config.n_embd, config.n_embd)
|
| 124 |
+
self.c_proj.MEMGPT_SCALE_INIT = 1
|
| 125 |
+
|
| 126 |
+
self.rope = RotaryPositionalEncoding(self.head_dim, config.block_size)
|
| 127 |
+
|
| 128 |
+
def forward(self, x, xl_memory=None):
|
| 129 |
+
B, T, C = x.size()
|
| 130 |
+
|
| 131 |
+
qkv = self.c_attn(x) # (B,T,3C)
|
| 132 |
+
q, k, v = qkv.split(self.n_embd, dim=2) # (B,T,C)
|
| 133 |
+
|
| 134 |
+
# Handle XL memory
|
| 135 |
+
if xl_memory is not None:
|
| 136 |
+
k_xl, v_xl = xl_memory.unbind(dim=-2)
|
| 137 |
+
k = torch.cat((k_xl, k), dim=1)
|
| 138 |
+
v = torch.cat((v_xl, v), dim=1)
|
| 139 |
+
xl_seq_len = k_xl.shape[1]
|
| 140 |
+
|
| 141 |
+
# Reshape for multi-head attention
|
| 142 |
+
q = q.view(B, T, self.n_head, self.head_dim).transpose(1, 2) # (B, nh, T, hs)
|
| 143 |
+
k = k.view(B, -1, self.n_head, self.head_dim).transpose(1, 2) # (B, nh, T+xl, hs)
|
| 144 |
+
v = v.view(B, -1, self.n_head, self.head_dim).transpose(1, 2) # (B, nh, T+xl, hs)
|
| 145 |
+
|
| 146 |
+
# Apply rotary positional encoding
|
| 147 |
+
seq_len = k.shape[2]
|
| 148 |
+
q, k = self.rope.apply_rotary_pos_emb(q, k, seq_len)
|
| 149 |
+
|
| 150 |
+
# Attention computation
|
| 151 |
+
att = (q @ k.transpose(-2, -1)) * self.scale
|
| 152 |
+
|
| 153 |
+
# Causal mask
|
| 154 |
+
mask = torch.tril(torch.ones(T, seq_len, device=x.device, dtype=torch.bool))
|
| 155 |
+
att = att.masked_fill(~mask, float('-inf'))
|
| 156 |
+
|
| 157 |
+
att = F.softmax(att, dim=-1)
|
| 158 |
+
att = self.dropout(att)
|
| 159 |
+
|
| 160 |
+
y = att @ v # (B, nh, T, hs)
|
| 161 |
+
y = y.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)
|
| 162 |
+
|
| 163 |
+
y = self.c_proj(y)
|
| 164 |
+
|
| 165 |
+
# Prepare new XL memories
|
| 166 |
+
k = k.transpose(1, 2).contiguous().view(B, -1, C)
|
| 167 |
+
v = v.transpose(1, 2).contiguous().view(B, -1, C)
|
| 168 |
+
kv_memories = torch.stack((k, v), dim=-2)
|
| 169 |
+
|
| 170 |
+
if xl_memory is not None:
|
| 171 |
+
current_kv = kv_memories[:, -xl_seq_len:]
|
| 172 |
+
else:
|
| 173 |
+
current_kv = kv_memories
|
| 174 |
+
|
| 175 |
+
return y, current_kv
|
| 176 |
+
|
| 177 |
+
|
| 178 |
|
| 179 |
|
| 180 |
|