Nicholasbass commited on
Commit
0992e4e
·
verified ·
1 Parent(s): 35e1568

Create saat_transformer.py

Browse files
Files changed (1) hide show
  1. saat_transformer.py +191 -0
saat_transformer.py ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Standalone SAAT Transformer
4
+ - No Qwen / HF deps
5
+ - Transformer + per-layer Working Memory + Progress/Halt heads
6
+ - Causal mask inside forward
7
+ """
8
+
9
+ import math
10
+ import torch
11
+ import torch.nn as nn
12
+ import torch.nn.functional as F
13
+
14
+
15
+ def build_causal_mask(T: int, device):
16
+ # [1, 1, T, T] to broadcast over (B, H, T, T)
17
+ mask = torch.full((T, T), float("-inf"), device=device)
18
+ mask = torch.triu(mask, diagonal=1) # disallow attending to future
19
+ return mask
20
+
21
+
22
+ class MultiHeadAttention(nn.Module):
23
+ def __init__(self, d_model: int, num_heads: int, dropout: float = 0.1):
24
+ super().__init__()
25
+ assert d_model % num_heads == 0
26
+ self.h = num_heads
27
+ self.d_head = d_model // num_heads
28
+
29
+ self.qkv = nn.Linear(d_model, 3 * d_model)
30
+ self.proj = nn.Linear(d_model, d_model)
31
+ self.drop = nn.Dropout(dropout)
32
+
33
+ def forward(self, x, attn_bias=None):
34
+ # x: [B, T, C]
35
+ B, T, C = x.shape
36
+ qkv = self.qkv(x) # [B, T, 3C]
37
+ qkv = qkv.view(B, T, 3, self.h, self.d_head).permute(2, 0, 3, 1, 4)
38
+ q, k, v = qkv[0], qkv[1], qkv[2] # each [B, H, T, Dh]
39
+
40
+ att = (q @ k.transpose(-2, -1)) / math.sqrt(self.d_head) # [B,H,T,T]
41
+ if attn_bias is not None:
42
+ att = att + attn_bias # add causal mask (-inf above diagonal)
43
+ att = F.softmax(att, dim=-1)
44
+ att = self.drop(att)
45
+
46
+ y = att @ v # [B,H,T,Dh]
47
+ y = y.transpose(1, 2).contiguous().view(B, T, C) # [B,T,C]
48
+ y = self.proj(y)
49
+ return y
50
+
51
+
52
+ class FeedForward(nn.Module):
53
+ def __init__(self, d_model: int, d_ff: int, dropout: float = 0.1):
54
+ super().__init__()
55
+ self.fc1 = nn.Linear(d_model, d_ff)
56
+ self.fc2 = nn.Linear(d_ff, d_model)
57
+ self.drop = nn.Dropout(dropout)
58
+
59
+ def forward(self, x):
60
+ return self.fc2(self.drop(F.gelu(self.fc1(x))))
61
+
62
+
63
+ class WorkingMemory(nn.Module):
64
+ """
65
+ Per-layer learnable working-memory slots, appended to the sequence.
66
+ Slot width == d_model so downstream modules don't need adapters.
67
+ """
68
+ def __init__(self, num_slots: int, d_model: int):
69
+ super().__init__()
70
+ self.slots = nn.Parameter(torch.randn(num_slots, d_model) * 0.02)
71
+
72
+ def forward(self, x):
73
+ # x: [B,T,C] -> concat slots: [B, T+S, C]
74
+ B, _, _ = x.shape
75
+ s = self.slots.unsqueeze(0).expand(B, -1, -1) # [B,S,C]
76
+ return torch.cat([x, s], dim=1), s.shape[1] # return S for masking
77
+
78
+
79
+ class SAATLayer(nn.Module):
80
+ def __init__(self, d_model: int, num_heads: int, d_ff: int, wm_slots: int, dropout: float = 0.1):
81
+ super().__init__()
82
+ self.wm = WorkingMemory(num_slots=wm_slots, d_model=d_model)
83
+
84
+ self.norm1 = nn.LayerNorm(d_model)
85
+ self.attn = MultiHeadAttention(d_model, num_heads, dropout)
86
+ self.norm2 = nn.LayerNorm(d_model)
87
+ self.ff = FeedForward(d_model, d_ff, dropout)
88
+
89
+ # Control heads (average across tokens later)
90
+ self.progress_head = nn.Linear(d_model, 1)
91
+ self.halt_head = nn.Linear(d_model, 1)
92
+
93
+ def forward(self, x, causal_bias):
94
+ # Append WM
95
+ x, S = self.wm(x) # x: [B, T+S, C]
96
+
97
+ # Build an extended causal mask for T+S
98
+ # New tokens (the WM) are appended at the end; they can attend to past, but not "future".
99
+ TplusS = x.size(1)
100
+ dev = x.device
101
+ ext_bias = build_causal_mask(TplusS, dev) # [T+S,T+S]
102
+ ext_bias = ext_bias.unsqueeze(0).unsqueeze(0) # [1,1,T+S,T+S]
103
+
104
+ # Attention + residual
105
+ h = self.attn(self.norm1(x), ext_bias)
106
+ x = x + h
107
+
108
+ # FFN + residual
109
+ h = self.ff(self.norm2(x))
110
+ x = x + h
111
+
112
+ # Control signals (mean over sequence)
113
+ prog = torch.sigmoid(self.progress_head(x)).mean(dim=1) # [B,1]
114
+ halt = torch.sigmoid(self.halt_head(x)).mean(dim=1) # [B,1]
115
+ return x, {"progress": prog, "halt": halt}
116
+
117
+
118
+ class SAATTransformer(nn.Module):
119
+ def __init__(
120
+ self,
121
+ vocab_size: int,
122
+ d_model: int = 512,
123
+ n_layers: int = 12,
124
+ num_heads: int = 8,
125
+ d_ff: int = 2048,
126
+ wm_slots: int = 4,
127
+ dropout: float = 0.1,
128
+ max_len: int = 2048,
129
+ pad_id: int = 256,
130
+ eos_id: int = 257,
131
+ ):
132
+ super().__init__()
133
+ self.vocab_size = vocab_size
134
+ self.pad_id = pad_id
135
+ self.eos_id = eos_id
136
+ self.max_len = max_len
137
+
138
+ self.tok_embed = nn.Embedding(vocab_size, d_model)
139
+ self.pos_embed = nn.Parameter(torch.zeros(1, max_len, d_model))
140
+ nn.init.normal_(self.pos_embed, std=0.01)
141
+
142
+ self.layers = nn.ModuleList([
143
+ SAATLayer(d_model, num_heads, d_ff, wm_slots, dropout)
144
+ for _ in range(n_layers)
145
+ ])
146
+ self.norm = nn.LayerNorm(d_model)
147
+ self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
148
+
149
+ @torch.no_grad()
150
+ def generate(self, input_ids, max_new_tokens=200, temperature=0.8, top_p=0.9):
151
+ self.eval()
152
+ out = input_ids
153
+ for _ in range(max_new_tokens):
154
+ logits = self.forward(out)["logits"][:, -1, :] # [B,V]
155
+ logits = logits / max(1e-6, temperature)
156
+
157
+ # nucleus sampling
158
+ probs = F.softmax(logits, dim=-1)
159
+ sorted_probs, sorted_idx = torch.sort(probs, descending=True, dim=-1)
160
+ cumsum = torch.cumsum(sorted_probs, dim=-1)
161
+ keep = cumsum <= top_p
162
+ keep[..., 0] = True
163
+ filtered = torch.full_like(sorted_probs, 0.0)
164
+ filtered[keep] = sorted_probs[keep]
165
+ filtered = filtered / filtered.sum(dim=-1, keepdim=True)
166
+ next_sorted = torch.multinomial(filtered, num_samples=1) # [B,1]
167
+ next_token = sorted_idx.gather(-1, next_sorted) # [B,1]
168
+
169
+ out = torch.cat([out, next_token], dim=1)
170
+ if (next_token == self.eos_id).all():
171
+ break
172
+ return out
173
+
174
+ def forward(self, input_ids):
175
+ # input_ids: [B,T]
176
+ B, T = input_ids.shape
177
+ assert T <= self.max_len, f"Sequence length {T} > max_len {self.max_len}"
178
+
179
+ x = self.tok_embed(input_ids) + self.pos_embed[:, :T, :]
180
+
181
+ # base causal mask (for original T). Each layer builds its own extended mask after WM append.
182
+ causal_bias = build_causal_mask(T, x.device).unsqueeze(0).unsqueeze(0)
183
+
184
+ signals = {}
185
+ for i, layer in enumerate(self.layers):
186
+ x, sig = layer(x, causal_bias)
187
+ signals[f"layer_{i}"] = sig
188
+
189
+ x = self.norm(x)
190
+ logits = self.lm_head(x) # [B, T+S, V]
191
+ return {"logits": logits, "signals": signals}