File size: 26,556 Bytes
2dc5456
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
diff --git a/nanochat/checkpoint_manager.py b/nanochat/checkpoint_manager.py
index bafcdee..1e7ce9e 100644
--- a/nanochat/checkpoint_manager.py
+++ b/nanochat/checkpoint_manager.py
@@ -28,6 +28,8 @@ def _patch_missing_config_keys(model_config_kwargs):
 
 def _patch_missing_keys(model_data, model_config):
     """Add default values for new parameters that may be missing in old checkpoints."""
+    if getattr(model_config, "attn_res", False):
+        return # AttnRes models have no resid_lambdas / x0_lambdas
     n_layer = model_config.n_layer
     # resid_lambdas defaults to 1.0 (identity scaling)
     if "resid_lambdas" not in model_data:
diff --git a/nanochat/gpt.py b/nanochat/gpt.py
index 1e89378..a4ab2eb 100644
--- a/nanochat/gpt.py
+++ b/nanochat/gpt.py
@@ -25,6 +25,12 @@ from nanochat.optim import MuonAdamW
 # Our custom Flash Attention module that automatically uses FA3 when compatible and SDPA fallback otherwise
 from nanochat.flash_attention import flash_attn
 
+# Attention Residuals (ported from dl_pro): Triton kernels for CUDA; AttnResTorch below is the reference elsewhere
+try:
+    from nanochat.attn_res import AttnRes
+except ImportError: # no triton (e.g. CPU/MPS-only installs)
+    AttnRes = None
+
 @dataclass
 class GPTConfig:
     sequence_len: int = 2048
@@ -37,6 +43,15 @@ class GPTConfig:
     # Characters: L=long (full context), S=short (quarter context)
     # Examples: "L"=all full context, "SL"=alternating, "SSL"=two short then one long
     window_pattern: str = "SSSL"
+    # Attention Residuals: each sublayer reads a learned softmax mixture over depth (the normed embedding and every
+    # earlier sublayer output) instead of the running residual sum. Replaces resid_lambdas, x0_lambdas and backout,
+    # which are defined on the residual sum; adds 2 * n_layer + 1 zero-init pseudo-query vectors (Adam, own LR, no decay)
+    attn_res: bool = False
+    # Full-bandwidth transformer (dl_pro's FBT): number of trunk passes (1 = off). Pass k > 1 reads
+    # norm(W_t e + W_h norm(h_{t-1})); W_t, W_h: (n_embd, n_embd) Muon matrices, init identity / zero
+    fbt_passes: int = 1
+    # looped FBT: a pass k > 1 also reads its own position's previous-pass state, + W_p norm(h_t) (W_p zero init: FBT)
+    fbt_inplace: bool = False
 
 
 def norm(x):
@@ -153,6 +168,24 @@ class Block(nn.Module):
         return x
 
 
+class AttnResTorch:
+    """AttnRes in plain PyTorch (nanochat.attn_res.reference: stacks every source, O(L^2) memory), with AttnRes's
+    interface, for devices without the Triton kernels."""
+    def __init__(self, x, queries):
+        from nanochat.attn_res import reference
+        self.reference = reference
+        self.us = list(queries)
+        self.v = []
+        self.add(x)
+
+    def add(self, v):
+        assert len(self.v) < len(self.us), "more sources than queries"
+        self.v.append(v)
+
+    def mix(self):
+        return self.reference(self.v, self.us[len(self.v) - 1]).to(self.v[0].dtype)
+
+
 class GPT(nn.Module):
     def __init__(self, config, pad_vocab_size_to=64):
         """
@@ -175,17 +208,30 @@ class GPT(nn.Module):
             "h": nn.ModuleList([Block(config, layer_idx) for layer_idx in range(config.n_layer)]),
         })
         self.lm_head = Linear(config.n_embd, padded_vocab_size, bias=False)
+        # FBT fusion of the token input with the previous pass's state (only with fbt_passes > 1)
+        if config.fbt_passes > 1:
+            self.fbt_t = Linear(config.n_embd, config.n_embd, bias=False)
+            self.fbt_h = Linear(config.n_embd, config.n_embd, bias=False)
+            if config.fbt_inplace:
+                self.fbt_p = Linear(config.n_embd, config.n_embd, bias=False)
         # Per-layer learnable scalars (inspired by modded-nanogpt)
         # resid_lambdas: scales the residual stream at each layer (init 1.0 = neutral)
         # x0_lambdas: blends initial embedding back in at each layer (init 0.0 = disabled)
         # Separate parameters so they can have different optimizer treatment
-        self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer))   # fake init, real init in init_weights()
-        self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer))     # fake init, real init in init_weights()
+        # (with attn_res these act on a residual sum that no longer exists: not created, and every parameter keeps a grad)
+        if not config.attn_res:
+            self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer))   # fake init, real init in init_weights()
+            self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer))     # fake init, real init in init_weights()
         # Smear: mix previous token's embedding into current token (cheap bigram-like info)
         self.smear_gate = Linear(24, 1, bias=False)
         self.smear_lambda = nn.Parameter(torch.zeros(1))
         # Backout: subtract cached mid-layer residual before final norm to remove low-level features
-        self.backout_lambda = nn.Parameter(0.2 * torch.ones(1))
+        if not config.attn_res:
+            self.backout_lambda = nn.Parameter(0.2 * torch.ones(1))
+        # AttnRes pseudo-queries: one per sublayer (attn, mlp of each block), then the readout. Separate (C,) vectors
+        # (as dl_pro): each stays a "small" param for MuonAdamW's distributed AdamW (no shape[0] % world_size rule)
+        if config.attn_res:
+            self.attn_res_queries = nn.ParameterList([nn.Parameter(torch.zeros(config.n_embd)) for _ in range(2 * config.n_layer + 1)])
         # Value embeddings (ResFormer-style): alternating layers, last layer always included
         head_dim = config.n_embd // config.n_head
         kv_dim = config.n_kv_head * head_dim
@@ -231,18 +277,31 @@ class GPT(nn.Module):
             torch.nn.init.uniform_(block.mlp.c_fc.weight, -s * 0.4, s * 0.4)  # 0.4x init scale for c_fc
             torch.nn.init.zeros_(block.mlp.c_proj.weight)
 
+        # FBT fusion: identity on the token, zero on the fed-back state (every pass starts as the plain trunk)
+        if self.config.fbt_passes > 1:
+            torch.nn.init.eye_(self.fbt_t.weight)
+            torch.nn.init.zeros_(self.fbt_h.weight)
+            if self.config.fbt_inplace:
+                torch.nn.init.zeros_(self.fbt_p.weight)
+
         # Per-layer scalars
         # Per-layer resid init: stronger residual at early layers, weaker at deep layers
         n_layer = self.config.n_layer
-        for i in range(n_layer):
-            self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1))
-        # Decaying x0 init: earlier layers get more input embedding blending
-        for i in range(n_layer):
-            self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1))
+        if not self.config.attn_res:
+            for i in range(n_layer):
+                self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1))
+            # Decaying x0 init: earlier layers get more input embedding blending
+            for i in range(n_layer):
+                self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1))
+        else:
+            # AttnRes pseudo-queries: zero = a uniform mixture over the sources at step 0
+            for q in self.attn_res_queries:
+                torch.nn.init.zeros_(q)
 
-        # Smear/backout scalars and smear gate must be explicitly initialized 
+        # Smear/backout scalars and smear gate must be explicitly initialized
         torch.nn.init.zeros_(self.smear_lambda)
-        torch.nn.init.constant_(self.backout_lambda, 0.2)
+        if not self.config.attn_res:
+            torch.nn.init.constant_(self.backout_lambda, 0.2)
         torch.nn.init.uniform_(self.smear_gate.weight, 0.0, 0.02)
 
         # Value embeddings (init like c_v: uniform with same std)
@@ -335,7 +394,7 @@ class GPT(nn.Module):
             window = window_size[0]  # (left, right) tuple, we use left
             effective_seq = t if window < 0 else min(window, t)
             attn_flops += 12 * h * q * effective_seq
-        num_flops_per_token = 6 * self.num_matmul_params() + attn_flops
+        num_flops_per_token = (6 * self.num_matmul_params() + attn_flops) * self.config.fbt_passes # (FBT: every pass, head included)
         return num_flops_per_token
 
     def num_matmul_params(self):
@@ -404,8 +463,13 @@ class GPT(nn.Module):
         value_embeds = sum(p.numel() for p in self.value_embeds.parameters())
         lm_head = sum(p.numel() for p in self.lm_head.parameters())
         transformer_matrices = sum(p.numel() for p in self.transformer.h.parameters())
-        scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel()
-        total = wte + value_embeds + lm_head + transformer_matrices + scalars
+        # FBT fusion matrices: counted apart, so the scaling params (and the token budget) stay the baseline's
+        fbt = sum(p.numel() for n, p in self.named_parameters() if n.startswith("fbt_"))
+        if self.config.attn_res: # (the pseudo-queries count as scalars)
+            scalars = sum(q.numel() for q in self.attn_res_queries) + self.smear_gate.weight.numel() + self.smear_lambda.numel()
+        else:
+            scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel()
+        total = wte + value_embeds + lm_head + transformer_matrices + scalars + fbt
         assert total == sum(p.numel() for p in self.parameters()), "Parameter count mismatch"
         return {
             'wte': wte,
@@ -413,21 +477,30 @@ class GPT(nn.Module):
             'lm_head': lm_head,
             'transformer_matrices': transformer_matrices,
             'scalars': scalars,
+            **({'fbt': fbt} if fbt else {}),
             'total': total,
         }
 
-    def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5):
+    def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5, attn_res_lr=1e-3, attn_res_betas=(0.8, 0.95)):
         model_dim = self.config.n_embd
 
         # Separate out all parameters into groups
         matrix_params = list(self.transformer.h.parameters())
+        if self.config.fbt_passes > 1: # FBT fusion matrices on Muon, as dl_pro (grouped by shape with the others)
+            matrix_params += [self.fbt_t.weight, self.fbt_h.weight] + ([self.fbt_p.weight] if self.config.fbt_inplace else [])
         value_embeds_params = list(self.value_embeds.parameters())
         embedding_params = list(self.transformer.wte.parameters())
         lm_head_params = list(self.lm_head.parameters())
-        resid_params = [self.resid_lambdas]
-        x0_params = [self.x0_lambdas]
-        smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda]
-        assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params)
+        if self.config.attn_res:
+            resid_params, x0_params = [], []
+            smear_params = [self.smear_gate.weight, self.smear_lambda]
+            attn_res_params = list(self.attn_res_queries)
+        else:
+            resid_params = [self.resid_lambdas]
+            x0_params = [self.x0_lambdas]
+            smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda]
+            attn_res_params = []
+        assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params) + len(attn_res_params)
 
         # Scale the LR for the AdamW parameters by ∝1/√dmodel (tuned for 768 dim model)
         dmodel_lr_scale = (model_dim / 768) ** -0.5
@@ -439,10 +512,18 @@ class GPT(nn.Module):
             dict(kind='adamw', params=lm_head_params, lr=unembedding_lr * dmodel_lr_scale, betas=(0.8, 0.96), eps=1e-10, weight_decay=0.01),
             dict(kind='adamw', params=embedding_params, lr=embedding_lr * dmodel_lr_scale, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.001),
             dict(kind='adamw', params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale * 0.5, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.01),
-            dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05),
-            dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0),  # higher beta1 for x0
-            dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0),
         ]
+        if not self.config.attn_res:
+            param_groups += [
+                dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05),
+                dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0),  # higher beta1 for x0
+            ]
+        param_groups.append(dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0))
+        if attn_res_params:
+            # AttnRes pseudo-queries: Adam at attn_res_lr (1e-3, dl_pro's), no weight decay, attn_res_betas (default
+            # (0.8, 0.95): beta1 as nanochat's other Adam groups; dl_pro used 0.65). The LR is used as given (no dmodel /
+            # batch scaling); the schedule multiplier applies as to every group.
+            param_groups.append(dict(kind='adamw', params=attn_res_params, lr=attn_res_lr, betas=tuple(attn_res_betas), eps=1e-10, weight_decay=0.0))
         # Muon groups (matrix params, grouped by shape for stacking)
         for shape in sorted({p.shape for p in matrix_params}):
             group_params = [p for p in matrix_params if p.shape == shape]
@@ -491,22 +572,56 @@ class GPT(nn.Module):
                 gate = self.smear_lambda.to(x.dtype) * torch.sigmoid(self.smear_gate(x[:, :, :24]))
                 x = x + gate * x_pre_smear
 
+        # Full-bandwidth transformer (dl_pro's FBT): fbt_passes > 1 runs the trunk again, each pass on the fusion
+        # norm(W_t e + W_h norm(h_{t-1})) of the token input e with the previous pass's final hidden state one position
+        # back (the first pass sees h = 0). Training: the mean of the passes' losses (as dl_pro); otherwise: the last pass
+        if self.config.fbt_passes > 1:
+            assert kv_cache is None, "FBT has no KV-cache inference path"
+            e, h, losses = x, None, []
+            for _ in range(self.config.fbt_passes):
+                h_prev = torch.zeros_like(e) if h is None else torch.cat([torch.zeros_like(h[:, :1]), h[:, :-1]], dim=1)
+                f = self.fbt_t(e) + self.fbt_h(norm(h_prev))
+                if self.config.fbt_inplace and h is not None:  # looped FBT: + W_p norm(h_t) (not in the first pass)
+                    f = f + self.fbt_p(norm(h))
+                h = self._trunk(norm(f), idx, cos_sin, None)
+                if targets is not None and self.training:
+                    losses.append(self._head(h, targets, loss_reduction))
+            return torch.stack(losses).mean() if losses else self._head(h, targets, loss_reduction)
+
+        x = self._trunk(x, idx, cos_sin, kv_cache)
+        return self._head(x, targets, loss_reduction)
+
+    def _trunk(self, x, idx, cos_sin, kv_cache):
+        """The transformer blocks on the input x (B, T, C): the final normed hidden state."""
         # Forward the trunk of the Transformer
-        x0 = x  # save initial normalized embedding for x0 residual
-        n_layer = self.config.n_layer
-        backout_layer = n_layer // 2  # cache at halfway point
-        x_backout = None
-        for i, block in enumerate(self.transformer.h):
-            x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
-            ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None
-            x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache)
-            if i == backout_layer:
-                x_backout = x
-        # Subtract mid-layer residual to remove low-level features before logit projection
-        if x_backout is not None:
-            x = x - self.backout_lambda.to(x.dtype) * x_backout
-        x = norm(x)
+        if self.config.attn_res:
+            # Attention Residuals: sources = the embedding and every sublayer output; each sublayer (and the readout)
+            # reads norm(softmax-over-depth mixture) with its own pseudo-query, so no pre-norm, x0 blend or backout
+            res = (AttnRes if x.is_cuda and AttnRes is not None else AttnResTorch)(x, self.attn_res_queries)
+            for i, block in enumerate(self.transformer.h):
+                ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None
+                res.add(block.attn(res.mix(), ve, cos_sin, self.window_sizes[i], kv_cache))
+                res.add(block.mlp(res.mix()))
+            x = res.mix() # the readout, already normed
+        else:
+            x0 = x  # save initial normalized embedding for x0 residual
+            n_layer = self.config.n_layer
+            backout_layer = n_layer // 2  # cache at halfway point
+            x_backout = None
+            for i, block in enumerate(self.transformer.h):
+                x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
+                ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None
+                x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache)
+                if i == backout_layer:
+                    x_backout = x
+            # Subtract mid-layer residual to remove low-level features before logit projection
+            if x_backout is not None:
+                x = x - self.backout_lambda.to(x.dtype) * x_backout
+            x = norm(x)
+        return x
 
+    def _head(self, x, targets, loss_reduction):
+        """Logits of the final hidden state x, or the loss when targets are given."""
         # Forward the lm_head (compute logits)
         softcap = 15 # smoothly cap the logits to the range [-softcap, softcap]
         logits = self.lm_head(x) # (B, T, padded_vocab_size) <- very big tensor, large amount of memory
diff --git a/scripts/base_train.py b/scripts/base_train.py
index d1af271..7c0cd69 100644
--- a/scripts/base_train.py
+++ b/scripts/base_train.py
@@ -52,6 +52,12 @@ parser.add_argument("--aspect-ratio", type=int, default=64, help="model_dim = de
 parser.add_argument("--head-dim", type=int, default=128, help="target head dimension for attention")
 parser.add_argument("--max-seq-len", type=int, default=2048, help="max context length")
 parser.add_argument("--window-pattern", type=str, default="SSSL", help="sliding window pattern tiled across layers: L=full, S=half context (e.g. 'SSL')")
+parser.add_argument("--attn-res", action="store_true", help="Attention Residuals (dl_pro port): softmax-over-depth mixtures replace the residual stream")
+parser.add_argument("--attn-res-lr", type=float, default=1e-3, help="Adam LR for the AttnRes pseudo-queries (no weight decay, not batch/dmodel-scaled)")
+parser.add_argument("--attn-res-beta1", type=float, default=0.8, help="Adam beta1 for the AttnRes pseudo-queries")
+parser.add_argument("--attn-res-beta2", type=float, default=0.95, help="Adam beta2 for the AttnRes pseudo-queries")
+parser.add_argument("--fbt-passes", type=int, default=1, help="full-bandwidth transformer (dl_pro FBT): trunk passes, each later one fed the previous pass's state at t-1 (1 = off)")
+parser.add_argument("--fbt-inplace", action="store_true", help="looped FBT: a feedback pass also fuses its own position's previous-pass state (+ W_p norm(h_t), zero init)")
 # Training horizon (only one used, in order of precedence)
 parser.add_argument("--num-iterations", type=int, default=-1, help="explicit number of optimization steps (-1 = disable)")
 parser.add_argument("--target-flops", type=float, default=-1.0, help="calculate num_iterations to reach target_flops (-1 = disable)")
@@ -75,6 +81,7 @@ parser.add_argument("--core-metric-every", type=int, default=2000, help="evaluat
 parser.add_argument("--core-metric-max-per-task", type=int, default=500, help="examples per task for CORE metric")
 parser.add_argument("--sample-every", type=int, default=2000, help="sample from model every N steps (-1 = disable)")
 parser.add_argument("--save-every", type=int, default=-1, help="save checkpoints every N steps (-1 = only at end)")
+parser.add_argument("--log-every", type=int, default=1, help="log training metrics to wandb every N steps (0 = never)")
 # Output
 parser.add_argument("--model-tag", type=str, default=None, help="override model tag for checkpoint directory name")
 args = parser.parse_args()
@@ -98,6 +105,8 @@ print0(f"COMPUTE_DTYPE: {COMPUTE_DTYPE} ({COMPUTE_DTYPE_REASON})")
 # wandb logging init
 use_dummy_wandb = args.run == "dummy" or not master_process
 wandb_run = DummyWandb() if use_dummy_wandb else wandb.init(project="nanochat", name=args.run, config=user_config)
+# Every wandb_run.log below passes step=step, so wandb's own step (the default x-axis) is the training step. Otherwise it
+# counts log calls (train, val and CORE logs at different cadences put e.g. the step-750 val point at x=753).
 
 # Flash Attention status
 from nanochat.flash_attention import USE_FA3
@@ -136,7 +145,7 @@ def build_model_meta(depth):
     config = GPTConfig(
         sequence_len=args.max_seq_len, vocab_size=vocab_size,
         n_layer=depth, n_head=num_heads, n_kv_head=num_heads, n_embd=model_dim,
-        window_pattern=args.window_pattern,
+        window_pattern=args.window_pattern, attn_res=args.attn_res, fbt_passes=args.fbt_passes, fbt_inplace=args.fbt_inplace,
     )
     with torch.device("meta"):
         model_meta = GPT(config)
@@ -313,6 +322,9 @@ optimizer = model.setup_optimizer(
     # Muon hyperparameters
     matrix_lr=args.matrix_lr * batch_lr_scale,
     weight_decay=weight_decay_scaled,
+    # AttnRes pseudo-queries (only with --attn-res)
+    attn_res_lr=args.attn_res_lr,
+    attn_res_betas=(args.attn_res_beta1, args.attn_res_beta2),
 )
 
 if resuming:
@@ -432,7 +444,7 @@ while True:
             "total_training_flops": flops_so_far,
             "total_training_time": total_training_time,
             "val/bpb": val_bpb,
-        })
+        }, step=step)
         model.train()
 
     # once in a while: estimate the CORE metric (all ranks participate)
@@ -449,12 +461,12 @@ while True:
             "total_training_flops": flops_so_far,
             "core_metric": results["core_metric"],
             "centered_results": results["centered_results"],
-        })
+        }, step=step)
         model.train()
 
     # once in a while: sample from the model (only on master process)
     # use the original uncompiled model because the inputs keep changing shape
-    if args.sample_every > 0 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)):
+    if args.sample_every > 0 and args.fbt_passes == 1 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)): # (FBT: no KV-cache sampler)
         model.eval()
         prompts = [
             "The capital of France is",
@@ -507,9 +519,11 @@ while True:
     # evaluate the gradient
     synchronize()
     t0 = time.time()
+    loss_sum = 0 # this step's loss summed over micro-batches (rank 0's data), for logging
     for micro_step in range(grad_accum_steps):
         loss = model(x, y)
         train_loss = loss.detach() # for logging
+        loss_sum = loss_sum + train_loss
         loss = loss / grad_accum_steps # each .backward() is a grad sum => normalize loss here
         if scaler is not None:
             scaler.scale(loss).backward()
@@ -538,7 +552,7 @@ while True:
     else:
         optimizer.step()
     model.zero_grad(set_to_none=True)
-    train_loss_f = train_loss.item() # .item() is a CPU-GPU sync point
+    train_loss_f, step_loss_f = torch.stack([train_loss, loss_sum / grad_accum_steps]).tolist() # one CPU-GPU sync point
     synchronize()
     t1 = time.time()
     dt = t1 - t0
@@ -565,19 +579,32 @@ while True:
         eta_str = ""
     epoch = f"{dataloader_state_dict['epoch']} pq: {dataloader_state_dict['pq_idx']} rg: {dataloader_state_dict['rg_idx']}"
     print0(f"step {step:05d}/{num_iterations:05d} ({pct_done:.2f}%) | loss: {debiased_smooth_loss:.6f} | lrm: {lrm:.2f} | dt: {dt * 1000:.2f}ms | tok/sec: {tok_per_sec:,} | bf16_mfu: {mfu:.2f} | epoch: {epoch} | total time: {total_training_time/60:.2f}m{eta_str}")
-    if step % 100 == 0:
+    if args.log_every > 0 and step % args.log_every == 0:
+        # every metric of the step, at the step (see the wandb init above); master process only (others: dummy)
         log_data = {
             "step": step,
             "total_training_flops": flops_so_far,
             "total_training_time": total_training_time,
-            "train/loss": debiased_smooth_loss,
+            "train/loss": debiased_smooth_loss, # EMA (as printed)
+            "train/loss_step": step_loss_f, # this step's mean loss over its micro-batches, unsmoothed
+            "train/tokens": total_batch_size * (step + 1),
             "train/lrm": lrm,
+            "train/lr_matrix": args.matrix_lr * batch_lr_scale * lrm,
+            "train/muon_momentum": muon_momentum,
+            "train/muon_weight_decay": muon_weight_decay,
             "train/dt": dt,
             "train/tok_per_sec": tok_per_sec,
             "train/mfu": mfu,
-            "train/epoch": epoch,
+            "train/peak_mem_gib": get_max_memory() / 2**30,
+            "data/epoch": dataloader_state_dict["epoch"],
+            "data/pq_idx": dataloader_state_dict["pq_idx"],
+            "data/rg_idx": dataloader_state_dict["rg_idx"],
         }
-        wandb_run.log(log_data)
+        if args.attn_res:
+            queries = torch.stack(list(orig_model.attn_res_queries))
+            log_data["train/lr_attn_res"] = args.attn_res_lr * lrm
+            log_data["attn_res/query_rms"] = queries.float().square().mean().sqrt().item()
+        wandb_run.log(log_data, step=step)
 
     # state update
     first_step_of_run = (step == 0) or (resuming and step == args.resume_from_step)