diff --git a/nanochat/checkpoint_manager.py b/nanochat/checkpoint_manager.py index bafcdee..1e7ce9e 100644 --- a/nanochat/checkpoint_manager.py +++ b/nanochat/checkpoint_manager.py @@ -28,6 +28,8 @@ def _patch_missing_config_keys(model_config_kwargs): def _patch_missing_keys(model_data, model_config): """Add default values for new parameters that may be missing in old checkpoints.""" + if getattr(model_config, "attn_res", False): + return # AttnRes models have no resid_lambdas / x0_lambdas n_layer = model_config.n_layer # resid_lambdas defaults to 1.0 (identity scaling) if "resid_lambdas" not in model_data: diff --git a/nanochat/gpt.py b/nanochat/gpt.py index 1e89378..a4ab2eb 100644 --- a/nanochat/gpt.py +++ b/nanochat/gpt.py @@ -25,6 +25,12 @@ from nanochat.optim import MuonAdamW # Our custom Flash Attention module that automatically uses FA3 when compatible and SDPA fallback otherwise from nanochat.flash_attention import flash_attn +# Attention Residuals (ported from dl_pro): Triton kernels for CUDA; AttnResTorch below is the reference elsewhere +try: + from nanochat.attn_res import AttnRes +except ImportError: # no triton (e.g. CPU/MPS-only installs) + AttnRes = None + @dataclass class GPTConfig: sequence_len: int = 2048 @@ -37,6 +43,15 @@ class GPTConfig: # Characters: L=long (full context), S=short (quarter context) # Examples: "L"=all full context, "SL"=alternating, "SSL"=two short then one long window_pattern: str = "SSSL" + # Attention Residuals: each sublayer reads a learned softmax mixture over depth (the normed embedding and every + # earlier sublayer output) instead of the running residual sum. Replaces resid_lambdas, x0_lambdas and backout, + # which are defined on the residual sum; adds 2 * n_layer + 1 zero-init pseudo-query vectors (Adam, own LR, no decay) + attn_res: bool = False + # Full-bandwidth transformer (dl_pro's FBT): number of trunk passes (1 = off). Pass k > 1 reads + # norm(W_t e + W_h norm(h_{t-1})); W_t, W_h: (n_embd, n_embd) Muon matrices, init identity / zero + fbt_passes: int = 1 + # looped FBT: a pass k > 1 also reads its own position's previous-pass state, + W_p norm(h_t) (W_p zero init: FBT) + fbt_inplace: bool = False def norm(x): @@ -153,6 +168,24 @@ class Block(nn.Module): return x +class AttnResTorch: + """AttnRes in plain PyTorch (nanochat.attn_res.reference: stacks every source, O(L^2) memory), with AttnRes's + interface, for devices without the Triton kernels.""" + def __init__(self, x, queries): + from nanochat.attn_res import reference + self.reference = reference + self.us = list(queries) + self.v = [] + self.add(x) + + def add(self, v): + assert len(self.v) < len(self.us), "more sources than queries" + self.v.append(v) + + def mix(self): + return self.reference(self.v, self.us[len(self.v) - 1]).to(self.v[0].dtype) + + class GPT(nn.Module): def __init__(self, config, pad_vocab_size_to=64): """ @@ -175,17 +208,30 @@ class GPT(nn.Module): "h": nn.ModuleList([Block(config, layer_idx) for layer_idx in range(config.n_layer)]), }) self.lm_head = Linear(config.n_embd, padded_vocab_size, bias=False) + # FBT fusion of the token input with the previous pass's state (only with fbt_passes > 1) + if config.fbt_passes > 1: + self.fbt_t = Linear(config.n_embd, config.n_embd, bias=False) + self.fbt_h = Linear(config.n_embd, config.n_embd, bias=False) + if config.fbt_inplace: + self.fbt_p = Linear(config.n_embd, config.n_embd, bias=False) # Per-layer learnable scalars (inspired by modded-nanogpt) # resid_lambdas: scales the residual stream at each layer (init 1.0 = neutral) # x0_lambdas: blends initial embedding back in at each layer (init 0.0 = disabled) # Separate parameters so they can have different optimizer treatment - self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) # fake init, real init in init_weights() - self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) # fake init, real init in init_weights() + # (with attn_res these act on a residual sum that no longer exists: not created, and every parameter keeps a grad) + if not config.attn_res: + self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) # fake init, real init in init_weights() + self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) # fake init, real init in init_weights() # Smear: mix previous token's embedding into current token (cheap bigram-like info) self.smear_gate = Linear(24, 1, bias=False) self.smear_lambda = nn.Parameter(torch.zeros(1)) # Backout: subtract cached mid-layer residual before final norm to remove low-level features - self.backout_lambda = nn.Parameter(0.2 * torch.ones(1)) + if not config.attn_res: + self.backout_lambda = nn.Parameter(0.2 * torch.ones(1)) + # AttnRes pseudo-queries: one per sublayer (attn, mlp of each block), then the readout. Separate (C,) vectors + # (as dl_pro): each stays a "small" param for MuonAdamW's distributed AdamW (no shape[0] % world_size rule) + if config.attn_res: + self.attn_res_queries = nn.ParameterList([nn.Parameter(torch.zeros(config.n_embd)) for _ in range(2 * config.n_layer + 1)]) # Value embeddings (ResFormer-style): alternating layers, last layer always included head_dim = config.n_embd // config.n_head kv_dim = config.n_kv_head * head_dim @@ -231,18 +277,31 @@ class GPT(nn.Module): torch.nn.init.uniform_(block.mlp.c_fc.weight, -s * 0.4, s * 0.4) # 0.4x init scale for c_fc torch.nn.init.zeros_(block.mlp.c_proj.weight) + # FBT fusion: identity on the token, zero on the fed-back state (every pass starts as the plain trunk) + if self.config.fbt_passes > 1: + torch.nn.init.eye_(self.fbt_t.weight) + torch.nn.init.zeros_(self.fbt_h.weight) + if self.config.fbt_inplace: + torch.nn.init.zeros_(self.fbt_p.weight) + # Per-layer scalars # Per-layer resid init: stronger residual at early layers, weaker at deep layers n_layer = self.config.n_layer - for i in range(n_layer): - self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1)) - # Decaying x0 init: earlier layers get more input embedding blending - for i in range(n_layer): - self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1)) + if not self.config.attn_res: + for i in range(n_layer): + self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1)) + # Decaying x0 init: earlier layers get more input embedding blending + for i in range(n_layer): + self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1)) + else: + # AttnRes pseudo-queries: zero = a uniform mixture over the sources at step 0 + for q in self.attn_res_queries: + torch.nn.init.zeros_(q) - # Smear/backout scalars and smear gate must be explicitly initialized + # Smear/backout scalars and smear gate must be explicitly initialized torch.nn.init.zeros_(self.smear_lambda) - torch.nn.init.constant_(self.backout_lambda, 0.2) + if not self.config.attn_res: + torch.nn.init.constant_(self.backout_lambda, 0.2) torch.nn.init.uniform_(self.smear_gate.weight, 0.0, 0.02) # Value embeddings (init like c_v: uniform with same std) @@ -335,7 +394,7 @@ class GPT(nn.Module): window = window_size[0] # (left, right) tuple, we use left effective_seq = t if window < 0 else min(window, t) attn_flops += 12 * h * q * effective_seq - num_flops_per_token = 6 * self.num_matmul_params() + attn_flops + num_flops_per_token = (6 * self.num_matmul_params() + attn_flops) * self.config.fbt_passes # (FBT: every pass, head included) return num_flops_per_token def num_matmul_params(self): @@ -404,8 +463,13 @@ class GPT(nn.Module): value_embeds = sum(p.numel() for p in self.value_embeds.parameters()) lm_head = sum(p.numel() for p in self.lm_head.parameters()) transformer_matrices = sum(p.numel() for p in self.transformer.h.parameters()) - scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel() - total = wte + value_embeds + lm_head + transformer_matrices + scalars + # FBT fusion matrices: counted apart, so the scaling params (and the token budget) stay the baseline's + fbt = sum(p.numel() for n, p in self.named_parameters() if n.startswith("fbt_")) + if self.config.attn_res: # (the pseudo-queries count as scalars) + scalars = sum(q.numel() for q in self.attn_res_queries) + self.smear_gate.weight.numel() + self.smear_lambda.numel() + else: + scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel() + total = wte + value_embeds + lm_head + transformer_matrices + scalars + fbt assert total == sum(p.numel() for p in self.parameters()), "Parameter count mismatch" return { 'wte': wte, @@ -413,21 +477,30 @@ class GPT(nn.Module): 'lm_head': lm_head, 'transformer_matrices': transformer_matrices, 'scalars': scalars, + **({'fbt': fbt} if fbt else {}), 'total': total, } - def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5): + def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5, attn_res_lr=1e-3, attn_res_betas=(0.8, 0.95)): model_dim = self.config.n_embd # Separate out all parameters into groups matrix_params = list(self.transformer.h.parameters()) + if self.config.fbt_passes > 1: # FBT fusion matrices on Muon, as dl_pro (grouped by shape with the others) + matrix_params += [self.fbt_t.weight, self.fbt_h.weight] + ([self.fbt_p.weight] if self.config.fbt_inplace else []) value_embeds_params = list(self.value_embeds.parameters()) embedding_params = list(self.transformer.wte.parameters()) lm_head_params = list(self.lm_head.parameters()) - resid_params = [self.resid_lambdas] - x0_params = [self.x0_lambdas] - smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda] - assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params) + if self.config.attn_res: + resid_params, x0_params = [], [] + smear_params = [self.smear_gate.weight, self.smear_lambda] + attn_res_params = list(self.attn_res_queries) + else: + resid_params = [self.resid_lambdas] + x0_params = [self.x0_lambdas] + smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda] + attn_res_params = [] + assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params) + len(attn_res_params) # Scale the LR for the AdamW parameters by ∝1/√dmodel (tuned for 768 dim model) dmodel_lr_scale = (model_dim / 768) ** -0.5 @@ -439,10 +512,18 @@ class GPT(nn.Module): dict(kind='adamw', params=lm_head_params, lr=unembedding_lr * dmodel_lr_scale, betas=(0.8, 0.96), eps=1e-10, weight_decay=0.01), dict(kind='adamw', params=embedding_params, lr=embedding_lr * dmodel_lr_scale, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.001), dict(kind='adamw', params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale * 0.5, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.01), - dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05), - dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), # higher beta1 for x0 - dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0), ] + if not self.config.attn_res: + param_groups += [ + dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05), + dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), # higher beta1 for x0 + ] + param_groups.append(dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0)) + if attn_res_params: + # AttnRes pseudo-queries: Adam at attn_res_lr (1e-3, dl_pro's), no weight decay, attn_res_betas (default + # (0.8, 0.95): beta1 as nanochat's other Adam groups; dl_pro used 0.65). The LR is used as given (no dmodel / + # batch scaling); the schedule multiplier applies as to every group. + param_groups.append(dict(kind='adamw', params=attn_res_params, lr=attn_res_lr, betas=tuple(attn_res_betas), eps=1e-10, weight_decay=0.0)) # Muon groups (matrix params, grouped by shape for stacking) for shape in sorted({p.shape for p in matrix_params}): group_params = [p for p in matrix_params if p.shape == shape] @@ -491,22 +572,56 @@ class GPT(nn.Module): gate = self.smear_lambda.to(x.dtype) * torch.sigmoid(self.smear_gate(x[:, :, :24])) x = x + gate * x_pre_smear + # Full-bandwidth transformer (dl_pro's FBT): fbt_passes > 1 runs the trunk again, each pass on the fusion + # norm(W_t e + W_h norm(h_{t-1})) of the token input e with the previous pass's final hidden state one position + # back (the first pass sees h = 0). Training: the mean of the passes' losses (as dl_pro); otherwise: the last pass + if self.config.fbt_passes > 1: + assert kv_cache is None, "FBT has no KV-cache inference path" + e, h, losses = x, None, [] + for _ in range(self.config.fbt_passes): + h_prev = torch.zeros_like(e) if h is None else torch.cat([torch.zeros_like(h[:, :1]), h[:, :-1]], dim=1) + f = self.fbt_t(e) + self.fbt_h(norm(h_prev)) + if self.config.fbt_inplace and h is not None: # looped FBT: + W_p norm(h_t) (not in the first pass) + f = f + self.fbt_p(norm(h)) + h = self._trunk(norm(f), idx, cos_sin, None) + if targets is not None and self.training: + losses.append(self._head(h, targets, loss_reduction)) + return torch.stack(losses).mean() if losses else self._head(h, targets, loss_reduction) + + x = self._trunk(x, idx, cos_sin, kv_cache) + return self._head(x, targets, loss_reduction) + + def _trunk(self, x, idx, cos_sin, kv_cache): + """The transformer blocks on the input x (B, T, C): the final normed hidden state.""" # Forward the trunk of the Transformer - x0 = x # save initial normalized embedding for x0 residual - n_layer = self.config.n_layer - backout_layer = n_layer // 2 # cache at halfway point - x_backout = None - for i, block in enumerate(self.transformer.h): - x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0 - ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None - x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache) - if i == backout_layer: - x_backout = x - # Subtract mid-layer residual to remove low-level features before logit projection - if x_backout is not None: - x = x - self.backout_lambda.to(x.dtype) * x_backout - x = norm(x) + if self.config.attn_res: + # Attention Residuals: sources = the embedding and every sublayer output; each sublayer (and the readout) + # reads norm(softmax-over-depth mixture) with its own pseudo-query, so no pre-norm, x0 blend or backout + res = (AttnRes if x.is_cuda and AttnRes is not None else AttnResTorch)(x, self.attn_res_queries) + for i, block in enumerate(self.transformer.h): + ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None + res.add(block.attn(res.mix(), ve, cos_sin, self.window_sizes[i], kv_cache)) + res.add(block.mlp(res.mix())) + x = res.mix() # the readout, already normed + else: + x0 = x # save initial normalized embedding for x0 residual + n_layer = self.config.n_layer + backout_layer = n_layer // 2 # cache at halfway point + x_backout = None + for i, block in enumerate(self.transformer.h): + x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0 + ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None + x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache) + if i == backout_layer: + x_backout = x + # Subtract mid-layer residual to remove low-level features before logit projection + if x_backout is not None: + x = x - self.backout_lambda.to(x.dtype) * x_backout + x = norm(x) + return x + def _head(self, x, targets, loss_reduction): + """Logits of the final hidden state x, or the loss when targets are given.""" # Forward the lm_head (compute logits) softcap = 15 # smoothly cap the logits to the range [-softcap, softcap] logits = self.lm_head(x) # (B, T, padded_vocab_size) <- very big tensor, large amount of memory diff --git a/scripts/base_train.py b/scripts/base_train.py index d1af271..7c0cd69 100644 --- a/scripts/base_train.py +++ b/scripts/base_train.py @@ -52,6 +52,12 @@ parser.add_argument("--aspect-ratio", type=int, default=64, help="model_dim = de parser.add_argument("--head-dim", type=int, default=128, help="target head dimension for attention") parser.add_argument("--max-seq-len", type=int, default=2048, help="max context length") parser.add_argument("--window-pattern", type=str, default="SSSL", help="sliding window pattern tiled across layers: L=full, S=half context (e.g. 'SSL')") +parser.add_argument("--attn-res", action="store_true", help="Attention Residuals (dl_pro port): softmax-over-depth mixtures replace the residual stream") +parser.add_argument("--attn-res-lr", type=float, default=1e-3, help="Adam LR for the AttnRes pseudo-queries (no weight decay, not batch/dmodel-scaled)") +parser.add_argument("--attn-res-beta1", type=float, default=0.8, help="Adam beta1 for the AttnRes pseudo-queries") +parser.add_argument("--attn-res-beta2", type=float, default=0.95, help="Adam beta2 for the AttnRes pseudo-queries") +parser.add_argument("--fbt-passes", type=int, default=1, help="full-bandwidth transformer (dl_pro FBT): trunk passes, each later one fed the previous pass's state at t-1 (1 = off)") +parser.add_argument("--fbt-inplace", action="store_true", help="looped FBT: a feedback pass also fuses its own position's previous-pass state (+ W_p norm(h_t), zero init)") # Training horizon (only one used, in order of precedence) parser.add_argument("--num-iterations", type=int, default=-1, help="explicit number of optimization steps (-1 = disable)") parser.add_argument("--target-flops", type=float, default=-1.0, help="calculate num_iterations to reach target_flops (-1 = disable)") @@ -75,6 +81,7 @@ parser.add_argument("--core-metric-every", type=int, default=2000, help="evaluat parser.add_argument("--core-metric-max-per-task", type=int, default=500, help="examples per task for CORE metric") parser.add_argument("--sample-every", type=int, default=2000, help="sample from model every N steps (-1 = disable)") parser.add_argument("--save-every", type=int, default=-1, help="save checkpoints every N steps (-1 = only at end)") +parser.add_argument("--log-every", type=int, default=1, help="log training metrics to wandb every N steps (0 = never)") # Output parser.add_argument("--model-tag", type=str, default=None, help="override model tag for checkpoint directory name") args = parser.parse_args() @@ -98,6 +105,8 @@ print0(f"COMPUTE_DTYPE: {COMPUTE_DTYPE} ({COMPUTE_DTYPE_REASON})") # wandb logging init use_dummy_wandb = args.run == "dummy" or not master_process wandb_run = DummyWandb() if use_dummy_wandb else wandb.init(project="nanochat", name=args.run, config=user_config) +# Every wandb_run.log below passes step=step, so wandb's own step (the default x-axis) is the training step. Otherwise it +# counts log calls (train, val and CORE logs at different cadences put e.g. the step-750 val point at x=753). # Flash Attention status from nanochat.flash_attention import USE_FA3 @@ -136,7 +145,7 @@ def build_model_meta(depth): config = GPTConfig( sequence_len=args.max_seq_len, vocab_size=vocab_size, n_layer=depth, n_head=num_heads, n_kv_head=num_heads, n_embd=model_dim, - window_pattern=args.window_pattern, + window_pattern=args.window_pattern, attn_res=args.attn_res, fbt_passes=args.fbt_passes, fbt_inplace=args.fbt_inplace, ) with torch.device("meta"): model_meta = GPT(config) @@ -313,6 +322,9 @@ optimizer = model.setup_optimizer( # Muon hyperparameters matrix_lr=args.matrix_lr * batch_lr_scale, weight_decay=weight_decay_scaled, + # AttnRes pseudo-queries (only with --attn-res) + attn_res_lr=args.attn_res_lr, + attn_res_betas=(args.attn_res_beta1, args.attn_res_beta2), ) if resuming: @@ -432,7 +444,7 @@ while True: "total_training_flops": flops_so_far, "total_training_time": total_training_time, "val/bpb": val_bpb, - }) + }, step=step) model.train() # once in a while: estimate the CORE metric (all ranks participate) @@ -449,12 +461,12 @@ while True: "total_training_flops": flops_so_far, "core_metric": results["core_metric"], "centered_results": results["centered_results"], - }) + }, step=step) model.train() # once in a while: sample from the model (only on master process) # use the original uncompiled model because the inputs keep changing shape - if args.sample_every > 0 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)): + if args.sample_every > 0 and args.fbt_passes == 1 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)): # (FBT: no KV-cache sampler) model.eval() prompts = [ "The capital of France is", @@ -507,9 +519,11 @@ while True: # evaluate the gradient synchronize() t0 = time.time() + loss_sum = 0 # this step's loss summed over micro-batches (rank 0's data), for logging for micro_step in range(grad_accum_steps): loss = model(x, y) train_loss = loss.detach() # for logging + loss_sum = loss_sum + train_loss loss = loss / grad_accum_steps # each .backward() is a grad sum => normalize loss here if scaler is not None: scaler.scale(loss).backward() @@ -538,7 +552,7 @@ while True: else: optimizer.step() model.zero_grad(set_to_none=True) - train_loss_f = train_loss.item() # .item() is a CPU-GPU sync point + train_loss_f, step_loss_f = torch.stack([train_loss, loss_sum / grad_accum_steps]).tolist() # one CPU-GPU sync point synchronize() t1 = time.time() dt = t1 - t0 @@ -565,19 +579,32 @@ while True: eta_str = "" epoch = f"{dataloader_state_dict['epoch']} pq: {dataloader_state_dict['pq_idx']} rg: {dataloader_state_dict['rg_idx']}" print0(f"step {step:05d}/{num_iterations:05d} ({pct_done:.2f}%) | loss: {debiased_smooth_loss:.6f} | lrm: {lrm:.2f} | dt: {dt * 1000:.2f}ms | tok/sec: {tok_per_sec:,} | bf16_mfu: {mfu:.2f} | epoch: {epoch} | total time: {total_training_time/60:.2f}m{eta_str}") - if step % 100 == 0: + if args.log_every > 0 and step % args.log_every == 0: + # every metric of the step, at the step (see the wandb init above); master process only (others: dummy) log_data = { "step": step, "total_training_flops": flops_so_far, "total_training_time": total_training_time, - "train/loss": debiased_smooth_loss, + "train/loss": debiased_smooth_loss, # EMA (as printed) + "train/loss_step": step_loss_f, # this step's mean loss over its micro-batches, unsmoothed + "train/tokens": total_batch_size * (step + 1), "train/lrm": lrm, + "train/lr_matrix": args.matrix_lr * batch_lr_scale * lrm, + "train/muon_momentum": muon_momentum, + "train/muon_weight_decay": muon_weight_decay, "train/dt": dt, "train/tok_per_sec": tok_per_sec, "train/mfu": mfu, - "train/epoch": epoch, + "train/peak_mem_gib": get_max_memory() / 2**30, + "data/epoch": dataloader_state_dict["epoch"], + "data/pq_idx": dataloader_state_dict["pq_idx"], + "data/rg_idx": dataloader_state_dict["rg_idx"], } - wandb_run.log(log_data) + if args.attn_res: + queries = torch.stack(list(orig_model.attn_res_queries)) + log_data["train/lr_attn_res"] = args.attn_res_lr * lrm + log_data["attn_res/query_rms"] = queries.float().square().mean().sqrt().item() + wandb_run.log(log_data, step=step) # state update first_step_of_run = (step == 0) or (resuming and step == args.resume_from_step)