Download code/nanochat_dev.patch from xTimeCrystal/nanochat-d16-loopfbt2-ar: direct link, hf CLI and curl.
- Browser
- Download file 26.6 kB
-
https://huggingface.co/xTimeCrystal/nanochat-d16-loopfbt2-ar/resolve/main/code/nanochat_dev.patch
- Command line
-
hf download hf://xTimeCrystal/nanochat-d16-loopfbt2-ar/code/nanochat_dev.patch
-
curl -L -o nanochat_dev.patch https://huggingface.co/xTimeCrystal/nanochat-d16-loopfbt2-ar/resolve/main/code/nanochat_dev.patch
26.6 kB
| diff --git a/nanochat/checkpoint_manager.py b/nanochat/checkpoint_manager.py | |
| index bafcdee..1e7ce9e 100644 | |
| --- a/nanochat/checkpoint_manager.py | |
| +++ b/nanochat/checkpoint_manager.py | |
| def _patch_missing_config_keys(model_config_kwargs): | |
| def _patch_missing_keys(model_data, model_config): | |
| """Add default values for new parameters that may be missing in old checkpoints.""" | |
| + if getattr(model_config, "attn_res", False): | |
| + return # AttnRes models have no resid_lambdas / x0_lambdas | |
| n_layer = model_config.n_layer | |
| # resid_lambdas defaults to 1.0 (identity scaling) | |
| if "resid_lambdas" not in model_data: | |
| diff --git a/nanochat/gpt.py b/nanochat/gpt.py | |
| index 1e89378..a4ab2eb 100644 | |
| --- a/nanochat/gpt.py | |
| +++ b/nanochat/gpt.py | |
| from nanochat.optim import MuonAdamW | |
| # Our custom Flash Attention module that automatically uses FA3 when compatible and SDPA fallback otherwise | |
| from nanochat.flash_attention import flash_attn | |
| +# Attention Residuals (ported from dl_pro): Triton kernels for CUDA; AttnResTorch below is the reference elsewhere | |
| +try: | |
| + from nanochat.attn_res import AttnRes | |
| +except ImportError: # no triton (e.g. CPU/MPS-only installs) | |
| + AttnRes = None | |
| + | |
| @dataclass | |
| class GPTConfig: | |
| sequence_len: int = 2048 | |
| class GPTConfig: | |
| # Characters: L=long (full context), S=short (quarter context) | |
| # Examples: "L"=all full context, "SL"=alternating, "SSL"=two short then one long | |
| window_pattern: str = "SSSL" | |
| + # Attention Residuals: each sublayer reads a learned softmax mixture over depth (the normed embedding and every | |
| + # earlier sublayer output) instead of the running residual sum. Replaces resid_lambdas, x0_lambdas and backout, | |
| + # which are defined on the residual sum; adds 2 * n_layer + 1 zero-init pseudo-query vectors (Adam, own LR, no decay) | |
| + attn_res: bool = False | |
| + # Full-bandwidth transformer (dl_pro's FBT): number of trunk passes (1 = off). Pass k > 1 reads | |
| + # norm(W_t e + W_h norm(h_{t-1})); W_t, W_h: (n_embd, n_embd) Muon matrices, init identity / zero | |
| + fbt_passes: int = 1 | |
| + # looped FBT: a pass k > 1 also reads its own position's previous-pass state, + W_p norm(h_t) (W_p zero init: FBT) | |
| + fbt_inplace: bool = False | |
| def norm(x): | |
| class Block(nn.Module): | |
| return x | |
| +class AttnResTorch: | |
| + """AttnRes in plain PyTorch (nanochat.attn_res.reference: stacks every source, O(L^2) memory), with AttnRes's | |
| + interface, for devices without the Triton kernels.""" | |
| + def __init__(self, x, queries): | |
| + from nanochat.attn_res import reference | |
| + self.reference = reference | |
| + self.us = list(queries) | |
| + self.v = [] | |
| + self.add(x) | |
| + | |
| + def add(self, v): | |
| + assert len(self.v) < len(self.us), "more sources than queries" | |
| + self.v.append(v) | |
| + | |
| + def mix(self): | |
| + return self.reference(self.v, self.us[len(self.v) - 1]).to(self.v[0].dtype) | |
| + | |
| + | |
| class GPT(nn.Module): | |
| def __init__(self, config, pad_vocab_size_to=64): | |
| """ | |
| class GPT(nn.Module): | |
| "h": nn.ModuleList([Block(config, layer_idx) for layer_idx in range(config.n_layer)]), | |
| }) | |
| self.lm_head = Linear(config.n_embd, padded_vocab_size, bias=False) | |
| + # FBT fusion of the token input with the previous pass's state (only with fbt_passes > 1) | |
| + if config.fbt_passes > 1: | |
| + self.fbt_t = Linear(config.n_embd, config.n_embd, bias=False) | |
| + self.fbt_h = Linear(config.n_embd, config.n_embd, bias=False) | |
| + if config.fbt_inplace: | |
| + self.fbt_p = Linear(config.n_embd, config.n_embd, bias=False) | |
| # Per-layer learnable scalars (inspired by modded-nanogpt) | |
| # resid_lambdas: scales the residual stream at each layer (init 1.0 = neutral) | |
| # x0_lambdas: blends initial embedding back in at each layer (init 0.0 = disabled) | |
| # Separate parameters so they can have different optimizer treatment | |
| - self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) # fake init, real init in init_weights() | |
| - self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) # fake init, real init in init_weights() | |
| + # (with attn_res these act on a residual sum that no longer exists: not created, and every parameter keeps a grad) | |
| + if not config.attn_res: | |
| + self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) # fake init, real init in init_weights() | |
| + self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) # fake init, real init in init_weights() | |
| # Smear: mix previous token's embedding into current token (cheap bigram-like info) | |
| self.smear_gate = Linear(24, 1, bias=False) | |
| self.smear_lambda = nn.Parameter(torch.zeros(1)) | |
| # Backout: subtract cached mid-layer residual before final norm to remove low-level features | |
| - self.backout_lambda = nn.Parameter(0.2 * torch.ones(1)) | |
| + if not config.attn_res: | |
| + self.backout_lambda = nn.Parameter(0.2 * torch.ones(1)) | |
| + # AttnRes pseudo-queries: one per sublayer (attn, mlp of each block), then the readout. Separate (C,) vectors | |
| + # (as dl_pro): each stays a "small" param for MuonAdamW's distributed AdamW (no shape[0] % world_size rule) | |
| + if config.attn_res: | |
| + self.attn_res_queries = nn.ParameterList([nn.Parameter(torch.zeros(config.n_embd)) for _ in range(2 * config.n_layer + 1)]) | |
| # Value embeddings (ResFormer-style): alternating layers, last layer always included | |
| head_dim = config.n_embd // config.n_head | |
| kv_dim = config.n_kv_head * head_dim | |
| class GPT(nn.Module): | |
| torch.nn.init.uniform_(block.mlp.c_fc.weight, -s * 0.4, s * 0.4) # 0.4x init scale for c_fc | |
| torch.nn.init.zeros_(block.mlp.c_proj.weight) | |
| + # FBT fusion: identity on the token, zero on the fed-back state (every pass starts as the plain trunk) | |
| + if self.config.fbt_passes > 1: | |
| + torch.nn.init.eye_(self.fbt_t.weight) | |
| + torch.nn.init.zeros_(self.fbt_h.weight) | |
| + if self.config.fbt_inplace: | |
| + torch.nn.init.zeros_(self.fbt_p.weight) | |
| + | |
| # Per-layer scalars | |
| # Per-layer resid init: stronger residual at early layers, weaker at deep layers | |
| n_layer = self.config.n_layer | |
| - for i in range(n_layer): | |
| - self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1)) | |
| - # Decaying x0 init: earlier layers get more input embedding blending | |
| - for i in range(n_layer): | |
| - self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1)) | |
| + if not self.config.attn_res: | |
| + for i in range(n_layer): | |
| + self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1)) | |
| + # Decaying x0 init: earlier layers get more input embedding blending | |
| + for i in range(n_layer): | |
| + self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1)) | |
| + else: | |
| + # AttnRes pseudo-queries: zero = a uniform mixture over the sources at step 0 | |
| + for q in self.attn_res_queries: | |
| + torch.nn.init.zeros_(q) | |
| - # Smear/backout scalars and smear gate must be explicitly initialized | |
| + # Smear/backout scalars and smear gate must be explicitly initialized | |
| torch.nn.init.zeros_(self.smear_lambda) | |
| - torch.nn.init.constant_(self.backout_lambda, 0.2) | |
| + if not self.config.attn_res: | |
| + torch.nn.init.constant_(self.backout_lambda, 0.2) | |
| torch.nn.init.uniform_(self.smear_gate.weight, 0.0, 0.02) | |
| # Value embeddings (init like c_v: uniform with same std) | |
| class GPT(nn.Module): | |
| window = window_size[0] # (left, right) tuple, we use left | |
| effective_seq = t if window < 0 else min(window, t) | |
| attn_flops += 12 * h * q * effective_seq | |
| - num_flops_per_token = 6 * self.num_matmul_params() + attn_flops | |
| + num_flops_per_token = (6 * self.num_matmul_params() + attn_flops) * self.config.fbt_passes # (FBT: every pass, head included) | |
| return num_flops_per_token | |
| def num_matmul_params(self): | |
| class GPT(nn.Module): | |
| value_embeds = sum(p.numel() for p in self.value_embeds.parameters()) | |
| lm_head = sum(p.numel() for p in self.lm_head.parameters()) | |
| transformer_matrices = sum(p.numel() for p in self.transformer.h.parameters()) | |
| - scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel() | |
| - total = wte + value_embeds + lm_head + transformer_matrices + scalars | |
| + # FBT fusion matrices: counted apart, so the scaling params (and the token budget) stay the baseline's | |
| + fbt = sum(p.numel() for n, p in self.named_parameters() if n.startswith("fbt_")) | |
| + if self.config.attn_res: # (the pseudo-queries count as scalars) | |
| + scalars = sum(q.numel() for q in self.attn_res_queries) + self.smear_gate.weight.numel() + self.smear_lambda.numel() | |
| + else: | |
| + scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel() | |
| + total = wte + value_embeds + lm_head + transformer_matrices + scalars + fbt | |
| assert total == sum(p.numel() for p in self.parameters()), "Parameter count mismatch" | |
| return { | |
| 'wte': wte, | |
| class GPT(nn.Module): | |
| 'lm_head': lm_head, | |
| 'transformer_matrices': transformer_matrices, | |
| 'scalars': scalars, | |
| + **({'fbt': fbt} if fbt else {}), | |
| 'total': total, | |
| } | |
| - def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5): | |
| + def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5, attn_res_lr=1e-3, attn_res_betas=(0.8, 0.95)): | |
| model_dim = self.config.n_embd | |
| # Separate out all parameters into groups | |
| matrix_params = list(self.transformer.h.parameters()) | |
| + if self.config.fbt_passes > 1: # FBT fusion matrices on Muon, as dl_pro (grouped by shape with the others) | |
| + matrix_params += [self.fbt_t.weight, self.fbt_h.weight] + ([self.fbt_p.weight] if self.config.fbt_inplace else []) | |
| value_embeds_params = list(self.value_embeds.parameters()) | |
| embedding_params = list(self.transformer.wte.parameters()) | |
| lm_head_params = list(self.lm_head.parameters()) | |
| - resid_params = [self.resid_lambdas] | |
| - x0_params = [self.x0_lambdas] | |
| - smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda] | |
| - assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params) | |
| + if self.config.attn_res: | |
| + resid_params, x0_params = [], [] | |
| + smear_params = [self.smear_gate.weight, self.smear_lambda] | |
| + attn_res_params = list(self.attn_res_queries) | |
| + else: | |
| + resid_params = [self.resid_lambdas] | |
| + x0_params = [self.x0_lambdas] | |
| + smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda] | |
| + attn_res_params = [] | |
| + assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params) + len(attn_res_params) | |
| # Scale the LR for the AdamW parameters by ∝1/√dmodel (tuned for 768 dim model) | |
| dmodel_lr_scale = (model_dim / 768) ** -0.5 | |
| class GPT(nn.Module): | |
| dict(kind='adamw', params=lm_head_params, lr=unembedding_lr * dmodel_lr_scale, betas=(0.8, 0.96), eps=1e-10, weight_decay=0.01), | |
| dict(kind='adamw', params=embedding_params, lr=embedding_lr * dmodel_lr_scale, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.001), | |
| dict(kind='adamw', params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale * 0.5, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.01), | |
| - dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05), | |
| - dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), # higher beta1 for x0 | |
| - dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0), | |
| ] | |
| + if not self.config.attn_res: | |
| + param_groups += [ | |
| + dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05), | |
| + dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), # higher beta1 for x0 | |
| + ] | |
| + param_groups.append(dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0)) | |
| + if attn_res_params: | |
| + # AttnRes pseudo-queries: Adam at attn_res_lr (1e-3, dl_pro's), no weight decay, attn_res_betas (default | |
| + # (0.8, 0.95): beta1 as nanochat's other Adam groups; dl_pro used 0.65). The LR is used as given (no dmodel / | |
| + # batch scaling); the schedule multiplier applies as to every group. | |
| + param_groups.append(dict(kind='adamw', params=attn_res_params, lr=attn_res_lr, betas=tuple(attn_res_betas), eps=1e-10, weight_decay=0.0)) | |
| # Muon groups (matrix params, grouped by shape for stacking) | |
| for shape in sorted({p.shape for p in matrix_params}): | |
| group_params = [p for p in matrix_params if p.shape == shape] | |
| class GPT(nn.Module): | |
| gate = self.smear_lambda.to(x.dtype) * torch.sigmoid(self.smear_gate(x[:, :, :24])) | |
| x = x + gate * x_pre_smear | |
| + # Full-bandwidth transformer (dl_pro's FBT): fbt_passes > 1 runs the trunk again, each pass on the fusion | |
| + # norm(W_t e + W_h norm(h_{t-1})) of the token input e with the previous pass's final hidden state one position | |
| + # back (the first pass sees h = 0). Training: the mean of the passes' losses (as dl_pro); otherwise: the last pass | |
| + if self.config.fbt_passes > 1: | |
| + assert kv_cache is None, "FBT has no KV-cache inference path" | |
| + e, h, losses = x, None, [] | |
| + for _ in range(self.config.fbt_passes): | |
| + h_prev = torch.zeros_like(e) if h is None else torch.cat([torch.zeros_like(h[:, :1]), h[:, :-1]], dim=1) | |
| + f = self.fbt_t(e) + self.fbt_h(norm(h_prev)) | |
| + if self.config.fbt_inplace and h is not None: # looped FBT: + W_p norm(h_t) (not in the first pass) | |
| + f = f + self.fbt_p(norm(h)) | |
| + h = self._trunk(norm(f), idx, cos_sin, None) | |
| + if targets is not None and self.training: | |
| + losses.append(self._head(h, targets, loss_reduction)) | |
| + return torch.stack(losses).mean() if losses else self._head(h, targets, loss_reduction) | |
| + | |
| + x = self._trunk(x, idx, cos_sin, kv_cache) | |
| + return self._head(x, targets, loss_reduction) | |
| + | |
| + def _trunk(self, x, idx, cos_sin, kv_cache): | |
| + """The transformer blocks on the input x (B, T, C): the final normed hidden state.""" | |
| # Forward the trunk of the Transformer | |
| - x0 = x # save initial normalized embedding for x0 residual | |
| - n_layer = self.config.n_layer | |
| - backout_layer = n_layer // 2 # cache at halfway point | |
| - x_backout = None | |
| - for i, block in enumerate(self.transformer.h): | |
| - x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0 | |
| - ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None | |
| - x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache) | |
| - if i == backout_layer: | |
| - x_backout = x | |
| - # Subtract mid-layer residual to remove low-level features before logit projection | |
| - if x_backout is not None: | |
| - x = x - self.backout_lambda.to(x.dtype) * x_backout | |
| - x = norm(x) | |
| + if self.config.attn_res: | |
| + # Attention Residuals: sources = the embedding and every sublayer output; each sublayer (and the readout) | |
| + # reads norm(softmax-over-depth mixture) with its own pseudo-query, so no pre-norm, x0 blend or backout | |
| + res = (AttnRes if x.is_cuda and AttnRes is not None else AttnResTorch)(x, self.attn_res_queries) | |
| + for i, block in enumerate(self.transformer.h): | |
| + ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None | |
| + res.add(block.attn(res.mix(), ve, cos_sin, self.window_sizes[i], kv_cache)) | |
| + res.add(block.mlp(res.mix())) | |
| + x = res.mix() # the readout, already normed | |
| + else: | |
| + x0 = x # save initial normalized embedding for x0 residual | |
| + n_layer = self.config.n_layer | |
| + backout_layer = n_layer // 2 # cache at halfway point | |
| + x_backout = None | |
| + for i, block in enumerate(self.transformer.h): | |
| + x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0 | |
| + ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None | |
| + x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache) | |
| + if i == backout_layer: | |
| + x_backout = x | |
| + # Subtract mid-layer residual to remove low-level features before logit projection | |
| + if x_backout is not None: | |
| + x = x - self.backout_lambda.to(x.dtype) * x_backout | |
| + x = norm(x) | |
| + return x | |
| + def _head(self, x, targets, loss_reduction): | |
| + """Logits of the final hidden state x, or the loss when targets are given.""" | |
| # Forward the lm_head (compute logits) | |
| softcap = 15 # smoothly cap the logits to the range [-softcap, softcap] | |
| logits = self.lm_head(x) # (B, T, padded_vocab_size) <- very big tensor, large amount of memory | |
| diff --git a/scripts/base_train.py b/scripts/base_train.py | |
| index d1af271..7c0cd69 100644 | |
| --- a/scripts/base_train.py | |
| +++ b/scripts/base_train.py | |
| parser.add_argument("--aspect-ratio", type=int, default=64, help="model_dim = de | |
| parser.add_argument("--head-dim", type=int, default=128, help="target head dimension for attention") | |
| parser.add_argument("--max-seq-len", type=int, default=2048, help="max context length") | |
| parser.add_argument("--window-pattern", type=str, default="SSSL", help="sliding window pattern tiled across layers: L=full, S=half context (e.g. 'SSL')") | |
| +parser.add_argument("--attn-res", action="store_true", help="Attention Residuals (dl_pro port): softmax-over-depth mixtures replace the residual stream") | |
| +parser.add_argument("--attn-res-lr", type=float, default=1e-3, help="Adam LR for the AttnRes pseudo-queries (no weight decay, not batch/dmodel-scaled)") | |
| +parser.add_argument("--attn-res-beta1", type=float, default=0.8, help="Adam beta1 for the AttnRes pseudo-queries") | |
| +parser.add_argument("--attn-res-beta2", type=float, default=0.95, help="Adam beta2 for the AttnRes pseudo-queries") | |
| +parser.add_argument("--fbt-passes", type=int, default=1, help="full-bandwidth transformer (dl_pro FBT): trunk passes, each later one fed the previous pass's state at t-1 (1 = off)") | |
| +parser.add_argument("--fbt-inplace", action="store_true", help="looped FBT: a feedback pass also fuses its own position's previous-pass state (+ W_p norm(h_t), zero init)") | |
| # Training horizon (only one used, in order of precedence) | |
| parser.add_argument("--num-iterations", type=int, default=-1, help="explicit number of optimization steps (-1 = disable)") | |
| parser.add_argument("--target-flops", type=float, default=-1.0, help="calculate num_iterations to reach target_flops (-1 = disable)") | |
| parser.add_argument("--core-metric-every", type=int, default=2000, help="evaluat | |
| parser.add_argument("--core-metric-max-per-task", type=int, default=500, help="examples per task for CORE metric") | |
| parser.add_argument("--sample-every", type=int, default=2000, help="sample from model every N steps (-1 = disable)") | |
| parser.add_argument("--save-every", type=int, default=-1, help="save checkpoints every N steps (-1 = only at end)") | |
| +parser.add_argument("--log-every", type=int, default=1, help="log training metrics to wandb every N steps (0 = never)") | |
| # Output | |
| parser.add_argument("--model-tag", type=str, default=None, help="override model tag for checkpoint directory name") | |
| args = parser.parse_args() | |
| print0(f"COMPUTE_DTYPE: {COMPUTE_DTYPE} ({COMPUTE_DTYPE_REASON})") | |
| # wandb logging init | |
| use_dummy_wandb = args.run == "dummy" or not master_process | |
| wandb_run = DummyWandb() if use_dummy_wandb else wandb.init(project="nanochat", name=args.run, config=user_config) | |
| +# Every wandb_run.log below passes step=step, so wandb's own step (the default x-axis) is the training step. Otherwise it | |
| +# counts log calls (train, val and CORE logs at different cadences put e.g. the step-750 val point at x=753). | |
| # Flash Attention status | |
| from nanochat.flash_attention import USE_FA3 | |
| def build_model_meta(depth): | |
| config = GPTConfig( | |
| sequence_len=args.max_seq_len, vocab_size=vocab_size, | |
| n_layer=depth, n_head=num_heads, n_kv_head=num_heads, n_embd=model_dim, | |
| - window_pattern=args.window_pattern, | |
| + window_pattern=args.window_pattern, attn_res=args.attn_res, fbt_passes=args.fbt_passes, fbt_inplace=args.fbt_inplace, | |
| ) | |
| with torch.device("meta"): | |
| model_meta = GPT(config) | |
| optimizer = model.setup_optimizer( | |
| # Muon hyperparameters | |
| matrix_lr=args.matrix_lr * batch_lr_scale, | |
| weight_decay=weight_decay_scaled, | |
| + # AttnRes pseudo-queries (only with --attn-res) | |
| + attn_res_lr=args.attn_res_lr, | |
| + attn_res_betas=(args.attn_res_beta1, args.attn_res_beta2), | |
| ) | |
| if resuming: | |
| while True: | |
| "total_training_flops": flops_so_far, | |
| "total_training_time": total_training_time, | |
| "val/bpb": val_bpb, | |
| - }) | |
| + }, step=step) | |
| model.train() | |
| # once in a while: estimate the CORE metric (all ranks participate) | |
| while True: | |
| "total_training_flops": flops_so_far, | |
| "core_metric": results["core_metric"], | |
| "centered_results": results["centered_results"], | |
| - }) | |
| + }, step=step) | |
| model.train() | |
| # once in a while: sample from the model (only on master process) | |
| # use the original uncompiled model because the inputs keep changing shape | |
| - if args.sample_every > 0 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)): | |
| + if args.sample_every > 0 and args.fbt_passes == 1 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)): # (FBT: no KV-cache sampler) | |
| model.eval() | |
| prompts = [ | |
| "The capital of France is", | |
| while True: | |
| # evaluate the gradient | |
| synchronize() | |
| t0 = time.time() | |
| + loss_sum = 0 # this step's loss summed over micro-batches (rank 0's data), for logging | |
| for micro_step in range(grad_accum_steps): | |
| loss = model(x, y) | |
| train_loss = loss.detach() # for logging | |
| + loss_sum = loss_sum + train_loss | |
| loss = loss / grad_accum_steps # each .backward() is a grad sum => normalize loss here | |
| if scaler is not None: | |
| scaler.scale(loss).backward() | |
| while True: | |
| else: | |
| optimizer.step() | |
| model.zero_grad(set_to_none=True) | |
| - train_loss_f = train_loss.item() # .item() is a CPU-GPU sync point | |
| + train_loss_f, step_loss_f = torch.stack([train_loss, loss_sum / grad_accum_steps]).tolist() # one CPU-GPU sync point | |
| synchronize() | |
| t1 = time.time() | |
| dt = t1 - t0 | |
| while True: | |
| eta_str = "" | |
| epoch = f"{dataloader_state_dict['epoch']} pq: {dataloader_state_dict['pq_idx']} rg: {dataloader_state_dict['rg_idx']}" | |
| print0(f"step {step:05d}/{num_iterations:05d} ({pct_done:.2f}%) | loss: {debiased_smooth_loss:.6f} | lrm: {lrm:.2f} | dt: {dt * 1000:.2f}ms | tok/sec: {tok_per_sec:,} | bf16_mfu: {mfu:.2f} | epoch: {epoch} | total time: {total_training_time/60:.2f}m{eta_str}") | |
| - if step % 100 == 0: | |
| + if args.log_every > 0 and step % args.log_every == 0: | |
| + # every metric of the step, at the step (see the wandb init above); master process only (others: dummy) | |
| log_data = { | |
| "step": step, | |
| "total_training_flops": flops_so_far, | |
| "total_training_time": total_training_time, | |
| - "train/loss": debiased_smooth_loss, | |
| + "train/loss": debiased_smooth_loss, # EMA (as printed) | |
| + "train/loss_step": step_loss_f, # this step's mean loss over its micro-batches, unsmoothed | |
| + "train/tokens": total_batch_size * (step + 1), | |
| "train/lrm": lrm, | |
| + "train/lr_matrix": args.matrix_lr * batch_lr_scale * lrm, | |
| + "train/muon_momentum": muon_momentum, | |
| + "train/muon_weight_decay": muon_weight_decay, | |
| "train/dt": dt, | |
| "train/tok_per_sec": tok_per_sec, | |
| "train/mfu": mfu, | |
| - "train/epoch": epoch, | |
| + "train/peak_mem_gib": get_max_memory() / 2**30, | |
| + "data/epoch": dataloader_state_dict["epoch"], | |
| + "data/pq_idx": dataloader_state_dict["pq_idx"], | |
| + "data/rg_idx": dataloader_state_dict["rg_idx"], | |
| } | |
| - wandb_run.log(log_data) | |
| + if args.attn_res: | |
| + queries = torch.stack(list(orig_model.attn_res_queries)) | |
| + log_data["train/lr_attn_res"] = args.attn_res_lr * lrm | |
| + log_data["attn_res/query_rms"] = queries.float().square().mean().sqrt().item() | |
| + wandb_run.log(log_data, step=step) | |
| # state update | |
| first_step_of_run = (step == 0) or (resuming and step == args.resume_from_step) | |