nanochat-d16-loopfbt2-ar / code /nanochat_dev.patch
xTimeCrystal's picture
nanochat d16: looped FBT (2 passes) + AttnRes, step 5376: weights
2dc5456 verified
Raw History Blame Contribute Delete
26.6 kB
diff --git a/nanochat/checkpoint_manager.py b/nanochat/checkpoint_manager.py
index bafcdee..1e7ce9e 100644
--- a/nanochat/checkpoint_manager.py
+++ b/nanochat/checkpoint_manager.py
@@ -28,6 +28,8 @@ def _patch_missing_config_keys(model_config_kwargs):
def _patch_missing_keys(model_data, model_config):
"""Add default values for new parameters that may be missing in old checkpoints."""
+ if getattr(model_config, "attn_res", False):
+ return # AttnRes models have no resid_lambdas / x0_lambdas
n_layer = model_config.n_layer
# resid_lambdas defaults to 1.0 (identity scaling)
if "resid_lambdas" not in model_data:
diff --git a/nanochat/gpt.py b/nanochat/gpt.py
index 1e89378..a4ab2eb 100644
--- a/nanochat/gpt.py
+++ b/nanochat/gpt.py
@@ -25,6 +25,12 @@ from nanochat.optim import MuonAdamW
# Our custom Flash Attention module that automatically uses FA3 when compatible and SDPA fallback otherwise
from nanochat.flash_attention import flash_attn
+# Attention Residuals (ported from dl_pro): Triton kernels for CUDA; AttnResTorch below is the reference elsewhere
+try:
+ from nanochat.attn_res import AttnRes
+except ImportError: # no triton (e.g. CPU/MPS-only installs)
+ AttnRes = None
+
@dataclass
class GPTConfig:
sequence_len: int = 2048
@@ -37,6 +43,15 @@ class GPTConfig:
# Characters: L=long (full context), S=short (quarter context)
# Examples: "L"=all full context, "SL"=alternating, "SSL"=two short then one long
window_pattern: str = "SSSL"
+ # Attention Residuals: each sublayer reads a learned softmax mixture over depth (the normed embedding and every
+ # earlier sublayer output) instead of the running residual sum. Replaces resid_lambdas, x0_lambdas and backout,
+ # which are defined on the residual sum; adds 2 * n_layer + 1 zero-init pseudo-query vectors (Adam, own LR, no decay)
+ attn_res: bool = False
+ # Full-bandwidth transformer (dl_pro's FBT): number of trunk passes (1 = off). Pass k > 1 reads
+ # norm(W_t e + W_h norm(h_{t-1})); W_t, W_h: (n_embd, n_embd) Muon matrices, init identity / zero
+ fbt_passes: int = 1
+ # looped FBT: a pass k > 1 also reads its own position's previous-pass state, + W_p norm(h_t) (W_p zero init: FBT)
+ fbt_inplace: bool = False
def norm(x):
@@ -153,6 +168,24 @@ class Block(nn.Module):
return x
+class AttnResTorch:
+ """AttnRes in plain PyTorch (nanochat.attn_res.reference: stacks every source, O(L^2) memory), with AttnRes's
+ interface, for devices without the Triton kernels."""
+ def __init__(self, x, queries):
+ from nanochat.attn_res import reference
+ self.reference = reference
+ self.us = list(queries)
+ self.v = []
+ self.add(x)
+
+ def add(self, v):
+ assert len(self.v) < len(self.us), "more sources than queries"
+ self.v.append(v)
+
+ def mix(self):
+ return self.reference(self.v, self.us[len(self.v) - 1]).to(self.v[0].dtype)
+
+
class GPT(nn.Module):
def __init__(self, config, pad_vocab_size_to=64):
"""
@@ -175,17 +208,30 @@ class GPT(nn.Module):
"h": nn.ModuleList([Block(config, layer_idx) for layer_idx in range(config.n_layer)]),
})
self.lm_head = Linear(config.n_embd, padded_vocab_size, bias=False)
+ # FBT fusion of the token input with the previous pass's state (only with fbt_passes > 1)
+ if config.fbt_passes > 1:
+ self.fbt_t = Linear(config.n_embd, config.n_embd, bias=False)
+ self.fbt_h = Linear(config.n_embd, config.n_embd, bias=False)
+ if config.fbt_inplace:
+ self.fbt_p = Linear(config.n_embd, config.n_embd, bias=False)
# Per-layer learnable scalars (inspired by modded-nanogpt)
# resid_lambdas: scales the residual stream at each layer (init 1.0 = neutral)
# x0_lambdas: blends initial embedding back in at each layer (init 0.0 = disabled)
# Separate parameters so they can have different optimizer treatment
- self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) # fake init, real init in init_weights()
- self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) # fake init, real init in init_weights()
+ # (with attn_res these act on a residual sum that no longer exists: not created, and every parameter keeps a grad)
+ if not config.attn_res:
+ self.resid_lambdas = nn.Parameter(torch.ones(config.n_layer)) # fake init, real init in init_weights()
+ self.x0_lambdas = nn.Parameter(torch.zeros(config.n_layer)) # fake init, real init in init_weights()
# Smear: mix previous token's embedding into current token (cheap bigram-like info)
self.smear_gate = Linear(24, 1, bias=False)
self.smear_lambda = nn.Parameter(torch.zeros(1))
# Backout: subtract cached mid-layer residual before final norm to remove low-level features
- self.backout_lambda = nn.Parameter(0.2 * torch.ones(1))
+ if not config.attn_res:
+ self.backout_lambda = nn.Parameter(0.2 * torch.ones(1))
+ # AttnRes pseudo-queries: one per sublayer (attn, mlp of each block), then the readout. Separate (C,) vectors
+ # (as dl_pro): each stays a "small" param for MuonAdamW's distributed AdamW (no shape[0] % world_size rule)
+ if config.attn_res:
+ self.attn_res_queries = nn.ParameterList([nn.Parameter(torch.zeros(config.n_embd)) for _ in range(2 * config.n_layer + 1)])
# Value embeddings (ResFormer-style): alternating layers, last layer always included
head_dim = config.n_embd // config.n_head
kv_dim = config.n_kv_head * head_dim
@@ -231,18 +277,31 @@ class GPT(nn.Module):
torch.nn.init.uniform_(block.mlp.c_fc.weight, -s * 0.4, s * 0.4) # 0.4x init scale for c_fc
torch.nn.init.zeros_(block.mlp.c_proj.weight)
+ # FBT fusion: identity on the token, zero on the fed-back state (every pass starts as the plain trunk)
+ if self.config.fbt_passes > 1:
+ torch.nn.init.eye_(self.fbt_t.weight)
+ torch.nn.init.zeros_(self.fbt_h.weight)
+ if self.config.fbt_inplace:
+ torch.nn.init.zeros_(self.fbt_p.weight)
+
# Per-layer scalars
# Per-layer resid init: stronger residual at early layers, weaker at deep layers
n_layer = self.config.n_layer
- for i in range(n_layer):
- self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1))
- # Decaying x0 init: earlier layers get more input embedding blending
- for i in range(n_layer):
- self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1))
+ if not self.config.attn_res:
+ for i in range(n_layer):
+ self.resid_lambdas.data[i] = 1.15 - (0.10 * i / max(n_layer - 1, 1))
+ # Decaying x0 init: earlier layers get more input embedding blending
+ for i in range(n_layer):
+ self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1))
+ else:
+ # AttnRes pseudo-queries: zero = a uniform mixture over the sources at step 0
+ for q in self.attn_res_queries:
+ torch.nn.init.zeros_(q)
- # Smear/backout scalars and smear gate must be explicitly initialized
+ # Smear/backout scalars and smear gate must be explicitly initialized
torch.nn.init.zeros_(self.smear_lambda)
- torch.nn.init.constant_(self.backout_lambda, 0.2)
+ if not self.config.attn_res:
+ torch.nn.init.constant_(self.backout_lambda, 0.2)
torch.nn.init.uniform_(self.smear_gate.weight, 0.0, 0.02)
# Value embeddings (init like c_v: uniform with same std)
@@ -335,7 +394,7 @@ class GPT(nn.Module):
window = window_size[0] # (left, right) tuple, we use left
effective_seq = t if window < 0 else min(window, t)
attn_flops += 12 * h * q * effective_seq
- num_flops_per_token = 6 * self.num_matmul_params() + attn_flops
+ num_flops_per_token = (6 * self.num_matmul_params() + attn_flops) * self.config.fbt_passes # (FBT: every pass, head included)
return num_flops_per_token
def num_matmul_params(self):
@@ -404,8 +463,13 @@ class GPT(nn.Module):
value_embeds = sum(p.numel() for p in self.value_embeds.parameters())
lm_head = sum(p.numel() for p in self.lm_head.parameters())
transformer_matrices = sum(p.numel() for p in self.transformer.h.parameters())
- scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel()
- total = wte + value_embeds + lm_head + transformer_matrices + scalars
+ # FBT fusion matrices: counted apart, so the scaling params (and the token budget) stay the baseline's
+ fbt = sum(p.numel() for n, p in self.named_parameters() if n.startswith("fbt_"))
+ if self.config.attn_res: # (the pseudo-queries count as scalars)
+ scalars = sum(q.numel() for q in self.attn_res_queries) + self.smear_gate.weight.numel() + self.smear_lambda.numel()
+ else:
+ scalars = self.resid_lambdas.numel() + self.x0_lambdas.numel() + self.smear_gate.weight.numel() + self.smear_lambda.numel() + self.backout_lambda.numel()
+ total = wte + value_embeds + lm_head + transformer_matrices + scalars + fbt
assert total == sum(p.numel() for p in self.parameters()), "Parameter count mismatch"
return {
'wte': wte,
@@ -413,21 +477,30 @@ class GPT(nn.Module):
'lm_head': lm_head,
'transformer_matrices': transformer_matrices,
'scalars': scalars,
+ **({'fbt': fbt} if fbt else {}),
'total': total,
}
- def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5):
+ def setup_optimizer(self, unembedding_lr=0.004, embedding_lr=0.2, matrix_lr=0.02, weight_decay=0.0, scalar_lr=0.5, attn_res_lr=1e-3, attn_res_betas=(0.8, 0.95)):
model_dim = self.config.n_embd
# Separate out all parameters into groups
matrix_params = list(self.transformer.h.parameters())
+ if self.config.fbt_passes > 1: # FBT fusion matrices on Muon, as dl_pro (grouped by shape with the others)
+ matrix_params += [self.fbt_t.weight, self.fbt_h.weight] + ([self.fbt_p.weight] if self.config.fbt_inplace else [])
value_embeds_params = list(self.value_embeds.parameters())
embedding_params = list(self.transformer.wte.parameters())
lm_head_params = list(self.lm_head.parameters())
- resid_params = [self.resid_lambdas]
- x0_params = [self.x0_lambdas]
- smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda]
- assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params)
+ if self.config.attn_res:
+ resid_params, x0_params = [], []
+ smear_params = [self.smear_gate.weight, self.smear_lambda]
+ attn_res_params = list(self.attn_res_queries)
+ else:
+ resid_params = [self.resid_lambdas]
+ x0_params = [self.x0_lambdas]
+ smear_params = [self.smear_gate.weight, self.smear_lambda, self.backout_lambda]
+ attn_res_params = []
+ assert len(list(self.parameters())) == len(matrix_params) + len(embedding_params) + len(lm_head_params) + len(value_embeds_params) + len(resid_params) + len(x0_params) + len(smear_params) + len(attn_res_params)
# Scale the LR for the AdamW parameters by ∝1/√dmodel (tuned for 768 dim model)
dmodel_lr_scale = (model_dim / 768) ** -0.5
@@ -439,10 +512,18 @@ class GPT(nn.Module):
dict(kind='adamw', params=lm_head_params, lr=unembedding_lr * dmodel_lr_scale, betas=(0.8, 0.96), eps=1e-10, weight_decay=0.01),
dict(kind='adamw', params=embedding_params, lr=embedding_lr * dmodel_lr_scale, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.001),
dict(kind='adamw', params=value_embeds_params, lr=embedding_lr * dmodel_lr_scale * 0.5, betas=(0.8, 0.995), eps=1e-10, weight_decay=0.01),
- dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05),
- dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), # higher beta1 for x0
- dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0),
]
+ if not self.config.attn_res:
+ param_groups += [
+ dict(kind='adamw', params=resid_params, lr=scalar_lr * 0.01, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.05),
+ dict(kind='adamw', params=x0_params, lr=scalar_lr, betas=(0.96, 0.95), eps=1e-10, weight_decay=0.0), # higher beta1 for x0
+ ]
+ param_groups.append(dict(kind='adamw', params=smear_params, lr=0.2, betas=(0.8, 0.95), eps=1e-10, weight_decay=0.0))
+ if attn_res_params:
+ # AttnRes pseudo-queries: Adam at attn_res_lr (1e-3, dl_pro's), no weight decay, attn_res_betas (default
+ # (0.8, 0.95): beta1 as nanochat's other Adam groups; dl_pro used 0.65). The LR is used as given (no dmodel /
+ # batch scaling); the schedule multiplier applies as to every group.
+ param_groups.append(dict(kind='adamw', params=attn_res_params, lr=attn_res_lr, betas=tuple(attn_res_betas), eps=1e-10, weight_decay=0.0))
# Muon groups (matrix params, grouped by shape for stacking)
for shape in sorted({p.shape for p in matrix_params}):
group_params = [p for p in matrix_params if p.shape == shape]
@@ -491,22 +572,56 @@ class GPT(nn.Module):
gate = self.smear_lambda.to(x.dtype) * torch.sigmoid(self.smear_gate(x[:, :, :24]))
x = x + gate * x_pre_smear
+ # Full-bandwidth transformer (dl_pro's FBT): fbt_passes > 1 runs the trunk again, each pass on the fusion
+ # norm(W_t e + W_h norm(h_{t-1})) of the token input e with the previous pass's final hidden state one position
+ # back (the first pass sees h = 0). Training: the mean of the passes' losses (as dl_pro); otherwise: the last pass
+ if self.config.fbt_passes > 1:
+ assert kv_cache is None, "FBT has no KV-cache inference path"
+ e, h, losses = x, None, []
+ for _ in range(self.config.fbt_passes):
+ h_prev = torch.zeros_like(e) if h is None else torch.cat([torch.zeros_like(h[:, :1]), h[:, :-1]], dim=1)
+ f = self.fbt_t(e) + self.fbt_h(norm(h_prev))
+ if self.config.fbt_inplace and h is not None: # looped FBT: + W_p norm(h_t) (not in the first pass)
+ f = f + self.fbt_p(norm(h))
+ h = self._trunk(norm(f), idx, cos_sin, None)
+ if targets is not None and self.training:
+ losses.append(self._head(h, targets, loss_reduction))
+ return torch.stack(losses).mean() if losses else self._head(h, targets, loss_reduction)
+
+ x = self._trunk(x, idx, cos_sin, kv_cache)
+ return self._head(x, targets, loss_reduction)
+
+ def _trunk(self, x, idx, cos_sin, kv_cache):
+ """The transformer blocks on the input x (B, T, C): the final normed hidden state."""
# Forward the trunk of the Transformer
- x0 = x # save initial normalized embedding for x0 residual
- n_layer = self.config.n_layer
- backout_layer = n_layer // 2 # cache at halfway point
- x_backout = None
- for i, block in enumerate(self.transformer.h):
- x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
- ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None
- x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache)
- if i == backout_layer:
- x_backout = x
- # Subtract mid-layer residual to remove low-level features before logit projection
- if x_backout is not None:
- x = x - self.backout_lambda.to(x.dtype) * x_backout
- x = norm(x)
+ if self.config.attn_res:
+ # Attention Residuals: sources = the embedding and every sublayer output; each sublayer (and the readout)
+ # reads norm(softmax-over-depth mixture) with its own pseudo-query, so no pre-norm, x0 blend or backout
+ res = (AttnRes if x.is_cuda and AttnRes is not None else AttnResTorch)(x, self.attn_res_queries)
+ for i, block in enumerate(self.transformer.h):
+ ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None
+ res.add(block.attn(res.mix(), ve, cos_sin, self.window_sizes[i], kv_cache))
+ res.add(block.mlp(res.mix()))
+ x = res.mix() # the readout, already normed
+ else:
+ x0 = x # save initial normalized embedding for x0 residual
+ n_layer = self.config.n_layer
+ backout_layer = n_layer // 2 # cache at halfway point
+ x_backout = None
+ for i, block in enumerate(self.transformer.h):
+ x = self.resid_lambdas[i] * x + self.x0_lambdas[i] * x0
+ ve = self.value_embeds[str(i)](idx).to(x.dtype) if str(i) in self.value_embeds else None
+ x = block(x, ve, cos_sin, self.window_sizes[i], kv_cache)
+ if i == backout_layer:
+ x_backout = x
+ # Subtract mid-layer residual to remove low-level features before logit projection
+ if x_backout is not None:
+ x = x - self.backout_lambda.to(x.dtype) * x_backout
+ x = norm(x)
+ return x
+ def _head(self, x, targets, loss_reduction):
+ """Logits of the final hidden state x, or the loss when targets are given."""
# Forward the lm_head (compute logits)
softcap = 15 # smoothly cap the logits to the range [-softcap, softcap]
logits = self.lm_head(x) # (B, T, padded_vocab_size) <- very big tensor, large amount of memory
diff --git a/scripts/base_train.py b/scripts/base_train.py
index d1af271..7c0cd69 100644
--- a/scripts/base_train.py
+++ b/scripts/base_train.py
@@ -52,6 +52,12 @@ parser.add_argument("--aspect-ratio", type=int, default=64, help="model_dim = de
parser.add_argument("--head-dim", type=int, default=128, help="target head dimension for attention")
parser.add_argument("--max-seq-len", type=int, default=2048, help="max context length")
parser.add_argument("--window-pattern", type=str, default="SSSL", help="sliding window pattern tiled across layers: L=full, S=half context (e.g. 'SSL')")
+parser.add_argument("--attn-res", action="store_true", help="Attention Residuals (dl_pro port): softmax-over-depth mixtures replace the residual stream")
+parser.add_argument("--attn-res-lr", type=float, default=1e-3, help="Adam LR for the AttnRes pseudo-queries (no weight decay, not batch/dmodel-scaled)")
+parser.add_argument("--attn-res-beta1", type=float, default=0.8, help="Adam beta1 for the AttnRes pseudo-queries")
+parser.add_argument("--attn-res-beta2", type=float, default=0.95, help="Adam beta2 for the AttnRes pseudo-queries")
+parser.add_argument("--fbt-passes", type=int, default=1, help="full-bandwidth transformer (dl_pro FBT): trunk passes, each later one fed the previous pass's state at t-1 (1 = off)")
+parser.add_argument("--fbt-inplace", action="store_true", help="looped FBT: a feedback pass also fuses its own position's previous-pass state (+ W_p norm(h_t), zero init)")
# Training horizon (only one used, in order of precedence)
parser.add_argument("--num-iterations", type=int, default=-1, help="explicit number of optimization steps (-1 = disable)")
parser.add_argument("--target-flops", type=float, default=-1.0, help="calculate num_iterations to reach target_flops (-1 = disable)")
@@ -75,6 +81,7 @@ parser.add_argument("--core-metric-every", type=int, default=2000, help="evaluat
parser.add_argument("--core-metric-max-per-task", type=int, default=500, help="examples per task for CORE metric")
parser.add_argument("--sample-every", type=int, default=2000, help="sample from model every N steps (-1 = disable)")
parser.add_argument("--save-every", type=int, default=-1, help="save checkpoints every N steps (-1 = only at end)")
+parser.add_argument("--log-every", type=int, default=1, help="log training metrics to wandb every N steps (0 = never)")
# Output
parser.add_argument("--model-tag", type=str, default=None, help="override model tag for checkpoint directory name")
args = parser.parse_args()
@@ -98,6 +105,8 @@ print0(f"COMPUTE_DTYPE: {COMPUTE_DTYPE} ({COMPUTE_DTYPE_REASON})")
# wandb logging init
use_dummy_wandb = args.run == "dummy" or not master_process
wandb_run = DummyWandb() if use_dummy_wandb else wandb.init(project="nanochat", name=args.run, config=user_config)
+# Every wandb_run.log below passes step=step, so wandb's own step (the default x-axis) is the training step. Otherwise it
+# counts log calls (train, val and CORE logs at different cadences put e.g. the step-750 val point at x=753).
# Flash Attention status
from nanochat.flash_attention import USE_FA3
@@ -136,7 +145,7 @@ def build_model_meta(depth):
config = GPTConfig(
sequence_len=args.max_seq_len, vocab_size=vocab_size,
n_layer=depth, n_head=num_heads, n_kv_head=num_heads, n_embd=model_dim,
- window_pattern=args.window_pattern,
+ window_pattern=args.window_pattern, attn_res=args.attn_res, fbt_passes=args.fbt_passes, fbt_inplace=args.fbt_inplace,
)
with torch.device("meta"):
model_meta = GPT(config)
@@ -313,6 +322,9 @@ optimizer = model.setup_optimizer(
# Muon hyperparameters
matrix_lr=args.matrix_lr * batch_lr_scale,
weight_decay=weight_decay_scaled,
+ # AttnRes pseudo-queries (only with --attn-res)
+ attn_res_lr=args.attn_res_lr,
+ attn_res_betas=(args.attn_res_beta1, args.attn_res_beta2),
)
if resuming:
@@ -432,7 +444,7 @@ while True:
"total_training_flops": flops_so_far,
"total_training_time": total_training_time,
"val/bpb": val_bpb,
- })
+ }, step=step)
model.train()
# once in a while: estimate the CORE metric (all ranks participate)
@@ -449,12 +461,12 @@ while True:
"total_training_flops": flops_so_far,
"core_metric": results["core_metric"],
"centered_results": results["centered_results"],
- })
+ }, step=step)
model.train()
# once in a while: sample from the model (only on master process)
# use the original uncompiled model because the inputs keep changing shape
- if args.sample_every > 0 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)):
+ if args.sample_every > 0 and args.fbt_passes == 1 and master_process and (last_step or (step > 0 and step % args.sample_every == 0)): # (FBT: no KV-cache sampler)
model.eval()
prompts = [
"The capital of France is",
@@ -507,9 +519,11 @@ while True:
# evaluate the gradient
synchronize()
t0 = time.time()
+ loss_sum = 0 # this step's loss summed over micro-batches (rank 0's data), for logging
for micro_step in range(grad_accum_steps):
loss = model(x, y)
train_loss = loss.detach() # for logging
+ loss_sum = loss_sum + train_loss
loss = loss / grad_accum_steps # each .backward() is a grad sum => normalize loss here
if scaler is not None:
scaler.scale(loss).backward()
@@ -538,7 +552,7 @@ while True:
else:
optimizer.step()
model.zero_grad(set_to_none=True)
- train_loss_f = train_loss.item() # .item() is a CPU-GPU sync point
+ train_loss_f, step_loss_f = torch.stack([train_loss, loss_sum / grad_accum_steps]).tolist() # one CPU-GPU sync point
synchronize()
t1 = time.time()
dt = t1 - t0
@@ -565,19 +579,32 @@ while True:
eta_str = ""
epoch = f"{dataloader_state_dict['epoch']} pq: {dataloader_state_dict['pq_idx']} rg: {dataloader_state_dict['rg_idx']}"
print0(f"step {step:05d}/{num_iterations:05d} ({pct_done:.2f}%) | loss: {debiased_smooth_loss:.6f} | lrm: {lrm:.2f} | dt: {dt * 1000:.2f}ms | tok/sec: {tok_per_sec:,} | bf16_mfu: {mfu:.2f} | epoch: {epoch} | total time: {total_training_time/60:.2f}m{eta_str}")
- if step % 100 == 0:
+ if args.log_every > 0 and step % args.log_every == 0:
+ # every metric of the step, at the step (see the wandb init above); master process only (others: dummy)
log_data = {
"step": step,
"total_training_flops": flops_so_far,
"total_training_time": total_training_time,
- "train/loss": debiased_smooth_loss,
+ "train/loss": debiased_smooth_loss, # EMA (as printed)
+ "train/loss_step": step_loss_f, # this step's mean loss over its micro-batches, unsmoothed
+ "train/tokens": total_batch_size * (step + 1),
"train/lrm": lrm,
+ "train/lr_matrix": args.matrix_lr * batch_lr_scale * lrm,
+ "train/muon_momentum": muon_momentum,
+ "train/muon_weight_decay": muon_weight_decay,
"train/dt": dt,
"train/tok_per_sec": tok_per_sec,
"train/mfu": mfu,
- "train/epoch": epoch,
+ "train/peak_mem_gib": get_max_memory() / 2**30,
+ "data/epoch": dataloader_state_dict["epoch"],
+ "data/pq_idx": dataloader_state_dict["pq_idx"],
+ "data/rg_idx": dataloader_state_dict["rg_idx"],
}
- wandb_run.log(log_data)
+ if args.attn_res:
+ queries = torch.stack(list(orig_model.attn_res_queries))
+ log_data["train/lr_attn_res"] = args.attn_res_lr * lrm
+ log_data["attn_res/query_rms"] = queries.float().square().mean().sqrt().item()
+ wandb_run.log(log_data, step=step)
# state update
first_step_of_run = (step == 0) or (resuming and step == args.resume_from_step)