"""Train one FFN ablation for Part 1. Every configuration sees the same data in the same order for the same number of tokens, so the only thing that differs between runs is the feed-forward block. Usage: python -m src.part1.train --config c3_moe_4e2a python -m src.part1.train --all """ from __future__ import annotations import argparse import math import time import numpy as np import torch from ..utils import RUNS, human, save_json, set_seed from .config import build_configs from .data import PackedBlocks, get_streams, get_tokenizer, make_loader from .evaluate import compute_bleu, evaluate_loss, expert_usage, usage_summary from .model import Transformer def lr_at(step, total, base_lr, warmup, floor_frac=0.1): if step < warmup: return base_lr * (step + 1) / warmup t = (step - warmup) / max(total - warmup, 1) return base_lr * (floor_frac + (1 - floor_frac) * 0.5 * (1 + math.cos(math.pi * t))) def train_one(cfg, args, streams, device): set_seed(args.seed) model = Transformer(cfg).to(device) total_p, active_p = cfg.param_counts() print(f"\n=== {cfg.name} | total {human(total_p)} | active {human(active_p)} " f"| d_ff_expert {cfg.d_ff_expert} ===", flush=True) train_ds = PackedBlocks(*streams["train"], cfg.n_ctx) val_ds = PackedBlocks(*streams["validation"], cfg.n_ctx) test_ds = PackedBlocks(*streams["test"], cfg.n_ctx) train_loader = make_loader(train_ds, args.batch_size, True, args.seed) val_loader = make_loader(val_ds, args.batch_size, False, num_workers=1) test_loader = make_loader(test_ds, args.batch_size, False, num_workers=1) # Decay only matrices; norms and embeddings are left alone. decay = [p for n, p in model.named_parameters() if p.dim() >= 2] nodecay = [p for n, p in model.named_parameters() if p.dim() < 2] opt = torch.optim.AdamW( [{"params": decay, "weight_decay": args.weight_decay}, {"params": nodecay, "weight_decay": 0.0}], lr=args.lr, betas=(0.9, 0.95), eps=1e-8) tok_per_step = args.batch_size * cfg.n_ctx total_steps = args.token_budget // tok_per_step print(f" {total_steps} steps x {tok_per_step} tok = " f"{human(total_steps * tok_per_step)} tokens", flush=True) hist = {"step": [], "tokens": [], "train_loss": [], "val_loss": [], "val_ppl": [], "aux": []} it = iter(train_loader) t0 = time.perf_counter() running, seen = 0.0, 0 for step in range(total_steps): for g in opt.param_groups: g["lr"] = lr_at(step, total_steps, args.lr, args.warmup) try: x, y, m, _ = next(it) except StopIteration: it = iter(train_loader) x, y, m, _ = next(it) x, y, m = x.to(device, non_blocking=True), y.to(device, non_blocking=True), m.to(device, non_blocking=True) with torch.autocast("cuda", dtype=torch.bfloat16, enabled=device.type == "cuda"): _, loss, (aux, z) = model(x, y, m) total_loss = loss + aux + z opt.zero_grad(set_to_none=True) total_loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) opt.step() running += loss.item() seen += 1 if (step + 1) % args.eval_every == 0 or step == total_steps - 1: vl, vp = evaluate_loss(model, val_loader, device, args.eval_batches) el = time.perf_counter() - t0 hist["step"].append(step + 1) hist["tokens"].append((step + 1) * tok_per_step) hist["train_loss"].append(running / max(seen, 1)) hist["val_loss"].append(vl) hist["val_ppl"].append(vp) hist["aux"].append(float(aux)) print(f" step {step+1:5d}/{total_steps} " f"train {running/max(seen,1):.4f} val {vl:.4f} ppl {vp:8.2f} " f"aux {float(aux):.3f} {el/60:.1f}m", flush=True) running, seen = 0.0, 0 train_minutes = (time.perf_counter() - t0) / 60 # ---- final evaluation ------------------------------------------------- test_loss, test_ppl = evaluate_loss(model, test_loader, device, args.test_batches) tok = streams["tokenizer"] bleu, samples = compute_bleu(model, tok, device, "test", args.bleu_pairs, n_ctx=cfg.n_ctx) counts = expert_usage(model, test_loader, device, max_batches=args.usage_batches) summary = usage_summary(counts) print(f" TEST loss {test_loss:.4f} ppl {test_ppl:.2f} " f"BLEU {bleu['bleu_all']:.2f} " f"(vi {bleu.get('bleu_vi', float('nan')):.2f} / " f"ja {bleu.get('bleu_ja', float('nan')):.2f})", flush=True) if summary: print(f" expert balance: mean entropy {summary['mean_entropy']:.3f} " f"(1.0 = uniform), max share {summary['max_share']:.3f}", flush=True) out = RUNS / "part1" / cfg.name out.mkdir(parents=True, exist_ok=True) torch.save({"model": model.state_dict(), "config": cfg.to_dict()}, out / "model.pt") if counts is not None: np.save(out / "expert_counts.npy", counts) save_json({ "config": cfg.to_dict(), "params_total": total_p, "params_active": active_p, "params_active_frac": active_p / total_p, "tokens_trained": total_steps * tok_per_step, "train_minutes": train_minutes, "history": hist, "test_loss": test_loss, "test_ppl": test_ppl, **bleu, "expert_usage": summary, "samples": samples, }, out / "metrics.json") del model, opt torch.cuda.empty_cache() def main(): ap = argparse.ArgumentParser() ap.add_argument("--config", default=None) ap.add_argument("--all", action="store_true") ap.add_argument("--token_budget", type=int, default=40_000_000) ap.add_argument("--batch_size", type=int, default=32) ap.add_argument("--n_ctx", type=int, default=256) ap.add_argument("--lr", type=float, default=6e-4) ap.add_argument("--weight_decay", type=float, default=0.1) ap.add_argument("--warmup", type=int, default=200) ap.add_argument("--eval_every", type=int, default=250) ap.add_argument("--eval_batches", type=int, default=60) ap.add_argument("--test_batches", type=int, default=200) ap.add_argument("--bleu_pairs", type=int, default=400) ap.add_argument("--usage_batches", type=int, default=40) ap.add_argument("--train_rows", type=int, default=250_000) ap.add_argument("--seed", type=int, default=1337) ap.add_argument("--vocab_size", type=int, default=16000) args = ap.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True tok = get_tokenizer(args.vocab_size, args.train_rows) streams = {"tokenizer": tok} for split, limit in (("train", args.train_rows), ("validation", None), ("test", None)): streams[split] = get_streams(tok, split, limit) print(f"[part1] {split}: {human(len(streams[split][0]))} tokens " f"({streams[split][1].mean()*100:.1f}% supervised)", flush=True) cfgs = build_configs(n_ctx=args.n_ctx, vocab_size=args.vocab_size) names = list(cfgs) if args.all else [args.config] for n in names: train_one(cfgs[n], args, streams, device) if __name__ == "__main__": main()