Download src/part1/train.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 7.47 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part1/train.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part1/train.py
-
curl -L -o train.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part1/train.py
7.47 kB
| """Train one FFN ablation for Part 1. | |
| Every configuration sees the same data in the same order for the same number | |
| of tokens, so the only thing that differs between runs is the feed-forward | |
| block. Usage: | |
| python -m src.part1.train --config c3_moe_4e2a | |
| python -m src.part1.train --all | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import math | |
| import time | |
| import numpy as np | |
| import torch | |
| from ..utils import RUNS, human, save_json, set_seed | |
| from .config import build_configs | |
| from .data import PackedBlocks, get_streams, get_tokenizer, make_loader | |
| from .evaluate import compute_bleu, evaluate_loss, expert_usage, usage_summary | |
| from .model import Transformer | |
| def lr_at(step, total, base_lr, warmup, floor_frac=0.1): | |
| if step < warmup: | |
| return base_lr * (step + 1) / warmup | |
| t = (step - warmup) / max(total - warmup, 1) | |
| return base_lr * (floor_frac + (1 - floor_frac) * 0.5 * (1 + math.cos(math.pi * t))) | |
| def train_one(cfg, args, streams, device): | |
| set_seed(args.seed) | |
| model = Transformer(cfg).to(device) | |
| total_p, active_p = cfg.param_counts() | |
| print(f"\n=== {cfg.name} | total {human(total_p)} | active {human(active_p)} " | |
| f"| d_ff_expert {cfg.d_ff_expert} ===", flush=True) | |
| train_ds = PackedBlocks(*streams["train"], cfg.n_ctx) | |
| val_ds = PackedBlocks(*streams["validation"], cfg.n_ctx) | |
| test_ds = PackedBlocks(*streams["test"], cfg.n_ctx) | |
| train_loader = make_loader(train_ds, args.batch_size, True, args.seed) | |
| val_loader = make_loader(val_ds, args.batch_size, False, num_workers=1) | |
| test_loader = make_loader(test_ds, args.batch_size, False, num_workers=1) | |
| # Decay only matrices; norms and embeddings are left alone. | |
| decay = [p for n, p in model.named_parameters() if p.dim() >= 2] | |
| nodecay = [p for n, p in model.named_parameters() if p.dim() < 2] | |
| opt = torch.optim.AdamW( | |
| [{"params": decay, "weight_decay": args.weight_decay}, | |
| {"params": nodecay, "weight_decay": 0.0}], | |
| lr=args.lr, betas=(0.9, 0.95), eps=1e-8) | |
| tok_per_step = args.batch_size * cfg.n_ctx | |
| total_steps = args.token_budget // tok_per_step | |
| print(f" {total_steps} steps x {tok_per_step} tok = " | |
| f"{human(total_steps * tok_per_step)} tokens", flush=True) | |
| hist = {"step": [], "tokens": [], "train_loss": [], "val_loss": [], | |
| "val_ppl": [], "aux": []} | |
| it = iter(train_loader) | |
| t0 = time.perf_counter() | |
| running, seen = 0.0, 0 | |
| for step in range(total_steps): | |
| for g in opt.param_groups: | |
| g["lr"] = lr_at(step, total_steps, args.lr, args.warmup) | |
| try: | |
| x, y, m, _ = next(it) | |
| except StopIteration: | |
| it = iter(train_loader) | |
| x, y, m, _ = next(it) | |
| x, y, m = x.to(device, non_blocking=True), y.to(device, non_blocking=True), m.to(device, non_blocking=True) | |
| with torch.autocast("cuda", dtype=torch.bfloat16, enabled=device.type == "cuda"): | |
| _, loss, (aux, z) = model(x, y, m) | |
| total_loss = loss + aux + z | |
| opt.zero_grad(set_to_none=True) | |
| total_loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| opt.step() | |
| running += loss.item() | |
| seen += 1 | |
| if (step + 1) % args.eval_every == 0 or step == total_steps - 1: | |
| vl, vp = evaluate_loss(model, val_loader, device, args.eval_batches) | |
| el = time.perf_counter() - t0 | |
| hist["step"].append(step + 1) | |
| hist["tokens"].append((step + 1) * tok_per_step) | |
| hist["train_loss"].append(running / max(seen, 1)) | |
| hist["val_loss"].append(vl) | |
| hist["val_ppl"].append(vp) | |
| hist["aux"].append(float(aux)) | |
| print(f" step {step+1:5d}/{total_steps} " | |
| f"train {running/max(seen,1):.4f} val {vl:.4f} ppl {vp:8.2f} " | |
| f"aux {float(aux):.3f} {el/60:.1f}m", flush=True) | |
| running, seen = 0.0, 0 | |
| train_minutes = (time.perf_counter() - t0) / 60 | |
| # ---- final evaluation ------------------------------------------------- | |
| test_loss, test_ppl = evaluate_loss(model, test_loader, device, args.test_batches) | |
| tok = streams["tokenizer"] | |
| bleu, samples = compute_bleu(model, tok, device, "test", args.bleu_pairs, | |
| n_ctx=cfg.n_ctx) | |
| counts = expert_usage(model, test_loader, device, max_batches=args.usage_batches) | |
| summary = usage_summary(counts) | |
| print(f" TEST loss {test_loss:.4f} ppl {test_ppl:.2f} " | |
| f"BLEU {bleu['bleu_all']:.2f} " | |
| f"(vi {bleu.get('bleu_vi', float('nan')):.2f} / " | |
| f"ja {bleu.get('bleu_ja', float('nan')):.2f})", flush=True) | |
| if summary: | |
| print(f" expert balance: mean entropy {summary['mean_entropy']:.3f} " | |
| f"(1.0 = uniform), max share {summary['max_share']:.3f}", flush=True) | |
| out = RUNS / "part1" / cfg.name | |
| out.mkdir(parents=True, exist_ok=True) | |
| torch.save({"model": model.state_dict(), "config": cfg.to_dict()}, | |
| out / "model.pt") | |
| if counts is not None: | |
| np.save(out / "expert_counts.npy", counts) | |
| save_json({ | |
| "config": cfg.to_dict(), | |
| "params_total": total_p, "params_active": active_p, | |
| "params_active_frac": active_p / total_p, | |
| "tokens_trained": total_steps * tok_per_step, | |
| "train_minutes": train_minutes, | |
| "history": hist, | |
| "test_loss": test_loss, "test_ppl": test_ppl, | |
| **bleu, | |
| "expert_usage": summary, | |
| "samples": samples, | |
| }, out / "metrics.json") | |
| del model, opt | |
| torch.cuda.empty_cache() | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--config", default=None) | |
| ap.add_argument("--all", action="store_true") | |
| ap.add_argument("--token_budget", type=int, default=40_000_000) | |
| ap.add_argument("--batch_size", type=int, default=32) | |
| ap.add_argument("--n_ctx", type=int, default=256) | |
| ap.add_argument("--lr", type=float, default=6e-4) | |
| ap.add_argument("--weight_decay", type=float, default=0.1) | |
| ap.add_argument("--warmup", type=int, default=200) | |
| ap.add_argument("--eval_every", type=int, default=250) | |
| ap.add_argument("--eval_batches", type=int, default=60) | |
| ap.add_argument("--test_batches", type=int, default=200) | |
| ap.add_argument("--bleu_pairs", type=int, default=400) | |
| ap.add_argument("--usage_batches", type=int, default=40) | |
| ap.add_argument("--train_rows", type=int, default=250_000) | |
| ap.add_argument("--seed", type=int, default=1337) | |
| ap.add_argument("--vocab_size", type=int, default=16000) | |
| args = ap.parse_args() | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| torch.backends.cuda.matmul.allow_tf32 = True | |
| torch.backends.cudnn.allow_tf32 = True | |
| tok = get_tokenizer(args.vocab_size, args.train_rows) | |
| streams = {"tokenizer": tok} | |
| for split, limit in (("train", args.train_rows), ("validation", None), | |
| ("test", None)): | |
| streams[split] = get_streams(tok, split, limit) | |
| print(f"[part1] {split}: {human(len(streams[split][0]))} tokens " | |
| f"({streams[split][1].mean()*100:.1f}% supervised)", flush=True) | |
| cfgs = build_configs(n_ctx=args.n_ctx, vocab_size=args.vocab_size) | |
| names = list(cfgs) if args.all else [args.config] | |
| for n in names: | |
| train_one(cfgs[n], args, streams, device) | |
| if __name__ == "__main__": | |
| main() | |