anlp-assignment2 / src /part1 /train.py
sahoosk's picture
Upload folder using huggingface_hub
5407bdc verified
Raw History Blame Contribute Delete
7.47 kB
"""Train one FFN ablation for Part 1.
Every configuration sees the same data in the same order for the same number
of tokens, so the only thing that differs between runs is the feed-forward
block. Usage:
python -m src.part1.train --config c3_moe_4e2a
python -m src.part1.train --all
"""
from __future__ import annotations
import argparse
import math
import time
import numpy as np
import torch
from ..utils import RUNS, human, save_json, set_seed
from .config import build_configs
from .data import PackedBlocks, get_streams, get_tokenizer, make_loader
from .evaluate import compute_bleu, evaluate_loss, expert_usage, usage_summary
from .model import Transformer
def lr_at(step, total, base_lr, warmup, floor_frac=0.1):
if step < warmup:
return base_lr * (step + 1) / warmup
t = (step - warmup) / max(total - warmup, 1)
return base_lr * (floor_frac + (1 - floor_frac) * 0.5 * (1 + math.cos(math.pi * t)))
def train_one(cfg, args, streams, device):
set_seed(args.seed)
model = Transformer(cfg).to(device)
total_p, active_p = cfg.param_counts()
print(f"\n=== {cfg.name} | total {human(total_p)} | active {human(active_p)} "
f"| d_ff_expert {cfg.d_ff_expert} ===", flush=True)
train_ds = PackedBlocks(*streams["train"], cfg.n_ctx)
val_ds = PackedBlocks(*streams["validation"], cfg.n_ctx)
test_ds = PackedBlocks(*streams["test"], cfg.n_ctx)
train_loader = make_loader(train_ds, args.batch_size, True, args.seed)
val_loader = make_loader(val_ds, args.batch_size, False, num_workers=1)
test_loader = make_loader(test_ds, args.batch_size, False, num_workers=1)
# Decay only matrices; norms and embeddings are left alone.
decay = [p for n, p in model.named_parameters() if p.dim() >= 2]
nodecay = [p for n, p in model.named_parameters() if p.dim() < 2]
opt = torch.optim.AdamW(
[{"params": decay, "weight_decay": args.weight_decay},
{"params": nodecay, "weight_decay": 0.0}],
lr=args.lr, betas=(0.9, 0.95), eps=1e-8)
tok_per_step = args.batch_size * cfg.n_ctx
total_steps = args.token_budget // tok_per_step
print(f" {total_steps} steps x {tok_per_step} tok = "
f"{human(total_steps * tok_per_step)} tokens", flush=True)
hist = {"step": [], "tokens": [], "train_loss": [], "val_loss": [],
"val_ppl": [], "aux": []}
it = iter(train_loader)
t0 = time.perf_counter()
running, seen = 0.0, 0
for step in range(total_steps):
for g in opt.param_groups:
g["lr"] = lr_at(step, total_steps, args.lr, args.warmup)
try:
x, y, m, _ = next(it)
except StopIteration:
it = iter(train_loader)
x, y, m, _ = next(it)
x, y, m = x.to(device, non_blocking=True), y.to(device, non_blocking=True), m.to(device, non_blocking=True)
with torch.autocast("cuda", dtype=torch.bfloat16, enabled=device.type == "cuda"):
_, loss, (aux, z) = model(x, y, m)
total_loss = loss + aux + z
opt.zero_grad(set_to_none=True)
total_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
opt.step()
running += loss.item()
seen += 1
if (step + 1) % args.eval_every == 0 or step == total_steps - 1:
vl, vp = evaluate_loss(model, val_loader, device, args.eval_batches)
el = time.perf_counter() - t0
hist["step"].append(step + 1)
hist["tokens"].append((step + 1) * tok_per_step)
hist["train_loss"].append(running / max(seen, 1))
hist["val_loss"].append(vl)
hist["val_ppl"].append(vp)
hist["aux"].append(float(aux))
print(f" step {step+1:5d}/{total_steps} "
f"train {running/max(seen,1):.4f} val {vl:.4f} ppl {vp:8.2f} "
f"aux {float(aux):.3f} {el/60:.1f}m", flush=True)
running, seen = 0.0, 0
train_minutes = (time.perf_counter() - t0) / 60
# ---- final evaluation -------------------------------------------------
test_loss, test_ppl = evaluate_loss(model, test_loader, device, args.test_batches)
tok = streams["tokenizer"]
bleu, samples = compute_bleu(model, tok, device, "test", args.bleu_pairs,
n_ctx=cfg.n_ctx)
counts = expert_usage(model, test_loader, device, max_batches=args.usage_batches)
summary = usage_summary(counts)
print(f" TEST loss {test_loss:.4f} ppl {test_ppl:.2f} "
f"BLEU {bleu['bleu_all']:.2f} "
f"(vi {bleu.get('bleu_vi', float('nan')):.2f} / "
f"ja {bleu.get('bleu_ja', float('nan')):.2f})", flush=True)
if summary:
print(f" expert balance: mean entropy {summary['mean_entropy']:.3f} "
f"(1.0 = uniform), max share {summary['max_share']:.3f}", flush=True)
out = RUNS / "part1" / cfg.name
out.mkdir(parents=True, exist_ok=True)
torch.save({"model": model.state_dict(), "config": cfg.to_dict()},
out / "model.pt")
if counts is not None:
np.save(out / "expert_counts.npy", counts)
save_json({
"config": cfg.to_dict(),
"params_total": total_p, "params_active": active_p,
"params_active_frac": active_p / total_p,
"tokens_trained": total_steps * tok_per_step,
"train_minutes": train_minutes,
"history": hist,
"test_loss": test_loss, "test_ppl": test_ppl,
**bleu,
"expert_usage": summary,
"samples": samples,
}, out / "metrics.json")
del model, opt
torch.cuda.empty_cache()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--config", default=None)
ap.add_argument("--all", action="store_true")
ap.add_argument("--token_budget", type=int, default=40_000_000)
ap.add_argument("--batch_size", type=int, default=32)
ap.add_argument("--n_ctx", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--weight_decay", type=float, default=0.1)
ap.add_argument("--warmup", type=int, default=200)
ap.add_argument("--eval_every", type=int, default=250)
ap.add_argument("--eval_batches", type=int, default=60)
ap.add_argument("--test_batches", type=int, default=200)
ap.add_argument("--bleu_pairs", type=int, default=400)
ap.add_argument("--usage_batches", type=int, default=40)
ap.add_argument("--train_rows", type=int, default=250_000)
ap.add_argument("--seed", type=int, default=1337)
ap.add_argument("--vocab_size", type=int, default=16000)
args = ap.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
tok = get_tokenizer(args.vocab_size, args.train_rows)
streams = {"tokenizer": tok}
for split, limit in (("train", args.train_rows), ("validation", None),
("test", None)):
streams[split] = get_streams(tok, split, limit)
print(f"[part1] {split}: {human(len(streams[split][0]))} tokens "
f"({streams[split][1].mean()*100:.1f}% supervised)", flush=True)
cfgs = build_configs(n_ctx=args.n_ctx, vocab_size=args.vocab_size)
names = list(cfgs) if args.all else [args.config]
for n in names:
train_one(cfgs[n], args, streams, device)
if __name__ == "__main__":
main()