"""Bonus: peak learning-rate sweep for each optimizer. Each optimizer is run on a short slice of the full regime (the low-compute trick from the assignment brief: tune the non-model hyperparameters cheaply, then spend the real budget once). Only the peak learning rate is swept -- it is far and away the most sensitive knob, and the four optimizers disagree about it by two orders of magnitude, so inheriting AdamW's value would have handicapped Lion and Muon for reasons that have nothing to do with their update rules. python -m src.part2.sweep --budget 4000000 """ from __future__ import annotations import argparse import torch import torch.nn.functional as F from ..utils import RUNS, human, load_json, save_json, set_seed from .data import Blocks, get_stream, get_tokenizer, make_loader from .optimizers import DEFAULT_HPARAMS, OPTIMIZERS from .optimizers.sophia import SophiaG from .train import build_optimizer, eval_loss, lr_at, model_config # Grids were extended after a first pass put MARS and Muon at the top of their # range and Sophia at the bottom of its. A selected value sitting on a grid # edge means the sweep found the edge, not the optimum, so comparing optimizers # there would report grid design rather than update rules. GRIDS = { "adamw": [1e-4, 3e-4, 6e-4, 1e-3, 3e-3], "mars": [1e-4, 3e-4, 6e-4, 1e-3, 3e-3, 6e-3, 1e-2], "lion": [3e-5, 1e-4, 3e-4, 1e-3], "muon": [5e-3, 1e-2, 2e-2, 5e-2, 1e-1, 2e-1], "sophia": [1e-5, 3e-5, 1e-4, 3e-4, 6e-4, 1e-3, 3e-3], } def short_run(name, lr, args, streams, device): set_seed(args.seed) cfg = model_config(args.vocab_size, args.n_ctx) from ..part1.model import Transformer model = Transformer(cfg).to(device) hp = dict(DEFAULT_HPARAMS[name]) hp["lr"] = lr opt = build_optimizer(name, model, hp) tok_per_step = args.batch_size * args.n_ctx steps = args.budget // tok_per_step loader = make_loader(Blocks(streams["train"], args.n_ctx), args.batch_size, True, args.seed) val_loader = make_loader(Blocks(streams["val"], args.n_ctx), args.batch_size, False, num_workers=1) it = iter(loader) is_sophia = isinstance(opt, SophiaG) diverged = False for step in range(steps): cur = lr_at(step, steps, lr, min(args.warmup, steps // 5 + 1)) mult = cur / lr for g in opt.param_groups: g["lr"] = cur if "adamw_lr" in g: # Muon's internal AdamW rate g["adamw_lr"] = hp.get("adamw_lr", 3e-4) * mult try: x, y = next(it) except StopIteration: it = iter(loader) x, y = next(it) x, y = x.to(device), y.to(device) with torch.autocast("cuda", dtype=torch.bfloat16, enabled=device.type == "cuda"): logits, _, _ = model(x) loss = F.cross_entropy(logits.view(-1, logits.size(-1)).float(), y.reshape(-1)) if not torch.isfinite(loss): diverged = True break opt.zero_grad(set_to_none=True) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) if is_sophia: opt.step(bs_scale=tok_per_step) else: opt.step() # Sophia's curvature estimate must be refreshed here exactly as it is # during training. Without it ``h`` stays at zero, the denominator # collapses to eps, the clip saturates on every coordinate, and the # sweep would be tuning sign-SGD rather than Sophia -- handing the real # run a learning rate chosen for a different algorithm. if is_sophia and (step + 1) % args.sophia_k == 0: with torch.no_grad(): samp = torch.multinomial( logits.detach().float().view(-1, logits.size(-1)).softmax(-1), 1).squeeze(-1) with torch.autocast("cuda", dtype=torch.bfloat16, enabled=device.type == "cuda"): logits2, _, _ = model(x) l2 = F.cross_entropy(logits2.view(-1, logits2.size(-1)).float(), samp) opt.zero_grad(set_to_none=True) l2.backward() opt.update_hessian() opt.zero_grad(set_to_none=True) vl = float("nan") if diverged else eval_loss(model, val_loader, device, 30)[0] del model, opt torch.cuda.empty_cache() return vl, diverged def main(): ap = argparse.ArgumentParser() ap.add_argument("--budget", type=int, default=4_000_000) ap.add_argument("--batch_size", type=int, default=24) ap.add_argument("--n_ctx", type=int, default=512) ap.add_argument("--vocab_size", type=int, default=16000) ap.add_argument("--warmup", type=int, default=100) ap.add_argument("--sophia_k", type=int, default=10, help="curvature refresh interval; must match training") ap.add_argument("--seed", type=int, default=1337) ap.add_argument("--optimizers", default=None, help="comma-separated subset; default all") args = ap.parse_args() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") tok = get_tokenizer(args.vocab_size) streams = {s: get_stream(tok, s) for s in ("train", "val")} names = args.optimizers.split(",") if args.optimizers else list(OPTIMIZERS) print(f"[sweep] {human(args.budget)} tokens per point " f"({args.budget // (args.batch_size*args.n_ctx)} steps)", flush=True) # Resume: reuse points already measured so an extended grid only pays for # the values that are new. sweep_path = RUNS / "part2" / "sweep.json" out = load_json(sweep_path) if sweep_path.exists() else {} for name in names: done = {r["lr"]: r for r in out.get(name, [])} out[name] = list(done.values()) for lr in GRIDS[name]: if lr in done: print(f" {name:8s} lr={lr:8.1e} (cached) " f"val {done[lr]['val_loss']:.4f}", flush=True) continue vl, div = short_run(name, lr, args, streams, device) tag = "DIVERGED" if div else f"val {vl:.4f}" print(f" {name:8s} lr={lr:8.1e} {tag}", flush=True) out[name].append({"lr": lr, "val_loss": vl, "diverged": div}) ok = [r for r in out[name] if not r["diverged"]] if ok: best = min(ok, key=lambda r: r["val_loss"]) print(f" -> best lr for {name}: {best['lr']:.1e} " f"(val {best['val_loss']:.4f})", flush=True) save_json(out, RUNS / "part2" / "sweep.json") print(f"\nwrote {RUNS/'part2'/'sweep.json'}") if __name__ == "__main__": main()