Download src/part2/sweep.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 6.79 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/sweep.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part2/sweep.py
-
curl -L -o sweep.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/sweep.py
6.79 kB
| """Bonus: peak learning-rate sweep for each optimizer. | |
| Each optimizer is run on a short slice of the full regime (the low-compute | |
| trick from the assignment brief: tune the non-model hyperparameters cheaply, | |
| then spend the real budget once). Only the peak learning rate is swept -- it | |
| is far and away the most sensitive knob, and the four optimizers disagree | |
| about it by two orders of magnitude, so inheriting AdamW's value would have | |
| handicapped Lion and Muon for reasons that have nothing to do with their | |
| update rules. | |
| python -m src.part2.sweep --budget 4000000 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import torch | |
| import torch.nn.functional as F | |
| from ..utils import RUNS, human, load_json, save_json, set_seed | |
| from .data import Blocks, get_stream, get_tokenizer, make_loader | |
| from .optimizers import DEFAULT_HPARAMS, OPTIMIZERS | |
| from .optimizers.sophia import SophiaG | |
| from .train import build_optimizer, eval_loss, lr_at, model_config | |
| # Grids were extended after a first pass put MARS and Muon at the top of their | |
| # range and Sophia at the bottom of its. A selected value sitting on a grid | |
| # edge means the sweep found the edge, not the optimum, so comparing optimizers | |
| # there would report grid design rather than update rules. | |
| GRIDS = { | |
| "adamw": [1e-4, 3e-4, 6e-4, 1e-3, 3e-3], | |
| "mars": [1e-4, 3e-4, 6e-4, 1e-3, 3e-3, 6e-3, 1e-2], | |
| "lion": [3e-5, 1e-4, 3e-4, 1e-3], | |
| "muon": [5e-3, 1e-2, 2e-2, 5e-2, 1e-1, 2e-1], | |
| "sophia": [1e-5, 3e-5, 1e-4, 3e-4, 6e-4, 1e-3, 3e-3], | |
| } | |
| def short_run(name, lr, args, streams, device): | |
| set_seed(args.seed) | |
| cfg = model_config(args.vocab_size, args.n_ctx) | |
| from ..part1.model import Transformer | |
| model = Transformer(cfg).to(device) | |
| hp = dict(DEFAULT_HPARAMS[name]) | |
| hp["lr"] = lr | |
| opt = build_optimizer(name, model, hp) | |
| tok_per_step = args.batch_size * args.n_ctx | |
| steps = args.budget // tok_per_step | |
| loader = make_loader(Blocks(streams["train"], args.n_ctx), | |
| args.batch_size, True, args.seed) | |
| val_loader = make_loader(Blocks(streams["val"], args.n_ctx), | |
| args.batch_size, False, num_workers=1) | |
| it = iter(loader) | |
| is_sophia = isinstance(opt, SophiaG) | |
| diverged = False | |
| for step in range(steps): | |
| cur = lr_at(step, steps, lr, min(args.warmup, steps // 5 + 1)) | |
| mult = cur / lr | |
| for g in opt.param_groups: | |
| g["lr"] = cur | |
| if "adamw_lr" in g: # Muon's internal AdamW rate | |
| g["adamw_lr"] = hp.get("adamw_lr", 3e-4) * mult | |
| try: | |
| x, y = next(it) | |
| except StopIteration: | |
| it = iter(loader) | |
| x, y = next(it) | |
| x, y = x.to(device), y.to(device) | |
| with torch.autocast("cuda", dtype=torch.bfloat16, | |
| enabled=device.type == "cuda"): | |
| logits, _, _ = model(x) | |
| loss = F.cross_entropy(logits.view(-1, logits.size(-1)).float(), | |
| y.reshape(-1)) | |
| if not torch.isfinite(loss): | |
| diverged = True | |
| break | |
| opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) | |
| if is_sophia: | |
| opt.step(bs_scale=tok_per_step) | |
| else: | |
| opt.step() | |
| # Sophia's curvature estimate must be refreshed here exactly as it is | |
| # during training. Without it ``h`` stays at zero, the denominator | |
| # collapses to eps, the clip saturates on every coordinate, and the | |
| # sweep would be tuning sign-SGD rather than Sophia -- handing the real | |
| # run a learning rate chosen for a different algorithm. | |
| if is_sophia and (step + 1) % args.sophia_k == 0: | |
| with torch.no_grad(): | |
| samp = torch.multinomial( | |
| logits.detach().float().view(-1, logits.size(-1)).softmax(-1), | |
| 1).squeeze(-1) | |
| with torch.autocast("cuda", dtype=torch.bfloat16, | |
| enabled=device.type == "cuda"): | |
| logits2, _, _ = model(x) | |
| l2 = F.cross_entropy(logits2.view(-1, logits2.size(-1)).float(), samp) | |
| opt.zero_grad(set_to_none=True) | |
| l2.backward() | |
| opt.update_hessian() | |
| opt.zero_grad(set_to_none=True) | |
| vl = float("nan") if diverged else eval_loss(model, val_loader, device, 30)[0] | |
| del model, opt | |
| torch.cuda.empty_cache() | |
| return vl, diverged | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--budget", type=int, default=4_000_000) | |
| ap.add_argument("--batch_size", type=int, default=24) | |
| ap.add_argument("--n_ctx", type=int, default=512) | |
| ap.add_argument("--vocab_size", type=int, default=16000) | |
| ap.add_argument("--warmup", type=int, default=100) | |
| ap.add_argument("--sophia_k", type=int, default=10, | |
| help="curvature refresh interval; must match training") | |
| ap.add_argument("--seed", type=int, default=1337) | |
| ap.add_argument("--optimizers", default=None, | |
| help="comma-separated subset; default all") | |
| args = ap.parse_args() | |
| device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
| tok = get_tokenizer(args.vocab_size) | |
| streams = {s: get_stream(tok, s) for s in ("train", "val")} | |
| names = args.optimizers.split(",") if args.optimizers else list(OPTIMIZERS) | |
| print(f"[sweep] {human(args.budget)} tokens per point " | |
| f"({args.budget // (args.batch_size*args.n_ctx)} steps)", flush=True) | |
| # Resume: reuse points already measured so an extended grid only pays for | |
| # the values that are new. | |
| sweep_path = RUNS / "part2" / "sweep.json" | |
| out = load_json(sweep_path) if sweep_path.exists() else {} | |
| for name in names: | |
| done = {r["lr"]: r for r in out.get(name, [])} | |
| out[name] = list(done.values()) | |
| for lr in GRIDS[name]: | |
| if lr in done: | |
| print(f" {name:8s} lr={lr:8.1e} (cached) " | |
| f"val {done[lr]['val_loss']:.4f}", flush=True) | |
| continue | |
| vl, div = short_run(name, lr, args, streams, device) | |
| tag = "DIVERGED" if div else f"val {vl:.4f}" | |
| print(f" {name:8s} lr={lr:8.1e} {tag}", flush=True) | |
| out[name].append({"lr": lr, "val_loss": vl, "diverged": div}) | |
| ok = [r for r in out[name] if not r["diverged"]] | |
| if ok: | |
| best = min(ok, key=lambda r: r["val_loss"]) | |
| print(f" -> best lr for {name}: {best['lr']:.1e} " | |
| f"(val {best['val_loss']:.4f})", flush=True) | |
| save_json(out, RUNS / "part2" / "sweep.json") | |
| print(f"\nwrote {RUNS/'part2'/'sweep.json'}") | |
| if __name__ == "__main__": | |
| main() | |