anlp-assignment2 / src /part2 /sweep.py
sahoosk's picture
Upload folder using huggingface_hub
5407bdc verified
Raw History Blame Contribute Delete
6.79 kB
"""Bonus: peak learning-rate sweep for each optimizer.
Each optimizer is run on a short slice of the full regime (the low-compute
trick from the assignment brief: tune the non-model hyperparameters cheaply,
then spend the real budget once). Only the peak learning rate is swept -- it
is far and away the most sensitive knob, and the four optimizers disagree
about it by two orders of magnitude, so inheriting AdamW's value would have
handicapped Lion and Muon for reasons that have nothing to do with their
update rules.
python -m src.part2.sweep --budget 4000000
"""
from __future__ import annotations
import argparse
import torch
import torch.nn.functional as F
from ..utils import RUNS, human, load_json, save_json, set_seed
from .data import Blocks, get_stream, get_tokenizer, make_loader
from .optimizers import DEFAULT_HPARAMS, OPTIMIZERS
from .optimizers.sophia import SophiaG
from .train import build_optimizer, eval_loss, lr_at, model_config
# Grids were extended after a first pass put MARS and Muon at the top of their
# range and Sophia at the bottom of its. A selected value sitting on a grid
# edge means the sweep found the edge, not the optimum, so comparing optimizers
# there would report grid design rather than update rules.
GRIDS = {
"adamw": [1e-4, 3e-4, 6e-4, 1e-3, 3e-3],
"mars": [1e-4, 3e-4, 6e-4, 1e-3, 3e-3, 6e-3, 1e-2],
"lion": [3e-5, 1e-4, 3e-4, 1e-3],
"muon": [5e-3, 1e-2, 2e-2, 5e-2, 1e-1, 2e-1],
"sophia": [1e-5, 3e-5, 1e-4, 3e-4, 6e-4, 1e-3, 3e-3],
}
def short_run(name, lr, args, streams, device):
set_seed(args.seed)
cfg = model_config(args.vocab_size, args.n_ctx)
from ..part1.model import Transformer
model = Transformer(cfg).to(device)
hp = dict(DEFAULT_HPARAMS[name])
hp["lr"] = lr
opt = build_optimizer(name, model, hp)
tok_per_step = args.batch_size * args.n_ctx
steps = args.budget // tok_per_step
loader = make_loader(Blocks(streams["train"], args.n_ctx),
args.batch_size, True, args.seed)
val_loader = make_loader(Blocks(streams["val"], args.n_ctx),
args.batch_size, False, num_workers=1)
it = iter(loader)
is_sophia = isinstance(opt, SophiaG)
diverged = False
for step in range(steps):
cur = lr_at(step, steps, lr, min(args.warmup, steps // 5 + 1))
mult = cur / lr
for g in opt.param_groups:
g["lr"] = cur
if "adamw_lr" in g: # Muon's internal AdamW rate
g["adamw_lr"] = hp.get("adamw_lr", 3e-4) * mult
try:
x, y = next(it)
except StopIteration:
it = iter(loader)
x, y = next(it)
x, y = x.to(device), y.to(device)
with torch.autocast("cuda", dtype=torch.bfloat16,
enabled=device.type == "cuda"):
logits, _, _ = model(x)
loss = F.cross_entropy(logits.view(-1, logits.size(-1)).float(),
y.reshape(-1))
if not torch.isfinite(loss):
diverged = True
break
opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
if is_sophia:
opt.step(bs_scale=tok_per_step)
else:
opt.step()
# Sophia's curvature estimate must be refreshed here exactly as it is
# during training. Without it ``h`` stays at zero, the denominator
# collapses to eps, the clip saturates on every coordinate, and the
# sweep would be tuning sign-SGD rather than Sophia -- handing the real
# run a learning rate chosen for a different algorithm.
if is_sophia and (step + 1) % args.sophia_k == 0:
with torch.no_grad():
samp = torch.multinomial(
logits.detach().float().view(-1, logits.size(-1)).softmax(-1),
1).squeeze(-1)
with torch.autocast("cuda", dtype=torch.bfloat16,
enabled=device.type == "cuda"):
logits2, _, _ = model(x)
l2 = F.cross_entropy(logits2.view(-1, logits2.size(-1)).float(), samp)
opt.zero_grad(set_to_none=True)
l2.backward()
opt.update_hessian()
opt.zero_grad(set_to_none=True)
vl = float("nan") if diverged else eval_loss(model, val_loader, device, 30)[0]
del model, opt
torch.cuda.empty_cache()
return vl, diverged
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--budget", type=int, default=4_000_000)
ap.add_argument("--batch_size", type=int, default=24)
ap.add_argument("--n_ctx", type=int, default=512)
ap.add_argument("--vocab_size", type=int, default=16000)
ap.add_argument("--warmup", type=int, default=100)
ap.add_argument("--sophia_k", type=int, default=10,
help="curvature refresh interval; must match training")
ap.add_argument("--seed", type=int, default=1337)
ap.add_argument("--optimizers", default=None,
help="comma-separated subset; default all")
args = ap.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
tok = get_tokenizer(args.vocab_size)
streams = {s: get_stream(tok, s) for s in ("train", "val")}
names = args.optimizers.split(",") if args.optimizers else list(OPTIMIZERS)
print(f"[sweep] {human(args.budget)} tokens per point "
f"({args.budget // (args.batch_size*args.n_ctx)} steps)", flush=True)
# Resume: reuse points already measured so an extended grid only pays for
# the values that are new.
sweep_path = RUNS / "part2" / "sweep.json"
out = load_json(sweep_path) if sweep_path.exists() else {}
for name in names:
done = {r["lr"]: r for r in out.get(name, [])}
out[name] = list(done.values())
for lr in GRIDS[name]:
if lr in done:
print(f" {name:8s} lr={lr:8.1e} (cached) "
f"val {done[lr]['val_loss']:.4f}", flush=True)
continue
vl, div = short_run(name, lr, args, streams, device)
tag = "DIVERGED" if div else f"val {vl:.4f}"
print(f" {name:8s} lr={lr:8.1e} {tag}", flush=True)
out[name].append({"lr": lr, "val_loss": vl, "diverged": div})
ok = [r for r in out[name] if not r["diverged"]]
if ok:
best = min(ok, key=lambda r: r["val_loss"])
print(f" -> best lr for {name}: {best['lr']:.1e} "
f"(val {best['val_loss']:.4f})", flush=True)
save_json(out, RUNS / "part2" / "sweep.json")
print(f"\nwrote {RUNS/'part2'/'sweep.json'}")
if __name__ == "__main__":
main()