"""Figures for Part 2. Run after training: ``python -m src.part2.plots``.""" from __future__ import annotations import numpy as np from ..utils import ASSETS, LABELS, PALETTE, RUNS, load_json, use_plot_style ORDER = ["adamw", "mars", "lion", "muon", "sophia"] def load_all(): out = {} for n in ORDER: f = RUNS / "part2" / n / "metrics.json" if f.exists(): out[n] = load_json(f) return out def fig_dynamics(M, plt): """Validation loss and test BLEU against tokens -- Figure 2 of the paper.""" fig, axes = plt.subplots(1, 2, figsize=(10.2, 3.8)) for n, m in M.items(): h = m["history"] x = np.array(h["tokens"]) / 1e6 axes[0].plot(x, h["val_loss"], color=PALETTE[n], lw=2, marker="o", ms=3.5, label=LABELS[n]) axes[1].plot(x, h["bleu"], color=PALETTE[n], lw=2, marker="o", ms=3.5, label=LABELS[n]) for ax, key in ((axes[0], "val_loss"), (axes[1], "bleu")): ax.annotate(LABELS[n], (x[-1], h[key][-1]), textcoords="offset points", xytext=(4, 0), fontsize=7, color=PALETTE[n], va="center") axes[0].set_ylabel("Validation loss") axes[1].set_ylabel("Test BLEU (greedy continuations)") for ax in axes: ax.set_xlabel("Training tokens (millions)") ax.set_xlim(0, x[-1] * 1.22) axes[0].legend(fontsize=7) # Second x-axis in Chinchilla units, which is how the paper reports budget. n_params = next(iter(M.values()))["n_params"] top = axes[0].secondary_xaxis( "top", functions=(lambda v: v * 1e6 / (20 * n_params), lambda v: v * 20 * n_params / 1e6)) top.set_xlabel("Chinchilla ratio (tokens / 20N)", fontsize=8) fig.suptitle("Part 2 - training dynamics by optimizer, up to 1x the corpus", fontsize=10, y=1.06) fig.savefig(ASSETS / "p2_dynamics.png", bbox_inches="tight") plt.close(fig) def fig_summary(M, plt): """Final quality, per-step cost and optimizer-state memory. Cost comes from src/part2/benchmark.py, not from the training runs' own wall-clock: those runs shared the GPU with other jobs, so their totals reflect what else the device happened to be doing rather than the update rule. The benchmark times every optimizer back-to-back under identical conditions instead. """ names = list(M) x = np.arange(len(names)) colors = [PALETTE[n] for n in names] ppl = [M[n]["test_ppl"] for n in names] bleu = [M[n]["test_bleu"] for n in names] mem = [M[n]["state_multiple"] for n in names] bench_f = RUNS / "part2" / "benchmark.json" if bench_f.exists(): B = {r["optimizer"]: r for r in load_json(bench_f)} cost = [B[n]["step_ms"] if n in B else 0.0 for n in names] cost_lab = "Optimizer step (ms), controlled" cost_fmt = "{:.1f}" else: cost = [M[n]["minutes"] for n in names] cost_lab = "Wall-clock minutes (shared GPU)" cost_fmt = "{:.1f}" fig, axes = plt.subplots(1, 4, figsize=(14, 3.6)) for ax, vals, lab, fmt in ( (axes[0], ppl, "Test perplexity (lower better)", "{:.2f}"), (axes[1], bleu, "Test BLEU (higher better)", "{:.2f}"), (axes[2], cost, cost_lab, cost_fmt), (axes[3], mem, "Optimizer state (x model size)", "{:.1f}x")): ax.bar(x, vals, color=colors, width=0.62) for i, v in enumerate(vals): ax.text(i, v, fmt.format(v), ha="center", va="bottom", fontsize=8) ax.set_ylabel(lab, fontsize=8) ax.set_xticks(x) ax.set_xticklabels([LABELS[n] for n in names], fontsize=7, rotation=20, ha="right") ax.set_ylim(0, max(vals) * 1.2) ax.grid(axis="x", visible=False) fig.suptitle("Part 2 - final quality, per-step cost and memory by optimizer", fontsize=10, y=1.04) fig.savefig(ASSETS / "p2_summary.png", bbox_inches="tight") plt.close(fig) def fig_sweep(plt): """Learning-rate sweep, if one was run.""" f = RUNS / "part2" / "sweep.json" if not f.exists(): return S = load_json(f) fig, ax = plt.subplots(figsize=(5.4, 3.6)) for n in ORDER: # Skip optimizers with no usable points: an entry can be empty (its # grid was reset) or hold only diverged runs. runs = [r for r in S.get(n, []) if not r.get("diverged") and r["val_loss"] == r["val_loss"]] if not runs: continue lrs = [r["lr"] for r in runs] vl = [r["val_loss"] for r in runs] o = np.argsort(lrs) lrs, vl = np.array(lrs)[o], np.array(vl)[o] ax.plot(lrs, vl, color=PALETTE[n], lw=2, marker="o", ms=4, label=LABELS[n]) best = int(np.argmin(vl)) ax.scatter([lrs[best]], [vl[best]], s=110, facecolor="none", edgecolor=PALETTE[n], linewidth=1.8, zorder=4) ax.set_xscale("log") ax.set_xlabel("Peak learning rate") ax.set_ylabel("Validation loss after short run") ax.legend(fontsize=7) ax.set_title("Part 2 - learning-rate sweep (circled = selected)", fontsize=9) fig.savefig(ASSETS / "p2_sweep.png", bbox_inches="tight") plt.close(fig) def write_table(M): bench_f = RUNS / "part2" / "benchmark.json" B = {r["optimizer"]: r for r in load_json(bench_f)} if bench_f.exists() else {} lines = [f"{'optimizer':10s} {'category':34s} {'lr':>8s} {'ppl':>8s} " f"{'BLEU':>6s} {'step_ms':>8s} {'state':>7s}"] for n, m in M.items(): step_ms = B[n]["step_ms"] if n in B else float("nan") lines.append(f"{LABELS[n]:10s} {m['category']:34s} " f"{m['hparams']['lr']:8.1e} {m['test_ppl']:8.3f} " f"{m['test_bleu']:6.2f} {step_ms:8.2f} " f"{m['state_multiple']:6.1f}x") txt = "\n".join(lines) (ASSETS / "p2_table.txt").write_text(txt + "\n") print(txt) def main(): plt = use_plot_style() ASSETS.mkdir(parents=True, exist_ok=True) M = load_all() if not M: print("no part2 metrics found") return fig_dynamics(M, plt) fig_summary(M, plt) fig_sweep(plt) write_table(M) print(f"\nwrote figures to {ASSETS}") if __name__ == "__main__": main()