Download src/part2/plots.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 6.39 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/plots.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part2/plots.py
-
curl -L -o plots.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part2/plots.py
6.39 kB
| """Figures for Part 2. Run after training: ``python -m src.part2.plots``.""" | |
| from __future__ import annotations | |
| import numpy as np | |
| from ..utils import ASSETS, LABELS, PALETTE, RUNS, load_json, use_plot_style | |
| ORDER = ["adamw", "mars", "lion", "muon", "sophia"] | |
| def load_all(): | |
| out = {} | |
| for n in ORDER: | |
| f = RUNS / "part2" / n / "metrics.json" | |
| if f.exists(): | |
| out[n] = load_json(f) | |
| return out | |
| def fig_dynamics(M, plt): | |
| """Validation loss and test BLEU against tokens -- Figure 2 of the paper.""" | |
| fig, axes = plt.subplots(1, 2, figsize=(10.2, 3.8)) | |
| for n, m in M.items(): | |
| h = m["history"] | |
| x = np.array(h["tokens"]) / 1e6 | |
| axes[0].plot(x, h["val_loss"], color=PALETTE[n], lw=2, marker="o", | |
| ms=3.5, label=LABELS[n]) | |
| axes[1].plot(x, h["bleu"], color=PALETTE[n], lw=2, marker="o", ms=3.5, | |
| label=LABELS[n]) | |
| for ax, key in ((axes[0], "val_loss"), (axes[1], "bleu")): | |
| ax.annotate(LABELS[n], (x[-1], h[key][-1]), | |
| textcoords="offset points", xytext=(4, 0), | |
| fontsize=7, color=PALETTE[n], va="center") | |
| axes[0].set_ylabel("Validation loss") | |
| axes[1].set_ylabel("Test BLEU (greedy continuations)") | |
| for ax in axes: | |
| ax.set_xlabel("Training tokens (millions)") | |
| ax.set_xlim(0, x[-1] * 1.22) | |
| axes[0].legend(fontsize=7) | |
| # Second x-axis in Chinchilla units, which is how the paper reports budget. | |
| n_params = next(iter(M.values()))["n_params"] | |
| top = axes[0].secondary_xaxis( | |
| "top", functions=(lambda v: v * 1e6 / (20 * n_params), | |
| lambda v: v * 20 * n_params / 1e6)) | |
| top.set_xlabel("Chinchilla ratio (tokens / 20N)", fontsize=8) | |
| fig.suptitle("Part 2 - training dynamics by optimizer, up to 1x the corpus", | |
| fontsize=10, y=1.06) | |
| fig.savefig(ASSETS / "p2_dynamics.png", bbox_inches="tight") | |
| plt.close(fig) | |
| def fig_summary(M, plt): | |
| """Final quality, per-step cost and optimizer-state memory. | |
| Cost comes from src/part2/benchmark.py, not from the training runs' own | |
| wall-clock: those runs shared the GPU with other jobs, so their totals | |
| reflect what else the device happened to be doing rather than the update | |
| rule. The benchmark times every optimizer back-to-back under identical | |
| conditions instead. | |
| """ | |
| names = list(M) | |
| x = np.arange(len(names)) | |
| colors = [PALETTE[n] for n in names] | |
| ppl = [M[n]["test_ppl"] for n in names] | |
| bleu = [M[n]["test_bleu"] for n in names] | |
| mem = [M[n]["state_multiple"] for n in names] | |
| bench_f = RUNS / "part2" / "benchmark.json" | |
| if bench_f.exists(): | |
| B = {r["optimizer"]: r for r in load_json(bench_f)} | |
| cost = [B[n]["step_ms"] if n in B else 0.0 for n in names] | |
| cost_lab = "Optimizer step (ms), controlled" | |
| cost_fmt = "{:.1f}" | |
| else: | |
| cost = [M[n]["minutes"] for n in names] | |
| cost_lab = "Wall-clock minutes (shared GPU)" | |
| cost_fmt = "{:.1f}" | |
| fig, axes = plt.subplots(1, 4, figsize=(14, 3.6)) | |
| for ax, vals, lab, fmt in ( | |
| (axes[0], ppl, "Test perplexity (lower better)", "{:.2f}"), | |
| (axes[1], bleu, "Test BLEU (higher better)", "{:.2f}"), | |
| (axes[2], cost, cost_lab, cost_fmt), | |
| (axes[3], mem, "Optimizer state (x model size)", "{:.1f}x")): | |
| ax.bar(x, vals, color=colors, width=0.62) | |
| for i, v in enumerate(vals): | |
| ax.text(i, v, fmt.format(v), ha="center", va="bottom", fontsize=8) | |
| ax.set_ylabel(lab, fontsize=8) | |
| ax.set_xticks(x) | |
| ax.set_xticklabels([LABELS[n] for n in names], fontsize=7, rotation=20, | |
| ha="right") | |
| ax.set_ylim(0, max(vals) * 1.2) | |
| ax.grid(axis="x", visible=False) | |
| fig.suptitle("Part 2 - final quality, per-step cost and memory by optimizer", | |
| fontsize=10, y=1.04) | |
| fig.savefig(ASSETS / "p2_summary.png", bbox_inches="tight") | |
| plt.close(fig) | |
| def fig_sweep(plt): | |
| """Learning-rate sweep, if one was run.""" | |
| f = RUNS / "part2" / "sweep.json" | |
| if not f.exists(): | |
| return | |
| S = load_json(f) | |
| fig, ax = plt.subplots(figsize=(5.4, 3.6)) | |
| for n in ORDER: | |
| # Skip optimizers with no usable points: an entry can be empty (its | |
| # grid was reset) or hold only diverged runs. | |
| runs = [r for r in S.get(n, []) | |
| if not r.get("diverged") and r["val_loss"] == r["val_loss"]] | |
| if not runs: | |
| continue | |
| lrs = [r["lr"] for r in runs] | |
| vl = [r["val_loss"] for r in runs] | |
| o = np.argsort(lrs) | |
| lrs, vl = np.array(lrs)[o], np.array(vl)[o] | |
| ax.plot(lrs, vl, color=PALETTE[n], lw=2, marker="o", ms=4, | |
| label=LABELS[n]) | |
| best = int(np.argmin(vl)) | |
| ax.scatter([lrs[best]], [vl[best]], s=110, facecolor="none", | |
| edgecolor=PALETTE[n], linewidth=1.8, zorder=4) | |
| ax.set_xscale("log") | |
| ax.set_xlabel("Peak learning rate") | |
| ax.set_ylabel("Validation loss after short run") | |
| ax.legend(fontsize=7) | |
| ax.set_title("Part 2 - learning-rate sweep (circled = selected)", | |
| fontsize=9) | |
| fig.savefig(ASSETS / "p2_sweep.png", bbox_inches="tight") | |
| plt.close(fig) | |
| def write_table(M): | |
| bench_f = RUNS / "part2" / "benchmark.json" | |
| B = {r["optimizer"]: r for r in load_json(bench_f)} if bench_f.exists() else {} | |
| lines = [f"{'optimizer':10s} {'category':34s} {'lr':>8s} {'ppl':>8s} " | |
| f"{'BLEU':>6s} {'step_ms':>8s} {'state':>7s}"] | |
| for n, m in M.items(): | |
| step_ms = B[n]["step_ms"] if n in B else float("nan") | |
| lines.append(f"{LABELS[n]:10s} {m['category']:34s} " | |
| f"{m['hparams']['lr']:8.1e} {m['test_ppl']:8.3f} " | |
| f"{m['test_bleu']:6.2f} {step_ms:8.2f} " | |
| f"{m['state_multiple']:6.1f}x") | |
| txt = "\n".join(lines) | |
| (ASSETS / "p2_table.txt").write_text(txt + "\n") | |
| print(txt) | |
| def main(): | |
| plt = use_plot_style() | |
| ASSETS.mkdir(parents=True, exist_ok=True) | |
| M = load_all() | |
| if not M: | |
| print("no part2 metrics found") | |
| return | |
| fig_dynamics(M, plt) | |
| fig_summary(M, plt) | |
| fig_sweep(plt) | |
| write_table(M) | |
| print(f"\nwrote figures to {ASSETS}") | |
| if __name__ == "__main__": | |
| main() | |