anlp-assignment2 / src /part2 /plots.py
sahoosk's picture
Upload folder using huggingface_hub
5407bdc verified
Raw History Blame Contribute Delete
6.39 kB
"""Figures for Part 2. Run after training: ``python -m src.part2.plots``."""
from __future__ import annotations
import numpy as np
from ..utils import ASSETS, LABELS, PALETTE, RUNS, load_json, use_plot_style
ORDER = ["adamw", "mars", "lion", "muon", "sophia"]
def load_all():
out = {}
for n in ORDER:
f = RUNS / "part2" / n / "metrics.json"
if f.exists():
out[n] = load_json(f)
return out
def fig_dynamics(M, plt):
"""Validation loss and test BLEU against tokens -- Figure 2 of the paper."""
fig, axes = plt.subplots(1, 2, figsize=(10.2, 3.8))
for n, m in M.items():
h = m["history"]
x = np.array(h["tokens"]) / 1e6
axes[0].plot(x, h["val_loss"], color=PALETTE[n], lw=2, marker="o",
ms=3.5, label=LABELS[n])
axes[1].plot(x, h["bleu"], color=PALETTE[n], lw=2, marker="o", ms=3.5,
label=LABELS[n])
for ax, key in ((axes[0], "val_loss"), (axes[1], "bleu")):
ax.annotate(LABELS[n], (x[-1], h[key][-1]),
textcoords="offset points", xytext=(4, 0),
fontsize=7, color=PALETTE[n], va="center")
axes[0].set_ylabel("Validation loss")
axes[1].set_ylabel("Test BLEU (greedy continuations)")
for ax in axes:
ax.set_xlabel("Training tokens (millions)")
ax.set_xlim(0, x[-1] * 1.22)
axes[0].legend(fontsize=7)
# Second x-axis in Chinchilla units, which is how the paper reports budget.
n_params = next(iter(M.values()))["n_params"]
top = axes[0].secondary_xaxis(
"top", functions=(lambda v: v * 1e6 / (20 * n_params),
lambda v: v * 20 * n_params / 1e6))
top.set_xlabel("Chinchilla ratio (tokens / 20N)", fontsize=8)
fig.suptitle("Part 2 - training dynamics by optimizer, up to 1x the corpus",
fontsize=10, y=1.06)
fig.savefig(ASSETS / "p2_dynamics.png", bbox_inches="tight")
plt.close(fig)
def fig_summary(M, plt):
"""Final quality, per-step cost and optimizer-state memory.
Cost comes from src/part2/benchmark.py, not from the training runs' own
wall-clock: those runs shared the GPU with other jobs, so their totals
reflect what else the device happened to be doing rather than the update
rule. The benchmark times every optimizer back-to-back under identical
conditions instead.
"""
names = list(M)
x = np.arange(len(names))
colors = [PALETTE[n] for n in names]
ppl = [M[n]["test_ppl"] for n in names]
bleu = [M[n]["test_bleu"] for n in names]
mem = [M[n]["state_multiple"] for n in names]
bench_f = RUNS / "part2" / "benchmark.json"
if bench_f.exists():
B = {r["optimizer"]: r for r in load_json(bench_f)}
cost = [B[n]["step_ms"] if n in B else 0.0 for n in names]
cost_lab = "Optimizer step (ms), controlled"
cost_fmt = "{:.1f}"
else:
cost = [M[n]["minutes"] for n in names]
cost_lab = "Wall-clock minutes (shared GPU)"
cost_fmt = "{:.1f}"
fig, axes = plt.subplots(1, 4, figsize=(14, 3.6))
for ax, vals, lab, fmt in (
(axes[0], ppl, "Test perplexity (lower better)", "{:.2f}"),
(axes[1], bleu, "Test BLEU (higher better)", "{:.2f}"),
(axes[2], cost, cost_lab, cost_fmt),
(axes[3], mem, "Optimizer state (x model size)", "{:.1f}x")):
ax.bar(x, vals, color=colors, width=0.62)
for i, v in enumerate(vals):
ax.text(i, v, fmt.format(v), ha="center", va="bottom", fontsize=8)
ax.set_ylabel(lab, fontsize=8)
ax.set_xticks(x)
ax.set_xticklabels([LABELS[n] for n in names], fontsize=7, rotation=20,
ha="right")
ax.set_ylim(0, max(vals) * 1.2)
ax.grid(axis="x", visible=False)
fig.suptitle("Part 2 - final quality, per-step cost and memory by optimizer",
fontsize=10, y=1.04)
fig.savefig(ASSETS / "p2_summary.png", bbox_inches="tight")
plt.close(fig)
def fig_sweep(plt):
"""Learning-rate sweep, if one was run."""
f = RUNS / "part2" / "sweep.json"
if not f.exists():
return
S = load_json(f)
fig, ax = plt.subplots(figsize=(5.4, 3.6))
for n in ORDER:
# Skip optimizers with no usable points: an entry can be empty (its
# grid was reset) or hold only diverged runs.
runs = [r for r in S.get(n, [])
if not r.get("diverged") and r["val_loss"] == r["val_loss"]]
if not runs:
continue
lrs = [r["lr"] for r in runs]
vl = [r["val_loss"] for r in runs]
o = np.argsort(lrs)
lrs, vl = np.array(lrs)[o], np.array(vl)[o]
ax.plot(lrs, vl, color=PALETTE[n], lw=2, marker="o", ms=4,
label=LABELS[n])
best = int(np.argmin(vl))
ax.scatter([lrs[best]], [vl[best]], s=110, facecolor="none",
edgecolor=PALETTE[n], linewidth=1.8, zorder=4)
ax.set_xscale("log")
ax.set_xlabel("Peak learning rate")
ax.set_ylabel("Validation loss after short run")
ax.legend(fontsize=7)
ax.set_title("Part 2 - learning-rate sweep (circled = selected)",
fontsize=9)
fig.savefig(ASSETS / "p2_sweep.png", bbox_inches="tight")
plt.close(fig)
def write_table(M):
bench_f = RUNS / "part2" / "benchmark.json"
B = {r["optimizer"]: r for r in load_json(bench_f)} if bench_f.exists() else {}
lines = [f"{'optimizer':10s} {'category':34s} {'lr':>8s} {'ppl':>8s} "
f"{'BLEU':>6s} {'step_ms':>8s} {'state':>7s}"]
for n, m in M.items():
step_ms = B[n]["step_ms"] if n in B else float("nan")
lines.append(f"{LABELS[n]:10s} {m['category']:34s} "
f"{m['hparams']['lr']:8.1e} {m['test_ppl']:8.3f} "
f"{m['test_bleu']:6.2f} {step_ms:8.2f} "
f"{m['state_multiple']:6.1f}x")
txt = "\n".join(lines)
(ASSETS / "p2_table.txt").write_text(txt + "\n")
print(txt)
def main():
plt = use_plot_style()
ASSETS.mkdir(parents=True, exist_ok=True)
M = load_all()
if not M:
print("no part2 metrics found")
return
fig_dynamics(M, plt)
fig_summary(M, plt)
fig_sweep(plt)
write_table(M)
print(f"\nwrote figures to {ASSETS}")
if __name__ == "__main__":
main()