world-pretraining-runs / code /wp /report_figs.py
Broyojo's picture
Update analysis code
2ea7404 verified
Raw History Blame Contribute Delete
12.7 kB
"""Report figures (report/figs/fig_*.pdf) from pulled metrics.
Every figure is drawn at its exact printed size (NeurIPS text width 5.5in) so fonts print at
7pt, in Arial to match the TikZ diagrams. Colors follow the validated categorical palette
(reference order, adjacent pairs); all lines are solid, and markers separate conditions where points matter.
"""
import json
import sys
from collections import defaultdict
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
sys.path.insert(0, str(Path(__file__).parent))
from analyze import load_runs # noqa: E402
TEXTW = 5.5
INK, INK2, GRID = "#2b2a27", "#73716b", "#e6e3dc"
plt.rcParams.update({
"font.family": "sans-serif", "font.sans-serif": ["Arial"], "font.size": 7, "axes.labelsize": 7, "xtick.labelsize": 6.5,
"ytick.labelsize": 6.5, "legend.fontsize": 6.5, "axes.linewidth": 0.6, "axes.edgecolor": INK2,
"axes.labelcolor": INK, "xtick.color": INK2, "ytick.color": INK2, "xtick.major.width": 0.6,
"ytick.major.width": 0.6, "xtick.major.size": 2.5, "ytick.major.size": 2.5,
"axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "axes.axisbelow": True,
"grid.color": GRID, "grid.linewidth": 0.5, "lines.linewidth": 1.3, "savefig.bbox": "tight",
"savefig.pad_inches": 0.02, "pdf.fonttype": 42, "figure.dpi": 200,
})
ARM = { # key: (label, color, marker); all lines are solid
"scratch": ("scratch", "#2a78d6", "o"),
"ivgpt": ("pretrained", "#eb6834", "s"),
"shuffled": ("shuffled", "#1baf7a", "^"),
"rescaled": ("rescaled", "#eda100", "D"),
"sp0.4": ("S&P 位=0.4", "#e87ba4", "v"),
"spshuf0.4": ("shuffled S&P", "#008300", "x"),
}
OUT = Path("report/figs")
def plain_log_ticks(ax, ticks):
"""Log axis with plain tick labels (e.g. 50, 100, 200) instead of powers of ten."""
ax.set_xticks(ticks, [f"{t:g}" for t in ticks])
ax.minorticks_off()
def lr_str(lr):
return f"{lr:.0e}".replace("e-0", "e-")
def runs_for(runs, key, budget, freeze, lr=None):
return [r for r in runs if r["init"] == key and r["budget"] == budget and r["freeze"] == freeze and r["done"]
and (lr is None or r["lr"] == lr)]
def best_lr(runs, key, budget, freeze):
"""The LR with the most seeds, then the lowest mean final loss (same rule as the tables)."""
groups = defaultdict(list)
for r in runs_for(runs, key, budget, freeze):
groups[r["lr"]].append(r["val"][-1])
return min(groups, key=lambda l: (-len(groups[l]), np.mean(groups[l]))) if groups else None
def seed_curves(runs, k, budget):
"""Mean and range over seeds at the condition's best LR, on the eval steps all seeds share."""
reps = runs_for(runs, k, budget, "none", best_lr(runs, k, budget, "none"))
if not reps:
return None
toks = sorted(set.intersection(*[set(r["tokens"]) for r in reps]) - {0})
v = np.array([[dict(zip(r["tokens"], r["val"]))[t] for t in toks] for r in reps])
return np.array(toks) / 1e6, v.mean(0), v.min(0), v.max(0), len(reps)
def loss_panel(ax, runs, budget, keys, xlim=None, ylim=None, label=False):
ends = []
for k in keys:
c = seed_curves(runs, k, budget)
if c is None:
continue
x, m, lo, hi, n = c
lab, col, _ = ARM[k]
sel = np.ones_like(x, bool) if xlim is None else (x >= xlim[0] * 0.98)
ax.plot(x[sel], m[sel], "-", color=col, lw=1.4, label=lab)
if n > 1:
ax.fill_between(x[sel], lo[sel], hi[sel], color=col, alpha=0.16, lw=0)
ends.append((m[-1], f"{lab} {m[-1]:.3f}", col))
ax.set_xscale("log")
if xlim:
ax.set_xlim(*xlim)
if ylim:
ax.set_ylim(*ylim)
ax.set_xlabel("text tokens (millions)")
ax.set_ylabel("validation loss")
def fig_curves500(runs):
"""(a) Validation loss over 500M tokens. (b) Zoom on the second half of training."""
keys = ["scratch", "ivgpt", "sp0.4", "spshuf0.4"]
fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.8, 2.2), gridspec_kw={"wspace": 0.35, "width_ratios": [1, 1]})
loss_panel(a, runs, "500M", keys, ylim=(3.3, 6.6))
plain_log_ticks(a, [50, 100, 200, 500])
a.set_title("(a) Full run", fontsize=7)
loss_panel(b, runs, "500M", keys, xlim=(250, 640), ylim=(3.3, 3.95))
plain_log_ticks(b, [300, 400, 500])
b.set_title("(b) Second half", fontsize=7)
# arrow marking the final gap between scratch and S&P
ys, yp = seed_curves(runs, "scratch", "500M")[1][-1], seed_curves(runs, "sp0.4", "500M")[1][-1]
xa = 512 # just past the last point, where no line runs
b.annotate("", xy=(xa, yp), xytext=(xa, ys),
arrowprops=dict(arrowstyle="<->", color=INK, lw=0.8, shrinkA=0, shrinkB=0))
b.text(xa * 1.02, (ys + yp) / 2, f"{yp - ys:+.3f}\n({100 * (yp - ys) / ys:+.1f}%)", ha="left", va="center",
fontsize=6.5, color=INK)
fig.legend(*a.get_legend_handles_labels(), loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=4,
frameon=False, handlelength=1.8, columnspacing=1.4)
fig.savefig(OUT / "fig_curves500.pdf")
plt.close(fig)
return {k: float(seed_curves(runs, k, "500M")[1][-1]) for k in keys if seed_curves(runs, k, "500M")}
def fig_lr_and_2b(runs):
"""(a) LR sweep at 500M tokens. (b) Phase 2 curves relative to scratch over 2B tokens."""
fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.0), gridspec_kw={"wspace": 0.4})
for k in ("scratch", "ivgpt", "shuffled", "rescaled", "sp0.4", "spshuf0.4"):
pts = sorted((r["lr"], r["val"][-1]) for r in runs_for(runs, k, "500M", "none") if r["seed"] == 0)
if pts:
x, y = zip(*pts)
lab, col, mk = ARM[k]
a.plot(x, y, "-", marker=mk, ms=3, color=col, label=lab)
a.set_xscale("log")
a.set_xticks([3e-4, 1e-3, 3e-3, 1e-2], [lr_str(t) for t in (3e-4, 1e-3, 3e-3, 1e-2)])
a.minorticks_off()
a.set_ylim(3.3, 4.4)
a.set_xlabel("peak learning rate")
a.set_ylabel("validation loss")
a.set_title("(a) Learning-rate sweep, 500M tokens", fontsize=7)
fig.legend(*a.get_legend_handles_labels(), loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=6,
frameon=False, handlelength=1.8, columnspacing=1.1)
loss_panel(b, runs, "2000M", ["scratch", "ivgpt", "sp0.4"], xlim=(300, 2100), ylim=(2.9, 3.85))
plain_log_ticks(b, [300, 500, 1000, 2000])
b.set_title("(b) Training to 2B tokens", fontsize=7)
rows = {k: float(seed_curves(runs, k, "2000M")[1][-1]) for k in ("scratch", "ivgpt", "sp0.4") if seed_curves(runs, k, "2000M")}
fig.savefig(OUT / "fig_lr_2b.pdf")
plt.close(fig)
return rows
def final_points(runs, k):
"""(tokens, mean final loss, n) per budget at the condition's best 500M learning rate."""
lr = best_lr(runs, k, "500M", "none")
pts = defaultdict(list)
for r in runs:
if r["init"] == k and r["freeze"] == "none" and r["done"] and r["lr"] == lr and r["budget"] != "100M":
pts[int(r["budget"][:-1])].append(r["val"][-1])
return sorted((b, float(np.mean(v)), len(v)) for b, v in pts.items())
def effective_tokens(scratch_pts, loss):
"""Tokens scratch needs to reach `loss`, interpolating log(loss) linearly in log(tokens)."""
for (b0, l0, _), (b1, l1, _) in zip(scratch_pts, scratch_pts[1:]):
if l1 <= loss <= l0:
t = (np.log(l0) - np.log(loss)) / (np.log(l0) - np.log(l1))
return float(np.exp(np.log(b0) + t * (np.log(b1) - np.log(b0))))
return None
def fig_sp(runs):
"""(a) Final loss against the S&P mixing weight at LR 2e-3. (b) Final loss against token budget."""
fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.1), gridspec_kw={"wspace": 0.4})
lr = 2e-3
def final(init, budget="500M"):
v = [r["val"][-1] for r in runs if r["init"] == init and r["freeze"] == "none" and r["budget"] == budget
and r["done"] and r["lr"] == lr]
return float(np.mean(v)) if v else None
lam = [(0.0, final("scratch"))] + [(l, final(f"sp{l}")) for l in (0.1, 0.2, 0.4, 0.7)] + [(1.0, final("ivgpt"))]
lam = [(l, v) for l, v in lam if v is not None]
x, y = zip(*lam)
a.plot(x, y, "-", marker=ARM["sp0.4"][2], ms=3, color=ARM["sp0.4"][1], label="S&P")
a.plot([0], [lam[0][1]], ARM["scratch"][2], ms=4, color=ARM["scratch"][1], label="scratch (位=0)", zorder=3)
a.plot([1], [lam[-1][1]], ARM["ivgpt"][2], ms=4, color=ARM["ivgpt"][1], label="pretrained (位=1)", zorder=3)
shuf = final("spshuf0.4")
if shuf:
a.plot([0.4], [shuf], ARM["spshuf0.4"][2], ms=4, mew=1.2, color=ARM["spshuf0.4"][1], label="shuffled S&P")
a.set_xlabel("fraction of video weights kept (位)")
a.set_ylabel("validation loss")
a.set_title("(a) Mixing weight", fontsize=7)
out = {"lambda": lam, "shuffled": shuf}
pts = {k: final_points(runs, k) for k in ("scratch", "ivgpt", "sp0.4")}
for k, ps in pts.items():
if ps:
lab, col, mk = ARM[k]
bx, by, _ = zip(*ps)
b.plot(bx, by, "-", marker=mk, ms=3, color=col)
sp = dict((t, l) for t, l, _ in pts.get("sp0.4", []))
if 500 in sp and len(pts["scratch"]) >= 2:
eff = effective_tokens(pts["scratch"], sp[500])
out["sp500_effective_scratch_tokens_M"] = eff
if eff:
b.annotate("", xy=(eff, sp[500]), xytext=(500, sp[500]),
arrowprops=dict(arrowstyle="->", color=INK, lw=0.8, shrinkA=2, shrinkB=2))
b.text(470, sp[500], f"{eff / 500:.2f}x tokens", ha="right", va="center", fontsize=6.5, color=INK)
b.set_xscale("log")
xs = sorted({t for ps in pts.values() for t, _, _ in ps})
plain_log_ticks(b, xs)
b.set_xlabel("text tokens (millions)")
b.set_ylabel("validation loss")
b.set_title("(b) Token budget", fontsize=7)
out["points"] = pts
handles = [plt.Line2D([], [], color=ARM[k][1], marker=ARM[k][2], ms=3, ls="-") for k in ("scratch", "ivgpt", "sp0.4")]
handles.append(plt.Line2D([], [], color=ARM["spshuf0.4"][1], marker=ARM["spshuf0.4"][2], ms=4, mew=1.2, ls="none"))
fig.legend(handles, ["scratch", "pretrained", "S&P 位=0.4", "shuffled S&P"], loc="upper center",
bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=2)
fig.savefig(OUT / "fig_sp.pdf")
plt.close(fig)
return out
def fig_budget(runs):
"""Final validation loss against token budget for both models, at each condition's chosen LR."""
import act_table
fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.1), gridspec_kw={"wspace": 0.35})
out = {}
for k in ("scratch", "ivgpt", "sp0.4"):
ps = final_points(runs, k)
lab, col, mk = ARM[k]
bx, by, _ = zip(*ps)
a.plot(bx, by, "-", marker=mk, ms=3, color=col, label=lab)
out[f"medium/{k}"] = ps
root = Path("results/ice")
for (src, init, lr), key in [(("sm", "scratch", 3e-3), "scratch"), (("base", "shuffled", 5e-3), "shuffled"),
(("base", "ivgpt", 8e-3), "ivgpt")]:
ps = []
for name, t in [("500M", 500), ("1000M", 1000), ("2000M", 2000), ("3000M", 3000)]:
rs = act_table.load(root, name).get((src, init, lr))
if rs:
ps.append((t, float(np.mean([r["val"][-1] for r in rs])), len(rs)))
lab, col, mk = ARM[key]
bx, by, _ = zip(*ps)
b.plot(bx, by, "-", marker=mk, ms=3, color=col, label=lab)
out[f"small/{key}"] = ps
for ax, title in ((a, "(a) Medium model (400M backbone)"), (b, "(b) Small model (113M backbone)")):
ax.set_xscale("log")
plain_log_ticks(ax, [250, 500, 1000, 2000, 3000] if ax is a else [500, 1000, 2000, 3000])
ax.set_xlabel("text tokens (millions)")
ax.set_ylabel("validation loss")
ax.set_title(title, fontsize=7)
handles = [plt.Line2D([], [], color=ARM[k][1], marker=ARM[k][2], ms=3, ls="-") for k in ("scratch", "ivgpt", "shuffled", "sp0.4")]
fig.legend(handles, [ARM[k][0] for k in ("scratch", "ivgpt", "shuffled", "sp0.4")], loc="upper center",
bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=2)
fig.savefig(OUT / "fig_budget.pdf")
plt.close(fig)
return out
def main():
OUT.mkdir(parents=True, exist_ok=True)
runs = load_runs(["results/ice", "results/dm_backup"])
out = {"sp": fig_sp(runs), "curves500": fig_curves500(runs), "delta2B": fig_lr_and_2b(runs), "budget": fig_budget(runs)}
Path("report/fig_numbers.json").write_text(json.dumps(out, indent=1, default=str))
print(json.dumps(out, indent=1, default=str))
if __name__ == "__main__":
main()