Download code/wp/report_figs.py from Broyojo/world-pretraining-runs: direct link, hf CLI and curl.
- Browser
- Download file 12.7 kB
-
https://huggingface.co/Broyojo/world-pretraining-runs/resolve/main/code/wp/report_figs.py
- Command line
-
hf download hf://Broyojo/world-pretraining-runs/code/wp/report_figs.py
-
curl -L -o report_figs.py https://huggingface.co/Broyojo/world-pretraining-runs/resolve/main/code/wp/report_figs.py
12.7 kB
| """Report figures (report/figs/fig_*.pdf) from pulled metrics. | |
| Every figure is drawn at its exact printed size (NeurIPS text width 5.5in) so fonts print at | |
| 7pt, in Arial to match the TikZ diagrams. Colors follow the validated categorical palette | |
| (reference order, adjacent pairs); all lines are solid, and markers separate conditions where points matter. | |
| """ | |
| import json | |
| import sys | |
| from collections import defaultdict | |
| from pathlib import Path | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| sys.path.insert(0, str(Path(__file__).parent)) | |
| from analyze import load_runs # noqa: E402 | |
| TEXTW = 5.5 | |
| INK, INK2, GRID = "#2b2a27", "#73716b", "#e6e3dc" | |
| plt.rcParams.update({ | |
| "font.family": "sans-serif", "font.sans-serif": ["Arial"], "font.size": 7, "axes.labelsize": 7, "xtick.labelsize": 6.5, | |
| "ytick.labelsize": 6.5, "legend.fontsize": 6.5, "axes.linewidth": 0.6, "axes.edgecolor": INK2, | |
| "axes.labelcolor": INK, "xtick.color": INK2, "ytick.color": INK2, "xtick.major.width": 0.6, | |
| "ytick.major.width": 0.6, "xtick.major.size": 2.5, "ytick.major.size": 2.5, | |
| "axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "axes.axisbelow": True, | |
| "grid.color": GRID, "grid.linewidth": 0.5, "lines.linewidth": 1.3, "savefig.bbox": "tight", | |
| "savefig.pad_inches": 0.02, "pdf.fonttype": 42, "figure.dpi": 200, | |
| }) | |
| ARM = { # key: (label, color, marker); all lines are solid | |
| "scratch": ("scratch", "#2a78d6", "o"), | |
| "ivgpt": ("pretrained", "#eb6834", "s"), | |
| "shuffled": ("shuffled", "#1baf7a", "^"), | |
| "rescaled": ("rescaled", "#eda100", "D"), | |
| "sp0.4": ("S&P 位=0.4", "#e87ba4", "v"), | |
| "spshuf0.4": ("shuffled S&P", "#008300", "x"), | |
| } | |
| OUT = Path("report/figs") | |
| def plain_log_ticks(ax, ticks): | |
| """Log axis with plain tick labels (e.g. 50, 100, 200) instead of powers of ten.""" | |
| ax.set_xticks(ticks, [f"{t:g}" for t in ticks]) | |
| ax.minorticks_off() | |
| def lr_str(lr): | |
| return f"{lr:.0e}".replace("e-0", "e-") | |
| def runs_for(runs, key, budget, freeze, lr=None): | |
| return [r for r in runs if r["init"] == key and r["budget"] == budget and r["freeze"] == freeze and r["done"] | |
| and (lr is None or r["lr"] == lr)] | |
| def best_lr(runs, key, budget, freeze): | |
| """The LR with the most seeds, then the lowest mean final loss (same rule as the tables).""" | |
| groups = defaultdict(list) | |
| for r in runs_for(runs, key, budget, freeze): | |
| groups[r["lr"]].append(r["val"][-1]) | |
| return min(groups, key=lambda l: (-len(groups[l]), np.mean(groups[l]))) if groups else None | |
| def seed_curves(runs, k, budget): | |
| """Mean and range over seeds at the condition's best LR, on the eval steps all seeds share.""" | |
| reps = runs_for(runs, k, budget, "none", best_lr(runs, k, budget, "none")) | |
| if not reps: | |
| return None | |
| toks = sorted(set.intersection(*[set(r["tokens"]) for r in reps]) - {0}) | |
| v = np.array([[dict(zip(r["tokens"], r["val"]))[t] for t in toks] for r in reps]) | |
| return np.array(toks) / 1e6, v.mean(0), v.min(0), v.max(0), len(reps) | |
| def loss_panel(ax, runs, budget, keys, xlim=None, ylim=None, label=False): | |
| ends = [] | |
| for k in keys: | |
| c = seed_curves(runs, k, budget) | |
| if c is None: | |
| continue | |
| x, m, lo, hi, n = c | |
| lab, col, _ = ARM[k] | |
| sel = np.ones_like(x, bool) if xlim is None else (x >= xlim[0] * 0.98) | |
| ax.plot(x[sel], m[sel], "-", color=col, lw=1.4, label=lab) | |
| if n > 1: | |
| ax.fill_between(x[sel], lo[sel], hi[sel], color=col, alpha=0.16, lw=0) | |
| ends.append((m[-1], f"{lab} {m[-1]:.3f}", col)) | |
| ax.set_xscale("log") | |
| if xlim: | |
| ax.set_xlim(*xlim) | |
| if ylim: | |
| ax.set_ylim(*ylim) | |
| ax.set_xlabel("text tokens (millions)") | |
| ax.set_ylabel("validation loss") | |
| def fig_curves500(runs): | |
| """(a) Validation loss over 500M tokens. (b) Zoom on the second half of training.""" | |
| keys = ["scratch", "ivgpt", "sp0.4", "spshuf0.4"] | |
| fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.8, 2.2), gridspec_kw={"wspace": 0.35, "width_ratios": [1, 1]}) | |
| loss_panel(a, runs, "500M", keys, ylim=(3.3, 6.6)) | |
| plain_log_ticks(a, [50, 100, 200, 500]) | |
| a.set_title("(a) Full run", fontsize=7) | |
| loss_panel(b, runs, "500M", keys, xlim=(250, 640), ylim=(3.3, 3.95)) | |
| plain_log_ticks(b, [300, 400, 500]) | |
| b.set_title("(b) Second half", fontsize=7) | |
| # arrow marking the final gap between scratch and S&P | |
| ys, yp = seed_curves(runs, "scratch", "500M")[1][-1], seed_curves(runs, "sp0.4", "500M")[1][-1] | |
| xa = 512 # just past the last point, where no line runs | |
| b.annotate("", xy=(xa, yp), xytext=(xa, ys), | |
| arrowprops=dict(arrowstyle="<->", color=INK, lw=0.8, shrinkA=0, shrinkB=0)) | |
| b.text(xa * 1.02, (ys + yp) / 2, f"{yp - ys:+.3f}\n({100 * (yp - ys) / ys:+.1f}%)", ha="left", va="center", | |
| fontsize=6.5, color=INK) | |
| fig.legend(*a.get_legend_handles_labels(), loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=4, | |
| frameon=False, handlelength=1.8, columnspacing=1.4) | |
| fig.savefig(OUT / "fig_curves500.pdf") | |
| plt.close(fig) | |
| return {k: float(seed_curves(runs, k, "500M")[1][-1]) for k in keys if seed_curves(runs, k, "500M")} | |
| def fig_lr_and_2b(runs): | |
| """(a) LR sweep at 500M tokens. (b) Phase 2 curves relative to scratch over 2B tokens.""" | |
| fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.0), gridspec_kw={"wspace": 0.4}) | |
| for k in ("scratch", "ivgpt", "shuffled", "rescaled", "sp0.4", "spshuf0.4"): | |
| pts = sorted((r["lr"], r["val"][-1]) for r in runs_for(runs, k, "500M", "none") if r["seed"] == 0) | |
| if pts: | |
| x, y = zip(*pts) | |
| lab, col, mk = ARM[k] | |
| a.plot(x, y, "-", marker=mk, ms=3, color=col, label=lab) | |
| a.set_xscale("log") | |
| a.set_xticks([3e-4, 1e-3, 3e-3, 1e-2], [lr_str(t) for t in (3e-4, 1e-3, 3e-3, 1e-2)]) | |
| a.minorticks_off() | |
| a.set_ylim(3.3, 4.4) | |
| a.set_xlabel("peak learning rate") | |
| a.set_ylabel("validation loss") | |
| a.set_title("(a) Learning-rate sweep, 500M tokens", fontsize=7) | |
| fig.legend(*a.get_legend_handles_labels(), loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=6, | |
| frameon=False, handlelength=1.8, columnspacing=1.1) | |
| loss_panel(b, runs, "2000M", ["scratch", "ivgpt", "sp0.4"], xlim=(300, 2100), ylim=(2.9, 3.85)) | |
| plain_log_ticks(b, [300, 500, 1000, 2000]) | |
| b.set_title("(b) Training to 2B tokens", fontsize=7) | |
| rows = {k: float(seed_curves(runs, k, "2000M")[1][-1]) for k in ("scratch", "ivgpt", "sp0.4") if seed_curves(runs, k, "2000M")} | |
| fig.savefig(OUT / "fig_lr_2b.pdf") | |
| plt.close(fig) | |
| return rows | |
| def final_points(runs, k): | |
| """(tokens, mean final loss, n) per budget at the condition's best 500M learning rate.""" | |
| lr = best_lr(runs, k, "500M", "none") | |
| pts = defaultdict(list) | |
| for r in runs: | |
| if r["init"] == k and r["freeze"] == "none" and r["done"] and r["lr"] == lr and r["budget"] != "100M": | |
| pts[int(r["budget"][:-1])].append(r["val"][-1]) | |
| return sorted((b, float(np.mean(v)), len(v)) for b, v in pts.items()) | |
| def effective_tokens(scratch_pts, loss): | |
| """Tokens scratch needs to reach `loss`, interpolating log(loss) linearly in log(tokens).""" | |
| for (b0, l0, _), (b1, l1, _) in zip(scratch_pts, scratch_pts[1:]): | |
| if l1 <= loss <= l0: | |
| t = (np.log(l0) - np.log(loss)) / (np.log(l0) - np.log(l1)) | |
| return float(np.exp(np.log(b0) + t * (np.log(b1) - np.log(b0)))) | |
| return None | |
| def fig_sp(runs): | |
| """(a) Final loss against the S&P mixing weight at LR 2e-3. (b) Final loss against token budget.""" | |
| fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.1), gridspec_kw={"wspace": 0.4}) | |
| lr = 2e-3 | |
| def final(init, budget="500M"): | |
| v = [r["val"][-1] for r in runs if r["init"] == init and r["freeze"] == "none" and r["budget"] == budget | |
| and r["done"] and r["lr"] == lr] | |
| return float(np.mean(v)) if v else None | |
| lam = [(0.0, final("scratch"))] + [(l, final(f"sp{l}")) for l in (0.1, 0.2, 0.4, 0.7)] + [(1.0, final("ivgpt"))] | |
| lam = [(l, v) for l, v in lam if v is not None] | |
| x, y = zip(*lam) | |
| a.plot(x, y, "-", marker=ARM["sp0.4"][2], ms=3, color=ARM["sp0.4"][1], label="S&P") | |
| a.plot([0], [lam[0][1]], ARM["scratch"][2], ms=4, color=ARM["scratch"][1], label="scratch (位=0)", zorder=3) | |
| a.plot([1], [lam[-1][1]], ARM["ivgpt"][2], ms=4, color=ARM["ivgpt"][1], label="pretrained (位=1)", zorder=3) | |
| shuf = final("spshuf0.4") | |
| if shuf: | |
| a.plot([0.4], [shuf], ARM["spshuf0.4"][2], ms=4, mew=1.2, color=ARM["spshuf0.4"][1], label="shuffled S&P") | |
| a.set_xlabel("fraction of video weights kept (位)") | |
| a.set_ylabel("validation loss") | |
| a.set_title("(a) Mixing weight", fontsize=7) | |
| out = {"lambda": lam, "shuffled": shuf} | |
| pts = {k: final_points(runs, k) for k in ("scratch", "ivgpt", "sp0.4")} | |
| for k, ps in pts.items(): | |
| if ps: | |
| lab, col, mk = ARM[k] | |
| bx, by, _ = zip(*ps) | |
| b.plot(bx, by, "-", marker=mk, ms=3, color=col) | |
| sp = dict((t, l) for t, l, _ in pts.get("sp0.4", [])) | |
| if 500 in sp and len(pts["scratch"]) >= 2: | |
| eff = effective_tokens(pts["scratch"], sp[500]) | |
| out["sp500_effective_scratch_tokens_M"] = eff | |
| if eff: | |
| b.annotate("", xy=(eff, sp[500]), xytext=(500, sp[500]), | |
| arrowprops=dict(arrowstyle="->", color=INK, lw=0.8, shrinkA=2, shrinkB=2)) | |
| b.text(470, sp[500], f"{eff / 500:.2f}x tokens", ha="right", va="center", fontsize=6.5, color=INK) | |
| b.set_xscale("log") | |
| xs = sorted({t for ps in pts.values() for t, _, _ in ps}) | |
| plain_log_ticks(b, xs) | |
| b.set_xlabel("text tokens (millions)") | |
| b.set_ylabel("validation loss") | |
| b.set_title("(b) Token budget", fontsize=7) | |
| out["points"] = pts | |
| handles = [plt.Line2D([], [], color=ARM[k][1], marker=ARM[k][2], ms=3, ls="-") for k in ("scratch", "ivgpt", "sp0.4")] | |
| handles.append(plt.Line2D([], [], color=ARM["spshuf0.4"][1], marker=ARM["spshuf0.4"][2], ms=4, mew=1.2, ls="none")) | |
| fig.legend(handles, ["scratch", "pretrained", "S&P 位=0.4", "shuffled S&P"], loc="upper center", | |
| bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=2) | |
| fig.savefig(OUT / "fig_sp.pdf") | |
| plt.close(fig) | |
| return out | |
| def fig_budget(runs): | |
| """Final validation loss against token budget for both models, at each condition's chosen LR.""" | |
| import act_table | |
| fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.1), gridspec_kw={"wspace": 0.35}) | |
| out = {} | |
| for k in ("scratch", "ivgpt", "sp0.4"): | |
| ps = final_points(runs, k) | |
| lab, col, mk = ARM[k] | |
| bx, by, _ = zip(*ps) | |
| a.plot(bx, by, "-", marker=mk, ms=3, color=col, label=lab) | |
| out[f"medium/{k}"] = ps | |
| root = Path("results/ice") | |
| for (src, init, lr), key in [(("sm", "scratch", 3e-3), "scratch"), (("base", "shuffled", 5e-3), "shuffled"), | |
| (("base", "ivgpt", 8e-3), "ivgpt")]: | |
| ps = [] | |
| for name, t in [("500M", 500), ("1000M", 1000), ("2000M", 2000), ("3000M", 3000)]: | |
| rs = act_table.load(root, name).get((src, init, lr)) | |
| if rs: | |
| ps.append((t, float(np.mean([r["val"][-1] for r in rs])), len(rs))) | |
| lab, col, mk = ARM[key] | |
| bx, by, _ = zip(*ps) | |
| b.plot(bx, by, "-", marker=mk, ms=3, color=col, label=lab) | |
| out[f"small/{key}"] = ps | |
| for ax, title in ((a, "(a) Medium model (400M backbone)"), (b, "(b) Small model (113M backbone)")): | |
| ax.set_xscale("log") | |
| plain_log_ticks(ax, [250, 500, 1000, 2000, 3000] if ax is a else [500, 1000, 2000, 3000]) | |
| ax.set_xlabel("text tokens (millions)") | |
| ax.set_ylabel("validation loss") | |
| ax.set_title(title, fontsize=7) | |
| handles = [plt.Line2D([], [], color=ARM[k][1], marker=ARM[k][2], ms=3, ls="-") for k in ("scratch", "ivgpt", "shuffled", "sp0.4")] | |
| fig.legend(handles, [ARM[k][0] for k in ("scratch", "ivgpt", "shuffled", "sp0.4")], loc="upper center", | |
| bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=2) | |
| fig.savefig(OUT / "fig_budget.pdf") | |
| plt.close(fig) | |
| return out | |
| def main(): | |
| OUT.mkdir(parents=True, exist_ok=True) | |
| runs = load_runs(["results/ice", "results/dm_backup"]) | |
| out = {"sp": fig_sp(runs), "curves500": fig_curves500(runs), "delta2B": fig_lr_and_2b(runs), "budget": fig_budget(runs)} | |
| Path("report/fig_numbers.json").write_text(json.dumps(out, indent=1, default=str)) | |
| print(json.dumps(out, indent=1, default=str)) | |
| if __name__ == "__main__": | |
| main() | |