"""Report figures (report/figs/fig_*.pdf) from pulled metrics. Every figure is drawn at its exact printed size (NeurIPS text width 5.5in) so fonts print at 7pt, in Arial to match the TikZ diagrams. Colors follow the validated categorical palette (reference order, adjacent pairs); all lines are solid, and markers separate conditions where points matter. """ import json import sys from collections import defaultdict from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy as np sys.path.insert(0, str(Path(__file__).parent)) from analyze import load_runs # noqa: E402 TEXTW = 5.5 INK, INK2, GRID = "#2b2a27", "#73716b", "#e6e3dc" plt.rcParams.update({ "font.family": "sans-serif", "font.sans-serif": ["Arial"], "font.size": 7, "axes.labelsize": 7, "xtick.labelsize": 6.5, "ytick.labelsize": 6.5, "legend.fontsize": 6.5, "axes.linewidth": 0.6, "axes.edgecolor": INK2, "axes.labelcolor": INK, "xtick.color": INK2, "ytick.color": INK2, "xtick.major.width": 0.6, "ytick.major.width": 0.6, "xtick.major.size": 2.5, "ytick.major.size": 2.5, "axes.spines.top": False, "axes.spines.right": False, "axes.grid": True, "axes.axisbelow": True, "grid.color": GRID, "grid.linewidth": 0.5, "lines.linewidth": 1.3, "savefig.bbox": "tight", "savefig.pad_inches": 0.02, "pdf.fonttype": 42, "figure.dpi": 200, }) ARM = { # key: (label, color, marker); all lines are solid "scratch": ("scratch", "#2a78d6", "o"), "ivgpt": ("pretrained", "#eb6834", "s"), "shuffled": ("shuffled", "#1baf7a", "^"), "rescaled": ("rescaled", "#eda100", "D"), "sp0.4": ("S&P λ=0.4", "#e87ba4", "v"), "spshuf0.4": ("shuffled S&P", "#008300", "x"), } OUT = Path("report/figs") def plain_log_ticks(ax, ticks): """Log axis with plain tick labels (e.g. 50, 100, 200) instead of powers of ten.""" ax.set_xticks(ticks, [f"{t:g}" for t in ticks]) ax.minorticks_off() def lr_str(lr): return f"{lr:.0e}".replace("e-0", "e-") def runs_for(runs, key, budget, freeze, lr=None): return [r for r in runs if r["init"] == key and r["budget"] == budget and r["freeze"] == freeze and r["done"] and (lr is None or r["lr"] == lr)] def best_lr(runs, key, budget, freeze): """The LR with the most seeds, then the lowest mean final loss (same rule as the tables).""" groups = defaultdict(list) for r in runs_for(runs, key, budget, freeze): groups[r["lr"]].append(r["val"][-1]) return min(groups, key=lambda l: (-len(groups[l]), np.mean(groups[l]))) if groups else None def seed_curves(runs, k, budget): """Mean and range over seeds at the condition's best LR, on the eval steps all seeds share.""" reps = runs_for(runs, k, budget, "none", best_lr(runs, k, budget, "none")) if not reps: return None toks = sorted(set.intersection(*[set(r["tokens"]) for r in reps]) - {0}) v = np.array([[dict(zip(r["tokens"], r["val"]))[t] for t in toks] for r in reps]) return np.array(toks) / 1e6, v.mean(0), v.min(0), v.max(0), len(reps) def loss_panel(ax, runs, budget, keys, xlim=None, ylim=None, label=False): ends = [] for k in keys: c = seed_curves(runs, k, budget) if c is None: continue x, m, lo, hi, n = c lab, col, _ = ARM[k] sel = np.ones_like(x, bool) if xlim is None else (x >= xlim[0] * 0.98) ax.plot(x[sel], m[sel], "-", color=col, lw=1.4, label=lab) if n > 1: ax.fill_between(x[sel], lo[sel], hi[sel], color=col, alpha=0.16, lw=0) ends.append((m[-1], f"{lab} {m[-1]:.3f}", col)) ax.set_xscale("log") if xlim: ax.set_xlim(*xlim) if ylim: ax.set_ylim(*ylim) ax.set_xlabel("text tokens (millions)") ax.set_ylabel("validation loss") def fig_curves500(runs): """(a) Validation loss over 500M tokens. (b) Zoom on the second half of training.""" keys = ["scratch", "ivgpt", "sp0.4", "spshuf0.4"] fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.8, 2.2), gridspec_kw={"wspace": 0.35, "width_ratios": [1, 1]}) loss_panel(a, runs, "500M", keys, ylim=(3.3, 6.6)) plain_log_ticks(a, [50, 100, 200, 500]) a.set_title("(a) Full run", fontsize=7) loss_panel(b, runs, "500M", keys, xlim=(250, 640), ylim=(3.3, 3.95)) plain_log_ticks(b, [300, 400, 500]) b.set_title("(b) Second half", fontsize=7) # arrow marking the final gap between scratch and S&P ys, yp = seed_curves(runs, "scratch", "500M")[1][-1], seed_curves(runs, "sp0.4", "500M")[1][-1] xa = 512 # just past the last point, where no line runs b.annotate("", xy=(xa, yp), xytext=(xa, ys), arrowprops=dict(arrowstyle="<->", color=INK, lw=0.8, shrinkA=0, shrinkB=0)) b.text(xa * 1.02, (ys + yp) / 2, f"{yp - ys:+.3f}\n({100 * (yp - ys) / ys:+.1f}%)", ha="left", va="center", fontsize=6.5, color=INK) fig.legend(*a.get_legend_handles_labels(), loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=1.8, columnspacing=1.4) fig.savefig(OUT / "fig_curves500.pdf") plt.close(fig) return {k: float(seed_curves(runs, k, "500M")[1][-1]) for k in keys if seed_curves(runs, k, "500M")} def fig_lr_and_2b(runs): """(a) LR sweep at 500M tokens. (b) Phase 2 curves relative to scratch over 2B tokens.""" fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.0), gridspec_kw={"wspace": 0.4}) for k in ("scratch", "ivgpt", "shuffled", "rescaled", "sp0.4", "spshuf0.4"): pts = sorted((r["lr"], r["val"][-1]) for r in runs_for(runs, k, "500M", "none") if r["seed"] == 0) if pts: x, y = zip(*pts) lab, col, mk = ARM[k] a.plot(x, y, "-", marker=mk, ms=3, color=col, label=lab) a.set_xscale("log") a.set_xticks([3e-4, 1e-3, 3e-3, 1e-2], [lr_str(t) for t in (3e-4, 1e-3, 3e-3, 1e-2)]) a.minorticks_off() a.set_ylim(3.3, 4.4) a.set_xlabel("peak learning rate") a.set_ylabel("validation loss") a.set_title("(a) Learning-rate sweep, 500M tokens", fontsize=7) fig.legend(*a.get_legend_handles_labels(), loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=6, frameon=False, handlelength=1.8, columnspacing=1.1) loss_panel(b, runs, "2000M", ["scratch", "ivgpt", "sp0.4"], xlim=(300, 2100), ylim=(2.9, 3.85)) plain_log_ticks(b, [300, 500, 1000, 2000]) b.set_title("(b) Training to 2B tokens", fontsize=7) rows = {k: float(seed_curves(runs, k, "2000M")[1][-1]) for k in ("scratch", "ivgpt", "sp0.4") if seed_curves(runs, k, "2000M")} fig.savefig(OUT / "fig_lr_2b.pdf") plt.close(fig) return rows def final_points(runs, k): """(tokens, mean final loss, n) per budget at the condition's best 500M learning rate.""" lr = best_lr(runs, k, "500M", "none") pts = defaultdict(list) for r in runs: if r["init"] == k and r["freeze"] == "none" and r["done"] and r["lr"] == lr and r["budget"] != "100M": pts[int(r["budget"][:-1])].append(r["val"][-1]) return sorted((b, float(np.mean(v)), len(v)) for b, v in pts.items()) def effective_tokens(scratch_pts, loss): """Tokens scratch needs to reach `loss`, interpolating log(loss) linearly in log(tokens).""" for (b0, l0, _), (b1, l1, _) in zip(scratch_pts, scratch_pts[1:]): if l1 <= loss <= l0: t = (np.log(l0) - np.log(loss)) / (np.log(l0) - np.log(l1)) return float(np.exp(np.log(b0) + t * (np.log(b1) - np.log(b0)))) return None def fig_sp(runs): """(a) Final loss against the S&P mixing weight at LR 2e-3. (b) Final loss against token budget.""" fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.1), gridspec_kw={"wspace": 0.4}) lr = 2e-3 def final(init, budget="500M"): v = [r["val"][-1] for r in runs if r["init"] == init and r["freeze"] == "none" and r["budget"] == budget and r["done"] and r["lr"] == lr] return float(np.mean(v)) if v else None lam = [(0.0, final("scratch"))] + [(l, final(f"sp{l}")) for l in (0.1, 0.2, 0.4, 0.7)] + [(1.0, final("ivgpt"))] lam = [(l, v) for l, v in lam if v is not None] x, y = zip(*lam) a.plot(x, y, "-", marker=ARM["sp0.4"][2], ms=3, color=ARM["sp0.4"][1], label="S&P") a.plot([0], [lam[0][1]], ARM["scratch"][2], ms=4, color=ARM["scratch"][1], label="scratch (λ=0)", zorder=3) a.plot([1], [lam[-1][1]], ARM["ivgpt"][2], ms=4, color=ARM["ivgpt"][1], label="pretrained (λ=1)", zorder=3) shuf = final("spshuf0.4") if shuf: a.plot([0.4], [shuf], ARM["spshuf0.4"][2], ms=4, mew=1.2, color=ARM["spshuf0.4"][1], label="shuffled S&P") a.set_xlabel("fraction of video weights kept (λ)") a.set_ylabel("validation loss") a.set_title("(a) Mixing weight", fontsize=7) out = {"lambda": lam, "shuffled": shuf} pts = {k: final_points(runs, k) for k in ("scratch", "ivgpt", "sp0.4")} for k, ps in pts.items(): if ps: lab, col, mk = ARM[k] bx, by, _ = zip(*ps) b.plot(bx, by, "-", marker=mk, ms=3, color=col) sp = dict((t, l) for t, l, _ in pts.get("sp0.4", [])) if 500 in sp and len(pts["scratch"]) >= 2: eff = effective_tokens(pts["scratch"], sp[500]) out["sp500_effective_scratch_tokens_M"] = eff if eff: b.annotate("", xy=(eff, sp[500]), xytext=(500, sp[500]), arrowprops=dict(arrowstyle="->", color=INK, lw=0.8, shrinkA=2, shrinkB=2)) b.text(470, sp[500], f"{eff / 500:.2f}x tokens", ha="right", va="center", fontsize=6.5, color=INK) b.set_xscale("log") xs = sorted({t for ps in pts.values() for t, _, _ in ps}) plain_log_ticks(b, xs) b.set_xlabel("text tokens (millions)") b.set_ylabel("validation loss") b.set_title("(b) Token budget", fontsize=7) out["points"] = pts handles = [plt.Line2D([], [], color=ARM[k][1], marker=ARM[k][2], ms=3, ls="-") for k in ("scratch", "ivgpt", "sp0.4")] handles.append(plt.Line2D([], [], color=ARM["spshuf0.4"][1], marker=ARM["spshuf0.4"][2], ms=4, mew=1.2, ls="none")) fig.legend(handles, ["scratch", "pretrained", "S&P λ=0.4", "shuffled S&P"], loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=2) fig.savefig(OUT / "fig_sp.pdf") plt.close(fig) return out def fig_budget(runs): """Final validation loss against token budget for both models, at each condition's chosen LR.""" import act_table fig, (a, b) = plt.subplots(1, 2, figsize=(TEXTW * 0.86, 2.1), gridspec_kw={"wspace": 0.35}) out = {} for k in ("scratch", "ivgpt", "sp0.4"): ps = final_points(runs, k) lab, col, mk = ARM[k] bx, by, _ = zip(*ps) a.plot(bx, by, "-", marker=mk, ms=3, color=col, label=lab) out[f"medium/{k}"] = ps root = Path("results/ice") for (src, init, lr), key in [(("sm", "scratch", 3e-3), "scratch"), (("base", "shuffled", 5e-3), "shuffled"), (("base", "ivgpt", 8e-3), "ivgpt")]: ps = [] for name, t in [("500M", 500), ("1000M", 1000), ("2000M", 2000), ("3000M", 3000)]: rs = act_table.load(root, name).get((src, init, lr)) if rs: ps.append((t, float(np.mean([r["val"][-1] for r in rs])), len(rs))) lab, col, mk = ARM[key] bx, by, _ = zip(*ps) b.plot(bx, by, "-", marker=mk, ms=3, color=col, label=lab) out[f"small/{key}"] = ps for ax, title in ((a, "(a) Medium model (400M backbone)"), (b, "(b) Small model (113M backbone)")): ax.set_xscale("log") plain_log_ticks(ax, [250, 500, 1000, 2000, 3000] if ax is a else [500, 1000, 2000, 3000]) ax.set_xlabel("text tokens (millions)") ax.set_ylabel("validation loss") ax.set_title(title, fontsize=7) handles = [plt.Line2D([], [], color=ARM[k][1], marker=ARM[k][2], ms=3, ls="-") for k in ("scratch", "ivgpt", "shuffled", "sp0.4")] fig.legend(handles, [ARM[k][0] for k in ("scratch", "ivgpt", "shuffled", "sp0.4")], loc="upper center", bbox_to_anchor=(0.5, -0.07), ncol=4, frameon=False, handlelength=2) fig.savefig(OUT / "fig_budget.pdf") plt.close(fig) return out def main(): OUT.mkdir(parents=True, exist_ok=True) runs = load_runs(["results/ice", "results/dm_backup"]) out = {"sp": fig_sp(runs), "curves500": fig_curves500(runs), "delta2B": fig_lr_and_2b(runs), "budget": fig_budget(runs)} Path("report/fig_numbers.json").write_text(json.dumps(out, indent=1, default=str)) print(json.dumps(out, indent=1, default=str)) if __name__ == "__main__": main()