"""Figures for Part 3. Run after evaluation: ``python -m src.part3.plots``.""" from __future__ import annotations import numpy as np from ..utils import ASSETS, RUNS, load_json, use_plot_style # Fixed identity colours: one per strategy family, assigned in order. FAMILY = {"greedy": "#2a78d6", "top_k": "#eda100", "top_p": "#e87ba4", "beam": "#008300"} def family_of(name): if name.startswith("beam"): return "beam" if name.startswith("top_k"): return "top_k" if name.startswith("top_p"): return "top_p" return "greedy" def fig_metrics(R, plt): names = [r["strategy"] for r in R] x = np.arange(len(names)) colors = [FAMILY[family_of(n)] for n in names] fig, axes = plt.subplots(1, 4, figsize=(14, 3.7)) for ax, key, lab in ((axes[0], "precision", "Unigram precision"), (axes[1], "recall", "Unigram recall"), (axes[2], "f1", "Unigram F1"), (axes[3], "accuracy", "Positional token accuracy")): vals = [r[key] for r in R] ax.bar(x, vals, color=colors, width=0.66) for i, v in enumerate(vals): ax.text(i, v, f"{v:.3f}", ha="center", va="bottom", fontsize=7) ax.set_ylabel(lab, fontsize=8) ax.set_xticks(x) ax.set_xticklabels(names, fontsize=6.5, rotation=35, ha="right") ax.set_ylim(0, max(vals) * 1.2) ax.grid(axis="x", visible=False) handles = [plt.Rectangle((0, 0), 1, 1, color=c) for c in FAMILY.values()] axes[0].legend(handles, FAMILY.keys(), fontsize=6.5, ncol=2) fig.suptitle("Part 3 - overlap with the reference continuation " "(ROCStories, Pythia-160M)", fontsize=10, y=1.04) fig.savefig(ASSETS / "p3_metrics.png", bbox_inches="tight") plt.close(fig) def fig_tradeoff(R, plt): """The axis the overlap scores cannot see: diversity. greedy and beam_1 are the same algorithm, so they land on exactly the same point in both panels; their labels are merged rather than drawn on top of one another, which also makes the equivalence visible in the figure. """ fig, axes = plt.subplots(1, 2, figsize=(9.8, 3.8)) for ax, xk, yk, xlab, ylab in ( (axes[0], "distinct_2", "f1", "distinct-2 (higher = more varied)", "Unigram F1 vs reference"), (axes[1], "repetition_rate", "gen_ppl", "Bigram repetition rate", "Perplexity of the model's own output")): groups = {} for r in R: key = (round(r[xk], 4), round(r[yk], 4)) groups.setdefault(key, []).append(r) for (x, y), rs in groups.items(): c = FAMILY[family_of(rs[0]["strategy"])] ax.scatter(x, y, s=95, color=c, zorder=3, edgecolor="white", linewidth=1.3) ax.annotate(" = ".join(r["strategy"] for r in rs), (x, y), textcoords="offset points", xytext=(0, 9), ha="center", fontsize=6.5, color=c) ax.set_xlabel(xlab) ax.set_ylabel(ylab) axes[1].axhline(R[0]["ref_ppl"], color="#9b9a93", ls="--", lw=1) axes[1].text(axes[1].get_xlim()[0], R[0]["ref_ppl"], f" human reference ppl = {R[0]['ref_ppl']:.1f}", fontsize=6.5, color="#52514e", va="bottom") fig.suptitle("Part 3 - the quality/diversity trade-off", fontsize=10, y=1.03) fig.savefig(ASSETS / "p3_tradeoff.png", bbox_inches="tight") plt.close(fig) def fig_beam_time(R, plt): """Beam-search cost against width, with the linear reference.""" beams = sorted([r for r in R if r["strategy"].startswith("beam")], key=lambda r: int(r["strategy"].split("_")[1])) if len(beams) < 2: return w = np.array([int(b["strategy"].split("_")[1]) for b in beams]) t = np.array([b["seconds"] for b in beams]) greedy = [r for r in R if r["strategy"] == "greedy"] fig, axes = plt.subplots(1, 2, figsize=(9.8, 3.7)) axes[0].plot(w, t, color="#008300", lw=2, marker="o", ms=7, zorder=3, label="measured") axes[0].plot(w, t[0] * w, color="#9b9a93", lw=1.4, ls="--", label=f"exactly linear (t={t[0]:.0f}s x W)") for wi, ti in zip(w, t): axes[0].annotate(f"{ti:.0f}s", (wi, ti), textcoords="offset points", xytext=(0, 9), ha="center", fontsize=7.5) if greedy: axes[0].axhline(greedy[0]["seconds"], color="#2a78d6", lw=1.4, ls=":", label=f"greedy ({greedy[0]['seconds']:.0f}s)") axes[0].set_xlabel("Beam width W") axes[0].set_ylabel("Wall-clock seconds for the full test split") axes[0].set_xticks(w) axes[0].legend(fontsize=7) axes[0].set_title("Cost of beam search", fontsize=9) axes[1].plot(w, t / t[0], color="#008300", lw=2, marker="o", ms=7, zorder=3, label="measured slowdown") axes[1].plot(w, w, color="#9b9a93", lw=1.4, ls="--", label="W (linear)") for wi, ti in zip(w, t / t[0]): axes[1].annotate(f"{ti:.2f}x", (wi, ti), textcoords="offset points", xytext=(0, 9), ha="center", fontsize=7.5) axes[1].set_xlabel("Beam width W") axes[1].set_ylabel("Slowdown relative to W=1") axes[1].set_xticks(w) axes[1].legend(fontsize=7) axes[1].set_title("Measured vs theoretical scaling", fontsize=9) fig.suptitle("Part 3 - beam search time complexity in the beam width", fontsize=10, y=1.03) fig.savefig(ASSETS / "p3_beam_time.png", bbox_inches="tight") plt.close(fig) def write_table(R): hdr = (f"{'strategy':11s} {'P':>6s} {'R':>6s} {'F1':>6s} {'acc':>6s} " f"{'genppl':>8s} {'d1':>6s} {'d2':>6s} {'rep':>6s} {'len':>6s} " f"{'sec':>7s}") lines = [hdr] for r in R: lines.append( f"{r['strategy']:11s} {r['precision']:6.3f} {r['recall']:6.3f} " f"{r['f1']:6.3f} {r['accuracy']:6.3f} {r['gen_ppl']:8.2f} " f"{r['distinct_1']:6.3f} {r['distinct_2']:6.3f} " f"{r['repetition_rate']:6.3f} {r['mean_len']:6.1f} " f"{r['seconds']:7.1f}") txt = "\n".join(lines) (ASSETS / "p3_table.txt").write_text(txt + "\n") print(txt) def main(): plt = use_plot_style() ASSETS.mkdir(parents=True, exist_ok=True) f = RUNS / "part3" / "metrics.json" if not f.exists(): print("no part3 metrics found") return M = load_json(f) R = M["results"] fig_metrics(R, plt) fig_tradeoff(R, plt) fig_beam_time(R, plt) write_table(R) print(f"\nreference perplexity: {M['reference_ppl']:.2f}") print(f"wrote figures to {ASSETS}") if __name__ == "__main__": main()