Download src/part3/plots.py from sahoosk/anlp-assignment2: direct link, hf CLI and curl.
- Browser
- Download file 6.76 kB
-
https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part3/plots.py
- Command line
-
hf download hf://sahoosk/anlp-assignment2/src/part3/plots.py
-
curl -L -o plots.py https://huggingface.co/sahoosk/anlp-assignment2/resolve/main/src/part3/plots.py
6.76 kB
| """Figures for Part 3. Run after evaluation: ``python -m src.part3.plots``.""" | |
| from __future__ import annotations | |
| import numpy as np | |
| from ..utils import ASSETS, RUNS, load_json, use_plot_style | |
| # Fixed identity colours: one per strategy family, assigned in order. | |
| FAMILY = {"greedy": "#2a78d6", "top_k": "#eda100", "top_p": "#e87ba4", | |
| "beam": "#008300"} | |
| def family_of(name): | |
| if name.startswith("beam"): | |
| return "beam" | |
| if name.startswith("top_k"): | |
| return "top_k" | |
| if name.startswith("top_p"): | |
| return "top_p" | |
| return "greedy" | |
| def fig_metrics(R, plt): | |
| names = [r["strategy"] for r in R] | |
| x = np.arange(len(names)) | |
| colors = [FAMILY[family_of(n)] for n in names] | |
| fig, axes = plt.subplots(1, 4, figsize=(14, 3.7)) | |
| for ax, key, lab in ((axes[0], "precision", "Unigram precision"), | |
| (axes[1], "recall", "Unigram recall"), | |
| (axes[2], "f1", "Unigram F1"), | |
| (axes[3], "accuracy", "Positional token accuracy")): | |
| vals = [r[key] for r in R] | |
| ax.bar(x, vals, color=colors, width=0.66) | |
| for i, v in enumerate(vals): | |
| ax.text(i, v, f"{v:.3f}", ha="center", va="bottom", fontsize=7) | |
| ax.set_ylabel(lab, fontsize=8) | |
| ax.set_xticks(x) | |
| ax.set_xticklabels(names, fontsize=6.5, rotation=35, ha="right") | |
| ax.set_ylim(0, max(vals) * 1.2) | |
| ax.grid(axis="x", visible=False) | |
| handles = [plt.Rectangle((0, 0), 1, 1, color=c) for c in FAMILY.values()] | |
| axes[0].legend(handles, FAMILY.keys(), fontsize=6.5, ncol=2) | |
| fig.suptitle("Part 3 - overlap with the reference continuation " | |
| "(ROCStories, Pythia-160M)", fontsize=10, y=1.04) | |
| fig.savefig(ASSETS / "p3_metrics.png", bbox_inches="tight") | |
| plt.close(fig) | |
| def fig_tradeoff(R, plt): | |
| """The axis the overlap scores cannot see: diversity. | |
| greedy and beam_1 are the same algorithm, so they land on exactly the same | |
| point in both panels; their labels are merged rather than drawn on top of | |
| one another, which also makes the equivalence visible in the figure. | |
| """ | |
| fig, axes = plt.subplots(1, 2, figsize=(9.8, 3.8)) | |
| for ax, xk, yk, xlab, ylab in ( | |
| (axes[0], "distinct_2", "f1", | |
| "distinct-2 (higher = more varied)", "Unigram F1 vs reference"), | |
| (axes[1], "repetition_rate", "gen_ppl", | |
| "Bigram repetition rate", "Perplexity of the model's own output")): | |
| groups = {} | |
| for r in R: | |
| key = (round(r[xk], 4), round(r[yk], 4)) | |
| groups.setdefault(key, []).append(r) | |
| for (x, y), rs in groups.items(): | |
| c = FAMILY[family_of(rs[0]["strategy"])] | |
| ax.scatter(x, y, s=95, color=c, zorder=3, edgecolor="white", | |
| linewidth=1.3) | |
| ax.annotate(" = ".join(r["strategy"] for r in rs), (x, y), | |
| textcoords="offset points", xytext=(0, 9), | |
| ha="center", fontsize=6.5, color=c) | |
| ax.set_xlabel(xlab) | |
| ax.set_ylabel(ylab) | |
| axes[1].axhline(R[0]["ref_ppl"], color="#9b9a93", ls="--", lw=1) | |
| axes[1].text(axes[1].get_xlim()[0], R[0]["ref_ppl"], | |
| f" human reference ppl = {R[0]['ref_ppl']:.1f}", | |
| fontsize=6.5, color="#52514e", va="bottom") | |
| fig.suptitle("Part 3 - the quality/diversity trade-off", fontsize=10, y=1.03) | |
| fig.savefig(ASSETS / "p3_tradeoff.png", bbox_inches="tight") | |
| plt.close(fig) | |
| def fig_beam_time(R, plt): | |
| """Beam-search cost against width, with the linear reference.""" | |
| beams = sorted([r for r in R if r["strategy"].startswith("beam")], | |
| key=lambda r: int(r["strategy"].split("_")[1])) | |
| if len(beams) < 2: | |
| return | |
| w = np.array([int(b["strategy"].split("_")[1]) for b in beams]) | |
| t = np.array([b["seconds"] for b in beams]) | |
| greedy = [r for r in R if r["strategy"] == "greedy"] | |
| fig, axes = plt.subplots(1, 2, figsize=(9.8, 3.7)) | |
| axes[0].plot(w, t, color="#008300", lw=2, marker="o", ms=7, zorder=3, | |
| label="measured") | |
| axes[0].plot(w, t[0] * w, color="#9b9a93", lw=1.4, ls="--", | |
| label=f"exactly linear (t={t[0]:.0f}s x W)") | |
| for wi, ti in zip(w, t): | |
| axes[0].annotate(f"{ti:.0f}s", (wi, ti), textcoords="offset points", | |
| xytext=(0, 9), ha="center", fontsize=7.5) | |
| if greedy: | |
| axes[0].axhline(greedy[0]["seconds"], color="#2a78d6", lw=1.4, ls=":", | |
| label=f"greedy ({greedy[0]['seconds']:.0f}s)") | |
| axes[0].set_xlabel("Beam width W") | |
| axes[0].set_ylabel("Wall-clock seconds for the full test split") | |
| axes[0].set_xticks(w) | |
| axes[0].legend(fontsize=7) | |
| axes[0].set_title("Cost of beam search", fontsize=9) | |
| axes[1].plot(w, t / t[0], color="#008300", lw=2, marker="o", ms=7, | |
| zorder=3, label="measured slowdown") | |
| axes[1].plot(w, w, color="#9b9a93", lw=1.4, ls="--", label="W (linear)") | |
| for wi, ti in zip(w, t / t[0]): | |
| axes[1].annotate(f"{ti:.2f}x", (wi, ti), textcoords="offset points", | |
| xytext=(0, 9), ha="center", fontsize=7.5) | |
| axes[1].set_xlabel("Beam width W") | |
| axes[1].set_ylabel("Slowdown relative to W=1") | |
| axes[1].set_xticks(w) | |
| axes[1].legend(fontsize=7) | |
| axes[1].set_title("Measured vs theoretical scaling", fontsize=9) | |
| fig.suptitle("Part 3 - beam search time complexity in the beam width", | |
| fontsize=10, y=1.03) | |
| fig.savefig(ASSETS / "p3_beam_time.png", bbox_inches="tight") | |
| plt.close(fig) | |
| def write_table(R): | |
| hdr = (f"{'strategy':11s} {'P':>6s} {'R':>6s} {'F1':>6s} {'acc':>6s} " | |
| f"{'genppl':>8s} {'d1':>6s} {'d2':>6s} {'rep':>6s} {'len':>6s} " | |
| f"{'sec':>7s}") | |
| lines = [hdr] | |
| for r in R: | |
| lines.append( | |
| f"{r['strategy']:11s} {r['precision']:6.3f} {r['recall']:6.3f} " | |
| f"{r['f1']:6.3f} {r['accuracy']:6.3f} {r['gen_ppl']:8.2f} " | |
| f"{r['distinct_1']:6.3f} {r['distinct_2']:6.3f} " | |
| f"{r['repetition_rate']:6.3f} {r['mean_len']:6.1f} " | |
| f"{r['seconds']:7.1f}") | |
| txt = "\n".join(lines) | |
| (ASSETS / "p3_table.txt").write_text(txt + "\n") | |
| print(txt) | |
| def main(): | |
| plt = use_plot_style() | |
| ASSETS.mkdir(parents=True, exist_ok=True) | |
| f = RUNS / "part3" / "metrics.json" | |
| if not f.exists(): | |
| print("no part3 metrics found") | |
| return | |
| M = load_json(f) | |
| R = M["results"] | |
| fig_metrics(R, plt) | |
| fig_tradeoff(R, plt) | |
| fig_beam_time(R, plt) | |
| write_table(R) | |
| print(f"\nreference perplexity: {M['reference_ppl']:.2f}") | |
| print(f"wrote figures to {ASSETS}") | |
| if __name__ == "__main__": | |
| main() | |