anlp-assignment2 / src /part3 /plots.py
sahoosk's picture
Upload folder using huggingface_hub
5407bdc verified
Raw History Blame Contribute Delete
6.76 kB
"""Figures for Part 3. Run after evaluation: ``python -m src.part3.plots``."""
from __future__ import annotations
import numpy as np
from ..utils import ASSETS, RUNS, load_json, use_plot_style
# Fixed identity colours: one per strategy family, assigned in order.
FAMILY = {"greedy": "#2a78d6", "top_k": "#eda100", "top_p": "#e87ba4",
"beam": "#008300"}
def family_of(name):
if name.startswith("beam"):
return "beam"
if name.startswith("top_k"):
return "top_k"
if name.startswith("top_p"):
return "top_p"
return "greedy"
def fig_metrics(R, plt):
names = [r["strategy"] for r in R]
x = np.arange(len(names))
colors = [FAMILY[family_of(n)] for n in names]
fig, axes = plt.subplots(1, 4, figsize=(14, 3.7))
for ax, key, lab in ((axes[0], "precision", "Unigram precision"),
(axes[1], "recall", "Unigram recall"),
(axes[2], "f1", "Unigram F1"),
(axes[3], "accuracy", "Positional token accuracy")):
vals = [r[key] for r in R]
ax.bar(x, vals, color=colors, width=0.66)
for i, v in enumerate(vals):
ax.text(i, v, f"{v:.3f}", ha="center", va="bottom", fontsize=7)
ax.set_ylabel(lab, fontsize=8)
ax.set_xticks(x)
ax.set_xticklabels(names, fontsize=6.5, rotation=35, ha="right")
ax.set_ylim(0, max(vals) * 1.2)
ax.grid(axis="x", visible=False)
handles = [plt.Rectangle((0, 0), 1, 1, color=c) for c in FAMILY.values()]
axes[0].legend(handles, FAMILY.keys(), fontsize=6.5, ncol=2)
fig.suptitle("Part 3 - overlap with the reference continuation "
"(ROCStories, Pythia-160M)", fontsize=10, y=1.04)
fig.savefig(ASSETS / "p3_metrics.png", bbox_inches="tight")
plt.close(fig)
def fig_tradeoff(R, plt):
"""The axis the overlap scores cannot see: diversity.
greedy and beam_1 are the same algorithm, so they land on exactly the same
point in both panels; their labels are merged rather than drawn on top of
one another, which also makes the equivalence visible in the figure.
"""
fig, axes = plt.subplots(1, 2, figsize=(9.8, 3.8))
for ax, xk, yk, xlab, ylab in (
(axes[0], "distinct_2", "f1",
"distinct-2 (higher = more varied)", "Unigram F1 vs reference"),
(axes[1], "repetition_rate", "gen_ppl",
"Bigram repetition rate", "Perplexity of the model's own output")):
groups = {}
for r in R:
key = (round(r[xk], 4), round(r[yk], 4))
groups.setdefault(key, []).append(r)
for (x, y), rs in groups.items():
c = FAMILY[family_of(rs[0]["strategy"])]
ax.scatter(x, y, s=95, color=c, zorder=3, edgecolor="white",
linewidth=1.3)
ax.annotate(" = ".join(r["strategy"] for r in rs), (x, y),
textcoords="offset points", xytext=(0, 9),
ha="center", fontsize=6.5, color=c)
ax.set_xlabel(xlab)
ax.set_ylabel(ylab)
axes[1].axhline(R[0]["ref_ppl"], color="#9b9a93", ls="--", lw=1)
axes[1].text(axes[1].get_xlim()[0], R[0]["ref_ppl"],
f" human reference ppl = {R[0]['ref_ppl']:.1f}",
fontsize=6.5, color="#52514e", va="bottom")
fig.suptitle("Part 3 - the quality/diversity trade-off", fontsize=10, y=1.03)
fig.savefig(ASSETS / "p3_tradeoff.png", bbox_inches="tight")
plt.close(fig)
def fig_beam_time(R, plt):
"""Beam-search cost against width, with the linear reference."""
beams = sorted([r for r in R if r["strategy"].startswith("beam")],
key=lambda r: int(r["strategy"].split("_")[1]))
if len(beams) < 2:
return
w = np.array([int(b["strategy"].split("_")[1]) for b in beams])
t = np.array([b["seconds"] for b in beams])
greedy = [r for r in R if r["strategy"] == "greedy"]
fig, axes = plt.subplots(1, 2, figsize=(9.8, 3.7))
axes[0].plot(w, t, color="#008300", lw=2, marker="o", ms=7, zorder=3,
label="measured")
axes[0].plot(w, t[0] * w, color="#9b9a93", lw=1.4, ls="--",
label=f"exactly linear (t={t[0]:.0f}s x W)")
for wi, ti in zip(w, t):
axes[0].annotate(f"{ti:.0f}s", (wi, ti), textcoords="offset points",
xytext=(0, 9), ha="center", fontsize=7.5)
if greedy:
axes[0].axhline(greedy[0]["seconds"], color="#2a78d6", lw=1.4, ls=":",
label=f"greedy ({greedy[0]['seconds']:.0f}s)")
axes[0].set_xlabel("Beam width W")
axes[0].set_ylabel("Wall-clock seconds for the full test split")
axes[0].set_xticks(w)
axes[0].legend(fontsize=7)
axes[0].set_title("Cost of beam search", fontsize=9)
axes[1].plot(w, t / t[0], color="#008300", lw=2, marker="o", ms=7,
zorder=3, label="measured slowdown")
axes[1].plot(w, w, color="#9b9a93", lw=1.4, ls="--", label="W (linear)")
for wi, ti in zip(w, t / t[0]):
axes[1].annotate(f"{ti:.2f}x", (wi, ti), textcoords="offset points",
xytext=(0, 9), ha="center", fontsize=7.5)
axes[1].set_xlabel("Beam width W")
axes[1].set_ylabel("Slowdown relative to W=1")
axes[1].set_xticks(w)
axes[1].legend(fontsize=7)
axes[1].set_title("Measured vs theoretical scaling", fontsize=9)
fig.suptitle("Part 3 - beam search time complexity in the beam width",
fontsize=10, y=1.03)
fig.savefig(ASSETS / "p3_beam_time.png", bbox_inches="tight")
plt.close(fig)
def write_table(R):
hdr = (f"{'strategy':11s} {'P':>6s} {'R':>6s} {'F1':>6s} {'acc':>6s} "
f"{'genppl':>8s} {'d1':>6s} {'d2':>6s} {'rep':>6s} {'len':>6s} "
f"{'sec':>7s}")
lines = [hdr]
for r in R:
lines.append(
f"{r['strategy']:11s} {r['precision']:6.3f} {r['recall']:6.3f} "
f"{r['f1']:6.3f} {r['accuracy']:6.3f} {r['gen_ppl']:8.2f} "
f"{r['distinct_1']:6.3f} {r['distinct_2']:6.3f} "
f"{r['repetition_rate']:6.3f} {r['mean_len']:6.1f} "
f"{r['seconds']:7.1f}")
txt = "\n".join(lines)
(ASSETS / "p3_table.txt").write_text(txt + "\n")
print(txt)
def main():
plt = use_plot_style()
ASSETS.mkdir(parents=True, exist_ok=True)
f = RUNS / "part3" / "metrics.json"
if not f.exists():
print("no part3 metrics found")
return
M = load_json(f)
R = M["results"]
fig_metrics(R, plt)
fig_tradeoff(R, plt)
fig_beam_time(R, plt)
write_table(R)
print(f"\nreference perplexity: {M['reference_ppl']:.2f}")
print(f"wrote figures to {ASSETS}")
if __name__ == "__main__":
main()