| """F1 against labelling budget for the overnight picks (reports/nnlab/confirmation.csv, view = all 12 draws).""" |
| from pathlib import Path |
| import numpy as np, pandas as pd |
| import matplotlib |
| matplotlib.use("Agg") |
| import matplotlib.pyplot as plt |
|
|
| R = Path(__file__).resolve().parent.parent.parent / "reports" / "nnlab" |
| df = pd.read_csv(R / "confirmation.csv") |
| N = [5, 25, 50, 100, 200] |
| INK, INK2, MUTED, GRID, SURF = "#0b0b0b", "#52514e", "#898781", "#e1e0d9", "#fcfcfb" |
| def row(m, k, view="all"): |
| r = df[(df.method == m) & (df.decision == k) & (df["view"] == view)].iloc[0] |
| return np.array([r[f"n{n}"] for n in N]), np.array([r[f"sd{n}"] for n in N]) |
| series = [ |
| ("ens_e12", "rule_smooth", "Zero-shot + LightGBM with domain copies", "#2a78d6", 6), |
| ("ens_e4", "rule_smooth", "Five-model ensemble", "#eb6834", -8), |
| ("lgbm_upw", "rule_smooth", "Retrained LightGBM", "#1baf7a", -3), |
| ] |
| fig, ax = plt.subplots(figsize=(9, 5.2), facecolor=SURF) |
| ax.set_facecolor(SURF) |
| x = np.log2(N) |
| for m, k, name, col, dy in series: |
| mean, sd = row(m, k) |
| ax.fill_between(x, mean - sd, mean + sd, color=col, alpha=0.12, lw=0) |
| ax.plot(x, mean, color=col, lw=2, marker="o", ms=8, mec=SURF, mew=2, label=name) |
| ax.annotate(name, (x[-1], mean[-1]), xytext=(10, dy), textcoords="offset points", va="center", color=INK2, fontsize=9) |
| for m, k, name, ls, dy in [("lgbm_upw", "rule", "Retrained LightGBM, no smoothing (previous best)", "--", -14), |
| ("zs_xgb_dann", "rule_smooth", "Teammate zero-shot (no labels)", ":", 0)]: |
| mean, _ = row(m, k) |
| ax.plot(x, mean, color=MUTED, lw=1.5, ls=ls, label=name) |
| ax.annotate(name, (x[-1], mean[-1]), xytext=(10, dy), textcoords="offset points", va="center", color=MUTED, fontsize=9) |
| ax.set_xticks(x, [str(n) for n in N]) |
| ax.set_xlabel("Labelled Amsterdam pixels per class (log scale)", color=INK2) |
| ax.set_ylabel("Macro F1, all of Amsterdam (12 draws per n)", color=INK2) |
| ax.text(0, 1.02, "All solid lines and the dotted line use neighbour smoothing; bands are one standard deviation over draws", transform=ax.transAxes, color=MUTED, fontsize=8) |
| ax.set_title("Macro F1 against labelling budget", loc="left", color=INK, fontsize=12, pad=18) |
| ax.grid(axis="y", color=GRID, lw=0.8); ax.set_axisbelow(True) |
| for sp in ("top", "right"): ax.spines[sp].set_visible(False) |
| for sp in ("left", "bottom"): ax.spines[sp].set_color(GRID) |
| ax.tick_params(colors=MUTED) |
| ax.set_xlim(x[0] - 0.3, x[-1] + 3.9) |
| ax.legend(loc="lower right", frameon=False, fontsize=8, labelcolor=INK2) |
| plt.tight_layout() |
| fig.savefig(R / "f1_vs_n_overnight.png", dpi=160, facecolor=SURF) |
| print("saved") |
|
|
