pakflood-lora / code /plot_results.py
msherry's picture
project code, results and figures
17285c6 verified
Raw History Blame Contribute Delete
10.2 kB
"""Local result figures from results/results.json (no server needed).
python3 pakflood/plot_results.py -> writes pakflood/figs/fig_*.png
Colors: validated categorical palette, one fixed hue per model (never cycled).
"""
import json
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
HERE = Path(__file__).resolve().parent
R = json.loads((HERE / "results" / "results.json").read_text())
OUT = HERE / "figs"
OUT.mkdir(exist_ok=True)
COL = {"baseline_frozen": "#2a78d6", "lora_r16": "#eb6834", "lora_r32_all": "#6250d6", "full_finetune": "#1baf7a",
"lora_r16_cedice": "#e87ba4", "sa_holdout": "#008300", "sa_fewshot": "#e34948", "official": "#52514e"}
LAB = {k: v["label"] for k, v in R["models"].items()}
LAB.update({"sa_holdout": "LoRA r=16 hold-out\n(never saw South Asia)", "sa_fewshot": "LoRA r=16 few-shot\n(+26 SA tiles)", "official": "CEMS official"})
SHORT = {"baseline_frozen": "Frozen", "lora_r16": "LoRA r16", "lora_r32_all": "LoRA r32\n(all linear)", "full_finetune": "Full FT",
"lora_r16_cedice": "LoRA r16\nCE+Dice", "sa_holdout": "Hold-out\n(no S. Asia)", "sa_fewshot": "Few-shot\n(+26 tiles)", "official": "CEMS official"}
INK, INK2, GRID = "#0b0b0b", "#52514e", "#e6e5e1"
plt.rcParams.update({"font.size": 9, "axes.edgecolor": INK2, "axes.labelcolor": INK, "xtick.color": INK2, "ytick.color": INK2,
"axes.spines.top": False, "axes.spines.right": False, "axes.titlesize": 10, "axes.titleweight": "bold",
"figure.facecolor": "#fcfcfb", "axes.facecolor": "#fcfcfb", "savefig.facecolor": "#fcfcfb"})
def tidy(ax, ylabel=None):
ax.grid(axis="y", color=GRID, linewidth=0.8)
ax.set_axisbelow(True)
if ylabel:
ax.set_ylabel(ylabel)
def bars(ax, keys, vals, los=None, his=None, fmt="{:.3f}", width=0.62):
x = np.arange(len(keys))
for i, k in enumerate(keys):
ax.bar(x[i], vals[i], width, color=COL[k], linewidth=0)
if los is not None and los[i] is not None:
ax.errorbar(x[i], vals[i], yerr=[[vals[i] - los[i]], [his[i] - vals[i]]], fmt="none", ecolor=INK, elinewidth=1.2, capsize=3)
ax.text(x[i], (his[i] if his is not None and his[i] is not None else vals[i]) + 0.012, fmt.format(vals[i]), ha="center", va="bottom", fontsize=8, color=INK)
ax.set_xticks(x)
ax.set_xticklabels([SHORT[k] for k in keys], fontsize=8)
return x
# ---------- Fig 1: validation curves ----------
def fig_curves():
fig, ax = plt.subplots(figsize=(9.5, 3.8))
order = ["baseline_frozen", "lora_r16", "lora_r32_all", "full_finetune", "lora_r16_cedice"]
for k in order:
y = R["val_curves"][k]
x = np.arange(1, len(y) + 1)
ax.plot(x, y, color=COL[k], linewidth=2, marker="o", markersize=5, markerfacecolor="#fcfcfb", markeredgewidth=1.6, label=f"{LAB[k]} (best {max(y):.3f})")
b = int(np.argmax(y))
ax.plot(x[b], y[b], marker="o", markersize=7, color=COL[k])
ax.set_xlim(0.7, 10.3)
ax.set_xticks(range(1, 11))
ax.set_xlabel("epoch")
tidy(ax, "validation flood IoU")
ax.set_title("Validation flood IoU per epoch (filled marker = best epoch, used for testing)")
ax.legend(loc="center left", bbox_to_anchor=(1.01, 0.5), frameon=False, fontsize=8)
fig.tight_layout()
fig.savefig(OUT / "fig_val_curves.png", dpi=200)
plt.close(fig)
# ---------- Fig 2: H1 accuracy + cost ----------
def fig_h1():
h = R["h1_official_test"]["full_tile"]
keys = [k for k in ["baseline_frozen", "lora_r16", "lora_r32_all", "full_finetune", "lora_r16_cedice"] if h.get(k)]
fig, axes = plt.subplots(1, 3, figsize=(14, 4.2), gridspec_kw={"width_ratios": [1.5, 1, 1]})
ax = axes[0]
vals = [h[k]["iou"] for k in keys]
bars(ax, keys, vals, [h[k]["ci_tile"][0] for k in keys], [h[k]["ci_tile"][1] for k in keys])
ax.set_ylim(0, 0.7)
tidy(ax, "flood IoU (official test split)")
ax.set_title(f"H1: accuracy on the official test split ({R['h1_official_test']['n_tiles']} tiles, {R['h1_official_test']['n_events']} events)")
ax.text(0.01, 0.97, "error bars: 95% paired bootstrap CI over tiles", transform=ax.transAxes, fontsize=7.5, color=INK2, va="top")
ax = axes[1]
m = R["models"]
vals = [m[k]["adapter_M"] if m[k]["adapter_M"] > 0 else 0.05 for k in keys]
x = bars(ax, keys, vals, fmt="{:.1f} M")
for i, k in enumerate(keys):
if m[k]["adapter_M"] == 0:
ax.texts[-len(keys) + i].set_text("0 (decoder only)")
ax.set_yscale("log")
ax.set_ylim(0.03, 400)
tidy(ax, "encoder parameters trained / stored (M, log)")
ax.set_title("What has to be stored per task")
ax = axes[2]
vals = [m[k]["peak_mem_GiB"] for k in keys]
x = bars(ax, keys, vals, fmt="{:.1f} GiB")
for i, k in enumerate(keys):
ax.text(x[i], 0.3, f"{m[k]['it_per_s']:.1f} it/s", ha="center", fontsize=7.5, color="#fcfcfb", fontweight="bold")
ax.set_ylim(0, 10.5)
tidy(ax, "peak GPU memory, batch 32 bf16 (GiB)")
ax.set_title("Training cost (RTX PRO 6000)")
fig.tight_layout()
fig.savefig(OUT / "fig_h1_accuracy_cost.png", dpi=200)
plt.close(fig)
# ---------- Fig 3: H3 transfer ----------
def fig_h3():
h3, pk = R["h3"], R["pk_official_test"]
fig, axes = plt.subplots(1, 3, figsize=(14, 4.4), gridspec_kw={"width_ratios": [0.7, 2.0, 1.4]})
ax = axes[0]
keys = ["sa_holdout", "sa_fewshot"]
s = h3["sa_test_238"]
bars(ax, keys, [s[k]["full_tile"] for k in keys], [s[k]["ci_tile"][0] for k in keys], [s[k]["ci_tile"][1] for k in keys])
d = s["diff_fewshot_minus_holdout"]["tile"]
ax.annotate(f"few-shot - hold-out: +{d[0]:.3f}\n95% CI [{d[1]:+.3f}, {d[2]:+.3f}]", xy=(0.5, 0.86), ha="center", fontsize=8, color=INK)
ax.set_ylim(0, 0.97)
tidy(ax, "flood IoU")
ax.set_title("South-Asia test\n(238 tiles, 6 events)")
ax = axes[1]
keys = ["sa_holdout", "sa_fewshot", "baseline_frozen", "lora_r16", "lora_r32_all", "full_finetune", "lora_r16_cedice"]
v = pk["full_tile"]
bars(ax, keys, [v[k][0] for k in keys], [v[k][1] for k in keys], [v[k][2] for k in keys])
ax.axvspan(-0.5, 1.5, color="#e6e5e1", alpha=0.5, lw=0, zorder=0)
ax.text(0.5, 0.91, "never trained on Pakistan\n(or only 26 tiles)", ha="center", fontsize=7.5, color=INK2)
ax.text(4.0, 0.91, "trained on 208 Pakistan tiles (EMSR629/631/838 train split)", ha="center", fontsize=7.5, color=INK2)
ax.set_ylim(0, 1.0)
tidy(ax)
ax.set_title(f"Pakistan official test tiles (EMSR838, n={pk['n_tiles']})")
ax = axes[2]
rows = [(k, v) for k, v in h3["per_aoi"].items() if v[2] >= 3]
rows.sort(key=lambda kv: kv[1][0])
y = np.arange(len(rows))
for i, (k, (a, b, n)) in enumerate(rows):
ax.plot([a, b], [i, i], color=INK2, linewidth=1.2, zorder=1)
ax.plot(a, i, "o", color=COL["sa_holdout"], markersize=6, zorder=2)
ax.plot(b, i, "o", color=COL["sa_fewshot"], markersize=6, zorder=2)
ax.text(max(a, b) + 0.02, i, f"n={n}", va="center", fontsize=7, color=INK2)
ax.set_yticks(y)
ax.set_yticklabels([k for k, _ in rows], fontsize=7.5)
ax.set_xlim(-0.02, 0.95)
ax.grid(axis="x", color=GRID)
ax.set_axisbelow(True)
ax.set_xlabel("flood IoU per AOI (>= 3 test tiles)")
ax.set_title("Per AOI: hold-out -> few-shot")
ax.legend(handles=[plt.Line2D([], [], marker="o", color=COL["sa_holdout"], lw=0, label="hold-out"),
plt.Line2D([], [], marker="o", color=COL["sa_fewshot"], lw=0, label="few-shot")], loc="lower right", frameon=False, fontsize=8)
fig.suptitle("H3: transfer to South Asia (train on all non-South-Asian events, then adapt with 26 tiles)", fontsize=10, fontweight="bold")
fig.tight_layout()
fig.savefig(OUT / "fig_h3_transfer.png", dpi=200)
plt.close(fig)
# ---------- Fig 4: H4 population ----------
def fig_h4():
h4 = R["h4_emsr838"]
off = h4["official_pop"]
fig, axes = plt.subplots(1, 2, figsize=(14, 4.4), gridspec_kw={"width_ratios": [1.4, 1.2]})
ax = axes[0]
keys = ["sa_holdout", "sa_fewshot", "baseline_frozen", "lora_r16", "lora_r32_all", "full_finetune", "lora_r16_cedice"]
vals = [h4["models"][k]["pop"] / 1e3 for k in keys]
x = bars(ax, keys, vals, fmt="{:,.0f} k")
ax.axhline(off / 1e3, color=COL["official"], linewidth=1.5, linestyle="--")
ax.text(0.01, 0.97, f"dashed line = CEMS official outline, {off / 1e3:,.0f} k people\ninside bars: total error, and population-weighted mean error per AOI", transform=ax.transAxes, fontsize=7.5, color=INK2, va="top")
for i, k in enumerate(keys):
m = h4["models"][k]
ax.text(x[i], 30, f"{100 * (m['pop'] / off - 1):+.1f}%", ha="center", fontsize=8, color="#fcfcfb", fontweight="bold")
ax.text(x[i], 110, f"AOI\n{m['aoi_mape_w']:.0f}%", ha="center", fontsize=7.5, color="#fcfcfb")
ax.set_ylim(0, 1250)
tidy(ax, "affected population (thousands)")
ax.set_title(f"H4: people inside the flood extent, EMSR838 ({h4['n_tiles']} tiles)")
ax = axes[1]
aois = list(h4["per_aoi_holdout"].keys())
aois.sort(key=lambda a: -h4["per_aoi_holdout"][a][1])
xx = np.arange(len(aois))
w = 0.27
for j, (k, src) in enumerate([("official", None), ("sa_holdout", "per_aoi_holdout"), ("sa_fewshot", "per_aoi_fewshot")]):
vals = [h4["per_aoi_holdout"][a][1] if src is None else h4[src][a][0] for a in aois]
ax.bar(xx + (j - 1) * w, np.maximum(vals, 1), w - 0.03, color=COL[k], linewidth=0, label=LAB[k].split("\n")[0])
ax.set_yscale("log")
ax.set_ylim(50, 2e6)
ax.set_xticks(xx)
ax.set_xticklabels([a.replace("EMSR838-", "AOI ") for a in aois], fontsize=8)
tidy(ax, "affected people per AOI (log)")
ax.set_title("Per AOI: official vs models that never (or barely) saw Pakistan")
ax.legend(frameon=False, fontsize=8, loc="upper right")
fig.tight_layout()
fig.savefig(OUT / "fig_h4_population.png", dpi=200)
plt.close(fig)
if __name__ == "__main__":
fig_curves(); fig_h1(); fig_h3(); fig_h4()
print("wrote", sorted(p.name for p in OUT.glob("fig_*.png")))