File size: 3,923 Bytes
1ea7ba6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | """
Assemble the cross-source benchmark: every backbone vs the studio->wild collapse.
Reads bench_<model>.json (from eval_backbone_shortcut.py) plus SpiceFusionNet
(ours, from shortcut_multiseed_aggregate.json) and GranuFormer, and writes a
comparison table (markdown + LaTeX) and a clean collapse-leaderboard figure. The
takeaway: the collapse is architecture-universal.
"""
import sys, os, glob, json
from pathlib import Path
_base = "/mnt/d/SpiceNet" if os.path.exists("/mnt/d/SpiceNet") else "D:/SpiceNet"
sys.path.insert(0, _base)
import numpy as np
import figstyle
ROOT = Path(_base)
OUT = ROOT / "outputs" / "benchmark_collapse"
def collect():
rows = [] # (name, studio_within, studio_to_wild, tax, macroF1_cross)
# ours (3-seed)
agg = ROOT / "outputs" / "shortcut_multiseed_aggregate.json"
if agg.exists():
a = json.load(open(agg))["aggregate"]
rows.append(("SpiceFusionNet (ours)", a["in_within"]["mean"], a["in_cross"]["mean"],
a["in_tax_pp"]["mean"], None))
gf = ROOT / "outputs" / "shortcut_test_matrix_granuformer.json"
if gf.exists():
r = json.load(open(gf))
rows.append(("GranuFormer", r["in"]["in"]["acc"], r["in"]["ss"]["acc"],
(r["in"]["in"]["acc"] - r["in"]["ss"]["acc"]) * 100, None))
for f in sorted(glob.glob(str(ROOT / "outputs" / "bench_*.json"))):
r = json.load(open(f))
rows.append((r["model"], r["in"]["in"]["acc"], r["in"]["ss"]["acc"],
r["in_tax_pp"], r["in"]["ss"].get("macro_f1")))
return rows
def main():
rows = collect()
if not rows:
raise SystemExit("no benchmark results yet — run run_benchmark.py first")
rows.sort(key=lambda r: r[3]) # by collapse tax ascending (least collapse first)
md = ["# Cross-source benchmark (Indian/studio-trained -> SS/wild test)\n",
"| Model | Studio within | Studio->Wild | Collapse (pp) | Macro-F1 (wild) |",
"|---|---|---|---|---|"]
tex = [r"\begin{tabular}{lrrrr}", r"\hline",
r"Model & Studio within & Studio$\rightarrow$Wild & Collapse (pp) & Macro-F1 \\", r"\hline"]
for name, sw, stw, tax, f1 in rows:
f1s = f"{f1*100:.1f}" if f1 is not None else "--"
md.append(f"| {name} | {sw*100:.1f} | {stw*100:.1f} | {tax:.1f} | {f1s} |")
tex.append(f"{name} & {sw*100:.1f} & {stw*100:.1f} & {tax:.1f} & {f1s} \\\\")
tex += [r"\hline", r"\end{tabular}"]
(ROOT / "outputs" / "benchmark_table.md").write_text("\n".join(md), encoding="utf-8")
(ROOT / "outputs" / "benchmark_table.tex").write_text("\n".join(tex), encoding="utf-8")
# ── clean collapse-leaderboard figure ───────────────────────────────────
figstyle.apply()
import matplotlib.pyplot as plt
P = figstyle.PALETTE
names = [r[0] for r in rows]
within = np.array([r[1] for r in rows]) * 100
cross = np.array([r[2] for r in rows]) * 100
y = np.arange(len(names))[::-1]
fig, ax = plt.subplots(figsize=(7.2, 0.5 * len(names) + 1.5))
ax.barh(y + 0.18, within, 0.34, color=P["studio"], label="studio test (within)")
ax.barh(y - 0.18, cross, 0.34, color=P["cross_broken"], label="studio→wild (cross)")
for yi, c in zip(y, cross):
ax.text(c + 1, yi - 0.18, f"{c:.0f}", va="center", fontsize=8)
ax.axvline(12.5, color=P["chance"], ls=":", lw=1, label="random (1/8)")
ax.set_yticks(y); ax.set_yticklabels(names, fontsize=9)
ax.set_xlabel("Top-1 accuracy (%)"); ax.set_xlim(0, 108)
ax.set_title("Studio-trained models all collapse on wild data", fontsize=11)
ax.legend(loc="lower right", fontsize=8)
fig.tight_layout()
figstyle.save(fig, str(OUT))
print("\n".join(md))
if __name__ == "__main__":
main()
|