""" Cross-source confusion matrices — the 2x2 as four confusion heatmaps. Kills the "accuracy is an artifact" objection: shows WHERE the studio-trained model's predictions collapse in the wild (per-class error structure), not just a scalar. Reads a shortcut_evidence_*.json (written by eval_confusion.py). """ import sys, os, glob, json from pathlib import Path _base = "/mnt/d/SpiceNet" if os.path.exists("/mnt/d/SpiceNet") else "D:/SpiceNet" sys.path.insert(0, _base) import numpy as np import figstyle ROOT = Path(_base) OUT = ROOT / "outputs" / "confusion_cross_source" def _pick(): ev = [p for p in sorted(glob.glob(str(ROOT / "outputs" / "shortcut_evidence_s*.json"))) if "smoke" not in Path(p).name.lower() and "dedup" not in Path(p).name.lower()] return ev[0] if ev else None def main(): src = _pick() if not src: raise SystemExit("no shortcut_evidence_s*.json found — run eval_confusion.py first") figstyle.apply() import matplotlib.pyplot as plt r = json.load(open(src)) classes = r["classes"] cells = [("ss", "ss", "Wild-trained / wild-test (within)"), ("ss", "in", "Wild-trained / studio-test (free)"), ("in", "ss", "Studio-trained / wild-test (COLLAPSE)"), ("in", "in", "Studio-trained / studio-test (within)")] fig, axes = plt.subplots(2, 2, figsize=(12, 11)) for ax, (mt, tt, title) in zip(axes.flat, cells): cm = np.array(r[mt][tt]["confusion"], float) cmn = cm / np.clip(cm.sum(1, keepdims=True), 1, None) im = ax.imshow(cmn, cmap=figstyle.MATRIX_CMAP, vmin=0, vmax=1, aspect="auto") acc = r[mt][tt]["acc"]; f1 = r[mt][tt]["macro_f1"] ax.set_title(f"{title}\nacc={acc*100:.1f}% macro-F1={f1*100:.1f}%", fontsize=10.5, fontweight="bold") ax.set_xticks(range(len(classes))); ax.set_yticks(range(len(classes))) ax.set_xticklabels(classes, rotation=90, fontsize=7) ax.set_yticklabels(classes, fontsize=7) ax.set_xlabel("predicted", fontsize=9); ax.set_ylabel("true", fontsize=9) for i in range(len(classes)): for j in range(len(classes)): v = cmn[i, j] if v > 0.02: ax.text(j, i, f"{v*100:.0f}", ha="center", va="center", fontsize=6, color="white" if v < 0.6 else "black") fig.suptitle("Cross-source confusion structure (row-normalized)", fontsize=13, fontweight="bold", y=1.0) fig.tight_layout(rect=[0, 0, 1, 0.97]) figstyle.save(fig, str(OUT)) if __name__ == "__main__": main()