File size: 4,083 Bytes
1ea7ba6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | """
Calibration under cross-source shift — is the studio model confidently WRONG?
For each model x test-set, dumps per-sample (max-softmax confidence, correct)
and computes ECE. The headline: the studio-trained model, tested in the wild,
stays HIGH-confidence while its accuracy collapses — overconfidence under shift.
This turns the "100% accuracy is a red flag" concern into evidence.
python eval_calibration.py \
--ss_ckpt outputs/checkpoints/overlap_ss_s42/p1_best.pth \
--in_ckpt outputs/checkpoints/overlap_indian_s42/p1_best.pth --out_suffix s42
GPU eval-only. Run after run_q1_anchor.py.
"""
import sys, os, argparse, json
from pathlib import Path
_base = "/mnt/d/SpiceNet" if os.path.exists("/mnt/d/SpiceNet") else "D:/SpiceNet"
sys.path.insert(0, _base)
import numpy as np
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
import config
from src.model import SpiceFusionNet
from src.dataset import SpiceDataset, get_val_transform, load_manifest_splits
ROOT = Path(_base)
SS_MANIFEST = str(ROOT / "outputs" / "manifest_overlap_ss.json")
IN_MANIFEST = str(ROOT / "outputs" / "manifest_overlap_indian.json")
def _loader(paths, labels):
ds = SpiceDataset(paths, labels, get_val_transform(), multimodal=False)
return DataLoader(ds, batch_size=64, num_workers=2, pin_memory=True)
def _load(ckpt, n, device):
m = SpiceFusionNet(num_classes=n).to(device)
ck = torch.load(ckpt, map_location=device, weights_only=False)
m.load_state_dict(ck["model_state"]); m.eval()
return m
@torch.no_grad()
def _eval(model, loader, device):
conf, correct = [], []
for imgs, tex, col, labels in loader:
probs = F.softmax(model.forward_image(imgs.to(device)), dim=1)
p, pred = probs.max(1)
conf.extend(p.cpu().tolist())
correct.extend((pred.cpu() == labels).int().tolist())
return np.asarray(conf), np.asarray(correct)
def ece(conf, correct, n_bins=15):
"""Expected Calibration Error."""
bins = np.linspace(0, 1, n_bins + 1)
e, N = 0.0, len(conf)
for i in range(n_bins):
m = (conf > bins[i]) & (conf <= bins[i + 1])
if m.sum():
e += (m.sum() / N) * abs(correct[m].mean() - conf[m].mean())
return float(e)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--ss_ckpt", required=True)
ap.add_argument("--in_ckpt", required=True)
ap.add_argument("--ss_manifest", default=SS_MANIFEST)
ap.add_argument("--in_manifest", default=IN_MANIFEST)
ap.add_argument("--out_suffix", required=True)
args = ap.parse_args()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
ss_splits, classes = load_manifest_splits(args.ss_manifest)
in_splits, _ = load_manifest_splits(args.in_manifest)
n = len(classes)
ss_loader, in_loader = _loader(*ss_splits["test"]), _loader(*in_splits["test"])
models = {"ss": _load(args.ss_ckpt, n, device), "in": _load(args.in_ckpt, n, device)}
loaders = {"ss": ss_loader, "in": in_loader}
out = {}
for mt in ("ss", "in"):
out[mt] = {}
for tt in ("ss", "in"):
conf, correct = _eval(models[mt], loaders[tt], device)
out[mt][tt] = {"confidence": conf.tolist(), "correct": correct.astype(int).tolist(),
"acc": float(correct.mean()), "mean_conf": float(conf.mean()),
"ece": ece(conf, correct)}
out["classes"] = classes
p = ROOT / "outputs" / f"calibration_{args.out_suffix}.json"
json.dump(out, open(p, "w"))
# headline: studio-trained (in) on wild-test (ss)
h = out["in"]["ss"]
direction = "over-confident" if h["mean_conf"] > h["acc"] else "under-confident"
print(f"Studio-trained on WILD test: acc={h['acc']*100:.1f}% mean-conf={h['mean_conf']*100:.1f}% "
f"ECE={h['ece']*100:.1f}% ({direction})")
print(f"saved -> {p}")
if __name__ == "__main__":
main()
|