File size: 3,571 Bytes
63fc39b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Regenerate research_v2_latest/kfold/cnn_clip/mcnemar.json for ALL NINE models.
Under the uniform pooled-ensemble protocol every model carries one aligned
per-image prediction on the fixed 3,208-image test set: the five convolutional
networks and CLIP from kfold/cnn_clip/<model>_test_preds.json, the three
foundation models from the probability average of their five
kfold/foundation_fold<k>_<model>_preds.json checkpoints. Because all nine share
the same test images in the same order, the paired McNemar test now covers every
pair (36 in total).
Test: exact binomial two-sided on the discordant pairs (b, c), identical to
research_v2_latest/code/ensemble_and_stats.py. The file keeps the original
"<a>_vs_<b>": {"b","c","p"} layout used by thesis_build/figures/make_fig4_5.py;
a sibling key "_meta" records the model order, pair count and Bonferroni factor.
Run with the project .venv interpreter:
.venv/bin/python thesis_build/make_mcnemar9.py
"""
import json
import numpy as np
from pathlib import Path
from scipy.stats import binom
ROOT = Path(__file__).resolve().parent.parent
KF = ROOT / "kfold"
OUT = KF / "cnn_clip" / "mcnemar.json"
# accuracy-descending order (matches Table 4.1)
ORDER = ["inception_v3", "clip_openai", "vgg19", "resnet101", "dinov2_l",
"densenet121", "resnet50", "swin_b", "retfound"]
CNN_CLIP = {"inception_v3", "clip_openai", "vgg19", "resnet101", "densenet121", "resnet50"}
def preds(model):
if model in CNN_CLIP:
d = json.load(open(KF / "cnn_clip" / f"{model}_test_preds.json"))
return np.array(d["labels"]), np.array(d["preds"])
probs, labels = [], None
for k in range(5):
d = json.load(open(KF / f"foundation_fold{k}_{model}_preds.json"))
probs.append(np.array(d["probs"])); labels = np.array(d["labels"])
return labels, np.mean(probs, axis=0).argmax(1)
def main():
P = {m: preds(m) for m in ORDER}
L = P[ORDER[0]][0]
for m in ORDER:
assert np.array_equal(P[m][0], L), f"{m} label order differs"
out = {}
pairs = []
for i in range(len(ORDER)):
for j in range(i + 1, len(ORDER)):
a, b = ORDER[i], ORDER[j]
ca = P[a][1] == L; cb = P[b][1] == L
bb = int(np.sum(ca & ~cb)); cc = int(np.sum(~ca & cb)); n = bb + cc
p = 1.0 if n == 0 else float(min(1.0, 2 * binom.cdf(min(bb, cc), n, 0.5)))
out[f"{a}_vs_{b}"] = {"b": bb, "c": cc, "p": p}
pairs.append((a, b, bb, cc, p))
npairs = len(pairs)
out["_meta"] = {"models": ORDER, "n_pairs": npairs, "bonferroni_factor": npairs,
"test": "exact binomial two-sided McNemar"}
with open(OUT, "w") as fh:
json.dump(out, fh, indent=1)
lines = []
nsig = 0
for a, b, bb, cc, p in sorted(pairs, key=lambda x: x[4]):
adj = min(1.0, p * npairs); sig = adj < 0.05; nsig += sig
lines.append(f"{'*' if sig else ' '} {a:13s} vs {b:13s} b={bb:4d} c={cc:4d} p={p:.3g} adj={adj:.3g}")
top = [x for x in pairs if x[0] not in {"swin_b", "retfound"} and x[1] not in {"swin_b", "retfound"}]
with open("/tmp/mcnemar9_check.txt", "w") as fh:
fh.write(f"wrote {OUT.relative_to(ROOT)} n_pairs={npairs} significant(Bonferroni)={nsig}\n")
fh.write(f"top-7 pairs={len(top)} min raw p among top-7={min(x[4] for x in top):.4f} "
f"significant among top-7={sum(1 for x in top if x[4]*npairs<0.05)}\n\n")
fh.write("\n".join(lines) + "\n")
print("wrote", OUT.relative_to(ROOT))
if __name__ == "__main__":
main()
|