code/bundle/analyze_paper_table.py

#!/usr/bin/env python3
"""Sanity-check: recompute Table 2 correlations from published Table 3 numbers (arXiv v3)."""

from __future__ import annotations

import json
from pathlib import Path

from analyze import analyze

# From paper Table 3 (arXiv:2602.03282v3)
PAPER_ROWS = [
    {"model": "Barlow Twins", "family": "Var-Decorr", "G.PR": 0.11, "G.Iso": 0.91, "L.Iso": 0.82, "JER": 28.8, "disc": 0.750, "binding": 0.446},
    {"model": "VICReg", "family": "Var-Decorr", "G.PR": 0.06, "G.Iso": 0.91, "L.Iso": 0.82, "JER": 28.8, "disc": 0.750, "binding": 0.436},
    {"model": "SwAV", "family": "Clustering", "G.PR": 0.08, "G.Iso": 0.89, "L.Iso": 0.80, "JER": 29.9, "disc": 0.750, "binding": 0.340},
    {"model": "DINOv2 ViT-B/14", "family": "Self-Distill", "G.PR": 0.22, "G.Iso": 0.94, "L.Iso": 0.82, "JER": 24.2, "disc": 0.746, "binding": 0.302},
    {"model": "DINOv2 ViT-S/14", "family": "Self-Distill", "G.PR": 0.29, "G.Iso": 0.94, "L.Iso": 0.82, "JER": 26.8, "disc": 0.750, "binding": 0.270},
    {"model": "DINOv2 ViT-L/14", "family": "Self-Distill", "G.PR": 0.21, "G.Iso": 0.94, "L.Iso": 0.83, "JER": 23.1, "disc": 0.724, "binding": 0.238},
    {"model": "MoCo v3", "family": "Contrastive", "G.PR": 0.11, "G.Iso": 0.90, "L.Iso": 0.85, "JER": 16.3, "disc": 0.750, "binding": 0.224},
    {"model": "MAE ViT-L/16", "family": "Masked", "G.PR": 0.01, "G.Iso": 0.71, "L.Iso": 0.72, "JER": 25.5, "disc": 0.500, "binding": 0.220},
    {"model": "DINO ViT-S/16", "family": "Self-Distill", "G.PR": 0.21, "G.Iso": 0.90, "L.Iso": 0.81, "JER": 26.3, "disc": 0.686, "binding": 0.196},
    {"model": "DINO ViT-B/16", "family": "Self-Distill", "G.PR": 0.11, "G.Iso": 0.90, "L.Iso": 0.83, "JER": 24.0, "disc": 0.696, "binding": 0.196},
    {"model": "DINOv2 ViT-g/14", "family": "Self-Distill", "G.PR": 0.14, "G.Iso": 0.93, "L.Iso": 0.82, "JER": 21.7, "disc": 0.710, "binding": 0.196},
    {"model": "MAE ViT-B/16", "family": "Masked", "G.PR": 0.00, "G.Iso": 0.33, "L.Iso": 0.56, "JER": 27.9, "disc": 0.500, "binding": 0.182},
    {"model": "ViT-L/16", "family": "Supervised", "G.PR": 0.11, "G.Iso": 0.90, "L.Iso": 0.80, "JER": 14.9, "disc": 0.646, "binding": 0.182},
    {"model": "BEiTv2", "family": "Masked", "G.PR": 0.11, "G.Iso": 0.90, "L.Iso": 0.79, "JER": 19.8, "disc": 0.714, "binding": 0.178},
    {"model": "ConvNeXt Large", "family": "Supervised", "G.PR": 0.12, "G.Iso": 0.89, "L.Iso": 0.79, "JER": 14.1, "disc": 0.702, "binding": 0.160},
    {"model": "ViT-B/16", "family": "Supervised", "G.PR": 0.19, "G.Iso": 0.94, "L.Iso": 0.79, "JER": 17.1, "disc": 0.648, "binding": 0.156},
    {"model": "ConvNeXt Base", "family": "Supervised", "G.PR": 0.00, "G.Iso": 0.22, "L.Iso": 0.79, "JER": 18.7, "disc": 0.586, "binding": 0.146},
    {"model": "CLIP ViT-B/16", "family": "Vision-Lang", "G.PR": 0.14, "G.Iso": 0.91, "L.Iso": 0.85, "JER": 20.8, "disc": 0.500, "binding": 0.138},
    {"model": "CLIP ViT-L/14", "family": "Vision-Lang", "G.PR": 0.13, "G.Iso": 0.92, "L.Iso": 0.86, "JER": 18.7, "disc": 0.500, "binding": 0.130},
    {"model": "SigLIP ViT-B/16", "family": "Vision-Lang", "G.PR": 0.09, "G.Iso": 0.91, "L.Iso": 0.83, "JER": 23.1, "disc": 0.500, "binding": 0.128},
    {"model": "SigLIP SoViT-400M", "family": "Vision-Lang", "G.PR": 0.07, "G.Iso": 0.90, "L.Iso": 0.82, "JER": 19.0, "disc": 0.500, "binding": 0.128},
    {"model": "BEiT", "family": "Masked", "G.PR": 0.07, "G.Iso": 0.90, "L.Iso": 0.72, "JER": 20.9, "disc": 0.500, "binding": 0.126},
    {"model": "CLIP ViT-B/32", "family": "Vision-Lang", "G.PR": 0.14, "G.Iso": 0.91, "L.Iso": 0.85, "JER": 21.7, "disc": 0.500, "binding": 0.126},
    {"model": "EVA-CLIP ViT-E/14", "family": "Vision-Lang", "G.PR": 0.08, "G.Iso": 0.92, "L.Iso": 0.85, "JER": 19.4, "disc": 0.500, "binding": 0.126},
    {"model": "EVA-CLIP ViT-B/16", "family": "Vision-Lang", "G.PR": 0.14, "G.Iso": 0.92, "L.Iso": 0.84, "JER": 19.0, "disc": 0.500, "binding": 0.124},
    {"model": "EVA-CLIP ViT-L/14", "family": "Vision-Lang", "G.PR": 0.10, "G.Iso": 0.91, "L.Iso": 0.83, "JER": 18.1, "disc": 0.500, "binding": 0.090},
]


def main():
    for r in PAPER_ROWS:
        r["error"] = None
    summary = analyze(PAPER_ROWS)
    # Paper-reported targets (v3)
    summary["paper_reported"] = {
        "G.PR": {"r": 0.10, "p": 0.64},
        "G.Iso": {"r": 0.11, "p": 0.59},
        "L.Iso": {"r": 0.02, "p": 0.93},
        "JER": {"r": 0.69, "p": 0.0001},
        "JER+Disc_R2": 0.78,
        "n_models": 26,
        "challenge_claim_note": (
            "Challenge claims cite 21 encoders with r≈0 for G.PR and r=0.65 for JER; "
            "arXiv v3 reports 26 encoders with r=0.10 (G.PR) and r=0.69 (JER)."
        ),
    }
    out = Path("outputs/paper_table_analysis.json")
    out.parent.mkdir(exist_ok=True)
    out.write_text(json.dumps(summary, indent=2))
    print(json.dumps(summary, indent=2))


if __name__ == "__main__":
    main()